2023-10-09T04:58:39.349 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T04:58:39.349 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492 --verbose --name teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492/orig.config.yaml 2023-10-09T04:58:39.408 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T04:58:39.502 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7417492' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 3687 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi008.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi191.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T04:58:39.502 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T04:58:39.503 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T04:58:39.503 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T04:58:39.505 INFO:teuthology.task.internal:Checking packages... 2023-10-09T04:58:39.529 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T04:58:39.530 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T04:58:39.530 INFO:teuthology.packaging:ref: None 2023-10-09T04:58:39.530 INFO:teuthology.packaging:tag: None 2023-10-09T04:58:39.530 INFO:teuthology.packaging:branch: reef 2023-10-09T04:58:39.530 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:58:39.530 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-10-09T04:58:39.735 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610.g397de1b0 2023-10-09T04:58:39.735 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T04:58:39.742 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T04:58:39.743 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T04:58:39.756 INFO:teuthology.task.internal:Saving configuration 2023-10-09T04:58:39.781 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T04:58:39.788 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T04:58:39.815 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi008.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 04:53:34.770646', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T04:58:39.836 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi191.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 04:53:34.772192', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T04:58:39.836 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T04:58:39.842 INFO:teuthology.task.internal:roles: ubuntu@smithi008.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T04:58:39.842 INFO:teuthology.task.internal:roles: ubuntu@smithi191.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T04:58:39.842 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T04:58:39.914 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f7dee480af0>, signals=[15]) 2023-10-09T04:58:39.915 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T04:58:39.921 INFO:teuthology.task.internal:Opening connections... 2023-10-09T04:58:39.921 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi008.front.sepia.ceph.com 2023-10-09T04:58:39.922 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi008.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T04:58:39.990 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi191.front.sepia.ceph.com 2023-10-09T04:58:39.991 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T04:58:40.057 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T04:58:40.064 DEBUG:teuthology.orchestra.run.smithi008:> uname -m 2023-10-09T04:58:40.084 INFO:teuthology.orchestra.run.smithi008.stdout:x86_64 2023-10-09T04:58:40.084 DEBUG:teuthology.orchestra.run.smithi008:> cat /etc/os-release 2023-10-09T04:58:40.145 INFO:teuthology.orchestra.run.smithi008.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T04:58:40.145 INFO:teuthology.orchestra.run.smithi008.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T04:58:40.145 INFO:teuthology.orchestra.run.smithi008.stdout:ID="rhel" 2023-10-09T04:58:40.145 INFO:teuthology.orchestra.run.smithi008.stdout:ID_LIKE="fedora" 2023-10-09T04:58:40.145 INFO:teuthology.orchestra.run.smithi008.stdout:VERSION_ID="8.6" 2023-10-09T04:58:40.145 INFO:teuthology.orchestra.run.smithi008.stdout:PLATFORM_ID="platform:el8" 2023-10-09T04:58:40.145 INFO:teuthology.orchestra.run.smithi008.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T04:58:40.146 INFO:teuthology.orchestra.run.smithi008.stdout:ANSI_COLOR="0;31" 2023-10-09T04:58:40.146 INFO:teuthology.orchestra.run.smithi008.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T04:58:40.146 INFO:teuthology.orchestra.run.smithi008.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T04:58:40.146 INFO:teuthology.orchestra.run.smithi008.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T04:58:40.146 INFO:teuthology.orchestra.run.smithi008.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T04:58:40.146 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T04:58:40.146 INFO:teuthology.orchestra.run.smithi008.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T04:58:40.146 INFO:teuthology.orchestra.run.smithi008.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T04:58:40.146 INFO:teuthology.orchestra.run.smithi008.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T04:58:40.147 INFO:teuthology.orchestra.run.smithi008.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T04:58:40.147 INFO:teuthology.lock.ops:Updating smithi008.front.sepia.ceph.com on lock server 2023-10-09T04:58:40.170 DEBUG:teuthology.orchestra.run.smithi191:> uname -m 2023-10-09T04:58:40.190 INFO:teuthology.orchestra.run.smithi191.stdout:x86_64 2023-10-09T04:58:40.191 DEBUG:teuthology.orchestra.run.smithi191:> cat /etc/os-release 2023-10-09T04:58:40.250 INFO:teuthology.orchestra.run.smithi191.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T04:58:40.250 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T04:58:40.250 INFO:teuthology.orchestra.run.smithi191.stdout:ID="rhel" 2023-10-09T04:58:40.250 INFO:teuthology.orchestra.run.smithi191.stdout:ID_LIKE="fedora" 2023-10-09T04:58:40.250 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION_ID="8.6" 2023-10-09T04:58:40.250 INFO:teuthology.orchestra.run.smithi191.stdout:PLATFORM_ID="platform:el8" 2023-10-09T04:58:40.250 INFO:teuthology.orchestra.run.smithi191.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T04:58:40.250 INFO:teuthology.orchestra.run.smithi191.stdout:ANSI_COLOR="0;31" 2023-10-09T04:58:40.251 INFO:teuthology.orchestra.run.smithi191.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T04:58:40.251 INFO:teuthology.orchestra.run.smithi191.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T04:58:40.251 INFO:teuthology.orchestra.run.smithi191.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T04:58:40.251 INFO:teuthology.orchestra.run.smithi191.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T04:58:40.251 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T04:58:40.251 INFO:teuthology.orchestra.run.smithi191.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T04:58:40.251 INFO:teuthology.orchestra.run.smithi191.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T04:58:40.251 INFO:teuthology.orchestra.run.smithi191.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T04:58:40.251 INFO:teuthology.orchestra.run.smithi191.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T04:58:40.251 INFO:teuthology.lock.ops:Updating smithi191.front.sepia.ceph.com on lock server 2023-10-09T04:58:40.270 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T04:58:40.279 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T04:58:40.285 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T04:58:40.285 DEBUG:teuthology.orchestra.run.smithi008:> test '!' -e /home/ubuntu/cephtest 2023-10-09T04:58:40.288 DEBUG:teuthology.orchestra.run.smithi191:> test '!' -e /home/ubuntu/cephtest 2023-10-09T04:58:40.309 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T04:58:40.314 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T04:58:40.315 DEBUG:teuthology.orchestra.run.smithi008:> test -z $(ls -A /var/lib/ceph) 2023-10-09T04:58:40.348 DEBUG:teuthology.orchestra.run.smithi191:> test -z $(ls -A /var/lib/ceph) 2023-10-09T04:58:40.374 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T04:58:40.427 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T04:58:40.441 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T04:58:40.441 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-10-09T04:58:40.441 DEBUG:teuthology.orchestra.run.smithi008:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T04:58:40.441 DEBUG:teuthology.orchestra.run.smithi191:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T04:58:40.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:40.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:40.460 DEBUG:teuthology.orchestra.run.smithi008:> uname -r 2023-10-09T04:58:40.461 DEBUG:teuthology.orchestra.run.smithi191:> uname -r 2023-10-09T04:58:40.518 INFO:teuthology.orchestra.run.smithi191.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T04:58:40.519 INFO:teuthology.task.kernel:Running kernel on smithi191: 4.18.0-372.9.1.el8.x86_64 2023-10-09T04:58:40.519 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum install -y kernel 2023-10-09T04:58:40.520 INFO:teuthology.orchestra.run.smithi008.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T04:58:40.520 INFO:teuthology.task.kernel:Running kernel on smithi008: 4.18.0-372.9.1.el8.x86_64 2023-10-09T04:58:40.520 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum install -y kernel 2023-10-09T04:58:41.734 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T04:58:41.734 INFO:teuthology.orchestra.run.smithi008.stdout:Unable to read consumer identity 2023-10-09T04:58:41.900 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T04:58:41.900 INFO:teuthology.orchestra.run.smithi191.stdout:Unable to read consumer identity 2023-10-09T04:58:43.161 INFO:teuthology.orchestra.run.smithi008.stdout:Copr repo for python3-asyncssh owned by ceph 21 kB/s | 3.5 kB 00:00 2023-10-09T04:58:43.508 INFO:teuthology.orchestra.run.smithi191.stdout:Copr repo for python3-asyncssh owned by ceph 14 kB/s | 3.5 kB 00:00 2023-10-09T04:58:43.862 INFO:teuthology.orchestra.run.smithi008.stdout:Extra Packages for Enterprise Linux 23 MB/s | 16 MB 00:00 2023-10-09T04:58:44.401 INFO:teuthology.orchestra.run.smithi191.stdout:Extra Packages for Enterprise Linux 18 MB/s | 16 MB 00:00 2023-10-09T04:58:47.646 INFO:teuthology.orchestra.run.smithi008.stdout:lab-extras 436 kB/s | 24 kB 00:00 2023-10-09T04:58:47.868 INFO:teuthology.orchestra.run.smithi191.stdout:lab-extras 537 kB/s | 24 kB 00:00 2023-10-09T04:58:48.356 INFO:teuthology.orchestra.run.smithi008.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T04:58:48.409 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T04:58:48.410 INFO:teuthology.orchestra.run.smithi008.stdout:Nothing to do. 2023-10-09T04:58:48.410 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T04:58:48.537 DEBUG:teuthology.orchestra.run.smithi008:> echo no | sudo yum reinstall kernel || true 2023-10-09T04:58:48.554 INFO:teuthology.orchestra.run.smithi191.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T04:58:48.612 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T04:58:48.613 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-10-09T04:58:48.613 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T04:58:48.726 DEBUG:teuthology.orchestra.run.smithi191:> echo no | sudo yum reinstall kernel || true 2023-10-09T04:58:48.885 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T04:58:48.885 INFO:teuthology.orchestra.run.smithi008.stdout:Unable to read consumer identity 2023-10-09T04:58:49.058 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T04:58:49.058 INFO:teuthology.orchestra.run.smithi191.stdout:Unable to read consumer identity 2023-10-09T04:58:49.159 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 04:58:47 AM UTC. 2023-10-09T04:58:49.204 INFO:teuthology.orchestra.run.smithi008.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T04:58:49.236 INFO:teuthology.orchestra.run.smithi008.stderr:Error: No packages marked for reinstall. 2023-10-09T04:58:49.283 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum reinstall -y kernel || true 2023-10-09T04:58:49.318 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 04:58:47 AM UTC. 2023-10-09T04:58:49.359 INFO:teuthology.orchestra.run.smithi191.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T04:58:49.401 INFO:teuthology.orchestra.run.smithi191.stderr:Error: No packages marked for reinstall. 2023-10-09T04:58:49.447 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum reinstall -y kernel || true 2023-10-09T04:58:49.633 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T04:58:49.634 INFO:teuthology.orchestra.run.smithi008.stdout:Unable to read consumer identity 2023-10-09T04:58:49.775 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T04:58:49.776 INFO:teuthology.orchestra.run.smithi191.stdout:Unable to read consumer identity 2023-10-09T04:58:49.901 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 04:58:47 AM UTC. 2023-10-09T04:58:49.945 INFO:teuthology.orchestra.run.smithi008.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T04:58:49.967 INFO:teuthology.orchestra.run.smithi008.stderr:Error: No packages marked for reinstall. 2023-10-09T04:58:50.015 DEBUG:teuthology.orchestra.run.smithi008:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T04:58:50.030 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:03 ago on Mon 09 Oct 2023 04:58:47 AM UTC. 2023-10-09T04:58:50.072 INFO:teuthology.orchestra.run.smithi191.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T04:58:50.095 INFO:teuthology.orchestra.run.smithi191.stderr:Error: No packages marked for reinstall. 2023-10-09T04:58:50.098 INFO:teuthology.orchestra.run.smithi008.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T04:58:50.099 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T04:58:50.099 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T04:58:50.099 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T04:58:50.099 DEBUG:teuthology.orchestra.run.smithi008:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T04:58:50.132 INFO:teuthology.orchestra.run.smithi008.stdout:ttyS1 2023-10-09T04:58:50.141 DEBUG:teuthology.orchestra.run.smithi191:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T04:58:50.146 DEBUG:teuthology.parallel:result is None 2023-10-09T04:58:50.220 INFO:teuthology.orchestra.run.smithi191.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T04:58:50.221 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T04:58:50.221 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T04:58:50.221 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T04:58:50.221 DEBUG:teuthology.orchestra.run.smithi191:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T04:58:50.256 INFO:teuthology.orchestra.run.smithi191.stdout:ttyS1 2023-10-09T04:58:50.268 DEBUG:teuthology.parallel:result is None 2023-10-09T04:58:50.268 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T04:58:50.276 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T04:58:50.277 DEBUG:teuthology.orchestra.run.smithi008:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T04:58:50.278 DEBUG:teuthology.orchestra.run.smithi191:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T04:58:50.324 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T04:58:50.331 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T04:58:50.339 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T04:58:50.339 DEBUG:teuthology.orchestra.run.smithi008:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T04:58:50.342 DEBUG:teuthology.orchestra.run.smithi191:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T04:58:50.387 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T04:58:50.395 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T04:58:50.395 DEBUG:teuthology.orchestra.run.smithi008:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T04:58:50.400 DEBUG:teuthology.orchestra.run.smithi191:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T04:58:50.444 INFO:teuthology.orchestra.run.smithi008.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T04:58:50.455 INFO:teuthology.orchestra.run.smithi008.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T04:58:50.490 INFO:teuthology.orchestra.run.smithi191.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T04:58:50.501 INFO:teuthology.orchestra.run.smithi191.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T04:58:50.502 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T04:58:50.509 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T04:58:50.510 DEBUG:teuthology.orchestra.run.smithi008:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T04:58:50.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T04:58:50.539 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T04:58:50.547 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T04:58:50.548 DEBUG:teuthology.orchestra.run.smithi008:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T04:58:50.581 DEBUG:teuthology.orchestra.run.smithi191:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T04:58:50.597 DEBUG:teuthology.orchestra.run.smithi008:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T04:58:50.681 DEBUG:teuthology.orchestra.run.smithi008:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T04:58:50.733 DEBUG:teuthology.orchestra.run.smithi008:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T04:58:50.817 DEBUG:teuthology.orchestra.run.smithi008:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T04:58:50.859 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T04:58:50.859 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T04:58:50.927 DEBUG:teuthology.orchestra.run.smithi191:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T04:58:50.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T04:58:51.021 DEBUG:teuthology.orchestra.run.smithi191:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T04:58:51.103 DEBUG:teuthology.orchestra.run.smithi191:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T04:58:51.147 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T04:58:51.147 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T04:58:51.216 DEBUG:teuthology.orchestra.run.smithi008:> sudo service rsyslog restart 2023-10-09T04:58:51.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo service rsyslog restart 2023-10-09T04:58:51.277 INFO:teuthology.orchestra.run.smithi008.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T04:58:51.332 INFO:teuthology.orchestra.run.smithi191.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T04:58:51.727 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T04:58:51.736 INFO:teuthology.task.internal:Starting timer... 2023-10-09T04:58:51.736 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T04:58:51.755 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T04:58:51.763 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-10-09T04:58:51.763 DEBUG:teuthology.orchestra.run.smithi008:> sudo service auditd rotate 2023-10-09T04:58:51.828 INFO:teuthology.orchestra.run.smithi008.stdout:Rotating logs: 2023-10-09T04:58:51.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo service auditd rotate 2023-10-09T04:58:51.905 INFO:teuthology.orchestra.run.smithi191.stdout:Rotating logs: 2023-10-09T04:58:51.907 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T04:58:51.908 DEBUG:teuthology.orchestra.run.smithi008:> /usr/sbin/getenforce 2023-10-09T04:58:51.929 INFO:teuthology.orchestra.run.smithi008.stdout:Permissive 2023-10-09T04:58:51.929 DEBUG:teuthology.orchestra.run.smithi191:> /usr/sbin/getenforce 2023-10-09T04:58:51.950 INFO:teuthology.orchestra.run.smithi191.stdout:Permissive 2023-10-09T04:58:51.951 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi008.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi191.front.sepia.ceph.com': 'permissive'} 2023-10-09T04:58:51.951 DEBUG:teuthology.orchestra.run.smithi008:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T04:58:51.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:51.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T04:58:52.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:58:52.026 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T04:58:52.026 DEBUG:teuthology.orchestra.run.smithi008:> sudo /usr/sbin/setenforce permissive 2023-10-09T04:58:52.064 DEBUG:teuthology.orchestra.run.smithi191:> sudo /usr/sbin/setenforce permissive 2023-10-09T04:58:52.095 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T04:58:52.105 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-10-09T04:58:52.105 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T04:58:52.120 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T04:58:52.121 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi008.front.sepia.ceph.com,smithi191.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T05:07:52.553 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi008.front.sepia.ceph.com'), Remote(name='ubuntu@smithi191.front.sepia.ceph.com')] 2023-10-09T05:07:52.554 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi008.front.sepia.ceph.com' 2023-10-09T05:07:52.555 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi008.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:07:52.624 DEBUG:teuthology.orchestra.run.smithi008:> true 2023-10-09T05:07:52.693 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi008.front.sepia.ceph.com' 2023-10-09T05:07:52.693 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2023-10-09T05:07:52.693 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:07:52.761 DEBUG:teuthology.orchestra.run.smithi191:> true 2023-10-09T05:07:52.827 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi191.front.sepia.ceph.com' 2023-10-09T05:07:52.828 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T05:07:52.838 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T05:07:52.839 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T05:07:52.839 DEBUG:teuthology.orchestra.run.smithi008:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:07:52.841 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T05:07:52.841 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:07:52.876 INFO:teuthology.orchestra.run.smithi008.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T05:07:52.892 INFO:teuthology.orchestra.run.smithi008.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T05:07:52.899 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T05:07:52.914 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T05:07:52.988 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: ntpd: command not found 2023-10-09T05:07:52.989 INFO:teuthology.orchestra.run.smithi008.stderr:sudo: ntpd: command not found 2023-10-09T05:07:52.999 INFO:teuthology.orchestra.run.smithi191.stdout:506 Cannot talk to daemon 2023-10-09T05:07:53.002 INFO:teuthology.orchestra.run.smithi008.stdout:506 Cannot talk to daemon 2023-10-09T05:07:53.014 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T05:07:53.018 INFO:teuthology.orchestra.run.smithi008.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T05:07:53.027 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T05:07:53.033 INFO:teuthology.orchestra.run.smithi008.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T05:07:53.108 INFO:teuthology.orchestra.run.smithi191.stderr:bash: ntpq: command not found 2023-10-09T05:07:53.112 INFO:teuthology.orchestra.run.smithi191.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T05:07:53.112 INFO:teuthology.orchestra.run.smithi191.stdout:=============================================================================== 2023-10-09T05:07:53.112 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:07:53.112 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:07:53.112 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:07:53.112 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:07:53.117 INFO:teuthology.orchestra.run.smithi008.stderr:bash: ntpq: command not found 2023-10-09T05:07:53.121 INFO:teuthology.orchestra.run.smithi008.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T05:07:53.121 INFO:teuthology.orchestra.run.smithi008.stdout:=============================================================================== 2023-10-09T05:07:53.121 INFO:teuthology.orchestra.run.smithi008.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:07:53.121 INFO:teuthology.orchestra.run.smithi008.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:07:53.121 INFO:teuthology.orchestra.run.smithi008.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:07:53.121 INFO:teuthology.orchestra.run.smithi008.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:07:53.122 INFO:teuthology.run_tasks:Running task pexec... 2023-10-09T05:07:53.131 INFO:teuthology.task.pexec:Executing custom commands... 2023-10-09T05:07:53.132 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi008.front.sepia.ceph.com 2023-10-09T05:07:53.132 DEBUG:teuthology.orchestra.run.smithi008:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T05:07:53.132 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi191.front.sepia.ceph.com 2023-10-09T05:07:53.132 DEBUG:teuthology.orchestra.run.smithi191:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T05:07:53.525 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:07:53.555 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:07:54.306 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:40 ago on Mon 09 Oct 2023 05:06:13 AM UTC. 2023-10-09T05:07:54.377 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:24 ago on Mon 09 Oct 2023 05:06:29 AM UTC. 2023-10-09T05:07:56.023 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:07:56.024 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:07:56.024 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2023-10-09T05:07:56.024 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:07:56.024 INFO:teuthology.orchestra.run.smithi191.stdout:Resetting modules: 2023-10-09T05:07:56.024 INFO:teuthology.orchestra.run.smithi191.stdout: container-tools 2023-10-09T05:07:56.024 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:07:56.025 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:07:56.025 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:07:56.025 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:07:56.310 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:07:56.312 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:07:56.312 INFO:teuthology.orchestra.run.smithi008.stdout: Package Architecture Version Repository Size 2023-10-09T05:07:56.312 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:07:56.312 INFO:teuthology.orchestra.run.smithi008.stdout:Resetting modules: 2023-10-09T05:07:56.312 INFO:teuthology.orchestra.run.smithi008.stdout: container-tools 2023-10-09T05:07:56.312 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:07:56.312 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:07:56.312 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:07:56.312 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:07:56.993 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:43 ago on Mon 09 Oct 2023 05:06:13 AM UTC. 2023-10-09T05:07:57.334 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:27 ago on Mon 09 Oct 2023 05:06:29 AM UTC. 2023-10-09T05:07:59.695 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:08:00.143 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:08:00.220 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:08:00.696 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:08:00.962 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:47 ago on Mon 09 Oct 2023 05:06:13 AM UTC. 2023-10-09T05:08:01.561 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:32 ago on Mon 09 Oct 2023 05:06:29 AM UTC. 2023-10-09T05:08:02.802 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:08:02.893 INFO:teuthology.orchestra.run.smithi191.stdout:========================================================================================================================== 2023-10-09T05:08:02.893 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-10-09T05:08:02.894 INFO:teuthology.orchestra.run.smithi191.stdout:========================================================================================================================== 2023-10-09T05:08:02.894 INFO:teuthology.orchestra.run.smithi191.stdout:Installing group/module packages: 2023-10-09T05:08:02.894 INFO:teuthology.orchestra.run.smithi191.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-09T05:08:02.894 INFO:teuthology.orchestra.run.smithi191.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-09T05:08:02.894 INFO:teuthology.orchestra.run.smithi191.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-09T05:08:02.894 INFO:teuthology.orchestra.run.smithi191.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-09T05:08:02.895 INFO:teuthology.orchestra.run.smithi191.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T05:08:02.895 INFO:teuthology.orchestra.run.smithi191.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:08:02.895 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-10-09T05:08:02.895 INFO:teuthology.orchestra.run.smithi191.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T05:08:02.895 INFO:teuthology.orchestra.run.smithi191.stdout:Downgrading: 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-09T05:08:02.896 INFO:teuthology.orchestra.run.smithi191.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:08:02.897 INFO:teuthology.orchestra.run.smithi191.stdout:Installing module profiles: 2023-10-09T05:08:02.897 INFO:teuthology.orchestra.run.smithi191.stdout: container-tools/common 2023-10-09T05:08:02.897 INFO:teuthology.orchestra.run.smithi191.stdout:Enabling module streams: 2023-10-09T05:08:02.897 INFO:teuthology.orchestra.run.smithi191.stdout: container-tools 3.0 2023-10-09T05:08:02.897 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:08:02.897 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:08:02.897 INFO:teuthology.orchestra.run.smithi191.stdout:========================================================================================================================== 2023-10-09T05:08:02.897 INFO:teuthology.orchestra.run.smithi191.stdout:Install 7 Packages 2023-10-09T05:08:02.897 INFO:teuthology.orchestra.run.smithi191.stdout:Downgrade 12 Packages 2023-10-09T05:08:02.897 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:08:02.898 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 51 M 2023-10-09T05:08:02.898 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:08:03.009 INFO:teuthology.orchestra.run.smithi191.stdout:(1/19): containers-common-1.2.4-1.module+el8.6. 812 kB/s | 91 kB 00:00 2023-10-09T05:08:03.041 INFO:teuthology.orchestra.run.smithi191.stdout:(2/19): container-selinux-2.178.0-2.module+el8. 368 kB/s | 53 kB 00:00 2023-10-09T05:08:03.066 INFO:teuthology.orchestra.run.smithi191.stdout:(3/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 304 kB/s | 51 kB 00:00 2023-10-09T05:08:03.100 INFO:teuthology.orchestra.run.smithi191.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 5.5 MB/s | 511 kB 00:00 2023-10-09T05:08:03.125 INFO:teuthology.orchestra.run.smithi191.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.2 MB/s | 69 kB 00:00 2023-10-09T05:08:03.150 INFO:teuthology.orchestra.run.smithi191.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.4 MB/s | 72 kB 00:00 2023-10-09T05:08:03.209 INFO:teuthology.orchestra.run.smithi191.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 963 kB/s | 56 kB 00:00 2023-10-09T05:08:03.293 INFO:teuthology.orchestra.run.smithi191.stdout:(8/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 13 MB/s | 2.9 MB 00:00 2023-10-09T05:08:03.318 INFO:teuthology.orchestra.run.smithi191.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 471 kB/s | 51 kB 00:00 2023-10-09T05:08:03.418 INFO:teuthology.orchestra.run.smithi191.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 3.1 MB/s | 322 kB 00:00 2023-10-09T05:08:03.455 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:08:03.477 INFO:teuthology.orchestra.run.smithi191.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 702 kB/s | 41 kB 00:00 2023-10-09T05:08:03.556 INFO:teuthology.orchestra.run.smithi008.stdout:========================================================================================================================== 2023-10-09T05:08:03.556 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repository Size 2023-10-09T05:08:03.556 INFO:teuthology.orchestra.run.smithi008.stdout:========================================================================================================================== 2023-10-09T05:08:03.556 INFO:teuthology.orchestra.run.smithi008.stdout:Installing group/module packages: 2023-10-09T05:08:03.557 INFO:teuthology.orchestra.run.smithi008.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-09T05:08:03.557 INFO:teuthology.orchestra.run.smithi008.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-09T05:08:03.557 INFO:teuthology.orchestra.run.smithi008.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-09T05:08:03.557 INFO:teuthology.orchestra.run.smithi008.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-09T05:08:03.557 INFO:teuthology.orchestra.run.smithi008.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T05:08:03.557 INFO:teuthology.orchestra.run.smithi008.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:08:03.557 INFO:teuthology.orchestra.run.smithi008.stdout:Installing dependencies: 2023-10-09T05:08:03.557 INFO:teuthology.orchestra.run.smithi008.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T05:08:03.557 INFO:teuthology.orchestra.run.smithi008.stdout:Downgrading: 2023-10-09T05:08:03.557 INFO:teuthology.orchestra.run.smithi008.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:08:03.557 INFO:teuthology.orchestra.run.smithi008.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-09T05:08:03.558 INFO:teuthology.orchestra.run.smithi008.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-09T05:08:03.558 INFO:teuthology.orchestra.run.smithi008.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-09T05:08:03.558 INFO:teuthology.orchestra.run.smithi008.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-09T05:08:03.558 INFO:teuthology.orchestra.run.smithi008.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-09T05:08:03.558 INFO:teuthology.orchestra.run.smithi008.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-09T05:08:03.558 INFO:teuthology.orchestra.run.smithi008.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-09T05:08:03.558 INFO:teuthology.orchestra.run.smithi008.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-09T05:08:03.558 INFO:teuthology.orchestra.run.smithi008.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-09T05:08:03.558 INFO:teuthology.orchestra.run.smithi008.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-09T05:08:03.558 INFO:teuthology.orchestra.run.smithi008.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:08:03.558 INFO:teuthology.orchestra.run.smithi008.stdout:Installing module profiles: 2023-10-09T05:08:03.559 INFO:teuthology.orchestra.run.smithi008.stdout: container-tools/common 2023-10-09T05:08:03.559 INFO:teuthology.orchestra.run.smithi008.stdout:Enabling module streams: 2023-10-09T05:08:03.559 INFO:teuthology.orchestra.run.smithi008.stdout: container-tools 3.0 2023-10-09T05:08:03.559 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:08:03.559 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:08:03.559 INFO:teuthology.orchestra.run.smithi008.stdout:========================================================================================================================== 2023-10-09T05:08:03.559 INFO:teuthology.orchestra.run.smithi008.stdout:Install 7 Packages 2023-10-09T05:08:03.559 INFO:teuthology.orchestra.run.smithi008.stdout:Downgrade 12 Packages 2023-10-09T05:08:03.559 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:08:03.560 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 51 M 2023-10-09T05:08:03.560 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:08:03.602 INFO:teuthology.orchestra.run.smithi191.stdout:(12/19): podman-3.0.1-9.module+el8.6.0+14874+64 23 MB/s | 11 MB 00:00 2023-10-09T05:08:03.660 INFO:teuthology.orchestra.run.smithi008.stdout:(1/19): container-selinux-2.178.0-2.module+el8. 534 kB/s | 53 kB 00:00 2023-10-09T05:08:03.669 INFO:teuthology.orchestra.run.smithi191.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 11 MB/s | 2.2 MB 00:00 2023-10-09T05:08:03.685 INFO:teuthology.orchestra.run.smithi008.stdout:(2/19): containers-common-1.2.4-1.module+el8.6. 732 kB/s | 91 kB 00:00 2023-10-09T05:08:03.710 INFO:teuthology.orchestra.run.smithi008.stdout:(3/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 343 kB/s | 51 kB 00:00 2023-10-09T05:08:03.802 INFO:teuthology.orchestra.run.smithi008.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 3.5 MB/s | 511 kB 00:00 2023-10-09T05:08:03.827 INFO:teuthology.orchestra.run.smithi008.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 486 kB/s | 69 kB 00:00 2023-10-09T05:08:03.878 INFO:teuthology.orchestra.run.smithi008.stdout:(6/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 17 MB/s | 2.9 MB 00:00 2023-10-09T05:08:03.886 INFO:teuthology.orchestra.run.smithi191.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 23 MB/s | 6.4 MB 00:00 2023-10-09T05:08:03.920 INFO:teuthology.orchestra.run.smithi008.stdout:(7/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 614 kB/s | 72 kB 00:00 2023-10-09T05:08:03.978 INFO:teuthology.orchestra.run.smithi191.stdout:(15/19): buildah-1.19.9-3.module+el8.6.0+14874+ 22 MB/s | 6.8 MB 00:00 2023-10-09T05:08:04.128 INFO:teuthology.orchestra.run.smithi008.stdout:(8/19): podman-3.0.1-9.module+el8.6.0+14874+644 36 MB/s | 11 MB 00:00 2023-10-09T05:08:04.153 INFO:teuthology.orchestra.run.smithi008.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 218 kB/s | 51 kB 00:00 2023-10-09T05:08:04.178 INFO:teuthology.orchestra.run.smithi191.stdout:(16/19): containernetworking-plugins-0.9.1-1.mo 21 MB/s | 19 MB 00:00 2023-10-09T05:08:04.179 INFO:teuthology.orchestra.run.smithi008.stdout:(10/19): podman-docker-3.0.1-9.module+el8.6.0+1 186 kB/s | 56 kB 00:00 2023-10-09T05:08:04.203 INFO:teuthology.orchestra.run.smithi191.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 822 kB/s | 185 kB 00:00 2023-10-09T05:08:04.271 INFO:teuthology.orchestra.run.smithi008.stdout:(11/19): podman-catatonit-3.0.1-9.module+el8.6. 2.7 MB/s | 322 kB 00:00 2023-10-09T05:08:04.273 INFO:teuthology.orchestra.run.smithi191.stdout:(18/19): cockpit-podman-29-2.module+el8.6.0+148 2.8 MB/s | 1.1 MB 00:00 2023-10-09T05:08:04.421 INFO:teuthology.orchestra.run.smithi008.stdout:(12/19): yajl-2.1.0-10.el8.x86_64.rpm 168 kB/s | 41 kB 00:00 2023-10-09T05:08:04.445 INFO:teuthology.orchestra.run.smithi191.stdout:(19/19): udica-0.2.4-1.module+el8.6.0+14874+644 189 kB/s | 51 kB 00:00 2023-10-09T05:08:04.446 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:08:04.447 INFO:teuthology.orchestra.run.smithi191.stdout:Total 33 MB/s | 51 MB 00:01 2023-10-09T05:08:04.496 INFO:teuthology.orchestra.run.smithi008.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 9.7 MB/s | 2.2 MB 00:00 2023-10-09T05:08:04.682 INFO:teuthology.orchestra.run.smithi008.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 25 MB/s | 6.4 MB 00:00 2023-10-09T05:08:04.747 INFO:teuthology.orchestra.run.smithi008.stdout:(15/19): cockpit-podman-29-2.module+el8.6.0+148 16 MB/s | 1.1 MB 00:00 2023-10-09T05:08:04.831 INFO:teuthology.orchestra.run.smithi008.stdout:(16/19): buildah-1.19.9-3.module+el8.6.0+14874+ 20 MB/s | 6.8 MB 00:00 2023-10-09T05:08:04.856 INFO:teuthology.orchestra.run.smithi008.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 1.7 MB/s | 185 kB 00:00 2023-10-09T05:08:04.881 INFO:teuthology.orchestra.run.smithi008.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.0 MB/s | 51 kB 00:00 2023-10-09T05:08:04.928 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:08:04.970 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:08:04.971 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:08:05.073 INFO:teuthology.orchestra.run.smithi008.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 20 MB/s | 19 MB 00:00 2023-10-09T05:08:05.074 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:08:05.074 INFO:teuthology.orchestra.run.smithi008.stdout:Total 33 MB/s | 51 MB 00:01 2023-10-09T05:08:05.392 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:08:05.395 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:08:05.569 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:08:05.615 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:08:05.615 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:08:06.069 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:08:06.073 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:08:06.682 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:08:06.737 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-09T05:08:06.948 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:08:06.957 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:08:06.978 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:08:07.189 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:08:07.259 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:08:07.314 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-09T05:08:07.455 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:08:07.465 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:08:07.486 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:08:07.672 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:08:24.444 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:08:25.641 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:08:26.260 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-09T05:08:26.555 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-09T05:08:26.845 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-09T05:08:27.025 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-09T05:08:27.270 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-09T05:08:27.270 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-09T05:08:27.270 INFO:teuthology.orchestra.run.smithi191.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-09T05:08:27.270 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:08:27.543 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-09T05:08:27.800 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-09T05:08:27.997 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-09T05:08:28.148 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-09T05:08:28.176 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-09T05:08:28.352 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-09T05:08:28.397 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-09T05:08:28.397 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-09T05:08:28.397 INFO:teuthology.orchestra.run.smithi008.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-09T05:08:28.397 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:08:29.177 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-09T05:08:29.421 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-09T05:08:29.882 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-09T05:08:30.264 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-09T05:08:30.676 INFO:teuthology.orchestra.run.smithi191.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-09T05:08:30.967 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-09T05:08:30.967 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-09T05:08:31.708 INFO:teuthology.orchestra.run.smithi008.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-09T05:08:31.727 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-09T05:08:31.982 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-09T05:08:32.455 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-09T05:08:32.693 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-09T05:08:32.751 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-09T05:08:32.925 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-09T05:08:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-09T05:08:33.282 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-09T05:08:33.282 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:08:33.508 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-09T05:08:33.627 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:08:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-09T05:08:33.826 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-09T05:08:33.826 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:08:33.943 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-09T05:08:34.002 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:08:34.152 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-09T05:08:34.168 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:08:34.260 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-09T05:08:34.260 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:08:34.377 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-09T05:08:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-09T05:08:34.617 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:08:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-09T05:08:34.833 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-09T05:08:34.833 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:08:34.868 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-09T05:08:34.993 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-09T05:08:35.012 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:08:35.161 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:08:35.170 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-09T05:08:35.171 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:08:35.307 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-09T05:08:35.376 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:08:35.453 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-09T05:08:35.632 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-09T05:08:35.753 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-09T05:08:35.861 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-09T05:08:35.918 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-09T05:08:36.052 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-09T05:08:36.053 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:08:36.227 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:08:36.328 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:08:36.328 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-09T05:08:36.328 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-09T05:08:36.328 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-09T05:08:36.328 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-09T05:08:36.328 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-09T05:08:36.328 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-09T05:08:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-09T05:08:36.330 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-09T05:08:36.330 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-09T05:08:36.330 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-09T05:08:36.330 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-09T05:08:36.330 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-09T05:08:36.330 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:08:36.330 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-09T05:08:36.330 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-09T05:08:36.330 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-09T05:08:36.330 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-09T05:08:36.331 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-09T05:08:36.331 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-09T05:08:36.772 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-09T05:08:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:08:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-09T05:08:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-09T05:08:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-09T05:08:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-09T05:08:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-09T05:08:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-09T05:08:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-09T05:08:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-09T05:08:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-09T05:08:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-09T05:08:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-09T05:08:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-09T05:08:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-09T05:08:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-09T05:08:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-09T05:08:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-09T05:08:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-09T05:08:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-09T05:08:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-09T05:08:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-09T05:08:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-09T05:08:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-09T05:08:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-09T05:08:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:08:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-09T05:08:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-09T05:08:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-09T05:08:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-09T05:08:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-09T05:08:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-09T05:08:37.792 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-09T05:08:37.792 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:24 ago on Mon 09 Oct 2023 05:06:13 AM UTC. 2023-10-09T05:08:38.642 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-09T05:08:38.642 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:02:09 ago on Mon 09 Oct 2023 05:06:29 AM UTC. 2023-10-09T05:08:40.991 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:08:41.184 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:08:41.184 INFO:teuthology.orchestra.run.smithi191.stdout:Downgraded: 2023-10-09T05:08:41.184 INFO:teuthology.orchestra.run.smithi191.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.185 INFO:teuthology.orchestra.run.smithi191.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:08:41.185 INFO:teuthology.orchestra.run.smithi191.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.185 INFO:teuthology.orchestra.run.smithi191.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.185 INFO:teuthology.orchestra.run.smithi191.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.185 INFO:teuthology.orchestra.run.smithi191.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.185 INFO:teuthology.orchestra.run.smithi191.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.185 INFO:teuthology.orchestra.run.smithi191.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.185 INFO:teuthology.orchestra.run.smithi191.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.185 INFO:teuthology.orchestra.run.smithi191.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:08:41.185 INFO:teuthology.orchestra.run.smithi191.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.185 INFO:teuthology.orchestra.run.smithi191.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.186 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:08:41.186 INFO:teuthology.orchestra.run.smithi191.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.186 INFO:teuthology.orchestra.run.smithi191.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:08:41.186 INFO:teuthology.orchestra.run.smithi191.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.186 INFO:teuthology.orchestra.run.smithi191.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.186 INFO:teuthology.orchestra.run.smithi191.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:41.186 INFO:teuthology.orchestra.run.smithi191.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:08:41.186 INFO:teuthology.orchestra.run.smithi191.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T05:08:41.187 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:08:41.187 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:08:41.371 DEBUG:teuthology.parallel:result is None 2023-10-09T05:08:41.803 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:08:42.022 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:08:42.022 INFO:teuthology.orchestra.run.smithi008.stdout:Downgraded: 2023-10-09T05:08:42.022 INFO:teuthology.orchestra.run.smithi008.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.022 INFO:teuthology.orchestra.run.smithi008.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:08:42.022 INFO:teuthology.orchestra.run.smithi008.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.022 INFO:teuthology.orchestra.run.smithi008.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:08:42.023 INFO:teuthology.orchestra.run.smithi008.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.024 INFO:teuthology.orchestra.run.smithi008.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.024 INFO:teuthology.orchestra.run.smithi008.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:08:42.024 INFO:teuthology.orchestra.run.smithi008.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:08:42.024 INFO:teuthology.orchestra.run.smithi008.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T05:08:42.024 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:08:42.024 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:08:42.230 DEBUG:teuthology.parallel:result is None 2023-10-09T05:08:42.230 INFO:teuthology.run_tasks:Running task install... 2023-10-09T05:08:42.240 DEBUG:teuthology.task.install:project ceph 2023-10-09T05:08:42.240 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T05:08:42.240 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T05:08:42.240 INFO:teuthology.task.install:Using flavor: default 2023-10-09T05:08:42.247 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T05:08:42.247 INFO:teuthology.task.install:extra packages: [] 2023-10-09T05:08:42.247 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T05:08:42.247 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:08:42.248 INFO:teuthology.packaging:ref: None 2023-10-09T05:08:42.248 INFO:teuthology.packaging:tag: None 2023-10-09T05:08:42.248 INFO:teuthology.packaging:branch: pacific 2023-10-09T05:08:42.248 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:08:42.248 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-09T05:08:42.250 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T05:08:42.250 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:08:42.250 INFO:teuthology.packaging:ref: None 2023-10-09T05:08:42.250 INFO:teuthology.packaging:tag: None 2023-10-09T05:08:42.250 INFO:teuthology.packaging:branch: pacific 2023-10-09T05:08:42.251 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:08:42.251 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-09T05:08:42.476 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/ 2023-10-09T05:08:42.476 INFO:teuthology.task.install.rpm:Package version is 16.2.14-48.gaeb2e425 2023-10-09T05:08:42.554 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/ 2023-10-09T05:08:42.554 INFO:teuthology.task.install.rpm:Package version is 16.2.14-48.gaeb2e425 2023-10-09T05:08:42.650 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T05:08:42.650 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:08:42.650 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T05:08:42.659 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T05:08:42.659 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T05:08:42.659 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T05:08:42.680 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T05:08:42.680 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:08:42.680 INFO:teuthology.packaging:ref: None 2023-10-09T05:08:42.680 INFO:teuthology.packaging:tag: None 2023-10-09T05:08:42.680 INFO:teuthology.packaging:branch: pacific 2023-10-09T05:08:42.680 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:08:42.681 DEBUG:teuthology.orchestra.run.smithi008:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T05:08:42.692 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T05:08:42.692 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:08:42.692 INFO:teuthology.packaging:ref: None 2023-10-09T05:08:42.692 INFO:teuthology.packaging:tag: None 2023-10-09T05:08:42.692 INFO:teuthology.packaging:branch: pacific 2023-10-09T05:08:42.692 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:08:42.692 DEBUG:teuthology.orchestra.run.smithi191:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T05:08:42.750 DEBUG:teuthology.orchestra.run.smithi008:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T05:08:42.764 DEBUG:teuthology.orchestra.run.smithi191:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T05:08:42.828 DEBUG:teuthology.orchestra.run.smithi008:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T05:08:42.845 DEBUG:teuthology.orchestra.run.smithi191:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T05:08:42.896 INFO:teuthology.orchestra.run.smithi008.stdout:check_obsoletes = 1 2023-10-09T05:08:42.897 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum clean all 2023-10-09T05:08:42.912 INFO:teuthology.orchestra.run.smithi191.stdout:check_obsoletes = 1 2023-10-09T05:08:42.914 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum clean all 2023-10-09T05:08:43.286 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:08:43.291 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:08:43.631 INFO:teuthology.orchestra.run.smithi191.stdout:44 files removed 2023-10-09T05:08:43.661 INFO:teuthology.orchestra.run.smithi008.stdout:44 files removed 2023-10-09T05:08:43.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-radosgw 2023-10-09T05:08:43.708 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install ceph-radosgw 2023-10-09T05:08:44.011 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:08:44.059 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:08:44.844 INFO:teuthology.orchestra.run.smithi191.stdout:ceph packages for x86_64 198 kB/s | 76 kB 00:00 2023-10-09T05:08:45.113 INFO:teuthology.orchestra.run.smithi008.stdout:ceph packages for x86_64 137 kB/s | 76 kB 00:00 2023-10-09T05:08:45.142 INFO:teuthology.orchestra.run.smithi191.stdout:ceph noarch packages 57 kB/s | 15 kB 00:00 2023-10-09T05:08:45.363 INFO:teuthology.orchestra.run.smithi008.stdout:ceph noarch packages 71 kB/s | 15 kB 00:00 2023-10-09T05:08:45.550 INFO:teuthology.orchestra.run.smithi191.stdout:ceph source packages 3.7 kB/s | 1.4 kB 00:00 2023-10-09T05:08:45.656 INFO:teuthology.orchestra.run.smithi008.stdout:ceph source packages 5.4 kB/s | 1.4 kB 00:00 2023-10-09T05:08:45.718 INFO:teuthology.orchestra.run.smithi191.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-10-09T05:08:45.798 INFO:teuthology.orchestra.run.smithi008.stdout:Copr repo for python3-asyncssh owned by ceph 29 kB/s | 3.5 kB 00:00 2023-10-09T05:08:46.970 INFO:teuthology.orchestra.run.smithi008.stdout:Extra Packages for Enterprise Linux 14 MB/s | 16 MB 00:01 2023-10-09T05:08:47.138 INFO:teuthology.orchestra.run.smithi191.stdout:Extra Packages for Enterprise Linux 11 MB/s | 16 MB 00:01 2023-10-09T05:08:50.643 INFO:teuthology.orchestra.run.smithi008.stdout:lab-extras 409 kB/s | 24 kB 00:00 2023-10-09T05:08:50.794 INFO:teuthology.orchestra.run.smithi191.stdout:lab-extras 378 kB/s | 24 kB 00:00 2023-10-09T05:08:51.694 INFO:teuthology.orchestra.run.smithi008.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 54 MB/s | 53 MB 00:00 2023-10-09T05:08:51.795 INFO:teuthology.orchestra.run.smithi191.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 57 MB/s | 53 MB 00:00 2023-10-09T05:09:00.347 INFO:teuthology.orchestra.run.smithi191.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 38 MB/s | 47 MB 00:01 2023-10-09T05:09:00.462 INFO:teuthology.orchestra.run.smithi008.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 53 MB/s | 47 MB 00:00 2023-10-09T05:09:11.249 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:09:11.252 INFO:teuthology.orchestra.run.smithi191.stdout:=================================================================================================== 2023-10-09T05:09:11.252 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-10-09T05:09:11.252 INFO:teuthology.orchestra.run.smithi191.stdout:=================================================================================================== 2023-10-09T05:09:11.252 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:09:11.252 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-radosgw x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 12 M 2023-10-09T05:09:11.252 INFO:teuthology.orchestra.run.smithi191.stdout:Upgrading: 2023-10-09T05:09:11.252 INFO:teuthology.orchestra.run.smithi191.stdout: librados2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.7 M 2023-10-09T05:09:11.252 INFO:teuthology.orchestra.run.smithi191.stdout: librbd1 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 4.2 M 2023-10-09T05:09:11.253 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-10-09T05:09:11.253 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-base x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 6.1 M 2023-10-09T05:09:11.253 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-common x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 24 M 2023-10-09T05:09:11.253 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-selinux x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 26 k 2023-10-09T05:09:11.253 INFO:teuthology.orchestra.run.smithi191.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T05:09:11.253 INFO:teuthology.orchestra.run.smithi191.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-10-09T05:09:11.253 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 824 k 2023-10-09T05:09:11.253 INFO:teuthology.orchestra.run.smithi191.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T05:09:11.253 INFO:teuthology.orchestra.run.smithi191.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T05:09:11.253 INFO:teuthology.orchestra.run.smithi191.stdout: libradosstriper1 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 493 k 2023-10-09T05:09:11.253 INFO:teuthology.orchestra.run.smithi191.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T05:09:11.254 INFO:teuthology.orchestra.run.smithi191.stdout: librgw2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.8 M 2023-10-09T05:09:11.254 INFO:teuthology.orchestra.run.smithi191.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T05:09:11.254 INFO:teuthology.orchestra.run.smithi191.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T05:09:11.254 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-argparse x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 45 k 2023-10-09T05:09:11.254 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-common x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 99 k 2023-10-09T05:09:11.254 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cephfs x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 215 k 2023-10-09T05:09:11.254 INFO:teuthology.orchestra.run.smithi191.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T05:09:11.254 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rados x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 390 k 2023-10-09T05:09:11.254 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rbd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 369 k 2023-10-09T05:09:11.254 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rgw x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 114 k 2023-10-09T05:09:11.254 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:09:11.255 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:09:11.255 INFO:teuthology.orchestra.run.smithi191.stdout:=================================================================================================== 2023-10-09T05:09:11.255 INFO:teuthology.orchestra.run.smithi191.stdout:Install 21 Packages 2023-10-09T05:09:11.255 INFO:teuthology.orchestra.run.smithi191.stdout:Upgrade 2 Packages 2023-10-09T05:09:11.255 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:09:11.255 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 57 M 2023-10-09T05:09:11.255 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:09:12.249 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:09:12.252 INFO:teuthology.orchestra.run.smithi008.stdout:=================================================================================================== 2023-10-09T05:09:12.252 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repository Size 2023-10-09T05:09:12.252 INFO:teuthology.orchestra.run.smithi008.stdout:=================================================================================================== 2023-10-09T05:09:12.252 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:09:12.252 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-radosgw x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 12 M 2023-10-09T05:09:12.252 INFO:teuthology.orchestra.run.smithi008.stdout:Upgrading: 2023-10-09T05:09:12.252 INFO:teuthology.orchestra.run.smithi008.stdout: librados2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.7 M 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout: librbd1 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 4.2 M 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout:Installing dependencies: 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-base x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 6.1 M 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-common x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 24 M 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-selinux x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 26 k 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout: libcephfs2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 824 k 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout: libradosstriper1 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 493 k 2023-10-09T05:09:12.253 INFO:teuthology.orchestra.run.smithi008.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout: librgw2 x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.8 M 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout: python3-ceph-argparse x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 45 k 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout: python3-ceph-common x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 99 k 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cephfs x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 215 k 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout: python3-rados x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 390 k 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout: python3-rbd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 369 k 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout: python3-rgw x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 114 k 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:09:12.254 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:09:12.255 INFO:teuthology.orchestra.run.smithi008.stdout:=================================================================================================== 2023-10-09T05:09:12.255 INFO:teuthology.orchestra.run.smithi008.stdout:Install 21 Packages 2023-10-09T05:09:12.255 INFO:teuthology.orchestra.run.smithi008.stdout:Upgrade 2 Packages 2023-10-09T05:09:12.255 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:09:12.255 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 57 M 2023-10-09T05:09:12.255 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:09:12.322 INFO:teuthology.orchestra.run.smithi191.stdout:(1/23): ceph-base-16.2.14-48.gaeb2e425.el8.x86_ 8.5 MB/s | 6.1 MB 00:00 2023-10-09T05:09:12.389 INFO:teuthology.orchestra.run.smithi191.stdout:(2/23): ceph-selinux-16.2.14-48.gaeb2e425.el8.x 384 kB/s | 26 kB 00:00 2023-10-09T05:09:12.589 INFO:teuthology.orchestra.run.smithi191.stdout:(3/23): ceph-radosgw-16.2.14-48.gaeb2e425.el8.x 12 MB/s | 12 MB 00:00 2023-10-09T05:09:12.614 INFO:teuthology.orchestra.run.smithi191.stdout:(4/23): libcephfs2-16.2.14-48.gaeb2e425.el8.x86 3.6 MB/s | 824 kB 00:00 2023-10-09T05:09:12.648 INFO:teuthology.orchestra.run.smithi191.stdout:(5/23): libradosstriper1-16.2.14-48.gaeb2e425.e 8.2 MB/s | 493 kB 00:00 2023-10-09T05:09:12.690 INFO:teuthology.orchestra.run.smithi191.stdout:(6/23): python3-ceph-argparse-16.2.14-48.gaeb2e 1.1 MB/s | 45 kB 00:00 2023-10-09T05:09:12.765 INFO:teuthology.orchestra.run.smithi191.stdout:(7/23): python3-ceph-common-16.2.14-48.gaeb2e42 1.3 MB/s | 99 kB 00:00 2023-10-09T05:09:12.849 INFO:teuthology.orchestra.run.smithi191.stdout:(8/23): librgw2-16.2.14-48.gaeb2e425.el8.x86_64 16 MB/s | 3.8 MB 00:00 2023-10-09T05:09:12.874 INFO:teuthology.orchestra.run.smithi191.stdout:(9/23): python3-cephfs-16.2.14-48.gaeb2e425.el8 1.9 MB/s | 215 kB 00:00 2023-10-09T05:09:12.908 INFO:teuthology.orchestra.run.smithi191.stdout:(10/23): python3-rados-16.2.14-48.gaeb2e425.el8 6.5 MB/s | 390 kB 00:00 2023-10-09T05:09:12.933 INFO:teuthology.orchestra.run.smithi191.stdout:(11/23): python3-rbd-16.2.14-48.gaeb2e425.el8.x 6.2 MB/s | 369 kB 00:00 2023-10-09T05:09:12.959 INFO:teuthology.orchestra.run.smithi191.stdout:(12/23): python3-rgw-16.2.14-48.gaeb2e425.el8.x 2.2 MB/s | 114 kB 00:00 2023-10-09T05:09:13.167 INFO:teuthology.orchestra.run.smithi191.stdout:(13/23): leveldb-1.22-1.el8.x86_64.rpm 869 kB/s | 181 kB 00:00 2023-10-09T05:09:13.201 INFO:teuthology.orchestra.run.smithi191.stdout:(14/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.1 MB/s | 306 kB 00:00 2023-10-09T05:09:13.249 INFO:teuthology.orchestra.run.smithi191.stdout:(15/23): liboath-2.6.2-3.el8.x86_64.rpm 717 kB/s | 59 kB 00:00 2023-10-09T05:09:13.285 INFO:teuthology.orchestra.run.smithi191.stdout:(16/23): libunwind-1.3.1-3.el8.x86_64.rpm 899 kB/s | 75 kB 00:00 2023-10-09T05:09:13.352 INFO:teuthology.orchestra.run.smithi191.stdout:(17/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 459 kB/s | 47 kB 00:00 2023-10-09T05:09:13.377 INFO:teuthology.orchestra.run.smithi191.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.8 MB/s | 259 kB 00:00 2023-10-09T05:09:13.402 INFO:teuthology.orchestra.run.smithi191.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 877 kB/s | 44 kB 00:00 2023-10-09T05:09:13.428 INFO:teuthology.orchestra.run.smithi191.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 6.9 MB/s | 354 kB 00:00 2023-10-09T05:09:13.538 INFO:teuthology.orchestra.run.smithi008.stdout:(1/23): ceph-base-16.2.14-48.gaeb2e425.el8.x86_ 6.5 MB/s | 6.1 MB 00:00 2023-10-09T05:09:13.655 INFO:teuthology.orchestra.run.smithi008.stdout:(2/23): ceph-selinux-16.2.14-48.gaeb2e425.el8.x 220 kB/s | 26 kB 00:00 2023-10-09T05:09:14.005 INFO:teuthology.orchestra.run.smithi191.stdout:(21/23): librados2-16.2.14-48.gaeb2e425.el8.x86 6.2 MB/s | 3.7 MB 00:00 2023-10-09T05:09:14.170 INFO:teuthology.orchestra.run.smithi191.stdout:(22/23): librbd1-16.2.14-48.gaeb2e425.el8.x86_6 5.6 MB/s | 4.2 MB 00:00 2023-10-09T05:09:14.331 INFO:teuthology.orchestra.run.smithi008.stdout:(3/23): ceph-radosgw-16.2.14-48.gaeb2e425.el8.x 6.7 MB/s | 12 MB 00:01 2023-10-09T05:09:14.364 INFO:teuthology.orchestra.run.smithi008.stdout:(4/23): libcephfs2-16.2.14-48.gaeb2e425.el8.x86 1.1 MB/s | 824 kB 00:00 2023-10-09T05:09:14.448 INFO:teuthology.orchestra.run.smithi008.stdout:(5/23): libradosstriper1-16.2.14-48.gaeb2e425.e 4.1 MB/s | 493 kB 00:00 2023-10-09T05:09:14.523 INFO:teuthology.orchestra.run.smithi008.stdout:(6/23): python3-ceph-argparse-16.2.14-48.gaeb2e 602 kB/s | 45 kB 00:00 2023-10-09T05:09:14.598 INFO:teuthology.orchestra.run.smithi008.stdout:(7/23): python3-ceph-common-16.2.14-48.gaeb2e42 1.3 MB/s | 99 kB 00:00 2023-10-09T05:09:14.674 INFO:teuthology.orchestra.run.smithi008.stdout:(8/23): python3-cephfs-16.2.14-48.gaeb2e425.el8 2.8 MB/s | 215 kB 00:00 2023-10-09T05:09:14.724 INFO:teuthology.orchestra.run.smithi008.stdout:(9/23): librgw2-16.2.14-48.gaeb2e425.el8.x86_64 11 MB/s | 3.8 MB 00:00 2023-10-09T05:09:14.749 INFO:teuthology.orchestra.run.smithi008.stdout:(10/23): python3-rados-16.2.14-48.gaeb2e425.el8 5.1 MB/s | 390 kB 00:00 2023-10-09T05:09:14.791 INFO:teuthology.orchestra.run.smithi008.stdout:(11/23): python3-rbd-16.2.14-48.gaeb2e425.el8.x 5.4 MB/s | 369 kB 00:00 2023-10-09T05:09:14.817 INFO:teuthology.orchestra.run.smithi008.stdout:(12/23): python3-rgw-16.2.14-48.gaeb2e425.el8.x 1.7 MB/s | 114 kB 00:00 2023-10-09T05:09:15.034 INFO:teuthology.orchestra.run.smithi008.stdout:(13/23): leveldb-1.22-1.el8.x86_64.rpm 835 kB/s | 181 kB 00:00 2023-10-09T05:09:15.067 INFO:teuthology.orchestra.run.smithi008.stdout:(14/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.1 MB/s | 306 kB 00:00 2023-10-09T05:09:15.092 INFO:teuthology.orchestra.run.smithi008.stdout:(15/23): liboath-2.6.2-3.el8.x86_64.rpm 1.0 MB/s | 59 kB 00:00 2023-10-09T05:09:15.134 INFO:teuthology.orchestra.run.smithi008.stdout:(16/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.1 MB/s | 75 kB 00:00 2023-10-09T05:09:15.151 INFO:teuthology.orchestra.run.smithi008.stdout:(17/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 798 kB/s | 47 kB 00:00 2023-10-09T05:09:15.219 INFO:teuthology.orchestra.run.smithi008.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 757 kB/s | 44 kB 00:00 2023-10-09T05:09:15.244 INFO:teuthology.orchestra.run.smithi008.stdout:(19/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.3 MB/s | 259 kB 00:00 2023-10-09T05:09:15.278 INFO:teuthology.orchestra.run.smithi008.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.9 MB/s | 354 kB 00:00 2023-10-09T05:09:15.686 INFO:teuthology.orchestra.run.smithi008.stdout:(21/23): librados2-16.2.14-48.gaeb2e425.el8.x86 8.5 MB/s | 3.7 MB 00:00 2023-10-09T05:09:15.878 INFO:teuthology.orchestra.run.smithi008.stdout:(22/23): librbd1-16.2.14-48.gaeb2e425.el8.x86_6 7.0 MB/s | 4.2 MB 00:00 2023-10-09T05:09:16.387 INFO:teuthology.orchestra.run.smithi191.stdout:(23/23): ceph-common-16.2.14-48.gaeb2e425.el8.x 5.1 MB/s | 24 MB 00:04 2023-10-09T05:09:16.389 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:09:16.390 INFO:teuthology.orchestra.run.smithi191.stdout:Total 11 MB/s | 57 MB 00:05 2023-10-09T05:09:16.436 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:09:16.482 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:09:16.482 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:09:16.810 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:09:16.813 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:09:17.886 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:09:17.987 INFO:teuthology.orchestra.run.smithi008.stdout:(23/23): ceph-common-16.2.14-48.gaeb2e425.el8.x 4.5 MB/s | 24 MB 00:05 2023-10-09T05:09:17.989 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:09:17.990 INFO:teuthology.orchestra.run.smithi008.stdout:Total 10 MB/s | 57 MB 00:05 2023-10-09T05:09:18.038 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:09:18.089 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:09:18.089 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:09:18.126 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:09:18.430 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:09:18.434 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:09:18.493 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:09:18.693 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T05:09:18.786 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T05:09:18.957 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 3/25 2023-10-09T05:09:19.232 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T05:09:19.442 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T05:09:19.472 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:09:19.655 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T05:09:20.075 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:09:20.109 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T05:09:20.208 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T05:09:20.254 INFO:teuthology.orchestra.run.smithi008.stdout: Upgrading : librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T05:09:20.358 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T05:09:20.401 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T05:09:20.469 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T05:09:20.554 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 3/25 2023-10-09T05:09:20.765 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T05:09:20.952 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-10-09T05:09:20.955 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T05:09:21.383 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T05:09:21.401 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 9/25 2023-10-09T05:09:21.467 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 9/25 2023-10-09T05:09:21.563 INFO:teuthology.orchestra.run.smithi008.stdout: Upgrading : librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T05:09:21.665 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-10-09T05:09:21.680 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T05:09:21.782 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x 11/25 2023-10-09T05:09:21.829 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T05:09:21.895 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T05:09:21.962 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 12/25 2023-10-09T05:09:22.210 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 13/25 2023-10-09T05:09:22.380 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-10-09T05:09:22.392 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 14/25 2023-10-09T05:09:22.542 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 15/25 2023-10-09T05:09:22.611 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 15/25 2023-10-09T05:09:22.704 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 9/25 2023-10-09T05:09:22.777 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T05:09:22.785 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 9/25 2023-10-09T05:09:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-10-09T05:09:22.996 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T05:09:23.129 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x 11/25 2023-10-09T05:09:23.255 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T05:09:23.283 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 12/25 2023-10-09T05:09:23.426 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86 19/25 2023-10-09T05:09:23.508 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 13/25 2023-10-09T05:09:23.749 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 14/25 2023-10-09T05:09:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 15/25 2023-10-09T05:09:23.963 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 15/25 2023-10-09T05:09:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T05:09:24.452 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T05:09:24.671 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T05:09:24.821 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86 19/25 2023-10-09T05:09:27.105 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T05:09:27.391 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T05:09:28.093 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T05:09:28.457 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 21/25 2023-10-09T05:09:28.539 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 21/25 2023-10-09T05:09:28.835 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T05:09:28.835 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T05:09:28.937 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T05:09:29.684 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 20/25 2023-10-09T05:09:30.108 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 21/25 2023-10-09T05:09:30.191 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 21/25 2023-10-09T05:09:30.303 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T05:09:47.275 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T05:09:47.276 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /sys 2023-10-09T05:09:47.276 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /proc 2023-10-09T05:09:47.276 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /dev 2023-10-09T05:09:47.276 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /run 2023-10-09T05:09:47.276 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /mnt 2023-10-09T05:09:47.276 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /var/tmp 2023-10-09T05:09:47.276 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /home 2023-10-09T05:09:47.276 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /tmp 2023-10-09T05:09:47.276 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /dev 2023-10-09T05:09:47.277 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:09:48.470 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 23/25 2023-10-09T05:09:48.580 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 23/25 2023-10-09T05:09:48.580 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:09:48.831 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:09:48.832 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:09:49.227 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T05:09:49.227 INFO:teuthology.orchestra.run.smithi008.stdout:skipping the directory /sys 2023-10-09T05:09:49.227 INFO:teuthology.orchestra.run.smithi008.stdout:skipping the directory /proc 2023-10-09T05:09:49.227 INFO:teuthology.orchestra.run.smithi008.stdout:skipping the directory /dev 2023-10-09T05:09:49.227 INFO:teuthology.orchestra.run.smithi008.stdout:skipping the directory /run 2023-10-09T05:09:49.227 INFO:teuthology.orchestra.run.smithi008.stdout:skipping the directory /mnt 2023-10-09T05:09:49.227 INFO:teuthology.orchestra.run.smithi008.stdout:skipping the directory /var/tmp 2023-10-09T05:09:49.228 INFO:teuthology.orchestra.run.smithi008.stdout:skipping the directory /home 2023-10-09T05:09:49.228 INFO:teuthology.orchestra.run.smithi008.stdout:skipping the directory /tmp 2023-10-09T05:09:49.228 INFO:teuthology.orchestra.run.smithi008.stdout:skipping the directory /dev 2023-10-09T05:09:49.228 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:09:49.461 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:09:49.461 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 1/25 2023-10-09T05:09:49.461 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T05:09:49.461 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 3/25 2023-10-09T05:09:49.461 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 4/25 2023-10-09T05:09:49.461 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 7/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x 8/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86 9/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 10/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 11/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 12/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 13/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-10-09T05:09:49.462 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T05:09:49.463 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-10-09T05:09:49.463 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-10-09T05:09:49.463 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-10-09T05:09:49.463 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-10-09T05:09:49.463 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T05:09:49.463 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T05:09:49.463 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 24/25 2023-10-09T05:09:50.431 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 23/25 2023-10-09T05:09:50.515 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 23/25 2023-10-09T05:09:50.515 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:09:50.711 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:09:50.712 INFO:teuthology.orchestra.run.smithi008.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:09:50.900 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:09:50.900 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:51 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:09:51.364 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:09:51.364 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 1/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 3/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 4/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 5/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 6/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 7/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x 8/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86 9/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 10/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 11/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 12/25 2023-10-09T05:09:51.365 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 13/25 2023-10-09T05:09:51.366 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T05:09:51.366 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-10-09T05:09:51.366 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-10-09T05:09:51.366 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T05:09:51.366 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-10-09T05:09:51.366 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-10-09T05:09:51.366 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-10-09T05:09:51.366 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-10-09T05:09:51.366 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 22/25 2023-10-09T05:09:51.366 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T05:09:51.367 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 24/25 2023-10-09T05:09:52.749 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:09:52.750 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:00:53 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:09:53.206 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:09:53.390 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:09:53.390 INFO:teuthology.orchestra.run.smithi191.stdout:Upgraded: 2023-10-09T05:09:53.391 INFO:teuthology.orchestra.run.smithi191.stdout: librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.391 INFO:teuthology.orchestra.run.smithi191.stdout: librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.391 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:09:53.391 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.391 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.391 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.391 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.391 INFO:teuthology.orchestra.run.smithi191.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T05:09:53.391 INFO:teuthology.orchestra.run.smithi191.stdout: leveldb-1.22-1.el8.x86_64 2023-10-09T05:09:53.391 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.391 INFO:teuthology.orchestra.run.smithi191.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.392 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.393 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:53.393 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:09:53.393 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:09:53.553 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-test 2023-10-09T05:09:53.900 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:09:54.861 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:55 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:09:55.278 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:09:55.483 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi008.stdout:Upgraded: 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi008.stdout: librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi008.stdout: librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-radosgw-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-selinux-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi008.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi008.stdout: leveldb-1.22-1.el8.x86_64 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi008.stdout: libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi008.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi008.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi008.stdout: libradosstriper1-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi008.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi008.stdout: librgw2-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi008.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi008.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi008.stdout: python3-ceph-argparse-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi008.stdout: python3-ceph-common-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi008.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi008.stdout: python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi008.stdout: python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi008.stdout: python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:09:55.757 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install ceph-test 2023-10-09T05:09:56.129 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:09:56.260 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:09:56.261 INFO:teuthology.orchestra.run.smithi191.stdout:=========================================================================================== 2023-10-09T05:09:56.261 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-10-09T05:09:56.261 INFO:teuthology.orchestra.run.smithi191.stdout:=========================================================================================== 2023-10-09T05:09:56.261 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:09:56.261 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-test x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 52 M 2023-10-09T05:09:56.262 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-10-09T05:09:56.262 INFO:teuthology.orchestra.run.smithi191.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T05:09:56.262 INFO:teuthology.orchestra.run.smithi191.stdout: libcephsqlite x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 174 k 2023-10-09T05:09:56.262 INFO:teuthology.orchestra.run.smithi191.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T05:09:56.262 INFO:teuthology.orchestra.run.smithi191.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T05:09:56.262 INFO:teuthology.orchestra.run.smithi191.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T05:09:56.262 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:09:56.262 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:09:56.262 INFO:teuthology.orchestra.run.smithi191.stdout:=========================================================================================== 2023-10-09T05:09:56.262 INFO:teuthology.orchestra.run.smithi191.stdout:Install 6 Packages 2023-10-09T05:09:56.262 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:09:56.263 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 53 M 2023-10-09T05:09:56.263 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 195 M 2023-10-09T05:09:56.263 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:09:56.296 INFO:teuthology.orchestra.run.smithi191.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.1 MB/s | 70 kB 00:00 2023-10-09T05:09:56.379 INFO:teuthology.orchestra.run.smithi191.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 3.8 MB/s | 323 kB 00:00 2023-10-09T05:09:56.438 INFO:teuthology.orchestra.run.smithi191.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 3.1 MB/s | 187 kB 00:00 2023-10-09T05:09:56.497 INFO:teuthology.orchestra.run.smithi191.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 3.4 MB/s | 202 kB 00:00 2023-10-09T05:09:56.556 INFO:teuthology.orchestra.run.smithi191.stdout:(5/6): libcephsqlite-16.2.14-48.gaeb2e425.el8.x 594 kB/s | 174 kB 00:00 2023-10-09T05:09:56.993 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:00:57 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:09:58.521 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:09:58.522 INFO:teuthology.orchestra.run.smithi008.stdout:=========================================================================================== 2023-10-09T05:09:58.522 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repository Size 2023-10-09T05:09:58.522 INFO:teuthology.orchestra.run.smithi008.stdout:=========================================================================================== 2023-10-09T05:09:58.522 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:09:58.522 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-test x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 52 M 2023-10-09T05:09:58.522 INFO:teuthology.orchestra.run.smithi008.stdout:Installing dependencies: 2023-10-09T05:09:58.522 INFO:teuthology.orchestra.run.smithi008.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T05:09:58.522 INFO:teuthology.orchestra.run.smithi008.stdout: libcephsqlite x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 174 k 2023-10-09T05:09:58.522 INFO:teuthology.orchestra.run.smithi008.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T05:09:58.522 INFO:teuthology.orchestra.run.smithi008.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T05:09:58.523 INFO:teuthology.orchestra.run.smithi008.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T05:09:58.523 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:09:58.523 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:09:58.523 INFO:teuthology.orchestra.run.smithi008.stdout:=========================================================================================== 2023-10-09T05:09:58.523 INFO:teuthology.orchestra.run.smithi008.stdout:Install 6 Packages 2023-10-09T05:09:58.523 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:09:58.524 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 53 M 2023-10-09T05:09:58.524 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 195 M 2023-10-09T05:09:58.524 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:09:58.557 INFO:teuthology.orchestra.run.smithi008.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.1 MB/s | 70 kB 00:00 2023-10-09T05:09:58.641 INFO:teuthology.orchestra.run.smithi008.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.3 MB/s | 323 kB 00:00 2023-10-09T05:09:58.683 INFO:teuthology.orchestra.run.smithi008.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-10-09T05:09:58.725 INFO:teuthology.orchestra.run.smithi008.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-10-09T05:09:58.767 INFO:teuthology.orchestra.run.smithi008.stdout:(5/6): libcephsqlite-16.2.14-48.gaeb2e425.el8.x 717 kB/s | 174 kB 00:00 2023-10-09T05:09:59.022 INFO:teuthology.orchestra.run.smithi191.stdout:(6/6): ceph-test-16.2.14-48.gaeb2e425.el8.x86_6 19 MB/s | 52 MB 00:02 2023-10-09T05:09:59.024 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:09:59.024 INFO:teuthology.orchestra.run.smithi191.stdout:Total 19 MB/s | 53 MB 00:02 2023-10-09T05:09:59.061 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:09:59.081 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:09:59.082 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:09:59.353 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:09:59.357 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:09:59.850 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:10:00.035 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:10:00.112 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:10:00.355 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T05:10:00.535 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T05:10:00.691 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T05:10:00.877 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 5/6 2023-10-09T05:10:01.025 INFO:teuthology.orchestra.run.smithi008.stdout:(6/6): ceph-test-16.2.14-48.gaeb2e425.el8.x86_6 21 MB/s | 52 MB 00:02 2023-10-09T05:10:01.027 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:10:01.028 INFO:teuthology.orchestra.run.smithi008.stdout:Total 21 MB/s | 53 MB 00:02 2023-10-09T05:10:01.067 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:10:01.090 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:10:01.090 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:10:01.378 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:10:01.382 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:10:01.854 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:10:02.099 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:10:02.191 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:10:02.373 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T05:10:02.525 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T05:10:02.681 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T05:10:02.833 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 5/6 2023-10-09T05:10:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 5/6 2023-10-09T05:10:05.706 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 6/6 2023-10-09T05:10:05.957 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 6/6 2023-10-09T05:10:05.958 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 1/6 2023-10-09T05:10:05.958 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 2/6 2023-10-09T05:10:05.958 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T05:10:05.958 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T05:10:05.958 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T05:10:07.126 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T05:10:07.126 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:08 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:07.701 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 5/6 2023-10-09T05:10:07.904 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 6/6 2023-10-09T05:10:08.152 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 6/6 2023-10-09T05:10:08.152 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 1/6 2023-10-09T05:10:08.152 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 2/6 2023-10-09T05:10:08.153 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T05:10:08.153 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T05:10:08.153 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T05:10:09.402 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T05:10:09.402 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:10 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:09.579 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:10:09.764 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:09.764 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:10:09.764 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:09.764 INFO:teuthology.orchestra.run.smithi191.stdout: jq-1.6-3.el8.x86_64 2023-10-09T05:10:09.765 INFO:teuthology.orchestra.run.smithi191.stdout: libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:09.765 INFO:teuthology.orchestra.run.smithi191.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T05:10:09.765 INFO:teuthology.orchestra.run.smithi191.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T05:10:09.765 INFO:teuthology.orchestra.run.smithi191.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T05:10:09.765 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:09.765 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:10:09.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph 2023-10-09T05:10:10.263 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:10:11.086 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:12 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:11.891 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:10:12.089 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:12.089 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:10:12.089 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-test-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:12.090 INFO:teuthology.orchestra.run.smithi008.stdout: jq-1.6-3.el8.x86_64 2023-10-09T05:10:12.090 INFO:teuthology.orchestra.run.smithi008.stdout: libcephsqlite-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:12.090 INFO:teuthology.orchestra.run.smithi008.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T05:10:12.090 INFO:teuthology.orchestra.run.smithi008.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T05:10:12.090 INFO:teuthology.orchestra.run.smithi008.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T05:10:12.090 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:12.090 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:10:12.236 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install ceph 2023-10-09T05:10:12.444 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:10:12.448 INFO:teuthology.orchestra.run.smithi191.stdout:====================================================================================================== 2023-10-09T05:10:12.448 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-10-09T05:10:12.448 INFO:teuthology.orchestra.run.smithi191.stdout:====================================================================================================== 2023-10-09T05:10:12.448 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:10:12.448 INFO:teuthology.orchestra.run.smithi191.stdout: ceph x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 6.4 k 2023-10-09T05:10:12.448 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-10-09T05:10:12.448 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mds x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 2.4 M 2023-10-09T05:10:12.448 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 1.6 M 2023-10-09T05:10:12.448 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-modules-core noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 244 k 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mon x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 5.0 M 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-osd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 18 M 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T05:10:12.449 INFO:teuthology.orchestra.run.smithi191.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T05:10:12.450 INFO:teuthology.orchestra.run.smithi191.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T05:10:12.450 INFO:teuthology.orchestra.run.smithi191.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T05:10:12.450 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T05:10:12.450 INFO:teuthology.orchestra.run.smithi191.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T05:10:12.450 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T05:10:12.450 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T05:10:12.450 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T05:10:12.450 INFO:teuthology.orchestra.run.smithi191.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T05:10:12.450 INFO:teuthology.orchestra.run.smithi191.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T05:10:12.450 INFO:teuthology.orchestra.run.smithi191.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T05:10:12.450 INFO:teuthology.orchestra.run.smithi191.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T05:10:12.451 INFO:teuthology.orchestra.run.smithi191.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T05:10:12.451 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T05:10:12.451 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T05:10:12.451 INFO:teuthology.orchestra.run.smithi191.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T05:10:12.451 INFO:teuthology.orchestra.run.smithi191.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T05:10:12.451 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:12.451 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:10:12.451 INFO:teuthology.orchestra.run.smithi191.stdout:====================================================================================================== 2023-10-09T05:10:12.451 INFO:teuthology.orchestra.run.smithi191.stdout:Install 32 Packages 2023-10-09T05:10:12.451 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:12.452 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 31 M 2023-10-09T05:10:12.452 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 101 M 2023-10-09T05:10:12.452 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:10:12.592 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:10:12.822 INFO:teuthology.orchestra.run.smithi191.stdout:(1/32): ceph-16.2.14-48.gaeb2e425.el8.x86_64.rp 33 kB/s | 6.4 kB 00:00 2023-10-09T05:10:12.906 INFO:teuthology.orchestra.run.smithi191.stdout:(2/32): ceph-mgr-16.2.14-48.gaeb2e425.el8.x86_6 5.8 MB/s | 1.6 MB 00:00 2023-10-09T05:10:13.458 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:14 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:13.940 INFO:teuthology.orchestra.run.smithi191.stdout:(3/32): ceph-osd-16.2.14-48.gaeb2e425.el8.x86_6 18 MB/s | 18 MB 00:01 2023-10-09T05:10:13.990 INFO:teuthology.orchestra.run.smithi191.stdout:(4/32): ceph-mds-16.2.14-48.gaeb2e425.el8.x86_6 1.8 MB/s | 2.4 MB 00:01 2023-10-09T05:10:14.015 INFO:teuthology.orchestra.run.smithi191.stdout:(5/32): ceph-mgr-modules-core-16.2.14-48.gaeb2e 3.2 MB/s | 244 kB 00:00 2023-10-09T05:10:14.258 INFO:teuthology.orchestra.run.smithi191.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 163 kB/s | 44 kB 00:00 2023-10-09T05:10:14.398 INFO:teuthology.orchestra.run.smithi191.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 484 kB/s | 185 kB 00:00 2023-10-09T05:10:14.458 INFO:teuthology.orchestra.run.smithi191.stdout:(8/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 862 kB/s | 173 kB 00:00 2023-10-09T05:10:14.525 INFO:teuthology.orchestra.run.smithi191.stdout:(9/32): ceph-mon-16.2.14-48.gaeb2e425.el8.x86_6 3.0 MB/s | 5.0 MB 00:01 2023-10-09T05:10:14.551 INFO:teuthology.orchestra.run.smithi191.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 120 kB/s | 11 kB 00:00 2023-10-09T05:10:14.577 INFO:teuthology.orchestra.run.smithi191.stdout:(11/32): python3-cherrypy-18.4.0-1.el8.noarch.r 2.1 MB/s | 384 kB 00:00 2023-10-09T05:10:14.602 INFO:teuthology.orchestra.run.smithi191.stdout:(12/32): python3-jaraco-functools-2.0-4.el8.noa 232 kB/s | 18 kB 00:00 2023-10-09T05:10:14.668 INFO:teuthology.orchestra.run.smithi191.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 638 kB/s | 59 kB 00:00 2023-10-09T05:10:14.736 INFO:teuthology.orchestra.run.smithi191.stdout:(14/32): python3-portend-2.6-1.el8.noarch.rpm 235 kB/s | 16 kB 00:00 2023-10-09T05:10:14.761 INFO:teuthology.orchestra.run.smithi191.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.8 MB/s | 283 kB 00:00 2023-10-09T05:10:14.803 INFO:teuthology.orchestra.run.smithi191.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 279 kB/s | 19 kB 00:00 2023-10-09T05:10:14.820 INFO:teuthology.orchestra.run.smithi191.stdout:(17/32): python3-logutils-0.3.5-11.el8.noarch.r 181 kB/s | 49 kB 00:00 2023-10-09T05:10:14.837 INFO:teuthology.orchestra.run.smithi191.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 316 kB/s | 24 kB 00:00 2023-10-09T05:10:14.905 INFO:teuthology.orchestra.run.smithi191.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 286 kB/s | 29 kB 00:00 2023-10-09T05:10:14.930 INFO:teuthology.orchestra.run.smithi191.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 252 kB/s | 27 kB 00:00 2023-10-09T05:10:14.955 INFO:teuthology.orchestra.run.smithi191.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.1 MB/s | 255 kB 00:00 2023-10-09T05:10:14.998 INFO:teuthology.orchestra.run.smithi191.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.7 MB/s | 251 kB 00:00 2023-10-09T05:10:15.015 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:10:15.019 INFO:teuthology.orchestra.run.smithi008.stdout:====================================================================================================== 2023-10-09T05:10:15.019 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repository Size 2023-10-09T05:10:15.019 INFO:teuthology.orchestra.run.smithi008.stdout:====================================================================================================== 2023-10-09T05:10:15.019 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:10:15.019 INFO:teuthology.orchestra.run.smithi008.stdout: ceph x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 6.4 k 2023-10-09T05:10:15.019 INFO:teuthology.orchestra.run.smithi008.stdout:Installing dependencies: 2023-10-09T05:10:15.019 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mds x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 2.4 M 2023-10-09T05:10:15.019 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 1.6 M 2023-10-09T05:10:15.019 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr-modules-core noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 244 k 2023-10-09T05:10:15.019 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mon x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 5.0 M 2023-10-09T05:10:15.019 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-osd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 18 M 2023-10-09T05:10:15.020 INFO:teuthology.orchestra.run.smithi008.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T05:10:15.020 INFO:teuthology.orchestra.run.smithi008.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T05:10:15.020 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T05:10:15.020 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T05:10:15.020 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T05:10:15.020 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T05:10:15.020 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T05:10:15.020 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T05:10:15.020 INFO:teuthology.orchestra.run.smithi008.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T05:10:15.020 INFO:teuthology.orchestra.run.smithi008.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T05:10:15.020 INFO:teuthology.orchestra.run.smithi008.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T05:10:15.021 INFO:teuthology.orchestra.run.smithi008.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T05:10:15.022 INFO:teuthology.orchestra.run.smithi008.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T05:10:15.022 INFO:teuthology.orchestra.run.smithi008.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T05:10:15.022 INFO:teuthology.orchestra.run.smithi008.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T05:10:15.022 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:15.022 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:10:15.022 INFO:teuthology.orchestra.run.smithi008.stdout:====================================================================================================== 2023-10-09T05:10:15.022 INFO:teuthology.orchestra.run.smithi008.stdout:Install 32 Packages 2023-10-09T05:10:15.023 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:15.023 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 31 M 2023-10-09T05:10:15.023 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 101 M 2023-10-09T05:10:15.023 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:10:15.030 INFO:teuthology.orchestra.run.smithi191.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 304 kB/s | 23 kB 00:00 2023-10-09T05:10:15.073 INFO:teuthology.orchestra.run.smithi191.stdout:(24/32): python3-webtest-2.0.33-1.el8.noarch.rp 598 kB/s | 85 kB 00:00 2023-10-09T05:10:15.106 INFO:teuthology.orchestra.run.smithi191.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 1.0 MB/s | 109 kB 00:00 2023-10-09T05:10:15.173 INFO:teuthology.orchestra.run.smithi191.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 4.0 MB/s | 559 kB 00:00 2023-10-09T05:10:15.199 INFO:teuthology.orchestra.run.smithi191.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 625 kB/s | 54 kB 00:00 2023-10-09T05:10:15.224 INFO:teuthology.orchestra.run.smithi191.stdout:(28/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.6 MB/s | 238 kB 00:00 2023-10-09T05:10:15.250 INFO:teuthology.orchestra.run.smithi191.stdout:(29/32): python3-markupsafe-0.23-19.el8.x86_64. 511 kB/s | 39 kB 00:00 2023-10-09T05:10:15.269 INFO:teuthology.orchestra.run.smithi008.stdout:(1/32): ceph-16.2.14-48.gaeb2e425.el8.x86_64.rp 36 kB/s | 6.4 kB 00:00 2023-10-09T05:10:15.275 INFO:teuthology.orchestra.run.smithi191.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.0 MB/s | 157 kB 00:00 2023-10-09T05:10:15.300 INFO:teuthology.orchestra.run.smithi191.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-10-09T05:10:15.326 INFO:teuthology.orchestra.run.smithi191.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.4 MB/s | 457 kB 00:00 2023-10-09T05:10:15.330 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:10:15.330 INFO:teuthology.orchestra.run.smithi191.stdout:Total 11 MB/s | 31 MB 00:02 2023-10-09T05:10:15.420 INFO:teuthology.orchestra.run.smithi008.stdout:(2/32): ceph-mds-16.2.14-48.gaeb2e425.el8.x86_6 7.4 MB/s | 2.4 MB 00:00 2023-10-09T05:10:15.422 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:10:15.461 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:10:15.461 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:10:15.677 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:10:15.680 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:10:15.805 INFO:teuthology.orchestra.run.smithi008.stdout:(3/32): ceph-mgr-16.2.14-48.gaeb2e425.el8.x86_6 2.3 MB/s | 1.6 MB 00:00 2023-10-09T05:10:15.903 INFO:teuthology.orchestra.run.smithi008.stdout:(4/32): ceph-mgr-modules-core-16.2.14-48.gaeb2e 2.4 MB/s | 244 kB 00:00 2023-10-09T05:10:16.162 INFO:teuthology.orchestra.run.smithi008.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 169 kB/s | 44 kB 00:00 2023-10-09T05:10:16.529 INFO:teuthology.orchestra.run.smithi008.stdout:(6/32): ceph-osd-16.2.14-48.gaeb2e425.el8.x86_6 17 MB/s | 18 MB 00:01 2023-10-09T05:10:16.554 INFO:teuthology.orchestra.run.smithi008.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 473 kB/s | 185 kB 00:00 2023-10-09T05:10:16.730 INFO:teuthology.orchestra.run.smithi008.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 2.1 MB/s | 384 kB 00:00 2023-10-09T05:10:16.846 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:10:16.846 INFO:teuthology.orchestra.run.smithi008.stdout:(9/32): ceph-mon-16.2.14-48.gaeb2e425.el8.x86_6 3.2 MB/s | 5.0 MB 00:01 2023-10-09T05:10:16.872 INFO:teuthology.orchestra.run.smithi008.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 78 kB/s | 11 kB 00:00 2023-10-09T05:10:16.897 INFO:teuthology.orchestra.run.smithi008.stdout:(11/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 470 kB/s | 173 kB 00:00 2023-10-09T05:10:16.922 INFO:teuthology.orchestra.run.smithi008.stdout:(12/32): python3-jaraco-functools-2.0-4.el8.noa 233 kB/s | 18 kB 00:00 2023-10-09T05:10:16.981 INFO:teuthology.orchestra.run.smithi008.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 703 kB/s | 59 kB 00:00 2023-10-09T05:10:17.023 INFO:teuthology.orchestra.run.smithi008.stdout:(14/32): python3-pecan-1.3.2-9.el8.noarch.rpm 2.9 MB/s | 283 kB 00:00 2023-10-09T05:10:17.056 INFO:teuthology.orchestra.run.smithi008.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 209 kB/s | 16 kB 00:00 2023-10-09T05:10:17.087 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T05:10:17.107 INFO:teuthology.orchestra.run.smithi008.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 224 kB/s | 19 kB 00:00 2023-10-09T05:10:17.124 INFO:teuthology.orchestra.run.smithi008.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 357 kB/s | 24 kB 00:00 2023-10-09T05:10:17.141 INFO:teuthology.orchestra.run.smithi008.stdout:(18/32): python3-logutils-0.3.5-11.el8.noarch.r 181 kB/s | 49 kB 00:00 2023-10-09T05:10:17.183 INFO:teuthology.orchestra.run.smithi008.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 386 kB/s | 29 kB 00:00 2023-10-09T05:10:17.208 INFO:teuthology.orchestra.run.smithi008.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 327 kB/s | 27 kB 00:00 2023-10-09T05:10:17.275 INFO:teuthology.orchestra.run.smithi008.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.7 MB/s | 251 kB 00:00 2023-10-09T05:10:17.292 INFO:teuthology.orchestra.run.smithi008.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.0 MB/s | 85 kB 00:00 2023-10-09T05:10:17.351 INFO:teuthology.orchestra.run.smithi008.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 304 kB/s | 23 kB 00:00 2023-10-09T05:10:17.376 INFO:teuthology.orchestra.run.smithi008.stdout:(24/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.1 MB/s | 255 kB 00:00 2023-10-09T05:10:17.410 INFO:teuthology.orchestra.run.smithi008.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 931 kB/s | 109 kB 00:00 2023-10-09T05:10:17.415 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T05:10:17.453 INFO:teuthology.orchestra.run.smithi008.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 5.4 MB/s | 559 kB 00:00 2023-10-09T05:10:17.469 INFO:teuthology.orchestra.run.smithi008.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 1.1 MB/s | 54 kB 00:00 2023-10-09T05:10:17.502 INFO:teuthology.orchestra.run.smithi008.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 772 kB/s | 39 kB 00:00 2023-10-09T05:10:17.528 INFO:teuthology.orchestra.run.smithi008.stdout:(29/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.6 MB/s | 238 kB 00:00 2023-10-09T05:10:17.553 INFO:teuthology.orchestra.run.smithi008.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.8 MB/s | 157 kB 00:00 2023-10-09T05:10:17.578 INFO:teuthology.orchestra.run.smithi008.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-10-09T05:10:17.612 INFO:teuthology.orchestra.run.smithi008.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.1 MB/s | 457 kB 00:00 2023-10-09T05:10:17.616 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:10:17.616 INFO:teuthology.orchestra.run.smithi008.stdout:Total 12 MB/s | 31 MB 00:02 2023-10-09T05:10:17.627 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-10-09T05:10:17.710 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:10:17.752 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:10:17.752 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:10:17.859 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-10-09T05:10:17.980 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:10:17.985 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:10:18.111 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-10-09T05:10:18.319 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-10-09T05:10:18.584 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-10-09T05:10:18.962 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-10-09T05:10:19.204 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:10:19.285 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-10-09T05:10:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T05:10:19.464 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-10-09T05:10:19.640 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-10-09T05:10:19.686 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T05:10:19.857 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-10-09T05:10:19.901 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-10-09T05:10:20.094 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T05:10:20.144 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-10-09T05:10:20.306 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T05:10:20.391 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-10-09T05:10:20.527 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T05:10:20.570 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-10-09T05:10:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T05:10:20.794 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-10-09T05:10:20.909 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T05:10:21.070 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T05:10:21.160 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-10-09T05:10:21.301 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T05:10:21.409 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-10-09T05:10:21.544 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T05:10:21.605 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-10-09T05:10:21.777 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-10-09T05:10:21.964 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-10-09T05:10:22.183 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T05:10:22.195 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T05:10:22.362 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T05:10:22.484 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T05:10:22.526 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T05:10:22.668 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T05:10:22.804 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T05:10:22.866 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T05:10:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T05:10:23.016 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T05:10:23.187 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T05:10:23.346 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T05:10:23.359 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T05:10:23.544 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T05:10:23.726 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T05:10:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T05:10:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.n 27/32 2023-10-09T05:10:24.255 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 28/32 2023-10-09T05:10:24.482 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T05:10:24.675 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T05:10:24.999 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T05:10:25.300 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T05:10:25.645 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.n 27/32 2023-10-09T05:10:25.747 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 28/32 2023-10-09T05:10:25.776 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 28/32 2023-10-09T05:10:25.997 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 29/32 2023-10-09T05:10:26.422 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 29/32 2023-10-09T05:10:26.605 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 30/32 2023-10-09T05:10:26.822 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 30/32 2023-10-09T05:10:26.980 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 31/32 2023-10-09T05:10:26.998 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 31/32 2023-10-09T05:10:27.161 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 32/32 2023-10-09T05:10:27.369 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 28/32 2023-10-09T05:10:27.513 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 32/32 2023-10-09T05:10:27.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 1/32 2023-10-09T05:10:27.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 2/32 2023-10-09T05:10:27.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 3/32 2023-10-09T05:10:27.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 4/32 2023-10-09T05:10:27.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 5/32 2023-10-09T05:10:27.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.n 6/32 2023-10-09T05:10:27.513 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T05:10:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T05:10:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T05:10:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T05:10:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T05:10:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T05:10:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T05:10:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-10-09T05:10:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-10-09T05:10:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-10-09T05:10:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-10-09T05:10:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-10-09T05:10:27.518 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-10-09T05:10:27.518 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-10-09T05:10:27.579 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 29/32 2023-10-09T05:10:28.031 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 29/32 2023-10-09T05:10:28.171 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 30/32 2023-10-09T05:10:28.401 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 30/32 2023-10-09T05:10:28.539 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 31/32 2023-10-09T05:10:28.557 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 31/32 2023-10-09T05:10:28.719 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 32/32 2023-10-09T05:10:29.042 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-10-09T05:10:29.042 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:30 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:29.099 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 32/32 2023-10-09T05:10:29.099 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 1/32 2023-10-09T05:10:29.099 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 2/32 2023-10-09T05:10:29.099 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 3/32 2023-10-09T05:10:29.099 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 4/32 2023-10-09T05:10:29.100 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 5/32 2023-10-09T05:10:29.100 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.n 6/32 2023-10-09T05:10:29.100 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T05:10:29.100 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T05:10:29.100 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T05:10:29.100 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T05:10:29.100 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T05:10:29.100 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T05:10:29.100 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T05:10:29.100 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T05:10:29.100 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T05:10:29.101 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T05:10:29.101 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T05:10:29.101 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T05:10:29.101 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T05:10:29.101 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T05:10:29.101 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T05:10:29.101 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T05:10:29.101 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T05:10:29.101 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T05:10:29.101 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-10-09T05:10:29.101 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-10-09T05:10:29.102 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-10-09T05:10:29.102 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-10-09T05:10:29.102 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-10-09T05:10:29.102 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-10-09T05:10:29.103 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-10-09T05:10:30.596 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-10-09T05:10:30.596 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:31 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:31.454 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:10:31.640 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:31.641 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:10:31.641 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:31.641 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:31.641 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:31.641 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:10:31.641 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:31.641 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:31.641 INFO:teuthology.orchestra.run.smithi191.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T05:10:31.641 INFO:teuthology.orchestra.run.smithi191.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T05:10:31.642 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T05:10:31.642 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T05:10:31.642 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T05:10:31.642 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T05:10:31.642 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T05:10:31.642 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T05:10:31.642 INFO:teuthology.orchestra.run.smithi191.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T05:10:31.642 INFO:teuthology.orchestra.run.smithi191.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T05:10:31.642 INFO:teuthology.orchestra.run.smithi191.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T05:10:31.643 INFO:teuthology.orchestra.run.smithi191.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T05:10:31.643 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T05:10:31.643 INFO:teuthology.orchestra.run.smithi191.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T05:10:31.643 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T05:10:31.643 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T05:10:31.643 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T05:10:31.643 INFO:teuthology.orchestra.run.smithi191.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T05:10:31.643 INFO:teuthology.orchestra.run.smithi191.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T05:10:31.643 INFO:teuthology.orchestra.run.smithi191.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T05:10:31.643 INFO:teuthology.orchestra.run.smithi191.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T05:10:31.644 INFO:teuthology.orchestra.run.smithi191.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T05:10:31.644 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T05:10:31.644 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T05:10:31.644 INFO:teuthology.orchestra.run.smithi191.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T05:10:31.644 INFO:teuthology.orchestra.run.smithi191.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T05:10:31.644 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:31.644 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:10:31.796 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-base 2023-10-09T05:10:32.143 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:10:33.160 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:34 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:33.415 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:10:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:33.624 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:10:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mds-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr-modules-core-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:10:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mon-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-osd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T05:10:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T05:10:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T05:10:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T05:10:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T05:10:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T05:10:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T05:10:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T05:10:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T05:10:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T05:10:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T05:10:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T05:10:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T05:10:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T05:10:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T05:10:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:33.627 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:10:33.792 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install ceph-base 2023-10-09T05:10:34.159 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:10:34.426 INFO:teuthology.orchestra.run.smithi191.stdout:Package ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:10:34.534 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:10:34.535 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-10-09T05:10:34.535 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:10:34.660 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install cephadm 2023-10-09T05:10:35.002 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:10:35.121 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:36 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:36.045 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:37 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:36.532 INFO:teuthology.orchestra.run.smithi008.stdout:Package ceph-base-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:10:36.653 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:10:36.654 INFO:teuthology.orchestra.run.smithi008.stdout:Nothing to do. 2023-10-09T05:10:36.654 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:10:36.764 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install cephadm 2023-10-09T05:10:37.129 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:10:37.407 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:10:37.407 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:10:37.407 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-10-09T05:10:37.407 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:10:37.407 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:10:37.408 INFO:teuthology.orchestra.run.smithi191.stdout: cephadm noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 84 k 2023-10-09T05:10:37.408 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:37.408 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:10:37.408 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:10:37.408 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-10-09T05:10:37.408 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:37.408 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 84 k 2023-10-09T05:10:37.409 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 348 k 2023-10-09T05:10:37.409 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:10:37.588 INFO:teuthology.orchestra.run.smithi191.stdout:cephadm-16.2.14-48.gaeb2e425.el8.noarch.rpm 469 kB/s | 84 kB 00:00 2023-10-09T05:10:37.589 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:10:37.589 INFO:teuthology.orchestra.run.smithi191.stdout:Total 466 kB/s | 84 kB 00:00 2023-10-09T05:10:37.589 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:10:37.596 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:10:37.596 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:10:37.598 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:10:37.602 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:10:37.645 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:10:38.695 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:39 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:40.228 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:10:40.229 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:10:40.229 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repository Size 2023-10-09T05:10:40.229 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:10:40.229 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:10:40.229 INFO:teuthology.orchestra.run.smithi008.stdout: cephadm noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 84 k 2023-10-09T05:10:40.230 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:40.230 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:10:40.230 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:10:40.230 INFO:teuthology.orchestra.run.smithi008.stdout:Install 1 Package 2023-10-09T05:10:40.230 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:40.230 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 84 k 2023-10-09T05:10:40.230 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 348 k 2023-10-09T05:10:40.230 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:10:40.713 INFO:teuthology.orchestra.run.smithi008.stdout:cephadm-16.2.14-48.gaeb2e425.el8.noarch.rpm 174 kB/s | 84 kB 00:00 2023-10-09T05:10:40.714 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:10:40.714 INFO:teuthology.orchestra.run.smithi008.stdout:Total 174 kB/s | 84 kB 00:00 2023-10-09T05:10:40.714 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:10:40.723 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:10:40.724 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:10:40.726 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:10:40.730 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:10:40.755 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T05:10:40.914 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T05:10:41.083 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T05:10:41.140 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:10:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T05:10:42.212 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:43 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:42.945 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T05:10:43.069 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T05:10:43.243 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T05:10:44.348 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/1 2023-10-09T05:10:44.348 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:45 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:46.439 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:10:46.621 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:46.621 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:10:46.621 INFO:teuthology.orchestra.run.smithi191.stdout: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:10:46.621 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:46.621 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:10:46.772 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T05:10:47.130 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:10:48.665 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:49 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:50.033 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:10:50.034 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:10:50.034 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repo Size 2023-10-09T05:10:50.034 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:10:50.034 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:10:50.034 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 161 k 2023-10-09T05:10:50.034 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:50.035 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:10:50.035 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:10:50.035 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-10-09T05:10:50.035 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:50.035 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 161 k 2023-10-09T05:10:50.035 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 470 k 2023-10-09T05:10:50.035 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:10:50.342 INFO:teuthology.orchestra.run.smithi191.stdout:ceph-immutable-object-cache-16.2.14-48.gaeb2e42 523 kB/s | 161 kB 00:00 2023-10-09T05:10:50.343 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:10:50.343 INFO:teuthology.orchestra.run.smithi191.stdout:Total 520 kB/s | 161 kB 00:00 2023-10-09T05:10:50.344 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:10:50.355 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:10:50.355 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:10:50.408 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:10:50.411 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:10:50.535 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:10:50.776 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T05:10:51.110 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T05:10:51.127 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:10:51.326 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:51.326 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:10:51.326 INFO:teuthology.orchestra.run.smithi008.stdout: cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:10:51.326 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:51.326 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:10:51.464 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T05:10:51.846 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:10:52.223 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T05:10:52.224 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:53 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:53.476 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:54 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:54.998 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:10:54.999 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:10:54.999 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repo Size 2023-10-09T05:10:54.999 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:10:55.000 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:10:55.000 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 161 k 2023-10-09T05:10:55.000 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:55.000 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:10:55.000 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:10:55.000 INFO:teuthology.orchestra.run.smithi008.stdout:Install 1 Package 2023-10-09T05:10:55.000 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:10:55.001 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 161 k 2023-10-09T05:10:55.001 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 470 k 2023-10-09T05:10:55.001 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:10:55.187 INFO:teuthology.orchestra.run.smithi008.stdout:ceph-immutable-object-cache-16.2.14-48.gaeb2e42 863 kB/s | 161 kB 00:00 2023-10-09T05:10:55.188 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:10:55.188 INFO:teuthology.orchestra.run.smithi008.stdout:Total 857 kB/s | 161 kB 00:00 2023-10-09T05:10:55.188 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:10:55.200 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:10:55.201 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:10:55.258 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:10:55.262 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:10:55.382 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:10:55.550 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T05:10:55.950 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T05:10:56.930 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:10:57.039 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.e 1/1 2023-10-09T05:10:57.040 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:58 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:10:57.110 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:57.111 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:10:57.111 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:10:57.111 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:10:57.111 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:10:57.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr 2023-10-09T05:10:57.591 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:10:59.211 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:00 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:00.469 INFO:teuthology.orchestra.run.smithi191.stdout:Package ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:11:00.579 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:11:00.580 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-10-09T05:11:00.580 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:11:00.867 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T05:11:01.213 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:11:02.612 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:03 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:04.010 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:11:04.011 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================================== 2023-10-09T05:11:04.011 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-10-09T05:11:04.011 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================================== 2023-10-09T05:11:04.011 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:11:04.011 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-dashboard noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 3.1 M 2023-10-09T05:11:04.011 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-10-09T05:11:04.012 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-grafana-dashboards noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 24 k 2023-10-09T05:11:04.012 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-prometheus-alerts noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 15 k 2023-10-09T05:11:04.012 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:11:04.012 INFO:teuthology.orchestra.run.smithi191.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T05:11:04.012 INFO:teuthology.orchestra.run.smithi191.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T05:11:04.012 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:04.012 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:11:04.012 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================================== 2023-10-09T05:11:04.012 INFO:teuthology.orchestra.run.smithi191.stdout:Install 6 Packages 2023-10-09T05:11:04.012 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:04.013 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 3.4 M 2023-10-09T05:11:04.013 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 22 M 2023-10-09T05:11:04.013 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:11:04.344 INFO:teuthology.orchestra.run.smithi191.stdout:(1/6): ceph-prometheus-alerts-16.2.14-48.gaeb2e 87 kB/s | 15 kB 00:00 2023-10-09T05:11:04.369 INFO:teuthology.orchestra.run.smithi191.stdout:(2/6): ceph-grafana-dashboards-16.2.14-48.gaeb2 124 kB/s | 24 kB 00:00 2023-10-09T05:11:04.611 INFO:teuthology.orchestra.run.smithi191.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 126 kB/s | 34 kB 00:00 2023-10-09T05:11:04.720 INFO:teuthology.orchestra.run.smithi191.stdout:(4/6): ceph-mgr-dashboard-16.2.14-48.gaeb2e425. 5.7 MB/s | 3.1 MB 00:00 2023-10-09T05:11:04.726 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:11:04.737 INFO:teuthology.orchestra.run.smithi191.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 534 kB/s | 196 kB 00:00 2023-10-09T05:11:04.854 INFO:teuthology.orchestra.run.smithi191.stdout:(6/6): python3-jwt-1.6.1-2.el8.noarch.rpm 176 kB/s | 43 kB 00:00 2023-10-09T05:11:04.857 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:11:04.857 INFO:teuthology.orchestra.run.smithi191.stdout:Total 4.1 MB/s | 3.4 MB 00:00 2023-10-09T05:11:04.869 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:11:04.880 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:11:04.880 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:11:04.926 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:04.926 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:11:04.926 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-immutable-object-cache-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:11:04.926 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:04.926 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:11:04.951 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:11:04.955 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:11:05.053 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install ceph-mgr 2023-10-09T05:11:05.288 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:11:05.421 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:11:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T05:11:05.679 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T05:11:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T05:11:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.no 4/6 2023-10-09T05:11:06.986 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:02:07 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:07.137 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.n 5/6 2023-10-09T05:11:07.442 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T05:11:07.689 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T05:11:07.689 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.n 1/6 2023-10-09T05:11:07.689 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 2/6 2023-10-09T05:11:07.689 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.no 3/6 2023-10-09T05:11:07.689 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T05:11:07.689 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T05:11:08.405 INFO:teuthology.orchestra.run.smithi008.stdout:Package ceph-mgr-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:11:08.525 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:11:08.527 INFO:teuthology.orchestra.run.smithi008.stdout:Nothing to do. 2023-10-09T05:11:08.527 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:11:08.645 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T05:11:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T05:11:08.736 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:09 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:09.014 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:11:10.483 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:02:11 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:12.015 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:11:12.016 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================================== 2023-10-09T05:11:12.016 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repository Size 2023-10-09T05:11:12.016 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================================== 2023-10-09T05:11:12.016 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:11:12.016 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr-dashboard noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 3.1 M 2023-10-09T05:11:12.017 INFO:teuthology.orchestra.run.smithi008.stdout:Installing dependencies: 2023-10-09T05:11:12.017 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-grafana-dashboards noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 24 k 2023-10-09T05:11:12.017 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-prometheus-alerts noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 15 k 2023-10-09T05:11:12.017 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:11:12.017 INFO:teuthology.orchestra.run.smithi008.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T05:11:12.017 INFO:teuthology.orchestra.run.smithi008.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T05:11:12.017 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:12.017 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:11:12.017 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================================== 2023-10-09T05:11:12.017 INFO:teuthology.orchestra.run.smithi008.stdout:Install 6 Packages 2023-10-09T05:11:12.017 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:12.018 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 3.4 M 2023-10-09T05:11:12.018 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 22 M 2023-10-09T05:11:12.018 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:11:12.393 INFO:teuthology.orchestra.run.smithi008.stdout:(1/6): ceph-grafana-dashboards-16.2.14-48.gaeb2 107 kB/s | 24 kB 00:00 2023-10-09T05:11:12.418 INFO:teuthology.orchestra.run.smithi008.stdout:(2/6): ceph-prometheus-alerts-16.2.14-48.gaeb2e 59 kB/s | 15 kB 00:00 2023-10-09T05:11:12.511 INFO:teuthology.orchestra.run.smithi008.stdout:(3/6): ceph-mgr-dashboard-16.2.14-48.gaeb2e425. 9.1 MB/s | 3.1 MB 00:00 2023-10-09T05:11:12.619 INFO:teuthology.orchestra.run.smithi008.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 431 kB/s | 43 kB 00:00 2023-10-09T05:11:12.828 INFO:teuthology.orchestra.run.smithi008.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 77 kB/s | 34 kB 00:00 2023-10-09T05:11:13.128 INFO:teuthology.orchestra.run.smithi008.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 277 kB/s | 196 kB 00:00 2023-10-09T05:11:13.131 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:11:13.132 INFO:teuthology.orchestra.run.smithi008.stdout:Total 3.1 MB/s | 3.4 MB 00:01 2023-10-09T05:11:13.146 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:11:13.159 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:11:13.159 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:11:13.237 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:11:13.241 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:11:13.550 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:11:14.035 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T05:11:14.255 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:11:14.268 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T05:11:14.437 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:14.437 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:11:14.437 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:11:14.437 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:11:14.437 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:11:14.437 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T05:11:14.437 INFO:teuthology.orchestra.run.smithi191.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T05:11:14.437 INFO:teuthology.orchestra.run.smithi191.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T05:11:14.437 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:14.438 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:11:14.467 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T05:11:14.567 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T05:11:14.581 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.no 4/6 2023-10-09T05:11:14.913 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:11:15.661 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.n 5/6 2023-10-09T05:11:16.006 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T05:11:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T05:11:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.n 1/6 2023-10-09T05:11:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 2/6 2023-10-09T05:11:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.no 3/6 2023-10-09T05:11:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T05:11:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T05:11:16.499 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:17 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:17.383 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T05:11:17.384 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:02:18 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:17.856 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:11:17.857 INFO:teuthology.orchestra.run.smithi191.stdout:======================================================================================================================= 2023-10-09T05:11:17.857 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-10-09T05:11:17.857 INFO:teuthology.orchestra.run.smithi191.stdout:======================================================================================================================= 2023-10-09T05:11:17.857 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:11:17.858 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 8.6 M 2023-10-09T05:11:17.858 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-10-09T05:11:17.858 INFO:teuthology.orchestra.run.smithi191.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T05:11:17.858 INFO:teuthology.orchestra.run.smithi191.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T05:11:17.858 INFO:teuthology.orchestra.run.smithi191.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T05:11:17.858 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T05:11:17.858 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T05:11:17.858 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T05:11:17.858 INFO:teuthology.orchestra.run.smithi191.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T05:11:17.858 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:17.858 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:11:17.859 INFO:teuthology.orchestra.run.smithi191.stdout:======================================================================================================================= 2023-10-09T05:11:17.859 INFO:teuthology.orchestra.run.smithi191.stdout:Install 8 Packages 2023-10-09T05:11:17.859 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:17.860 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 37 M 2023-10-09T05:11:17.860 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 210 M 2023-10-09T05:11:17.860 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:11:18.053 INFO:teuthology.orchestra.run.smithi191.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 885 kB/s | 171 kB 00:00 2023-10-09T05:11:18.127 INFO:teuthology.orchestra.run.smithi191.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 2.4 MB/s | 644 kB 00:00 2023-10-09T05:11:18.479 INFO:teuthology.orchestra.run.smithi191.stdout:(3/8): ceph-mgr-diskprediction-local-16.2.14-48 14 MB/s | 8.6 MB 00:00 2023-10-09T05:11:18.625 INFO:teuthology.orchestra.run.smithi191.stdout:(4/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 7.5 MB/s | 3.7 MB 00:00 2023-10-09T05:11:18.663 INFO:teuthology.orchestra.run.smithi191.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.2 MB/s | 225 kB 00:00 2023-10-09T05:11:18.797 INFO:teuthology.orchestra.run.smithi191.stdout:(6/8): python3-scipy-1.0.0-21.module+el8.5.0+10 18 MB/s | 14 MB 00:00 2023-10-09T05:11:18.948 INFO:teuthology.orchestra.run.smithi191.stdout:(7/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 15 MB/s | 4.9 MB 00:00 2023-10-09T05:11:19.056 INFO:teuthology.orchestra.run.smithi191.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 12 MB/s | 4.7 MB 00:00 2023-10-09T05:11:19.057 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:11:19.057 INFO:teuthology.orchestra.run.smithi191.stdout:Total 31 MB/s | 37 MB 00:01 2023-10-09T05:11:19.296 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:11:19.313 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:11:19.313 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:11:19.614 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:11:19.616 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:11:20.163 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:11:20.295 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:11:20.470 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:11:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:11:21.300 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:11:21.479 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:11:22.080 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:11:22.286 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:11:22.980 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:11:23.331 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T05:11:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:11:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:11:26.240 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:11:26.442 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:26.442 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:11:26.442 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-grafana-dashboards-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:11:26.442 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr-dashboard-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:11:26.442 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-prometheus-alerts-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:11:26.442 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T05:11:26.442 INFO:teuthology.orchestra.run.smithi008.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T05:11:26.442 INFO:teuthology.orchestra.run.smithi008.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T05:11:26.442 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:26.443 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:11:26.576 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T05:11:26.709 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T05:11:26.943 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:11:26.977 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 8/8 2023-10-09T05:11:27.262 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 8/8 2023-10-09T05:11:27.262 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 1/8 2023-10-09T05:11:27.262 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:11:27.262 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-10-09T05:11:27.262 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-10-09T05:11:27.262 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T05:11:27.262 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:11:27.262 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-10-09T05:11:28.311 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-10-09T05:11:28.312 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:29 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:28.574 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:02:29 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:30.112 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:11:30.113 INFO:teuthology.orchestra.run.smithi008.stdout:======================================================================================================================= 2023-10-09T05:11:30.113 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repository Size 2023-10-09T05:11:30.113 INFO:teuthology.orchestra.run.smithi008.stdout:======================================================================================================================= 2023-10-09T05:11:30.113 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:11:30.113 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 8.6 M 2023-10-09T05:11:30.113 INFO:teuthology.orchestra.run.smithi008.stdout:Installing dependencies: 2023-10-09T05:11:30.114 INFO:teuthology.orchestra.run.smithi008.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T05:11:30.114 INFO:teuthology.orchestra.run.smithi008.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T05:11:30.114 INFO:teuthology.orchestra.run.smithi008.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T05:11:30.114 INFO:teuthology.orchestra.run.smithi008.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T05:11:30.114 INFO:teuthology.orchestra.run.smithi008.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T05:11:30.114 INFO:teuthology.orchestra.run.smithi008.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T05:11:30.114 INFO:teuthology.orchestra.run.smithi008.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T05:11:30.114 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:30.114 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:11:30.114 INFO:teuthology.orchestra.run.smithi008.stdout:======================================================================================================================= 2023-10-09T05:11:30.115 INFO:teuthology.orchestra.run.smithi008.stdout:Install 8 Packages 2023-10-09T05:11:30.115 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:30.115 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 37 M 2023-10-09T05:11:30.115 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 210 M 2023-10-09T05:11:30.115 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:11:30.240 INFO:teuthology.orchestra.run.smithi008.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.3 MB/s | 171 kB 00:00 2023-10-09T05:11:30.307 INFO:teuthology.orchestra.run.smithi008.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 3.3 MB/s | 644 kB 00:00 2023-10-09T05:11:30.658 INFO:teuthology.orchestra.run.smithi008.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 33 MB/s | 14 MB 00:00 2023-10-09T05:11:30.741 INFO:teuthology.orchestra.run.smithi008.stdout:(4/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 8.6 MB/s | 3.7 MB 00:00 2023-10-09T05:11:30.775 INFO:teuthology.orchestra.run.smithi008.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.9 MB/s | 225 kB 00:00 2023-10-09T05:11:30.933 INFO:teuthology.orchestra.run.smithi008.stdout:(6/8): ceph-mgr-diskprediction-local-16.2.14-48 11 MB/s | 8.6 MB 00:00 2023-10-09T05:11:31.059 INFO:teuthology.orchestra.run.smithi008.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 17 MB/s | 4.7 MB 00:00 2023-10-09T05:11:31.125 INFO:teuthology.orchestra.run.smithi008.stdout:(8/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 13 MB/s | 4.9 MB 00:00 2023-10-09T05:11:31.127 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:11:31.127 INFO:teuthology.orchestra.run.smithi008.stdout:Total 36 MB/s | 37 MB 00:01 2023-10-09T05:11:31.371 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:11:31.390 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:11:31.391 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:11:31.706 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:11:31.709 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:11:32.275 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:11:32.521 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:11:32.643 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:11:32.820 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:11:33.466 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:11:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:11:33.951 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:11:34.235 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:34.236 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:11:34.236 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:11:34.236 INFO:teuthology.orchestra.run.smithi191.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:11:34.236 INFO:teuthology.orchestra.run.smithi191.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:11:34.236 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T05:11:34.236 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T05:11:34.236 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T05:11:34.236 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T05:11:34.236 INFO:teuthology.orchestra.run.smithi191.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T05:11:34.237 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:34.237 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:11:34.264 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:11:34.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-rook 2023-10-09T05:11:34.554 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:11:34.649 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:11:35.364 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:11:35.654 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T05:11:35.845 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:11:36.340 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:37 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:37.713 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:11:37.782 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:11:37.783 INFO:teuthology.orchestra.run.smithi191.stdout:======================================================================================================= 2023-10-09T05:11:37.783 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-10-09T05:11:37.783 INFO:teuthology.orchestra.run.smithi191.stdout:======================================================================================================= 2023-10-09T05:11:37.783 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:11:37.783 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-rook noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 27 k 2023-10-09T05:11:37.783 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-10-09T05:11:37.784 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T05:11:37.784 INFO:teuthology.orchestra.run.smithi191.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T05:11:37.784 INFO:teuthology.orchestra.run.smithi191.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T05:11:37.784 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T05:11:37.784 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T05:11:37.784 INFO:teuthology.orchestra.run.smithi191.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T05:11:37.784 INFO:teuthology.orchestra.run.smithi191.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T05:11:37.784 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T05:11:37.784 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T05:11:37.785 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:11:37.785 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T05:11:37.785 INFO:teuthology.orchestra.run.smithi191.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T05:11:37.785 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:37.785 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:11:37.785 INFO:teuthology.orchestra.run.smithi191.stdout:======================================================================================================= 2023-10-09T05:11:37.785 INFO:teuthology.orchestra.run.smithi191.stdout:Install 13 Packages 2023-10-09T05:11:37.785 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:37.786 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 1.9 M 2023-10-09T05:11:37.786 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 24 M 2023-10-09T05:11:37.786 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:11:38.064 INFO:teuthology.orchestra.run.smithi191.stdout:(1/13): ceph-mgr-rook-16.2.14-48.gaeb2e425.el8. 196 kB/s | 27 kB 00:00 2023-10-09T05:11:38.215 INFO:teuthology.orchestra.run.smithi191.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 56 kB/s | 16 kB 00:00 2023-10-09T05:11:38.240 INFO:teuthology.orchestra.run.smithi191.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 104 kB/s | 33 kB 00:00 2023-10-09T05:11:38.448 INFO:teuthology.orchestra.run.smithi191.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 211 kB/s | 81 kB 00:00 2023-10-09T05:11:38.474 INFO:teuthology.orchestra.run.smithi191.stdout:(5/13): python3-rsa-4.9-2.el8.noarch.rpm 275 kB/s | 64 kB 00:00 2023-10-09T05:11:38.608 INFO:teuthology.orchestra.run.smithi191.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 386 kB/s | 61 kB 00:00 2023-10-09T05:11:38.708 INFO:teuthology.orchestra.run.smithi191.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 182 kB/s | 43 kB 00:00 2023-10-09T05:11:38.742 INFO:teuthology.orchestra.run.smithi191.stdout:(8/13): python3-kubernetes-11.0.0-6.el8.noarch. 2.2 MB/s | 1.2 MB 00:00 2023-10-09T05:11:38.767 INFO:teuthology.orchestra.run.smithi191.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 973 kB/s | 155 kB 00:00 2023-10-09T05:11:38.926 INFO:teuthology.orchestra.run.smithi191.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 505 kB/s | 110 kB 00:00 2023-10-09T05:11:39.070 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T05:11:39.234 INFO:teuthology.orchestra.run.smithi191.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 54 kB/s | 27 kB 00:00 2023-10-09T05:11:39.260 INFO:teuthology.orchestra.run.smithi191.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 259 kB/s | 126 kB 00:00 2023-10-09T05:11:39.261 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 8/8 2023-10-09T05:11:39.368 INFO:teuthology.orchestra.run.smithi191.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 45 kB/s | 20 kB 00:00 2023-10-09T05:11:39.369 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:11:39.369 INFO:teuthology.orchestra.run.smithi191.stdout:Total 1.2 MB/s | 1.9 MB 00:01 2023-10-09T05:11:39.435 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:11:39.451 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:11:39.451 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:11:39.522 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:11:39.527 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 8/8 2023-10-09T05:11:39.527 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425 1/8 2023-10-09T05:11:39.527 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:11:39.527 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-10-09T05:11:39.527 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-10-09T05:11:39.527 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T05:11:39.528 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:11:39.528 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-10-09T05:11:39.528 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:11:40.071 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:11:40.341 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T05:11:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T05:11:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-10-09T05:11:40.718 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:02:41 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:40.823 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T05:11:41.026 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-10-09T05:11:41.209 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-10-09T05:11:41.446 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-10-09T05:11:41.711 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-10-09T05:11:41.923 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T05:11:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T05:11:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T05:11:43.450 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T05:11:43.698 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T05:11:43.893 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 13/13 2023-10-09T05:11:44.128 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 13/13 2023-10-09T05:11:44.128 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 1/13 2023-10-09T05:11:44.128 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T05:11:44.128 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T05:11:44.128 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T05:11:44.129 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T05:11:44.129 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T05:11:44.129 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T05:11:44.129 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-10-09T05:11:44.129 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-10-09T05:11:44.129 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-10-09T05:11:44.129 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-10-09T05:11:44.129 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-10-09T05:11:45.235 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-10-09T05:11:45.235 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:46 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:47.150 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:11:47.352 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:47.352 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:11:47.352 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr-diskprediction-local-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:11:47.352 INFO:teuthology.orchestra.run.smithi008.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:11:47.353 INFO:teuthology.orchestra.run.smithi008.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:11:47.353 INFO:teuthology.orchestra.run.smithi008.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T05:11:47.353 INFO:teuthology.orchestra.run.smithi008.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T05:11:47.353 INFO:teuthology.orchestra.run.smithi008.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T05:11:47.353 INFO:teuthology.orchestra.run.smithi008.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T05:11:47.353 INFO:teuthology.orchestra.run.smithi008.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T05:11:47.353 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:47.353 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:11:47.490 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install ceph-mgr-rook 2023-10-09T05:11:47.859 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:11:49.412 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:02:50 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:50.946 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:11:50.947 INFO:teuthology.orchestra.run.smithi008.stdout:======================================================================================================= 2023-10-09T05:11:50.947 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repository Size 2023-10-09T05:11:50.948 INFO:teuthology.orchestra.run.smithi008.stdout:======================================================================================================= 2023-10-09T05:11:50.948 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:11:50.948 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr-rook noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 27 k 2023-10-09T05:11:50.948 INFO:teuthology.orchestra.run.smithi008.stdout:Installing dependencies: 2023-10-09T05:11:50.948 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T05:11:50.948 INFO:teuthology.orchestra.run.smithi008.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T05:11:50.948 INFO:teuthology.orchestra.run.smithi008.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T05:11:50.948 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T05:11:50.948 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T05:11:50.948 INFO:teuthology.orchestra.run.smithi008.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T05:11:50.949 INFO:teuthology.orchestra.run.smithi008.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T05:11:50.949 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T05:11:50.949 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T05:11:50.949 INFO:teuthology.orchestra.run.smithi008.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:11:50.949 INFO:teuthology.orchestra.run.smithi008.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T05:11:50.949 INFO:teuthology.orchestra.run.smithi008.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T05:11:50.949 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:50.949 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:11:50.949 INFO:teuthology.orchestra.run.smithi008.stdout:======================================================================================================= 2023-10-09T05:11:50.949 INFO:teuthology.orchestra.run.smithi008.stdout:Install 13 Packages 2023-10-09T05:11:50.949 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:11:50.950 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 1.9 M 2023-10-09T05:11:50.950 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 24 M 2023-10-09T05:11:50.950 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:11:51.018 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:11:51.202 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:51.202 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:11:51.202 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:11:51.202 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T05:11:51.202 INFO:teuthology.orchestra.run.smithi191.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T05:11:51.203 INFO:teuthology.orchestra.run.smithi191.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T05:11:51.203 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T05:11:51.203 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T05:11:51.203 INFO:teuthology.orchestra.run.smithi191.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T05:11:51.203 INFO:teuthology.orchestra.run.smithi191.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T05:11:51.203 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T05:11:51.203 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T05:11:51.204 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T05:11:51.204 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T05:11:51.204 INFO:teuthology.orchestra.run.smithi191.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T05:11:51.204 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:51.204 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:11:51.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T05:11:51.527 INFO:teuthology.orchestra.run.smithi008.stdout:(1/13): ceph-mgr-rook-16.2.14-48.gaeb2e425.el8. 117 kB/s | 27 kB 00:00 2023-10-09T05:11:51.698 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:11:51.744 INFO:teuthology.orchestra.run.smithi008.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 36 kB/s | 16 kB 00:00 2023-10-09T05:11:51.786 INFO:teuthology.orchestra.run.smithi008.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 67 kB/s | 33 kB 00:00 2023-10-09T05:11:52.028 INFO:teuthology.orchestra.run.smithi008.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 266 kB/s | 64 kB 00:00 2023-10-09T05:11:52.103 INFO:teuthology.orchestra.run.smithi008.stdout:(5/13): python3-google-auth-1.1.1-10.el8.noarch 141 kB/s | 81 kB 00:00 2023-10-09T05:11:52.187 INFO:teuthology.orchestra.run.smithi008.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 386 kB/s | 61 kB 00:00 2023-10-09T05:11:52.287 INFO:teuthology.orchestra.run.smithi008.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 232 kB/s | 43 kB 00:00 2023-10-09T05:11:52.320 INFO:teuthology.orchestra.run.smithi008.stdout:(8/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.1 MB/s | 155 kB 00:00 2023-10-09T05:11:52.404 INFO:teuthology.orchestra.run.smithi008.stdout:(9/13): python3-pyasn1-modules-0.3.7-6.el8.noar 943 kB/s | 110 kB 00:00 2023-10-09T05:11:52.446 INFO:teuthology.orchestra.run.smithi008.stdout:(10/13): python3-kubernetes-11.0.0-6.el8.noarch 1.7 MB/s | 1.2 MB 00:00 2023-10-09T05:11:52.489 INFO:teuthology.orchestra.run.smithi008.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 158 kB/s | 27 kB 00:00 2023-10-09T05:11:52.547 INFO:teuthology.orchestra.run.smithi008.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 887 kB/s | 126 kB 00:00 2023-10-09T05:11:52.597 INFO:teuthology.orchestra.run.smithi008.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 133 kB/s | 20 kB 00:00 2023-10-09T05:11:52.600 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:11:52.600 INFO:teuthology.orchestra.run.smithi008.stdout:Total 1.2 MB/s | 1.9 MB 00:01 2023-10-09T05:11:52.670 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:11:52.687 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:11:52.687 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:11:52.766 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:11:52.770 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:11:53.103 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:54 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:11:53.587 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T05:11:53.799 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T05:11:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T05:11:54.159 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-10-09T05:11:54.354 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-10-09T05:11:54.534 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:11:54.535 INFO:teuthology.orchestra.run.smithi191.stdout:============================================================================================== 2023-10-09T05:11:54.536 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-10-09T05:11:54.536 INFO:teuthology.orchestra.run.smithi191.stdout:============================================================================================== 2023-10-09T05:11:54.536 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:11:54.536 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-cephadm noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 116 k 2023-10-09T05:11:54.536 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-10-09T05:11:54.536 INFO:teuthology.orchestra.run.smithi191.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-10-09T05:11:54.536 INFO:teuthology.orchestra.run.smithi191.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T05:11:54.536 INFO:teuthology.orchestra.run.smithi191.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-10-09T05:11:54.536 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T05:11:54.536 INFO:teuthology.orchestra.run.smithi191.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-10-09T05:11:54.536 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:54.537 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:11:54.537 INFO:teuthology.orchestra.run.smithi191.stdout:============================================================================================== 2023-10-09T05:11:54.537 INFO:teuthology.orchestra.run.smithi191.stdout:Install 6 Packages 2023-10-09T05:11:54.537 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:11:54.537 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 5.8 M 2023-10-09T05:11:54.537 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 24 M 2023-10-09T05:11:54.538 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:11:54.566 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-10-09T05:11:54.748 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-10-09T05:11:54.926 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T05:11:54.932 INFO:teuthology.orchestra.run.smithi191.stdout:(1/6): python3-apipkg-1.5-6.el8.noarch.rpm 85 kB/s | 21 kB 00:00 2023-10-09T05:11:54.966 INFO:teuthology.orchestra.run.smithi191.stdout:(2/6): ceph-mgr-cephadm-16.2.14-48.gaeb2e425.el 411 kB/s | 116 kB 00:00 2023-10-09T05:11:55.049 INFO:teuthology.orchestra.run.smithi191.stdout:(3/6): python3-remoto-1.2.1-1.el8.noarch.rpm 417 kB/s | 49 kB 00:00 2023-10-09T05:11:55.091 INFO:teuthology.orchestra.run.smithi191.stdout:(4/6): python3-execnet-1.7.1-1.el8.noarch.rpm 871 kB/s | 355 kB 00:00 2023-10-09T05:11:55.101 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T05:11:55.216 INFO:teuthology.orchestra.run.smithi191.stdout:(5/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.1 MB/s | 538 kB 00:00 2023-10-09T05:11:55.312 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T05:11:55.467 INFO:teuthology.orchestra.run.smithi191.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 11 MB/s | 4.8 MB 00:00 2023-10-09T05:11:55.469 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:11:55.469 INFO:teuthology.orchestra.run.smithi191.stdout:Total 6.3 MB/s | 5.8 MB 00:00 2023-10-09T05:11:55.523 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:11:55.533 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:11:55.533 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:11:55.612 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:11:55.615 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:11:56.421 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T05:11:56.617 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:11:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T05:11:56.759 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 13/13 2023-10-09T05:11:57.013 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 13/13 2023-10-09T05:11:57.013 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 1/13 2023-10-09T05:11:57.013 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T05:11:57.013 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T05:11:57.013 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T05:11:57.013 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T05:11:57.014 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T05:11:57.014 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T05:11:57.014 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-10-09T05:11:57.014 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-10-09T05:11:57.014 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-10-09T05:11:57.014 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-10-09T05:11:57.014 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-10-09T05:11:57.092 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-10-09T05:11:57.378 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-10-09T05:11:57.691 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-10-09T05:11:58.085 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-10-09T05:11:58.193 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-10-09T05:11:58.193 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:02:59 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:11:58.819 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-10-09T05:11:59.830 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T05:12:00.080 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T05:12:00.080 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/6 2023-10-09T05:12:00.080 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-10-09T05:12:00.080 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-10-09T05:12:00.080 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-10-09T05:12:00.080 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-10-09T05:12:01.142 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-10-09T05:12:01.142 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:02 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:02.431 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:12:02.633 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:02.633 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:12:02.633 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr-rook-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:12:02.634 INFO:teuthology.orchestra.run.smithi008.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T05:12:02.634 INFO:teuthology.orchestra.run.smithi008.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T05:12:02.634 INFO:teuthology.orchestra.run.smithi008.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T05:12:02.634 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T05:12:02.634 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T05:12:02.634 INFO:teuthology.orchestra.run.smithi008.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T05:12:02.634 INFO:teuthology.orchestra.run.smithi008.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T05:12:02.634 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T05:12:02.634 INFO:teuthology.orchestra.run.smithi008.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T05:12:02.635 INFO:teuthology.orchestra.run.smithi008.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T05:12:02.635 INFO:teuthology.orchestra.run.smithi008.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T05:12:02.635 INFO:teuthology.orchestra.run.smithi008.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T05:12:02.635 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:02.635 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:12:02.780 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T05:12:03.153 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:03.917 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:12:04.102 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:04.102 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:12:04.103 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:12:04.103 INFO:teuthology.orchestra.run.smithi191.stdout: python3-apipkg-1.5-6.el8.noarch 2023-10-09T05:12:04.103 INFO:teuthology.orchestra.run.smithi191.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T05:12:04.103 INFO:teuthology.orchestra.run.smithi191.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-10-09T05:12:04.103 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T05:12:04.103 INFO:teuthology.orchestra.run.smithi191.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-10-09T05:12:04.103 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:04.103 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:12:04.140 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:05 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:04.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-fuse 2023-10-09T05:12:04.613 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:05.679 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:12:05.680 INFO:teuthology.orchestra.run.smithi008.stdout:============================================================================================== 2023-10-09T05:12:05.680 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repository Size 2023-10-09T05:12:05.680 INFO:teuthology.orchestra.run.smithi008.stdout:============================================================================================== 2023-10-09T05:12:05.680 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:12:05.680 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr-cephadm noarch 2:16.2.14-48.gaeb2e425.el8 ceph-noarch 116 k 2023-10-09T05:12:05.680 INFO:teuthology.orchestra.run.smithi008.stdout:Installing dependencies: 2023-10-09T05:12:05.680 INFO:teuthology.orchestra.run.smithi008.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-10-09T05:12:05.680 INFO:teuthology.orchestra.run.smithi008.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T05:12:05.680 INFO:teuthology.orchestra.run.smithi008.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-10-09T05:12:05.680 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T05:12:05.681 INFO:teuthology.orchestra.run.smithi008.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-10-09T05:12:05.681 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:05.681 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:12:05.681 INFO:teuthology.orchestra.run.smithi008.stdout:============================================================================================== 2023-10-09T05:12:05.681 INFO:teuthology.orchestra.run.smithi008.stdout:Install 6 Packages 2023-10-09T05:12:05.681 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:05.681 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 5.8 M 2023-10-09T05:12:05.682 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 24 M 2023-10-09T05:12:05.682 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:12:06.057 INFO:teuthology.orchestra.run.smithi008.stdout:(1/6): ceph-mgr-cephadm-16.2.14-48.gaeb2e425.el 498 kB/s | 116 kB 00:00 2023-10-09T05:12:06.058 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:07 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:06.234 INFO:teuthology.orchestra.run.smithi008.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 52 kB/s | 21 kB 00:00 2023-10-09T05:12:06.426 INFO:teuthology.orchestra.run.smithi008.stdout:(3/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.8 MB/s | 538 kB 00:00 2023-10-09T05:12:06.476 INFO:teuthology.orchestra.run.smithi008.stdout:(4/6): python3-remoto-1.2.1-1.el8.noarch.rpm 117 kB/s | 49 kB 00:00 2023-10-09T05:12:06.543 INFO:teuthology.orchestra.run.smithi008.stdout:(5/6): python3-execnet-1.7.1-1.el8.noarch.rpm 494 kB/s | 355 kB 00:00 2023-10-09T05:12:06.669 INFO:teuthology.orchestra.run.smithi008.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 20 MB/s | 4.8 MB 00:00 2023-10-09T05:12:06.671 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:12:06.672 INFO:teuthology.orchestra.run.smithi008.stdout:Total 5.9 MB/s | 5.8 MB 00:00 2023-10-09T05:12:06.728 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:12:06.739 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:12:06.740 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:12:06.826 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:12:06.830 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:12:07.420 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:12:07.421 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:07.421 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-10-09T05:12:07.421 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:07.421 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:12:07.421 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-fuse 2023-10-09T05:12:07.421 INFO:teuthology.orchestra.run.smithi191.stdout: x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 910 k 2023-10-09T05:12:07.421 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-10-09T05:12:07.421 INFO:teuthology.orchestra.run.smithi191.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T05:12:07.422 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:07.422 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:12:07.422 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:07.422 INFO:teuthology.orchestra.run.smithi191.stdout:Install 2 Packages 2023-10-09T05:12:07.422 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:07.423 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 993 k 2023-10-09T05:12:07.423 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 2.8 M 2023-10-09T05:12:07.423 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:12:07.666 INFO:teuthology.orchestra.run.smithi191.stdout:(1/2): ceph-fuse-16.2.14-48.gaeb2e425.el8.x86_6 3.7 MB/s | 910 kB 00:00 2023-10-09T05:12:07.725 INFO:teuthology.orchestra.run.smithi191.stdout:(2/2): fuse-2.9.7-15.el8.x86_64.rpm 274 kB/s | 83 kB 00:00 2023-10-09T05:12:07.725 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:12:07.726 INFO:teuthology.orchestra.run.smithi191.stdout:Total 3.2 MB/s | 993 kB 00:00 2023-10-09T05:12:07.736 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:12:07.757 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:12:07.757 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:12:07.833 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:12:07.837 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:12:07.859 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:12:08.006 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:12:08.289 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-10-09T05:12:08.311 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T05:12:08.470 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2/2 2023-10-09T05:12:08.489 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-10-09T05:12:08.772 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-10-09T05:12:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2/2 2023-10-09T05:12:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/2 2023-10-09T05:12:08.996 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-10-09T05:12:09.193 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-10-09T05:12:09.391 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T05:12:09.645 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 6/6 2023-10-09T05:12:09.645 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 1/6 2023-10-09T05:12:09.646 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-10-09T05:12:09.646 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-10-09T05:12:09.646 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-10-09T05:12:09.646 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-10-09T05:12:09.898 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T05:12:09.898 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:10 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-10-09T05:12:10.762 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:11 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:13.502 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:12:13.686 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:13.686 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:12:13.686 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T05:12:13.686 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:13.686 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:12:13.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install librados-devel 2023-10-09T05:12:14.178 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:14.238 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:12:14.444 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:14.444 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:12:14.445 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-mgr-cephadm-2:16.2.14-48.gaeb2e425.el8.noarch 2023-10-09T05:12:14.445 INFO:teuthology.orchestra.run.smithi008.stdout: python3-apipkg-1.5-6.el8.noarch 2023-10-09T05:12:14.445 INFO:teuthology.orchestra.run.smithi008.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T05:12:14.445 INFO:teuthology.orchestra.run.smithi008.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-10-09T05:12:14.445 INFO:teuthology.orchestra.run.smithi008.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T05:12:14.445 INFO:teuthology.orchestra.run.smithi008.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-10-09T05:12:14.445 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:14.445 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:12:14.603 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install ceph-fuse 2023-10-09T05:12:14.979 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:15.019 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:16 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:15.886 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:16 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:16.396 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:12:16.396 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:16.396 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repo Size 2023-10-09T05:12:16.397 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:16.397 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:12:16.397 INFO:teuthology.orchestra.run.smithi191.stdout: librados-devel x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 131 k 2023-10-09T05:12:16.397 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:16.397 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:12:16.397 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:16.397 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-10-09T05:12:16.397 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:16.398 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 131 k 2023-10-09T05:12:16.398 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 432 k 2023-10-09T05:12:16.398 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:12:16.612 INFO:teuthology.orchestra.run.smithi191.stdout:librados-devel-16.2.14-48.gaeb2e425.el8.x86_64. 613 kB/s | 131 kB 00:00 2023-10-09T05:12:16.613 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:12:16.613 INFO:teuthology.orchestra.run.smithi191.stdout:Total 610 kB/s | 131 kB 00:00 2023-10-09T05:12:16.613 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:12:16.618 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:12:16.618 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:12:16.672 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:12:16.675 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:12:16.790 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:12:16.982 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:17.218 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:17.414 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:12:17.414 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:12:17.414 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repository Size 2023-10-09T05:12:17.414 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:12:17.414 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:12:17.415 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-fuse 2023-10-09T05:12:17.415 INFO:teuthology.orchestra.run.smithi008.stdout: x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 910 k 2023-10-09T05:12:17.415 INFO:teuthology.orchestra.run.smithi008.stdout:Installing dependencies: 2023-10-09T05:12:17.415 INFO:teuthology.orchestra.run.smithi008.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T05:12:17.415 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:17.415 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:12:17.415 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:12:17.415 INFO:teuthology.orchestra.run.smithi008.stdout:Install 2 Packages 2023-10-09T05:12:17.415 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:17.416 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 993 k 2023-10-09T05:12:17.416 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 2.8 M 2023-10-09T05:12:17.416 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:12:17.588 INFO:teuthology.orchestra.run.smithi008.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 480 kB/s | 83 kB 00:00 2023-10-09T05:12:18.282 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:18.282 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:19 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:18.339 INFO:teuthology.orchestra.run.smithi008.stdout:(2/2): ceph-fuse-16.2.14-48.gaeb2e425.el8.x86_6 987 kB/s | 910 kB 00:00 2023-10-09T05:12:18.339 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:12:18.339 INFO:teuthology.orchestra.run.smithi008.stdout:Total 1.0 MB/s | 993 kB 00:00 2023-10-09T05:12:18.352 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:12:18.375 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:12:18.375 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:12:18.461 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:12:18.465 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:12:18.639 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:12:18.952 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T05:12:19.088 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2/2 2023-10-09T05:12:19.432 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2/2 2023-10-09T05:12:19.432 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/2 2023-10-09T05:12:20.537 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T05:12:20.537 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:21 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:22.392 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:12:22.577 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:22.577 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:12:22.577 INFO:teuthology.orchestra.run.smithi191.stdout: librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:12:22.577 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:22.577 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:12:22.709 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install libcephfs2 2023-10-09T05:12:23.067 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:23.390 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:12:23.589 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:23.589 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:12:23.589 INFO:teuthology.orchestra.run.smithi008.stdout: ceph-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T05:12:23.589 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:23.590 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:12:23.729 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install librados-devel 2023-10-09T05:12:23.940 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:24 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:24.105 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:24.992 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:25 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:25.230 INFO:teuthology.orchestra.run.smithi191.stdout:Package libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:12:25.337 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:12:25.338 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-10-09T05:12:25.338 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:12:25.467 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install libcephfs-devel 2023-10-09T05:12:25.821 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:26.534 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:12:26.534 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:12:26.535 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repo Size 2023-10-09T05:12:26.535 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:12:26.535 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:12:26.535 INFO:teuthology.orchestra.run.smithi008.stdout: librados-devel x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 131 k 2023-10-09T05:12:26.535 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:26.535 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:12:26.535 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:12:26.535 INFO:teuthology.orchestra.run.smithi008.stdout:Install 1 Package 2023-10-09T05:12:26.535 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:26.536 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 131 k 2023-10-09T05:12:26.536 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 432 k 2023-10-09T05:12:26.536 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:12:26.727 INFO:teuthology.orchestra.run.smithi008.stdout:librados-devel-16.2.14-48.gaeb2e425.el8.x86_64. 687 kB/s | 131 kB 00:00 2023-10-09T05:12:26.727 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:12:26.728 INFO:teuthology.orchestra.run.smithi008.stdout:Total 682 kB/s | 131 kB 00:00 2023-10-09T05:12:26.728 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:12:26.734 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:12:26.734 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:12:26.794 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:12:26.798 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:12:26.912 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:12:27.078 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:27.288 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:28 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:27.342 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:28.538 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:28.538 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:29 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:28.646 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:12:28.647 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repo Size 2023-10-09T05:12:28.647 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:28.648 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:12:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs-devel x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 25 k 2023-10-09T05:12:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:28.648 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:12:28.648 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:28.649 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-10-09T05:12:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:28.649 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 25 k 2023-10-09T05:12:28.649 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 107 k 2023-10-09T05:12:28.649 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:12:28.799 INFO:teuthology.orchestra.run.smithi191.stdout:libcephfs-devel-16.2.14-48.gaeb2e425.el8.x86_64 168 kB/s | 25 kB 00:00 2023-10-09T05:12:28.800 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:12:28.800 INFO:teuthology.orchestra.run.smithi191.stdout:Total 167 kB/s | 25 kB 00:00 2023-10-09T05:12:28.800 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:12:28.804 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:12:28.804 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:12:28.806 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:12:28.810 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:12:28.860 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:12:29.039 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:29.239 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:30.252 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:30.252 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:31 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:31.892 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:12:32.090 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:32.090 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:12:32.090 INFO:teuthology.orchestra.run.smithi008.stdout: librados-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:12:32.090 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:32.091 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:12:32.226 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install libcephfs2 2023-10-09T05:12:32.602 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:34.186 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:35 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:34.302 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:12:34.486 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:34.487 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:12:34.487 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:12:34.487 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:34.487 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:12:34.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install librados2 2023-10-09T05:12:35.002 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:35.556 INFO:teuthology.orchestra.run.smithi008.stdout:Package libcephfs2-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:12:35.676 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:12:35.678 INFO:teuthology.orchestra.run.smithi008.stdout:Nothing to do. 2023-10-09T05:12:35.678 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:12:35.797 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install libcephfs-devel 2023-10-09T05:12:36.171 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:36.538 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:37 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:37.791 INFO:teuthology.orchestra.run.smithi191.stdout:Package librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:12:37.795 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:38 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:37.900 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:12:37.901 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-10-09T05:12:37.901 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:12:38.023 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install librbd1 2023-10-09T05:12:38.377 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:39.319 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:12:39.319 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:12:39.320 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repo Size 2023-10-09T05:12:39.320 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:12:39.320 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:12:39.320 INFO:teuthology.orchestra.run.smithi008.stdout: libcephfs-devel x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 25 k 2023-10-09T05:12:39.320 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:39.320 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:12:39.320 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:12:39.320 INFO:teuthology.orchestra.run.smithi008.stdout:Install 1 Package 2023-10-09T05:12:39.320 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:39.321 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 25 k 2023-10-09T05:12:39.321 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 107 k 2023-10-09T05:12:39.321 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:12:39.467 INFO:teuthology.orchestra.run.smithi008.stdout:libcephfs-devel-16.2.14-48.gaeb2e425.el8.x86_64 174 kB/s | 25 kB 00:00 2023-10-09T05:12:39.468 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:12:39.468 INFO:teuthology.orchestra.run.smithi008.stdout:Total 173 kB/s | 25 kB 00:00 2023-10-09T05:12:39.468 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:12:39.472 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:12:39.472 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:12:39.475 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:12:39.479 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:12:39.526 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:12:39.693 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:39.845 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:40 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:39.922 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:40.996 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:40.996 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:41 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:41.094 INFO:teuthology.orchestra.run.smithi191.stdout:Package librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:12:41.202 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:12:41.203 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-10-09T05:12:41.203 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:12:41.324 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-rados 2023-10-09T05:12:41.676 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:42.602 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:43 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:43.854 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:12:43.964 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:12:43.965 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-10-09T05:12:43.965 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:12:44.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-rgw 2023-10-09T05:12:44.441 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:45.927 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:46 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:47.172 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:12:47.281 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:12:47.282 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-10-09T05:12:47.282 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:12:47.401 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-cephfs 2023-10-09T05:12:47.761 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:48.012 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:12:48.213 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:48.213 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:12:48.213 INFO:teuthology.orchestra.run.smithi008.stdout: libcephfs-devel-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:12:48.213 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:12:48.213 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:12:48.373 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install librados2 2023-10-09T05:12:48.750 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:49.257 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:50 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:50.303 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:51 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:50.537 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:12:50.645 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:12:50.646 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-10-09T05:12:50.646 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:12:50.772 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-rbd 2023-10-09T05:12:51.126 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:51.714 INFO:teuthology.orchestra.run.smithi008.stdout:Package librados2-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:12:51.835 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:12:51.836 INFO:teuthology.orchestra.run.smithi008.stdout:Nothing to do. 2023-10-09T05:12:51.836 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:12:51.960 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install librbd1 2023-10-09T05:12:52.341 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:52.559 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:53 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:53.470 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:54 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:53.801 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:12:53.909 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:12:53.910 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-10-09T05:12:53.911 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:12:54.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install rbd-fuse 2023-10-09T05:12:54.381 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:54.873 INFO:teuthology.orchestra.run.smithi008.stdout:Package librbd1-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:12:54.990 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:12:54.992 INFO:teuthology.orchestra.run.smithi008.stdout:Nothing to do. 2023-10-09T05:12:54.992 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:12:55.099 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install python3-rados 2023-10-09T05:12:55.488 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:55.975 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:03:56 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:57.102 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:03:58 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:12:57.333 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:12:57.334 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:57.335 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2023-10-09T05:12:57.335 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:57.335 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:12:57.335 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-fuse x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 88 k 2023-10-09T05:12:57.335 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:57.336 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:12:57.336 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:12:57.336 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-10-09T05:12:57.336 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:12:57.336 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 88 k 2023-10-09T05:12:57.336 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 240 k 2023-10-09T05:12:57.336 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:12:57.603 INFO:teuthology.orchestra.run.smithi191.stdout:rbd-fuse-16.2.14-48.gaeb2e425.el8.x86_64.rpm 329 kB/s | 88 kB 00:00 2023-10-09T05:12:57.604 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:12:57.604 INFO:teuthology.orchestra.run.smithi191.stdout:Total 328 kB/s | 88 kB 00:00 2023-10-09T05:12:57.604 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:12:57.614 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:12:57.614 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:12:57.670 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:12:57.673 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:12:57.779 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:12:58.009 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:58.273 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:58.509 INFO:teuthology.orchestra.run.smithi008.stdout:Package python3-rados-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:12:58.629 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:12:58.630 INFO:teuthology.orchestra.run.smithi008.stdout:Nothing to do. 2023-10-09T05:12:58.630 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:12:58.749 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install python3-rgw 2023-10-09T05:12:59.126 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:12:59.385 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:12:59.385 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:04:00 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:00.647 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:04:01 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:02.063 INFO:teuthology.orchestra.run.smithi008.stdout:Package python3-rgw-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:13:02.185 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:13:02.186 INFO:teuthology.orchestra.run.smithi008.stdout:Nothing to do. 2023-10-09T05:13:02.187 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:13:02.309 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install python3-cephfs 2023-10-09T05:13:02.684 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:13:04.339 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:04:05 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:05.102 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:13:05.286 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:13:05.286 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:13:05.286 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:13:05.286 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:13:05.286 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:13:05.452 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install rbd-mirror 2023-10-09T05:13:05.747 INFO:teuthology.orchestra.run.smithi008.stdout:Package python3-cephfs-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:13:05.808 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:13:05.865 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:13:05.866 INFO:teuthology.orchestra.run.smithi008.stdout:Nothing to do. 2023-10-09T05:13:05.866 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:13:05.990 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install python3-rbd 2023-10-09T05:13:06.367 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:13:07.169 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:04:08 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:07.910 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:04:08 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:08.531 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:13:08.531 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:13:08.532 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repo Size 2023-10-09T05:13:08.532 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:13:08.532 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:13:08.532 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-mirror x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.7 M 2023-10-09T05:13:08.532 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:13:08.532 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:13:08.532 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:13:08.532 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-10-09T05:13:08.532 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:13:08.533 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 3.7 M 2023-10-09T05:13:08.533 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 15 M 2023-10-09T05:13:08.533 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:13:08.914 INFO:teuthology.orchestra.run.smithi191.stdout:rbd-mirror-16.2.14-48.gaeb2e425.el8.x86_64.rpm 9.6 MB/s | 3.7 MB 00:00 2023-10-09T05:13:08.915 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:13:08.915 INFO:teuthology.orchestra.run.smithi191.stdout:Total 9.6 MB/s | 3.7 MB 00:00 2023-10-09T05:13:08.915 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:13:08.927 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:13:08.927 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:13:08.996 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:13:08.999 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:13:09.320 INFO:teuthology.orchestra.run.smithi008.stdout:Package python3-rbd-2:16.2.14-48.gaeb2e425.el8.x86_64 is already installed. 2023-10-09T05:13:09.440 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:13:09.441 INFO:teuthology.orchestra.run.smithi008.stdout:Nothing to do. 2023-10-09T05:13:09.441 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:13:09.442 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:13:09.564 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install rbd-fuse 2023-10-09T05:13:09.626 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:09.938 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:13:09.985 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:11.026 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:11.026 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:04:12 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:11.567 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:04:12 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:13.056 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:13:13.057 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:13:13.057 INFO:teuthology.orchestra.run.smithi008.stdout: Package Architecture Version Repository Size 2023-10-09T05:13:13.057 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:13:13.057 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:13:13.058 INFO:teuthology.orchestra.run.smithi008.stdout: rbd-fuse x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 88 k 2023-10-09T05:13:13.058 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:13.058 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:13:13.058 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:13:13.058 INFO:teuthology.orchestra.run.smithi008.stdout:Install 1 Package 2023-10-09T05:13:13.058 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:13.059 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 88 k 2023-10-09T05:13:13.059 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 240 k 2023-10-09T05:13:13.059 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:13:13.389 INFO:teuthology.orchestra.run.smithi008.stdout:rbd-fuse-16.2.14-48.gaeb2e425.el8.x86_64.rpm 267 kB/s | 88 kB 00:00 2023-10-09T05:13:13.390 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:13:13.390 INFO:teuthology.orchestra.run.smithi008.stdout:Total 266 kB/s | 88 kB 00:00 2023-10-09T05:13:13.390 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:13:13.401 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:13:13.401 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:13:13.459 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:13:13.463 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:13:13.574 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:13:13.739 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:13.986 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:15.065 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:15.066 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:04:16 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:15.970 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:13:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:13:16.152 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:13:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:13:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:13:16.153 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:13:16.311 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install rbd-nbd 2023-10-09T05:13:16.667 INFO:teuthology.orchestra.run.smithi191.stdout:Updating Subscription Management repositories. 2023-10-09T05:13:17.889 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:04:18 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:19.274 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-10-09T05:13:19.275 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:13:19.275 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2023-10-09T05:13:19.275 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:13:19.275 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-10-09T05:13:19.276 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-nbd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 173 k 2023-10-09T05:13:19.276 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:13:19.276 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-10-09T05:13:19.276 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-10-09T05:13:19.276 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-10-09T05:13:19.276 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:13:19.276 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 173 k 2023-10-09T05:13:19.276 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 515 k 2023-10-09T05:13:19.277 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-10-09T05:13:19.567 INFO:teuthology.orchestra.run.smithi191.stdout:rbd-nbd-16.2.14-48.gaeb2e425.el8.x86_64.rpm 597 kB/s | 173 kB 00:00 2023-10-09T05:13:19.567 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:13:19.568 INFO:teuthology.orchestra.run.smithi191.stdout:Total 594 kB/s | 173 kB 00:00 2023-10-09T05:13:19.568 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-10-09T05:13:19.579 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-10-09T05:13:19.579 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-10-09T05:13:19.633 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-10-09T05:13:19.637 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-10-09T05:13:19.766 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-10-09T05:13:19.963 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:20.189 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:20.676 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:13:20.876 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:20.876 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:13:20.876 INFO:teuthology.orchestra.run.smithi008.stdout: rbd-fuse-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:13:20.876 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:20.877 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:13:21.030 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install rbd-mirror 2023-10-09T05:13:21.280 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:21.280 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:04:22 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:21.408 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:13:22.969 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:04:23 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:24.489 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:13:24.490 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:13:24.490 INFO:teuthology.orchestra.run.smithi008.stdout: Package Arch Version Repo Size 2023-10-09T05:13:24.490 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:13:24.490 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:13:24.490 INFO:teuthology.orchestra.run.smithi008.stdout: rbd-mirror x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 3.7 M 2023-10-09T05:13:24.490 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:24.490 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:13:24.491 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:13:24.491 INFO:teuthology.orchestra.run.smithi008.stdout:Install 1 Package 2023-10-09T05:13:24.491 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:24.491 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 3.7 M 2023-10-09T05:13:24.491 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 15 M 2023-10-09T05:13:24.491 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:13:25.068 INFO:teuthology.orchestra.run.smithi008.stdout:rbd-mirror-16.2.14-48.gaeb2e425.el8.x86_64.rpm 6.4 MB/s | 3.7 MB 00:00 2023-10-09T05:13:25.068 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:13:25.069 INFO:teuthology.orchestra.run.smithi008.stdout:Total 6.4 MB/s | 3.7 MB 00:00 2023-10-09T05:13:25.069 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:13:25.082 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:13:25.082 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:13:25.155 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:13:25.159 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:13:25.625 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:13:25.782 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:26.159 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:26.477 INFO:teuthology.orchestra.run.smithi191.stdout:Installed products updated. 2023-10-09T05:13:26.658 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:13:26.659 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-10-09T05:13:26.659 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:13:26.659 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:13:26.659 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-10-09T05:13:26.806 DEBUG:teuthology.parallel:result is None 2023-10-09T05:13:27.280 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:27.280 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:04:28 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:31.992 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:13:32.192 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:32.193 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:13:32.193 INFO:teuthology.orchestra.run.smithi008.stdout: rbd-mirror-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:13:32.193 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:32.193 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:13:32.333 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum -y install rbd-nbd 2023-10-09T05:13:32.710 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2023-10-09T05:13:34.275 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:04:35 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:35.807 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2023-10-09T05:13:35.808 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:13:35.808 INFO:teuthology.orchestra.run.smithi008.stdout: Package Architecture Version Repository Size 2023-10-09T05:13:35.808 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:13:35.808 INFO:teuthology.orchestra.run.smithi008.stdout:Installing: 2023-10-09T05:13:35.808 INFO:teuthology.orchestra.run.smithi008.stdout: rbd-nbd x86_64 2:16.2.14-48.gaeb2e425.el8 ceph 173 k 2023-10-09T05:13:35.808 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:35.808 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction Summary 2023-10-09T05:13:35.809 INFO:teuthology.orchestra.run.smithi008.stdout:================================================================================ 2023-10-09T05:13:35.809 INFO:teuthology.orchestra.run.smithi008.stdout:Install 1 Package 2023-10-09T05:13:35.809 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:35.809 INFO:teuthology.orchestra.run.smithi008.stdout:Total download size: 173 k 2023-10-09T05:13:35.809 INFO:teuthology.orchestra.run.smithi008.stdout:Installed size: 515 k 2023-10-09T05:13:35.809 INFO:teuthology.orchestra.run.smithi008.stdout:Downloading Packages: 2023-10-09T05:13:35.988 INFO:teuthology.orchestra.run.smithi008.stdout:rbd-nbd-16.2.14-48.gaeb2e425.el8.x86_64.rpm 969 kB/s | 173 kB 00:00 2023-10-09T05:13:35.989 INFO:teuthology.orchestra.run.smithi008.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:13:35.989 INFO:teuthology.orchestra.run.smithi008.stdout:Total 962 kB/s | 173 kB 00:00 2023-10-09T05:13:35.989 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction check 2023-10-09T05:13:36.001 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction check succeeded. 2023-10-09T05:13:36.001 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction test 2023-10-09T05:13:36.060 INFO:teuthology.orchestra.run.smithi008.stdout:Transaction test succeeded. 2023-10-09T05:13:36.064 INFO:teuthology.orchestra.run.smithi008.stdout:Running transaction 2023-10-09T05:13:36.182 INFO:teuthology.orchestra.run.smithi008.stdout: Preparing : 1/1 2023-10-09T05:13:36.369 INFO:teuthology.orchestra.run.smithi008.stdout: Installing : rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: Running scriptlet: rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:37.773 INFO:teuthology.orchestra.run.smithi008.stdout: Verifying : rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 1/1 2023-10-09T05:13:37.773 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:04:38 ago on Mon 09 Oct 2023 05:08:59 AM UTC. 2023-10-09T05:13:42.943 INFO:teuthology.orchestra.run.smithi008.stdout:Installed products updated. 2023-10-09T05:13:43.142 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:43.143 INFO:teuthology.orchestra.run.smithi008.stdout:Installed: 2023-10-09T05:13:43.143 INFO:teuthology.orchestra.run.smithi008.stdout: rbd-nbd-2:16.2.14-48.gaeb2e425.el8.x86_64 2023-10-09T05:13:43.143 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:13:43.143 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2023-10-09T05:13:43.436 DEBUG:teuthology.parallel:result is None 2023-10-09T05:13:43.437 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:13:43.437 INFO:teuthology.packaging:ref: None 2023-10-09T05:13:43.437 INFO:teuthology.packaging:tag: None 2023-10-09T05:13:43.437 INFO:teuthology.packaging:branch: pacific 2023-10-09T05:13:43.437 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:13:43.437 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-09T05:13:43.749 DEBUG:teuthology.orchestra.run.smithi008:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T05:13:43.773 INFO:teuthology.orchestra.run.smithi008.stdout:16.2.14-48.gaeb2e425.el8 2023-10-09T05:13:43.773 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48.gaeb2e425.el8 2023-10-09T05:13:43.773 INFO:teuthology.task.install:The correct ceph version 16.2.14-48.gaeb2e425 is installed. 2023-10-09T05:13:43.774 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:13:43.774 INFO:teuthology.packaging:ref: None 2023-10-09T05:13:43.774 INFO:teuthology.packaging:tag: None 2023-10-09T05:13:43.774 INFO:teuthology.packaging:branch: pacific 2023-10-09T05:13:43.774 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:13:43.774 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-09T05:13:43.988 DEBUG:teuthology.orchestra.run.smithi191:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T05:13:44.014 INFO:teuthology.orchestra.run.smithi191.stdout:16.2.14-48.gaeb2e425.el8 2023-10-09T05:13:44.014 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48.gaeb2e425.el8 2023-10-09T05:13:44.014 INFO:teuthology.task.install:The correct ceph version 16.2.14-48.gaeb2e425 is installed. 2023-10-09T05:13:44.015 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T05:13:44.015 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:13:44.015 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T05:13:44.044 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T05:13:44.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T05:13:44.087 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T05:13:44.087 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:13:44.087 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T05:13:44.116 DEBUG:teuthology.orchestra.run.smithi008:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T05:13:44.185 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T05:13:44.185 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T05:13:44.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T05:13:44.283 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T05:13:44.283 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:13:44.283 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T05:13:44.311 DEBUG:teuthology.orchestra.run.smithi008:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T05:13:44.381 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T05:13:44.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T05:13:44.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T05:13:44.477 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T05:13:44.477 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:13:44.478 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T05:13:44.505 DEBUG:teuthology.orchestra.run.smithi008:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T05:13:44.573 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T05:13:44.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T05:13:44.602 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T05:13:44.670 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T05:13:44.745 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T05:13:44.745 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T05:13:44.745 INFO:tasks.cephadm:Cluster fsid is 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:13:44.745 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T05:13:44.746 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.8', 'mon.c': '[v2:172.21.15.8:3301,v1:172.21.15.8:6790]', 'mon.b': '172.21.15.191'} 2023-10-09T05:13:44.746 INFO:tasks.cephadm:First mon is mon.a on smithi008 2023-10-09T05:13:44.746 INFO:tasks.cephadm:First mgr is y 2023-10-09T05:13:44.746 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T05:13:44.746 DEBUG:teuthology.orchestra.run.smithi008:> sudo hostname $(hostname -s) 2023-10-09T05:13:44.775 DEBUG:teuthology.orchestra.run.smithi191:> sudo hostname $(hostname -s) 2023-10-09T05:13:44.803 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-10-09T05:13:44.803 DEBUG:teuthology.orchestra.run.smithi008:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T05:13:45.070 INFO:teuthology.orchestra.run.smithi008.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Oct 9 05:13 /home/ubuntu/cephtest/cephadm 2023-10-09T05:13:45.071 DEBUG:teuthology.orchestra.run.smithi191:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T05:13:45.210 INFO:teuthology.orchestra.run.smithi191.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Oct 9 05:13 /home/ubuntu/cephtest/cephadm 2023-10-09T05:13:45.210 DEBUG:teuthology.orchestra.run.smithi008:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T05:13:45.230 DEBUG:teuthology.orchestra.run.smithi191:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T05:13:45.259 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-10-09T05:13:45.259 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T05:13:45.273 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T05:13:46.427 INFO:teuthology.orchestra.run.smithi191.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T05:13:46.470 INFO:teuthology.orchestra.run.smithi008.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T05:14:43.172 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:14:43.173 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T05:14:43.173 INFO:teuthology.orchestra.run.smithi008.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T05:14:43.173 INFO:teuthology.orchestra.run.smithi008.stdout: "repo_digests": [ 2023-10-09T05:14:43.173 INFO:teuthology.orchestra.run.smithi008.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T05:14:43.173 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:14:43.173 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:14:47.686 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:14:47.686 INFO:teuthology.orchestra.run.smithi191.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T05:14:47.686 INFO:teuthology.orchestra.run.smithi191.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T05:14:47.686 INFO:teuthology.orchestra.run.smithi191.stdout: "repo_digests": [ 2023-10-09T05:14:47.686 INFO:teuthology.orchestra.run.smithi191.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T05:14:47.686 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:14:47.686 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:14:47.712 DEBUG:teuthology.orchestra.run.smithi008:> sudo mkdir -p /etc/ceph 2023-10-09T05:14:47.749 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /etc/ceph 2023-10-09T05:14:47.781 DEBUG:teuthology.orchestra.run.smithi008:> sudo chmod 777 /etc/ceph 2023-10-09T05:14:47.821 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod 777 /etc/ceph 2023-10-09T05:14:47.851 INFO:tasks.cephadm:Writing seed config... 2023-10-09T05:14:47.852 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T05:14:47.852 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T05:14:47.852 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T05:14:47.852 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T05:14:47.852 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T05:14:47.852 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-10-09T05:14:47.852 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T05:14:47.852 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T05:14:47.853 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T05:14:47.853 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T05:14:47.853 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T05:14:47.853 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:14:47.853 DEBUG:teuthology.orchestra.run.smithi008:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T05:14:47.881 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 9e85b874-6662-11ee-8db5-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T05:14:47.881 DEBUG:teuthology.orchestra.run.smithi008:mon.a> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T05:14:47.923 DEBUG:teuthology.orchestra.run.smithi008:mgr.y> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T05:14:47.952 INFO:journalctl@ceph.mon.a.smithi008.stdout:-- Logs begin at Mon 2023-10-09 04:57:44 UTC. -- 2023-10-09T05:14:47.953 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T05:14:47.953 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.8 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:14:47.981 INFO:journalctl@ceph.mgr.y.smithi008.stdout:-- Logs begin at Mon 2023-10-09 04:57:44 UTC. -- 2023-10-09T05:14:48.167 INFO:teuthology.orchestra.run.smithi008.stderr:-------------------------------------------------------------------------------- 2023-10-09T05:14:48.167 INFO:teuthology.orchestra.run.smithi008.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', '9e85b874-6662-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.8', '--skip-admin-label'] 2023-10-09T05:14:48.216 INFO:teuthology.orchestra.run.smithi008.stderr:/bin/podman: 3.0.1 2023-10-09T05:14:48.234 INFO:teuthology.orchestra.run.smithi008.stderr:Verifying podman|docker is present... 2023-10-09T05:14:48.280 INFO:teuthology.orchestra.run.smithi008.stderr:/bin/podman: 3.0.1 2023-10-09T05:14:48.294 INFO:teuthology.orchestra.run.smithi008.stderr:Verifying lvm2 is present... 2023-10-09T05:14:48.294 INFO:teuthology.orchestra.run.smithi008.stderr:Verifying time synchronization is in place... 2023-10-09T05:14:48.301 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T05:14:48.306 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: inactive 2023-10-09T05:14:48.312 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: enabled 2023-10-09T05:14:48.318 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: active 2023-10-09T05:14:48.319 INFO:teuthology.orchestra.run.smithi008.stderr:Unit chronyd.service is enabled and running 2023-10-09T05:14:48.319 INFO:teuthology.orchestra.run.smithi008.stderr:Repeating the final host check... 2023-10-09T05:14:48.363 INFO:teuthology.orchestra.run.smithi008.stderr:/bin/podman: 3.0.1 2023-10-09T05:14:48.379 INFO:teuthology.orchestra.run.smithi008.stderr:podman (/bin/podman) version 3.0.1 is present 2023-10-09T05:14:48.379 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl is present 2023-10-09T05:14:48.379 INFO:teuthology.orchestra.run.smithi008.stderr:lvcreate is present 2023-10-09T05:14:48.384 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T05:14:48.390 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: inactive 2023-10-09T05:14:48.396 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: enabled 2023-10-09T05:14:48.401 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: active 2023-10-09T05:14:48.402 INFO:teuthology.orchestra.run.smithi008.stderr:Unit chronyd.service is enabled and running 2023-10-09T05:14:48.402 INFO:teuthology.orchestra.run.smithi008.stderr:Host looks OK 2023-10-09T05:14:48.402 INFO:teuthology.orchestra.run.smithi008.stderr:Cluster fsid: 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:14:48.402 INFO:teuthology.orchestra.run.smithi008.stderr:Acquiring lock 140416551905768 on /run/cephadm/9e85b874-6662-11ee-8db5-212e2dc638e7.lock 2023-10-09T05:14:48.403 INFO:teuthology.orchestra.run.smithi008.stderr:Lock 140416551905768 acquired on /run/cephadm/9e85b874-6662-11ee-8db5-212e2dc638e7.lock 2023-10-09T05:14:48.403 INFO:teuthology.orchestra.run.smithi008.stderr:Verifying IP 172.21.15.8 port 3300 ... 2023-10-09T05:14:48.403 INFO:teuthology.orchestra.run.smithi008.stderr:Verifying IP 172.21.15.8 port 6789 ... 2023-10-09T05:14:48.404 INFO:teuthology.orchestra.run.smithi008.stderr:Base mon IP(s) is [172.21.15.8:3300, 172.21.15.8:6789], mon addrv is [v2:172.21.15.8:3300,v1:172.21.15.8:6789] 2023-10-09T05:14:48.406 INFO:teuthology.orchestra.run.smithi008.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp metric 100 2023-10-09T05:14:48.406 INFO:teuthology.orchestra.run.smithi008.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.8 metric 100 2023-10-09T05:14:48.409 INFO:teuthology.orchestra.run.smithi008.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T05:14:48.409 INFO:teuthology.orchestra.run.smithi008.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-10-09T05:14:48.409 INFO:teuthology.orchestra.run.smithi008.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1762sec hoplimit 64 pref medium 2023-10-09T05:14:48.412 INFO:teuthology.orchestra.run.smithi008.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T05:14:48.412 INFO:teuthology.orchestra.run.smithi008.stderr:/sbin/ip: inet6 ::1/128 scope host 2023-10-09T05:14:48.412 INFO:teuthology.orchestra.run.smithi008.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T05:14:48.412 INFO:teuthology.orchestra.run.smithi008.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2023-10-09T05:14:48.412 INFO:teuthology.orchestra.run.smithi008.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fef8:25fe/64 scope link 2023-10-09T05:14:48.412 INFO:teuthology.orchestra.run.smithi008.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T05:14:48.413 INFO:teuthology.orchestra.run.smithi008.stderr:Mon IP `172.21.15.8` is in CIDR network `172.21.0.0/20` 2023-10-09T05:14:48.413 INFO:teuthology.orchestra.run.smithi008.stderr:Mon IP `172.21.15.8` is in CIDR network `172.21.0.0/20` 2023-10-09T05:14:48.413 INFO:teuthology.orchestra.run.smithi008.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-10-09T05:14:48.414 INFO:teuthology.orchestra.run.smithi008.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T05:14:48.414 INFO:teuthology.orchestra.run.smithi008.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T05:14:48.488 INFO:teuthology.orchestra.run.smithi008.stderr:/bin/podman: Trying to pull quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T05:14:48.569 INFO:teuthology.orchestra.run.smithi008.stderr:/bin/podman: Getting image source signatures 2023-10-09T05:14:48.569 INFO:teuthology.orchestra.run.smithi008.stderr:/bin/podman: Copying blob sha256:24d54b00114a652388987a20e62df7ca52e6b59afd272176be9f22a3eeefd892 2023-10-09T05:14:48.569 INFO:teuthology.orchestra.run.smithi008.stderr:/bin/podman: Copying blob sha256:be9ed931e0f930233626f1c4d225977fbcc33e056e1a8fc67c7f63553d9455a2 2023-10-09T05:14:48.569 INFO:teuthology.orchestra.run.smithi008.stderr:/bin/podman: Copying config sha256:83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352 2023-10-09T05:14:48.624 INFO:teuthology.orchestra.run.smithi008.stderr:/bin/podman: Writing manifest to image destination 2023-10-09T05:14:48.624 INFO:teuthology.orchestra.run.smithi008.stderr:/bin/podman: Storing signatures 2023-10-09T05:14:48.773 INFO:teuthology.orchestra.run.smithi008.stderr:/bin/podman: 83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352 2023-10-09T05:14:49.245 INFO:teuthology.orchestra.run.smithi008.stderr:ceph: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T05:14:49.757 INFO:teuthology.orchestra.run.smithi008.stderr:Ceph version: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T05:14:49.757 INFO:teuthology.orchestra.run.smithi008.stderr:Extracting ceph user uid/gid from container image... 2023-10-09T05:14:50.107 INFO:teuthology.orchestra.run.smithi008.stderr:stat: 167 167 2023-10-09T05:14:50.606 INFO:teuthology.orchestra.run.smithi008.stderr:Creating initial keys... 2023-10-09T05:14:50.983 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-authtool: AQBKjCNlyqrmORAAz+uWL0E017eqarHrKrmw8Q== 2023-10-09T05:14:51.815 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-authtool: AQBLjCNlOUnxLxAARdtJKMlvjF1hfdWivP0fsw== 2023-10-09T05:14:52.649 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-authtool: AQBMjCNly3PEJRAAO1sXFtKLKNIcwDUg3b2hsg== 2023-10-09T05:14:53.166 INFO:teuthology.orchestra.run.smithi008.stderr:Creating initial monmap... 2023-10-09T05:14:53.565 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T05:14:53.566 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:14:53.566 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T05:14:54.065 INFO:teuthology.orchestra.run.smithi008.stderr:monmaptool for a [v2:172.21.15.8:3300,v1:172.21.15.8:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T05:14:54.065 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/monmaptool: set fsid to 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:14:54.065 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T05:14:54.065 INFO:teuthology.orchestra.run.smithi008.stderr: 2023-10-09T05:14:54.065 INFO:teuthology.orchestra.run.smithi008.stderr:Creating mon... 2023-10-09T05:14:54.531 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.494+0000 7f22331b2880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:14:54.531 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 1 imported monmap: 2023-10-09T05:14:54.531 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: epoch 0 2023-10-09T05:14:54.531 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:14:54.531 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: last_changed 2023-10-09T05:14:53.552466+0000 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: created 2023-10-09T05:14:53.552466+0000 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0] mon.a 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 0 /usr/bin/ceph-mon: set fsid to 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: DB SUMMARY 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.532 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-10-09T05:14:54.533 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.533 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-10-09T05:14:54.533 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.533 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T05:14:54.533 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.create_if_missing: 1 2023-10-09T05:14:54.533 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T05:14:54.533 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.env: 0x557d43419080 2023-10-09T05:14:54.533 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.fs: Posix File System 2023-10-09T05:14:54.533 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.info_log: 0x557d43fabaa0 2023-10-09T05:14:54.533 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:14:54.533 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.statistics: (nil) 2023-10-09T05:14:54.534 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.use_fsync: 0 2023-10-09T05:14:54.534 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T05:14:54.534 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:14:54.534 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:14:54.534 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:14:54.534 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:14:54.534 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T05:14:54.534 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:14:54.534 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.db_log_dir: 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:14:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.write_buffer_manager: 0x557d43fb3890 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:14:54.536 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:14:54.537 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:14:54.537 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.unordered_write: 0 2023-10-09T05:14:54.537 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:14:54.537 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:14:54.537 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:14:54.537 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:14:54.537 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.row_cache: None 2023-10-09T05:14:54.537 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.wal_filter: None 2023-10-09T05:14:54.537 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:14:54.537 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:14:54.538 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T05:14:54.538 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T05:14:54.538 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:14:54.538 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T05:14:54.538 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:14:54.538 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:14:54.538 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:14:54.538 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T05:14:54.538 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T05:14:54.538 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T05:14:54.538 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T05:14:54.539 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:14:54.539 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:14:54.539 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:14:54.539 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:14:54.539 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:14:54.539 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:14:54.539 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:14:54.539 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:14:54.539 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.max_open_files: -1 2023-10-09T05:14:54.540 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:14:54.540 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:14:54.540 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:14:54.540 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:14:54.540 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Compression algorithms supported: 2023-10-09T05:14:54.540 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:14:54.540 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: kZSTD supported: 0 2023-10-09T05:14:54.540 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T05:14:54.540 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.495+0000 7f22331b2880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.merge_operator: 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_filter: None 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557d43ebdd20) 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: index_type: 0 2023-10-09T05:14:54.542 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-10-09T05:14:54.543 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-10-09T05:14:54.543 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:14:54.543 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-10-09T05:14:54.543 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: checksum: 1 2023-10-09T05:14:54.543 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-10-09T05:14:54.543 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: block_cache: 0x557d43ef6d10 2023-10-09T05:14:54.543 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-10-09T05:14:54.543 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-10-09T05:14:54.543 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-10-09T05:14:54.543 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-10-09T05:14:54.544 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:14:54.545 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-10-09T05:14:54.545 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-10-09T05:14:54.545 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-10-09T05:14:54.545 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: format_version: 2 2023-10-09T05:14:54.545 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-10-09T05:14:54.545 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: block_align: 0 2023-10-09T05:14:54.545 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.545 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:14:54.545 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:14:54.545 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compression: NoCompression 2023-10-09T05:14:54.545 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:14:54.546 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:14:54.546 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:14:54.546 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.num_levels: 7 2023-10-09T05:14:54.546 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:14:54.546 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:14:54.546 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:14:54.546 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:14:54.546 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:14:54.546 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:14:54.546 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:14:54.546 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:14:54.547 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:14:54.548 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:14:54.549 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:14:54.550 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:14:54.550 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:14:54.550 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T05:14:54.550 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T05:14:54.550 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:14:54.550 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:14:54.550 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:14:54.550 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:14:54.550 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T05:14:54.550 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T05:14:54.550 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.ttl: 2592000 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.496+0000 7f22331b2880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.497+0000 7f22331b2880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.497+0000 7f22331b2880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.497+0000 7f22331b2880 4 rocksdb: DB pointer 0x557d43fc1800 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.497+0000 7f221c03a700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.497+0000 7f221c03a700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T05:14:54.551 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:14:54.552 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:14:54.553 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:14:54.553 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.553 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T05:14:54.553 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:14:54.553 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:14:54.553 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:14:54.553 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:14:54.553 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:14:54.553 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:14:54.553 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:14:54.553 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:14:54.554 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:14:54.554 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:14:54.554 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:14:54.554 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.554 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:14:54.554 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.554 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T05:14:54.554 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:14:54.554 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:14:54.554 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:14:54.554 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:14:54.555 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:14:54.556 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:14:54.556 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.556 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:14:54.556 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: 2023-10-09T05:14:54.556 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.499+0000 7f22331b2880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-10-09T05:14:54.556 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.499+0000 7f22331b2880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-10-09T05:14:54.556 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:14:54.499+0000 7f22331b2880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-10-09T05:14:55.057 INFO:teuthology.orchestra.run.smithi008.stderr:create mon.a on 2023-10-09T05:14:55.071 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T05:14:55.167 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-10-09T05:14:55.263 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-9e85b874-6662-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-9e85b874-6662-11ee-8db5-212e2dc638e7.target. 2023-10-09T05:14:55.263 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-9e85b874-6662-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-9e85b874-6662-11ee-8db5-212e2dc638e7.target. 2023-10-09T05:14:55.476 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: Failed to reset failed state of unit ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T05:14:55.483 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9e85b874-6662-11ee-8db5-212e2dc638e7.target.wants/ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@.service. 2023-10-09T05:14:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:55 smithi008 systemd[1]: Starting Ceph mon.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:14:56.138 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: disabled 2023-10-09T05:14:56.145 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: inactive 2023-10-09T05:14:56.146 INFO:teuthology.orchestra.run.smithi008.stderr:firewalld.service is not enabled 2023-10-09T05:14:56.146 INFO:teuthology.orchestra.run.smithi008.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T05:14:56.147 INFO:teuthology.orchestra.run.smithi008.stderr:Waiting for mon to start... 2023-10-09T05:14:56.147 INFO:teuthology.orchestra.run.smithi008.stderr:Waiting for mon... 2023-10-09T05:14:56.374 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:56 smithi008 bash[100018]: 48b04e768411bcaeb60d234b1f5cb4891b5516c614dbf829d86f7008ac5b78ed 2023-10-09T05:14:56.374 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:56 smithi008 systemd[1]: Started Ceph mon.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:14:56.374 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:56 smithi008 conmon[100059]: cluster 2023-10-09T05:14:56.161084+0000 mon.a 2023-10-09T05:14:56.374 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:56 smithi008 conmon[100059]: (mon.0) 0 : [INF] mkfs 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:14:56.374 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:56 smithi008 conmon[100059]: cluster 2023-10-09T05 2023-10-09T05:14:56.374 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:56 smithi008 conmon[100059]: :14:56.132771+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:14:56.774 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: cluster: 2023-10-09T05:14:56.774 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: id: 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:14:56.774 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-10-09T05:14:56.774 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:14:56.774 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: services: 2023-10-09T05:14:56.774 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.624324s) 2023-10-09T05:14:56.775 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: mgr: no daemons active 2023-10-09T05:14:56.775 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-10-09T05:14:56.775 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:14:56.775 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: data: 2023-10-09T05:14:56.775 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-10-09T05:14:56.775 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-10-09T05:14:56.775 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-10-09T05:14:56.775 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: pgs: 2023-10-09T05:14:56.775 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:14:57.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:57 smithi008 conmon[100059]: cluster 2023-10-09T05:14:56.148577+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:14:57.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:57 smithi008 conmon[100059]: cluster 2023-10-09T05:14:56.148794+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0]} removed_ranks: {} 2023-10-09T05:14:57.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:57 smithi008 conmon[100059]: cluster 2023-10-09T05:14:56.162726+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-10-09T05:14:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:57 smithi008 conmon[100059]: cluster 2023-10-09T05:14:56.165104+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T05:14:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:57 smithi008 conmon[100059]: cluster 2023-10-09T05:14:56.165627+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T05:14:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:57 smithi008 conmon[100059]: audit 2023-10-09T05:14:56.772853+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.8:0/842860820' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T05:14:57.478 INFO:teuthology.orchestra.run.smithi008.stderr:mon is available 2023-10-09T05:14:57.479 INFO:teuthology.orchestra.run.smithi008.stderr:Assimilating anything we can from ceph.conf... 2023-10-09T05:14:58.063 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:14:58.064 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: [global] 2023-10-09T05:14:58.064 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: fsid = 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:14:58.064 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T05:14:58.064 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.8:3300,v1:172.21.15.8:6789] 2023-10-09T05:14:58.064 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T05:14:58.064 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T05:14:58.064 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T05:14:58.064 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T05:14:58.064 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:14:58.064 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: [mgr] 2023-10-09T05:14:58.064 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T05:14:58.065 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:14:58.065 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: [osd] 2023-10-09T05:14:58.065 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T05:14:58.065 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T05:14:58.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:58 smithi008 conmon[100059]: audit 2023-10-09T05:14:58.059504+0000 2023-10-09T05:14:58.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:58 smithi008 conmon[100059]: mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.8:0/982618095' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T05:14:58.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:58 smithi008 conmon[100059]: audit 2023-10-09T05:14:58.062279+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.8:0/982618095' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T05:14:58.659 INFO:teuthology.orchestra.run.smithi008.stderr:Generating new minimal ceph.conf... 2023-10-09T05:14:59.649 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:59 smithi008 conmon[100059]: audit 2023-10-09T05 2023-10-09T05:14:59.650 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:14:59 smithi008 conmon[100059]: :14:59.321129+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.8:0/2959780224' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:15:00.059 INFO:teuthology.orchestra.run.smithi008.stderr:Restarting the monitor... 2023-10-09T05:15:00.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:00 smithi008 systemd[1]: Stopping Ceph mon.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:15:00.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:00 smithi008 bash[100565]: Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-mon.a found: no such container 2023-10-09T05:15:00.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:00 smithi008 conmon[100059]: debug 2023-10-09T05:15:00.365+0000 7f6381b31700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T05:15:00.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:00 smithi008 conmon[100059]: debug 2023-10-09T05:15:00.365+0000 7f6381b31700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T05:15:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:00 smithi008 bash[100681]: 48b04e768411bcaeb60d234b1f5cb4891b5516c614dbf829d86f7008ac5b78ed 2023-10-09T05:15:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T05:15:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 systemd[1]: Stopped Ceph mon.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:15:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 systemd[1]: Starting Ceph mon.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:15:01.670 INFO:teuthology.orchestra.run.smithi008.stderr:Setting mon public_network to 172.21.0.0/20 2023-10-09T05:15:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.659+0000 7ff4bbbfa880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:15:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.659+0000 7ff4bbbfa880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T05:15:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.659+0000 7ff4bbbfa880 0 pidfile_write: ignore empty --pid-file 2023-10-09T05:15:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 bash[101012]: d155e1e1e35f7314228013264251d26590c9ba27beb998fb6950b033f07c090a 2023-10-09T05:15:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.663+0000 7ff4bbbfa880 0 load: jerasure load: lrc load: isa 2023-10-09T05:15:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T05:15:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:15:01.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T05:15:01.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: DB SUMMARY 2023-10-09T05:15:01.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T05:15:01.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:15:01.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-10-09T05:15:01.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-10-09T05:15:01.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86633 ; 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.env: 0x55a3525f2080 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.fs: Posix File System 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.info_log: 0x55a353f33a80 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.statistics: (nil) 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.use_fsync: 0 2023-10-09T05:15:01.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.db_log_dir: 2023-10-09T05:15:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.write_buffer_manager: 0x55a354c2e330 2023-10-09T05:15:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:15:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:15:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:15:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:15:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:15:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:15:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:15:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:15:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:15:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.unordered_write: 0 2023-10-09T05:15:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:15:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:15:01.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:15:01.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:15:01.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.row_cache: None 2023-10-09T05:15:01.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.wal_filter: None 2023-10-09T05:15:01.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:15:01.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:15:01.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T05:15:01.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T05:15:01.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:15:01.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T05:15:01.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:15:01.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:15:01.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:15:01.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T05:15:01.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T05:15:01.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T05:15:01.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T05:15:01.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:15:01.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:15:01.974 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:15:01.974 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:15:01.974 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:15:01.974 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:15:01.974 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:15:01.974 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:15:01.974 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.max_open_files: -1 2023-10-09T05:15:01.974 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:15:01.974 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:15:01.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:15:01.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:15:01.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Compression algorithms supported: 2023-10-09T05:15:01.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:15:01.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: kZSTD supported: 0 2023-10-09T05:15:01.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T05:15:01.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:15:01.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T05:15:01.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T05:15:01.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T05:15:01.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.664+0000 7ff4bbbfa880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.merge_operator: 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_filter: None 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:15:01.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a353eddd18) 2023-10-09T05:15:01.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: cache_index_and_filter_blocks: 1 2023-10-09T05:15:01.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:15:01.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:15:01.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: pin_top_level_index_and_filter: 1 2023-10-09T05:15:01.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: index_type: 0 2023-10-09T05:15:01.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: data_block_index_type: 0 2023-10-09T05:15:01.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: index_shortening: 1 2023-10-09T05:15:01.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:15:01.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: hash_index_allow_collision: 1 2023-10-09T05:15:01.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: checksum: 1 2023-10-09T05:15:01.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: no_block_cache: 0 2023-10-09T05:15:01.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: block_cache: 0x55a353f16f10 2023-10-09T05:15:01.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: block_cache_name: BinnedLRUCache 2023-10-09T05:15:01.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: block_cache_options: 2023-10-09T05:15:01.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: capacity : 536870912 2023-10-09T05:15:01.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: num_shard_bits : 4 2023-10-09T05:15:01.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: strict_capacity_limit : 0 2023-10-09T05:15:01.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: high_pri_pool_ratio: 0.000 2023-10-09T05:15:01.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: block_cache_compressed: (nil) 2023-10-09T05:15:01.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: persistent_cache: (nil) 2023-10-09T05:15:01.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: block_size: 4096 2023-10-09T05:15:01.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: block_size_deviation: 10 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: block_restart_interval: 16 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: index_block_restart_interval: 1 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: metadata_block_size: 4096 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: partition_filters: 0 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: use_delta_encoding: 1 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: whole_key_filtering: 1 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: verify_compression: 0 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: read_amp_bytes_per_bit: 0 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: format_version: 2 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: enable_index_compression: 1 2023-10-09T05:15:01.979 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: block_align: 0 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compression: NoCompression 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.num_levels: 7 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:15:01.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:15:01.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:15:01.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:15:01.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:15:01.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:15:01.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:15:01.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:15:01.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:15:01.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:15:01.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:15:01.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:15:01.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:15:01.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:15:01.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:15:01.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:15:01.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:15:01.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:15:01.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:15:01.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:15:01.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:15:01.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:15:01.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:15:01.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:15:01.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4b 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: bbfa880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T05:15:01.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T05:15:01.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:15:01.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:15:01.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:15:01.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:15:01.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T05:15:01.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T05:15:01.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:15:01.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:15:01.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T05:15:01.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:15:01.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.ttl: 2592000 2023-10-09T05:15:01.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:15:01.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:15:01.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-10-09T05:15:01.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696828501667118, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-10-09T05:15:01.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.665+0000 7ff4bbbfa880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-10-09T05:15:01.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.666+0000 7ff4bbbfa880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T05:15:01.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.666+0000 7ff4bbbfa880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696828501668033, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83233, "table_properties": {"data_size": 81506, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75827, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696828501, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T05:15:01.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.666+0000 7ff4bbbfa880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-10-09T05:15:01.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 systemd[1]: Started Ceph mon.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.668+0000 7ff4bbbfa880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696828501669352, "job": 1, "event": "recovery_finished"} 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.668+0000 7ff4bbbfa880 4 rocksdb: DB pointer 0x55a353fe1800 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.669+0000 7ff4a2a01700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.669+0000 7ff4a2a01700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: ** DB Stats ** 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:15:01.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:15:01.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:15:01.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: ** Compaction Stats [default] ** 2023-10-09T05:15:01.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:15:01.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:15:01.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: L0 2/0 82.94 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.5 0.00 0.00 1 0.001 0 0 2023-10-09T05:15:01.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Sum 2/0 82.94 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.5 0.00 0.00 1 0.001 0 0 2023-10-09T05:15:01.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.5 0.00 0.00 1 0.001 0 0 2023-10-09T05:15:01.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: ** Compaction Stats [default] ** 2023-10-09T05:15:01.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 134.5 0.00 0.00 1 0.001 0 0 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Cumulative compaction: 0.00 GB write, 20.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Interval compaction: 0.00 GB write, 20.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:15:01.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.990 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:15:01.990 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.990 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: ** Compaction Stats [default] ** 2023-10-09T05:15:01.990 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:15:01.990 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:15:01.990 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: L0 2/0 82.94 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.5 0.00 0.00 1 0.001 0 0 2023-10-09T05:15:01.990 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Sum 2/0 82.94 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.5 0.00 0.00 1 0.001 0 0 2023-10-09T05:15:01.990 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:15:01.991 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.991 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: ** Compaction Stats [default] ** 2023-10-09T05:15:01.991 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:15:01.991 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:15:01.991 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 134.5 0.00 0.00 1 0.001 0 0 2023-10-09T05:15:01.991 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:15:01.991 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:15:01.991 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:15:01.991 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:15:01.991 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:15:01.991 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:15:01.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Cumulative compaction: 0.00 GB write, 19.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:15:01.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:15:01.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:15:01.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:15:01.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.669+0000 7ff4bbbfa880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0] at bind addrs [v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:15:01.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.670+0000 7ff4bbbfa880 1 mon.a@-1(???) e1 preinit fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:15:01.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.670+0000 7ff4bbbfa880 0 mon.a@-1(???).mds e1 new map 2023-10-09T05:15:01.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.670+0000 7ff4bbbfa880 0 mon.a@-1(???).mds e1 print_map 2023-10-09T05:15:01.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: e1 2023-10-09T05:15:01.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:15:01.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:15:01.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: legacy client fscid: -1 2023-10-09T05:15:01.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: No filesystems configured 2023-10-09T05:15:01.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:15:01.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.670+0000 7ff4bbbfa880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T05:15:01.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.670+0000 7ff4bbbfa880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:15:01.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.670+0000 7ff4bbbfa880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:15:01.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.670+0000 7ff4bbbfa880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:15:01.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: audit 2023-10-09T05:15:01.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 2023-10-09T05:14:59. 2023-10-09T05:15:01.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: 321129+0000 mon.a (mon.0) 10 2023-10-09T05:15:01.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: : audit [DBG] from='client.? 172.21.15.8:0/2959780224' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:15:01.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:01 smithi008 conmon[101053]: debug 2023-10-09T05:15:01.670+0000 7ff4bbbfa880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T05:15:02.916 INFO:teuthology.orchestra.run.smithi008.stderr:Wrote config to /etc/ceph/ceph.conf 2023-10-09T05:15:02.917 INFO:teuthology.orchestra.run.smithi008.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:15:02.917 INFO:teuthology.orchestra.run.smithi008.stderr:Creating mgr... 2023-10-09T05:15:02.917 INFO:teuthology.orchestra.run.smithi008.stderr:Verifying port 9283 ... 2023-10-09T05:15:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:02 smithi008 conmon[101053]: cluster 2023-10-09T05:15:01.686900+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:15:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:02 smithi008 conmon[101053]: cluster 2023-10-09T05:15:01.686988+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0]} removed_ranks: {} 2023-10-09T05:15:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:02 smithi008 conmon[101053]: cluster 2023-10-09T05:15:01.698581+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-10-09T05:15:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:02 smithi008 conmon[101053]: cluster 2023-10-09T05:15:01.698619+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T05:15:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:02 smithi008 conmon[101053]: cluster 2023-10-09T05:15:01.699006+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T05:15:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:02 smithi008 conmon[101053]: audit 2023-10-09T05:15:02.266113+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.8:0/3548988359' entity='client.admin' 2023-10-09T05:15:03.061 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: Failed to reset failed state of unit ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T05:15:03.069 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9e85b874-6662-11ee-8db5-212e2dc638e7.target.wants/ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@.service. 2023-10-09T05:15:03.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:03 smithi008 systemd[1]: Starting Ceph mgr.y for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:15:03.778 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: disabled 2023-10-09T05:15:03.786 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: inactive 2023-10-09T05:15:03.786 INFO:teuthology.orchestra.run.smithi008.stderr:firewalld.service is not enabled 2023-10-09T05:15:03.786 INFO:teuthology.orchestra.run.smithi008.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T05:15:03.794 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: disabled 2023-10-09T05:15:03.801 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: inactive 2023-10-09T05:15:03.803 INFO:teuthology.orchestra.run.smithi008.stderr:firewalld.service is not enabled 2023-10-09T05:15:03.803 INFO:teuthology.orchestra.run.smithi008.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T05:15:03.803 INFO:teuthology.orchestra.run.smithi008.stderr:Waiting for mgr to start... 2023-10-09T05:15:03.803 INFO:teuthology.orchestra.run.smithi008.stderr:Waiting for mgr... 2023-10-09T05:15:04.044 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:03 smithi008 bash[101468]: b54a8f2f116cf016eccb9bf719501d5f484ac3ca286dce557f90525646c77184 2023-10-09T05:15:04.045 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:03 smithi008 systemd[1]: Started Ceph mgr.y for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:15:04.045 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:03 smithi008 conmon[101509]: debug 2023-10-09T05:15:03.911+0000 7fcd888ac680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:15:04.045 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:04 smithi008 conmon[101509]: debug 2023-10-09T05:15:04.010+0000 7fcd888ac680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:15:04.474 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:15:04.474 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: { 2023-10-09T05:15:04.474 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "fsid": "9e85b874-6662-11ee-8db5-212e2dc638e7", 2023-10-09T05:15:04.474 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "health": { 2023-10-09T05:15:04.474 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T05:15:04.474 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T05:15:04.474 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T05:15:04.474 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 0 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "a" 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T05:15:04.475 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T05:15:04.476 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T05:15:04.477 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:04.478 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T05:15:04.478 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "available": false, 2023-10-09T05:15:04.478 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T05:15:04.478 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T05:15:04.478 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "iostat", 2023-10-09T05:15:04.478 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "nfs", 2023-10-09T05:15:04.478 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "restful" 2023-10-09T05:15:04.478 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:04.478 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:15:04.478 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:04.478 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T05:15:04.479 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:04.479 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "modified": "2023-10-09T05:14:56.161557+0000", 2023-10-09T05:15:04.479 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:15:04.479 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:04.479 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T05:15:04.479 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: } 2023-10-09T05:15:04.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:04 smithi008 conmon[101509]: debug 2023-10-09T05:15:04.474+0000 7fcd888ac680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:15:04.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:04 smithi008 conmon[101053]: audit 2023-10-09T05:15:04.468494+0000 2023-10-09T05:15:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:04 smithi008 conmon[101053]: mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.8:0/159304251' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:15:05.091 INFO:teuthology.orchestra.run.smithi008.stderr:mgr not available, waiting (1/15)... 2023-10-09T05:15:05.873 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:05 smithi008 conmon[101509]: debug 2023-10-09T05:15:05.533+0000 7fcd888ac680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:15:05.873 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:05 smithi008 conmon[101509]: debug 2023-10-09T05:15:05.628+0000 7fcd888ac680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:15:05.873 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:05 smithi008 conmon[101509]: debug 2023-10-09T05:15:05.873+0000 7fcd888ac680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:15:07.270 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:07 smithi008 conmon[101509]: debug 2023-10-09T05:15:07.008+0000 7fcd888ac680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:15:07.270 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:07 smithi008 conmon[101509]: debug 2023-10-09T05:15:07.176+0000 7fcd888ac680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:15:07.603 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:07 smithi008 conmon[101509]: debug 2023-10-09T05:15:07.270+0000 7fcd888ac680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:15:07.603 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:07 smithi008 conmon[101509]: debug 2023-10-09T05:15:07.384+0000 7fcd888ac680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:15:07.761 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:15:07.761 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: { 2023-10-09T05:15:07.761 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "fsid": "9e85b874-6662-11ee-8db5-212e2dc638e7", 2023-10-09T05:15:07.761 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "health": { 2023-10-09T05:15:07.761 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T05:15:07.761 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T05:15:07.761 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T05:15:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T05:15:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T05:15:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 0 2023-10-09T05:15:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T05:15:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "a" 2023-10-09T05:15:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum_age": 6, 2023-10-09T05:15:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T05:15:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:07.763 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T05:15:07.763 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T05:15:07.763 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:07.763 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T05:15:07.763 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:07.763 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T05:15:07.763 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T05:15:07.763 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T05:15:07.766 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T05:15:07.766 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T05:15:07.766 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T05:15:07.767 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:07.767 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T05:15:07.767 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T05:15:07.767 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T05:15:07.767 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T05:15:07.767 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T05:15:07.767 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T05:15:07.767 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T05:15:07.767 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T05:15:07.767 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T05:15:07.767 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "available": false, 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "iostat", 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "nfs", 2023-10-09T05:15:07.768 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "restful" 2023-10-09T05:15:07.769 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:07.769 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:15:07.769 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:07.769 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T05:15:07.769 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:07.769 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "modified": "2023-10-09T05:14:56.161557+0000", 2023-10-09T05:15:07.769 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:15:07.769 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:07.769 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T05:15:07.769 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: } 2023-10-09T05:15:07.922 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:07 smithi008 conmon[101509]: debug 2023-10-09T05:15:07.603+0000 7fcd888ac680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:15:07.923 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:07 smithi008 conmon[101053]: audit 2023-10-09T05:15:07.758844+0000 mon.a 2023-10-09T05:15:07.923 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:07 smithi008 conmon[101053]: (mon.0) 8 : audit [DBG] from='client.? 172.21.15.8:0/3021388541' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:15:08.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:07 smithi008 conmon[101509]: debug 2023-10-09T05:15:07.975+0000 7fcd888ac680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:15:08.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:08 smithi008 conmon[101509]: debug 2023-10-09T05:15:08.091+0000 7fcd888ac680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:15:08.282 INFO:teuthology.orchestra.run.smithi008.stderr:mgr not available, waiting (2/15)... 2023-10-09T05:15:09.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:08 smithi008 conmon[101509]: debug 2023-10-09T05:15:08.963+0000 7fcd888ac680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:15:09.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:09 smithi008 conmon[101509]: debug 2023-10-09T05:15:09.060+0000 7fcd888ac680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:15:09.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:09 smithi008 conmon[101509]: debug 2023-10-09T05:15:09.170+0000 7fcd888ac680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:15:09.663 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:09 smithi008 conmon[101509]: debug 2023-10-09T05:15:09.378+0000 7fcd888ac680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:15:09.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:09 smithi008 conmon[101509]: debug 2023-10-09T05:15:09.663+0000 7fcd888ac680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:15:09.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:09 smithi008 conmon[101509]: debug 2023-10-09T05:15:09.823+0000 7fcd888ac680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:15:10.306 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:09 smithi008 conmon[101509]: debug 2023-10-09T05:15:09.990+0000 7fcd888ac680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:15:10.307 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101509]: debug 2023-10-09T05:15:10.203+0000 7fcd888ac680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:15:10.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101509]: debug 2023-10-09T05:15:10.306+0000 7fcd888ac680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:15:10.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: cluster 2023-10-09T05:15:10.320391+0000 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: cluster 2023-10-09T05:15:10.322913+0000 mon.a (mon 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: .0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00260848s) 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: audit 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: 2023-10-09T05:15:10.324027+0000 mon.a (mon.0) 11 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: : audit [DBG] from='mgr.14100 172.21.15.8:0/98714652' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: 05:15:10.324218+0000 mon.a (mon.0) 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: 12 : audit [DBG] from='mgr.14100 172.21.15.8:0/98714652' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: -09T05:15:10.324371+0000 mon.a ( 2023-10-09T05:15:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.8:0/98714652' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: audit 2023 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: -10-09T05:15:10.324448+0000 mon.a 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.8:0/98714652' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: audit 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: 2023-10-09T05:15:10.325091+0000 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.8:0/98714652' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: -10-09T05:15:10.339448+0000 mon.a (mon.0) 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: 16 : cluster [INF] Manager daemon y is now available 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: 05:15:10.348572+0000 mon.a (mon.0 2023-10-09T05:15:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: ) 17 : audit [INF] from='mgr.14100 172.21.15.8:0/98714652' entity='mgr.y' 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: -09T05:15:10.354633+0000 mon.a ( 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.8:0/98714652' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: audit 2023 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: -10-09T05:15:10.357670+0000 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.8:0/98714652' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: 05:15:10.361036+0000 mon.a (mon.0) 20 : audit 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: [INF] from='mgr.14100 172.21.15.8:0/98714652' entity='mgr.y' 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: audit 2023-10-09T05:15 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: :10.363133+0000 mon.a (mon.0) 21 : 2023-10-09T05:15:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:10 smithi008 conmon[101053]: audit [INF] from='mgr.14100 172.21.15.8:0/98714652' entity='mgr.y' 2023-10-09T05:15:10.878 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: { 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "fsid": "9e85b874-6662-11ee-8db5-212e2dc638e7", 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "health": { 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 0 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:10.879 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T05:15:10.880 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "a" 2023-10-09T05:15:10.880 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:10.880 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum_age": 9, 2023-10-09T05:15:10.880 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T05:15:10.880 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:10.880 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T05:15:10.880 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T05:15:10.880 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:10.880 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T05:15:10.880 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:10.880 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T05:15:10.884 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T05:15:10.884 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T05:15:10.884 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T05:15:10.884 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T05:15:10.884 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T05:15:10.884 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:10.884 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T05:15:10.884 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T05:15:10.884 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T05:15:10.884 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T05:15:10.885 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "available": false, 2023-10-09T05:15:10.886 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T05:15:10.886 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T05:15:10.886 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "iostat", 2023-10-09T05:15:10.886 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "nfs", 2023-10-09T05:15:10.886 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "restful" 2023-10-09T05:15:10.886 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:10.886 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:15:10.886 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:10.886 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T05:15:10.886 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:10.886 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "modified": "2023-10-09T05:14:56.161557+0000", 2023-10-09T05:15:10.887 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:15:10.887 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:10.887 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T05:15:10.887 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: } 2023-10-09T05:15:11.431 INFO:teuthology.orchestra.run.smithi008.stderr:mgr not available, waiting (3/15)... 2023-10-09T05:15:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:11 smithi008 conmon[101053]: audit 2023-10-09T05:15:10.876798+0000 2023-10-09T05:15:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:11 smithi008 conmon[101053]: mon.a (mon.0) 22 : audit [DBG] from='client.? 172.21.15.8:0/3815925285' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:15:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:11 smithi008 conmon[101053]: cluster 2023-10-09T05:15:11.326111+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e3: y(active, since 1.0058s) 2023-10-09T05:15:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:13 smithi008 conmon[101053]: cluster 2023-10-09T05:15:12.679155 2023-10-09T05:15:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:13 smithi008 conmon[101053]: +0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 2s) 2023-10-09T05:15:14.393 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:15:14.393 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: { 2023-10-09T05:15:14.393 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "fsid": "9e85b874-6662-11ee-8db5-212e2dc638e7", 2023-10-09T05:15:14.393 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "health": { 2023-10-09T05:15:14.393 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 0 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "a" 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "quorum_age": 12, 2023-10-09T05:15:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T05:15:14.395 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:14.396 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "available": true, 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "iostat", 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "nfs", 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "restful" 2023-10-09T05:15:14.397 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ], 2023-10-09T05:15:14.398 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:15:14.398 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:14.398 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T05:15:14.398 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:15:14.398 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "modified": "2023-10-09T05:14:56.161557+0000", 2023-10-09T05:15:14.398 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:15:14.398 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: }, 2023-10-09T05:15:14.398 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T05:15:14.398 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: } 2023-10-09T05:15:14.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:14 smithi008 conmon[101053]: audit 2023-10-09T05:15:14.387242+0000 mon.a (mon 2023-10-09T05:15:14.980 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:14 smithi008 conmon[101053]: .0) 25 : audit [DBG] from='client.? 172.21.15.8:0/3775754611' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:15:14.997 INFO:teuthology.orchestra.run.smithi008.stderr:mgr is available 2023-10-09T05:15:15.796 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:15:15.796 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: [global] 2023-10-09T05:15:15.796 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: fsid = 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:15:15.796 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T05:15:15.796 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T05:15:15.796 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T05:15:15.796 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T05:15:15.796 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T05:15:15.797 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:15:15.797 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: [mgr] 2023-10-09T05:15:15.797 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T05:15:15.797 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 2023-10-09T05:15:15.797 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: [osd] 2023-10-09T05:15:15.797 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T05:15:15.797 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T05:15:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:15 smithi008 conmon[101053]: audit 2023-10-09T05:15:15.794669+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.8:0/515230522' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T05:15:16.473 INFO:teuthology.orchestra.run.smithi008.stderr:Enabling cephadm module... 2023-10-09T05:15:17.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:17 smithi008 conmon[101053]: audit 2023-10-09T05:15:17.336791 2023-10-09T05:15:17.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:17 smithi008 conmon[101053]: +0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.8:0/2222206385' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T05:15:17.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:17 smithi008 conmon[101509]: ignoring --setuser ceph since I am not root 2023-10-09T05:15:17.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:17 smithi008 conmon[101509]: ignoring --setgroup ceph since I am not root 2023-10-09T05:15:17.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:17 smithi008 conmon[101509]: debug 2023-10-09T05:15:17.869+0000 7f2543363680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:15:18.232 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:17 smithi008 conmon[101509]: debug 2023-10-09T05:15:17.969+0000 7f2543363680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:15:18.679 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:18 smithi008 conmon[101509]: debug 2023-10-09T05:15:18.432+0000 7f2543363680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:15:18.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:18 smithi008 conmon[101053]: audit 2023-10-09T05:15:17.682267 2023-10-09T05:15:18.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:18 smithi008 conmon[101053]: +0000 mon.a (mon.0) 28 : audit [INF] from='client.? 172.21.15.8:0/2222206385' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T05:15:18.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:18 smithi008 conmon[101053]: cluster 2023-10-09T05:15:17.682337+0000 mon.a (mon.0) 2023-10-09T05:15:18.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:18 smithi008 conmon[101053]: 29 : cluster [DBG] mgrmap e5: y(active, since 7s) 2023-10-09T05:15:19.144 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: { 2023-10-09T05:15:19.144 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 5, 2023-10-09T05:15:19.144 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "available": true, 2023-10-09T05:15:19.144 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T05:15:19.145 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T05:15:19.145 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: } 2023-10-09T05:15:19.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:19 smithi008 conmon[101509]: debug 2023-10-09T05:15:19.385+0000 7f2543363680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:15:19.733 INFO:teuthology.orchestra.run.smithi008.stderr:Waiting for the mgr to restart... 2023-10-09T05:15:19.733 INFO:teuthology.orchestra.run.smithi008.stderr:Waiting for mgr epoch 5... 2023-10-09T05:15:19.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:19 smithi008 conmon[101053]: audit 2023-10-09T05:15:19.143816+0000 mon.a ( 2023-10-09T05:15:19.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:19 smithi008 conmon[101053]: mon.0) 30 : audit [DBG] from='client.? 172.21.15.8:0/3567062295' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T05:15:19.734 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:19 smithi008 conmon[101509]: debug 2023-10-09T05:15:19.480+0000 7f2543363680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:15:19.735 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:19 smithi008 conmon[101509]: debug 2023-10-09T05:15:19.730+0000 7f2543363680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:15:21.135 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:20 smithi008 conmon[101509]: debug 2023-10-09T05:15:20.878+0000 7f2543363680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:15:21.135 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:21 smithi008 conmon[101509]: debug 2023-10-09T05:15:21.039+0000 7f2543363680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:15:21.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:21 smithi008 conmon[101509]: debug 2023-10-09T05:15:21.134+0000 7f2543363680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:15:21.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:21 smithi008 conmon[101509]: debug 2023-10-09T05:15:21.243+0000 7f2543363680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:15:21.849 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:21 smithi008 conmon[101509]: debug 2023-10-09T05:15:21.482+0000 7f2543363680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:15:22.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:21 smithi008 conmon[101509]: debug 2023-10-09T05:15:21.848+0000 7f2543363680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:15:22.215 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:21 smithi008 conmon[101509]: debug 2023-10-09T05:15:21.968+0000 7f2543363680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:15:23.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:22 smithi008 conmon[101509]: debug 2023-10-09T05:15:22.840+0000 7f2543363680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:15:23.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:22 smithi008 conmon[101509]: debug 2023-10-09T05:15:22.938+0000 7f2543363680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:15:23.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:23 smithi008 conmon[101509]: debug 2023-10-09T05:15:23.049+0000 7f2543363680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:15:23.546 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:23 smithi008 conmon[101509]: debug 2023-10-09T05:15:23.260+0000 7f2543363680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:15:23.876 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:23 smithi008 conmon[101509]: debug 2023-10-09T05:15:23.545+0000 7f2543363680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:15:23.876 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:23 smithi008 conmon[101509]: debug 2023-10-09T05:15:23.708+0000 7f2543363680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:15:24.191 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:23 smithi008 conmon[101509]: debug 2023-10-09T05:15:23.875+0000 7f2543363680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:15:24.191 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101509]: debug 2023-10-09T05:15:24.090+0000 7f2543363680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:15:24.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101509]: debug 2023-10-09T05:15:24.190+0000 7f2543363680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:15:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: cluster 2023-10-09T05:15:24.205175+0000 mon.a ( 2023-10-09T05:15:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: mon.0) 31 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:15:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: cluster 2023-10-09T05:15:24.205390+0000 mon.a (mon.0) 32 : cluster [INF] Activating manager daemon y 2023-10-09T05:15:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: cluster 2023-10-09T05:15:24.208423+0000 mon.a 2023-10-09T05:15:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: (mon.0) 33 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-10-09T05:15:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: cluster 2023-10-09T05:15:24.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: 2023-10-09T05:15:24.208542+0000 mon.a (mon.0) 34 : cluster [DBG] 2023-10-09T05:15:24.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: mgrmap e6: y(active, starting, since 0.00323016s) 2023-10-09T05:15:24.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:15:24.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: -09T05:15:24.209533+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:15:24.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:15:24.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: 05:15:24.210176+0000 mon.a (mon.0) 36 : audit [DBG] 2023-10-09T05:15:24.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:15:24.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:15:24.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: 09T05:15:24.211015+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:15:24.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:15:24.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: 05:15:24.211302+0000 mon.a (mon.0) 38 : audit 2023-10-09T05:15:24.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: [DBG] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:15:24.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: audit 2023-10-09T05:15:24.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: 2023-10-09T05:15:24.211523+0000 mon.a (mon.0) 39 : audit [DBG] 2023-10-09T05:15:24.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:15:24.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: cluster 2023- 2023-10-09T05:15:24.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: 10-09T05:15:24.225906+0000 mon.a (mon.0) 40 : cluster [INF] 2023-10-09T05:15:24.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: Manager daemon y is now available 2023-10-09T05:15:24.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: audit 2023 2023-10-09T05:15:24.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: -10-09T05:15:24.240192+0000 mon.a (mon.0) 41 : audit 2023-10-09T05:15:24.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:24 smithi008 conmon[101053]: [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' 2023-10-09T05:15:25.222 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: { 2023-10-09T05:15:25.222 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-10-09T05:15:25.222 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T05:15:25.223 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: } 2023-10-09T05:15:25.512 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: cephadm 2023-10-09T05:15:24.237691+0000 mgr.y (mgr.14118) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-10-09T05:15:25.512 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: audit 2023-10-09T05:15:24.357712+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' 2023-10-09T05:15:25.512 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: audit 2023 2023-10-09T05:15:25.512 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: -10-09T05:15:24. 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: 358889+0000 mon.a (mon.0) 43 : audit 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: [DBG] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: -09T05:15:24. 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: 360474+0000 mon.a (mon.0) 44 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: : audit [DBG] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: audit 2023 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: -10-09T05:15: 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: 24.362257+0000 mon.a (mon.0) 45 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: : audit [DBG] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: audit 2023 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: -10-09T05:15: 2023-10-09T05:15:25.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: 24.376855+0000 mon.a (mon.0) 46 2023-10-09T05:15:25.514 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: : audit [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:15:25.514 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:15:25.514 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: -09T05:15:24.380127 2023-10-09T05:15:25.514 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: +0000 mon.a (mon.0) 47 : audit 2023-10-09T05:15:25.514 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:15:25.514 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: cluster 2023-10- 2023-10-09T05:15:25.514 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: 09T05:15:25.211232 2023-10-09T05:15:25.514 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: +0000 mon.a (mon.0) 48 : cluster 2023-10-09T05:15:25.514 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:25 smithi008 conmon[101053]: [DBG] mgrmap e7: y(active, since 1.00591s) 2023-10-09T05:15:25.848 INFO:teuthology.orchestra.run.smithi008.stderr:mgr epoch 5 is available 2023-10-09T05:15:25.848 INFO:teuthology.orchestra.run.smithi008.stderr:Setting orchestrator backend to cephadm... 2023-10-09T05:15:26.709 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:26 smithi008 conmon[101053]: audit 2023-10-09T05:15:25.210495+0000 mgr.y (mgr.14118) 2 : 2023-10-09T05:15:26.709 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:26 smithi008 conmon[101053]: audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T05:15:26.709 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:26 smithi008 conmon[101053]: audit 2023-10-09T05:15:25.221829+0000 mgr.y (mgr.14118) 3 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T05:15:27.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:27 smithi008 conmon[101053]: cluster 2023-10-09T05:15:26.360105+0000 mon.a (mon 2023-10-09T05:15:27.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:27 smithi008 conmon[101053]: .0) 49 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-10-09T05:15:27.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:27 smithi008 conmon[101053]: audit 2023-10-09T05:15:26.672045+0000 mgr.y (mgr.14118) 4 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:15:27.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:27 smithi008 conmon[101053]: audit 2023-10-09T05:15:26.676173+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' 2023-10-09T05:15:27.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:27 smithi008 conmon[101053]: audit 2023-10-09T05:15:26.735316+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:15:27.974 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: value unchanged 2023-10-09T05:15:28.588 INFO:teuthology.orchestra.run.smithi008.stderr:Generating ssh key... 2023-10-09T05:15:28.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:28 smithi008 conmon[101053]: audit 2023-10-09T05:15:27.974051+0000 mgr.y 2023-10-09T05:15:28.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:28 smithi008 conmon[101053]: (mgr.14118) 5 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:15:30.169 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101053]: audit 2023-10-09T05:15:29.393987+0000 2023-10-09T05:15:30.170 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101053]: mgr.y (mgr.14118) 6 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:15:30.170 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101053]: cephadm 2023-10-09T05:15:29.394203+0000 mgr.y (mgr.14118) 7 : cephadm [INF] Generating ssh key... 2023-10-09T05:15:30.170 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: Generating public/private rsa key pair. 2023-10-09T05:15:30.170 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: Your identification has been saved in /tmp/tmpn28gycnt/key. 2023-10-09T05:15:30.170 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: Your public key has been saved in /tmp/tmpn28gycnt/key.pub. 2023-10-09T05:15:30.170 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: The key fingerprint is: 2023-10-09T05:15:30.170 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: SHA256:qkja7yMiUpJWoyMTRN8IbydPMC5MuUO43OiH+mPH5+0 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: The key's randomart image is: 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: +---[RSA 3072]----+ 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: |o+.o | 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: |++= = | 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: |+++B + | 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: |o=o+= | 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: |.o= .. S | 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: |=*.. . | 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: |+++. . | 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: |=++.+... | 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: |+++*++..E | 2023-10-09T05:15:30.171 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:29 smithi008 conmon[101509]: +----[SHA256]-----+ 2023-10-09T05:15:31.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:30 smithi008 conmon[101053]: audit 2023-10-09T05:15:29.960313 2023-10-09T05:15:31.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:30 smithi008 conmon[101053]: +0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' 2023-10-09T05:15:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:30 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:15:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:30 smithi008 conmon[101053]: -09T05:15:29.962009+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' 2023-10-09T05:15:31.429 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3R1LU96+u6MxB3eYLPnlNhhMVPcNcuS3L/0Nk3AEHMMpmpkSkFBQasuBJaW2Wje7jT9BEuENCcSsIFwN37aYeRXxBydmdpxCUUB3TiL38nIGWWXGs09yUxSehISR6gTFJzdzWcjr5zC/BvpXvdfgP++AC+NUQybZ5HauxEh5bV7okIUa41JqJjJPaBahtA7mFDKrvW5rTB4CMmQkntEJLT7I632IIkOU6q1l0IyTXlezQTcBL8jAHBvoS7DQ4/FIKmHvorFoJCtMOMEBkuRsPlfW+3pLfphCpaRqUglNkTqmZwPRzSKLZiY5rqcm49GNMtZ3JdYsHmDyeGCaeXMLKr60RrNMaP2gfc1fFfMtGrxzN4NaWKlgV9pLRgTdYX3maxXEia+sjBWaKT2RoWPeSh6JhpgwHFaVza7DaHxVZsQd8gJ6HsqCRe72xXejMBW0XANls6nk091DOmn80ghZLzMgFMhGcIieIB0zxJ4FA8SOIoigR5zxBfdflVvqgbQ0= ceph-9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:15:32.054 INFO:teuthology.orchestra.run.smithi008.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T05:15:32.055 INFO:teuthology.orchestra.run.smithi008.stderr:Adding key to root@localhost authorized_keys... 2023-10-09T05:15:32.056 INFO:teuthology.orchestra.run.smithi008.stderr:Adding host smithi008... 2023-10-09T05:15:32.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:32 smithi008 conmon[101053]: audit 2023-10-09T05:15:31.429031+0000 mgr.y (mgr.14118) 8 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:15:32.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:32 smithi008 conmon[101509]: [78B blob data] 2023-10-09T05:15:33.660 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: Added host 'smithi008' with addr '172.21.15.8' 2023-10-09T05:15:34.312 INFO:teuthology.orchestra.run.smithi008.stderr:Deploying unmanaged mon service... 2023-10-09T05:15:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:34 smithi008 conmon[101053]: audit 2023-10-09T05:15:32.868221+0000 mgr.y (mgr.14118) 9 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi008", "addr": "172.21.15.8", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:15:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:34 smithi008 conmon[101053]: audit 2023-10-09T05:15:33.659308+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' 2023-10-09T05:15:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:34 smithi008 conmon[101053]: cephadm 2023-10-09T05:15:33.659632+0000 mgr.y (mgr.14118) 10 : cephadm [INF] Added host smithi008 2023-10-09T05:15:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:34 smithi008 conmon[101053]: audit 2023-10-09T05:15:33.664293+0000 mon.a (mon.0) 55 : audit [DBG] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:15:35.232 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: Scheduled mon update... 2023-10-09T05:15:36.214 INFO:teuthology.orchestra.run.smithi008.stderr:Deploying unmanaged mgr service... 2023-10-09T05:15:36.498 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:36 smithi008 conmon[101053]: audit 2023-10-09T05:15:35.226922+0000 mgr.y ( 2023-10-09T05:15:36.498 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:36 smithi008 conmon[101053]: mgr.14118) 11 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:15:36.498 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:36 smithi008 conmon[101053]: cephadm 2023-10-09T05:15:35.228353+0000 mgr.y (mgr.14118) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-10-09T05:15:36.498 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:36 smithi008 conmon[101053]: audit 2023-10-09T05:15:35.231496+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' 2023-10-09T05:15:37.146 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-10-09T05:15:37.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:37 smithi008 conmon[101053]: audit 2023-10-09T05:15:36.370031+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' 2023-10-09T05:15:37.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:37 smithi008 conmon[101053]: audit 2023-10-09T05:15:36.649772+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' 2023-10-09T05:15:37.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:37 smithi008 conmon[101053]: audit 2023-10-09T05:15:37.145819+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' 2023-10-09T05:15:38.645 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:38 smithi008 conmon[101053]: audit 2023-10-09T05:15:37.142453 2023-10-09T05:15:38.645 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:38 smithi008 conmon[101053]: +0000 mgr.y (mgr.14118) 13 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:15:38.645 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:38 smithi008 conmon[101053]: cephadm 2023-10-09T05:15:37.143522+0000 mgr.y (mgr.14118) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-10-09T05:15:38.645 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:38 smithi008 conmon[101053]: audit 2023-10-09T05:15:37.914827+0000 mon.a (mon.0) 2023-10-09T05:15:38.645 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:38 smithi008 conmon[101053]: 60 : audit [INF] from='mgr.14118 172.21.15.8:0/2115149619' entity='mgr.y' 2023-10-09T05:15:40.092 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:39 smithi008 conmon[101053]: audit 2023-10-09T05:15:38.794200+0000 2023-10-09T05:15:40.093 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:39 smithi008 conmon[101053]: mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.8:0/1474268673' entity='client.admin' 2023-10-09T05:15:41.336 INFO:teuthology.orchestra.run.smithi008.stderr:Enabling the dashboard module... 2023-10-09T05:15:41.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:41 smithi008 conmon[101053]: audit 2023-10-09T05:15:40.713898+0000 mon.a (mon.0 2023-10-09T05:15:41.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:41 smithi008 conmon[101053]: ) 62 : audit [INF] from='client.? 172.21.15.8:0/1526280844' entity='client.admin' 2023-10-09T05:15:42.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:42 smithi008 conmon[101053]: audit 2023-10-09T05:15:42.290754+0000 mon.a (mon. 2023-10-09T05:15:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:42 smithi008 conmon[101053]: 0) 63 : audit [INF] from='client.? 172.21.15.8:0/648943292' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T05:15:42.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:42 smithi008 conmon[101509]: ignoring --setuser ceph since I am not root 2023-10-09T05:15:42.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:42 smithi008 conmon[101509]: ignoring --setgroup ceph since I am not root 2023-10-09T05:15:42.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:42 smithi008 conmon[101509]: debug 2023-10-09T05:15:42.905+0000 7fe2b7c0b680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:15:43.264 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:43 smithi008 conmon[101509]: debug 2023-10-09T05:15:43.005+0000 7fe2b7c0b680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:15:43.521 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:43 smithi008 conmon[101509]: debug 2023-10-09T05:15:43.464+0000 7fe2b7c0b680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:15:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:43 smithi008 conmon[101053]: audit 2023-10-09T05:15:42.717421+0000 mon.a (mon. 2023-10-09T05:15:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:43 smithi008 conmon[101053]: 0) 64 : audit [INF] from='client.? 172.21.15.8:0/648943292' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T05:15:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:43 smithi008 conmon[101053]: cluster 2023-10-09T05:15:42.717501+0000 mon.a (mon.0) 65 : cluster [DBG] mgrmap e9: y(active, since 18s) 2023-10-09T05:15:44.152 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: { 2023-10-09T05:15:44.152 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "epoch": 9, 2023-10-09T05:15:44.152 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "available": true, 2023-10-09T05:15:44.152 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T05:15:44.152 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T05:15:44.152 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: } 2023-10-09T05:15:44.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:44 smithi008 conmon[101509]: debug 2023-10-09T05:15:44.417+0000 7fe2b7c0b680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:15:44.636 INFO:teuthology.orchestra.run.smithi008.stderr:Waiting for the mgr to restart... 2023-10-09T05:15:44.636 INFO:teuthology.orchestra.run.smithi008.stderr:Waiting for mgr epoch 9... 2023-10-09T05:15:44.770 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:44 smithi008 conmon[101509]: debug 2023-10-09T05:15:44.512+0000 7fe2b7c0b680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:15:44.770 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:44 smithi008 conmon[101053]: audit 2023-10-09T05:15:44.151258+0000 2023-10-09T05:15:44.770 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:44 smithi008 conmon[101053]: mon.a (mon.0) 66 : audit [DBG] from='client.? 172.21.15.8:0/4085743734' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T05:15:45.217 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:44 smithi008 conmon[101509]: debug 2023-10-09T05:15:44.769+0000 7fe2b7c0b680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:15:46.165 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:45 smithi008 conmon[101509]: debug 2023-10-09T05:15:45.907+0000 7fe2b7c0b680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:15:46.165 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:46 smithi008 conmon[101509]: debug 2023-10-09T05:15:46.069+0000 7fe2b7c0b680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:15:46.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:46 smithi008 conmon[101509]: debug 2023-10-09T05:15:46.164+0000 7fe2b7c0b680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:15:46.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:46 smithi008 conmon[101509]: debug 2023-10-09T05:15:46.273+0000 7fe2b7c0b680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:15:46.880 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:46 smithi008 conmon[101509]: debug 2023-10-09T05:15:46.512+0000 7fe2b7c0b680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:15:47.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:46 smithi008 conmon[101509]: debug 2023-10-09T05:15:46.879+0000 7fe2b7c0b680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:15:47.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:47 smithi008 conmon[101509]: debug 2023-10-09T05:15:46.998+0000 7fe2b7c0b680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:15:48.080 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:47 smithi008 conmon[101509]: debug 2023-10-09T05:15:47.869+0000 7fe2b7c0b680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:15:48.081 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:47 smithi008 conmon[101509]: debug 2023-10-09T05:15:47.967+0000 7fe2b7c0b680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:15:48.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:48 smithi008 conmon[101509]: debug 2023-10-09T05:15:48.080+0000 7fe2b7c0b680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:15:48.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:48 smithi008 conmon[101509]: debug 2023-10-09T05:15:48.293+0000 7fe2b7c0b680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:15:48.940 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:48 smithi008 conmon[101509]: debug 2023-10-09T05:15:48.605+0000 7fe2b7c0b680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:15:48.940 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:48 smithi008 conmon[101509]: debug 2023-10-09T05:15:48.768+0000 7fe2b7c0b680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:15:49.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:48 smithi008 conmon[101509]: debug 2023-10-09T05:15:48.939+0000 7fe2b7c0b680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:15:49.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101509]: debug 2023-10-09T05:15:49.155+0000 7fe2b7c0b680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:15:49.516 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101509]: debug 2023-10-09T05:15:49.255+0000 7fe2b7c0b680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:15:49.516 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101509]: [78B blob data] 2023-10-09T05:15:49.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: cluster 2023-10-09T05:15:49. 2023-10-09T05:15:49.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: 269990+0000 mon.a (mon.0) 67 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:15:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: cluster 2023-10-09T05: 2023-10-09T05:15:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: 15:49.270221+0000 mon.a (mon.0) 68 : cluster [INF] Activating manager daemon y 2023-10-09T05:15:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:15:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: -10-09T05:15:49.272793+0000 2023-10-09T05:15:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: mon.a (mon.0) 69 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-10-09T05:15:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: cluster 2023-10-09 2023-10-09T05:15:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: T05:15:49.272889+0000 mon.a (mon.0) 70 2023-10-09T05:15:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: : cluster [DBG] mgrmap e10: y(active, starting, since 0.002761s) 2023-10-09T05:15:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:15:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: -09T05:15:49.273688+0000 mon.a ( 2023-10-09T05:15:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: mon.0) 71 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: 10-09T05:15:49.274475+0000 mon.a 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: (mon.0) 72 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: audit 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: 2023-10-09T05:15:49.275376+0000 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: mon.a (mon.0) 73 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: audit 2023-10-09T05:15:49 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: .275575+0000 mon.a (mon.0) 74 : audit 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: audit 2023-10-09T05:15:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: 2023-10-09T05:15:49.275713 2023-10-09T05:15:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: +0000 mon.a (mon.0) 75 : audit [DBG] 2023-10-09T05:15:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:15:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: cluster 2023-10-09T05:15 2023-10-09T05:15:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: :49.278272+0000 mon.a (mon.0) 76 : 2023-10-09T05:15:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:49 smithi008 conmon[101053]: cluster [INF] Manager daemon y is now available 2023-10-09T05:15:50.285 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: { 2023-10-09T05:15:50.286 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-10-09T05:15:50.286 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T05:15:50.286 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: } 2023-10-09T05:15:50.592 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:50 smithi008 conmon[101053]: audit 2023-10-09T05:15:49.407257+0000 mon.a (mon.0 2023-10-09T05:15:50.592 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:50 smithi008 conmon[101053]: ) 77 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:15:50.593 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:50 smithi008 conmon[101053]: audit 2023-10-09T05:15:49.408579+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:15:50.593 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:50 smithi008 conmon[101053]: audit 2023-10-09T05:15:49.436877+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:15:50.593 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:50 smithi008 conmon[101053]: audit 2023-10-09T05:15:49.440257+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:15:50.593 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:50 smithi008 conmon[101053]: cluster 2023-10-09T05:15:50.275112+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e11: y(active, since 1.00498s) 2023-10-09T05:15:50.968 INFO:teuthology.orchestra.run.smithi008.stderr:mgr epoch 9 is available 2023-10-09T05:15:50.968 INFO:teuthology.orchestra.run.smithi008.stderr:Generating a dashboard self-signed certificate... 2023-10-09T05:15:51.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:51 smithi008 conmon[101053]: audit 2023-10-09T05:15:50.274658+0000 mgr.y (mgr.14148) 1 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T05:15:51.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:51 smithi008 conmon[101053]: audit 2023-10-09T05:15:50.285323+0000 mgr.y (mgr.14148) 2 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T05:15:51.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:51 smithi008 conmon[101053]: audit 2023-10-09T05:15:50.954491+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:15:52.073 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: Self-signed certificate created 2023-10-09T05:15:52.835 INFO:teuthology.orchestra.run.smithi008.stderr:Creating initial admin user... 2023-10-09T05:15:53.112 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:52 smithi008 conmon[101053]: audit 2023-10-09T05:15:51.816313+0000 2023-10-09T05:15:53.112 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:52 smithi008 conmon[101053]: mgr.y (mgr.14148) 3 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:15:53.112 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:52 smithi008 conmon[101053]: cluster 2023-10-09T05:15:51.956469+0000 mon.a (mon.0) 83 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-10-09T05:15:53.112 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:52 smithi008 conmon[101053]: audit 2023-10-09T05:15:52.071313+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:15:53.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:52 smithi008 conmon[101053]: audit 2023-10-09T05:15:53.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:52 smithi008 conmon[101053]: 2023-10-09T05:15:52.073321+0000 mon.a (mon. 2023-10-09T05:15:53.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:52 smithi008 conmon[101053]: 0) 85 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:15:53.944 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$6.cNUbiVRgr/bts/ugGTjuL5wukjVVjqoK7vkiz6t7muKE9VhKquO", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696828553, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T05:15:54.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:53 smithi008 conmon[101053]: audit 2023-10-09T05:15:53.699485+0000 mgr.y (mgr.14148) 4 : audit [DBG] from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:15:54.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:53 smithi008 conmon[101053]: audit 2023-10-09T05:15:53.944579+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:15:54.595 INFO:teuthology.orchestra.run.smithi008.stderr:Fetching dashboard port number... 2023-10-09T05:15:55.426 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: 8443 2023-10-09T05:15:55.986 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: disabled 2023-10-09T05:15:55.993 INFO:teuthology.orchestra.run.smithi008.stderr:systemctl: inactive 2023-10-09T05:15:55.994 INFO:teuthology.orchestra.run.smithi008.stderr:firewalld.service is not enabled 2023-10-09T05:15:55.994 INFO:teuthology.orchestra.run.smithi008.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T05:15:55.996 INFO:teuthology.orchestra.run.smithi008.stderr:Ceph Dashboard is now available at: 2023-10-09T05:15:55.996 INFO:teuthology.orchestra.run.smithi008.stderr: 2023-10-09T05:15:55.996 INFO:teuthology.orchestra.run.smithi008.stderr: URL: https://smithi008:8443/ 2023-10-09T05:15:55.996 INFO:teuthology.orchestra.run.smithi008.stderr: User: admin 2023-10-09T05:15:55.996 INFO:teuthology.orchestra.run.smithi008.stderr: Password: oyyxloislb 2023-10-09T05:15:55.996 INFO:teuthology.orchestra.run.smithi008.stderr: 2023-10-09T05:15:55.997 INFO:teuthology.orchestra.run.smithi008.stderr:Enabling autotune for osd_memory_target 2023-10-09T05:15:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:56 smithi008 conmon[101053]: cluster 2023-10-09T05:15:55.336697+0000 mon.a ( 2023-10-09T05:15:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:56 smithi008 conmon[101053]: mon.0) 87 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-10-09T05:15:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:56 smithi008 conmon[101053]: audit 2023-10-09T05:15:55.425966+0000 mon.a (mon.0) 88 : audit [DBG] from='client.? 172.21.15.8:0/1820687189' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T05:15:58.337 INFO:teuthology.orchestra.run.smithi008.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-10-09T05:15:58.943 INFO:teuthology.orchestra.run.smithi008.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T05:15:58.943 INFO:teuthology.orchestra.run.smithi008.stderr: 2023-10-09T05:15:58.943 INFO:teuthology.orchestra.run.smithi008.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:15:58.943 INFO:teuthology.orchestra.run.smithi008.stderr: 2023-10-09T05:15:58.943 INFO:teuthology.orchestra.run.smithi008.stderr:Or, if you are only running a single cluster on this host: 2023-10-09T05:15:58.943 INFO:teuthology.orchestra.run.smithi008.stderr: 2023-10-09T05:15:58.943 INFO:teuthology.orchestra.run.smithi008.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T05:15:58.943 INFO:teuthology.orchestra.run.smithi008.stderr: 2023-10-09T05:15:58.943 INFO:teuthology.orchestra.run.smithi008.stderr:Please consider enabling telemetry to help improve Ceph: 2023-10-09T05:15:58.944 INFO:teuthology.orchestra.run.smithi008.stderr: 2023-10-09T05:15:58.944 INFO:teuthology.orchestra.run.smithi008.stderr: ceph telemetry on 2023-10-09T05:15:58.944 INFO:teuthology.orchestra.run.smithi008.stderr: 2023-10-09T05:15:58.944 INFO:teuthology.orchestra.run.smithi008.stderr:For more information see: 2023-10-09T05:15:58.944 INFO:teuthology.orchestra.run.smithi008.stderr: 2023-10-09T05:15:58.944 INFO:teuthology.orchestra.run.smithi008.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-10-09T05:15:58.944 INFO:teuthology.orchestra.run.smithi008.stderr: 2023-10-09T05:15:58.944 INFO:teuthology.orchestra.run.smithi008.stderr:Bootstrap complete. 2023-10-09T05:15:58.995 INFO:tasks.cephadm:Fetching config... 2023-10-09T05:15:58.995 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:15:58.995 DEBUG:teuthology.orchestra.run.smithi008:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T05:15:59.013 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T05:15:59.013 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:15:59.013 DEBUG:teuthology.orchestra.run.smithi008:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T05:15:59.071 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T05:15:59.071 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:15:59.071 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd if=/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T05:15:59.141 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T05:15:59.141 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:15:59.141 DEBUG:teuthology.orchestra.run.smithi008:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T05:15:59.199 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T05:15:59.199 DEBUG:teuthology.orchestra.run.smithi008:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3R1LU96+u6MxB3eYLPnlNhhMVPcNcuS3L/0Nk3AEHMMpmpkSkFBQasuBJaW2Wje7jT9BEuENCcSsIFwN37aYeRXxBydmdpxCUUB3TiL38nIGWWXGs09yUxSehISR6gTFJzdzWcjr5zC/BvpXvdfgP++AC+NUQybZ5HauxEh5bV7okIUa41JqJjJPaBahtA7mFDKrvW5rTB4CMmQkntEJLT7I632IIkOU6q1l0IyTXlezQTcBL8jAHBvoS7DQ4/FIKmHvorFoJCtMOMEBkuRsPlfW+3pLfphCpaRqUglNkTqmZwPRzSKLZiY5rqcm49GNMtZ3JdYsHmDyeGCaeXMLKr60RrNMaP2gfc1fFfMtGrxzN4NaWKlgV9pLRgTdYX3maxXEia+sjBWaKT2RoWPeSh6JhpgwHFaVza7DaHxVZsQd8gJ6HsqCRe72xXejMBW0XANls6nk091DOmn80ghZLzMgFMhGcIieIB0zxJ4FA8SOIoigR5zxBfdflVvqgbQ0= ceph-9e85b874-6662-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T05:15:59.280 INFO:teuthology.orchestra.run.smithi008.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3R1LU96+u6MxB3eYLPnlNhhMVPcNcuS3L/0Nk3AEHMMpmpkSkFBQasuBJaW2Wje7jT9BEuENCcSsIFwN37aYeRXxBydmdpxCUUB3TiL38nIGWWXGs09yUxSehISR6gTFJzdzWcjr5zC/BvpXvdfgP++AC+NUQybZ5HauxEh5bV7okIUa41JqJjJPaBahtA7mFDKrvW5rTB4CMmQkntEJLT7I632IIkOU6q1l0IyTXlezQTcBL8jAHBvoS7DQ4/FIKmHvorFoJCtMOMEBkuRsPlfW+3pLfphCpaRqUglNkTqmZwPRzSKLZiY5rqcm49GNMtZ3JdYsHmDyeGCaeXMLKr60RrNMaP2gfc1fFfMtGrxzN4NaWKlgV9pLRgTdYX3maxXEia+sjBWaKT2RoWPeSh6JhpgwHFaVza7DaHxVZsQd8gJ6HsqCRe72xXejMBW0XANls6nk091DOmn80ghZLzMgFMhGcIieIB0zxJ4FA8SOIoigR5zxBfdflVvqgbQ0= ceph-9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:15:59.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3R1LU96+u6MxB3eYLPnlNhhMVPcNcuS3L/0Nk3AEHMMpmpkSkFBQasuBJaW2Wje7jT9BEuENCcSsIFwN37aYeRXxBydmdpxCUUB3TiL38nIGWWXGs09yUxSehISR6gTFJzdzWcjr5zC/BvpXvdfgP++AC+NUQybZ5HauxEh5bV7okIUa41JqJjJPaBahtA7mFDKrvW5rTB4CMmQkntEJLT7I632IIkOU6q1l0IyTXlezQTcBL8jAHBvoS7DQ4/FIKmHvorFoJCtMOMEBkuRsPlfW+3pLfphCpaRqUglNkTqmZwPRzSKLZiY5rqcm49GNMtZ3JdYsHmDyeGCaeXMLKr60RrNMaP2gfc1fFfMtGrxzN4NaWKlgV9pLRgTdYX3maxXEia+sjBWaKT2RoWPeSh6JhpgwHFaVza7DaHxVZsQd8gJ6HsqCRe72xXejMBW0XANls6nk091DOmn80ghZLzMgFMhGcIieIB0zxJ4FA8SOIoigR5zxBfdflVvqgbQ0= ceph-9e85b874-6662-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T05:15:59.350 INFO:teuthology.orchestra.run.smithi191.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3R1LU96+u6MxB3eYLPnlNhhMVPcNcuS3L/0Nk3AEHMMpmpkSkFBQasuBJaW2Wje7jT9BEuENCcSsIFwN37aYeRXxBydmdpxCUUB3TiL38nIGWWXGs09yUxSehISR6gTFJzdzWcjr5zC/BvpXvdfgP++AC+NUQybZ5HauxEh5bV7okIUa41JqJjJPaBahtA7mFDKrvW5rTB4CMmQkntEJLT7I632IIkOU6q1l0IyTXlezQTcBL8jAHBvoS7DQ4/FIKmHvorFoJCtMOMEBkuRsPlfW+3pLfphCpaRqUglNkTqmZwPRzSKLZiY5rqcm49GNMtZ3JdYsHmDyeGCaeXMLKr60RrNMaP2gfc1fFfMtGrxzN4NaWKlgV9pLRgTdYX3maxXEia+sjBWaKT2RoWPeSh6JhpgwHFaVza7DaHxVZsQd8gJ6HsqCRe72xXejMBW0XANls6nk091DOmn80ghZLzMgFMhGcIieIB0zxJ4FA8SOIoigR5zxBfdflVvqgbQ0= ceph-9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:15:59.363 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T05:15:59.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:15:59 smithi008 conmon[101053]: audit 2023-10-09T05:15:58.334472+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.8:0/2961963461' entity='client.admin' 2023-10-09T05:16:02.661 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T05:16:02.661 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T05:16:02.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:02 smithi008 conmon[101053]: audit 2023-10-09T05:16:01.818299+0000 mon.a (mon.0) 90 : audit [INF] from='client.? 172.21.15.8:0/575579286' entity='client.admin' 2023-10-09T05:16:04.170 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:03 smithi008 conmon[101053]: audit 2023-10-09T05:16:02.910523+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:04.171 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:03 smithi008 conmon[101053]: audit 2023-10-09T05:16:03.319432+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi008", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:16:04.171 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:03 smithi008 conmon[101053]: audit 2023-10-09T05:16:03.322658+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:04.171 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:03 smithi008 conmon[101053]: audit 2023-10-09T05:16:03.324191+0000 mon.a (mon.0) 94 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:04.171 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:03 smithi008 conmon[101053]: audit 2023-10-09T05:16:03.327250+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:05.165 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi191 2023-10-09T05:16:05.165 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T05:16:05.165 DEBUG:teuthology.orchestra.run.smithi191:> dd of=/etc/ceph/ceph.conf 2023-10-09T05:16:05.186 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T05:16:05.186 DEBUG:teuthology.orchestra.run.smithi191:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:16:05.245 INFO:tasks.cephadm:Adding host smithi191 to orchestrator... 2023-10-09T05:16:05.245 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi191 2023-10-09T05:16:05.868 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:05 smithi008 conmon[101053]: audit 2023-10-09T05:16: 2023-10-09T05:16:05.868 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:05 smithi008 conmon[101053]: 04.512287+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:05.868 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:05 smithi008 conmon[101053]: audit 2023-10-09T05:16:04.543858+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:05.868 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:05 smithi008 conmon[101053]: audit 2023-10-09T05:16:04.545364+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:05.868 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:05 smithi008 conmon[101053]: audit 2023-10-09T05:16:04.546129+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:05.868 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:05 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:04.547285+0000 mgr.y (mgr.14148) 6 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:16:05.869 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:05 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:04.584142+0000 mgr.y (mgr.14148) 7 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:16:05.869 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:05 smithi008 conmon[101053]: audit 2023-10-09T05:16:04.661861+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:05.869 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:05 smithi008 conmon[101053]: audit 2023-10-09T05:16:04.664147+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:07.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:16:07 smithi008 conmon[101509]: [90B blob data] 2023-10-09T05:16:07.733 INFO:teuthology.orchestra.run.smithi008.stdout:Added host 'smithi191' with addr '172.21.15.191' 2023-10-09T05:16:08.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:16:07 smithi008 conmon[101509]: [80B blob data] 2023-10-09T05:16:08.367 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T05:16:09.006 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:08 smithi008 conmon[101053]: audit 2023-10-09T05:16:07. 2023-10-09T05:16:09.006 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:08 smithi008 conmon[101053]: 025097+0000 mgr.y (mgr.14148) 8 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi191", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:16:09.007 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:08 smithi008 conmon[101053]: audit 2023-10-09T05:16:07.732057+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:09.007 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:08 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:07.732417+0000 mgr.y (mgr.14148) 9 : cephadm [INF] Added host smithi191 2023-10-09T05:16:09.007 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:08 smithi008 conmon[101053]: audit 2023-10-09T05:16:07.737077+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:09.007 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:08 smithi008 conmon[101053]: audit 2023-10-09T05:16:08.212039+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:09.007 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:08 smithi008 conmon[101053]: audit 2023-10-09T05:16:08.569401+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:10.221 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:16:10.221 INFO:teuthology.orchestra.run.smithi008.stdout:[{"addr": "172.21.15.8", "hostname": "smithi008", "labels": [], "status": ""}, {"addr": "172.21.15.191", "hostname": "smithi191", "labels": [], "status": ""}] 2023-10-09T05:16:10.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:10 smithi008 conmon[101053]: cluster 2023-10-09T05:16:09.276805+0000 mgr.y (mgr.14148) 10 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:10.816 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T05:16:10.817 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T05:16:11.669 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:11 smithi008 conmon[101053]: audit 2023-10-09T05:16:10.220757 2023-10-09T05:16:11.669 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:11 smithi008 conmon[101053]: +0000 mgr.y (mgr.14148) 11 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:16:12.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:12 smithi008 conmon[101053]: cluster 2023-10-09T05:16:11.277131+0000 mgr.y (mgr.14148) 12 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:12 smithi008 conmon[101053]: audit 2023-10-09T05:16:11.601432+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:12 smithi008 conmon[101053]: audit 2023-10-09T05:16:11.929742+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:12 smithi008 conmon[101053]: audit 2023-10-09T05:16:11.930426+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi191", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:16:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:12 smithi008 conmon[101053]: audit 2023-10-09T05:16:11.931338+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:12 smithi008 conmon[101053]: audit 2023-10-09T05:16:11.932032+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:12 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:16:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:12 smithi008 conmon[101053]: 09T05:16:11.987895+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:12 smithi008 conmon[101053]: audit 2023-10-09T05:16:11.990247+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:13.608 INFO:teuthology.orchestra.run.smithi008.stderr:adjusted tunables profile to default 2023-10-09T05:16:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:13 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:11.933146+0000 mgr.y (mgr 2023-10-09T05:16:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:13 smithi008 conmon[101053]: .14148) 13 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:16:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:13 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:11.960412+0000 mgr.y (mgr.14148 2023-10-09T05:16:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:13 smithi008 conmon[101053]: ) 14 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:16:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:13 smithi008 conmon[101053]: audit 2023-10-09T05:16:12.647273+0000 mon.a ( 2023-10-09T05:16:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:13 smithi008 conmon[101053]: mon.0) 113 : audit [INF] from='client.? 172.21.15.8:0/4122434991' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T05:16:14.185 INFO:tasks.cephadm:Adding mon.a on smithi008 2023-10-09T05:16:14.185 INFO:tasks.cephadm:Adding mon.c on smithi008 2023-10-09T05:16:14.186 INFO:tasks.cephadm:Adding mon.b on smithi191 2023-10-09T05:16:14.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi008:172.21.15.8=a;smithi008:[v2:172.21.15.8:3301,v1:172.21.15.8:6790]=c;smithi191:172.21.15.191=b' 2023-10-09T05:16:14.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:14 smithi008 conmon[101053]: cluster 2023-10-09T05:16:13.277783+0000 mgr.y (mgr. 2023-10-09T05:16:14.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:14 smithi008 conmon[101053]: 14148) 15 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:14.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:14 smithi008 conmon[101053]: audit 2023-10-09T05:16:13.605314+0000 mon.a (mon.0) 114 : audit [INF] from='client.? 172.21.15.8:0/4122434991' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T05:16:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:14 smithi008 conmon[101053]: cluster 2023-10-09T05:16:13.605387+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:16:16.180 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled mon update... 2023-10-09T05:16:16.791 DEBUG:teuthology.orchestra.run.smithi008:mon.c> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T05:16:17.416 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: cluster 2023-10-09T05:16:15.278149+0000 mgr.y (mgr.14148) 2023-10-09T05:16:17.416 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: 16 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:17.416 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: audit 2023-10-09T05:16:16.179803+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:17.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: audit 2023-10-09T05:16:16.215380+0000 mon.a (mon.0 2023-10-09T05:16:17.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: ) 117 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:17.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:16:17.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: 09T05:16:16.217343+0000 mon.a (mon.0) 118 : 2023-10-09T05:16:17.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:17.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: audit 2023-10-09T05:16:17.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: 2023-10-09T05:16:16.218234+0000 2023-10-09T05:16:17.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: mon.a (mon.0) 119 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:17.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:16:17.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: 09T05:16:16.221774+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:17.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: 2023-10-09T05:16:17.418 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: audit 2023-10-09T05:16:16. 2023-10-09T05:16:17.418 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: 223879+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:17.418 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: audit 2023-10-09T05:16:17.418 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: 2023-10-09T05:16:16.224820+0000 mon.a (mon 2023-10-09T05:16:17.418 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:16 smithi008 conmon[101053]: .0) 122 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:17.422 DEBUG:teuthology.orchestra.run.smithi191:mon.b> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T05:16:17.425 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T05:16:17.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T05:16:17.455 INFO:journalctl@ceph.mon.b.smithi191.stdout:-- Logs begin at Mon 2023-10-09 04:57:51 UTC. -- 2023-10-09T05:16:17.458 INFO:journalctl@ceph.mon.c.smithi008.stdout:-- Logs begin at Mon 2023-10-09 04:57:44 UTC. -- 2023-10-09T05:16:17.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:17 smithi008 conmon[101053]: audit 2023-10-09T05:16:16.174419+0000 mgr.y (mgr.14148) 17 2023-10-09T05:16:17.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:17 smithi008 conmon[101053]: : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi008:172.21.15.8=a;smithi008:[v2:172.21.15.8:3301,v1:172.21.15.8:6790]=c;smithi191:172.21.15.191=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:16:17.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:17 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:16.177209+0000 mgr.y (mgr.14148) 18 : cephadm [INF] Saving service mon spec with placement smithi008:172.21.15.8=a;smithi008:[v2:172.21.15.8:3301,v1:172.21.15.8:6790]=c;smithi191:172.21.15.191=b;count:3 2023-10-09T05:16:17.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:17 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:16.225685+0000 mgr.y (mgr.14148) 19 : cephadm [INF] Deploying daemon mon.b on smithi191 2023-10-09T05:16:18.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:18 smithi008 conmon[101053]: cluster 2023-10-09T05 2023-10-09T05:16:18.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:18 smithi008 conmon[101053]: :16:17.278576+0000 mgr.y (mgr.14148) 20 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:19.784 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:16:19.785 INFO:teuthology.orchestra.run.smithi191.stdout:{"epoch":1,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","modified":"2023-10-09T05:14:53.552466Z","created":"2023-10-09T05:14:53.552466Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:3300","nonce":0},{"type":"v1","addr":"172.21.15.8:6789","nonce":0}]},"addr":"172.21.15.8:6789/0","public_addr":"172.21.15.8:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-09T05:16:19.786 INFO:teuthology.orchestra.run.smithi191.stderr:dumped monmap epoch 1 2023-10-09T05:16:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:20 smithi008 conmon[101053]: cluster 2023-10-09T05:16:19.278992+0000 mgr.y (mgr.14148) 2023-10-09T05:16:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:20 smithi008 conmon[101053]: 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:20 smithi008 conmon[101053]: audit 2023-10-09T05:16:19.784203+0000 mon.a (mon.0) 123 : audit [DBG] from='client.? 172.21.15.191:0/1127711612' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:16:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:20 smithi191 systemd[1]: Starting Ceph mon.b for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:16:21.521 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T05:16:21.521 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T05:16:21.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.388+0000 7f08ee96f880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:16:21.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.388+0000 7f08ee96f880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T05:16:21.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.388+0000 7f08ee96f880 0 pidfile_write: ignore empty --pid-file 2023-10-09T05:16:21.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 0 load: jerasure load: lrc load: isa 2023-10-09T05:16:21.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T05:16:21.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: DB SUMMARY 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-10-09T05:16:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000003.log size: 511 ; 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.env: 0x561696ce3080 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.fs: Posix File System 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.info_log: 0x561698ceda80 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.statistics: (nil) 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.use_fsync: 0 2023-10-09T05:16:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T05:16:21.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:16:21.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:16:21.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:16:21.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:16:21.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T05:16:21.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:16:21.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:16:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T05:16:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:16:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:16:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.db_log_dir: 2023-10-09T05:16:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-10-09T05:16:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:16:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T05:16:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T05:16:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:16:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:16:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.write_buffer_manager: 0x5616999e8330 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:16:21.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.unordered_write: 0 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.row_cache: None 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.wal_filter: None 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T05:16:21.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T05:16:21.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:16:21.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T05:16:21.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:16:21.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:16:21.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:16:21.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T05:16:21.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T05:16:21.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T05:16:21.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T05:16:21.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:16:21.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_open_files: -1 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:16:21.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Compression algorithms supported: 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: kZSTD supported: 0 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.merge_operator: 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_filter: None 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561698c97d18) 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: cache_index_and_filter_blocks: 1 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:16:21.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: pin_top_level_index_and_filter: 1 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: index_type: 0 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: data_block_index_type: 0 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: index_shortening: 1 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: hash_index_allow_collision: 1 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: checksum: 1 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: no_block_cache: 0 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: block_cache: 0x561698cd0f10 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: block_cache_name: BinnedLRUCache 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: block_cache_options: 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: capacity : 536870912 2023-10-09T05:16:21.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: num_shard_bits : 4 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: strict_capacity_limit : 0 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: high_pri_pool_ratio: 0.000 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: block_cache_compressed: (nil) 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: persistent_cache: (nil) 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: block_size: 4096 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: block_size_deviation: 10 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: block_restart_interval: 16 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: index_block_restart_interval: 1 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: metadata_block_size: 4096 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: partition_filters: 0 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: use_delta_encoding: 1 2023-10-09T05:16:21.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: whole_key_filtering: 1 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: verify_compression: 0 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: read_amp_bytes_per_bit: 0 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: format_version: 2 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: enable_index_compression: 1 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: block_align: 0 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compression: NoCompression 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:16:21.563 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.num_levels: 7 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:16:21.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:16:21.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:16:21.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:16:21.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.393+0000 7f08ee96f880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:16:21.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.ttl: 2592000 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696828581395205, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.394+0000 7f08ee96f880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.395+0000 7f08ee96f880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696828581395672, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696828581, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T05:16:21.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.395+0000 7f08ee96f880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.395+0000 7f08ee96f880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696828581396023, "job": 1, "event": "recovery_finished"} 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.396+0000 7f08ee96f880 4 rocksdb: DB pointer 0x561698d9b800 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.396+0000 7f08d5776700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.396+0000 7f08d5776700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: ** DB Stats ** 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:16:21.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: ** Compaction Stats [default] ** 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: ** Compaction Stats [default] ** 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:16:21.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:21.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:16:21.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:16:21.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:16:21.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:16:21.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:16:21.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:16:21.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:16:21.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Interval compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:16:21.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:16:21.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:16:21.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: ** Compaction Stats [default] ** 2023-10-09T05:16:21.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:16:21.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:16:21.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:21.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:21.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:16:21.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: ** Compaction Stats [default] ** 2023-10-09T05:16:21.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:16:21.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.396+0000 7f08ee96f880 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.396+0000 7f08ee96f880 0 using public_addr v2:172.21.15.191:0/0 -> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.396+0000 7f08ee96f880 0 starting mon.b rank -1 at public addrs [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] at bind addrs [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.397+0000 7f08ee96f880 1 mon.b@-1(???) e0 preinit fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 bash[101259]: 2fa5874c9b2ca9ba6b36d9f5a8c7c8de1eaeb99bf15bca7164d05f48542f0f4d 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 systemd[1]: Started Ceph mon.b for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.416+0000 7f08d7f7b700 0 mon.b@-1(synchronizing).mds e1 new map 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.416+0000 7f08d7f7b700 0 mon.b@-1(synchronizing).mds e1 print_map 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: e1 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:16:21.575 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: legacy client fscid: -1 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: No filesystems configured 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.416+0000 7f08d7f7b700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.416+0000 7f08d7f7b700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.417+0000 7f08d7f7b700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.417+0000 7f08d7f7b700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.417+0000 7f08d7f7b700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.417+0000 7f08d7f7b700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.417+0000 7f08d7f7b700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.417+0000 7f08d7f7b700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:16:21.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.417+0000 7f08d7f7b700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:16:21.577 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.417+0000 7f08d7f7b700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:16:21.577 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: cluster 2023- 2023-10-09T05:16:21.577 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 10-09T05:16:19.278992+0000 mgr.y (mgr.14148) 21 : cluster 2023-10-09T05:16:21.577 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:21.577 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: audit 2023-10-09T05:16:21.577 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: 2023-10-09T05:16:19.784203+0000 mon.a (mon.0) 123 : audit 2023-10-09T05:16:21.577 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: [DBG] from='client.? 172.21.15.191:0/1127711612' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:16:21.577 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:21 smithi191 conmon[101299]: debug 2023-10-09T05:16:21.417+0000 7f08d7f7b700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T05:16:26.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: cluster 2023-10-09T05:16:21.279306+0000 mgr.y (mgr.14148) 2023-10-09T05:16:26.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: 22 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:26.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: audit 2023-10-09T05:16:21.422065+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: audit 2023-10-09T05:16:21.424482+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:16:26.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: cluster 2023-10-09T05:16:21.424677+0000 mon.a (mon.0) 126 : cluster [INF] mon.a calling monitor election 2023-10-09T05:16:26.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: audit 2023-10-09T05:16:21.438442+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: audit 2023-10-09T05:16:22.421746+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: cluster 2023-10-09T05:16:23.279559+0000 mgr.y (mgr.14148) 23 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:26.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: audit 2023-10-09T05:16:23.421606+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: cluster 2023-10-09T05:16:23.422578+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T05:16:26.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: audit 2023-10-09T05:16:24.421588+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: cluster 2023-10-09T05:16:25.279882+0000 mgr.y (mgr.14148) 24 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:26.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: audit 2023-10-09T05:16:25.421814+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: audit 2023-10-09T05:16:26.422020+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: cluster 2023-10-09T05:16:26.451693+0000 mon.a (mon.0) 133 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:16:26.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: cluster 2023-10-09T05:16:26.460348+0000 mon.a (mon.0) 134 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0]} removed_ranks: {} 2023-10-09T05:16:26.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: cluster 2023-10-09T05:16:26.472763+0000 mon.a (mon.0) 135 : cluster [DBG] fsmap 2023-10-09T05:16:26.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: cluster 2023-10-09T05:16:26.472807+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:16:26.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: cluster 2023-10-09T05:16:26.473030+0000 mon.a (mon.0) 137 : cluster [DBG] mgrmap e13: y(active, since 37s) 2023-10-09T05:16:26.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: cluster 2023-10-09T05:16:26.476253+0000 mon.a (mon.0) 138 : cluster [INF] overall HEALTH_OK 2023-10-09T05:16:26.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: audit 2023-10-09T05:16:26.482058+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:26.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: audit 2023-10-09T05:16:26.483118+0000 mon.a (mon.0) 140 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:26.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:26 smithi191 conmon[101299]: audit 2023-10-09T05:16:26.484053+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:26.865 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:16:26.865 INFO:teuthology.orchestra.run.smithi191.stdout:{"epoch":2,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","modified":"2023-10-09T05:16:21.421728Z","created":"2023-10-09T05:14:53.552466Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:3300","nonce":0},{"type":"v1","addr":"172.21.15.8:6789","nonce":0}]},"addr":"172.21.15.8:6789/0","public_addr":"172.21.15.8:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:3300","nonce":0},{"type":"v1","addr":"172.21.15.191:6789","nonce":0}]},"addr":"172.21.15.191:6789/0","public_addr":"172.21.15.191:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-09T05:16:26.867 INFO:teuthology.orchestra.run.smithi191.stderr:dumped monmap epoch 2 2023-10-09T05:16:26.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster 2023-10-09T05:16:21.279306+0000 mgr.y (mgr.14148 2023-10-09T05:16:26.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: ) 22 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:26.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: audit 2023-10-09T05:16:21.422065+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: audit 2023-10-09T05:16:21.424482+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:16:26.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster 2023-10-09T05:16:21.424677+0000 mon.a (mon.0) 126 : cluster [INF] mon.a calling monitor election 2023-10-09T05:16:26.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: audit 2023-10-09T05:16:21.438442+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:16:26.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 10-09T05:16:22.421746+0000 2023-10-09T05:16:26.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: mon.a (mon.0) 128 : audit 2023-10-09T05:16:26.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster 2023- 2023-10-09T05:16:26.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 10-09T05:16:23.279559+0000 2023-10-09T05:16:26.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: mgr.y (mgr.14148) 23 : cluster 2023-10-09T05:16:26.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:26.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: audit 2023-10-09T05:16:26.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 2023-10-09T05:16:23 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: .421606+0000 mon.a (mon.0 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: ) 129 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster 2023- 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 10-09T05:16:23.422578 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: +0000 mon.b (mon.1) 1 : 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster [INF] mon.b calling monitor election 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: audit 2023 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: -10-09T05:16:24. 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 421588+0000 mon.a (mon.0) 130 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:16:26.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: -10-09T05:16:25 2023-10-09T05:16:26.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: .279882+0000 mgr.y (mgr.14148 2023-10-09T05:16:26.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: ) 24 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:26.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: audit 2023 2023-10-09T05:16:26.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: -10-09T05:16:25. 2023-10-09T05:16:26.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 421814+0000 mon.a (mon.0) 2023-10-09T05:16:26.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 131 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: audit 2023 2023-10-09T05:16:26.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: -10-09T05:16:26. 2023-10-09T05:16:26.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 422020+0000 mon.a (mon.0) 2023-10-09T05:16:26.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 132 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:26.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:16:26.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: -10-09T05:16:26. 2023-10-09T05:16:26.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 451693+0000 mon.a (mon.0 2023-10-09T05:16:26.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: ) 133 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:16:26.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:16:26.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: -10-09T05:16:26 2023-10-09T05:16:26.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: .460348+0000 mon.a (mon.0 2023-10-09T05:16:26.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: ) 134 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0]} removed_ranks: {} 2023-10-09T05:16:26.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:16:26.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: -10-09T05:16:26. 2023-10-09T05:16:26.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 472763+0000 mon.a (mon.0) 135 2023-10-09T05:16:26.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: : cluster [DBG] fsmap 2023-10-09T05:16:26.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster 2023- 2023-10-09T05:16:26.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 10-09T05:16:26.472807+0000 2023-10-09T05:16:26.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: mon.a (mon.0) 136 2023-10-09T05:16:26.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:16:26.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster 2023-10-09T05:16:26.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 2023-10-09T05:16:26 2023-10-09T05:16:26.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: .473030+0000 mon.a (mon. 2023-10-09T05:16:26.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 0) 137 : cluster [DBG] 2023-10-09T05:16:26.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: mgrmap e13: y(active, since 37s) 2023-10-09T05:16:26.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:16:26.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: -10-09T05:16:26 2023-10-09T05:16:26.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: .476253+0000 mon.a (mon.0 2023-10-09T05:16:26.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: ) 138 : cluster [INF] overall HEALTH_OK 2023-10-09T05:16:26.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: audit 2023 2023-10-09T05:16:26.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: -10-09T05:16:26. 2023-10-09T05:16:26.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 482058+0000 mon.a (mon.0 2023-10-09T05:16:26.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: ) 139 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:26.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: audit 2023 2023-10-09T05:16:26.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: -10-09T05:16:26 2023-10-09T05:16:26.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: .483118+0000 mon.a (mon. 2023-10-09T05:16:26.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 0) 140 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:26.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: 2023-10-09T05:16:26.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: audit 2023 2023-10-09T05:16:26.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: -10-09T05:16:26 2023-10-09T05:16:26.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: .484053+0000 mon.a (mon.0 2023-10-09T05:16:26.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:26 smithi008 conmon[101053]: ) 141 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:27 smithi191 conmon[101299]: cephadm 2023-10-09T05:16:26.484889+0000 mgr.y (mgr.14148) 25 : cephadm [INF] 2023-10-09T05:16:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:27 smithi191 conmon[101299]: Deploying daemon mon.c on smithi008 2023-10-09T05:16:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:27 smithi191 conmon[101299]: audit 2023-10-09T05:16:26.864949+0000 mon.a (mon.0) 142 : audit [DBG] from='client.? 172.21.15.191:0/3908539974' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:16:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:27 smithi191 conmon[101299]: audit 2023-10-09T05:16:27.422137+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:27.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:27 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:26.484889+0000 mgr.y (mgr.14148) 25 : cephadm [INF] Deploying daemon mon.c on smithi008 2023-10-09T05:16:27.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:27 smithi008 conmon[101053]: audit 2023-10-09T05:16:26.864949+0000 mon.a (mon.0) 142 : audit [DBG] from='client.? 172.21.15.191:0/3908539974' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:16:27.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:27 smithi008 conmon[101053]: audit 2023-10-09T05:16:27.422137+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:28.513 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T05:16:28.514 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T05:16:28.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:28 smithi008 conmon[101053]: cluster 2023-10-09T05:16:27.280149+0000 mgr.y (mgr.14148) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:28.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:28 smithi191 conmon[101299]: cluster 2023-10-09T05:16:27.280149+0000 mgr.y (mgr.14148) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:29.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 systemd[1]: Starting Ceph mon.c for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:16:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.639+0000 7f08b6437880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:16:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.639+0000 7f08b6437880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T05:16:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.639+0000 7f08b6437880 0 pidfile_write: ignore empty --pid-file 2023-10-09T05:16:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 0 load: jerasure load: lrc load: isa 2023-10-09T05:16:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T05:16:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:16:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T05:16:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: DB SUMMARY 2023-10-09T05:16:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T05:16:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:16:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-10-09T05:16:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-10-09T05:16:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000003.log size: 636 ; 2023-10-09T05:16:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T05:16:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T05:16:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T05:16:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.env: 0x56389eba2080 2023-10-09T05:16:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.fs: Posix File System 2023-10-09T05:16:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.info_log: 0x5638a0bbba80 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.statistics: (nil) 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.use_fsync: 0 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:16:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T05:16:29.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:16:29.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:16:29.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.db_log_dir: 2023-10-09T05:16:29.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-10-09T05:16:29.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:16:29.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T05:16:29.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T05:16:29.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:16:29.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:16:29.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:16:29.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.write_buffer_manager: 0x5638a18b6300 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:16:29.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.unordered_write: 0 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.row_cache: None 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.wal_filter: None 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T05:16:29.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:16:29.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_open_files: -1 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Compression algorithms supported: 2023-10-09T05:16:29.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: kZSTD supported: 0 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T05:16:29.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:16:29.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.merge_operator: 2023-10-09T05:16:29.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.compaction_filter: None 2023-10-09T05:16:29.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:16:29.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:16:29.976 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:16:29.976 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5638a0b65d18) 2023-10-09T05:16:29.976 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: cache_index_and_filter_blocks: 1 2023-10-09T05:16:29.976 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:16:29.976 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:16:29.976 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: pin_top_level_index_and_filter: 1 2023-10-09T05:16:29.976 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: index_type: 0 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: data_block_index_type: 0 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: index_shortening: 1 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: hash_index_allow_collision: 1 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: checksum: 1 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: no_block_cache: 0 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: block_cache: 0x5638a0b9ef10 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: block_cache_name: BinnedLRUCache 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: block_cache_options: 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: capacity : 536870912 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: num_shard_bits : 4 2023-10-09T05:16:29.977 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: strict_capacity_limit : 0 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: high_pri_pool_ratio: 0.000 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: block_cache_compressed: (nil) 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: persistent_cache: (nil) 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: block_size: 4096 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: block_size_deviation: 10 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: block_restart_interval: 16 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: index_block_restart_interval: 1 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: metadata_block_size: 4096 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: partition_filters: 0 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: use_delta_encoding: 1 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:16:29.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: whole_key_filtering: 1 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: verify_compression: 0 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: read_amp_bytes_per_bit: 0 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: format_version: 2 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: enable_index_compression: 1 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: block_align: 0 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.compression: NoCompression 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:16:29.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.num_levels: 7 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.644+0000 7f08b6437880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:16:29.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:16:29.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:16:29.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:16:29.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:16:29.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:16:29.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:16:29.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:16:29.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:16:29.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:16:29.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:16:29.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:16:29.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:16:29.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:16:29.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:16:29.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:16:29.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:16:29.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:16:29.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:16:29.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:16:29.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:16:29.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:16:29.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:16:29.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:16:29.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:16:29.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:16:29.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:16:29.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T05:16:29.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T05:16:29.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:16:29.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:16:29.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:16:29.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:16:29.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T05:16:29.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T05:16:29.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.ttl: 2592000 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696828589646899, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-10-09T05:16:29.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.645+0000 7f08b6437880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T05:16:29.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.646+0000 7f08b6437880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696828589647354, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1562, "table_properties": {"data_size": 648, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696828589, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T05:16:29.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.646+0000 7f08b6437880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-10-09T05:16:29.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.646+0000 7f08b6437880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696828589647735, "job": 1, "event": "recovery_finished"} 2023-10-09T05:16:29.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.647+0000 7f08b6437880 4 rocksdb: DB pointer 0x5638a0c69800 2023-10-09T05:16:29.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.647+0000 7f089d23e700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T05:16:29.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.647+0000 7f089d23e700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T05:16:29.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: ** DB Stats ** 2023-10-09T05:16:29.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:16:29.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:16:29.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:16:29.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:16:29.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:16:29.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:16:29.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:16:29.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: ** Compaction Stats [default] ** 2023-10-09T05:16:29.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:16:29.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:16:29.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:29.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:29.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: ** Compaction Stats [default] ** 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:16:29.988 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:16:29.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Interval compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:16:29.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:16:29.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:16:29.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: ** Compaction Stats [default] ** 2023-10-09T05:16:29.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:16:29.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:16:29.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:29.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:29.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: ** Compaction Stats [default] ** 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:16:29.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:16:29.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:16:29.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:16:29.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:16:29.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.647+0000 7f08b6437880 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2023-10-09T05:16:29.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.647+0000 7f08b6437880 0 using public_addrv [v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0] 2023-10-09T05:16:29.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.648+0000 7f08b6437880 0 starting mon.c rank -1 at public addrs [v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0] at bind addrs [v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:16:29.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.649+0000 7f08b6437880 1 mon.c@-1(???) e0 preinit fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:16:29.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 bash[110741]: bc3d5fa1338eeafeca9ac64bec49af7a55820b4d1016d95d0b8be0b43c45971e 2023-10-09T05:16:29.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.666+0000 7f089fa43700 0 mon.c@-1(synchronizing).mds e1 new map 2023-10-09T05:16:29.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.666+0000 7f089fa43700 0 mon.c@-1(synchronizing).mds e1 print_map 2023-10-09T05:16:29.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: e1 2023-10-09T05:16:29.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:16:29.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:16:29.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: legacy client fscid: -1 2023-10-09T05:16:29.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: No filesystems configured 2023-10-09T05:16:29.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 2023-10-09T05:16:29.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.667+0000 7f089fa43700 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T05:16:29.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.667+0000 7f089fa43700 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T05:16:29.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.667+0000 7f089fa43700 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T05:16:29.993 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.667+0000 7f089fa43700 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T05:16:29.993 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: cluster 2023-10- 2023-10-09T05:16:29.993 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: 09T05:16:27.280149+0000 mgr.y (mgr.14148) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:29.993 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 systemd[1]: Started Ceph mon.c for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:16:29.993 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.667+0000 7f089fa43700 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T05:16:29.993 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.667+0000 7f089fa43700 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T05:16:29.993 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.667+0000 7f089fa43700 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T05:16:29.993 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.667+0000 7f089fa43700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:16:29.993 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.667+0000 7f089fa43700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:16:29.993 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.667+0000 7f089fa43700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:16:29.993 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:29 smithi008 conmon[110782]: debug 2023-10-09T05:16:29.667+0000 7f089fa43700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T05:16:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:29.280409+0000 mgr.y (mgr.14148) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: audit 2023-10-09T05:16:29.676514+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:16:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:29.676692+0000 mon.a (mon.0) 146 : cluster [INF] mon.a calling monitor election 2023-10-09T05:16:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:29.677210+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T05:16:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: audit 2023-10-09T05:16:29.689127+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: audit 2023-10-09T05:16:29.689266+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: audit 2023-10-09T05:16:30.672772+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:31.280765+0000 mgr.y (mgr.14148) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: audit 2023-10-09T05:16:31.672589+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:31.675057+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: audit 2023-10-09T05:16:32.672901+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:33.281136+0000 mgr.y (mgr.14148) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: audit 2023-10-09T05:16:33.672858+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: audit 2023-10-09T05:16:34.673060+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:34.714839+0000 mon.a (mon.0) 154 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:34.718068+0000 mon.a (mon.0) 155 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:34.730833+0000 mon.a (mon.0) 156 : cluster [DBG] fsmap 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:34.730880+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:16:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:34.731110+0000 mon.a (mon.0) 158 : cluster [DBG] mgrmap e13: y(active, since 45s) 2023-10-09T05:16:35.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: cluster 2023-10-09T05:16:34.734752+0000 mon.a (mon.0) 159 : cluster [INF] overall HEALTH_OK 2023-10-09T05:16:35.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: audit 2023-10-09T05:16:34.737063+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:35.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:34 smithi191 conmon[101299]: audit 2023-10-09T05:16:34.738536+0000 mon.a (mon.0) 161 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023-10-09T05:16:29.280409 2023-10-09T05:16:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: +0000 mgr.y (mgr.14148) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 2023-10-09T05:16:29.676514+0000 mon.a (mon. 2023-10-09T05:16:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 0) 145 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:16:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:16:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: -10-09T05:16:29.676692+0000 mon.a ( 2023-10-09T05:16:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: mon.0) 146 : cluster [INF] mon.a calling monitor election 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 2023-10-09T05:16:29.677210+0000 mon.b 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 10-09T05:16:29.689127+0000 mon.a (mon 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: .0) 147 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: audit 2023 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: -10-09T05:16:29.689266+0000 mon.a 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: (mon.0) 148 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 09T05:16:30.672772+0000 mon.a (mon.0) 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 149 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:16:35.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: -10-09T05:16:31.280765+0000 mgr.y ( 2023-10-09T05:16:35.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: mgr.14148) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:35.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 2023-10-09T05:16:31.672589 2023-10-09T05:16:35.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: +0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:16:35.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 10-09T05:16:31.675057+0000 mon.c (mon 2023-10-09T05:16:35.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: .2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T05:16:35.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:16:35.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 10-09T05:16:32.672901+0000 mon.a (mon. 2023-10-09T05:16:35.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 0) 151 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 2023-10-09T05:16:33.281136+0000 mgr.y (mgr.14148) 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 2023-10-09T05:16:33.672858+0000 mon.a 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: (mon.0) 152 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: audit 2023 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: -10-09T05:16:34.673060+0000 mon.a (mon. 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 0) 153 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 2023-10-09T05:16:34.714839+0000 2023-10-09T05:16:35.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: mon.a (mon.0) 154 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023-10 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: -09T05:16:34.718068+0000 mon.a (mon.0) 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 155 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 2023-10-09T05:16:34.730833 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: +0000 mon.a (mon.0) 156 : cluster [DBG] fsmap 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023-10-09T05 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: :16:34.730880+0000 mon.a (mon.0) 157 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:16:35.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: -10-09T05:16:34.731110+0000 mon.a ( 2023-10-09T05:16:35.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: mon.0) 158 : cluster [DBG] mgrmap e13: y(active, since 45s) 2023-10-09T05:16:35.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: cluster 2023-10-09T05:16:35.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 2023-10-09T05:16:34.734752+0000 mon.a 2023-10-09T05:16:35.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: (mon.0) 159 : cluster [INF] overall HEALTH_OK 2023-10-09T05:16:35.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: audit 2023 2023-10-09T05:16:35.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: -10-09T05:16:34.737063+0000 2023-10-09T05:16:35.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: mon.a (mon.0) 160 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:35.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:16:35.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: 10-09T05:16:34.738536+0000 mon.a 2023-10-09T05:16:35.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[110782]: (mon.0) 161 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:35.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023-10-09T05:16:29. 2023-10-09T05:16:35.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 280409+0000 mgr.y (mgr.14148) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:35.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: audit 2023-10-09T05:16:29.676514+0000 mon.a ( 2023-10-09T05:16:35.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: mon.0) 145 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:16:35.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:16:35.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: -10-09T05:16:29.676692+0000 2023-10-09T05:16:35.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: mon.a (mon.0) 146 : cluster [INF] 2023-10-09T05:16:35.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: mon.a calling monitor election 2023-10-09T05:16:35.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023-10-09T05:16 2023-10-09T05:16:35.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: :29.677210+0000 mon.b (mon.1) 2 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: : cluster [INF] mon.b calling monitor election 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 09T05:16:29.689127+0000 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: mon.a (mon.0) 147 : audit [DBG] 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 16:29.689266+0000 mon.a (mon.0 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: ) 148 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 16:30.672772+0000 mon.a (mon.0 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: ) 149 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023-10- 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 09T05:16:31.280765+0000 mgr.y 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: (mgr.14148) 28 : cluster 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 2023-10-09T05: 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 16:31.672589+0000 mon.a (mon. 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 0) 150 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023-10-09 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: T05:16:31.675057+0000 mon.c 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: (mon.2) 1 : cluster 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: [INF] mon.c calling monitor election 2023-10-09T05:16:35.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:16:35.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 10-09T05:16:32.672901+0000 2023-10-09T05:16:35.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: mon.a (mon.0) 151 : audit 2023-10-09T05:16:35.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:16:35.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: -10-09T05:16:33.281136+0000 mgr.y ( 2023-10-09T05:16:35.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: mgr.14148) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:35.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: audit 2023-10-09T05:16:33.672858+0000 mon.a 2023-10-09T05:16:35.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: (mon.0) 152 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:16:35.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: -09T05:16:34.673060+0000 mon.a (mon.0) 153 : audit [DBG] 2023-10-09T05:16:35.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 2023-10-09T05:16:34.714839+0000 mon.a (mon.0) 154 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023-10-09T05:16: 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 34.718068+0000 mon.a (mon.0) 155 : cluster [DBG] 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: monmap e3: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023-10-09T05:16:34.730833+0000 mon.a (mon.0 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: ) 156 : cluster [DBG] fsmap 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023-10 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: -09T05:16:34.730880+0000 mon.a (mon.0) 157 : cluster 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023-10-09T05:16:34.731110+0000 2023-10-09T05:16:35.225 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: mon.a (mon.0) 158 : cluster [DBG] mgrmap e13: y(active, since 45s) 2023-10-09T05:16:35.226 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:16:35.226 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: -10-09T05:16:34.734752+0000 mon.a (mon.0) 159 2023-10-09T05:16:35.226 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: : cluster [INF] overall HEALTH_OK 2023-10-09T05:16:35.226 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.226 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 2023-10-09T05:16:34.737063+0000 mon.a (mon.0) 2023-10-09T05:16:35.226 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: 160 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:35.226 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: audit 2023-10-09T05:16:34.738536+0000 2023-10-09T05:16:35.226 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:34 smithi008 conmon[101053]: mon.a (mon.0) 161 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:36.435 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-10-09T05:16:36.435 INFO:teuthology.orchestra.run.smithi191.stdout:{"epoch":3,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","modified":"2023-10-09T05:16:29.673756Z","created":"2023-10-09T05:14:53.552466Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:3300","nonce":0},{"type":"v1","addr":"172.21.15.8:6789","nonce":0}]},"addr":"172.21.15.8:6789/0","public_addr":"172.21.15.8:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:3300","nonce":0},{"type":"v1","addr":"172.21.15.191:6789","nonce":0}]},"addr":"172.21.15.191:6789/0","public_addr":"172.21.15.191:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:3301","nonce":0},{"type":"v1","addr":"172.21.15.8:6790","nonce":0}]},"addr":"172.21.15.8:6790/0","public_addr":"172.21.15.8:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-10-09T05:16:36.437 INFO:teuthology.orchestra.run.smithi191.stderr:dumped monmap epoch 3 2023-10-09T05:16:36.712 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: cluster 2023-10-09T05:16:35.281395+0000 mgr.y (mgr.14148) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:36.712 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.528699+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.712 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.673107+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:36.712 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.811900+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.712 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.812558+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:36.712 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.813254+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:36.712 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:16:36.712 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: :16:35.850748+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.712 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.868126+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.712 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.871911+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.713 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.873387+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:36.713 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.874006+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:16:36.713 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05:16:35.874636+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:36.713 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[110782]: audit 2023-10-09T05:16:36.435274+0000 mon.a (mon.0) 173 : audit [DBG] from='client.? 172.21.15.191:0/3919025956' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:16:36.713 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: cluster 2023-10-09T05:16:35.281395+0000 mgr.y (mgr.14148) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:36.713 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.528699+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.713 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.673107+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:36.713 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.811900+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.713 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.812558+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:36.713 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.813254+0000 mon.a (mon.0) 166 : audit 2023-10-09T05:16:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.850748+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.868126+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.871911+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.873387+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.874006+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:16:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:35.874636+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:36 smithi008 conmon[101053]: audit 2023-10-09T05:16:36.435274+0000 mon.a (mon.0) 173 : audit [DBG] from='client.? 172.21.15.191:0/3919025956' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:16:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: cluster 2023-10-09T05:16:35.281395+0000 mgr.y (mgr.14148) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T05:16:35.528699+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T05:16:35.673107+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:16:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T05:16:35.811900+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T05:16:35.812558+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T05:16:35.813254+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T05:16: 2023-10-09T05:16:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: 35.850748+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T05:16:35.868126+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T05:16:35.871911+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:36.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T05:16:35.873387+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:36.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T05:16:35.874006+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:16:36.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T05:16:35.874636+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:36.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:16:36.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:36 smithi191 conmon[101299]: 05:16:36.435274+0000 mon.a (mon.0) 173 : audit [DBG] from='client.? 172.21.15.191:0/3919025956' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:16:37.068 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T05:16:37.068 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T05:16:37.798 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:35.814209+0000 mgr.y ( 2023-10-09T05:16:37.798 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[101053]: mgr.14148) 31 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:16:37.798 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:35.814427+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:16:37.798 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:35.873153+0000 mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:16:37.799 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:35.875215+0000 mgr.y (mgr.14148) 34 : cephadm [INF] Reconfiguring daemon mon.a on smithi008 2023-10-09T05:16:37.799 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[101053]: audit 2023-10-09T05:16:37.217006+0000 mon.a (mon.0) 174 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:37.799 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[101053]: audit 2023-10-09T05:16:37.217854+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:37.799 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[101053]: audit 2023-10-09T05:16:37.218523+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:16:37.799 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[101053]: audit 2023-10-09T05:16:37.219179+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:37.799 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:35. 2023-10-09T05:16:37.799 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[110782]: 814209+0000 mgr.y (mgr.14148) 31 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:16:37.799 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:35.814427+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:16:37.799 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:35.873153+0000 mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:16:37.800 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:35.875215+0000 mgr.y (mgr.14148) 34 : cephadm [INF] Reconfiguring daemon mon.a on smithi008 2023-10-09T05:16:37.800 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[110782]: audit 2023-10-09T05:16:37.217006+0000 mon.a (mon.0) 174 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:37.800 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[110782]: audit 2023-10-09T05:16:37.217854+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:37.800 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[110782]: audit 2023-10-09T05:16:37.218523+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:16:37.800 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:37 smithi008 conmon[110782]: audit 2023-10-09T05:16:37.219179+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: cephadm 2023-10-09T05:16:35.814209+0000 mgr.y (mgr. 2023-10-09T05:16:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: 14148) 31 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:16:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: cephadm 2023-10-09T05:16:35.814427+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:16:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: cephadm 2023-10-09T05:16:35.873153+0000 2023-10-09T05:16:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:16:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: cephadm 2023- 2023-10-09T05:16:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: 10-09T05:16:35.875215+0000 mgr.y (mgr. 2023-10-09T05:16:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: 14148) 34 : cephadm [INF] Reconfiguring daemon mon.a on smithi008 2023-10-09T05:16:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: audit 2023 2023-10-09T05:16:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: -10-09T05:16:37.217006+0000 mon.a 2023-10-09T05:16:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: (mon.0) 174 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: audit 2023 2023-10-09T05:16:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: -10-09T05:16:37.217854+0000 mon.a (mon 2023-10-09T05:16:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: .0) 175 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:16:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: 10-09T05:16:37.218523+0000 mon.a (mon 2023-10-09T05:16:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: .0) 176 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:16:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: audit 2023-10-09T05:16 2023-10-09T05:16:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: :37.219179+0000 mon.a (mon.0) 177 : audit 2023-10-09T05:16:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:37 smithi191 conmon[101299]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:38.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:38 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:37.217604+0000 mgr.y (mgr.14148) 35 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T05:16:38.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:38 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:37.219776+0000 mgr.y (mgr.14148) 36 : cephadm [INF] Reconfiguring daemon mon.c on smithi008 2023-10-09T05:16:38.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:38 smithi008 conmon[101053]: cluster 2023-10-09T05:16:37.281642+0000 mgr.y (mgr.14148) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:38.891 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:38 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:37.217604+0000 mgr.y (mgr.14148) 35 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T05:16:38.891 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:38 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:37.219776+0000 mgr.y (mgr.14148) 36 : cephadm [INF] Reconfiguring daemon mon.c on smithi008 2023-10-09T05:16:38.891 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:38 smithi008 conmon[110782]: cluster 2023-10-09T05:16:37.281642+0000 mgr.y (mgr.14148) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:39.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:38 smithi191 conmon[101299]: cephadm 2023-10-09T05:16:37.217604+0000 mgr.y 2023-10-09T05:16:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:38 smithi191 conmon[101299]: (mgr.14148) 35 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T05:16:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:38 smithi191 conmon[101299]: cephadm 2023-10-09T05:16:37.219776+0000 mgr.y ( 2023-10-09T05:16:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:38 smithi191 conmon[101299]: mgr.14148) 36 : cephadm [INF] Reconfiguring daemon mon.c on smithi008 2023-10-09T05:16:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:38 smithi191 conmon[101299]: cluster 2023-10-09T05:16:37.281642+0000 2023-10-09T05:16:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:38 smithi191 conmon[101299]: mgr.y (mgr.14148) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:39.326 INFO:teuthology.orchestra.run.smithi008.stdout:# minimal ceph.conf for 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:16:39.326 INFO:teuthology.orchestra.run.smithi008.stdout:[global] 2023-10-09T05:16:39.326 INFO:teuthology.orchestra.run.smithi008.stdout: fsid = 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:16:39.326 INFO:teuthology.orchestra.run.smithi008.stdout: mon_host = [v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0] [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] [v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0] 2023-10-09T05:16:39.929 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T05:16:39.929 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:16:39.930 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T05:16:39.963 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:16:39.963 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:16:40.032 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T05:16:40.032 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T05:16:40.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: audit 2023-10-09T05:16:38.934263+0000 mon.a (mon.0 2023-10-09T05:16:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: ) 178 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: cephadm 2023-10-09T05:16:38.934808+0000 mgr.y (mgr.14148) 38 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T05:16:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: audit 2023-10-09T05:16:38.935035+0000 mon.a (mon. 2023-10-09T05:16:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: 0) 179 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:16:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: T05:16:38.935671+0000 mon.a (mon. 2023-10-09T05:16:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: 0) 180 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:16:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:16:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: -09T05:16:38.936268+0000 mon.a ( 2023-10-09T05:16:40.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: mon.0) 181 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:40.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: cephadm 2023- 2023-10-09T05:16:40.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: 10-09T05:16:38.936818+0000 mgr.y ( 2023-10-09T05:16:40.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: mgr.14148) 39 : cephadm [INF] Reconfiguring daemon mon.b on smithi191 2023-10-09T05:16:40.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: cluster 2023- 2023-10-09T05:16:40.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: 10-09T05:16:39.281895+0000 2023-10-09T05:16:40.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: mgr.y (mgr.14148) 40 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:40.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: 2023-10-09T05:16:40.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: audit 2023-10-09T05:16:40.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: 2023-10-09T05:16:39.326589+0000 mon.c ( 2023-10-09T05:16:40.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: mon.2) 2 : audit [DBG] from='client.? 172.21.15.8:0/2857097947' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:40.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: audit 2023 2023-10-09T05:16:40.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: -10-09T05:16:39.440776+0000 2023-10-09T05:16:40.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: mon.a (mon.0) 182 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:40.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:39 smithi191 conmon[101299]: 2023-10-09T05:16:40.062 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T05:16:40.062 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:16:40.131 INFO:tasks.cephadm:Adding mgr.y on smithi008 2023-10-09T05:16:40.131 INFO:tasks.cephadm:Adding mgr.x on smithi191 2023-10-09T05:16:40.131 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi008=y;smithi191=x' 2023-10-09T05:16:40.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[101053]: audit 2023-10-09T05:16:38 2023-10-09T05:16:40.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[101053]: .934263+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:38.934808+0000 mgr.y (mgr.14148) 38 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T05:16:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[101053]: audit 2023-10-09T05:16:38.935035+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[101053]: audit 2023-10-09T05:16:38.935671+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:16:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[101053]: audit 2023-10-09T05:16:38.936268+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:38.936818+0000 mgr.y (mgr.14148) 39 : cephadm [INF] Reconfiguring daemon mon.b on smithi191 2023-10-09T05:16:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[101053]: cluster 2023-10-09T05:16:39.281895+0000 mgr.y (mgr.14148) 40 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[101053]: audit 2023-10-09T05:16:39.326589+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.8:0/2857097947' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[101053]: audit 2023-10-09T05:16:39.440776+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:40.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: audit 2023-10-09T05:16:38.934263+0000 mon.a 2023-10-09T05:16:40.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: (mon.0) 178 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:38.934808+0000 mgr.y (mgr.14148) 38 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T05:16:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: audit 2023-10-09T05:16:38.935035+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:16:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: audit 2023-10-09T05:16:38.935671+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:16:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: audit 2023-10-09T05:16:38.936268+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:38.936818+0000 mgr.y (mgr.14148) 39 : cephadm [INF] Reconfiguring daemon mon.b on smithi191 2023-10-09T05:16:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: cluster 2023-10-09T05:16:39.281895+0000 mgr.y (mgr.14148) 40 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:16:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: 09T05:16:39.326589+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.8:0/2857097947' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: audit 2023-10-09T05:16:39.440776+0000 mon.a (mon.0) 182 : 2023-10-09T05:16:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:39 smithi008 conmon[110782]: audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:41 smithi191 conmon[101299]: audit 2023-10-09T05:16:40.180101+0000 mon.a (mon 2023-10-09T05:16:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:41 smithi191 conmon[101299]: .0) 183 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:41 smithi191 conmon[101299]: audit 2023-10-09T05:16:40.181038+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:41 smithi191 conmon[101299]: audit 2023-10-09T05:16:40.182427+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:41 smithi191 conmon[101299]: audit 2023-10-09T05:16:40.183092+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:41.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:41 smithi191 conmon[101299]: audit 2023-10-09T05:16:40.186774+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:41.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[101053]: audit 2023-10-09T05:16:40.180101+0000 mon.a (mon.0 2023-10-09T05:16:41.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[101053]: ) 183 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:41.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[101053]: audit 2023-10-09T05:16:40.181038+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[101053]: audit 2023-10-09T05:16:40.182427+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[101053]: audit 2023-10-09T05:16:40.183092+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[101053]: audit 2023-10-09T05:16:40.186774+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:41.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: audit 2023-10-09T05:16:40.180101+0000 mon.a 2023-10-09T05:16:41.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: (mon.0) 183 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:41.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:16:41.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: -09T05:16:40.181038+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:41.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:16:41.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: T05:16:40.182427+0000 mon.a (mon.0) 185 : audit 2023-10-09T05:16:41.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: audit 2023-10-09T05:16:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: 2023-10-09T05:16:40.183092 2023-10-09T05:16:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: +0000 mon.a (mon.0) 186 : audit [INF] 2023-10-09T05:16:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: audit 2023-10-09T05:16 2023-10-09T05:16:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: :40.186774+0000 mon.a (mon.0) 187 : audit 2023-10-09T05:16:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:41 smithi008 conmon[110782]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:41.974 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled mgr update... 2023-10-09T05:16:42.640 DEBUG:teuthology.orchestra.run.smithi191:mgr.x> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T05:16:42.642 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T05:16:42.642 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:16:42.642 DEBUG:teuthology.orchestra.run.smithi008:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T05:16:42.663 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T05:16:42.663 DEBUG:teuthology.orchestra.run.smithi008:> stat /dev/vg_nvme/lv_1 2023-10-09T05:16:42.672 INFO:journalctl@ceph.mgr.x.smithi191.stdout:-- Logs begin at Mon 2023-10-09 04:57:51 UTC. -- 2023-10-09T05:16:42.725 INFO:teuthology.orchestra.run.smithi008.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T05:16:42.726 INFO:teuthology.orchestra.run.smithi008.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:16:42.726 INFO:teuthology.orchestra.run.smithi008.stdout:Device: 6h/6d Inode: 385949 Links: 1 2023-10-09T05:16:42.726 INFO:teuthology.orchestra.run.smithi008.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:16:42.726 INFO:teuthology.orchestra.run.smithi008.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:16:42.726 INFO:teuthology.orchestra.run.smithi008.stdout:Access: 2023-10-09 05:16:29.057212493 +0000 2023-10-09T05:16:42.726 INFO:teuthology.orchestra.run.smithi008.stdout:Modify: 2023-10-09 05:07:23.806559069 +0000 2023-10-09T05:16:42.726 INFO:teuthology.orchestra.run.smithi008.stdout:Change: 2023-10-09 05:07:23.806559069 +0000 2023-10-09T05:16:42.726 INFO:teuthology.orchestra.run.smithi008.stdout: Birth: - 2023-10-09T05:16:42.727 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T05:16:42.797 INFO:teuthology.orchestra.run.smithi008.stderr:1+0 records in 2023-10-09T05:16:42.797 INFO:teuthology.orchestra.run.smithi008.stderr:1+0 records out 2023-10-09T05:16:42.797 INFO:teuthology.orchestra.run.smithi008.stderr:512 bytes copied, 0.000124808 s, 4.1 MB/s 2023-10-09T05:16:42.798 DEBUG:teuthology.orchestra.run.smithi008:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T05:16:42.860 DEBUG:teuthology.orchestra.run.smithi008:> stat /dev/vg_nvme/lv_2 2023-10-09T05:16:42.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:42 smithi191 conmon[101299]: cluster 2023-10-09T05:16:41.282295+0000 mgr.y (mgr.14148) 41 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:42.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:42 smithi191 conmon[101299]: audit 2023-10-09T05:16:41.973734+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:42.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:42 smithi191 conmon[101299]: audit 2023-10-09T05:16:42.007354+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:42.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:42 smithi191 conmon[101299]: audit 2023-10-09T05:16:42.009111+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:42.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:42 smithi191 conmon[101299]: audit 2023-10-09T05:16:42.010044+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:42.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:42 smithi191 conmon[101299]: audit 2023-10-09T05:16:42.014593+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:42.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:42 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:16:42.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:42 smithi191 conmon[101299]: T05:16:42.016229+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:16:42.884 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:42 smithi191 conmon[101299]: audit 2023-10-09T05:16:42.019095+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:16:42.884 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:42 smithi191 conmon[101299]: audit 2023-10-09T05:16:42.019938+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:16:42.884 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:42 smithi191 conmon[101299]: audit 2023-10-09T05:16:42.020930+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:42.920 INFO:teuthology.orchestra.run.smithi008.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T05:16:42.920 INFO:teuthology.orchestra.run.smithi008.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:16:42.921 INFO:teuthology.orchestra.run.smithi008.stdout:Device: 6h/6d Inode: 385987 Links: 1 2023-10-09T05:16:42.921 INFO:teuthology.orchestra.run.smithi008.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:16:42.921 INFO:teuthology.orchestra.run.smithi008.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:16:42.921 INFO:teuthology.orchestra.run.smithi008.stdout:Access: 2023-10-09 05:16:29.057212493 +0000 2023-10-09T05:16:42.921 INFO:teuthology.orchestra.run.smithi008.stdout:Modify: 2023-10-09 05:07:24.327553345 +0000 2023-10-09T05:16:42.921 INFO:teuthology.orchestra.run.smithi008.stdout:Change: 2023-10-09 05:07:24.327553345 +0000 2023-10-09T05:16:42.921 INFO:teuthology.orchestra.run.smithi008.stdout: Birth: - 2023-10-09T05:16:42.921 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T05:16:42.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: cluster 2023-10-09T05:16:41.282295+0000 mgr.y (mgr.14148) 41 : 2023-10-09T05:16:42.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: audit 2023-10-09T05:16:41.973734+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:16:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: -09T05:16:42.007354+0000 mon.a (mon.0) 189 2023-10-09T05:16:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:16:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: 09T05:16:42.009111+0000 mon.a (mon.0) 190 2023-10-09T05:16:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:16:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: -09T05:16:42.010044+0000 mon.a (mon.0) 191 2023-10-09T05:16:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:16:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: 09T05:16:42.014593+0000 mon.a (mon.0 2023-10-09T05:16:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: ) 192 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: audit 2023 2023-10-09T05:16:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: -10-09T05:16:42.016229+0000 mon.a (mon 2023-10-09T05:16:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: .0) 193 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:16:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:16:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: -09T05:16:42.019095+0000 mon.a (mon. 2023-10-09T05:16:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: 0) 194 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:16:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:16:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: 09T05:16:42.019938+0000 mon.a (mon.0) 2023-10-09T05:16:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: 195 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:16:42.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: audit 2023-10-09T05:16: 2023-10-09T05:16:42.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: 42.020930+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:42.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[101053]: 2023-10-09T05:16:42.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: cluster 2023-10-09T05:16: 2023-10-09T05:16:42.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: 41.282295+0000 mgr.y (mgr.14148) 41 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:42.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: audit 2023 2023-10-09T05:16:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: -10-09T05:16:41.973734 2023-10-09T05:16:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: +0000 mon.a (mon.0) 2023-10-09T05:16:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: 188 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:42.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: audit 2023 2023-10-09T05:16:42.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: -10-09T05:16:42. 2023-10-09T05:16:42.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: 007354+0000 mon.a (mon.0) 2023-10-09T05:16:42.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: 189 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:42.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: audit 2023 2023-10-09T05:16:42.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: -10-09T05:16:42.009111+0000 mon.a 2023-10-09T05:16:42.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: (mon.0) 190 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:42.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:16:42.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: 10-09T05:16:42.010044+0000 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: mon.a (mon.0) 191 : audit [INF] 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: T05:16:42.014593+0000 mon.a 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: (mon.0) 192 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: audit 2023-10-09T05:16:42 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: .016229+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: 16:42.019095+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: :16:42.019938+0000 mon.a 2023-10-09T05:16:42.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: (mon.0) 195 : 2023-10-09T05:16:42.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:16:42.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: audit 2023-10-09T05:16:42.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: 2023-10-09T05:16:42.020930+0000 mon.a (mon.0) 196 2023-10-09T05:16:42.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:42 smithi008 conmon[110782]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:42.991 INFO:teuthology.orchestra.run.smithi008.stderr:1+0 records in 2023-10-09T05:16:42.991 INFO:teuthology.orchestra.run.smithi008.stderr:1+0 records out 2023-10-09T05:16:42.991 INFO:teuthology.orchestra.run.smithi008.stderr:512 bytes copied, 0.000127613 s, 4.0 MB/s 2023-10-09T05:16:42.992 DEBUG:teuthology.orchestra.run.smithi008:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T05:16:43.054 DEBUG:teuthology.orchestra.run.smithi008:> stat /dev/vg_nvme/lv_3 2023-10-09T05:16:43.113 INFO:teuthology.orchestra.run.smithi008.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T05:16:43.114 INFO:teuthology.orchestra.run.smithi008.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:16:43.114 INFO:teuthology.orchestra.run.smithi008.stdout:Device: 6h/6d Inode: 395320 Links: 1 2023-10-09T05:16:43.114 INFO:teuthology.orchestra.run.smithi008.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:16:43.114 INFO:teuthology.orchestra.run.smithi008.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:16:43.114 INFO:teuthology.orchestra.run.smithi008.stdout:Access: 2023-10-09 05:16:29.057212493 +0000 2023-10-09T05:16:43.114 INFO:teuthology.orchestra.run.smithi008.stdout:Modify: 2023-10-09 05:07:24.835547765 +0000 2023-10-09T05:16:43.114 INFO:teuthology.orchestra.run.smithi008.stdout:Change: 2023-10-09 05:07:24.835547765 +0000 2023-10-09T05:16:43.114 INFO:teuthology.orchestra.run.smithi008.stdout: Birth: - 2023-10-09T05:16:43.115 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T05:16:43.183 INFO:teuthology.orchestra.run.smithi008.stderr:1+0 records in 2023-10-09T05:16:43.184 INFO:teuthology.orchestra.run.smithi008.stderr:1+0 records out 2023-10-09T05:16:43.184 INFO:teuthology.orchestra.run.smithi008.stderr:512 bytes copied, 0.000112487 s, 4.6 MB/s 2023-10-09T05:16:43.185 DEBUG:teuthology.orchestra.run.smithi008:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T05:16:43.245 DEBUG:teuthology.orchestra.run.smithi008:> stat /dev/vg_nvme/lv_4 2023-10-09T05:16:43.305 INFO:teuthology.orchestra.run.smithi008.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T05:16:43.305 INFO:teuthology.orchestra.run.smithi008.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:16:43.305 INFO:teuthology.orchestra.run.smithi008.stdout:Device: 6h/6d Inode: 394973 Links: 1 2023-10-09T05:16:43.305 INFO:teuthology.orchestra.run.smithi008.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:16:43.305 INFO:teuthology.orchestra.run.smithi008.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:16:43.306 INFO:teuthology.orchestra.run.smithi008.stdout:Access: 2023-10-09 05:16:29.057212493 +0000 2023-10-09T05:16:43.306 INFO:teuthology.orchestra.run.smithi008.stdout:Modify: 2023-10-09 05:07:25.366541931 +0000 2023-10-09T05:16:43.306 INFO:teuthology.orchestra.run.smithi008.stdout:Change: 2023-10-09 05:07:25.366541931 +0000 2023-10-09T05:16:43.306 INFO:teuthology.orchestra.run.smithi008.stdout: Birth: - 2023-10-09T05:16:43.306 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T05:16:43.373 INFO:teuthology.orchestra.run.smithi008.stderr:1+0 records in 2023-10-09T05:16:43.374 INFO:teuthology.orchestra.run.smithi008.stderr:1+0 records out 2023-10-09T05:16:43.374 INFO:teuthology.orchestra.run.smithi008.stderr:512 bytes copied, 0.000121114 s, 4.2 MB/s 2023-10-09T05:16:43.375 DEBUG:teuthology.orchestra.run.smithi008:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T05:16:43.435 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T05:16:43.435 DEBUG:teuthology.orchestra.run.smithi191:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T05:16:43.453 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T05:16:43.454 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_1 2023-10-09T05:16:43.511 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T05:16:43.511 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:16:43.511 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 6h/6d Inode: 388862 Links: 1 2023-10-09T05:16:43.511 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:16:43.511 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:16:43.511 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2023-10-09 05:16:20.752064871 +0000 2023-10-09T05:16:43.511 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2023-10-09 05:07:07.921915634 +0000 2023-10-09T05:16:43.511 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2023-10-09 05:07:07.921915634 +0000 2023-10-09T05:16:43.511 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: - 2023-10-09T05:16:43.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T05:16:43.578 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2023-10-09T05:16:43.579 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2023-10-09T05:16:43.579 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.00022814 s, 2.2 MB/s 2023-10-09T05:16:43.579 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T05:16:43.639 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_2 2023-10-09T05:16:43.696 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T05:16:43.696 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:16:43.696 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 6h/6d Inode: 388044 Links: 1 2023-10-09T05:16:43.696 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:16:43.696 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:16:43.696 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2023-10-09 05:16:20.752064871 +0000 2023-10-09T05:16:43.696 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2023-10-09 05:07:08.450906555 +0000 2023-10-09T05:16:43.696 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2023-10-09 05:07:08.450906555 +0000 2023-10-09T05:16:43.696 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: - 2023-10-09T05:16:43.697 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T05:16:43.779 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2023-10-09T05:16:43.780 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2023-10-09T05:16:43.780 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000207505 s, 2.5 MB/s 2023-10-09T05:16:43.782 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T05:16:43.798 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_3 2023-10-09T05:16:43.856 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T05:16:43.856 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:16:43.856 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 6h/6d Inode: 392546 Links: 1 2023-10-09T05:16:43.856 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:16:43.856 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:16:43.857 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2023-10-09 05:16:43.733671778 +0000 2023-10-09T05:16:43.857 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2023-10-09 05:07:08.978897493 +0000 2023-10-09T05:16:43.857 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2023-10-09 05:07:08.978897493 +0000 2023-10-09T05:16:43.857 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: - 2023-10-09T05:16:43.857 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T05:16:43.913 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:43 smithi191 conmon[101299]: audit 2023-10-09T05:16:41. 2023-10-09T05:16:43.914 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:43 smithi191 conmon[101299]: 968356+0000 mgr.y (mgr.14148) 42 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi008=y;smithi191=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:16:43.914 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:43 smithi191 conmon[101299]: cephadm 2023-10-09T05:16:41.970066+0000 mgr.y (mgr.14148) 43 : cephadm [INF] Saving service mgr spec with placement smithi008=y;smithi191=x;count:2 2023-10-09T05:16:43.914 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:43 smithi191 conmon[101299]: cephadm 2023-10-09T05:16:42.021748+0000 mgr.y (mgr.14148) 44 : cephadm [INF] Deploying daemon mgr.x on smithi191 2023-10-09T05:16:43.924 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2023-10-09T05:16:43.924 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2023-10-09T05:16:43.924 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000230644 s, 2.2 MB/s 2023-10-09T05:16:43.925 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T05:16:43.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:43 smithi008 conmon[110782]: audit 2023-10-09T05:16:41.968356+0000 2023-10-09T05:16:43.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:43 smithi008 conmon[110782]: mgr.y (mgr.14148) 42 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi008=y;smithi191=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:16:43.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:43 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:41. 2023-10-09T05:16:43.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:43 smithi008 conmon[110782]: 970066+0000 mgr.y (mgr.14148) 43 : cephadm [INF] Saving service mgr spec with placement smithi008=y;smithi191=x;count:2 2023-10-09T05:16:43.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:43 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:42.021748+0000 mgr.y (mgr.14148) 44 : cephadm [INF] Deploying daemon mgr.x on smithi191 2023-10-09T05:16:43.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:43 smithi008 conmon[101053]: audit 2023-10-09T05:16:41.968356+0000 mgr.y ( 2023-10-09T05:16:43.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:43 smithi008 conmon[101053]: mgr.14148) 42 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi008=y;smithi191=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:16:43.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:43 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:41.970066+0000 mgr.y (mgr.14148) 43 : cephadm [INF] Saving service mgr spec with placement smithi008=y;smithi191=x;count:2 2023-10-09T05:16:43.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:43 smithi008 conmon[101053]: cephadm 2023-10-09T05:16: 2023-10-09T05:16:43.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:43 smithi008 conmon[101053]: 42.021748+0000 mgr.y (mgr.14148) 44 : cephadm [INF] Deploying daemon mgr.x on smithi191 2023-10-09T05:16:43.986 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_4 2023-10-09T05:16:44.043 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T05:16:44.044 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:16:44.044 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 6h/6d Inode: 397457 Links: 1 2023-10-09T05:16:44.044 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:16:44.044 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:16:44.044 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2023-10-09 05:16:43.868669468 +0000 2023-10-09T05:16:44.044 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2023-10-09 05:07:09.491888689 +0000 2023-10-09T05:16:44.044 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2023-10-09 05:07:09.491888689 +0000 2023-10-09T05:16:44.044 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: - 2023-10-09T05:16:44.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T05:16:44.110 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2023-10-09T05:16:44.111 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2023-10-09T05:16:44.111 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000183566 s, 2.8 MB/s 2023-10-09T05:16:44.112 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T05:16:44.170 INFO:tasks.cephadm:Deploying osd.0 on smithi008 with /dev/vg_nvme/lv_4... 2023-10-09T05:16:44.171 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T05:16:44.299 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:43 smithi191 systemd[1]: Starting Ceph mgr.x for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:16:44.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:44 smithi008 conmon[101053]: cluster 2023-10-09T05:16:43.282682+0000 mgr.y (mgr.14148) 45 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:44 smithi008 conmon[101053]: audit 2023-10-09T05:16:44.658885+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:44 smithi008 conmon[101053]: audit 2023-10-09T05:16:44.660757+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:44 smithi008 conmon[110782]: cluster 2023-10-09T05:16:43.282682+0000 mgr.y (mgr.14148) 45 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:44 smithi008 conmon[110782]: audit 2023-10-09T05:16:44.658885+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:44 smithi008 conmon[110782]: audit 2023-10-09T05:16:44.660757+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:45.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:44 smithi191 conmon[103609]: debug 2023-10-09T05:16:44.578+0000 7fdd15a44700 1 -- 172.21.15.191:0/866429658 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5601c773e4e0 con 0x5601c7714c00 2023-10-09T05:16:45.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:44 smithi191 bash[103546]: 720487b0c8fc00403d16f7a979fed18b8846a006571325059325e175c6898ff1 2023-10-09T05:16:45.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:44 smithi191 systemd[1]: Started Ceph mgr.x for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:16:45.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:44 smithi191 conmon[103609]: debug 2023-10-09T05:16:44.708+0000 7fdd2602a680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:16:45.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:44 smithi191 conmon[103609]: debug 2023-10-09T05:16:44.792+0000 7fdd2602a680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:16:45.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:44 smithi191 conmon[101299]: cluster 2023-10-09T05:16:43.282682+0000 mgr.y (mgr.14148 2023-10-09T05:16:45.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:44 smithi191 conmon[101299]: ) 45 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:45.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:44 smithi191 conmon[101299]: audit 2023-10-09T05:16:44.658885+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:45.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:44 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:16:45.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:44 smithi191 conmon[101299]: :16:44.660757+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:45.549 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:45 smithi191 conmon[103609]: debug 2023-10-09T05:16:45.203+0000 7fdd2602a680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:16:46.122 INFO:teuthology.orchestra.run.smithi008.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T05:16:46.123 INFO:teuthology.orchestra.run.smithi008.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T05:16:46.123 INFO:teuthology.orchestra.run.smithi008.stderr: stderr: 10+0 records in 2023-10-09T05:16:46.123 INFO:teuthology.orchestra.run.smithi008.stderr:10+0 records out 2023-10-09T05:16:46.123 INFO:teuthology.orchestra.run.smithi008.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00796637 s, 1.3 GB/s 2023-10-09T05:16:46.123 INFO:teuthology.orchestra.run.smithi008.stderr:--> Zapping successful for: 2023-10-09T05:16:46.271 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:45 smithi191 conmon[103609]: debug 2023-10-09T05:16:45.976+0000 7fdd2602a680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:16:46.271 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[103609]: debug 2023-10-09T05:16:46.059+0000 7fdd2602a680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:16:46.549 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[103609]: debug 2023-10-09T05:16:46.270+0000 7fdd2602a680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:16:46.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[101053]: cluster 2023-10-09T05:16:45.283113+0000 mgr.y (mgr.14148) 46 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[101053]: audit 2023-10-09T05:16:45.582225+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[101053]: audit 2023-10-09T05:16:45.582969+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[101053]: audit 2023-10-09T05:16:45.583778+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[101053]: audit 2023-10-09T05:16:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[101053]: 2023-10-09T05:16:45.587791+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:45.589358+0000 mgr.y (mgr.14148) 47 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:16:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[101053]: audit 2023-10-09T05:16:45.589618+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:16:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[101053]: audit 2023-10-09T05:16:45.590321+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:16:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[101053]: audit 2023-10-09T05:16:45.590872+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:45.591409+0000 mgr.y (mgr.14148) 48 : cephadm [INF] Reconfiguring daemon mgr.y on smithi008 2023-10-09T05:16:46.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[110782]: cluster 2023-10-09T05:16:45.283113+0000 mgr.y (mgr.14148) 46 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:46.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[110782]: audit 2023-10-09T05:16:45.582225+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:46.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[110782]: audit 2023-10-09T05:16:45.582969+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:46.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[110782]: audit 2023-10-09T05:16:45.583778+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:46.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[110782]: audit 2023-10-09T05:16:45.587791+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:46.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:45.589358+0000 mgr.y (mgr.14148) 47 : 2023-10-09T05:16:46.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[110782]: cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:16:46.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[110782]: audit 2023-10-09T05:16:45.589618+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:16:46.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[110782]: audit 2023-10-09T05:16:45.590321+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:16:46.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[110782]: audit 2023-10-09T05:16:45.590872+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:46.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:46 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:45.591409+0000 mgr.y (mgr.14148) 48 : cephadm [INF] Reconfiguring daemon mgr.y on smithi008 2023-10-09T05:16:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: cluster 2023-10-09T05:16:45.283113+0000 mgr.y (mgr.14148) 46 : cluster 2023-10-09T05:16:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: audit 2023-10-09T05:16:45.582225+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: audit 2023-10-09T05:16:45.582969+0000 mon.a ( 2023-10-09T05:16:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: mon.0) 200 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: audit 2023-10-09T05:16:45.583778+0000 mon.a (mon.0) 2023-10-09T05:16:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: 201 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:16:47.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: T05:16:45.587791+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:47.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: cephadm 2023-10-09T05:16: 2023-10-09T05:16:47.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: 45.589358+0000 mgr.y (mgr.14148) 47 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:16:47.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:16:47.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: -09T05:16:45.589618+0000 mon.a (mon.0) 203 : audit 2023-10-09T05:16:47.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:16:47.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: audit 2023 2023-10-09T05:16:47.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: -10-09T05:16:45.590321+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:16:47.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: 2023-10-09T05:16:47.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:16:47.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: T05:16:45.590872+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:47.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: cephadm 2023-10-09T05:16:45.591409+0000 mgr.y ( 2023-10-09T05:16:47.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:46 smithi191 conmon[101299]: mgr.14148) 48 : cephadm [INF] Reconfiguring daemon mgr.y on smithi008 2023-10-09T05:16:47.210 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi008:vg_nvme/lv_4 2023-10-09T05:16:47.502 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:47 smithi191 conmon[103609]: debug 2023-10-09T05:16:47.208+0000 7fdd2602a680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:16:47.502 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:47 smithi191 conmon[103609]: debug 2023-10-09T05:16:47.334+0000 7fdd2602a680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:16:47.502 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:47 smithi191 conmon[103609]: debug 2023-10-09T05:16:47.412+0000 7fdd2602a680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:16:47.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:47 smithi191 conmon[103609]: debug 2023-10-09T05:16:47.501+0000 7fdd2602a680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:16:47.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:47 smithi191 conmon[103609]: debug 2023-10-09T05:16:47.696+0000 7fdd2602a680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:16:48.299 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:47 smithi191 conmon[103609]: debug 2023-10-09T05:16:47.971+0000 7fdd2602a680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:16:48.300 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[103609]: debug 2023-10-09T05:16:48.068+0000 7fdd2602a680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:16:48.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[110782]: cluster 2023-10-09T05:16:47.283433+0000 mgr.y (mgr.14148) 49 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:48.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[110782]: audit 2023-10-09T05:16:47.311699+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:48.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[110782]: audit 2023-10-09T05:16:47.312805+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:48.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[110782]: audit 2023-10-09T05:16:47.314148+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:48.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[110782]: audit 2023-10-09T05:16:47.314794+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:48.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[110782]: audit 2023-10-09T05:16:47.318528+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:48.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[101053]: cluster 2023-10-09T05:16: 2023-10-09T05:16:48.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[101053]: 47.283433+0000 mgr.y (mgr.14148) 49 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:48.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[101053]: audit 2023-10-09T05:16:47.311699+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:48.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[101053]: audit 2023-10-09T05:16:47.312805+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:48.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[101053]: audit 2023-10-09T05:16:47.314148+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:48.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[101053]: audit 2023-10-09T05:16:47.314794+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:48.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:48 smithi008 conmon[101053]: audit 2023-10-09T05:16:47.318528+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:48.788 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: cluster 2023-10-09T05:16:47.283433+0000 2023-10-09T05:16:48.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: mgr.y (mgr.14148) 49 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:48.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:16:48.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: T05:16:47.311699+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:48.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: audit 2023 2023-10-09T05:16:48.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: -10-09T05:16:47.312805+0000 mon.a (mon.0 2023-10-09T05:16:48.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: ) 207 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:16:48.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: audit 2023-10-09T05:16:48.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: 2023-10-09T05:16: 2023-10-09T05:16:48.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: 47.314148+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:48.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: 2023-10-09T05:16:48.790 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: audit 2023-10-09T05:16:47. 2023-10-09T05:16:48.790 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: 314794+0000 mon.a (mon.0) 209 : audit 2023-10-09T05:16:48.790 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:16:48.790 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: audit 2023-10-09T05:16:48.790 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: 2023-10 2023-10-09T05:16:48.790 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: -09T05:16:47.318528 2023-10-09T05:16:48.790 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: +0000 mon.a (mon.0) 210 : audit 2023-10-09T05:16:48.790 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[101299]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:49.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[103609]: debug 2023-10-09T05:16:48.787+0000 7fdd2602a680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:16:49.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[103609]: debug 2023-10-09T05:16:48.867+0000 7fdd2602a680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:16:49.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:48 smithi191 conmon[103609]: debug 2023-10-09T05:16:48.959+0000 7fdd2602a680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:16:49.392 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[103609]: debug 2023-10-09T05:16:49.131+0000 7fdd2602a680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:16:49.392 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[101299]: audit 2023-10-09T05:16:49.123986 2023-10-09T05:16:49.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[101299]: +0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:16:49.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:16:49.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[101299]: 05:16:49.126287+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:16:49.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:16:49.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[101299]: 09T05:16:49.126885+0000 mon.a (mon.0) 2023-10-09T05:16:49.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[101299]: 213 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:49.653 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[103609]: debug 2023-10-09T05:16:49.391+0000 7fdd2602a680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:16:49.653 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[103609]: debug 2023-10-09T05:16:49.521+0000 7fdd2602a680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:16:49.660 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:49 smithi008 conmon[101053]: audit 2023-10-09T05:16: 2023-10-09T05:16:49.660 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:49 smithi008 conmon[101053]: 49.123986+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:16:49.660 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:49 smithi008 conmon[101053]: audit 2023-10-09T05:16:49.126287+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:16:49.660 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:49 smithi008 conmon[101053]: audit 2023-10-09T05:16:49.126885+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:49.661 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:49 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:16:49.661 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:49 smithi008 conmon[110782]: 16:49.123986+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:16:49.661 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:49 smithi008 conmon[110782]: audit 2023-10-09T05:16:49.126287+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:16:49.661 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:49 smithi008 conmon[110782]: audit 2023-10-09T05:16:49.126885+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:49.906 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[103609]: debug 2023-10-09T05:16:49.653+0000 7fdd2602a680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:16:49.907 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[103609]: debug 2023-10-09T05:16:49.823+0000 7fdd2602a680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:16:50.657 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: audit 2023-10-09T05:16:49.121179 2023-10-09T05:16:50.657 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: +0000 mgr.y (mgr.14148) 50 : audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:16:50.657 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: cluster 2023-10-09T05:16:49.283646+0000 mgr.y (mgr.14148) 51 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:50.657 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: audit 2023-10-09T05:16:49.445034+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:50.657 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: cluster 2023-10-09T05:16:49.920323+0000 mon.a (mon.0) 215 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:16:50.657 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: audit 2023-10-09T05:16:49.921129+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:16:50.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: audit 2023-10-09T05:16:49.921455+0000 2023-10-09T05:16:50.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:16:50.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: audit 2023-10-09T05:16: 2023-10-09T05:16:50.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: 49.922377+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:16:50.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:16:50.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: 16:49.922637+0000 mon.b (mon.1) 6 : audit [DBG] 2023-10-09T05:16:50.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[110782]: from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:16:50.658 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[101053]: audit 2023-10-09T05:16:49.121179+0000 mgr.y (mgr.14148 2023-10-09T05:16:50.658 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[101053]: ) 50 : audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:16:50.659 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[101053]: cluster 2023-10-09T05:16:49.283646+0000 mgr.y (mgr.14148) 51 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:50.659 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[101053]: audit 2023-10-09T05:16:49.445034+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:50.659 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[101053]: cluster 2023-10-09T05:16:49.920323+0000 mon.a (mon.0) 215 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:16:50.659 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[101053]: audit 2023-10-09T05:16:49.921129+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:16:50.659 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[101053]: audit 2023-10-09T05:16:49.921455+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:16:50.659 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[101053]: audit 2023-10-09T05:16:49.922377+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:16:50.659 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:50 smithi008 conmon[101053]: audit 2023-10-09T05:16:49.922637+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:16:50.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:16:49 smithi191 conmon[103609]: debug 2023-10-09T05:16:49.906+0000 7fdd2602a680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:16:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: audit 2023-10-09T05:16:49.121179+0000 mgr.y (mgr.14148) 50 : 2023-10-09T05:16:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:16:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: cluster 2023-10-09T05:16:49.283646+0000 mgr.y (mgr.14148) 51 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:16:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: 16:49.445034+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:16:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: cluster 2023-10-09T05:16:49.920323+0000 mon.a 2023-10-09T05:16:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: (mon.0) 215 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:16:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:16:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: -09T05:16:49.921129+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:16:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: audit 2023-10-09T05:16:49.921455 2023-10-09T05:16:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: +0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:16:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: audit 2023-10-09T05:16:49.922377+0000 mon.b (mon 2023-10-09T05:16:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: .1) 5 : audit [DBG] from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:16:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: audit 2023-10-09T05:16:49.922637+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.191:0/1381624916' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:16:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:50 smithi191 conmon[101299]: 2023-10-09T05:16:51.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[101053]: cluster 2023-10-09T05:16:50.453442+0000 mon.a (mon.0) 216 : cluster [DBG] mgrmap e14: y(active, since 61s), standbys: x 2023-10-09T05:16:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[101053]: audit 2023-10-09T05:16:50.453580+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:16:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[101053]: audit 2023-10-09T05:16:51.326979+0000 mon.a (mon.0) 218 : audit [INF] from='client.? 172.21.15.8:0/89704178' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc"}]: dispatch 2023-10-09T05:16:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[101053]: audit 2023-10-09T05:16:51.330698+0000 mon.a (mon.0) 219 : audit [INF] from='client.? 172.21.15.8:0/89704178' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc"}]': finished 2023-10-09T05:16:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[101053]: cluster 2023-10-09T05:16:51.330746+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:16:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[101053]: audit 2023-10-09T05:16:51.330862+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:16:51.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[110782]: cluster 2023-10-09T05:16:50.453442+0000 mon.a (mon.0) 216 : cluster [DBG] mgrmap e14: y(active, since 61s), standbys: x 2023-10-09T05:16:51.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[110782]: audit 2023-10-09T05:16:50.453580+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:16:51.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[110782]: audit 2023-10-09T05:16:51.326979+0000 mon.a (mon.0) 218 : audit [INF] from='client.? 172.21.15.8:0/89704178' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc"}]: dispatch 2023-10-09T05:16:51.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[110782]: audit 2023-10-09T05:16:51.330698+0000 mon.a (mon.0) 219 : audit [INF] from='client.? 172.21.15.8:0/89704178' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc"}]': finished 2023-10-09T05:16:51.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[110782]: cluster 2023-10-09T05:16:51.330746+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:16:51.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:16:51.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:51 smithi008 conmon[110782]: 05:16:51.330862+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:16:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: cluster 2023-10-09T05:16:50.453442+0000 mon.a (mon.0) 216 : cluster 2023-10-09T05:16:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: [DBG] mgrmap e14: y(active, since 61s), standbys: x 2023-10-09T05:16:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: audit 2023-10-09T05:16:50.453580+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:16:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: audit 2023-10-09T05:16:51.326979+0000 mon.a (mon.0) 218 : audit [INF] 2023-10-09T05:16:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: from='client.? 172.21.15.8:0/89704178' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc"}]: dispatch 2023-10-09T05:16:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: audit 2023-10-09T05:16:51. 2023-10-09T05:16:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: 330698+0000 mon.a (mon.0) 219 : audit [INF] from='client.? 172.21.15.8:0/89704178' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc"}]': finished 2023-10-09T05:16:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: 2023-10-09T05:16:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: cluster 2023-10-09T05:16:51.330746+0000 2023-10-09T05:16:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: mon.a (mon.0) 220 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:16:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: audit 2023-10-09T05:16:51 2023-10-09T05:16:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: .330862+0000 mon.a (mon.0) 221 : audit [DBG] 2023-10-09T05:16:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:51 smithi191 conmon[101299]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:16:52.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:52 smithi008 conmon[101053]: cluster 2023-10-09T05:16:51.283896+0000 mgr.y (mgr.14148) 52 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:52.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:52 smithi008 conmon[101053]: audit 2023-10-09T05:16:51.945469+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.8:0/4097333858' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:16:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:52 smithi008 conmon[110782]: cluster 2023-10-09T05:16:51.283896+0000 mgr.y (mgr.14148) 52 : cluster 2023-10-09T05:16:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:52 smithi008 conmon[110782]: [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:52 smithi008 conmon[110782]: audit 2023-10-09T05:16:51.945469+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.8:0/4097333858' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:16:53.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:52 smithi191 conmon[101299]: cluster 2023-10-09T05:16:51.283896+0000 mgr.y (mgr.14148 2023-10-09T05:16:53.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:52 smithi191 conmon[101299]: ) 52 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:53.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:52 smithi191 conmon[101299]: audit 2023-10-09T05:16:51.945469+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.8:0/4097333858' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:16:53.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:52 smithi191 conmon[101299]: 2023-10-09T05:16:54.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:54 smithi008 conmon[101053]: cluster 2023-10-09T05:16:53. 2023-10-09T05:16:54.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:54 smithi008 conmon[101053]: 284201+0000 mgr.y (mgr.14148) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:54.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:54 smithi008 conmon[110782]: cluster 2023-10-09T05:16:53.284201+0000 2023-10-09T05:16:54.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:54 smithi008 conmon[110782]: mgr.y (mgr.14148) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:55.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:54 smithi191 conmon[101299]: cluster 2023-10-09T05:16:53.284201+0000 mgr.y (mgr. 2023-10-09T05:16:55.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:54 smithi191 conmon[101299]: 14148) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:56 smithi008 conmon[101053]: cluster 2023-10-09T05:16:55.284415+0000 mgr.y (mgr.14148) 54 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:56 smithi008 conmon[110782]: cluster 2023-10-09T05:16:55.284415+0000 mgr.y (mgr.14148) 54 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:57.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:56 smithi191 conmon[101299]: cluster 2023-10-09T05:16:55.284415+0000 mgr.y (mgr. 2023-10-09T05:16:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:56 smithi191 conmon[101299]: 14148) 54 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:57.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:57 smithi008 conmon[110782]: audit 2023-10-09T05:16:57.424002+0000 2023-10-09T05:16:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:57 smithi008 conmon[110782]: mon.a (mon.0) 222 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:16:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:57 smithi008 conmon[110782]: audit 2023-10-09T05:16:57.424695+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:57.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:57 smithi008 conmon[101053]: audit 2023-10-09T05:16:57 2023-10-09T05:16:57.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:57 smithi008 conmon[101053]: .424002+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:16:57.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:57 smithi008 conmon[101053]: audit 2023-10-09T05:16:57.424695+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:57 smithi191 conmon[101299]: audit 2023-10-09T05:16:57.424002+0000 2023-10-09T05:16:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:57 smithi191 conmon[101299]: mon.a (mon.0) 222 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:16:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:57 smithi191 conmon[101299]: audit 2023-10-09T05:16: 2023-10-09T05:16:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:57 smithi191 conmon[101299]: 57.424695+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:16:58.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:58 smithi008 conmon[101053]: cluster 2023-10-09T05:16:57.284685+0000 mgr.y (mgr.14148) 55 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:58.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:16:58 smithi008 conmon[101053]: cephadm 2023-10-09T05:16:57.425354+0000 mgr.y (mgr.14148) 56 : cephadm [INF] Deploying daemon osd.0 on smithi008 2023-10-09T05:16:58.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:58 smithi008 conmon[110782]: cluster 2023-10-09T05:16:57.284685+0000 mgr.y (mgr.14148) 55 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:58.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:16:58 smithi008 conmon[110782]: cephadm 2023-10-09T05:16:57.425354+0000 mgr.y (mgr.14148) 56 : cephadm [INF] Deploying daemon osd.0 on smithi008 2023-10-09T05:16:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:58 smithi191 conmon[101299]: cluster 2023-10-09T05:16:57.284685+0000 mgr.y (mgr.14148) 55 : cluster [DBG] 2023-10-09T05:16:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:58 smithi191 conmon[101299]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:16:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:16:58 smithi191 conmon[101299]: cephadm 2023-10-09T05:16:57.425354+0000 mgr.y (mgr.14148) 56 : cephadm [INF] Deploying daemon osd.0 on smithi008 2023-10-09T05:17:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:00 smithi008 conmon[101053]: cluster 2023-10-09T05:16: 2023-10-09T05:17:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:00 smithi008 conmon[101053]: 59.285069+0000 mgr.y (mgr.14148) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:00.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:00 smithi008 conmon[110782]: cluster 2023-10-09T05:16:59.285069+0000 2023-10-09T05:17:00.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:00 smithi008 conmon[110782]: mgr.y (mgr.14148) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:01.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:00 smithi191 conmon[101299]: cluster 2023-10-09T05:16:59.285069+0000 mgr.y (mgr.14148) 57 : cluster [DBG] 2023-10-09T05:17:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:00 smithi191 conmon[101299]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:02.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:02 smithi008 conmon[101053]: cluster 2023-10-09T05:17:01.285330+0000 mgr.y ( 2023-10-09T05:17:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:02 smithi008 conmon[101053]: mgr.14148) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:02 smithi008 conmon[101053]: audit 2023-10-09T05:17:02.318116+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:02 smithi008 conmon[101053]: audit 2023-10-09T05:17:02.318368+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:02 smithi008 conmon[110782]: cluster 2023-10-09T05:17:01.285330+0000 mgr.y (mgr 2023-10-09T05:17:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:02 smithi008 conmon[110782]: .14148) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:02 smithi008 conmon[110782]: audit 2023-10-09T05:17:02.318116+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:02 smithi008 conmon[110782]: audit 2023-10-09T05:17:02.318368+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:03.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:02 smithi191 conmon[101299]: cluster 2023-10-09T05:17:01.285330+0000 mgr.y (mgr.14148 2023-10-09T05:17:03.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:02 smithi191 conmon[101299]: ) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:03.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:02 smithi191 conmon[101299]: audit 2023-10-09T05:17:02.318116+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:03.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:02 smithi191 conmon[101299]: audit 2023-10-09T05:17: 2023-10-09T05:17:03.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:02 smithi191 conmon[101299]: 02.318368+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:03.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:02 smithi191 conmon[101299]: 2023-10-09T05:17:04.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:04 smithi008 conmon[101053]: cluster 2023-10-09T05:17:03.285691+0000 mgr.y (mgr.14148) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:04.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:04 smithi008 conmon[101053]: audit 2023-10-09T05:17:04.064695+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:04.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:04 smithi008 conmon[101053]: audit 2023-10-09T05:17:04.410541+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:04.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:04 smithi008 conmon[101053]: audit 2023-10-09T05:17:04.411271+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:04 smithi008 conmon[101053]: audit 2023-10-09T05:17:04.415842+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:04.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:04 smithi008 conmon[110782]: cluster 2023-10-09T05:17:03.285691+0000 mgr.y (mgr.14148) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:04.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:04 smithi008 conmon[110782]: audit 2023-10-09T05:17:04.064695+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:04.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:04 smithi008 conmon[110782]: audit 2023-10-09T05:17:04.410541+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:04.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:04 smithi008 conmon[110782]: audit 2023-10-09T05:17:04.411271+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:04.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:04 smithi008 conmon[110782]: audit 2023-10-09T05:17:04.415842+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:04 smithi191 conmon[101299]: cluster 2023-10-09T05:17:03.285691+0000 mgr.y (mgr.14148 2023-10-09T05:17:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:04 smithi191 conmon[101299]: ) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:04 smithi191 conmon[101299]: audit 2023-10-09T05:17:04.064695+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:04 smithi191 conmon[101299]: audit 2023-10-09T05:17:04 2023-10-09T05:17:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:04 smithi191 conmon[101299]: .410541+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:04 smithi191 conmon[101299]: audit 2023-10-09T05:17:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:04 smithi191 conmon[101299]: 2023-10-09T05:17:04.411271+0000 mon.a (mon 2023-10-09T05:17:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:04 smithi191 conmon[101299]: .0) 228 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:04 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:17:05.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:04 smithi191 conmon[101299]: 10-09T05:17:04.415842+0000 mon.a (mon 2023-10-09T05:17:05.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:04 smithi191 conmon[101299]: .0) 229 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:05.228 INFO:teuthology.orchestra.run.smithi008.stdout:Created osd(s) 0 on host 'smithi008' 2023-10-09T05:17:05.836 DEBUG:teuthology.orchestra.run.smithi008:osd.0> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T05:17:05.838 INFO:tasks.cephadm:Deploying osd.1 on smithi008 with /dev/vg_nvme/lv_3... 2023-10-09T05:17:05.838 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T05:17:05.871 INFO:journalctl@ceph.osd.0.smithi008.stdout:-- Logs begin at Mon 2023-10-09 04:57:44 UTC. -- 2023-10-09T05:17:06.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:17:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[110782]: :17:05.227902+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[110782]: audit 2023-10-09T05:17:05.233183+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[110782]: cluster 2023-10-09T05:17:05.286249+0000 mgr.y (mgr.14148) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[110782]: audit 2023-10-09T05:17:05.524606+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.8:6802/791166472,v1:172.21.15.8:6803/791166472]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:17:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[110782]: audit 2023-10-09T05:17:05.525035+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:17:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[101053]: audit 2023-10-09T05:17:05. 2023-10-09T05:17:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[101053]: 227902+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[101053]: audit 2023-10-09T05:17:05.233183+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[101053]: cluster 2023-10-09T05:17:05.286249+0000 mgr.y (mgr.14148) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[101053]: audit 2023-10-09T05:17:05.524606+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.8:6802/791166472,v1:172.21.15.8:6803/791166472]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:17:06.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:06 smithi008 conmon[101053]: audit 2023-10-09T05:17:05.525035+0000 mon.a (mon.0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:17:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:06 smithi191 conmon[101299]: audit 2023-10-09T05:17:05.227902+0000 mon.a (mon.0) 230 2023-10-09T05:17:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:06 smithi191 conmon[101299]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:06 smithi191 conmon[101299]: audit 2023-10-09T05:17:05.233183+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:06 smithi191 conmon[101299]: cluster 2023-10-09T05:17:05.286249 2023-10-09T05:17:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:06 smithi191 conmon[101299]: +0000 mgr.y (mgr.14148) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:06 smithi191 conmon[101299]: audit 2023-10-09T05:17:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:06 smithi191 conmon[101299]: 2023-10-09T05:17:05.524606+0000 mon.c (mon 2023-10-09T05:17:06.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:06 smithi191 conmon[101299]: .2) 4 : audit [INF] from='osd.0 [v2:172.21.15.8:6802/791166472,v1:172.21.15.8:6803/791166472]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:17:06.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:06 smithi191 conmon[101299]: audit 2023-10-09T05:17:06.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:06 smithi191 conmon[101299]: 2023-10-09T05:17:05.525035+0000 mon.a (mon 2023-10-09T05:17:06.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:06 smithi191 conmon[101299]: .0) 232 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:17:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: audit 2023-10-09T05:17:06.232686+0000 mon.a 2023-10-09T05:17:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:17:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: cluster 2023-10-09T05:17:06.232790+0000 2023-10-09T05:17:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: mon.a (mon.0) 234 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:17:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: audit 2023-10-09T05:17:06.233078 2023-10-09T05:17:07.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: +0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:07.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: 2023-10-09T05:17:07.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: audit 2023-10-09T05:17:06.233371 2023-10-09T05:17:07.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: +0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.8:6802/791166472,v1:172.21.15.8:6803/791166472]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:07.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: 2023-10-09T05:17:07.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: audit 2023-10-09T05:17:06.233725 2023-10-09T05:17:07.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: +0000 mon.a (mon.0) 236 : audit [INF] 2023-10-09T05:17:07.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:07 smithi191 conmon[101299]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:07.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[101053]: audit 2023-10-09T05:17:06.232686+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:17:07.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[101053]: cluster 2023-10-09T05:17:06.232790+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:17:07.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[101053]: audit 2023-10-09T05:17:06.233078+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:07.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[101053]: audit 2023-10-09T05:17:06.233371+0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.8:6802/791166472,v1:172.21.15.8:6803/791166472]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:07.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[101053]: audit 2023-10-09T05:17:06.233725+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:07.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[110782]: audit 2023-10-09T05:17:06.232686+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:17:07.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[110782]: cluster 2023-10-09T05:17:06.232790+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:17:07.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[110782]: audit 2023-10-09T05:17:06.233078+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:07.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:17:07.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[110782]: :17:06.233371+0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.8:6802/791166472,v1:172.21.15.8:6803/791166472]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:07.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[110782]: audit 2023-10-09T05:17:06.233725+0000 mon.a (mon.0) 236 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:07.718 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[115909]: debug 2023-10-09T05:17:07.236+0000 7fd2e850e700 -1 osd.0 0 waiting for initial osdmap 2023-10-09T05:17:07.718 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:17:07 smithi008 conmon[115909]: debug 2023-10-09T05:17:07.238+0000 7fd2e1483700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:17:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[101053]: audit 2023-10-09T05:17:07.232940+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:17:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[101053]: cluster 2023-10-09T05:17:07.233044+0000 mon.a (mon.0) 238 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:17:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[101053]: audit 2023-10-09T05:17:07.233344+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[101053]: audit 2023-10-09T05:17:07.237063+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[101053]: cluster 2023-10-09T05:17:07.286722+0000 mgr.y (mgr.14148) 61 : 2023-10-09T05:17:08.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[101053]: cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[110782]: audit 2023-10-09T05:17:07.232940+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:17:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[110782]: cluster 2023-10-09T05:17:07.233044+0000 mon.a (mon.0) 238 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:17:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[110782]: audit 2023-10-09T05:17:07.233344+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[110782]: audit 2023-10-09T05:17:07.237063+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[110782]: cluster 2023-10-09T05:17:07.286722+0000 mgr.y (mgr.14148) 61 2023-10-09T05:17:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:08 smithi008 conmon[110782]: : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:08.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: audit 2023-10-09T05:17:07.232940+0000 mon.a (mon 2023-10-09T05:17:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: .0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:17:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: cluster 2023-10-09T05:17:07.233044+0000 mon.a (mon.0) 2023-10-09T05:17:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: 238 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:17:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:17:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: 10-09T05:17:07.233344+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:17:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: -09T05:17:07.237063+0000 mon.a (mon.0) 2023-10-09T05:17:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: 240 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: cluster 2023-10-09T05: 2023-10-09T05:17:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: 17:07.286722+0000 mgr.y (mgr.14148) 61 : cluster 2023-10-09T05:17:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:08 smithi191 conmon[101299]: [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:17:08.746 INFO:teuthology.orchestra.run.smithi008.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T05:17:08.746 INFO:teuthology.orchestra.run.smithi008.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T05:17:08.746 INFO:teuthology.orchestra.run.smithi008.stderr: stderr: 10+0 records in 2023-10-09T05:17:08.746 INFO:teuthology.orchestra.run.smithi008.stderr:10+0 records out 2023-10-09T05:17:08.746 INFO:teuthology.orchestra.run.smithi008.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0081947 s, 1.3 GB/s 2023-10-09T05:17:08.747 INFO:teuthology.orchestra.run.smithi008.stderr:--> Zapping successful for: 2023-10-09T05:17:09.360 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi008:vg_nvme/lv_3 2023-10-09T05:17:09.387 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[101053]: cluster 2023-10-09T05:17:06.542629+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:17:09.387 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[101053]: cluster 2023-10-09T05:17:06.542684+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:17:09.388 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[101053]: audit 2023-10-09T05:17:08.237440+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:09.388 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[101053]: cluster 2023-10-09T05:17:08.241715+0000 mon.a (mon.0) 242 : cluster [INF] osd.0 [v2:172.21.15.8:6802/791166472,v1:172.21.15.8:6803/791166472] boot 2023-10-09T05:17:09.388 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[101053]: cluster 2023-10-09T05:17:08.241759+0000 mon.a (mon.0) 243 : 2023-10-09T05:17:09.388 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[101053]: cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:17:09.388 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[101053]: audit 2023-10-09T05:17:08.242057+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:09.388 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[110782]: cluster 2023-10-09T05:17:06.542629+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:17:09.388 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[110782]: cluster 2023-10-09T05:17:06.542684+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:17:09.388 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[110782]: audit 2023-10-09T05:17:08.237440+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:09.388 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[110782]: cluster 2023-10-09T05:17:08.241715+0000 mon.a (mon.0) 242 : cluster [INF] osd.0 [v2:172.21.15.8:6802/791166472,v1:172.21.15.8:6803/791166472] boot 2023-10-09T05:17:09.388 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[110782]: cluster 2023-10-09T05:17:08.241759+0000 mon.a (mon.0) 2023-10-09T05:17:09.389 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[110782]: 243 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:17:09.389 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:09 smithi008 conmon[110782]: audit 2023-10-09T05:17:08.242057+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:09.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: cluster 2023-10-09T05:17:06.542629+0000 osd.0 2023-10-09T05:17:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:17:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: cluster 2023-10-09T05:17:06 2023-10-09T05:17:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: .542684+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:17:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: audit 2023-10-09T05:17 2023-10-09T05:17:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: :08.237440+0000 mon.a (mon.0) 241 : audit 2023-10-09T05:17:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: cluster 2023-10-09T05:17 2023-10-09T05:17:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: :08.241715+0000 mon.a (mon.0) 242 : cluster 2023-10-09T05:17:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: [INF] osd.0 [v2:172.21.15.8:6802/791166472,v1:172.21.15.8:6803/791166472] boot 2023-10-09T05:17:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: cluster 2023-10-09T05: 2023-10-09T05:17:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: 17:08.241759+0000 mon.a (mon.0) 243 : 2023-10-09T05:17:09.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:17:09.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:17:09.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: T05:17:08.242057+0000 mon.a (mon.0) 2023-10-09T05:17:09.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:09 smithi191 conmon[101299]: 244 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:17:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: cluster 2023-10-09T05:17:09.246182+0000 mon.a (mon.0 2023-10-09T05:17:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: ) 245 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:17:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: cluster 2023-10-09T05:17:09.286989+0000 mgr.y (mgr.14148) 62 : cluster [DBG] 2023-10-09T05:17:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: cephadm 2023-10-09T 2023-10-09T05:17:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: 05:17:09.417368+0000 mgr.y (mgr.14148) 63 : cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:17:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: audit 2023-10-09T05:17:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: 2023-10-09T05:17:09.422502+0000 mon.a (mon 2023-10-09T05:17:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: .0) 246 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: audit 2023 2023-10-09T05:17:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: -10-09T05:17:09.423374+0000 mon.a (mon 2023-10-09T05:17:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: .0) 247 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: cephadm 2023 2023-10-09T05:17:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: -10-09T05:17:09.423919+0000 mgr.y (mgr 2023-10-09T05:17:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: .14148) 64 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 16149M 2023-10-09T05:17:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: audit 2023-10-09T05:17 2023-10-09T05:17:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: :09.428443+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:17:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: -09T05:17:09.429435+0000 mon.a (mon.0) 2023-10-09T05:17:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: 249 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:17:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: 10-09T05:17:09.430064+0000 mon.a ( 2023-10-09T05:17:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: mon.0) 250 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:17:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: :17:09.434970+0000 mon.a (mon.0) 251 2023-10-09T05:17:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:10 smithi191 conmon[101299]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:10.581 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[101053]: cluster 2023-10-09T 2023-10-09T05:17:10.581 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[101053]: 05:17:09.246182+0000 mon.a (mon.0) 245 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:17:10.581 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[101053]: cluster 2023-10-09T05:17:09.286989+0000 mgr.y (mgr.14148) 62 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:10.581 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[101053]: cephadm 2023-10-09T05:17:09.417368+0000 mgr.y (mgr.14148) 63 : cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:17:10.581 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[101053]: audit 2023-10-09T05:17:09.422502+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:10.581 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[101053]: audit 2023-10-09T05:17:09.423374+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:10.582 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[101053]: cephadm 2023-10-09T05:17:09.423919+0000 mgr.y (mgr.14148) 64 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 16149M 2023-10-09T05:17:10.582 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[101053]: audit 2023-10-09T05:17:09.428443+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:10.582 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[101053]: audit 2023-10-09T05:17:09.429435+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:10.582 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[101053]: audit 2023-10-09T05:17:09.430064+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:10.582 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[101053]: audit 2023-10-09T05:17:09.434970+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:10.582 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: cluster 2023-10-09T05 2023-10-09T05:17:10.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: :17:09.246182+0000 mon.a (mon.0) 245 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:17:10.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: cluster 2023-10-09T05:17:09.286989+0000 mgr.y (mgr.14148) 62 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:10.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: cephadm 2023-10-09T05:17:09.417368+0000 mgr.y (mgr.14148) 63 : cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:17:10.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: audit 2023-10-09T05:17:09.422502+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:10.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: audit 2023-10-09T05:17:09.423374+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:10.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: cephadm 2023-10-09T05:17:09.423919+0000 mgr.y (mgr.14148) 64 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 16149M 2023-10-09T05:17:10.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: audit 2023-10-09T05:17:09.428443+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:10.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: audit 2023-10-09T05:17:09.429435+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:10.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: audit 2023-10-09T05:17:10.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: 2023-10-09T05:17:09.430064+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:10.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: audit 2023-10-09T05:17:10.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:17:10.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: 17:09.434970+0000 mon.a ( 2023-10-09T05:17:10.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: mon.0) 251 : audit [INF] 2023-10-09T05:17:10.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:10 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:11 smithi191 conmon[101299]: audit 2023-10-09T05:17:11.224356+0000 mon.a 2023-10-09T05:17:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:11 smithi191 conmon[101299]: (mon.0) 252 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:17:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:11 smithi191 conmon[101299]: audit 2023-10-09T05:17:11. 2023-10-09T05:17:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:11 smithi191 conmon[101299]: 226582+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:17:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:11 smithi191 conmon[101299]: audit 2023-10-09T05:17:11.227134+0000 2023-10-09T05:17:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:11 smithi191 conmon[101299]: mon.a (mon.0) 254 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:11 smithi008 conmon[101053]: audit 2023-10-09T05:17:11.224356+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:17:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:11 smithi008 conmon[101053]: audit 2023-10-09T05:17:11.226582+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:17:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:11 smithi008 conmon[101053]: audit 2023-10-09T05:17:11.227134+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:11 smithi008 conmon[110782]: audit 2023-10-09T05:17:11.224356+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:17:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:11 smithi008 conmon[110782]: audit 2023-10-09T05:17:11.226582+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:17:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:11 smithi008 conmon[110782]: audit 2023-10-09T05:17:11.227134+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:12.787 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:12 smithi008 conmon[101053]: audit 2023-10-09T05:17:11.221725+0000 mgr.y (mgr 2023-10-09T05:17:12.787 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:12 smithi008 conmon[101053]: .14148) 65 : audit [DBG] from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:17:12.787 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:12 smithi008 conmon[101053]: cluster 2023-10-09T05:17:11.287304+0000 mgr.y (mgr.14148) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:12.788 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:12 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:17:12.788 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:12 smithi008 conmon[110782]: 17:11.221725+0000 mgr.y (mgr.14148) 65 : audit [DBG] from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:17:12.788 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:12 smithi008 conmon[110782]: cluster 2023-10-09T05:17:11.287304+0000 mgr.y (mgr.14148 2023-10-09T05:17:12.788 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:12 smithi008 conmon[110782]: ) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:12 smithi191 conmon[101299]: audit 2023-10-09T05:17:11.221725+0000 mgr.y (mgr.14148) 2023-10-09T05:17:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:12 smithi191 conmon[101299]: 65 : audit [DBG] from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:17:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:12 smithi191 conmon[101299]: cluster 2023-10-09T05:17:11.287304+0000 mgr.y (mgr.14148) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:13 smithi008 conmon[101053]: audit 2023-10-09T05:17:13.494442+0000 mon.a (mon. 2023-10-09T05:17:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:13 smithi008 conmon[101053]: 0) 255 : audit [INF] from='client.? 172.21.15.8:0/1957162356' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55f3290e-b82d-4839-a28d-1123a20c3a4c"}]: dispatch 2023-10-09T05:17:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:13 smithi008 conmon[101053]: audit 2023-10-09T05:17:13.498102+0000 mon.a (mon.0) 256 : audit [INF] from='client.? 172.21.15.8:0/1957162356' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55f3290e-b82d-4839-a28d-1123a20c3a4c"}]': finished 2023-10-09T05:17:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:13 smithi008 conmon[101053]: cluster 2023-10-09T05:17:13.498184+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:17:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:13 smithi008 conmon[101053]: audit 2023-10-09T05:17:13.498315+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:13.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:13 smithi008 conmon[110782]: audit 2023-10-09T05:17:13.494442+0000 2023-10-09T05:17:13.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:13 smithi008 conmon[110782]: mon.a (mon.0) 255 : audit [INF] from='client.? 172.21.15.8:0/1957162356' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55f3290e-b82d-4839-a28d-1123a20c3a4c"}]: dispatch 2023-10-09T05:17:13.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:13 smithi008 conmon[110782]: audit 2023-10-09T05:17:13.498102+0000 mon.a (mon.0) 256 : audit [INF] from='client.? 172.21.15.8:0/1957162356' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55f3290e-b82d-4839-a28d-1123a20c3a4c"}]': finished 2023-10-09T05:17:13.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:13 smithi008 conmon[110782]: cluster 2023-10-09T05:17:13.498184+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:17:13.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:13 smithi008 conmon[110782]: audit 2023-10-09T05:17:13.498315+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:13 smithi191 conmon[101299]: audit 2023-10-09T05:17:13.494442+0000 mon.a (mon 2023-10-09T05:17:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:13 smithi191 conmon[101299]: .0) 255 : audit [INF] from='client.? 172.21.15.8:0/1957162356' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55f3290e-b82d-4839-a28d-1123a20c3a4c"}]: dispatch 2023-10-09T05:17:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:13 smithi191 conmon[101299]: audit 2023-10-09T05:17:13.498102+0000 mon.a ( 2023-10-09T05:17:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:13 smithi191 conmon[101299]: mon.0) 256 : audit [INF] from='client.? 172.21.15.8:0/1957162356' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55f3290e-b82d-4839-a28d-1123a20c3a4c"}]': finished 2023-10-09T05:17:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:13 smithi191 conmon[101299]: cluster 2023-10-09T05:17:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:13 smithi191 conmon[101299]: 2023-10-09T05:17:13.498184+0000 mon.a (mon.0) 257 : cluster [DBG] 2023-10-09T05:17:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:13 smithi191 conmon[101299]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:17:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:13 smithi191 conmon[101299]: audit 2023-10-09T05:17:13. 2023-10-09T05:17:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:13 smithi191 conmon[101299]: 498315+0000 mon.a (mon.0) 258 : audit [DBG] 2023-10-09T05:17:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:13 smithi191 conmon[101299]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:14.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:14 smithi008 conmon[101053]: cluster 2023-10-09T05:17:13.287610+0000 mgr.y (mgr.14148) 67 2023-10-09T05:17:14.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:14 smithi008 conmon[101053]: : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:14.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:14 smithi008 conmon[101053]: audit 2023-10-09T05:17:14.107233+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.8:0/1018862821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:17:14.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:14 smithi008 conmon[110782]: cluster 2023-10-09T05:17:13. 2023-10-09T05:17:14.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:14 smithi008 conmon[110782]: 287610+0000 mgr.y (mgr.14148) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:14.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:14 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:17:14.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:14 smithi008 conmon[110782]: T05:17:14.107233+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.8:0/1018862821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:17:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:14 smithi191 conmon[101299]: cluster 2023-10-09T05:17:13.287610+0000 mgr.y (mgr 2023-10-09T05:17:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:14 smithi191 conmon[101299]: .14148) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:14 smithi191 conmon[101299]: audit 2023-10-09T05:17:14.107233+0000 mon.c (mon. 2023-10-09T05:17:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:14 smithi191 conmon[101299]: 2) 6 : audit [DBG] from='client.? 172.21.15.8:0/1018862821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:17:16.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:16 smithi008 conmon[101053]: cluster 2023-10-09T05:17:15.288072 2023-10-09T05:17:16.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:16 smithi008 conmon[101053]: +0000 mgr.y (mgr.14148) 68 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:16.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:16 smithi008 conmon[110782]: cluster 2023-10-09T05:17:15. 2023-10-09T05:17:16.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:16 smithi008 conmon[110782]: 288072+0000 mgr.y (mgr.14148) 68 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:17.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:16 smithi191 conmon[101299]: cluster 2023-10-09T05:17:15.288072+0000 mgr.y (mgr.14148 2023-10-09T05:17:17.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:16 smithi191 conmon[101299]: ) 68 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:18.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:18 smithi008 conmon[101053]: cluster 2023-10-09T05:17:17.288321+0000 mgr.y 2023-10-09T05:17:18.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:18 smithi008 conmon[101053]: (mgr.14148) 69 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:18.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:18 smithi008 conmon[110782]: cluster 2023-10-09T05:17 2023-10-09T05:17:18.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:18 smithi008 conmon[110782]: :17.288321+0000 mgr.y (mgr.14148) 69 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:19.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:18 smithi191 conmon[101299]: cluster 2023-10-09T05:17:17.288321+0000 mgr.y (mgr 2023-10-09T05:17:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:18 smithi191 conmon[101299]: .14148) 69 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:19.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:19 smithi008 conmon[101053]: audit 2023-10-09T05:17:19.581040+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:17:19.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:19 smithi008 conmon[101053]: audit 2023-10-09T05:17:19.581788+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:19.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:19 smithi008 conmon[110782]: audit 2023-10-09T05:17:19.581040+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:17:19.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:19 smithi008 conmon[110782]: audit 2023-10-09T05:17:19.581788+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:20.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:19 smithi191 conmon[101299]: audit 2023-10-09T05:17:19.581040+0000 mon.a (mon.0) 259 : audit 2023-10-09T05:17:20.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:19 smithi191 conmon[101299]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:17:20.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:19 smithi191 conmon[101299]: audit 2023-10-09T05:17:19.581788+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:20.836 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:20 smithi008 conmon[101053]: cluster 2023- 2023-10-09T05:17:20.836 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:20 smithi008 conmon[101053]: 10-09T05:17:19.288560+0000 mgr.y (mgr.14148) 70 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:20.836 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:20 smithi008 conmon[101053]: cephadm 2023-10-09T05:17:19.582400+0000 mgr.y (mgr.14148) 71 : cephadm [INF] Deploying daemon osd.1 on smithi008 2023-10-09T05:17:20.836 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:20 smithi008 conmon[110782]: cluster 2023-10-09T05:17:19.288560 2023-10-09T05:17:20.837 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:20 smithi008 conmon[110782]: +0000 mgr.y (mgr.14148) 70 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:20.837 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:20 smithi008 conmon[110782]: cephadm 2023-10-09T05:17:19.582400+0000 mgr.y (mgr.14148) 71 : cephadm [INF] Deploying daemon osd.1 on smithi008 2023-10-09T05:17:21.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:20 smithi191 conmon[101299]: cluster 2023-10-09T05:17:19.288560+0000 2023-10-09T05:17:21.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:20 smithi191 conmon[101299]: mgr.y (mgr.14148) 70 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:21.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:20 smithi191 conmon[101299]: cephadm 2023-10-09T05:17: 2023-10-09T05:17:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:20 smithi191 conmon[101299]: 19.582400+0000 mgr.y (mgr.14148) 71 : cephadm [INF] Deploying daemon osd.1 on smithi008 2023-10-09T05:17:23.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:22 smithi191 conmon[101299]: cluster 2023-10-09T05:17:21.288831+0000 mgr.y (mgr. 2023-10-09T05:17:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:22 smithi191 conmon[101299]: 14148) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:23.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:22 smithi008 conmon[101053]: cluster 2023-10-09T05:17:21. 2023-10-09T05:17:23.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:22 smithi008 conmon[101053]: 288831+0000 mgr.y (mgr.14148) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:23.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:22 smithi008 conmon[110782]: cluster 2023-10-09T05:17:21.288831+0000 mgr.y (mgr.14148) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:25.039 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:24 smithi008 conmon[101053]: cluster 2023-10-09T05:17:23.289520+0000 mgr.y (mgr 2023-10-09T05:17:25.040 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:24 smithi008 conmon[101053]: .14148) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:25.040 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:24 smithi008 conmon[101053]: audit 2023-10-09T05:17:24.474415+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:25.040 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:24 smithi008 conmon[101053]: audit 2023-10-09T05:17:24.479337+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:25.040 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:24 smithi008 conmon[110782]: cluster 2023-10-09T05:17:23.289520+0000 2023-10-09T05:17:25.040 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:24 smithi008 conmon[110782]: mgr.y (mgr.14148) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:25.041 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:24 smithi008 conmon[110782]: audit 2023-10-09T05:17:24.474415+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:25.041 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:24 smithi008 conmon[110782]: audit 2023-10-09T05:17:24.479337+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:25.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:24 smithi191 conmon[101299]: cluster 2023-10-09T05:17:23.289520+0000 mgr.y (mgr.14148) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:25.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:24 smithi191 conmon[101299]: audit 2023-10-09T05:17:24.474415+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:24 smithi191 conmon[101299]: audit 2023-10-09T05:17:24.479337+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:27.038 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:26 smithi008 conmon[101053]: cluster 2023-10-09T05:17:25.289772+0000 mgr.y (mgr.14148) 74 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:27.038 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:26 smithi008 conmon[101053]: audit 2023-10-09T05:17:26.568998+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:27.039 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:26 smithi008 conmon[101053]: audit 2023-10-09T05:17:26.569784+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:27.039 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:26 smithi008 conmon[101053]: audit 2023-10-09T05:17:26.570601+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:27.039 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:26 smithi008 conmon[101053]: audit 2023-10-09T05:17:26.575251+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:27.039 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:26 smithi008 conmon[110782]: cluster 2023-10-09T05:17:25.289772+0000 mgr.y (mgr.14148) 74 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:27.039 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:26 smithi008 conmon[110782]: audit 2023-10-09T05:17:26.568998+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:27.039 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:26 smithi008 conmon[110782]: audit 2023-10-09T05:17:26.569784+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:27.039 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:26 smithi008 conmon[110782]: audit 2023-10-09T05:17:26.570601+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:27.039 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:26 smithi008 conmon[110782]: audit 2023-10-09T05:17:26.575251+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:27.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:26 smithi191 conmon[101299]: cluster 2023-10-09T05:17:25.289772+0000 mgr.y (mgr.14148) 74 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:27.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:26 smithi191 conmon[101299]: audit 2023-10-09T05:17:26.568998+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:27.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:26 smithi191 conmon[101299]: audit 2023-10-09T05:17:26.569784+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:27.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:26 smithi191 conmon[101299]: audit 2023-10-09T05:17:26.570601+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:27.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:26 smithi191 conmon[101299]: audit 2023-10-09T05:17:26.575251+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:27.499 INFO:teuthology.orchestra.run.smithi008.stdout:Created osd(s) 1 on host 'smithi008' 2023-10-09T05:17:28.036 DEBUG:teuthology.orchestra.run.smithi008:osd.1> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T05:17:28.080 INFO:tasks.cephadm:Deploying osd.2 on smithi008 with /dev/vg_nvme/lv_2... 2023-10-09T05:17:28.080 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T05:17:28.112 INFO:journalctl@ceph.osd.1.smithi008.stdout:-- Logs begin at Mon 2023-10-09 04:57:44 UTC. -- 2023-10-09T05:17:28.613 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[101053]: cluster 2023-10-09T05:17:27.290063+0000 mgr.y (mgr.14148) 75 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:28.613 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[101053]: audit 2023-10-09T05:17:27.498491+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:28.613 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[101053]: audit 2023-10-09T05:17:27.542967+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:28.613 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[101053]: audit 2023-10-09T05:17:27.682257+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.8:6810/809058426,v1:172.21.15.8:6811/809058426]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:17:28.613 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[101053]: audit 2023-10-09T05:17 2023-10-09T05:17:28.613 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[101053]: :27.682659+0000 mon.a (mon.0) 269 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:17:28.613 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[110782]: cluster 2023-10-09T05:17:27.290063+0000 mgr.y (mgr.14148) 75 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:28.614 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[110782]: audit 2023-10-09T05:17:27.498491+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:28.614 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[110782]: audit 2023-10-09T05:17:27.542967+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:28.614 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[110782]: audit 2023-10-09T05:17:27.682257+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.8:6810/809058426,v1:172.21.15.8:6811/809058426]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:17:28.614 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[110782]: audit 2023-10-09T05:17:28.614 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:28 smithi008 conmon[110782]: 2023-10-09T05:17:27.682659+0000 mon.a (mon.0) 269 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:17:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: cluster 2023-10-09T05:17:27.290063+0000 mgr.y (mgr.14148) 75 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: audit 2023-10-09T05:17:27.498491+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:17:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: 05:17:27.542967+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: audit 2023-10-09T05:17:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: 2023-10-09T05:17: 2023-10-09T05:17:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: 27.682257+0000 mon.c (mon.2 2023-10-09T05:17:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: ) 7 : audit [INF] from='osd.1 [v2:172.21.15.8:6810/809058426,v1:172.21.15.8:6811/809058426]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:17:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: audit 2023 2023-10-09T05:17:28.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: -10-09T05:17:27. 2023-10-09T05:17:28.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: 682659+0000 mon.a (mon.0) 269 : audit [INF] 2023-10-09T05:17:28.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:28 smithi191 conmon[101299]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:17:29.715 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[121813]: debug 2023-10-09T05:17:29.508+0000 7f04542d6700 -1 osd.1 0 waiting for initial osdmap 2023-10-09T05:17:29.715 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[121813]: debug 2023-10-09T05:17:29.511+0000 7f044ea4e700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:17:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[110782]: audit 2023-10-09T05:17:28.500103+0000 mon.a ( 2023-10-09T05:17:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[110782]: mon.0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:17:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[110782]: cluster 2023-10-09T05:17:28.500195+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:17:29.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[110782]: audit 2023-10-09T05:17:28.500571+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:29.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[110782]: audit 2023-10-09T05:17:28.500876+0000 mon.c (mon.2) 8 : audit [INF] from='osd.1 [v2:172.21.15.8:6810/809058426,v1:172.21.15.8:6811/809058426]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:29.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[110782]: audit 2023-10-09T05:17:28.504285+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:29.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[101053]: audit 2023-10-09T05:17:28.500103+0000 mon.a (mon 2023-10-09T05:17:29.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[101053]: .0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:17:29.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[101053]: cluster 2023-10-09T05:17:28.500195+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:17:29.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[101053]: audit 2023-10-09T05:17:28.500571+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:29.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[101053]: audit 2023-10-09T05:17:28.500876+0000 mon.c (mon.2) 8 : audit [INF] from='osd.1 [v2:172.21.15.8:6810/809058426,v1:172.21.15.8:6811/809058426]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:29.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:29 smithi008 conmon[101053]: audit 2023-10-09T05:17:28.504285+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:29 smithi191 conmon[101299]: audit 2023-10-09T05:17:28.500103+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:17:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:29 smithi191 conmon[101299]: cluster 2023-10-09T05:17:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:29 smithi191 conmon[101299]: 2023-10-09T05:17:28.500195+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:17:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:29 smithi191 conmon[101299]: audit 2023-10-09T05:17:28.500571+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:29 smithi191 conmon[101299]: audit 2023-10-09T05:17:28.500876+0000 mon.c (mon.2) 8 : audit [INF] from='osd.1 [v2:172.21.15.8:6810/809058426,v1:172.21.15.8:6811/809058426]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:29 smithi191 conmon[101299]: audit 2023-10-09T05:17:28.504285+0000 mon.a (mon.0) 2023-10-09T05:17:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:29 smithi191 conmon[101299]: 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:30.596 INFO:teuthology.orchestra.run.smithi008.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T05:17:30.596 INFO:teuthology.orchestra.run.smithi008.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T05:17:30.596 INFO:teuthology.orchestra.run.smithi008.stderr: stderr: 10+0 records in 2023-10-09T05:17:30.596 INFO:teuthology.orchestra.run.smithi008.stderr:10+0 records out 2023-10-09T05:17:30.597 INFO:teuthology.orchestra.run.smithi008.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00796341 s, 1.3 GB/s 2023-10-09T05:17:30.597 INFO:teuthology.orchestra.run.smithi008.stderr:--> Zapping successful for: 2023-10-09T05:17:30.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:30 smithi008 conmon[101053]: cluster 2023-10-09T05:17:29.290354+0000 mgr.y (mgr.14148) 76 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:30.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:30 smithi008 conmon[101053]: audit 2023-10-09T05:17:29.505241+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:17:30.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:30 smithi008 conmon[101053]: cluster 2023-10-09T05:17:29.505321+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:17:30.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:30 smithi008 conmon[101053]: audit 2023-10-09T05:17:29.505690+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:30.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:30 smithi008 conmon[101053]: audit 2023-10-09T05:17:29.509346+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:30 smithi008 conmon[110782]: cluster 2023-10-09T05:17:29.290354+0000 mgr.y (mgr.14148) 76 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:30 smithi008 conmon[110782]: audit 2023-10-09T05:17:29.505241+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:17:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:30 smithi008 conmon[110782]: cluster 2023-10-09T05:17:29.505321+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:17:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:30 smithi008 conmon[110782]: audit 2023-10-09T05:17:29.505690+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:30 smithi008 conmon[110782]: audit 2023-10-09T05:17:29.509346+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:30 smithi191 conmon[101299]: cluster 2023-10-09T05:17:29.290354+0000 mgr.y (mgr.14148) 76 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:30 smithi191 conmon[101299]: audit 2023-10-09T05:17:29.505241+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:17:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:30 smithi191 conmon[101299]: cluster 2023-10-09T05:17:29.505321+0000 2023-10-09T05:17:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:30 smithi191 conmon[101299]: mon.a (mon.0) 275 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:17:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:30 smithi191 conmon[101299]: audit 2023-10-09T05:17:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:30 smithi191 conmon[101299]: 2023-10-09T05:17:29.505690+0000 2023-10-09T05:17:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:30 smithi191 conmon[101299]: mon.a (mon.0) 276 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:30 smithi191 conmon[101299]: audit 2023-10-09T05:17:30.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:30 smithi191 conmon[101299]: 2023-10-09T05:17:29 2023-10-09T05:17:30.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:30 smithi191 conmon[101299]: .509346+0000 mon.a (mon.0) 277 : audit 2023-10-09T05:17:30.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:30 smithi191 conmon[101299]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:31.500 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi008:vg_nvme/lv_2 2023-10-09T05:17:31.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:31 smithi191 conmon[101299]: cluster 2023-10-09T05:17:28.674352+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:17:31.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:31 smithi191 conmon[101299]: cluster 2023-10-09T05:17:28.674403+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:17:31.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:31 smithi191 conmon[101299]: cluster 2023-10-09T05:17:30. 2023-10-09T05:17:31.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:31 smithi191 conmon[101299]: 510559+0000 mon.a (mon.0) 278 : cluster [INF] osd.1 [v2:172.21.15.8:6810/809058426,v1:172.21.15.8:6811/809058426] boot 2023-10-09T05:17:31.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:31 smithi191 conmon[101299]: 2023-10-09T05:17:31.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:31 smithi191 conmon[101299]: cluster 2023-10-09T05: 2023-10-09T05:17:31.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:31 smithi191 conmon[101299]: 17:30.510613+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:17:31.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:31 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:17:31.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:31 smithi191 conmon[101299]: 10-09T05:17:30. 2023-10-09T05:17:31.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:31 smithi191 conmon[101299]: 510938+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:31.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:31 smithi191 conmon[101299]: 2023-10-09T05:17:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:31 smithi008 conmon[101053]: cluster 2023-10-09T05:17:28.674352+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:17:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:31 smithi008 conmon[101053]: cluster 2023-10-09T05:17:28.674403+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:17:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:31 smithi008 conmon[101053]: cluster 2023-10-09T05:17:30.510559+0000 mon.a (mon.0) 278 : cluster [INF] osd.1 [v2:172.21.15.8:6810/809058426,v1:172.21.15.8:6811/809058426] boot 2023-10-09T05:17:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:31 smithi008 conmon[101053]: cluster 2023-10-09T05:17:30.510613+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:17:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:31 smithi008 conmon[101053]: audit 2023-10-09T05:17:30.510938+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:31 smithi008 conmon[110782]: cluster 2023-10-09T05:17:28.674352+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:17:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:31 smithi008 conmon[110782]: cluster 2023-10-09T05:17:28.674403+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:17:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:31 smithi008 conmon[110782]: cluster 2023-10-09T05:17:30.510559+0000 mon.a (mon.0) 278 : cluster [INF] osd.1 [v2:172.21.15.8:6810/809058426,v1:172.21.15.8:6811/809058426] boot 2023-10-09T05:17:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:31 smithi008 conmon[110782]: cluster 2023-10-09T05:17:30.510613+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:17:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:31 smithi008 conmon[110782]: audit 2023-10-09T05:17:30.510938+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:17:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: cluster 2023-10-09T05:17:31 2023-10-09T05:17:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: .290804+0000 mgr.y (mgr.14148) 77 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: cluster 2023-10-09T05:17:31.519377+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:17:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: cephadm 2023-10-09T05:17:31.609459+0000 mgr.y (mgr.14148) 78 : cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:17:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: audit 2023-10-09T05:17:31.613819+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: audit 2023-10-09T05:17:31.614861+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: audit 2023-10-09T05:17:31.615572+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: cephadm 2023-10-09T05:17:31.616093+0000 mgr.y (mgr.14148) 79 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 8074M 2023-10-09T05:17:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: audit 2023-10-09T05:17:31.619702+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: audit 2023-10-09T05:17:31.621174+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: audit 2023-10-09T05:17:31.621883+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[101053]: audit 2023-10-09T05:17:31.627200+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:32.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: cluster 2023-10-09T05:17:31.290804+0000 mgr.y (mgr 2023-10-09T05:17:32.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: .14148) 77 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:32.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: cluster 2023-10-09T05:17:31.519377+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:17:32.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: cephadm 2023-10-09T05:17:31.609459+0000 mgr.y (mgr.14148) 78 : cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:17:32.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: audit 2023-10-09T05:17:31.613819+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:32.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: audit 2023-10-09T05:17:31.614861+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:32.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: audit 2023-10-09T05:17:31.615572+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:32.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: cephadm 2023-10-09T05:17:31.616093+0000 mgr.y (mgr.14148) 79 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 8074M 2023-10-09T05:17:32.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: audit 2023-10-09T05:17:31.619702+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:32.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: audit 2023-10-09T05:17:31.621174+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:32.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: audit 2023-10-09T05:17:31.621883+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:32.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:32 smithi008 conmon[110782]: audit 2023-10-09T05:17:31.627200+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: cluster 2023-10-09T05:17:31.290804+0000 mgr.y (mgr.14148) 77 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:17:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: cluster 2023-10-09T05:17:31.519377+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:17:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: cephadm 2023-10-09T05 2023-10-09T05:17:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: :17:31.609459+0000 mgr.y (mgr.14148) 78 : cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:17:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:17:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: T05:17:31.613819+0000 2023-10-09T05:17:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: mon.a (mon.0) 282 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:17:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: 05:17:31.614861+0000 mon.a (mon.0) 283 2023-10-09T05:17:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: 09T05:17:31.615572+0000 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: mon.a (mon.0) 284 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: cephadm 2023 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: -10-09T05:17:31. 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: 616093+0000 mgr.y (mgr.14148) 79 : cephadm 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: [INF] Adjusting osd_memory_target on smithi008 to 8074M 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: 05:17:31.619702+0000 mon.a (mon 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: .0) 285 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: audit 2023 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: -10-09T05:17: 2023-10-09T05:17:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: 31.621174+0000 mon.a (mon.0) 286 : 2023-10-09T05:17:32.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:32.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:17:32.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: 09T05:17:31.621883+0000 2023-10-09T05:17:32.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: mon.a (mon.0) 287 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:32.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: 2023-10-09T05:17:32.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:17:32.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: :17:31.627200+0000 mon.a (mon 2023-10-09T05:17:32.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:32 smithi191 conmon[101299]: .0) 288 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:33 smithi191 conmon[101299]: audit 2023-10-09T05:17:33.426259+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:17:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:33 smithi191 conmon[101299]: audit 2023-10-09T05:17:33.428548+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:17:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:33 smithi191 conmon[101299]: audit 2023-10-09T05:17:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:33 smithi191 conmon[101299]: 2023-10-09T05:17:33.429124+0000 mon.a (mon.0) 291 : audit 2023-10-09T05:17:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:33 smithi191 conmon[101299]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:33.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[101053]: audit 2023-10-09T05:17 2023-10-09T05:17:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[101053]: :33.426259+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:17:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[101053]: audit 2023-10-09T05:17:33.428548+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:17:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[101053]: audit 2023-10-09T05:17:33.429124+0000 mon.a ( 2023-10-09T05:17:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[101053]: mon.0) 291 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[110782]: audit 2023-10-09T05:17: 2023-10-09T05:17:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[110782]: 33.426259+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:17:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[110782]: audit 2023-10-09T05:17:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[110782]: 2023-10-09T05:17:33.428548 2023-10-09T05:17:33.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[110782]: +0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:17:33.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:17:33.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[110782]: 05:17:33.429124+0000 mon.a (mon.0) 291 2023-10-09T05:17:33.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:33 smithi008 conmon[110782]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:34.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:34 smithi008 conmon[101053]: cluster 2023-10-09T05:17:33.291125+0000 mgr.y (mgr.14148) 80 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:34.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:34 smithi008 conmon[101053]: audit 2023-10-09T05:17:33.423646+0000 mgr.y (mgr.14148) 81 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:17:34.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:34 smithi008 conmon[110782]: cluster 2023-10-09T05:17:33.291125+0000 mgr.y (mgr.14148) 80 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:34.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:34 smithi008 conmon[110782]: audit 2023-10-09T05:17:33.423646+0000 mgr.y (mgr.14148) 81 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:17:34.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:34 smithi191 conmon[101299]: cluster 2023-10-09T05:17:33.291125+0000 mgr.y 2023-10-09T05:17:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:34 smithi191 conmon[101299]: (mgr.14148) 80 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:34 smithi191 conmon[101299]: audit 2023-10-09T05:17:33.423646+0000 mgr.y ( 2023-10-09T05:17:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:34 smithi191 conmon[101299]: mgr.14148) 81 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:17:36.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: cluster 2023-10-09T05:17:35.291378+0000 mgr.y (mgr.14148) 82 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: audit 2023-10-09T05:17:35.751059+0000 mon.a (mon.0) 292 : audit [INF] from='client.? 172.21.15.8:0/260553007' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "720b47df-d743-46cf-ad04-36e113c8fcb3"}]: dispatch 2023-10-09T05:17:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:17:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: 17:35.755082+0000 mon.a (mon.0) 293 : audit [INF] from='client.? 172.21.15.8:0/260553007' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "720b47df-d743-46cf-ad04-36e113c8fcb3"}]': finished 2023-10-09T05:17:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: cluster 2023-10- 2023-10-09T05:17:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: 09T05:17:35.755150+0000 2023-10-09T05:17:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: mon.a (mon.0) 294 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:17:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:17:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: 05:17:35.755269+0000 mon.a (mon. 2023-10-09T05:17:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: 0) 295 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: audit 2023-10-09T05:17:36.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: 2023-10-09T05:17:36 2023-10-09T05:17:36.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: .400314+0000 mon.a (mon.0) 296 : audit 2023-10-09T05:17:36.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:36 smithi191 conmon[101299]: [DBG] from='client.? 172.21.15.8:0/113640391' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:17:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[101053]: cluster 2023-10-09T05:17:35.291378+0000 mgr.y 2023-10-09T05:17:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[101053]: (mgr.14148) 82 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[101053]: audit 2023-10-09T05:17:35.751059+0000 mon.a (mon.0) 292 : audit [INF] from='client.? 172.21.15.8:0/260553007' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "720b47df-d743-46cf-ad04-36e113c8fcb3"}]: dispatch 2023-10-09T05:17:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[101053]: audit 2023-10-09T05:17:35.755082+0000 mon.a (mon.0) 293 : audit [INF] from='client.? 172.21.15.8:0/260553007' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "720b47df-d743-46cf-ad04-36e113c8fcb3"}]': finished 2023-10-09T05:17:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[101053]: cluster 2023-10-09T05:17:35.755150+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:17:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[101053]: audit 2023-10-09T05:17:35.755269+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:36.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[101053]: audit 2023-10-09T05:17:36.400314+0000 mon.a (mon.0) 296 : audit [DBG] from='client.? 172.21.15.8:0/113640391' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:17:36.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[110782]: cluster 2023-10-09T05:17:35.291378+0000 mgr.y (mgr 2023-10-09T05:17:36.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[110782]: .14148) 82 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:36.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[110782]: audit 2023-10-09T05:17:35.751059+0000 mon.a (mon.0) 292 : audit [INF] from='client.? 172.21.15.8:0/260553007' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "720b47df-d743-46cf-ad04-36e113c8fcb3"}]: dispatch 2023-10-09T05:17:36.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[110782]: audit 2023-10-09T05:17:35.755082+0000 mon.a (mon.0) 293 : audit [INF] from='client.? 172.21.15.8:0/260553007' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "720b47df-d743-46cf-ad04-36e113c8fcb3"}]': finished 2023-10-09T05:17:36.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[110782]: cluster 2023-10-09T05:17:35.755150+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:17:36.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[110782]: audit 2023-10-09T05:17:35.755269+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:36.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:36 smithi008 conmon[110782]: audit 2023-10-09T05:17:36.400314+0000 mon.a (mon.0) 296 : audit [DBG] from='client.? 172.21.15.8:0/113640391' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:17:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:38 smithi008 conmon[101053]: cluster 2023-10-09T05:17:37.291675+0000 mgr.y (mgr.14148) 83 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:38 smithi008 conmon[110782]: cluster 2023-10-09T05:17:37.291675+0000 mgr.y (mgr.14148) 83 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:39.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:38 smithi191 conmon[101299]: cluster 2023-10-09T05:17:37.291675+0000 mgr.y (mgr.14148) 83 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:40 smithi008 conmon[101053]: cluster 2023-10-09T05:17:39.291989 2023-10-09T05:17:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:40 smithi008 conmon[101053]: +0000 mgr.y (mgr.14148) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:40.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:40 smithi008 conmon[110782]: cluster 2023-10-09T05:17:39.291989+0000 mgr.y ( 2023-10-09T05:17:40.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:40 smithi008 conmon[110782]: mgr.14148) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:41.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:40 smithi191 conmon[101299]: cluster 2023-10-09T05:17:39.291989+0000 mgr.y (mgr.14148) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:42.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:42 smithi008 conmon[101053]: cluster 2023-10-09T05:17:41.292296+0000 mgr.y (mgr.14148) 85 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:42.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:42 smithi008 conmon[101053]: audit 2023-10-09T05:17:41.877981+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:17:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:42 smithi008 conmon[101053]: audit 2023-10-09T05:17:41.878714+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:42.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:42 smithi008 conmon[110782]: cluster 2023-10-09T05:17:41.292296+0000 mgr.y (mgr.14148) 85 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:42.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:42 smithi008 conmon[110782]: audit 2023-10-09T05:17:41.877981+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:17:42.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:42 smithi008 conmon[110782]: audit 2023-10-09T05:17:41.878714+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:43.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:42 smithi191 conmon[101299]: cluster 2023-10-09T05:17:41.292296+0000 mgr.y (mgr.14148) 85 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:43.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:42 smithi191 conmon[101299]: audit 2023-10-09T05:17:41.877981+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:17:43.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:42 smithi191 conmon[101299]: audit 2023-10-09T05:17:43.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:42 smithi191 conmon[101299]: 2023-10-09T05:17:41.878714+0000 mon.a (mon.0) 298 : 2023-10-09T05:17:43.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:42 smithi191 conmon[101299]: audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:44.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:43 smithi191 conmon[101299]: cephadm 2023-10-09T05:17:41.879322+0000 mgr.y (mgr.14148) 86 : cephadm [INF] Deploying daemon osd.2 on smithi008 2023-10-09T05:17:44.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:43 smithi008 conmon[101053]: cephadm 2023-10-09T05:17:41.879322+0000 mgr.y ( 2023-10-09T05:17:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:43 smithi008 conmon[101053]: mgr.14148) 86 : cephadm [INF] Deploying daemon osd.2 on smithi008 2023-10-09T05:17:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:43 smithi008 conmon[110782]: cephadm 2023-10-09T05:17 2023-10-09T05:17:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:43 smithi008 conmon[110782]: :41.879322+0000 mgr.y (mgr.14148) 86 : cephadm [INF] Deploying daemon osd.2 on smithi008 2023-10-09T05:17:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:44 smithi008 conmon[101053]: cluster 2023-10-09T05:17:43 2023-10-09T05:17:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:44 smithi008 conmon[101053]: .292711+0000 mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:44 smithi008 conmon[110782]: cluster 2023-10-09T05 2023-10-09T05:17:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:44 smithi008 conmon[110782]: :17:43.292711+0000 mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:45.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:44 smithi191 conmon[101299]: cluster 2023-10-09T05:17:43.292711+0000 mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:47.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:46 smithi191 conmon[101299]: cluster 2023-10-09T05:17:45.293173+0000 mgr.y (mgr.14148 2023-10-09T05:17:47.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:46 smithi191 conmon[101299]: ) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:47.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:46 smithi008 conmon[101053]: cluster 2023-10-09T05:17:45 2023-10-09T05:17:47.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:46 smithi008 conmon[101053]: .293173+0000 mgr.y (mgr.14148) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:46 smithi008 conmon[110782]: cluster 2023-10-09T05:17:45.293173+0000 mgr.y 2023-10-09T05:17:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:46 smithi008 conmon[110782]: (mgr.14148) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:47.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:47 smithi008 conmon[101053]: audit 2023-10-09T05:17:46.826149+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:47.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:47 smithi008 conmon[101053]: audit 2023-10-09T05:17:46.829949+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:47.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:47 smithi008 conmon[110782]: audit 2023-10-09T05:17:46.826149+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:47.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:47 smithi008 conmon[110782]: audit 2023-10-09T05:17:46.829949+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:48.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:47 smithi191 conmon[101299]: audit 2023-10-09T05:17:46.826149+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:47 smithi191 conmon[101299]: audit 2023-10-09T05:17:46.829949+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:48.788 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:48 smithi008 conmon[101053]: cluster 2023-10-09T05: 2023-10-09T05:17:48.789 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:48 smithi008 conmon[101053]: 17:47.293450+0000 mgr.y (mgr.14148) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:48.789 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:48 smithi008 conmon[110782]: cluster 2023-10-09T05:17:47 2023-10-09T05:17:48.789 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:48 smithi008 conmon[110782]: .293450+0000 mgr.y (mgr.14148) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:49.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:48 smithi191 conmon[101299]: cluster 2023-10-09T05:17:47.293450+0000 mgr.y (mgr.14148) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:49.913 INFO:teuthology.orchestra.run.smithi008.stdout:Created osd(s) 2 on host 'smithi008' 2023-10-09T05:17:50.119 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[101053]: audit 2023-10-09T05:17:48. 2023-10-09T05:17:50.119 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[101053]: 865943+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:50.119 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[101053]: audit 2023-10-09T05:17:48.866708+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:50.120 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[101053]: audit 2023-10-09T05:17:48.867381+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:50.120 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[101053]: audit 2023-10-09T05:17:48.872260+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:50.120 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[101053]: cluster 2023-10-09T05:17:49.293867+0000 mgr.y (mgr.14148) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:50.120 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[110782]: audit 2023-10-09T05:17:48.865943+0000 2023-10-09T05:17:50.120 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[110782]: mon.a (mon.0) 301 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:50.120 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[110782]: audit 2023-10-09T05:17:48.866708+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:50.120 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[110782]: audit 2023-10-09T05:17:48.867381+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:50.120 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[110782]: audit 2023-10-09T05:17:48.872260+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:50.120 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:49 smithi008 conmon[110782]: cluster 2023-10-09T05:17:49.293867+0000 mgr.y (mgr.14148) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: audit 2023-10-09T05:17:48.865943+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: audit 2023-10-09T05:17:48.866708+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: audit 2023-10-09T05:17: 2023-10-09T05:17:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: 48.867381+0000 mon.a (mon.0) 303 : audit 2023-10-09T05:17:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:17:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: 05:17:48.872260+0000 mon.a (mon 2023-10-09T05:17:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: .0) 304 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:50.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: cluster 2023 2023-10-09T05:17:50.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: -10-09T05:17:49.293867 2023-10-09T05:17:50.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: +0000 mgr.y (mgr.14148) 90 : cluster [DBG] 2023-10-09T05:17:50.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:49 smithi191 conmon[101299]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:50.530 DEBUG:teuthology.orchestra.run.smithi008:osd.2> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T05:17:50.532 INFO:tasks.cephadm:Deploying osd.3 on smithi008 with /dev/vg_nvme/lv_1... 2023-10-09T05:17:50.532 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T05:17:50.566 INFO:journalctl@ceph.osd.2.smithi008.stdout:-- Logs begin at Mon 2023-10-09 04:57:44 UTC. -- 2023-10-09T05:17:51.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:50 smithi008 conmon[101053]: audit 2023-10-09T05:17:49.912345+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:51.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:50 smithi008 conmon[101053]: audit 2023-10-09T05:17:49.940206+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:51.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:50 smithi008 conmon[101053]: audit 2023-10-09T05:17:50.007132+0000 mon.a (mon.0) 307 : audit [INF] from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:17:51.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:50 smithi008 conmon[110782]: audit 2023-10-09T05:17:49.912345+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:51.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:50 smithi008 conmon[110782]: audit 2023-10-09T05:17:49.940206+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:51.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:50 smithi008 conmon[110782]: audit 2023-10-09T05:17:50.007132+0000 mon.a (mon.0) 307 : audit [INF] from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:17:51.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:50 smithi191 conmon[101299]: audit 2023-10-09T05:17:49.912345+0000 mon.a (mon.0) 2023-10-09T05:17:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:50 smithi191 conmon[101299]: 305 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:50 smithi191 conmon[101299]: audit 2023-10-09T05:17:49.940206+0000 mon.a (mon.0) 306 : audit [DBG] 2023-10-09T05:17:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:50 smithi191 conmon[101299]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:17:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:50 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:17:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:50 smithi191 conmon[101299]: 09T05:17:50.007132+0000 mon.a (mon.0) 307 : 2023-10-09T05:17:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:50 smithi191 conmon[101299]: audit [INF] from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:17:52.214 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[127626]: debug 2023-10-09T05:17:51.921+0000 7fb649b67700 -1 osd.2 0 waiting for initial osdmap 2023-10-09T05:17:52.214 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[127626]: debug 2023-10-09T05:17:51.925+0000 7fb6422db700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:17:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[101053]: audit 2023-10-09T05:17:50.917086 2023-10-09T05:17:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[101053]: +0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:17:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[101053]: cluster 2023-10-09T05:17:50.917147+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:17:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[101053]: audit 2023-10-09T05:17:50.917313+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[101053]: audit 2023-10-09T05:17:50.917729+0000 mon.a (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[101053]: cluster 2023-10-09T05:17:51.294393 2023-10-09T05:17:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[101053]: +0000 mgr.y (mgr.14148) 91 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:52.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: audit 2023-10-09T05:17:50. 2023-10-09T05:17:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: 917086+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:17:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: cluster 2023-10-09T05:17:50.917147+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:17:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: audit 2023-10-09T05:17:50.917313+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: 2023-10-09T05:17:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: audit 2023-10-09T05:17:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: 2023-10-09T05:17:50.917729 2023-10-09T05:17:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: +0000 mon.a (mon.0) 311 : audit [INF] 2023-10-09T05:17:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: cluster 2023-10-09T05:17: 2023-10-09T05:17:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: 51.294393+0000 mgr.y (mgr.14148) 91 2023-10-09T05:17:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:51 smithi008 conmon[110782]: : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:52.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: audit 2023-10-09T05:17:50.917086+0000 mon.a (mon. 2023-10-09T05:17:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: 0) 308 : audit [INF] from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:17:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: cluster 2023-10-09T05:17:50.917147+0000 mon.a 2023-10-09T05:17:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: (mon.0) 309 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:17:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: audit 2023-10-09T05:17 2023-10-09T05:17:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: :50.917313+0000 mon.a (mon.0 2023-10-09T05:17:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: ) 310 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: audit 2023 2023-10-09T05:17:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: -10-09T05:17:50. 2023-10-09T05:17:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: 917729+0000 mon.a (mon.0) 311 : audit [INF] 2023-10-09T05:17:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:17:52.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: cluster 2023-10-09T05 2023-10-09T05:17:52.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: :17:51.294393+0000 mgr.y (mgr.14148 2023-10-09T05:17:52.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:51 smithi191 conmon[101299]: ) 91 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:53.398 INFO:teuthology.orchestra.run.smithi008.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T05:17:53.398 INFO:teuthology.orchestra.run.smithi008.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T05:17:53.398 INFO:teuthology.orchestra.run.smithi008.stderr: stderr: 10+0 records in 2023-10-09T05:17:53.398 INFO:teuthology.orchestra.run.smithi008.stderr:10+0 records out 2023-10-09T05:17:53.398 INFO:teuthology.orchestra.run.smithi008.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00941415 s, 1.1 GB/s 2023-10-09T05:17:53.398 INFO:teuthology.orchestra.run.smithi008.stderr:--> Zapping successful for: 2023-10-09T05:17:53.443 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:53 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:17:53.443 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:53 smithi008 conmon[101053]: 17:51.918532+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:17:53.443 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:53 smithi008 conmon[101053]: cluster 2023-10-09T05:17:51.918618+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:17:53.443 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:53 smithi008 conmon[101053]: audit 2023-10-09T05:17:51.919001+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:53.443 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:53 smithi008 conmon[101053]: audit 2023-10-09T05:17:51.923240+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:53.443 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:53 smithi008 conmon[110782]: audit 2023-10-09T05:17:51.918532 2023-10-09T05:17:53.443 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:53 smithi008 conmon[110782]: +0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:17:53.444 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:53 smithi008 conmon[110782]: cluster 2023-10-09T05:17:51.918618+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:17:53.444 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:53 smithi008 conmon[110782]: audit 2023-10-09T05:17:51.919001+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:53.444 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:53 smithi008 conmon[110782]: audit 2023-10-09T05:17:51.923240+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:53 smithi191 conmon[101299]: audit 2023-10-09T05:17:51.918532+0000 mon.a (mon.0) 312 : audit 2023-10-09T05:17:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:53 smithi191 conmon[101299]: [INF] from='osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:17:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:53 smithi191 conmon[101299]: cluster 2023-10-09T05:17:51.918618+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:17:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:53 smithi191 conmon[101299]: audit 2023-10-09T05:17:51.919001+0000 mon.a 2023-10-09T05:17:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:53 smithi191 conmon[101299]: (mon.0) 314 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:53.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:53 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:17:53.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:53 smithi191 conmon[101299]: 09T05:17:51.923240+0000 mon.a (mon 2023-10-09T05:17:53.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:53 smithi191 conmon[101299]: .0) 315 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:54.350 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi008:vg_nvme/lv_1 2023-10-09T05:17:54.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: cluster 2023-10-09T05:17:51.044897+0000 osd.2 (osd. 2023-10-09T05:17:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: 2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:17:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: cluster 2023-10-09T05:17:51.044952+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:17:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: cluster 2023-10-09T05:17:52.923996+0000 mon.a (mon.0) 316 : cluster [INF] osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415] boot 2023-10-09T05:17:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: cluster 2023-10-09T05:17:52.924028+0000 mon.a (mon.0) 317 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:17:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: audit 2023-10-09T05:17:52.924691+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: audit 2023-10-09T05:17:52.925045+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:17:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: cluster 2023-10-09T05:17:53.294645+0000 mgr.y (mgr.14148) 92 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: audit 2023-10-09T05:17:53.927425+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:17:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: cluster 2023-10-09T05:17:53.927518+0000 mon.a (mon.0) 321 : 2023-10-09T05:17:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:17:54.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:17:54.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[110782]: 17:53.928179+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:17:54.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: cluster 2023-10-09T05:17:51.044897+0000 2023-10-09T05:17:54.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:17:54.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: cluster 2023-10-09T05:17:51.044952+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:17:54.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: cluster 2023-10-09T05:17:52.923996+0000 mon.a (mon.0) 316 : cluster [INF] osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415] boot 2023-10-09T05:17:54.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: cluster 2023-10-09T05:17:52.924028+0000 mon.a (mon.0) 317 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:17:54.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: audit 2023-10-09T05:17:52.924691+0000 mon.a (mon.0) 318 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:54.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: audit 2023-10-09T05:17:52.925045+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:17:54.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: cluster 2023-10-09T05:17:53.294645+0000 mgr.y (mgr.14148) 92 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:54.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: audit 2023-10-09T05:17:53.927425+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:17:54.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: cluster 2023-10-09T05:17:53.927518+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:17:54.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:17:54.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: -09T05:17:53.928179+0000 mon.a 2023-10-09T05:17:54.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:54 smithi008 conmon[101053]: (mon.0) 322 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:17:54.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: cluster 2023-10-09T05:17:51.044897+0000 osd.2 (osd.2) 1 2023-10-09T05:17:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:17:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: cluster 2023-10-09T05:17:51.044952+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:17:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: cluster 2023-10-09T05:17:52.923996+0000 mon.a (mon.0) 316 : cluster 2023-10-09T05:17:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: [INF] osd.2 [v2:172.21.15.8:6818/3035534415,v1:172.21.15.8:6819/3035534415] boot 2023-10-09T05:17:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: cluster 2023-10-09T05:17: 2023-10-09T05:17:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: 52.924028+0000 mon.a (mon.0) 317 : cluster 2023-10-09T05:17:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:17:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: audit 2023-10-09T05:17:52. 2023-10-09T05:17:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: 924691+0000 mon.a (mon.0) 318 : audit [DBG] 2023-10-09T05:17:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:17:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: audit 2023-10-09T05:17:52.925045 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: +0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: cluster 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: 2023-10-09T05:17:53.294645+0000 mgr.y (mgr.14148 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: ) 92 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: 17:53.927425+0000 mon.a (mon.0) 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: 320 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: cluster 2023-10 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: -09T05:17:53.927518+0000 mon.a ( 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: mon.0) 321 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:17:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:17:54.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: 17:53.928179+0000 mon.a (mon.0) 322 : audit 2023-10-09T05:17:54.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:54 smithi191 conmon[101299]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:17:55.660 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: cephadm 2023-10-09T05:17:54.450882+0000 mgr.y (mgr.14148) 93 : cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:17:55.660 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: audit 2023-10-09T05:17:54.456751+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:55.660 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: audit 2023-10-09T05:17:54.457815+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:55.660 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: audit 2023-10-09T05:17:54.458576+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:55.660 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: audit 2023-10-09T05:17:54.459297+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:55.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: cephadm 2023-10-09T05:17:54.459903+0000 mgr.y (mgr.14148) 94 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 5383M 2023-10-09T05:17:55.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: audit 2023-10-09T05:17:54.465043+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:55.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: audit 2023-10-09T05:17:54.465969+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:55.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: audit 2023-10-09T05:17:54.466616+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:55.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: audit 2023-10-09T05:17:54.470798+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:55.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: audit 2023-10-09T05:17:54.929345+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:17:55.662 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[101053]: cluster 2023-10-09T05:17:54.929407+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:17:55.662 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: cephadm 2023-10-09T05:17:54.450882+0000 mgr.y (mgr.14148) 93 : cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:17:55.662 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: audit 2023-10-09T05:17:54.456751+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:55.662 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: audit 2023-10-09T05:17:54.457815+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:55.662 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: audit 2023-10-09T05:17:54.458576+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:55.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: audit 2023-10-09T05:17:54.459297+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:55.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: cephadm 2023-10-09T05:17:54.459903+0000 mgr.y (mgr.14148) 94 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 5383M 2023-10-09T05:17:55.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: audit 2023-10-09T05:17 2023-10-09T05:17:55.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: :54.465043+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:55.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: audit 2023-10-09T05:17:54.465969+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:55.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: audit 2023-10-09T05:17:54.466616+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:55.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: audit 2023-10-09T05:17:54.470798+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:55.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: audit 2023-10-09T05:17:54.929345+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:17:55.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:55 smithi008 conmon[110782]: cluster 2023-10-09T05:17:54.929407+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:17:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: cephadm 2023-10-09T05:17:54.450882+0000 mgr.y (mgr.14148) 93 : 2023-10-09T05:17:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:17:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: audit 2023-10-09T05:17:54.456751+0000 mon.a (mon.0) 323 2023-10-09T05:17:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:17:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: 09T05:17:54.457815+0000 mon.a (mon.0) 324 : audit [INF] 2023-10-09T05:17:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: audit 2023-10-09T05:17:54.458576+0000 mon.a (mon.0) 325 : 2023-10-09T05:17:55.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:55.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: audit 2023-10-09T05:17:54.459297 2023-10-09T05:17:55.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: +0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:17:55.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: 2023-10-09T05:17:55.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: cephadm 2023-10-09T05:17:55.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: 2023-10-09T05:17:54.459903+0000 mgr.y (mgr.14148 2023-10-09T05:17:55.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: ) 94 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 5383M 2023-10-09T05:17:55.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:17:55.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: :17:54.465043+0000 mon.a (mon.0) 2023-10-09T05:17:55.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: 327 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:55.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:17:55.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: 05:17:54.465969+0000 mon.a (mon.0 2023-10-09T05:17:55.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: ) 328 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:55.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:17:55.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: 17:54.466616+0000 mon.a (mon.0) 329 2023-10-09T05:17:55.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:17:55.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:17:55.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: :17:54.470798+0000 mon.a (mon.0 2023-10-09T05:17:55.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: ) 330 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:17:55.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:17:55.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: T05:17:54.929345+0000 mon.a (mon 2023-10-09T05:17:55.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: .0) 331 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:17:55.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: cluster 2023-10-09 2023-10-09T05:17:55.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: T05:17:54.929407+0000 mon.a (mon. 2023-10-09T05:17:55.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:55 smithi191 conmon[101299]: 0) 332 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:17:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[101053]: cluster 2023-10-09T05:17:55.294906+0000 mgr.y (mgr.14148 2023-10-09T05:17:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[101053]: ) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:17:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[101053]: cluster 2023-10-09T05:17:55.932858+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:17:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[101053]: audit 2023-10-09T05:17:56.302553+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:17:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[101053]: audit 2023-10-09T05:17:56.305070+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:17:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[101053]: audit 2023-10-09T05:17:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[101053]: 2023-10-09T05:17:56.305688+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: cluster 2023-10-09T05:17:55.294906 2023-10-09T05:17:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: +0000 mgr.y (mgr.14148) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:17:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: cluster 2023-10-09T05:17:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: 2023-10-09T05:17:55.932858+0000 mon.a (mon.0) 333 : cluster [DBG] 2023-10-09T05:17:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:17:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:17:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: 05:17:56.302553+0000 mon.a (mon.0 2023-10-09T05:17:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: ) 334 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:17:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: audit 2023-10-09T05:17:56. 2023-10-09T05:17:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: 305070+0000 mon.a (mon.0) 335 : audit [INF] 2023-10-09T05:17:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:17:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: audit 2023-10-09T05:17:56.305688 2023-10-09T05:17:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: +0000 mon.a (mon.0) 336 : audit 2023-10-09T05:17:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:56 smithi008 conmon[110782]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: cluster 2023-10-09T05:17:55.294906+0000 mgr.y (mgr.14148) 95 : cluster 2023-10-09T05:17:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: [DBG] pgmap v73: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:17:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: cluster 2023-10-09T05:17:55.932858+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:17:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: audit 2023-10-09T05:17:56.302553+0000 mon.a 2023-10-09T05:17:56.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: (mon.0) 334 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:17:56.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: audit 2023 2023-10-09T05:17:56.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: -10-09T05:17:56.305070+0000 2023-10-09T05:17:56.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: mon.a (mon.0) 335 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:17:56.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: 2023-10-09T05:17:56.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: audit 2023-10-09T05:17:56.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: 2023-10-09T05:17:56.305688 2023-10-09T05:17:56.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: +0000 mon.a (mon.0) 336 : audit [DBG] 2023-10-09T05:17:56.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:56 smithi191 conmon[101299]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:17:57.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:57 smithi008 conmon[101053]: audit 2023-10-09T05:17:56.299908+0000 mgr.y ( 2023-10-09T05:17:57.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:57 smithi008 conmon[101053]: mgr.14148) 96 : audit [DBG] from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:17:57.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:57 smithi008 conmon[110782]: audit 2023-10-09T05:17:56.299908+0000 mgr.y 2023-10-09T05:17:57.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:57 smithi008 conmon[110782]: (mgr.14148) 96 : audit [DBG] from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:17:58.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:57 smithi191 conmon[101299]: audit 2023-10-09T05:17:56.299908+0000 mgr.y (mgr.14148) 96 : audit [DBG] from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi008:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:17:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: cluster 2023-10-09T05:17:57.295228+0000 mgr.y (mgr.14148) 97 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:17:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: audit 2023-10-09T05:17:58.636777+0000 mon.c (mon.2) 9 : audit [INF] from='client.? 172.21.15.8:0/3072837055' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "861bf05d-69f8-4000-999e-ab708531edb0"}]: dispatch 2023-10-09T05:17:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: audit 2023-10-09T05:17:58.637135+0000 mon.a (mon.0) 337 : 2023-10-09T05:17:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "861bf05d-69f8-4000-999e-ab708531edb0"}]: dispatch 2023-10-09T05:17:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:17:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: -09T05:17:58.641289+0000 mon.a (mon.0) 2023-10-09T05:17:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: 338 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "861bf05d-69f8-4000-999e-ab708531edb0"}]': finished 2023-10-09T05:17:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: cluster 2023- 2023-10-09T05:17:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: 10-09T05:17:58. 2023-10-09T05:17:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: 641372+0000 mon.a (mon.0) 339 : cluster 2023-10-09T05:17:59.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:17:59.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:17:59.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: -09T05:17:58.641500 2023-10-09T05:17:59.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: +0000 mon.a (mon.0) 340 : audit [DBG] 2023-10-09T05:17:59.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:58 smithi191 conmon[101299]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:17:59.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[101053]: cluster 2023-10-09T05:17:57.295228+0000 mgr.y (mgr.14148) 97 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:17:59.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[101053]: audit 2023-10-09T05:17:58.636777+0000 mon.c (mon.2) 9 : audit [INF] from='client.? 172.21.15.8:0/3072837055' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "861bf05d-69f8-4000-999e-ab708531edb0"}]: dispatch 2023-10-09T05:17:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[101053]: audit 2023-10-09T05:17:58.637135+0000 mon.a (mon.0) 337 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "861bf05d-69f8-4000-999e-ab708531edb0"}]: dispatch 2023-10-09T05:17:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[101053]: audit 2023-10-09T05:17:58.641289+0000 mon.a (mon.0) 338 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "861bf05d-69f8-4000-999e-ab708531edb0"}]': finished 2023-10-09T05:17:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[101053]: cluster 2023-10-09T05:17:58.641372+0000 mon.a (mon.0) 339 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:17:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[101053]: audit 2023-10-09T05:17:58.641500+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:17:59.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[110782]: cluster 2023-10-09T05:17:57.295228+0000 mgr.y (mgr.14148) 97 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:17:59.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[110782]: audit 2023-10-09T05:17:58.636777+0000 mon.c (mon.2) 9 : audit [INF] from='client.? 172.21.15.8:0/3072837055' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "861bf05d-69f8-4000-999e-ab708531edb0"}]: dispatch 2023-10-09T05:17:59.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[110782]: audit 2023-10-09T05:17:58.637135+0000 mon.a (mon.0) 337 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "861bf05d-69f8-4000-999e-ab708531edb0"}]: dispatch 2023-10-09T05:17:59.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[110782]: audit 2023-10-09T05:17:58.641289+0000 mon.a (mon.0) 338 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "861bf05d-69f8-4000-999e-ab708531edb0"}]': finished 2023-10-09T05:17:59.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[110782]: cluster 2023-10-09T05:17:58.641372+0000 mon.a (mon.0) 339 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:17:59.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:58 smithi008 conmon[110782]: audit 2023-10-09T05:17:58.641500+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:00.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:17:59 smithi191 conmon[101299]: audit 2023-10-09T05:17:59.242914+0000 mon.a (mon.0) 341 : audit [DBG] from='client.? 172.21.15.8:0/1368097236' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:18:00.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:59 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:18:00.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:17:59 smithi008 conmon[110782]: 17:59.242914+0000 mon.a (mon.0) 341 : audit [DBG] from='client.? 172.21.15.8:0/1368097236' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:18:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:59 smithi008 conmon[101053]: audit 2023-10-09T05:17:59. 2023-10-09T05:18:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:17:59 smithi008 conmon[101053]: 242914+0000 mon.a (mon.0) 341 : audit [DBG] from='client.? 172.21.15.8:0/1368097236' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:18:01.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:00 smithi191 conmon[101299]: cluster 2023-10-09T05:17:59.295720+0000 mgr.y (mgr. 2023-10-09T05:18:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:00 smithi191 conmon[101299]: 14148) 98 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:01.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:00 smithi008 conmon[101053]: cluster 2023-10-09T05:18:01.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:00 smithi008 conmon[101053]: 2023-10-09T05:17:59.295720+0000 mgr.y (mgr.14148) 98 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:00 smithi008 conmon[110782]: cluster 2023-10-09T05:17:59 2023-10-09T05:18:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:00 smithi008 conmon[110782]: .295720+0000 mgr.y (mgr.14148) 98 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:03.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:02 smithi191 conmon[101299]: cluster 2023-10-09T05:18:01.296255+0000 mgr.y (mgr.14148) 99 2023-10-09T05:18:03.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:02 smithi191 conmon[101299]: : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:03.097 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:02 smithi008 conmon[101053]: cluster 2023-10-09T05:18:01.296255+0000 mgr.y ( 2023-10-09T05:18:03.097 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:02 smithi008 conmon[101053]: mgr.14148) 99 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:03.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:02 smithi008 conmon[110782]: cluster 2023-10-09T05:18:01.296255+0000 2023-10-09T05:18:03.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:02 smithi008 conmon[110782]: mgr.y (mgr.14148) 99 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:05.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:04 smithi191 conmon[101299]: cluster 2023-10-09T05:18:03.296573+0000 mgr.y (mgr.14148) 100 2023-10-09T05:18:05.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:04 smithi191 conmon[101299]: : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:05.075 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:04 smithi008 conmon[101053]: cluster 2023-10-09T05:18:03.296573+0000 mgr.y (mgr.14148) 100 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:05.076 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:04 smithi008 conmon[110782]: cluster 2023-10-09T05:18:03.296573+0000 mgr.y (mgr.14148) 100 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:05 smithi008 conmon[101053]: audit 2023-10-09T05:18:05.095476+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:18:05.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:05 smithi008 conmon[101053]: audit 2023-10-09T05:18:05.096236+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:05.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:05 smithi008 conmon[110782]: audit 2023-10-09T05:18:05.095476+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:18:05.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:05 smithi008 conmon[110782]: audit 2023-10-09T05:18:05.096236+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:05 smithi191 conmon[101299]: audit 2023-10-09T05:18:05.095476+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:18:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:05 smithi191 conmon[101299]: audit 2023-10-09T05:18:05.096236+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:06.782 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:06 smithi008 conmon[101053]: cephadm 2023-10-09T05:18:05.096872+0000 2023-10-09T05:18:06.782 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:06 smithi008 conmon[101053]: mgr.y (mgr.14148) 101 : cephadm [INF] Deploying daemon osd.3 on smithi008 2023-10-09T05:18:06.782 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:06 smithi008 conmon[101053]: cluster 2023-10-09T05:18:05.296856+0000 mgr.y (mgr.14148) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:06.783 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:06 smithi008 conmon[110782]: cephadm 2023-10-09T05:18:05.096872+0000 mgr.y (mgr 2023-10-09T05:18:06.783 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:06 smithi008 conmon[110782]: .14148) 101 : cephadm [INF] Deploying daemon osd.3 on smithi008 2023-10-09T05:18:06.783 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:06 smithi008 conmon[110782]: cluster 2023-10- 2023-10-09T05:18:06.783 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:06 smithi008 conmon[110782]: 09T05:18:05.296856+0000 mgr.y (mgr 2023-10-09T05:18:06.783 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:06 smithi008 conmon[110782]: .14148) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:07.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:06 smithi191 conmon[101299]: cephadm 2023-10-09T05:18:05.096872+0000 mgr.y (mgr.14148) 101 : cephadm [INF] Deploying daemon osd.3 on smithi008 2023-10-09T05:18:07.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:06 smithi191 conmon[101299]: cluster 2023-10-09T05:18:05.296856+0000 mgr.y (mgr.14148) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:08.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:08 smithi008 conmon[101053]: cluster 2023-10-09T05:18: 2023-10-09T05:18:08.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:08 smithi008 conmon[101053]: 07.297187+0000 mgr.y (mgr.14148) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:08.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:08 smithi008 conmon[110782]: cluster 2023-10-09T05:18:07.297187+0000 mgr.y (mgr.14148) 103 : 2023-10-09T05:18:08.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:08 smithi008 conmon[110782]: cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:09.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:08 smithi191 conmon[101299]: cluster 2023-10-09T05:18:07.297187+0000 mgr.y (mgr.14148) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:10.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[101053]: cluster 2023-10-09T05:18:09.297467+0000 mgr.y (mgr.14148 2023-10-09T05:18:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[101053]: ) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[101053]: audit 2023-10-09T05:18:10.227260+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[101053]: audit 2023-10-09T05:18:10.267664+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[110782]: cluster 2023-10-09T05:18:09 2023-10-09T05:18:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[110782]: .297467+0000 mgr.y (mgr.14148) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[110782]: audit 2023-10-09T05:18:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[110782]: 2023-10-09T05:18:10.227260 2023-10-09T05:18:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[110782]: +0000 mon.a (mon.0) 344 : audit [INF] 2023-10-09T05:18:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:10.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:18:10.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[110782]: 09T05:18:10.267664+0000 mon.a (mon.0) 2023-10-09T05:18:10.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:10 smithi008 conmon[110782]: 345 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:11.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:10 smithi191 conmon[101299]: cluster 2023-10-09T05:18:09.297467+0000 mgr.y (mgr.14148) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:10 smithi191 conmon[101299]: audit 2023-10-09T05:18:10.227260+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:10 smithi191 conmon[101299]: audit 2023-10-09T05:18:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:10 smithi191 conmon[101299]: 2023-10-09T05:18:10.267664+0000 mon.a (mon.0) 345 : 2023-10-09T05:18:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:10 smithi191 conmon[101299]: audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:13.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:12 smithi191 conmon[101299]: cluster 2023-10-09T05:18:11.297901+0000 mgr.y (mgr.14148) 105 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:12 smithi191 conmon[101299]: audit 2023-10-09T05:18:12.504862 2023-10-09T05:18:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:12 smithi191 conmon[101299]: +0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:13.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:12 smithi008 conmon[101053]: cluster 2023-10-09T05:18:11.297901+0000 mgr.y (mgr.14148) 105 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:12 smithi008 conmon[101053]: audit 2023-10-09T05:18:12.504862+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:12 smithi008 conmon[110782]: cluster 2023-10-09T05:18:11.297901+0000 mgr.y (mgr.14148) 105 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:12 smithi008 conmon[110782]: audit 2023-10-09T05:18:12.504862+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:13.368 INFO:teuthology.orchestra.run.smithi008.stdout:Created osd(s) 3 on host 'smithi008' 2023-10-09T05:18:13.959 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[101053]: audit 2023-10-09T05:18:12.829212 2023-10-09T05:18:13.960 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[101053]: +0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:13.960 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[101053]: audit 2023-10-09T05:18:12.830058+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:13.960 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[101053]: audit 2023-10-09T05:18:12.834748+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:13.960 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[101053]: audit 2023-10-09T05:18:13.368137+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:13.960 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[101053]: audit 2023-10-09T05:18:13.401823+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:13.960 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[101053]: audit 2023-10-09T05:18:13.408585+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.8:6826/626394183,v1:172.21.15.8:6827/626394183]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:18:13.960 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[101053]: audit 2023-10-09T05:18:13.408983+0000 mon.a (mon.0) 352 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:18:13.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: audit 2023-10-09T05:18:12.829212+0000 mon.a (mon.0) 2023-10-09T05:18:13.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: 347 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:13.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:18:13.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: 05:18:12.830058+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:13.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:18:13.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: 10-09T05:18:12.834748+0000 mon.a (mon.0) 349 : audit 2023-10-09T05:18:13.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:13.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: audit 2023-10-09T05:18:13. 2023-10-09T05:18:13.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: 368137+0000 mon.a (mon.0) 350 : audit [INF] 2023-10-09T05:18:13.962 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:13.962 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: audit 2023-10-09T05:18:13.962 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: 2023-10-09T05:18:13.401823+0000 mon.a 2023-10-09T05:18:13.962 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: (mon.0) 351 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:13.962 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:18:13.962 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: T05:18:13.408585+0000 mon.c (mon.2 2023-10-09T05:18:13.962 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: ) 10 : audit [INF] from='osd.3 [v2:172.21.15.8:6826/626394183,v1:172.21.15.8:6827/626394183]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:18:13.963 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:18:13.963 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: :18:13.408983+0000 mon.a (mon.0) 352 2023-10-09T05:18:13.963 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:13 smithi008 conmon[110782]: : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:18:13.980 DEBUG:teuthology.orchestra.run.smithi008:osd.3> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T05:18:13.982 INFO:tasks.cephadm:Deploying osd.4 on smithi191 with /dev/vg_nvme/lv_4... 2023-10-09T05:18:13.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T05:18:14.013 INFO:journalctl@ceph.osd.3.smithi008.stdout:-- Logs begin at Mon 2023-10-09 04:57:44 UTC. -- 2023-10-09T05:18:14.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: audit 2023-10-09T05:18:12.829212+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:14.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: audit 2023-10-09T05:18:12.830058+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:14.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: audit 2023-10-09T05:18:12. 2023-10-09T05:18:14.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: 834748+0000 mon.a (mon.0) 349 : audit [INF] 2023-10-09T05:18:14.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:14.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:18:14.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: :18:13.368137+0000 mon.a (mon.0 2023-10-09T05:18:14.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: ) 350 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:14.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: audit 2023 2023-10-09T05:18:14.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: -10-09T05:18:13. 2023-10-09T05:18:14.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: 401823+0000 mon.a (mon.0) 351 : audit [DBG] 2023-10-09T05:18:14.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:14.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:18:14.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: 05:18:13.408585+0000 mon.c (mon 2023-10-09T05:18:14.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: .2) 10 : audit [INF] from='osd.3 [v2:172.21.15.8:6826/626394183,v1:172.21.15.8:6827/626394183]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:18:14.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: audit 2023 2023-10-09T05:18:14.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: -10-09T05:18:13.408983 2023-10-09T05:18:14.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: +0000 mon.a (mon.0) 352 : audit [INF] 2023-10-09T05:18:14.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:13 smithi191 conmon[101299]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:18:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[101053]: cluster 2023-10-09T05:18:13.298276+0000 mgr.y (mgr.14148) 106 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[101053]: audit 2023-10-09T05:18:14.369919+0000 mon.a (mon.0) 353 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:18:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[101053]: cluster 2023-10-09T05:18:14.370015+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:18:14.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[101053]: audit 2023-10-09T05:18:14.370189+0000 mon.a (mon 2023-10-09T05:18:14.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[101053]: .0) 355 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:14.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[101053]: audit 2023-10-09T05:18:14.370587+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.8:6826/626394183,v1:172.21.15.8:6827/626394183]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:18:14.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[101053]: audit 2023-10-09T05:18:14.370921+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:18:14.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[110782]: cluster 2023-10-09T05:18:13.298276+0000 mgr.y (mgr.14148) 106 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:14.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[110782]: audit 2023-10-09T05:18:14.369919+0000 mon.a (mon.0) 353 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:18:14.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[110782]: cluster 2023-10-09T05:18:14.370015+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:18:14.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[110782]: audit 2023-10-09T05:18:14.370189+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:14.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[110782]: audit 2023-10-09T05:18:14.370587+0000 mon.c (mon.2) 11 : audit 2023-10-09T05:18:14.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[110782]: [INF] from='osd.3 [v2:172.21.15.8:6826/626394183,v1:172.21.15.8:6827/626394183]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:18:14.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:14 smithi008 conmon[110782]: audit 2023-10-09T05:18:14.370921+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:18:15.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:14 smithi191 conmon[101299]: cluster 2023-10-09T05:18:13 2023-10-09T05:18:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:14 smithi191 conmon[101299]: .298276+0000 mgr.y (mgr.14148) 106 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:14 smithi191 conmon[101299]: audit 2023-10-09T05:18:14.369919+0000 mon.a (mon.0) 353 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:18:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:14 smithi191 conmon[101299]: cluster 2023-10-09T05:18:14.370015+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:18:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:14 smithi191 conmon[101299]: audit 2023-10-09T05:18:14.370189+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:14 smithi191 conmon[101299]: audit 2023-10-09T05:18:14.370587+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.8:6826/626394183,v1:172.21.15.8:6827/626394183]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:18:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:14 smithi191 conmon[101299]: audit 2023-10-09T05:18:14.370921+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:18:15.715 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:18:15 smithi008 conmon[133577]: debug 2023-10-09T05:18:15.373+0000 7fd584010700 -1 osd.3 0 waiting for initial osdmap 2023-10-09T05:18:15.715 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:18:15 smithi008 conmon[133577]: debug 2023-10-09T05:18:15.378+0000 7fd58078c700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:18:15.994 INFO:teuthology.orchestra.run.smithi191.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T05:18:15.994 INFO:teuthology.orchestra.run.smithi191.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T05:18:15.994 INFO:teuthology.orchestra.run.smithi191.stderr: stderr: 10+0 records in 2023-10-09T05:18:15.994 INFO:teuthology.orchestra.run.smithi191.stderr:10+0 records out 2023-10-09T05:18:15.994 INFO:teuthology.orchestra.run.smithi191.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0154671 s, 678 MB/s 2023-10-09T05:18:15.994 INFO:teuthology.orchestra.run.smithi191.stderr:--> Zapping successful for: 2023-10-09T05:18:16.584 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi191:vg_nvme/lv_4 2023-10-09T05:18:16.650 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:16 smithi191 conmon[101299]: cluster 2023-10-09T05:18:15.298534+0000 mgr.y (mgr.14148) 107 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:16.650 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:16 smithi191 conmon[101299]: audit 2023-10-09T05:18:15.371089+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:18:16.650 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:16 smithi191 conmon[101299]: cluster 2023-10-09T05:18:15.371188+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:18:16.651 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:16 smithi191 conmon[101299]: audit 2023-10-09T05:18:15.371295+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:16.651 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:16 smithi191 conmon[101299]: audit 2023-10-09T05:18:15.374514+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:16.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:16 smithi008 conmon[101053]: cluster 2023-10-09T05:18:15.298534+0000 mgr.y (mgr.14148) 107 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:16.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:16 smithi008 conmon[101053]: audit 2023-10-09T05:18:15.371089+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:18:16.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:16 smithi008 conmon[101053]: cluster 2023-10-09T05:18:15.371188+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:18:16.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:16 smithi008 conmon[101053]: audit 2023-10-09T05:18:15.371295+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:16.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:16 smithi008 conmon[101053]: audit 2023-10-09T05:18:15.374514+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:16.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:16 smithi008 conmon[110782]: cluster 2023-10-09T05:18:15.298534+0000 mgr.y (mgr.14148) 107 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:18:16.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:16 smithi008 conmon[110782]: audit 2023-10-09T05:18:15.371089+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]': finished 2023-10-09T05:18:16.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:16 smithi008 conmon[110782]: cluster 2023-10-09T05:18:15.371188+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:18:16.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:16 smithi008 conmon[110782]: audit 2023-10-09T05:18:15.371295+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:16.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:16 smithi008 conmon[110782]: audit 2023-10-09T05:18:15.374514+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: cluster 2023-10-09T05:18:14.395309+0000 osd.3 (osd.3) 1 : cluster 2023-10-09T05:18:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: [DBG] purged_snaps scrub starts 2023-10-09T05:18:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: cluster 2023-10-09T05:18:14.395351+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:18:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: cluster 2023-10-09T05:18:16.372373+0000 mon.a (mon.0) 361 : cluster [INF] osd.3 [v2:172.21.15.8:6826/626394183,v1:172.21.15.8:6827/626394183] boot 2023-10-09T05:18:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: cluster 2023-10-09T05:18:16.372416+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:18:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: audit 2023-10-09T05:18:16.372760+0000 mon.a (mon.0 2023-10-09T05:18:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: ) 363 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: cephadm 2023-10-09T05:18:16.757486+0000 mgr.y (mgr.14148) 108 : cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:18:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: audit 2023-10-09T05:18:16.762121+0000 mon.a 2023-10-09T05:18:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: (mon.0) 364 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: audit 2023-10-09T05:18:16 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: .763136+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: 18:16.763806+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: T05:18:16.764424+0000 mon.a (mon.0) 367 : audit 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: 10-09T05:18:16.765020+0000 mon.a (mon.0) 368 : audit [INF] 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: cephadm 2023- 2023-10-09T05:18:17.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: 10-09T05:18:16.765509+0000 mgr.y (mgr.14148) 109 : cephadm [INF] 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: Adjusting osd_memory_target on smithi008 to 4037M 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: 09T05:18:16.768657+0000 mon.a (mon.0) 369 : audit 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: audit 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: 2023-10-09T05:18:16.770238+0000 mon.a (mon.0) 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: 370 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: audit 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: 2023-10-09T05:18:16.770955+0000 mon.a (mon.0) 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: 371 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: audit 2023-10-09T05:18:16.774732+0000 2023-10-09T05:18:17.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[101053]: mon.a (mon.0) 372 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:17.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: cluster 2023-10-09T05:18:14.395309 2023-10-09T05:18:17.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: +0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:18:17.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: cluster 2023-10-09T05:18:14.395351+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:18:17.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: cluster 2023-10-09T05:18:16.372373+0000 mon.a (mon.0) 361 : cluster [INF] osd.3 [v2:172.21.15.8:6826/626394183,v1:172.21.15.8:6827/626394183] boot 2023-10-09T05:18:17.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: cluster 2023-10-09T05:18:16.372416+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:18:17.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: audit 2023-10-09T05:18 2023-10-09T05:18:17.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: :16.372760+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:17.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: cephadm 2023-10-09T05: 2023-10-09T05:18:17.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: 18:16.757486+0000 mgr.y (mgr.14148) 108 : cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:18:17.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: audit 2023-10-09T05:18:16 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: .762121+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: T05:18:16.763136+0000 mon.a (mon.0) 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: 365 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: audit 2023-10-09T05:18:16 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: .763806+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: :18:16.764424+0000 mon.a (mon.0) 367 : audit [INF] 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: 09T05:18:16.765020+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: cephadm 2023-10-09T05:18 2023-10-09T05:18:17.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: :16.765509+0000 mgr.y (mgr.14148) 109 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 4037M 2023-10-09T05:18:17.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:18:17.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: 05:18:16.768657+0000 mon.a (mon.0) 369 : audit [INF] 2023-10-09T05:18:17.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:17.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:18:17.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: 09T05:18:16.770238+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:17.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: audit 2023-10-09T05:18 2023-10-09T05:18:17.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: :16.770955+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:17.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:18:17.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: 18:16.774732+0000 mon.a (mon.0) 372 : audit [INF] 2023-10-09T05:18:17.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:17 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: cluster 2023-10-09T05:18:14.395309+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:18:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: cluster 2023-10-09T05:18:14.395351+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:18:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: cluster 2023-10-09T05:18:16.372373+0000 mon.a (mon.0) 361 : cluster [INF] osd.3 [v2:172.21.15.8:6826/626394183,v1:172.21.15.8:6827/626394183] boot 2023-10-09T05:18:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: cluster 2023-10-09T05:18:16.372416+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:18:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: audit 2023-10-09T05:18:16.372760+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:18:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: cephadm 2023-10-09T05:18:16.757486+0000 mgr.y (mgr.14148) 108 : cephadm [INF] Detected new or changed devices on smithi008 2023-10-09T05:18:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: audit 2023-10-09T05:18:16.762121+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:18:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: 05:18:16.763136+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: audit 2023-10-09T05:18:16.763806+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: audit 2023-10-09T05:18:16.764424+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: audit 2023-10-09T05:18:16.765020+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:17.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: cephadm 2023-10-09T05:18:16.765509+0000 mgr.y (mgr.14148) 109 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 4037M 2023-10-09T05:18:17.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: audit 2023-10-09T05:18:16.768657+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:17.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: audit 2023-10-09T05:18:16.770238+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:17.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: audit 2023-10-09T05:18:16.770955+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:17.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:17 smithi191 conmon[101299]: audit 2023-10-09T05:18:16.774732+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:18.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:18 smithi191 conmon[101299]: cluster 2023-10-09T05:18:17.298974+0000 mgr.y (mgr.14148) 110 : cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:18.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:18 smithi191 conmon[101299]: cluster 2023- 2023-10-09T05:18:18.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:18 smithi191 conmon[101299]: 10-09T05:18:17.377730+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:18:18.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:18 smithi008 conmon[101053]: cluster 2023-10-09T05:18:17.298974+0000 mgr.y (mgr.14148) 110 : 2023-10-09T05:18:18.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:18 smithi008 conmon[101053]: cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:18.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:18 smithi008 conmon[101053]: cluster 2023-10-09T05:18:17.377730+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:18:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:18 smithi008 conmon[110782]: cluster 2023-10-09T05:18:17.298974+0000 2023-10-09T05:18:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:18 smithi008 conmon[110782]: mgr.y (mgr.14148) 110 : cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:18 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:18:18.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:18 smithi008 conmon[110782]: -10-09T05:18:17.377730+0000 mon.a (mon 2023-10-09T05:18:18.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:18 smithi008 conmon[110782]: .0) 373 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:18:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[101053]: audit 2023-10-09T05:18:18.525285+0000 2023-10-09T05:18:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[101053]: mgr.y (mgr.14148) 111 : audit [DBG] from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:18:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[101053]: audit 2023-10-09T05:18:18.528465+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:18:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[101053]: audit 2023-10-09T05:18:18.531377+0000 mon.a 2023-10-09T05:18:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[101053]: (mon.0) 375 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:18:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:18:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[101053]: -09T05:18:18.532267+0000 mon.a (mon.0) 376 2023-10-09T05:18:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[101053]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: audit 2023-10-09T05:18:18. 2023-10-09T05:18:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: 525285+0000 mgr.y (mgr.14148) 111 : audit [DBG] from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:18:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: audit 2023-10-09T05:18:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: 2023-10-09T05:18:18.528465+0000 mon.a (mon.0) 374 : audit [DBG] 2023-10-09T05:18:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:18:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: audit 2023-10-09T05:18:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: 2023-10-09T05:18:18.531377 2023-10-09T05:18:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: +0000 mon.a (mon.0) 375 : audit [INF] 2023-10-09T05:18:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:18:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:18:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: 05:18:18.532267+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:19 smithi008 conmon[110782]: 2023-10-09T05:18:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:19 smithi191 conmon[101299]: audit 2023-10-09T05:18:18.525285+0000 mgr.y (mgr.14148) 111 : audit [DBG] from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:18:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:19 smithi191 conmon[101299]: audit 2023-10-09T05:18:18.528465+0000 mon.a (mon.0) 374 : audit [DBG] 2023-10-09T05:18:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:19 smithi191 conmon[101299]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:18:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:19 smithi191 conmon[101299]: audit 2023-10-09T05:18:18.531377+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:18:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:19 smithi191 conmon[101299]: audit 2023-10-09T05:18:18.532267+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:20.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:20 smithi191 conmon[101299]: cluster 2023-10-09T05:18:19.299408+0000 mgr.y (mgr.14148) 112 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:20.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:20 smithi008 conmon[110782]: cluster 2023-10-09T05:18:19.299408+0000 2023-10-09T05:18:20.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:20 smithi008 conmon[110782]: mgr.y (mgr.14148) 112 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:20.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:20 smithi008 conmon[101053]: cluster 2023-10-09T05:18:19 2023-10-09T05:18:20.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:20 smithi008 conmon[101053]: .299408+0000 mgr.y (mgr.14148) 112 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:21.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: audit 2023-10-09T05:18:20.882031+0000 mon.b (mon.1 2023-10-09T05:18:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: ) 7 : audit [INF] from='client.? 172.21.15.191:0/1190485150' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9"}]: dispatch 2023-10-09T05:18:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:18:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: 18:20.882657+0000 mon.a (mon.0) 377 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9"}]: dispatch 2023-10-09T05:18:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: audit 2023-10-09T05:18:20.888701+0000 2023-10-09T05:18:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: mon.a (mon.0) 378 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9"}]': finished 2023-10-09T05:18:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: cluster 2023-10-09T05 2023-10-09T05:18:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: :18:20.888828+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:18:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: audit 2023 2023-10-09T05:18:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: -10-09T05:18:20.889000 2023-10-09T05:18:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: +0000 mon.a (mon.0) 380 : 2023-10-09T05:18:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[110782]: audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:21.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: audit 2023-10-09T05:18:20.882031+0000 mon.b 2023-10-09T05:18:21.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: (mon.1) 7 : audit [INF] from='client.? 172.21.15.191:0/1190485150' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9"}]: dispatch 2023-10-09T05:18:21.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: audit 2023-10-09T05:18:20.882657+0000 mon.a (mon.0) 377 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9"}]: dispatch 2023-10-09T05:18:21.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: audit 2023-10-09T05:18:20 2023-10-09T05:18:21.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: .888701+0000 mon.a (mon.0) 378 : 2023-10-09T05:18:21.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9"}]': finished 2023-10-09T05:18:21.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: cluster 2023-10- 2023-10-09T05:18:21.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: 09T05:18:20.888828+0000 mon.a 2023-10-09T05:18:21.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: (mon.0) 379 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:18:21.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:18:21.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: 10-09T05:18:20.889000+0000 2023-10-09T05:18:21.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: mon.a (mon.0) 380 : audit [DBG] 2023-10-09T05:18:21.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:21 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:21 smithi191 conmon[101299]: audit 2023-10-09T05:18:20.882031+0000 mon.b (mon.1 2023-10-09T05:18:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:21 smithi191 conmon[101299]: ) 7 : audit [INF] from='client.? 172.21.15.191:0/1190485150' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9"}]: dispatch 2023-10-09T05:18:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:21 smithi191 conmon[101299]: audit 2023-10-09T05:18:20.882657+0000 mon.a (mon.0) 377 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9"}]: dispatch 2023-10-09T05:18:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:21 smithi191 conmon[101299]: audit 2023-10-09T05:18:20.888701+0000 mon.a (mon.0) 378 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9"}]': finished 2023-10-09T05:18:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:21 smithi191 conmon[101299]: cluster 2023-10-09T05:18:20.888828+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:18:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:21 smithi191 conmon[101299]: audit 2023-10-09T05:18:20.889000+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:22.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:22 smithi008 conmon[101053]: cluster 2023-10-09T05:18:21.299747+0000 2023-10-09T05:18:22.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:22 smithi008 conmon[101053]: mgr.y (mgr.14148) 113 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:22.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:22 smithi008 conmon[101053]: audit 2023-10-09T05:18: 2023-10-09T05:18:22.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:22 smithi008 conmon[101053]: 21.461286+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.191:0/2502770105' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:18:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:22 smithi008 conmon[110782]: cluster 2023-10-09T05:18:21 2023-10-09T05:18:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:22 smithi008 conmon[110782]: .299747+0000 mgr.y (mgr.14148) 113 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:22 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:18:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:22 smithi008 conmon[110782]: 18:21.461286+0000 mon.b (mon. 2023-10-09T05:18:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:22 smithi008 conmon[110782]: 1) 8 : audit [DBG] from='client.? 172.21.15.191:0/2502770105' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:18:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:22 smithi008 conmon[110782]: 2023-10-09T05:18:22.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:22 smithi191 conmon[101299]: cluster 2023-10-09T05:18:21.299747+0000 mgr.y (mgr.14148) 113 : cluster [DBG] 2023-10-09T05:18:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:22 smithi191 conmon[101299]: pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:22 smithi191 conmon[101299]: audit 2023-10-09T05:18:21.461286+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.191:0/2502770105' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:18:24.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:24 smithi008 conmon[101053]: cluster 2023-10-09T05:18:23.300221 2023-10-09T05:18:24.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:24 smithi008 conmon[101053]: +0000 mgr.y (mgr.14148) 114 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:24.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:24 smithi008 conmon[110782]: cluster 2023-10-09T05:18:23. 2023-10-09T05:18:24.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:24 smithi008 conmon[110782]: 300221+0000 mgr.y (mgr.14148) 114 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:24 smithi191 conmon[101299]: cluster 2023-10-09T05:18:23.300221+0000 mgr.y (mgr.14148) 114 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:26.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:26 smithi008 conmon[101053]: cluster 2023- 2023-10-09T05:18:26.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:26 smithi008 conmon[101053]: 10-09T05:18:25.300727+0000 mgr.y (mgr.14148) 115 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:26.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:26 smithi008 conmon[110782]: cluster 2023-10-09T05:18:25.300727 2023-10-09T05:18:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:26 smithi008 conmon[110782]: +0000 mgr.y (mgr.14148) 115 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:26.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:26 smithi191 conmon[101299]: cluster 2023-10-09T05:18:25.300727+0000 mgr.y (mgr.14148) 115 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:27 smithi191 conmon[101299]: audit 2023-10-09T05:18:27.176112+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:18:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:27 smithi191 conmon[101299]: audit 2023-10-09T05:18:27.177092+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:28.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:27 smithi008 conmon[101053]: audit 2023-10-09T05:18:27.176112+0000 mon.a (mon.0) 2023-10-09T05:18:28.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:27 smithi008 conmon[101053]: 381 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:18:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:27 smithi008 conmon[101053]: audit 2023-10-09T05:18:27.177092+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:27 smithi008 conmon[110782]: audit 2023-10-09T05:18:27.176112+0000 mon.a (mon.0) 381 : audit 2023-10-09T05:18:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:27 smithi008 conmon[110782]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:18:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:27 smithi008 conmon[110782]: audit 2023-10-09T05:18:27.177092+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:28.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:28 smithi191 conmon[101299]: cephadm 2023-10-09T05:18:27.177934+0000 mgr.y 2023-10-09T05:18:28.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:28 smithi191 conmon[101299]: (mgr.14148) 116 : cephadm [INF] Deploying daemon osd.4 on smithi191 2023-10-09T05:18:28.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:28 smithi191 conmon[101299]: cluster 2023-10-09T05:18:27.301310+0000 mgr.y (mgr.14148) 117 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:29.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:28 smithi008 conmon[101053]: cephadm 2023-10-09T05:18:27.177934+0000 mgr.y (mgr.14148) 2023-10-09T05:18:29.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:28 smithi008 conmon[101053]: 116 : cephadm [INF] Deploying daemon osd.4 on smithi191 2023-10-09T05:18:29.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:28 smithi008 conmon[101053]: cluster 2023-10-09T05:18:27.301310+0000 mgr.y (mgr.14148) 117 : cluster [DBG] 2023-10-09T05:18:29.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:28 smithi008 conmon[101053]: pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:29.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:28 smithi008 conmon[110782]: cephadm 2023-10-09T05:18:27 2023-10-09T05:18:29.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:28 smithi008 conmon[110782]: .177934+0000 mgr.y (mgr.14148) 116 : cephadm [INF] Deploying daemon osd.4 on smithi191 2023-10-09T05:18:29.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:28 smithi008 conmon[110782]: cluster 2023-10-09T05:18:27.301310+0000 mgr.y (mgr.14148) 117 : cluster [DBG] 2023-10-09T05:18:29.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:28 smithi008 conmon[110782]: pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:30 smithi191 conmon[101299]: cluster 2023-10-09T05:18:29.301726+0000 mgr.y (mgr.14148) 2023-10-09T05:18:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:30 smithi191 conmon[101299]: 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:31.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:30 smithi008 conmon[101053]: cluster 2023-10-09T05:18:29.301726+0000 mgr.y (mgr. 2023-10-09T05:18:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:30 smithi008 conmon[101053]: 14148) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:31.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:30 smithi008 conmon[110782]: cluster 2023-10-09T05:18:29.301726+0000 mgr.y ( 2023-10-09T05:18:31.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:30 smithi008 conmon[110782]: mgr.14148) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:33.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:32 smithi191 conmon[101299]: cluster 2023-10-09T05:18:31.302203+0000 mgr.y 2023-10-09T05:18:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:32 smithi191 conmon[101299]: (mgr.14148) 119 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:32 smithi191 conmon[101299]: audit 2023-10-09T05:18:32.301044+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:32 smithi191 conmon[101299]: audit 2023-10-09T05:18:32.330434+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:33.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[110782]: cluster 2023-10-09T05:18:31.302203+0000 mgr.y ( 2023-10-09T05:18:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[110782]: mgr.14148) 119 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[110782]: audit 2023-10-09T05:18:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[110782]: 2023-10-09T05:18:32.301044+0000 mon.a (mon.0) 383 : audit 2023-10-09T05:18:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[110782]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:18:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[110782]: -09T05:18:32.330434+0000 mon.a (mon.0) 2023-10-09T05:18:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[110782]: 384 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[101053]: cluster 2023-10-09T05:18: 2023-10-09T05:18:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[101053]: 31.302203+0000 mgr.y (mgr.14148) 119 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:18:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[101053]: -09T05:18:32.301044+0000 mon.a (mon.0) 383 : audit 2023-10-09T05:18:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[101053]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:18:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[101053]: 10-09T05:18:32.330434+0000 2023-10-09T05:18:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[101053]: mon.a (mon.0) 384 : audit [DBG] 2023-10-09T05:18:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:32 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:34 smithi191 conmon[101299]: cluster 2023-10-09T05:18:33.302779+0000 mgr.y (mgr.14148) 120 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:34 smithi191 conmon[101299]: audit 2023-10-09T05:18:33.972713+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:34 smithi191 conmon[101299]: audit 2023-10-09T05:18:34.291051+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:34 smithi191 conmon[101299]: audit 2023-10-09T05:18:34.292103+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:35.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:34 smithi191 conmon[101299]: audit 2023-10-09T05:18:34.297688+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:35.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: cluster 2023-10-09T05:18:33.302779+0000 mgr.y (mgr.14148) 120 2023-10-09T05:18:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: audit 2023-10-09T05:18:33.972713+0000 mon.a (mon 2023-10-09T05:18:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: .0) 385 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: audit 2023-10-09T05:18:34.291051+0000 mon.a (mon. 2023-10-09T05:18:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: 0) 386 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:18:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: -09T05:18:34.292103+0000 mon.a (mon.0) 387 2023-10-09T05:18:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:18:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: :18:34.297688+0000 mon.a (mon.0) 388 : audit 2023-10-09T05:18:35.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[110782]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:35.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: cluster 2023-10-09T05:18:33.302779+0000 mgr.y (mgr.14148) 120 2023-10-09T05:18:35.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:35.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: audit 2023-10-09T05:18:33.972713+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:35.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: audit 2023-10-09T05:18:34. 2023-10-09T05:18:35.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: 291051+0000 mon.a (mon.0) 386 : audit 2023-10-09T05:18:35.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:35.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: audit 2023 2023-10-09T05:18:35.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: -10-09T05:18:34 2023-10-09T05:18:35.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: .292103+0000 mon.a (mon.0) 387 2023-10-09T05:18:35.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:35.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:18:35.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: 10-09T05:18:34 2023-10-09T05:18:35.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: .297688+0000 mon.a (mon.0) 388 : 2023-10-09T05:18:35.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:34 smithi008 conmon[101053]: audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:35.309 INFO:teuthology.orchestra.run.smithi191.stdout:Created osd(s) 4 on host 'smithi191' 2023-10-09T05:18:35.930 DEBUG:teuthology.orchestra.run.smithi191:osd.4> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T05:18:35.987 INFO:tasks.cephadm:Deploying osd.5 on smithi191 with /dev/vg_nvme/lv_3... 2023-10-09T05:18:35.987 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T05:18:36.017 INFO:journalctl@ceph.osd.4.smithi191.stdout:-- Logs begin at Mon 2023-10-09 04:57:51 UTC. -- 2023-10-09T05:18:36.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:36 smithi191 conmon[101299]: cluster 2023-10-09T05:18:35.303113+0000 mgr.y (mgr. 2023-10-09T05:18:36.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:36 smithi191 conmon[101299]: 14148) 121 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:36.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:36 smithi191 conmon[101299]: audit 2023-10-09T05:18:35.308545+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:36.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:36 smithi191 conmon[101299]: audit 2023-10-09T05:18:35.316917+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:36.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:36 smithi191 conmon[101299]: audit 2023-10-09T05:18:35.531296+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.191:6800/3473028939,v1:172.21.15.191:6801/3473028939]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:18:36.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:36 smithi191 conmon[101299]: audit 2023-10-09T05:18:35.531740+0000 mon.a (mon.0) 391 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:18:36.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: cluster 2023-10-09T05:18:35.303113+0000 mgr.y ( 2023-10-09T05:18:36.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: mgr.14148) 121 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:36.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: audit 2023-10-09T05:18:35.308545+0000 2023-10-09T05:18:36.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: mon.a (mon.0) 389 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:36.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: audit 2023 2023-10-09T05:18:36.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: -10-09T05:18:35.316917+0000 2023-10-09T05:18:36.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: mon.a (mon.0) 390 : 2023-10-09T05:18:36.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:36.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: audit 2023 2023-10-09T05:18:36.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: -10-09T05:18:35.531296 2023-10-09T05:18:36.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: +0000 mon.b (mon.1) 9 2023-10-09T05:18:36.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: : audit [INF] from='osd.4 [v2:172.21.15.191:6800/3473028939,v1:172.21.15.191:6801/3473028939]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:18:36.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: audit 2023 2023-10-09T05:18:36.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: -10-09T05:18:35. 2023-10-09T05:18:36.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: 531740+0000 mon.a (mon.0) 2023-10-09T05:18:36.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[101053]: 391 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:18:36.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: cluster 2023-10-09T05:18:35.303113+0000 2023-10-09T05:18:36.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: mgr.y (mgr.14148) 121 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:36.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:18:36.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: -09T05:18:35.308545+0000 mon.a (mon.0) 389 : audit 2023-10-09T05:18:36.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:36.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: audit 2023-10-09T05:18 2023-10-09T05:18:36.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: :35.316917+0000 mon.a (mon.0) 390 : 2023-10-09T05:18:36.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:36.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:18:36.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: 09T05:18:35.531296+0000 mon.b (mon.1) 9 2023-10-09T05:18:36.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: : audit [INF] from='osd.4 [v2:172.21.15.191:6800/3473028939,v1:172.21.15.191:6801/3473028939]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:18:36.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:18:36.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: 09T05:18:35.531740+0000 mon.a 2023-10-09T05:18:36.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:36 smithi008 conmon[110782]: (mon.0) 391 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:18:37.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[110782]: audit 2023-10-09T05:18:36.310460+0000 mon.a (mon.0) 392 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:18:37.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[110782]: cluster 2023-10-09T05:18:36.310596+0000 mon.a (mon.0) 393 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:18:37.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[110782]: audit 2023-10-09T05:18:36.311025+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:37.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[110782]: audit 2023-10-09T05:18:36.311160+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.191:6800/3473028939,v1:172.21.15.191:6801/3473028939]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:37.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[110782]: audit 2023-10-09T05:18:36.311518+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:37.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:18:37.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[101053]: :18:36.310460+0000 mon.a (mon.0) 392 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:18:37.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[101053]: cluster 2023-10-09T05:18:36.310596+0000 mon.a (mon.0) 393 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:18:37.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[101053]: audit 2023-10-09T05:18:36.311025+0000 mon.a ( 2023-10-09T05:18:37.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[101053]: mon.0) 394 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:37.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[101053]: audit 2023-10-09T05:18:36 2023-10-09T05:18:37.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[101053]: .311160+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.191:6800/3473028939,v1:172.21.15.191:6801/3473028939]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:37.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:18:37.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[101053]: 10-09T05:18:36.311518+0000 mon.a (mon.0) 395 : 2023-10-09T05:18:37.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:37 smithi008 conmon[101053]: audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:37.784 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 05:18:37 smithi191 conmon[107592]: debug 2023-10-09T05:18:37.316+0000 7f5b38ade700 -1 osd.4 0 waiting for initial osdmap 2023-10-09T05:18:37.784 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 05:18:37 smithi191 conmon[107592]: debug 2023-10-09T05:18:37.320+0000 7f5b33256700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:18:37.784 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:37 smithi191 conmon[101299]: audit 2023-10-09T05:18:36.310460+0000 mon.a (mon.0) 392 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:18:37.785 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:37 smithi191 conmon[101299]: cluster 2023-10-09T05:18:36.310596+0000 mon.a (mon.0) 393 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:18:37.785 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:37 smithi191 conmon[101299]: audit 2023-10-09T05:18:36.311025+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:37.785 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:37 smithi191 conmon[101299]: audit 2023-10-09T05:18:36.311160+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.191:6800/3473028939,v1:172.21.15.191:6801/3473028939]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:37.785 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:37 smithi191 conmon[101299]: audit 2023-10-09T05:18:36.311518+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:38.524 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:38 smithi191 conmon[101299]: cluster 2023-10-09T05:18:37.303430+0000 mgr.y (mgr.14148) 122 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:38.524 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:38 smithi191 conmon[101299]: audit 2023-10-09T05:18:37.312456+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:18:38.524 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:38 smithi191 conmon[101299]: cluster 2023-10-09T05:18:37.312583+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:18:38.524 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:38 smithi191 conmon[101299]: audit 2023-10-09T05:18:37.312761+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:38.524 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:38 smithi191 conmon[101299]: audit 2023-10-09T05:18:37.318531+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:38.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[101053]: cluster 2023-10-09T05:18:37.303430+0000 mgr.y 2023-10-09T05:18:38.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[101053]: (mgr.14148) 122 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:38.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[101053]: audit 2023-10-09T05:18:37.312456+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:18:38.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[101053]: cluster 2023-10-09T05:18:37.312583 2023-10-09T05:18:38.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[101053]: +0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:18:38.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:18:38.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[101053]: 10-09T05:18:37.312761+0000 mon.a (mon.0) 398 : audit [DBG] 2023-10-09T05:18:38.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:38.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[101053]: audit 2023-10-09T05:18:37.318531+0000 2023-10-09T05:18:38.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[101053]: mon.a (mon.0) 399 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:38.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: cluster 2023-10-09T05:18:37.303430+0000 mgr.y (mgr 2023-10-09T05:18:38.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: .14148) 122 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:38.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:18:38.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: 05:18:37.312456+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:18:38.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: cluster 2023-10-09T05:18:38.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: 2023-10-09T05:18: 2023-10-09T05:18:38.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: 37.312583+0000 mon.a (mon.0) 397 : cluster [DBG] 2023-10-09T05:18:38.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:18:38.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:18:38.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: :18:37.312761+0000 mon.a (mon.0) 398 2023-10-09T05:18:38.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:38.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:18:38.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: T05:18:37.318531+0000 mon.a (mon.0) 2023-10-09T05:18:38.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:38 smithi008 conmon[110782]: 399 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:38.742 INFO:teuthology.orchestra.run.smithi191.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T05:18:38.742 INFO:teuthology.orchestra.run.smithi191.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T05:18:38.742 INFO:teuthology.orchestra.run.smithi191.stderr: stderr: 10+0 records in 2023-10-09T05:18:38.742 INFO:teuthology.orchestra.run.smithi191.stderr:10+0 records out 2023-10-09T05:18:38.742 INFO:teuthology.orchestra.run.smithi191.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011057 s, 948 MB/s 2023-10-09T05:18:38.743 INFO:teuthology.orchestra.run.smithi191.stderr:--> Zapping successful for: 2023-10-09T05:18:39.453 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:39 smithi191 conmon[101299]: cluster 2023-10-09T05:18:36.486041+0000 osd.4 (osd. 2023-10-09T05:18:39.453 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:39 smithi191 conmon[101299]: 4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:18:39.454 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:39 smithi191 conmon[101299]: cluster 2023-10-09T05:18:36.486096+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:18:39.454 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:39 smithi191 conmon[101299]: cluster 2023-10-09T05:18:38.320512+0000 mon.a (mon.0) 400 : cluster [INF] osd.4 [v2:172.21.15.191:6800/3473028939,v1:172.21.15.191:6801/3473028939] boot 2023-10-09T05:18:39.454 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:39 smithi191 conmon[101299]: cluster 2023-10-09T05:18:38.320592+0000 mon.a (mon.0) 401 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:18:39.454 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:39 smithi191 conmon[101299]: audit 2023-10-09T05:18:38.321151+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:39.454 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:39 smithi191 conmon[101299]: cluster 2023-10-09T05:18:39.321470+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:18:39.474 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi191:vg_nvme/lv_3 2023-10-09T05:18:39.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[101053]: cluster 2023-10-09T05:18:36.486041+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:18:39.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[101053]: cluster 2023-10-09T05:18:39.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[101053]: 2023-10-09T05:18:36.486096+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:18:39.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[101053]: cluster 2023-10-09T05:18:38.320512+0000 mon.a (mon.0) 400 : cluster [INF] osd.4 [v2:172.21.15.191:6800/3473028939,v1:172.21.15.191:6801/3473028939] boot 2023-10-09T05:18:39.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:18:39.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[101053]: -10-09T05:18:38.320592+0000 mon.a (mon.0) 401 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:18:39.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[101053]: audit 2023-10-09T05:18:39.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[101053]: 2023-10-09T05:18:38.321151+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:39.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[101053]: cluster 2023-10-09T05:18:39.321470+0000 mon.a (mon.0) 403 : cluster 2023-10-09T05:18:39.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[101053]: [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:18:39.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: cluster 2023-10-09T05:18:36. 2023-10-09T05:18:39.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: 486041+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: cluster 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: 2023-10-09T05:18: 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: 36.486096+0000 osd.4 (osd. 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: 4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: cluster 2023-10- 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: 09T05:18:38.320512+0000 mon.a (mon.0) 400 : 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: cluster [INF] osd.4 [v2:172.21.15.191:6800/3473028939,v1:172.21.15.191:6801/3473028939] boot 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: cluster 2023-10-09T05:18:38.320592+0000 mon.a (mon. 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: 0) 401 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: audit 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: 2023-10-09T05:18:38.321151+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:18:39.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: cluster 2023-10-09T05:18:39. 2023-10-09T05:18:39.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:39 smithi008 conmon[110782]: 321470+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:18:40.892 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:40 smithi191 conmon[101299]: cluster 2023-10-09T05:18:39.303928+0000 mgr.y (mgr.14148) 123 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:40.892 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:40 smithi191 conmon[101299]: cephadm 2023-10-09T05:18:39.592597+0000 mgr.y (mgr.14148) 124 : cephadm [INF] Detected new or changed devices on smithi191 2023-10-09T05:18:40.892 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:40 smithi191 conmon[101299]: audit 2023-10-09T05:18:39.597638+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:40.892 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:40 smithi191 conmon[101299]: audit 2023-10-09T05:18:39.598691+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:40.892 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:40 smithi191 conmon[101299]: cephadm 2023-10-09T05:18:39.599463+0000 mgr.y (mgr.14148) 125 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 17168M 2023-10-09T05:18:40.892 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:40 smithi191 conmon[101299]: audit 2023-10-09T05:18:39.603572+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:40.892 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:40 smithi191 conmon[101299]: audit 2023-10-09T05:18:39.604683+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:40.892 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:40 smithi191 conmon[101299]: audit 2023-10-09T05:18:39.605625+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:40.892 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:40 smithi191 conmon[101299]: audit 2023-10-09T05:18:39.610336 2023-10-09T05:18:40.893 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:40 smithi191 conmon[101299]: +0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:40.893 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:40 smithi191 conmon[101299]: cluster 2023-10-09T05:18:40.327920+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:18:40.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: cluster 2023-10-09T05:18:39.303928+0000 mgr.y (mgr. 2023-10-09T05:18:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: 14148) 123 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: cephadm 2023-10-09T05:18:39.592597+0000 mgr.y (mgr.14148) 124 : cephadm [INF] Detected new or changed devices on smithi191 2023-10-09T05:18:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: audit 2023-10-09T05:18:39.597638+0000 mon.a (mon.0) 404 : audit [INF] 2023-10-09T05:18:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: audit 2023 2023-10-09T05:18:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: -10-09T05:18:39.598691+0000 mon.a (mon.0) 405 : audit 2023-10-09T05:18:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: cephadm 2023-10-09T05:18:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: 2023-10-09T05:18:39.599463+0000 mgr.y (mgr.14148) 125 : cephadm 2023-10-09T05:18:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: [INF] Adjusting osd_memory_target on smithi191 to 17168M 2023-10-09T05:18:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: audit 2023 2023-10-09T05:18:40.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: -10-09T05:18:39.603572+0000 mon.a (mon.0) 406 : audit 2023-10-09T05:18:40.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:40.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: audit 2023-10-09T05:18:40.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: 2023-10-09T05:18:39.604683+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:40.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:18:40.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: :18:39.605625+0000 mon.a (mon.0) 408 : audit [INF] 2023-10-09T05:18:40.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:40.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: audit 2023 2023-10-09T05:18:40.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: -10-09T05:18:39.610336+0000 mon.a (mon.0) 409 : audit [INF] 2023-10-09T05:18:40.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:40.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: cluster 2023-10-09T05:18:40.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: 2023-10-09T05:18:40.327920+0000 mon.a (mon.0) 410 2023-10-09T05:18:40.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[101053]: : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:18:40.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: cluster 2023-10-09T05:18:39.303928+0000 mgr.y (mgr 2023-10-09T05:18:40.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: .14148) 123 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:18:40.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: cephadm 2023-10-09T05:18:39.592597+0000 2023-10-09T05:18:40.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: mgr.y (mgr.14148) 124 : cephadm [INF] Detected new or changed devices on smithi191 2023-10-09T05:18:40.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:18:40.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: 18:39.597638+0000 mon.a (mon 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: .0) 404 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: audit 2023 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: -10-09T05:18:39 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: .598691+0000 mon.a (mon.0) 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: 405 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: cephadm 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: 2023-10-09T05:18: 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: 39.599463+0000 mgr.y (mgr.14148) 125 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 17168M 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: -09T05:18:39.603572+0000 mon.a (mon.0) 406 : 2023-10-09T05:18:40.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: audit 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: 2023-10-09T05:18 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: :39.604683+0000 mon.a (mon.0) 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: 407 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: audit 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: 2023-10-09T05:18:39 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: .605625+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: audit 2023-10-09T05:18:39.610336+0000 mon.a 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: (mon.0) 409 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: 10-09T05:18:40.327920+0000 mon.a (mon.0 2023-10-09T05:18:40.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:40 smithi008 conmon[110782]: ) 410 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:18:41.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:41 smithi191 conmon[101299]: audit 2023-10-09T05:18:41.303690+0000 mon.a (mon 2023-10-09T05:18:41.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:41 smithi191 conmon[101299]: .0) 411 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:18:41.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:41 smithi191 conmon[101299]: audit 2023-10-09T05:18:41.306723+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:18:41.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:41 smithi191 conmon[101299]: audit 2023-10-09T05:18:41.307593+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:41.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[101053]: audit 2023-10-09T05:18:41.303690+0000 2023-10-09T05:18:41.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[101053]: mon.a (mon.0) 411 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:18:41.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[101053]: audit 2023-10-09T05:18:41.306723+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:18:41.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[101053]: audit 2023-10-09T05:18:41.307593+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:41.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[101053]: 2023-10-09T05:18:41.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[110782]: audit 2023-10-09T05:18:41.303690+0000 mon.a 2023-10-09T05:18:41.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[110782]: (mon.0) 411 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:18:41.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:18:41.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[110782]: -09T05:18:41.306723+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:18:41.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:18:41.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[110782]: -09T05:18:41.307593+0000 mon.a (mon.0) 2023-10-09T05:18:41.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:41 smithi008 conmon[110782]: 413 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:42.824 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:42 smithi191 conmon[101299]: audit 2023-10-09T05:18:41.300698+0000 mgr.y (mgr.14148) 126 : audit [DBG] from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:18:42.824 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:42 smithi191 conmon[101299]: cluster 2023-10-09T05:18:41.304268+0000 mgr.y (mgr.14148) 127 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:43.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:42 smithi008 conmon[101053]: audit 2023-10-09T05:18:41.300698+0000 mgr.y 2023-10-09T05:18:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:42 smithi008 conmon[101053]: (mgr.14148) 126 : audit [DBG] from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:18:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:42 smithi008 conmon[101053]: cluster 2023-10-09T05:18:41.304268+0000 mgr.y (mgr.14148) 127 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:43.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:42 smithi008 conmon[110782]: audit 2023-10-09T05:18:41.300698+0000 mgr.y (mgr. 2023-10-09T05:18:43.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:42 smithi008 conmon[110782]: 14148) 126 : audit [DBG] from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:18:43.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:42 smithi008 conmon[110782]: cluster 2023-10-09T05: 2023-10-09T05:18:43.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:42 smithi008 conmon[110782]: 18:41.304268+0000 mgr.y (mgr.14148) 127 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:44.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:43 smithi191 conmon[101299]: audit 2023-10-09T05:18:43.510464+0000 mon.b (mon.1) 2023-10-09T05:18:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:43 smithi191 conmon[101299]: 11 : audit [INF] from='client.? 172.21.15.191:0/3008250488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01507478-481e-4646-a30d-542184240f07"}]: dispatch 2023-10-09T05:18:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:43 smithi191 conmon[101299]: audit 2023-10-09T05:18:43.510944+0000 mon.a (mon.0) 414 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01507478-481e-4646-a30d-542184240f07"}]: dispatch 2023-10-09T05:18:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:43 smithi191 conmon[101299]: audit 2023-10-09T05:18:43.515965+0000 mon.a (mon.0) 415 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "01507478-481e-4646-a30d-542184240f07"}]': finished 2023-10-09T05:18:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:43 smithi191 conmon[101299]: cluster 2023-10-09T05:18:43.516037+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T05:18:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:43 smithi191 conmon[101299]: audit 2023-10-09T05:18:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:43 smithi191 conmon[101299]: 2023-10-09T05:18:43.516225+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:18:44.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[101053]: audit 2023-10-09T05:18:43.510464+0000 mon.b (mon.1) 2023-10-09T05:18:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[101053]: 11 : audit [INF] from='client.? 172.21.15.191:0/3008250488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01507478-481e-4646-a30d-542184240f07"}]: dispatch 2023-10-09T05:18:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[101053]: audit 2023-10-09T05:18:43.510944+0000 mon.a (mon.0) 414 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01507478-481e-4646-a30d-542184240f07"}]: dispatch 2023-10-09T05:18:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[101053]: audit 2023-10-09T05:18:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[101053]: 2023-10-09T05:18:43.515965+0000 mon.a (mon.0) 415 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "01507478-481e-4646-a30d-542184240f07"}]': finished 2023-10-09T05:18:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[101053]: cluster 2023-10-09T05:18:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[101053]: 2023-10-09T05:18:43.516037+0000 mon.a (mon.0) 2023-10-09T05:18:44.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[101053]: 416 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T05:18:44.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[101053]: audit 2023-10-09T05:18: 2023-10-09T05:18:44.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[101053]: 43.516225+0000 mon.a (mon.0) 417 : audit [DBG] 2023-10-09T05:18:44.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:18:44.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: audit 2023-10-09T05:18:43.510464+0000 2023-10-09T05:18:44.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.191:0/3008250488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01507478-481e-4646-a30d-542184240f07"}]: dispatch 2023-10-09T05:18:44.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: audit 2023-10-09T05:18 2023-10-09T05:18:44.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: :43.510944+0000 mon.a (mon.0) 414 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01507478-481e-4646-a30d-542184240f07"}]: dispatch 2023-10-09T05:18:44.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: 2023-10-09T05:18:44.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:18:44.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: 10-09T05:18:43.515965+0000 mon.a (mon.0) 415 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "01507478-481e-4646-a30d-542184240f07"}]': finished 2023-10-09T05:18:44.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: cluster 2023-10 2023-10-09T05:18:44.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: -09T05:18:43.516037+0000 mon.a ( 2023-10-09T05:18:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: mon.0) 416 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T05:18:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:18:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: 05:18:43.516225+0000 mon.a (mon.0) 417 : audit [DBG] 2023-10-09T05:18:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:43 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:18:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:44 smithi191 conmon[101299]: cluster 2023-10-09T05:18:43.304687+0000 mgr.y (mgr.14148) 128 2023-10-09T05:18:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:44 smithi191 conmon[101299]: : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:44 smithi191 conmon[101299]: audit 2023-10-09T05:18:44.114347+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.191:0/4119698045' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:18:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:44 smithi008 conmon[101053]: cluster 2023-10-09T05:18:43.304687+0000 mgr.y ( 2023-10-09T05:18:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:44 smithi008 conmon[101053]: mgr.14148) 128 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:44 smithi008 conmon[101053]: audit 2023-10-09T05:18:44.114347+0000 mon.b (mon.1) 2023-10-09T05:18:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:44 smithi008 conmon[101053]: 12 : audit [DBG] from='client.? 172.21.15.191:0/4119698045' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:18:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:44 smithi008 conmon[110782]: cluster 2023-10-09T05:18:43.304687 2023-10-09T05:18:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:44 smithi008 conmon[110782]: +0000 mgr.y (mgr.14148) 128 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:44 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:18:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:44 smithi008 conmon[110782]: 10-09T05:18:44.114347+0000 mon.b (mon.1) 12 : audit 2023-10-09T05:18:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:44 smithi008 conmon[110782]: [DBG] from='client.? 172.21.15.191:0/4119698045' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:18:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:46 smithi191 conmon[101299]: cluster 2023-10-09T05:18:45.305118+0000 mgr.y (mgr.14148) 129 : 2023-10-09T05:18:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:46 smithi191 conmon[101299]: cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:47.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:46 smithi008 conmon[101053]: cluster 2023-10-09T05:18:45.305118+0000 mgr.y ( 2023-10-09T05:18:47.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:46 smithi008 conmon[101053]: mgr.14148) 129 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:46 smithi008 conmon[110782]: cluster 2023-10-09T05:18:45.305118+0000 mgr.y (mgr.14148) 129 : cluster [DBG] 2023-10-09T05:18:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:46 smithi008 conmon[110782]: pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:49.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:48 smithi191 conmon[101299]: cluster 2023-10-09T05:18:47.305550+0000 mgr.y (mgr.14148) 130 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:49.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:48 smithi008 conmon[101053]: cluster 2023-10-09T05:18:47.305550+0000 mgr.y (mgr 2023-10-09T05:18:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:48 smithi008 conmon[101053]: .14148) 130 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:49.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:48 smithi008 conmon[110782]: cluster 2023-10-09T05:18:47.305550+0000 mgr.y (mgr. 2023-10-09T05:18:49.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:48 smithi008 conmon[110782]: 14148) 130 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:49 smithi191 conmon[101299]: audit 2023-10-09T05:18:49.520115+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:18:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:49 smithi191 conmon[101299]: audit 2023-10-09T05:18:49.521104+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:50.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:49 smithi008 conmon[101053]: audit 2023-10-09T05:18:49.520115+0000 mon.a (mon. 2023-10-09T05:18:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:49 smithi008 conmon[101053]: 0) 418 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:18:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:49 smithi008 conmon[101053]: audit 2023-10-09T05:18:49.521104+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:50.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:49 smithi008 conmon[110782]: audit 2023-10-09T05:18:49.520115+0000 mon.a ( 2023-10-09T05:18:50.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:49 smithi008 conmon[110782]: mon.0) 418 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:18:50.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:49 smithi008 conmon[110782]: audit 2023-10-09T05:18:49.521104+0000 2023-10-09T05:18:50.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:49 smithi008 conmon[110782]: mon.a (mon.0) 419 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:50.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:50 smithi191 conmon[101299]: cluster 2023-10-09T05:18:49.306052+0000 2023-10-09T05:18:50.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:50 smithi191 conmon[101299]: mgr.y (mgr.14148) 131 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:50.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:50 smithi191 conmon[101299]: cephadm 2023-10-09T05:18:49.521940+0000 mgr.y (mgr.14148) 132 : cephadm [INF] Deploying daemon osd.5 on smithi191 2023-10-09T05:18:51.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:50 smithi008 conmon[101053]: cluster 2023-10-09T05:18:49.306052+0000 mgr.y ( 2023-10-09T05:18:51.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:50 smithi008 conmon[101053]: mgr.14148) 131 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:51.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:50 smithi008 conmon[101053]: cephadm 2023-10-09T05:18:49.521940+0000 mgr.y (mgr.14148) 2023-10-09T05:18:51.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:50 smithi008 conmon[101053]: 132 : cephadm [INF] Deploying daemon osd.5 on smithi191 2023-10-09T05:18:51.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:50 smithi008 conmon[110782]: cluster 2023-10-09T05:18:49.306052+0000 2023-10-09T05:18:51.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:50 smithi008 conmon[110782]: mgr.y (mgr.14148) 131 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:51.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:50 smithi008 conmon[110782]: cephadm 2023-10-09T05:18:51.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:50 smithi008 conmon[110782]: 2023-10-09T05:18:49.521940+0000 mgr.y (mgr. 2023-10-09T05:18:51.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:50 smithi008 conmon[110782]: 14148) 132 : cephadm [INF] Deploying daemon osd.5 on smithi191 2023-10-09T05:18:53.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:52 smithi191 conmon[101299]: cluster 2023-10-09T05:18:51.306636+0000 mgr.y (mgr.14148 2023-10-09T05:18:53.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:52 smithi191 conmon[101299]: ) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:53.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:52 smithi008 conmon[101053]: cluster 2023-10-09T05:18:51.306636+0000 mgr.y (mgr.14148) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:52 smithi008 conmon[110782]: cluster 2023-10-09T05:18:51.306636+0000 mgr.y (mgr.14148) 133 : 2023-10-09T05:18:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:52 smithi008 conmon[110782]: cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:54 smithi191 conmon[101299]: cluster 2023-10-09T05:18:53.307253+0000 mgr.y (mgr.14148) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:54 smithi191 conmon[101299]: audit 2023-10-09T05:18:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:54 smithi191 conmon[101299]: 2023-10-09T05:18:54.289259+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:54 smithi191 conmon[101299]: audit 2023-10-09T05:18:54.317094+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:55.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[101053]: cluster 2023-10-09T05:18:53.307253 2023-10-09T05:18:55.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[101053]: +0000 mgr.y (mgr.14148) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:55.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[101053]: audit 2023-10-09T05:18:54.289259+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:55.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[101053]: audit 2023-10-09T05:18:54.317094+0000 mon.a (mon. 2023-10-09T05:18:55.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[101053]: 0) 421 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:55.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[110782]: cluster 2023-10-09T05:18:53.307253+0000 mgr.y ( 2023-10-09T05:18:55.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[110782]: mgr.14148) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:55.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:18:55.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[110782]: T05:18:54.289259+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:55.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:18:55.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[110782]: 10-09T05:18:54.317094+0000 mon.a (mon.0) 421 2023-10-09T05:18:55.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:54 smithi008 conmon[110782]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:56 smithi191 conmon[101299]: cluster 2023-10-09T05:18:55.307860+0000 mgr.y (mgr.14148) 135 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:56 smithi191 conmon[101299]: audit 2023-10-09T05:18:56.082288+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:56 smithi191 conmon[101299]: audit 2023-10-09T05:18:56.083219+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:56 smithi191 conmon[101299]: audit 2023-10-09T05:18:56.084224+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:56 smithi191 conmon[101299]: audit 2023-10-09T05:18:56.089051+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:57.165 INFO:teuthology.orchestra.run.smithi191.stdout:Created osd(s) 5 on host 'smithi191' 2023-10-09T05:18:57.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[101053]: cluster 2023-10-09T05:18:55.307860+0000 mgr.y ( 2023-10-09T05:18:57.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[101053]: mgr.14148) 135 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[101053]: audit 2023-10-09T05:18:56.082288+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[101053]: audit 2023-10-09T05:18:56 2023-10-09T05:18:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[101053]: .083219+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[101053]: audit 2023-10-09T05:18:56.084224+0000 mon.a (mon.0) 424 : audit 2023-10-09T05:18:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[101053]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[101053]: audit 2023-10-09T05:18:56.089051+0000 mon.a (mon.0) 425 2023-10-09T05:18:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[101053]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:57.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[110782]: cluster 2023-10-09T05:18:55.307860+0000 mgr.y ( 2023-10-09T05:18:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[110782]: mgr.14148) 135 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[110782]: audit 2023-10-09T05:18:56.082288+0000 mon.a (mon.0) 422 : audit [INF] 2023-10-09T05:18:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[110782]: audit 2023-10-09T05:18:56.083219+0000 mon.a (mon.0) 423 : audit 2023-10-09T05:18:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[110782]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:18:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[110782]: audit 2023-10-09T05:18 2023-10-09T05:18:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[110782]: :56.084224+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:18:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[110782]: audit 2023-10-09T05:18:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:56 smithi008 conmon[110782]: 2023-10-09T05:18:56.089051+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:57.658 DEBUG:teuthology.orchestra.run.smithi191:osd.5> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T05:18:57.660 INFO:tasks.cephadm:Deploying osd.6 on smithi191 with /dev/vg_nvme/lv_2... 2023-10-09T05:18:57.660 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T05:18:57.688 INFO:journalctl@ceph.osd.5.smithi191.stdout:-- Logs begin at Mon 2023-10-09 04:57:51 UTC. -- 2023-10-09T05:18:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:58 smithi191 conmon[101299]: audit 2023-10-09T05:18:57.164121+0000 2023-10-09T05:18:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:58 smithi191 conmon[101299]: mon.a (mon.0) 426 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:58 smithi191 conmon[101299]: audit 2023-10-09T05:18:57.209066+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:58 smithi191 conmon[101299]: cluster 2023-10-09T05:18:57.308219+0000 mgr.y (mgr.14148) 136 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:58 smithi191 conmon[101299]: audit 2023-10-09T05:18:57.512235+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.191:6808/1592222523,v1:172.21.15.191:6809/1592222523]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:18:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:58 smithi191 conmon[101299]: audit 2023-10-09T05:18:57.512651+0000 mon.a (mon.0) 428 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:18:58.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[101053]: audit 2023-10-09T05:18:57.164121+0000 mon.a (mon.0) 426 : audit 2023-10-09T05:18:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[101053]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[101053]: audit 2023-10-09T05:18:57.209066+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[101053]: cluster 2023-10-09T05:18:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[101053]: 2023-10-09T05:18:57.308219+0000 mgr.y (mgr.14148) 136 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:18:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[101053]: -09T05:18:57.512235+0000 mon.b (mon.1) 13 2023-10-09T05:18:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[101053]: : audit [INF] from='osd.5 [v2:172.21.15.191:6808/1592222523,v1:172.21.15.191:6809/1592222523]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:18:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:18:58.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[101053]: 10-09T05:18:57.512651+0000 mon.a (mon.0) 2023-10-09T05:18:58.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[101053]: 428 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:18:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: audit 2023-10-09T05:18:57.164121+0000 2023-10-09T05:18:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: mon.a (mon.0) 426 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:18:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: audit 2023-10-09T05:18:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: 2023-10-09T05:18:57.209066+0000 mon.a (mon.0) 427 : audit [DBG] 2023-10-09T05:18:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:18:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: cluster 2023-10-09T05:18:57 2023-10-09T05:18:58.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: .308219+0000 mgr.y (mgr.14148) 136 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:18:58.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:18:58.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: 05:18:57.512235+0000 mon.b (mon.1) 13 : 2023-10-09T05:18:58.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: audit [INF] from='osd.5 [v2:172.21.15.191:6808/1592222523,v1:172.21.15.191:6809/1592222523]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:18:58.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: audit 2023-10-09T05:18:57.512651 2023-10-09T05:18:58.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:58 smithi008 conmon[110782]: +0000 mon.a (mon.0) 428 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:18:59.438 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 05:18:59 smithi191 conmon[113449]: debug 2023-10-09T05:18:59.176+0000 7fed1964f700 -1 osd.5 0 waiting for initial osdmap 2023-10-09T05:18:59.438 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 05:18:59 smithi191 conmon[113449]: debug 2023-10-09T05:18:59.181+0000 7fed135c6700 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:18:59.439 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:59 smithi191 conmon[101299]: audit 2023-10-09T05:18:58.170036+0000 mon.a (mon.0) 429 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:18:59.439 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:59 smithi191 conmon[101299]: cluster 2023-10-09T05:18:58.170155+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T05:18:59.439 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:59 smithi191 conmon[101299]: audit 2023-10-09T05:18:58.170505+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:18:59.439 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:59 smithi191 conmon[101299]: audit 2023-10-09T05:18:58.170760+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.191:6808/1592222523,v1:172.21.15.191:6809/1592222523]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:59.439 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:18:59 smithi191 conmon[101299]: audit 2023-10-09T05:18:58.171051+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:59.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[101053]: audit 2023-10-09T05:18:58.170036+0000 mon.a (mon.0) 429 : 2023-10-09T05:18:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[101053]: audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:18:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[101053]: cluster 2023-10-09T05:18:58.170155+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T05:18:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[101053]: audit 2023-10-09T05:18:58.170505+0000 mon.a (mon.0 2023-10-09T05:18:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[101053]: ) 431 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:18:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:18:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[101053]: 10-09T05:18:58.170760+0000 mon.b 2023-10-09T05:18:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[101053]: (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.191:6808/1592222523,v1:172.21.15.191:6809/1592222523]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[101053]: audit 2023-10-09T05:18:58.171051+0000 mon.a (mon.0 2023-10-09T05:18:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[101053]: ) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:59.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: audit 2023-10-09T05:18:58.170036+0000 mon.a (mon.0 2023-10-09T05:18:59.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: ) 429 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:18:59.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: cluster 2023-10-09T05:18:58.170155+0000 mon.a (mon.0) 430 2023-10-09T05:18:59.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T05:18:59.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: audit 2023-10-09T05:18: 2023-10-09T05:18:59.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: 58.170505+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:18:59.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:18:59.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: 09T05:18:58.170760+0000 mon.b (mon.1) 14 2023-10-09T05:18:59.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: : audit [INF] from='osd.5 [v2:172.21.15.191:6808/1592222523,v1:172.21.15.191:6809/1592222523]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:59.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: audit 2023-10-09T05:18:58.171051 2023-10-09T05:18:59.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: +0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:18:59.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:18:59 smithi008 conmon[110782]: 2023-10-09T05:19:00.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[101053]: audit 2023-10-09T05:18:59.171206+0000 mon.a (mon.0) 433 2023-10-09T05:19:00.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[101053]: : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:19:00.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[101053]: cluster 2023-10-09T05:18:59.171330+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:19:00.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[101053]: audit 2023-10-09T05:18:59.171875+0000 mon.a (mon.0) 435 : audit [DBG] 2023-10-09T05:19:00.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:00.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:19:00.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[101053]: 18:59.176795+0000 mon.a (mon.0) 436 : audit [DBG] 2023-10-09T05:19:00.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:00.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:19:00.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[101053]: -10-09T05:18:59.308705+0000 mgr.y 2023-10-09T05:19:00.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[101053]: (mgr.14148) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:19:00.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: audit 2023-10-09T05:18:59.171206+0000 2023-10-09T05:19:00.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:19:00.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: cluster 2023-10-09T05:19:00.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: 2023-10-09T05:18:59.171330+0000 mon.a (mon.0) 434 : cluster [DBG] 2023-10-09T05:19:00.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:19:00.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:19:00.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: T05:18:59.171875+0000 mon.a (mon.0 2023-10-09T05:19:00.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: ) 435 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:00.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:19:00.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: 05:18:59.176795+0000 mon.a (mon.0 2023-10-09T05:19:00.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: ) 436 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:00.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: cluster 2023-10-09T05:18:59. 2023-10-09T05:19:00.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:00 smithi008 conmon[110782]: 308705+0000 mgr.y (mgr.14148) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:19:00.520 INFO:teuthology.orchestra.run.smithi191.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T05:19:00.520 INFO:teuthology.orchestra.run.smithi191.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T05:19:00.520 INFO:teuthology.orchestra.run.smithi191.stderr: stderr: 10+0 records in 2023-10-09T05:19:00.520 INFO:teuthology.orchestra.run.smithi191.stderr:10+0 records out 2023-10-09T05:19:00.520 INFO:teuthology.orchestra.run.smithi191.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110366 s, 950 MB/s 2023-10-09T05:19:00.520 INFO:teuthology.orchestra.run.smithi191.stderr:--> Zapping successful for: 2023-10-09T05:19:00.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:00 smithi191 conmon[101299]: audit 2023-10-09T05:18:59.171206+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:19:00.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:00 smithi191 conmon[101299]: cluster 2023-10-09T05:18:59.171330+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:19:00.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:00 smithi191 conmon[101299]: audit 2023-10-09T05:18:59.171875+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:00.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:00 smithi191 conmon[101299]: audit 2023-10-09T05:18:59.176795+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:00.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:00 smithi191 conmon[101299]: cluster 2023-10-09T05:18:59.308705+0000 mgr.y (mgr.14148) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:19:01.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: cluster 2023-10-09T05:18:58.485342+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:19:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: 2023-10-09T05:19:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: cluster 2023-10-09T05:18:58.485437+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:19:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: audit 2023-10-09T05:19:00.177356+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: cluster 2023-10-09T05:19:00.182189+0000 mon.a (mon.0) 438 : cluster [INF] osd.5 [v2:172.21.15.191:6808/1592222523,v1:172.21.15.191:6809/1592222523] boot 2023-10-09T05:19:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: cluster 2023-10-09T05:19:00.182261+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T05:19:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: audit 2023-10-09T05:19:00.182488+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: audit 2023-10-09T05:19:01.165359+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: audit 2023-10-09T05:19:01.166483+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: audit 2023-10-09T05:19:01.167471+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: audit 2023-10-09T05:19:01.172628+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: audit 2023-10-09T05:19:01.173786+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:01 smithi191 conmon[101299]: audit 2023-10-09T05:19:01.174829+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:01.351 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi191:vg_nvme/lv_2 2023-10-09T05:19:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: cluster 2023-10-09T05:18:58.485342+0000 2023-10-09T05:19:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:19:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: cluster 2023-10-09 2023-10-09T05:19:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: T05:18:58.485437+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:19:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: audit 2023-10-09T05:19:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: 2023-10-09T05:19:00.177356 2023-10-09T05:19:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: +0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: 2023-10-09T05:19:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:19:01.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: -10-09T05:19:00.182189+0000 mon.a (mon.0) 2023-10-09T05:19:01.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: 438 : cluster [INF] osd.5 [v2:172.21.15.191:6808/1592222523,v1:172.21.15.191:6809/1592222523] boot 2023-10-09T05:19:01.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: cluster 2023-10-09T 2023-10-09T05:19:01.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: 05:19:00.182261+0000 mon.a (mon.0) 2023-10-09T05:19:01.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: 439 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T05:19:01.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: audit 2023-10-09T05:19:00 2023-10-09T05:19:01.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: .182488+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:01.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:19:01.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: T05:19:01.165359+0000 mon.a (mon.0) 441 : audit [INF] 2023-10-09T05:19:01.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:01.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:19:01.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: 05:19:01.166483+0000 mon.a (mon.0) 442 : 2023-10-09T05:19:01.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:01.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: audit 2023-10-09T05:19:01.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: 2023-10-09T05:19:01.167471+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:01.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: 2023-10-09T05:19:01.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: audit 2023-10-09T05:19:01.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: 2023-10-09T05:19:01.172628+0000 mon.a (mon.0) 2023-10-09T05:19:01.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: 444 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:01.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:19:01.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: 19:01.173786+0000 mon.a (mon.0) 445 : audit 2023-10-09T05:19:01.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:01.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: audit 2023-10-09T05:19:01.174829+0000 2023-10-09T05:19:01.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[110782]: mon.a (mon.0) 446 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:01.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: cluster 2023-10-09T05:18:58.485342+0000 osd.5 ( 2023-10-09T05:19:01.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:19:01.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: cluster 2023-10-09T05:18:58.485437+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:19:01.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: 2023-10-09T05:19:01.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: audit 2023-10-09T05:19 2023-10-09T05:19:01.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: :00.177356+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:01.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: cluster 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: 2023-10-09T05:19:00.182189+0000 mon.a 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: (mon.0) 438 : cluster [INF] osd.5 [v2:172.21.15.191:6808/1592222523,v1:172.21.15.191:6809/1592222523] boot 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: cluster 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: 2023-10-09T05:19:00.182261+0000 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: mon.a (mon.0) 439 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: audit 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: 2023-10-09T05:19:00. 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: 182488+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: audit 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: 2023-10-09T05:19:01.165359+0000 mon.a (mon. 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: 0) 441 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:01.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: audit 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: 2023-10-09T05:19:01.166483+0000 mon.a 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: (mon.0) 442 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: audit 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: 2023-10-09T05:19:01.167471+0000 mon.a (mon.0 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: ) 443 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: 05:19:01.172628+0000 mon.a (mon.0) 444 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: audit 2023-10-09T05:19:01 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: .173786+0000 mon.a (mon.0) 445 : audit [DBG] 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:01.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:19:01.471 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: T05:19:01.174829+0000 mon.a (mon 2023-10-09T05:19:01.471 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:01 smithi008 conmon[101053]: .0) 446 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:02.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[101053]: cephadm 2023-10-09T05:19:01.160327+0000 mgr.y (mgr.14148) 138 : cephadm [INF] 2023-10-09T05:19:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[101053]: Detected new or changed devices on smithi191 2023-10-09T05:19:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[101053]: cephadm 2023-10-09T05:19:01.168252+0000 mgr.y (mgr.14148) 139 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 8584M 2023-10-09T05:19:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[101053]: audit 2023-10-09T05:19:01.179829+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[101053]: cluster 2023-10-09T05:19:01.189323+0000 mon.a (mon.0) 2023-10-09T05:19:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[101053]: 448 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T05:19:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[101053]: cluster 2023-10-09T 2023-10-09T05:19:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[101053]: 05:19:01.309232+0000 mgr.y (mgr. 2023-10-09T05:19:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[101053]: 14148) 140 : cluster [DBG] pgmap v123: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:02.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: cephadm 2023-10-09T05:19:01.160327+0000 2023-10-09T05:19:02.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: mgr.y (mgr.14148) 138 : cephadm [INF] Detected new or changed devices on smithi191 2023-10-09T05:19:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: cephadm 2023-10-09 2023-10-09T05:19:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: T05:19:01.168252+0000 mgr.y (mgr.14148) 139 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 8584M 2023-10-09T05:19:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: audit 2023-10-09T05:19:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: 2023-10-09T05 2023-10-09T05:19:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: :19:01.179829+0000 mon.a (mon.0) 447 : 2023-10-09T05:19:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: cluster 2023-10-09T05:19:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: 2023-10-09T05:19:01.189323 2023-10-09T05:19:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: +0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T05:19:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: 2023-10-09T05:19:02.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: cluster 2023-10-09T05:19:01. 2023-10-09T05:19:02.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: 309232+0000 mgr.y (mgr.14148) 140 : cluster [DBG] pgmap v123: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:02.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:02 smithi008 conmon[110782]: 2023-10-09T05:19:02.469 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:02 smithi191 conmon[101299]: cephadm 2023-10-09T05:19:01.160327+0000 mgr.y (mgr.14148) 138 : cephadm [INF] Detected new or changed devices on smithi191 2023-10-09T05:19:02.469 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:02 smithi191 conmon[101299]: cephadm 2023-10-09T05:19:01.168252+0000 mgr.y (mgr.14148) 139 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 8584M 2023-10-09T05:19:02.469 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:02 smithi191 conmon[101299]: audit 2023-10-09T05:19:01.179829+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:02.469 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:02 smithi191 conmon[101299]: cluster 2023-10-09T05:19:01.189323+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T05:19:02.469 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:02 smithi191 conmon[101299]: cluster 2023-10-09T05:19:01.309232+0000 mgr.y (mgr.14148) 140 : cluster [DBG] pgmap v123: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:03.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[101053]: cluster 2023-10-09T05:19:02.192308+0000 mon.a ( 2023-10-09T05:19:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[101053]: mon.0) 449 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:19:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[101053]: audit 2023-10-09T05:19:03.048528+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:19:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[101053]: audit 2023-10-09T05:19:03.051502+0000 mon.a 2023-10-09T05:19:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[101053]: (mon.0) 451 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:19:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:19:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[101053]: 10-09T05:19:03.052430+0000 mon.a (mon.0) 2023-10-09T05:19:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[101053]: 452 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[110782]: cluster 2023-10-09T05:19:02.192308+0000 mon.a (mon.0) 449 : cluster 2023-10-09T05:19:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[110782]: [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:19:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[110782]: audit 2023-10-09T05:19:03.048528+0000 mon.a (mon.0) 450 2023-10-09T05:19:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[110782]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:19:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[110782]: audit 2023-10-09T05:19:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[110782]: 2023-10-09T05:19:03.051502+0000 mon.a ( 2023-10-09T05:19:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[110782]: mon.0) 451 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:19:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[110782]: audit 2023-10-09T05:19:03.052430 2023-10-09T05:19:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:03 smithi008 conmon[110782]: +0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:03.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:03 smithi191 conmon[101299]: cluster 2023-10-09T05:19:02. 2023-10-09T05:19:03.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:03 smithi191 conmon[101299]: 192308+0000 mon.a (mon.0) 449 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:19:03.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:03 smithi191 conmon[101299]: audit 2023-10-09T05:19:03.048528+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:19:03.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:03 smithi191 conmon[101299]: audit 2023-10-09T05:19:03.051502+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:19:03.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:03 smithi191 conmon[101299]: audit 2023-10-09T05:19:03.052430+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:04.453 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:04 smithi191 conmon[101299]: audit 2023-10-09T05:19:03.045533+0000 mgr.y (mgr.14148) 141 : audit [DBG] from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:04.453 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:04 smithi191 conmon[101299]: cluster 2023-10-09T05:19:03.309841+0000 mgr.y (mgr.14148) 142 : cluster [DBG] pgmap v125: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:04.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:04 smithi008 conmon[101053]: audit 2023-10-09T05:19:03.045533+0000 mgr.y (mgr. 2023-10-09T05:19:04.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:04 smithi008 conmon[101053]: 14148) 141 : audit [DBG] from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:04.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:04 smithi008 conmon[101053]: cluster 2023-10-09T05:19:03.309841+0000 mgr.y (mgr.14148) 142 : cluster [DBG] pgmap v125: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:04 smithi008 conmon[110782]: audit 2023-10-09T05:19:03.045533+0000 mgr.y (mgr. 2023-10-09T05:19:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:04 smithi008 conmon[110782]: 14148) 141 : audit [DBG] from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:04 smithi008 conmon[110782]: cluster 2023-10-09T05:19:03.309841+0000 mgr.y (mgr.14148) 142 : cluster [DBG] pgmap v125: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:05.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[101053]: audit 2023-10-09T05:19:05.160172+0000 mon.b (mon.1 2023-10-09T05:19:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[101053]: ) 15 : audit [INF] from='client.? 172.21.15.191:0/1342287718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8bc75ec3-46ee-4d36-989b-4e65e330cfb9"}]: dispatch 2023-10-09T05:19:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[101053]: audit 2023-10-09T05:19:05.160868+0000 mon.a (mon.0) 453 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8bc75ec3-46ee-4d36-989b-4e65e330cfb9"}]: dispatch 2023-10-09T05:19:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[101053]: audit 2023-10-09T05:19:05.165920+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8bc75ec3-46ee-4d36-989b-4e65e330cfb9"}]': finished 2023-10-09T05:19:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[101053]: cluster 2023-10-09T05:19:05.166038+0000 mon.a 2023-10-09T05:19:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[101053]: (mon.0) 455 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T05:19:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[101053]: audit 2023-10-09T05:19:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[101053]: 2023-10-09T05:19:05.166219+0000 mon.a (mon 2023-10-09T05:19:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[101053]: .0) 456 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:05.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[110782]: audit 2023-10-09T05:19:05.160172+0000 mon.b ( 2023-10-09T05:19:05.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[110782]: mon.1) 15 : audit [INF] from='client.? 172.21.15.191:0/1342287718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8bc75ec3-46ee-4d36-989b-4e65e330cfb9"}]: dispatch 2023-10-09T05:19:05.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[110782]: audit 2023-10-09T05:19: 2023-10-09T05:19:05.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[110782]: 05.160868+0000 mon.a (mon.0) 453 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8bc75ec3-46ee-4d36-989b-4e65e330cfb9"}]: dispatch 2023-10-09T05:19:05.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[110782]: audit 2023-10-09T05:19:05.165920+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8bc75ec3-46ee-4d36-989b-4e65e330cfb9"}]': finished 2023-10-09T05:19:05.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[110782]: cluster 2023-10-09T05:19:05.166038+0000 mon.a (mon.0) 455 : cluster 2023-10-09T05:19:05.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[110782]: [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T05:19:05.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[110782]: audit 2023-10-09T05:19:05.166219+0000 mon.a (mon.0) 456 2023-10-09T05:19:05.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:05 smithi008 conmon[110782]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:05 smithi191 conmon[101299]: audit 2023-10-09T05:19:05.160172+0000 mon.b (mon.1) 15 : 2023-10-09T05:19:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:05 smithi191 conmon[101299]: audit [INF] from='client.? 172.21.15.191:0/1342287718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8bc75ec3-46ee-4d36-989b-4e65e330cfb9"}]: dispatch 2023-10-09T05:19:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:05 smithi191 conmon[101299]: audit 2023-10-09T05:19:05.160868+0000 mon.a (mon.0) 453 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8bc75ec3-46ee-4d36-989b-4e65e330cfb9"}]: dispatch 2023-10-09T05:19:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:05 smithi191 conmon[101299]: audit 2023-10-09T05:19:05.165920+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8bc75ec3-46ee-4d36-989b-4e65e330cfb9"}]': finished 2023-10-09T05:19:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:05 smithi191 conmon[101299]: cluster 2023-10-09T05:19:05.166038+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T05:19:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:05 smithi191 conmon[101299]: audit 2023-10-09T05:19:05.166219+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:06 smithi008 conmon[101053]: cluster 2023-10-09T05:19 2023-10-09T05:19:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:06 smithi008 conmon[101053]: :05.310449+0000 mgr.y (mgr.14148) 143 : cluster [DBG] pgmap v127: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:06 smithi008 conmon[101053]: audit 2023-10-09T05:19:05 2023-10-09T05:19:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:06 smithi008 conmon[101053]: .777037+0000 mon.b (mon.1) 16 2023-10-09T05:19:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:06 smithi008 conmon[101053]: : audit [DBG] from='client.? 172.21.15.191:0/2121737705' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:19:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:06 smithi008 conmon[110782]: cluster 2023-10-09T05:19:05 2023-10-09T05:19:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:06 smithi008 conmon[110782]: .310449+0000 mgr.y (mgr.14148) 143 : cluster [DBG] pgmap v127: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:06 smithi008 conmon[110782]: audit 2023 2023-10-09T05:19:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:06 smithi008 conmon[110782]: -10-09T05:19:05 2023-10-09T05:19:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:06 smithi008 conmon[110782]: .777037+0000 mon.b (mon.1 2023-10-09T05:19:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:06 smithi008 conmon[110782]: ) 16 : audit [DBG] from='client.? 172.21.15.191:0/2121737705' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:19:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:06 smithi191 conmon[101299]: cluster 2023-10-09T05:19:05.310449+0000 mgr.y (mgr.14148) 2023-10-09T05:19:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:06 smithi191 conmon[101299]: 143 : cluster [DBG] pgmap v127: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:06 smithi191 conmon[101299]: audit 2023-10-09T05:19:05.777037+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.191:0/2121737705' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:19:09.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:08 smithi191 conmon[101299]: cluster 2023-10-09T05:19:07.311023+0000 mgr.y (mgr.14148) 144 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:08 smithi008 conmon[101053]: cluster 2023-10 2023-10-09T05:19:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:08 smithi008 conmon[101053]: -09T05:19:07.311023+0000 mgr.y (mgr.14148) 144 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:08 smithi008 conmon[110782]: cluster 2023-10-09T05:19:07. 2023-10-09T05:19:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:08 smithi008 conmon[110782]: 311023+0000 mgr.y (mgr.14148) 144 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:10 smithi191 conmon[101299]: cluster 2023-10-09T05:19:09.311520+0000 mgr.y (mgr.14148) 145 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:11.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:10 smithi008 conmon[101053]: cluster 2023-10-09T05:19:09.311520+0000 mgr.y (mgr. 2023-10-09T05:19:11.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:10 smithi008 conmon[101053]: 14148) 145 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:11.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:10 smithi008 conmon[110782]: cluster 2023-10-09T05:19:09 2023-10-09T05:19:11.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:10 smithi008 conmon[110782]: .311520+0000 mgr.y (mgr.14148) 145 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:12.034 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:11 smithi191 conmon[101299]: audit 2023-10-09T05:19:11.274897+0000 mon.a ( 2023-10-09T05:19:12.034 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:11 smithi191 conmon[101299]: mon.0) 457 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:19:12.035 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:11 smithi191 conmon[101299]: audit 2023-10-09T05:19:11.275890+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:12.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:11 smithi008 conmon[101053]: audit 2023-10-09T05:19:11.274897+0000 mon.a (mon.0) 457 : 2023-10-09T05:19:12.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:11 smithi008 conmon[101053]: audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:19:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:11 smithi008 conmon[101053]: audit 2023-10-09T05:19:11.275890+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:12.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:11 smithi008 conmon[110782]: audit 2023-10-09T05:19:11. 2023-10-09T05:19:12.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:11 smithi008 conmon[110782]: 274897+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:19:12.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:11 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:19:12.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:11 smithi008 conmon[110782]: 09T05:19:11.275890+0000 2023-10-09T05:19:12.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:11 smithi008 conmon[110782]: mon.a (mon.0) 458 : audit 2023-10-09T05:19:12.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:11 smithi008 conmon[110782]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:13.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:12 smithi191 conmon[101299]: cephadm 2023-10-09T05:19:11.276766+0000 mgr.y (mgr 2023-10-09T05:19:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:12 smithi191 conmon[101299]: .14148) 146 : cephadm [INF] Deploying daemon osd.6 on smithi191 2023-10-09T05:19:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:12 smithi191 conmon[101299]: cluster 2023-10-09T05:19:11.312037+0000 mgr.y (mgr.14148) 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:13.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:12 smithi008 conmon[101053]: cephadm 2023-10-09T05:19:11.276766+0000 mgr.y (mgr.14148 2023-10-09T05:19:13.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:12 smithi008 conmon[101053]: ) 146 : cephadm [INF] Deploying daemon osd.6 on smithi191 2023-10-09T05:19:13.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:12 smithi008 conmon[101053]: cluster 2023-10-09T05:19:11.312037+0000 mgr.y ( 2023-10-09T05:19:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:12 smithi008 conmon[101053]: mgr.14148) 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:12 smithi008 conmon[110782]: cephadm 2023-10-09T05:19:11 2023-10-09T05:19:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:12 smithi008 conmon[110782]: .276766+0000 mgr.y (mgr.14148) 146 : cephadm [INF] Deploying daemon osd.6 on smithi191 2023-10-09T05:19:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:12 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:19:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:12 smithi008 conmon[110782]: 10-09T05:19:11. 2023-10-09T05:19:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:12 smithi008 conmon[110782]: 312037+0000 mgr.y (mgr.14148) 2023-10-09T05:19:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:12 smithi008 conmon[110782]: 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:14 smithi191 conmon[101299]: cluster 2023-10-09T05:19:13. 2023-10-09T05:19:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:14 smithi191 conmon[101299]: 312457+0000 mgr.y (mgr.14148) 148 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:14 smithi008 conmon[101053]: cluster 2023-10-09T05:19:13.312457+0000 mgr.y 2023-10-09T05:19:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:14 smithi008 conmon[101053]: (mgr.14148) 148 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:14 smithi008 conmon[110782]: cluster 2023-10-09T05:19:13.312457+0000 mgr.y ( 2023-10-09T05:19:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:14 smithi008 conmon[110782]: mgr.14148) 148 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:17.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:16 smithi191 conmon[101299]: cluster 2023-10-09T05:19:15.312976+0000 mgr.y (mgr 2023-10-09T05:19:17.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:16 smithi191 conmon[101299]: .14148) 149 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:17.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:16 smithi191 conmon[101299]: audit 2023-10-09T05:19:15.861330+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:17.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:16 smithi191 conmon[101299]: audit 2023-10-09T05:19:15.887915+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:16 smithi008 conmon[101053]: cluster 2023-10-09T05:19:15.312976+0000 mgr.y (mgr.14148 2023-10-09T05:19:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:16 smithi008 conmon[101053]: ) 149 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:16 smithi008 conmon[101053]: audit 2023-10-09T05:19:15.861330+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:16 smithi008 conmon[101053]: audit 2023-10-09T05:19:15.887915+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:16 smithi008 conmon[110782]: cluster 2023-10-09T05:19:15.312976+0000 mgr.y (mgr.14148) 149 : cluster 2023-10-09T05:19:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:16 smithi008 conmon[110782]: [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:16 smithi008 conmon[110782]: audit 2023-10-09T05:19:15.861330+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:16 smithi008 conmon[110782]: audit 2023-10-09T05:19:15.887915+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:18.670 INFO:teuthology.orchestra.run.smithi191.stdout:Created osd(s) 6 on host 'smithi191' 2023-10-09T05:19:18.908 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:18 smithi191 conmon[101299]: cluster 2023-10-09T05:19:17.313459+0000 mgr.y (mgr.14148) 150 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:18.908 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:18 smithi191 conmon[101299]: audit 2023-10-09T05:19:17.674885+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:18.909 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:18 smithi191 conmon[101299]: audit 2023-10-09T05:19:17.675817+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:18.909 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:18 smithi191 conmon[101299]: audit 2023-10-09T05:19:17.676872+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:18.909 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:18 smithi191 conmon[101299]: audit 2023-10-09T05:19:17.681985+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:18.909 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:18 smithi191 conmon[101299]: audit 2023-10-09T05:19:18.669330+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:18.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: cluster 2023-10-09T05:19:17.313459+0000 2023-10-09T05:19:18.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: mgr.y (mgr.14148) 150 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:18.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: audit 2023-10-09T05:19:18.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: 2023-10-09T05:19:17.674885+0000 mon.a (mon 2023-10-09T05:19:18.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: .0) 461 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:18.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:19:18.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: 05:19:17.675817+0000 mon.a (mon.0) 462 : audit [DBG] 2023-10-09T05:19:18.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:18.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: audit 2023-10-09T05:19:18.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: 2023-10-09T05:19:17.676872 2023-10-09T05:19:18.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: +0000 mon.a (mon.0) 463 : 2023-10-09T05:19:18.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:18.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:19:18.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: -09T05:19:17.681985+0000 mon.a (mon. 2023-10-09T05:19:18.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: 0) 464 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:18.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: 2023-10-09T05:19:18.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: audit 2023-10-09T05:19:18.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: 2023-10-09T05 2023-10-09T05:19:18.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: :19:18.669330+0000 mon.a (mon 2023-10-09T05:19:18.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: .0) 465 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:18.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[110782]: 2023-10-09T05:19:18.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: cluster 2023-10-09T05:19:17.313459+0000 mgr.y (mgr.14148 2023-10-09T05:19:18.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: ) 150 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:18.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: audit 2023-10-09T05:19:17.674885+0000 mon.a (mon.0) 461 2023-10-09T05:19:18.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:18.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:19:18.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: 05:19:17.675817+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:18.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: audit 2023-10-09T05:19:17.676872+0000 mon.a (mon. 2023-10-09T05:19:18.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: 0) 463 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:18.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:19:18.971 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: T05:19:17.681985+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:18.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: audit 2023 2023-10-09T05:19:18.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: -10-09T05:19:18 2023-10-09T05:19:18.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: .669330+0000 mon.a (mon.0) 465 : audit 2023-10-09T05:19:18.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:18 smithi008 conmon[101053]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:19.356 DEBUG:teuthology.orchestra.run.smithi191:osd.6> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T05:19:19.362 INFO:tasks.cephadm:Deploying osd.7 on smithi191 with /dev/vg_nvme/lv_1... 2023-10-09T05:19:19.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T05:19:19.389 INFO:journalctl@ceph.osd.6.smithi191.stdout:-- Logs begin at Mon 2023-10-09 04:57:51 UTC. -- 2023-10-09T05:19:19.925 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:19 smithi191 conmon[101299]: audit 2023-10-09T05:19:18.701393+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:19.925 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:19 smithi191 conmon[101299]: 2023-10-09T05:19:19.925 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:19 smithi191 conmon[101299]: audit 2023-10-09T05:19:19.060538+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.191:6816/4267099596,v1:172.21.15.191:6817/4267099596]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:19:19.925 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:19 smithi191 conmon[101299]: audit 2023-10-09T05:19:19.060908+0000 mon.a (mon.0) 467 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:19:19.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[101053]: audit 2023-10-09T05:19:18.701393+0000 mon.a (mon.0) 466 : audit 2023-10-09T05:19:19.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[101053]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[101053]: audit 2023-10-09T05:19:19.060538+0000 mon.b (mon. 2023-10-09T05:19:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[101053]: 1) 17 : audit [INF] from='osd.6 [v2:172.21.15.191:6816/4267099596,v1:172.21.15.191:6817/4267099596]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:19:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[101053]: audit 2023-10-09T05:19:19.060908 2023-10-09T05:19:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[101053]: +0000 mon.a (mon.0) 467 : audit [INF] 2023-10-09T05:19:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[101053]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:19:19.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[110782]: audit 2023-10-09T05:19:18.701393+0000 mon.a 2023-10-09T05:19:19.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[110782]: (mon.0) 466 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:19.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[110782]: audit 2023-10-09T05:19:19.060538 2023-10-09T05:19:19.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[110782]: +0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.191:6816/4267099596,v1:172.21.15.191:6817/4267099596]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:19:19.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[110782]: audit 2023-10-09T05:19:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[110782]: 2023-10-09T05:19: 2023-10-09T05:19:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[110782]: 19.060908+0000 mon.a (mon.0) 467 : audit 2023-10-09T05:19:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:19 smithi008 conmon[110782]: [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:19:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: cluster 2023-10-09T05:19:19.313974+0000 2023-10-09T05:19:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: mgr.y (mgr.14148) 151 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: audit 2023-10-09T05:19:19.681684+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:19:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: 2023-10-09T05:19:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: cluster 2023-10-09T05:19:19.681817+0000 mon.a ( 2023-10-09T05:19:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: mon.0) 469 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T05:19:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: audit 2023-10-09T05:19:19.682175 2023-10-09T05:19:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: +0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: audit 2023-10-09T05:19:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: 2023-10-09T05:19:19.684079+0000 mon.b (mon 2023-10-09T05:19:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: .1) 18 : audit [INF] from='osd.6 [v2:172.21.15.191:6816/4267099596,v1:172.21.15.191:6817/4267099596]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:20.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:19:20.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[101053]: 09T05:19:19.684760+0000 mon.a (mon.0) 471 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:20.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: cluster 2023-10-09T05:19:19.313974+0000 mgr.y (mgr.14148) 2023-10-09T05:19:20.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: 151 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:20.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:19:20.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: T05:19:19.681684+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:19:20.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:19:20.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: 10-09T05:19:19.681817+0000 2023-10-09T05:19:20.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: mon.a (mon.0) 2023-10-09T05:19:20.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: 469 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: audit 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: 2023-10-09T05:19:19. 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: 682175+0000 mon.a (mon.0) 470 : 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: 10-09T05:19:19.684079+0000 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: mon.b (mon.1) 18 : audit [INF] 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: from='osd.6 [v2:172.21.15.191:6816/4267099596,v1:172.21.15.191:6817/4267099596]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: audit 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: 2023-10-09T05:19:19 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: .684760+0000 mon.a (mon.0) 2023-10-09T05:19:20.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:20 smithi008 conmon[110782]: 471 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:21.050 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 05:19:20 smithi191 conmon[119259]: debug 2023-10-09T05:19:20.684+0000 7f4755b32700 -1 osd.6 0 waiting for initial osdmap 2023-10-09T05:19:21.050 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 05:19:20 smithi191 conmon[119259]: debug 2023-10-09T05:19:20.691+0000 7f47502aa700 -1 osd.6 41 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:19:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:20 smithi191 conmon[101299]: cluster 2023-10-09T05:19:19.313974+0000 mgr.y (mgr.14148) 151 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:20 smithi191 conmon[101299]: audit 2023-10-09T05:19:19.681684+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:19:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:20 smithi191 conmon[101299]: cluster 2023-10-09T05:19:19.681817+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T05:19:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:20 smithi191 conmon[101299]: audit 2023-10-09T05:19:19.682175+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:21.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:20 smithi191 conmon[101299]: audit 2023-10-09T05:19:19.684079+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.191:6816/4267099596,v1:172.21.15.191:6817/4267099596]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:21.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:20 smithi191 conmon[101299]: audit 2023-10-09T05:19:19.684760+0000 mon.a (mon.0) 471 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:21.693 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:21 smithi191 conmon[101299]: audit 2023-10-09T05:19:20.683189+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:19:21.693 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:21 smithi191 conmon[101299]: cluster 2023-10-09T05:19:20.683368+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T05:19:21.694 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:21 smithi191 conmon[101299]: audit 2023-10-09T05:19:20.683868+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:21.694 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:21 smithi191 conmon[101299]: audit 2023-10-09T05:19:20.688549+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:21.882 INFO:teuthology.orchestra.run.smithi191.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T05:19:21.883 INFO:teuthology.orchestra.run.smithi191.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T05:19:21.883 INFO:teuthology.orchestra.run.smithi191.stderr: stderr: 10+0 records in 2023-10-09T05:19:21.883 INFO:teuthology.orchestra.run.smithi191.stderr:10+0 records out 2023-10-09T05:19:21.883 INFO:teuthology.orchestra.run.smithi191.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0148494 s, 706 MB/s 2023-10-09T05:19:21.883 INFO:teuthology.orchestra.run.smithi191.stderr:--> Zapping successful for: 2023-10-09T05:19:21.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[101053]: audit 2023-10-09T05:19:20.683189+0000 mon.a (mon. 2023-10-09T05:19:21.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[101053]: 0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:19:21.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[101053]: cluster 2023-10-09T05:19:20.683368+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T05:19:21.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[101053]: audit 2023-10-09T05:19:20.683868+0000 mon.a (mon. 2023-10-09T05:19:21.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[101053]: 0) 474 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:21.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:19:21.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[101053]: :19:20.688549+0000 mon.a (mon.0) 475 : 2023-10-09T05:19:21.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[101053]: audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[110782]: audit 2023-10-09T05:19:20.683189+0000 mon.a (mon 2023-10-09T05:19:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[110782]: .0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:19:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[110782]: cluster 2023-10-09T05:19:20.683368+0000 mon.a (mon 2023-10-09T05:19:21.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[110782]: .0) 473 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T05:19:21.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[110782]: audit 2023-10-09T05:19:21.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[110782]: 2023-10-09T05:19:20.683868+0000 mon.a (mon.0) 2023-10-09T05:19:21.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[110782]: 474 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:21.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:19:21.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[110782]: T05:19:20.688549+0000 mon.a (mon.0) 475 2023-10-09T05:19:21.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:21 smithi008 conmon[110782]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:22.863 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi191:vg_nvme/lv_1 2023-10-09T05:19:22.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: cluster 2023-10-09T05:19:20.015379+0000 osd.6 (osd.6) 1 2023-10-09T05:19:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:19:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: cluster 2023-10-09T05:19:20.015427+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:19:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: cluster 2023-10-09T05:19:21.314350+0000 mgr.y (mgr.14148) 152 : cluster [DBG] 2023-10-09T05:19:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: audit 2023-10-09T05:19:21. 2023-10-09T05:19:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: 690097+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: cluster 2023-10-09T05:19:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: 2023-10-09T05:19:21.694097+0000 mon.a (mon.0) 477 : cluster 2023-10-09T05:19:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: [INF] osd.6 [v2:172.21.15.191:6816/4267099596,v1:172.21.15.191:6817/4267099596] boot 2023-10-09T05:19:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: cluster 2023-10-09T05:19: 2023-10-09T05:19:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: 21.694175+0000 mon.a (mon.0) 478 : cluster [DBG] 2023-10-09T05:19:22.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: osdmap e42: 7 total, 7 up, 7 in 2023-10-09T05:19:22.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: audit 2023-10-09T05:19:21 2023-10-09T05:19:22.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: .694627+0000 mon.a (mon.0) 479 : audit [DBG] 2023-10-09T05:19:22.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:22.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: cluster 2023-10-09T05:19:21 2023-10-09T05:19:22.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: .729474+0000 mon.a (mon.0) 480 : cluster [DBG] 2023-10-09T05:19:22.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[101053]: osdmap e43: 7 total, 7 up, 7 in 2023-10-09T05:19:22.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: cluster 2023-10-09T05:19:20.015379 2023-10-09T05:19:22.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: +0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:19:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: cluster 2023-10-09T05:19:20.015427+0000 osd.6 (osd.6) 2023-10-09T05:19:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:19:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: cluster 2023-10- 2023-10-09T05:19:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: 09T05:19:21.314350+0000 mgr.y (mgr.14148) 152 : 2023-10-09T05:19:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: audit 2023-10-09T05:19:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: 2023-10-09T05:19:21.690097+0000 2023-10-09T05:19:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: mon.a (mon.0) 476 : audit 2023-10-09T05:19:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: cluster 2023-10-09T05:19:21.694097 2023-10-09T05:19:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: +0000 mon.a (mon.0) 477 : cluster [INF] osd.6 [v2:172.21.15.191:6816/4267099596,v1:172.21.15.191:6817/4267099596] boot 2023-10-09T05:19:22.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: cluster 2023-10-09T05:19:21. 2023-10-09T05:19:22.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: 694175+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T05:19:22.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:19:22.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: :19:21.694627+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:22.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: cluster 2023-10- 2023-10-09T05:19:22.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:22 smithi008 conmon[110782]: 09T05:19:21.729474+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T05:19:23.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:22 smithi191 conmon[101299]: cluster 2023-10-09T05:19:20.015379+0000 osd.6 2023-10-09T05:19:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:22 smithi191 conmon[101299]: (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:19:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:22 smithi191 conmon[101299]: cluster 2023-10-09T05:19:20.015427+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:19:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:22 smithi191 conmon[101299]: cluster 2023-10-09T05:19:21.314350+0000 mgr.y (mgr.14148) 152 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:19:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:22 smithi191 conmon[101299]: audit 2023-10-09T05:19:21.690097+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:22 smithi191 conmon[101299]: cluster 2023-10-09T05:19:21.694097+0000 mon.a (mon.0) 477 : cluster [INF] osd.6 [v2:172.21.15.191:6816/4267099596,v1:172.21.15.191:6817/4267099596] boot 2023-10-09T05:19:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:22 smithi191 conmon[101299]: cluster 2023-10-09T05:19:21.694175+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T05:19:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:22 smithi191 conmon[101299]: audit 2023-10-09T05:19:21.694627+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:19:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:22 smithi191 conmon[101299]: cluster 2023-10-09T05:19:21.729474+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T05:19:24.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:23 smithi191 conmon[101299]: cluster 2023-10-09T05:19:22.734701+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T05:19:24.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:23 smithi191 conmon[101299]: audit 2023-10-09T05:19:22.964000+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:24.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:23 smithi191 conmon[101299]: audit 2023-10-09T05:19:22.965326+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:24.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:23 smithi191 conmon[101299]: audit 2023-10-09T05:19:22.966298+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:24.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:23 smithi191 conmon[101299]: audit 2023-10-09T05:19:22.967202+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:24.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:23 smithi191 conmon[101299]: audit 2023-10-09T05:19:22.971797+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:24.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:23 smithi191 conmon[101299]: audit 2023-10-09T05:19:22.972890+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:24.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:23 smithi191 conmon[101299]: audit 2023-10-09T05:19:22.973790 2023-10-09T05:19:24.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:23 smithi191 conmon[101299]: +0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:24.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:23 smithi191 conmon[101299]: audit 2023-10-09T05:19:22.978400+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: cluster 2023-10-09T05:19:22.734701+0000 mon.a (mon.0) 2023-10-09T05:19:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: 481 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T05:19:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: audit 2023-10-09T05:19:22.964000+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: audit 2023-10-09T05:19:22.965326+0000 mon.a (mon.0 2023-10-09T05:19:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: ) 483 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:19:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: 19:22.966298+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: audit 2023-10-09T05:19:22.967202+0000 mon.a (mon.0) 485 : 2023-10-09T05:19:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:19:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: 19:22.971797+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:19:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: 10-09T05:19:22.972890+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: audit 2023-10-09T05:19:22.973790+0000 mon.a ( 2023-10-09T05:19:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: mon.0) 488 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: audit 2023-10-09T05:19:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[101053]: 2023-10-09T05:19:22.978400+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:24.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: cluster 2023-10-09T05:19:22.734701+0000 mon.a (mon.0 2023-10-09T05:19:24.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: ) 481 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: audit 2023-10-09T05:19:22.964000+0000 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: mon.a (mon.0) 482 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: 19:22.965326+0000 mon.a (mon.0) 483 : audit 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: 05:19:22.966298+0000 mon.a (mon.0) 484 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: audit 2023-10-09T05:19:22.967202+0000 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: mon.a (mon.0) 485 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: audit 2023-10-09T05:19:22. 2023-10-09T05:19:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: 971797+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:19:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: :19:22.972890+0000 mon.a (mon.0) 487 : audit 2023-10-09T05:19:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:19:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: 19:22.973790+0000 mon.a (mon.0) 488 : audit 2023-10-09T05:19:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:19:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: 19:22.978400+0000 mon.a (mon.0) 489 : audit 2023-10-09T05:19:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:23 smithi008 conmon[110782]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:25.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:24 smithi191 conmon[101299]: cephadm 2023-10-09T05:19: 2023-10-09T05:19:25.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:24 smithi191 conmon[101299]: 22.958724+0000 mgr.y (mgr.14148) 153 : cephadm [INF] Detected new or changed devices on smithi191 2023-10-09T05:19:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:24 smithi191 conmon[101299]: cephadm 2023-10-09T05:19:22.967894+0000 mgr.y (mgr.14148) 154 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 5722M 2023-10-09T05:19:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:24 smithi191 conmon[101299]: cluster 2023-10-09T05:19:23.314718+0000 mgr.y (mgr.14148) 155 : cluster [DBG] pgmap v141: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:24 smithi191 conmon[101299]: audit 2023-10-09T05:19:24.616715+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:19:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:24 smithi191 conmon[101299]: audit 2023-10-09T05:19:24.619659+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:19:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:24 smithi191 conmon[101299]: audit 2023-10-09T05:19:24.620506+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:25.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[101053]: cephadm 2023-10-09T05:19:22.958724+0000 mgr.y (mgr.14148) 153 : cephadm [INF] Detected new or changed devices on smithi191 2023-10-09T05:19:25.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[101053]: cephadm 2023-10-09T05: 2023-10-09T05:19:25.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[101053]: 19:22.967894+0000 mgr.y (mgr.14148) 154 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 5722M 2023-10-09T05:19:25.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[101053]: cluster 2023-10-09T05:19:23.314718+0000 mgr.y (mgr.14148) 155 : cluster [DBG] pgmap v141: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:25.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[101053]: audit 2023-10-09T05:19:24.616715+0000 2023-10-09T05:19:25.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[101053]: mon.a (mon.0) 490 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:19:25.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[101053]: audit 2023-10-09T05:19:24.619659+0000 mon.a (mon 2023-10-09T05:19:25.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[101053]: .0) 491 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:19:25.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:19:25.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[101053]: :19:24.620506+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: cephadm 2023-10-09T05:19:22 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: .958724+0000 mgr.y (mgr.14148) 153 : cephadm [INF] Detected new or changed devices on smithi191 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: cephadm 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: 2023-10-09T05:19:22.967894 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: +0000 mgr.y (mgr.14148) 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: 154 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 5722M 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: 10-09T05:19:23.314718+0000 mgr.y (mgr.14148) 155 : 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: cluster [DBG] pgmap v141: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: audit 2023-10-09T05:19:24.616715+0000 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: mon.a (mon.0) 490 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:19:25.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: audit 2023-10-09T05:19:24. 2023-10-09T05:19:25.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: 619659+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:19:25.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: audit 2023-10-09T05:19:24. 2023-10-09T05:19:25.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:24 smithi008 conmon[110782]: 620506+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:26.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:25 smithi191 conmon[101299]: audit 2023-10-09T05:19:24.613745+0000 mgr.y (mgr.14148) 2023-10-09T05:19:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:25 smithi191 conmon[101299]: 156 : audit [DBG] from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:26.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:25 smithi008 conmon[101053]: audit 2023-10-09T05:19:24.613745+0000 mgr.y (mgr.14148) 2023-10-09T05:19:26.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:25 smithi008 conmon[101053]: 156 : audit [DBG] from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:26.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:25 smithi008 conmon[110782]: audit 2023-10-09T05:19:24. 2023-10-09T05:19:26.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:25 smithi008 conmon[110782]: 613745+0000 mgr.y (mgr.14148) 156 : audit [DBG] from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:27.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:26 smithi191 conmon[101299]: cluster 2023-10-09T05:19:25.315117+0000 mgr.y (mgr.14148) 157 : cluster [DBG] pgmap v142: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:27.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:26 smithi008 conmon[101053]: cluster 2023-10-09T05:19:25.315117+0000 mgr.y (mgr.14148) 157 : cluster [DBG] pgmap v142: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:27.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:26 smithi008 conmon[110782]: cluster 2023-10-09T05:19:25.315117+0000 mgr.y (mgr.14148) 157 : cluster 2023-10-09T05:19:27.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:26 smithi008 conmon[110782]: [DBG] pgmap v142: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: audit 2023-10-09T05:19:26.800493+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.191:0/2690741533' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5ac2e382-7341-47dd-b6c9-0fbdba12829c"}]: dispatch 2023-10-09T05:19:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: audit 2023-10-09T05:19:26.800965+0000 mon.a (mon.0) 493 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5ac2e382-7341-47dd-b6c9-0fbdba12829c"}]: dispatch 2023-10-09T05:19:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: audit 2023-10-09T05:19:26.805531+0000 mon.a 2023-10-09T05:19:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: (mon.0) 494 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5ac2e382-7341-47dd-b6c9-0fbdba12829c"}]': finished 2023-10-09T05:19:28.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: cluster 2023-10-09T05:19:28.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: 2023-10-09T05:19:26.805631+0000 mon.a (mon.0 2023-10-09T05:19:28.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: ) 495 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T05:19:28.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: audit 2023-10-09T05:19 2023-10-09T05:19:28.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: :26.805852+0000 mon.a (mon.0) 496 2023-10-09T05:19:28.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:28.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:19:28.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: 19:27.420140+0000 mon.b (mon.1) 2023-10-09T05:19:28.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:27 smithi191 conmon[101299]: 20 : audit [DBG] from='client.? 172.21.15.191:0/3584196722' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:19:28.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: audit 2023-10-09T05:19:26.800493+0000 mon.b (mon.1) 2023-10-09T05:19:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: 19 : audit [INF] from='client.? 172.21.15.191:0/2690741533' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5ac2e382-7341-47dd-b6c9-0fbdba12829c"}]: dispatch 2023-10-09T05:19:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: audit 2023-10-09T05:19:26.800965+0000 mon.a (mon.0) 493 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5ac2e382-7341-47dd-b6c9-0fbdba12829c"}]: dispatch 2023-10-09T05:19:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: audit 2023-10-09T05:19:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: 2023-10-09T05:19:26.805531+0000 mon.a (mon.0) 494 2023-10-09T05:19:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5ac2e382-7341-47dd-b6c9-0fbdba12829c"}]': finished 2023-10-09T05:19:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: cluster 2023-10-09T05:19:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: 2023-10- 2023-10-09T05:19:28.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: 09T05:19:26.805631+0000 mon.a (mon.0) 2023-10-09T05:19:28.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: 495 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T05:19:28.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: audit 2023-10-09T05:19:28.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: 2023-10-09T05:19:26.805852+0000 mon.a ( 2023-10-09T05:19:28.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: mon.0) 496 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:28.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:19:28.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: 10-09T05:19:27.420140+0000 mon.b (mon.1 2023-10-09T05:19:28.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[101053]: ) 20 : audit [DBG] from='client.? 172.21.15.191:0/3584196722' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:19:28.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: audit 2023-10-09T05:19:26.800493 2023-10-09T05:19:28.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: +0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.191:0/2690741533' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5ac2e382-7341-47dd-b6c9-0fbdba12829c"}]: dispatch 2023-10-09T05:19:28.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:19:28.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: T05:19:26.800965+0000 mon.a (mon.0) 493 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5ac2e382-7341-47dd-b6c9-0fbdba12829c"}]: dispatch 2023-10-09T05:19:28.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: audit 2023 2023-10-09T05:19:28.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: -10-09T05: 2023-10-09T05:19:28.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: 19:26.805531+0000 mon.a ( 2023-10-09T05:19:28.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: mon.0) 494 : audit [INF] 2023-10-09T05:19:28.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5ac2e382-7341-47dd-b6c9-0fbdba12829c"}]': finished 2023-10-09T05:19:28.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: cluster 2023-10-09T05:19:28.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: 2023-10-09T05:19:26.805631+0000 mon.a 2023-10-09T05:19:28.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: (mon.0) 495 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T05:19:28.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: 2023-10-09T05:19:28.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:19:28.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: 09T05:19:26.805852+0000 mon.a (mon.0) 496 : audit [DBG] 2023-10-09T05:19:28.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:28.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:19:28.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:27 smithi008 conmon[110782]: 05:19:27.420140+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.191:0/3584196722' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:19:29.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:28 smithi191 conmon[101299]: cluster 2023-10-09T05:19:27.315542+0000 mgr.y (mgr.14148) 158 : cluster [DBG] pgmap v144: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:29.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:28 smithi008 conmon[101053]: cluster 2023-10-09T05:19:27.315542+0000 mgr.y (mgr.14148 2023-10-09T05:19:29.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:28 smithi008 conmon[101053]: ) 158 : cluster [DBG] pgmap v144: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:29.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:28 smithi008 conmon[110782]: cluster 2023-10-09T05:19:27.315542+0000 mgr.y (mgr.14148 2023-10-09T05:19:29.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:28 smithi008 conmon[110782]: ) 158 : cluster [DBG] pgmap v144: 1 pgs: 1 peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:30 smithi191 conmon[101299]: cluster 2023-10-09T05:19:29.316059+0000 mgr.y (mgr.14148 2023-10-09T05:19:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:30 smithi191 conmon[101299]: ) 159 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:31.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:30 smithi008 conmon[101053]: cluster 2023-10-09T05:19:29.316059+0000 mgr.y (mgr.14148 2023-10-09T05:19:31.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:30 smithi008 conmon[101053]: ) 159 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:31.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:30 smithi008 conmon[110782]: cluster 2023-10-09T05:19:29.316059+0000 mgr.y (mgr.14148) 159 : cluster [DBG] 2023-10-09T05:19:31.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:30 smithi008 conmon[110782]: pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:33.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:32 smithi191 conmon[101299]: cluster 2023-10-09T05:19:31.316415+0000 mgr.y (mgr.14148) 160 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:33.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:32 smithi008 conmon[101053]: cluster 2023-10-09T05:19:31.316415+0000 mgr.y (mgr 2023-10-09T05:19:33.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:32 smithi008 conmon[101053]: .14148) 160 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:32 smithi008 conmon[110782]: cluster 2023-10-09T05:19:31.316415+0000 mgr.y (mgr.14148) 160 : cluster [DBG] 2023-10-09T05:19:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:32 smithi008 conmon[110782]: pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:33 smithi191 conmon[101299]: audit 2023-10-09T05:19:32.913982+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:19:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:33 smithi191 conmon[101299]: audit 2023-10-09T05:19:32.914860+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:34.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:33 smithi008 conmon[101053]: audit 2023-10-09T05:19:32.913982+0000 mon.a (mon.0) 497 2023-10-09T05:19:34.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:33 smithi008 conmon[101053]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:19:34.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:33 smithi008 conmon[101053]: audit 2023-10-09T05:19:32.914860+0000 mon.a (mon.0) 498 : audit [DBG] 2023-10-09T05:19:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:33 smithi008 conmon[101053]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:33 smithi008 conmon[110782]: audit 2023-10-09T05:19:32.913982+0000 mon.a (mon.0) 497 : audit [INF] 2023-10-09T05:19:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:33 smithi008 conmon[110782]: from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:19:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:33 smithi008 conmon[110782]: audit 2023-10-09T05:19:32.914860+0000 mon.a (mon.0) 498 : audit 2023-10-09T05:19:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:33 smithi008 conmon[110782]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:35.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:34 smithi191 conmon[101299]: cephadm 2023-10-09T05:19:32.915614+0000 2023-10-09T05:19:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:34 smithi191 conmon[101299]: mgr.y (mgr.14148) 161 : cephadm [INF] Deploying daemon osd.7 on smithi191 2023-10-09T05:19:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:34 smithi191 conmon[101299]: cluster 2023-10-09T05:19:33.316952+0000 mgr.y (mgr.14148) 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:35.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:34 smithi008 conmon[101053]: cephadm 2023-10-09T05:19:32.915614 2023-10-09T05:19:35.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:34 smithi008 conmon[101053]: +0000 mgr.y (mgr.14148) 161 : cephadm [INF] Deploying daemon osd.7 on smithi191 2023-10-09T05:19:35.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:34 smithi008 conmon[101053]: cluster 2023-10-09T05:19:33. 2023-10-09T05:19:35.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:34 smithi008 conmon[101053]: 316952+0000 mgr.y (mgr.14148) 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:34 smithi008 conmon[110782]: cephadm 2023-10-09T05:19:32.915614+0000 mgr.y (mgr 2023-10-09T05:19:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:34 smithi008 conmon[110782]: .14148) 161 : cephadm [INF] Deploying daemon osd.7 on smithi191 2023-10-09T05:19:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:34 smithi008 conmon[110782]: cluster 2023-10 2023-10-09T05:19:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:34 smithi008 conmon[110782]: -09T05:19:33.316952+0000 mgr.y (mgr.14148) 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:36 smithi191 conmon[101299]: cluster 2023-10-09T05:19:35.317527+0000 mgr.y (mgr.14148) 163 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:37.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:36 smithi008 conmon[110782]: cluster 2023-10-09T05:19:35.317527+0000 mgr.y (mgr.14148) 163 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:36 smithi008 conmon[101053]: cluster 2023-10-09T05:19:35.317527+0000 mgr.y (mgr.14148) 2023-10-09T05:19:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:36 smithi008 conmon[101053]: 163 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:38.855 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:38 smithi191 conmon[101299]: cluster 2023-10-09T05:19:37.317970+0000 mgr.y (mgr.14148) 164 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:38.855 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:38 smithi191 conmon[101299]: audit 2023-10-09T05:19:37.814947+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:38.855 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:38 smithi191 conmon[101299]: audit 2023-10-09T05:19:37.834654+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:39.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[101053]: cluster 2023-10-09T05:19:37.317970+0000 mgr.y ( 2023-10-09T05:19:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[101053]: mgr.14148) 164 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[101053]: audit 2023-10-09T05:19 2023-10-09T05:19:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[101053]: :37.814947+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[101053]: audit 2023-10-09T05:19:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[101053]: 2023-10-09T05:19: 2023-10-09T05:19:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[101053]: 37.834654+0000 mon.a (mon.0) 2023-10-09T05:19:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[101053]: 500 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[110782]: cluster 2023-10-09T05:19:37.317970+0000 mgr.y ( 2023-10-09T05:19:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[110782]: mgr.14148) 164 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:19:39.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[110782]: 05:19:37.814947+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:39.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:19:39.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[110782]: -09T05:19:37.834654+0000 mon.a (mon.0 2023-10-09T05:19:39.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:38 smithi008 conmon[110782]: ) 500 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:40.815 INFO:teuthology.orchestra.run.smithi191.stdout:Created osd(s) 7 on host 'smithi191' 2023-10-09T05:19:41.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:40 smithi191 conmon[101299]: cluster 2023-10-09T05:19:39.318492+0000 mgr.y (mgr.14148) 165 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:41.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:40 smithi191 conmon[101299]: audit 2023-10-09T05:19:40.001227+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:41.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:40 smithi191 conmon[101299]: audit 2023-10-09T05:19:40.326375+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:41.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:40 smithi191 conmon[101299]: audit 2023-10-09T05:19:40.327473+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:41.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:40 smithi191 conmon[101299]: audit 2023-10-09T05:19:40.333269+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:41.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[101053]: cluster 2023-10-09T05:19:39.318492+0000 mgr.y (mgr.14148) 2023-10-09T05:19:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[101053]: 165 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[101053]: audit 2023-10-09T05:19:40.001227+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:19:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[101053]: :19:40.326375+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[101053]: audit 2023-10-09T05:19:40.327473+0000 mon.a (mon.0 2023-10-09T05:19:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[101053]: ) 503 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[101053]: audit 2023-10-09T05:19:40.333269+0000 2023-10-09T05:19:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[101053]: mon.a (mon.0) 504 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: cluster 2023-10-09T05:19:39 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: .318492+0000 mgr.y (mgr.14148) 165 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: audit 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: 2023-10-09T05:19:40.001227 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: +0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: audit 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: 2023-10-09T05:19:40.326375 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: +0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: -09T05:19:40.327473+0000 mon.a (mon.0) 503 : 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: audit 2023-10-09T05:19:40. 2023-10-09T05:19:41.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:40 smithi008 conmon[110782]: 333269+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:41.444 DEBUG:teuthology.orchestra.run.smithi191:osd.7> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T05:19:41.446 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T05:19:41.446 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T05:19:41.476 INFO:journalctl@ceph.osd.7.smithi191.stdout:-- Logs begin at Mon 2023-10-09 04:57:51 UTC. -- 2023-10-09T05:19:42.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:41 smithi191 conmon[101299]: audit 2023-10-09T05:19:40.814170+0000 2023-10-09T05:19:42.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:41 smithi191 conmon[101299]: mon.a (mon.0) 505 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:42.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:41 smithi191 conmon[101299]: audit 2023-10-09T05:19:40.851300+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:42.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:41 smithi191 conmon[101299]: audit 2023-10-09T05:19:41.054422+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.191:6824/1787611943,v1:172.21.15.191:6825/1787611943]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:19:42.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:41 smithi191 conmon[101299]: audit 2023-10-09T05:19:41.054957+0000 mon.a (mon.0) 507 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:19:42.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:41 smithi191 conmon[101299]: cluster 2023-10-09T05:19:41.318912+0000 mgr.y (mgr.14148) 166 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:42.091 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[101053]: audit 2023-10-09T05:19:40.814170+0000 mon.a ( 2023-10-09T05:19:42.091 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[101053]: mon.0) 505 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:42.091 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[101053]: audit 2023-10-09T05:19:40.851300+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:42.091 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[101053]: audit 2023-10-09T05:19:41.054422+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.191:6824/1787611943,v1:172.21.15.191:6825/1787611943]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:19:42.091 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[101053]: audit 2023-10-09T05:19:41.054957+0000 mon.a (mon.0) 507 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:19:42.091 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[101053]: cluster 2023-10-09T05:19:41.318912+0000 mgr.y (mgr.14148) 166 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:42.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: audit 2023-10-09T05:19:40.814170+0000 mon.a 2023-10-09T05:19:42.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: (mon.0) 505 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:42.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: audit 2023-10-09T05:19 2023-10-09T05:19:42.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: :40.851300+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:42.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: audit 2023 2023-10-09T05:19:42.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: -10-09T05:19:41.054422+0000 mon.b (mon.1 2023-10-09T05:19:42.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: ) 21 : audit [INF] from='osd.7 [v2:172.21.15.191:6824/1787611943,v1:172.21.15.191:6825/1787611943]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:19:42.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:19:42.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: 05:19:41.054957+0000 mon.a (mon.0) 507 2023-10-09T05:19:42.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:19:42.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: cluster 2023-10-09T05:19:41.318912+0000 2023-10-09T05:19:42.093 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:41 smithi008 conmon[110782]: mgr.y (mgr.14148) 166 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:42.937 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:42 smithi008 conmon[101053]: audit 2023-10-09T05:19:41.819319+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:19:42.937 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:42 smithi008 conmon[101053]: cluster 2023-10-09T05:19:41.819385+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T05:19:42.937 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:42 smithi008 conmon[101053]: audit 2023-10-09T05:19:41.819519+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:42.937 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:42 smithi008 conmon[101053]: audit 2023-10-09T05:19:41.819932+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.191:6824/1787611943,v1:172.21.15.191:6825/1787611943]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:42.937 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:42 smithi008 conmon[101053]: audit 2023-10-09T05:19:41.820213+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:42.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:42 smithi008 conmon[110782]: audit 2023-10-09T05:19:41.819319+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:19:42.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:42 smithi008 conmon[110782]: cluster 2023-10-09T05:19:41.819385+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T05:19:42.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:42 smithi008 conmon[110782]: audit 2023-10-09T05:19:41.819519+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:42.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:42 smithi008 conmon[110782]: audit 2023-10-09T05:19:41.819932+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.191:6824/1787611943,v1:172.21.15.191:6825/1787611943]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:42.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:42 smithi008 conmon[110782]: audit 2023-10-09T05:19:41.820213+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:43.245 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:19:42 smithi191 conmon[125089]: debug 2023-10-09T05:19:42.823+0000 7fee8a090700 -1 osd.7 0 waiting for initial osdmap 2023-10-09T05:19:43.245 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:19:42 smithi191 conmon[125089]: debug 2023-10-09T05:19:42.830+0000 7fee85009700 -1 osd.7 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:19:43.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:42 smithi191 conmon[101299]: audit 2023-10-09T05:19:41.819319+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:19:43.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:42 smithi191 conmon[101299]: cluster 2023-10-09T05:19:41.819385+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T05:19:43.246 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:42 smithi191 conmon[101299]: audit 2023-10-09T05:19:41.819519+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:43.246 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:42 smithi191 conmon[101299]: audit 2023-10-09T05:19:41.819932+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.191:6824/1787611943,v1:172.21.15.191:6825/1787611943]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:43.246 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:42 smithi191 conmon[101299]: audit 2023-10-09T05:19:41.820213+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:19:43.253 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:19:43.870 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":47,"num_osds":8,"num_up_osds":7,"osd_up_since":1696828761,"num_in_osds":8,"osd_in_since":1696828766,"num_remapped_pgs":0} 2023-10-09T05:19:44.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[101053]: audit 2023-10-09T05:19:42.820321+0000 mon.a (mon 2023-10-09T05:19:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[101053]: .0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:19:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[101053]: cluster 2023-10-09T05:19:42.820420+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T05:19:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[101053]: audit 2023-10-09T05:19:42.820792+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[101053]: audit 2023-10-09T05:19:42.824918+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[101053]: audit 2023-10-09T05:19:43.254204+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.8:0/4235291181' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:19:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[101053]: cluster 2023-10-09T05:19:43.319229+0000 mgr.y (mgr.14148) 167 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:19:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[110782]: 05:19:42.820321+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:19:44.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[110782]: cluster 2023-10-09T05:19:42.820420+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T05:19:44.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[110782]: audit 2023-10-09T05:19:42.820792+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:44.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[110782]: audit 2023-10-09T05:19:42.824918+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:44.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[110782]: audit 2023-10-09T05:19:43.254204+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.8:0/4235291181' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:19:44.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:43 smithi008 conmon[110782]: cluster 2023-10-09T05:19:43.319229+0000 mgr.y (mgr.14148) 167 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:44.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:43 smithi191 conmon[101299]: audit 2023-10-09T05:19:42.820321+0000 mon.a (mon.0) 512 : audit [INF] 2023-10-09T05:19:44.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:43 smithi191 conmon[101299]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2023-10-09T05:19:44.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:43 smithi191 conmon[101299]: cluster 2023-10-09T05:19:42.820420+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T05:19:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:43 smithi191 conmon[101299]: audit 2023-10-09T05:19:42.820792+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:43 smithi191 conmon[101299]: audit 2023-10-09T05:19:42.824918+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:43 smithi191 conmon[101299]: audit 2023-10-09T05:19:43.254204+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.8:0/4235291181' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:19:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:43 smithi191 conmon[101299]: cluster 2023-10-09T05:19:43.319229+0000 mgr.y (mgr.14148) 167 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:19:44.871 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T05:19:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: cluster 2023-10-09T05:19:42.014715+0000 osd.7 (osd.7 2023-10-09T05:19:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: ) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:19:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: cluster 2023-10-09T05:19:42.014769+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:19:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: audit 2023-10-09T05:19:43.824885+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: cluster 2023-10-09T05:19:43.828810+0000 mon.a 2023-10-09T05:19:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: (mon.0) 517 : cluster [INF] osd.7 [v2:172.21.15.191:6824/1787611943,v1:172.21.15.191:6825/1787611943] boot 2023-10-09T05:19:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: cluster 2023-10-09T 2023-10-09T05:19:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: 05:19:43.828866+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T05:19:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: audit 2023-10-09T05:19:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: 2023-10-09T05:19:43.829271+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: cephadm 2023-10-09T05 2023-10-09T05:19:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: :19:44.062700+0000 mgr.y (mgr.14148) 168 : cephadm [INF] Detected new or changed devices on smithi191 2023-10-09T05:19:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:19:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: :19:44.068470+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: audit 2023-10-09T05:19: 2023-10-09T05:19:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: 44.069444+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: audit 2023-10-09T05:19:44.070264+0000 2023-10-09T05:19:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: mon.a (mon.0) 522 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: audit 2023-10-09T05:19:44.070995+0000 mon.a (mon.0) 2023-10-09T05:19:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: 523 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: audit 2023-10-09T05:19:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: 2023-10-09T05:19:44.071729+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: cephadm 2023-10-09T05:19:44. 2023-10-09T05:19:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: 072309+0000 mgr.y (mgr.14148) 169 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 4292M 2023-10-09T05:19:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: audit 2023-10-09T05:19:44.076833+0000 mon.a ( 2023-10-09T05:19:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: mon.0) 525 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: audit 2023-10-09T05:19:44.077682+0000 mon.a (mon.0) 2023-10-09T05:19:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: 526 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: audit 2023 2023-10-09T05:19:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: -10-09T05:19:44.078384+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: audit 2023-10-09T05:19: 2023-10-09T05:19:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[110782]: 44.082521+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:45.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: cluster 2023-10-09T05:19:42.014715+0000 osd.7 (osd.7) 1 : cluster 2023-10-09T05:19:45.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: [DBG] purged_snaps scrub starts 2023-10-09T05:19:45.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: cluster 2023-10-09T05:19:42.014769+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:19:45.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: audit 2023-10-09T05:19:43.824885+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:45.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: cluster 2023-10-09T05:19:43.828810+0000 mon.a (mon.0) 517 : cluster [INF] osd.7 [v2:172.21.15.191:6824/1787611943,v1:172.21.15.191:6825/1787611943] boot 2023-10-09T05:19:45.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: cluster 2023-10-09T05:19:43.828866+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T05:19:45.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: audit 2023-10-09T05:19:45.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: 2023-10-09T05:19:43.829271+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:45.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: cephadm 2023-10-09T05:19:44.062700+0000 mgr.y (mgr 2023-10-09T05:19:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: .14148) 168 : cephadm [INF] Detected new or changed devices on smithi191 2023-10-09T05:19:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:19:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: :19:44.068470+0000 mon.a (mon.0) 520 : audit 2023-10-09T05:19:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: audit 2023-10-09T05:19:44.069444 2023-10-09T05:19:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: +0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:19:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: 10-09T05:19:44.070264+0000 mon.a (mon.0) 522 2023-10-09T05:19:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: audit 2023-10-09T05:19:44.070995 2023-10-09T05:19:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: +0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: 10-09T05:19:44.071729+0000 mon.a (mon.0) 524 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: cephadm 2023-10-09T05:19:44. 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: 072309+0000 mgr.y (mgr.14148) 169 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 4292M 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: audit 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: 2023-10-09T05:19:44.076833+0000 mon.a (mon.0 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: ) 525 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: audit 2023-10-09T05:19: 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: 44.077682+0000 mon.a (mon.0) 526 : audit 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:45.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: audit 2023-10-09T05:19:44.078384+0000 2023-10-09T05:19:45.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: mon.a (mon.0) 527 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:45.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:19:45.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: 10-09T05:19:44.082521+0000 mon.a (mon.0) 2023-10-09T05:19:45.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:44 smithi008 conmon[101053]: 528 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: cluster 2023-10-09T05:19:42.014715+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:19:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: cluster 2023-10-09T05 2023-10-09T05:19:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: :19:42.014769+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:19:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit 2023-10-09T05:19:43.824885+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: cluster 2023-10-09T05:19:43.828810+0000 mon.a (mon.0) 517 : cluster [INF] 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: osd.7 [v2:172.21.15.191:6824/1787611943,v1:172.21.15.191:6825/1787611943] boot 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: cluster 2023-10-09T05:19:43.828866+0000 mon.a 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: (mon.0) 518 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: 19:43.829271+0000 mon.a (mon.0) 519 : 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: cephadm 2023-10-09T05:19: 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: 44.062700+0000 mgr.y (mgr.14148) 168 : cephadm 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: [INF] Detected new or changed devices on smithi191 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit 2023-10-09T05:19:44. 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: 068470+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: 2023-10-09T05:19:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: 2023-10-09T05:19:44.069444+0000 mon.a (mon.0 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: ) 521 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: 05:19:44.070264+0000 mon.a (mon.0 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: ) 522 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: 05:19:44.070995+0000 mon.a (mon.0 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: ) 523 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit 2023-10-09T05:19:44. 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: 071729+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: cephadm 2023-10 2023-10-09T05:19:45.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: -09T05:19:44.072309+0000 mgr.y ( 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: mgr.14148) 169 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 4292M 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: 10-09T05:19:44.076833+0000 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: mon.a (mon.0) 525 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: -09T05:19:44.077682+0000 mon.a ( 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: mon.0) 526 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit 2023-10-09T05:19:44.078384+0000 mon.a (mon.0) 527 : audit 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: T05:19:44.082521+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:44 smithi191 conmon[101299]: 2023-10-09T05:19:45.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:45 smithi008 conmon[101053]: cluster 2023-10-09T05:19:44.830458+0000 mon.a (mon.0) 529 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T05:19:45.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:45 smithi008 conmon[101053]: cluster 2023-10-09T05:19:45.319555+0000 mgr.y (mgr.14148) 170 : cluster [DBG] pgmap v157: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:45.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:45 smithi008 conmon[110782]: cluster 2023-10-09T05:19:44.830458+0000 mon.a (mon.0) 529 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T05:19:45.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:45 smithi008 conmon[110782]: cluster 2023-10-09T05:19:45.319555+0000 mgr.y (mgr.14148) 170 : cluster [DBG] pgmap v157: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:46.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:45 smithi191 conmon[101299]: cluster 2023-10-09T05:19:44.830458+0000 mon.a (mon.0) 529 : cluster 2023-10-09T05:19:46.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:45 smithi191 conmon[101299]: [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T05:19:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:45 smithi191 conmon[101299]: cluster 2023-10-09T05:19:45.319555+0000 mgr.y (mgr.14148) 170 : cluster [DBG] pgmap v157: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:46.710 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:19:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:46 smithi008 conmon[101053]: cluster 2023-10-09T05:19:45.833509 2023-10-09T05:19:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:46 smithi008 conmon[101053]: +0000 mon.a (mon.0) 530 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:19:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:46 smithi008 conmon[101053]: cluster 2023-10-09T05:19:45.841199+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T05:19:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:46 smithi008 conmon[101053]: audit 2023-10-09T05:19:46.710464+0000 mon.a (mon.0) 532 : audit [DBG] from='client.? 172.21.15.8:0/2658969258' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:19:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:46 smithi008 conmon[110782]: cluster 2023-10-09T05:19:45.833509+0000 mon.a ( 2023-10-09T05:19:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:46 smithi008 conmon[110782]: mon.0) 530 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:19:47.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:46 smithi008 conmon[110782]: cluster 2023-10-09T05:19:45.841199+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T05:19:47.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:46 smithi008 conmon[110782]: audit 2023-10-09T05:19:46.710464+0000 mon.a (mon.0) 532 : audit [DBG] from='client.? 172.21.15.8:0/2658969258' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:19:47.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:46 smithi191 conmon[101299]: cluster 2023-10-09T05:19:45.833509+0000 mon.a (mon.0) 530 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:19:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:46 smithi191 conmon[101299]: cluster 2023-10-09T05:19:45.841199+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T05:19:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:46 smithi191 conmon[101299]: audit 2023-10-09T05:19:46.710464+0000 mon.a (mon.0) 532 : audit [DBG] from='client.? 172.21.15.8:0/2658969258' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:19:47.354 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1696828783,"num_in_osds":8,"osd_in_since":1696828766,"num_remapped_pgs":0} 2023-10-09T05:19:47.354 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:19:47.715 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:19:48.135 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:47 smithi008 conmon[101053]: cluster 2023-10-09T05:19:47. 2023-10-09T05:19:48.135 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:47 smithi008 conmon[101053]: 319991+0000 mgr.y (mgr.14148) 171 : cluster [DBG] pgmap v159: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:48.136 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:47 smithi008 conmon[110782]: cluster 2023-10-09T05:19:47.319991+0000 mgr.y (mgr 2023-10-09T05:19:48.136 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:47 smithi008 conmon[110782]: .14148) 171 : cluster [DBG] pgmap v159: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:47 smithi191 conmon[101299]: cluster 2023-10-09T05:19:47.319991+0000 mgr.y (mgr.14148) 171 : cluster 2023-10-09T05:19:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:47 smithi191 conmon[101299]: [DBG] pgmap v159: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:49.327 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:19:49.327 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":50,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:14:56.160913+0000","modified":"2023-10-09T05:19:45.833740+0000","last_up_change":"2023-10-09T05:19:43.821876+0000","last_in_change":"2023-10-09T05:19:26.801646+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:17:53.386952+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6802","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6803","nonce":791166472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6804","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6805","nonce":791166472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6808","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6809","nonce":791166472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6806","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6807","nonce":791166472}]},"public_addr":"172.21.15.8:6803/791166472","cluster_addr":"172.21.15.8:6805/791166472","heartbeat_back_addr":"172.21.15.8:6809/791166472","heartbeat_front_addr":"172.21.15.8:6807/791166472","state":["exists","up"]},{"osd":1,"uuid":"55f3290e-b82d-4839-a28d-1123a20c3a4c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6810","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6811","nonce":809058426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6812","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6813","nonce":809058426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6816","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6817","nonce":809058426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6814","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6815","nonce":809058426}]},"public_addr":"172.21.15.8:6811/809058426","cluster_addr":"172.21.15.8:6813/809058426","heartbeat_back_addr":"172.21.15.8:6817/809058426","heartbeat_front_addr":"172.21.15.8:6815/809058426","state":["exists","up"]},{"osd":2,"uuid":"720b47df-d743-46cf-ad04-36e113c8fcb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6818","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6819","nonce":3035534415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6820","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6821","nonce":3035534415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6824","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6825","nonce":3035534415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6822","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6823","nonce":3035534415}]},"public_addr":"172.21.15.8:6819/3035534415","cluster_addr":"172.21.15.8:6821/3035534415","heartbeat_back_addr":"172.21.15.8:6825/3035534415","heartbeat_front_addr":"172.21.15.8:6823/3035534415","state":["exists","up"]},{"osd":3,"uuid":"861bf05d-69f8-4000-999e-ab708531edb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6826","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6827","nonce":626394183}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6828","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6829","nonce":626394183}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6832","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6833","nonce":626394183}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6830","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6831","nonce":626394183}]},"public_addr":"172.21.15.8:6827/626394183","cluster_addr":"172.21.15.8:6829/626394183","heartbeat_back_addr":"172.21.15.8:6833/626394183","heartbeat_front_addr":"172.21.15.8:6831/626394183","state":["exists","up"]},{"osd":4,"uuid":"b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6801","nonce":3473028939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6803","nonce":3473028939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6807","nonce":3473028939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6805","nonce":3473028939}]},"public_addr":"172.21.15.191:6801/3473028939","cluster_addr":"172.21.15.191:6803/3473028939","heartbeat_back_addr":"172.21.15.191:6807/3473028939","heartbeat_front_addr":"172.21.15.191:6805/3473028939","state":["exists","up"]},{"osd":5,"uuid":"01507478-481e-4646-a30d-542184240f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6809","nonce":1592222523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6811","nonce":1592222523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6815","nonce":1592222523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6813","nonce":1592222523}]},"public_addr":"172.21.15.191:6809/1592222523","cluster_addr":"172.21.15.191:6811/1592222523","heartbeat_back_addr":"172.21.15.191:6815/1592222523","heartbeat_front_addr":"172.21.15.191:6813/1592222523","state":["exists","up"]},{"osd":6,"uuid":"8bc75ec3-46ee-4d36-989b-4e65e330cfb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6817","nonce":4267099596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6819","nonce":4267099596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6823","nonce":4267099596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6821","nonce":4267099596}]},"public_addr":"172.21.15.191:6817/4267099596","cluster_addr":"172.21.15.191:6819/4267099596","heartbeat_back_addr":"172.21.15.191:6823/4267099596","heartbeat_front_addr":"172.21.15.191:6821/4267099596","state":["exists","up"]},{"osd":7,"uuid":"5ac2e382-7341-47dd-b6c9-0fbdba12829c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6825","nonce":1787611943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6827","nonce":1787611943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6831","nonce":1787611943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6829","nonce":1787611943}]},"public_addr":"172.21.15.191:6825/1787611943","cluster_addr":"172.21.15.191:6827/1787611943","heartbeat_back_addr":"172.21.15.191:6831/1787611943","heartbeat_front_addr":"172.21.15.191:6829/1787611943","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:06.542686+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:28.674405+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:51.044955+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:14.395353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:36.486099+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:58.485442+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:20.015429+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:42.014771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.8:6800/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/1062877078":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2096086544":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2360193392":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/69531294":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6800/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6801/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:6801/2430836187":"2023-10-10T05:15:24.205194+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:19:49.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:49 smithi008 conmon[101053]: audit 2023-10-09T05:19:49.326560+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.8:0/2999977149' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:19:49.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:49 smithi008 conmon[110782]: audit 2023-10-09T05:19:49.326560+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.8:0/2999977149' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:19:49.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:49 smithi191 conmon[101299]: audit 2023-10-09T05:19:49.326560+0000 mon.c (mon.2) 13 : audit 2023-10-09T05:19:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:49 smithi191 conmon[101299]: [DBG] from='client.? 172.21.15.8:0/2999977149' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:19:49.887 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-10-09T05:17:53.386952+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-10-09T05:19:49.887 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-10-09T05:19:50.243 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:19:50.674 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:50 smithi008 conmon[101053]: cluster 2023-10-09T05:19:49.320304+0000 mgr.y (mgr.14148) 172 : cluster [DBG] pgmap v160: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:50.675 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:50 smithi008 conmon[110782]: cluster 2023-10-09T05:19 2023-10-09T05:19:50.675 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:50 smithi008 conmon[110782]: :49.320304+0000 mgr.y (mgr.14148) 172 : cluster [DBG] pgmap v160: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:50.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:50 smithi191 conmon[101299]: cluster 2023-10-09T05:19:49.320304+0000 mgr.y (mgr.14148) 2023-10-09T05:19:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:50 smithi191 conmon[101299]: 172 : cluster [DBG] pgmap v160: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:51.549 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:51 smithi008 conmon[101053]: cluster 2023-10-09T05:19:51.380374+0000 mon.a (mon.0) 533 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:19:51.549 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:51 smithi008 conmon[101053]: cluster 2023-10-09T05:19:51.380406+0000 mon.a (mon.0) 534 : cluster [INF] Cluster is now healthy 2023-10-09T05:19:51.550 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:51 smithi008 conmon[110782]: cluster 2023-10-09T05:19:51.380374+0000 mon.a (mon.0) 533 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:19:51.550 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:51 smithi008 conmon[110782]: cluster 2023-10-09T05:19:51.380406+0000 mon.a (mon.0) 534 : cluster [INF] Cluster is now healthy 2023-10-09T05:19:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:51 smithi191 conmon[101299]: cluster 2023-10-09T05:19:51.380374+0000 mon.a (mon.0) 533 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:19:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:51 smithi191 conmon[101299]: 2023-10-09T05:19:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:51 smithi191 conmon[101299]: cluster 2023-10-09T05:19:51.380406+0000 mon.a (mon.0) 534 : cluster [INF] Cluster is now healthy 2023-10-09T05:19:51.883 INFO:teuthology.orchestra.run.smithi008.stdout:pg_num: 1 2023-10-09T05:19:52.489 INFO:tasks.cephadm:Adding prometheus.a on smithi191 2023-10-09T05:19:52.489 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi191=a' 2023-10-09T05:19:52.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:52 smithi191 conmon[101299]: cluster 2023-10-09T05:19:51.320608+0000 mgr.y ( 2023-10-09T05:19:52.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:52 smithi191 conmon[101299]: mgr.14148) 173 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:52 smithi191 conmon[101299]: audit 2023-10-09T05:19:51.883342+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.8:0/3725544983' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T05:19:53.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:52 smithi008 conmon[101053]: cluster 2023-10- 2023-10-09T05:19:53.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:52 smithi008 conmon[101053]: 09T05:19:51.320608+0000 mgr.y (mgr.14148) 173 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:53.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:52 smithi008 conmon[101053]: audit 2023-10-09T05:19:51.883342+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.8:0/3725544983' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T05:19:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:52 smithi008 conmon[110782]: cluster 2023-10-09T05:19:51.320608+0000 mgr.y (mgr.14148) 173 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:52 smithi008 conmon[110782]: audit 2023-10-09T05:19:51.883342+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.8:0/3725544983' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T05:19:54.413 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled prometheus update... 2023-10-09T05:19:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:54 smithi191 conmon[101299]: cluster 2023-10-09T05:19:53.321103+0000 mgr.y (mgr.14148) 174 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:54 smithi191 conmon[101299]: audit 2023-10-09T05:19:54.412500+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:54 smithi191 conmon[101299]: audit 2023-10-09T05:19:54.419269+0000 mon.a (mon.0) 536 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:54 smithi191 conmon[101299]: audit 2023-10-09T05:19:54.421096+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:54 smithi191 conmon[101299]: audit 2023-10-09T05:19:54.422113+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:54 smithi191 conmon[101299]: audit 2023-10-09T05:19:54.428073+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:54 smithi191 conmon[101299]: audit 2023-10-09T05:19:54.434368+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:19:55.115 DEBUG:teuthology.orchestra.run.smithi191:prometheus.a> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T05:19:55.117 INFO:tasks.cephadm:Adding node-exporter.a on smithi008 2023-10-09T05:19:55.117 INFO:tasks.cephadm:Adding node-exporter.b on smithi191 2023-10-09T05:19:55.118 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi008=a;smithi191=b' 2023-10-09T05:19:55.146 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:-- Logs begin at Mon 2023-10-09 04:57:51 UTC. -- 2023-10-09T05:19:55.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: cluster 2023-10-09T05:19 2023-10-09T05:19:55.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: :53.321103+0000 mgr.y (mgr.14148) 174 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:55.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: 10-09T05:19:54.412500+0000 mon.a 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: (mon.0) 535 : audit 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: audit 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: 2023-10-09T05:19:54.419269+0000 mon.a ( 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: mon.0) 536 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: 10-09T05:19:54.421096+0000 mon.a (mon.0 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: ) 537 : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: -09T05:19:54.422113+0000 mon.a (mon.0) 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: 538 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:19:55.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: -09T05:19:54.428073+0000 mon.a (mon.0 2023-10-09T05:19:55.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: ) 539 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:55.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:19:55.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: -09T05:19:54.434368+0000 mon.a (mon.0 2023-10-09T05:19:55.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[110782]: ) 540 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:19:55.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: cluster 2023-10-09T05:19:53.321103+0000 2023-10-09T05:19:55.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: mgr.y (mgr.14148) 174 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:19:55.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: audit 2023 2023-10-09T05:19:55.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: -10-09T05:19:54.412500+0000 mon.a (mon. 2023-10-09T05:19:55.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: 0) 535 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:55.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: 10-09T05:19:54.419269+0000 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: mon.a (mon.0) 536 : audit 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: -09T05:19:54.421096+0000 mon.a (mon.0) 537 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: : audit [DBG] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: audit 2023 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: -10-09T05:19:54. 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: 422113+0000 mon.a (mon.0) 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: 538 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: 2023-10-09T05:19:55.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: audit 2023-10-09T05:19:55.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: 2023-10- 2023-10-09T05:19:55.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: 09T05:19:54.428073 2023-10-09T05:19:55.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: +0000 mon.a (mon.0) 539 2023-10-09T05:19:55.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' 2023-10-09T05:19:55.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: audit 2023 2023-10-09T05:19:55.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: -10-09T05:19:54.434368 2023-10-09T05:19:55.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: +0000 mon.a (mon.0) 540 2023-10-09T05:19:55.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:54 smithi008 conmon[101053]: : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:19:55.723 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101509]: ignoring --setuser ceph since I am not root 2023-10-09T05:19:55.723 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101509]: ignoring --setgroup ceph since I am not root 2023-10-09T05:19:55.723 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101509]: debug 2023-10-09T05:19:55.478+0000 7fe19715f700 1 -- 172.21.15.8:0/214326553 <== mon.2 v2:172.21.15.8:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b3181204e0 con 0x55b3180f8c00 2023-10-09T05:19:55.723 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101509]: debug 2023-10-09T05:19:55.624+0000 7fe1a7327680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:19:55.723 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101509]: debug 2023-10-09T05:19:55.722+0000 7fe1a7327680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:19:55.742 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[103609]: ignoring --setuser ceph since I am not root 2023-10-09T05:19:55.742 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[103609]: ignoring --setgroup ceph since I am not root 2023-10-09T05:19:55.742 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[103609]: debug 2023-10-09T05:19:55.466+0000 7f9f4ee38700 1 -- 172.21.15.191:0/2157501538 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561f8ba1c4e0 con 0x561f8b9f5000 2023-10-09T05:19:55.743 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[103609]: debug 2023-10-09T05:19:55.594+0000 7f9f5f000680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:19:55.743 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[103609]: debug 2023-10-09T05:19:55.685+0000 7f9f5f000680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:19:56.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[101299]: audit 2023-10-09T05:19:54.406021+0000 mgr.y (mgr.14148 2023-10-09T05:19:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[101299]: ) 175 : audit [DBG] from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[101299]: cephadm 2023-10-09T05:19:54.407709+0000 mgr.y (mgr.14148) 176 : cephadm [INF] Saving service prometheus spec with placement smithi191=a;count:1 2023-10-09T05:19:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[101299]: audit 2023-10-09T05:19:55.434740+0000 mon.a (mon. 2023-10-09T05:19:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[101299]: 0) 541 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:19:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[101299]: cluster 2023-10-09T05:19 2023-10-09T05:19:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[101299]: :55.434827+0000 mon.a (mon.0) 542 : cluster 2023-10-09T05:19:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:19:55 smithi191 conmon[101299]: [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-10-09T05:19:56.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[110782]: audit 2023-10-09T05:19:54.406021 2023-10-09T05:19:56.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[110782]: +0000 mgr.y (mgr.14148) 175 : audit [DBG] from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:56.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[110782]: cephadm 2023-10- 2023-10-09T05:19:56.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[110782]: 09T05:19:54.407709+0000 mgr.y (mgr.14148) 176 : cephadm [INF] Saving service prometheus spec with placement smithi191=a;count:1 2023-10-09T05:19:56.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[110782]: audit 2023 2023-10-09T05:19:56.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[110782]: -10-09T05:19:55.434740+0000 mon.a ( 2023-10-09T05:19:56.172 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[110782]: mon.0) 541 : audit [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:19:56.172 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:19:56.172 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[110782]: -10-09T05:19:55.434827+0000 mon.a 2023-10-09T05:19:56.172 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[110782]: (mon.0) 542 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-10-09T05:19:56.172 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101053]: audit 2023-10-09T05:19: 2023-10-09T05:19:56.172 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101053]: 54.406021+0000 mgr.y (mgr.14148) 175 : audit [DBG] from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:19:56.172 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101053]: cephadm 2023-10-09T05:19 2023-10-09T05:19:56.172 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101053]: :54.407709+0000 mgr.y (mgr.14148) 176 : cephadm [INF] Saving service prometheus spec with placement smithi191=a;count:1 2023-10-09T05:19:56.172 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101053]: audit 2023-10-09T05:19:56.173 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101053]: 2023-10-09T05:19:55.434740 2023-10-09T05:19:56.173 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101053]: +0000 mon.a (mon.0) 541 : audit 2023-10-09T05:19:56.173 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101053]: [INF] from='mgr.14148 172.21.15.8:0/2848612120' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:19:56.173 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101053]: cluster 2023-10-09T05:19 2023-10-09T05:19:56.173 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101053]: :55.434827+0000 mon.a (mon.0) 542 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-10-09T05:19:56.173 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:19:55 smithi008 conmon[101053]: 2023-10-09T05:19:56.355 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:56 smithi191 conmon[103609]: debug 2023-10-09T05:19:56.074+0000 7f9f5f000680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:19:56.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:56 smithi008 conmon[101509]: debug 2023-10-09T05:19:56.170+0000 7fe1a7327680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:19:57.154 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:56 smithi191 conmon[103609]: debug 2023-10-09T05:19:56.857+0000 7f9f5f000680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:19:57.154 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:56 smithi191 conmon[103609]: debug 2023-10-09T05:19:56.943+0000 7f9f5f000680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:19:57.154 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:57 smithi191 conmon[103609]: debug 2023-10-09T05:19:57.153+0000 7f9f5f000680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:19:57.440 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:57 smithi008 conmon[101509]: debug 2023-10-09T05:19:57.096+0000 7fe1a7327680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:19:57.440 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:57 smithi008 conmon[101509]: debug 2023-10-09T05:19:57.191+0000 7fe1a7327680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:19:58.373 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:58 smithi191 conmon[103609]: debug 2023-10-09T05:19:58.081+0000 7f9f5f000680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:19:58.373 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:58 smithi191 conmon[103609]: debug 2023-10-09T05:19:58.207+0000 7f9f5f000680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:19:58.373 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:58 smithi191 conmon[103609]: debug 2023-10-09T05:19:58.285+0000 7f9f5f000680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:19:58.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:58 smithi191 conmon[103609]: debug 2023-10-09T05:19:58.373+0000 7f9f5f000680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:19:58.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:58 smithi191 conmon[103609]: debug 2023-10-09T05:19:58.569+0000 7f9f5f000680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:19:58.825 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:57 smithi008 conmon[101509]: debug 2023-10-09T05:19:57.439+0000 7fe1a7327680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:19:58.825 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:58 smithi008 conmon[101509]: debug 2023-10-09T05:19:58.567+0000 7fe1a7327680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:19:58.825 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:58 smithi008 conmon[101509]: debug 2023-10-09T05:19:58.729+0000 7fe1a7327680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:19:59.154 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:58 smithi008 conmon[101509]: debug 2023-10-09T05:19:58.824+0000 7fe1a7327680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:19:59.154 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:58 smithi008 conmon[101509]: debug 2023-10-09T05:19:58.934+0000 7fe1a7327680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:19:59.299 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:58 smithi191 conmon[103609]: debug 2023-10-09T05:19:58.848+0000 7f9f5f000680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:19:59.299 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:58 smithi191 conmon[103609]: debug 2023-10-09T05:19:58.945+0000 7f9f5f000680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:19:59.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:59 smithi008 conmon[101509]: debug 2023-10-09T05:19:59.153+0000 7fe1a7327680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:19:59.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:59 smithi008 conmon[101509]: debug 2023-10-09T05:19:59.520+0000 7fe1a7327680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:19:59.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:19:59 smithi008 conmon[101509]: debug 2023-10-09T05:19:59.637+0000 7fe1a7327680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:20:00.009 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:59 smithi191 conmon[103609]: debug 2023-10-09T05:19:59.664+0000 7f9f5f000680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:20:00.009 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:59 smithi191 conmon[103609]: debug 2023-10-09T05:19:59.745+0000 7f9f5f000680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:20:00.009 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:19:59 smithi191 conmon[103609]: debug 2023-10-09T05:19:59.837+0000 7f9f5f000680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:20:00.268 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: debug 2023-10-09T05:20:00.009+0000 7f9f5f000680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:20:00.268 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[101299]: cluster 2023-10-09T05:20:00.000132+0000 mon.a (mon.0 2023-10-09T05:20:00.268 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[101299]: ) 543 : cluster [INF] overall HEALTH_OK 2023-10-09T05:20:00.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:00 smithi008 conmon[101053]: cluster 2023-10 2023-10-09T05:20:00.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:00 smithi008 conmon[101053]: -09T05:20:00.000132+0000 mon.a (mon.0) 543 : cluster [INF] overall HEALTH_OK 2023-10-09T05:20:00.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:00 smithi008 conmon[110782]: cluster 2023-10-09T05:20:00.000132+0000 2023-10-09T05:20:00.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:00 smithi008 conmon[110782]: mon.a (mon.0) 543 : cluster [INF] overall HEALTH_OK 2023-10-09T05:20:00.529 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: debug 2023-10-09T05:20:00.268+0000 7f9f5f000680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:20:00.529 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: debug 2023-10-09T05:20:00.397+0000 7f9f5f000680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:20:00.784 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: debug 2023-10-09T05:20:00.528+0000 7f9f5f000680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:20:00.785 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: debug 2023-10-09T05:20:00.700+0000 7f9f5f000680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:20:00.929 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:00 smithi008 conmon[101509]: debug 2023-10-09T05:20:00.507+0000 7fe1a7327680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:20:00.929 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:00 smithi008 conmon[101509]: debug 2023-10-09T05:20:00.605+0000 7fe1a7327680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:20:00.929 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:00 smithi008 conmon[101509]: debug 2023-10-09T05:20:00.717+0000 7fe1a7327680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:20:01.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: debug 2023-10-09T05:20:00.784+0000 7f9f5f000680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:20:01.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: [09/Oct/2023:05:20:00] ENGINE Bus STARTING 2023-10-09T05:20:01.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: CherryPy Checker: 2023-10-09T05:20:01.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: The Application mounted at '' has an empty config. 2023-10-09T05:20:01.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: 2023-10-09T05:20:01.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: [09/Oct/2023:05:20:00] ENGINE Serving on http://:::9283 2023-10-09T05:20:01.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:00 smithi191 conmon[103609]: [09/Oct/2023:05:20:00] ENGINE Bus STARTED 2023-10-09T05:20:01.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:00 smithi008 conmon[101509]: debug 2023-10-09T05:20:00.928+0000 7fe1a7327680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:20:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: cluster 2023-10-09T05:20:00.799316+0000 mon.a 2023-10-09T05:20:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: (mon.0) 544 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T05:20:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: cluster 2023-10-09T05:20:00.799414+0000 mon.a (mon.0) 545 : cluster 2023-10-09T05:20:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: [DBG] Standby manager daemon x started 2023-10-09T05:20:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: audit 2023-10-09T05:20:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: 2023-10-09T05:20:00.800737+0000 mon.b 2023-10-09T05:20:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:20:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:20:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: 05:20:00.801248+0000 mon.b (mon.1) 24 2023-10-09T05:20:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: : audit [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:20:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:20:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: 10-09T05:20:00 2023-10-09T05:20:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: .803107+0000 mon.b (mon.1) 25 : 2023-10-09T05:20:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: audit [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:20:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:20:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: 10-09T05:20:00 2023-10-09T05:20:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: .804096+0000 mon.b (mon.1) 26 2023-10-09T05:20:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101053]: : audit [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:20:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: cluster 2023-10-09T05:20:00 2023-10-09T05:20:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: .799316+0000 mon.a (mon.0) 544 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T05:20:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: cluster 2023-10-09T05:20:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: 2023-10-09T05:20: 2023-10-09T05:20:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: 00.799414+0000 mon.a (mon.0) 545 2023-10-09T05:20:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: : cluster [DBG] Standby manager daemon x started 2023-10-09T05:20:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: audit 2023-10-09T05:20:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: 2023-10-09T05:20 2023-10-09T05:20:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: :00.800737+0000 mon.b (mon.1 2023-10-09T05:20:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: ) 23 : audit [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:20:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:20:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: 10-09T05:20: 2023-10-09T05:20:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: 00.801248+0000 mon.b (mon.1) 24 2023-10-09T05:20:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: : audit [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:20:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:20:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: 10-09T05:20:00 2023-10-09T05:20:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: .803107+0000 mon.b (mon.1) 25 : 2023-10-09T05:20:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: audit [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:20:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:20:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: -09T05:20:00. 2023-10-09T05:20:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: 804096+0000 mon.b (mon.1) 26 : audit 2023-10-09T05:20:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[110782]: [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:20:01.547 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101509]: debug 2023-10-09T05:20:01.218+0000 7fe1a7327680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:20:01.548 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101509]: debug 2023-10-09T05:20:01.379+0000 7fe1a7327680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:20:01.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: cluster 2023-10-09T05:20:00.799316+0000 mon.a 2023-10-09T05:20:01.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: (mon.0) 544 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T05:20:01.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: cluster 2023-10-09T05:20:00.799414+0000 mon.a (mon.0) 545 : cluster [DBG] 2023-10-09T05:20:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: Standby manager daemon x started 2023-10-09T05:20:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: audit 2023-10-09T05:20:00.800737+0000 mon.b (mon. 2023-10-09T05:20:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: 1) 23 : audit [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:20:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:20:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: 10-09T05:20:00.801248+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:20:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: audit 2023-10-09T05:20: 2023-10-09T05:20:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: 00.803107+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:20:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: audit 2023-10-09T05:20:00.804096 2023-10-09T05:20:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:01 smithi191 conmon[101299]: +0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.191:0/907719638' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:20:01.864 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101509]: debug 2023-10-09T05:20:01.547+0000 7fe1a7327680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:20:01.865 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101509]: debug 2023-10-09T05:20:01.762+0000 7fe1a7327680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:20:02.147 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:01 smithi008 conmon[101509]: debug 2023-10-09T05:20:01.863+0000 7fe1a7327680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:20:02.147 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101509]: [09/Oct/2023:05:20:02] ENGINE Bus STARTING 2023-10-09T05:20:02.147 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101509]: [80B blob data] 2023-10-09T05:20:02.147 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101509]: [78B blob data] 2023-10-09T05:20:02.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: cluster 2023-10-09T05:20:01. 2023-10-09T05:20:02.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 059859+0000 mon.a (mon.0) 546 : cluster [DBG] mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T05:20:02.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: cluster 2023-10-09T05:20:02.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05:20 2023-10-09T05:20:02.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: :01.878246+0000 mon.a ( 2023-10-09T05:20:02.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: mon.0) 547 : cluster 2023-10-09T05:20:02.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: [INF] Active manager daemon y restarted 2023-10-09T05:20:02.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: cluster 2023-10 2023-10-09T05:20:02.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: -09T05:20:01. 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 878526+0000 mon.a (mon.0 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: ) 548 : cluster [INF] Activating manager daemon y 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: -10-09T05:20: 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 01.884702+0000 mon.a (mon. 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 0) 549 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: -10-09T05:20:01.885186+0000 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: mon.a (mon.0) 550 : cluster [DBG] 2023-10-09T05:20:02.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: mgrmap e17: y(active, starting, since 0.00675524s), standbys: x 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 05:20:01.886018+0000 mon.c (mon. 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2) 15 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 10-09T05:20:01.886246+0000 mon.c ( 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: mon.2) 16 : audit [DBG] 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 05:20:01.886456+0000 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: mon.c (mon.2) 17 : audit 2023-10-09T05:20:02.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: -09T05:20:01.887440+0000 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: mon.c (mon.2) 18 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: :20:01.887693+0000 mon.c (mon 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: .2) 19 : audit 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05: 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 20:01.887940+0000 mon.c 2023-10-09T05:20:02.151 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: (mon.2) 20 : 2023-10-09T05:20:02.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:20:02.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:20:02.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: -09T05:20:01.888243+0000 2023-10-09T05:20:02.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: mon.c (mon.2) 21 : 2023-10-09T05:20:02.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:20:02.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:20:02.152 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: cluster 2023-10-09T05: 2023-10-09T05:20:02.152 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 20:01.059859+0000 mon.a (mon.0) 546 : cluster [DBG] mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T05:20:02.152 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:20:02.152 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: -10-09T05:20:01. 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 878246+0000 mon.a (mon.0) 547 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: : cluster [INF] Active manager daemon y restarted 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: cluster 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 2023-10-09T05 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: :20:01.878526+0000 mon.a (mon 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: .0) 548 : cluster [INF] Activating manager daemon y 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 10-09T05:20:01. 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 884702+0000 mon.a (mon.0) 549 : cluster 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T05:20:02.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: cluster 2023-10- 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 09T05:20:01.885186+0000 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: mon.a (mon.0) 550 : cluster 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: [DBG] mgrmap e17: y(active, starting, since 0.00675524s), standbys: x 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 09T05:20:01.886018+0000 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: mon.c (mon.2) 15 : audit 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 20:01.886246+0000 mon.c (mon. 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 2) 16 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:20:02.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 2023-10-09 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: T05:20:01.886456+0000 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: mon.c (mon.2) 17 : audit [DBG] 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: T05:20:01.887440+0000 mon.c 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: (mon.2) 18 : audit [DBG] 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 05:20:01.887693+0000 mon.c ( 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: mon.2) 19 : audit [DBG] 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:20:02.155 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023 2023-10-09T05:20:02.156 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: -10-09T05:20:01.887940 2023-10-09T05:20:02.156 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: +0000 mon.c (mon.2) 20 : audit 2023-10-09T05:20:02.156 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:20:02.156 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:20:02.156 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: -09T05:20:01.888243+0000 2023-10-09T05:20:02.156 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: mon.c (mon.2) 21 : audit [DBG] 2023-10-09T05:20:02.156 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:20:02.156 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023 2023-10-09T05:20:02.156 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: -10-09T05:20:01.888524+0000 mon.c (mon.2 2023-10-09T05:20:02.156 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 10-09T05:20:01.888524 2023-10-09T05:20:02.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: +0000 mon.c (mon.2) 2023-10-09T05:20:02.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 22 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:20:02.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05:20:02.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05:20 2023-10-09T05:20:02.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: :01.888803+0000 mon.c ( 2023-10-09T05:20:02.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: mon.2) 23 : audit 2023-10-09T05:20:02.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:20:02.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05:20:01 2023-10-09T05:20:02.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: .889213+0000 mon.c (mon 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: .2) 24 : audit [DBG] 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05:20:01 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: .889513+0000 mon.c (mon.2 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: ) 25 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 10-09T05:20:01.889763 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: +0000 mon.c (mon.2) 26 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:20:02.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: -10-09T05:20:01. 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 890015+0000 mon.c (mon.2) 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 27 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 10-09T05:20:01.890326 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: +0000 mon.c (mon.2) 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 28 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: -10-09T05:20:01. 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 890524+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05:20:02.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05:20:01.890934+0000 mon.c (mon.2) 30 : audit 2023-10-09T05:20:02.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:20:02.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: cluster 2023-10-09T05:20:01.906764 2023-10-09T05:20:02.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: +0000 mon.a (mon.0) 551 : cluster 2023-10-09T05:20:02.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: [INF] Manager daemon y is now available 2023-10-09T05:20:02.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: 2023-10-09T05 2023-10-09T05:20:02.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: :20:02.037678+0000 2023-10-09T05:20:02.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: mon.c (mon.2) 31 : audit 2023-10-09T05:20:02.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:02.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.039282+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: ) 22 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 2023-10-09 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: T05:20:01.888803+0000 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: mon.c (mon.2) 23 : audit [DBG] 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: -09T05:20:01.889213+0000 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: mon.c (mon.2) 24 : audit 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023 2023-10-09T05:20:02.161 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: -10-09T05:20:01.889513 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: +0000 mon.c (mon.2) 25 : audit 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 10-09T05:20:01.889763+0000 mon.c (mon 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: .2) 26 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 20:01.890015+0000 mon.c (mon.2) 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 27 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 2023-10-09T05:20 2023-10-09T05:20:02.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: :01.890326+0000 mon.c (mon.2) 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 28 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 2023-10-09T05:20:01 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: .890524+0000 mon.c (mon.2) 29 : audit 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: -10-09T05:20 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: :01.890934+0000 mon.c (mon.2) 30 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: cluster 2023-10-09T05 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: :20:01.906764+0000 mon.a (mon. 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 0) 551 : cluster [INF] Manager daemon y is now available 2023-10-09T05:20:02.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:20:02.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 10-09T05:20:02.037678+0000 2023-10-09T05:20:02.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: mon.c (mon.2) 31 : audit 2023-10-09T05:20:02.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:02.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:20:02.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 05:20:02.039282+0000 mon.c ( 2023-10-09T05:20:02.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: mon.2) 32 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:02.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[110782]: 2023-10-09T05:20:02.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101509]: CherryPy Checker: 2023-10-09T05:20:02.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101509]: The Application mounted at '' has an empty config. 2023-10-09T05:20:02.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101509]: 2023-10-09T05:20:02.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101509]: [09/Oct/2023:05:20:02] ENGINE Serving on http://:::9283 2023-10-09T05:20:02.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:02 smithi008 conmon[101509]: [09/Oct/2023:05:20:02] ENGINE Bus STARTED 2023-10-09T05:20:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: cluster 2023-10-09T05:20:01.059859+0000 mon.a (mon.0) 546 : cluster [DBG] mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T05:20:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: 2023-10-09T05:20:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: cluster 2023-10-09T05:20:01.878246+0000 mon.a (mon.0) 547 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:20:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: cluster 2023-10-09T05:20:01.878526+0000 mon.a (mon.0) 548 : cluster [INF] Activating manager daemon y 2023-10-09T05:20:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: cluster 2023-10-09T 2023-10-09T05:20:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: 05:20:01.884702+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: cluster 2023-10-09T05:20:01.885186+0000 mon.a (mon.0) 550 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: : cluster [DBG] mgrmap e17: y(active, starting, since 0.00675524s), standbys: x 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:01.886018 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: +0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: -10-09T05:20:01.886246+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:01.886456+0000 mon.c (mon. 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: 2) 17 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: 2023-10-09T05:20:01.887440+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:01.887693+0000 mon.c (mon.2) 19 : 2023-10-09T05:20:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: -09T05:20:01.887940+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:01.888243+0000 mon.c ( 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: mon.2) 21 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: 09T05:20:01.888524+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:01.888803+0000 mon.c (mon.2) 23 : 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: -10-09T05:20:01.889213+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:01.889513+0000 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: mon.c (mon.2) 25 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:20:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: :20:01.889763+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:01. 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: 890015+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:01.890326+0000 mon.c (mon.2) 28 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: -10-09T05:20:01.890524+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:01.890934+0000 mon.c (mon 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: .2) 30 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: cluster 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: 2023-10-09T05:20:01.906764+0000 mon.a (mon.0) 551 : cluster [INF] Manager daemon y is now available 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:02. 2023-10-09T05:20:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: 037678+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: audit 2023-10-09T05:20:02.039282+0000 mon.c ( 2023-10-09T05:20:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:02 smithi191 conmon[101299]: mon.2) 32 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:02.898 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled node-exporter update... 2023-10-09T05:20:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:03 smithi191 conmon[101299]: audit 2023-10-09T05:20:02.079057+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:03 smithi191 conmon[101299]: audit 2023-10-09T05:20:02.091475+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:20:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:03 smithi191 conmon[101299]: audit 2023-10-09T05:20:02.091842+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:20:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:03 smithi191 conmon[101299]: audit 2023-10-09T05:20:02.102273+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:20:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:03 smithi191 conmon[101299]: audit 2023-10-09T05:20:02.102660+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:20:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:03 smithi191 conmon[101299]: cluster 2023-10-09T05:20:02.887567+0000 mon.a (mon.0) 555 : cluster [DBG] mgrmap e18: y(active, since 1.00914s), standbys: x 2023-10-09T05:20:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:03 smithi191 conmon[101299]: audit 2023-10-09T05:20:02.897046+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:03.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.079057+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.091475+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:20:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.091842+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:20:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.102273+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:20:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.102660+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:20:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[101053]: cluster 2023-10-09T05:20:02.887567+0000 mon.a (mon.0) 555 : cluster [DBG] mgrmap e18: y(active, since 1.00914s), standbys: x 2023-10-09T05:20:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[101053]: audit 2023-10-09T05:20:02.897046+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: audit 2023-10-09T05:20:02.079057+0000 mon.a (mon.0 2023-10-09T05:20:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: ) 552 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: audit 2023-10-09T05:20:02.091475+0000 mon.c (mon.2) 2023-10-09T05:20:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: 33 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:20:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: audit 2023-10-09T05:20:02.091842 2023-10-09T05:20:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: +0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:20:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: audit 2023-10-09T05:20:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: 2023-10-09T05:20:02.102273+0000 mon.c (mon.2) 34 2023-10-09T05:20:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:20:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: audit 2023-10-09T05:20:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: 2023-10-09T05:20:02.102660+0000 mon.a (mon.0) 554 : audit 2023-10-09T05:20:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:20:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:20:03.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: 10-09T05:20:02.887567+0000 mon.a (mon.0) 555 : cluster [DBG] mgrmap e18: y(active, since 1.00914s), standbys: x 2023-10-09T05:20:03.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: 2023-10-09T05:20:03.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:20:03.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: 09T05:20:02.897046+0000 mon.a (mon.0) 556 : audit [INF] 2023-10-09T05:20:03.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:03 smithi008 conmon[110782]: from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:03.685 DEBUG:teuthology.orchestra.run.smithi008:node-exporter.a> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T05:20:03.687 DEBUG:teuthology.orchestra.run.smithi191:node-exporter.b> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T05:20:03.689 INFO:tasks.cephadm:Adding alertmanager.a on smithi008 2023-10-09T05:20:03.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi008=a' 2023-10-09T05:20:03.718 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:-- Logs begin at Mon 2023-10-09 04:57:44 UTC. -- 2023-10-09T05:20:03.719 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:-- Logs begin at Mon 2023-10-09 04:57:51 UTC. -- 2023-10-09T05:20:04.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:04 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:02.893681+0000 mgr.y (mgr.14442) 2 : cephadm [INF] Saving service node-exporter spec with placement smithi008=a;smithi191=b;count:2 2023-10-09T05:20:04.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:04 smithi191 conmon[101299]: cluster 2023-10-09T05: 2023-10-09T05:20:04.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:04 smithi191 conmon[101299]: 20:02.893718+0000 mgr.y (mgr.14442) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:04.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:04 smithi191 conmon[101299]: audit 2023-10-09T05:20:03.639751+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:04.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:04 smithi191 conmon[101299]: audit 2023-10-09T05:20:04.055888+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:04.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:04 smithi008 conmon[110782]: cephadm 2023-10-09T05:20:02.893681+0000 mgr.y (mgr.14442) 2 : cephadm [INF] Saving service node-exporter spec with placement smithi008=a;smithi191=b;count:2 2023-10-09T05:20:04.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:04 smithi008 conmon[110782]: cluster 2023-10-09T05:20:02.893718+0000 mgr.y (mgr.14442) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:04 smithi008 conmon[110782]: audit 2023-10-09T05:20:03.639751+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:04 smithi008 conmon[110782]: audit 2023-10-09T05:20:04.055888+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:04.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:04 smithi008 conmon[101053]: cephadm 2023-10-09T05:20:02.893681+0000 mgr.y (mgr.14442) 2 : cephadm [INF] Saving service node-exporter spec with placement smithi008=a;smithi191=b;count:2 2023-10-09T05:20:04.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:04 smithi008 conmon[101053]: cluster 2023-10-09T05:20:02.893718+0000 mgr.y (mgr.14442) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:04.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:04 smithi008 conmon[101053]: audit 2023-10-09T05:20:03.639751+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:04.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:04 smithi008 conmon[101053]: audit 2023-10-09T05:20:04.055888+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:05.351 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: cluster 2023-10-09T05:20:03.888988+0000 mgr.y (mgr.14442) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:05.351 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: audit 2023-10-09T05:20:04.371178+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:20:05.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: audit 2023-10-09T05:20:04.372094+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:20:05.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:04.373449+0000 mgr.y (mgr.14442) 5 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:20:05.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:04.373709+0000 mgr.y (mgr.14442) 6 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:20:05.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:04.389013+0000 mgr.y (mgr.14442) 7 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:20:05.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:04.402995+0000 mgr.y (mgr.14442) 8 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:20:05.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: audit 2023-10-09T05:20:04.419579+0000 mon.a (mon.0) 559 : audit 2023-10-09T05:20:05.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:05.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: audit 2023-10-09T05:20:04.445786+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:05.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: audit 2023-10-09T05:20:04.450424+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:05.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: audit 2023-10-09T05:20:04.455789+0000 mon.c (mon.2) 37 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:20:05.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: audit 2023-10-09T05:20:04.456136+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:20:05.353 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: audit 2023-10-09T05:20:04.895979+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:20:05.353 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:05 smithi191 conmon[101299]: cluster 2023-10-09T05:20:04.896075+0000 mon.a (mon.0) 564 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T05:20:05.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: cluster 2023-10-09T05:20:03.888988+0000 mgr.y (mgr.14442) 4 : 2023-10-09T05:20:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: audit 2023-10-09T05:20:04.371178+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:20:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: audit 2023-10-09T05:20:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: 2023-10-09T05:20:04.372094+0000 mon.c (mon.2) 36 : 2023-10-09T05:20:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:20:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: cephadm 2023-10 2023-10-09T05:20:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: -09T05:20:04.373449+0000 mgr.y 2023-10-09T05:20:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: (mgr.14442) 5 : cephadm [INF] 2023-10-09T05:20:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:20:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: cephadm 2023- 2023-10-09T05:20:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: 10-09T05:20:04.373709+0000 2023-10-09T05:20:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: mgr.y (mgr.14442) 6 : cephadm 2023-10-09T05:20:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:20:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: cephadm 2023-10- 2023-10-09T05:20:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: 09T05:20:04.389013+0000 mgr.y (mgr.14442) 7 : cephadm [INF] 2023-10-09T05:20:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:20:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: cephadm 2023-10-09T05: 2023-10-09T05:20:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: 20:04.402995+0000 mgr.y (mgr.14442) 8 : cephadm 2023-10-09T05:20:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:20:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: audit 2023-10-09T05:20:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: 2023-10-09T05:20:04.419579 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: +0000 mon.a (mon.0) 559 : audit [INF] 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: -09T05:20:04.445786+0000 mon.a (mon.0) 560 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: -09T05:20:04.450424+0000 mon.a (mon.0) 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: 561 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: -09T05:20:04.455789+0000 mon.c (mon.2) 37 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:20:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: audit 2023-10-09T05:20:05.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: 2023-10-09T05:20 2023-10-09T05:20:05.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: :04.456136+0000 mon.a (mon.0) 2023-10-09T05:20:05.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: 562 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:20:05.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:20:05.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: -09T05:20:04.895979+0000 mon.a (mon.0 2023-10-09T05:20:05.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: ) 563 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:20:05.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: cluster 2023-10 2023-10-09T05:20:05.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: -09T05:20:04.896075+0000 mon.a (mon 2023-10-09T05:20:05.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[101053]: .0) 564 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T05:20:05.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: cluster 2023-10-09T05:20:03.888988+0000 mgr.y 2023-10-09T05:20:05.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: (mgr.14442) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:05.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: audit 2023-10-09T05:20:04 2023-10-09T05:20:05.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: .371178+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:20:05.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:20:05.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 10-09T05:20:04.372094+0000 mon.c 2023-10-09T05:20:05.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: (mon.2) 36 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:20:05.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 2023-10-09T05:20:05.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: cephadm 2023-10-09T05:20:05.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 2023-10-09T05:20: 2023-10-09T05:20:05.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 04.373449+0000 mgr.y (mgr.14442) 5 2023-10-09T05:20:05.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:20:05.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: cephadm 2023-10-09T05:20:05.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 2023-10-09T05:20: 2023-10-09T05:20:05.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 04.373709+0000 mgr.y (mgr.14442) 2023-10-09T05:20:05.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 6 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:20:05.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: cephadm 2023-10-09T05:20:05.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 2023-10-09T05:20: 2023-10-09T05:20:05.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 04.389013+0000 mgr.y (mgr.14442) 2023-10-09T05:20:05.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 7 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:20:05.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: cephadm 2023- 2023-10-09T05:20:05.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 10-09T05:20:04. 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 402995+0000 mgr.y (mgr.14442) 8 : cephadm 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 05:20:04.419579+0000 mon.a ( 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: mon.0) 559 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: audit 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 2023-10-09T05:20: 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 04.445786+0000 mon.a (mon.0) 560 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 09T05:20:04.450424+0000 mon.a (mon.0 2023-10-09T05:20:05.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: ) 561 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: audit 2023 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: -10-09T05:20:04 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: .455789+0000 mon.c (mon.2) 37 : audit 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 05:20:04.456136+0000 mon.a 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: (mon.0) 562 : audit 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: audit 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 2023-10-09T05:20: 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 04.895979+0000 mon.a (mon.0) 2023-10-09T05:20:05.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 563 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:20:05.473 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:20:05.473 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: -10-09T05:20: 2023-10-09T05:20:05.473 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: 04.896075+0000 mon.a (mon.0) 564 2023-10-09T05:20:05.473 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:05 smithi008 conmon[110782]: : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T05:20:05.738 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled alertmanager update... 2023-10-09T05:20:06.434 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:06 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:04.911854+0000 mgr.y (mgr.14442) 9 : cephadm [INF] Deploying daemon prometheus.a on smithi191 2023-10-09T05:20:06.434 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:06 smithi191 conmon[101299]: audit 2023-10-09T05:20:05.737348+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:06.455 DEBUG:teuthology.orchestra.run.smithi008:alertmanager.a> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T05:20:06.459 INFO:tasks.cephadm:Adding grafana.a on smithi191 2023-10-09T05:20:06.459 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi191=a' 2023-10-09T05:20:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:06 smithi008 conmon[101053]: cephadm 2023-10-09T05:20:04.911854+0000 2023-10-09T05:20:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:06 smithi008 conmon[101053]: mgr.y (mgr.14442) 9 : cephadm [INF] Deploying daemon prometheus.a on smithi191 2023-10-09T05:20:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:06 smithi008 conmon[101053]: audit 2023-10-09T05:20:05.737348+0000 mon.a ( 2023-10-09T05:20:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:06 smithi008 conmon[101053]: mon.0) 565 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:06 smithi008 conmon[110782]: cephadm 2023-10-09T05:20:04. 2023-10-09T05:20:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:06 smithi008 conmon[110782]: 911854+0000 mgr.y (mgr.14442) 9 : cephadm [INF] Deploying daemon prometheus.a on smithi191 2023-10-09T05:20:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:06 smithi008 conmon[110782]: audit 2023-10-09T05:20:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:06 smithi008 conmon[110782]: 2023-10-09T05:20:05 2023-10-09T05:20:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:06 smithi008 conmon[110782]: .737348+0000 mon.a (mon.0) 565 : audit [INF] 2023-10-09T05:20:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:06 smithi008 conmon[110782]: from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:06.493 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:-- Logs begin at Mon 2023-10-09 04:57:44 UTC. -- 2023-10-09T05:20:07.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[110782]: audit 2023-10-09T05:20:05.731068+0000 mgr.y (mgr.14442) 10 : 2023-10-09T05:20:07.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[110782]: audit [DBG] from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi008=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[110782]: cephadm 2023-10-09T05:20:05.732726+0000 2023-10-09T05:20:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[110782]: mgr.y (mgr.14442) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi008=a;count:1 2023-10-09T05:20:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[110782]: cluster 2023-10 2023-10-09T05:20:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[110782]: -09T05:20:05.889375+0000 mgr.y (mgr.14442) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[101053]: audit 2023-10-09T05:20 2023-10-09T05:20:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[101053]: :05.731068+0000 mgr.y (mgr.14442) 10 : audit [DBG] from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi008=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[101053]: cephadm 2023-10-09T05:20:05.732726+0000 mgr.y (mgr.14442 2023-10-09T05:20:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[101053]: ) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi008=a;count:1 2023-10-09T05:20:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[101053]: cluster 2023-10-09T05:20:05.889375+0000 mgr.y ( 2023-10-09T05:20:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:07 smithi008 conmon[101053]: mgr.14442) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:07.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:07 smithi191 conmon[101299]: audit 2023-10-09T05:20:05.731068+0000 mgr.y (mgr.14442) 10 : audit [DBG] from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi008=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:07.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:07 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:05.732726+0000 mgr.y (mgr.14442) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi008=a;count:1 2023-10-09T05:20:07.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:07 smithi191 conmon[101299]: cluster 2023-10-09T05:20:05.889375+0000 mgr.y (mgr.14442) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:09.088 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:09 smithi191 conmon[101299]: cluster 2023-10-09T05:20:07.889816+0000 mgr.y (mgr.14442) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:09.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:09 smithi008 conmon[101053]: cluster 2023-10-09T05:20:07.889816+0000 mgr.y (mgr.14442) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:09 smithi008 conmon[110782]: cluster 2023-10-09T05:20:07.889816+0000 mgr.y ( 2023-10-09T05:20:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:09 smithi008 conmon[110782]: mgr.14442) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:09.492 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled grafana update... 2023-10-09T05:20:10.760 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:10 smithi191 conmon[101299]: audit 2023-10-09T05:20:09.485713+0000 mgr.y (mgr.14442) 14 2023-10-09T05:20:10.761 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:10 smithi191 conmon[101299]: : audit [DBG] from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:10.761 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:10 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:09.487394+0000 mgr.y (mgr.14442) 15 : cephadm [INF] Saving service grafana spec with placement smithi191=a;count:1 2023-10-09T05:20:10.761 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:10 smithi191 conmon[101299]: audit 2023-10-09T05:20:09.491675+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:10.761 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:10 smithi191 conmon[101299]: cluster 2023-10-09T05:20:09.890369+0000 mgr.y (mgr.14442) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:10.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[110782]: audit 2023-10-09T05:20:09.485713+0000 mgr.y (mgr.14442) 14 : audit 2023-10-09T05:20:10.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[110782]: [DBG] from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[110782]: cephadm 2023-10-09T05:20:09.487394+0000 mgr.y (mgr.14442) 15 : cephadm [INF] Saving service grafana spec with placement smithi191=a;count:1 2023-10-09T05:20:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[110782]: audit 2023-10-09T05:20:09.491675+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[110782]: 2023-10-09T05:20:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[110782]: cluster 2023-10-09T 2023-10-09T05:20:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[110782]: 05:20:09.890369+0000 mgr.y (mgr.14442) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[101053]: audit 2023-10-09T05:20:09.485713+0000 mgr.y (mgr.14442) 14 : audit 2023-10-09T05:20:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[101053]: [DBG] from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:20:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[101053]: cephadm 2023-10-09T05:20:09.487394+0000 mgr.y (mgr.14442) 15 : cephadm [INF] Saving service grafana spec with placement smithi191=a;count:1 2023-10-09T05:20:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[101053]: audit 2023-10-09T05:20: 2023-10-09T05:20:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[101053]: 09.491675+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:10.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[101053]: cluster 2023-10-09T05: 2023-10-09T05:20:10.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:10 smithi008 conmon[101053]: 20:09.890369+0000 mgr.y (mgr.14442) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:12.834 DEBUG:teuthology.orchestra.run.smithi191:grafana.a> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T05:20:12.836 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T05:20:12.836 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T05:20:12.868 INFO:journalctl@ceph.grafana.a.smithi191.stdout:-- Logs begin at Mon 2023-10-09 04:57:51 UTC. -- 2023-10-09T05:20:13.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:12 smithi191 conmon[101299]: cluster 2023-10-09T05:20:11.890825+0000 mgr.y (mgr.14442) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:13.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:12 smithi008 conmon[101053]: cluster 2023-10-09T05:20:11.890825+0000 mgr.y (mgr 2023-10-09T05:20:13.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:12 smithi008 conmon[101053]: .14442) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:12 smithi008 conmon[110782]: cluster 2023-10-09T05:20:11.890825+0000 mgr.y (mgr.14442) 17 : cluster 2023-10-09T05:20:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:12 smithi008 conmon[110782]: [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:14.760 INFO:teuthology.orchestra.run.smithi008.stdout:[client.0] 2023-10-09T05:20:14.761 INFO:teuthology.orchestra.run.smithi008.stdout: key = AQCOjSNljdUnLRAA/5D5UZI8q7aLt78C4udXvw== 2023-10-09T05:20:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:14 smithi008 conmon[101053]: cluster 2023-10-09T05:20:13.891285+0000 mgr.y (mgr.14442) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:14 smithi008 conmon[101053]: audit 2023-10-09T05:20:14.757423+0000 mon.a (mon.0) 567 : audit [INF] from='client.? 172.21.15.8:0/1807603127' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:20:15.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:14 smithi008 conmon[101053]: audit 2023-10-09T05:20:14.760815+0000 mon.a (mon.0) 568 : audit [INF] from='client.? 172.21.15.8:0/1807603127' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:20:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:14 smithi008 conmon[110782]: cluster 2023-10-09T05:20:13.891285+0000 mgr.y (mgr.14442) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:14 smithi008 conmon[110782]: audit 2023-10-09T05:20:14.757423+0000 mon.a (mon.0) 567 : audit [INF] from='client.? 172.21.15.8:0/1807603127' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:20:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:14 smithi008 conmon[110782]: audit 2023-10-09T05:20:14.760815+0000 mon.a (mon.0) 568 : audit [INF] from='client.? 172.21.15.8:0/1807603127' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:20:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:14 smithi191 conmon[101299]: cluster 2023-10-09T05:20:13.891285+0000 mgr.y (mgr.14442) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:14 smithi191 conmon[101299]: audit 2023-10-09T05:20:14.757423+0000 mon.a (mon.0) 567 : audit [INF] from='client.? 172.21.15.8:0/1807603127' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:20:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:14 smithi191 conmon[101299]: audit 2023-10-09T05:20:14.760815+0000 mon.a (mon.0) 568 : audit [INF] from='client.? 172.21.15.8:0/1807603127' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:20:15.416 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:20:15.417 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T05:20:15.417 DEBUG:teuthology.orchestra.run.smithi008:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T05:20:15.459 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T05:20:17.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:16 smithi191 conmon[101299]: cluster 2023-10-09T05:20:15.891599+0000 mgr.y (mgr.14442) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:16 smithi008 conmon[101053]: cluster 2023-10-09T05:20: 2023-10-09T05:20:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:16 smithi008 conmon[101053]: 15.891599+0000 mgr.y (mgr.14442) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:16 smithi008 conmon[110782]: cluster 2023-10-09T05:20:15.891599+0000 mgr.y (mgr.14442) 19 2023-10-09T05:20:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:16 smithi008 conmon[110782]: : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:18.294 INFO:teuthology.orchestra.run.smithi191.stdout:[client.1] 2023-10-09T05:20:18.295 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQCSjSNl5DdQERAA17zMUpeZzzREcAyYD7pxUg== 2023-10-09T05:20:18.800 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[130670]: ts=2023-10-09T05:20:18.546Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T05:20:19.080 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-10-09T05:20:19.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T05:20:19.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T05:20:19.123 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:20:18 smithi191 bash[130626]: f5f1d0f2fc50bbd5908daecb525bb3393d313d6ef81df154605ffc739bc6c05a 2023-10-09T05:20:19.124 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:20:18 smithi191 systemd[1]: Started Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:20:19.124 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[101299]: cluster 2023-10-09T05:20:17.891916+0000 mgr.y (mgr.14442 2023-10-09T05:20:19.124 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[101299]: ) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:19.124 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[101299]: audit 2023-10-09T05:20:18.289723+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.191:0/4160281084' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:20:19.124 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:20:19.124 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[101299]: 10-09T05:20:18.290275+0000 mon.a (mon.0) 569 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:20:19.124 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:20:19.124 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[101299]: 20:18.294134+0000 mon.a (mon.0) 570 2023-10-09T05:20:19.125 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[101299]: : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:20:19.125 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:20:19.125 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[101299]: 20:18.936942+0000 mon.a (mon.0) 571 2023-10-09T05:20:19.125 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:18 smithi191 conmon[101299]: : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:19.126 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T05:20:19.126 DEBUG:teuthology.orchestra.run.smithi008:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T05:20:19.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[101053]: cluster 2023-10-09T05:20:17.891916+0000 mgr.y (mgr.14442) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[101053]: audit 2023-10-09T05:20:18.289723+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.191:0/4160281084' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:20:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[101053]: audit 2023-10-09T05:20:18.290275+0000 mon.a 2023-10-09T05:20:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[101053]: (mon.0) 569 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:20:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[101053]: audit 2023-10-09T05:20:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[101053]: 2023-10-09T05:20:18.294134+0000 mon.a (mon.0) 570 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:20:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[101053]: audit 2023-10-09T05:20:18.936942+0000 mon.a (mon. 2023-10-09T05:20:19.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[101053]: 0) 571 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:19.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: cluster 2023-10-09T05:20:17.891916+0000 mgr.y (mgr.14442) 20 2023-10-09T05:20:19.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:19.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: audit 2023-10-09T05:20:18.289723+0000 mon.b (mon. 2023-10-09T05:20:19.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: 1) 27 : audit [INF] from='client.? 172.21.15.191:0/4160281084' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:20:19.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:20:19.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: -09T05:20:18.290275+0000 mon.a (mon.0) 569 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:20:19.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: 2023-10-09T05:20:19.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:20:19.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: -09T05:20:18.294134+0000 mon.a (mon.0) 570 : 2023-10-09T05:20:19.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:20:19.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: audit 2023-10-09T05:20:18 2023-10-09T05:20:19.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:18 smithi008 conmon[110782]: .936942+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:19.508 INFO:teuthology.misc.health.smithi008.stdout: 2023-10-09T05:20:19.509 INFO:teuthology.misc.health.smithi008.stdout:{"epoch":51,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:14:56.160913+0000","modified":"2023-10-09T05:20:01.878299+0000","last_up_change":"2023-10-09T05:19:43.821876+0000","last_in_change":"2023-10-09T05:19:26.801646+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:17:53.386952+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6802","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6803","nonce":791166472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6804","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6805","nonce":791166472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6808","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6809","nonce":791166472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6806","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6807","nonce":791166472}]},"public_addr":"172.21.15.8:6803/791166472","cluster_addr":"172.21.15.8:6805/791166472","heartbeat_back_addr":"172.21.15.8:6809/791166472","heartbeat_front_addr":"172.21.15.8:6807/791166472","state":["exists","up"]},{"osd":1,"uuid":"55f3290e-b82d-4839-a28d-1123a20c3a4c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6810","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6811","nonce":809058426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6812","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6813","nonce":809058426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6816","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6817","nonce":809058426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6814","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6815","nonce":809058426}]},"public_addr":"172.21.15.8:6811/809058426","cluster_addr":"172.21.15.8:6813/809058426","heartbeat_back_addr":"172.21.15.8:6817/809058426","heartbeat_front_addr":"172.21.15.8:6815/809058426","state":["exists","up"]},{"osd":2,"uuid":"720b47df-d743-46cf-ad04-36e113c8fcb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6818","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6819","nonce":3035534415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6820","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6821","nonce":3035534415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6824","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6825","nonce":3035534415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6822","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6823","nonce":3035534415}]},"public_addr":"172.21.15.8:6819/3035534415","cluster_addr":"172.21.15.8:6821/3035534415","heartbeat_back_addr":"172.21.15.8:6825/3035534415","heartbeat_front_addr":"172.21.15.8:6823/3035534415","state":["exists","up"]},{"osd":3,"uuid":"861bf05d-69f8-4000-999e-ab708531edb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6826","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6827","nonce":626394183}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6828","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6829","nonce":626394183}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6832","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6833","nonce":626394183}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6830","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6831","nonce":626394183}]},"public_addr":"172.21.15.8:6827/626394183","cluster_addr":"172.21.15.8:6829/626394183","heartbeat_back_addr":"172.21.15.8:6833/626394183","heartbeat_front_addr":"172.21.15.8:6831/626394183","state":["exists","up"]},{"osd":4,"uuid":"b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6801","nonce":3473028939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6803","nonce":3473028939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6807","nonce":3473028939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6805","nonce":3473028939}]},"public_addr":"172.21.15.191:6801/3473028939","cluster_addr":"172.21.15.191:6803/3473028939","heartbeat_back_addr":"172.21.15.191:6807/3473028939","heartbeat_front_addr":"172.21.15.191:6805/3473028939","state":["exists","up"]},{"osd":5,"uuid":"01507478-481e-4646-a30d-542184240f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6809","nonce":1592222523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6811","nonce":1592222523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6815","nonce":1592222523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6813","nonce":1592222523}]},"public_addr":"172.21.15.191:6809/1592222523","cluster_addr":"172.21.15.191:6811/1592222523","heartbeat_back_addr":"172.21.15.191:6815/1592222523","heartbeat_front_addr":"172.21.15.191:6813/1592222523","state":["exists","up"]},{"osd":6,"uuid":"8bc75ec3-46ee-4d36-989b-4e65e330cfb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6817","nonce":4267099596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6819","nonce":4267099596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6823","nonce":4267099596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6821","nonce":4267099596}]},"public_addr":"172.21.15.191:6817/4267099596","cluster_addr":"172.21.15.191:6819/4267099596","heartbeat_back_addr":"172.21.15.191:6823/4267099596","heartbeat_front_addr":"172.21.15.191:6821/4267099596","state":["exists","up"]},{"osd":7,"uuid":"5ac2e382-7341-47dd-b6c9-0fbdba12829c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6825","nonce":1787611943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6827","nonce":1787611943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6831","nonce":1787611943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6829","nonce":1787611943}]},"public_addr":"172.21.15.191:6825/1787611943","cluster_addr":"172.21.15.191:6827/1787611943","heartbeat_back_addr":"172.21.15.191:6831/1787611943","heartbeat_front_addr":"172.21.15.191:6829/1787611943","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:06.542686+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:28.674405+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:51.044955+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:14.395353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:36.486099+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:58.485442+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:20.015429+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:42.014771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.8:0/2178079664":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6801/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/1936221066":"2023-10-10T05:20:01.878268+0000","172.21.15.8:0/1062877078":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2096086544":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2360193392":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/69531294":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6800/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6801/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:6801/2430836187":"2023-10-10T05:15:24.205194+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:20:19.521 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T05:20:19.521 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T05:20:19.522 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T05:20:19.898 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:19 smithi008 systemd[1]: Starting Ceph node-exporter.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:20:20.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:19 smithi008 conmon[101053]: cephadm 2023-10-09T05:20:18.939623 2023-10-09T05:20:20.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:19 smithi008 conmon[101053]: +0000 mgr.y (mgr.14442) 21 : cephadm [INF] Deploying daemon node-exporter.a on smithi008 2023-10-09T05:20:20.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:19 smithi008 conmon[101053]: audit 2023-10-09T05:20:19.508559+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.8:0/2708992269' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:20:20.158 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:19 smithi008 conmon[110782]: cephadm 2023-10-09T05:20:18.939623+0000 mgr.y (mgr.14442) 21 2023-10-09T05:20:20.158 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:19 smithi008 conmon[110782]: : cephadm [INF] Deploying daemon node-exporter.a on smithi008 2023-10-09T05:20:20.158 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:19 smithi008 conmon[110782]: audit 2023-10-09T05:20:19.508559+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.8:0/2708992269' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:20:20.158 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:19 smithi008 bash[139291]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-10-09T05:20:20.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:19 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:18.939623+0000 mgr.y (mgr.14442) 21 : cephadm 2023-10-09T05:20:20.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:19 smithi191 conmon[101299]: [INF] Deploying daemon node-exporter.a on smithi008 2023-10-09T05:20:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:19 smithi191 conmon[101299]: audit 2023-10-09T05:20:19.508559+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.8:0/2708992269' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:20:20.714 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:20 smithi008 bash[139291]: Getting image source signatures 2023-10-09T05:20:21.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:20 smithi008 conmon[101053]: cluster 2023-10-09T05:20:19.892251+0000 mgr.y (mgr.14442) 22 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:21.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:20 smithi008 conmon[110782]: cluster 2023-10-09T05:20:19.892251+0000 mgr.y (mgr.14442) 22 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:21.215 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:20 smithi008 bash[139291]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-10-09T05:20:21.215 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:20 smithi008 bash[139291]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-10-09T05:20:21.215 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:20 smithi008 bash[139291]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-10-09T05:20:21.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:20 smithi191 conmon[101299]: cluster 2023-10-09T05:20:19.892251+0000 mgr.y (mgr.14442) 22 : 2023-10-09T05:20:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:20 smithi191 conmon[101299]: cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:21.714 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:21 smithi008 bash[139291]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-10-09T05:20:21.714 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:21 smithi008 bash[139291]: Writing manifest to image destination 2023-10-09T05:20:21.714 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:21 smithi008 bash[139291]: Storing signatures 2023-10-09T05:20:22.225 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'rbd' created 2023-10-09T05:20:22.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:22 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:20:22.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:22 smithi008 conmon[101053]: 05:20:21.707610+0000 mon.a (mon.0) 572 : audit [INF] from='client.? 172.21.15.8:0/1443286228' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:20:22.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:22 smithi008 conmon[101053]: audit 2023-10-09T05:20:22.172843+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:22.518 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:22 smithi008 conmon[110782]: audit 2023-10-09T05:20:21. 2023-10-09T05:20:22.518 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:22 smithi008 conmon[110782]: 707610+0000 mon.a (mon.0) 572 : audit [INF] from='client.? 172.21.15.8:0/1443286228' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:20:22.518 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:22 smithi008 conmon[110782]: audit 2023-10-09T05:20:22.518 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:22 smithi008 conmon[110782]: 2023-10-09T05:20:22.172843+0000 mon.a (mon.0) 573 2023-10-09T05:20:22.518 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:22 smithi008 conmon[110782]: : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:22.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:22 smithi191 conmon[101299]: audit 2023-10-09T05:20:21.707610+0000 mon.a (mon.0) 572 : audit [INF] 2023-10-09T05:20:22.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:22 smithi191 conmon[101299]: from='client.? 172.21.15.8:0/1443286228' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:20:22.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:22 smithi191 conmon[101299]: audit 2023-10-09T05:20:22.172843+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:23.409 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T05:20:23.535 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: cluster 2023-10-09T05:20:21.892538+0000 mgr.y (mgr.14442) 23 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:23.535 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:20:23.535 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: 09T05:20:22.220449+0000 mon.a (mon.0) 574 : audit [INF] from='client.? 172.21.15.8:0/1443286228' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:20:23.535 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: cluster 2023-10-09T05:20:22.220530+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T05:20:23.535 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:20:23.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: 20:22.221045+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:20:23.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: audit 2023-10-09T05:20: 2023-10-09T05:20:23.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: 22.221744+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:20:23.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: audit 2023-10-09T05:20:23.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: 2023-10-09T05:20:22.228551+0000 2023-10-09T05:20:23.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: mon.c (mon.2) 40 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:20:23.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:20:23.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: 05:20:22.228889+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:20:23.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: 2023-10-09T05:20:23.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: audit 2023-10-09T05:20 2023-10-09T05:20:23.537 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[101053]: :23.223009+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:20:23.537 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: cluster 2023-10-09T05:20:21.892538+0000 mgr.y (mgr.14442 2023-10-09T05:20:23.537 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: ) 23 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:23.537 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: audit 2023-10-09T05:20:22.220449+0000 mon.a (mon.0) 574 : audit 2023-10-09T05:20:23.537 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: [INF] from='client.? 172.21.15.8:0/1443286228' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:20:23.537 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: cluster 2023-10-09T05:20:22.220530+0000 2023-10-09T05:20:23.537 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: mon.a (mon.0) 575 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T05:20:23.537 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: audit 2023-10-09T05:20: 2023-10-09T05:20:23.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: 22.221045+0000 mon.c (mon.2) 39 : audit 2023-10-09T05:20:23.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:20:23.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: audit 2023-10-09T05:20:22.221744+0000 mon.a 2023-10-09T05:20:23.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: (mon.0) 576 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:20:23.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:20:23.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: 09T05:20:22.228551+0000 2023-10-09T05:20:23.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: mon.c (mon.2) 40 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:20:23.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: 2023-10-09T05:20:23.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: audit 2023 2023-10-09T05:20:23.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: -10-09T05:20:22.228889+0000 mon.a (mon.0) 577 2023-10-09T05:20:23.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:20:23.539 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:20:23.539 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: 09T05:20:23.223009+0000 mon.a ( 2023-10-09T05:20:23.539 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: mon.0) 578 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:20:23.539 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[110782]: 2023-10-09T05:20:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: cluster 2023-10-09T05:20:21.892538+0000 mgr.y (mgr.14442) 23 : cluster 2023-10-09T05:20:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: audit 2023-10-09T05:20:22.220449+0000 mon.a (mon.0) 574 : audit [INF] from='client.? 172.21.15.8:0/1443286228' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:20:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: cluster 2023-10-09T05: 2023-10-09T05:20:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: 20:22.220530+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T05:20:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: audit 2023-10-09T05:20:22.221045+0000 2023-10-09T05:20:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: mon.c (mon.2) 39 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:20:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:20:23.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: T05:20:22.221744+0000 mon.a (mon.0) 576 : audit [INF] 2023-10-09T05:20:23.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:20:23.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: audit 2023-10-09T05:20:22.228551+0000 mon.c ( 2023-10-09T05:20:23.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: mon.2) 40 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:20:23.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:20:23.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: 20:22.228889+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:20:23.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: audit 2023-10-09T05:20:23.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: 2023-10-09T05:20:23.223009+0000 mon.a (mon 2023-10-09T05:20:23.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:23 smithi191 conmon[101299]: .0) 578 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:20:23.964 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.709Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T05:20:23.965 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.709Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T05:20:23.965 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.710Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T05:20:23.965 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.710Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:20:23.965 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.710Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T05:20:23.965 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T05:20:23.965 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T05:20:23.965 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T05:20:23.965 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T05:20:23.965 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T05:20:23.965 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T05:20:23.966 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T05:20:23.967 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T05:20:23.968 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T05:20:23.969 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T05:20:23.969 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 conmon[139688]: ts=2023-10-09T05:20:23.711Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T05:20:23.969 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 bash[139291]: ec044fd127f69e102100b2227d1907773aa174965ce1d359c873b3d61522967e 2023-10-09T05:20:23.969 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:20:23 smithi008 systemd[1]: Started Ceph node-exporter.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:20:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:24 smithi008 conmon[101053]: cluster 2023-10-09T05:20:23 2023-10-09T05:20:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:24 smithi008 conmon[101053]: .223128+0000 mon.a (mon.0) 579 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T05:20:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:24 smithi008 conmon[101053]: audit 2023-10-09T05:20:23.808913+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:24 smithi008 conmon[110782]: cluster 2023-10-09T05:20:23.223128+0000 2023-10-09T05:20:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:24 smithi008 conmon[110782]: mon.a (mon.0) 579 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T05:20:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:24 smithi008 conmon[110782]: audit 2023-10-09T05:20:23.808913+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:24.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:24 smithi191 systemd[1]: Starting Ceph node-exporter.b for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:20:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:24 smithi191 conmon[101299]: cluster 2023-10-09T05:20:23.223128+0000 mon.a (mon.0) 579 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T05:20:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:24 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:20:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:24 smithi191 conmon[101299]: 20:23.808913+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:25.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:24 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:20:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:20:25.050 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:24 smithi191 bash[131117]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-10-09T05:20:25.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:25 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:23.810367+0000 mgr.y (mgr.14442) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi191 2023-10-09T05:20:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:25 smithi191 conmon[101299]: cluster 2023-10-09T05:20:23.892872+0000 mgr.y (mgr.14442) 25 : cluster [DBG] pgmap v16: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:25 smithi191 conmon[101299]: cluster 2023-10-09T05:20:24.232533+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:20:25.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:25 smithi191 bash[131117]: Getting image source signatures 2023-10-09T05:20:25.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:25 smithi191 bash[131117]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-10-09T05:20:25.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:25 smithi191 bash[131117]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-10-09T05:20:25.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:25 smithi191 bash[131117]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-10-09T05:20:25.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:25 smithi008 conmon[101053]: cephadm 2023-10-09T05 2023-10-09T05:20:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:25 smithi008 conmon[101053]: :20:23.810367+0000 mgr.y (mgr.14442) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi191 2023-10-09T05:20:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:25 smithi008 conmon[101053]: cluster 2023-10-09T05:20:23.892872+0000 mgr.y (mgr.14442) 25 : cluster [DBG] pgmap v16: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:25 smithi008 conmon[101053]: cluster 2023-10-09T05:20:24.232533+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:20:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:25 smithi008 conmon[110782]: cephadm 2023-10-09T05:20:23.810367 2023-10-09T05:20:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:25 smithi008 conmon[110782]: +0000 mgr.y (mgr.14442) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi191 2023-10-09T05:20:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:25 smithi008 conmon[110782]: cluster 2023-10-09T05:20:23.892872+0000 mgr.y (mgr.14442) 25 : cluster [DBG] pgmap v16: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:25 smithi008 conmon[110782]: cluster 2023-10-09T05:20:24.232533+0000 mon.a (mon.0) 581 : cluster 2023-10-09T05:20:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:25 smithi008 conmon[110782]: [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:20:26.233 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:25 smithi191 bash[131117]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-10-09T05:20:26.233 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:25 smithi191 bash[131117]: Writing manifest to image destination 2023-10-09T05:20:26.233 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:25 smithi191 bash[131117]: Storing signatures 2023-10-09T05:20:26.243 INFO:teuthology.orchestra.run.smithi008.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T05:20:26.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:26 smithi191 conmon[101299]: audit 2023-10-09T05:20:25.337303+0000 mon.c (mon 2023-10-09T05:20:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:26 smithi191 conmon[101299]: .2) 41 : audit [INF] from='client.? 172.21.15.8:0/4069999433' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:20:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:26 smithi191 conmon[101299]: audit 2023-10-09T05:20:25.337650+0000 mon.a (mon.0) 582 : audit 2023-10-09T05:20:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:26 smithi191 conmon[101299]: [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:20:26.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:26 smithi008 conmon[101053]: audit 2023-10-09T05:20:25.337303+0000 mon.c (mon.2) 41 : 2023-10-09T05:20:26.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:26 smithi008 conmon[101053]: audit [INF] from='client.? 172.21.15.8:0/4069999433' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:20:26.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:26 smithi008 conmon[101053]: audit 2023-10-09T05:20:25.337650+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:20:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:26 smithi008 conmon[110782]: audit 2023-10-09T05:20:25.337303+0000 mon.c (mon. 2023-10-09T05:20:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:26 smithi008 conmon[110782]: 2) 41 : audit [INF] from='client.? 172.21.15.8:0/4069999433' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:20:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:26 smithi008 conmon[110782]: audit 2023-10-09T05:20:25.337650+0000 mon.a 2023-10-09T05:20:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:26 smithi008 conmon[110782]: (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:20:26.935 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T05:20:26.936 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T05:20:26.936 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T05:20:27.295 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:27.515 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:27 smithi008 conmon[101053]: cluster 2023-10-09T05:20:25.893488 2023-10-09T05:20:27.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:27 smithi008 conmon[101053]: +0000 mgr.y (mgr.14442) 26 : cluster [DBG] pgmap v18: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:27.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:27 smithi008 conmon[101053]: audit 2023-10-09T05:20:26.239604+0000 mon.a (mon.0) 583 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:20:27.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:27 smithi008 conmon[101053]: cluster 2023-10-09T05:20:26.239700+0000 mon.a (mon.0) 584 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T05:20:27.516 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:27 smithi008 conmon[110782]: cluster 2023-10-09T05:20:25.893488 2023-10-09T05:20:27.516 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:27 smithi008 conmon[110782]: +0000 mgr.y (mgr.14442) 26 : cluster [DBG] pgmap v18: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:27.516 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:27 smithi008 conmon[110782]: audit 2023-10-09T05:20:26.239604+0000 mon.a (mon.0) 2023-10-09T05:20:27.516 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:27 smithi008 conmon[110782]: 583 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:20:27.516 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:27 smithi008 conmon[110782]: cluster 2023-10-09T05:20:26.239700+0000 mon.a (mon.0) 584 : 2023-10-09T05:20:27.516 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:27 smithi008 conmon[110782]: cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T05:20:27.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[101299]: cluster 2023-10-09T05:20:25.893488+0000 mgr.y (mgr.14442) 2023-10-09T05:20:27.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[101299]: 26 : cluster [DBG] pgmap v18: 9 pgs: 8 creating+peering, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[101299]: audit 2023-10-09T05:20:26.239604+0000 mon.a (mon.0) 583 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:20:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[101299]: cluster 2023-10-09T05:20:26.239700+0000 mon.a (mon.0 2023-10-09T05:20:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[101299]: ) 584 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T05:20:27.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.346Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T05:20:27.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.346Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T05:20:27.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T05:20:27.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T05:20:27.551 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T05:20:27.552 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T05:20:27.553 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T05:20:27.554 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T05:20:27.554 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T05:20:27.554 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T05:20:27.554 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T05:20:27.554 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T05:20:27.554 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T05:20:27.554 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T05:20:27.554 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.347Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T05:20:27.554 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 conmon[131193]: ts=2023-10-09T05:20:27.348Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T05:20:27.554 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 bash[131117]: 715f54230f50b0f311e290e1674519233761a672cd2bd4c7d7dece7d263b4767 2023-10-09T05:20:27.554 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:20:27 smithi191 systemd[1]: Started Ceph node-exporter.b for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:20:28.626 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:28 smithi008 conmon[101053]: audit 2023-10-09T05:20 2023-10-09T05:20:28.626 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:28 smithi008 conmon[101053]: :27.425749+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:28.626 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:28 smithi008 conmon[101053]: audit 2023-10-09T05:20:27.427261+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:28.627 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:28 smithi008 conmon[110782]: audit 2023-10-09T05:20:27 2023-10-09T05:20:28.627 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:28 smithi008 conmon[110782]: .425749+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:28.627 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:28 smithi008 conmon[110782]: audit 2023-10-09T05:20:27.427261+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:28 smithi191 conmon[101299]: audit 2023-10-09T05:20:27.425749+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:28 smithi191 conmon[101299]: audit 2023-10-09T05:20:27.427261+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:29.131 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:20:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:29 smithi008 conmon[101053]: cluster 2023-10-09T05:20:27.893816+0000 mgr.y (mgr.14442) 27 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:29 smithi008 conmon[101053]: audit 2023-10-09T05:20:29.126331+0000 mon.a (mon.0) 586 : audit [DBG] from='client.? 172.21.15.8:0/2204472487' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:20:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:29 smithi008 conmon[110782]: cluster 2023-10-09T05:20:27.893816+0000 mgr.y (mgr.14442) 27 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:29 smithi008 conmon[110782]: audit 2023-10-09T05:20:29.126331+0000 mon.a (mon.0) 586 : audit [DBG] from='client.? 172.21.15.8:0/2204472487' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:20:29.765 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":19,"active_gid":14442,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6800","nonce":2854879000},{"type":"v1","addr":"172.21.15.8:6801","nonce":2854879000}]},"active_addr":"172.21.15.8:6801/2854879000","active_change":"2023-10-09T05:20:01.878417+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.8:8443/","prometheus":"http://172.21.15.8:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.8:0","nonce":357500347}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.8:0","nonce":1669399762}]}]} 2023-10-09T05:20:29.769 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T05:20:29.769 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:20:29.769 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:20:29.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:29 smithi191 conmon[101299]: cluster 2023-10-09T05:20:27.893816+0000 2023-10-09T05:20:29.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:29 smithi191 conmon[101299]: mgr.y (mgr.14442) 27 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:29.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:29 smithi191 conmon[101299]: audit 2023-10-09T05:20:29.126331+0000 mon.a (mon.0) 586 : audit [DBG] from='client.? 172.21.15.8:0/2204472487' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:20:30.342 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:30.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[101053]: audit 2023-10-09T05:20: 2023-10-09T05:20:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[101053]: 29.595288+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[101053]: cluster 2023-10-09T05:20:29.894107+0000 mgr.y (mgr.14442) 28 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[101053]: audit 2023-10-09T05:20:30.462975+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[101053]: audit 2023-10-09T05:20:30.464085+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:20:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[101053]: audit 2023-10-09T05:20:30.464808+0000 mon.c (mon.2) 44 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:20:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[101053]: audit 2023-10-09T05:20:30.469679+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[101053]: cephadm 2023-10-09T05:20:30.479348+0000 mgr.y (mgr.14442) 29 : cephadm [INF] Deploying daemon alertmanager.a on smithi008 2023-10-09T05:20:30.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: audit 2023-10-09T05:20 2023-10-09T05:20:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: :29.595288+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: cluster 2023-10-09T05:20:29.894107+0000 mgr.y (mgr.14442) 28 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: audit 2023-10-09T05:20:30.462975+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: audit 2023-10-09T05:20:30.464085+0000 mon.c (mon.2) 43 : audit [DBG] 2023-10-09T05:20:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:20:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: audit 2023-10-09T05:20:30.464808+0000 2023-10-09T05:20:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: mon.c (mon.2) 44 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:20:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: audit 2023-10-09T05:20:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: 2023-10-09T05:20 2023-10-09T05:20:30.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: :30.469679+0000 mon.a ( 2023-10-09T05:20:30.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: mon.0) 589 : audit 2023-10-09T05:20:30.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:30.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: cephadm 2023-10 2023-10-09T05:20:30.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: -09T05:20:30. 2023-10-09T05:20:30.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: 479348+0000 mgr.y (mgr.14442 2023-10-09T05:20:30.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:30 smithi008 conmon[110782]: ) 29 : cephadm [INF] Deploying daemon alertmanager.a on smithi008 2023-10-09T05:20:31.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: audit 2023-10-09T05:20:29.595288+0000 mon.a (mon.0) 587 : audit 2023-10-09T05:20:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: cluster 2023-10-09T05:20:29.894107+0000 mgr.y (mgr.14442) 28 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: audit 2023-10-09T05:20:30.462975+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: audit 2023-10-09T05:20:30. 2023-10-09T05:20:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: 464085+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:20:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: audit 2023 2023-10-09T05:20:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: -10-09T05:20:30.464808+0000 mon.c (mon.2) 44 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:20:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: audit 2023-10-09T05:20:30.469679+0000 2023-10-09T05:20:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: mon.a (mon.0) 589 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:31.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:30.479348+0000 mgr.y (mgr.14442) 29 : cephadm 2023-10-09T05:20:31.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:30 smithi191 conmon[101299]: [INF] Deploying daemon alertmanager.a on smithi008 2023-10-09T05:20:32.045 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:20:32.046 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":55,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:14:56.160913+0000","modified":"2023-10-09T05:20:26.232555+0000","last_up_change":"2023-10-09T05:19:43.821876+0000","last_in_change":"2023-10-09T05:19:26.801646+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:17:53.386952+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:20:22.166676+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6802","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6803","nonce":791166472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6804","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6805","nonce":791166472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6808","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6809","nonce":791166472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6806","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6807","nonce":791166472}]},"public_addr":"172.21.15.8:6803/791166472","cluster_addr":"172.21.15.8:6805/791166472","heartbeat_back_addr":"172.21.15.8:6809/791166472","heartbeat_front_addr":"172.21.15.8:6807/791166472","state":["exists","up"]},{"osd":1,"uuid":"55f3290e-b82d-4839-a28d-1123a20c3a4c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6810","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6811","nonce":809058426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6812","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6813","nonce":809058426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6816","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6817","nonce":809058426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6814","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6815","nonce":809058426}]},"public_addr":"172.21.15.8:6811/809058426","cluster_addr":"172.21.15.8:6813/809058426","heartbeat_back_addr":"172.21.15.8:6817/809058426","heartbeat_front_addr":"172.21.15.8:6815/809058426","state":["exists","up"]},{"osd":2,"uuid":"720b47df-d743-46cf-ad04-36e113c8fcb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6818","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6819","nonce":3035534415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6820","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6821","nonce":3035534415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6824","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6825","nonce":3035534415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6822","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6823","nonce":3035534415}]},"public_addr":"172.21.15.8:6819/3035534415","cluster_addr":"172.21.15.8:6821/3035534415","heartbeat_back_addr":"172.21.15.8:6825/3035534415","heartbeat_front_addr":"172.21.15.8:6823/3035534415","state":["exists","up"]},{"osd":3,"uuid":"861bf05d-69f8-4000-999e-ab708531edb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6826","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6827","nonce":626394183}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6828","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6829","nonce":626394183}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6832","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6833","nonce":626394183}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6830","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6831","nonce":626394183}]},"public_addr":"172.21.15.8:6827/626394183","cluster_addr":"172.21.15.8:6829/626394183","heartbeat_back_addr":"172.21.15.8:6833/626394183","heartbeat_front_addr":"172.21.15.8:6831/626394183","state":["exists","up"]},{"osd":4,"uuid":"b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6801","nonce":3473028939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6803","nonce":3473028939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6807","nonce":3473028939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6805","nonce":3473028939}]},"public_addr":"172.21.15.191:6801/3473028939","cluster_addr":"172.21.15.191:6803/3473028939","heartbeat_back_addr":"172.21.15.191:6807/3473028939","heartbeat_front_addr":"172.21.15.191:6805/3473028939","state":["exists","up"]},{"osd":5,"uuid":"01507478-481e-4646-a30d-542184240f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6809","nonce":1592222523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6811","nonce":1592222523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6815","nonce":1592222523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6813","nonce":1592222523}]},"public_addr":"172.21.15.191:6809/1592222523","cluster_addr":"172.21.15.191:6811/1592222523","heartbeat_back_addr":"172.21.15.191:6815/1592222523","heartbeat_front_addr":"172.21.15.191:6813/1592222523","state":["exists","up"]},{"osd":6,"uuid":"8bc75ec3-46ee-4d36-989b-4e65e330cfb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6817","nonce":4267099596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6819","nonce":4267099596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6823","nonce":4267099596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6821","nonce":4267099596}]},"public_addr":"172.21.15.191:6817/4267099596","cluster_addr":"172.21.15.191:6819/4267099596","heartbeat_back_addr":"172.21.15.191:6823/4267099596","heartbeat_front_addr":"172.21.15.191:6821/4267099596","state":["exists","up"]},{"osd":7,"uuid":"5ac2e382-7341-47dd-b6c9-0fbdba12829c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6825","nonce":1787611943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6827","nonce":1787611943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6831","nonce":1787611943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6829","nonce":1787611943}]},"public_addr":"172.21.15.191:6825/1787611943","cluster_addr":"172.21.15.191:6827/1787611943","heartbeat_back_addr":"172.21.15.191:6831/1787611943","heartbeat_front_addr":"172.21.15.191:6829/1787611943","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:06.542686+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:28.674405+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:51.044955+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:14.395353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:36.486099+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:58.485442+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:20.015429+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:42.014771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.8:0/2178079664":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6801/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/1936221066":"2023-10-10T05:20:01.878268+0000","172.21.15.8:0/1062877078":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2096086544":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2360193392":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/69531294":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6800/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6801/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:6801/2430836187":"2023-10-10T05:15:24.205194+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:20:33.000 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:20:33.000 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:20:33.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:32 smithi008 conmon[101053]: cluster 2023-10-09T05:20:31.894397+0000 mgr.y (mgr.14442) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:32 smithi008 conmon[101053]: audit 2023-10-09T05:20:32.045401+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.8:0/2044210427' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:20:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:32 smithi008 conmon[101053]: audit 2023-10-09T05:20:32.180662+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:32 smithi008 conmon[110782]: cluster 2023-10-09T05:20:31.894397+0000 mgr.y (mgr.14442) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:32 smithi008 conmon[110782]: audit 2023-10-09T05:20:32.045401+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.8:0/2044210427' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:20:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:32 smithi008 conmon[110782]: audit 2023-10-09T05:20:32.180662+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:33.215 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:33 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:20:33] "GET /metrics HTTP/1.1" 200 155635 "" "Prometheus/2.33.4" 2023-10-09T05:20:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:32 smithi191 conmon[101299]: cluster 2023-10-09T05:20:31.894397+0000 mgr.y (mgr.14442) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:32 smithi191 conmon[101299]: audit 2023-10-09T05:20:32.045401+0000 mon.c (mon 2023-10-09T05:20:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:32 smithi191 conmon[101299]: .2) 45 : audit [DBG] from='client.? 172.21.15.8:0/2044210427' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:20:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:32 smithi191 conmon[101299]: audit 2023-10-09T05:20:32.180662+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:34.170 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:34.950 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:34 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:20:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:20:35.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:34 smithi008 conmon[101053]: cluster 2023-10-09T05:20:33.894752+0000 mgr.y (mgr.14442) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:35.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:34 smithi008 conmon[110782]: cluster 2023-10-09T05:20:33.894752+0000 mgr.y (mgr.14442) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:34 smithi191 conmon[101299]: cluster 2023-10-09T05:20:33.894752+0000 mgr.y (mgr.14442) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:34 smithi191 conmon[101299]: 2023-10-09T05:20:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:36 smithi008 conmon[101053]: cluster 2023-10-09T05:20:35 2023-10-09T05:20:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:36 smithi008 conmon[101053]: .895234+0000 mgr.y (mgr.14442) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:36 smithi008 conmon[110782]: cluster 2023-10-09T05:20:35.895234+0000 2023-10-09T05:20:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:36 smithi008 conmon[110782]: mgr.y (mgr.14442) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:37.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:36 smithi191 conmon[101299]: cluster 2023-10-09T05:20:35.895234+0000 mgr.y (mgr.14442) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:37.354 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:20:37.355 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":55,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:14:56.160913+0000","modified":"2023-10-09T05:20:26.232555+0000","last_up_change":"2023-10-09T05:19:43.821876+0000","last_in_change":"2023-10-09T05:19:26.801646+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:17:53.386952+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:20:22.166676+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6802","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6803","nonce":791166472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6804","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6805","nonce":791166472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6808","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6809","nonce":791166472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6806","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6807","nonce":791166472}]},"public_addr":"172.21.15.8:6803/791166472","cluster_addr":"172.21.15.8:6805/791166472","heartbeat_back_addr":"172.21.15.8:6809/791166472","heartbeat_front_addr":"172.21.15.8:6807/791166472","state":["exists","up"]},{"osd":1,"uuid":"55f3290e-b82d-4839-a28d-1123a20c3a4c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6810","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6811","nonce":809058426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6812","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6813","nonce":809058426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6816","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6817","nonce":809058426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6814","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6815","nonce":809058426}]},"public_addr":"172.21.15.8:6811/809058426","cluster_addr":"172.21.15.8:6813/809058426","heartbeat_back_addr":"172.21.15.8:6817/809058426","heartbeat_front_addr":"172.21.15.8:6815/809058426","state":["exists","up"]},{"osd":2,"uuid":"720b47df-d743-46cf-ad04-36e113c8fcb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6818","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6819","nonce":3035534415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6820","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6821","nonce":3035534415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6824","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6825","nonce":3035534415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6822","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6823","nonce":3035534415}]},"public_addr":"172.21.15.8:6819/3035534415","cluster_addr":"172.21.15.8:6821/3035534415","heartbeat_back_addr":"172.21.15.8:6825/3035534415","heartbeat_front_addr":"172.21.15.8:6823/3035534415","state":["exists","up"]},{"osd":3,"uuid":"861bf05d-69f8-4000-999e-ab708531edb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6826","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6827","nonce":626394183}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6828","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6829","nonce":626394183}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6832","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6833","nonce":626394183}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6830","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6831","nonce":626394183}]},"public_addr":"172.21.15.8:6827/626394183","cluster_addr":"172.21.15.8:6829/626394183","heartbeat_back_addr":"172.21.15.8:6833/626394183","heartbeat_front_addr":"172.21.15.8:6831/626394183","state":["exists","up"]},{"osd":4,"uuid":"b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6801","nonce":3473028939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6803","nonce":3473028939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6807","nonce":3473028939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6805","nonce":3473028939}]},"public_addr":"172.21.15.191:6801/3473028939","cluster_addr":"172.21.15.191:6803/3473028939","heartbeat_back_addr":"172.21.15.191:6807/3473028939","heartbeat_front_addr":"172.21.15.191:6805/3473028939","state":["exists","up"]},{"osd":5,"uuid":"01507478-481e-4646-a30d-542184240f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6809","nonce":1592222523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6811","nonce":1592222523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6815","nonce":1592222523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6813","nonce":1592222523}]},"public_addr":"172.21.15.191:6809/1592222523","cluster_addr":"172.21.15.191:6811/1592222523","heartbeat_back_addr":"172.21.15.191:6815/1592222523","heartbeat_front_addr":"172.21.15.191:6813/1592222523","state":["exists","up"]},{"osd":6,"uuid":"8bc75ec3-46ee-4d36-989b-4e65e330cfb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6817","nonce":4267099596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6819","nonce":4267099596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6823","nonce":4267099596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6821","nonce":4267099596}]},"public_addr":"172.21.15.191:6817/4267099596","cluster_addr":"172.21.15.191:6819/4267099596","heartbeat_back_addr":"172.21.15.191:6823/4267099596","heartbeat_front_addr":"172.21.15.191:6821/4267099596","state":["exists","up"]},{"osd":7,"uuid":"5ac2e382-7341-47dd-b6c9-0fbdba12829c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6825","nonce":1787611943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6827","nonce":1787611943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6831","nonce":1787611943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6829","nonce":1787611943}]},"public_addr":"172.21.15.191:6825/1787611943","cluster_addr":"172.21.15.191:6827/1787611943","heartbeat_back_addr":"172.21.15.191:6831/1787611943","heartbeat_front_addr":"172.21.15.191:6829/1787611943","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:06.542686+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:28.674405+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:51.044955+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:14.395353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:36.486099+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:58.485442+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:20.015429+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:42.014771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.8:0/2178079664":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6801/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/1936221066":"2023-10-10T05:20:01.878268+0000","172.21.15.8:0/1062877078":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2096086544":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2360193392":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/69531294":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6800/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6801/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:6801/2430836187":"2023-10-10T05:15:24.205194+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:20:38.278 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:20:38.279 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:20:38.279 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:20:38.279 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:20:38.280 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:20:38.280 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:20:38.281 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:20:38.281 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:20:38.337 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:38 smithi008 conmon[101053]: audit 2023-10-09T05:20:37.354471+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.8:0/3004196611' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:20:38.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:38 smithi008 conmon[110782]: audit 2023-10-09T05:20:37.354471+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.8:0/3004196611' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:20:38.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:38 smithi191 conmon[101299]: audit 2023-10-09T05:20:37.354471+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.8:0/3004196611' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:20:38.716 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:38 smithi008 systemd[1]: Starting Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:20:39.447 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[101053]: cluster 2023-10-09T 2023-10-09T05:20:39.447 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[101053]: 05:20:37.895583+0000 mgr.y (mgr.14442) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:39.447 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[110782]: cluster 2023-10-09T05:20:37.895583+0000 mgr.y ( 2023-10-09T05:20:39.447 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[110782]: mgr.14442) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:39.533 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:39.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:39 smithi191 conmon[101299]: cluster 2023-10-09T05:20:37.895583+0000 mgr.y (mgr.14442) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:39.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:39 smithi191 conmon[101299]: 2023-10-09T05:20:39.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[101509]: [09/Oct/2023:05:20:39] ENGINE Bus STOPPING 2023-10-09T05:20:39.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[101509]: [09/Oct/2023:05:20:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:20:39.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[101509]: [09/Oct/2023:05:20:39] ENGINE Bus STOPPED 2023-10-09T05:20:39.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[101509]: [09/Oct/2023:05:20:39] ENGINE Bus STARTING 2023-10-09T05:20:39.715 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[143429]: level=info ts=2023-10-09T05:20:39.447Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T05:20:39.715 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[143429]: level=info ts=2023-10-09T05:20:39.447Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T05:20:39.715 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[143429]: level=info ts=2023-10-09T05:20:39.448Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.8 port=9094 2023-10-09T05:20:39.715 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[143429]: level=info ts=2023-10-09T05:20:39.450Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T05:20:39.715 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:39 smithi008 bash[143263]: 041d72a037cc263a9fde86d1542d5c272e59afc1036ce9382be4d9351a948225 2023-10-09T05:20:39.715 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:39 smithi008 systemd[1]: Started Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:20:39.715 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[143429]: level=info ts=2023-10-09T05:20:39.483Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:20:39.716 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[143429]: level=info ts=2023-10-09T05:20:39.483Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:20:39.716 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[143429]: level=info ts=2023-10-09T05:20:39.487Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T05:20:39.716 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[143429]: level=info ts=2023-10-09T05:20:39.487Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T05:20:39.832 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:40.147 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:40.151 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[101509]: [09/Oct/2023:05:20:39] ENGINE Serving on http://:::9283 2023-10-09T05:20:40.151 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:39 smithi008 conmon[101509]: [09/Oct/2023:05:20:39] ENGINE Bus STARTED 2023-10-09T05:20:40.422 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:40.429 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:40.429 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:40.429 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: audit 2023-10-09T05:20:39.550409+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:20:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: :20:39.645225+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: audit 2023-10-09T05:20:39.648586+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:20:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: 05:20:39.651789+0000 mon.c (mon.2) 47 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:20:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: audit 2023 2023-10-09T05:20:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: -10-09T05:20:39.652249+0000 mgr.y (mgr.14442) 34 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:20:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:20:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: :20:39.656790+0000 mon.a (mon.0) 594 : audit 2023-10-09T05:20:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: cephadm 2023-10-09T05:20:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: 2023-10-09T05:20:39.673411+0000 mgr.y (mgr.14442) 35 : 2023-10-09T05:20:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: cephadm [INF] Deploying daemon grafana.a on smithi191 2023-10-09T05:20:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: cluster 2023-10-09T05:20:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: 2023-10-09T05:20:39.895893+0000 mgr.y (mgr.14442) 2023-10-09T05:20:40.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:40 smithi191 conmon[101299]: 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:40.802 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:40.806 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[101053]: audit 2023-10-09T05:20:39.550409+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.806 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[101053]: audit 2023-10-09T05:20:39.645225+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.806 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[101053]: audit 2023-10-09T05:20:39.648586+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.806 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[101053]: audit 2023-10-09T05:20:39.651789+0000 mon.c (mon.2) 47 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:20:40.806 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[101053]: audit 2023-10-09T05:20:39.652249+0000 mgr.y (mgr.14442) 34 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:20:40.807 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[101053]: audit 2023-10-09T05:20:39.656790+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.807 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[101053]: cephadm 2023-10-09T05:20:39.673411+0000 mgr.y (mgr.14442) 35 : cephadm [INF] Deploying daemon grafana.a on smithi191 2023-10-09T05:20:40.807 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[101053]: cluster 2023-10-09T05:20:39.895893+0000 mgr.y (mgr.14442) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:40.807 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[110782]: audit 2023-10-09T05:20:39.550409+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.807 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[110782]: audit 2023-10-09T05:20:39.645225+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.807 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[110782]: audit 2023-10-09T05:20:39.648586+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.807 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[110782]: audit 2023-10-09T05:20:39.651789+0000 mon.c (mon.2) 47 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:20:40.807 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[110782]: audit 2023-10-09T05:20:39.652249+0000 mgr.y (mgr.14442) 34 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:20:40.807 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[110782]: audit 2023-10-09T05:20:39.656790+0000 mon.a (mon.0) 594 : audit 2023-10-09T05:20:40.808 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[110782]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:40.808 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[110782]: cephadm 2023-10-09T05:20:39.673411+0000 mgr.y (mgr.14442) 35 : cephadm [INF] Deploying daemon grafana.a on smithi191 2023-10-09T05:20:40.808 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:40 smithi008 conmon[110782]: cluster 2023-10-09T05:20:39.895893+0000 mgr.y (mgr.14442) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:41.648 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:41 smithi008 conmon[143429]: level=info ts=2023-10-09T05:20:41.451Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000667886s 2023-10-09T05:20:43.167 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:43 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:20:43] "GET /metrics HTTP/1.1" 200 155635 "" "Prometheus/2.33.4" 2023-10-09T05:20:43.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:43 smithi008 conmon[101053]: cluster 2023-10-09T05:20:41.896283 2023-10-09T05:20:43.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:43 smithi008 conmon[101053]: +0000 mgr.y (mgr.14442) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:43.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:43 smithi008 conmon[101053]: audit 2023-10-09T05:20:42.187871+0000 mon.a (mon 2023-10-09T05:20:43.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:43 smithi008 conmon[101053]: .0) 595 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:43.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:43 smithi008 conmon[110782]: cluster 2023-10-09T05:20:41.896283+0000 2023-10-09T05:20:43.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:43 smithi008 conmon[110782]: mgr.y (mgr.14442) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:43.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:43 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:20:43.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:43 smithi008 conmon[110782]: 20:42.187871+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:43.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:43 smithi191 conmon[101299]: cluster 2023-10-09T05:20:41.896283+0000 mgr.y ( 2023-10-09T05:20:43.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:43 smithi191 conmon[101299]: mgr.14442) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:43 smithi191 conmon[101299]: audit 2023-10-09T05:20:42.187871+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:45.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:44 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:20:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:20:45.376 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:45 smithi008 conmon[110782]: cluster 2023-10-09T05:20:43. 2023-10-09T05:20:45.376 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:45 smithi008 conmon[110782]: 896960+0000 mgr.y (mgr.14442) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:45.377 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:45 smithi008 conmon[101053]: cluster 2023-10-09T05:20:43.896960+0000 2023-10-09T05:20:45.377 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:45 smithi008 conmon[101053]: mgr.y (mgr.14442) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:45.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:45 smithi191 conmon[101299]: cluster 2023-10-09T05:20:43.896960+0000 mgr.y (mgr 2023-10-09T05:20:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:45 smithi191 conmon[101299]: .14442) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:46.274 INFO:teuthology.orchestra.run.smithi008.stdout:107374182430 2023-10-09T05:20:46.274 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:20:46.365 INFO:teuthology.orchestra.run.smithi008.stdout:34359738414 2023-10-09T05:20:46.365 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:20:47.297 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:47 smithi008 conmon[101053]: cluster 2023-10-09T05:20:45.897325+0000 mgr.y (mgr.14442) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:47.297 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:47 smithi008 conmon[110782]: cluster 2023-10-09T05:20:45.897325+0000 mgr.y (mgr.14442) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:47.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:47 smithi191 conmon[101299]: cluster 2023-10-09T05:20:45. 2023-10-09T05:20:47.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:47 smithi191 conmon[101299]: 897325+0000 mgr.y (mgr.14442) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:48.369 INFO:teuthology.orchestra.run.smithi008.stdout:180388626451 2023-10-09T05:20:48.369 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:20:49.196 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:49 smithi008 conmon[110782]: cluster 2023-10-09T05:20:47.897673+0000 mgr.y (mgr.14442) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:49.258 INFO:teuthology.orchestra.run.smithi008.stdout:206158430224 2023-10-09T05:20:49.258 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:20:49.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:49 smithi008 conmon[101053]: cluster 2023-10-09T05:20:47.897673+0000 mgr.y (mgr.14442) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:49 smithi191 conmon[101299]: cluster 2023-10-09T05:20:47.897673 2023-10-09T05:20:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:49 smithi191 conmon[101299]: +0000 mgr.y (mgr.14442) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:49.714 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:20:49 smithi008 conmon[143429]: level=info ts=2023-10-09T05:20:49.452Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002253855s 2023-10-09T05:20:49.792 INFO:teuthology.orchestra.run.smithi008.stdout:154618822679 2023-10-09T05:20:49.793 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:20:50.054 INFO:teuthology.orchestra.run.smithi008.stdout:55834574889 2023-10-09T05:20:50.054 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:20:50.926 INFO:teuthology.orchestra.run.smithi008.stdout:128849018908 2023-10-09T05:20:50.926 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:20:50.960 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:51.051 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:51.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:51 smithi008 conmon[101053]: cluster 2023-10-09T05:20:49.898046+0000 mgr.y (mgr.14442) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:51.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:51 smithi008 conmon[110782]: cluster 2023-10-09T05:20:49.898046+0000 mgr.y (mgr.14442) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:51.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:51 smithi191 conmon[101299]: cluster 2023-10-09T05:20:49.898046+0000 mgr.y (mgr.14442) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:52.108 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:52.139 INFO:teuthology.orchestra.run.smithi008.stdout:77309411365 2023-10-09T05:20:52.140 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:20:52.339 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:52.947 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:53.120 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:53.348 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:53 smithi008 conmon[101053]: cluster 2023-10-09T05:20:51. 2023-10-09T05:20:53.348 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:53 smithi008 conmon[101053]: 898444+0000 mgr.y (mgr.14442) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:53.348 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:20:53 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:20:53] "GET /metrics HTTP/1.1" 200 155629 "" "Prometheus/2.33.4" 2023-10-09T05:20:53.349 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:53 smithi008 conmon[110782]: cluster 2023-10-09T05:20:51.898444+0000 mgr.y (mgr.14442) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:53 smithi191 conmon[101299]: cluster 2023-10-09T05:20:51.898444+0000 mgr.y (mgr.14442) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:54.024 INFO:teuthology.orchestra.run.smithi008.stdout:34359738415 2023-10-09T05:20:54.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:54 smithi008 conmon[110782]: audit 2023-10-09T05:20:54.024239+0000 mon.a (mon.0) 596 : audit [DBG] from='client.? 172.21.15.8:0/824635721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:20:54.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:54 smithi008 conmon[101053]: audit 2023-10-09T05:20:54 2023-10-09T05:20:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:54 smithi008 conmon[101053]: .024239+0000 mon.a (mon.0) 596 : audit [DBG] from='client.? 172.21.15.8:0/824635721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:20:54.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:54 smithi191 conmon[101299]: audit 2023-10-09T05:20:54.024239+0000 mon.a (mon.0) 596 : audit [DBG] from='client.? 172.21.15.8:0/824635721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:20:55.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:20:54 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:20:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:20:55.135 INFO:teuthology.orchestra.run.smithi008.stdout:180388626453 2023-10-09T05:20:55.209 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738415 for osd.0 2023-10-09T05:20:55.210 DEBUG:teuthology.parallel:result is None 2023-10-09T05:20:55.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:55 smithi008 conmon[101053]: cluster 2023-10-09T05: 2023-10-09T05:20:55.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:55 smithi008 conmon[101053]: 20:53.898801+0000 mgr.y (mgr.14442) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:55.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:55 smithi008 conmon[101053]: audit 2023-10-09T05:20:54.786223+0000 mon.a (mon.0) 597 : audit [DBG] from='client.? 172.21.15.8:0/177454518' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:20:55.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:55 smithi008 conmon[110782]: cluster 2023-10-09T05:20:53.898801 2023-10-09T05:20:55.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:55 smithi008 conmon[110782]: +0000 mgr.y (mgr.14442) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:55.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:55 smithi008 conmon[110782]: audit 2023-10-09T05:20:54.786223+0000 mon.a 2023-10-09T05:20:55.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:55 smithi008 conmon[110782]: (mon.0) 597 : audit [DBG] from='client.? 172.21.15.8:0/177454518' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:20:55.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:55 smithi191 conmon[101299]: cluster 2023-10-09T05:20:53.898801+0000 mgr.y (mgr.14442) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:55.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:55 smithi191 conmon[101299]: audit 2023-10-09T05:20:54.786223+0000 mon.a 2023-10-09T05:20:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:55 smithi191 conmon[101299]: (mon.0) 597 : audit [DBG] from='client.? 172.21.15.8:0/177454518' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:20:56.605 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:56.860 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626451 got 180388626453 for osd.6 2023-10-09T05:20:56.860 DEBUG:teuthology.parallel:result is None 2023-10-09T05:20:56.997 INFO:teuthology.orchestra.run.smithi008.stdout:154618822681 2023-10-09T05:20:57.462 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:57 smithi008 conmon[101053]: cluster 2023-10-09T05:20:55.899253+0000 mgr.y (mgr 2023-10-09T05:20:57.462 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:57 smithi008 conmon[101053]: .14442) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:57.462 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:57 smithi008 conmon[101053]: audit 2023-10-09T05:20:56.997556+0000 mon.a (mon.0) 598 : audit [DBG] from='client.? 172.21.15.8:0/965282352' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:20:57.463 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:57 smithi008 conmon[110782]: cluster 2023-10-09T05:20:55.899253+0000 mgr.y (mgr.14442) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:57.463 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:57 smithi008 conmon[110782]: audit 2023-10-09T05:20:57.463 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:57 smithi008 conmon[110782]: 2023-10-09T05:20:56. 2023-10-09T05:20:57.463 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:57 smithi008 conmon[110782]: 997556+0000 mon.a (mon.0) 598 : audit [DBG] from='client.? 172.21.15.8:0/965282352' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:20:57.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:57 smithi191 conmon[101299]: cluster 2023-10-09T05:20:55.899253+0000 mgr.y (mgr.14442) 44 2023-10-09T05:20:57.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:57 smithi191 conmon[101299]: : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:57.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:57 smithi191 conmon[101299]: audit 2023-10-09T05:20:56.997556+0000 mon.a (mon.0) 598 : audit [DBG] from='client.? 172.21.15.8:0/965282352' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:20:57.574 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:20:57.965 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822679 got 154618822681 for osd.5 2023-10-09T05:20:57.965 DEBUG:teuthology.parallel:result is None 2023-10-09T05:20:58.299 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 systemd[1]: Starting Ceph grafana.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:20:58.462 INFO:teuthology.orchestra.run.smithi008.stdout:107374182433 2023-10-09T05:20:58.943 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 bash[132593]: abc49ed66cfa0a2f68459a00637373e2667488e6ea6f44e9a01aad40286ca11a 2023-10-09T05:20:58.943 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 systemd[1]: Started Ceph grafana.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:20:58.943 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-09T05:20:58.943 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-09T05:20:58.943 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-09T05:20:58.945 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-09T05:20:58.945 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T05:20:58.945 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T05:20:58.945 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T05:20:58.945 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T05:20:58.945 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-09T05:20:58.945 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-09T05:20:58.945 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-09T05:20:58.945 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-09T05:20:58.946 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-09T05:20:58.946 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="App mode production" logger=settings 2023-10-09T05:20:58.946 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-09T05:20:58.946 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T05:20:58.946 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-09T05:20:58.946 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-10-09T05:20:58.946 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-10-09T05:20:58.946 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:58 smithi191 conmon[132642]: t=2023-10-09T05:20:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-10-09T05:20:59.203 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-10-09T05:20:59.203 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-10-09T05:20:59.203 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-10-09T05:20:59.204 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-10-09T05:20:59.204 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-10-09T05:20:59.455 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-10-09T05:20:59.455 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-10-09T05:20:59.455 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T05:20:59.455 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-10-09T05:20:59.455 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[101299]: cluster 2023-10-09T05:20:57.899608+0000 mgr.y ( 2023-10-09T05:20:59.455 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[101299]: mgr.14442) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:59.455 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[101299]: audit 2023-10-09T05:20:58.463218+0000 mon.a (mon.0) 599 : audit [DBG] 2023-10-09T05:20:59.456 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[101299]: from='client.? 172.21.15.8:0/3392233233' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:20:59.456 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[101299]: audit 2023 2023-10-09T05:20:59.456 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[101299]: -10-09T05:20:58.920288+0000 mon.a (mon.0) 600 2023-10-09T05:20:59.456 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[101299]: : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:59.456 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[101299]: audit 2023-10-09T05:20:59.456 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[101299]: 2023-10-09T05: 2023-10-09T05:20:59.456 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[101299]: 20:58.922057+0000 mon.c (mon.2 2023-10-09T05:20:59.456 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[101299]: ) 48 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:59.462 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:59 smithi008 conmon[101053]: cluster 2023-10-09T05:20:57.899608+0000 mgr.y (mgr.14442) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:59.462 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:59 smithi008 conmon[101053]: audit 2023-10-09T05:20:58.463218+0000 mon.a (mon.0) 599 : audit [DBG] from='client.? 172.21.15.8:0/3392233233' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:20:59.462 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:59 smithi008 conmon[101053]: audit 2023-10-09T05:20:58.920288+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:59.462 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:59 smithi008 conmon[101053]: audit 2023-10-09T05:20:58.922057+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:59.463 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:20:59 smithi008 conmon[101053]: 2023-10-09T05:20:59.463 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:59 smithi008 conmon[110782]: cluster 2023-10-09T05:20:57.899608+0000 mgr.y (mgr.14442) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:20:59.463 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:59 smithi008 conmon[110782]: audit 2023-10-09T05:20:58.463218+0000 mon.a (mon.0) 599 : audit [DBG] from='client.? 172.21.15.8:0/3392233233' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:20:59.463 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:59 smithi008 conmon[110782]: audit 2023-10-09T05:20:58.920288+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:20:59.463 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:20:59 smithi008 conmon[110782]: audit 2023-10-09T05:20:58.922057+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:20:59.514 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182430 got 107374182433 for osd.3 2023-10-09T05:20:59.514 DEBUG:teuthology.parallel:result is None 2023-10-09T05:20:59.740 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-10-09T05:20:59.740 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-10-09T05:20:59.740 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-10-09T05:20:59.885 INFO:teuthology.orchestra.run.smithi008.stdout:206158430226 2023-10-09T05:21:00.025 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-10-09T05:21:00.025 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-10-09T05:21:00.025 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-10-09T05:21:00.025 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-10-09T05:21:00.026 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:20:59 smithi191 conmon[132642]: t=2023-10-09T05:20:59+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-10-09T05:21:00.077 INFO:teuthology.orchestra.run.smithi008.stdout:55834574891 2023-10-09T05:21:00.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[101299]: audit 2023-10-09T05:20:59.759606+0000 mon.c (mon.2) 49 : 2023-10-09T05:21:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[101299]: audit [DBG] from='client.? 172.21.15.8:0/848389445' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:21:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[101299]: audit 2023-10-09T05:21:00.077871+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.8:0/4093621499' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:21:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[101299]: 2023-10-09T05:21:00.300 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-10-09T05:21:00.300 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T05:21:00.300 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-10-09T05:21:00.300 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-10-09T05:21:00.300 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-10-09T05:21:00.301 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-10-09T05:21:00.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:00 smithi008 conmon[101053]: audit 2023-10-09T05:20:59.759606+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.8:0/848389445' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:21:00.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:00 smithi008 conmon[101053]: audit 2023-10-09T05:21:00.077871+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.8:0/4093621499' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:21:00.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:00 smithi008 conmon[110782]: audit 2023-10-09T05:20:59.759606+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.8:0/848389445' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:21:00.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:00 smithi008 conmon[110782]: audit 2023-10-09T05:21:00.077871+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.8:0/4093621499' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:21:00.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-10-09T05:21:00.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-10-09T05:21:00.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-10-09T05:21:00.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T05:21:00.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-10-09T05:21:00.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-10-09T05:21:00.897 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-10-09T05:21:00.898 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-10-09T05:21:00.898 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-10-09T05:21:00.898 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-10-09T05:21:01.150 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-10-09T05:21:01.150 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T05:21:01.150 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:00 smithi191 conmon[132642]: t=2023-10-09T05:21:00+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-10-09T05:21:01.150 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-10-09T05:21:01.150 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-10-09T05:21:01.229 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430224 got 206158430226 for osd.7 2023-10-09T05:21:01.229 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:01.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-10-09T05:21:01.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-10-09T05:21:01.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-10-09T05:21:01.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T05:21:01.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-10-09T05:21:01.411 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[101299]: cluster 2023-10-09T05:20:59.899968+0000 mgr.y (mgr.14442) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:01.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:01 smithi008 conmon[101053]: cluster 2023-10-09T05:20:59 2023-10-09T05:21:01.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:01 smithi008 conmon[101053]: .899968+0000 mgr.y (mgr.14442) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:01.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:01 smithi008 conmon[110782]: cluster 2023-10-09T05:20:59 2023-10-09T05:21:01.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:01 smithi008 conmon[110782]: .899968+0000 mgr.y (mgr.14442) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:01.474 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574889 got 55834574891 for osd.1 2023-10-09T05:21:01.475 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:01.687 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-10-09T05:21:01.687 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-10-09T05:21:01.687 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-10-09T05:21:01.687 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-10-09T05:21:01.767 INFO:teuthology.orchestra.run.smithi008.stdout:77309411367 2023-10-09T05:21:01.947 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-10-09T05:21:01.947 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-10-09T05:21:01.947 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-10-09T05:21:02.145 INFO:teuthology.orchestra.run.smithi008.stdout:128849018910 2023-10-09T05:21:02.208 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:01 smithi191 conmon[132642]: t=2023-10-09T05:21:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T05:21:02.208 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T05:21:02.208 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T05:21:02.208 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-10-09T05:21:02.208 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-10-09T05:21:02.208 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T05:21:02.208 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-10-09T05:21:02.208 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-10-09T05:21:02.208 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-10-09T05:21:02.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:02 smithi008 conmon[110782]: audit 2023-10-09T05:21:01.767334+0000 mon.b (mon.1) 28 : 2023-10-09T05:21:02.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:02 smithi008 conmon[110782]: audit [DBG] from='client.? 172.21.15.8:0/2733239703' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:21:02.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:02 smithi008 conmon[110782]: audit 2023-10-09T05:21:01.991397+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:02.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:02 smithi008 conmon[110782]: audit 2023-10-09T05:21:02.145358+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.8:0/1892098662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:21:02.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:02 smithi008 conmon[110782]: audit 2023-10-09T05:21:02.198293+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:02 smithi008 conmon[101053]: audit 2023-10-09T05:21:01.767334 2023-10-09T05:21:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:02 smithi008 conmon[101053]: +0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.8:0/2733239703' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:21:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:02 smithi008 conmon[101053]: audit 2023-10-09T05:21:01.991397+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:02 smithi008 conmon[101053]: audit 2023-10-09T05:21:02.145358+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.8:0/1892098662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:21:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:02 smithi008 conmon[101053]: audit 2023-10-09T05:21:02.198293+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:02.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[101299]: audit 2023-10-09T05:21:01.767334+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.8:0/2733239703' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:21:02.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[101299]: audit 2023-10-09T05:21: 2023-10-09T05:21:02.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[101299]: 01.991397+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:02.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[101299]: audit 2023-10-09T05:21:02.145358+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.8:0/1892098662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:21:02.469 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[101299]: audit 2023-10-09T05:21:02.198293+0000 mon.a (mon.0 2023-10-09T05:21:02.469 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[101299]: ) 602 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:02.469 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-10-09T05:21:02.469 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-10-09T05:21:02.469 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-10-09T05:21:02.469 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-10-09T05:21:02.469 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-10-09T05:21:02.469 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-10-09T05:21:02.469 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-10-09T05:21:02.469 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-10-09T05:21:02.469 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-10-09T05:21:02.470 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-10-09T05:21:02.728 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-10-09T05:21:02.728 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-10-09T05:21:02.729 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-10-09T05:21:02.729 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-10-09T05:21:02.729 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-10-09T05:21:02.729 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-10-09T05:21:02.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-10-09T05:21:02.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T05:21:02.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-10-09T05:21:02.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T05:21:02.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-10-09T05:21:02.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T05:21:02.998 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:02 smithi191 conmon[132642]: t=2023-10-09T05:21:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T05:21:03.047 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411367 for osd.2 2023-10-09T05:21:03.047 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:03.087 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018908 got 128849018910 for osd.4 2023-10-09T05:21:03.087 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:03.087 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:21:03.087 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:21:03.266 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[101299]: cluster 2023-10-09T05:21:01.900306+0000 mgr.y (mgr.14442) 47 2023-10-09T05:21:03.266 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[101299]: : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:03.267 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-10-09T05:21:03.267 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T05:21:03.267 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-10-09T05:21:03.267 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-10-09T05:21:03.267 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-10-09T05:21:03.267 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-10-09T05:21:03.267 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-10-09T05:21:03.267 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-10-09T05:21:03.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:03 smithi008 conmon[101053]: cluster 2023-10-09T05:21: 2023-10-09T05:21:03.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:03 smithi008 conmon[101053]: 01.900306+0000 mgr.y (mgr.14442) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:03.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:03 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:03] "GET /metrics HTTP/1.1" 200 155628 "" "Prometheus/2.33.4" 2023-10-09T05:21:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:03 smithi008 conmon[110782]: cluster 2023-10-09T05:21:01.900306+0000 mgr.y ( 2023-10-09T05:21:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:03 smithi008 conmon[110782]: mgr.14442) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:03.535 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-10-09T05:21:03.535 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-10-09T05:21:03.535 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-10-09T05:21:03.535 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T05:21:03.535 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-10-09T05:21:03.535 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-10-09T05:21:03.535 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-10-09T05:21:03.548 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:03.799 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-10-09T05:21:03.799 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T05:21:03.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-10-09T05:21:03.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-10-09T05:21:03.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-10-09T05:21:04.105 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-10-09T05:21:04.105 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-10-09T05:21:04.105 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-10-09T05:21:04.105 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:03 smithi191 conmon[132642]: t=2023-10-09T05:21:03+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-10-09T05:21:04.106 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-10-09T05:21:04.106 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-10-09T05:21:04.106 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-10-09T05:21:04.106 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-10-09T05:21:04.357 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-10-09T05:21:04.357 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-10-09T05:21:04.357 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-10-09T05:21:04.357 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-10-09T05:21:04.357 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-10-09T05:21:04.357 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-10-09T05:21:04.358 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-10-09T05:21:04.643 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:21:04.643 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T05:21:04.643 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-10-09T05:21:04.643 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-10-09T05:21:04.643 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-10-09T05:21:04.643 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-10-09T05:21:04.643 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-10-09T05:21:04.644 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-10-09T05:21:04.895 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-10-09T05:21:04.895 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-10-09T05:21:04.895 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-10-09T05:21:04.895 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-10-09T05:21:04.896 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-10-09T05:21:04.896 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-10-09T05:21:04.896 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-10-09T05:21:04.896 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T05:21:04.896 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T05:21:04.896 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T05:21:05.163 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:04 smithi191 conmon[132642]: t=2023-10-09T05:21:04+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T05:21:05.163 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-10-09T05:21:05.163 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T05:21:05.164 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-10-09T05:21:05.164 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T05:21:05.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[101299]: cluster 2023-10-09T05:21:03.900810+0000 mgr.y (mgr.14442) 48 : cluster [DBG] 2023-10-09T05:21:05.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[101299]: pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:05.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[101299]: audit 2023-10-09T05:21:05.010818+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:05.440 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-10-09T05:21:05.440 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T05:21:05.440 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-10-09T05:21:05.441 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-10-09T05:21:05.441 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T05:21:05.441 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-10-09T05:21:05.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:05 smithi008 conmon[101053]: cluster 2023-10-09T05:21:03.900810+0000 mgr.y (mgr.14442) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:05.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:05 smithi008 conmon[101053]: audit 2023-10-09T05:21:05.010818+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:05.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:05 smithi008 conmon[110782]: cluster 2023-10-09T05:21:03.900810+0000 mgr.y (mgr.14442) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:05.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:05 smithi008 conmon[110782]: audit 2023-10-09T05:21:05.010818+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:05.600 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:21:05.600 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:21:05.767 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-10-09T05:21:05.767 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-10-09T05:21:05.767 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-10-09T05:21:05.768 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-10-09T05:21:05.768 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-10-09T05:21:05.768 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-10-09T05:21:05.768 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-10-09T05:21:05.768 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-10-09T05:21:05.768 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-10-09T05:21:06.019 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-10-09T05:21:06.020 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-10-09T05:21:06.020 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-10-09T05:21:06.020 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-10-09T05:21:06.020 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-10-09T05:21:06.020 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:05 smithi191 conmon[132642]: t=2023-10-09T05:21:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-10-09T05:21:06.020 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-10-09T05:21:06.280 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-10-09T05:21:06.280 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-10-09T05:21:06.280 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-10-09T05:21:06.280 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-10-09T05:21:06.280 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T05:21:06.280 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T05:21:06.280 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: audit 2023-10-09T05:21:05.339386+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:06.280 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:21:06.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: 09T05:21:05.340205+0000 mon.c (mon.2) 53 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:06.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: audit 2023-10-09T05:21:05.345101 2023-10-09T05:21:06.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: +0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:06.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: cephadm 2023-10-09T 2023-10-09T05:21:06.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: 05:21:05.348731+0000 mgr.y (mgr 2023-10-09T05:21:06.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: .14442) 49 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:21:06.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: cephadm 2023 2023-10-09T05:21:06.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: -10-09T05:21:05. 2023-10-09T05:21:06.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: 351992+0000 mgr.y (mgr.14442) 50 : cephadm [INF] 2023-10-09T05:21:06.282 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[101299]: Reconfiguring daemon alertmanager.a on smithi008 2023-10-09T05:21:06.454 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2023-10-09T05:21:03.900483+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380980,"kb_used_data":4212,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580236,"statfs":{"total":767960285184,"available":765522161664,"internally_reserved":0,"allocated":4313088,"data_stored":1763225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.646809+0000","last_change":"2023-10-09T05:20:23.226324+0000","last_active":"2023-10-09T05:20:24.646809+0000","last_peered":"2023-10-09T05:20:24.646809+0000","last_clean":"2023-10-09T05:20:24.646809+0000","last_became_active":"2023-10-09T05:20:23.225950+0000","last_became_peered":"2023-10-09T05:20:23.225950+0000","last_unstale":"2023-10-09T05:20:24.646809+0000","last_undegraded":"2023-10-09T05:20:24.646809+0000","last_fullsized":"2023-10-09T05:20:24.646809+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.646620+0000","last_change":"2023-10-09T05:20:23.589992+0000","last_active":"2023-10-09T05:20:24.646620+0000","last_peered":"2023-10-09T05:20:24.646620+0000","last_clean":"2023-10-09T05:20:24.646620+0000","last_became_active":"2023-10-09T05:20:23.589676+0000","last_became_peered":"2023-10-09T05:20:23.589676+0000","last_unstale":"2023-10-09T05:20:24.646620+0000","last_undegraded":"2023-10-09T05:20:24.646620+0000","last_fullsized":"2023-10-09T05:20:24.646620+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.646969+0000","last_change":"2023-10-09T05:20:23.732093+0000","last_active":"2023-10-09T05:20:24.646969+0000","last_peered":"2023-10-09T05:20:24.646969+0000","last_clean":"2023-10-09T05:20:24.646969+0000","last_became_active":"2023-10-09T05:20:23.731684+0000","last_became_peered":"2023-10-09T05:20:23.731684+0000","last_unstale":"2023-10-09T05:20:24.646969+0000","last_undegraded":"2023-10-09T05:20:24.646969+0000","last_fullsized":"2023-10-09T05:20:24.646969+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.646916+0000","last_change":"2023-10-09T05:20:23.731911+0000","last_active":"2023-10-09T05:20:24.646916+0000","last_peered":"2023-10-09T05:20:24.646916+0000","last_clean":"2023-10-09T05:20:24.646916+0000","last_became_active":"2023-10-09T05:20:23.731530+0000","last_became_peered":"2023-10-09T05:20:23.731530+0000","last_unstale":"2023-10-09T05:20:24.646916+0000","last_undegraded":"2023-10-09T05:20:24.646916+0000","last_fullsized":"2023-10-09T05:20:24.646916+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.647001+0000","last_change":"2023-10-09T05:19:46.035865+0000","last_active":"2023-10-09T05:20:24.647001+0000","last_peered":"2023-10-09T05:20:24.647001+0000","last_clean":"2023-10-09T05:20:24.647001+0000","last_became_active":"2023-10-09T05:19:46.035493+0000","last_became_peered":"2023-10-09T05:19:46.035493+0000","last_unstale":"2023-10-09T05:20:24.647001+0000","last_undegraded":"2023-10-09T05:20:24.647001+0000","last_fullsized":"2023-10-09T05:20:24.647001+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.322281+0000","last_change":"2023-10-09T05:20:23.227498+0000","last_active":"2023-10-09T05:20:24.322281+0000","last_peered":"2023-10-09T05:20:24.322281+0000","last_clean":"2023-10-09T05:20:24.322281+0000","last_became_active":"2023-10-09T05:20:23.227147+0000","last_became_peered":"2023-10-09T05:20:23.227147+0000","last_unstale":"2023-10-09T05:20:24.322281+0000","last_undegraded":"2023-10-09T05:20:24.322281+0000","last_fullsized":"2023-10-09T05:20:24.322281+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.231399+0000","last_change":"2023-10-09T05:20:23.731595+0000","last_active":"2023-10-09T05:20:24.231399+0000","last_peered":"2023-10-09T05:20:24.231399+0000","last_clean":"2023-10-09T05:20:24.231399+0000","last_became_active":"2023-10-09T05:20:23.731240+0000","last_became_peered":"2023-10-09T05:20:23.731240+0000","last_unstale":"2023-10-09T05:20:24.231399+0000","last_undegraded":"2023-10-09T05:20:24.231399+0000","last_fullsized":"2023-10-09T05:20:24.231399+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.322273+0000","last_change":"2023-10-09T05:20:23.227732+0000","last_active":"2023-10-09T05:20:24.322273+0000","last_peered":"2023-10-09T05:20:24.322273+0000","last_clean":"2023-10-09T05:20:24.322273+0000","last_became_active":"2023-10-09T05:20:23.227348+0000","last_became_peered":"2023-10-09T05:20:23.227348+0000","last_unstale":"2023-10-09T05:20:24.322273+0000","last_undegraded":"2023-10-09T05:20:24.322273+0000","last_fullsized":"2023-10-09T05:20:24.322273+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.646607+0000","last_change":"2023-10-09T05:20:23.731592+0000","last_active":"2023-10-09T05:20:24.646607+0000","last_peered":"2023-10-09T05:20:24.646607+0000","last_clean":"2023-10-09T05:20:24.646607+0000","last_became_active":"2023-10-09T05:20:23.731263+0000","last_became_peered":"2023-10-09T05:20:23.731263+0000","last_unstale":"2023-10-09T05:20:24.646607+0000","last_undegraded":"2023-10-09T05:20:24.646607+0000","last_fullsized":"2023-10-09T05:20:24.646607+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430227,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.8300000000000001,"5min":2.8300000000000001,"15min":2.8300000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.53000000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":2.2400000000000002,"5min":2.2400000000000002,"15min":2.2400000000000002},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.4809999999999999,"5min":2.4809999999999999,"15min":2.4809999999999999},"last":0.64500000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.7029999999999998,"5min":2.7029999999999998,"15min":2.7029999999999998},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.117,"5min":2.117,"15min":2.117},"last":0.57799999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.6669999999999998,"5min":2.6669999999999998,"15min":2.6669999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":2.1379999999999999,"5min":2.1379999999999999,"15min":2.1379999999999999},"last":0.73099999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.38}]},{"osd":5,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.502},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.41899999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.44500000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626455,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.40300000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.48499999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.38}]},{"osd":3,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.502}]},{"osd":4,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"last":0.45600000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.44},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.432}]},{"osd":7,"last update":"Mon Oct 9 05:20:44 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.47499999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574892,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.32500000000000001},"min":{"1min":0.223,"5min":0.218,"15min":0.218},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.34899999999999998,"15min":0.33200000000000002},"min":{"1min":0.23799999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.247}]},{"osd":2,"last update":"Mon Oct 9 05:20:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.442,"15min":0.41499999999999998},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8460000000000001,"5min":2.8460000000000001,"15min":2.8460000000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.44500000000000001,"15min":0.40899999999999997},"min":{"1min":0.25700000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.9290000000000003,"5min":5.9290000000000003,"15min":5.9290000000000003},"last":0.35099999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:20:24 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.47599999999999998,"15min":0.47799999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.314},{"interface":"front","average":{"1min":0.44,"5min":0.45400000000000001,"15min":0.45600000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.629,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.38300000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:20:40 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.59399999999999997,"15min":0.59199999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.64700000000000002,"15min":0.61199999999999999},"min":{"1min":0.46400000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.3330000000000002,"5min":6.3330000000000002,"15min":6.3330000000000002},"last":0.56200000000000006}]},{"osd":5,"last update":"Mon Oct 9 05:21:02 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.61599999999999999,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.3639999999999999,"5min":6.3639999999999999,"15min":6.3639999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.59099999999999997,"15min":0.57399999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":2.8799999999999999,"5min":2.8799999999999999,"15min":2.8799999999999999},"last":0.57999999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:20:24 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.61399999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":2.8069999999999999,"5min":2.8069999999999999,"15min":2.8069999999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.0920000000000001,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.629}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.37,"15min":0.34799999999999998},"min":{"1min":0.23599999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.311},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.40300000000000002,"15min":0.36699999999999999},"min":{"1min":0.27100000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.484,"5min":2.484,"15min":2.484},"last":0.441}]},{"osd":2,"last update":"Mon Oct 9 05:19:58 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.42499999999999999,"15min":0.42099999999999999},"min":{"1min":0.35199999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.41399999999999998,"15min":0.41099999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.34200000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:20:18 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.47099999999999997,"15min":0.47299999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.46500000000000002,"15min":0.46300000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.41899999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:20:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.47599999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:20:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.57399999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:20:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.627}]},{"osd":7,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":2.5830000000000002,"5min":2.5830000000000002,"15min":2.5830000000000002},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.53600000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:19:59 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.39200000000000002,"15min":0.39400000000000002},"min":{"1min":0.26800000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.57899999999999996,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.40300000000000002,"15min":0.40000000000000002},"min":{"1min":0.29199999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.61499999999999999,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.38600000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:19:59 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.433,"15min":0.42899999999999999},"min":{"1min":0.30599999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.46200000000000002,"15min":0.45700000000000002},"min":{"1min":0.29599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.73699999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.46200000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:20:17 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.45300000000000001,"15min":0.46200000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.58899999999999997,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.371},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.503,"15min":0.50800000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.69299999999999995,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.42599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:20:39 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.64500000000000002,"15min":0.63900000000000001},"min":{"1min":0.502,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.6339999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":0.63800000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:20:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.57099999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.54200000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:20:44 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":1.661,"5min":1.661,"15min":1.661},"last":0.66900000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182434,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:20:19 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.434},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.436,"15min":0.438},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.48999999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:20:19 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.45000000000000001,"15min":0.45400000000000001},"min":{"1min":0.315,"5min":0.312,"15min":0.312},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.438,"5min":0.45700000000000002,"15min":0.46000000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.54900000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:20:19 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.499,"15min":0.505},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.70099999999999996,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.48899999999999999,"15min":0.49199999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.63600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:20:41 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.60599999999999998,"15min":0.60899999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.86399999999999999,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.61499999999999999,"15min":0.61899999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.88900000000000001,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.66300000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.57699999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:20:21 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":3.181,"5min":3.181,"15min":3.181},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.59699999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.629},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.69599999999999995}]}]},{"osd":4,"up_from":30,"seq":128849018911,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.57799999999999996,"15min":0.58699999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53000000000000003,"15min":0.52900000000000003},"min":{"1min":0.32100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.53500000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56299999999999994,"15min":0.56599999999999995},"min":{"1min":0.32700000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.379,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.56399999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54200000000000004,"15min":0.54300000000000004},"min":{"1min":0.311,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.73499999999999999,"5min":0.75,"15min":0.75},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56899999999999995,"15min":0.57399999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.73599999999999999,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.57299999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.74299999999999999,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.56999999999999995,"15min":0.57499999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.5}]},{"osd":5,"last update":"Mon Oct 9 05:21:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.41699999999999998,"15min":0.41199999999999998},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.502,"5min":0.48499999999999999,"15min":0.48199999999999998},"min":{"1min":0.35999999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.35999999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.32000000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.34499999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822683,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.439}]},{"osd":1,"last update":"Mon Oct 9 05:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.375}]},{"osd":2,"last update":"Mon Oct 9 05:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.39600000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.47399999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.36299999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:20:24 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.51500000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.143,"5min":1.143,"15min":1.143},"last":0.499},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.46300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:21:06.457 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:21:06.483 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[101053]: audit 2023-10-09T05:21:05.339386+0000 mon.c 2023-10-09T05:21:06.484 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[101053]: (mon.2) 52 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:06.484 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[101053]: audit 2023-10-09T05:21:05.340205+0000 mon.c (mon.2) 53 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:06.484 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[101053]: audit 2023-10-09T05:21:05.345101+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:06.484 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[101053]: cephadm 2023-10-09T05:21:05.348731+0000 mgr.y (mgr.14442) 49 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:21:06.484 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[101053]: cephadm 2023-10-09T05:21:05.351992+0000 mgr.y (mgr.14442) 50 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi008 2023-10-09T05:21:06.484 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[110782]: audit 2023-10-09T05:21:05.339386+0000 2023-10-09T05:21:06.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[110782]: mon.c (mon.2) 52 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:06.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[110782]: audit 2023-10-09T05:21:05.340205+0000 mon.c (mon.2) 53 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:06.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[110782]: audit 2023-10-09T05:21:05.345101+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:06.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[110782]: cephadm 2023-10-09T05:21:05.348731+0000 mgr.y (mgr.14442) 49 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:21:06.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:06 smithi008 conmon[110782]: cephadm 2023-10-09T05:21:05.351992+0000 mgr.y (mgr.14442) 50 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi008 2023-10-09T05:21:06.540 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T05:21:06.540 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-10-09T05:21:06.540 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T05:21:06.540 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-10-09T05:21:06.540 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-10-09T05:21:06.799 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-10-09T05:21:06.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-10-09T05:21:06.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-10-09T05:21:06.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-10-09T05:21:06.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-10-09T05:21:06.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-10-09T05:21:07.069 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-10-09T05:21:07.069 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-10-09T05:21:07.069 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-10-09T05:21:07.069 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T05:21:07.069 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-10-09T05:21:07.069 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-10-09T05:21:07.069 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T05:21:07.070 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-10-09T05:21:07.070 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:06 smithi191 conmon[132642]: t=2023-10-09T05:21:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-10-09T05:21:07.321 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-10-09T05:21:07.321 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-10-09T05:21:07.321 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-10-09T05:21:07.321 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-10-09T05:21:07.321 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-10-09T05:21:07.321 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T05:21:07.359 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:07.521 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:07 smithi008 conmon[101053]: audit 2023-10-09T05:21:05.593499+0000 mgr.y (mgr.14442) 51 : audit [DBG] from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:21:07.521 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:07 smithi008 conmon[101053]: cluster 2023-10-09T05:21:05.901331+0000 mgr.y (mgr.14442) 52 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:07.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:07 smithi008 conmon[110782]: audit 2023-10-09T05:21:05.593499+0000 mgr.y (mgr.14442) 51 : audit [DBG] from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:21:07.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:07 smithi008 conmon[110782]: cluster 2023-10-09T05:21:05.901331+0000 mgr.y (mgr.14442) 52 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:07.589 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[101299]: audit 2023-10-09T05:21:05.593499+0000 mgr.y (mgr.14442) 51 : audit [DBG] from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:21:07.589 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[101299]: cluster 2023-10-09T05:21: 2023-10-09T05:21:07.589 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[101299]: 05.901331+0000 mgr.y (mgr.14442) 52 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:07.589 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T05:21:07.590 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-10-09T05:21:07.590 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-10-09T05:21:07.590 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-10-09T05:21:07.590 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-10-09T05:21:07.590 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-10-09T05:21:07.590 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-10-09T05:21:07.849 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-10-09T05:21:07.849 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-10-09T05:21:07.849 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-10-09T05:21:07.849 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-10-09T05:21:07.849 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-10-09T05:21:07.849 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-10-09T05:21:07.849 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T05:21:08.109 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T05:21:08.109 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T05:21:08.110 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-10-09T05:21:08.110 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:07 smithi191 conmon[132642]: t=2023-10-09T05:21:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T05:21:08.110 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-10-09T05:21:08.110 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-10-09T05:21:08.110 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-10-09T05:21:08.378 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-10-09T05:21:08.378 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-10-09T05:21:08.378 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-10-09T05:21:08.378 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-10-09T05:21:08.378 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-10-09T05:21:08.378 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-10-09T05:21:08.378 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-10-09T05:21:08.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-10-09T05:21:08.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-10-09T05:21:08.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-10-09T05:21:08.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T05:21:08.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T05:21:08.638 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T05:21:08.715 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:08 smithi008 systemd[1]: Stopping Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:21:08.715 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:08 smithi008 bash[150933]: Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-alertmanager.a found: no such container 2023-10-09T05:21:08.890 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T05:21:08.890 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-10-09T05:21:08.890 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-10-09T05:21:08.890 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-10-09T05:21:08.890 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-10-09T05:21:08.890 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T05:21:08.890 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-10-09T05:21:09.158 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-10-09T05:21:09.158 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:08 smithi191 conmon[132642]: t=2023-10-09T05:21:08+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-10-09T05:21:09.159 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-10-09T05:21:09.159 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-10-09T05:21:09.159 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-10-09T05:21:09.214 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:09 smithi008 conmon[143429]: level=info ts=2023-10-09T05:21:09.027Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:21:09.418 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[101299]: cluster 2023-10-09T05:21:07.901859+0000 mgr.y (mgr.14442) 53 : cluster [DBG] 2023-10-09T05:21:09.419 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[101299]: pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:09.419 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-10-09T05:21:09.419 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-10-09T05:21:09.419 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T05:21:09.419 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-10-09T05:21:09.419 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-10-09T05:21:09.419 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-10-09T05:21:09.419 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-10-09T05:21:09.670 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-10-09T05:21:09.670 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-10-09T05:21:09.670 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T05:21:09.670 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T05:21:09.670 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-10-09T05:21:09.679 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:09 smithi008 conmon[101053]: cluster 2023-10-09T05:21:07.901859 2023-10-09T05:21:09.680 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:09 smithi008 conmon[101053]: +0000 mgr.y (mgr.14442) 53 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:09.680 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:09 smithi008 conmon[110782]: cluster 2023-10-09T05:21:07. 2023-10-09T05:21:09.680 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:09 smithi008 conmon[110782]: 901859+0000 mgr.y (mgr.14442) 53 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:09.680 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:09 smithi008 bash[151062]: 041d72a037cc263a9fde86d1542d5c272e59afc1036ce9382be4d9351a948225 2023-10-09T05:21:09.930 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-10-09T05:21:09.930 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-10-09T05:21:09.930 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-10-09T05:21:09.930 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-10-09T05:21:09.930 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-10-09T05:21:09.931 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-10-09T05:21:10.141 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:21:10.142 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:21:10.190 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-10-09T05:21:10.190 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:09 smithi191 conmon[132642]: t=2023-10-09T05:21:09+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-10-09T05:21:10.190 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-10-09T05:21:10.190 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-10-09T05:21:10.191 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T05:21:10.191 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-10-09T05:21:10.336 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:10 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T05:21:10.336 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:10 smithi008 systemd[1]: Stopped Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:21:10.336 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:10 smithi008 systemd[1]: Starting Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:21:10.442 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-10-09T05:21:10.442 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-10-09T05:21:10.442 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-10-09T05:21:10.442 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-10-09T05:21:10.442 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T05:21:10.443 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-10-09T05:21:10.694 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-10-09T05:21:10.694 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-10-09T05:21:10.694 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-10-09T05:21:10.694 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-10-09T05:21:10.694 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-10-09T05:21:10.694 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-10-09T05:21:10.695 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-10-09T05:21:10.695 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-10-09T05:21:10.775 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":41,"stamp":"2023-10-09T05:21:09.902521+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380980,"kb_used_data":4212,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580236,"statfs":{"total":767960285184,"available":765522161664,"internally_reserved":0,"allocated":4313088,"data_stored":1763225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.646809+0000","last_change":"2023-10-09T05:20:23.226324+0000","last_active":"2023-10-09T05:20:24.646809+0000","last_peered":"2023-10-09T05:20:24.646809+0000","last_clean":"2023-10-09T05:20:24.646809+0000","last_became_active":"2023-10-09T05:20:23.225950+0000","last_became_peered":"2023-10-09T05:20:23.225950+0000","last_unstale":"2023-10-09T05:20:24.646809+0000","last_undegraded":"2023-10-09T05:20:24.646809+0000","last_fullsized":"2023-10-09T05:20:24.646809+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.646620+0000","last_change":"2023-10-09T05:20:23.589992+0000","last_active":"2023-10-09T05:20:24.646620+0000","last_peered":"2023-10-09T05:20:24.646620+0000","last_clean":"2023-10-09T05:20:24.646620+0000","last_became_active":"2023-10-09T05:20:23.589676+0000","last_became_peered":"2023-10-09T05:20:23.589676+0000","last_unstale":"2023-10-09T05:20:24.646620+0000","last_undegraded":"2023-10-09T05:20:24.646620+0000","last_fullsized":"2023-10-09T05:20:24.646620+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.646969+0000","last_change":"2023-10-09T05:20:23.732093+0000","last_active":"2023-10-09T05:20:24.646969+0000","last_peered":"2023-10-09T05:20:24.646969+0000","last_clean":"2023-10-09T05:20:24.646969+0000","last_became_active":"2023-10-09T05:20:23.731684+0000","last_became_peered":"2023-10-09T05:20:23.731684+0000","last_unstale":"2023-10-09T05:20:24.646969+0000","last_undegraded":"2023-10-09T05:20:24.646969+0000","last_fullsized":"2023-10-09T05:20:24.646969+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.646916+0000","last_change":"2023-10-09T05:20:23.731911+0000","last_active":"2023-10-09T05:20:24.646916+0000","last_peered":"2023-10-09T05:20:24.646916+0000","last_clean":"2023-10-09T05:20:24.646916+0000","last_became_active":"2023-10-09T05:20:23.731530+0000","last_became_peered":"2023-10-09T05:20:23.731530+0000","last_unstale":"2023-10-09T05:20:24.646916+0000","last_undegraded":"2023-10-09T05:20:24.646916+0000","last_fullsized":"2023-10-09T05:20:24.646916+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.647001+0000","last_change":"2023-10-09T05:19:46.035865+0000","last_active":"2023-10-09T05:20:24.647001+0000","last_peered":"2023-10-09T05:20:24.647001+0000","last_clean":"2023-10-09T05:20:24.647001+0000","last_became_active":"2023-10-09T05:19:46.035493+0000","last_became_peered":"2023-10-09T05:19:46.035493+0000","last_unstale":"2023-10-09T05:20:24.647001+0000","last_undegraded":"2023-10-09T05:20:24.647001+0000","last_fullsized":"2023-10-09T05:20:24.647001+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.322281+0000","last_change":"2023-10-09T05:20:23.227498+0000","last_active":"2023-10-09T05:20:24.322281+0000","last_peered":"2023-10-09T05:20:24.322281+0000","last_clean":"2023-10-09T05:20:24.322281+0000","last_became_active":"2023-10-09T05:20:23.227147+0000","last_became_peered":"2023-10-09T05:20:23.227147+0000","last_unstale":"2023-10-09T05:20:24.322281+0000","last_undegraded":"2023-10-09T05:20:24.322281+0000","last_fullsized":"2023-10-09T05:20:24.322281+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.231399+0000","last_change":"2023-10-09T05:20:23.731595+0000","last_active":"2023-10-09T05:20:24.231399+0000","last_peered":"2023-10-09T05:20:24.231399+0000","last_clean":"2023-10-09T05:20:24.231399+0000","last_became_active":"2023-10-09T05:20:23.731240+0000","last_became_peered":"2023-10-09T05:20:23.731240+0000","last_unstale":"2023-10-09T05:20:24.231399+0000","last_undegraded":"2023-10-09T05:20:24.231399+0000","last_fullsized":"2023-10-09T05:20:24.231399+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.322273+0000","last_change":"2023-10-09T05:20:23.227732+0000","last_active":"2023-10-09T05:20:24.322273+0000","last_peered":"2023-10-09T05:20:24.322273+0000","last_clean":"2023-10-09T05:20:24.322273+0000","last_became_active":"2023-10-09T05:20:23.227348+0000","last_became_peered":"2023-10-09T05:20:23.227348+0000","last_unstale":"2023-10-09T05:20:24.322273+0000","last_undegraded":"2023-10-09T05:20:24.322273+0000","last_fullsized":"2023-10-09T05:20:24.322273+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:20:24.646607+0000","last_change":"2023-10-09T05:20:23.731592+0000","last_active":"2023-10-09T05:20:24.646607+0000","last_peered":"2023-10-09T05:20:24.646607+0000","last_clean":"2023-10-09T05:20:24.646607+0000","last_became_active":"2023-10-09T05:20:23.731263+0000","last_became_peered":"2023-10-09T05:20:23.731263+0000","last_unstale":"2023-10-09T05:20:24.646607+0000","last_undegraded":"2023-10-09T05:20:24.646607+0000","last_fullsized":"2023-10-09T05:20:24.646607+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430228,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.8300000000000001,"5min":2.8300000000000001,"15min":2.8300000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.46200000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":2.2400000000000002,"5min":2.2400000000000002,"15min":2.2400000000000002},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.4809999999999999,"5min":2.4809999999999999,"15min":2.4809999999999999},"last":0.443}]},{"osd":2,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.7029999999999998,"5min":2.7029999999999998,"15min":2.7029999999999998},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.117,"5min":2.117,"15min":2.117},"last":0.59299999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.6669999999999998,"5min":2.6669999999999998,"15min":2.6669999999999998},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":2.1379999999999999,"5min":2.1379999999999999,"15min":2.1379999999999999},"last":0.628}]},{"osd":4,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.61599999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.64900000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.70199999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626456,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.61299999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.45500000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.65800000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.50600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"last":0.52200000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:20:23 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.47599999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:20:44 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.82699999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.32500000000000001},"min":{"1min":0.223,"5min":0.218,"15min":0.218},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.34899999999999998,"15min":0.33200000000000002},"min":{"1min":0.23799999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.33600000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:20:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.442,"15min":0.41499999999999998},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8460000000000001,"5min":2.8460000000000001,"15min":2.8460000000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.44500000000000001,"15min":0.40899999999999997},"min":{"1min":0.25700000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.9290000000000003,"5min":5.9290000000000003,"15min":5.9290000000000003},"last":0.29099999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:20:24 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.47599999999999998,"15min":0.47799999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.44,"5min":0.45400000000000001,"15min":0.45600000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.629,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.26700000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:20:40 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.59399999999999997,"15min":0.59199999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.64700000000000002,"15min":0.61199999999999999},"min":{"1min":0.46400000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.3330000000000002,"5min":6.3330000000000002,"15min":6.3330000000000002},"last":0.64700000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:21:02 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.61599999999999999,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.3639999999999999,"5min":6.3639999999999999,"15min":6.3639999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.59099999999999997,"15min":0.57399999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":2.8799999999999999,"5min":2.8799999999999999,"15min":2.8799999999999999},"last":0.63800000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:20:24 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.68600000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":2.8069999999999999,"5min":2.8069999999999999,"15min":2.8069999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.0920000000000001,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.65300000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.37,"15min":0.34799999999999998},"min":{"1min":0.23599999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.40300000000000002,"15min":0.36699999999999999},"min":{"1min":0.27100000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.484,"5min":2.484,"15min":2.484},"last":0.38500000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.45300000000000001,"15min":0.42999999999999999},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.4359999999999999,"5min":2.4359999999999999,"15min":2.4359999999999999},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41499999999999998,"15min":0.41099999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.98599999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:20:18 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.47099999999999997,"15min":0.47299999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.46500000000000002,"15min":0.46300000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.48199999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:20:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.78100000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:20:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.622},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.85999999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:20:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.69099999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":2.5830000000000002,"5min":2.5830000000000002,"15min":2.5830000000000002},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.65700000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411369,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.42799999999999999,"15min":0.40600000000000003},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.253},"max":{"1min":3.657,"5min":3.657,"15min":3.657},"last":0.317},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.40500000000000003,"15min":0.40100000000000002},"min":{"1min":0.246,"5min":0.222,"15min":0.222},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.34300000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.44700000000000001,"15min":0.433},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.6859999999999999,"5min":1.6859999999999999,"15min":1.6859999999999999},"last":0.376},{"interface":"front","average":{"1min":0.495,"5min":0.46999999999999997,"15min":0.46000000000000002},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.45500000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:20:17 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.45300000000000001,"15min":0.46200000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.58899999999999997,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.503,"15min":0.50800000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.69299999999999995,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.41299999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:20:39 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.64500000000000002,"15min":0.63900000000000001},"min":{"1min":0.502,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.6339999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":0.52100000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.607,"5min":1.607,"15min":1.607},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.60999999999999999,"15min":0.60599999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":0.55000000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.58399999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:20:44 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":1.661,"5min":1.661,"15min":1.661},"last":0.55900000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182435,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:20:19 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.436,"15min":0.438},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.34100000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:20:19 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.45000000000000001,"15min":0.45400000000000001},"min":{"1min":0.315,"5min":0.312,"15min":0.312},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.438,"5min":0.45700000000000002,"15min":0.46000000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.29999999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:20:19 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.499,"15min":0.505},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.70099999999999996,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.48899999999999999,"15min":0.49199999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.40999999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:20:41 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.60599999999999998,"15min":0.60899999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.86399999999999999,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.61499999999999999,"15min":0.61899999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.88900000000000001,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.68500000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.627,"15min":0.63100000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79900000000000004,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77900000000000003,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.58199999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:20:21 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":3.181,"5min":3.181,"15min":3.181},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.70299999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.64800000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018912,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.57799999999999996,"15min":0.58699999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53000000000000003,"15min":0.52900000000000003},"min":{"1min":0.32100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.71799999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56299999999999994,"15min":0.56599999999999995},"min":{"1min":0.32700000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.379,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.75900000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54200000000000004,"15min":0.54300000000000004},"min":{"1min":0.311,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.73499999999999999,"5min":0.75,"15min":0.75},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56899999999999995,"15min":0.57399999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.73599999999999999,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.61799999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.74299999999999999,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.56999999999999995,"15min":0.57499999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.66100000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:21:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.41699999999999998,"15min":0.41199999999999998},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.502,"5min":0.48499999999999999,"15min":0.48199999999999998},"min":{"1min":0.35999999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.73699999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.67300000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.71199999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822684,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.55900000000000005,"15min":0.56799999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70699999999999996,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.59399999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.54800000000000004,"15min":0.55200000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.66100000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73799999999999999,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.60799999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.53100000000000003,"15min":0.53800000000000003},"min":{"1min":0.39500000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.67700000000000005,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.498},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.59899999999999998,"15min":0.59699999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.708,"5min":1.708,"15min":1.708},"last":0.52200000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.57199999999999995,"15min":0.57799999999999996},"min":{"1min":0.39000000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.65500000000000003,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.623,"15min":0.629},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.79500000000000004,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.627}]},{"osd":4,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.40100000000000002,"15min":0.38700000000000001},"min":{"1min":0.27500000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.43099999999999999,"15min":0.41699999999999998},"min":{"1min":0.28999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.54400000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:20:24 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.65100000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.143,"5min":1.143,"15min":1.143},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.63700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:21:10.777 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:21:10.777 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T05:21:10.778 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T05:21:10.778 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T05:21:10.971 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-10-09T05:21:10.971 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-10-09T05:21:10.971 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-10-09T05:21:10.971 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-10-09T05:21:10.971 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-10-09T05:21:10.971 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-10-09T05:21:10.971 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-10-09T05:21:11.248 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:10 smithi191 conmon[132642]: t=2023-10-09T05:21:10+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-10-09T05:21:11.248 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-10-09T05:21:11.248 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-10-09T05:21:11.248 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-10-09T05:21:11.248 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-10-09T05:21:11.249 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T05:21:11.249 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-10-09T05:21:11.324 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:11.510 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[101053]: cluster 2023-10-09T05:21:11.510 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[101053]: 2023-10-09T05:21:09.902695+0000 mgr.y (mgr.14442) 54 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:11.510 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[101053]: audit 2023-10-09T05:21:10.136462+0000 mgr.y (mgr.14442) 55 : audit [DBG] from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:21:11.510 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[101053]: audit 2023-10-09T05:21:11.287167+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:11.511 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[151620]: level=info ts=2023-10-09T05:21:11.227Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T05:21:11.511 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[151620]: level=info ts=2023-10-09T05:21:11.227Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T05:21:11.511 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[151620]: level=info ts=2023-10-09T05:21:11.228Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.8 port=9094 2023-10-09T05:21:11.511 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[151620]: level=info ts=2023-10-09T05:21:11.230Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T05:21:11.511 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:11 smithi008 bash[151544]: 50083941a19709d3ff3906ce448387e3291040008ae4264f75bbcd1cb94d1201 2023-10-09T05:21:11.511 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:11 smithi008 systemd[1]: Started Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:21:11.511 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[151620]: level=info ts=2023-10-09T05:21:11.263Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:21:11.511 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[151620]: level=info ts=2023-10-09T05:21:11.263Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:21:11.512 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[151620]: level=info ts=2023-10-09T05:21:11.266Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T05:21:11.512 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[151620]: level=info ts=2023-10-09T05:21:11.266Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T05:21:11.512 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[110782]: cluster 2023-10-09T05:21 2023-10-09T05:21:11.512 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[110782]: :09.902695+0000 mgr.y (mgr.14442) 54 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:11.512 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[110782]: audit 2023-10-09T05:21:10.136462+0000 mgr.y (mgr.14442) 55 : audit [DBG] from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:21:11.512 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:11 smithi008 conmon[110782]: audit 2023-10-09T05:21:11.287167+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[101299]: cluster 2023-10-09T05:21:09.902695+0000 mgr.y (mgr.14442) 54 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[101299]: audit 2023-10-09T05:21:10.136462+0000 mgr.y (mgr.14442) 55 : audit [DBG] from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:21:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[101299]: audit 2023-10-09T05:21:11.287167+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:11.551 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-10-09T05:21:11.551 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-10-09T05:21:11.551 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T05:21:11.551 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-10-09T05:21:11.551 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-10-09T05:21:11.885 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-10-09T05:21:11.885 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-10-09T05:21:11.885 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T05:21:11.885 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T05:21:11.885 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T05:21:12.139 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:11 smithi191 conmon[132642]: t=2023-10-09T05:21:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-10-09T05:21:12.139 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[132642]: t=2023-10-09T05:21:12+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-10-09T05:21:12.139 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[132642]: t=2023-10-09T05:21:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T05:21:12.447 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[132642]: t=2023-10-09T05:21:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T05:21:12.448 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[132642]: t=2023-10-09T05:21:12+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-10-09T05:21:12.448 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[132642]: t=2023-10-09T05:21:12+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T05:21:12.448 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[132642]: t=2023-10-09T05:21:12+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-10-09T05:21:12.448 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[132642]: t=2023-10-09T05:21:12+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T05:21:12.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[101299]: cephadm 2023-10-09T05:21:11.288745+0000 mgr.y ( 2023-10-09T05:21:12.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[101299]: mgr.14442) 56 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:21:12.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[101299]: cephadm 2023-10-09T05:21:11.292559+0000 mgr.y (mgr.14442) 57 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:21:12.545 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:12 smithi008 conmon[101053]: cephadm 2023-10-09T05:21:11.288745+0000 mgr.y (mgr.14442) 56 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:21:12.545 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:12 smithi008 conmon[101053]: cephadm 2023-10-09T05:21:11.292559+0000 mgr.y (mgr.14442) 57 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:21:12.545 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:12 smithi008 conmon[110782]: cephadm 2023-10-09T05:21:11.288745+0000 mgr.y (mgr.14442) 56 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:21:12.545 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:12 smithi008 conmon[110782]: cephadm 2023-10-09T05:21:11.292559+0000 mgr.y (mgr.14442) 57 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:21:12.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[132642]: t=2023-10-09T05:21:12+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-10-09T05:21:12.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[132642]: t=2023-10-09T05:21:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T05:21:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:21:13.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:13] "GET /metrics HTTP/1.1" 200 155628 "" "Prometheus/2.33.4" 2023-10-09T05:21:13.253 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:12 smithi191 conmon[132642]: t=2023-10-09T05:21:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-10-09T05:21:13.424 INFO:teuthology.orchestra.run.smithi008.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T05:21:13.425 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T05:21:13.425 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T05:21:13.425 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T05:21:13.434 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi008.front.sepia.ceph.com 2023-10-09T05:21:13.434 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T05:21:13.544 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[132642]: t=2023-10-09T05:21:13+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-10-09T05:21:13.545 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[132642]: t=2023-10-09T05:21:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-10-09T05:21:13.545 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[132642]: t=2023-10-09T05:21:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-10-09T05:21:13.545 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[101299]: cluster 2023-10-09T05:21:11.903039+0000 mgr.y (mgr.14442) 58 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:13.545 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[101299]: audit 2023-10-09T05:21:12. 2023-10-09T05:21:13.545 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[101299]: 912287+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.8:0/74466117' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:21:13.714 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[151620]: level=info ts=2023-10-09T05:21:13.230Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000254462s 2023-10-09T05:21:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[110782]: cluster 2023-10-09T05:21:11.903039+0000 2023-10-09T05:21:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[110782]: mgr.y (mgr.14442) 58 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:21:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[110782]: :21:12.912287+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.8:0/74466117' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:21:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[110782]: 2023-10-09T05:21:13.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[101053]: cluster 2023-10-09T05:21:11. 2023-10-09T05:21:13.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[101053]: 903039+0000 mgr.y (mgr.14442) 58 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:13.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:21:13.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[101053]: T05:21:12.912287+0000 mon.c (mon.2) 54 : 2023-10-09T05:21:13.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:13 smithi008 conmon[101053]: audit [DBG] from='client.? 172.21.15.8:0/74466117' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:21:13.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[132642]: t=2023-10-09T05:21:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T05:21:13.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[132642]: t=2023-10-09T05:21:13+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T05:21:13.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[132642]: t=2023-10-09T05:21:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-10-09T05:21:13.800 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[132642]: t=2023-10-09T05:21:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-10-09T05:21:14.075 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[132642]: t=2023-10-09T05:21:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-10-09T05:21:14.075 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[132642]: t=2023-10-09T05:21:13+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T05:21:14.075 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:13 smithi191 conmon[132642]: t=2023-10-09T05:21:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T05:21:14.075 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[132642]: t=2023-10-09T05:21:14+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-10-09T05:21:14.075 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[132642]: t=2023-10-09T05:21:14+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-10-09T05:21:14.075 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[132642]: t=2023-10-09T05:21:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T05:21:14.344 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[132642]: t=2023-10-09T05:21:14+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-10-09T05:21:14.344 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[132642]: t=2023-10-09T05:21:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T05:21:14.344 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[132642]: t=2023-10-09T05:21:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T05:21:14.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[132642]: t=2023-10-09T05:21:14+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T05:21:14.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[132642]: t=2023-10-09T05:21:14+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-10-09T05:21:14.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[132642]: t=2023-10-09T05:21:14+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-10-09T05:21:14.755 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:21:15.031 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[132642]: t=2023-10-09T05:21:14+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-10-09T05:21:15.031 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:14 smithi191 conmon[132642]: t=2023-10-09T05:21:14+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-10-09T05:21:15.291 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 systemd[1]: Stopping Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:21:15.291 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 bash[133929]: Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-prometheus.a found: no such container 2023-10-09T05:21:15.291 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-10-09T05:21:15.291 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T05:21:15.291 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-10-09T05:21:15.291 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-10-09T05:21:15.550 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-10-09T05:21:15.550 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-10-09T05:21:15.550 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-10-09T05:21:15.550 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-10-09T05:21:15.550 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-10-09T05:21:15.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.478Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:21:15.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.478Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T05:21:15.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.478Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T05:21:15.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.478Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T05:21:15.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.478Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T05:21:15.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.478Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T05:21:15.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.478Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T05:21:15.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.478Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T05:21:15.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.478Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T05:21:15.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.479Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T05:21:15.552 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.479Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T05:21:15.552 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[130670]: ts=2023-10-09T05:21:15.479Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T05:21:15.931 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-10-09T05:21:15.931 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-10-09T05:21:15.931 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T05:21:15.931 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[101299]: cluster 2023-10-09T05:21:13.903386+0000 mgr.y (mgr 2023-10-09T05:21:15.931 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[101299]: .14442) 59 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:15.931 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[101299]: audit 2023-10-09T05:21:15.276599+0000 2023-10-09T05:21:15.931 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[101299]: mon.c (mon.2) 55 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:21:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:15 smithi008 conmon[101053]: cluster 2023-10-09T05:21:13.903386+0000 mgr.y (mgr.14442) 59 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:15 smithi008 conmon[101053]: audit 2023-10-09T05:21:15.276599+0000 mon.c (mon.2) 55 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:21:16.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:15 smithi008 conmon[110782]: cluster 2023-10-09T05:21:13.903386+0000 mgr.y (mgr.14442) 59 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:16.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:15 smithi008 conmon[110782]: audit 2023-10-09T05:21:15.276599+0000 mon.c (mon.2) 55 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:21:16.245 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:15 smithi191 conmon[132642]: t=2023-10-09T05:21:15+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-10-09T05:21:16.245 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-10-09T05:21:16.245 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-10-09T05:21:16.497 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:16 smithi191 bash[134044]: f5f1d0f2fc50bbd5908daecb525bb3393d313d6ef81df154605ffc739bc6c05a 2023-10-09T05:21:16.498 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:16 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T05:21:16.498 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:16 smithi191 systemd[1]: Stopped Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:21:16.498 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:16 smithi191 systemd[1]: Starting Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:21:16.498 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-10-09T05:21:16.498 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-10-09T05:21:16.498 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T05:21:16.498 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-10-09T05:21:16.498 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-10-09T05:21:16.498 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-10-09T05:21:16.498 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-10-09T05:21:16.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[101299]: audit 2023-10-09T05:21:15.276029+0000 mgr.y (mgr.14442) 60 : audit [DBG] from='client.24524 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:21:16.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[101299]: audit 2023-10-09T05:21:15.277008+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:21:16.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[101299]: audit 2023-10-09T05:21:15.766747+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T05:21:16.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[101299]: cluster 2023-10-09T05:21:15.766812+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T05:21:16.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[101299]: audit 2023-10-09T05:21:15.771771+0000 2023-10-09T05:21:16.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[101299]: mon.c (mon.2) 56 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:16.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[101299]: audit 2023-10-09T05:21:15.772074 2023-10-09T05:21:16.789 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[101299]: +0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:16.790 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[101299]: cluster 2023-10-09T05:21:15.903842+0000 mgr.y (mgr 2023-10-09T05:21:16.790 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[101299]: .14442) 61 : cluster [DBG] pgmap v45: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:16.790 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-10-09T05:21:16.790 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-10-09T05:21:16.790 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-10-09T05:21:16.790 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-10-09T05:21:16.790 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-10-09T05:21:16.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: audit 2023-10-09T05:21:15.276029+0000 mgr.y (mgr.14442) 2023-10-09T05:21:16.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: 60 : audit [DBG] from='client.24524 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:21:16.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: audit 2023-10-09T05:21:15.277008+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:21:16.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: audit 2023-10-09T05:21:15.766747+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T05:21:16.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: cluster 2023-10-09T05:21:15.766812+0000 2023-10-09T05:21:16.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: mon.a (mon.0) 608 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T05:21:16.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: audit 2023-10-09T05:21:15.771771+0000 mon.c (mon 2023-10-09T05:21:16.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: .2) 56 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:16.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:21:16.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: 10-09T05:21:15.772074+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:16.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: cluster 2023-10-09T05:21:15.903842+0000 2023-10-09T05:21:16.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: mgr.y (mgr.14442) 61 : cluster 2023-10-09T05:21:16.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[110782]: [DBG] pgmap v45: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:16.985 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: audit 2023-10-09T05:21:15. 2023-10-09T05:21:16.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: 276029+0000 mgr.y (mgr.14442) 60 : audit [DBG] from='client.24524 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:21:16.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: audit 2023-10-09T05:21:16.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: 2023-10-09T05:21:15.277008+0000 mon.a 2023-10-09T05:21:16.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: (mon.0) 606 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:21:16.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:21:16.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: 10-09T05:21:15.766747+0000 mon.a (mon.0 2023-10-09T05:21:16.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: ) 607 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T05:21:16.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: cluster 2023-10- 2023-10-09T05:21:16.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: 09T05:21:15.766812+0000 mon.a ( 2023-10-09T05:21:16.986 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: mon.0) 608 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: 09T05:21:15.771771+0000 mon.c (mon. 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: 2) 56 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: 09T05:21:15.772074+0000 mon.a 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: (mon.0) 609 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: cluster 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: 2023-10-09T05:21:15. 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: 903842+0000 mgr.y (mgr.14442) 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: 61 : cluster [DBG] pgmap v45: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:16.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:16 smithi008 conmon[101053]: debug 2023-10-09T05:21:16.789+0000 7ff4a5206700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:21:17.049 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-10-09T05:21:17.050 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-10-09T05:21:17.050 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:16 smithi191 conmon[132642]: t=2023-10-09T05:21:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T05:21:17.050 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-10-09T05:21:17.050 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-10-09T05:21:17.266 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STOPPING 2023-10-09T05:21:17.421 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-10-09T05:21:17.422 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T05:21:17.422 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-10-09T05:21:17.422 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-10-09T05:21:17.422 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-10-09T05:21:17.422 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-10-09T05:21:17.422 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-10-09T05:21:17.422 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.183Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T05:21:17.423 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.183Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T05:21:17.423 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.183Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi191 (none))" 2023-10-09T05:21:17.423 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.183Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T05:21:17.423 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.183Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T05:21:17.423 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.185Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T05:21:17.423 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.186Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T05:21:17.423 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.187Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T05:21:17.423 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.189Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T05:21:17.423 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.189Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.925µs 2023-10-09T05:21:17.423 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.189Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T05:21:17.423 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 bash[134370]: 0e52be64285ae43ecb53bab5b2d0295241d343e8e87ed562bfe55de45515a516 2023-10-09T05:21:17.424 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 systemd[1]: Started Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:21:17.424 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.373Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T05:21:17.424 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.373Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T05:21:17.424 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.373Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.407µs wal_replay_duration=184.177758ms total_replay_duration=184.227602ms 2023-10-09T05:21:17.424 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.375Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T05:21:17.424 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.375Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T05:21:17.424 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.375Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T05:21:17.424 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.391Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.602509ms db_storage=999ns remote_storage=1.333µs web_handler=327ns query_engine=672ns scrape=402.141µs scrape_sd=27.24µs notify=27.252µs notify_sd=7.552µs rules=15.912068ms 2023-10-09T05:21:17.424 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[134411]: ts=2023-10-09T05:21:17.391Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T05:21:17.451 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T05:21:17.613 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:21:17.614 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STOPPED 2023-10-09T05:21:17.614 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STARTING 2023-10-09T05:21:17.614 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Serving on http://:::9283 2023-10-09T05:21:17.614 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STARTED 2023-10-09T05:21:17.614 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STOPPING 2023-10-09T05:21:17.614 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:21:17.614 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STOPPED 2023-10-09T05:21:17.614 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STARTING 2023-10-09T05:21:17.614 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Serving on http://:::9283 2023-10-09T05:21:17.614 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STARTED 2023-10-09T05:21:17.615 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STOPPING 2023-10-09T05:21:17.615 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:21:17.615 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STOPPED 2023-10-09T05:21:17.615 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STARTING 2023-10-09T05:21:17.615 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Serving on http://:::9283 2023-10-09T05:21:17.615 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101509]: [09/Oct/2023:05:21:17] ENGINE Bus STARTED 2023-10-09T05:21:17.690 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-10-09T05:21:17.690 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T05:21:17.691 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-10-09T05:21:17.691 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-10-09T05:21:17.691 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-10-09T05:21:17.926 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21:16.771966+0000 mon.a ( 2023-10-09T05:21:17.926 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: mon.0) 610 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T05:21:17.926 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: cluster 2023-10-09T05:21:16.772052+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T05:21:17.926 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21:16.787689+0000 mon.c (mon.2) 57 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:17.926 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21:16. 2023-10-09T05:21:17.926 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 788643+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:17.926 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: cluster 2023-10-09T05:21 2023-10-09T05:21:17.926 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :16.789545+0000 mon.a (mon.0) 613 : cluster [ERR] 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: cluster 2023- 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 10-09T05:21:16.789572+0000 mon.a (mon. 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 0) 614 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: cluster 2023-10-09T05 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :21:16.800186+0000 mon.a (mon. 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 0) 615 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 05:21:16.800319+0000 mon.a (mon 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: .0) 616 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: cluster 2023-10-09 2023-10-09T05:21:17.927 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: T05:21:16.800432+0000 mon.a ( 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: mon.0) 617 : cluster [DBG] fsmap foo:0 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: cephadm 2023-10- 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 09T05:21:16.804281+0000 mgr.y 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: (mgr.14442) 62 : cephadm 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: [INF] Saving service mds.foo spec with placement count:2 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 2023-10-09T05:21:16.811772+0000 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: mon.a (mon.0) 618 : audit [INF] 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 10-09T05:21:17.088575+0000 mon.a (mon 2023-10-09T05:21:17.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: .0) 619 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: T05:21:17.250265+0000 mon.a ( 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: mon.0) 620 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 05:21:17.253875+0000 mon.c (mon. 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 2) 58 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :21:17.254318+0000 mgr.y (mgr. 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 14442) 63 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :21:17.255187+0000 mon.c (mon. 2023-10-09T05:21:17.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 2) 59 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008:9093"}]: dispatch 2023-10-09T05:21:17.930 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:21:17.930 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :21:17.255561+0000 mgr.y (mgr.14442 2023-10-09T05:21:17.930 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: ) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008:9093"}]: dispatch 2023-10-09T05:21:17.930 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:21:17.930 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 21:17.259523+0000 mon.a (mon.0) 2023-10-09T05:21:17.930 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 621 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.930 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21 2023-10-09T05:21:17.930 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :17.264625+0000 mon.c (mon.2) 2023-10-09T05:21:17.930 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 60 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:21:17.930 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21 2023-10-09T05:21:17.930 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :17.265202+0000 mgr.y (mgr.14442) 2023-10-09T05:21:17.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:21:17.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21: 2023-10-09T05:21:17.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 17.267475+0000 mon.c (mon.2) 61 2023-10-09T05:21:17.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:21:17.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21 2023-10-09T05:21:17.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :17.268009+0000 mgr.y (mgr.14442) 2023-10-09T05:21:17.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:21:17.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :17.272700+0000 mon.a (mon.0) 622 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :17.277628+0000 mon.c (mon.2) 62 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21: 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 17.278132+0000 mgr.y (mgr.14442) 67 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :17.283623+0000 mon.c (mon.2) 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 63 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21 2023-10-09T05:21:17.932 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :17.284028+0000 mgr.y (mgr.14442) 2023-10-09T05:21:17.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:21:17.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21 2023-10-09T05:21:17.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :17.288540+0000 mon.a (mon.0) 2023-10-09T05:21:17.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 623 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: audit 2023-10-09T05:21 2023-10-09T05:21:17.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: :17.292443+0000 mon.c (mon.2) 2023-10-09T05:21:17.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[101053]: 64 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:21:17.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T05:21:16.771966+0000 mon.a 2023-10-09T05:21:17.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: (mon.0) 610 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T05:21:17.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: cluster 2023-10-09T05:21:16. 2023-10-09T05:21:17.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 772052+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T05:21:17.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:21:17.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 05:21:16.787689+0000 mon.c (mon.2) 2023-10-09T05:21:17.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 57 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:17.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:21:17.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 09T05:21:16.788643+0000 mon.a (mon 2023-10-09T05:21:17.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: .0) 612 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:17.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:21:17.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: -10-09T05:21:16.789545+0000 2023-10-09T05:21:17.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: mon.a (mon.0) 613 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:21:17.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 2023-10-09T05:21:17.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: cluster 2023-10-09T05:21:17.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 2023-10-09T05:21:16.789572+0000 mon.a ( 2023-10-09T05:21:17.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: mon.0) 614 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T05:21:17.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:21:17.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 10-09T05:21:16.800186+0000 mon.a ( 2023-10-09T05:21:17.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: mon.0) 615 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T05:21:17.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023 2023-10-09T05:21:17.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: -10-09T05:21:16.800319+0000 mon.a 2023-10-09T05:21:17.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: (mon.0) 616 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T05:21:17.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:21:17.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: -10-09T05:21:16.800432+0000 mon.a 2023-10-09T05:21:17.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: (mon.0) 617 : cluster [DBG] fsmap foo:0 2023-10-09T05:21:17.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: cephadm 2023-10-09T05:21:17.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 2023-10-09T05:21:16.804281+0000 2023-10-09T05:21:17.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: mgr.y (mgr.14442) 62 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T05:21:16. 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 811772+0000 mon.a (mon.0) 618 : audit 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T05:21 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: :17.088575+0000 mon.a (mon.0) 619 : 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 05:21:17.250265+0000 mon.a (mon.0 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: ) 620 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:21:17.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 09T05:21:17.253875+0000 mon.c (mon 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: .2) 58 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: :21:17.254318+0000 mgr.y (mgr.14442) 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 63 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 05:21:17.255187+0000 mon.c (mon.2 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: ) 59 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008:9093"}]: dispatch 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 2023-10-09T05:21:17.255561+0000 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: mgr.y (mgr.14442) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008:9093"}]: dispatch 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 2023-10-09T05:21:17.938 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T05:21:17. 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 259523+0000 mon.a (mon.0) 621 : audit 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T05:21 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: :17.264625+0000 mon.c (mon.2) 60 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: :21:17.265202+0000 mgr.y (mgr.14442) 65 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: T05:21:17.267475+0000 mon.c (mon 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: .2) 61 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:21:17.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 10-09T05:21:17.268009+0000 mgr.y ( 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: mgr.14442) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 10-09T05:21:17.272700+0000 mon.a ( 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: mon.0) 622 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: -10-09T05:21:17.277628+0000 mon.c 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: (mon.2) 62 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 2023-10-09T05:21:17.278132+0000 mgr.y (mgr. 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 14442) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:21:17.940 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: -09T05:21:17.283623+0000 mon.c ( 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: mon.2) 63 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 10-09T05:21:17.284028+0000 mgr.y 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: (mgr.14442) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 2023-10-09T05:21:17.288540+0000 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: mon.a (mon.0) 623 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: audit 2023-10-09T05:21:17. 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: 292443+0000 mon.c (mon.2) 64 : audit [DBG] 2023-10-09T05:21:17.941 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:17 smithi008 conmon[110782]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:21:17.972 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-10-09T05:21:17.972 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=18.878859987s 2023-10-09T05:21:17.972 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T05:21:17.972 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T05:21:17.973 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T05:21:17.973 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T05:21:17.973 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T05:21:17.973 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T05:21:17.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05:21:16.771966+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T05:21:17.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 2023-10-09T05:21:17.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: cluster 2023-10-09T05:21:16.772052+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T05:21:17.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05:21:16.787689+0000 mon.c (mon.2) 57 : audit [INF] 2023-10-09T05:21:17.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:17.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05:21:16.788643+0000 mon.a (mon.0) 612 : audit [INF] 2023-10-09T05:21:17.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:21:17.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: cluster 2023-10-09T 2023-10-09T05:21:17.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 05:21:16.789545+0000 mon.a (mon.0) 613 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:21:17.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: cluster 2023-10-09T05:21: 2023-10-09T05:21:17.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 16.789572+0000 mon.a (mon.0) 614 : cluster 2023-10-09T05:21:17.976 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T05:21:17.976 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: cluster 2023- 2023-10-09T05:21:17.976 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 10-09T05:21:16.800186+0000 2023-10-09T05:21:17.976 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: mon.a (mon.0) 615 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T05:21:17.976 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:21:17.976 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 21:16.800319+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T05:21:17.976 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 2023-10-09T05:21:17.976 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: cluster 2023-10-09T05: 2023-10-09T05:21:17.977 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 21:16.800432+0000 mon.a (mon.0) 617 : cluster [DBG] fsmap foo:0 2023-10-09T05:21:17.977 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: cephadm 2023-10-09T05:21:16.804281+0000 mgr.y 2023-10-09T05:21:17.977 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: (mgr.14442) 62 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T05:21:17.977 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05:21:16 2023-10-09T05:21:17.977 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: .811772+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.977 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 2023-10-09T05:21:17.977 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05:21:17.978 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 2023-10-09T05:21:17.088575+0000 mon.a (mon. 2023-10-09T05:21:17.978 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 0) 619 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.978 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05:21:17.978 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 2023-10-09T05:21:17.250265+0000 mon.a (mon.0) 620 : 2023-10-09T05:21:17.978 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.978 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023 2023-10-09T05:21:17.978 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: -10-09T05:21:17.253875+0000 mon.c (mon.2) 58 : audit [DBG] 2023-10-09T05:21:17.979 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:21:17.981 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:21:17.981 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: T05:21:17.254318+0000 mgr.y (mgr.14442) 63 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:21:17.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05:21: 2023-10-09T05:21:17.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 17.255187+0000 mon.c (mon.2) 59 : audit [INF] 2023-10-09T05:21:17.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008:9093"}]: dispatch 2023-10-09T05:21:17.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:21:17.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 10-09T05:21:17.255561+0000 mgr.y (mgr.14442) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008:9093"}]: dispatch 2023-10-09T05:21:17.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 2023-10-09T05:21:17.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:21:17.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 09T05:21:17.259523+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:21:17.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 05:21:17.264625+0000 mon.c (mon.2) 60 : audit [DBG] 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: -09T05:21:17.265202+0000 mgr.y (mgr.14442) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 05:21:17.267475+0000 mon.c (mon.2) 61 : audit [INF] 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 09T05:21:17.268009+0000 mgr.y (mgr.14442) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05:21 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: :17.272700+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:21:17.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 05:21:17.277628+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: -10-09T05:21:17.278132+0000 mgr.y (mgr.14442) 67 : audit [DBG] 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 09T05:21:17.283623+0000 mon.c (mon.2) 63 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 21:17.284028+0000 mgr.y (mgr.14442) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05:21: 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 17.288540+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit 2023-10-09T05:21:17.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: 2023-10-09T05:21:17.292443+0000 mon.c (mon.2) 64 : 2023-10-09T05:21:17.985 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[101299]: audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:21:18.299 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T05:21:18.299 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:17 smithi191 conmon[132642]: t=2023-10-09T05:21:17+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T05:21:18.300 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:18 smithi191 conmon[132642]: t=2023-10-09T05:21:18+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T05:21:18.300 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:18 smithi191 conmon[132642]: t=2023-10-09T05:21:18+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T05:21:18.802 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:21:18 smithi191 conmon[132642]: t=2023-10-09T05:21:18+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T05:21:18.802 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:18 smithi008 conmon[110782]: cluster 2023-10-09T05:21:17.800029+0000 mon.a (mon.0) 624 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T05:21:18.802 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:18 smithi008 conmon[110782]: cluster 2023-10-09T05:21:17.904320+0000 mgr.y (mgr.14442) 69 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:19.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:18 smithi008 conmon[101053]: cluster 2023-10-09T05:21:17.800029+0000 mon.a (mon.0) 624 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T05:21:19.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:18 smithi008 conmon[101053]: cluster 2023-10-09T05:21:17.904320+0000 mgr.y (mgr.14442) 69 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:19.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:18 smithi191 conmon[101299]: cluster 2023-10-09T05:21:17.800029+0000 mon.a (mon.0) 2023-10-09T05:21:19.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:18 smithi191 conmon[101299]: 624 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T05:21:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:18 smithi191 conmon[101299]: cluster 2023-10-09T05:21:17.904320+0000 mgr.y (mgr.14442) 69 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:20.372 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T05:21:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: cluster 2023-10-09T05:21:19.904720+0000 mgr.y (mgr.14442) 70 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: audit 2023-10-09T05:21:20.236392 2023-10-09T05:21:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: +0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: audit 2023-10-09T05:21:20.757146+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: audit 2023-10-09T05:21:20.758428+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: audit 2023-10-09T05:21:20.759195+0000 mon.c (mon.2) 66 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: audit 2023-10-09T05:21:20.764238+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: audit 2023-10-09T05:21:20.773212+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: audit 2023-10-09T05:21:20.775200+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: audit 2023-10-09T05:21:20.775513+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: audit 2023-10-09T05:21:20.779026+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:21:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:21 smithi191 conmon[101299]: audit 2023-10-09T05:21:20.779981+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:21.714 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[151620]: level=info ts=2023-10-09T05:21:21.233Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003102392s 2023-10-09T05:21:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: cluster 2023-10-09T05:21:19.904720+0000 mgr.y ( 2023-10-09T05:21:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: mgr.14442) 70 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit 2023-10-09T05:21: 2023-10-09T05:21:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: 20.236392+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit 2023-10-09T05:21:20.757146 2023-10-09T05:21:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: +0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:21:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: -09T05:21:20.758428+0000 mon.c (mon 2023-10-09T05:21:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: .2) 65 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit 2023-10-09T05:21 2023-10-09T05:21:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: :20.759195+0000 mon.c (mon.2) 66 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: 2023-10-09T05:21:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:21:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: 09T05:21:20.764238+0000 mon.a (mon.0) 627 : audit 2023-10-09T05:21:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit 2023 2023-10-09T05:21:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: -10-09T05:21:20.773212+0000 mon.a (mon.0) 628 : audit 2023-10-09T05:21:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit 2023 2023-10-09T05:21:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: -10-09T05:21:20.775200+0000 mon.c (mon.2) 67 : 2023-10-09T05:21:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit 2023-10-09T05:21:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: 2023-10-09T05:21:20.775513+0000 mon.a (mon.0) 629 : 2023-10-09T05:21:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit 2023-10-09T05:21:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: 2023-10-09T05:21:20.779026+0000 mon.a (mon.0) 630 : audit 2023-10-09T05:21:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:21:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: audit 2023-10-09T05:21 2023-10-09T05:21:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: :20.779981+0000 mon.c (mon.2) 68 2023-10-09T05:21:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[110782]: : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:21.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: cluster 2023-10-09T05:21:19.904720+0000 2023-10-09T05:21:21.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: mgr.y (mgr.14442) 70 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:21.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: audit 2023-10-09T05:21:20 2023-10-09T05:21:21.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: .236392+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:21:21.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: 21:20.757146+0000 mon.a (mon.0) 2023-10-09T05:21:21.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: 626 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:21:21.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: 21:20.758428+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:21.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: 2023-10-09T05:21:21.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: audit 2023 2023-10-09T05:21:21.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: -10-09T05:21:20.759195+0000 mon.c (mon. 2023-10-09T05:21:21.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: 2) 66 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: T05:21:20.764238+0000 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: mon.a (mon.0) 627 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: audit 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: 2023-10-09T05:21 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: :20.773212+0000 mon.a (mon.0) 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: 628 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: audit 2023 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: -10-09T05:21: 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: 20.775200+0000 mon.c (mon.2) 67 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:21.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:21:21.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: -09T05:21:20.775513 2023-10-09T05:21:21.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: +0000 mon.a (mon.0) 629 : audit 2023-10-09T05:21:21.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:21.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:21:21.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: T05:21:20.779026 2023-10-09T05:21:21.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: +0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:21:21.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:21:21.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: 10-09T05:21: 2023-10-09T05:21:21.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: 20.779981+0000 mon.c (mon.2) 68 : audit [DBG] 2023-10-09T05:21:21.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:21 smithi008 conmon[101053]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:22.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:22 smithi008 conmon[101053]: cephadm 2023-10-09T05:21:20.780672+0000 mgr.y ( 2023-10-09T05:21:22.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:22 smithi008 conmon[101053]: mgr.14442) 71 : cephadm [INF] Deploying daemon mds.foo.smithi008.zfejpl on smithi008 2023-10-09T05:21:22.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:22 smithi008 conmon[110782]: cephadm 2023-10-09T05:21:20.780672+0000 mgr.y (mgr.14442) 71 2023-10-09T05:21:22.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:22 smithi008 conmon[110782]: : cephadm [INF] Deploying daemon mds.foo.smithi008.zfejpl on smithi008 2023-10-09T05:21:22.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:22 smithi191 conmon[101299]: cephadm 2023-10-09T05:21:20.780672+0000 mgr.y (mgr.14442) 71 : cephadm [INF] Deploying daemon mds.foo.smithi008.zfejpl on smithi008 2023-10-09T05:21:23.345 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T05:21:23.355 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T05:21:23.355 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T05:21:23.356 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T05:21:23.416 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:23 smithi008 conmon[101053]: cluster 2023-10-09T05:21: 2023-10-09T05:21:23.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:23 smithi008 conmon[101053]: 21.905038+0000 mgr.y (mgr.14442) 72 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:23.417 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:23 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:23] "GET /metrics HTTP/1.1" 200 160649 "" "Prometheus/2.33.4" 2023-10-09T05:21:23.417 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:23 smithi008 conmon[110782]: cluster 2023-10-09T05:21:21.905038+0000 2023-10-09T05:21:23.417 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:23 smithi008 conmon[110782]: mgr.y (mgr.14442) 72 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:23.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:23 smithi191 conmon[101299]: cluster 2023-10-09T05:21:21.905038+0000 mgr.y (mgr.14442) 72 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:23.788 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:25.015 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:21:24 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:21:25.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: cluster 2023-10-09T05:21:23.905700+0000 mgr.y (mgr.14442) 73 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: audit 2023-10-09T05:21:24.431824+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: audit 2023-10-09T05:21:24.433326+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: audit 2023-10-09T05:21:24.433608+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: audit 2023-10-09T05:21:24.436552+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:21:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: audit 2023-10-09T05:21:24.437367+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: cephadm 2023-10-09T05:21:24.437983+0000 mgr.y (mgr.14442) 74 : cephadm [INF] Deploying daemon mds.foo.smithi191.walafj on smithi191 2023-10-09T05:21:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: cluster 2023-10-09T05:21:25.245858+0000 mon.a (mon.0) 634 : cluster [DBG] mds.? [v2:172.21.15.8:6834/560051770,v1:172.21.15.8:6835/560051770] up:boot 2023-10-09T05:21:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: cluster 2023-10-09T05:21:25.245917+0000 mon.a (mon.0) 635 : cluster [INF] daemon mds.foo.smithi008.zfejpl assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T05:21:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: cluster 2023-10-09T05:21:25.245994+0000 mon.a (mon.0) 636 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T05:21:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: cluster 2023-10-09T05:21:25.246020+0000 mon.a (mon.0) 637 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T05:21:25.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: cluster 2023-10-09T05:21:25.246031+0000 mon.a (mon.0) 638 : cluster [INF] Cluster is now healthy 2023-10-09T05:21:25.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: audit 2023-10-09T05:21:25.246031+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi008.zfejpl"}]: dispatch 2023-10-09T05:21:25.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: cluster 2023-10-09T05:21:25.246225+0000 mon.a (mon.0) 639 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T05:21:25.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: cluster 2023-10-09T05:21:25.249020+0000 mon.a (mon.0) 640 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:creating} 2023-10-09T05:21:25.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[101053]: cluster 2023-10-09T05:21:25.260678+0000 mon.a (mon.0) 641 : cluster [INF] daemon mds.foo.smithi008.zfejpl is now active in filesystem foo as rank 0 2023-10-09T05:21:25.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: cluster 2023-10-09T05:21:23.905700+0000 mgr.y (mgr.14442) 73 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:25.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: audit 2023-10-09T05:21:24.431824+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:25.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: audit 2023-10-09T05:21:24.433326+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:25.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: audit 2023-10-09T05:21:24.433608+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:25.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: audit 2023-10-09T05:21:24.436552+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: :21:24.437367+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: cephadm 2023-10-09T05:21:24.437983+0000 mgr.y (mgr.14442) 74 : cephadm [INF] Deploying daemon mds.foo.smithi191.walafj on smithi191 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: cluster 2023-10-09T05:21:25.245858+0000 mon.a (mon.0) 634 : cluster [DBG] mds.? [v2:172.21.15.8:6834/560051770,v1:172.21.15.8:6835/560051770] up:boot 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: cluster 2023-10-09T05:21:25.245917+0000 mon.a (mon.0) 635 : cluster [INF] daemon mds.foo.smithi008.zfejpl assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: cluster 2023-10-09T05:21:25.245994+0000 mon.a (mon.0) 636 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: cluster 2023-10-09T05:21:25.246020+0000 mon.a (mon.0) 637 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: cluster 2023-10-09T05:21:25.246031+0000 mon.a (mon.0) 638 : cluster [INF] Cluster is now healthy 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: audit 2023-10-09T05:21:25.246031+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi008.zfejpl"}]: dispatch 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: cluster 2023-10-09T05:21:25.246225+0000 mon.a (mon.0) 639 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: cluster 2023-10-09T05:21:25.249020+0000 mon.a (mon.0) 640 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:creating} 2023-10-09T05:21:25.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:25 smithi008 conmon[110782]: cluster 2023-10-09T05:21:25.260678+0000 mon.a (mon.0) 641 : cluster [INF] daemon mds.foo.smithi008.zfejpl is now active in filesystem foo as rank 0 2023-10-09T05:21:25.795 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:21:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: cluster 2023-10-09T05:21:23.905700+0000 mgr.y (mgr.14442) 73 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: audit 2023-10-09T05:21:24.431824+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: audit 2023-10-09T05:21:24.433326+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: audit 2023-10-09T05:21:24.433608+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:21:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: audit 2023-10-09T05:21:24.436552+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:21:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: audit 2023-10-09T05:21:24.437367+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: cephadm 2023-10-09T05:21:24.437983+0000 mgr.y (mgr.14442) 74 : cephadm [INF] Deploying daemon mds.foo.smithi191.walafj on smithi191 2023-10-09T05:21:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: cluster 2023-10-09T05:21:25.245858+0000 mon.a (mon.0) 634 : cluster [DBG] mds.? [v2:172.21.15.8:6834/560051770,v1:172.21.15.8:6835/560051770] up:boot 2023-10-09T05:21:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: cluster 2023-10-09T05:21:25.245917+0000 mon.a (mon.0) 635 : cluster [INF] daemon mds.foo.smithi008.zfejpl assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T05:21:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: cluster 2023-10 2023-10-09T05:21:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: -09T05:21:25.245994+0000 mon.a (mon.0) 636 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T05:21:25.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: cluster 2023-10-09T05:21:25.246020+0000 mon.a (mon.0) 637 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T05:21:25.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: cluster 2023-10-09T05:21:25.246031+0000 mon.a (mon.0) 638 : cluster [INF] Cluster is now healthy 2023-10-09T05:21:25.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: audit 2023-10-09T05:21:25.246031+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi008.zfejpl"}]: dispatch 2023-10-09T05:21:25.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: cluster 2023-10-09T05:21:25.246225+0000 mon.a (mon.0) 639 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T05:21:25.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: cluster 2023-10-09T05:21:25.249020+0000 mon.a (mon.0) 640 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:creating} 2023-10-09T05:21:25.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:25 smithi191 conmon[101299]: cluster 2023-10-09T05:21:25.260678+0000 mon.a (mon.0) 641 : cluster [INF] daemon mds.foo.smithi008.zfejpl is now active in filesystem foo as rank 0 2023-10-09T05:21:26.485 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":19,"active_gid":14442,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6800","nonce":2854879000},{"type":"v1","addr":"172.21.15.8:6801","nonce":2854879000}]},"active_addr":"172.21.15.8:6801/2854879000","active_change":"2023-10-09T05:20:01.878417+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.8:8443/","prometheus":"http://172.21.15.8:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.8:0","nonce":357500347}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.8:0","nonce":1669399762}]}]} 2023-10-09T05:21:26.491 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T05:21:26.491 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:21:26.491 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:21:26.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[101053]: audit 2023-10-09T05:21:25.791315+0000 mon.a 2023-10-09T05:21:26.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[101053]: (mon.0) 642 : audit [DBG] from='client.? 172.21.15.8:0/4032260769' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:21:26.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[101053]: cluster 2023-10-09T05:21:26.249991+0000 mon.a (mon.0) 643 : cluster [DBG] mds.? [v2:172.21.15.8:6834/560051770,v1:172.21.15.8:6835/560051770] up:active 2023-10-09T05:21:26.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[101053]: cluster 2023-10-09T05:21:26.250044+0000 mon.a (mon.0) 644 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 2023-10-09T05:21:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[110782]: audit 2023-10-09T05:21:25 2023-10-09T05:21:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[110782]: .791315+0000 mon.a (mon.0) 642 : audit [DBG] from='client.? 172.21.15.8:0/4032260769' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:21:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[110782]: cluster 2023-10-09T05:21:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[110782]: 2023-10-09T05:21:26.249991 2023-10-09T05:21:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[110782]: +0000 mon.a (mon.0) 643 : 2023-10-09T05:21:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[110782]: cluster [DBG] mds.? [v2:172.21.15.8:6834/560051770,v1:172.21.15.8:6835/560051770] up:active 2023-10-09T05:21:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:21:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[110782]: 10-09T05:21:26.250044+0000 mon.a (mon 2023-10-09T05:21:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[110782]: .0) 644 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 2023-10-09T05:21:26.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:26 smithi008 conmon[110782]: 2023-10-09T05:21:26.745 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:26 smithi191 conmon[101299]: audit 2023-10-09T05:21:25. 2023-10-09T05:21:26.745 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:26 smithi191 conmon[101299]: 791315+0000 mon.a (mon.0) 642 : audit [DBG] from='client.? 172.21.15.8:0/4032260769' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:21:26.745 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:26 smithi191 conmon[101299]: cluster 2023-10-09T05:21:26.249991+0000 mon.a (mon.0) 643 : cluster [DBG] mds.? [v2:172.21.15.8:6834/560051770,v1:172.21.15.8:6835/560051770] up:active 2023-10-09T05:21:26.745 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:26 smithi191 conmon[101299]: cluster 2023-10-09T05:21:26.250044+0000 mon.a (mon.0) 644 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 2023-10-09T05:21:26.859 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:27.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[101053]: cluster 2023-10-09T05:21:25.906059+0000 mgr.y (mgr.14442) 75 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:27.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[101053]: audit 2023-10-09T05:21:26.922035+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:27.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[101053]: audit 2023-10-09T05:21:26.923678+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:21:27.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[101053]: cluster 2023-10-09T05:21:27.187532+0000 mon.a (mon.0) 646 : cluster [DBG] mds.? [v2:172.21.15.191:6832/1976404494,v1:172.21.15.191:6833/1976404494] up:boot 2023-10-09T05:21:27.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[101053]: audit 2023-10-09T05:21:27.187600+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.walafj"}]: dispatch 2023-10-09T05:21:27.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[101053]: cluster 2023-10-09T05:21:27.187602+0000 mon.a ( 2023-10-09T05:21:27.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[101053]: mon.0) 647 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:21:27.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[101053]: audit 2023-10-09T05:21:27.210628+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:27.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[110782]: cluster 2023-10-09T05:21:25.906059+0000 mgr.y (mgr.14442) 75 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:27.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[110782]: audit 2023-10-09T05:21:26.922035+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:27.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[110782]: audit 2023-10-09T05:21:26.923678+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:21:27.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[110782]: cluster 2023-10-09T05:21:27.187532+0000 mon.a (mon.0) 646 : cluster [DBG] mds.? [v2:172.21.15.191:6832/1976404494,v1:172.21.15.191:6833/1976404494] up:boot 2023-10-09T05:21:27.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[110782]: audit 2023-10-09T05:21:27.187600+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.walafj"}]: dispatch 2023-10-09T05:21:27.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[110782]: cluster 2023-10-09T05:21:27.187602+0000 mon.a (mon.0) 647 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:21:27.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[110782]: audit 2023-10-09T05:21:27.210628+0000 mon.a (mon.0) 648 : audit 2023-10-09T05:21:27.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:27 smithi008 conmon[110782]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:27 smithi191 conmon[101299]: cluster 2023-10-09T05:21:25.906059+0000 mgr.y (mgr.14442) 75 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:27 smithi191 conmon[101299]: audit 2023-10-09T05:21:26.922035+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:27 smithi191 conmon[101299]: audit 2023-10-09T05:21:26.923678+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:21:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:27 smithi191 conmon[101299]: cluster 2023-10-09T05:21:27.187532+0000 mon.a (mon.0) 646 : cluster [DBG] mds.? [v2:172.21.15.191:6832/1976404494,v1:172.21.15.191:6833/1976404494] up:boot 2023-10-09T05:21:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:27 smithi191 conmon[101299]: audit 2023-10-09T05:21:27.187600+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.walafj"}]: dispatch 2023-10-09T05:21:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:27 smithi191 conmon[101299]: cluster 2023-10-09T05:21:27.187602+0000 mon.a (mon.0) 647 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:21:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:27 smithi191 conmon[101299]: audit 2023-10-09T05:21:27.210628+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:28.405 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:21:28.405 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":59,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:14:56.160913+0000","modified":"2023-10-09T05:21:17.795799+0000","last_up_change":"2023-10-09T05:19:43.821876+0000","last_in_change":"2023-10-09T05:19:26.801646+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:17:53.386952+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:20:22.166676+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:21:15.712971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:21:16.208554+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6802","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6803","nonce":791166472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6804","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6805","nonce":791166472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6808","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6809","nonce":791166472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6806","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6807","nonce":791166472}]},"public_addr":"172.21.15.8:6803/791166472","cluster_addr":"172.21.15.8:6805/791166472","heartbeat_back_addr":"172.21.15.8:6809/791166472","heartbeat_front_addr":"172.21.15.8:6807/791166472","state":["exists","up"]},{"osd":1,"uuid":"55f3290e-b82d-4839-a28d-1123a20c3a4c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6810","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6811","nonce":809058426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6812","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6813","nonce":809058426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6816","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6817","nonce":809058426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6814","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6815","nonce":809058426}]},"public_addr":"172.21.15.8:6811/809058426","cluster_addr":"172.21.15.8:6813/809058426","heartbeat_back_addr":"172.21.15.8:6817/809058426","heartbeat_front_addr":"172.21.15.8:6815/809058426","state":["exists","up"]},{"osd":2,"uuid":"720b47df-d743-46cf-ad04-36e113c8fcb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6818","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6819","nonce":3035534415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6820","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6821","nonce":3035534415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6824","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6825","nonce":3035534415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6822","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6823","nonce":3035534415}]},"public_addr":"172.21.15.8:6819/3035534415","cluster_addr":"172.21.15.8:6821/3035534415","heartbeat_back_addr":"172.21.15.8:6825/3035534415","heartbeat_front_addr":"172.21.15.8:6823/3035534415","state":["exists","up"]},{"osd":3,"uuid":"861bf05d-69f8-4000-999e-ab708531edb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6826","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6827","nonce":626394183}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6828","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6829","nonce":626394183}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6832","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6833","nonce":626394183}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6830","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6831","nonce":626394183}]},"public_addr":"172.21.15.8:6827/626394183","cluster_addr":"172.21.15.8:6829/626394183","heartbeat_back_addr":"172.21.15.8:6833/626394183","heartbeat_front_addr":"172.21.15.8:6831/626394183","state":["exists","up"]},{"osd":4,"uuid":"b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6801","nonce":3473028939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6803","nonce":3473028939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6807","nonce":3473028939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6805","nonce":3473028939}]},"public_addr":"172.21.15.191:6801/3473028939","cluster_addr":"172.21.15.191:6803/3473028939","heartbeat_back_addr":"172.21.15.191:6807/3473028939","heartbeat_front_addr":"172.21.15.191:6805/3473028939","state":["exists","up"]},{"osd":5,"uuid":"01507478-481e-4646-a30d-542184240f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6809","nonce":1592222523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6811","nonce":1592222523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6815","nonce":1592222523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6813","nonce":1592222523}]},"public_addr":"172.21.15.191:6809/1592222523","cluster_addr":"172.21.15.191:6811/1592222523","heartbeat_back_addr":"172.21.15.191:6815/1592222523","heartbeat_front_addr":"172.21.15.191:6813/1592222523","state":["exists","up"]},{"osd":6,"uuid":"8bc75ec3-46ee-4d36-989b-4e65e330cfb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6817","nonce":4267099596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6819","nonce":4267099596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6823","nonce":4267099596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6821","nonce":4267099596}]},"public_addr":"172.21.15.191:6817/4267099596","cluster_addr":"172.21.15.191:6819/4267099596","heartbeat_back_addr":"172.21.15.191:6823/4267099596","heartbeat_front_addr":"172.21.15.191:6821/4267099596","state":["exists","up"]},{"osd":7,"uuid":"5ac2e382-7341-47dd-b6c9-0fbdba12829c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6825","nonce":1787611943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6827","nonce":1787611943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6831","nonce":1787611943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6829","nonce":1787611943}]},"public_addr":"172.21.15.191:6825/1787611943","cluster_addr":"172.21.15.191:6827/1787611943","heartbeat_back_addr":"172.21.15.191:6831/1787611943","heartbeat_front_addr":"172.21.15.191:6829/1787611943","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:06.542686+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:28.674405+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:51.044955+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:14.395353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:36.486099+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:58.485442+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:20.015429+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:42.014771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.8:0/2178079664":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6801/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/1936221066":"2023-10-10T05:20:01.878268+0000","172.21.15.8:0/1062877078":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2096086544":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2360193392":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/69531294":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6800/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6801/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:6801/2430836187":"2023-10-10T05:15:24.205194+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:21:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:28 smithi008 conmon[101053]: audit 2023-10-09T05:21:28.404969+0000 mon.a (mon.0) 649 : audit [DBG] from='client.? 172.21.15.8:0/1824865420' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:21:28.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:28 smithi008 conmon[110782]: audit 2023-10-09T05:21 2023-10-09T05:21:28.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:28 smithi008 conmon[110782]: :28.404969+0000 mon.a (mon.0) 649 : audit [DBG] from='client.? 172.21.15.8:0/1824865420' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:21:28.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:28 smithi191 conmon[101299]: audit 2023-10-09T05:21:28.404969+0000 mon.a (mon.0 2023-10-09T05:21:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:28 smithi191 conmon[101299]: ) 649 : audit [DBG] from='client.? 172.21.15.8:0/1824865420' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:21:29.046 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:21:29.046 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:21:29.473 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:29 smithi008 conmon[101053]: cluster 2023-10-09T05:21:27.906657+0000 mgr.y (mgr.14442) 76 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:29.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:29 smithi008 conmon[110782]: cluster 2023-10-09T05:21:27.906657+0000 mgr.y (mgr.14442) 76 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:29.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:29 smithi191 conmon[101299]: cluster 2023-10-09T05:21:27.906657+0000 mgr.y (mgr.14442 2023-10-09T05:21:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:29 smithi191 conmon[101299]: ) 76 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:30.959 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:30 smithi008 conmon[101053]: audit 2023-10-09T05:21:29.565557 2023-10-09T05:21:30.959 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:30 smithi008 conmon[101053]: +0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:30.959 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:30 smithi008 conmon[101053]: cluster 2023-10-09T05:21:29.907248+0000 mgr.y (mgr.14442) 77 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:30.959 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:30 smithi008 conmon[110782]: audit 2023-10-09T05:21:29. 2023-10-09T05:21:30.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:30 smithi008 conmon[110782]: 565557+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:30.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:30 smithi008 conmon[110782]: cluster 2023-10-09T05:21 2023-10-09T05:21:30.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:30 smithi008 conmon[110782]: :29.907248+0000 mgr.y (mgr.14442) 77 2023-10-09T05:21:30.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:30 smithi008 conmon[110782]: : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:31.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:30 smithi191 conmon[101299]: audit 2023-10-09T05:21:29.565557+0000 mon.a (mon.0) 650 : audit [INF] 2023-10-09T05:21:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:30 smithi191 conmon[101299]: from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:30 smithi191 conmon[101299]: cluster 2023-10-09T05:21:29.907248+0000 mgr.y (mgr.14442) 77 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:21:31.443 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":59,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:14:56.160913+0000","modified":"2023-10-09T05:21:17.795799+0000","last_up_change":"2023-10-09T05:19:43.821876+0000","last_in_change":"2023-10-09T05:19:26.801646+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:17:53.386952+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:20:22.166676+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:21:15.712971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:21:16.208554+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6802","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6803","nonce":791166472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6804","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6805","nonce":791166472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6808","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6809","nonce":791166472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6806","nonce":791166472},{"type":"v1","addr":"172.21.15.8:6807","nonce":791166472}]},"public_addr":"172.21.15.8:6803/791166472","cluster_addr":"172.21.15.8:6805/791166472","heartbeat_back_addr":"172.21.15.8:6809/791166472","heartbeat_front_addr":"172.21.15.8:6807/791166472","state":["exists","up"]},{"osd":1,"uuid":"55f3290e-b82d-4839-a28d-1123a20c3a4c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6810","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6811","nonce":809058426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6812","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6813","nonce":809058426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6816","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6817","nonce":809058426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6814","nonce":809058426},{"type":"v1","addr":"172.21.15.8:6815","nonce":809058426}]},"public_addr":"172.21.15.8:6811/809058426","cluster_addr":"172.21.15.8:6813/809058426","heartbeat_back_addr":"172.21.15.8:6817/809058426","heartbeat_front_addr":"172.21.15.8:6815/809058426","state":["exists","up"]},{"osd":2,"uuid":"720b47df-d743-46cf-ad04-36e113c8fcb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6818","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6819","nonce":3035534415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6820","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6821","nonce":3035534415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6824","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6825","nonce":3035534415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6822","nonce":3035534415},{"type":"v1","addr":"172.21.15.8:6823","nonce":3035534415}]},"public_addr":"172.21.15.8:6819/3035534415","cluster_addr":"172.21.15.8:6821/3035534415","heartbeat_back_addr":"172.21.15.8:6825/3035534415","heartbeat_front_addr":"172.21.15.8:6823/3035534415","state":["exists","up"]},{"osd":3,"uuid":"861bf05d-69f8-4000-999e-ab708531edb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6826","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6827","nonce":626394183}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6828","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6829","nonce":626394183}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6832","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6833","nonce":626394183}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6830","nonce":626394183},{"type":"v1","addr":"172.21.15.8:6831","nonce":626394183}]},"public_addr":"172.21.15.8:6827/626394183","cluster_addr":"172.21.15.8:6829/626394183","heartbeat_back_addr":"172.21.15.8:6833/626394183","heartbeat_front_addr":"172.21.15.8:6831/626394183","state":["exists","up"]},{"osd":4,"uuid":"b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6801","nonce":3473028939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6803","nonce":3473028939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6807","nonce":3473028939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6805","nonce":3473028939}]},"public_addr":"172.21.15.191:6801/3473028939","cluster_addr":"172.21.15.191:6803/3473028939","heartbeat_back_addr":"172.21.15.191:6807/3473028939","heartbeat_front_addr":"172.21.15.191:6805/3473028939","state":["exists","up"]},{"osd":5,"uuid":"01507478-481e-4646-a30d-542184240f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6809","nonce":1592222523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6811","nonce":1592222523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6815","nonce":1592222523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6813","nonce":1592222523}]},"public_addr":"172.21.15.191:6809/1592222523","cluster_addr":"172.21.15.191:6811/1592222523","heartbeat_back_addr":"172.21.15.191:6815/1592222523","heartbeat_front_addr":"172.21.15.191:6813/1592222523","state":["exists","up"]},{"osd":6,"uuid":"8bc75ec3-46ee-4d36-989b-4e65e330cfb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6817","nonce":4267099596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6819","nonce":4267099596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6823","nonce":4267099596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6821","nonce":4267099596}]},"public_addr":"172.21.15.191:6817/4267099596","cluster_addr":"172.21.15.191:6819/4267099596","heartbeat_back_addr":"172.21.15.191:6823/4267099596","heartbeat_front_addr":"172.21.15.191:6821/4267099596","state":["exists","up"]},{"osd":7,"uuid":"5ac2e382-7341-47dd-b6c9-0fbdba12829c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6825","nonce":1787611943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6827","nonce":1787611943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6831","nonce":1787611943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6829","nonce":1787611943}]},"public_addr":"172.21.15.191:6825/1787611943","cluster_addr":"172.21.15.191:6827/1787611943","heartbeat_back_addr":"172.21.15.191:6831/1787611943","heartbeat_front_addr":"172.21.15.191:6829/1787611943","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:06.542686+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:28.674405+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:51.044955+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:14.395353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:36.486099+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:58.485442+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:20.015429+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:42.014771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.8:0/2178079664":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6801/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/1936221066":"2023-10-10T05:20:01.878268+0000","172.21.15.8:0/1062877078":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2096086544":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/2360193392":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/69531294":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6800/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6801/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:6801/2430836187":"2023-10-10T05:15:24.205194+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:21:32.191 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:21:32.191 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:21:32.192 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:21:32.192 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:21:32.193 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:21:32.193 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:21:32.193 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:21:32.194 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:21:32.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[101053]: audit 2023-10-09T05:21:31.272032+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:32.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[101053]: audit 2023-10-09T05:21:31.274065+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:32.471 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[101053]: audit 2023-10-09T05:21:31.274940+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:32.471 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[101053]: audit 2023-10-09T05:21:31.279372+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:32.471 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[101053]: audit 2023-10-09T05:21:31.285291+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:21:32.471 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[101053]: audit 2023-10-09T05:21:31.442444+0000 mon.a (mon.0) 653 : audit [DBG] from='client.? 172.21.15.8:0/3314637577' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:21:32.471 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[101053]: audit 2023-10-09T05:21:32.088720+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:32.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[110782]: audit 2023-10-09T05:21:31.272032+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:32.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[110782]: audit 2023-10-09T05:21:31.274065+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:32.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[110782]: audit 2023-10-09T05:21:31.274940+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:32.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[110782]: audit 2023-10-09T05:21:31.279372+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:32.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[110782]: audit 2023-10-09T05:21:31.285291+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:21:32.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[110782]: audit 2023-10-09T05:21:31.442444+0000 mon.a (mon.0) 653 : audit [DBG] from='client.? 172.21.15.8:0/3314637577' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:21:32.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:32 smithi008 conmon[110782]: audit 2023-10-09T05:21:32.088720+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:32.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: audit 2023-10-09T05:21:31.272032+0000 mon.a (mon.0) 651 : 2023-10-09T05:21:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: audit 2023-10-09T05:21:31.274065+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:21:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: -09T05:21:31.274940+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: audit 2023-10-09T05:21:31.279372 2023-10-09T05:21:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: +0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:21:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: -09T05:21:31.285291+0000 mon.c (mon 2023-10-09T05:21:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: .2) 76 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:21:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:21:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: 09T05:21:31.442444+0000 mon.a (mon. 2023-10-09T05:21:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: 0) 653 : audit [DBG] from='client.? 172.21.15.8:0/3314637577' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:21:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:21:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: -09T05:21:32.088720+0000 mon.a ( 2023-10-09T05:21:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:32 smithi191 conmon[101299]: mon.0) 654 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:33.024 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:33.074 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:33.144 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:33.432 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:33.434 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:33.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:33 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:33] "GET /metrics HTTP/1.1" 200 191452 "" "Prometheus/2.33.4" 2023-10-09T05:21:33.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:33 smithi008 conmon[101053]: cluster 2023-10-09T05:21:31.907636+0000 mgr.y (mgr.14442 2023-10-09T05:21:33.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:33 smithi008 conmon[101053]: ) 78 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:33.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:33 smithi008 conmon[110782]: cluster 2023-10-09T05:21:31.907636 2023-10-09T05:21:33.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:33 smithi008 conmon[110782]: +0000 mgr.y (mgr.14442) 78 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:33.482 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:33.702 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:33.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:33 smithi191 conmon[101299]: cluster 2023-10-09T05:21:31.907636+0000 mgr.y (mgr.14442 2023-10-09T05:21:33.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:33 smithi191 conmon[101299]: ) 78 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:34.418 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:35.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:21:34 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:21:35.339 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:35 smithi008 conmon[101053]: cluster 2023-10-09T05:21:33.908389+0000 mgr.y (mgr.14442) 79 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:35.339 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:35 smithi008 conmon[101053]: audit 2023-10-09T05:21:34.079975+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:35.339 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:35 smithi008 conmon[101053]: audit 2023-10-09T05:21:34.081260+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:35.339 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:35 smithi008 conmon[101053]: audit 2023-10-09T05:21:34.081993+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:35.339 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:35 smithi008 conmon[101053]: audit 2023-10-09T05:21:34.086404+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:35.340 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:35 smithi008 conmon[110782]: cluster 2023-10-09T05:21:33.908389+0000 mgr.y (mgr.14442) 79 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:35.340 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:35 smithi008 conmon[110782]: audit 2023-10-09T05:21:34.079975+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:35.340 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:35 smithi008 conmon[110782]: audit 2023-10-09T05:21:34.081260+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:35.340 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:35 smithi008 conmon[110782]: audit 2023-10-09T05:21:34.081993+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:35.340 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:35 smithi008 conmon[110782]: audit 2023-10-09T05:21:34.086404+0000 mon.a (mon.0) 656 : audit [INF] 2023-10-09T05:21:35.341 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:35 smithi008 conmon[110782]: from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:35.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:35 smithi191 conmon[101299]: cluster 2023-10-09T05:21:33.908389+0000 mgr.y (mgr.14442) 79 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:35 smithi191 conmon[101299]: audit 2023-10-09T05:21:34.079975+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:35 smithi191 conmon[101299]: audit 2023-10-09T05:21:34.081260+0000 mon.c (mon 2023-10-09T05:21:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:35 smithi191 conmon[101299]: .2) 77 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:21:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:35 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:21:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:35 smithi191 conmon[101299]: T05:21:34.081993+0000 mon.c (mon.2) 78 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:21:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:35 smithi191 conmon[101299]: 2023-10-09T05:21:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:35 smithi191 conmon[101299]: audit 2023-10-09T05:21:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:35 smithi191 conmon[101299]: 2023-10-09T05:21:34.086404+0000 mon.a (mon.0 2023-10-09T05:21:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:35 smithi191 conmon[101299]: ) 656 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:21:37.311 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:37 smithi008 conmon[101053]: cluster 2023-10-09T05:21:35 2023-10-09T05:21:37.311 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:37 smithi008 conmon[101053]: .908727+0000 mgr.y (mgr.14442) 80 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:37.312 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:37 smithi008 conmon[110782]: cluster 2023-10-09T05:21:35.908727+0000 mgr.y ( 2023-10-09T05:21:37.312 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:37 smithi008 conmon[110782]: mgr.14442) 80 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:37.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:37 smithi191 conmon[101299]: cluster 2023-10-09T05:21:35.908727+0000 mgr.y (mgr.14442) 80 : cluster 2023-10-09T05:21:37.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:37 smithi191 conmon[101299]: [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:39.400 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:39 smithi008 conmon[101053]: cluster 2023-10-09T05:21:37.909484+0000 mgr.y (mgr.14442) 81 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:39.400 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:39 smithi008 conmon[110782]: cluster 2023-10-09T05:21:37.909484+0000 mgr.y (mgr.14442) 81 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:39 smithi191 conmon[101299]: cluster 2023-10-09T05:21:37.909484+0000 mgr.y (mgr.14442) 81 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:39 smithi191 conmon[101299]: 2023-10-09T05:21:40.633 INFO:teuthology.orchestra.run.smithi008.stdout:154618822691 2023-10-09T05:21:40.634 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:21:41.097 INFO:teuthology.orchestra.run.smithi008.stdout:77309411376 2023-10-09T05:21:41.097 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:21:41.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:41 smithi008 conmon[101053]: cluster 2023-10-09T05:21:39.909863+0000 mgr.y (mgr.14442) 82 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:41.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:41 smithi008 conmon[110782]: cluster 2023-10-09T05:21:39.909863+0000 mgr.y (mgr.14442) 82 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:41.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:41 smithi191 conmon[101299]: cluster 2023-10-09T05:21:39.909863+0000 mgr.y (mgr.14442) 82 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:41.933 INFO:teuthology.orchestra.run.smithi008.stdout:55834574900 2023-10-09T05:21:41.933 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:21:42.823 INFO:teuthology.orchestra.run.smithi008.stdout:107374182441 2023-10-09T05:21:42.823 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:21:43.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:43 smithi008 conmon[101053]: cluster 2023- 2023-10-09T05:21:43.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:43 smithi008 conmon[101053]: 10-09T05:21:41.910289+0000 mgr.y (mgr.14442) 83 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:43.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:43 smithi008 conmon[110782]: cluster 2023-10-09T 2023-10-09T05:21:43.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:43 smithi008 conmon[110782]: 05:21:41.910289+0000 mgr.y (mgr.14442) 83 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:43.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:43 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:43] "GET /metrics HTTP/1.1" 200 191452 "" "Prometheus/2.33.4" 2023-10-09T05:21:43.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:43 smithi191 conmon[101299]: cluster 2023-10-09T05:21:41.910289+0000 mgr.y (mgr.14442) 83 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:43 smithi191 conmon[101299]: 2023-10-09T05:21:44.072 INFO:teuthology.orchestra.run.smithi008.stdout:34359738426 2023-10-09T05:21:44.072 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:21:44.366 INFO:teuthology.orchestra.run.smithi008.stdout:128849018919 2023-10-09T05:21:44.367 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:21:44.431 INFO:teuthology.orchestra.run.smithi008.stdout:206158430236 2023-10-09T05:21:44.431 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:21:45.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:21:44 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:21:45.068 INFO:teuthology.orchestra.run.smithi008.stdout:180388626464 2023-10-09T05:21:45.069 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:21:45.096 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:45.104 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:45.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:45 smithi008 conmon[101053]: cluster 2023-10-09T05:21:43.911054 2023-10-09T05:21:45.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:45 smithi008 conmon[101053]: +0000 mgr.y (mgr.14442) 84 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:45.333 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:45 smithi008 conmon[110782]: cluster 2023-10-09T05:21:43.911054+0000 mgr.y (mgr.14442) 84 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:45.417 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:45.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:45 smithi191 conmon[101299]: cluster 2023-10-09T05:21:43.911054+0000 mgr.y (mgr.14442) 84 : cluster [DBG] 2023-10-09T05:21:45.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:45 smithi191 conmon[101299]: pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:21:46.679 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:46.855 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:47 smithi008 conmon[101053]: cluster 2023-10-09T05:21:45.911486+0000 mgr.y (mgr.14442) 85 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:47 smithi008 conmon[110782]: cluster 2023-10-09T05:21:45.911486+0000 mgr.y (mgr.14442) 85 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:47.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:47 smithi191 conmon[101299]: cluster 2023-10-09T05:21:45.911486+0000 mgr.y (mgr.14442) 85 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:47.811 INFO:teuthology.orchestra.run.smithi008.stdout:154618822692 2023-10-09T05:21:48.376 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:48.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:48 smithi008 conmon[101053]: audit 2023-10-09T05:21:47. 2023-10-09T05:21:48.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:48 smithi008 conmon[101053]: 811455+0000 mon.a (mon.0) 657 : audit [DBG] from='client.? 172.21.15.8:0/4011705807' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:21:48.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:48 smithi008 conmon[110782]: audit 2023-10-09T05:21:47.811455+0000 2023-10-09T05:21:48.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:48 smithi008 conmon[110782]: mon.a (mon.0) 657 : audit [DBG] from='client.? 172.21.15.8:0/4011705807' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:21:48.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:48 smithi191 conmon[101299]: audit 2023-10-09T05:21:47.811455+0000 mon.a (mon.0) 657 : audit [DBG] 2023-10-09T05:21:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:48 smithi191 conmon[101299]: from='client.? 172.21.15.8:0/4011705807' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:21:48.939 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822691 got 154618822692 for osd.5 2023-10-09T05:21:48.940 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:48.944 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:48.950 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:49.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:49 smithi008 conmon[101053]: cluster 2023-10-09T05:21:47.911928+0000 mgr.y (mgr.14442) 86 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:49.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:49 smithi008 conmon[110782]: cluster 2023-10-09T05:21:47.911928+0000 mgr.y (mgr.14442) 86 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:49 smithi191 conmon[101299]: cluster 2023-10-09T05:21:47.911928+0000 mgr.y (mgr.14442) 86 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:50.195 INFO:teuthology.orchestra.run.smithi008.stdout:55834574902 2023-10-09T05:21:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:50 smithi008 conmon[101053]: audit 2023-10-09T05:21:50.032102+0000 2023-10-09T05:21:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:50 smithi008 conmon[101053]: mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.8:0/3300276805' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:21:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:50 smithi008 conmon[110782]: audit 2023-10-09T05:21:50.032102 2023-10-09T05:21:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:50 smithi008 conmon[110782]: +0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.8:0/3300276805' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:21:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:50 smithi191 conmon[101299]: audit 2023-10-09T05:21:50.032102+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.8:0/3300276805' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:21:51.348 INFO:teuthology.orchestra.run.smithi008.stdout:77309411378 2023-10-09T05:21:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:51 smithi191 conmon[101299]: cluster 2023-10-09T05:21:49.912258+0000 mgr.y (mgr.14442) 87 : cluster 2023-10-09T05:21:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:51 smithi191 conmon[101299]: [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:51.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:51 smithi008 conmon[101053]: cluster 2023-10-09T05:21:49.912258+0000 mgr.y (mgr.14442) 87 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:51.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:51 smithi008 conmon[110782]: cluster 2023-10-09T05:21:49.912258+0000 mgr.y (mgr.14442) 87 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:52.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:52 smithi191 conmon[101299]: audit 2023-10-09T05:21:51.347823+0000 mon.a (mon.0) 658 : audit [DBG] from='client.? 172.21.15.8:0/3632084205' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:21:52.642 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:52 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:21:52.643 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:52 smithi008 conmon[101053]: 21:51.347823+0000 mon.a (mon.0) 658 : audit [DBG] from='client.? 172.21.15.8:0/3632084205' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:21:52.643 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:52 smithi008 conmon[110782]: audit 2023-10-09T05:21:51.347823+0000 mon.a (mon.0) 658 : 2023-10-09T05:21:52.643 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:52 smithi008 conmon[110782]: audit [DBG] from='client.? 172.21.15.8:0/3632084205' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:21:53.313 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:53] "GET /metrics HTTP/1.1" 200 191455 "" "Prometheus/2.33.4" 2023-10-09T05:21:53.313 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[101053]: cluster 2023-10-09T05:21:51.912777+0000 mgr.y (mgr. 2023-10-09T05:21:53.313 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[101053]: 14442) 88 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:53.313 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[101053]: audit 2023-10-09T05:21:52.736154+0000 mon.c (mon.2) 80 : 2023-10-09T05:21:53.314 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[101053]: audit [DBG] from='client.? 172.21.15.8:0/1806444885' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:21:53.314 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[101053]: audit 2023-10-09T05:21: 2023-10-09T05:21:53.314 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[101053]: 52.823619+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.8:0/3286294360' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:21:53.314 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[110782]: cluster 2023-10-09T05:21:51.912777 2023-10-09T05:21:53.314 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[110782]: +0000 mgr.y (mgr.14442) 88 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:53.314 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:21:53.314 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[110782]: 09T05:21:52.736154+0000 mon.c (mon.2) 80 : audit 2023-10-09T05:21:53.314 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[110782]: [DBG] from='client.? 172.21.15.8:0/1806444885' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:21:53.314 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[110782]: audit 2023-10-09T05:21: 2023-10-09T05:21:53.314 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[110782]: 52.823619+0000 mon.c (mon.2) 81 : 2023-10-09T05:21:53.315 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:53 smithi008 conmon[110782]: audit [DBG] from='client.? 172.21.15.8:0/3286294360' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:21:53.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:53 smithi191 conmon[101299]: cluster 2023-10-09T05:21:51.912777+0000 mgr.y ( 2023-10-09T05:21:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:53 smithi191 conmon[101299]: mgr.14442) 88 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:53 smithi191 conmon[101299]: audit 2023-10-09T05:21:52.736154+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.8:0/1806444885' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:21:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:53 smithi191 conmon[101299]: audit 2023-10-09T05:21:52.823619+0000 mon.c (mon.2) 81 : audit 2023-10-09T05:21:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:53 smithi191 conmon[101299]: [DBG] from='client.? 172.21.15.8:0/3286294360' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:21:53.728 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574900 got 55834574902 for osd.1 2023-10-09T05:21:53.729 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:54.019 INFO:teuthology.orchestra.run.smithi008.stdout:107374182444 2023-10-09T05:21:54.445 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411376 got 77309411378 for osd.2 2023-10-09T05:21:54.446 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:54.655 INFO:teuthology.orchestra.run.smithi008.stdout:34359738428 2023-10-09T05:21:54.767 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182441 got 107374182444 for osd.3 2023-10-09T05:21:54.768 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:55.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:21:54 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:21:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:21:55.422 INFO:teuthology.orchestra.run.smithi008.stdout:180388626466 2023-10-09T05:21:55.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:55 smithi191 conmon[101299]: cluster 2023-10-09T05:21:53.913298+0000 mgr.y (mgr.14442) 89 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:55 smithi008 conmon[101053]: cluster 2023-10-09T05:21:53. 2023-10-09T05:21:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:55 smithi008 conmon[101053]: 913298+0000 mgr.y (mgr.14442) 89 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:55 smithi008 conmon[110782]: cluster 2023-10-09T05:21 2023-10-09T05:21:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:55 smithi008 conmon[110782]: :53.913298+0000 mgr.y (mgr.14442) 89 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:55.735 INFO:teuthology.orchestra.run.smithi008.stdout:128849018922 2023-10-09T05:21:55.969 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738426 got 34359738428 for osd.0 2023-10-09T05:21:55.970 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:56.312 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:56 smithi008 conmon[101053]: audit 2023-10-09T05:21:55.421765+0000 mon.a 2023-10-09T05:21:56.312 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:56 smithi008 conmon[101053]: (mon.0) 659 : audit [DBG] from='client.? 172.21.15.8:0/393780444' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:21:56.313 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:56 smithi008 conmon[101053]: audit 2023-10-09T05:21:55.736442+0000 mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.8:0/3212687822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:21:56.313 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:56 smithi008 conmon[110782]: audit 2023-10-09T05:21:55.421765+0000 mon.a ( 2023-10-09T05:21:56.313 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:56 smithi008 conmon[110782]: mon.0) 659 : audit [DBG] from='client.? 172.21.15.8:0/393780444' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:21:56.313 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:56 smithi008 conmon[110782]: audit 2023-10-09T05:21 2023-10-09T05:21:56.313 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:56 smithi008 conmon[110782]: :55.736442+0000 2023-10-09T05:21:56.313 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:56 smithi008 conmon[110782]: mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.8:0/3212687822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:21:56.530 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626464 got 180388626466 for osd.6 2023-10-09T05:21:56.530 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:56.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:56 smithi191 conmon[101299]: audit 2023-10-09T05:21:55.421765+0000 mon.a (mon.0) 659 : audit [DBG] from='client.? 172.21.15.8:0/393780444' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:21:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:56 smithi191 conmon[101299]: audit 2023-10-09T05:21:55.736442+0000 mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.8:0/3212687822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:21:56.909 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018919 got 128849018922 for osd.4 2023-10-09T05:21:56.909 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:57.255 INFO:teuthology.orchestra.run.smithi008.stdout:206158430238 2023-10-09T05:21:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:57 smithi191 conmon[101299]: cluster 2023-10-09T05:21:55.913765+0000 mgr.y (mgr.14442) 90 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:57.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:57 smithi008 conmon[101053]: cluster 2023-10-09T05:21:55.913765+0000 mgr.y (mgr.14442) 90 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:57.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:57 smithi008 conmon[110782]: cluster 2023-10-09T05:21:55.913765+0000 mgr.y (mgr.14442) 90 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:57.886 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430236 got 206158430238 for osd.7 2023-10-09T05:21:57.886 DEBUG:teuthology.parallel:result is None 2023-10-09T05:21:57.886 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:21:57.887 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:21:58.253 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:21:58.495 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:58 smithi008 conmon[101053]: audit 2023-10-09T05:21:57.256309+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.8:0/3968407104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:21:58.495 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:58 smithi008 conmon[110782]: audit 2023-10-09T05:21:57.256309 2023-10-09T05:21:58.495 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:58 smithi008 conmon[110782]: +0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.8:0/3968407104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:21:58.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:58 smithi191 conmon[101299]: audit 2023-10-09T05:21:57.256309+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.8:0/3968407104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:21:59.506 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:59 smithi008 conmon[101053]: cluster 2023-10-09T05:21 2023-10-09T05:21:59.506 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:21:59 smithi008 conmon[101053]: :57.914274+0000 mgr.y (mgr.14442) 91 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:59.506 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:59 smithi008 conmon[110782]: cluster 2023-10-09T05:21:57.914274+0000 mgr.y (mgr 2023-10-09T05:21:59.506 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:21:59 smithi008 conmon[110782]: .14442) 91 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:59 smithi191 conmon[101299]: cluster 2023-10-09T05:21:57.914274+0000 mgr.y (mgr.14442) 91 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:21:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:21:59 smithi191 conmon[101299]: 2023-10-09T05:21:59.916 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:21:59.917 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:22:00.545 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":69,"stamp":"2023-10-09T05:21:57.913927+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381460,"kb_used_data":4692,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579756,"statfs":{"total":767960285184,"available":765521670144,"internally_reserved":0,"allocated":4804608,"data_stored":2001039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002599"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.813369+0000","last_change":"2023-10-09T05:21:16.812916+0000","last_active":"2023-10-09T05:21:16.813369+0000","last_peered":"2023-10-09T05:21:16.813369+0000","last_clean":"2023-10-09T05:21:16.813369+0000","last_became_active":"2023-10-09T05:21:16.812631+0000","last_became_peered":"2023-10-09T05:21:16.812631+0000","last_unstale":"2023-10-09T05:21:16.813369+0000","last_undegraded":"2023-10-09T05:21:16.813369+0000","last_fullsized":"2023-10-09T05:21:16.813369+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.256702+0000","last_change":"2023-10-09T05:21:16.789949+0000","last_active":"2023-10-09T05:21:25.256702+0000","last_peered":"2023-10-09T05:21:25.256702+0000","last_clean":"2023-10-09T05:21:25.256702+0000","last_became_active":"2023-10-09T05:21:16.789627+0000","last_became_peered":"2023-10-09T05:21:16.789627+0000","last_unstale":"2023-10-09T05:21:25.256702+0000","last_undegraded":"2023-10-09T05:21:25.256702+0000","last_fullsized":"2023-10-09T05:21:25.256702+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.811047+0000","last_change":"2023-10-09T05:21:16.806637+0000","last_active":"2023-10-09T05:21:16.811047+0000","last_peered":"2023-10-09T05:21:16.811047+0000","last_clean":"2023-10-09T05:21:16.811047+0000","last_became_active":"2023-10-09T05:21:16.806265+0000","last_became_peered":"2023-10-09T05:21:16.806265+0000","last_unstale":"2023-10-09T05:21:16.811047+0000","last_undegraded":"2023-10-09T05:21:16.811047+0000","last_fullsized":"2023-10-09T05:21:16.811047+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.799419+0000","last_change":"2023-10-09T05:21:16.788923+0000","last_active":"2023-10-09T05:21:17.799419+0000","last_peered":"2023-10-09T05:21:17.799419+0000","last_clean":"2023-10-09T05:21:17.799419+0000","last_became_active":"2023-10-09T05:21:16.788574+0000","last_became_peered":"2023-10-09T05:21:16.788574+0000","last_unstale":"2023-10-09T05:21:17.799419+0000","last_undegraded":"2023-10-09T05:21:17.799419+0000","last_fullsized":"2023-10-09T05:21:17.799419+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.806582+0000","last_change":"2023-10-09T05:21:16.806322+0000","last_active":"2023-10-09T05:21:16.806582+0000","last_peered":"2023-10-09T05:21:16.806582+0000","last_clean":"2023-10-09T05:21:16.806582+0000","last_became_active":"2023-10-09T05:21:16.806079+0000","last_became_peered":"2023-10-09T05:21:16.806079+0000","last_unstale":"2023-10-09T05:21:16.806582+0000","last_undegraded":"2023-10-09T05:21:16.806582+0000","last_fullsized":"2023-10-09T05:21:16.806582+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.257373+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:21:25.257373+0000","last_peered":"2023-10-09T05:21:25.257373+0000","last_clean":"2023-10-09T05:21:25.257373+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:21:25.257373+0000","last_undegraded":"2023-10-09T05:21:25.257373+0000","last_fullsized":"2023-10-09T05:21:25.257373+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.813408+0000","last_change":"2023-10-09T05:21:16.812942+0000","last_active":"2023-10-09T05:21:16.813408+0000","last_peered":"2023-10-09T05:21:16.813408+0000","last_clean":"2023-10-09T05:21:16.813408+0000","last_became_active":"2023-10-09T05:21:16.812652+0000","last_became_peered":"2023-10-09T05:21:16.812652+0000","last_unstale":"2023-10-09T05:21:16.813408+0000","last_undegraded":"2023-10-09T05:21:16.813408+0000","last_fullsized":"2023-10-09T05:21:16.813408+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.257325+0000","last_change":"2023-10-09T05:21:16.787852+0000","last_active":"2023-10-09T05:21:25.257325+0000","last_peered":"2023-10-09T05:21:25.257325+0000","last_clean":"2023-10-09T05:21:25.257325+0000","last_became_active":"2023-10-09T05:21:16.787524+0000","last_became_peered":"2023-10-09T05:21:16.787524+0000","last_unstale":"2023-10-09T05:21:25.257325+0000","last_undegraded":"2023-10-09T05:21:25.257325+0000","last_fullsized":"2023-10-09T05:21:25.257325+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.815314+0000","last_change":"2023-10-09T05:21:16.814044+0000","last_active":"2023-10-09T05:21:16.815314+0000","last_peered":"2023-10-09T05:21:16.815314+0000","last_clean":"2023-10-09T05:21:16.815314+0000","last_became_active":"2023-10-09T05:21:16.813764+0000","last_became_peered":"2023-10-09T05:21:16.813764+0000","last_unstale":"2023-10-09T05:21:16.815314+0000","last_undegraded":"2023-10-09T05:21:16.815314+0000","last_fullsized":"2023-10-09T05:21:16.815314+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.838306+0000","last_change":"2023-10-09T05:21:16.786540+0000","last_active":"2023-10-09T05:21:17.838306+0000","last_peered":"2023-10-09T05:21:17.838306+0000","last_clean":"2023-10-09T05:21:17.838306+0000","last_became_active":"2023-10-09T05:21:16.786188+0000","last_became_peered":"2023-10-09T05:21:16.786188+0000","last_unstale":"2023-10-09T05:21:17.838306+0000","last_undegraded":"2023-10-09T05:21:17.838306+0000","last_fullsized":"2023-10-09T05:21:17.838306+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.804530+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:21:16.804530+0000","last_peered":"2023-10-09T05:21:16.804530+0000","last_clean":"2023-10-09T05:21:16.804530+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:21:16.804530+0000","last_undegraded":"2023-10-09T05:21:16.804530+0000","last_fullsized":"2023-10-09T05:21:16.804530+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.838480+0000","last_change":"2023-10-09T05:21:16.784232+0000","last_active":"2023-10-09T05:21:17.838480+0000","last_peered":"2023-10-09T05:21:17.838480+0000","last_clean":"2023-10-09T05:21:17.838480+0000","last_became_active":"2023-10-09T05:21:16.783878+0000","last_became_peered":"2023-10-09T05:21:16.783878+0000","last_unstale":"2023-10-09T05:21:17.838480+0000","last_undegraded":"2023-10-09T05:21:17.838480+0000","last_fullsized":"2023-10-09T05:21:17.838480+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.803803+0000","last_change":"2023-10-09T05:20:23.226324+0000","last_active":"2023-10-09T05:21:16.803803+0000","last_peered":"2023-10-09T05:21:16.803803+0000","last_clean":"2023-10-09T05:21:16.803803+0000","last_became_active":"2023-10-09T05:20:23.225950+0000","last_became_peered":"2023-10-09T05:20:23.225950+0000","last_unstale":"2023-10-09T05:21:16.803803+0000","last_undegraded":"2023-10-09T05:21:16.803803+0000","last_fullsized":"2023-10-09T05:21:16.803803+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.467741+0000","last_change":"2023-10-09T05:21:17.467374+0000","last_active":"2023-10-09T05:21:17.467741+0000","last_peered":"2023-10-09T05:21:17.467741+0000","last_clean":"2023-10-09T05:21:17.467741+0000","last_became_active":"2023-10-09T05:21:17.467039+0000","last_became_peered":"2023-10-09T05:21:17.467039+0000","last_unstale":"2023-10-09T05:21:17.467741+0000","last_undegraded":"2023-10-09T05:21:17.467741+0000","last_fullsized":"2023-10-09T05:21:17.467741+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.813515+0000","last_change":"2023-10-09T05:20:23.589992+0000","last_active":"2023-10-09T05:21:16.813515+0000","last_peered":"2023-10-09T05:21:16.813515+0000","last_clean":"2023-10-09T05:21:16.813515+0000","last_became_active":"2023-10-09T05:20:23.589676+0000","last_became_peered":"2023-10-09T05:20:23.589676+0000","last_unstale":"2023-10-09T05:21:16.813515+0000","last_undegraded":"2023-10-09T05:21:16.813515+0000","last_fullsized":"2023-10-09T05:21:16.813515+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.256118+0000","last_change":"2023-10-09T05:21:16.790617+0000","last_active":"2023-10-09T05:21:25.256118+0000","last_peered":"2023-10-09T05:21:25.256118+0000","last_clean":"2023-10-09T05:21:25.256118+0000","last_became_active":"2023-10-09T05:21:16.789963+0000","last_became_peered":"2023-10-09T05:21:16.789963+0000","last_unstale":"2023-10-09T05:21:25.256118+0000","last_undegraded":"2023-10-09T05:21:25.256118+0000","last_fullsized":"2023-10-09T05:21:25.256118+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.465497+0000","last_change":"2023-10-09T05:21:17.465241+0000","last_active":"2023-10-09T05:21:17.465497+0000","last_peered":"2023-10-09T05:21:17.465497+0000","last_clean":"2023-10-09T05:21:17.465497+0000","last_became_active":"2023-10-09T05:21:17.464876+0000","last_became_peered":"2023-10-09T05:21:17.464876+0000","last_unstale":"2023-10-09T05:21:17.465497+0000","last_undegraded":"2023-10-09T05:21:17.465497+0000","last_fullsized":"2023-10-09T05:21:17.465497+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.807624+0000","last_change":"2023-10-09T05:21:16.791273+0000","last_active":"2023-10-09T05:21:16.807624+0000","last_peered":"2023-10-09T05:21:16.807624+0000","last_clean":"2023-10-09T05:21:16.807624+0000","last_became_active":"2023-10-09T05:21:16.790683+0000","last_became_peered":"2023-10-09T05:21:16.790683+0000","last_unstale":"2023-10-09T05:21:16.807624+0000","last_undegraded":"2023-10-09T05:21:16.807624+0000","last_fullsized":"2023-10-09T05:21:16.807624+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.463723+0000","last_change":"2023-10-09T05:20:23.732093+0000","last_active":"2023-10-09T05:21:17.463723+0000","last_peered":"2023-10-09T05:21:17.463723+0000","last_clean":"2023-10-09T05:21:17.463723+0000","last_became_active":"2023-10-09T05:20:23.731684+0000","last_became_peered":"2023-10-09T05:20:23.731684+0000","last_unstale":"2023-10-09T05:21:17.463723+0000","last_undegraded":"2023-10-09T05:21:17.463723+0000","last_fullsized":"2023-10-09T05:21:17.463723+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.815961+0000","last_change":"2023-10-09T05:21:16.813087+0000","last_active":"2023-10-09T05:21:16.815961+0000","last_peered":"2023-10-09T05:21:16.815961+0000","last_clean":"2023-10-09T05:21:16.815961+0000","last_became_active":"2023-10-09T05:21:16.812633+0000","last_became_peered":"2023-10-09T05:21:16.812633+0000","last_unstale":"2023-10-09T05:21:16.815961+0000","last_undegraded":"2023-10-09T05:21:16.815961+0000","last_fullsized":"2023-10-09T05:21:16.815961+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.802155+0000","last_change":"2023-10-09T05:21:16.787709+0000","last_active":"2023-10-09T05:21:16.802155+0000","last_peered":"2023-10-09T05:21:16.802155+0000","last_clean":"2023-10-09T05:21:16.802155+0000","last_became_active":"2023-10-09T05:21:16.787359+0000","last_became_peered":"2023-10-09T05:21:16.787359+0000","last_unstale":"2023-10-09T05:21:16.802155+0000","last_undegraded":"2023-10-09T05:21:16.802155+0000","last_fullsized":"2023-10-09T05:21:16.802155+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.463532+0000","last_change":"2023-10-09T05:20:23.731911+0000","last_active":"2023-10-09T05:21:17.463532+0000","last_peered":"2023-10-09T05:21:17.463532+0000","last_clean":"2023-10-09T05:21:17.463532+0000","last_became_active":"2023-10-09T05:20:23.731530+0000","last_became_peered":"2023-10-09T05:20:23.731530+0000","last_unstale":"2023-10-09T05:21:17.463532+0000","last_undegraded":"2023-10-09T05:21:17.463532+0000","last_fullsized":"2023-10-09T05:21:17.463532+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.466449+0000","last_change":"2023-10-09T05:21:17.465038+0000","last_active":"2023-10-09T05:21:17.466449+0000","last_peered":"2023-10-09T05:21:17.466449+0000","last_clean":"2023-10-09T05:21:17.466449+0000","last_became_active":"2023-10-09T05:21:17.464695+0000","last_became_peered":"2023-10-09T05:21:17.464695+0000","last_unstale":"2023-10-09T05:21:17.466449+0000","last_undegraded":"2023-10-09T05:21:17.466449+0000","last_fullsized":"2023-10-09T05:21:17.466449+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.798895+0000","last_change":"2023-10-09T05:21:16.793476+0000","last_active":"2023-10-09T05:21:16.798895+0000","last_peered":"2023-10-09T05:21:16.798895+0000","last_clean":"2023-10-09T05:21:16.798895+0000","last_became_active":"2023-10-09T05:21:16.792461+0000","last_became_peered":"2023-10-09T05:21:16.792461+0000","last_unstale":"2023-10-09T05:21:16.798895+0000","last_undegraded":"2023-10-09T05:21:16.798895+0000","last_fullsized":"2023-10-09T05:21:16.798895+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.463545+0000","last_change":"2023-10-09T05:19:46.035865+0000","last_active":"2023-10-09T05:21:17.463545+0000","last_peered":"2023-10-09T05:21:17.463545+0000","last_clean":"2023-10-09T05:21:17.463545+0000","last_became_active":"2023-10-09T05:19:46.035493+0000","last_became_peered":"2023-10-09T05:19:46.035493+0000","last_unstale":"2023-10-09T05:21:17.463545+0000","last_undegraded":"2023-10-09T05:21:17.463545+0000","last_fullsized":"2023-10-09T05:21:17.463545+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.802935+0000","last_change":"2023-10-09T05:20:23.227498+0000","last_active":"2023-10-09T05:21:16.802935+0000","last_peered":"2023-10-09T05:21:16.802935+0000","last_clean":"2023-10-09T05:21:16.802935+0000","last_became_active":"2023-10-09T05:20:23.227147+0000","last_became_peered":"2023-10-09T05:20:23.227147+0000","last_unstale":"2023-10-09T05:21:16.802935+0000","last_undegraded":"2023-10-09T05:21:16.802935+0000","last_fullsized":"2023-10-09T05:21:16.802935+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.816333+0000","last_change":"2023-10-09T05:21:16.815975+0000","last_active":"2023-10-09T05:21:16.816333+0000","last_peered":"2023-10-09T05:21:16.816333+0000","last_clean":"2023-10-09T05:21:16.816333+0000","last_became_active":"2023-10-09T05:21:16.815543+0000","last_became_peered":"2023-10-09T05:21:16.815543+0000","last_unstale":"2023-10-09T05:21:16.816333+0000","last_undegraded":"2023-10-09T05:21:16.816333+0000","last_fullsized":"2023-10-09T05:21:16.816333+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.803140+0000","last_change":"2023-10-09T05:21:16.789949+0000","last_active":"2023-10-09T05:21:16.803140+0000","last_peered":"2023-10-09T05:21:16.803140+0000","last_clean":"2023-10-09T05:21:16.803140+0000","last_became_active":"2023-10-09T05:21:16.789627+0000","last_became_peered":"2023-10-09T05:21:16.789627+0000","last_unstale":"2023-10-09T05:21:16.803140+0000","last_undegraded":"2023-10-09T05:21:16.803140+0000","last_fullsized":"2023-10-09T05:21:16.803140+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.797873+0000","last_change":"2023-10-09T05:20:23.731595+0000","last_active":"2023-10-09T05:21:16.797873+0000","last_peered":"2023-10-09T05:21:16.797873+0000","last_clean":"2023-10-09T05:21:16.797873+0000","last_became_active":"2023-10-09T05:20:23.731240+0000","last_became_peered":"2023-10-09T05:20:23.731240+0000","last_unstale":"2023-10-09T05:21:16.797873+0000","last_undegraded":"2023-10-09T05:21:16.797873+0000","last_fullsized":"2023-10-09T05:21:16.797873+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.815322+0000","last_change":"2023-10-09T05:21:16.814836+0000","last_active":"2023-10-09T05:21:16.815322+0000","last_peered":"2023-10-09T05:21:16.815322+0000","last_clean":"2023-10-09T05:21:16.815322+0000","last_became_active":"2023-10-09T05:21:16.814444+0000","last_became_peered":"2023-10-09T05:21:16.814444+0000","last_unstale":"2023-10-09T05:21:16.815322+0000","last_undegraded":"2023-10-09T05:21:16.815322+0000","last_fullsized":"2023-10-09T05:21:16.815322+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.798704+0000","last_change":"2023-10-09T05:21:16.783355+0000","last_active":"2023-10-09T05:21:16.798704+0000","last_peered":"2023-10-09T05:21:16.798704+0000","last_clean":"2023-10-09T05:21:16.798704+0000","last_became_active":"2023-10-09T05:21:16.783076+0000","last_became_peered":"2023-10-09T05:21:16.783076+0000","last_unstale":"2023-10-09T05:21:16.798704+0000","last_undegraded":"2023-10-09T05:21:16.798704+0000","last_fullsized":"2023-10-09T05:21:16.798704+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.802845+0000","last_change":"2023-10-09T05:20:23.227732+0000","last_active":"2023-10-09T05:21:16.802845+0000","last_peered":"2023-10-09T05:21:16.802845+0000","last_clean":"2023-10-09T05:21:16.802845+0000","last_became_active":"2023-10-09T05:20:23.227348+0000","last_became_peered":"2023-10-09T05:20:23.227348+0000","last_unstale":"2023-10-09T05:21:16.802845+0000","last_undegraded":"2023-10-09T05:21:16.802845+0000","last_fullsized":"2023-10-09T05:21:16.802845+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.814978+0000","last_change":"2023-10-09T05:21:16.814020+0000","last_active":"2023-10-09T05:21:16.814978+0000","last_peered":"2023-10-09T05:21:16.814978+0000","last_clean":"2023-10-09T05:21:16.814978+0000","last_became_active":"2023-10-09T05:21:16.813570+0000","last_became_peered":"2023-10-09T05:21:16.813570+0000","last_unstale":"2023-10-09T05:21:16.814978+0000","last_undegraded":"2023-10-09T05:21:16.814978+0000","last_fullsized":"2023-10-09T05:21:16.814978+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.803404+0000","last_change":"2023-10-09T05:21:16.785050+0000","last_active":"2023-10-09T05:21:16.803404+0000","last_peered":"2023-10-09T05:21:16.803404+0000","last_clean":"2023-10-09T05:21:16.803404+0000","last_became_active":"2023-10-09T05:21:16.784746+0000","last_became_peered":"2023-10-09T05:21:16.784746+0000","last_unstale":"2023-10-09T05:21:16.803404+0000","last_undegraded":"2023-10-09T05:21:16.803404+0000","last_fullsized":"2023-10-09T05:21:16.803404+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.813883+0000","last_change":"2023-10-09T05:20:23.731592+0000","last_active":"2023-10-09T05:21:16.813883+0000","last_peered":"2023-10-09T05:21:16.813883+0000","last_clean":"2023-10-09T05:21:16.813883+0000","last_became_active":"2023-10-09T05:20:23.731263+0000","last_became_peered":"2023-10-09T05:20:23.731263+0000","last_unstale":"2023-10-09T05:21:16.813883+0000","last_undegraded":"2023-10-09T05:21:16.813883+0000","last_fullsized":"2023-10-09T05:21:16.813883+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.816182+0000","last_change":"2023-10-09T05:21:16.815915+0000","last_active":"2023-10-09T05:21:16.816182+0000","last_peered":"2023-10-09T05:21:16.816182+0000","last_clean":"2023-10-09T05:21:16.816182+0000","last_became_active":"2023-10-09T05:21:16.814425+0000","last_became_peered":"2023-10-09T05:21:16.814425+0000","last_unstale":"2023-10-09T05:21:16.816182+0000","last_undegraded":"2023-10-09T05:21:16.816182+0000","last_fullsized":"2023-10-09T05:21:16.816182+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.799357+0000","last_change":"2023-10-09T05:21:16.793014+0000","last_active":"2023-10-09T05:21:16.799357+0000","last_peered":"2023-10-09T05:21:16.799357+0000","last_clean":"2023-10-09T05:21:16.799357+0000","last_became_active":"2023-10-09T05:21:16.792440+0000","last_became_peered":"2023-10-09T05:21:16.792440+0000","last_unstale":"2023-10-09T05:21:16.799357+0000","last_undegraded":"2023-10-09T05:21:16.799357+0000","last_fullsized":"2023-10-09T05:21:16.799357+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.809606+0000","last_change":"2023-10-09T05:21:16.809188+0000","last_active":"2023-10-09T05:21:16.809606+0000","last_peered":"2023-10-09T05:21:16.809606+0000","last_clean":"2023-10-09T05:21:16.809606+0000","last_became_active":"2023-10-09T05:21:16.808804+0000","last_became_peered":"2023-10-09T05:21:16.808804+0000","last_unstale":"2023-10-09T05:21:16.809606+0000","last_undegraded":"2023-10-09T05:21:16.809606+0000","last_fullsized":"2023-10-09T05:21:16.809606+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.798921+0000","last_change":"2023-10-09T05:21:16.781454+0000","last_active":"2023-10-09T05:21:16.798921+0000","last_peered":"2023-10-09T05:21:16.798921+0000","last_clean":"2023-10-09T05:21:16.798921+0000","last_became_active":"2023-10-09T05:21:16.781163+0000","last_became_peered":"2023-10-09T05:21:16.781163+0000","last_unstale":"2023-10-09T05:21:16.798921+0000","last_undegraded":"2023-10-09T05:21:16.798921+0000","last_fullsized":"2023-10-09T05:21:16.798921+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.807615+0000","last_change":"2023-10-09T05:21:16.807405+0000","last_active":"2023-10-09T05:21:16.807615+0000","last_peered":"2023-10-09T05:21:16.807615+0000","last_clean":"2023-10-09T05:21:16.807615+0000","last_became_active":"2023-10-09T05:21:16.807120+0000","last_became_peered":"2023-10-09T05:21:16.807120+0000","last_unstale":"2023-10-09T05:21:16.807615+0000","last_undegraded":"2023-10-09T05:21:16.807615+0000","last_fullsized":"2023-10-09T05:21:16.807615+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.801599+0000","last_change":"2023-10-09T05:21:16.789029+0000","last_active":"2023-10-09T05:21:16.801599+0000","last_peered":"2023-10-09T05:21:16.801599+0000","last_clean":"2023-10-09T05:21:16.801599+0000","last_became_active":"2023-10-09T05:21:16.788674+0000","last_became_peered":"2023-10-09T05:21:16.788674+0000","last_unstale":"2023-10-09T05:21:16.801599+0000","last_undegraded":"2023-10-09T05:21:16.801599+0000","last_fullsized":"2023-10-09T05:21:16.801599+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.811702+0000","last_change":"2023-10-09T05:21:16.811469+0000","last_active":"2023-10-09T05:21:16.811702+0000","last_peered":"2023-10-09T05:21:16.811702+0000","last_clean":"2023-10-09T05:21:16.811702+0000","last_became_active":"2023-10-09T05:21:16.811167+0000","last_became_peered":"2023-10-09T05:21:16.811167+0000","last_unstale":"2023-10-09T05:21:16.811702+0000","last_undegraded":"2023-10-09T05:21:16.811702+0000","last_fullsized":"2023-10-09T05:21:16.811702+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.259074+0000","last_change":"2023-10-09T05:21:16.791185+0000","last_active":"2023-10-09T05:21:25.259074+0000","last_peered":"2023-10-09T05:21:25.259074+0000","last_clean":"2023-10-09T05:21:25.259074+0000","last_became_active":"2023-10-09T05:21:16.790365+0000","last_became_peered":"2023-10-09T05:21:16.790365+0000","last_unstale":"2023-10-09T05:21:25.259074+0000","last_undegraded":"2023-10-09T05:21:25.259074+0000","last_fullsized":"2023-10-09T05:21:25.259074+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.815021+0000","last_change":"2023-10-09T05:21:16.813006+0000","last_active":"2023-10-09T05:21:16.815021+0000","last_peered":"2023-10-09T05:21:16.815021+0000","last_clean":"2023-10-09T05:21:16.815021+0000","last_became_active":"2023-10-09T05:21:16.812658+0000","last_became_peered":"2023-10-09T05:21:16.812658+0000","last_unstale":"2023-10-09T05:21:16.815021+0000","last_undegraded":"2023-10-09T05:21:16.815021+0000","last_fullsized":"2023-10-09T05:21:16.815021+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.801635+0000","last_change":"2023-10-09T05:21:16.789068+0000","last_active":"2023-10-09T05:21:16.801635+0000","last_peered":"2023-10-09T05:21:16.801635+0000","last_clean":"2023-10-09T05:21:16.801635+0000","last_became_active":"2023-10-09T05:21:16.788751+0000","last_became_peered":"2023-10-09T05:21:16.788751+0000","last_unstale":"2023-10-09T05:21:16.801635+0000","last_undegraded":"2023-10-09T05:21:16.801635+0000","last_fullsized":"2023-10-09T05:21:16.801635+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.465630+0000","last_change":"2023-10-09T05:21:17.464207+0000","last_active":"2023-10-09T05:21:17.465630+0000","last_peered":"2023-10-09T05:21:17.465630+0000","last_clean":"2023-10-09T05:21:17.465630+0000","last_became_active":"2023-10-09T05:21:17.463892+0000","last_became_peered":"2023-10-09T05:21:17.463892+0000","last_unstale":"2023-10-09T05:21:17.465630+0000","last_undegraded":"2023-10-09T05:21:17.465630+0000","last_fullsized":"2023-10-09T05:21:17.465630+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.840209+0000","last_change":"2023-10-09T05:21:16.781147+0000","last_active":"2023-10-09T05:21:17.840209+0000","last_peered":"2023-10-09T05:21:17.840209+0000","last_clean":"2023-10-09T05:21:17.840209+0000","last_became_active":"2023-10-09T05:21:16.780755+0000","last_became_peered":"2023-10-09T05:21:16.780755+0000","last_unstale":"2023-10-09T05:21:17.840209+0000","last_undegraded":"2023-10-09T05:21:17.840209+0000","last_fullsized":"2023-10-09T05:21:17.840209+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.811827+0000","last_change":"2023-10-09T05:21:16.811562+0000","last_active":"2023-10-09T05:21:16.811827+0000","last_peered":"2023-10-09T05:21:16.811827+0000","last_clean":"2023-10-09T05:21:16.811827+0000","last_became_active":"2023-10-09T05:21:16.811269+0000","last_became_peered":"2023-10-09T05:21:16.811269+0000","last_unstale":"2023-10-09T05:21:16.811827+0000","last_undegraded":"2023-10-09T05:21:16.811827+0000","last_fullsized":"2023-10-09T05:21:16.811827+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.258538+0000","last_change":"2023-10-09T05:21:16.781889+0000","last_active":"2023-10-09T05:21:25.258538+0000","last_peered":"2023-10-09T05:21:25.258538+0000","last_clean":"2023-10-09T05:21:25.258538+0000","last_became_active":"2023-10-09T05:21:16.781645+0000","last_became_peered":"2023-10-09T05:21:16.781645+0000","last_unstale":"2023-10-09T05:21:25.258538+0000","last_undegraded":"2023-10-09T05:21:25.258538+0000","last_fullsized":"2023-10-09T05:21:25.258538+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.466018+0000","last_change":"2023-10-09T05:21:17.464398+0000","last_active":"2023-10-09T05:21:17.466018+0000","last_peered":"2023-10-09T05:21:17.466018+0000","last_clean":"2023-10-09T05:21:17.466018+0000","last_became_active":"2023-10-09T05:21:17.464086+0000","last_became_peered":"2023-10-09T05:21:17.464086+0000","last_unstale":"2023-10-09T05:21:17.466018+0000","last_undegraded":"2023-10-09T05:21:17.466018+0000","last_fullsized":"2023-10-09T05:21:17.466018+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.838784+0000","last_change":"2023-10-09T05:21:16.783643+0000","last_active":"2023-10-09T05:21:17.838784+0000","last_peered":"2023-10-09T05:21:17.838784+0000","last_clean":"2023-10-09T05:21:17.838784+0000","last_became_active":"2023-10-09T05:21:16.783225+0000","last_became_peered":"2023-10-09T05:21:16.783225+0000","last_unstale":"2023-10-09T05:21:17.838784+0000","last_undegraded":"2023-10-09T05:21:17.838784+0000","last_fullsized":"2023-10-09T05:21:17.838784+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.464962+0000","last_change":"2023-10-09T05:21:17.464241+0000","last_active":"2023-10-09T05:21:17.464962+0000","last_peered":"2023-10-09T05:21:17.464962+0000","last_clean":"2023-10-09T05:21:17.464962+0000","last_became_active":"2023-10-09T05:21:17.463854+0000","last_became_peered":"2023-10-09T05:21:17.463854+0000","last_unstale":"2023-10-09T05:21:17.464962+0000","last_undegraded":"2023-10-09T05:21:17.464962+0000","last_fullsized":"2023-10-09T05:21:17.464962+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.258732+0000","last_change":"2023-10-09T05:21:16.783235+0000","last_active":"2023-10-09T05:21:25.258732+0000","last_peered":"2023-10-09T05:21:25.258732+0000","last_clean":"2023-10-09T05:21:25.258732+0000","last_became_active":"2023-10-09T05:21:16.782671+0000","last_became_peered":"2023-10-09T05:21:16.782671+0000","last_unstale":"2023-10-09T05:21:25.258732+0000","last_undegraded":"2023-10-09T05:21:25.258732+0000","last_fullsized":"2023-10-09T05:21:25.258732+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.467215+0000","last_change":"2023-10-09T05:21:17.466925+0000","last_active":"2023-10-09T05:21:17.467215+0000","last_peered":"2023-10-09T05:21:17.467215+0000","last_clean":"2023-10-09T05:21:17.467215+0000","last_became_active":"2023-10-09T05:21:17.466588+0000","last_became_peered":"2023-10-09T05:21:17.466588+0000","last_unstale":"2023-10-09T05:21:17.467215+0000","last_undegraded":"2023-10-09T05:21:17.467215+0000","last_fullsized":"2023-10-09T05:21:17.467215+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.463681+0000","last_change":"2023-10-09T05:21:16.784983+0000","last_active":"2023-10-09T05:21:17.463681+0000","last_peered":"2023-10-09T05:21:17.463681+0000","last_clean":"2023-10-09T05:21:17.463681+0000","last_became_active":"2023-10-09T05:21:16.784699+0000","last_became_peered":"2023-10-09T05:21:16.784699+0000","last_unstale":"2023-10-09T05:21:17.463681+0000","last_undegraded":"2023-10-09T05:21:17.463681+0000","last_fullsized":"2023-10-09T05:21:17.463681+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.466350+0000","last_change":"2023-10-09T05:21:17.465091+0000","last_active":"2023-10-09T05:21:17.466350+0000","last_peered":"2023-10-09T05:21:17.466350+0000","last_clean":"2023-10-09T05:21:17.466350+0000","last_became_active":"2023-10-09T05:21:17.464814+0000","last_became_peered":"2023-10-09T05:21:17.464814+0000","last_unstale":"2023-10-09T05:21:17.466350+0000","last_undegraded":"2023-10-09T05:21:17.466350+0000","last_fullsized":"2023-10-09T05:21:17.466350+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.803910+0000","last_change":"2023-10-09T05:21:16.786350+0000","last_active":"2023-10-09T05:21:16.803910+0000","last_peered":"2023-10-09T05:21:16.803910+0000","last_clean":"2023-10-09T05:21:16.803910+0000","last_became_active":"2023-10-09T05:21:16.785976+0000","last_became_peered":"2023-10-09T05:21:16.785976+0000","last_unstale":"2023-10-09T05:21:16.803910+0000","last_undegraded":"2023-10-09T05:21:16.803910+0000","last_fullsized":"2023-10-09T05:21:16.803910+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.465266+0000","last_change":"2023-10-09T05:21:17.464752+0000","last_active":"2023-10-09T05:21:17.465266+0000","last_peered":"2023-10-09T05:21:17.465266+0000","last_clean":"2023-10-09T05:21:17.465266+0000","last_became_active":"2023-10-09T05:21:17.464439+0000","last_became_peered":"2023-10-09T05:21:17.464439+0000","last_unstale":"2023-10-09T05:21:17.465266+0000","last_undegraded":"2023-10-09T05:21:17.465266+0000","last_fullsized":"2023-10-09T05:21:17.465266+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.259103+0000","last_change":"2023-10-09T05:21:16.789819+0000","last_active":"2023-10-09T05:21:25.259103+0000","last_peered":"2023-10-09T05:21:25.259103+0000","last_clean":"2023-10-09T05:21:25.259103+0000","last_became_active":"2023-10-09T05:21:16.789507+0000","last_became_peered":"2023-10-09T05:21:16.789507+0000","last_unstale":"2023-10-09T05:21:25.259103+0000","last_undegraded":"2023-10-09T05:21:25.259103+0000","last_fullsized":"2023-10-09T05:21:25.259103+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.813428+0000","last_change":"2023-10-09T05:21:16.813026+0000","last_active":"2023-10-09T05:21:16.813428+0000","last_peered":"2023-10-09T05:21:16.813428+0000","last_clean":"2023-10-09T05:21:16.813428+0000","last_became_active":"2023-10-09T05:21:16.812743+0000","last_became_peered":"2023-10-09T05:21:16.812743+0000","last_unstale":"2023-10-09T05:21:16.813428+0000","last_undegraded":"2023-10-09T05:21:16.813428+0000","last_fullsized":"2023-10-09T05:21:16.813428+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"59'6","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.259176+0000","last_change":"2023-10-09T05:21:16.778811+0000","last_active":"2023-10-09T05:21:25.259176+0000","last_peered":"2023-10-09T05:21:25.259176+0000","last_clean":"2023-10-09T05:21:25.259176+0000","last_became_active":"2023-10-09T05:21:16.778515+0000","last_became_peered":"2023-10-09T05:21:16.778515+0000","last_unstale":"2023-10-09T05:21:25.259176+0000","last_undegraded":"2023-10-09T05:21:25.259176+0000","last_fullsized":"2023-10-09T05:21:25.259176+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.814690+0000","last_change":"2023-10-09T05:21:16.813124+0000","last_active":"2023-10-09T05:21:16.814690+0000","last_peered":"2023-10-09T05:21:16.814690+0000","last_clean":"2023-10-09T05:21:16.814690+0000","last_became_active":"2023-10-09T05:21:16.812491+0000","last_became_peered":"2023-10-09T05:21:16.812491+0000","last_unstale":"2023-10-09T05:21:16.814690+0000","last_undegraded":"2023-10-09T05:21:16.814690+0000","last_fullsized":"2023-10-09T05:21:16.814690+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":14,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.258628+0000","last_change":"2023-10-09T05:21:16.791216+0000","last_active":"2023-10-09T05:21:25.258628+0000","last_peered":"2023-10-09T05:21:25.258628+0000","last_clean":"2023-10-09T05:21:25.258628+0000","last_became_active":"2023-10-09T05:21:16.790923+0000","last_became_peered":"2023-10-09T05:21:16.790923+0000","last_unstale":"2023-10-09T05:21:25.258628+0000","last_undegraded":"2023-10-09T05:21:25.258628+0000","last_fullsized":"2023-10-09T05:21:25.258628+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.465297+0000","last_change":"2023-10-09T05:21:17.464837+0000","last_active":"2023-10-09T05:21:17.465297+0000","last_peered":"2023-10-09T05:21:17.465297+0000","last_clean":"2023-10-09T05:21:17.465297+0000","last_became_active":"2023-10-09T05:21:17.464513+0000","last_became_peered":"2023-10-09T05:21:17.464513+0000","last_unstale":"2023-10-09T05:21:17.465297+0000","last_undegraded":"2023-10-09T05:21:17.465297+0000","last_fullsized":"2023-10-09T05:21:17.465297+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.803253+0000","last_change":"2023-10-09T05:21:16.785017+0000","last_active":"2023-10-09T05:21:16.803253+0000","last_peered":"2023-10-09T05:21:16.803253+0000","last_clean":"2023-10-09T05:21:16.803253+0000","last_became_active":"2023-10-09T05:21:16.784698+0000","last_became_peered":"2023-10-09T05:21:16.784698+0000","last_unstale":"2023-10-09T05:21:16.803253+0000","last_undegraded":"2023-10-09T05:21:16.803253+0000","last_fullsized":"2023-10-09T05:21:16.803253+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.817272+0000","last_change":"2023-10-09T05:21:16.813333+0000","last_active":"2023-10-09T05:21:16.817272+0000","last_peered":"2023-10-09T05:21:16.817272+0000","last_clean":"2023-10-09T05:21:16.817272+0000","last_became_active":"2023-10-09T05:21:16.812960+0000","last_became_peered":"2023-10-09T05:21:16.812960+0000","last_unstale":"2023-10-09T05:21:16.817272+0000","last_undegraded":"2023-10-09T05:21:16.817272+0000","last_fullsized":"2023-10-09T05:21:16.817272+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.805390+0000","last_change":"2023-10-09T05:21:16.788367+0000","last_active":"2023-10-09T05:21:16.805390+0000","last_peered":"2023-10-09T05:21:16.805390+0000","last_clean":"2023-10-09T05:21:16.805390+0000","last_became_active":"2023-10-09T05:21:16.787793+0000","last_became_peered":"2023-10-09T05:21:16.787793+0000","last_unstale":"2023-10-09T05:21:16.805390+0000","last_undegraded":"2023-10-09T05:21:16.805390+0000","last_fullsized":"2023-10-09T05:21:16.805390+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.811930+0000","last_change":"2023-10-09T05:21:16.811697+0000","last_active":"2023-10-09T05:21:16.811930+0000","last_peered":"2023-10-09T05:21:16.811930+0000","last_clean":"2023-10-09T05:21:16.811930+0000","last_became_active":"2023-10-09T05:21:16.811357+0000","last_became_peered":"2023-10-09T05:21:16.811357+0000","last_unstale":"2023-10-09T05:21:16.811930+0000","last_undegraded":"2023-10-09T05:21:16.811930+0000","last_fullsized":"2023-10-09T05:21:16.811930+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.799583+0000","last_change":"2023-10-09T05:21:16.792789+0000","last_active":"2023-10-09T05:21:16.799583+0000","last_peered":"2023-10-09T05:21:16.799583+0000","last_clean":"2023-10-09T05:21:16.799583+0000","last_became_active":"2023-10-09T05:21:16.791974+0000","last_became_peered":"2023-10-09T05:21:16.791974+0000","last_unstale":"2023-10-09T05:21:16.799583+0000","last_undegraded":"2023-10-09T05:21:16.799583+0000","last_fullsized":"2023-10-09T05:21:16.799583+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.464529+0000","last_change":"2023-10-09T05:21:17.464218+0000","last_active":"2023-10-09T05:21:17.464529+0000","last_peered":"2023-10-09T05:21:17.464529+0000","last_clean":"2023-10-09T05:21:17.464529+0000","last_became_active":"2023-10-09T05:21:17.463847+0000","last_became_peered":"2023-10-09T05:21:17.463847+0000","last_unstale":"2023-10-09T05:21:17.464529+0000","last_undegraded":"2023-10-09T05:21:17.464529+0000","last_fullsized":"2023-10-09T05:21:17.464529+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.802492+0000","last_change":"2023-10-09T05:21:16.793454+0000","last_active":"2023-10-09T05:21:16.802492+0000","last_peered":"2023-10-09T05:21:16.802492+0000","last_clean":"2023-10-09T05:21:16.802492+0000","last_became_active":"2023-10-09T05:21:16.793136+0000","last_became_peered":"2023-10-09T05:21:16.793136+0000","last_unstale":"2023-10-09T05:21:16.802492+0000","last_undegraded":"2023-10-09T05:21:16.802492+0000","last_fullsized":"2023-10-09T05:21:16.802492+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.812141+0000","last_change":"2023-10-09T05:21:16.811697+0000","last_active":"2023-10-09T05:21:16.812141+0000","last_peered":"2023-10-09T05:21:16.812141+0000","last_clean":"2023-10-09T05:21:16.812141+0000","last_became_active":"2023-10-09T05:21:16.811360+0000","last_became_peered":"2023-10-09T05:21:16.811360+0000","last_unstale":"2023-10-09T05:21:16.812141+0000","last_undegraded":"2023-10-09T05:21:16.812141+0000","last_fullsized":"2023-10-09T05:21:16.812141+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.839170+0000","last_change":"2023-10-09T05:21:16.784096+0000","last_active":"2023-10-09T05:21:17.839170+0000","last_peered":"2023-10-09T05:21:17.839170+0000","last_clean":"2023-10-09T05:21:17.839170+0000","last_became_active":"2023-10-09T05:21:16.783768+0000","last_became_peered":"2023-10-09T05:21:16.783768+0000","last_unstale":"2023-10-09T05:21:17.839170+0000","last_undegraded":"2023-10-09T05:21:17.839170+0000","last_fullsized":"2023-10-09T05:21:17.839170+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430239,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297700,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447452,"statfs":{"total":95995035648,"available":95690190848,"internally_reserved":0,"allocated":618496,"data_stored":251129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.61499999999999999,"15min":0.63500000000000001},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.88400000000000001,"5min":2.8300000000000001,"15min":2.8300000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53800000000000003,"15min":0.54300000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.874,"5min":1.389,"15min":1.389},"last":0.46100000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.58899999999999997,"15min":0.59899999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.92800000000000005,"5min":2.2400000000000002,"15min":2.2400000000000002},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.57499999999999996,"15min":0.58499999999999996},"min":{"1min":0.36699999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.83199999999999996,"5min":2.4809999999999999,"15min":2.4809999999999999},"last":0.49399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.61099999999999999,"15min":0.625},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.91500000000000004,"5min":2.7029999999999998,"15min":2.7029999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.58299999999999996,"15min":0.59299999999999997},"min":{"1min":0.36399999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.83599999999999997,"5min":2.117,"15min":2.117},"last":0.51800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.63900000000000001,"15min":0.63500000000000001},"min":{"1min":0.40799999999999997,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.6560000000000001,"5min":2.6669999999999998,"15min":2.6669999999999998},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61599999999999999,"15min":0.622},"min":{"1min":0.441,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.90100000000000002,"5min":2.1379999999999999,"15min":2.1379999999999999},"last":0.54700000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.45000000000000001,"15min":0.44400000000000001},"min":{"1min":0.307,"5min":0.25,"15min":0.25},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.621},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.49099999999999999,"15min":0.48499999999999999},"min":{"1min":0.311,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.57099999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.49099999999999999,"15min":0.48599999999999999},"min":{"1min":0.32400000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.54500000000000004,"15min":0.53900000000000003},"min":{"1min":0.35199999999999998,"5min":0.314,"15min":0.314},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.64000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.52500000000000002,"15min":0.52000000000000002},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.57599999999999996,"15min":0.57399999999999995},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80400000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.65900000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626467,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":250718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.65800000000000003,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.626,"5min":0.55000000000000004,"15min":0.53700000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.9500000000000002,"5min":2.9500000000000002,"15min":2.9500000000000002},"last":0.56699999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51700000000000002,"15min":0.51800000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.64900000000000002,"5min":0.747,"15min":0.747},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54000000000000004,"15min":0.54300000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.55000000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53900000000000003,"15min":0.54300000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.63500000000000001,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.56699999999999995,"15min":0.56399999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.101,"5min":1.101,"15min":1.101},"last":0.51900000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.56499999999999995,"15min":0.54200000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":3.9569999999999999,"5min":3.9569999999999999,"15min":3.9569999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.59199999999999997,"15min":0.57599999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.9049999999999998,"5min":2.9049999999999998,"15min":2.9049999999999998},"last":0.52900000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.434,"15min":0.42699999999999999},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.441,"15min":0.42799999999999999},"min":{"1min":0.373,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.42799999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.46800000000000003,"15min":0.46000000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.47099999999999997,"15min":0.46300000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.57899999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:21:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.52500000000000002,"15min":0.51900000000000002},"min":{"1min":0.41099999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.53600000000000003,"15min":0.52700000000000002},"min":{"1min":0.38400000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.60699999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574904,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297672,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447480,"statfs":{"total":95995035648,"available":95690219520,"internally_reserved":0,"allocated":589824,"data_stored":249377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.41299999999999998,"15min":0.34799999999999998},"min":{"1min":0.23899999999999999,"5min":0.218,"15min":0.218},"max":{"1min":6.2880000000000003,"5min":6.2880000000000003,"15min":6.2880000000000003},"last":0.378},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.378,"15min":0.34200000000000003},"min":{"1min":0.247,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.7679999999999998,"5min":2.7679999999999998,"15min":2.7679999999999998},"last":0.26700000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:20:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.442,"15min":0.41499999999999998},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8460000000000001,"5min":2.8460000000000001,"15min":2.8460000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.44500000000000001,"15min":0.40899999999999997},"min":{"1min":0.25700000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.9290000000000003,"5min":5.9290000000000003,"15min":5.9290000000000003},"last":0.29199999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:21:29 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.54200000000000004,"15min":0.5},"min":{"1min":0.307,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.007,"5min":10.007,"15min":10.007},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.49399999999999999,"15min":0.46999999999999997},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.7000000000000002,"5min":6.7000000000000002,"15min":6.7000000000000002},"last":0.42299999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:21:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.60799999999999998,"15min":0.59599999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.9139999999999999,"5min":1.9139999999999999,"15min":1.9139999999999999},"last":1.03},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.64200000000000002,"15min":0.60999999999999999},"min":{"1min":0.41999999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80600000000000005,"5min":6.3330000000000002,"15min":6.3330000000000002},"last":0.56299999999999994}]},{"osd":5,"last update":"Mon Oct 9 05:21:02 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.61599999999999999,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.3639999999999999,"5min":6.3639999999999999,"15min":6.3639999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.59099999999999997,"15min":0.57399999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":2.8799999999999999,"5min":2.8799999999999999,"15min":2.8799999999999999},"last":0.497}]},{"osd":6,"last update":"Mon Oct 9 05:21:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59599999999999997,"15min":0.59699999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.66600000000000004,"15min":0.63500000000000001},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":6.4400000000000004,"5min":6.4400000000000004,"15min":6.4400000000000004},"last":0.61499999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.68100000000000005,"15min":0.69999999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.75600000000000001,"5min":2.8069999999999999,"15min":2.8069999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.64200000000000002,"15min":0.64700000000000002},"min":{"1min":0.49099999999999999,"5min":0.432,"15min":0.432},"max":{"1min":0.79700000000000004,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":1.0149999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738430,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297676,"kb_used_data":580,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447476,"statfs":{"total":95995035648,"available":95690215424,"internally_reserved":0,"allocated":593920,"data_stored":249638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:21:41 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.377,"15min":0.35099999999999998},"min":{"1min":0.26800000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.137,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.253},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.43099999999999999,"15min":0.377},"min":{"1min":0.253,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.2000000000000002,"5min":3.2000000000000002,"15min":3.2000000000000002},"last":0.373}]},{"osd":2,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.45300000000000001,"15min":0.42999999999999999},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.4359999999999999,"5min":2.4359999999999999,"15min":2.4359999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41499999999999998,"15min":0.41099999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.27000000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.47799999999999998,"15min":0.47499999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.5179999999999998,"5min":2.5179999999999998,"15min":2.5179999999999998},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.45100000000000001,"15min":0.45800000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65400000000000003,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:21:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.61199999999999999,"15min":0.60699999999999998},"min":{"1min":0.38100000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.677,"5min":2.677,"15min":2.677},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.625,"15min":0.61299999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":2.4359999999999999,"5min":2.4359999999999999,"15min":2.4359999999999999},"last":0.59599999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:21:08 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.58799999999999997,"15min":0.59599999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.623,"15min":0.60699999999999998},"min":{"1min":0.45600000000000002,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":2.552,"5min":2.552,"15min":2.552},"last":0.53400000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.61499999999999999,"15min":0.60899999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.637,"5min":2.637,"15min":2.637},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":0.498}]},{"osd":7,"last update":"Mon Oct 9 05:21:50 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.72699999999999998,"15min":0.71799999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":3.7610000000000001,"5min":3.7610000000000001,"15min":3.7610000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.60999999999999999,"15min":0.60299999999999998},"min":{"1min":0.437,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":0.51700000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411380,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297680,"kb_used_data":584,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447472,"statfs":{"total":95995035648,"available":95690211328,"internally_reserved":0,"allocated":598016,"data_stored":249965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.42799999999999999,"15min":0.40600000000000003},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.253},"max":{"1min":3.657,"5min":3.657,"15min":3.657},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.40500000000000003,"15min":0.40100000000000002},"min":{"1min":0.246,"5min":0.222,"15min":0.222},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.36599999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.44700000000000001,"15min":0.433},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.6859999999999999,"5min":1.6859999999999999,"15min":1.6859999999999999},"last":0.317},{"interface":"front","average":{"1min":0.495,"5min":0.46999999999999997,"15min":0.46000000000000002},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.39700000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:21:17 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44800000000000001,"15min":0.46000000000000002},"min":{"1min":0.27300000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.528,"5min":1.528,"15min":1.528},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.496,"15min":0.50600000000000001},"min":{"1min":0.29799999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":0.32600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:21:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.61899999999999999,"15min":0.629},"min":{"1min":0.41199999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0069999999999999,"5min":1.264,"15min":1.264},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.63100000000000001,"15min":0.63500000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.73299999999999998,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":0.57199999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.607,"5min":1.607,"15min":1.607},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.63,"5min":0.60999999999999999,"15min":0.60599999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":0.505}]},{"osd":6,"last update":"Mon Oct 9 05:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.64200000000000002,"15min":0.64600000000000002},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.464,"5min":1.464,"15min":1.464},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.47899999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.66600000000000004,"15min":0.68400000000000005},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.78600000000000003,"5min":1.502,"15min":1.502},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.69199999999999995,"15min":0.70699999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.77500000000000002,"5min":1.661,"15min":1.661},"last":0.55100000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182446,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297676,"kb_used_data":580,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447476,"statfs":{"total":95995035648,"available":95690215424,"internally_reserved":0,"allocated":593920,"data_stored":249467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.38800000000000001,"15min":0.39700000000000002},"min":{"1min":0.26000000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.434,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.35099999999999998,"5min":0.41799999999999998,"15min":0.432},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.48999999999999999,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.28100000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.437,"15min":0.45000000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.53500000000000003,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.379,"5min":0.44,"15min":0.45400000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.54900000000000004,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.32900000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.47999999999999998,"15min":0.498},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.56000000000000005,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.47199999999999998,"15min":0.48599999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.63600000000000001,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.34899999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:21:42 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.60599999999999998,"15min":0.60899999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.441,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.72799999999999998,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.67900000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.627,"15min":0.63100000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79900000000000004,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77900000000000003,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.79700000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.68899999999999995,"15min":0.70299999999999996},"min":{"1min":0.41099999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81100000000000005,"5min":3.181,"15min":3.181},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61299999999999999,"15min":0.61399999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81999999999999995,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.85499999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:21:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.621,"15min":0.623},"min":{"1min":0.46800000000000003,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.73399999999999999,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.63500000000000001,"15min":0.63400000000000001},"min":{"1min":0.50600000000000001,"5min":0.437,"15min":0.437},"max":{"1min":1.335,"5min":1.335,"15min":1.335},"last":0.57599999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018923,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297696,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447456,"statfs":{"total":95995035648,"available":95690194944,"internally_reserved":0,"allocated":614400,"data_stored":251059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.56299999999999994,"15min":0.58199999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79000000000000004,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.52500000000000002,"15min":0.52700000000000002},"min":{"1min":0.36699999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.71799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.51900000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.54700000000000004,"15min":0.56100000000000005},"min":{"1min":0.28899999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.71599999999999997,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.54900000000000004,"15min":0.55900000000000005},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.75900000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.66100000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.53500000000000003,"15min":0.54100000000000004},"min":{"1min":0.36299999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69799999999999995,"5min":0.75,"15min":0.75},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.55700000000000005,"15min":0.56999999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72699999999999998,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.59899999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55700000000000005,"15min":0.56499999999999995},"min":{"1min":0.35899999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77500000000000002,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.55500000000000005,"15min":0.56999999999999995},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.68200000000000005,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.68600000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:21:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.41699999999999998,"15min":0.41199999999999998},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.502,"5min":0.48499999999999999,"15min":0.48199999999999998},"min":{"1min":0.35999999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.495}]},{"osd":6,"last update":"Mon Oct 9 05:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.48799999999999999,"15min":0.49299999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.80500000000000005,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.432},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.501,"15min":0.505},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.77700000000000002,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.45700000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.53900000000000003,"15min":0.54700000000000004},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.80300000000000005,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.53500000000000003,"15min":0.54300000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.74099999999999999,"5min":1.034,"15min":1.034},"last":0.57199999999999995}]}]},{"osd":5,"up_from":36,"seq":154618822695,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297676,"kb_used_data":580,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447476,"statfs":{"total":95995035648,"available":95690215424,"internally_reserved":0,"allocated":593920,"data_stored":249686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.55900000000000005,"15min":0.56799999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70699999999999996,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.53200000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.54800000000000004,"15min":0.55200000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.66100000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.371},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73799999999999999,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.45100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.53100000000000003,"15min":0.53800000000000003},"min":{"1min":0.39500000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.67700000000000005,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.59899999999999998,"15min":0.59699999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.708,"5min":1.708,"15min":1.708},"last":0.58699999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.57199999999999995,"15min":0.57799999999999996},"min":{"1min":0.39000000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.65500000000000003,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.623,"15min":0.629},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.79500000000000004,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.626}]},{"osd":4,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.40100000000000002,"15min":0.38700000000000001},"min":{"1min":0.27500000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.43099999999999999,"15min":0.41699999999999998},"min":{"1min":0.28999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.56100000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.433,"15min":0.42199999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.49199999999999999,"15min":0.48599999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.88100000000000001,"5min":1.109,"15min":1.109},"last":0.65000000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.56999999999999995,"15min":0.56799999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.90800000000000003,"5min":1.143,"15min":1.143},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.36399999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.84599999999999997,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.59699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:22:00.551 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:22:00.909 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:01.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:01 smithi008 conmon[101053]: audit 2023-10-09T05:21:59.905294+0000 mgr.y (mgr.14442) 92 : audit [DBG] from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:22:01.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:01 smithi008 conmon[101053]: cluster 2023-10-09T05:21:59.914560+0000 mgr.y (mgr.14442) 93 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:01 smithi008 conmon[110782]: audit 2023-10-09T05:21:59.905294+0000 mgr.y (mgr.14442) 92 : audit [DBG] from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:22:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:01 smithi008 conmon[110782]: cluster 2023-10-09T05:21:59.914560+0000 mgr.y (mgr.14442) 93 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:01 smithi191 conmon[101299]: audit 2023-10-09T05:21:59.905294+0000 mgr.y (mgr.14442) 92 : audit [DBG] from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:22:01.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:01 smithi191 conmon[101299]: cluster 2023-10-09T05:21:59.914560+0000 mgr.y (mgr.14442) 93 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[101053]: audit 2023-10-09T05:22:01.912437+0000 mon.c ( 2023-10-09T05:22:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[101053]: mon.2) 83 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:22:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[101053]: audit 2023-10-09T05:22:01.912735+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[101053]: audit 2023-10-09T05:22:01.912898+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:22:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[101053]: audit 2023-10-09T05:22:01.913034+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[101053]: audit 2023-10-09T05:22:01.913550+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[101053]: audit 2023-10-09T05:22:01.913878+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[110782]: audit 2023-10-09T05:22:01.912437+0000 mon.c (mon 2023-10-09T05:22:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[110782]: .2) 83 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:22:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[110782]: audit 2023-10-09T05:22:01.912735+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[110782]: audit 2023-10-09T05:22:01.912898+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:22:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[110782]: audit 2023-10-09T05:22:01.913034+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[110782]: audit 2023-10-09T05:22:01.913550+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:02 smithi008 conmon[110782]: audit 2023-10-09T05:22:01.913878+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:02 smithi191 conmon[101299]: audit 2023-10-09T05:22:01.912437+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:22:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:02 smithi191 conmon[101299]: 2023-10-09T05:22:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:02 smithi191 conmon[101299]: audit 2023-10-09T05:22:01.912735+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:02 smithi191 conmon[101299]: audit 2023-10-09T05:22:01.912898+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:22:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:02 smithi191 conmon[101299]: audit 2023-10-09T05:22:01.913034+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:02 smithi191 conmon[101299]: audit 2023-10-09T05:22:01.913550+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:02 smithi191 conmon[101299]: audit 2023-10-09T05:22:01.913878+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-10-09T05:22:02.623 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:22:02.626 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:22:03.279 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":72,"stamp":"2023-10-09T05:22:02.244903+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381460,"kb_used_data":4692,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579756,"statfs":{"total":767960285184,"available":765521670144,"internally_reserved":0,"allocated":4804608,"data_stored":2001039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.332385"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.813369+0000","last_change":"2023-10-09T05:21:16.812916+0000","last_active":"2023-10-09T05:21:16.813369+0000","last_peered":"2023-10-09T05:21:16.813369+0000","last_clean":"2023-10-09T05:21:16.813369+0000","last_became_active":"2023-10-09T05:21:16.812631+0000","last_became_peered":"2023-10-09T05:21:16.812631+0000","last_unstale":"2023-10-09T05:21:16.813369+0000","last_undegraded":"2023-10-09T05:21:16.813369+0000","last_fullsized":"2023-10-09T05:21:16.813369+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":15,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.256702+0000","last_change":"2023-10-09T05:21:16.789949+0000","last_active":"2023-10-09T05:21:25.256702+0000","last_peered":"2023-10-09T05:21:25.256702+0000","last_clean":"2023-10-09T05:21:25.256702+0000","last_became_active":"2023-10-09T05:21:16.789627+0000","last_became_peered":"2023-10-09T05:21:16.789627+0000","last_unstale":"2023-10-09T05:21:25.256702+0000","last_undegraded":"2023-10-09T05:21:25.256702+0000","last_fullsized":"2023-10-09T05:21:25.256702+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.811047+0000","last_change":"2023-10-09T05:21:16.806637+0000","last_active":"2023-10-09T05:21:16.811047+0000","last_peered":"2023-10-09T05:21:16.811047+0000","last_clean":"2023-10-09T05:21:16.811047+0000","last_became_active":"2023-10-09T05:21:16.806265+0000","last_became_peered":"2023-10-09T05:21:16.806265+0000","last_unstale":"2023-10-09T05:21:16.811047+0000","last_undegraded":"2023-10-09T05:21:16.811047+0000","last_fullsized":"2023-10-09T05:21:16.811047+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.799419+0000","last_change":"2023-10-09T05:21:16.788923+0000","last_active":"2023-10-09T05:21:17.799419+0000","last_peered":"2023-10-09T05:21:17.799419+0000","last_clean":"2023-10-09T05:21:17.799419+0000","last_became_active":"2023-10-09T05:21:16.788574+0000","last_became_peered":"2023-10-09T05:21:16.788574+0000","last_unstale":"2023-10-09T05:21:17.799419+0000","last_undegraded":"2023-10-09T05:21:17.799419+0000","last_fullsized":"2023-10-09T05:21:17.799419+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.806582+0000","last_change":"2023-10-09T05:21:16.806322+0000","last_active":"2023-10-09T05:21:16.806582+0000","last_peered":"2023-10-09T05:21:16.806582+0000","last_clean":"2023-10-09T05:21:16.806582+0000","last_became_active":"2023-10-09T05:21:16.806079+0000","last_became_peered":"2023-10-09T05:21:16.806079+0000","last_unstale":"2023-10-09T05:21:16.806582+0000","last_undegraded":"2023-10-09T05:21:16.806582+0000","last_fullsized":"2023-10-09T05:21:16.806582+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.257373+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:21:25.257373+0000","last_peered":"2023-10-09T05:21:25.257373+0000","last_clean":"2023-10-09T05:21:25.257373+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:21:25.257373+0000","last_undegraded":"2023-10-09T05:21:25.257373+0000","last_fullsized":"2023-10-09T05:21:25.257373+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.813408+0000","last_change":"2023-10-09T05:21:16.812942+0000","last_active":"2023-10-09T05:21:16.813408+0000","last_peered":"2023-10-09T05:21:16.813408+0000","last_clean":"2023-10-09T05:21:16.813408+0000","last_became_active":"2023-10-09T05:21:16.812652+0000","last_became_peered":"2023-10-09T05:21:16.812652+0000","last_unstale":"2023-10-09T05:21:16.813408+0000","last_undegraded":"2023-10-09T05:21:16.813408+0000","last_fullsized":"2023-10-09T05:21:16.813408+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.257325+0000","last_change":"2023-10-09T05:21:16.787852+0000","last_active":"2023-10-09T05:21:25.257325+0000","last_peered":"2023-10-09T05:21:25.257325+0000","last_clean":"2023-10-09T05:21:25.257325+0000","last_became_active":"2023-10-09T05:21:16.787524+0000","last_became_peered":"2023-10-09T05:21:16.787524+0000","last_unstale":"2023-10-09T05:21:25.257325+0000","last_undegraded":"2023-10-09T05:21:25.257325+0000","last_fullsized":"2023-10-09T05:21:25.257325+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.815314+0000","last_change":"2023-10-09T05:21:16.814044+0000","last_active":"2023-10-09T05:21:16.815314+0000","last_peered":"2023-10-09T05:21:16.815314+0000","last_clean":"2023-10-09T05:21:16.815314+0000","last_became_active":"2023-10-09T05:21:16.813764+0000","last_became_peered":"2023-10-09T05:21:16.813764+0000","last_unstale":"2023-10-09T05:21:16.815314+0000","last_undegraded":"2023-10-09T05:21:16.815314+0000","last_fullsized":"2023-10-09T05:21:16.815314+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.838306+0000","last_change":"2023-10-09T05:21:16.786540+0000","last_active":"2023-10-09T05:21:17.838306+0000","last_peered":"2023-10-09T05:21:17.838306+0000","last_clean":"2023-10-09T05:21:17.838306+0000","last_became_active":"2023-10-09T05:21:16.786188+0000","last_became_peered":"2023-10-09T05:21:16.786188+0000","last_unstale":"2023-10-09T05:21:17.838306+0000","last_undegraded":"2023-10-09T05:21:17.838306+0000","last_fullsized":"2023-10-09T05:21:17.838306+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.804530+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:21:16.804530+0000","last_peered":"2023-10-09T05:21:16.804530+0000","last_clean":"2023-10-09T05:21:16.804530+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:21:16.804530+0000","last_undegraded":"2023-10-09T05:21:16.804530+0000","last_fullsized":"2023-10-09T05:21:16.804530+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.838480+0000","last_change":"2023-10-09T05:21:16.784232+0000","last_active":"2023-10-09T05:21:17.838480+0000","last_peered":"2023-10-09T05:21:17.838480+0000","last_clean":"2023-10-09T05:21:17.838480+0000","last_became_active":"2023-10-09T05:21:16.783878+0000","last_became_peered":"2023-10-09T05:21:16.783878+0000","last_unstale":"2023-10-09T05:21:17.838480+0000","last_undegraded":"2023-10-09T05:21:17.838480+0000","last_fullsized":"2023-10-09T05:21:17.838480+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.803803+0000","last_change":"2023-10-09T05:20:23.226324+0000","last_active":"2023-10-09T05:21:16.803803+0000","last_peered":"2023-10-09T05:21:16.803803+0000","last_clean":"2023-10-09T05:21:16.803803+0000","last_became_active":"2023-10-09T05:20:23.225950+0000","last_became_peered":"2023-10-09T05:20:23.225950+0000","last_unstale":"2023-10-09T05:21:16.803803+0000","last_undegraded":"2023-10-09T05:21:16.803803+0000","last_fullsized":"2023-10-09T05:21:16.803803+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.467741+0000","last_change":"2023-10-09T05:21:17.467374+0000","last_active":"2023-10-09T05:21:17.467741+0000","last_peered":"2023-10-09T05:21:17.467741+0000","last_clean":"2023-10-09T05:21:17.467741+0000","last_became_active":"2023-10-09T05:21:17.467039+0000","last_became_peered":"2023-10-09T05:21:17.467039+0000","last_unstale":"2023-10-09T05:21:17.467741+0000","last_undegraded":"2023-10-09T05:21:17.467741+0000","last_fullsized":"2023-10-09T05:21:17.467741+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.813515+0000","last_change":"2023-10-09T05:20:23.589992+0000","last_active":"2023-10-09T05:21:16.813515+0000","last_peered":"2023-10-09T05:21:16.813515+0000","last_clean":"2023-10-09T05:21:16.813515+0000","last_became_active":"2023-10-09T05:20:23.589676+0000","last_became_peered":"2023-10-09T05:20:23.589676+0000","last_unstale":"2023-10-09T05:21:16.813515+0000","last_undegraded":"2023-10-09T05:21:16.813515+0000","last_fullsized":"2023-10-09T05:21:16.813515+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.256118+0000","last_change":"2023-10-09T05:21:16.790617+0000","last_active":"2023-10-09T05:21:25.256118+0000","last_peered":"2023-10-09T05:21:25.256118+0000","last_clean":"2023-10-09T05:21:25.256118+0000","last_became_active":"2023-10-09T05:21:16.789963+0000","last_became_peered":"2023-10-09T05:21:16.789963+0000","last_unstale":"2023-10-09T05:21:25.256118+0000","last_undegraded":"2023-10-09T05:21:25.256118+0000","last_fullsized":"2023-10-09T05:21:25.256118+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.465497+0000","last_change":"2023-10-09T05:21:17.465241+0000","last_active":"2023-10-09T05:21:17.465497+0000","last_peered":"2023-10-09T05:21:17.465497+0000","last_clean":"2023-10-09T05:21:17.465497+0000","last_became_active":"2023-10-09T05:21:17.464876+0000","last_became_peered":"2023-10-09T05:21:17.464876+0000","last_unstale":"2023-10-09T05:21:17.465497+0000","last_undegraded":"2023-10-09T05:21:17.465497+0000","last_fullsized":"2023-10-09T05:21:17.465497+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.807624+0000","last_change":"2023-10-09T05:21:16.791273+0000","last_active":"2023-10-09T05:21:16.807624+0000","last_peered":"2023-10-09T05:21:16.807624+0000","last_clean":"2023-10-09T05:21:16.807624+0000","last_became_active":"2023-10-09T05:21:16.790683+0000","last_became_peered":"2023-10-09T05:21:16.790683+0000","last_unstale":"2023-10-09T05:21:16.807624+0000","last_undegraded":"2023-10-09T05:21:16.807624+0000","last_fullsized":"2023-10-09T05:21:16.807624+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.463723+0000","last_change":"2023-10-09T05:20:23.732093+0000","last_active":"2023-10-09T05:21:17.463723+0000","last_peered":"2023-10-09T05:21:17.463723+0000","last_clean":"2023-10-09T05:21:17.463723+0000","last_became_active":"2023-10-09T05:20:23.731684+0000","last_became_peered":"2023-10-09T05:20:23.731684+0000","last_unstale":"2023-10-09T05:21:17.463723+0000","last_undegraded":"2023-10-09T05:21:17.463723+0000","last_fullsized":"2023-10-09T05:21:17.463723+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.815961+0000","last_change":"2023-10-09T05:21:16.813087+0000","last_active":"2023-10-09T05:21:16.815961+0000","last_peered":"2023-10-09T05:21:16.815961+0000","last_clean":"2023-10-09T05:21:16.815961+0000","last_became_active":"2023-10-09T05:21:16.812633+0000","last_became_peered":"2023-10-09T05:21:16.812633+0000","last_unstale":"2023-10-09T05:21:16.815961+0000","last_undegraded":"2023-10-09T05:21:16.815961+0000","last_fullsized":"2023-10-09T05:21:16.815961+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.802155+0000","last_change":"2023-10-09T05:21:16.787709+0000","last_active":"2023-10-09T05:21:16.802155+0000","last_peered":"2023-10-09T05:21:16.802155+0000","last_clean":"2023-10-09T05:21:16.802155+0000","last_became_active":"2023-10-09T05:21:16.787359+0000","last_became_peered":"2023-10-09T05:21:16.787359+0000","last_unstale":"2023-10-09T05:21:16.802155+0000","last_undegraded":"2023-10-09T05:21:16.802155+0000","last_fullsized":"2023-10-09T05:21:16.802155+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.463532+0000","last_change":"2023-10-09T05:20:23.731911+0000","last_active":"2023-10-09T05:21:17.463532+0000","last_peered":"2023-10-09T05:21:17.463532+0000","last_clean":"2023-10-09T05:21:17.463532+0000","last_became_active":"2023-10-09T05:20:23.731530+0000","last_became_peered":"2023-10-09T05:20:23.731530+0000","last_unstale":"2023-10-09T05:21:17.463532+0000","last_undegraded":"2023-10-09T05:21:17.463532+0000","last_fullsized":"2023-10-09T05:21:17.463532+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.466449+0000","last_change":"2023-10-09T05:21:17.465038+0000","last_active":"2023-10-09T05:21:17.466449+0000","last_peered":"2023-10-09T05:21:17.466449+0000","last_clean":"2023-10-09T05:21:17.466449+0000","last_became_active":"2023-10-09T05:21:17.464695+0000","last_became_peered":"2023-10-09T05:21:17.464695+0000","last_unstale":"2023-10-09T05:21:17.466449+0000","last_undegraded":"2023-10-09T05:21:17.466449+0000","last_fullsized":"2023-10-09T05:21:17.466449+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.798895+0000","last_change":"2023-10-09T05:21:16.793476+0000","last_active":"2023-10-09T05:21:16.798895+0000","last_peered":"2023-10-09T05:21:16.798895+0000","last_clean":"2023-10-09T05:21:16.798895+0000","last_became_active":"2023-10-09T05:21:16.792461+0000","last_became_peered":"2023-10-09T05:21:16.792461+0000","last_unstale":"2023-10-09T05:21:16.798895+0000","last_undegraded":"2023-10-09T05:21:16.798895+0000","last_fullsized":"2023-10-09T05:21:16.798895+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.463545+0000","last_change":"2023-10-09T05:19:46.035865+0000","last_active":"2023-10-09T05:21:17.463545+0000","last_peered":"2023-10-09T05:21:17.463545+0000","last_clean":"2023-10-09T05:21:17.463545+0000","last_became_active":"2023-10-09T05:19:46.035493+0000","last_became_peered":"2023-10-09T05:19:46.035493+0000","last_unstale":"2023-10-09T05:21:17.463545+0000","last_undegraded":"2023-10-09T05:21:17.463545+0000","last_fullsized":"2023-10-09T05:21:17.463545+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.802935+0000","last_change":"2023-10-09T05:20:23.227498+0000","last_active":"2023-10-09T05:21:16.802935+0000","last_peered":"2023-10-09T05:21:16.802935+0000","last_clean":"2023-10-09T05:21:16.802935+0000","last_became_active":"2023-10-09T05:20:23.227147+0000","last_became_peered":"2023-10-09T05:20:23.227147+0000","last_unstale":"2023-10-09T05:21:16.802935+0000","last_undegraded":"2023-10-09T05:21:16.802935+0000","last_fullsized":"2023-10-09T05:21:16.802935+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.816333+0000","last_change":"2023-10-09T05:21:16.815975+0000","last_active":"2023-10-09T05:21:16.816333+0000","last_peered":"2023-10-09T05:21:16.816333+0000","last_clean":"2023-10-09T05:21:16.816333+0000","last_became_active":"2023-10-09T05:21:16.815543+0000","last_became_peered":"2023-10-09T05:21:16.815543+0000","last_unstale":"2023-10-09T05:21:16.816333+0000","last_undegraded":"2023-10-09T05:21:16.816333+0000","last_fullsized":"2023-10-09T05:21:16.816333+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.803140+0000","last_change":"2023-10-09T05:21:16.789949+0000","last_active":"2023-10-09T05:21:16.803140+0000","last_peered":"2023-10-09T05:21:16.803140+0000","last_clean":"2023-10-09T05:21:16.803140+0000","last_became_active":"2023-10-09T05:21:16.789627+0000","last_became_peered":"2023-10-09T05:21:16.789627+0000","last_unstale":"2023-10-09T05:21:16.803140+0000","last_undegraded":"2023-10-09T05:21:16.803140+0000","last_fullsized":"2023-10-09T05:21:16.803140+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.797873+0000","last_change":"2023-10-09T05:20:23.731595+0000","last_active":"2023-10-09T05:21:16.797873+0000","last_peered":"2023-10-09T05:21:16.797873+0000","last_clean":"2023-10-09T05:21:16.797873+0000","last_became_active":"2023-10-09T05:20:23.731240+0000","last_became_peered":"2023-10-09T05:20:23.731240+0000","last_unstale":"2023-10-09T05:21:16.797873+0000","last_undegraded":"2023-10-09T05:21:16.797873+0000","last_fullsized":"2023-10-09T05:21:16.797873+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.815322+0000","last_change":"2023-10-09T05:21:16.814836+0000","last_active":"2023-10-09T05:21:16.815322+0000","last_peered":"2023-10-09T05:21:16.815322+0000","last_clean":"2023-10-09T05:21:16.815322+0000","last_became_active":"2023-10-09T05:21:16.814444+0000","last_became_peered":"2023-10-09T05:21:16.814444+0000","last_unstale":"2023-10-09T05:21:16.815322+0000","last_undegraded":"2023-10-09T05:21:16.815322+0000","last_fullsized":"2023-10-09T05:21:16.815322+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.798704+0000","last_change":"2023-10-09T05:21:16.783355+0000","last_active":"2023-10-09T05:21:16.798704+0000","last_peered":"2023-10-09T05:21:16.798704+0000","last_clean":"2023-10-09T05:21:16.798704+0000","last_became_active":"2023-10-09T05:21:16.783076+0000","last_became_peered":"2023-10-09T05:21:16.783076+0000","last_unstale":"2023-10-09T05:21:16.798704+0000","last_undegraded":"2023-10-09T05:21:16.798704+0000","last_fullsized":"2023-10-09T05:21:16.798704+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.802845+0000","last_change":"2023-10-09T05:20:23.227732+0000","last_active":"2023-10-09T05:21:16.802845+0000","last_peered":"2023-10-09T05:21:16.802845+0000","last_clean":"2023-10-09T05:21:16.802845+0000","last_became_active":"2023-10-09T05:20:23.227348+0000","last_became_peered":"2023-10-09T05:20:23.227348+0000","last_unstale":"2023-10-09T05:21:16.802845+0000","last_undegraded":"2023-10-09T05:21:16.802845+0000","last_fullsized":"2023-10-09T05:21:16.802845+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.814978+0000","last_change":"2023-10-09T05:21:16.814020+0000","last_active":"2023-10-09T05:21:16.814978+0000","last_peered":"2023-10-09T05:21:16.814978+0000","last_clean":"2023-10-09T05:21:16.814978+0000","last_became_active":"2023-10-09T05:21:16.813570+0000","last_became_peered":"2023-10-09T05:21:16.813570+0000","last_unstale":"2023-10-09T05:21:16.814978+0000","last_undegraded":"2023-10-09T05:21:16.814978+0000","last_fullsized":"2023-10-09T05:21:16.814978+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.803404+0000","last_change":"2023-10-09T05:21:16.785050+0000","last_active":"2023-10-09T05:21:16.803404+0000","last_peered":"2023-10-09T05:21:16.803404+0000","last_clean":"2023-10-09T05:21:16.803404+0000","last_became_active":"2023-10-09T05:21:16.784746+0000","last_became_peered":"2023-10-09T05:21:16.784746+0000","last_unstale":"2023-10-09T05:21:16.803404+0000","last_undegraded":"2023-10-09T05:21:16.803404+0000","last_fullsized":"2023-10-09T05:21:16.803404+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.813883+0000","last_change":"2023-10-09T05:20:23.731592+0000","last_active":"2023-10-09T05:21:16.813883+0000","last_peered":"2023-10-09T05:21:16.813883+0000","last_clean":"2023-10-09T05:21:16.813883+0000","last_became_active":"2023-10-09T05:20:23.731263+0000","last_became_peered":"2023-10-09T05:20:23.731263+0000","last_unstale":"2023-10-09T05:21:16.813883+0000","last_undegraded":"2023-10-09T05:21:16.813883+0000","last_fullsized":"2023-10-09T05:21:16.813883+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.816182+0000","last_change":"2023-10-09T05:21:16.815915+0000","last_active":"2023-10-09T05:21:16.816182+0000","last_peered":"2023-10-09T05:21:16.816182+0000","last_clean":"2023-10-09T05:21:16.816182+0000","last_became_active":"2023-10-09T05:21:16.814425+0000","last_became_peered":"2023-10-09T05:21:16.814425+0000","last_unstale":"2023-10-09T05:21:16.816182+0000","last_undegraded":"2023-10-09T05:21:16.816182+0000","last_fullsized":"2023-10-09T05:21:16.816182+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.799357+0000","last_change":"2023-10-09T05:21:16.793014+0000","last_active":"2023-10-09T05:21:16.799357+0000","last_peered":"2023-10-09T05:21:16.799357+0000","last_clean":"2023-10-09T05:21:16.799357+0000","last_became_active":"2023-10-09T05:21:16.792440+0000","last_became_peered":"2023-10-09T05:21:16.792440+0000","last_unstale":"2023-10-09T05:21:16.799357+0000","last_undegraded":"2023-10-09T05:21:16.799357+0000","last_fullsized":"2023-10-09T05:21:16.799357+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.809606+0000","last_change":"2023-10-09T05:21:16.809188+0000","last_active":"2023-10-09T05:21:16.809606+0000","last_peered":"2023-10-09T05:21:16.809606+0000","last_clean":"2023-10-09T05:21:16.809606+0000","last_became_active":"2023-10-09T05:21:16.808804+0000","last_became_peered":"2023-10-09T05:21:16.808804+0000","last_unstale":"2023-10-09T05:21:16.809606+0000","last_undegraded":"2023-10-09T05:21:16.809606+0000","last_fullsized":"2023-10-09T05:21:16.809606+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.798921+0000","last_change":"2023-10-09T05:21:16.781454+0000","last_active":"2023-10-09T05:21:16.798921+0000","last_peered":"2023-10-09T05:21:16.798921+0000","last_clean":"2023-10-09T05:21:16.798921+0000","last_became_active":"2023-10-09T05:21:16.781163+0000","last_became_peered":"2023-10-09T05:21:16.781163+0000","last_unstale":"2023-10-09T05:21:16.798921+0000","last_undegraded":"2023-10-09T05:21:16.798921+0000","last_fullsized":"2023-10-09T05:21:16.798921+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.807615+0000","last_change":"2023-10-09T05:21:16.807405+0000","last_active":"2023-10-09T05:21:16.807615+0000","last_peered":"2023-10-09T05:21:16.807615+0000","last_clean":"2023-10-09T05:21:16.807615+0000","last_became_active":"2023-10-09T05:21:16.807120+0000","last_became_peered":"2023-10-09T05:21:16.807120+0000","last_unstale":"2023-10-09T05:21:16.807615+0000","last_undegraded":"2023-10-09T05:21:16.807615+0000","last_fullsized":"2023-10-09T05:21:16.807615+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.801599+0000","last_change":"2023-10-09T05:21:16.789029+0000","last_active":"2023-10-09T05:21:16.801599+0000","last_peered":"2023-10-09T05:21:16.801599+0000","last_clean":"2023-10-09T05:21:16.801599+0000","last_became_active":"2023-10-09T05:21:16.788674+0000","last_became_peered":"2023-10-09T05:21:16.788674+0000","last_unstale":"2023-10-09T05:21:16.801599+0000","last_undegraded":"2023-10-09T05:21:16.801599+0000","last_fullsized":"2023-10-09T05:21:16.801599+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.811702+0000","last_change":"2023-10-09T05:21:16.811469+0000","last_active":"2023-10-09T05:21:16.811702+0000","last_peered":"2023-10-09T05:21:16.811702+0000","last_clean":"2023-10-09T05:21:16.811702+0000","last_became_active":"2023-10-09T05:21:16.811167+0000","last_became_peered":"2023-10-09T05:21:16.811167+0000","last_unstale":"2023-10-09T05:21:16.811702+0000","last_undegraded":"2023-10-09T05:21:16.811702+0000","last_fullsized":"2023-10-09T05:21:16.811702+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.259074+0000","last_change":"2023-10-09T05:21:16.791185+0000","last_active":"2023-10-09T05:21:25.259074+0000","last_peered":"2023-10-09T05:21:25.259074+0000","last_clean":"2023-10-09T05:21:25.259074+0000","last_became_active":"2023-10-09T05:21:16.790365+0000","last_became_peered":"2023-10-09T05:21:16.790365+0000","last_unstale":"2023-10-09T05:21:25.259074+0000","last_undegraded":"2023-10-09T05:21:25.259074+0000","last_fullsized":"2023-10-09T05:21:25.259074+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.815021+0000","last_change":"2023-10-09T05:21:16.813006+0000","last_active":"2023-10-09T05:21:16.815021+0000","last_peered":"2023-10-09T05:21:16.815021+0000","last_clean":"2023-10-09T05:21:16.815021+0000","last_became_active":"2023-10-09T05:21:16.812658+0000","last_became_peered":"2023-10-09T05:21:16.812658+0000","last_unstale":"2023-10-09T05:21:16.815021+0000","last_undegraded":"2023-10-09T05:21:16.815021+0000","last_fullsized":"2023-10-09T05:21:16.815021+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.801635+0000","last_change":"2023-10-09T05:21:16.789068+0000","last_active":"2023-10-09T05:21:16.801635+0000","last_peered":"2023-10-09T05:21:16.801635+0000","last_clean":"2023-10-09T05:21:16.801635+0000","last_became_active":"2023-10-09T05:21:16.788751+0000","last_became_peered":"2023-10-09T05:21:16.788751+0000","last_unstale":"2023-10-09T05:21:16.801635+0000","last_undegraded":"2023-10-09T05:21:16.801635+0000","last_fullsized":"2023-10-09T05:21:16.801635+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.465630+0000","last_change":"2023-10-09T05:21:17.464207+0000","last_active":"2023-10-09T05:21:17.465630+0000","last_peered":"2023-10-09T05:21:17.465630+0000","last_clean":"2023-10-09T05:21:17.465630+0000","last_became_active":"2023-10-09T05:21:17.463892+0000","last_became_peered":"2023-10-09T05:21:17.463892+0000","last_unstale":"2023-10-09T05:21:17.465630+0000","last_undegraded":"2023-10-09T05:21:17.465630+0000","last_fullsized":"2023-10-09T05:21:17.465630+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.840209+0000","last_change":"2023-10-09T05:21:16.781147+0000","last_active":"2023-10-09T05:21:17.840209+0000","last_peered":"2023-10-09T05:21:17.840209+0000","last_clean":"2023-10-09T05:21:17.840209+0000","last_became_active":"2023-10-09T05:21:16.780755+0000","last_became_peered":"2023-10-09T05:21:16.780755+0000","last_unstale":"2023-10-09T05:21:17.840209+0000","last_undegraded":"2023-10-09T05:21:17.840209+0000","last_fullsized":"2023-10-09T05:21:17.840209+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.811827+0000","last_change":"2023-10-09T05:21:16.811562+0000","last_active":"2023-10-09T05:21:16.811827+0000","last_peered":"2023-10-09T05:21:16.811827+0000","last_clean":"2023-10-09T05:21:16.811827+0000","last_became_active":"2023-10-09T05:21:16.811269+0000","last_became_peered":"2023-10-09T05:21:16.811269+0000","last_unstale":"2023-10-09T05:21:16.811827+0000","last_undegraded":"2023-10-09T05:21:16.811827+0000","last_fullsized":"2023-10-09T05:21:16.811827+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.258538+0000","last_change":"2023-10-09T05:21:16.781889+0000","last_active":"2023-10-09T05:21:25.258538+0000","last_peered":"2023-10-09T05:21:25.258538+0000","last_clean":"2023-10-09T05:21:25.258538+0000","last_became_active":"2023-10-09T05:21:16.781645+0000","last_became_peered":"2023-10-09T05:21:16.781645+0000","last_unstale":"2023-10-09T05:21:25.258538+0000","last_undegraded":"2023-10-09T05:21:25.258538+0000","last_fullsized":"2023-10-09T05:21:25.258538+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.466018+0000","last_change":"2023-10-09T05:21:17.464398+0000","last_active":"2023-10-09T05:21:17.466018+0000","last_peered":"2023-10-09T05:21:17.466018+0000","last_clean":"2023-10-09T05:21:17.466018+0000","last_became_active":"2023-10-09T05:21:17.464086+0000","last_became_peered":"2023-10-09T05:21:17.464086+0000","last_unstale":"2023-10-09T05:21:17.466018+0000","last_undegraded":"2023-10-09T05:21:17.466018+0000","last_fullsized":"2023-10-09T05:21:17.466018+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.838784+0000","last_change":"2023-10-09T05:21:16.783643+0000","last_active":"2023-10-09T05:21:17.838784+0000","last_peered":"2023-10-09T05:21:17.838784+0000","last_clean":"2023-10-09T05:21:17.838784+0000","last_became_active":"2023-10-09T05:21:16.783225+0000","last_became_peered":"2023-10-09T05:21:16.783225+0000","last_unstale":"2023-10-09T05:21:17.838784+0000","last_undegraded":"2023-10-09T05:21:17.838784+0000","last_fullsized":"2023-10-09T05:21:17.838784+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.464962+0000","last_change":"2023-10-09T05:21:17.464241+0000","last_active":"2023-10-09T05:21:17.464962+0000","last_peered":"2023-10-09T05:21:17.464962+0000","last_clean":"2023-10-09T05:21:17.464962+0000","last_became_active":"2023-10-09T05:21:17.463854+0000","last_became_peered":"2023-10-09T05:21:17.463854+0000","last_unstale":"2023-10-09T05:21:17.464962+0000","last_undegraded":"2023-10-09T05:21:17.464962+0000","last_fullsized":"2023-10-09T05:21:17.464962+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.258732+0000","last_change":"2023-10-09T05:21:16.783235+0000","last_active":"2023-10-09T05:21:25.258732+0000","last_peered":"2023-10-09T05:21:25.258732+0000","last_clean":"2023-10-09T05:21:25.258732+0000","last_became_active":"2023-10-09T05:21:16.782671+0000","last_became_peered":"2023-10-09T05:21:16.782671+0000","last_unstale":"2023-10-09T05:21:25.258732+0000","last_undegraded":"2023-10-09T05:21:25.258732+0000","last_fullsized":"2023-10-09T05:21:25.258732+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.467215+0000","last_change":"2023-10-09T05:21:17.466925+0000","last_active":"2023-10-09T05:21:17.467215+0000","last_peered":"2023-10-09T05:21:17.467215+0000","last_clean":"2023-10-09T05:21:17.467215+0000","last_became_active":"2023-10-09T05:21:17.466588+0000","last_became_peered":"2023-10-09T05:21:17.466588+0000","last_unstale":"2023-10-09T05:21:17.467215+0000","last_undegraded":"2023-10-09T05:21:17.467215+0000","last_fullsized":"2023-10-09T05:21:17.467215+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.463681+0000","last_change":"2023-10-09T05:21:16.784983+0000","last_active":"2023-10-09T05:21:17.463681+0000","last_peered":"2023-10-09T05:21:17.463681+0000","last_clean":"2023-10-09T05:21:17.463681+0000","last_became_active":"2023-10-09T05:21:16.784699+0000","last_became_peered":"2023-10-09T05:21:16.784699+0000","last_unstale":"2023-10-09T05:21:17.463681+0000","last_undegraded":"2023-10-09T05:21:17.463681+0000","last_fullsized":"2023-10-09T05:21:17.463681+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.466350+0000","last_change":"2023-10-09T05:21:17.465091+0000","last_active":"2023-10-09T05:21:17.466350+0000","last_peered":"2023-10-09T05:21:17.466350+0000","last_clean":"2023-10-09T05:21:17.466350+0000","last_became_active":"2023-10-09T05:21:17.464814+0000","last_became_peered":"2023-10-09T05:21:17.464814+0000","last_unstale":"2023-10-09T05:21:17.466350+0000","last_undegraded":"2023-10-09T05:21:17.466350+0000","last_fullsized":"2023-10-09T05:21:17.466350+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.803910+0000","last_change":"2023-10-09T05:21:16.786350+0000","last_active":"2023-10-09T05:21:16.803910+0000","last_peered":"2023-10-09T05:21:16.803910+0000","last_clean":"2023-10-09T05:21:16.803910+0000","last_became_active":"2023-10-09T05:21:16.785976+0000","last_became_peered":"2023-10-09T05:21:16.785976+0000","last_unstale":"2023-10-09T05:21:16.803910+0000","last_undegraded":"2023-10-09T05:21:16.803910+0000","last_fullsized":"2023-10-09T05:21:16.803910+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.465266+0000","last_change":"2023-10-09T05:21:17.464752+0000","last_active":"2023-10-09T05:21:17.465266+0000","last_peered":"2023-10-09T05:21:17.465266+0000","last_clean":"2023-10-09T05:21:17.465266+0000","last_became_active":"2023-10-09T05:21:17.464439+0000","last_became_peered":"2023-10-09T05:21:17.464439+0000","last_unstale":"2023-10-09T05:21:17.465266+0000","last_undegraded":"2023-10-09T05:21:17.465266+0000","last_fullsized":"2023-10-09T05:21:17.465266+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":13,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.259103+0000","last_change":"2023-10-09T05:21:16.789819+0000","last_active":"2023-10-09T05:21:25.259103+0000","last_peered":"2023-10-09T05:21:25.259103+0000","last_clean":"2023-10-09T05:21:25.259103+0000","last_became_active":"2023-10-09T05:21:16.789507+0000","last_became_peered":"2023-10-09T05:21:16.789507+0000","last_unstale":"2023-10-09T05:21:25.259103+0000","last_undegraded":"2023-10-09T05:21:25.259103+0000","last_fullsized":"2023-10-09T05:21:25.259103+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.813428+0000","last_change":"2023-10-09T05:21:16.813026+0000","last_active":"2023-10-09T05:21:16.813428+0000","last_peered":"2023-10-09T05:21:16.813428+0000","last_clean":"2023-10-09T05:21:16.813428+0000","last_became_active":"2023-10-09T05:21:16.812743+0000","last_became_peered":"2023-10-09T05:21:16.812743+0000","last_unstale":"2023-10-09T05:21:16.813428+0000","last_undegraded":"2023-10-09T05:21:16.813428+0000","last_fullsized":"2023-10-09T05:21:16.813428+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"59'6","reported_seq":17,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.259176+0000","last_change":"2023-10-09T05:21:16.778811+0000","last_active":"2023-10-09T05:21:25.259176+0000","last_peered":"2023-10-09T05:21:25.259176+0000","last_clean":"2023-10-09T05:21:25.259176+0000","last_became_active":"2023-10-09T05:21:16.778515+0000","last_became_peered":"2023-10-09T05:21:16.778515+0000","last_unstale":"2023-10-09T05:21:25.259176+0000","last_undegraded":"2023-10-09T05:21:25.259176+0000","last_fullsized":"2023-10-09T05:21:25.259176+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.814690+0000","last_change":"2023-10-09T05:21:16.813124+0000","last_active":"2023-10-09T05:21:16.814690+0000","last_peered":"2023-10-09T05:21:16.814690+0000","last_clean":"2023-10-09T05:21:16.814690+0000","last_became_active":"2023-10-09T05:21:16.812491+0000","last_became_peered":"2023-10-09T05:21:16.812491+0000","last_unstale":"2023-10-09T05:21:16.814690+0000","last_undegraded":"2023-10-09T05:21:16.814690+0000","last_fullsized":"2023-10-09T05:21:16.814690+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":14,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:25.258628+0000","last_change":"2023-10-09T05:21:16.791216+0000","last_active":"2023-10-09T05:21:25.258628+0000","last_peered":"2023-10-09T05:21:25.258628+0000","last_clean":"2023-10-09T05:21:25.258628+0000","last_became_active":"2023-10-09T05:21:16.790923+0000","last_became_peered":"2023-10-09T05:21:16.790923+0000","last_unstale":"2023-10-09T05:21:25.258628+0000","last_undegraded":"2023-10-09T05:21:25.258628+0000","last_fullsized":"2023-10-09T05:21:25.258628+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.465297+0000","last_change":"2023-10-09T05:21:17.464837+0000","last_active":"2023-10-09T05:21:17.465297+0000","last_peered":"2023-10-09T05:21:17.465297+0000","last_clean":"2023-10-09T05:21:17.465297+0000","last_became_active":"2023-10-09T05:21:17.464513+0000","last_became_peered":"2023-10-09T05:21:17.464513+0000","last_unstale":"2023-10-09T05:21:17.465297+0000","last_undegraded":"2023-10-09T05:21:17.465297+0000","last_fullsized":"2023-10-09T05:21:17.465297+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.803253+0000","last_change":"2023-10-09T05:21:16.785017+0000","last_active":"2023-10-09T05:21:16.803253+0000","last_peered":"2023-10-09T05:21:16.803253+0000","last_clean":"2023-10-09T05:21:16.803253+0000","last_became_active":"2023-10-09T05:21:16.784698+0000","last_became_peered":"2023-10-09T05:21:16.784698+0000","last_unstale":"2023-10-09T05:21:16.803253+0000","last_undegraded":"2023-10-09T05:21:16.803253+0000","last_fullsized":"2023-10-09T05:21:16.803253+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.817272+0000","last_change":"2023-10-09T05:21:16.813333+0000","last_active":"2023-10-09T05:21:16.817272+0000","last_peered":"2023-10-09T05:21:16.817272+0000","last_clean":"2023-10-09T05:21:16.817272+0000","last_became_active":"2023-10-09T05:21:16.812960+0000","last_became_peered":"2023-10-09T05:21:16.812960+0000","last_unstale":"2023-10-09T05:21:16.817272+0000","last_undegraded":"2023-10-09T05:21:16.817272+0000","last_fullsized":"2023-10-09T05:21:16.817272+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.805390+0000","last_change":"2023-10-09T05:21:16.788367+0000","last_active":"2023-10-09T05:21:16.805390+0000","last_peered":"2023-10-09T05:21:16.805390+0000","last_clean":"2023-10-09T05:21:16.805390+0000","last_became_active":"2023-10-09T05:21:16.787793+0000","last_became_peered":"2023-10-09T05:21:16.787793+0000","last_unstale":"2023-10-09T05:21:16.805390+0000","last_undegraded":"2023-10-09T05:21:16.805390+0000","last_fullsized":"2023-10-09T05:21:16.805390+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.811930+0000","last_change":"2023-10-09T05:21:16.811697+0000","last_active":"2023-10-09T05:21:16.811930+0000","last_peered":"2023-10-09T05:21:16.811930+0000","last_clean":"2023-10-09T05:21:16.811930+0000","last_became_active":"2023-10-09T05:21:16.811357+0000","last_became_peered":"2023-10-09T05:21:16.811357+0000","last_unstale":"2023-10-09T05:21:16.811930+0000","last_undegraded":"2023-10-09T05:21:16.811930+0000","last_fullsized":"2023-10-09T05:21:16.811930+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.799583+0000","last_change":"2023-10-09T05:21:16.792789+0000","last_active":"2023-10-09T05:21:16.799583+0000","last_peered":"2023-10-09T05:21:16.799583+0000","last_clean":"2023-10-09T05:21:16.799583+0000","last_became_active":"2023-10-09T05:21:16.791974+0000","last_became_peered":"2023-10-09T05:21:16.791974+0000","last_unstale":"2023-10-09T05:21:16.799583+0000","last_undegraded":"2023-10-09T05:21:16.799583+0000","last_fullsized":"2023-10-09T05:21:16.799583+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.464529+0000","last_change":"2023-10-09T05:21:17.464218+0000","last_active":"2023-10-09T05:21:17.464529+0000","last_peered":"2023-10-09T05:21:17.464529+0000","last_clean":"2023-10-09T05:21:17.464529+0000","last_became_active":"2023-10-09T05:21:17.463847+0000","last_became_peered":"2023-10-09T05:21:17.463847+0000","last_unstale":"2023-10-09T05:21:17.464529+0000","last_undegraded":"2023-10-09T05:21:17.464529+0000","last_fullsized":"2023-10-09T05:21:17.464529+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.802492+0000","last_change":"2023-10-09T05:21:16.793454+0000","last_active":"2023-10-09T05:21:16.802492+0000","last_peered":"2023-10-09T05:21:16.802492+0000","last_clean":"2023-10-09T05:21:16.802492+0000","last_became_active":"2023-10-09T05:21:16.793136+0000","last_became_peered":"2023-10-09T05:21:16.793136+0000","last_unstale":"2023-10-09T05:21:16.802492+0000","last_undegraded":"2023-10-09T05:21:16.802492+0000","last_fullsized":"2023-10-09T05:21:16.802492+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":10,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T05:21:16.812141+0000","last_change":"2023-10-09T05:21:16.811697+0000","last_active":"2023-10-09T05:21:16.812141+0000","last_peered":"2023-10-09T05:21:16.812141+0000","last_clean":"2023-10-09T05:21:16.812141+0000","last_became_active":"2023-10-09T05:21:16.811360+0000","last_became_peered":"2023-10-09T05:21:16.811360+0000","last_unstale":"2023-10-09T05:21:16.812141+0000","last_undegraded":"2023-10-09T05:21:16.812141+0000","last_fullsized":"2023-10-09T05:21:16.812141+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-09T05:21:17.839170+0000","last_change":"2023-10-09T05:21:16.784096+0000","last_active":"2023-10-09T05:21:17.839170+0000","last_peered":"2023-10-09T05:21:17.839170+0000","last_clean":"2023-10-09T05:21:17.839170+0000","last_became_active":"2023-10-09T05:21:16.783768+0000","last_became_peered":"2023-10-09T05:21:16.783768+0000","last_unstale":"2023-10-09T05:21:17.839170+0000","last_undegraded":"2023-10-09T05:21:17.839170+0000","last_fullsized":"2023-10-09T05:21:17.839170+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430239,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297700,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447452,"statfs":{"total":95995035648,"available":95690190848,"internally_reserved":0,"allocated":618496,"data_stored":251129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.61499999999999999,"15min":0.63500000000000001},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.88400000000000001,"5min":2.8300000000000001,"15min":2.8300000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53800000000000003,"15min":0.54300000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.874,"5min":1.389,"15min":1.389},"last":0.46100000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.58899999999999997,"15min":0.59899999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.92800000000000005,"5min":2.2400000000000002,"15min":2.2400000000000002},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.57499999999999996,"15min":0.58499999999999996},"min":{"1min":0.36699999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.83199999999999996,"5min":2.4809999999999999,"15min":2.4809999999999999},"last":0.49399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.61099999999999999,"15min":0.625},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.91500000000000004,"5min":2.7029999999999998,"15min":2.7029999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.58299999999999996,"15min":0.59299999999999997},"min":{"1min":0.36399999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.83599999999999997,"5min":2.117,"15min":2.117},"last":0.51800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.63900000000000001,"15min":0.63500000000000001},"min":{"1min":0.40799999999999997,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.6560000000000001,"5min":2.6669999999999998,"15min":2.6669999999999998},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61599999999999999,"15min":0.622},"min":{"1min":0.441,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.90100000000000002,"5min":2.1379999999999999,"15min":2.1379999999999999},"last":0.54700000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.45000000000000001,"15min":0.44400000000000001},"min":{"1min":0.307,"5min":0.25,"15min":0.25},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.621},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.49099999999999999,"15min":0.48499999999999999},"min":{"1min":0.311,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.57099999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.49099999999999999,"15min":0.48599999999999999},"min":{"1min":0.32400000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.54500000000000004,"15min":0.53900000000000003},"min":{"1min":0.35199999999999998,"5min":0.314,"15min":0.314},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.64000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.52500000000000002,"15min":0.52000000000000002},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.57599999999999996,"15min":0.57399999999999995},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80400000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.65900000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626467,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":250718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.65800000000000003,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.626,"5min":0.55000000000000004,"15min":0.53700000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.9500000000000002,"5min":2.9500000000000002,"15min":2.9500000000000002},"last":0.56699999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51700000000000002,"15min":0.51800000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.64900000000000002,"5min":0.747,"15min":0.747},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54000000000000004,"15min":0.54300000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.55000000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53900000000000003,"15min":0.54300000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.63500000000000001,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.56699999999999995,"15min":0.56399999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.101,"5min":1.101,"15min":1.101},"last":0.51900000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.56499999999999995,"15min":0.54200000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":3.9569999999999999,"5min":3.9569999999999999,"15min":3.9569999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.59199999999999997,"15min":0.57599999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.9049999999999998,"5min":2.9049999999999998,"15min":2.9049999999999998},"last":0.52900000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.434,"15min":0.42699999999999999},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.441,"15min":0.42799999999999999},"min":{"1min":0.373,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.42799999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.46800000000000003,"15min":0.46000000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.47099999999999997,"15min":0.46300000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.57899999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:21:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.52500000000000002,"15min":0.51900000000000002},"min":{"1min":0.41099999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.53600000000000003,"15min":0.52700000000000002},"min":{"1min":0.38400000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.60699999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574904,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297672,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447480,"statfs":{"total":95995035648,"available":95690219520,"internally_reserved":0,"allocated":589824,"data_stored":249377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.41299999999999998,"15min":0.34799999999999998},"min":{"1min":0.23899999999999999,"5min":0.218,"15min":0.218},"max":{"1min":6.2880000000000003,"5min":6.2880000000000003,"15min":6.2880000000000003},"last":0.378},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.378,"15min":0.34200000000000003},"min":{"1min":0.247,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.7679999999999998,"5min":2.7679999999999998,"15min":2.7679999999999998},"last":0.26700000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:20:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.442,"15min":0.41499999999999998},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8460000000000001,"5min":2.8460000000000001,"15min":2.8460000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.44500000000000001,"15min":0.40899999999999997},"min":{"1min":0.25700000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.9290000000000003,"5min":5.9290000000000003,"15min":5.9290000000000003},"last":0.29199999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:21:29 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.54200000000000004,"15min":0.5},"min":{"1min":0.307,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.007,"5min":10.007,"15min":10.007},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.49399999999999999,"15min":0.46999999999999997},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.7000000000000002,"5min":6.7000000000000002,"15min":6.7000000000000002},"last":0.42299999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:21:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.60799999999999998,"15min":0.59599999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.9139999999999999,"5min":1.9139999999999999,"15min":1.9139999999999999},"last":1.03},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.64200000000000002,"15min":0.60999999999999999},"min":{"1min":0.41999999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80600000000000005,"5min":6.3330000000000002,"15min":6.3330000000000002},"last":0.56299999999999994}]},{"osd":5,"last update":"Mon Oct 9 05:21:02 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.61599999999999999,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.3639999999999999,"5min":6.3639999999999999,"15min":6.3639999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.59099999999999997,"15min":0.57399999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":2.8799999999999999,"5min":2.8799999999999999,"15min":2.8799999999999999},"last":0.497}]},{"osd":6,"last update":"Mon Oct 9 05:21:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59599999999999997,"15min":0.59699999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.66600000000000004,"15min":0.63500000000000001},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":6.4400000000000004,"5min":6.4400000000000004,"15min":6.4400000000000004},"last":0.61499999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.68100000000000005,"15min":0.69999999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.75600000000000001,"5min":2.8069999999999999,"15min":2.8069999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.64200000000000002,"15min":0.64700000000000002},"min":{"1min":0.49099999999999999,"5min":0.432,"15min":0.432},"max":{"1min":0.79700000000000004,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":1.0149999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738430,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297676,"kb_used_data":580,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447476,"statfs":{"total":95995035648,"available":95690215424,"internally_reserved":0,"allocated":593920,"data_stored":249638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:21:41 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.377,"15min":0.35099999999999998},"min":{"1min":0.26800000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.137,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.253},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.43099999999999999,"15min":0.377},"min":{"1min":0.253,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.2000000000000002,"5min":3.2000000000000002,"15min":3.2000000000000002},"last":0.373}]},{"osd":2,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.45300000000000001,"15min":0.42999999999999999},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.4359999999999999,"5min":2.4359999999999999,"15min":2.4359999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41499999999999998,"15min":0.41099999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.27000000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.47799999999999998,"15min":0.47499999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.5179999999999998,"5min":2.5179999999999998,"15min":2.5179999999999998},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.45100000000000001,"15min":0.45800000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65400000000000003,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:21:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.61199999999999999,"15min":0.60699999999999998},"min":{"1min":0.38100000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.677,"5min":2.677,"15min":2.677},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.625,"15min":0.61299999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":2.4359999999999999,"5min":2.4359999999999999,"15min":2.4359999999999999},"last":0.59599999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:21:08 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.58799999999999997,"15min":0.59599999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.623,"15min":0.60699999999999998},"min":{"1min":0.45600000000000002,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":2.552,"5min":2.552,"15min":2.552},"last":0.53400000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.61499999999999999,"15min":0.60899999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.637,"5min":2.637,"15min":2.637},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":0.498}]},{"osd":7,"last update":"Mon Oct 9 05:21:50 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.72699999999999998,"15min":0.71799999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":3.7610000000000001,"5min":3.7610000000000001,"15min":3.7610000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.60999999999999999,"15min":0.60299999999999998},"min":{"1min":0.437,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":0.51700000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411380,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297680,"kb_used_data":584,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447472,"statfs":{"total":95995035648,"available":95690211328,"internally_reserved":0,"allocated":598016,"data_stored":249965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.42799999999999999,"15min":0.40600000000000003},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.253},"max":{"1min":3.657,"5min":3.657,"15min":3.657},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.40500000000000003,"15min":0.40100000000000002},"min":{"1min":0.246,"5min":0.222,"15min":0.222},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.36599999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.44700000000000001,"15min":0.433},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.6859999999999999,"5min":1.6859999999999999,"15min":1.6859999999999999},"last":0.317},{"interface":"front","average":{"1min":0.495,"5min":0.46999999999999997,"15min":0.46000000000000002},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.39700000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:21:17 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44800000000000001,"15min":0.46000000000000002},"min":{"1min":0.27300000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.528,"5min":1.528,"15min":1.528},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.496,"15min":0.50600000000000001},"min":{"1min":0.29799999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":0.32600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:21:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.61899999999999999,"15min":0.629},"min":{"1min":0.41199999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0069999999999999,"5min":1.264,"15min":1.264},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.63100000000000001,"15min":0.63500000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.73299999999999998,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":0.57199999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.607,"5min":1.607,"15min":1.607},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.63,"5min":0.60999999999999999,"15min":0.60599999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":0.505}]},{"osd":6,"last update":"Mon Oct 9 05:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.64200000000000002,"15min":0.64600000000000002},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.464,"5min":1.464,"15min":1.464},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.47899999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.66600000000000004,"15min":0.68400000000000005},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.78600000000000003,"5min":1.502,"15min":1.502},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.69199999999999995,"15min":0.70699999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.77500000000000002,"5min":1.661,"15min":1.661},"last":0.55100000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182446,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297676,"kb_used_data":580,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447476,"statfs":{"total":95995035648,"available":95690215424,"internally_reserved":0,"allocated":593920,"data_stored":249467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.38800000000000001,"15min":0.39700000000000002},"min":{"1min":0.26000000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.434,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.35099999999999998,"5min":0.41799999999999998,"15min":0.432},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.48999999999999999,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.28100000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.437,"15min":0.45000000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.53500000000000003,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.379,"5min":0.44,"15min":0.45400000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.54900000000000004,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.32900000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.47999999999999998,"15min":0.498},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.56000000000000005,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.47199999999999998,"15min":0.48599999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.63600000000000001,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.34899999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:21:42 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.60599999999999998,"15min":0.60899999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.441,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.72799999999999998,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.67900000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.627,"15min":0.63100000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79900000000000004,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77900000000000003,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.79700000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.68899999999999995,"15min":0.70299999999999996},"min":{"1min":0.41099999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81100000000000005,"5min":3.181,"15min":3.181},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61299999999999999,"15min":0.61399999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81999999999999995,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.85499999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:21:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.621,"15min":0.623},"min":{"1min":0.46800000000000003,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.73399999999999999,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.63500000000000001,"15min":0.63400000000000001},"min":{"1min":0.50600000000000001,"5min":0.437,"15min":0.437},"max":{"1min":1.335,"5min":1.335,"15min":1.335},"last":0.57599999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018923,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297696,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447456,"statfs":{"total":95995035648,"available":95690194944,"internally_reserved":0,"allocated":614400,"data_stored":251059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.56299999999999994,"15min":0.58199999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79000000000000004,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.52500000000000002,"15min":0.52700000000000002},"min":{"1min":0.36699999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.71799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.51900000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.54700000000000004,"15min":0.56100000000000005},"min":{"1min":0.28899999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.71599999999999997,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.54900000000000004,"15min":0.55900000000000005},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.75900000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.66100000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.53500000000000003,"15min":0.54100000000000004},"min":{"1min":0.36299999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69799999999999995,"5min":0.75,"15min":0.75},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.55700000000000005,"15min":0.56999999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72699999999999998,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.59899999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55700000000000005,"15min":0.56499999999999995},"min":{"1min":0.35899999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77500000000000002,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.55500000000000005,"15min":0.56999999999999995},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.68200000000000005,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.68600000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:21:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.41699999999999998,"15min":0.41199999999999998},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.502,"5min":0.48499999999999999,"15min":0.48199999999999998},"min":{"1min":0.35999999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.495}]},{"osd":6,"last update":"Mon Oct 9 05:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.48799999999999999,"15min":0.49299999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.80500000000000005,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.432},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.501,"15min":0.505},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.77700000000000002,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.45700000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.53900000000000003,"15min":0.54700000000000004},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.80300000000000005,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.53500000000000003,"15min":0.54300000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.74099999999999999,"5min":1.034,"15min":1.034},"last":0.57199999999999995}]}]},{"osd":5,"up_from":36,"seq":154618822695,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297676,"kb_used_data":580,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447476,"statfs":{"total":95995035648,"available":95690215424,"internally_reserved":0,"allocated":593920,"data_stored":249686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.55900000000000005,"15min":0.56799999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70699999999999996,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.53200000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.54800000000000004,"15min":0.55200000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.66100000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.371},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73799999999999999,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.45100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.53100000000000003,"15min":0.53800000000000003},"min":{"1min":0.39500000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.67700000000000005,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.59899999999999998,"15min":0.59699999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.708,"5min":1.708,"15min":1.708},"last":0.58699999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.57199999999999995,"15min":0.57799999999999996},"min":{"1min":0.39000000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.65500000000000003,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.623,"15min":0.629},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.79500000000000004,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.626}]},{"osd":4,"last update":"Mon Oct 9 05:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.40100000000000002,"15min":0.38700000000000001},"min":{"1min":0.27500000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.43099999999999999,"15min":0.41699999999999998},"min":{"1min":0.28999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.56100000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.433,"15min":0.42199999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.49199999999999999,"15min":0.48599999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.88100000000000001,"5min":1.109,"15min":1.109},"last":0.65000000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.56999999999999995,"15min":0.56799999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.90800000000000003,"5min":1.143,"15min":1.143},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.36399999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.84599999999999997,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.59699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:22:03.284 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:22:03.284 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T05:22:03.284 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T05:22:03.284 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T05:22:03.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:03] "GET /metrics HTTP/1.1" 200 191461 "" "Prometheus/2.33.4" 2023-10-09T05:22:03.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: cluster 2023-10-09T05:22:01.914858+0000 mgr.y 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: (mgr.14442) 94 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: audit 2023-10-09T05:22:02. 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: 246630+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: audit 2023-10-09T05:22:02 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: .246689+0000 mon.a (mon.0) 665 : audit [INF] 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: 10-09T05:22:02.246732+0000 mon.a (mon.0) 666 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: cluster 2023-10-09T05:22 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: :02.246789+0000 mon.a (mon.0) 667 2023-10-09T05:22:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[110782]: : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T05:22:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: cluster 2023-10-09T05:22:01.914858+0000 mgr.y 2023-10-09T05:22:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: (mgr.14442) 94 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: audit 2023-10-09T05:22:02. 2023-10-09T05:22:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: 246630+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:22:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: audit 2023-10-09T05:22:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: 2023-10- 2023-10-09T05:22:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: 09T05:22:02.246689 2023-10-09T05:22:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: +0000 mon.a (mon 2023-10-09T05:22:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: .0) 665 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T05:22:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: audit 2023-10-09T05:22:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: 2023-10-09T05:22:02.246732 2023-10-09T05:22:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: +0000 mon.a (mon.0) 666 : audit [INF] 2023-10-09T05:22:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-10-09T05:22:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: cluster 2023-10-09T05:22:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: 2023-10-09T05:22:02.246789+0000 mon.a (mon.0) 667 : 2023-10-09T05:22:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:03 smithi008 conmon[101053]: cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T05:22:03.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:03 smithi191 conmon[101299]: cluster 2023-10-09T05:22:01.914858+0000 mgr.y (mgr.14442) 94 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:03 smithi191 conmon[101299]: audit 2023-10-09T05:22:02.246630+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:22:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:03 smithi191 conmon[101299]: audit 2023-10-09T05:22:02.246689+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T05:22:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:03 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:22:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:03 smithi191 conmon[101299]: 09T05:22:02.246732+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-10-09T05:22:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:03 smithi191 conmon[101299]: cluster 2023-10 2023-10-09T05:22:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:03 smithi191 conmon[101299]: -09T05:22:02.246789+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T05:22:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:03 smithi191 conmon[101299]: 2023-10-09T05:22:03.645 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:22:04.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:04 smithi191 conmon[101299]: audit 2023-10-09T05:22:02.613521+0000 mgr.y (mgr.14442) 95 : audit [DBG] from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:22:04.730 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:04 smithi191 conmon[101299]: 2023-10-09T05:22:04.730 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:04 smithi191 conmon[101299]: cluster 2023-10-09T05:22:03.248623+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T05:22:04.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:04 smithi008 conmon[101053]: audit 2023-10-09T05:22:02.613521+0000 mgr.y (mgr.14442) 95 : audit [DBG] from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:22:04.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:04 smithi008 conmon[101053]: cluster 2023-10-09T05:22:03.248623+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T05:22:04.731 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:04 smithi008 conmon[110782]: audit 2023-10-09T05:22:02.613521+0000 mgr.y (mgr.14442) 95 : audit [DBG] from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:22:04.731 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:04 smithi008 conmon[110782]: cluster 2023-10-09T05:22:03.248623+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T05:22:04.992 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:22:04 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:22:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:05 smithi008 conmon[101053]: cluster 2023-10-09T05:22:03.915401+0000 mgr.y (mgr.14442) 2023-10-09T05:22:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:05 smithi008 conmon[101053]: 96 : cluster [DBG] pgmap v74: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:05.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:05 smithi008 conmon[110782]: cluster 2023-10-09T05:22:03.915401+0000 mgr.y ( 2023-10-09T05:22:05.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:05 smithi008 conmon[110782]: mgr.14442) 96 : cluster [DBG] pgmap v74: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:05.489 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:22:05.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:05 smithi191 conmon[101299]: cluster 2023-10-09T05:22:03.915401+0000 mgr.y (mgr.14442) 96 : cluster 2023-10-09T05:22:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:05 smithi191 conmon[101299]: [DBG] pgmap v74: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:06.084 INFO:teuthology.orchestra.run.smithi008.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T05:22:06.085 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T05:22:06.085 INFO:teuthology.run_tasks:Running task print... 2023-10-09T05:22:06.094 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T05:22:06.094 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T05:22:06.103 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T05:22:06.105 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T05:22:06.105 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T05:22:06.109 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T05:22:06.109 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T05:22:06.109 DEBUG:teuthology.orchestra.run.smithi008:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:22:06.110 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T05:22:06.110 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T05:22:06.110 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi008.front.sepia.ceph.com 2023-10-09T05:22:06.111 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T05:22:06.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:22:06.128 INFO:teuthology.orchestra.run.smithi008.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T05:22:06.129 DEBUG:teuthology.orchestra.run.smithi008:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:22:06.185 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T05:22:06.185 DEBUG:teuthology.orchestra.run.smithi008:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T05:22:06.242 INFO:tasks.workunit:timeout=3h 2023-10-09T05:22:06.243 INFO:tasks.workunit:cleanup=True 2023-10-09T05:22:06.243 DEBUG:teuthology.orchestra.run.smithi008:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:22:06.300 INFO:tasks.workunit.client.0.smithi008.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T05:22:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:06 smithi008 conmon[101053]: audit 2023-10-09T05:22:05.487261+0000 mon.c (mon.2) 86 : audit [DBG] from='client.? 172.21.15.8:0/2733196185' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:22:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:06 smithi008 conmon[110782]: audit 2023-10-09T05:22:05.487261+0000 mon.c (mon.2) 86 : audit [DBG] from='client.? 172.21.15.8:0/2733196185' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:22:06.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:06 smithi191 conmon[101299]: audit 2023-10-09T05:22:05.487261+0000 mon.c (mon.2) 86 : 2023-10-09T05:22:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:06 smithi191 conmon[101299]: audit [DBG] from='client.? 172.21.15.8:0/2733196185' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:22:07.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:07 smithi191 conmon[101299]: cluster 2023-10-09T05:22:05.915789+0000 mgr.y (mgr.14442) 97 : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:07.630 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:07 smithi008 conmon[101053]: cluster 2023-10-09T05:22: 2023-10-09T05:22:07.630 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:07 smithi008 conmon[101053]: 05.915789+0000 mgr.y (mgr.14442) 97 : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:07.630 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:07 smithi008 conmon[110782]: cluster 2023-10-09T05:22:05.915789 2023-10-09T05:22:07.630 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:07 smithi008 conmon[110782]: +0000 mgr.y (mgr.14442) 97 : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:08.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:08 smithi008 conmon[101509]: [09/Oct/2023:05:22:08] ENGINE Bus STOPPING 2023-10-09T05:22:08.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:08 smithi008 conmon[101509]: [09/Oct/2023:05:22:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:22:08.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:08 smithi008 conmon[101509]: [09/Oct/2023:05:22:08] ENGINE Bus STOPPED 2023-10-09T05:22:08.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:08 smithi008 conmon[101509]: [09/Oct/2023:05:22:08] ENGINE Bus STARTING 2023-10-09T05:22:08.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:08 smithi008 conmon[101509]: [09/Oct/2023:05:22:08] ENGINE Serving on http://:::9283 2023-10-09T05:22:08.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:08 smithi008 conmon[101509]: [09/Oct/2023:05:22:08] ENGINE Bus STARTED 2023-10-09T05:22:08.873 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T05:22:09.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[101053]: cluster 2023-10-09T05:22:07.916240+0000 mgr.y (mgr.14442) 98 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:09.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[101053]: audit 2023-10-09T05:22:08.009538+0000 mon.a (mon.0) 669 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T05:22:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[101053]: audit 2023-10-09T05:22:08.044508+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[101053]: audit 2023-10-09T05:22:08.704615+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:22:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[101053]: 09T05:22:08.705444+0000 mon.c (mon.2) 89 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[101053]: audit 2023-10-09T05:22:08.711534+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[110782]: cluster 2023-10-09T05:22:07.916240+0000 mgr.y (mgr.14442) 98 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[110782]: audit 2023-10-09T05:22:08.009538+0000 mon.a (mon.0) 669 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T05:22:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[110782]: audit 2023-10-09T05:22:08.044508+0000 mon.c (mon.2) 87 : audit [DBG] 2023-10-09T05:22:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[110782]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:09.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[110782]: audit 2023-10-09T05:22:08.704615+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:09.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[110782]: audit 2023-10-09T05:22:08.705444+0000 mon.c (mon.2) 89 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:09.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:09 smithi008 conmon[110782]: audit 2023-10-09T05:22:08.711534+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:09.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:09 smithi191 conmon[101299]: cluster 2023-10-09T05:22:07.916240+0000 mgr.y (mgr.14442) 98 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:09 smithi191 conmon[101299]: audit 2023-10-09T05:22:08.009538+0000 mon.a (mon.0) 669 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T05:22:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:09 smithi191 conmon[101299]: audit 2023-10-09T05:22:08.044508+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:09 smithi191 conmon[101299]: audit 2023-10-09T05:22:08.704615+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:09 smithi191 conmon[101299]: audit 2023-10-09T05:22:08.705444+0000 mon.c (mon.2) 89 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:09 smithi191 conmon[101299]: audit 2023-10-09T05:22:08.711534+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:10.818 INFO:tasks.workunit.client.0.smithi008.stderr:Updating files: 97% (10890/11158) Updating files: 98% (10935/11158) Updating files: 99% (11047/11158) Updating files: 100% (11158/11158) Updating files: 100% (11158/11158), done. 2023-10-09T05:22:10.835 DEBUG:teuthology.orchestra.run.smithi008:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T05:22:11.098 INFO:tasks.workunit.client.0.smithi008.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T05:22:11.100 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:22:11.100 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T05:22:11.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:11 smithi191 conmon[101299]: cluster 2023-10-09T05:22:09.916549+0000 mgr.y (mgr.14442) 99 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:11.309 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:11 smithi008 conmon[101053]: cluster 2023-10-09T05:22:09.916549+0000 mgr.y 2023-10-09T05:22:11.309 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:11 smithi008 conmon[101053]: (mgr.14442) 99 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:11.309 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:11 smithi008 conmon[110782]: cluster 2023-10-09T05:22:09.916549 2023-10-09T05:22:11.309 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:11 smithi008 conmon[110782]: +0000 mgr.y (mgr.14442) 99 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:11.968 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T05:22:12.028 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T05:22:12.065 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:22:12.067 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:22:12.067 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T05:22:12.107 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:22:12.111 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T05:22:12.111 DEBUG:teuthology.orchestra.run.smithi008:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T05:22:12.129 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T05:22:12.130 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T05:22:12.130 DEBUG:teuthology.orchestra.run.smithi008:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T05:22:12.191 INFO:tasks.workunit.client.0.smithi008.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T05:22:12.192 INFO:tasks.workunit.client.0.smithi008.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-10-09T05:22:12.192 INFO:tasks.workunit.client.0.smithi008.stderr:+ TMPDIR=/tmp/rbd_import_export_165117 2023-10-09T05:22:12.192 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm -rf /tmp/rbd_import_export_165117 2023-10-09T05:22:12.192 INFO:tasks.workunit.client.0.smithi008.stderr:+ mkdir /tmp/rbd_import_export_165117 2023-10-09T05:22:12.193 INFO:tasks.workunit.client.0.smithi008.stderr:+ trap 'rm -rf /tmp/rbd_import_export_165117' INT TERM EXIT 2023-10-09T05:22:12.193 INFO:tasks.workunit.client.0.smithi008.stderr:+ mkdir foo.165117 2023-10-09T05:22:12.193 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import foo.165117 foo.dir 2023-10-09T05:22:12.277 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: cannot import a directory 2023-10-09T05:22:12.277 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 0% complete...failed. 2023-10-09T05:22:12.277 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: import failed: (21) Is a directory 2023-10-09T05:22:12.279 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T05:22:12.279 INFO:tasks.workunit.client.0.smithi008.stderr:+ rmdir foo.165117 2023-10-09T05:22:12.287 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_165117/img bs=1k count=1 seek=10 2023-10-09T05:22:12.289 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records in 2023-10-09T05:22:12.289 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records out 2023-10-09T05:22:12.289 INFO:tasks.workunit.client.0.smithi008.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 5.4378e-05 s, 18.8 MB/s 2023-10-09T05:22:12.289 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_165117/img bs=1k count=10 seek=100 2023-10-09T05:22:12.289 INFO:tasks.workunit.client.0.smithi008.stderr:10+0 records in 2023-10-09T05:22:12.290 INFO:tasks.workunit.client.0.smithi008.stderr:10+0 records out 2023-10-09T05:22:12.290 INFO:tasks.workunit.client.0.smithi008.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.094e-05 s, 168 MB/s 2023-10-09T05:22:12.290 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_165117/img bs=1k count=100 seek=1000 2023-10-09T05:22:12.290 INFO:tasks.workunit.client.0.smithi008.stderr:70+1 records in 2023-10-09T05:22:12.291 INFO:tasks.workunit.client.0.smithi008.stderr:70+1 records out 2023-10-09T05:22:12.291 INFO:tasks.workunit.client.0.smithi008.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000201353 s, 358 MB/s 2023-10-09T05:22:12.291 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_165117/img bs=1k seek=10000 2023-10-09T05:22:12.291 INFO:tasks.workunit.client.0.smithi008.stderr:139+1 records in 2023-10-09T05:22:12.292 INFO:tasks.workunit.client.0.smithi008.stderr:139+1 records out 2023-10-09T05:22:12.292 INFO:tasks.workunit.client.0.smithi008.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000356588 s, 402 MB/s 2023-10-09T05:22:12.292 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_165117/img bs=1k seek=100000 2023-10-09T05:22:12.295 INFO:tasks.workunit.client.0.smithi008.stderr:70+1 records in 2023-10-09T05:22:12.295 INFO:tasks.workunit.client.0.smithi008.stderr:70+1 records out 2023-10-09T05:22:12.295 INFO:tasks.workunit.client.0.smithi008.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.00271264 s, 26.6 MB/s 2023-10-09T05:22:12.295 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_165117/img bs=1k seek=1000000 2023-10-09T05:22:12.296 INFO:tasks.workunit.client.0.smithi008.stderr:189+1 records in 2023-10-09T05:22:12.296 INFO:tasks.workunit.client.0.smithi008.stderr:189+1 records out 2023-10-09T05:22:12.297 INFO:tasks.workunit.client.0.smithi008.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000501937 s, 386 MB/s 2023-10-09T05:22:12.297 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm testimg 2023-10-09T05:22:12.334 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 0% complete...failed. 2023-10-09T05:22:12.334 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T05:22:12.337 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T05:22:12.337 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format /tmp/rbd_import_export_165117/img testimg 2023-10-09T05:22:12.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:12 smithi008 conmon[101509]: [09/Oct/2023:05:22:12] ENGINE Bus STOPPING 2023-10-09T05:22:12.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:12 smithi008 conmon[101509]: [09/Oct/2023:05:22:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:22:12.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:12 smithi008 conmon[101509]: [09/Oct/2023:05:22:12] ENGINE Bus STOPPED 2023-10-09T05:22:12.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:12 smithi008 conmon[101509]: [09/Oct/2023:05:22:12] ENGINE Bus STARTING 2023-10-09T05:22:12.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:12 smithi008 conmon[101509]: [09/Oct/2023:05:22:12] ENGINE Serving on http://:::9283 2023-10-09T05:22:12.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:12 smithi008 conmon[101509]: [09/Oct/2023:05:22:12] ENGINE Bus STARTED 2023-10-09T05:22:13.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:13] "GET /metrics HTTP/1.1" 200 191461 "" "Prometheus/2.33.4" 2023-10-09T05:22:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[101053]: cluster 2023-10-09T05:22:11.916922+0000 2023-10-09T05:22:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[101053]: mgr.y (mgr.14442) 100 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[101053]: audit 2023-10-09T05:22:12.093447+0000 mon.a (mon.0) 671 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T05:22:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[101053]: audit 2023-10-09T05:22:12.136305+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[101053]: audit 2023-10-09T05:22:12.137840+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[101053]: audit 2023-10-09T05:22:12.138528+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[101053]: audit 2023-10-09T05:22:12.143295+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:13.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[110782]: cluster 2023-10-09T05:22:11.916922+0000 mgr.y (mgr.14442 2023-10-09T05:22:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[110782]: ) 100 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[110782]: audit 2023-10-09T05:22:12.093447+0000 mon.a (mon.0) 671 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T05:22:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[110782]: audit 2023-10-09T05:22:12.136305+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[110782]: audit 2023-10-09T05:22:12.137840+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[110782]: audit 2023-10-09T05:22:12.138528+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:13 smithi008 conmon[110782]: audit 2023-10-09T05:22:12.143295+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:13.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:13 smithi191 conmon[101299]: cluster 2023-10-09T05:22:11.916922+0000 mgr.y (mgr.14442) 100 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:13 smithi191 conmon[101299]: audit 2023-10-09T05:22:12.093447+0000 mon.a (mon.0) 671 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T05:22:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:13 smithi191 conmon[101299]: audit 2023-10-09T05:22:12.136305+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:13 smithi191 conmon[101299]: audit 2023-10-09T05:22:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:13 smithi191 conmon[101299]: 2023-10-09T05:22:12.137840+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:13 smithi191 conmon[101299]: audit 2023-10-09T05:22 2023-10-09T05:22:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:13 smithi191 conmon[101299]: :12.138528+0000 mon.c (mon.2) 92 : 2023-10-09T05:22:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:13 smithi191 conmon[101299]: audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:13 smithi191 conmon[101299]: audit 2023-10-09T05:22: 2023-10-09T05:22:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:13 smithi191 conmon[101299]: 12.143295+0000 mon.a (mon.0) 672 : audit 2023-10-09T05:22:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:13 smithi191 conmon[101299]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:14.276 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T05:22:14.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:14 smithi008 conmon[101053]: cluster 2023-10-09T05:22:13.202422+0000 mon.a ( 2023-10-09T05:22:14.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:14 smithi008 conmon[101053]: mon.0) 673 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T05:22:14.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:14 smithi008 conmon[110782]: cluster 2023-10-09T05:22:13.202422 2023-10-09T05:22:14.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:14 smithi008 conmon[110782]: +0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T05:22:14.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:14 smithi191 conmon[101299]: cluster 2023-10-09T05:22:13.202422+0000 mon.a (mon.0) 673 : cluster [DBG] 2023-10-09T05:22:14.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:14 smithi191 conmon[101299]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T05:22:14.746 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T05:22:14.747 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:22:14.749 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg /tmp/rbd_import_export_165117/img2 2023-10-09T05:22:15.004 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:15.007 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg - 2023-10-09T05:22:15.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:22:14 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:22:15.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:15 smithi008 conmon[101053]: cluster 2023-10-09T05:22:13.917411+0000 mgr.y (mgr.14442 2023-10-09T05:22:15.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:15 smithi008 conmon[101053]: ) 101 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:15.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:15 smithi008 conmon[101053]: cluster 2023-10-09T05:22:14.208439+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T05:22:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:15 smithi008 conmon[110782]: cluster 2023-10-09T05:22:13. 2023-10-09T05:22:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:15 smithi008 conmon[110782]: 917411+0000 mgr.y (mgr.14442) 101 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:15 smithi008 conmon[110782]: cluster 2023-10-09T05:22:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:15 smithi008 conmon[110782]: 2023-10-09T05:22:14.208439+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T05:22:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:15 smithi008 conmon[110782]: 2023-10-09T05:22:15.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:15 smithi191 conmon[101299]: cluster 2023-10-09T05:22:13.917411+0000 mgr.y (mgr.14442) 101 : cluster [DBG] 2023-10-09T05:22:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:15 smithi191 conmon[101299]: pgmap v80: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:15 smithi191 conmon[101299]: cluster 2023-10-09T05:22:14.208439+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T05:22:15.718 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:15.721 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm testimg 2023-10-09T05:22:15.809 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T05:22:15.812 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp /tmp/rbd_import_export_165117/img /tmp/rbd_import_export_165117/img2 2023-10-09T05:22:16.497 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp /tmp/rbd_import_export_165117/img /tmp/rbd_import_export_165117/img3 2023-10-09T05:22:16.971 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/img2 /tmp/rbd_import_export_165117/img3 2023-10-09T05:22:17.193 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format - testimg 2023-10-09T05:22:17.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:17 smithi008 conmon[101053]: cluster 2023-10-09T05:22:15.917779+0000 mgr.y (mgr.14442) 2023-10-09T05:22:17.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:17 smithi008 conmon[101053]: 102 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:17 smithi008 conmon[110782]: cluster 2023-10-09T05:22:15.917779+0000 2023-10-09T05:22:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:17 smithi008 conmon[110782]: mgr.y (mgr.14442) 102 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:17.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:17 smithi191 conmon[101299]: cluster 2023-10-09T05:22:15.917779+0000 mgr.y (mgr.14442) 102 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:17.581 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T05:22:17.581 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:22:17.584 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg /tmp/rbd_import_export_165117/img2 2023-10-09T05:22:17.877 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:17.880 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg - 2023-10-09T05:22:18.492 INFO:teuthology.orchestra.run.smithi008.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:22:18.604 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:18.608 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm testimg 2023-10-09T05:22:18.702 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T05:22:18.705 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp /tmp/rbd_import_export_165117/img /tmp/rbd_import_export_165117/img2 2023-10-09T05:22:19.367 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp /tmp/rbd_import_export_165117/img /tmp/rbd_import_export_165117/img3 2023-10-09T05:22:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: cluster 2023-10-09T05:22:17.918329+0000 mgr.y (mgr.14442) 103 : cluster [DBG] 2023-10-09T05:22:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: pgmap v83: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 129 KiB/s wr, 77 op/s 2023-10-09T05:22:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: audit 2023-10-09T05:22:18.486917+0000 mgr.y (mgr.14442) 104 : audit [DBG] from='client.24752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:22:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: cephadm 2023-10-09T05:22:18.487680+0000 2023-10-09T05:22:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: mgr.y (mgr.14442) 105 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:22:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: audit 2023-10-09T05:22 2023-10-09T05:22:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: :18.491868+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: audit 2023 2023-10-09T05:22:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: -10-09T05:22:18.524830+0000 mon.c 2023-10-09T05:22:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: (mon.2) 93 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: audit 2023-10-09T05:22:18.526633+0000 mon.c ( 2023-10-09T05:22:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: mon.2) 94 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:22:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: 05:22:18.527510+0000 mon.c (mon.2) 95 2023-10-09T05:22:19.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:19.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:22:19.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: 22:18.532884+0000 mon.a (mon.0) 676 2023-10-09T05:22:19.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:19 smithi191 conmon[101299]: : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:19.846 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/img /tmp/rbd_import_export_165117/img2 /tmp/rbd_import_export_165117/img3 2023-10-09T05:22:19.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[101053]: cluster 2023-10-09T05:22:17.918329+0000 mgr.y (mgr.14442) 103 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 129 KiB/s wr, 77 op/s 2023-10-09T05:22:19.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[101053]: audit 2023-10-09T05:22:18.486917+0000 mgr.y (mgr.14442) 104 : audit [DBG] from='client.24752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:22:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[101053]: cephadm 2023-10-09T05:22:18.487680+0000 mgr.y (mgr.14442) 105 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:22:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[101053]: audit 2023-10-09T05:22:18.491868+0000 mon.a (mon.0) 675 : audit 2023-10-09T05:22:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[101053]: [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[101053]: audit 2023-10-09T05:22:18.524830+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[101053]: audit 2023-10-09T05:22:18.526633+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[101053]: audit 2023-10-09T05:22:18.527510+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[101053]: audit 2023-10-09T05:22:18.532884+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[110782]: cluster 2023-10-09T05:22:17.918329+0000 mgr.y (mgr.14442) 103 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 129 KiB/s wr, 77 op/s 2023-10-09T05:22:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[110782]: audit 2023-10-09T05:22:18.486917+0000 mgr.y (mgr.14442) 104 : audit [DBG] from='client.24752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:22:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[110782]: cephadm 2023-10-09T05:22:18.487680+0000 mgr.y (mgr.14442) 105 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:22:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[110782]: audit 2023-10-09T05:22:18.491868+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[110782]: audit 2023-10-09T05:22:18.524830+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:22:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[110782]: audit 2023-10-09T05:22:18.526633+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:22:19.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[110782]: audit 2023-10-09T05:22:18.527510+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:22:19.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[110782]: audit 2023-10-09T05:22 2023-10-09T05:22:19.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:19 smithi008 conmon[110782]: :18.532884+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:22:20.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:20 smithi191 conmon[101299]: cephadm 2023-10-09T05:22:18.538884+0000 mgr.y (mgr.14442) 106 : cephadm [INF] 2023-10-09T05:22:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:20 smithi191 conmon[101299]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:22:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:20 smithi008 conmon[101053]: cephadm 2023-10-09T05:22:18.538884+0000 mgr.y (mgr.14442 2023-10-09T05:22:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:20 smithi008 conmon[101053]: ) 106 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:22:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:20 smithi008 conmon[110782]: cephadm 2023-10-09T05:22:18.538884+0000 mgr.y (mgr.14442) 2023-10-09T05:22:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:20 smithi008 conmon[110782]: 106 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:22:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:21 smithi191 conmon[101299]: cluster 2023-10-09T05:22:19.918674+0000 mgr.y (mgr.14442) 107 : cluster [DBG] 2023-10-09T05:22:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:21 smithi191 conmon[101299]: pgmap v84: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 129 KiB/s wr, 77 op/s 2023-10-09T05:22:21.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:21 smithi008 conmon[101053]: cluster 2023-10-09T05:22:19.918674+0000 2023-10-09T05:22:21.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:21 smithi008 conmon[101053]: mgr.y (mgr.14442) 107 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 129 KiB/s wr, 77 op/s 2023-10-09T05:22:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:21 smithi008 conmon[110782]: cluster 2023-10-09T05:22:19.918674+0000 mgr.y ( 2023-10-09T05:22:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:21 smithi008 conmon[110782]: mgr.14442) 107 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 129 KiB/s wr, 77 op/s 2023-10-09T05:22:23.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:23 smithi008 conmon[101053]: cluster 2023-10-09T05:22:21.919105+0000 mgr.y ( 2023-10-09T05:22:23.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:23 smithi008 conmon[101053]: mgr.14442) 108 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 218 KiB/s rd, 118 KiB/s wr, 70 op/s 2023-10-09T05:22:23.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:23 smithi008 conmon[101053]: cluster 2023-10-09T05:22:22.203127+0000 mon.a (mon 2023-10-09T05:22:23.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:23 smithi008 conmon[101053]: .0) 677 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T05:22:23.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:23 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:23] "GET /metrics HTTP/1.1" 200 191521 "" "Prometheus/2.33.4" 2023-10-09T05:22:23.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:23 smithi008 conmon[110782]: cluster 2023-10-09T05:22: 2023-10-09T05:22:23.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:23 smithi008 conmon[110782]: 21.919105+0000 mgr.y (mgr.14442) 108 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 218 KiB/s rd, 118 KiB/s wr, 70 op/s 2023-10-09T05:22:23.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:23 smithi008 conmon[110782]: cluster 2023-10-09T05:22:23.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:23 smithi008 conmon[110782]: 2023-10-09T05:22:22.203127 2023-10-09T05:22:23.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:23 smithi008 conmon[110782]: +0000 mon.a (mon.0) 677 2023-10-09T05:22:23.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:23 smithi008 conmon[110782]: : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T05:22:23.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:23 smithi191 conmon[101299]: cluster 2023-10-09T05:22:21.919105+0000 mgr.y (mgr.14442) 108 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 218 KiB/s rd, 118 KiB/s wr, 70 op/s 2023-10-09T05:22:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:23 smithi191 conmon[101299]: cluster 2023-10-09T05:22:22.203127+0000 mon.a (mon.0) 677 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T05:22:24.915 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd help export 2023-10-09T05:22:24.915 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -q export-format 2023-10-09T05:22:24.935 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_165117/img bs=1k count=10 seek=100 2023-10-09T05:22:24.936 INFO:tasks.workunit.client.0.smithi008.stderr:10+0 records in 2023-10-09T05:22:24.936 INFO:tasks.workunit.client.0.smithi008.stderr:10+0 records out 2023-10-09T05:22:24.936 INFO:tasks.workunit.client.0.smithi008.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.3663e-05 s, 139 MB/s 2023-10-09T05:22:24.936 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format /tmp/rbd_import_export_165117/img testimg 2023-10-09T05:22:24.997 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T05:22:24.997 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:22:24.999 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap create testimg@snap 2023-10-09T05:22:25.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:22:24 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:22:25.217 INFO:tasks.workunit.client.0.smithi008.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T05:22:25.223 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T05:22:25.272 INFO:tasks.workunit.client.0.smithi008.stderr:++ rbd image-meta list testimg 2023-10-09T05:22:25.311 INFO:tasks.workunit.client.0.smithi008.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-10-09T05:22:25.311 INFO:tasks.workunit.client.0.smithi008.stderr: 2023-10-09T05:22:25.311 INFO:tasks.workunit.client.0.smithi008.stderr:Key Value 2023-10-09T05:22:25.311 INFO:tasks.workunit.client.0.smithi008.stderr:key1 value1' 2023-10-09T05:22:25.311 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_165117/img_v2 2023-10-09T05:22:25.352 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:25.354 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_165117/img_v2 testimg_import 2023-10-09T05:22:25.386 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:22:25.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:25 smithi008 conmon[101053]: cluster 2023-10-09T05:22:23.919804+0000 mgr.y 2023-10-09T05:22:25.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:25 smithi008 conmon[101053]: (mgr.14442) 109 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 260 KiB/s rd, 107 KiB/s wr, 82 op/s 2023-10-09T05:22:25.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:25 smithi008 conmon[110782]: cluster 2023-10-09T05:22:23. 2023-10-09T05:22:25.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:25 smithi008 conmon[110782]: 919804+0000 mgr.y (mgr.14442) 109 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 260 KiB/s rd, 107 KiB/s wr, 82 op/s 2023-10-09T05:22:25.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:25 smithi191 conmon[101299]: cluster 2023-10-09T05:22:23.919804+0000 mgr.y (mgr.14442) 109 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 260 KiB/s rd, 107 KiB/s wr, 82 op/s 2023-10-09T05:22:26.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:26 smithi008 conmon[110782]: cluster 2023-10-09T05:22 2023-10-09T05:22:26.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:26 smithi008 conmon[110782]: :25.207485+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T05:22:26.233 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:22:26.236 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import 2023-10-09T05:22:26.270 INFO:tasks.workunit.client.0.smithi008.stdout:rbd image 'testimg_import': 2023-10-09T05:22:26.271 INFO:tasks.workunit.client.0.smithi008.stdout: size 110 KiB in 1 objects 2023-10-09T05:22:26.271 INFO:tasks.workunit.client.0.smithi008.stdout: order 22 (4 MiB objects) 2023-10-09T05:22:26.271 INFO:tasks.workunit.client.0.smithi008.stdout: snapshot_count: 1 2023-10-09T05:22:26.271 INFO:tasks.workunit.client.0.smithi008.stdout: id: 60da20413cde 2023-10-09T05:22:26.271 INFO:tasks.workunit.client.0.smithi008.stdout: block_name_prefix: rbd_data.60da20413cde 2023-10-09T05:22:26.271 INFO:tasks.workunit.client.0.smithi008.stdout: format: 2 2023-10-09T05:22:26.271 INFO:tasks.workunit.client.0.smithi008.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T05:22:26.271 INFO:tasks.workunit.client.0.smithi008.stdout: op_features: 2023-10-09T05:22:26.271 INFO:tasks.workunit.client.0.smithi008.stdout: flags: 2023-10-09T05:22:26.271 INFO:tasks.workunit.client.0.smithi008.stdout: create_timestamp: Mon Oct 9 05:22:25 2023 2023-10-09T05:22:26.271 INFO:tasks.workunit.client.0.smithi008.stdout: access_timestamp: Mon Oct 9 05:22:25 2023 2023-10-09T05:22:26.272 INFO:tasks.workunit.client.0.smithi008.stdout: modify_timestamp: Mon Oct 9 05:22:25 2023 2023-10-09T05:22:26.273 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import@snap 2023-10-09T05:22:26.313 INFO:tasks.workunit.client.0.smithi008.stdout:rbd image 'testimg_import': 2023-10-09T05:22:26.313 INFO:tasks.workunit.client.0.smithi008.stdout: size 110 KiB in 1 objects 2023-10-09T05:22:26.314 INFO:tasks.workunit.client.0.smithi008.stdout: order 22 (4 MiB objects) 2023-10-09T05:22:26.314 INFO:tasks.workunit.client.0.smithi008.stdout: snapshot_count: 1 2023-10-09T05:22:26.314 INFO:tasks.workunit.client.0.smithi008.stdout: id: 60da20413cde 2023-10-09T05:22:26.314 INFO:tasks.workunit.client.0.smithi008.stdout: block_name_prefix: rbd_data.60da20413cde 2023-10-09T05:22:26.314 INFO:tasks.workunit.client.0.smithi008.stdout: format: 2 2023-10-09T05:22:26.314 INFO:tasks.workunit.client.0.smithi008.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T05:22:26.314 INFO:tasks.workunit.client.0.smithi008.stdout: op_features: 2023-10-09T05:22:26.314 INFO:tasks.workunit.client.0.smithi008.stdout: flags: 2023-10-09T05:22:26.314 INFO:tasks.workunit.client.0.smithi008.stdout: create_timestamp: Mon Oct 9 05:22:25 2023 2023-10-09T05:22:26.314 INFO:tasks.workunit.client.0.smithi008.stdout: access_timestamp: Mon Oct 9 05:22:25 2023 2023-10-09T05:22:26.314 INFO:tasks.workunit.client.0.smithi008.stdout: modify_timestamp: Mon Oct 9 05:22:25 2023 2023-10-09T05:22:26.315 INFO:tasks.workunit.client.0.smithi008.stdout: protected: False 2023-10-09T05:22:26.316 INFO:tasks.workunit.client.0.smithi008.stderr:++ rbd image-meta list testimg_import 2023-10-09T05:22:26.358 INFO:tasks.workunit.client.0.smithi008.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-10-09T05:22:26.358 INFO:tasks.workunit.client.0.smithi008.stderr: 2023-10-09T05:22:26.358 INFO:tasks.workunit.client.0.smithi008.stderr:Key Value 2023-10-09T05:22:26.358 INFO:tasks.workunit.client.0.smithi008.stderr:key1 value1' 2023-10-09T05:22:26.358 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 'There is 1 metadatum on this image: 2023-10-09T05:22:26.358 INFO:tasks.workunit.client.0.smithi008.stderr: 2023-10-09T05:22:26.358 INFO:tasks.workunit.client.0.smithi008.stderr:Key Value 2023-10-09T05:22:26.358 INFO:tasks.workunit.client.0.smithi008.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-10-09T05:22:26.358 INFO:tasks.workunit.client.0.smithi008.stderr: 2023-10-09T05:22:26.358 INFO:tasks.workunit.client.0.smithi008.stderr:Key Value 2023-10-09T05:22:26.359 INFO:tasks.workunit.client.0.smithi008.stderr:key1 value1' ']' 2023-10-09T05:22:26.359 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg_import /tmp/rbd_import_export_165117/img_import 2023-10-09T05:22:26.402 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete...done. 2023-10-09T05:22:26.403 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165117/img /tmp/rbd_import_export_165117/img_import 2023-10-09T05:22:26.403 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_165117/img /tmp/rbd_import_export_165117/img_import 2023-10-09T05:22:26.404 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/img --format %b 2023-10-09T05:22:26.406 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=24 2023-10-09T05:22:26.406 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/img_import --format %b 2023-10-09T05:22:26.407 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=224 2023-10-09T05:22:26.407 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=200 2023-10-09T05:22:26.407 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=200 2023-10-09T05:22:26.407 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T05:22:26.407 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_165117/img_snap 2023-10-09T05:22:26.442 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete...done. 2023-10-09T05:22:26.445 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_165117/img_snap_import 2023-10-09T05:22:26.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:26 smithi008 conmon[101053]: cluster 2023-10-09T05:22:25. 2023-10-09T05:22:26.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:26 smithi008 conmon[101053]: 207485+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T05:22:26.482 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete...done. 2023-10-09T05:22:26.485 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165117/img_snap /tmp/rbd_import_export_165117/img_snap_import 2023-10-09T05:22:26.485 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_165117/img_snap /tmp/rbd_import_export_165117/img_snap_import 2023-10-09T05:22:26.486 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/img_snap --format %b 2023-10-09T05:22:26.487 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=224 2023-10-09T05:22:26.487 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/img_snap_import --format %b 2023-10-09T05:22:26.488 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=224 2023-10-09T05:22:26.488 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T05:22:26.488 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T05:22:26.488 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:22:26.489 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/img_v2 2023-10-09T05:22:26.489 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/img_import 2023-10-09T05:22:26.490 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/img_snap 2023-10-09T05:22:26.490 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/img_snap_import 2023-10-09T05:22:26.491 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T05:22:26.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:26 smithi191 conmon[101299]: cluster 2023-10-09T05:22:25.207485+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T05:22:27.229 INFO:tasks.workunit.client.0.smithi008.stderr: Removing snap: 100% complete...done. 2023-10-09T05:22:27.236 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg_import 2023-10-09T05:22:27.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:27 smithi008 conmon[101053]: cluster 2023-10-09T05:22:25 2023-10-09T05:22:27.279 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:27 smithi008 conmon[101053]: .920151+0000 mgr.y (mgr.14442) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 895 B/s wr, 23 op/s 2023-10-09T05:22:27.279 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:27 smithi008 conmon[101053]: cluster 2023-10-09T05:22:26.216597+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T05:22:27.279 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:27 smithi008 conmon[110782]: cluster 2023-10-09T05:22:25.920151+0000 mgr.y (mgr 2023-10-09T05:22:27.279 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:27 smithi008 conmon[110782]: .14442) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 895 B/s wr, 23 op/s 2023-10-09T05:22:27.279 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:27 smithi008 conmon[110782]: cluster 2023-10-09T05:22:26.216597+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T05:22:27.335 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T05:22:27.338 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap rm testimg@snap 2023-10-09T05:22:27.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:27 smithi191 conmon[101299]: cluster 2023-10-09T05:22:25.920151+0000 mgr.y (mgr.14442) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 895 B/s wr, 23 op/s 2023-10-09T05:22:27.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:27 smithi191 conmon[101299]: 2023-10-09T05:22:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:27 smithi191 conmon[101299]: cluster 2023-10-09T05:22:26.216597+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T05:22:27.737 INFO:teuthology.orchestra.run.smithi008.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:22:27.737 INFO:teuthology.orchestra.run.smithi008.stdout:alertmanager.a smithi008 *:9093,9094 running (76s) 56s ago 108s 15.6M - 0.23.0 ba2b418f427c 50083941a197 2023-10-09T05:22:27.737 INFO:teuthology.orchestra.run.smithi008.stdout:grafana.a smithi191 *:3000 running (89s) 53s ago 88s 52.7M - 8.3.5 dad864ee21e9 abc49ed66cfa 2023-10-09T05:22:27.737 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi008.zfejpl smithi008 running (63s) 56s ago 63s 14.0M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5d86a4e2d3b 2023-10-09T05:22:27.737 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi191.walafj smithi191 running (61s) 53s ago 60s 12.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 93130380f297 2023-10-09T05:22:27.738 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.x smithi191 *:8443 running (5m) 53s ago 5m 392M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 720487b0c8fc 2023-10-09T05:22:27.738 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.y smithi008 *:9283 running (7m) 56s ago 7m 446M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b54a8f2f116c 2023-10-09T05:22:27.738 INFO:teuthology.orchestra.run.smithi008.stdout:mon.a smithi008 running (7m) 56s ago 7m 104M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d155e1e1e35f 2023-10-09T05:22:27.738 INFO:teuthology.orchestra.run.smithi008.stdout:mon.b smithi191 running (6m) 53s ago 6m 102M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 2fa5874c9b2c 2023-10-09T05:22:27.738 INFO:teuthology.orchestra.run.smithi008.stdout:mon.c smithi008 running (5m) 56s ago 5m 101M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 bc3d5fa1338e 2023-10-09T05:22:27.738 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.a smithi008 *:9100 running (2m) 56s ago 2m 19.9M - 1.3.1 1dbe0e931976 ec044fd127f6 2023-10-09T05:22:27.738 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.b smithi191 *:9100 running (2m) 53s ago 2m 18.7M - 1.3.1 1dbe0e931976 715f54230f50 2023-10-09T05:22:27.738 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0 smithi008 running (5m) 56s ago 5m 47.5M 4037M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 70d3660d55a1 2023-10-09T05:22:27.738 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 smithi008 running (5m) 56s ago 5m 47.3M 4037M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 312729dec19a 2023-10-09T05:22:27.738 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2 smithi008 running (4m) 56s ago 4m 44.1M 4037M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7c1912182283 2023-10-09T05:22:27.738 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 smithi008 running (4m) 56s ago 4m 46.2M 4037M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9c33044d2771 2023-10-09T05:22:27.739 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 smithi191 running (3m) 53s ago 3m 47.8M 4292M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 57a8d325e472 2023-10-09T05:22:27.739 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 smithi191 running (3m) 53s ago 3m 44.7M 4292M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 141ad4d0ecc5 2023-10-09T05:22:27.739 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6 smithi191 running (3m) 53s ago 3m 45.7M 4292M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0039e7e7a199 2023-10-09T05:22:27.739 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7 smithi191 running (2m) 53s ago 2m 48.2M 4292M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d71410a3a7bd 2023-10-09T05:22:27.739 INFO:teuthology.orchestra.run.smithi008.stdout:prometheus.a smithi191 *:9095 running (71s) 53s ago 2m 44.2M - 2.33.4 514e6a882f6e 0e52be64285a 2023-10-09T05:22:28.234 INFO:tasks.workunit.client.0.smithi008.stderr: Removing snap: 100% complete...done. 2023-10-09T05:22:28.239 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm testimg 2023-10-09T05:22:28.336 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T05:22:28.338 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --order 20 /tmp/rbd_import_export_165117/img testimg 2023-10-09T05:22:28.396 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T05:22:28.396 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:22:28.398 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_165117/img_v2 2023-10-09T05:22:28.438 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:28.441 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_165117/img_v2 testimg_import 2023-10-09T05:22:28.500 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:22:28.503 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import 2023-10-09T05:22:28.503 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep order 2023-10-09T05:22:28.503 INFO:tasks.workunit.client.0.smithi008.stderr:+ awk '{print $2}' 2023-10-09T05:22:28.503 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 20 2023-10-09T05:22:28.539 INFO:tasks.workunit.client.0.smithi008.stdout:20 2023-10-09T05:22:28.539 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/img_v2 2023-10-09T05:22:28.540 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg_import 2023-10-09T05:22:28.547 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:28 smithi008 conmon[101053]: cluster 2023-10-09T05:22:27.220492 2023-10-09T05:22:28.547 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:28 smithi008 conmon[101053]: +0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T05:22:28.547 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:28 smithi008 conmon[110782]: cluster 2023-10-09T05:22:27.220492+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T05:22:28.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:28 smithi191 conmon[101299]: cluster 2023-10-09T05:22:27.220492+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T05:22:28.571 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T05:22:28.638 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T05:22:28.641 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg 2023-10-09T05:22:28.735 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T05:22:28.737 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_165117/img testimg 2023-10-09T05:22:28.788 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T05:22:28.792 INFO:tasks.workunit.client.0.smithi008.stderr:++ rbd info testimg 2023-10-09T05:22:28.792 INFO:tasks.workunit.client.0.smithi008.stderr:++ grep features 2023-10-09T05:22:28.834 INFO:tasks.workunit.client.0.smithi008.stderr:+ FEATURES_BEFORE=' features: layering 2023-10-09T05:22:28.834 INFO:tasks.workunit.client.0.smithi008.stderr: op_features: ' 2023-10-09T05:22:28.834 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_165117/img_v2 2023-10-09T05:22:28.873 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:28.876 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_165117/img_v2 testimg_import 2023-10-09T05:22:28.939 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:22:28.943 INFO:tasks.workunit.client.0.smithi008.stderr:++ rbd info testimg_import 2023-10-09T05:22:28.943 INFO:tasks.workunit.client.0.smithi008.stderr:++ grep features 2023-10-09T05:22:28.987 INFO:tasks.workunit.client.0.smithi008.stderr:+ FEATURES_AFTER=' features: layering 2023-10-09T05:22:28.989 INFO:tasks.workunit.client.0.smithi008.stderr: op_features: ' 2023-10-09T05:22:28.989 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' ' features: layering 2023-10-09T05:22:28.989 INFO:tasks.workunit.client.0.smithi008.stderr: op_features: ' '!=' ' features: layering 2023-10-09T05:22:28.989 INFO:tasks.workunit.client.0.smithi008.stderr: op_features: ' ']' 2023-10-09T05:22:28.989 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/img_v2 2023-10-09T05:22:28.989 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg_import 2023-10-09T05:22:29.057 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T05:22:29.059 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg 2023-10-09T05:22:29.134 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T05:22:29.137 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_165117/img testimg 2023-10-09T05:22:29.204 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T05:22:29.206 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_165117/img_v2 2023-10-09T05:22:29.248 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:29.251 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_165117/img_v2 testimg_import 2023-10-09T05:22:29.315 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:22:29.318 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import 2023-10-09T05:22:29.319 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -Ei '(4 KiB|4096)' 2023-10-09T05:22:29.319 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 'stripe unit' 2023-10-09T05:22:29.357 INFO:tasks.workunit.client.0.smithi008.stdout: stripe unit: 4 KiB 2023-10-09T05:22:29.358 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import 2023-10-09T05:22:29.358 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 'stripe count' 2023-10-09T05:22:29.358 INFO:tasks.workunit.client.0.smithi008.stderr:+ awk '{print $3}' 2023-10-09T05:22:29.358 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 1000 2023-10-09T05:22:29.400 INFO:tasks.workunit.client.0.smithi008.stdout:1000 2023-10-09T05:22:29.400 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/img_v2 2023-10-09T05:22:29.400 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg_import 2023-10-09T05:22:29.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:29 smithi008 conmon[101053]: audit 2023-10-09T05:22:27.722012+0000 mgr.y (mgr 2023-10-09T05:22:29.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:29 smithi008 conmon[101053]: .14442) 111 : audit [DBG] from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:22:29.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:29 smithi008 conmon[101053]: cluster 2023-10-09T05:22:27.920656+0000 mgr.y (mgr.14442) 112 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 230 KiB/s rd, 12 KiB/s wr, 184 op/s 2023-10-09T05:22:29.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:29 smithi008 conmon[101053]: cluster 2023-10-09T05:22:28.225667+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T05:22:29.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:29 smithi008 conmon[110782]: audit 2023-10-09T05:22:27. 2023-10-09T05:22:29.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:29 smithi008 conmon[110782]: 722012+0000 mgr.y (mgr.14442) 111 : audit [DBG] from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:22:29.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:29 smithi008 conmon[110782]: cluster 2023-10-09T05:22:27.920656+0000 mgr.y (mgr.14442) 112 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 230 KiB/s rd, 12 KiB/s wr, 184 op/s 2023-10-09T05:22:29.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:29 smithi008 conmon[110782]: cluster 2023-10-09T05:22:28.225667+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T05:22:29.500 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T05:22:29.503 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg 2023-10-09T05:22:29.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:29 smithi191 conmon[101299]: audit 2023-10-09T05:22:27.722012+0000 mgr.y (mgr.14442) 111 : audit [DBG] from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:22:29.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:29 smithi191 conmon[101299]: cluster 2023-10-09T05:22:27.920656+0000 mgr.y (mgr.14442) 112 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 230 KiB/s rd, 12 KiB/s wr, 184 op/s 2023-10-09T05:22:29.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:29 smithi191 conmon[101299]: cluster 2023-10-09T05:22:28.225667+0000 2023-10-09T05:22:29.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:29 smithi191 conmon[101299]: mon.a (mon.0) 681 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T05:22:29.595 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T05:22:29.598 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_165117/img testimg 2023-10-09T05:22:29.655 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T05:22:29.657 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap create testimg@snap1 2023-10-09T05:22:30.241 INFO:tasks.workunit.client.0.smithi008.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T05:22:30.247 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap create testimg@snap2 2023-10-09T05:22:30.681 INFO:teuthology.orchestra.run.smithi008.stdout:wait for minority of mons to upgrade 2023-10-09T05:22:31.246 INFO:tasks.workunit.client.0.smithi008.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T05:22:31.252 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap protect testimg@snap2 2023-10-09T05:22:31.296 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_165117/snap_protect 2023-10-09T05:22:31.338 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:31.340 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_165117/snap_protect testimg_import 2023-10-09T05:22:31.490 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:31 smithi008 conmon[101053]: cluster 2023-10-09T05:22:29.921041+0000 mgr.y (mgr.14442) 113 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 13 KiB/s wr, 185 op/s 2023-10-09T05:22:31.490 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:31 smithi008 conmon[101053]: cluster 2023-10-09T05:22:30.230449+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T05:22:31.490 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:31 smithi008 conmon[110782]: cluster 2023-10-09T05:22:29.921041+0000 mgr.y (mgr.14442) 113 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 13 KiB/s wr, 185 op/s 2023-10-09T05:22:31.491 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:31 smithi008 conmon[110782]: cluster 2023-10-09T05:22:30.230449+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T05:22:31.492 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T05:22:31.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:31 smithi191 conmon[101299]: cluster 2023-10-09T05:22:29.921041+0000 mgr.y (mgr.14442) 113 : cluster [DBG] 2023-10-09T05:22:31.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:31 smithi191 conmon[101299]: pgmap v94: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 13 KiB/s wr, 185 op/s 2023-10-09T05:22:31.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:31 smithi191 conmon[101299]: cluster 2023-10-09T05:22:30.230449+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T05:22:32.458 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:32 smithi008 conmon[101053]: cluster 2023-10-09T05:22:31.236283+0000 2023-10-09T05:22:32.458 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:32 smithi008 conmon[101053]: mon.a (mon.0) 683 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T05:22:32.458 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:32 smithi008 conmon[101053]: cluster 2023-10-09T05:22:32.204677+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T05:22:32.458 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:32 smithi008 conmon[110782]: cluster 2023-10-09T05:22:31 2023-10-09T05:22:32.458 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:32 smithi008 conmon[110782]: .236283+0000 mon.a (mon.0) 683 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T05:22:32.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:32 smithi008 conmon[110782]: cluster 2023-10-09T05:22:32.204677+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T05:22:32.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:32 smithi191 conmon[101299]: cluster 2023-10-09T05:22:31.236283+0000 mon.a (mon.0) 683 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T05:22:32.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:32 smithi191 conmon[101299]: cluster 2023-10-09T05:22:32.204677+0000 mon.a ( 2023-10-09T05:22:32.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:32 smithi191 conmon[101299]: mon.0) 684 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T05:22:33.232 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:22:33.235 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import@snap1 2023-10-09T05:22:33.235 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 'protected: False' 2023-10-09T05:22:33.278 INFO:tasks.workunit.client.0.smithi008.stdout: protected: False 2023-10-09T05:22:33.279 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import@snap2 2023-10-09T05:22:33.279 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 'protected: True' 2023-10-09T05:22:33.325 INFO:tasks.workunit.client.0.smithi008.stdout: protected: True 2023-10-09T05:22:33.326 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/snap_protect 2023-10-09T05:22:33.326 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T05:22:33.371 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T05:22:33.416 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap purge testimg 2023-10-09T05:22:33.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:33 smithi008 conmon[101053]: cluster 2023-10-09T05:22:31. 2023-10-09T05:22:33.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:33 smithi008 conmon[101053]: 921416+0000 mgr.y (mgr.14442) 114 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 13 KiB/s wr, 185 op/s 2023-10-09T05:22:33.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:33 smithi008 conmon[101053]: cluster 2023-10-09T05:22:33.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:33 smithi008 conmon[101053]: 2023-10-09T05:22:33.209790+0000 mon.a (mon.0) 685 : cluster 2023-10-09T05:22:33.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:33 smithi008 conmon[101053]: [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T05:22:33.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:33 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:33] "GET /metrics HTTP/1.1" 200 192096 "" "Prometheus/2.33.4" 2023-10-09T05:22:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:33 smithi008 conmon[110782]: cluster 2023-10-09T05:22:31.921416 2023-10-09T05:22:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:33 smithi008 conmon[110782]: +0000 mgr.y (mgr.14442) 114 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 13 KiB/s wr, 185 op/s 2023-10-09T05:22:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:33 smithi008 conmon[110782]: cluster 2023-10-09T05:22:33.209790 2023-10-09T05:22:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:33 smithi008 conmon[110782]: +0000 mon.a (mon.0) 685 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T05:22:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:33 smithi008 conmon[110782]: 2023-10-09T05:22:33.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:33 smithi191 conmon[101299]: cluster 2023-10-09T05:22:31.921416+0000 mgr.y (mgr.14442) 114 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 13 KiB/s wr, 185 op/s 2023-10-09T05:22:33.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:33 smithi191 conmon[101299]: cluster 2023-10-09T05:22:33.209790+0000 mon.a (mon.0) 685 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T05:22:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:34 smithi191 conmon[101299]: audit 2023-10-09T05:22:34.038683+0000 mon.a (mon.0) 686 : audit [DBG] from='client.? 172.21.15.8:0/1511435456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:34.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:34 smithi008 conmon[101053]: audit 2023-10-09T05:22:34.038683+0000 2023-10-09T05:22:34.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:34 smithi008 conmon[101053]: mon.a (mon.0) 686 : audit [DBG] from='client.? 172.21.15.8:0/1511435456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:34.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:34 smithi008 conmon[110782]: audit 2023-10-09T05:22:34.038683+0000 mon.a (mon.0) 686 : audit [DBG] from='client.? 172.21.15.8:0/1511435456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:35.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:22:34 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:22:35.256 INFO:tasks.workunit.client.0.smithi008.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T05:22:35.264 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap purge testimg_import 2023-10-09T05:22:35.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:35 smithi191 conmon[101299]: cluster 2023-10-09T05:22:33.921823+0000 mgr.y (mgr.14442) 115 : cluster 2023-10-09T05:22:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:35 smithi191 conmon[101299]: [DBG] pgmap v100: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-10-09T05:22:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:35 smithi191 conmon[101299]: cluster 2023-10-09T05:22:34.247065+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T05:22:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:35 smithi191 conmon[101299]: 2023-10-09T05:22:35.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:35 smithi008 conmon[101053]: cluster 2023-10-09T05:22:33.921823+0000 mgr.y ( 2023-10-09T05:22:35.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:35 smithi008 conmon[101053]: mgr.14442) 115 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-10-09T05:22:35.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:35 smithi008 conmon[101053]: cluster 2023-10-09T05:22:34.247065+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T05:22:35.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:35 smithi008 conmon[110782]: cluster 2023-10-09T05:22:33. 2023-10-09T05:22:35.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:35 smithi008 conmon[110782]: 921823+0000 mgr.y (mgr.14442) 115 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-10-09T05:22:35.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:35 smithi008 conmon[110782]: cluster 2023-10-09T05:22:34.247065+0000 mon.a 2023-10-09T05:22:35.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:35 smithi008 conmon[110782]: (mon.0) 687 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T05:22:36.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:36 smithi191 conmon[101299]: cluster 2023-10-09T05:22:35.251807+0000 mon.a (mon.0) 688 : cluster 2023-10-09T05:22:36.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:36 smithi191 conmon[101299]: [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T05:22:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:36 smithi008 conmon[101053]: cluster 2023-10-09T05:22:35.251807+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T05:22:36.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:36 smithi008 conmon[110782]: cluster 2023-10-09T05:22:35.251807+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T05:22:37.213 INFO:tasks.workunit.client.0.smithi008.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T05:22:37.223 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg 2023-10-09T05:22:37.329 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T05:22:37.332 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg_import 2023-10-09T05:22:37.419 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T05:22:37.422 INFO:tasks.workunit.client.0.smithi008.stderr:+ tiered=0 2023-10-09T05:22:37.422 INFO:tasks.workunit.client.0.smithi008.stderr:+ ceph osd dump 2023-10-09T05:22:37.423 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep '^pool' 2023-10-09T05:22:37.423 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep ''\''rbd'\''' 2023-10-09T05:22:37.423 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep tier 2023-10-09T05:22:37.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:37 smithi191 conmon[101299]: cluster 2023-10-09T05:22:35.922171+0000 mgr.y (mgr.14442) 116 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-10-09T05:22:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:37 smithi191 conmon[101299]: 2023-10-09T05:22:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:37 smithi191 conmon[101299]: cluster 2023-10-09T05:22:36.257623+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T05:22:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:37 smithi191 conmon[101299]: audit 2023-10-09T05:22:36.483816+0000 mon.c (mon.2) 96 : audit [DBG] from='client.? 172.21.15.8:0/703086042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:37 smithi191 conmon[101299]: cluster 2023-10-09T05:22:37 2023-10-09T05:22:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:37 smithi191 conmon[101299]: .206154+0000 mon.a (mon.0) 690 : cluster 2023-10-09T05:22:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:37 smithi191 conmon[101299]: [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T05:22:37.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[101053]: cluster 2023-10-09T05:22:35.922171+0000 mgr.y (mgr.14442 2023-10-09T05:22:37.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[101053]: ) 116 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-10-09T05:22:37.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[101053]: cluster 2023-10-09T05:22:36.257623+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T05:22:37.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[101053]: audit 2023-10-09T05:22:36.483816+0000 mon.c ( 2023-10-09T05:22:37.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[101053]: mon.2) 96 : audit [DBG] from='client.? 172.21.15.8:0/703086042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:37.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[101053]: cluster 2023-10- 2023-10-09T05:22:37.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[101053]: 09T05:22:37.206154+0000 mon.a ( 2023-10-09T05:22:37.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[101053]: mon.0) 690 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T05:22:37.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: cluster 2023-10-09T05 2023-10-09T05:22:37.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: :22:35.922171+0000 mgr.y (mgr.14442) 116 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-10-09T05:22:37.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: cluster 2023-10- 2023-10-09T05:22:37.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: 09T05:22:36.257623+0000 mon.a (mon.0) 2023-10-09T05:22:37.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: 689 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T05:22:37.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: audit 2023-10-09T05:22:37.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: 2023-10-09T05:22:36. 2023-10-09T05:22:37.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: 483816+0000 mon.c (mon.2 2023-10-09T05:22:37.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: ) 96 : audit [DBG] from='client.? 172.21.15.8:0/703086042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:37.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: cluster 2023-10 2023-10-09T05:22:37.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: -09T05:22:37.206154+0000 mon.a ( 2023-10-09T05:22:37.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:37 smithi008 conmon[110782]: mon.0) 690 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T05:22:37.908 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_165117/sparse1 2023-10-09T05:22:37.914 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records in 2023-10-09T05:22:37.915 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records out 2023-10-09T05:22:37.915 INFO:tasks.workunit.client.0.smithi008.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00502287 s, 209 MB/s 2023-10-09T05:22:37.915 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_165117/sparse2 2023-10-09T05:22:37.920 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records in 2023-10-09T05:22:37.920 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records out 2023-10-09T05:22:37.920 INFO:tasks.workunit.client.0.smithi008.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0050192 s, 209 MB/s 2023-10-09T05:22:37.920 INFO:tasks.workunit.client.0.smithi008.stderr:+ truncate /tmp/rbd_import_export_165117/sparse2 -s 2M 2023-10-09T05:22:37.939 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse1 2023-10-09T05:22:37.983 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 0% complete...failed. 2023-10-09T05:22:37.983 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T05:22:37.986 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T05:22:37.986 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_165117/sparse1 2023-10-09T05:22:38.050 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T05:22:38.051 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:22:38.051 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:22:38.053 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd ls -l 2023-10-09T05:22:38.054 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep sparse1 2023-10-09T05:22:38.054 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T05:22:38.101 INFO:tasks.workunit.client.0.smithi008.stdout:sparse1 2 MiB 2 2023-10-09T05:22:38.101 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse1 2023-10-09T05:22:38.102 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse1 2023-10-09T05:22:38.102 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse1 2023-10-09T05:22:38.102 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T05:22:38.102 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T05:22:38.141 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.61fd6b16dfad 2023-10-09T05:22:38.142 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.61fd6b16dfad 2023-10-09T05:22:38.142 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse1 2023-10-09T05:22:38.142 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse1 2023-10-09T05:22:38.143 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T05:22:38.143 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.61fd6b16dfad\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:22:38.143 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:22:38.143 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:22:38.144 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse1 2023-10-09T05:22:38.182 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T05:22:38.182 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T05:22:38.182 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T05:22:38.182 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T05:22:38.182 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T05:22:38.214 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects=1 2023-10-09T05:22:38.214 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 1 2023-10-09T05:22:38.215 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T05:22:38.215 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export sparse1 /tmp/rbd_import_export_165117/sparse1.out 2023-10-09T05:22:38.256 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:38.260 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165117/sparse1 /tmp/rbd_import_export_165117/sparse1.out 2023-10-09T05:22:38.261 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_165117/sparse1 /tmp/rbd_import_export_165117/sparse1.out 2023-10-09T05:22:38.263 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/sparse1 --format %b 2023-10-09T05:22:38.264 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=2048 2023-10-09T05:22:38.264 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/sparse1.out --format %b 2023-10-09T05:22:38.266 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=2048 2023-10-09T05:22:38.266 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T05:22:38.266 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T05:22:38.266 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:22:38.266 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/sparse1.out 2023-10-09T05:22:38.267 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse1 2023-10-09T05:22:38.356 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T05:22:38.359 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse2 2023-10-09T05:22:38.397 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 0% complete...failed. 2023-10-09T05:22:38.397 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T05:22:38.400 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T05:22:38.400 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_165117/sparse2 2023-10-09T05:22:38.476 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T05:22:38.477 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:22:38.477 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:22:38.480 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd ls -l 2023-10-09T05:22:38.481 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep sparse2 2023-10-09T05:22:38.481 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T05:22:38.528 INFO:tasks.workunit.client.0.smithi008.stdout:sparse2 2 MiB 2 2023-10-09T05:22:38.529 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse2 2023-10-09T05:22:38.529 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse2 2023-10-09T05:22:38.529 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse2 2023-10-09T05:22:38.529 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T05:22:38.529 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T05:22:38.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:38 smithi191 conmon[101299]: audit 2023-10-09T05:22:37.763541+0000 mon.c (mon.2) 97 : audit 2023-10-09T05:22:38.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:38 smithi191 conmon[101299]: [DBG] from='client.? 172.21.15.8:0/2683845015' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:22:38.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:38 smithi191 conmon[101299]: audit 2023-10-09T05:22:37.893128+0000 mon.c (mon.2) 98 : audit [DBG] from='client.? 172.21.15.8:0/2683845015' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:22:38.572 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.3b8818e29cf6 2023-10-09T05:22:38.573 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.3b8818e29cf6 2023-10-09T05:22:38.573 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse2 2023-10-09T05:22:38.573 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.3b8818e29cf6\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:22:38.573 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse2 2023-10-09T05:22:38.573 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T05:22:38.574 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse2 2023-10-09T05:22:38.574 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:22:38.574 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:22:38.623 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T05:22:38.624 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T05:22:38.624 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T05:22:38.624 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T05:22:38.624 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T05:22:38.657 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects=0 2023-10-09T05:22:38.657 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 0 2023-10-09T05:22:38.658 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T05:22:38.658 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export sparse2 /tmp/rbd_import_export_165117/sparse2.out 2023-10-09T05:22:38.696 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:38.701 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165117/sparse2 /tmp/rbd_import_export_165117/sparse2.out 2023-10-09T05:22:38.701 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_165117/sparse2 /tmp/rbd_import_export_165117/sparse2.out 2023-10-09T05:22:38.703 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/sparse2 --format %b 2023-10-09T05:22:38.704 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=2048 2023-10-09T05:22:38.704 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/sparse2.out --format %b 2023-10-09T05:22:38.705 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=2048 2023-10-09T05:22:38.705 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T05:22:38.705 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T05:22:38.705 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:22:38.705 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/sparse2.out 2023-10-09T05:22:38.706 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse2 2023-10-09T05:22:38.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:38 smithi008 conmon[101053]: audit 2023-10-09T05:22:37.763541+0000 mon.c (mon.2) 97 : audit [DBG] from='client.? 172.21.15.8:0/2683845015' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:22:38.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:38 smithi008 conmon[101053]: audit 2023-10-09T05:22:37.893128+0000 mon.c (mon.2) 98 : audit [DBG] from='client.? 172.21.15.8:0/2683845015' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:22:38.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:38 smithi008 conmon[110782]: audit 2023-10-09T05:22:37.763541+0000 mon.c (mon.2) 2023-10-09T05:22:38.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:38 smithi008 conmon[110782]: 97 : audit [DBG] from='client.? 172.21.15.8:0/2683845015' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:22:38.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:38 smithi008 conmon[110782]: audit 2023-10-09T05:22:37.893128+0000 mon.c (mon.2) 98 : audit [DBG] from='client.? 172.21.15.8:0/2683845015' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:22:38.802 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T05:22:38.805 INFO:tasks.workunit.client.0.smithi008.stderr:+ truncate /tmp/rbd_import_export_165117/sparse1 -s 10M 2023-10-09T05:22:38.805 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T05:22:38.900 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T05:22:38.903 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:22:38.903 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:22:38.906 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd ls -l 2023-10-09T05:22:38.906 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep sparse1 2023-10-09T05:22:38.907 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-10-09T05:22:38.943 INFO:tasks.workunit.client.0.smithi008.stdout:sparse1 10 MiB 2 2023-10-09T05:22:38.944 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse1 2023-10-09T05:22:38.944 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse1 2023-10-09T05:22:38.944 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse1 2023-10-09T05:22:38.944 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T05:22:38.944 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T05:22:38.981 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.3bb892473b73 2023-10-09T05:22:38.982 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.3bb892473b73 2023-10-09T05:22:38.982 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse1 2023-10-09T05:22:38.982 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse1 2023-10-09T05:22:38.982 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.3bb892473b73\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:22:38.982 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T05:22:38.984 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse1 2023-10-09T05:22:38.984 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:22:38.984 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:22:39.022 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T05:22:39.022 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T05:22:39.023 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T05:22:39.023 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T05:22:39.023 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T05:22:39.058 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects=1 2023-10-09T05:22:39.058 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 1 2023-10-09T05:22:39.058 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T05:22:39.058 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export sparse1 /tmp/rbd_import_export_165117/sparse1.out 2023-10-09T05:22:39.098 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:39.102 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165117/sparse1 /tmp/rbd_import_export_165117/sparse1.out 2023-10-09T05:22:39.102 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_165117/sparse1 /tmp/rbd_import_export_165117/sparse1.out 2023-10-09T05:22:39.110 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/sparse1 --format %b 2023-10-09T05:22:39.111 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=2048 2023-10-09T05:22:39.111 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/sparse1.out --format %b 2023-10-09T05:22:39.112 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=2048 2023-10-09T05:22:39.112 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T05:22:39.112 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T05:22:39.112 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:22:39.113 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/sparse1.out 2023-10-09T05:22:39.113 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse1 2023-10-09T05:22:39.206 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T05:22:39.208 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_165117/sparse2 oflag=append conv=notrunc 2023-10-09T05:22:39.221 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records in 2023-10-09T05:22:39.221 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records out 2023-10-09T05:22:39.221 INFO:tasks.workunit.client.0.smithi008.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0115654 s, 181 MB/s 2023-10-09T05:22:39.221 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T05:22:39.304 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T05:22:39.304 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:22:39.304 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:22:39.307 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd ls -l 2023-10-09T05:22:39.307 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep sparse2 2023-10-09T05:22:39.307 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-10-09T05:22:39.344 INFO:tasks.workunit.client.0.smithi008.stdout:sparse2 4 MiB 2 2023-10-09T05:22:39.345 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse2 2023-10-09T05:22:39.345 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse2 2023-10-09T05:22:39.345 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse2 2023-10-09T05:22:39.345 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T05:22:39.346 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T05:22:39.383 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.3be2196e6da1 2023-10-09T05:22:39.384 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.3be2196e6da1 2023-10-09T05:22:39.384 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.3be2196e6da1\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:22:39.384 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse2 2023-10-09T05:22:39.384 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse2 2023-10-09T05:22:39.384 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T05:22:39.387 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse2 2023-10-09T05:22:39.387 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:22:39.387 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:22:39.422 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T05:22:39.423 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T05:22:39.423 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T05:22:39.423 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T05:22:39.423 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T05:22:39.455 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects='0 2023-10-09T05:22:39.455 INFO:tasks.workunit.client.0.smithi008.stderr:2 2023-10-09T05:22:39.456 INFO:tasks.workunit.client.0.smithi008.stderr:3' 2023-10-09T05:22:39.456 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 0 2 3 2023-10-09T05:22:39.456 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-10-09T05:22:39.456 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export sparse2 /tmp/rbd_import_export_165117/sparse2.out 2023-10-09T05:22:39.496 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T05:22:39.499 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165117/sparse2 /tmp/rbd_import_export_165117/sparse2.out 2023-10-09T05:22:39.499 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_165117/sparse2 /tmp/rbd_import_export_165117/sparse2.out 2023-10-09T05:22:39.503 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/sparse2 --format %b 2023-10-09T05:22:39.504 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=6144 2023-10-09T05:22:39.504 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/sparse2.out --format %b 2023-10-09T05:22:39.505 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=6144 2023-10-09T05:22:39.505 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T05:22:39.505 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T05:22:39.505 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:22:39.506 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/sparse2.out 2023-10-09T05:22:39.506 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse2 2023-10-09T05:22:39.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:39 smithi191 conmon[101299]: cluster 2023-10-09T05:22:37.922688+0000 2023-10-09T05:22:39.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:39 smithi191 conmon[101299]: mgr.y (mgr.14442) 117 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 150 KiB/s rd, 10 KiB/s wr, 203 op/s 2023-10-09T05:22:39.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:39 smithi191 conmon[101299]: audit 2023-10-09T05:22:39.024962+0000 mon.a (mon.0) 691 : 2023-10-09T05:22:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:39 smithi191 conmon[101299]: audit [DBG] from='client.? 172.21.15.8:0/979933952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:39.606 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T05:22:39.609 INFO:tasks.workunit.client.0.smithi008.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-10-09T05:22:39.609 INFO:tasks.workunit.client.0.smithi008.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T05:22:39.610 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_165117/sparse1 sparse 2023-10-09T05:22:39.672 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T05:22:39.673 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:22:39.673 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:22:39.675 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse 2023-10-09T05:22:39.676 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse 2023-10-09T05:22:39.676 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse 2023-10-09T05:22:39.676 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T05:22:39.676 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T05:22:39.714 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.3c0c839a46b3 2023-10-09T05:22:39.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:39 smithi008 conmon[101053]: cluster 2023-10-09T05:22:37.922688 2023-10-09T05:22:39.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:39 smithi008 conmon[101053]: +0000 mgr.y (mgr.14442) 117 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 150 KiB/s rd, 10 KiB/s wr, 203 op/s 2023-10-09T05:22:39.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:39 smithi008 conmon[101053]: audit 2023-10-09T05:22:39.024962+0000 2023-10-09T05:22:39.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:39 smithi008 conmon[101053]: mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.8:0/979933952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:39.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:39 smithi008 conmon[110782]: cluster 2023-10-09T05:22:37.922688+0000 mgr.y ( 2023-10-09T05:22:39.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:39 smithi008 conmon[110782]: mgr.14442) 117 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 150 KiB/s rd, 10 KiB/s wr, 203 op/s 2023-10-09T05:22:39.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:39 smithi008 conmon[110782]: audit 2023-10-09T05:22:39.024962+0000 2023-10-09T05:22:39.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:39 smithi008 conmon[110782]: mon.a (mon.0) 691 : audit [DBG] from='client.? 172.21.15.8:0/979933952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:39.718 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.3c0c839a46b3 2023-10-09T05:22:39.719 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse 2023-10-09T05:22:39.719 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.3c0c839a46b3\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:22:39.719 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse 2023-10-09T05:22:39.719 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T05:22:39.719 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse 2023-10-09T05:22:39.719 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:22:39.719 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:22:39.762 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T05:22:39.762 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T05:22:39.762 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T05:22:39.762 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T05:22:39.763 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T05:22:39.803 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects=1 2023-10-09T05:22:39.803 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 1 2023-10-09T05:22:39.803 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T05:22:39.803 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse 2023-10-09T05:22:39.904 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T05:22:39.907 INFO:tasks.workunit.client.0.smithi008.stdout:zeros import through stdin to sparse image 2023-10-09T05:22:39.908 INFO:tasks.workunit.client.0.smithi008.stderr:+ echo 'zeros import through stdin to sparse image' 2023-10-09T05:22:39.908 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T05:22:39.908 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format - sparse 2023-10-09T05:22:39.958 INFO:tasks.workunit.client.0.smithi008.stderr:4+0 records in 2023-10-09T05:22:39.958 INFO:tasks.workunit.client.0.smithi008.stderr:4+0 records out 2023-10-09T05:22:39.958 INFO:tasks.workunit.client.0.smithi008.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0493055 s, 85.1 MB/s 2023-10-09T05:22:39.969 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T05:22:39.969 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:22:39.972 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse 2023-10-09T05:22:39.972 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse 2023-10-09T05:22:39.973 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse 2023-10-09T05:22:39.973 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T05:22:39.973 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T05:22:40.011 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.3c2d5eb7029d 2023-10-09T05:22:40.012 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.3c2d5eb7029d 2023-10-09T05:22:40.012 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.3c2d5eb7029d\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:22:40.012 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse 2023-10-09T05:22:40.012 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse 2023-10-09T05:22:40.012 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T05:22:40.013 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse 2023-10-09T05:22:40.013 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:22:40.013 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:22:40.050 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T05:22:40.050 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T05:22:40.050 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T05:22:40.050 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T05:22:40.050 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T05:22:40.088 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects= 2023-10-09T05:22:40.088 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 2023-10-09T05:22:40.088 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-10-09T05:22:40.088 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse 2023-10-09T05:22:40.177 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T05:22:40.180 INFO:tasks.workunit.client.0.smithi008.stderr:+ echo 'zeros export to sparse file' 2023-10-09T05:22:40.180 INFO:tasks.workunit.client.0.smithi008.stdout:zeros export to sparse file 2023-10-09T05:22:40.181 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T05:22:40.219 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:22:40.221 INFO:tasks.workunit.client.0.smithi008.stderr:++ rbd info sparse 2023-10-09T05:22:40.221 INFO:tasks.workunit.client.0.smithi008.stderr:++ grep block_name_prefix 2023-10-09T05:22:40.221 INFO:tasks.workunit.client.0.smithi008.stderr:++ awk '{print $NF;}' 2023-10-09T05:22:40.259 INFO:tasks.workunit.client.0.smithi008.stderr:+ prefix=rbd_data.3c4e9a5327f3 2023-10-09T05:22:40.259 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T05:22:40.260 INFO:tasks.workunit.client.0.smithi008.stderr:++ get_image_data_pool sparse 2023-10-09T05:22:40.260 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse 2023-10-09T05:22:40.260 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse 2023-10-09T05:22:40.260 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep 'data_pool: ' 2023-10-09T05:22:40.260 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk -F: '{ print $NF }' 2023-10-09T05:22:40.299 INFO:tasks.workunit.client.0.smithi008.stderr:++ data_pool= 2023-10-09T05:22:40.299 INFO:tasks.workunit.client.0.smithi008.stderr:++ '[' -z ']' 2023-10-09T05:22:40.299 INFO:tasks.workunit.client.0.smithi008.stderr:++ data_pool=rbd 2023-10-09T05:22:40.299 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo rbd 2023-10-09T05:22:40.300 INFO:tasks.workunit.client.0.smithi008.stderr:+ rados -p rbd put rbd_data.3c4e9a5327f3.000000000000 - 2023-10-09T05:22:40.329 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records in 2023-10-09T05:22:40.330 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records out 2023-10-09T05:22:40.330 INFO:tasks.workunit.client.0.smithi008.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0685123 s, 61.2 MB/s 2023-10-09T05:22:40.353 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse 2023-10-09T05:22:40.353 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse 2023-10-09T05:22:40.353 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse 2023-10-09T05:22:40.353 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T05:22:40.353 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T05:22:40.391 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.3c4e9a5327f3 2023-10-09T05:22:40.392 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.3c4e9a5327f3 2023-10-09T05:22:40.392 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.3c4e9a5327f3\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:22:40.392 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse 2023-10-09T05:22:40.392 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse 2023-10-09T05:22:40.392 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T05:22:40.394 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse 2023-10-09T05:22:40.394 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:22:40.394 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:22:40.430 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T05:22:40.430 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T05:22:40.430 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T05:22:40.430 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T05:22:40.430 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T05:22:40.463 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects=0 2023-10-09T05:22:40.463 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 0 2023-10-09T05:22:40.463 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T05:22:40.463 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/sparse 2023-10-09T05:22:40.464 INFO:tasks.workunit.client.0.smithi008.stderr:rm: cannot remove '/tmp/rbd_import_export_165117/sparse': No such file or directory 2023-10-09T05:22:40.464 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T05:22:40.464 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export sparse /tmp/rbd_import_export_165117/sparse 2023-10-09T05:22:40.500 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete...done. 2023-10-09T05:22:40.504 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_165117/sparse --format=%b 2023-10-09T05:22:40.505 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 = 0 ']' 2023-10-09T05:22:40.505 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse 2023-10-09T05:22:40.592 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T05:22:40.595 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_165117/sparse /tmp/rbd_import_export_165117/sparse1 /tmp/rbd_import_export_165117/sparse2 /tmp/rbd_import_export_165117/sparse3 2023-10-09T05:22:40.597 INFO:tasks.workunit.client.0.smithi008.stderr:rm: cannot remove '/tmp/rbd_import_export_165117/sparse3': No such file or directory 2023-10-09T05:22:40.597 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T05:22:40.597 INFO:tasks.workunit.client.0.smithi008.stdout:OK 2023-10-09T05:22:40.598 INFO:tasks.workunit.client.0.smithi008.stderr:+ echo OK 2023-10-09T05:22:40.599 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm -rf /tmp/rbd_import_export_165117 2023-10-09T05:22:40.599 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T05:22:40.599 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T05:22:40.632 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T05:22:40.632 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:22:40.906 DEBUG:teuthology.parallel:result is None 2023-10-09T05:22:40.906 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:22:40.936 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:22:40.936 DEBUG:teuthology.orchestra.run.smithi008:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:22:40.993 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:22:40.994 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T05:22:40.994 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T05:22:40.994 DEBUG:teuthology.parallel:result is None 2023-10-09T05:22:41.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:41 smithi191 conmon[101299]: cluster 2023-10-09T05:22:39.923029+0000 mgr.y (mgr.14442) 118 : cluster [DBG] 2023-10-09T05:22:41.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:41 smithi191 conmon[101299]: pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 7.4 KiB/s wr, 143 op/s 2023-10-09T05:22:41.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:41 smithi008 conmon[101053]: cluster 2023-10-09T05:22:39.923029 2023-10-09T05:22:41.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:41 smithi008 conmon[101053]: +0000 mgr.y (mgr.14442) 118 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 7.4 KiB/s wr, 143 op/s 2023-10-09T05:22:41.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:41 smithi008 conmon[110782]: cluster 2023-10-09T05:22:39.923029+0000 mgr.y 2023-10-09T05:22:41.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:41 smithi008 conmon[110782]: (mgr.14442) 118 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 7.4 KiB/s wr, 143 op/s 2023-10-09T05:22:42.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:42 smithi191 conmon[101299]: audit 2023-10-09T05:22:41.492308+0000 mon.c (mon.2) 99 : audit [DBG] from='client.? 172.21.15.8:0/1501103165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:42 smithi191 conmon[101299]: cluster 2023-10-09T05:22:42.207062+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T05:22:42.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:42 smithi008 conmon[101053]: audit 2023-10-09T05:22: 2023-10-09T05:22:42.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:42 smithi008 conmon[101053]: 41.492308+0000 mon.c (mon.2) 99 : audit [DBG] from='client.? 172.21.15.8:0/1501103165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:42.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:42 smithi008 conmon[101053]: cluster 2023-10-09T05: 2023-10-09T05:22:42.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:42 smithi008 conmon[101053]: 22:42.207062+0000 mon.a (mon.0) 692 : cluster [DBG] 2023-10-09T05:22:42.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:42 smithi008 conmon[101053]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T05:22:42.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:42 smithi008 conmon[110782]: audit 2023-10-09T05:22:41. 2023-10-09T05:22:42.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:42 smithi008 conmon[110782]: 492308+0000 mon.c (mon.2) 99 : audit [DBG] from='client.? 172.21.15.8:0/1501103165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:42.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:42 smithi008 conmon[110782]: cluster 2023-10 2023-10-09T05:22:42.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:42 smithi008 conmon[110782]: -09T05:22:42.207062+0000 mon.a ( 2023-10-09T05:22:42.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:42 smithi008 conmon[110782]: mon.0) 692 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T05:22:43.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:43 smithi008 conmon[110782]: cluster 2023-10-09T05: 2023-10-09T05:22:43.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:43 smithi008 conmon[110782]: 22:41.923376+0000 mgr.y (mgr.14442) 119 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s rd, 6.3 KiB/s wr, 121 op/s 2023-10-09T05:22:43.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:43 smithi008 conmon[101053]: cluster 2023-10-09T05:22:41.923376+0000 2023-10-09T05:22:43.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:43 smithi008 conmon[101053]: mgr.y (mgr.14442) 119 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s rd, 6.3 KiB/s wr, 121 op/s 2023-10-09T05:22:43.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:43 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:43] "GET /metrics HTTP/1.1" 200 192096 "" "Prometheus/2.33.4" 2023-10-09T05:22:43.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:43 smithi191 conmon[101299]: cluster 2023-10-09T05:22:41.923376+0000 mgr.y (mgr.14442) 119 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s rd, 6.3 KiB/s wr, 121 op/s 2023-10-09T05:22:44.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:44 smithi191 conmon[101299]: audit 2023-10-09T05:22:43.868278+0000 mon.c (mon.2) 100 : audit [DBG] from='client.? 172.21.15.8:0/3421609111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:44 smithi008 conmon[101053]: audit 2023-10-09T05:22 2023-10-09T05:22:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:44 smithi008 conmon[101053]: :43.868278+0000 mon.c (mon.2) 100 : audit [DBG] from='client.? 172.21.15.8:0/3421609111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:44 smithi008 conmon[110782]: audit 2023-10-09T05:22:43. 2023-10-09T05:22:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:44 smithi008 conmon[110782]: 868278+0000 mon.c (mon.2) 100 : audit [DBG] from='client.? 172.21.15.8:0/3421609111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:45.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:22:44 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:22:45.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:45 smithi191 conmon[101299]: cluster 2023-10-09T05:22:43.923882+0000 mgr.y (mgr.14442) 120 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 403 op/s 2023-10-09T05:22:45.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:45 smithi008 conmon[101053]: cluster 2023-10-09T05:22:43.923882+0000 mgr.y (mgr. 2023-10-09T05:22:45.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:45 smithi008 conmon[101053]: 14442) 120 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 403 op/s 2023-10-09T05:22:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:45 smithi008 conmon[110782]: cluster 2023-10-09T05:22:43.923882 2023-10-09T05:22:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:45 smithi008 conmon[110782]: +0000 mgr.y (mgr.14442) 120 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 403 op/s 2023-10-09T05:22:47.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:47 smithi191 conmon[101299]: cluster 2023-10-09T05:22:45.924243+0000 mgr.y (mgr.14442) 121 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 970 KiB/s rd, 1.3 MiB/s wr, 354 op/s 2023-10-09T05:22:47.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:47 smithi191 conmon[101299]: audit 2023-10-09T05:22:46.346328+0000 mon.c ( 2023-10-09T05:22:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:47 smithi191 conmon[101299]: mon.2) 101 : audit [DBG] from='client.? 172.21.15.8:0/3343877129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:47.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:47 smithi008 conmon[101053]: cluster 2023-10-09T05:22:45.924243 2023-10-09T05:22:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:47 smithi008 conmon[101053]: +0000 mgr.y (mgr.14442) 121 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 970 KiB/s rd, 1.3 MiB/s wr, 354 op/s 2023-10-09T05:22:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:47 smithi008 conmon[101053]: audit 2023-10-09T05:22:46.346328+0000 2023-10-09T05:22:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:47 smithi008 conmon[101053]: mon.c (mon.2) 101 : audit [DBG] from='client.? 172.21.15.8:0/3343877129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:47.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:47 smithi008 conmon[110782]: cluster 2023-10-09T05:22:45.924243+0000 2023-10-09T05:22:47.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:47 smithi008 conmon[110782]: mgr.y (mgr.14442) 121 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 970 KiB/s rd, 1.3 MiB/s wr, 354 op/s 2023-10-09T05:22:47.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:47 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:22:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:47 smithi008 conmon[110782]: :22:46.346328+0000 mon.c (mon.2) 101 : audit [DBG] from='client.? 172.21.15.8:0/3343877129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:49 smithi191 conmon[101299]: cluster 2023-10-09T05:22:47.924737+0000 mgr.y (mgr.14442) 122 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-10-09T05:22:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:49 smithi191 conmon[101299]: audit 2023-10-09T05:22:48.760234+0000 mon.a (mon.0) 693 : audit [DBG] from='client.? 172.21.15.8:0/1282543951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:49.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[101053]: cluster 2023-10 2023-10-09T05:22:49.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[101053]: -09T05:22:47.924737+0000 mgr.y (mgr.14442) 122 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-10-09T05:22:49.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[101053]: audit 2023-10-09T05:22:49.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[101053]: 2023-10-09T05:22:48 2023-10-09T05:22:49.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[101053]: .760234+0000 mon.a (mon.0) 693 : audit 2023-10-09T05:22:49.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[101053]: [DBG] 2023-10-09T05:22:49.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[101053]: from='client.? 172.21.15.8:0/1282543951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:49.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[110782]: cluster 2023-10-09T05:22: 2023-10-09T05:22:49.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[110782]: 47.924737+0000 mgr.y (mgr.14442) 122 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-10-09T05:22:49.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[110782]: audit 2023-10-09T05:22:49.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[110782]: 2023-10-09T05:22:48.760234 2023-10-09T05:22:49.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[110782]: +0000 mon.a (mon.0) 2023-10-09T05:22:49.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:49 smithi008 conmon[110782]: 693 : audit [DBG] from='client.? 172.21.15.8:0/1282543951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:51 smithi191 conmon[101299]: cluster 2023-10-09T05:22:49.925057+0000 mgr.y (mgr.14442) 123 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-10-09T05:22:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:51 smithi191 conmon[101299]: 2023-10-09T05:22:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:51 smithi191 conmon[101299]: audit 2023-10-09T05:22:51.192554+0000 mon.a (mon.0) 694 : audit [DBG] from='client.? 172.21.15.8:0/1577988412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:51.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:51 smithi008 conmon[101053]: cluster 2023-10-09T05:22:49.925057+0000 mgr.y (mgr.14442) 123 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-10-09T05:22:51.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:51 smithi008 conmon[101053]: audit 2023-10-09T05:22:51.192554+0000 mon.a (mon.0) 694 : audit [DBG] from='client.? 172.21.15.8:0/1577988412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:51.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:51 smithi008 conmon[110782]: cluster 2023-10-09T05:22:49.925057+0000 mgr.y (mgr.14442) 123 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-10-09T05:22:51.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:51 smithi008 conmon[110782]: audit 2023-10-09T05:22:51.192554+0000 mon.a (mon.0) 694 : audit [DBG] from='client.? 172.21.15.8:0/1577988412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:53.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:53 smithi008 conmon[101053]: cluster 2023-10-09T05:22:51.925413+0000 2023-10-09T05:22:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:53 smithi008 conmon[101053]: mgr.y (mgr.14442) 124 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-10-09T05:22:53.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:22:53 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:53] "GET /metrics HTTP/1.1" 200 192185 "" "Prometheus/2.33.4" 2023-10-09T05:22:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:53 smithi008 conmon[110782]: cluster 2023-10-09T05:22:51.925413+0000 mgr.y 2023-10-09T05:22:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:53 smithi008 conmon[110782]: (mgr.14442) 124 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-10-09T05:22:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:53 smithi191 conmon[101299]: cluster 2023-10-09T05:22:51.925413+0000 mgr.y (mgr.14442) 124 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-10-09T05:22:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:54 smithi191 conmon[101299]: audit 2023-10-09T05:22:53.555797+0000 mon.a (mon.0) 695 : audit [DBG] from='client.? 172.21.15.8:0/177815497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:54.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:54 smithi008 conmon[101053]: audit 2023-10-09T05:22:53.555797+0000 mon.a (mon.0 2023-10-09T05:22:54.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:54 smithi008 conmon[101053]: ) 695 : audit [DBG] from='client.? 172.21.15.8:0/177815497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:54 smithi008 conmon[110782]: audit 2023-10-09T05:22:53.555797+0000 mon.a 2023-10-09T05:22:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:54 smithi008 conmon[110782]: (mon.0) 695 : audit [DBG] from='client.? 172.21.15.8:0/177815497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:55.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:22:54 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:22:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:22:55.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:55 smithi191 conmon[101299]: cluster 2023-10-09T05:22:53.926008+0000 mgr.y (mgr.14442) 125 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 670 KiB/s rd, 969 KiB/s wr, 194 op/s 2023-10-09T05:22:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:55 smithi008 conmon[101053]: cluster 2023-10-09T05:22:53.926008 2023-10-09T05:22:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:55 smithi008 conmon[101053]: +0000 mgr.y (mgr.14442) 125 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 670 KiB/s rd, 969 KiB/s wr, 194 op/s 2023-10-09T05:22:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:55 smithi008 conmon[110782]: cluster 2023-10-09T05:22:53.926008+0000 2023-10-09T05:22:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:55 smithi008 conmon[110782]: mgr.y (mgr.14442) 125 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 670 KiB/s rd, 969 KiB/s wr, 194 op/s 2023-10-09T05:22:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:56 smithi008 conmon[101053]: audit 2023-10-09T05:22:55.926133+0000 mon.a (mon.0 2023-10-09T05:22:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:56 smithi008 conmon[101053]: ) 696 : audit [DBG] from='client.? 172.21.15.8:0/76024434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:56 smithi008 conmon[110782]: audit 2023-10-09T05:22:55.926133+0000 mon.a (mon.0) 696 : audit 2023-10-09T05:22:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:56 smithi008 conmon[110782]: [DBG] from='client.? 172.21.15.8:0/76024434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:56.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:56 smithi191 conmon[101299]: audit 2023-10-09T05:22:55.926133+0000 mon.a (mon.0) 696 : audit [DBG] from='client.? 172.21.15.8:0/76024434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:57.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:57 smithi008 conmon[101053]: cluster 2023-10-09T05:22:55.926362+0000 mgr.y (mgr.14442) 126 2023-10-09T05:22:57.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:57 smithi008 conmon[101053]: : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:57.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:57 smithi008 conmon[110782]: cluster 2023-10-09T05:22:55.926362+0000 mgr.y ( 2023-10-09T05:22:57.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:57 smithi008 conmon[110782]: mgr.14442) 126 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:57 smithi191 conmon[101299]: cluster 2023-10-09T05:22:55.926362+0000 mgr.y (mgr.14442) 126 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:58.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:58 smithi008 conmon[101053]: audit 2023-10-09T05:22:58. 2023-10-09T05:22:58.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:58 smithi008 conmon[101053]: 294326+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.8:0/1784872584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:58.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:58 smithi008 conmon[110782]: audit 2023-10-09T05:22:58.294326+0000 mon.c (mon.2 2023-10-09T05:22:58.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:58 smithi008 conmon[110782]: ) 102 : audit [DBG] from='client.? 172.21.15.8:0/1784872584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:58 smithi191 conmon[101299]: audit 2023-10-09T05:22:58.294326+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.8:0/1784872584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:22:59.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:59 smithi008 conmon[101053]: cluster 2023-10-09T05:22:57.926905+0000 mgr.y (mgr.14442 2023-10-09T05:22:59.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:22:59 smithi008 conmon[101053]: ) 127 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:59.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:59 smithi008 conmon[110782]: cluster 2023-10-09T05:22:57.926905+0000 mgr.y (mgr.14442) 127 : cluster 2023-10-09T05:22:59.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:22:59 smithi008 conmon[110782]: [DBG] pgmap v117: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:22:59.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:59 smithi191 conmon[101299]: cluster 2023-10-09T05:22:57.926905+0000 mgr.y (mgr.14442) 127 : 2023-10-09T05:22:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:22:59 smithi191 conmon[101299]: cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:01.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:01 smithi008 conmon[101053]: cluster 2023-10-09T05:22:59.927401+0000 2023-10-09T05:23:01.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:01 smithi008 conmon[101053]: mgr.y (mgr.14442) 128 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:01 smithi008 conmon[101053]: audit 2023-10-09T05:23:00.663809+0000 mon.a ( 2023-10-09T05:23:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:01 smithi008 conmon[101053]: mon.0) 697 : audit [DBG] from='client.? 172.21.15.8:0/485670483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:01 smithi008 conmon[110782]: cluster 2023-10-09T05:22:59.927401+0000 mgr.y (mgr.14442) 128 : 2023-10-09T05:23:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:01 smithi008 conmon[110782]: cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:01 smithi008 conmon[110782]: audit 2023-10-09T05:23:00.663809+0000 mon.a (mon.0) 697 : audit 2023-10-09T05:23:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:01 smithi008 conmon[110782]: [DBG] from='client.? 172.21.15.8:0/485670483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:01.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:01 smithi191 conmon[101299]: cluster 2023-10-09T05:22:59.927401+0000 mgr.y (mgr.14442) 128 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:01 smithi191 conmon[101299]: audit 2023-10-09T05:23:00.663809+0000 mon.a (mon.0) 697 : audit [DBG] from='client.? 172.21.15.8:0/485670483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:03.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:23:03] "GET /metrics HTTP/1.1" 200 192182 "" "Prometheus/2.33.4" 2023-10-09T05:23:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: cluster 2023-10-09T05:23:01.927868+0000 2023-10-09T05:23:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: mgr.y (mgr.14442) 129 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: audit 2023-10-09T05:23:02. 2023-10-09T05:23:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: 409971+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: cephadm 2023-10-09T 2023-10-09T05:23:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: 05:23:02.410585+0000 mgr.y (mgr.14442) 2023-10-09T05:23:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: 130 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T05:23:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: cephadm 2023-10 2023-10-09T05:23:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: -09T05:23:02.410671+0000 mgr.y (mgr 2023-10-09T05:23:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: .14442) 131 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T05:23:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: audit 2023 2023-10-09T05:23:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: -10-09T05:23:02.411655+0000 mon.c ( 2023-10-09T05:23:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: mon.2) 103 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:23:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: 05:23:02.416988+0000 mon.a (mon.0) 2023-10-09T05:23:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: 699 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: cephadm 2023-10 2023-10-09T05:23:03.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: -09T05:23:02.417845+0000 mgr.y (mgr.14442 2023-10-09T05:23:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: ) 132 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T05:23:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: audit 2023-10-09T05:23:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: 2023-10-09T05:23: 2023-10-09T05:23:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: 03.036246+0000 mon.c (mon.2) 104 : 2023-10-09T05:23:03.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[101053]: audit [DBG] from='client.? 172.21.15.8:0/1161380804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:03.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: cluster 2023-10-09T05:23:01.927868+0000 mgr.y 2023-10-09T05:23:03.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: (mgr.14442) 129 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:03.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:23:03.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: 09T05:23:02.409971+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:03.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: cephadm 2023-10-09T05:23:03.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: 2023-10-09T05:23:02.410585+0000 mgr.y 2023-10-09T05:23:03.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: (mgr.14442) 130 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T05:23:03.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: cephadm 2023 2023-10-09T05:23:03.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: -10-09T05:23:02.410671+0000 mgr.y 2023-10-09T05:23:03.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: (mgr.14442) 131 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T05:23:03.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: audit 2023-10-09T05:23:03.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: 2023-10-09T05:23:02.411655+0000 mon.c 2023-10-09T05:23:03.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: (mon.2) 103 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:03.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:23:03.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: 10-09T05:23:02.416988+0000 mon.a (mon 2023-10-09T05:23:03.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: .0) 699 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:03.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: cephadm 2023-10-09T05:23:03.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: 2023-10-09T05:23:02.417845+0000 mgr.y 2023-10-09T05:23:03.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: (mgr.14442) 132 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T05:23:03.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: audit 2023-10-09T05:23:03.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: 2023-10-09T05:23:03.036246+0000 mon.c 2023-10-09T05:23:03.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:03 smithi008 conmon[110782]: (mon.2) 104 : audit [DBG] from='client.? 172.21.15.8:0/1161380804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:03.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: cluster 2023-10-09T05:23:01.927868+0000 mgr.y ( 2023-10-09T05:23:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: mgr.14442) 129 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: audit 2023-10-09T05:23:02.409971+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:02.410585+0000 mgr.y (mgr.14442) 130 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T05:23:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:02.410671+0000 mgr.y (mgr.14442) 131 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T05:23:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: audit 2023-10-09T05:23:02.411655+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:23:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: :23:02.416988+0000 mon.a (mon.0) 699 2023-10-09T05:23:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: cephadm 2023-10-09T05: 2023-10-09T05:23:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: 23:02.417845+0000 mgr.y (mgr.14442) 2023-10-09T05:23:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: 132 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T05:23:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:23:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: T05:23:03.036246+0000 mon.c (mon. 2023-10-09T05:23:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:03 smithi191 conmon[101299]: 2) 104 : audit [DBG] from='client.? 172.21.15.8:0/1161380804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:04.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:04 smithi008 conmon[101053]: cephadm 2023-10-09T05:23: 2023-10-09T05:23:04.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:04 smithi008 conmon[101053]: 02.715654+0000 mgr.y (mgr.14442) 133 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi191 2023-10-09T05:23:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:04 smithi008 conmon[110782]: cephadm 2023-10-09T05:23:02.715654+0000 mgr.y ( 2023-10-09T05:23:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:04 smithi008 conmon[110782]: mgr.14442) 133 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi191 2023-10-09T05:23:04.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:04 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:23:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:23:04.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:04 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:02.715654+0000 mgr.y ( 2023-10-09T05:23:04.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:04 smithi191 conmon[101299]: mgr.14442) 133 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi191 2023-10-09T05:23:05.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:05 smithi008 conmon[101053]: cluster 2023-10-09T05:23:03.928766+0000 mgr.y ( 2023-10-09T05:23:05.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:05 smithi008 conmon[101053]: mgr.14442) 134 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:05.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:05 smithi008 conmon[101053]: audit 2023-10-09T05:23:05.404965+0000 mon.a (mon.0) 700 : audit 2023-10-09T05:23:05.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:05 smithi008 conmon[101053]: [DBG] from='client.? 172.21.15.8:0/2801245069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:05 smithi008 conmon[110782]: cluster 2023-10-09T05:23:03. 2023-10-09T05:23:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:05 smithi008 conmon[110782]: 928766+0000 mgr.y (mgr.14442) 134 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:05 smithi008 conmon[110782]: audit 2023-10-09T05:23 2023-10-09T05:23:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:05 smithi008 conmon[110782]: :05.404965+0000 mon.a (mon.0) 700 2023-10-09T05:23:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:05 smithi008 conmon[110782]: : audit [DBG] from='client.? 172.21.15.8:0/2801245069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:05.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:05 smithi191 conmon[101299]: cluster 2023-10-09T05:23:03.928766+0000 mgr.y (mgr.14442) 134 : 2023-10-09T05:23:05.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:05 smithi191 conmon[101299]: cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:05.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:05 smithi191 conmon[101299]: audit 2023-10-09T05:23:05.404965+0000 mon.a (mon.0) 700 : audit [DBG] from='client.? 172.21.15.8:0/2801245069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:07.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:07 smithi008 conmon[101053]: cluster 2023-10-09T05:23:05.929256+0000 mgr.y 2023-10-09T05:23:07.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:07 smithi008 conmon[101053]: (mgr.14442) 135 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:07.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:07 smithi008 conmon[110782]: cluster 2023-10-09T05:23:05.929256+0000 mgr.y ( 2023-10-09T05:23:07.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:07 smithi008 conmon[110782]: mgr.14442) 135 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:07.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:07 smithi191 conmon[101299]: cluster 2023-10-09T05:23:05.929256+0000 mgr.y (mgr.14442) 135 : cluster 2023-10-09T05:23:07.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:07 smithi191 conmon[101299]: [DBG] pgmap v121: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:08.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:08 smithi008 conmon[101053]: audit 2023-10-09T05:23:07.772124+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.8:0/2335200088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:08.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:08 smithi008 conmon[101053]: 2023-10-09T05:23:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:08 smithi008 conmon[110782]: audit 2023-10-09T05:23:07.772124+0000 2023-10-09T05:23:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:08 smithi008 conmon[110782]: mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.8:0/2335200088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:08.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:08 smithi191 conmon[101299]: audit 2023-10-09T05:23:07.772124+0000 mon.c (mon. 2023-10-09T05:23:08.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:08 smithi191 conmon[101299]: 2) 105 : audit [DBG] from='client.? 172.21.15.8:0/2335200088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:09 smithi008 conmon[101053]: cluster 2023-10-09T05:23:07.929718+0000 mgr.y (mgr.14442) 2023-10-09T05:23:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:09 smithi008 conmon[101053]: 136 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:09 smithi008 conmon[110782]: cluster 2023-10-09T05:23:07.929718+0000 mgr.y 2023-10-09T05:23:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:09 smithi008 conmon[110782]: (mgr.14442) 136 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:09.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:09 smithi191 conmon[101299]: cluster 2023-10-09T05:23:07.929718 2023-10-09T05:23:09.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:09 smithi191 conmon[101299]: +0000 mgr.y (mgr.14442) 136 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:10.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:10 smithi008 conmon[101053]: audit 2023-10-09T05:23:10.138049+0000 mon.a (mon 2023-10-09T05:23:10.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:10 smithi008 conmon[101053]: .0) 701 : audit [DBG] from='client.? 172.21.15.8:0/3255891114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:10.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:10 smithi008 conmon[110782]: audit 2023-10-09T05:23:10.138049 2023-10-09T05:23:10.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:10 smithi008 conmon[110782]: +0000 mon.a (mon.0) 701 : audit [DBG] from='client.? 172.21.15.8:0/3255891114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:10.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:10 smithi191 conmon[101299]: audit 2023-10-09T05:23:10.138049+0000 mon.a (mon.0) 2023-10-09T05:23:10.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:10 smithi191 conmon[101299]: 701 : audit [DBG] from='client.? 172.21.15.8:0/3255891114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:11 smithi008 conmon[101053]: cluster 2023-10-09T05:23:09.930054+0000 mgr.y ( 2023-10-09T05:23:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:11 smithi008 conmon[101053]: mgr.14442) 137 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:11 smithi008 conmon[110782]: cluster 2023-10-09T05:23:09.930054+0000 mgr.y 2023-10-09T05:23:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:11 smithi008 conmon[110782]: (mgr.14442) 137 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:11 smithi191 conmon[101299]: cluster 2023-10-09T05:23:09.930054+0000 mgr.y (mgr.14442) 2023-10-09T05:23:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:11 smithi191 conmon[101299]: 137 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:13.432 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:13 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:23:13] "GET /metrics HTTP/1.1" 200 192182 "" "Prometheus/2.33.4" 2023-10-09T05:23:13.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:13 smithi008 conmon[101053]: cluster 2023-10-09T05:23:11.930473+0000 mgr.y (mgr.14442) 2023-10-09T05:23:13.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:13 smithi008 conmon[101053]: 138 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:13.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:13 smithi008 conmon[101053]: audit 2023-10-09T05:23:12.508775+0000 mon.c (mon.2) 106 : audit 2023-10-09T05:23:13.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:13 smithi008 conmon[101053]: [DBG] from='client.? 172.21.15.8:0/2563023454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:13 smithi008 conmon[110782]: cluster 2023-10-09T05:23:11.930473+0000 mgr.y (mgr.14442) 138 : 2023-10-09T05:23:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:13 smithi008 conmon[110782]: cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:13 smithi008 conmon[110782]: audit 2023-10-09T05:23:12.508775+0000 mon.c (mon 2023-10-09T05:23:13.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:13 smithi008 conmon[110782]: .2) 106 : audit [DBG] from='client.? 172.21.15.8:0/2563023454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:13.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:13 smithi191 conmon[101299]: cluster 2023-10-09T05:23:11.930473+0000 mgr.y (mgr.14442) 2023-10-09T05:23:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:13 smithi191 conmon[101299]: 138 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:13 smithi191 conmon[101299]: audit 2023-10-09T05:23:12.508775+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.8:0/2563023454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:15.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:14 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:23:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:23:15.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:15 smithi008 conmon[101053]: cluster 2023-10-09T05:23:13.931050+0000 mgr.y (mgr 2023-10-09T05:23:15.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:15 smithi008 conmon[101053]: .14442) 139 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:15.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:15 smithi008 conmon[101053]: audit 2023-10-09T05:23:14.875377+0000 mon.a (mon.0) 702 : audit [DBG] 2023-10-09T05:23:15.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:15 smithi008 conmon[101053]: from='client.? 172.21.15.8:0/3296188870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:15.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:15 smithi008 conmon[110782]: cluster 2023-10-09T05:23:13.931050+0000 2023-10-09T05:23:15.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:15 smithi008 conmon[110782]: mgr.y (mgr.14442) 139 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:15.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:15 smithi008 conmon[110782]: audit 2023-10-09T05:23:15.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:15 smithi008 conmon[110782]: 2023-10-09T05:23:14.875377+0000 mon.a (mon.0) 2023-10-09T05:23:15.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:15 smithi008 conmon[110782]: 702 : audit [DBG] from='client.? 172.21.15.8:0/3296188870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:15.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:15 smithi191 conmon[101299]: cluster 2023-10-09T05:23:13.931050+0000 mgr.y ( 2023-10-09T05:23:15.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:15 smithi191 conmon[101299]: mgr.14442) 139 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:15.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:15 smithi191 conmon[101299]: audit 2023-10-09T05:23:14.875377+0000 mon.a (mon.0) 702 : audit [DBG] from='client.? 172.21.15.8:0/3296188870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:17.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:17 smithi008 conmon[101053]: cluster 2023-10-09T05:23 2023-10-09T05:23:17.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:17 smithi008 conmon[101053]: :15.931608+0000 mgr.y (mgr.14442) 140 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:17.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:17 smithi008 conmon[101053]: audit 2023-10-09T05:23:17.245712 2023-10-09T05:23:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:17 smithi008 conmon[101053]: +0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.8:0/2665508280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:17.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:17 smithi008 conmon[110782]: cluster 2023-10-09T05:23:15. 2023-10-09T05:23:17.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:17 smithi008 conmon[110782]: 931608+0000 mgr.y (mgr.14442) 140 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:17.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:17 smithi008 conmon[110782]: audit 2023-10-09T05:23: 2023-10-09T05:23:17.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:17 smithi008 conmon[110782]: 17.245712+0000 mon.c (mon.2) 107 : audit 2023-10-09T05:23:17.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:17 smithi008 conmon[110782]: [DBG] from='client.? 172.21.15.8:0/2665508280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:17.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:17 smithi191 conmon[101299]: cluster 2023-10-09T05:23:15.931608+0000 mgr.y (mgr.14442) 2023-10-09T05:23:17.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:17 smithi191 conmon[101299]: 140 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:17 smithi191 conmon[101299]: audit 2023-10-09T05:23:17.245712+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.8:0/2665508280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:19 smithi008 conmon[101053]: cluster 2023-10-09T05:23:17. 2023-10-09T05:23:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:19 smithi008 conmon[101053]: 932301+0000 mgr.y (mgr.14442) 141 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:19 smithi008 conmon[110782]: cluster 2023-10-09T05: 2023-10-09T05:23:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:19 smithi008 conmon[110782]: 23:17.932301+0000 mgr.y (mgr.14442) 141 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:19.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:19 smithi191 conmon[101299]: cluster 2023-10-09T05:23:17.932301+0000 mgr.y (mgr.14442) 2023-10-09T05:23:19.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:19 smithi191 conmon[101299]: 141 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:20.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:20 smithi008 conmon[101053]: audit 2023-10-09T05:23:19.615507+0000 mon.a (mon.0 2023-10-09T05:23:20.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:20 smithi008 conmon[101053]: ) 703 : audit [DBG] from='client.? 172.21.15.8:0/3478304750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:20.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:20 smithi008 conmon[110782]: audit 2023-10-09T05:23: 2023-10-09T05:23:20.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:20 smithi008 conmon[110782]: 19.615507+0000 mon.a (mon.0) 703 : audit [DBG] from='client.? 172.21.15.8:0/3478304750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:20.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:20 smithi191 conmon[101299]: audit 2023-10-09T05:23:19.615507+0000 mon.a (mon.0 2023-10-09T05:23:20.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:20 smithi191 conmon[101299]: ) 703 : audit [DBG] from='client.? 172.21.15.8:0/3478304750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:21.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:21 smithi008 conmon[101053]: cluster 2023-10-09T05:23:19.932663+0000 mgr.y ( 2023-10-09T05:23:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:21 smithi008 conmon[101053]: mgr.14442) 142 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:21 smithi008 conmon[110782]: cluster 2023-10-09T05:23:19.932663+0000 2023-10-09T05:23:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:21 smithi008 conmon[110782]: mgr.y (mgr.14442) 142 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:21 smithi191 conmon[101299]: cluster 2023-10-09T05:23:19.932663+0000 2023-10-09T05:23:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:21 smithi191 conmon[101299]: mgr.y (mgr.14442) 142 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:22.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:22 smithi008 conmon[101053]: audit 2023-10-09T05:23:21.987627+0000 mon.a (mon. 2023-10-09T05:23:22.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:22 smithi008 conmon[101053]: 0) 704 : audit [DBG] from='client.? 172.21.15.8:0/3293255403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:22 smithi008 conmon[110782]: audit 2023-10-09T05:23:21.987627+0000 2023-10-09T05:23:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:22 smithi008 conmon[110782]: mon.a (mon.0) 704 : audit [DBG] from='client.? 172.21.15.8:0/3293255403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:22.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:22 smithi191 conmon[101299]: audit 2023-10-09T05:23:21.987627+0000 mon.a ( 2023-10-09T05:23:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:22 smithi191 conmon[101299]: mon.0) 704 : audit [DBG] from='client.? 172.21.15.8:0/3293255403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:23.454 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:23 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:23:23] "GET /metrics HTTP/1.1" 200 192184 "" "Prometheus/2.33.4" 2023-10-09T05:23:23.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:23 smithi008 conmon[110782]: cluster 2023-10-09T05:23:21.932991+0000 mgr.y (mgr.14442 2023-10-09T05:23:23.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:23 smithi008 conmon[110782]: ) 143 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:23 smithi008 conmon[101053]: cluster 2023-10-09T05:23:21.932991+0000 mgr.y ( 2023-10-09T05:23:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:23 smithi008 conmon[101053]: mgr.14442) 143 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:23.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:23 smithi191 conmon[101299]: cluster 2023-10-09T05:23:21.932991+0000 mgr.y (mgr.14442) 2023-10-09T05:23:23.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:23 smithi191 conmon[101299]: 143 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:24.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:24 smithi008 conmon[101053]: audit 2023-10-09T05:23:24.356506+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.8:0/700031461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:24.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:24 smithi008 conmon[110782]: audit 2023-10-09T05:23 2023-10-09T05:23:24.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:24 smithi008 conmon[110782]: :24.356506+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.8:0/700031461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:24.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:24 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:23:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:23:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:24 smithi191 conmon[101299]: audit 2023-10-09T05:23:24. 2023-10-09T05:23:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:24 smithi191 conmon[101299]: 356506+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.8:0/700031461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:25.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:25 smithi008 conmon[101053]: cluster 2023-10-09T05:23:23.933628+0000 mgr.y (mgr. 2023-10-09T05:23:25.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:25 smithi008 conmon[101053]: 14442) 144 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:25 smithi008 conmon[110782]: cluster 2023-10-09T05:23:23.933628+0000 mgr.y (mgr.14442 2023-10-09T05:23:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:25 smithi008 conmon[110782]: ) 144 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:25.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:25 smithi191 conmon[101299]: cluster 2023-10-09T05:23:23.933628+0000 mgr.y 2023-10-09T05:23:25.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:25 smithi191 conmon[101299]: (mgr.14442) 144 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:27.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:27 smithi008 conmon[101053]: cluster 2023-10-09T05:23:25.934092 2023-10-09T05:23:27.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:27 smithi008 conmon[101053]: +0000 mgr.y (mgr.14442) 145 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:27.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:27 smithi008 conmon[101053]: audit 2023-10-09T05:23:26.727794+0000 mon.a (mon.0) 706 : audit [DBG] from='client.? 172.21.15.8:0/1353034914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:27.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:27 smithi008 conmon[110782]: cluster 2023-10-09T05:23:25.934092+0000 mgr.y (mgr.14442 2023-10-09T05:23:27.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:27 smithi008 conmon[110782]: ) 145 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:27.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:27 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:23:27.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:27 smithi008 conmon[110782]: 23:26.727794+0000 mon.a (mon.0) 706 : audit [DBG] from='client.? 172.21.15.8:0/1353034914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:27 smithi191 conmon[101299]: cluster 2023-10-09T05:23:25.934092+0000 mgr.y ( 2023-10-09T05:23:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:27 smithi191 conmon[101299]: mgr.14442) 145 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:27 smithi191 conmon[101299]: audit 2023-10-09T05:23:26.727794+0000 mon.a (mon.0) 706 : audit [DBG] from='client.? 172.21.15.8:0/1353034914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:29 smithi191 conmon[101299]: cluster 2023-10-09T05:23:27.934833+0000 mgr.y (mgr.14442) 146 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:29 smithi191 conmon[101299]: audit 2023-10-09T05:23:29. 2023-10-09T05:23:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:29 smithi191 conmon[101299]: 096237+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.8:0/2530991423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:29 smithi008 conmon[101053]: cluster 2023-10-09T05:23:27.934833+0000 mgr.y ( 2023-10-09T05:23:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:29 smithi008 conmon[101053]: mgr.14442) 146 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:29.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:29 smithi008 conmon[101053]: audit 2023-10-09T05:23:29.096237+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.8:0/2530991423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:29 smithi008 conmon[110782]: cluster 2023-10-09T05:23:27.934833+0000 mgr.y (mgr 2023-10-09T05:23:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:29 smithi008 conmon[110782]: .14442) 146 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:29 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:23:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:29 smithi008 conmon[110782]: -09T05:23:29.096237+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.8:0/2530991423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:31.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:31 smithi191 conmon[101299]: cluster 2023-10-09T05:23:29.935331+0000 mgr.y (mgr.14442) 147 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:31.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:31 smithi191 conmon[101299]: audit 2023-10-09T05:23:31.465943+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.8:0/3265191715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:31 smithi008 conmon[101053]: cluster 2023-10-09T05:23:29.935331+0000 mgr.y (mgr.14442 2023-10-09T05:23:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:31 smithi008 conmon[101053]: ) 147 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:31 smithi008 conmon[101053]: audit 2023-10-09T05:23:31.465943+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.8:0/3265191715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:31 smithi008 conmon[110782]: cluster 2023-10-09T05:23 2023-10-09T05:23:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:31 smithi008 conmon[110782]: :29.935331+0000 mgr.y (mgr.14442) 147 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:31 smithi008 conmon[110782]: audit 2023-10-09T05:23:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:31 smithi008 conmon[110782]: 2023-10-09T05:23:31.465943 2023-10-09T05:23:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:31 smithi008 conmon[110782]: +0000 mon.c (mon.2) 109 : audit 2023-10-09T05:23:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:31 smithi008 conmon[110782]: [DBG] from='client.? 172.21.15.8:0/3265191715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:33.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:33 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:23:33] "GET /metrics HTTP/1.1" 200 192181 "" "Prometheus/2.33.4" 2023-10-09T05:23:33.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:33 smithi191 conmon[101299]: cluster 2023-10-09T05:23:31.935806+0000 mgr.y (mgr.14442) 148 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:33.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:33 smithi008 conmon[101053]: cluster 2023-10-09T05:23:31. 2023-10-09T05:23:33.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:33 smithi008 conmon[101053]: 935806+0000 mgr.y (mgr.14442) 148 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:33 smithi008 conmon[110782]: cluster 2023-10-09T05:23:31.935806+0000 mgr.y (mgr. 2023-10-09T05:23:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:33 smithi008 conmon[110782]: 14442) 148 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:34.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:34 smithi191 conmon[103609]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:23:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:23:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:34 smithi191 conmon[101299]: audit 2023-10-09T05:23:33.833243+0000 mon.c (mon.2) 110 : audit 2023-10-09T05:23:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:34 smithi191 conmon[101299]: [DBG] from='client.? 172.21.15.8:0/3267034831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:34 smithi008 conmon[101053]: audit 2023-10-09T05:23:33.833243+0000 2023-10-09T05:23:34.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:34 smithi008 conmon[101053]: mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.8:0/3267034831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:34.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:34 smithi008 conmon[110782]: audit 2023-10-09T05:23: 2023-10-09T05:23:34.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:34 smithi008 conmon[110782]: 33.833243+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.8:0/3267034831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:35.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:35 smithi191 conmon[101299]: cluster 2023-10-09T05:23:33.936556+0000 mgr.y (mgr.14442) 149 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:35.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:35 smithi008 conmon[101053]: cluster 2023-10-09T05:23:33.936556+0000 mgr.y (mgr 2023-10-09T05:23:35.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:35 smithi008 conmon[101053]: .14442) 149 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:35.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:35 smithi008 conmon[110782]: cluster 2023-10-09T05:23:33.936556+0000 mgr.y (mgr.14442) 149 : cluster [DBG] 2023-10-09T05:23:35.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:35 smithi008 conmon[110782]: pgmap v135: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:36 smithi191 conmon[101299]: audit 2023-10-09T05:23:36.204758+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.8:0/3440787612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:36.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:36 smithi008 conmon[101053]: audit 2023-10-09T05:23:36.204758+0000 mon.c ( 2023-10-09T05:23:36.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:36 smithi008 conmon[101053]: mon.2) 111 : audit [DBG] from='client.? 172.21.15.8:0/3440787612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:36 smithi008 conmon[110782]: audit 2023-10-09T05:23:36.204758+0000 2023-10-09T05:23:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:36 smithi008 conmon[110782]: mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.8:0/3440787612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:37.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:37 smithi191 conmon[101299]: cluster 2023-10-09T05:23:35.936915+0000 mgr.y (mgr.14442) 150 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:37.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:37 smithi008 conmon[101053]: cluster 2023-10-09T05:23:35.936915+0000 mgr.y (mgr.14442) 150 2023-10-09T05:23:37.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:37 smithi008 conmon[101053]: : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:37.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:37 smithi008 conmon[110782]: cluster 2023-10-09T05:23:35.936915+0000 mgr.y 2023-10-09T05:23:37.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:37 smithi008 conmon[110782]: (mgr.14442) 150 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:39.702 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:39 smithi191 conmon[101299]: cluster 2023-10-09T05:23:37.937625+0000 mgr.y (mgr.14442) 151 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:39.702 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:39 smithi191 conmon[101299]: audit 2023-10-09T05:23:38.574914+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.8:0/1259296998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:39.702 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:39 smithi191 conmon[101299]: audit 2023-10-09T05:23:38.877169+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:39.703 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:39 smithi191 conmon[101299]: audit 2023-10-09T05:23:38.878092+0000 mon.c (mon.2) 113 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:23:39.703 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:39 smithi191 conmon[101299]: audit 2023-10-09T05:23:38.878432+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:23:39.703 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:39 smithi191 conmon[101299]: audit 2023-10-09T05:23:38.879252+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:23:39.703 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:39 smithi191 conmon[101299]: audit 2023-10-09T05:23:38.880022+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:23:39.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: cluster 2023-10-09T05:23:37.937625+0000 mgr.y (mgr.14442) 2023-10-09T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: 151 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: audit 2023-10-09T05:23:38.574914+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.8:0/1259296998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: audit 2023-10-09T05:23:38.877169+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: audit 2023-10-09T05:23:38.878092+0000 mon.c (mon.2) 113 : audit [INF] 2023-10-09T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: audit 2023-10-09T05:23:38 2023-10-09T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: .878432+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: 09T05:23:38.879252+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: audit 2023-10-09T05:23:38.880022+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:23:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[101053]: 2023-10-09T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: cluster 2023-10-09T05:23:37.937625+0000 mgr.y (mgr 2023-10-09T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: .14442) 151 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: 23:38.574914+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.8:0/1259296998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: audit 2023-10-09T05:23 2023-10-09T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: :38.877169+0000 mon.a (mon.0) 707 : audit [INF] 2023-10-09T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: audit 2023-10-09T05:23:38 2023-10-09T05:23:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: .878092+0000 mon.c (mon.2) 113 : audit [INF] 2023-10-09T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: audit 2023-10-09T05:23:38. 2023-10-09T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: 878432+0000 mon.a (mon.0) 708 : audit [INF] 2023-10-09T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: audit 2023-10-09T05:23:38. 2023-10-09T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: 879252+0000 mon.c (mon.2) 114 : audit [DBG] 2023-10-09T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:23:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:39 smithi008 conmon[110782]: :23:38.880022+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:23:40.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:40 smithi191 systemd[1]: Stopping Ceph mgr.x for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:23:40.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:40 smithi191 bash[138563]: Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-mgr.x found: no such container 2023-10-09T05:23:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:40 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:38.872703+0000 mgr.y ( 2023-10-09T05:23:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:40 smithi191 conmon[101299]: mgr.14442) 152 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T05:23:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:40 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:38.880649+0000 mgr.y (mgr.14442) 153 : cephadm [INF] Deploying daemon mgr.x on smithi191 2023-10-09T05:23:40.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:40 smithi008 conmon[101053]: cephadm 2023-10-09T05:23:38.872703+0000 mgr.y ( 2023-10-09T05:23:40.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:40 smithi008 conmon[101053]: mgr.14442) 152 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T05:23:40.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:40 smithi008 conmon[101053]: cephadm 2023-10-09T05:23:38.880649+0000 mgr.y (mgr.14442) 153 : cephadm [INF] Deploying daemon mgr.x on smithi191 2023-10-09T05:23:40.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:40 smithi008 conmon[110782]: cephadm 2023-10-09T05:23:38.872703+0000 mgr.y (mgr.14442 2023-10-09T05:23:40.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:40 smithi008 conmon[110782]: ) 152 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T05:23:40.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:40 smithi008 conmon[110782]: cephadm 2023-10-09T05:23:38.880649+0000 mgr.y (mgr.14442 2023-10-09T05:23:40.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:40 smithi008 conmon[110782]: ) 153 : cephadm [INF] Deploying daemon mgr.x on smithi191 2023-10-09T05:23:41.641 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:41 smithi191 conmon[101299]: cluster 2023-10-09T05:23:39.938006+0000 mgr.y (mgr.14442) 154 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:41.641 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:41 smithi191 conmon[101299]: audit 2023-10-09T05:23:40.946413+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.8:0/3942675106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:41.641 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:41 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:23:41.641 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:41 smithi191 bash[138678]: 720487b0c8fc00403d16f7a979fed18b8846a006571325059325e175c6898ff1 2023-10-09T05:23:41.642 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:41 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T05:23:41.642 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:41 smithi191 systemd[1]: Stopped Ceph mgr.x for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:23:41.642 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:41 smithi191 systemd[1]: Starting Ceph mgr.x for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:23:41.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:41 smithi008 conmon[101053]: cluster 2023-10-09T05:23:39.938006+0000 mgr.y (mgr. 2023-10-09T05:23:41.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:41 smithi008 conmon[101053]: 14442) 154 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:41.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:41 smithi008 conmon[101053]: audit 2023-10-09T05:23:40.946413+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.8:0/3942675106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:41.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:41 smithi008 conmon[110782]: cluster 2023-10-09T05:23:39.938006+0000 mgr.y ( 2023-10-09T05:23:41.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:41 smithi008 conmon[110782]: mgr.14442) 154 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:41.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:41 smithi008 conmon[110782]: audit 2023-10-09T05:23:40 2023-10-09T05:23:41.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:41 smithi008 conmon[110782]: .946413+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.8:0/3942675106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:42.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:42 smithi191 conmon[139071]: debug 2023-10-09T05:23:42.292+0000 7f64990e9700 1 -- 172.21.15.191:0/1283955700 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f67e9691e0 con 0x55f67e941000 2023-10-09T05:23:42.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:42 smithi191 bash[139030]: 196b73760dba15a429271b13795cd8ae58babfa17ce98c38e245b7dd6728a780 2023-10-09T05:23:42.551 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:42 smithi191 systemd[1]: Started Ceph mgr.x for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:23:42.551 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:42 smithi191 conmon[139071]: debug 2023-10-09T05:23:42.414+0000 7f64a3237200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:23:42.551 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:42 smithi191 conmon[139071]: debug 2023-10-09T05:23:42.507+0000 7f64a3237200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:23:42.923 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:42 smithi191 conmon[139071]: debug 2023-10-09T05:23:42.605+0000 7f64a3237200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:23:42.923 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:42 smithi191 conmon[139071]: debug 2023-10-09T05:23:42.714+0000 7f64a3237200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:23:42.924 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:42 smithi191 conmon[139071]: debug 2023-10-09T05:23:42.923+0000 7f64a3237200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:23:43.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:43 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:23:43] "GET /metrics HTTP/1.1" 200 192181 "" "Prometheus/2.33.4" 2023-10-09T05:23:43.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:43 smithi008 conmon[101053]: cluster 2023-10-09T05:23:41.938637+0000 mgr.y (mgr.14442) 155 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:43.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:43 smithi008 conmon[101053]: audit 2023-10-09T05:23:42.395846+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:43.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:43 smithi008 conmon[101053]: audit 2023-10-09T05:23:42.397419+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:43.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:43 smithi008 conmon[101053]: audit 2023-10-09T05:23:43.311679+0000 mon.a (mon.0) 710 : audit [DBG] from='client.? 172.21.15.8:0/1265939539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:43.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:43 smithi008 conmon[110782]: cluster 2023-10-09T05:23:41.938637+0000 mgr.y (mgr.14442) 155 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:43.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:43 smithi008 conmon[110782]: audit 2023-10-09T05:23:42.395846+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:43.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:43 smithi008 conmon[110782]: audit 2023-10-09T05:23:42.397419+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:43.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:43 smithi008 conmon[110782]: audit 2023-10-09T05:23:43.311679+0000 mon.a (mon.0) 710 : audit [DBG] from='client.? 172.21.15.8:0/1265939539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:43.661 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:43 smithi191 conmon[101299]: cluster 2023-10-09T05:23:41.938637+0000 mgr.y (mgr.14442 2023-10-09T05:23:43.661 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:43 smithi191 conmon[101299]: ) 155 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:43.661 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:43 smithi191 conmon[101299]: audit 2023-10-09T05:23:42.395846+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:43.661 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:43 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:23:43.661 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:43 smithi191 conmon[101299]: 23:42.397419+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:43.661 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:43 smithi191 conmon[101299]: audit 2023-10-09T05:23: 2023-10-09T05:23:43.661 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:43 smithi191 conmon[101299]: 43.311679+0000 mon.a (mon.0) 710 : audit [DBG] from='client.? 172.21.15.8:0/1265939539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:43.662 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:43 smithi191 conmon[139071]: debug 2023-10-09T05:23:43.317+0000 7f64a3237200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:23:43.662 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:43 smithi191 conmon[139071]: debug 2023-10-09T05:23:43.467+0000 7f64a3237200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:23:43.662 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:43 smithi191 conmon[139071]: debug 2023-10-09T05:23:43.559+0000 7f64a3237200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:23:43.662 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:43 smithi191 conmon[139071]: debug 2023-10-09T05:23:43.660+0000 7f64a3237200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:23:44.549 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:44 smithi191 conmon[139071]: debug 2023-10-09T05:23:44.129+0000 7f64a3237200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:23:45.299 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:44 smithi191 conmon[139071]: debug 2023-10-09T05:23:44.862+0000 7f64a3237200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:23:45.299 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:44 smithi191 conmon[139071]: debug 2023-10-09T05:23:44.952+0000 7f64a3237200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:23:45.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:45 smithi008 conmon[101053]: cluster 2023-10-09T05:23:43.939137 2023-10-09T05:23:45.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:45 smithi008 conmon[101053]: +0000 mgr.y (mgr.14442) 156 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:45 smithi008 conmon[101053]: audit 2023-10-09T05:23:44.829929+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:45 smithi008 conmon[110782]: cluster 2023-10-09T05:23:43.939137+0000 2023-10-09T05:23:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:45 smithi008 conmon[110782]: mgr.y (mgr.14442) 156 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:45 smithi008 conmon[110782]: audit 2023-10-09T05:23:44.829929+0000 2023-10-09T05:23:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:45 smithi008 conmon[110782]: mon.a (mon.0) 711 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:45.793 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:45 smithi191 conmon[101299]: cluster 2023-10-09T05:23:43.939137+0000 mgr.y 2023-10-09T05:23:45.793 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:45 smithi191 conmon[101299]: (mgr.14442) 156 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:45.793 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:45 smithi191 conmon[101299]: audit 2023-10-09T05:23:44.829929+0000 mon.a (mon.0) 711 2023-10-09T05:23:45.793 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:45 smithi191 conmon[101299]: : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:46.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:45 smithi191 conmon[139071]: debug 2023-10-09T05:23:45.793+0000 7f64a3237200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:23:46.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:45 smithi191 conmon[139071]: debug 2023-10-09T05:23:45.889+0000 7f64a3237200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:23:46.374 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101509]: debug 2023-10-09T05:23:46.087+0000 7fe19495a700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T05:23:46.374 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101509]: ignoring --setuser ceph since I am not root 2023-10-09T05:23:46.374 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101509]: ignoring --setgroup ceph since I am not root 2023-10-09T05:23:46.374 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101509]: debug 2023-10-09T05:23:46.132+0000 7f25af5b3700 1 -- 172.21.15.8:0/278535700 <== mon.0 v2:172.21.15.8:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5567bdbd44e0 con 0x5567bdbad400 2023-10-09T05:23:46.374 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101509]: debug 2023-10-09T05:23:46.275+0000 7f25bf77b680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:23:46.374 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101509]: debug 2023-10-09T05:23:46.373+0000 7f25bf77b680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:23:46.376 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[139071]: debug 2023-10-09T05:23:46.091+0000 7f64a3237200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:23:46.377 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[139071]: debug 2023-10-09T05:23:46.376+0000 7f64a3237200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:23:46.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: audit 2023-10-09T05:23:45.676751 2023-10-09T05:23:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: +0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.8:0/1697458985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: audit 2023-10-09T05:23:45.737342+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: audit 2023-10-09T05:23:46.065245+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:23:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: audit 2023-10-09T05:23:46.066094+0000 mon.c 2023-10-09T05:23:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: (mon.2) 120 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:23:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: audit 2023-10-09T05:23:46.071023+0000 mon.a 2023-10-09T05:23:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: (mon.0) 713 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: audit 2023-10-09T05:23 2023-10-09T05:23:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: :46.077142+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: 2023-10-09T05:23:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:23:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: :23:46.082392+0000 mon.c (mon.2) 122 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T05:23:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: audit 2023-10-09T05:23:46.082685 2023-10-09T05:23:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: +0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T05:23:46.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: cluster 2023-10-09T05:23: 2023-10-09T05:23:46.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: 46.088131+0000 mon.a (mon.0) 715 : cluster 2023-10-09T05:23:46.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T05:23:46.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: audit 2023-10-09T05:23:46.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: 2023-10-09T05:23:46.088406+0000 mon.a ( 2023-10-09T05:23:46.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: mon.0) 716 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T05:23:46.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: cluster 2023-10-09T05:23:46.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: 2023-10-09T05:23:46.088493+0000 mon.a (mon.0 2023-10-09T05:23:46.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101053]: ) 717 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00534188s) 2023-10-09T05:23:46.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: audit 2023-10-09T05:23:45.676751 2023-10-09T05:23:46.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: +0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.8:0/1697458985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:46.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: audit 2023-10-09T05:23:46.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: 2023-10-09T05:23:45.737342+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:46.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:23:46.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: :23:46.065245+0000 mon.c (mon.2) 119 : audit [DBG] 2023-10-09T05:23:46.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:23:46.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:23:46.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: T05:23:46.066094+0000 mon.c (mon.2) 120 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:23:46.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: audit 2023-10-09T05:23:46. 2023-10-09T05:23:46.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: 071023+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:46.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: audit 2023-10-09T05:23: 2023-10-09T05:23:46.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: 46.077142+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:46.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: audit 2023-10-09T05:23: 2023-10-09T05:23:46.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: 46.082392+0000 mon.c (mon.2) 122 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T05:23:46.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: 2023-10-09T05:23:46.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: audit 2023-10-09T05:23:46.082685+0000 2023-10-09T05:23:46.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: mon.a (mon.0) 714 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T05:23:46.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:23:46.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: -10-09T05:23:46.088131+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T05:23:46.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: 2023-10-09T05:23:46.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:23:46.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: :23:46.088406+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T05:23:46.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: cluster 2023-10-09T05:23: 2023-10-09T05:23:46.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[110782]: 46.088493+0000 mon.a (mon.0) 717 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00534188s) 2023-10-09T05:23:46.755 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[139071]: debug 2023-10-09T05:23:46.606+0000 7f64a3237200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:23:46.755 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: audit 2023-10-09T05:23:45.676751+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.8:0/1697458985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:46.755 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: audit 2023-10-09T05:23:45.737342+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:46.755 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: audit 2023-10-09T05:23:46.065245+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: audit 2023-10-09T05:23:46.066094+0000 mon.c (mon.2) 120 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: audit 2023 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: -10-09T05:23:46.071023+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14442 ' entity='mgr.y' 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: audit 2023-10-09T05:23:46.077142+0000 mon.c (mon 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: .2) 121 : audit [DBG] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: audit 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: 2023-10-09T05:23:46.082392+0000 mon.c (mon.2) 122 : audit [INF] from='mgr.14442 172.21.15.8:0/3437872069' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: audit 2023-10-09T05:23:46. 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: 082685+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: cluster 2023-10-09T05:23:46.088131+0000 mon.a 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: (mon.0) 715 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T05:23:46.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: audit 2023-10-09T05:23:46.088406+0000 mon.a (mon 2023-10-09T05:23:46.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: .0) 716 : audit [INF] from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T05:23:46.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: cluster 2023 2023-10-09T05:23:46.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[101299]: -10-09T05:23:46.088493+0000 mon.a (mon.0) 717 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00534188s) 2023-10-09T05:23:47.020 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:46 smithi191 conmon[139071]: debug 2023-10-09T05:23:46.755+0000 7f64a3237200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:23:47.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:46 smithi008 conmon[101509]: debug 2023-10-09T05:23:46.824+0000 7f25bf77b680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:23:47.299 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:47 smithi191 conmon[139071]: debug 2023-10-09T05:23:47.020+0000 7f64a3237200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:23:48.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:47 smithi191 conmon[139071]: debug 2023-10-09T05:23:47.728+0000 7f64a3237200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:23:48.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:47 smithi191 conmon[139071]: debug 2023-10-09T05:23:47.860+0000 7f64a3237200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:23:48.106 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:47 smithi008 conmon[101509]: debug 2023-10-09T05:23:47.753+0000 7f25bf77b680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:23:48.106 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:47 smithi008 conmon[101509]: debug 2023-10-09T05:23:47.854+0000 7f25bf77b680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:23:48.390 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:48 smithi191 conmon[101299]: audit 2023-10-09T05:23:48.069047+0000 mon.c (mon.2) 2023-10-09T05:23:48.390 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:48 smithi191 conmon[101299]: 123 : audit [DBG] from='client.? 172.21.15.8:0/4079775575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:48.390 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:48 smithi191 conmon[139071]: debug 2023-10-09T05:23:48.056+0000 7f64a3237200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:23:48.390 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:48 smithi191 conmon[139071]: debug 2023-10-09T05:23:48.170+0000 7f64a3237200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:23:48.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:48 smithi008 conmon[101509]: debug 2023-10-09T05:23:48.105+0000 7f25bf77b680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:23:48.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:48 smithi008 conmon[101053]: audit 2023-10-09T05:23:48.069047+0000 mon.c ( 2023-10-09T05:23:48.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:48 smithi008 conmon[101053]: mon.2) 123 : audit [DBG] from='client.? 172.21.15.8:0/4079775575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:48.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:48 smithi008 conmon[110782]: audit 2023-10-09T05:23 2023-10-09T05:23:48.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:48 smithi008 conmon[110782]: :48.069047+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.8:0/4079775575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:48.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:48 smithi191 conmon[139071]: debug 2023-10-09T05:23:48.390+0000 7f64a3237200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:23:49.516 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101509]: debug 2023-10-09T05:23:49.246+0000 7f25bf77b680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:23:49.517 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101509]: debug 2023-10-09T05:23:49.416+0000 7f25bf77b680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:23:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: cluster 2023-10-09T05:23:49.274405+0000 mon.a (mon.0) 718 2023-10-09T05:23:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: : cluster [INF] Active manager daemon x restarted 2023-10-09T05:23:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: cluster 2023-10-09T05:23:49.274786+0000 mon.a (mon.0) 719 : cluster [INF] Activating manager daemon x 2023-10-09T05:23:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05:23:49.276496+0000 mon.b (mon.1) 29 : 2023-10-09T05:23:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit [DBG] from='mgr.? 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:23:49.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: cluster 2023-10-09T05:23 2023-10-09T05:23:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: :49.279807+0000 mon.a (mon.0) 720 : 2023-10-09T05:23:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T05:23:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: cluster 2023-10-09T05:23:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 2023-10-09T05:23:49.280070+0000 2023-10-09T05:23:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: mon.a (mon.0) 721 : cluster [DBG] mgrmap e21: x(active, starting, since 0.00545511s) 2023-10-09T05:23:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 2023-10-09T05:23:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05:23:49 2023-10-09T05:23:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: .285691+0000 mon.a (mon.0) 722 : audit [DBG] 2023-10-09T05:23:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:23:49.518 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 23:49.285834+0000 mon.a (mon.0) 723 : audit 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05:23 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: :49.285990+0000 mon.a (mon.0) 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 724 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05:23: 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 49.286205+0000 mon.a (mon.0) 725 : audit [DBG] 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 23:49.286404+0000 mon.a (mon.0) 726 : 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi008.zfejpl"}]: dispatch 2023-10-09T05:23:49.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 23:49.286693+0000 mon.a (mon.0) 727 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.walafj"}]: dispatch 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 09T05:23:49.286977+0000 mon.a (mon 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: .0) 728 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 10-09T05:23:49.287227+0000 mon.a ( 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: mon.0) 729 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 2023-10-09T05:23 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: :49.287512+0000 mon.a (mon.0) 730 2023-10-09T05:23:49.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:23:49.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: cluster 2023-10-09T05:23:49.274405 2023-10-09T05:23:49.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: +0000 mon.a (mon.0) 718 : cluster [INF] Active manager daemon x restarted 2023-10-09T05:23:49.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:23:49.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 10-09T05:23:49.274786+0000 mon.a (mon.0) 719 : cluster [INF] 2023-10-09T05:23:49.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: Activating manager daemon x 2023-10-09T05:23:49.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:23:49.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 23:49.276496+0000 mon.b (mon.1) 29 : 2023-10-09T05:23:49.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit [DBG] from='mgr.? 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:23:49.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: cluster 2023-10-09T05:23: 2023-10-09T05:23:49.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 49.279807+0000 mon.a (mon.0) 720 2023-10-09T05:23:49.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T05:23:49.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: cluster 2023-10-09T05: 2023-10-09T05:23:49.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 23:49.280070+0000 mon.a (mon.0) 721 : cluster [DBG] 2023-10-09T05:23:49.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: mgrmap e21: x(active, starting, since 0.00545511s) 2023-10-09T05:23:49.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T05:23 2023-10-09T05:23:49.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: :49.285691+0000 mon.a (mon.0) 722 : audit [DBG] 2023-10-09T05:23:49.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:23:49.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T05:23:49.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 2023-10-09T05:23:49.285834+0000 mon.a (mon. 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 0) 723 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: -09T05:23:49.285990+0000 mon.a (mon.0) 724 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T05:23:49. 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 286205+0000 mon.a (mon.0) 725 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 23:49.286404+0000 mon.a (mon.0) 726 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi008.zfejpl"}]: dispatch 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:23:49.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 05:23:49.286693+0000 mon.a (mon.0) 727 : audit 2023-10-09T05:23:49.524 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.walafj"}]: dispatch 2023-10-09T05:23:49.524 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T05:23:49.524 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 2023-10-09T05:23:49.286977+0000 mon.a (mon 2023-10-09T05:23:49.524 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: .0) 728 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:23:49.524 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T05:23 2023-10-09T05:23:49.524 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: :49.287227+0000 mon.a (mon.0) 729 : audit 2023-10-09T05:23:49.524 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:23:49.524 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T05:23 2023-10-09T05:23:49.524 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: :49.287512+0000 mon.a (mon.0) 730 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:23:49.525 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023 2023-10-09T05:23:49.525 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: -10-09T05:23:49.287789+0000 mon.a 2023-10-09T05:23:49.525 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: (mon.0) 731 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:23:49.525 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05:23:49.525 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 2023-10-09T05:23:49.288101+0000 2023-10-09T05:23:49.525 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: mon.a (mon.0) 732 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:23:49.525 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05:23: 2023-10-09T05:23:49.525 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 49.288389+0000 mon.a (mon.0) 733 : audit [DBG] 2023-10-09T05:23:49.525 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:23:49.525 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05:23: 2023-10-09T05:23:49.525 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 49.288670+0000 mon.a (mon.0) 734 : audit 2023-10-09T05:23:49.526 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:23:49.526 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:23:49.526 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: :23:49.288943+0000 mon.a (mon.0) 2023-10-09T05:23:49.526 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 735 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:23:49.526 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:23:49.526 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: -09T05:23:49.289234+0000 mon.a (mon 2023-10-09T05:23:49.526 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: .0) 736 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:23:49.526 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05:23:49.526 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 2023-10-09T05:23:49.289527+0000 2023-10-09T05:23:49.526 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: mon.a (mon.0) 737 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:23:49.526 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 09T05:23:49.289845+0000 mon.a (mon.0) 738 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: T05:23:49.290396+0000 mon.a (mon.0) 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 739 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: -09T05:23:49.290625+0000 mon.a 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: (mon.0) 740 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 10-09T05:23:49.292075+0000 mon.a ( 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: mon.0) 741 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:23:49.527 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: audit 2023-10-09T05:23:49.528 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: 2023-10-09T05:23:49.292629+0000 2023-10-09T05:23:49.528 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101053]: mon.a (mon.0) 742 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:23:49.528 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:23:49.528 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: T05:23:49.287789+0000 mon.a (mon. 2023-10-09T05:23:49.528 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 0) 731 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:23:49.528 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:23:49.528 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: -09T05:23:49.288101+0000 mon.a (mon. 2023-10-09T05:23:49.528 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 0) 732 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:23:49.528 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 09T05:23:49.288389+0000 mon.a (mon.0) 733 : 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: -09T05:23:49.288670+0000 mon.a (mon. 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 0) 734 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: T05:23:49.288943+0000 mon.a (mon 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: .0) 735 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 09T05:23:49.289234+0000 mon.a (mon.0) 736 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:23:49.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 2023-10-09T05:23:49.289527+0000 mon.a (mon 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: .0) 737 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: T05:23:49.289845+0000 mon.a (mon 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: .0) 738 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 09T05:23:49.290396+0000 mon.a (mon.0) 739 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: -09T05:23:49.290625+0000 mon.a (mon.0) 740 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:23:49.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T05:23:49. 2023-10-09T05:23:49.531 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: 292075+0000 mon.a (mon.0) 741 : audit 2023-10-09T05:23:49.531 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:23:49.531 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: audit 2023-10-09T05:23:49.292629+0000 mon.a ( 2023-10-09T05:23:49.531 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[110782]: mon.0) 742 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:23:49.546 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: cluster 2023-10-09T05:23:49.274405+0000 mon.a (mon. 2023-10-09T05:23:49.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 0) 718 : cluster [INF] Active manager daemon x restarted 2023-10-09T05:23:49.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: cluster 2023-10-09T05:23:49.274786+0000 mon.a (mon.0) 719 : cluster [INF] Activating manager daemon x 2023-10-09T05:23:49.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10-09T05:23 2023-10-09T05:23:49.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: :49.276496+0000 mon.b (mon.1) 29 : audit 2023-10-09T05:23:49.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: [DBG] from='mgr.? 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:23:49.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: cluster 2023-10-09T05: 2023-10-09T05:23:49.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 23:49.279807+0000 mon.a (mon.0) 720 : 2023-10-09T05:23:49.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T05:23:49.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: cluster 2023-10-09T05 2023-10-09T05:23:49.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: :23:49.280070+0000 mon.a (mon 2023-10-09T05:23:49.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: .0) 721 : cluster [DBG] mgrmap e21: x(active, starting, since 0.00545511s) 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: -10-09T05:23:49.285691+0000 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: mon.a (mon.0) 722 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 10-09T05:23:49.285834+0000 mon.a 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: (mon.0) 723 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 10-09T05:23:49.285990+0000 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: mon.a (mon.0) 724 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 23:49.286205+0000 mon.a (mon.0) 725 : 2023-10-09T05:23:49.548 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: :23:49.286404+0000 mon.a (mon.0) 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 726 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi008.zfejpl"}]: dispatch 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: T05:23:49.286693+0000 mon.a (mon 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: .0) 727 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.walafj"}]: dispatch 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: -09T05:23:49.286977+0000 mon.a ( 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: mon.0) 728 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 10-09T05:23:49.287227+0000 mon.a 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: (mon.0) 729 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:23:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:23:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 10-09T05:23:49.287512+0000 mon.a 2023-10-09T05:23:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: (mon.0) 730 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:23:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023 2023-10-09T05:23:49.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: debug 2023-10-09T05:23:49.260+0000 7f64a3237200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:23:49.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: [09/Oct/2023:05:23:49] ENGINE Bus STARTING 2023-10-09T05:23:49.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: CherryPy Checker: 2023-10-09T05:23:49.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: The Application mounted at '' has an empty config. 2023-10-09T05:23:49.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: 2023-10-09T05:23:49.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: [09/Oct/2023:05:23:49] ENGINE Serving on http://:::9283 2023-10-09T05:23:49.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: [09/Oct/2023:05:23:49] ENGINE Bus STARTED 2023-10-09T05:23:49.551 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: [09/Oct/2023:05:23:49] ENGINE Bus STOPPING 2023-10-09T05:23:49.551 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: [09/Oct/2023:05:23:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:23:49.551 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: [09/Oct/2023:05:23:49] ENGINE Bus STOPPED 2023-10-09T05:23:49.551 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: [09/Oct/2023:05:23:49] ENGINE Bus STARTING 2023-10-09T05:23:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: -10-09T05:23:49.287789+0000 2023-10-09T05:23:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: mon.a (mon.0) 731 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:23:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10-09T05:23:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 2023-10-09T05:23:49.288101+0000 2023-10-09T05:23:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: mon.a (mon.0) 732 : audit [DBG] 2023-10-09T05:23:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10-09T05:23:49 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: .288389+0000 mon.a (mon.0) 733 : audit 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 23:49.288670+0000 mon.a (mon.0) 734 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: :23:49.288943+0000 mon.a (mon.0 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: ) 735 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 09T05:23:49.289234+0000 mon.a ( 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: mon.0) 736 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:23:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 10-09T05:23:49.289527+0000 mon.a 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: (mon.0) 737 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 2023-10-09T05:23:49.289845+0000 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: mon.a (mon.0) 738 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 2023-10-09T05:23:49.290396+0000 mon.a ( 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: mon.0) 739 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 10-09T05:23:49.290625+0000 mon.a 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: (mon.0) 740 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:23:49.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023 2023-10-09T05:23:49.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: -10-09T05:23:49.292075+0000 2023-10-09T05:23:49.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: mon.a (mon.0) 741 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:23:49.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:23:49.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: 10-09T05:23:49.292629+0000 mon.a 2023-10-09T05:23:49.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[101299]: (mon.0) 742 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:23:49.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: debug 2023-10-09T05:23:49.548+0000 7f6400e70700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:23:49.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: debug 2023-10-09T05:23:49.548+0000 7f6400e70700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:23:49.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: debug 2023-10-09T05:23:49.548+0000 7f6400e70700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:23:49.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: debug 2023-10-09T05:23:49.548+0000 7f6400e70700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:23:49.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: debug 2023-10-09T05:23:49.548+0000 7f6400e70700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:23:49.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: debug 2023-10-09T05:23:49.548+0000 7f6403675700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:23:49.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: debug 2023-10-09T05:23:49.548+0000 7f6403675700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:23:49.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: debug 2023-10-09T05:23:49.548+0000 7f6403675700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:23:49.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: debug 2023-10-09T05:23:49.548+0000 7f6403675700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:23:49.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: debug 2023-10-09T05:23:49.548+0000 7f6403675700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:23:49.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: [09/Oct/2023:05:23:49] ENGINE Serving on http://:::9283 2023-10-09T05:23:49.801 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:49 smithi191 conmon[139071]: [09/Oct/2023:05:23:49] ENGINE Bus STARTED 2023-10-09T05:23:49.859 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101509]: debug 2023-10-09T05:23:49.516+0000 7f25bf77b680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:23:49.859 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101509]: debug 2023-10-09T05:23:49.634+0000 7f25bf77b680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:23:50.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:49 smithi008 conmon[101509]: debug 2023-10-09T05:23:49.858+0000 7f25bf77b680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:23:50.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101509]: debug 2023-10-09T05:23:50.237+0000 7f25bf77b680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:23:50.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101509]: debug 2023-10-09T05:23:50.361+0000 7f25bf77b680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:23:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: cluster 2023-10-09T05:23:49.447444+0000 mon.a (mon. 2023-10-09T05:23:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: 0) 743 : cluster [INF] Manager daemon x is now available 2023-10-09T05:23:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: audit 2023-10-09T05:23:49.467277+0000 mon.a (mon.0) 744 : audit [INF] 2023-10-09T05:23:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: audit 2023 2023-10-09T05:23:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: -10-09T05:23:49.471962+0000 mon.a (mon.0) 745 : 2023-10-09T05:23:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:23:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: -09T05:23:49.473815+0000 mon.a (mon. 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: 0) 746 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: audit 2023-10-09T05:23 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: :49.482450+0000 mon.a (mon.0) 747 : audit 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: audit 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: 2023-10-09T05:23:49.494318+0000 mon.a (mon.0) 748 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: 09T05:23:49.501897+0000 mon.a (mon. 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: 0) 749 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: audit 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: 2023-10-09T05:23:49.527116+0000 2023-10-09T05:23:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: mon.a (mon.0) 750 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:50.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: audit 2023 2023-10-09T05:23:50.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: -10-09T05:23:49.527456+0000 mon.a (mon 2023-10-09T05:23:50.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: .0) 751 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T05:23:50.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: cephadm 2023- 2023-10-09T05:23:50.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: 10-09T05:23:49.869704+0000 mgr.x 2023-10-09T05:23:50.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: (mgr.25453) 1 : cephadm [INF] Deploying cephadm binary to smithi191 2023-10-09T05:23:50.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: cephadm 2023-10- 2023-10-09T05:23:50.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: 09T05:23:49.877249+0000 mgr.x (mgr 2023-10-09T05:23:50.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[101053]: .25453) 2 : cephadm [INF] Deploying cephadm binary to smithi008 2023-10-09T05:23:50.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: cluster 2023-10-09T05:23:49.447444 2023-10-09T05:23:50.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: +0000 mon.a (mon.0) 743 : cluster [INF] Manager daemon x is now available 2023-10-09T05:23:50.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:23:50.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: 09T05:23:49.467277+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:50.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:23:50.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: 05:23:49.471962+0000 mon.a (mon.0) 745 : 2023-10-09T05:23:50.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:50.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:23:50.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: T05:23:49.473815+0000 mon.a (mon. 2023-10-09T05:23:50.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: 0) 746 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:50.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: audit 2023-10-09T05:23:50.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: 2023-10-09T05:23:49.482450+0000 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: mon.a (mon.0) 747 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: audit 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: 2023-10-09T05 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: :23:49.494318+0000 mon.a (mon.0) 748 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: 10-09T05:23:49.501897+0000 mon.a 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: (mon.0) 749 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:23:50.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: T05:23:49.527116+0000 mon.a (mon.0) 750 2023-10-09T05:23:50.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:50.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:23:50.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: 09T05:23:49.527456+0000 mon.a (mon.0 2023-10-09T05:23:50.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: ) 751 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T05:23:50.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: cephadm 2023- 2023-10-09T05:23:50.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: 10-09T05:23:49.869704+0000 mgr.x 2023-10-09T05:23:50.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: (mgr.25453) 1 : cephadm [INF] Deploying cephadm binary to smithi191 2023-10-09T05:23:50.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: cephadm 2023-10-09T05:23:50.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: 2023-10-09T05:23:49.877249+0000 2023-10-09T05:23:50.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:50 smithi008 conmon[110782]: mgr.x (mgr.25453) 2 : cephadm [INF] Deploying cephadm binary to smithi008 2023-10-09T05:23:50.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: cluster 2023-10-09T05:23:49.447444+0000 mon.a (mon.0) 2023-10-09T05:23:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: 743 : cluster [INF] Manager daemon x is now available 2023-10-09T05:23:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: audit 2023-10-09T05:23:49.467277+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: audit 2023-10-09T05:23:49.471962+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: audit 2023-10-09T05:23:49.473815+0000 mon.a (mon.0) 746 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:23:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: audit 2023 2023-10-09T05:23:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: -10-09T05:23:49.482450+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:23:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: audit 2023-10-09T05:23:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: 2023-10-09T05:23:49.494318+0000 mon.a (mon.0 2023-10-09T05:23:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: ) 748 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:23:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: T05:23:49.501897+0000 mon.a (mon.0 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: ) 749 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: T05:23:49.527116+0000 mon.a (mon. 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: 0) 750 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: -09T05:23:49.527456+0000 mon.a ( 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: mon.0) 751 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: cephadm 2023-10- 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: 09T05:23:49.869704+0000 mgr.x (mgr 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: .25453) 1 : cephadm [INF] Deploying cephadm binary to smithi191 2023-10-09T05:23:50.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: cephadm 2023- 2023-10-09T05:23:50.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: 10-09T05:23:49.877249+0000 2023-10-09T05:23:50.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: mgr.x (mgr.25453) 2023-10-09T05:23:50.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:50 smithi191 conmon[101299]: 2 : cephadm [INF] Deploying cephadm binary to smithi008 2023-10-09T05:23:51.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: cluster 2023-10-09T05:23:50.337824+0000 mon.a 2023-10-09T05:23:51.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: (mon.0) 752 : cluster [DBG] mgrmap e22: x(active, since 1.06321s) 2023-10-09T05:23:51.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: cluster 2023-10-09T05:23:50.349071+0000 mgr.x (mgr.25453) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:51.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: audit 2023-10-09T05:23:50.429646+0000 mon.a (mon.0) 753 : audit 2023-10-09T05:23:51.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: [DBG] from='client.? 172.21.15.8:0/3263249316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:51.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:23:51.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: T05:23:50.469350+0000 mon.a (mon.0) 754 : audit 2023-10-09T05:23:51.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:23:51.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: cluster 2023-10- 2023-10-09T05:23:51.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: 09T05:23:50.469446+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:23:51.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:23:51.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: :23:50.470752+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:23:51.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: 2023-10-09T05:23:51.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:23:51.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: 05:23:51.092738+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:51.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: audit 2023-10-09T05:23:51 2023-10-09T05:23:51.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: .109800+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:51.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: cluster 2023-10-09T05:23:51. 2023-10-09T05:23:51.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101053]: 289026+0000 mgr.x (mgr.25453) 4 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:51.709 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: cluster 2023-10-09T05:23 2023-10-09T05:23:51.709 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: :50.337824+0000 mon.a (mon.0) 752 : cluster [DBG] mgrmap e22: x(active, since 1.06321s) 2023-10-09T05:23:51.709 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: cluster 2023-10-09T05:23:50 2023-10-09T05:23:51.709 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: .349071+0000 mgr.x (mgr.25453) 3 : cluster 2023-10-09T05:23:51.709 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:51.709 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: audit 2023-10-09T05:23: 2023-10-09T05:23:51.709 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: 50.429646+0000 mon.a (mon.0) 753 2023-10-09T05:23:51.709 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: : audit [DBG] from='client.? 172.21.15.8:0/3263249316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:51.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:23:51.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: :23:50.469350+0000 mon.a (mon. 2023-10-09T05:23:51.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: 0) 754 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:23:51.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: cluster 2023-10-09 2023-10-09T05:23:51.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: T05:23:50.469446+0000 mon.a ( 2023-10-09T05:23:51.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: mon.0) 755 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:23:51.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:23:51.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: 10-09T05:23:50.470752+0000 2023-10-09T05:23:51.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: mon.a (mon.0) 756 : audit [INF] 2023-10-09T05:23:51.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:23:51.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: audit 2023 2023-10-09T05:23:51.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: -10-09T05:23:51.092738+0000 mon.a (mon 2023-10-09T05:23:51.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: .0) 757 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:51.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:23:51.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: 09T05:23:51.109800+0000 mon.a 2023-10-09T05:23:51.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: (mon.0) 758 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:51.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: 2023-10-09T05:23:51.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:23:51.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: -10-09T05:23:51.289026+0000 mgr.x ( 2023-10-09T05:23:51.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[110782]: mgr.25453) 4 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:51.711 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101509]: debug 2023-10-09T05:23:51.246+0000 7f25bf77b680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:23:51.712 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101509]: debug 2023-10-09T05:23:51.349+0000 7f25bf77b680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:23:51.712 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101509]: debug 2023-10-09T05:23:51.465+0000 7f25bf77b680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:23:51.712 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:51 smithi008 conmon[101509]: debug 2023-10-09T05:23:51.704+0000 7f25bf77b680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:23:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: cluster 2023-10-09T05:23:50.337824+0000 mon.a 2023-10-09T05:23:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: (mon.0) 752 : cluster [DBG] mgrmap e22: x(active, since 1.06321s) 2023-10-09T05:23:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: cluster 2023-10-09T05:23:50.349071+0000 mgr.x (mgr.25453) 3 : cluster [DBG] 2023-10-09T05:23:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: audit 2023-10-09T05:23:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: 2023-10-09T05:23:50.429646+0000 mon.a 2023-10-09T05:23:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: (mon.0) 753 : audit [DBG] from='client.? 172.21.15.8:0/3263249316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:23:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: :23:50.469350+0000 mon.a (mon.0) 2023-10-09T05:23:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: 754 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:23:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: cluster 2023- 2023-10-09T05:23:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: 10-09T05:23:50.469446+0000 2023-10-09T05:23:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: mon.a (mon.0) 755 : cluster [DBG] 2023-10-09T05:23:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:23:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: audit 2023-10-09T05:23 2023-10-09T05:23:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: :50.470752+0000 mon.a (mon.0) 2023-10-09T05:23:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: 756 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:23:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:23:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: T05:23:51.092738+0000 mon.a ( 2023-10-09T05:23:51.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: mon.0) 757 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:51.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: audit 2023 2023-10-09T05:23:51.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: -10-09T05:23:51.109800 2023-10-09T05:23:51.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: +0000 mon.a (mon.0) 758 : audit [INF] 2023-10-09T05:23:51.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:51.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: cluster 2023-10-09T05: 2023-10-09T05:23:51.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: 23:51.289026+0000 mgr.x (mgr.25453) 2023-10-09T05:23:51.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:51 smithi191 conmon[101299]: 4 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:52.396 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101509]: debug 2023-10-09T05:23:52.039+0000 7f25bf77b680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:23:52.396 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101509]: debug 2023-10-09T05:23:52.222+0000 7f25bf77b680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:23:52.396 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101509]: debug 2023-10-09T05:23:52.395+0000 7f25bf77b680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:23:52.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: cephadm 2023-10-09T05:23:51 2023-10-09T05:23:52.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: .432849+0000 mgr.x (mgr.25453) 5 : cephadm [INF] [09/Oct/2023:05:23:51] ENGINE Bus STARTING 2023-10-09T05:23:52.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: audit 2023-10-09T05:23:51.470101+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:23:52.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: cluster 2023-10-09T05:23:51.470200+0000 mon.a (mon 2023-10-09T05:23:52.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: .0) 760 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T05:23:52.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: cephadm 2023-10-09T 2023-10-09T05:23:52.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: 05:23:51.534592+0000 mgr.x (mgr 2023-10-09T05:23:52.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: .25453) 6 : cephadm [INF] 2023-10-09T05:23:52.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: [09/Oct/2023:05:23:51] ENGINE Serving on http://172.21.15.191:8765 2023-10-09T05:23:52.662 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: cephadm 2023-10-09T05:23:52.662 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: 2023-10-09T05: 2023-10-09T05:23:52.662 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: 23:51.651347+0000 mgr.x (mgr.25453 2023-10-09T05:23:52.662 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: ) 7 : cephadm [INF] [09/Oct/2023:05:23:51] ENGINE Serving on https://172.21.15.191:7150 2023-10-09T05:23:52.662 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: cephadm 2023-10-09T05:23:51.651483+0000 mgr.x (mgr.25453) 8 : cephadm [INF] [09/Oct/2023:05:23:51] ENGINE Bus STARTED 2023-10-09T05:23:52.662 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: cluster 2023-10-09T05:23:52.662 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101053]: 2023-10-09T05:23:52.112974+0000 mon.a (mon.0) 761 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-10-09T05:23:52.662 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: cephadm 2023-10-09T05:23:51 2023-10-09T05:23:52.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: .432849+0000 mgr.x (mgr.25453) 5 : cephadm [INF] [09/Oct/2023:05:23:51] ENGINE Bus STARTING 2023-10-09T05:23:52.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: audit 2023-10-09T05:23:51.470101+0000 2023-10-09T05:23:52.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: mon.a (mon.0) 759 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:23:52.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: 2023-10-09T05:23:52.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: cluster 2023-10-09T05:23:52.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: 2023-10-09T05:23:51.470200+0000 mon.a (mon.0 2023-10-09T05:23:52.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: ) 760 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T05:23:52.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: cephadm 2023-10-09T 2023-10-09T05:23:52.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: 05:23:51.534592+0000 mgr.x (mgr.25453 2023-10-09T05:23:52.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: ) 6 : cephadm [INF] [09/Oct/2023:05:23:51] ENGINE Serving on http://172.21.15.191:8765 2023-10-09T05:23:52.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: cephadm 2023-10-09T 2023-10-09T05:23:52.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: 05:23:51.651347+0000 mgr.x (mgr.25453 2023-10-09T05:23:52.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: ) 7 : cephadm [INF] [09/Oct/2023:05:23:51] ENGINE Serving on https://172.21.15.191:7150 2023-10-09T05:23:52.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: cephadm 2023-10-09T 2023-10-09T05:23:52.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: 05:23:51.651483+0000 mgr.x (mgr. 2023-10-09T05:23:52.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: 25453) 8 : cephadm [INF] [09/Oct/2023:05:23:51] ENGINE Bus STARTED 2023-10-09T05:23:52.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: cluster 2023-10-09 2023-10-09T05:23:52.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: T05:23:52.112974+0000 mon.a (mon. 2023-10-09T05:23:52.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[110782]: 0) 761 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-10-09T05:23:52.665 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101509]: debug 2023-10-09T05:23:52.659+0000 7f25bf77b680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:23:52.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:51.432849+0000 mgr.x (mgr.25453) 5 : cephadm [INF] [09/Oct/2023:05:23:51] ENGINE Bus STARTING 2023-10-09T05:23:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:52 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:23:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:52 smithi191 conmon[101299]: 23:51.470101+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:23:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:52 smithi191 conmon[101299]: cluster 2023-10-09T05:23:51.470200+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T05:23:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:51.534592+0000 mgr.x (mgr.25453) 6 : cephadm [INF] [09/Oct/2023:05:23:51] ENGINE Serving on http://172.21.15.191:8765 2023-10-09T05:23:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:51.651347+0000 mgr.x (mgr.25453) 7 : cephadm [INF] [09/Oct/2023:05:23:51] ENGINE Serving on https://172.21.15.191:7150 2023-10-09T05:23:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:51.651483+0000 mgr.x (mgr.25453) 8 : cephadm [INF] [09/Oct/2023:05:23:51] ENGINE Bus STARTED 2023-10-09T05:23:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:52 smithi191 conmon[101299]: cluster 2023-10-09T05:23:52.112974+0000 mon.a (mon.0) 761 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-10-09T05:23:52.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101509]: debug 2023-10-09T05:23:52.779+0000 7f25bf77b680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:23:52.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101509]: [09/Oct/2023:05:23:52] ENGINE Bus STARTING 2023-10-09T05:23:52.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101509]: CherryPy Checker: 2023-10-09T05:23:52.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101509]: The Application mounted at '' has an empty config. 2023-10-09T05:23:52.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101509]: 2023-10-09T05:23:52.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101509]: [09/Oct/2023:05:23:52] ENGINE Serving on http://:::9283 2023-10-09T05:23:52.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:52 smithi008 conmon[101509]: [09/Oct/2023:05:23:52] ENGINE Bus STARTED 2023-10-09T05:23:53.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:23:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:23:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:53 smithi191 conmon[101299]: audit 2023-10-09T05:23:52.522027+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:53 smithi191 conmon[101299]: cluster 2023-10-09T05:23:52.794281+0000 mon.a (mon.0) 763 : cluster [DBG] Standby manager daemon y started 2023-10-09T05:23:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:53 smithi191 conmon[101299]: audit 2023-10-09T05:23:52.798704+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:23:53.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:53 smithi191 conmon[101299]: audit 2023-10-09T05:23:52.799248+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:23:53.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:53 smithi191 conmon[101299]: audit 2023-10-09T05:23:52.803082+0000 mon.a (mon.0) 766 : audit [DBG] from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:23:53.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:53 smithi191 conmon[101299]: audit 2023-10-09T05:23:52.803636+0000 mon.a (mon.0) 767 : audit [DBG] from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:23:53.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:53 smithi191 conmon[101299]: audit 2023-10-09T05:23:52.912924+0000 mon.c (mon.2) 124 : audit [DBG] from='client.? 172.21.15.8:0/2453071470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:53.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:53 smithi191 conmon[101299]: cluster 2023-10-09T05:23:53.289556+0000 mgr.x (mgr.25453) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:53.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: audit 2023-10-09T05:23:52.522027 2023-10-09T05:23:53.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: +0000 mon.a (mon.0) 762 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: cluster 2023-10-09T05:23:52.794281+0000 mon.a (mon.0) 763 : cluster [DBG] Standby manager daemon y started 2023-10-09T05:23:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: audit 2023-10-09T05:23:52.798704+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:23:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:23:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: 09T05:23:52.799248+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:23:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:23:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: 10-09T05:23:52.803082+0000 mon.a 2023-10-09T05:23:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: (mon.0) 766 : audit [DBG] 2023-10-09T05:23:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:23:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: audit 2023 2023-10-09T05:23:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: -10-09T05:23:52 2023-10-09T05:23:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: .803636+0000 mon.a (mon.0 2023-10-09T05:23:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: ) 767 : audit [DBG] from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:23:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: 2023-10-09T05:23:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: audit 2023-10-09T05:23:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: 2023-10-09T05 2023-10-09T05:23:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: :23:52.912924+0000 2023-10-09T05:23:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: mon.c (mon.2) 124 : audit 2023-10-09T05:23:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: [DBG] from='client.? 172.21.15.8:0/2453071470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: cluster 2023-10-09T05:23:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: 2023-10-09T05:23:53.289556+0000 2023-10-09T05:23:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: mgr.x (mgr.25453) 9 : 2023-10-09T05:23:53.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[101053]: cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:53.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: audit 2023-10-09T05:23:52 2023-10-09T05:23:53.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: .522027+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:53.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:23:53.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: 10-09T05:23:52.794281+0000 mon.a (mon.0) 763 : cluster [DBG] Standby manager daemon y started 2023-10-09T05:23:53.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: audit 2023-10-09T05:23: 2023-10-09T05:23:53.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: 52.798704+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:23:53.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: 2023-10-09T05:23:53.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: T05:23:52.799248+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: 23:52.803082+0000 mon.a (mon.0) 766 : audit [DBG] 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: 10-09T05:23:52.803636+0000 mon.a (mon.0) 767 : audit [DBG] 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: from='mgr.? 172.21.15.8:0/3881978668' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: 10-09T05:23:52.912924+0000 mon.c (mon.2) 124 : audit [DBG] 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: from='client.? 172.21.15.8:0/2453071470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: -10-09T05:23:53.289556+0000 mgr.x (mgr.25453) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:53 smithi008 conmon[110782]: 2023-10-09T05:23:54.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[139071]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:23:54] "GET /metrics HTTP/1.1" 200 31025 "" "Prometheus/2.33.4" 2023-10-09T05:23:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[101299]: cluster 2023-10-09T05:23:53.530212+0000 2023-10-09T05:23:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[101299]: mon.a (mon.0) 768 : cluster [DBG] mgrmap e24: x(active, since 4s), standbys: y 2023-10-09T05:23:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[101299]: audit 2023-10-09T05:23:53.531317+0000 mon.a 2023-10-09T05:23:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[101299]: (mon.0) 769 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:23:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[101299]: audit 2023-10-09T05:23:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[101299]: 2023-10-09T05:23:54.083715+0000 2023-10-09T05:23:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[101299]: mon.a (mon.0) 770 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[101299]: audit 2023-10-09T05:23:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[101299]: 2023-10-09T05:23: 2023-10-09T05:23:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[101299]: 54.087583+0000 mon.a (mon.0) 771 : audit [INF] 2023-10-09T05:23:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:54 smithi191 conmon[101299]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:54.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[101053]: cluster 2023-10-09T05:23:53.530212+0000 mon.a (mon.0) 768 : cluster [DBG] mgrmap e24: x(active, since 4s), standbys: y 2023-10-09T05:23:54.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[101053]: audit 2023 2023-10-09T05:23:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[101053]: -10-09T05:23:53.531317+0000 mon.a (mon.0) 769 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:23:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[101053]: audit 2023-10-09T05:23:54.083715+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[101053]: audit 2023-10-09T05:23:54.087583+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:54.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: cluster 2023-10-09T05:23 2023-10-09T05:23:54.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: :53.530212+0000 mon.a (mon.0) 768 : cluster [DBG] mgrmap e24: x(active, since 4s), standbys: y 2023-10-09T05:23:54.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:23:54.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: 09T05:23:53.531317+0000 mon.a ( 2023-10-09T05:23:54.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: mon.0) 769 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:23:54.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: 2023-10-09T05:23:54.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: audit 2023-10-09T05:23:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: 2023-10-09T05:23:54.083715+0000 2023-10-09T05:23:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: mon.a (mon.0) 770 : audit [INF] 2023-10-09T05:23:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:23:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: 23:54.087583+0000 mon.a (mon. 2023-10-09T05:23:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:54 smithi008 conmon[110782]: 0) 771 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[101053]: audit 2023-10-09T05:23:54.606023+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[101053]: audit 2023-10-09T05:23:54.610854+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[101053]: audit 2023-10-09T05:23:54.828242+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[101053]: audit 2023-10-09T05:23:54.832522+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[101053]: audit 2023-10-09T05:23:55.249468+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[101053]: audit 2023-10-09T05:23:55.254302+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[101053]: cluster 2023-10-09T05:23:55.289963+0000 mgr.x (mgr.25453) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:55.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[101053]: audit 2023-10-09T05:23:55.370783+0000 mon.a (mon.0) 778 : audit [DBG] from='client.? 172.21.15.8:0/1523665501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[110782]: audit 2023-10-09T05:23:54.606023+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[110782]: audit 2023-10-09T05:23:54.610854+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[110782]: audit 2023-10-09T05:23:54.828242+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[110782]: audit 2023-10-09T05:23:54.832522+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[110782]: audit 2023-10-09T05:23: 2023-10-09T05:23:55.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[110782]: 55.249468+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[110782]: audit 2023-10-09T05:23:55.254302+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[110782]: cluster 2023-10-09T05:23:55.289963+0000 mgr.x (mgr.25453) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:55.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:55 smithi008 conmon[110782]: audit 2023-10-09T05:23:55.370783+0000 mon.a (mon.0) 778 : audit [DBG] from='client.? 172.21.15.8:0/1523665501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:55.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: audit 2023-10-09T05:23:54.606023+0000 mon.a (mon. 2023-10-09T05:23:55.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: 0) 772 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: audit 2023-10-09T05:23:54.610854+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: audit 2023-10-09T05:23:54.828242+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: audit 2023-10-09T05:23:54.832522+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: audit 2023-10-09T05:23:55.249468 2023-10-09T05:23:55.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: +0000 mon.a (mon.0) 776 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: audit 2023-10-09T05:23:55. 2023-10-09T05:23:55.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: 254302+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:55.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: cluster 2023-10-09T05:23:55.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: 2023-10-09T05:23:55.289963 2023-10-09T05:23:55.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: +0000 mgr.x (mgr.25453) 10 : cluster [DBG] 2023-10-09T05:23:55.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:23:55.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: audit 2023-10-09T05:23 2023-10-09T05:23:55.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: :55.370783+0000 mon.a (mon.0) 778 : 2023-10-09T05:23:55.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:55 smithi191 conmon[101299]: audit [DBG] from='client.? 172.21.15.8:0/1523665501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:23:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:56 smithi008 conmon[101053]: audit 2023-10-09T05:23: 2023-10-09T05:23:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:56 smithi008 conmon[101053]: 55.874207+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:56 smithi008 conmon[110782]: audit 2023-10-09T05:23:55. 2023-10-09T05:23:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:56 smithi008 conmon[110782]: 874207+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:57.233 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:56 smithi191 conmon[101299]: audit 2023-10-09T05:23:55.874207+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:23:58.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:58 smithi008 conmon[101053]: cluster 2023-10-09T05:23:57.290465+0000 mgr.x (mgr.25453) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 235 KiB/s wr, 31 op/s 2023-10-09T05:23:58.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:58 smithi008 conmon[101053]: audit 2023-10-09T05:23:57.426302+0000 mon.a (mon.0) 780 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:23:58.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:58 smithi008 conmon[110782]: cluster 2023-10-09T05:23:57.290465+0000 mgr.x (mgr.25453) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 235 KiB/s wr, 31 op/s 2023-10-09T05:23:58.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:58 smithi008 conmon[110782]: audit 2023-10-09T05:23:57.426302+0000 mon.a (mon.0) 780 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:23:58.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:58 smithi191 conmon[101299]: cluster 2023-10-09T05:23:57.290465+0000 mgr.x (mgr.25453) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 235 KiB/s wr, 31 op/s 2023-10-09T05:23:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:58 smithi191 conmon[101299]: audit 2023-10-09T05:23:57.426302+0000 mon.a (mon.0) 780 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:24:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05:23:57.784446+0000 mon.b (mon. 2023-10-09T05:24:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: 1) 30 : audit [DBG] from='client.? 172.21.15.8:0/1681817582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05:23:58.325693+0000 mon.a (mon.0) 781 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:24:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05:23:58.326823+0000 mon.a (mon.0) 782 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:24:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05:23:58.330827+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:24:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05:23:58.331018+0000 mon.a (mon.0) 784 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:24:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05:23:58.336637+0000 mon.b (mon.1) 31 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:24:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05:23:58.339010+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05:23:58.339223+0000 mon.a (mon.0) 786 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05:23:58.339427+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: 23:58.339565+0000 mon.a (mon.0) 788 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05:23:58.844706+0000 mon.b (mon.1) 32 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: 05:23:58.846962+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: T05:23:58.847231+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T05:23:58.847478+0000 2023-10-09T05:24:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: mon.a (mon.0) 791 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:24:00.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:24:00.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: 05:23:58.850908+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:00.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:24:00.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: 05:23:58.852199+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:00.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: cluster 2023-10-09T05:24:00.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: 2023-10-09T05:23:59.290883+0000 mgr.x (mgr. 2023-10-09T05:24:00.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:23:59 smithi191 conmon[101299]: 25453) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-10-09T05:24:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10-09T05:23:57.784446+0000 2023-10-09T05:24:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.8:0/1681817582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10-09T05:23: 2023-10-09T05:24:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: 58.325693+0000 mon.a (mon.0) 781 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:24:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: 2023-10-09T05:24:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:24:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: -09T05:23:58.326823 2023-10-09T05:24:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: +0000 mon.a (mon.0) 782 2023-10-09T05:24:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:24:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:24:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: -09T05:23:58.330827 2023-10-09T05:24:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: +0000 mon.a (mon.0) 783 : audit [DBG] 2023-10-09T05:24:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:24:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: :23:58.331018+0000 mon.a (mon.0) 784 : audit [DBG] 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: 09T05:23:58.336637+0000 mon.b (mon.1) 31 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10-09T05:23:58.339010+0000 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: mon.a (mon.0) 785 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10-09T05:23:58.339223+0000 mon.a (mon. 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: 0) 786 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: 09T05:23:58.339427+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: 2023-10-09T05:24:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:24:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: T05:23:58.339565+0000 mon.a (mon.0) 788 : 2023-10-09T05:24:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:24:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10-09T05:24:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: 2023-10-09T05:23:58.844706+0000 mon.b 2023-10-09T05:24:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: (mon.1) 32 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:24:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:24:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: 05:23:58.846962+0000 mon.a (mon 2023-10-09T05:24:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: .0) 789 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:24:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:24:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: :23:58.847231+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:24:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: 2023-10-09T05:24:00.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10-09T05:23:58.847478+0000 2023-10-09T05:24:00.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: mon.a (mon.0) 791 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:24:00.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:24:00.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: :23:57.784446+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.8:0/1681817582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:00.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023 2023-10-09T05:24:00.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: -10-09T05:23:58.325693+0000 mon.a ( 2023-10-09T05:24:00.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: mon.0) 781 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:24:00.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10-09T05:23 2023-10-09T05:24:00.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: :58.326823+0000 mon.a (mon.0) 782 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10-09T05:23:58.330827+0000 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: mon.a (mon.0) 783 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: -10-09T05:23:58.331018+0000 mon.a ( 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: mon.0) 784 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 2023-10-09T05:23: 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 58.336637+0000 mon.b (mon.1) 31 : audit 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 09T05:23:58.339010+0000 mon.a (mon.0 2023-10-09T05:24:00.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: ) 785 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:00.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10-09T05:24:00.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 2023-10-09T05:23:58.339223+0000 2023-10-09T05:24:00.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: mon.a (mon.0) 786 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:24:00.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:24:00.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 05:23:58.339427+0000 mon.a (mon.0) 787 2023-10-09T05:24:00.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:24:00.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:24:00.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 09T05:23:58.339565+0000 mon.a (mon. 2023-10-09T05:24:00.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 0) 788 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 23:58.844706+0000 mon.b (mon.1) 32 : 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 2023-10-09T05:23 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: :58.846962+0000 mon.a (mon.0) 789 : 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 09T05:23:58.847231+0000 mon.a (mon 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: .0) 790 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:24:00.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:24:00.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 10-09T05:23:58.847478+0000 mon.a (mon.0) 2023-10-09T05:24:00.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:24:00.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: -09T05:23:58.850908+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:00.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: audit 2023-10-09T05:23:58 2023-10-09T05:24:00.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: .852199+0000 mon.a (mon.0) 793 : audit [INF] 2023-10-09T05:24:00.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:00.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:24:00.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: -10-09T05:23:59.290883+0000 mgr.x (mgr.25453) 12 : cluster 2023-10-09T05:24:00.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[101053]: [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-10-09T05:24:00.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 791 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:24:00.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:24:00.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: -09T05:23:58.850908+0000 mon.a ( 2023-10-09T05:24:00.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: mon.0) 792 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:00.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: audit 2023-10-09T05:24:00.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: 2023-10-09T05:23:58.852199+0000 2023-10-09T05:24:00.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: mon.a (mon.0) 793 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:00.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:24:00.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: -10-09T05:23:59.290883+0000 mgr.x ( 2023-10-09T05:24:00.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:23:59 smithi008 conmon[110782]: mgr.25453) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-10-09T05:24:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05:23:58.846740+0000 mon.c 2023-10-09T05:24:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: (mon.2) 125 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:24:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05:23:59.742314+0000 mon.c (mon.2) 126 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:24:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05:23:59.748006+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05:23:59.749970+0000 mon.a (mon.0) 795 : audit [INF] 2023-10-09T05:24:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023 2023-10-09T05:24:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: -10-09T05:23:59.762672+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05:23 2023-10-09T05:24:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: :59.764045+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05:23: 2023-10-09T05:24:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: 59.764218+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05:23:59. 2023-10-09T05:24:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: 764785+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05:23:59 2023-10-09T05:24:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: .764927+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: cephadm 2023-10-09T05:23: 2023-10-09T05:24:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: 59.765357+0000 mgr.x (mgr.25453) 13 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 2501M 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: 23:59.765967+0000 mon.a (mon.0) 801 : audit [INF] 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: :23:59.772675+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: 10-09T05:23:59.774069+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:59. 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: 774756+0000 mgr.x (mgr.25453) 14 : cephadm [INF] Adjusting osd_memory_target on smithi191 to 2500M 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05:23:59 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: .779023+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: 05:23:59.780514+0000 mon.a (mon.0) 805 : audit 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: -09T05:23:59.781432+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:59. 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: 782980+0000 mgr.x (mgr.25453) 15 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: cephadm 2023-10-09T05:23:59.783175+0000 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: mgr.x (mgr.25453) 16 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:00.047287 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: +0000 mgr.x (mgr.25453) 17 : cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:00.047758+0000 2023-10-09T05:24:01.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: mgr.x (mgr.25453) 18 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:24:01.054 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: audit 2023-10-09T05:24:00.213292+0000 mon.c ( 2023-10-09T05:24:01.054 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:00 smithi191 conmon[101299]: mon.2) 127 : audit [DBG] from='client.? 172.21.15.8:0/1482361851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:01.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10-09T05:23:58.846740+0000 mon.c (mon.2) 125 : 2023-10-09T05:24:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:24:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:24:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 05:23:59.742314+0000 mon.c (mon.2) 126 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:24:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023 2023-10-09T05:24:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: -10-09T05:23:59 2023-10-09T05:24:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: .748006+0000 mon.a (mon. 2023-10-09T05:24:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 0) 794 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:24:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10-09T05:24:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:23 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: :59.749970+0000 mon.a ( 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: mon.0) 795 : audit [INF] 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: -09T05:23:59 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: .762672+0000 mon.a (mon.0 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: ) 796 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: -10-09T05:23:59 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: .764045+0000 mon.a (mon.0 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: ) 797 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 05:23:59.764218+0000 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: mon.a (mon.0) 798 : 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:23:59.764785 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: +0000 mon.a (mon.0) 799 : audit [INF] 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:23: 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 59.764927+0000 mon.a (mon. 2023-10-09T05:24:01.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 0) 800 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05: 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 23:59.765357+0000 mgr.x ( 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: mgr.25453) 13 : cephadm 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: [INF] Adjusting osd_memory_target on smithi008 to 2501M 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: -10-09T05:23:59.765967+0000 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: mon.a (mon.0) 801 : 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023 2023-10-09T05:24:01.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: -10-09T05:23: 2023-10-09T05:24:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023-10-09T05:23 2023-10-09T05:24:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: :58.846740+0000 mon.c (mon.2) 125 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:24:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023-10-09T05:24:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 2023-10-09T05:23:59.742314+0000 2023-10-09T05:24:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: mon.c (mon.2) 126 : audit 2023-10-09T05:24:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:24:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: -09T05:23:59.748006+0000 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: mon.a (mon.0) 794 : audit 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 10-09T05:23:59.749970+0000 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: mon.a (mon.0) 795 : audit 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: -09T05:23:59.762672+0000 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: mon.a (mon.0) 796 : audit 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 10-09T05:23:59.764045 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: +0000 mon.a (mon.0) 797 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 10-09T05:23:59.764218 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: +0000 mon.a (mon.0) 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 798 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 2023-10-09 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: T05:23:59.764785+0000 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: mon.a (mon.0) 799 : audit 2023-10-09T05:24:01.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023-10-09T05:24:01.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 2023-10-09 2023-10-09T05:24:01.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: T05:23:59.764927+0000 2023-10-09T05:24:01.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: mon.a (mon.0) 800 : 2023-10-09T05:24:01.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: cephadm 2023- 2023-10-09T05:24:01.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 10-09T05:23:59.765357 2023-10-09T05:24:01.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: +0000 mgr.x (mgr.25453) 2023-10-09T05:24:01.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 13 : cephadm [INF] Adjusting osd_memory_target on smithi008 to 2501M 2023-10-09T05:24:01.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023 2023-10-09T05:24:01.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: -10-09T05:23 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: :59.765967+0000 mon.a ( 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: mon.0) 801 : audit 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 2023-10-09T05 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: :23:59.772675+0000 mon.a 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: (mon.0) 802 : audit 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 23:59.774069+0000 2023-10-09T05:24:01.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: mon.a (mon.0 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: ) 803 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: cephadm 2023-10-09T 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 05:23:59.774756+0000 mgr.x (mgr.25453) 14 : cephadm 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: [INF] Adjusting osd_memory_target on smithi191 to 2500M 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 2023-10-09T05:23 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: :59.779023+0000 mon.a ( 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: mon.0) 804 : audit [INF] 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 2023-10-09T05:23:59.780514 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: +0000 mon.a (mon.0) 805 : audit [DBG] 2023-10-09T05:24:01.224 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 23:59.781432+0000 mon.a (mon.0) 806 : 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: cephadm 2023-10-09 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: T05:23:59.782980+0000 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: mgr.x (mgr.25453) 15 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: cephadm 2023- 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 10-09T05:23:59.783175 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: +0000 mgr.x (mgr.25453) 16 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:24:01.225 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: cephadm 2023 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: -10-09T05:24:00. 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 047287+0000 mgr.x (mgr.25453 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: ) 17 : cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: cephadm 2023-10- 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 09T05:24:00. 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 047758+0000 mgr.x (mgr.25453) 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 18 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: audit 2023 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: -10-09T05:24:00 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: .213292+0000 mon.c (mon. 2023-10-09T05:24:01.226 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 2) 127 : audit [DBG] from='client.? 172.21.15.8:0/1482361851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:01.227 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[110782]: 2023-10-09T05:24:01.227 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 59.772675+0000 mon.a (mon. 2023-10-09T05:24:01.227 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 0) 802 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.227 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:24:01.227 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10-09T05:24:01.227 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10- 2023-10-09T05:24:01.227 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 09T05:23:59.774069+0000 2023-10-09T05:24:01.227 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: mon.a (mon.0) 803 2023-10-09T05:24:01.227 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:24:01.227 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:23: 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 59.774756+0000 mgr.x (mgr 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: .25453) 14 : cephadm [INF] 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: Adjusting osd_memory_target on smithi191 to 2500M 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:23: 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 59.779023+0000 mon.a (mon 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: .0) 804 : audit 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: T05:23:59.780514 2023-10-09T05:24:01.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: +0000 mon.a (mon.0 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: ) 805 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: -09T05:23:59. 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 781432+0000 mon.a (mon.0) 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 806 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:23: 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 59.782980+0000 mgr.x (mgr 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: .25453) 15 : cephadm 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:24:01.229 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:23: 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 59.783175+0000 mgr.x (mgr 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: .25453) 16 : cephadm 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10- 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 09T05:24:00 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: .047287+0000 mgr.x (mgr 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: .25453) 17 : cephadm 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:01.230 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T05:24 2023-10-09T05:24:01.231 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: :00.047758+0000 mgr.x 2023-10-09T05:24:01.231 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: (mgr.25453) 18 : 2023-10-09T05:24:01.231 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:24:01.231 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: audit 2023-10-09T05:24:01.231 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 2023-10-09T 2023-10-09T05:24:01.231 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 05:24:00.213292 2023-10-09T05:24:01.231 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: +0000 mon.c (mon.2) 2023-10-09T05:24:01.231 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:00 smithi008 conmon[101053]: 127 : audit [DBG] from='client.? 172.21.15.8:0/1482361851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:01.656 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:01 smithi008 systemd[1]: Stopping Ceph node-exporter.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:01.656 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:01 smithi008 bash[174210]: Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-node-exporter.a found: no such container 2023-10-09T05:24:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: cephadm 2023-10-09T05: 2023-10-09T05:24:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: 24:00.312129+0000 mgr.x (mgr.25453) 19 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:24:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: cephadm 2023- 2023-10-09T05:24:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: 10-09T05:24:00.312380+0000 mgr.x 2023-10-09T05:24:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: (mgr.25453) 20 : cephadm 2023-10-09T05:24:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:24:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: cephadm 2023-10 2023-10-09T05:24:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: -09T05:24:00.576750+0000 mgr.x ( 2023-10-09T05:24:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: mgr.25453) 21 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:24:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: cephadm 2023- 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: 10-09T05:24:00.576944+0000 mgr.x (mgr.25453) 22 : 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: audit 2023-10-09T05:24:00.795005+0000 mon.a ( 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: mon.0) 807 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: audit 2023-10-09T05:24:00. 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: 799259+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: 05:24:00.845252+0000 mon.a (mon.0) 809 : audit 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: audit 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: 2023-10-09T05:24:00 2023-10-09T05:24:01.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: .850780+0000 mon.a (mon.0) 810 : audit [INF] 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: audit 2023-10-09T05:24:00. 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: 856124+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: audit 2023-10-09T05:24:00.925765 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: +0000 mon.a (mon.0) 812 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: audit 2023-10-09T05:24:00. 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: 931128+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: audit 2023-10-09T05:24: 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: 00.936962+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: audit 2023-10-09T05:24 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: :00.942070+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: cephadm 2023-10-09T05:24 2023-10-09T05:24:01.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: :00.943013+0000 mgr.x (mgr.25453) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T05:24:01.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: cephadm 2023-10-09T05: 2023-10-09T05:24:01.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: 24:00.943387+0000 mgr.x (mgr.25453) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi008 2023-10-09T05:24:01.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: 2023-10-09T05:24:01.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: cluster 2023-10-09T05 2023-10-09T05:24:01.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[110782]: :24:01.291532+0000 mgr.x (mgr.25453) 25 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-10-09T05:24:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:00. 2023-10-09T05:24:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: 312129+0000 mgr.x (mgr.25453) 19 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:24:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:00.312380+0000 mgr.x (mgr.25453) 20 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:24:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:00.576750+0000 mgr.x (mgr.25453) 21 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:24:01.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:00.576944+0000 mgr.x (mgr.25453) 22 : cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: audit 2023-10-09T05:24:00.795005+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: 09T05:24:00.799259+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: audit 2023-10-09T05:24:00.845252+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: audit 2023-10-09T05:24:00.850780+0000 mon.a (mon.0) 810 : audit 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: :24:00.856124+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: audit 2023-10-09T05:24:00.925765+0000 mon.a 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: (mon.0) 812 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: audit 2023-10-09T05:24:00.931128+0000 mon.a (mon. 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: 0) 813 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: audit 2023-10-09T05:24:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: 2023-10-09T05:24:00.936962+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: 2023-10-09T05:24:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: audit 2023-10-09T05:24 2023-10-09T05:24:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: :00.942070+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: 2023-10-09T05:24:00.943013+0000 mgr.x (mgr.25453) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T05:24:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: cephadm 2023-10-09T05:24: 2023-10-09T05:24:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: 00.943387+0000 mgr.x (mgr.25453) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi008 2023-10-09T05:24:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: cluster 2023-10-09T05:24:01.291532+0000 mgr.x 2023-10-09T05:24:01.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:01 smithi008 conmon[101053]: (mgr.25453) 25 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-10-09T05:24:02.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:00.312129+0000 mgr.x (mgr.25453) 19 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:24:02.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:00.312380+0000 mgr.x (mgr.25453) 20 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:00.576750+0000 mgr.x (mgr.25453) 21 : cephadm [INF] 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:00.576944+0000 mgr.x (mgr.25453) 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: 22 : cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: audit 2023-10-09T05:24 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: :00.795005+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: audit 2023-10-09T05:24:00.799259 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: +0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: audit 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: 2023-10-09T05:24:00.845252+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: audit 2023-10-09T05:24:00.850780 2023-10-09T05:24:02.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: +0000 mon.a (mon.0) 810 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: audit 2023-10-09T05:24:00.856124+0000 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: mon.a (mon.0) 811 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: audit 2023-10-09T05:24:00.925765+0000 mon.a ( 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: mon.0) 812 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: audit 2023-10-09T05:24:00.931128+0000 mon.a (mon.0) 813 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: 10-09T05:24:00.936962+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: audit 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: 2023-10-09T05:24:00.942070+0000 mon.a (mon. 2023-10-09T05:24:02.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: 0) 815 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:02.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: cephadm 2023- 2023-10-09T05:24:02.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: 10-09T05:24:00.943013+0000 mgr.x (mgr. 2023-10-09T05:24:02.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: 25453) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T05:24:02.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: cephadm 2023-10-09T05 2023-10-09T05:24:02.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: :24:00.943387+0000 mgr.x (mgr.25453) 24 : cephadm 2023-10-09T05:24:02.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: [INF] Reconfiguring daemon node-exporter.a on smithi008 2023-10-09T05:24:02.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: cluster 2023-10-09T05:24:02.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: 2023-10-09T05:24:01.291532+0000 mgr.x (mgr.25453) 2023-10-09T05:24:02.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:01 smithi191 conmon[101299]: 25 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-10-09T05:24:02.714 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:02 smithi008 bash[174327]: ec044fd127f69e102100b2227d1907773aa174965ce1d359c873b3d61522967e 2023-10-09T05:24:02.715 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:02 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:24:02.715 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:02 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T05:24:02.715 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:02 smithi008 systemd[1]: Stopped Ceph node-exporter.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:02.715 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:02 smithi008 systemd[1]: Starting Ceph node-exporter.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:03.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:24:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:24:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[101053]: audit 2023-10-09T05:24:02.654181+0000 mon.a 2023-10-09T05:24:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[101053]: (mon.0) 816 : audit [DBG] from='client.? 172.21.15.8:0/932205117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[101053]: audit 2023-10-09T05:24:03.065460+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[101053]: audit 2023-10-09T05:24:03.069115 2023-10-09T05:24:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[101053]: +0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:03.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[101053]: 2023-10-09T05:24:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:24:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: 24:02.654181+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.8:0/932205117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: audit 2023-10-09T05:24:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: 2023-10-09T05:24: 2023-10-09T05:24:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: 03.065460+0000 mon.a (mon 2023-10-09T05:24:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: .0) 817 : audit [INF] 2023-10-09T05:24:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: audit 2023 2023-10-09T05:24:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: -10-09T05:24:03 2023-10-09T05:24:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: .069115+0000 mon.a (mon.0 2023-10-09T05:24:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: ) 818 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:03.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[110782]: 2023-10-09T05:24:03.467 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.005Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T05:24:03.467 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.005Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T05:24:03.467 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T05:24:03.467 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:24:03.467 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T05:24:03.467 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T05:24:03.467 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T05:24:03.467 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T05:24:03.467 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.006Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T05:24:03.468 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T05:24:03.469 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T05:24:03.470 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T05:24:03.471 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T05:24:03.471 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T05:24:03.471 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T05:24:03.471 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 conmon[174716]: ts=2023-10-09T05:24:03.007Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T05:24:03.471 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 bash[174673]: 0b0d19fe4991526ba7d8d317472327fcdade1f66addd61b34db3a862e3e88c0e 2023-10-09T05:24:03.471 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 05:24:03 smithi008 systemd[1]: Started Ceph node-exporter.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:03.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:03 smithi191 conmon[101299]: audit 2023-10-09T05:24:02.654181+0000 mon.a (mon.0) 816 : audit [DBG] 2023-10-09T05:24:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:03 smithi191 conmon[101299]: from='client.? 172.21.15.8:0/932205117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:03 smithi191 conmon[101299]: audit 2023-10-09T05:24:03.065460+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:03 smithi191 conmon[101299]: audit 2023-10-09T05:24:03 2023-10-09T05:24:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:03 smithi191 conmon[101299]: .069115+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:04.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:04 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:03.112462+0000 mgr.x (mgr.25453) 26 : cephadm [INF] 2023-10-09T05:24:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:04 smithi191 conmon[101299]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:24:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:04 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:03.124233+0000 mgr.x (mgr.25453) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi008 2023-10-09T05:24:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:04 smithi191 conmon[101299]: cluster 2023-10-09T05:24:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:04 smithi191 conmon[101299]: 2023-10-09T05:24:03.291941+0000 mgr.x (mgr.25453) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-10-09T05:24:04.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:04 smithi008 conmon[101053]: cephadm 2023-10-09T05: 2023-10-09T05:24:04.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:04 smithi008 conmon[101053]: 24:03.112462+0000 mgr.x (mgr.25453) 26 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:24:04.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:04 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:03.124233+0000 mgr.x (mgr.25453) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi008 2023-10-09T05:24:04.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:04 smithi008 conmon[101053]: cluster 2023-10-09T05:24:03.291941+0000 mgr.x (mgr.25453) 28 : cluster 2023-10-09T05:24:04.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:04 smithi008 conmon[101053]: [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-10-09T05:24:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:04 smithi008 conmon[110782]: cephadm 2023-10-09T05:24:03.112462+0000 mgr.x 2023-10-09T05:24:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:04 smithi008 conmon[110782]: (mgr.25453) 26 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:24:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:04 smithi008 conmon[110782]: cephadm 2023-10-09T05:24:03.124233+0000 mgr.x (mgr.25453) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi008 2023-10-09T05:24:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:04 smithi008 conmon[110782]: cluster 2023-10-09T05:24:03.291941+0000 mgr.x (mgr.25453) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-10-09T05:24:05.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:04 smithi191 conmon[139071]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:24:04] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.33.4" 2023-10-09T05:24:05.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:05 smithi191 conmon[101299]: audit 2023-10-09T05:24:04.494753+0000 mon.a (mon.0) 819 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:05 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:24:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:05 smithi191 conmon[101299]: :24:05.065989+0000 mon.a (mon.0) 820 : audit [DBG] from='client.? 172.21.15.8:0/2884960082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:05.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:05 smithi008 conmon[101053]: audit 2023-10-09T05:24:04.494753+0000 mon.a (mon. 2023-10-09T05:24:05.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:05 smithi008 conmon[101053]: 0) 819 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:05.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:05 smithi008 conmon[101053]: audit 2023-10-09T05:24:05.065989+0000 mon.a (mon.0) 820 : audit [DBG] from='client.? 172.21.15.8:0/2884960082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:05 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:24:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:05 smithi008 conmon[110782]: 24:04.494753+0000 mon.a (mon.0) 819 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:05 smithi008 conmon[110782]: audit 2023 2023-10-09T05:24:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:05 smithi008 conmon[110782]: -10-09T05:24:05.065989 2023-10-09T05:24:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:05 smithi008 conmon[110782]: +0000 mon.a (mon.0) 820 2023-10-09T05:24:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:05 smithi008 conmon[110782]: : audit [DBG] from='client.? 172.21.15.8:0/2884960082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:06.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:06 smithi191 conmon[101299]: cluster 2023-10-09T05:24:05.292311+0000 mgr.x (mgr.25453) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T05:24:06.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:06 smithi008 conmon[101053]: cluster 2023-10-09T05:24:05.292311+0000 mgr.x (mgr.25453) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T05:24:06.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:06 smithi008 conmon[101053]: 2023-10-09T05:24:06.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:06 smithi008 conmon[110782]: cluster 2023-10-09T05:24: 2023-10-09T05:24:06.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:06 smithi008 conmon[110782]: 05.292311+0000 mgr.x (mgr.25453) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T05:24:08.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:08 smithi191 conmon[101299]: cluster 2023-10-09T05:24:07.292979+0000 mgr.x (mgr.25453) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T05:24:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:08 smithi191 conmon[101299]: 2023-10-09T05:24:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:08 smithi191 conmon[101299]: audit 2023-10-09T05:24:07.478421+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.8:0/2174291527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:08.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:08 smithi008 conmon[101053]: cluster 2023-10-09T05:24:07.292979+0000 mgr.x ( 2023-10-09T05:24:08.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:08 smithi008 conmon[101053]: mgr.25453) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T05:24:08.661 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:08 smithi008 conmon[101053]: audit 2023-10-09T05:24:07.478421+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.8:0/2174291527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:08.661 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:08 smithi008 conmon[110782]: cluster 2023-10-09T05:24:07.292979+0000 mgr.x 2023-10-09T05:24:08.662 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:08 smithi008 conmon[110782]: (mgr.25453) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T05:24:08.662 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:08 smithi008 conmon[110782]: audit 2023-10-09T05:24:07.478421+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.8:0/2174291527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:09 smithi008 conmon[101053]: cluster 2023-10-09T05:24:09.293416+0000 mgr.x (mgr.25453 2023-10-09T05:24:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:09 smithi008 conmon[101053]: ) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T05:24:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:09 smithi008 conmon[110782]: cluster 2023-10-09T05:24:09 2023-10-09T05:24:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:09 smithi008 conmon[110782]: .293416+0000 mgr.x (mgr.25453) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T05:24:09.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:09 smithi191 conmon[101299]: cluster 2023-10-09T05:24:09.293416+0000 mgr.x (mgr.25453) 31 : cluster [DBG] 2023-10-09T05:24:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:09 smithi191 conmon[101299]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T05:24:10.145 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:09 smithi008 systemd[1]: Stopping Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:10.145 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:09 smithi008 bash[175248]: Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-alertmanager.a found: no such container 2023-10-09T05:24:10.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:10 smithi008 conmon[101053]: audit 2023-10-09T05:24:09.889513+0000 mon.c 2023-10-09T05:24:10.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:10 smithi008 conmon[101053]: (mon.2) 129 : audit [DBG] from='client.? 172.21.15.8:0/394861846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:10.465 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:10 smithi008 conmon[151620]: level=info ts=2023-10-09T05:24:10.145Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:24:10.465 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:10 smithi008 bash[175369]: 50083941a19709d3ff3906ce448387e3291040008ae4264f75bbcd1cb94d1201 2023-10-09T05:24:10.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:10 smithi008 conmon[110782]: audit 2023-10-09T05:24:09.889513+0000 mon.c 2023-10-09T05:24:10.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:10 smithi008 conmon[110782]: (mon.2) 129 : audit [DBG] from='client.? 172.21.15.8:0/394861846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:10.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:10 smithi191 conmon[101299]: audit 2023-10-09T05:24:09.889513+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.8:0/394861846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:11.214 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:10 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T05:24:11.214 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:10 smithi008 systemd[1]: Stopped Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:11.214 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:10 smithi008 systemd[1]: Starting Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:11.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[101053]: cluster 2023-10-09T05:24:11 2023-10-09T05:24:11.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[101053]: .294239+0000 mgr.x (mgr.25453) 32 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T05:24:11.607 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[110782]: cluster 2023-10-09T05:24:11 2023-10-09T05:24:11.607 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[110782]: .294239+0000 mgr.x (mgr.25453) 32 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T05:24:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:11 smithi191 conmon[101299]: cluster 2023-10-09T05:24:11.294239+0000 mgr.x (mgr.25453) 32 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T05:24:11.964 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[175748]: level=info ts=2023-10-09T05:24:11.607Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T05:24:11.964 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[175748]: level=info ts=2023-10-09T05:24:11.607Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T05:24:11.965 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[175748]: level=info ts=2023-10-09T05:24:11.608Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.8 port=9094 2023-10-09T05:24:11.965 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[175748]: level=info ts=2023-10-09T05:24:11.609Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T05:24:11.965 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:11 smithi008 bash[175707]: 172e511a984d8f143d866d4460c2dcae82ad2ebf33dab50c4132adc1992badb8 2023-10-09T05:24:11.965 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:11 smithi008 systemd[1]: Started Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:11.965 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[175748]: level=info ts=2023-10-09T05:24:11.642Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:24:11.965 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[175748]: level=info ts=2023-10-09T05:24:11.642Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:24:11.965 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[175748]: level=info ts=2023-10-09T05:24:11.645Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T05:24:11.965 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:11 smithi008 conmon[175748]: level=info ts=2023-10-09T05:24:11.645Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T05:24:12.649 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[134411]: ts=2023-10-09T05:24:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.8:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:24:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: audit 2023-10-09T05:24:11.649903+0000 2023-10-09T05:24:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: mon.a (mon.0) 821 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: audit 2023-10-09T05:24:11.653811+0000 2023-10-09T05:24:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: mon.a (mon.0) 822 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:11 2023-10-09T05:24:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: .667338+0000 mgr.x (mgr.25453) 2023-10-09T05:24:12.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: 33 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:24:12.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: cephadm 2023-10 2023-10-09T05:24:12.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: -09T05:24:11.886967+0000 mgr.x (mgr 2023-10-09T05:24:12.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: .25453) 34 : cephadm [INF] 2023-10-09T05:24:12.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:24:12.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:24:12.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: 10-09T05:24:12.289407 2023-10-09T05:24:12.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: +0000 mon.c (mon.2) 2023-10-09T05:24:12.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[101053]: 130 : audit [DBG] from='client.? 172.21.15.8:0/4077753671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:12.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: audit 2023-10-09T05:24: 2023-10-09T05:24:12.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: 11.649903+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:12.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: audit 2023-10-09T05:24:11.653811+0000 mon.a (mon.0) 822 : audit [INF] 2023-10-09T05:24:12.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:12.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: cephadm 2023-10-09T05:24 2023-10-09T05:24:12.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: :11.667338+0000 mgr.x (mgr.25453) 33 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:24:12.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: cephadm 2023-10-09T05:24 2023-10-09T05:24:12.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: :11.886967+0000 mgr.x (mgr.25453) 2023-10-09T05:24:12.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: 34 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:24:12.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:24:12.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: T05:24:12.289407+0000 mon.c ( 2023-10-09T05:24:12.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:12 smithi008 conmon[110782]: mon.2) 130 : audit [DBG] from='client.? 172.21.15.8:0/4077753671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:13.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: audit 2023-10-09T05:24:11.649903+0000 mon.a (mon.0) 821 : audit 2023-10-09T05:24:13.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: audit 2023-10-09T05:24:11.653811+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: 2023-10-09T05:24: 2023-10-09T05:24:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: 11.667338+0000 mgr.x (mgr.25453) 33 : cephadm 2023-10-09T05:24:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:24:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: cephadm 2023-10 2023-10-09T05:24:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: -09T05:24:11.886967+0000 mgr.x (mgr 2023-10-09T05:24:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: .25453) 34 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:24:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:24:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: -09T05:24:12.289407+0000 mon.c ( 2023-10-09T05:24:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:12 smithi191 conmon[101299]: mon.2) 130 : audit [DBG] from='client.? 172.21.15.8:0/4077753671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:13.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:13 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:24:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:13 smithi008 conmon[101053]: cluster 2023-10-09T05:24:13.294567+0000 mgr.x (mgr.25453) 35 : cluster [DBG] 2023-10-09T05:24:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:13 smithi008 conmon[101053]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:13.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:13 smithi008 conmon[110782]: cluster 2023-10-09T05:24:13.294567+0000 mgr.x (mgr.25453) 35 : cluster [DBG] 2023-10-09T05:24:13.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:13 smithi008 conmon[110782]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:13.965 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:13 smithi008 conmon[175748]: level=info ts=2023-10-09T05:24:13.610Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000185716s 2023-10-09T05:24:14.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:13 smithi191 conmon[101299]: cluster 2023-10-09T05:24:13.294567+0000 mgr.x (mgr.25453) 35 2023-10-09T05:24:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:13 smithi191 conmon[101299]: : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:15.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:14 smithi191 conmon[139071]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:24:14] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.33.4" 2023-10-09T05:24:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:14 smithi191 conmon[101299]: audit 2023-10-09T05:24:14.694080+0000 mon.a (mon. 2023-10-09T05:24:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:14 smithi191 conmon[101299]: 0) 823 : audit [DBG] from='client.? 172.21.15.8:0/1260453351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:14 smithi008 conmon[101053]: audit 2023-10-09T05:24:14. 2023-10-09T05:24:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:14 smithi008 conmon[101053]: 694080+0000 mon.a (mon.0) 823 : audit [DBG] from='client.? 172.21.15.8:0/1260453351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:14 smithi008 conmon[110782]: audit 2023-10-09T05:24:14.694080+0000 mon.a ( 2023-10-09T05:24:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:14 smithi008 conmon[110782]: mon.0) 823 : audit [DBG] from='client.? 172.21.15.8:0/1260453351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:16.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:15 smithi191 conmon[101299]: cluster 2023-10-09T05:24:15.294864 2023-10-09T05:24:16.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:15 smithi191 conmon[101299]: +0000 mgr.x (mgr.25453) 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:16.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:15 smithi008 conmon[110782]: cluster 2023-10-09T05:24:15.294864+0000 mgr.x 2023-10-09T05:24:16.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:15 smithi008 conmon[110782]: (mgr.25453) 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:16.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:15 smithi008 conmon[101053]: cluster 2023-10-09T05:24:15.294864+0000 mgr.x (mgr.25453) 36 : 2023-10-09T05:24:16.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:15 smithi008 conmon[101053]: cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:17.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:17 smithi008 conmon[101053]: audit 2023-10-09T05:24:17.099063+0000 2023-10-09T05:24:17.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:17 smithi008 conmon[101053]: mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.8:0/3404255595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:17 smithi008 conmon[110782]: audit 2023-10-09T05:24:17.099063+0000 2023-10-09T05:24:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:17 smithi008 conmon[110782]: mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.8:0/3404255595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:17.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:17 smithi191 conmon[101299]: audit 2023-10-09T05:24:17.099063+0000 mon.c ( 2023-10-09T05:24:17.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:17 smithi191 conmon[101299]: mon.2) 131 : audit [DBG] from='client.? 172.21.15.8:0/3404255595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:18.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:18 smithi191 conmon[101299]: cluster 2023-10-09T05:24:17.295286+0000 mgr.x (mgr.25453) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:18.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:18 smithi008 conmon[101053]: cluster 2023-10-09T05:24:17.295286+0000 mgr.x (mgr. 2023-10-09T05:24:18.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:18 smithi008 conmon[101053]: 25453) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:18 smithi008 conmon[110782]: cluster 2023-10-09T05:24:17.295286+0000 mgr.x 2023-10-09T05:24:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:18 smithi008 conmon[110782]: (mgr.25453) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:19.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:19 smithi191 conmon[101299]: cluster 2023-10-09T05:24:19.295636+0000 mgr.x (mgr.25453) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:19 smithi008 conmon[101053]: cluster 2023-10-09T05:24:19 2023-10-09T05:24:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:19 smithi008 conmon[101053]: .295636+0000 mgr.x (mgr.25453) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:19 smithi008 conmon[110782]: cluster 2023-10-09T05:24:19.295636+0000 2023-10-09T05:24:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:19 smithi008 conmon[110782]: mgr.x (mgr.25453) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:20.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:20 smithi008 conmon[101053]: audit 2023-10-09T05:24:19. 2023-10-09T05:24:20.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:20 smithi008 conmon[101053]: 494824+0000 mon.a (mon.0) 824 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:20.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:20 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:24:20.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:20 smithi008 conmon[101053]: :24:19.505471+0000 mon.c (mon.2) 132 : audit [DBG] from='client.? 172.21.15.8:0/3980180101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:20.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:20 smithi008 conmon[110782]: audit 2023-10-09T05:24:19.494824+0000 mon.a (mon.0) 2023-10-09T05:24:20.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:20 smithi008 conmon[110782]: 824 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:20.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:20 smithi008 conmon[110782]: audit 2023-10-09T05:24:19.505471+0000 mon.c (mon.2 2023-10-09T05:24:20.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:20 smithi008 conmon[110782]: ) 132 : audit [DBG] from='client.? 172.21.15.8:0/3980180101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:20.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:20 smithi191 conmon[101299]: audit 2023-10-09T05:24:19.494824+0000 2023-10-09T05:24:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:20 smithi191 conmon[101299]: mon.a (mon.0) 824 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:20 smithi191 conmon[101299]: audit 2023-10-09T05:24:19.505471+0000 mon.c (mon.2) 132 : audit [DBG] from='client.? 172.21.15.8:0/3980180101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:21.612 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:21 smithi008 conmon[101053]: cluster 2023-10-09T05:24:21.296355+0000 mgr.x (mgr.25453) 39 2023-10-09T05:24:21.612 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:21 smithi008 conmon[101053]: : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:21.613 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:21 smithi008 conmon[110782]: cluster 2023-10-09T05:24:21.296355+0000 mgr.x (mgr.25453) 39 2023-10-09T05:24:21.613 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:21 smithi008 conmon[110782]: : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:21 smithi191 conmon[101299]: cluster 2023-10-09T05:24:21.296355+0000 mgr.x (mgr.25453) 39 : cluster [DBG] 2023-10-09T05:24:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:21 smithi191 conmon[101299]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:21.964 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 05:24:21 smithi008 conmon[175748]: level=info ts=2023-10-09T05:24:21.613Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003070473s 2023-10-09T05:24:22.359 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:22 smithi191 conmon[134411]: ts=2023-10-09T05:24:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.8:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:24:22.636 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:22 smithi191 conmon[101299]: audit 2023-10-09T05:24:21.912015+0000 mon.a (mon. 2023-10-09T05:24:22.636 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:22 smithi191 conmon[101299]: 0) 825 : audit [DBG] from='client.? 172.21.15.8:0/1812632115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:22.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:22 smithi008 conmon[101053]: audit 2023-10-09T05:24:21.912015+0000 mon.a ( 2023-10-09T05:24:22.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:22 smithi008 conmon[101053]: mon.0) 825 : audit [DBG] from='client.? 172.21.15.8:0/1812632115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:22 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:24:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:22 smithi008 conmon[110782]: :24:21.912015+0000 mon.a (mon.0) 825 : audit [DBG] from='client.? 172.21.15.8:0/1812632115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:23.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:23 smithi008 conmon[101509]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:24:23.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:23 smithi008 conmon[101053]: cluster 2023-10-09T05:24:23.296782+0000 mgr.x (mgr.25453) 40 : cluster 2023-10-09T05:24:23.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:23 smithi008 conmon[101053]: [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:23.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:23 smithi008 conmon[110782]: cluster 2023-10-09T05:24:23.296782+0000 mgr.x 2023-10-09T05:24:23.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:23 smithi008 conmon[110782]: (mgr.25453) 40 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:23.666 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 systemd[1]: Stopping Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:23.666 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 bash[144473]: Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-prometheus.a found: no such container 2023-10-09T05:24:23.666 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[101299]: cluster 2023-10-09T05:24:23.296782+0000 mgr.x (mgr.25453) 40 2023-10-09T05:24:23.666 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[101299]: : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:24.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.666Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:24:24.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.666Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T05:24:24.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.666Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T05:24:24.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.666Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T05:24:24.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.666Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T05:24:24.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.666Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T05:24:24.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.666Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T05:24:24.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.666Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T05:24:24.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.666Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T05:24:24.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.667Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T05:24:24.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.667Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T05:24:24.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:23 smithi191 conmon[134411]: ts=2023-10-09T05:24:23.667Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T05:24:24.549 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 bash[144589]: 0e52be64285ae43ecb53bab5b2d0295241d343e8e87ed562bfe55de45515a516 2023-10-09T05:24:24.549 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T05:24:24.550 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 systemd[1]: Stopped Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:24.550 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 systemd[1]: Starting Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[101299]: audit 2023-10-09T05:24:24.318670+0000 mon.c (mon 2023-10-09T05:24:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[101299]: .2) 133 : audit [DBG] from='client.? 172.21.15.8:0/2949448658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:24.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:24 smithi008 conmon[101053]: audit 2023-10-09T05:24:24.318670 2023-10-09T05:24:24.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:24 smithi008 conmon[101053]: +0000 mon.c (mon.2) 133 : audit [DBG] from='client.? 172.21.15.8:0/2949448658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:24.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:24 smithi008 conmon[110782]: audit 2023-10-09T05:24:24. 2023-10-09T05:24:24.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:24 smithi008 conmon[110782]: 318670+0000 mon.c (mon.2) 133 : audit [DBG] from='client.? 172.21.15.8:0/2949448658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:25.049 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 bash[144927]: 7a37068c353e0d679a45833949067530a077aeee2cf0dde29526afe3d84ccb06 2023-10-09T05:24:25.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[144968]: ts=2023-10-09T05:24:24.945Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T05:24:25.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[144968]: ts=2023-10-09T05:24:24.945Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T05:24:25.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[144968]: ts=2023-10-09T05:24:24.945Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi191 (none))" 2023-10-09T05:24:25.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[144968]: ts=2023-10-09T05:24:24.945Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T05:24:25.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[144968]: ts=2023-10-09T05:24:24.945Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T05:24:25.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[144968]: ts=2023-10-09T05:24:24.946Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T05:24:25.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[144968]: ts=2023-10-09T05:24:24.947Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T05:24:25.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 systemd[1]: Started Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:25.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[144968]: ts=2023-10-09T05:24:24.948Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T05:24:25.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[144968]: ts=2023-10-09T05:24:24.952Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T05:24:25.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[144968]: ts=2023-10-09T05:24:24.952Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.428µs 2023-10-09T05:24:25.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:24 smithi191 conmon[144968]: ts=2023-10-09T05:24:24.952Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T05:24:25.363 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:25 smithi191 systemd[1]: Stopping Ceph node-exporter.b for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:25.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[144968]: ts=2023-10-09T05:24:25.110Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T05:24:25.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[144968]: ts=2023-10-09T05:24:25.340Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T05:24:25.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[144968]: ts=2023-10-09T05:24:25.341Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T05:24:25.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[144968]: ts=2023-10-09T05:24:25.341Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=50.59µs wal_replay_duration=389.19637ms total_replay_duration=389.269943ms 2023-10-09T05:24:25.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[144968]: ts=2023-10-09T05:24:25.345Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T05:24:25.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[144968]: ts=2023-10-09T05:24:25.345Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T05:24:25.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[144968]: ts=2023-10-09T05:24:25.345Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T05:24:25.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[144968]: ts=2023-10-09T05:24:25.363Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.500754ms db_storage=655ns remote_storage=1.583µs web_handler=344ns query_engine=745ns scrape=590.498µs scrape_sd=122.849µs notify=17.106µs notify_sd=17.498µs rules=17.348701ms 2023-10-09T05:24:25.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[144968]: ts=2023-10-09T05:24:25.363Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T05:24:25.737 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:25 smithi191 bash[145126]: Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-node-exporter.b found: no such container 2023-10-09T05:24:26.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: audit 2023-10-09T05:24:24.978568+0000 mon.a (mon. 2023-10-09T05:24:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: 0) 826 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: audit 2023-10-09T05:24:24.983126+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:25.028016 2023-10-09T05:24:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: +0000 mgr.x (mgr.25453) 41 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T05:24:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: cephadm 2023 2023-10-09T05:24:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: -10-09T05:24:25.028511+0000 mgr.x ( 2023-10-09T05:24:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: mgr.25453) 42 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi191 2023-10-09T05:24:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: cluster 2023-10-09T05:24:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: 2023-10-09T05 2023-10-09T05:24:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: :24:25.297114+0000 mgr.x (mgr.25453) 2023-10-09T05:24:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:25 smithi191 conmon[101299]: 43 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[101053]: audit 2023-10-09T05:24:24. 2023-10-09T05:24:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[101053]: 978568+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[101053]: audit 2023-10-09T05:24:24.983126+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:25.028016+0000 mgr.x (mgr.25453) 41 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T05:24:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:25.028511+0000 mgr.x ( 2023-10-09T05:24:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[101053]: mgr.25453) 42 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi191 2023-10-09T05:24:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[101053]: cluster 2023-10-09T05:24:25.297114+0000 mgr.x (mgr.25453) 43 : cluster 2023-10-09T05:24:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[101053]: [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: audit 2023-10-09T05:24: 2023-10-09T05:24:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: 24.978568+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: audit 2023 2023-10-09T05:24:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: -10-09T05:24:24.983126+0000 2023-10-09T05:24:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: mon.a (mon.0) 827 : audit 2023-10-09T05:24:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: cephadm 2023- 2023-10-09T05:24:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: 10-09T05:24:25.028016 2023-10-09T05:24:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: +0000 mgr.x (mgr.25453) 41 : 2023-10-09T05:24:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T05:24:26.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: cephadm 2023 2023-10-09T05:24:26.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: -10-09T05:24:25.028511 2023-10-09T05:24:26.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: +0000 mgr.x (mgr. 2023-10-09T05:24:26.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: 25453) 42 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi191 2023-10-09T05:24:26.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:24:26.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: -10-09T05:24:25.297114+0000 2023-10-09T05:24:26.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: mgr.x (mgr.25453) 43 : cluster 2023-10-09T05:24:26.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:25 smithi008 conmon[110782]: [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:26.549 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:24:26.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 bash[145241]: 715f54230f50b0f311e290e1674519233761a672cd2bd4c7d7dece7d263b4767 2023-10-09T05:24:26.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T05:24:26.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 systemd[1]: Stopped Ceph node-exporter.b for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:26.550 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 systemd[1]: Starting Ceph node-exporter.b for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:27.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[101299]: audit 2023-10-09T05:24:26.726086+0000 mon.c (mon 2023-10-09T05:24:27.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[101299]: .2) 134 : audit [DBG] from='client.? 172.21.15.8:0/3467791138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:27.300 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.956Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T05:24:27.300 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.956Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T05:24:27.300 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.956Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T05:24:27.300 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.956Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:24:27.300 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T05:24:27.300 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T05:24:27.301 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T05:24:27.302 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T05:24:27.303 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T05:24:27.304 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T05:24:27.304 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T05:24:27.304 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T05:24:27.304 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T05:24:27.304 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T05:24:27.304 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T05:24:27.304 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T05:24:27.304 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 conmon[145615]: ts=2023-10-09T05:24:26.957Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T05:24:27.304 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 bash[145575]: 05f0cea760c5fae5d2f19492ac44e265c35330e0219c4170185d5aa8d94a88e1 2023-10-09T05:24:27.304 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 05:24:26 smithi191 systemd[1]: Started Ceph node-exporter.b for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:27.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:26 smithi008 conmon[101053]: audit 2023-10-09T05:24:26.726086+0000 mon.c (mon.2 2023-10-09T05:24:27.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:26 smithi008 conmon[101053]: ) 134 : audit [DBG] from='client.? 172.21.15.8:0/3467791138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:27.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:26 smithi008 conmon[110782]: audit 2023-10-09T05:24:26.726086+0000 mon.c (mon.2) 2023-10-09T05:24:27.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:26 smithi008 conmon[110782]: 134 : audit [DBG] from='client.? 172.21.15.8:0/3467791138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:28 smithi191 conmon[101299]: audit 2023-10-09T05:24:27.005648+0000 mon.a (mon.0) 828 2023-10-09T05:24:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:28 smithi191 conmon[101299]: : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:28 smithi191 conmon[101299]: audit 2023-10-09T05:24:27.010853+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:28 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:27.045303+0000 mgr.x (mgr.25453) 44 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T05:24:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:28 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:27.055975+0000 mgr.x (mgr.25453) 45 : cephadm [INF] Reconfiguring daemon grafana.a on smithi191 2023-10-09T05:24:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:28 smithi191 conmon[101299]: cluster 2023-10-09T05:24:27.297615+0000 mgr.x (mgr.25453) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:28.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[101053]: audit 2023-10-09T05:24: 2023-10-09T05:24:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[101053]: 27.005648+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:24:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[101053]: T05:24:27.010853+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:27.045303 2023-10-09T05:24:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[101053]: +0000 mgr.x (mgr.25453) 44 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T05:24:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[101053]: 2023-10-09T05:24:27.055975+0000 mgr.x (mgr.25453) 45 : cephadm [INF] Reconfiguring daemon grafana.a on smithi191 2023-10-09T05:24:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[101053]: cluster 2023-10-09T05:24:27. 2023-10-09T05:24:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[101053]: 297615+0000 mgr.x (mgr.25453) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[110782]: audit 2023-10-09T05:24:27.005648+0000 mon.a (mon.0) 828 : audit 2023-10-09T05:24:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[110782]: audit 2023-10-09T05:24:27.010853+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[110782]: cephadm 2023-10-09T05:24:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[110782]: 2023-10-09T05:24:27.045303+0000 mgr.x (mgr.25453) 44 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T05:24:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[110782]: cephadm 2023-10-09T05:24:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[110782]: 2023-10-09T05:24:27.055975+0000 mgr.x (mgr.25453) 45 : cephadm [INF] 2023-10-09T05:24:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[110782]: Reconfiguring daemon grafana.a on smithi191 2023-10-09T05:24:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[110782]: cluster 2023-10-09T05 2023-10-09T05:24:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:28 smithi008 conmon[110782]: :24:27.297615+0000 mgr.x (mgr.25453) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:29.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:29 smithi191 conmon[101299]: audit 2023-10-09T05:24:29. 2023-10-09T05:24:29.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:29 smithi191 conmon[101299]: 135425+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.8:0/421223334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:29 smithi008 conmon[101053]: audit 2023-10-09T05:24:29.135425+0000 mon.a (mon 2023-10-09T05:24:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:29 smithi008 conmon[101053]: .0) 830 : audit [DBG] from='client.? 172.21.15.8:0/421223334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:29 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:24:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:29 smithi008 conmon[110782]: 05:24:29.135425+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.8:0/421223334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:30.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:30 smithi191 conmon[101299]: cluster 2023-10-09T05:24:29.297899+0000 mgr.x (mgr. 2023-10-09T05:24:30.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:30 smithi191 conmon[101299]: 25453) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:30.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:30 smithi008 conmon[101053]: cluster 2023-10-09T05:24:29.297899 2023-10-09T05:24:30.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:30 smithi008 conmon[101053]: +0000 mgr.x (mgr.25453) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:30 smithi008 conmon[110782]: cluster 2023-10-09T05:24:29.297899+0000 mgr.x 2023-10-09T05:24:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:30 smithi008 conmon[110782]: (mgr.25453) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:31.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:31 smithi008 conmon[101053]: cluster 2023-10-09T05:24:31.298376 2023-10-09T05:24:31.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:31 smithi008 conmon[101053]: +0000 mgr.x (mgr.25453) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:31 smithi008 conmon[110782]: cluster 2023-10-09T05:24:31.298376 2023-10-09T05:24:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:31 smithi008 conmon[110782]: +0000 mgr.x (mgr.25453) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:31.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:31 smithi191 conmon[101299]: cluster 2023-10-09T05:24:31.298376 2023-10-09T05:24:31.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:31 smithi191 conmon[101299]: +0000 mgr.x (mgr.25453) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:32.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:32 smithi008 conmon[101053]: audit 2023-10-09T05:24:31.541415+0000 mon.a (mon.0) 831 2023-10-09T05:24:32.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:32 smithi008 conmon[101053]: : audit [DBG] from='client.? 172.21.15.8:0/1801694879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:32 smithi008 conmon[110782]: audit 2023-10-09T05:24:31.541415 2023-10-09T05:24:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:32 smithi008 conmon[110782]: +0000 mon.a (mon.0) 831 : audit [DBG] from='client.? 172.21.15.8:0/1801694879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:32 smithi191 conmon[101299]: audit 2023-10-09T05:24:31. 2023-10-09T05:24:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:32 smithi191 conmon[101299]: 541415+0000 mon.a (mon.0) 831 : audit [DBG] from='client.? 172.21.15.8:0/1801694879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:33.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:33 smithi008 conmon[101053]: cluster 2023-10-09T05:24:33.298729+0000 mgr.x ( 2023-10-09T05:24:33.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:33 smithi008 conmon[101053]: mgr.25453) 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:33.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:33 smithi008 conmon[110782]: cluster 2023-10-09T05:24:33.298729+0000 mgr.x ( 2023-10-09T05:24:33.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:33 smithi008 conmon[110782]: mgr.25453) 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:33.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:33 smithi191 conmon[101299]: cluster 2023-10-09T05:24:33. 2023-10-09T05:24:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:33 smithi191 conmon[101299]: 298729+0000 mgr.x (mgr.25453) 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:33.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:33 smithi191 conmon[139071]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:24:33] "GET /metrics HTTP/1.1" 200 33911 "" "Prometheus/2.33.4" 2023-10-09T05:24:34.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:34 smithi008 conmon[101053]: audit 2023-10-09T05:24:33. 2023-10-09T05:24:34.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:34 smithi008 conmon[101053]: 945436+0000 mon.a (mon.0) 832 : audit [DBG] from='client.? 172.21.15.8:0/2538954747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:34.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:34 smithi008 conmon[110782]: audit 2023-10-09T05:24:33.945436+0000 mon.a (mon 2023-10-09T05:24:34.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:34 smithi008 conmon[110782]: .0) 832 : audit [DBG] from='client.? 172.21.15.8:0/2538954747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:34.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:34 smithi191 conmon[101299]: audit 2023-10-09T05:24:33. 2023-10-09T05:24:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:34 smithi191 conmon[101299]: 945436+0000 mon.a (mon.0) 832 : audit [DBG] from='client.? 172.21.15.8:0/2538954747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:35.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:35 smithi008 conmon[101053]: audit 2023-10-09T05:24:34.495425+0000 mon.a ( 2023-10-09T05:24:35.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:35 smithi008 conmon[101053]: mon.0) 833 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:35.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:35 smithi008 conmon[101053]: cluster 2023-10-09T05:24:35.299116+0000 mgr.x (mgr.25453) 50 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:35.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:35 smithi008 conmon[110782]: audit 2023-10-09T05:24:34.495425+0000 mon.a (mon. 2023-10-09T05:24:35.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:35 smithi008 conmon[110782]: 0) 833 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:35.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:35 smithi008 conmon[110782]: cluster 2023-10-09T05:24:35.299116+0000 mgr.x (mgr. 2023-10-09T05:24:35.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:35 smithi008 conmon[110782]: 25453) 50 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:35.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:35 smithi191 conmon[101299]: audit 2023-10-09T05:24:34.495425+0000 mon.a (mon. 2023-10-09T05:24:35.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:35 smithi191 conmon[101299]: 0) 833 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:35.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:35 smithi191 conmon[101299]: cluster 2023-10-09T05:24:35.299116+0000 mgr.x (mgr.25453) 50 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:36 smithi008 conmon[101053]: audit 2023-10-09T05:24:36.350706 2023-10-09T05:24:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:36 smithi008 conmon[101053]: +0000 mon.a (mon.0) 834 : audit [DBG] from='client.? 172.21.15.8:0/3021229874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:36.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:36 smithi008 conmon[110782]: audit 2023-10-09T05:24:36.350706+0000 2023-10-09T05:24:36.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:36 smithi008 conmon[110782]: mon.a (mon.0) 834 : audit [DBG] from='client.? 172.21.15.8:0/3021229874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:36.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:36 smithi191 conmon[101299]: audit 2023-10-09T05:24:36.350706 2023-10-09T05:24:36.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:36 smithi191 conmon[101299]: +0000 mon.a (mon.0) 834 : audit [DBG] from='client.? 172.21.15.8:0/3021229874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:38.516 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:38 smithi191 conmon[101299]: cluster 2023-10-09T05:24:37.299618+0000 mgr.x (mgr.25453) 51 : cluster 2023-10-09T05:24:38.516 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:38 smithi191 conmon[101299]: [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:38.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:38 smithi008 conmon[101053]: cluster 2023-10-09T05:24:37.299618+0000 mgr.x (mgr.25453) 51 2023-10-09T05:24:38.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:38 smithi008 conmon[101053]: : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:38.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:38 smithi008 conmon[110782]: cluster 2023-10-09T05:24:37.299618+0000 2023-10-09T05:24:38.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:38 smithi008 conmon[110782]: mgr.x (mgr.25453) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:38.799 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:38 smithi191 conmon[144968]: ts=2023-10-09T05:24:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:24:39.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:39 smithi191 conmon[101299]: audit 2023-10-09T05:24:38.757386+0000 mon.c (mon.2) 135 : audit 2023-10-09T05:24:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:39 smithi191 conmon[101299]: [DBG] from='client.? 172.21.15.8:0/3373040131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:39.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:39 smithi008 conmon[101053]: audit 2023-10-09T05:24:38.757386+0000 mon.c (mon. 2023-10-09T05:24:39.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:39 smithi008 conmon[101053]: 2) 135 : audit [DBG] from='client.? 172.21.15.8:0/3373040131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:39.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:39 smithi008 conmon[110782]: audit 2023-10-09T05:24:38.757386+0000 2023-10-09T05:24:39.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:39 smithi008 conmon[110782]: mon.c (mon.2) 135 : audit [DBG] from='client.? 172.21.15.8:0/3373040131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:40.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:40 smithi191 conmon[101299]: cluster 2023-10-09T05:24:39.300062+0000 mgr.x (mgr.25453) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:40.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:40 smithi008 conmon[101053]: cluster 2023-10-09T05:24:39.300062+0000 mgr.x (mgr.25453 2023-10-09T05:24:40.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:40 smithi008 conmon[101053]: ) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:40.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:40 smithi008 conmon[110782]: cluster 2023-10-09T05:24:39.300062+0000 2023-10-09T05:24:40.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:40 smithi008 conmon[110782]: mgr.x (mgr.25453) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:41.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:41 smithi191 conmon[101299]: audit 2023-10-09T05:24:41.164615+0000 mon.c (mon.2) 136 : audit 2023-10-09T05:24:41.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:41 smithi191 conmon[101299]: [DBG] from='client.? 172.21.15.8:0/2130785747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:41.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:41 smithi008 conmon[101053]: audit 2023-10-09T05:24:41.164615+0000 mon.c (mon.2 2023-10-09T05:24:41.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:41 smithi008 conmon[101053]: ) 136 : audit [DBG] from='client.? 172.21.15.8:0/2130785747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:41.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:41 smithi008 conmon[110782]: audit 2023-10-09T05:24:41. 2023-10-09T05:24:41.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:41 smithi008 conmon[110782]: 164615+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.8:0/2130785747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:42.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:42 smithi191 conmon[101299]: cluster 2023-10-09T05:24:41.300943+0000 mgr.x (mgr.25453) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:42.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:42 smithi008 conmon[101053]: cluster 2023-10-09T05:24:41.300943+0000 mgr.x (mgr.25453) 2023-10-09T05:24:42.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:42 smithi008 conmon[101053]: 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:42.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:42 smithi008 conmon[110782]: cluster 2023-10-09T05:24:41.300943+0000 mgr.x 2023-10-09T05:24:42.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:42 smithi008 conmon[110782]: (mgr.25453) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:43.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:43 smithi008 conmon[101053]: cluster 2023-10-09T05:24:43.301435+0000 2023-10-09T05:24:43.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:43 smithi008 conmon[101053]: mgr.x (mgr.25453) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:43.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:43 smithi008 conmon[110782]: cluster 2023-10 2023-10-09T05:24:43.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:43 smithi008 conmon[110782]: -09T05:24:43.301435+0000 mgr.x (mgr.25453) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:43.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:43 smithi191 conmon[101299]: cluster 2023-10-09T05:24:43.301435+0000 mgr.x (mgr.25453) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:43.801 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:43 smithi191 conmon[139071]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:24:43] "GET /metrics HTTP/1.1" 200 33908 "" "Prometheus/2.33.4" 2023-10-09T05:24:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:44 smithi008 conmon[101053]: audit 2023-10-09T05:24 2023-10-09T05:24:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:44 smithi008 conmon[101053]: :43.574619+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.8:0/1258628345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:44 smithi008 conmon[110782]: audit 2023-10-09T05:24:43.574619+0000 2023-10-09T05:24:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:44 smithi008 conmon[110782]: mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.8:0/1258628345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:44.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:44 smithi191 conmon[101299]: audit 2023-10-09T05:24:43.574619+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.8:0/1258628345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:45.587 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:45 smithi191 conmon[101299]: cluster 2023-10-09T05:24:45.301949+0000 mgr.x (mgr.25453) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:45.587 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:45 smithi191 conmon[101299]: 2023-10-09T05:24:45.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:45 smithi008 conmon[101053]: cluster 2023-10-09T05:24:45.301949 2023-10-09T05:24:45.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:45 smithi008 conmon[101053]: +0000 mgr.x (mgr.25453) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:45 smithi008 conmon[110782]: cluster 2023-10-09T05:24:45.301949+0000 mgr.x (mgr. 2023-10-09T05:24:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:45 smithi008 conmon[110782]: 25453) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:46.620 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:46 smithi191 systemd[1]: Stopping Ceph grafana.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:46.620 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:46 smithi191 bash[146029]: Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-grafana.a found: no such container 2023-10-09T05:24:46.620 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:46 smithi191 conmon[101299]: audit 2023-10-09T05:24:45.984789+0000 mon.a (mon.0 2023-10-09T05:24:46.620 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:46 smithi191 conmon[101299]: ) 835 : audit [DBG] from='client.? 172.21.15.8:0/591363148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:46.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:46 smithi008 conmon[101053]: audit 2023-10-09T05:24: 2023-10-09T05:24:46.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:46 smithi008 conmon[101053]: 45.984789+0000 mon.a (mon.0) 835 : audit [DBG] from='client.? 172.21.15.8:0/591363148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:46 smithi008 conmon[110782]: audit 2023-10-09T05:24:45.984789+0000 mon.a ( 2023-10-09T05:24:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:46 smithi008 conmon[110782]: mon.0) 835 : audit [DBG] from='client.? 172.21.15.8:0/591363148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:47.049 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:46 smithi191 conmon[132642]: t=2023-10-09T05:24:46+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T05:24:47.550 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:47 smithi191 bash[146144]: abc49ed66cfa0a2f68459a00637373e2667488e6ea6f44e9a01aad40286ca11a 2023-10-09T05:24:47.550 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:47 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T05:24:47.550 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:47 smithi191 systemd[1]: Stopped Ceph grafana.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:47.550 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:47 smithi191 systemd[1]: Starting Ceph grafana.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:48.327 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STOPPING 2023-10-09T05:24:48.327 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:24:48.327 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STOPPED 2023-10-09T05:24:48.327 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STARTING 2023-10-09T05:24:48.327 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Serving on http://:::9283 2023-10-09T05:24:48.328 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STARTED 2023-10-09T05:24:48.328 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STOPPING 2023-10-09T05:24:48.328 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:24:48.328 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STOPPED 2023-10-09T05:24:48.328 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STARTING 2023-10-09T05:24:48.329 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 bash[146468]: 66b01c4fecc1c055ed262a624227b278f869cca63e49773a544458edb32fd83f 2023-10-09T05:24:48.329 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 systemd[1]: Started Ceph grafana.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:48.329 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-09T05:24:48.329 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-09T05:24:48.329 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-09T05:24:48.329 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-09T05:24:48.329 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T05:24:48.330 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T05:24:48.330 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T05:24:48.330 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T05:24:48.330 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-09T05:24:48.330 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-09T05:24:48.330 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-09T05:24:48.330 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-09T05:24:48.331 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-09T05:24:48.331 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="App mode production" logger=settings 2023-10-09T05:24:48.331 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-09T05:24:48.331 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T05:24:48.331 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-09T05:24:48.331 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=1.025799ms 2023-10-09T05:24:48.331 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T05:24:48.331 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T05:24:48.331 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T05:24:48.332 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T05:24:48.332 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T05:24:48.332 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T05:24:48.332 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-10-09T05:24:48.332 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T05:24:48.332 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-10-09T05:24:48.333 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: cluster 2023-10-09T05:24:47.302451+0000 mgr.x (mgr.25453) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:48.333 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:24:48.333 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: 24:48.087545+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.333 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023-10-09T05:24:48.091931+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.333 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023-10-09T05:24:48 2023-10-09T05:24:48.333 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: .111890+0000 mon.a (mon.0) 838 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:24:48.333 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: 2023-10-09T05:24:48.113356+0000 mon.a (mon.0 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: ) 839 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023-10-09T05:24 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: :48.117887+0000 mon.a (mon.0) 840 : 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023-10-09T05:24: 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: 48.122107+0000 mon.a (mon.0) 841 : 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023-10-09T05:24: 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: 48.126270+0000 mon.a (mon.0) 842 : 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191:9095"}]: dispatch 2023-10-09T05:24:48.334 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023-10-09T05:24: 2023-10-09T05:24:48.335 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: 48.131092+0000 mon.a (mon.0) 843 : audit 2023-10-09T05:24:48.335 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.335 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023-10-09T05:24:48 2023-10-09T05:24:48.335 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: .137617+0000 mon.a (mon.0) 844 : audit 2023-10-09T05:24:48.335 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:24:48.335 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023-10-09T05:24:48.335 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: 2023-10-09T05 2023-10-09T05:24:48.335 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: :24:48.140168+0000 mon.a (mon 2023-10-09T05:24:48.335 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: .0) 845 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191:3000"}]: dispatch 2023-10-09T05:24:48.336 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023-10-09T05:24:48.144798+0000 mon.a (mon.0) 846 2023-10-09T05:24:48.336 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.336 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:24:48.336 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: 10-09T05:24:48.156296+0000 mon.a (mon.0) 847 : 2023-10-09T05:24:48.336 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[101299]: audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:24:48.659 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T05:24:48.659 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T05:24:48.659 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[146508]: t=2023-10-09T05:24:48+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T05:24:48.660 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Serving on http://:::9283 2023-10-09T05:24:48.660 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STARTED 2023-10-09T05:24:48.660 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STOPPING 2023-10-09T05:24:48.660 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:24:48.660 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STOPPED 2023-10-09T05:24:48.660 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STARTING 2023-10-09T05:24:48.660 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Serving on http://:::9283 2023-10-09T05:24:48.660 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[139071]: [09/Oct/2023:05:24:48] ENGINE Bus STARTED 2023-10-09T05:24:48.674 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: cluster 2023-10-09T05:24:47.302451+0000 mgr.x (mgr.25453) 2023-10-09T05:24:48.674 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:48.674 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.087545+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.674 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.091931+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.674 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.111890+0000 mon.a (mon.0) 838 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:24:48.674 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.113356+0000 mon.a (mon.0) 839 : 2023-10-09T05:24:48.674 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:24:48.674 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.117887+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.675 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.122107+0000 mon.a 2023-10-09T05:24:48.675 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: (mon.0) 841 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:24:48.675 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:24:48.675 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: 24:48.126270+0000 mon.a (mon.0) 842 : audit [INF] 2023-10-09T05:24:48.675 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191:9095"}]: dispatch 2023-10-09T05:24:48.675 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.675 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: 2023-10-09T05:24:48.131092+0000 mon.a (mon.0) 843 2023-10-09T05:24:48.675 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.675 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.675 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: 2023-10-09T05:24:48.137617+0000 mon.a (mon 2023-10-09T05:24:48.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: .0) 844 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:24:48.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09T05:24: 2023-10-09T05:24:48.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: 48.140168+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191:3000"}]: dispatch 2023-10-09T05:24:48.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:24:48.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: T05:24:48.144798+0000 mon.a (mon.0) 846 : audit 2023-10-09T05:24:48.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: 2023-10-09T05:24:48.156296+0000 mon.a (mon.0) 847 2023-10-09T05:24:48.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[101053]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:24:48.676 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: cluster 2023-10-09T05:24:47.302451+0000 mgr.x (mgr.25453) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:48.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.087545+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.091931+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.111890+0000 mon.a (mon.0) 838 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:24:48.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: 2023-10-09T05:24:48.113356+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:24:48.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: 2023-10-09T05:24:48.117887 2023-10-09T05:24:48.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: +0000 mon.a (mon.0) 840 : audit [INF] 2023-10-09T05:24:48.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:24:48.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: -09T05:24:48.122107+0000 mon.a ( 2023-10-09T05:24:48.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: mon.0) 841 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:24:48.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: 2023-10-09T05:24:48.126270 2023-10-09T05:24:48.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: +0000 mon.a (mon.0) 842 : audit [INF] 2023-10-09T05:24:48.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191:9095"}]: dispatch 2023-10-09T05:24:48.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023-10-09T05:24:48. 2023-10-09T05:24:48.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: 131092+0000 mon.a (mon.0) 843 : audit 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: 10-09T05:24:48.137617+0000 mon.a (mon.0) 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: 844 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: 05:24:48.140168+0000 mon.a (mon.0 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: ) 845 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191:3000"}]: dispatch 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: -09T05:24:48.144798+0000 mon.a 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: (mon.0) 846 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: 05:24:48.156296+0000 mon.a (mon.0) 847 2023-10-09T05:24:48.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:48 smithi008 conmon[110782]: : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:24:49.049 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:48 smithi191 conmon[144968]: ts=2023-10-09T05:24:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:24:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: audit 2023-10-09T05:24:48.112424+0000 mgr.x (mgr.25453) 57 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:24:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: audit 2023-10-09T05:24:48.113681+0000 mgr.x (mgr.25453) 58 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:24:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:24:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: :24:48.122664+0000 mgr.x (mgr.25453) 59 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:24:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: audit 2023-10-09T05:24:48.126712+0000 mgr.x (mgr.25453) 60 : audit 2023-10-09T05:24:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191:9095"}]: dispatch 2023-10-09T05:24:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: audit 2023-10-09T05:24:48. 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: 138063+0000 mgr.x (mgr.25453) 61 : audit [DBG] 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: audit 2023-10-09T05:24:48 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: .140495+0000 mgr.x (mgr.25453) 62 : audit 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191:3000"}]: dispatch 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:48 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: .158055+0000 mgr.x (mgr.25453) 63 : cephadm 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:48 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: .158485+0000 mgr.x (mgr.25453) 64 : cephadm 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:24:49.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: audit 2023-10-09T05:24:48 2023-10-09T05:24:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: .407599+0000 mon.a (mon.0) 848 : audit 2023-10-09T05:24:49.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:49 smithi191 conmon[101299]: [DBG] from='client.? 172.21.15.8:0/4288682156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:49.662 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.112424+0000 mgr.x (mgr.25453) 57 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:24:49.662 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.113681+0000 mgr.x (mgr.25453) 58 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:24:49.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.122664+0000 mgr.x (mgr.25453) 59 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:24:49.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.126712+0000 mgr.x (mgr.25453) 60 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191:9095"}]: dispatch 2023-10-09T05:24:49.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.138063+0000 mgr.x ( 2023-10-09T05:24:49.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[101053]: mgr.25453) 61 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:24:49.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.140495+0000 mgr.x (mgr.25453) 62 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191:3000"}]: dispatch 2023-10-09T05:24:49.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:48.158055+0000 mgr.x (mgr.25453) 63 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:24:49.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:48.158485+0000 mgr.x (mgr.25453) 64 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:24:49.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[101053]: audit 2023-10-09T05:24:48.407599+0000 mon.a (mon.0) 848 : audit [DBG] from='client.? 172.21.15.8:0/4288682156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:49.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.112424+0000 mgr.x (mgr.25453) 57 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:24:49.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.113681+0000 mgr.x (mgr.25453) 58 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi008.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:24:49.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.122664+0000 mgr.x (mgr.25453) 59 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:24:49.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.126712+0000 mgr.x (mgr.25453) 60 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191:9095"}]: dispatch 2023-10-09T05:24:49.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.138063+0000 mgr.x (mgr.25453) 61 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:24:49.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.140495+0000 mgr.x (mgr.25453) 62 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191:3000"}]: dispatch 2023-10-09T05:24:49.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[110782]: cephadm 2023-10-09T05:24:48.158055+0000 mgr.x (mgr.25453) 63 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:24:49.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[110782]: cephadm 2023-10-09T05:24:48.158485+0000 mgr.x (mgr.25453) 64 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:24:49.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:49 smithi008 conmon[110782]: audit 2023-10-09T05:24:48.407599+0000 mon.a (mon.0) 848 : audit [DBG] from='client.? 172.21.15.8:0/4288682156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[101053]: cluster 2023-10-09T05:24:49.302940+0000 mgr.x ( 2023-10-09T05:24:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[101053]: mgr.25453) 65 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[101053]: audit 2023-10-09T05:24:49.499467+0000 mon.a (mon.0) 849 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[101053]: audit 2023-10-09T05:24:49.730597+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[101053]: audit 2023-10-09T05:24:49.731636+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:24:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[101053]: audit 2023-10-09T05:24:49.732553+0000 mon.a (mon.0) 852 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:24:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[101053]: audit 2023-10-09T05:24:49.733376+0000 mon.a (mon.0) 853 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:24:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: cluster 2023-10-09T05: 2023-10-09T05:24:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: 24:49.302940+0000 mgr.x (mgr.25453) 65 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:24:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: 10-09T05:24:49.499467+0000 mon.a (mon.0) 849 : audit 2023-10-09T05:24:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: audit 2023-10-09T05:24:49.730597+0000 mon.a (mon. 2023-10-09T05:24:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: 0) 850 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: audit 2023-10-09T05:24:49.731636 2023-10-09T05:24:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: +0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:24:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:24:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: 05:24:49.732553+0000 mon.a (mon.0) 852 : audit 2023-10-09T05:24:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:24:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: audit 2023-10-09T05:24:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: 2023-10-09T05:24:49.733376+0000 mon.a (mon.0) 853 : audit [DBG] 2023-10-09T05:24:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:50 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:24:50.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: cluster 2023-10-09T05:24:49.302940+0000 mgr.x (mgr.25453) 65 : cluster [DBG] 2023-10-09T05:24:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: audit 2023-10-09T05:24:49.499467+0000 mon.a (mon.0) 849 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:24:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: audit 2023-10-09T05:24:49.730597+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: audit 2023-10-09T05:24:49.731636+0000 mon.a (mon.0) 851 : audit 2023-10-09T05:24:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:24:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: audit 2023-10-09T05:24:49. 2023-10-09T05:24:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: 732553+0000 mon.a (mon.0) 852 : audit [DBG] 2023-10-09T05:24:50.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:24:50.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: audit 2023-10-09T05:24:50.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: 2023-10-09T05:24:49.733376+0000 mon.a (mon.0 2023-10-09T05:24:50.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:50 smithi191 conmon[101299]: ) 853 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:24:51.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:51 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:49.726019+0000 mgr.x (mgr.25453) 66 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T05:24:51.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:51 smithi008 conmon[101053]: cephadm 2023-10-09T05:24:49.734367+0000 mgr.x (mgr.25453) 67 : cephadm [INF] Deploying daemon mgr.y on smithi008 2023-10-09T05:24:51.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:51 smithi008 conmon[101053]: audit 2023-10-09T05:24:50.812848+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.8:0/4264179820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:51.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:51 smithi008 conmon[110782]: cephadm 2023-10-09T05:24:49.726019+0000 mgr.x (mgr.25453) 66 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T05:24:51.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:51 smithi008 conmon[110782]: cephadm 2023-10-09T05:24:49.734367+0000 mgr.x (mgr.25453) 67 : cephadm [INF] Deploying daemon mgr.y on smithi008 2023-10-09T05:24:51.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:51 smithi008 conmon[110782]: audit 2023-10-09T05:24:50.812848+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.8:0/4264179820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:51.466 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:51 smithi008 systemd[1]: Stopping Ceph mgr.y for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:51.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:51 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:49.726019+0000 mgr.x (mgr.25453) 66 2023-10-09T05:24:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:51 smithi191 conmon[101299]: : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T05:24:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:51 smithi191 conmon[101299]: cephadm 2023-10-09T05:24:49.734367+0000 mgr.x (mgr.25453) 67 : 2023-10-09T05:24:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:51 smithi191 conmon[101299]: cephadm [INF] Deploying daemon mgr.y on smithi008 2023-10-09T05:24:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:51 smithi191 conmon[101299]: audit 2023-10-09T05:24:50.812848 2023-10-09T05:24:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:51 smithi191 conmon[101299]: +0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.8:0/4264179820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:52.416 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:52 smithi008 conmon[101053]: cluster 2023-10-09T05:24: 2023-10-09T05:24:52.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:52 smithi008 conmon[101053]: 51.303786+0000 mgr.x (mgr.25453) 68 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:52.417 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:52 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:24:52.417 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:52 smithi008 bash[176828]: b54a8f2f116cf016eccb9bf719501d5f484ac3ca286dce557f90525646c77184 2023-10-09T05:24:52.417 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:52 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T05:24:52.417 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:52 smithi008 systemd[1]: Stopped Ceph mgr.y for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:52.417 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:52 smithi008 conmon[110782]: cluster 2023-10-09T05:24:51. 2023-10-09T05:24:52.417 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:52 smithi008 conmon[110782]: 303786+0000 mgr.x (mgr.25453) 68 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:52.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:52 smithi191 conmon[101299]: cluster 2023-10-09T05:24:51.303786+0000 mgr.x (mgr.25453) 68 : cluster [DBG] 2023-10-09T05:24:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:52 smithi191 conmon[101299]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:52.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:52 smithi008 systemd[1]: Starting Ceph mgr.y for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:24:53.317 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:53 smithi008 ceph-mgr[177273]: -- 172.21.15.8:0/1271053132 <== mon.0 v2:172.21.15.8:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5602a16bf1e0 con 0x5602a1697000 2023-10-09T05:24:53.318 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:53 smithi008 bash[177185]: b6adda8222c05e8930d6da978910b7940c48521a5171b88e38be39c4306ae92d 2023-10-09T05:24:53.318 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:53 smithi008 systemd[1]: Started Ceph mgr.y for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:24:53.318 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:53 smithi008 conmon[177229]: 2023-10-09T05:24:53.216+0000 7f529e205200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:24:53.318 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:53 smithi008 conmon[177229]: 2023-10-09T05:24:53.316+0000 7f529e205200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:24:53.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:53 smithi008 conmon[177229]: 2023-10-09T05:24:53.439+0000 7f529e205200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:24:53.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:53 smithi008 conmon[177229]: 2023-10-09T05:24:53.564+0000 7f529e205200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:24:53.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:24:53 smithi191 conmon[139071]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:24:53] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.33.4" 2023-10-09T05:24:54.145 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:53 smithi008 conmon[177229]: 2023-10-09T05:24:53.791+0000 7f529e205200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:24:54.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[177229]: 2023-10-09T05:24:54.305+0000 7f529e205200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:24:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[101053]: audit 2023-10-09T05:24:53.145036+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[101053]: audit 2023-10-09T05:24:53.150258+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[101053]: audit 2023-10-09T05:24:53.156431+0000 mon.a (mon.0) 856 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:24:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[101053]: audit 2023-10-09T05:24:53.219120+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.8:0/2756002065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[101053]: cluster 2023-10-09T05:24:53.304110+0000 mgr.x (mgr.25453) 69 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:24:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: 24:53.145036+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:24:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: -09T05:24:53.150258+0000 mon.a 2023-10-09T05:24:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: (mon.0) 855 : audit [INF] 2023-10-09T05:24:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: audit 2023-10-09T05:24 2023-10-09T05:24:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: :53.156431+0000 mon.a (mon.0) 2023-10-09T05:24:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: 856 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:24:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: audit 2023-10-09T05:24:53.219120+0000 mon.a (mon.0 2023-10-09T05:24:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: ) 857 : audit [DBG] from='client.? 172.21.15.8:0/2756002065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:24:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: 10-09T05:24:53.304110+0000 mgr.x (mgr 2023-10-09T05:24:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: .25453) 69 : cluster [DBG] 2023-10-09T05:24:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[110782]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:54.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:54 smithi191 conmon[101299]: audit 2023-10-09T05:24:53.145036+0000 mon.a ( 2023-10-09T05:24:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:54 smithi191 conmon[101299]: mon.0) 854 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:54 smithi191 conmon[101299]: audit 2023-10-09T05:24:53.150258+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:54 smithi191 conmon[101299]: audit 2023-10-09T05:24:53.156431+0000 mon.a (mon.0) 856 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:24:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:54 smithi191 conmon[101299]: audit 2023-10-09T05:24:53.219120+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.8:0/2756002065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:54 smithi191 conmon[101299]: cluster 2023-10-09T05:24:53.304110+0000 mgr.x (mgr.25453) 69 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:54.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[177229]: 2023-10-09T05:24:54.464+0000 7f529e205200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:24:54.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[177229]: 2023-10-09T05:24:54.574+0000 7f529e205200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:24:54.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:54 smithi008 conmon[177229]: 2023-10-09T05:24:54.694+0000 7f529e205200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:24:55.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:55 smithi008 conmon[177229]: 2023-10-09T05:24:55.275+0000 7f529e205200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:24:56.361 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[177229]: 2023-10-09T05:24:56.107+0000 7f529e205200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:24:56.361 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[177229]: 2023-10-09T05:24:56.217+0000 7f529e205200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:24:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[101053]: cluster 2023-10-09T05:24:55.304722+0000 mgr.x ( 2023-10-09T05:24:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[101053]: mgr.25453) 70 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[101053]: audit 2023-10-09T05:24:55.668062+0000 mon.c (mon.2) 139 : audit [DBG] 2023-10-09T05:24:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[101053]: from='client.? 172.21.15.8:0/2833521979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[101053]: audit 2023-10-09T05:24:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[101053]: 2023-10-09T05:24:56.231937+0000 mon.a ( 2023-10-09T05:24:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[101053]: mon.0) 858 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[101053]: audit 2023-10-09T05:24:56.236392+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: cluster 2023-10-09T05:24:55. 2023-10-09T05:24:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: 304722+0000 mgr.x (mgr.25453) 70 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:24:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: 24:55.668062+0000 mon.c (mon.2) 139 : 2023-10-09T05:24:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: audit [DBG] from='client.? 172.21.15.8:0/2833521979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: audit 2023-10-09T05:24:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: 2023-10-09T 2023-10-09T05:24:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: 05:24:56.231937+0000 mon.a (mon.0) 858 : audit 2023-10-09T05:24:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: audit 2023-10-09T05:24:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: 2023-10-09T05:24:56 2023-10-09T05:24:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: .236392+0000 mon.a (mon.0) 859 : audit 2023-10-09T05:24:56.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:56 smithi008 conmon[110782]: [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:56.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:56 smithi191 conmon[101299]: cluster 2023-10-09T05:24:55.304722 2023-10-09T05:24:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:56 smithi191 conmon[101299]: +0000 mgr.x (mgr.25453) 70 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:56 smithi191 conmon[101299]: audit 2023-10-09T05:24:55.668062+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.8:0/2833521979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:56 smithi191 conmon[101299]: audit 2023-10-09T05:24:56.231937+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:56 smithi191 conmon[101299]: audit 2023-10-09T05:24:56.236392+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.565 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[177229]: 2023-10-09T05:24:57.255+0000 7f529e205200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:24:57.565 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[177229]: 2023-10-09T05:24:57.393+0000 7f529e205200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:24:57.565 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[101053]: audit 2023-10-09T05:24:56.434323+0000 mon.a (mon.0) 2023-10-09T05:24:57.565 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[101053]: 860 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.565 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[101053]: audit 2023-10-09T05:24:56.439029+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:24:57.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[101053]: 24:56.972479+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[101053]: audit 2023-10-09T05:24:56. 2023-10-09T05:24:57.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[101053]: 977868+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.566 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:24:57.566 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: :24:56.434323+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.566 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: audit 2023 2023-10-09T05:24:57.566 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: -10-09T05:24:56. 2023-10-09T05:24:57.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: 439029+0000 mon.a (mon.0) 2023-10-09T05:24:57.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: 861 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: audit 2023 2023-10-09T05:24:57.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: -10-09T05:24:56 2023-10-09T05:24:57.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: .972479+0000 mon.a (mon. 2023-10-09T05:24:57.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: 0) 862 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: 2023-10-09T05:24:57.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: audit 2023 2023-10-09T05:24:57.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: -10-09T05:24: 2023-10-09T05:24:57.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: 56.977868+0000 mon.a (mon 2023-10-09T05:24:57.568 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: .0) 863 : audit [INF] 2023-10-09T05:24:57.568 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:57 smithi191 conmon[101299]: audit 2023-10-09T05:24:56.434323+0000 mon.a (mon. 2023-10-09T05:24:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:57 smithi191 conmon[101299]: 0) 860 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:57 smithi191 conmon[101299]: audit 2023-10-09T05:24:56.439029+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:57 smithi191 conmon[101299]: audit 2023-10-09T05:24:56.972479+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:57 smithi191 conmon[101299]: audit 2023-10-09T05:24:56.977868 2023-10-09T05:24:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:57 smithi191 conmon[101299]: +0000 mon.a (mon.0) 863 : audit [INF] 2023-10-09T05:24:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:57 smithi191 conmon[101299]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:24:57.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:57 smithi008 conmon[177229]: 2023-10-09T05:24:57.627+0000 7f529e205200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:24:58.280 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[177229]: 2023-10-09T05:24:58.015+0000 7f529e205200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:24:58.602 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[101053]: cluster 2023-10-09T05:24: 2023-10-09T05:24:58.603 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[101053]: 57.305398+0000 mgr.x (mgr.25453) 71 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:58.603 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[101053]: audit 2023-10-09T05:24:58.091576 2023-10-09T05:24:58.603 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[101053]: +0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.8:0/4245288950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:58.603 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[110782]: cluster 2023-10-09T05:24:57. 2023-10-09T05:24:58.603 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[110782]: 305398+0000 mgr.x (mgr.25453) 71 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:58.603 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[110782]: audit 2023-10-09T05:24: 2023-10-09T05:24:58.603 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[110782]: 58.091576+0000 mon.a (mon.0) 864 : audit 2023-10-09T05:24:58.604 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[110782]: [DBG] from='client.? 172.21.15.8:0/4245288950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:58.604 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[177229]: 2023-10-09T05:24:58.279+0000 7f529e205200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:24:58.604 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[177229]: 2023-10-09T05:24:58.467+0000 7f529e205200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:24:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:58 smithi191 conmon[101299]: cluster 2023-10-09T05:24:57.305398+0000 mgr.x (mgr.25453) 71 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:24:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:58 smithi191 conmon[101299]: audit 2023-10-09T05:24: 2023-10-09T05:24:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:24:58 smithi191 conmon[101299]: 58.091576+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.8:0/4245288950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:24:58.801 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:24:58 smithi191 conmon[144968]: ts=2023-10-09T05:24:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:24:58.912 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:58 smithi008 conmon[177229]: 2023-10-09T05:24:58.827+0000 7f529e205200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:24:59.881 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:59 smithi008 conmon[177229]: 2023-10-09T05:24:59.713+0000 7f529e205200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:25:00.136 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:24:59 smithi008 conmon[177229]: 2023-10-09T05:24:59.879+0000 7f529e205200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:25:00.136 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:00 smithi008 conmon[177229]: 2023-10-09T05:25:00.135+0000 7f529e205200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:25:00.442 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:00 smithi008 conmon[177229]: 2023-10-09T05:25:00.287+0000 7f529e205200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:25:00.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:00 smithi008 conmon[177229]: 2023-10-09T05:25:00.587+0000 7f529e205200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:25:00.747 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:00 smithi191 conmon[139071]: debug 2023-10-09T05:25:00.413+0000 7f64968e4700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T05:25:00.747 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:00 smithi191 conmon[139071]: ignoring --setuser ceph since I am not root 2023-10-09T05:25:00.747 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:00 smithi191 conmon[139071]: ignoring --setgroup ceph since I am not root 2023-10-09T05:25:00.747 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:00 smithi191 conmon[139071]: debug 2023-10-09T05:25:00.464+0000 7fc31e914700 1 -- 172.21.15.191:0/2180945577 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5618410171e0 con 0x561840feec00 2023-10-09T05:25:00.747 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:00 smithi191 conmon[139071]: debug 2023-10-09T05:25:00.579+0000 7fc328644200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:25:00.747 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:00 smithi191 conmon[139071]: debug 2023-10-09T05:25:00.658+0000 7fc328644200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:25:00.747 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:00 smithi191 conmon[139071]: debug 2023-10-09T05:25:00.747+0000 7fc328644200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:25:01.014 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:00 smithi191 conmon[139071]: debug 2023-10-09T05:25:00.844+0000 7fc328644200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:25:01.299 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[139071]: debug 2023-10-09T05:25:01.014+0000 7fc328644200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:25:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: cluster 2023-10-09T05:24:59.305801+0000 mgr.x (mgr.25453) 2023-10-09T05:25:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: 72 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: audit 2023-10-09T05:25:00.255314+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:25:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: audit 2023-10-09T05:25: 2023-10-09T05:25:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: 00.260534+0000 mon.a (mon.0) 866 : audit [INF] 2023-10-09T05:25:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:25:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: audit 2023-10-09T05:25:00 2023-10-09T05:25:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: .261556+0000 mon.a (mon.0) 867 : audit 2023-10-09T05:25:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: 25:00.262456+0000 mon.a (mon.0) 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: 868 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: 09T05:25:00.267017+0000 mon.a (mon 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: .0) 869 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: audit 2023 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: -10-09T05:25:00.404925+0000 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: mon.a (mon.0) 870 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: audit 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: 2023-10-09T05 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: :25:00.410028+0000 mon.a (mon.0) 2023-10-09T05:25:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: 871 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: cluster 2023-10- 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: 09T05:25:00.415838+0000 mon.a (mon 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: .0) 872 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: audit 2023 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: -10-09T05:25:00.416133+0000 mon.a 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: (mon.0) 873 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: cluster 2023 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: -10-09T05:25:00.416206+0000 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: mon.a (mon.0) 874 : cluster [DBG] mgrmap e25: y(active, starting, since 0.00571032s) 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: audit 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: 2023-10-09T05:25:00.503624+0000 2023-10-09T05:25:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.8:0/1069543717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:01.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[101299]: 2023-10-09T05:25:01.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: cluster 2023-10-09T05:24:59 2023-10-09T05:25:01.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: .305801+0000 mgr.x (mgr.25453) 72 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:01.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: audit 2023-10-09T05:25:00.255314+0000 mon.a 2023-10-09T05:25:01.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: (mon.0) 865 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:25:01.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:25:01.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 05:25:00.260534+0000 2023-10-09T05:25:01.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: mon.a (mon.0) 866 : audit [INF] 2023-10-09T05:25:01.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:25:01.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:25:01.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: -09T05:25:00.261556+0000 2023-10-09T05:25:01.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: mon.a (mon.0) 867 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:01.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: audit 2023-10-09T05:25:01.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 2023-10-09T05:25 2023-10-09T05:25:01.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: :00.262456+0000 mon.a (mon. 2023-10-09T05:25:01.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 0) 868 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:01.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: audit 2023-10-09T05:25:01.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 2023-10-09T05:25 2023-10-09T05:25:01.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: :00.267017+0000 mon.a (mon.0) 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 869 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: audit 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 2023-10-09T05:25 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: :00.404925+0000 mon.a (mon.0) 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 870 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: audit 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 2023-10-09T05:25 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: :00.410028+0000 mon.a (mon.0) 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 871 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: cluster 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 2023-10-09T05: 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 25:00.415838+0000 mon.a (mon.0) 872 : cluster 2023-10-09T05:25:01.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T05:25:01.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: audit 2023-10-09T05:25:01.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 2023-10-09T05:25:00.416133+0000 mon.a 2023-10-09T05:25:01.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: (mon.0) 873 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T05:25:01.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: cluster 2023-10-09T05: 2023-10-09T05:25:01.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 25:00.416206+0000 mon.a (mon.0) 874 : cluster [DBG] mgrmap e25: y(active, starting, since 0.00571032s) 2023-10-09T05:25:01.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 2023-10-09T05:25:01.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:25:01.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 09T05:25:00.503624+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.8:0/1069543717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:01.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[101053]: 2023-10-09T05:25:01.666 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: cluster 2023-10-09T05:24:59.305801 2023-10-09T05:25:01.666 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: +0000 mgr.x (mgr.25453) 72 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: audit 2023-10-09T05:25:00. 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 255314+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 10-09T05:25:00.260534+0000 mon.a (mon. 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 0) 866 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: audit 2023-10-09T05:25 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: :00.261556+0000 mon.a (mon.0) 867 : audit 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: -09T05:25:00. 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 262456+0000 mon.a (mon.0) 2023-10-09T05:25:01.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 868 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: audit 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 2023-10-09T05:25: 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 00.267017+0000 mon.a (mon 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: .0) 869 : audit [INF] 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: -09T05:25:00. 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 404925+0000 mon.a (mon.0) 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 870 : audit [DBG] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: audit 2023 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: -10-09T05:25:00 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: .410028+0000 mon.a (mon.0 2023-10-09T05:25:01.668 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: ) 871 : audit [INF] from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T05:25:01.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 2023-10-09T05:25:01.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: cluster 2023-10-09T05:25:01.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:25:01.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 25:00.415838+0000 mon.a 2023-10-09T05:25:01.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: (mon.0) 872 : cluster 2023-10-09T05:25:01.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T05:25:01.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: audit 2023 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: -10-09T05:25:00 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: .416133+0000 mon.a (mon. 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 0) 873 : audit [INF] 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: from='mgr.25453 172.21.15.191:0/2348084758' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: cluster 2023- 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 10-09T05:25: 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 00.416206+0000 mon.a (mon. 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 0) 874 : cluster [DBG] 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: mgrmap e25: y(active, starting, since 0.00571032s) 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: audit 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 2023-10 2023-10-09T05:25:01.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: -09T05:25:00. 2023-10-09T05:25:01.671 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: 503624+0000 mon.c (mon.2 2023-10-09T05:25:01.671 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[110782]: ) 140 : audit [DBG] from='client.? 172.21.15.8:0/1069543717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:01.695 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[139071]: debug 2023-10-09T05:25:01.377+0000 7fc328644200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:25:01.695 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[139071]: debug 2023-10-09T05:25:01.511+0000 7fc328644200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:25:01.695 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[139071]: debug 2023-10-09T05:25:01.598+0000 7fc328644200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:25:01.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.662+0000 7f529e205200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:25:01.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: [09/Oct/2023:05:25:01] ENGINE Bus STARTING 2023-10-09T05:25:01.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: CherryPy Checker: 2023-10-09T05:25:01.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: The Application mounted at '' has an empty config. 2023-10-09T05:25:01.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: [09/Oct/2023:05:25:01] ENGINE Serving on http://:::9283 2023-10-09T05:25:01.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: [09/Oct/2023:05:25:01] ENGINE Bus STARTED 2023-10-09T05:25:01.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: [09/Oct/2023:05:25:01] ENGINE Bus STOPPING 2023-10-09T05:25:01.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: [09/Oct/2023:05:25:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:25:01.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: [09/Oct/2023:05:25:01] ENGINE Bus STOPPED 2023-10-09T05:25:01.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: [09/Oct/2023:05:25:01] ENGINE Bus STARTING 2023-10-09T05:25:02.258 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:01 smithi191 conmon[139071]: debug 2023-10-09T05:25:01.695+0000 7fc328644200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:25:02.258 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[139071]: debug 2023-10-09T05:25:02.137+0000 7fc328644200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:25:02.261 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: cluster 2023-10-09T05:25 2023-10-09T05:25:02.261 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: :01.676505+0000 mon.a (mon.0) 875 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:25:02.262 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: cluster 2023-10-09T05:25:02.262 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09 2023-10-09T05:25:02.262 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: T05:25:01.676902+0000 2023-10-09T05:25:02.262 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.a (mon.0) 876 : cluster [INF] 2023-10-09T05:25:02.262 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: Activating manager daemon y 2023-10-09T05:25:02.262 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: cluster 2023-10-09T05:25:02.262 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:25:02.262 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 25:01.682757+0000 mon.a (mon. 2023-10-09T05:25:02.262 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 0) 877 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T05:25:02.262 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: cluster 2023-10-09T05:25:01.683136+0000 mon.a (mon.0) 878 : cluster [DBG] mgrmap e26: y(active, starting, since 0.006369s) 2023-10-09T05:25:02.262 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: :25:01.683293+0000 mon.a (mon.0) 879 : audit [DBG] 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: from='mgr.? 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05:25 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: :01.685596+0000 mon.a (mon 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: .0) 880 : audit [DBG] 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: -09T05:25:01.685844+0000 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.a (mon.0) 881 : 2023-10-09T05:25:02.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 10-09T05:25:01.686064+0000 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.a (mon.0) 882 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: :01.686287+0000 mon.a (mon.0) 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 883 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 05:25:01.686485+0000 mon.a (mon. 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 0) 884 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:25:02.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: T05:25:01.686687+0000 mon.a (mon 2023-10-09T05:25:02.265 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: .0) 885 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi008.zfejpl"}]: dispatch 2023-10-09T05:25:02.265 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:25:02.265 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.973+0000 7f51fb676700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:02.265 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.973+0000 7f51fb676700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:02.265 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.973+0000 7f51fb676700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:02.265 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.973+0000 7f51fb676700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:02.265 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.973+0000 7f51fb676700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:02.265 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.973+0000 7f51f8670700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:02.265 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.973+0000 7f51f8670700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:02.265 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.973+0000 7f51f8670700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:02.266 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.973+0000 7f51f8670700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:02.266 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:01 smithi008 conmon[177229]: 2023-10-09T05:25:01.973+0000 7f51f8670700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:02.266 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[177229]: [09/Oct/2023:05:25:02] ENGINE Serving on http://:::9283 2023-10-09T05:25:02.266 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[177229]: [09/Oct/2023:05:25:02] ENGINE Bus STARTED 2023-10-09T05:25:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: cluster 2023-10-09T05:25:01.676505+0000 mon.a (mon.0) 875 2023-10-09T05:25:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: : cluster [INF] Active manager daemon y restarted 2023-10-09T05:25:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: cluster 2023-10-09T05:25:01.676902+0000 mon.a (mon.0) 876 : cluster [INF] Activating manager daemon y 2023-10-09T05:25:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: cluster 2023-10-09 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: T05:25:01.682757+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: cluster 2023-10-09T05:25:01.683136+0000 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: mon.a (mon.0) 878 : cluster [DBG] mgrmap e26: y(active, starting, since 0.006369s) 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 05:25:01.683293+0000 mon.a (mon.0) 879 : 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit [DBG] from='mgr.? 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 2023-10-09T05:25:01.685596+0000 mon.a (mon.0) 880 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:01.685844 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: +0000 mon.a (mon.0) 881 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:02.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 2023-10-09T05:25:01.686064+0000 mon.a (mon.0) 882 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25: 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 01.686287+0000 mon.a (mon.0) 883 : audit [DBG] 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 2023-10-09T05:25:01.686485+0000 mon.a (mon. 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 0) 884 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: :25:01.686687+0000 mon.a (mon.0) 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 885 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi008.zfejpl"}]: dispatch 2023-10-09T05:25:02.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 2023-10-09T05:25:01.686987+0000 mon.a (mon. 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 0) 886 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.walafj"}]: dispatch 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: :01.687286+0000 mon.a (mon.0) 887 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:01 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: .687526+0000 mon.a (mon.0) 888 : audit [DBG] 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 2023-10-09T05:25:01.687818+0000 mon.a (mon.0) 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 889 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:25:02.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: :25:01.688103+0000 mon.a (mon.0) 890 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: :01.688446+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: :01.688813+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25: 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 01.689076+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:01.689307+0000 mon.a (mon.0) 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 894 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:25:02.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 10-09T05:25:01.689524+0000 mon.a (mon.0) 895 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:01. 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 689767+0000 mon.a (mon.0) 896 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:01.690011+0000 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: mon.a (mon.0) 897 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:01.690464+0000 mon.a (mon. 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 0) 898 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 2023-10-09T05:25:01.690678+0000 mon.a (mon.0) 899 : audit [DBG] 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: :25:01.690981+0000 mon.a (mon.0) 900 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:25:02.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: cluster 2023-10-09T05:25:01.850250+0000 mon.a (mon. 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 0) 901 : cluster [INF] Manager daemon y is now available 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 2023-10-09T05:25:01.874388+0000 mon.a (mon.0) 902 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:01 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: .878055+0000 mon.a (mon.0) 903 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:01.905181+0000 mon.a (mon.0 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: ) 904 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: 10-09T05:25:01.915261+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: audit 2023-10-09T05:25:01.946607+0000 mon.a 2023-10-09T05:25:02.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[101299]: (mon.0) 906 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:25:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: cluster 2023-10-09T05:25:01.676505+0000 mon.a ( 2023-10-09T05:25:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: mon.0) 875 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:25:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: cluster 2023-10-09T05:25:01.676902+0000 mon.a (mon.0) 876 : cluster [INF] Activating manager daemon y 2023-10-09T05:25:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: cluster 2023-10-09T05:25:01.682757+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T05:25:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: cluster 2023-10-09T05:25:01.683136+0000 mon.a (mon.0) 878 : cluster [DBG] 2023-10-09T05:25:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: mgrmap e26: y(active, starting, since 0.006369s) 2023-10-09T05:25:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25:01.683293+0000 2023-10-09T05:25:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: mon.a (mon.0) 879 : audit [DBG] from='mgr.? 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:25:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:25:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 10-09T05:25:01.685596+0000 mon.a (mon. 2023-10-09T05:25:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 0) 880 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: :25:01.685844+0000 mon.a (mon.0) 881 : audit 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 2023-10-09T05:25:01.686064+0000 mon.a (mon.0 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: ) 882 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: -10-09T05:25:01.686287+0000 mon.a 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: (mon.0) 883 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25:01.686485+0000 mon.a (mon.0) 884 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: :01.686687+0000 mon.a (mon.0) 885 : audit 2023-10-09T05:25:02.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi008.zfejpl"}]: dispatch 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25:01 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: .686987+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.walafj"}]: dispatch 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 10-09T05:25:01.687286+0000 mon.a ( 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: mon.0) 887 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25: 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 01.687526+0000 mon.a (mon.0) 888 : audit 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 2023-10-09T05:25:01.687818+0000 mon.a (mon.0 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: ) 889 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:25:02.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T 2023-10-09T05:25:02.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 05:25:01.686987+0000 mon.a (mon.0 2023-10-09T05:25:02.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: ) 886 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.walafj"}]: dispatch 2023-10-09T05:25:02.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:25:02.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 05:25:01.687286+0000 mon.a (mon. 2023-10-09T05:25:02.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 0) 887 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:25:02.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:25:02.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: -09T05:25:01.687526+0000 mon.a 2023-10-09T05:25:02.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: (mon.0) 888 : audit [DBG] 2023-10-09T05:25:02.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 09T05:25:01. 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 687818+0000 mon.a (mon.0 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: ) 889 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 09T05:25:01 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: .688103+0000 mon.a (mon. 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 0) 890 : audit [DBG] 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 10-09T05:25: 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 01.688446+0000 mon.a ( 2023-10-09T05:25:02.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.0) 891 : audit 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: -10-09T05:25 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: :01.688813+0000 mon.a ( 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.0) 892 : audit 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 25:01.689076+0000 mon.a 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: (mon.0) 893 : audit 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 25:01.689307+0000 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.a (mon.0) 894 : audit 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 25:01.689524+0000 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.a (mon.0) 895 : 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 25:01.689767+0000 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.a (mon.0) 896 : 2023-10-09T05:25:02.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:25:02.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 05:25:01.688103+0000 mon.a (mon.0) 890 2023-10-09T05:25:02.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:25:02.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:25:02.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 25:01.688446+0000 mon.a (mon.0) 891 : 2023-10-09T05:25:02.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:25:02.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:25:02.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 25:01.688813+0000 mon.a (mon.0) 892 : audit 2023-10-09T05:25:02.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:25:02.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25 2023-10-09T05:25:02.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: :01.689076+0000 mon.a (mon.0) 893 : audit 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25:01.689307 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: +0000 mon.a (mon.0) 894 : audit [DBG] 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25:01.689524 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: +0000 mon.a (mon.0) 895 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 2023-10-09T05:25:01.689767+0000 mon.a (mon.0 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: ) 896 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:25:02.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 25:01.690011+0000 mon.a (mon.0) 897 : audit 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: :01.690464+0000 mon.a (mon.0) 898 : audit 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09T05:25:01. 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 690678+0000 mon.a (mon.0) 899 : audit [DBG] 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: -09T05:25:01.690981+0000 mon.a (mon.0) 900 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: cluster 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 2023-10-09T05:25:01.850250+0000 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: mon.a (mon.0) 901 : cluster [INF] Manager daemon y is now available 2023-10-09T05:25:02.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023 2023-10-09T05:25:02.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: -10-09T05:25:01.874388+0000 2023-10-09T05:25:02.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: mon.a (mon.0) 902 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:02.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023 2023-10-09T05:25:02.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: -10-09T05:25:01.878055+0000 mon.a 2023-10-09T05:25:02.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05 2023-10-09T05:25:02.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: :25:01.690011+0000 2023-10-09T05:25:02.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.a (mon.0) 897 2023-10-09T05:25:02.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:25:02.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: :25:01.690464+0000 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.a (mon.0) 898 : 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05:25:01 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: .690678+0000 mon.a (mon.0) 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 899 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: T05:25:01.690981+0000 mon.a ( 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.0) 900 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:25:02.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: cluster 2023-10 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: -09T05:25:01.850250+0000 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.a (mon.0) 901 : cluster [INF] Manager daemon y is now available 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 09T05:25:01.874388+0000 mon.a (mon.0) 902 : audit 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: :25:01.878055+0000 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: mon.a (mon.0) 903 : audit 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05:25:01. 2023-10-09T05:25:02.727 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 905181+0000 mon.a (mon.0) 904 : audit [DBG] 2023-10-09T05:25:02.728 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:02.728 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.728 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 2023-10-09T05:25:01.915261+0000 mon.a (mon. 2023-10-09T05:25:02.728 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 0) 905 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:25:02.728 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: audit 2023 2023-10-09T05:25:02.728 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: -10-09T05:25: 2023-10-09T05:25:02.728 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: 01.946607+0000 mon.a (mon 2023-10-09T05:25:02.728 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: .0) 906 : audit [INF] 2023-10-09T05:25:02.728 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[110782]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:25:02.728 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: (mon.0) 903 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:02.728 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:25:02.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: -09T05:25:01.905181+0000 mon.a (mon 2023-10-09T05:25:02.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: .0) 904 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:02.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:25:02.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 09T05:25:01.915261+0000 mon.a (mon. 2023-10-09T05:25:02.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 0) 905 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:25:02.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:25:02.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: T05:25:01.946607+0000 mon.a (mon.0) 2023-10-09T05:25:02.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:02 smithi008 conmon[101053]: 906 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:25:03.300 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[139071]: debug 2023-10-09T05:25:02.853+0000 7fc328644200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:25:03.300 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:02 smithi191 conmon[139071]: debug 2023-10-09T05:25:02.962+0000 7fc328644200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:25:03.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[101053]: cluster 2023-10-09T05:25:02.685343 2023-10-09T05:25:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[101053]: +0000 mon.a (mon.0) 907 : cluster [DBG] mgrmap e27: y(active, since 1.00858s) 2023-10-09T05:25:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:02.793210+0000 mgr.y (mgr.15870) 2 : cephadm [INF] [09/Oct/2023:05:25:02] ENGINE Bus STARTING 2023-10-09T05:25:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:02.913229+0000 mgr.y (mgr.15870) 3 : cephadm [INF] [09/Oct/2023:05:25:02] ENGINE Serving on https://172.21.15.8:7150 2023-10-09T05:25:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[101053]: audit 2023-10-09T05:25:02.979587+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.8:0/1538862118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:03.014469+0000 mgr.y (mgr.15870) 4 : cephadm [INF] [09/Oct/2023:05:25:03] ENGINE Serving on http://172.21.15.8:8765 2023-10-09T05:25:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:03.014633+0000 mgr.y (mgr.15870) 5 : cephadm [INF] [09/Oct/2023:05:25:03] ENGINE Bus STARTED 2023-10-09T05:25:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[110782]: cluster 2023-10-09T05:25:02.685343+0000 mon.a ( 2023-10-09T05:25:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[110782]: mon.0) 907 : cluster [DBG] mgrmap e27: y(active, since 1.00858s) 2023-10-09T05:25:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:02.793210+0000 mgr.y (mgr.15870) 2 : cephadm [INF] [09/Oct/2023:05:25:02] ENGINE Bus STARTING 2023-10-09T05:25:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:02.913229+0000 mgr.y (mgr.15870) 3 : cephadm [INF] [09/Oct/2023:05:25:02] ENGINE Serving on https://172.21.15.8:7150 2023-10-09T05:25:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[110782]: audit 2023-10-09T05:25:02.979587+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.8:0/1538862118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:03.014469+0000 mgr.y (mgr.15870) 4 : cephadm [INF] [09/Oct/2023:05:25:03] ENGINE Serving on http://172.21.15.8:8765 2023-10-09T05:25:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:03 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:03.014633+0000 mgr.y (mgr.15870) 5 : cephadm [INF] [09/Oct/2023:05:25:03] ENGINE Bus STARTED 2023-10-09T05:25:04.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:03 smithi191 conmon[139071]: debug 2023-10-09T05:25:03.828+0000 7fc328644200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:25:04.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:03 smithi191 conmon[139071]: debug 2023-10-09T05:25:03.937+0000 7fc328644200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:25:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:03 smithi191 conmon[101299]: cluster 2023-10-09T05:25:02.685343+0000 mon.a (mon.0) 907 : cluster [DBG] mgrmap e27: y(active, since 1.00858s) 2023-10-09T05:25:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:03 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:02.793210+0000 mgr.y (mgr.15870) 2 : cephadm [INF] [09/Oct/2023:05:25:02] ENGINE Bus STARTING 2023-10-09T05:25:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:03 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:02.913229+0000 mgr.y (mgr.15870) 3 : cephadm [INF] [09/Oct/2023:05:25:02] ENGINE Serving on https://172.21.15.8:7150 2023-10-09T05:25:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:03 smithi191 conmon[101299]: audit 2023-10-09T05:25:02.979587+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.8:0/1538862118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:03 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:03.014469+0000 mgr.y (mgr.15870) 4 : cephadm [INF] [09/Oct/2023:05:25:03] ENGINE Serving on http://172.21.15.8:8765 2023-10-09T05:25:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:03 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:03.014633+0000 mgr.y (mgr.15870) 5 : cephadm [INF] [09/Oct/2023:05:25:03] ENGINE Bus STARTED 2023-10-09T05:25:04.443 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:04 smithi191 conmon[139071]: debug 2023-10-09T05:25:04.137+0000 7fc328644200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:25:04.443 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:04 smithi191 conmon[139071]: debug 2023-10-09T05:25:04.442+0000 7fc328644200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:25:04.701 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:04 smithi191 conmon[101299]: cluster 2023-10-09T05:25:03.688426+0000 mgr.y (mgr.15870) 6 2023-10-09T05:25:04.701 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:04 smithi191 conmon[101299]: : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:04.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:04 smithi008 conmon[101053]: cluster 2023-10-09T05:25:03.688426+0000 mgr.y (mgr.15870) 6 : 2023-10-09T05:25:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:04 smithi008 conmon[101053]: cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:04.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:04 smithi008 conmon[110782]: cluster 2023-10-09T05:25:03 2023-10-09T05:25:04.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:04 smithi008 conmon[110782]: .688426+0000 mgr.y (mgr.15870) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:05.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:04 smithi191 conmon[139071]: debug 2023-10-09T05:25:04.699+0000 7fc328644200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:25:05.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:04 smithi191 conmon[139071]: debug 2023-10-09T05:25:04.852+0000 7fc328644200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:25:05.549 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[139071]: debug 2023-10-09T05:25:05.148+0000 7fc328644200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:25:05.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: cluster 2023-10-09T05:25: 2023-10-09T05:25:05.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: 04.693042+0000 mon.a (mon.0) 909 : cluster [DBG] mgrmap e28: y(active, since 3s) 2023-10-09T05:25:05.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: audit 2023-10-09T05:25:05. 2023-10-09T05:25:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: 180803+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:25:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: -09T05:25:05.185373+0000 mon.a (mon 2023-10-09T05:25:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: .0) 911 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: audit 2023-10-09T05:25:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: 2023-10-09T05:25:05.392909+0000 2023-10-09T05:25:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.8:0/627797004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: audit 2023-10-09T05:25:05 2023-10-09T05:25:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: .491344+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:25:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: 09T05:25:05.495406+0000 mon.a (mon.0) 2023-10-09T05:25:05.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[101053]: 913 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:05.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: cluster 2023-10-09T05:25:04.693042 2023-10-09T05:25:05.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: +0000 mon.a (mon.0) 909 : cluster [DBG] mgrmap e28: y(active, since 3s) 2023-10-09T05:25:05.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:25:05.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: -09T05:25:05.180803+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:05.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:25:05.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: 10-09T05:25:05. 2023-10-09T05:25:05.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: 185373+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:05.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: 2023-10-09T05:25:05.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:25:05.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: 10-09T05:25:05.392909+0000 2023-10-09T05:25:05.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.8:0/627797004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:05.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: 2023-10-09T05:25:05.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: audit 2023-10-09T05:25:05. 2023-10-09T05:25:05.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: 491344+0000 mon.a (mon.0) 912 : audit [INF] 2023-10-09T05:25:05.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:05.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:25:05.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: 05:25:05.495406+0000 mon.a (mon.0 2023-10-09T05:25:05.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:05 smithi008 conmon[110782]: ) 913 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:06.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[139071]: debug 2023-10-09T05:25:05.911+0000 7fc328644200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:25:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[101299]: cluster 2023-10-09T05:25:04.693042+0000 mon.a (mon.0) 909 : cluster 2023-10-09T05:25:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[101299]: [DBG] mgrmap e28: y(active, since 3s) 2023-10-09T05:25:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[101299]: audit 2023-10-09T05:25:05.180803+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[101299]: audit 2023-10-09T05:25 2023-10-09T05:25:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[101299]: :05.185373+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:25:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[101299]: :25:05.392909+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.8:0/627797004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[101299]: audit 2023-10-09T05:25:05. 2023-10-09T05:25:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[101299]: 491344+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[101299]: audit 2023-10-09T05:25:05 2023-10-09T05:25:06.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:05 smithi191 conmon[101299]: .495406+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:06.381 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[139071]: debug 2023-10-09T05:25:06.053+0000 7fc328644200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:25:06.381 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[139071]: debug 2023-10-09T05:25:06.265+0000 7fc328644200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:25:06.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[139071]: debug 2023-10-09T05:25:06.379+0000 7fc328644200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:25:06.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[139071]: debug 2023-10-09T05:25:06.605+0000 7fc328644200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:25:07.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[101053]: cluster 2023-10-09T05:25:05.688734+0000 mgr.y (mgr.15870) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[101053]: audit 2023-10-09T05:25:05.873952+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[101053]: audit 2023-10-09T05:25:05.877672+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[101053]: audit 2023-10-09T05:25:06.236448+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:25:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[101053]: -09T05:25:06.240725+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[110782]: cluster 2023-10-09T05:25:05.688734+0000 mgr.y (mgr.15870) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[110782]: audit 2023-10-09T05:25:05.873952+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[110782]: audit 2023-10-09T05:25:05.877672+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[110782]: audit 2023-10-09T05:25:06.236448+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[110782]: audit 2023-10-09T05:25:06.240725 2023-10-09T05:25:07.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:06 smithi008 conmon[110782]: +0000 mon.a (mon.0) 917 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[101299]: cluster 2023-10-09T05:25:05.688734+0000 mgr.y (mgr.15870 2023-10-09T05:25:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[101299]: ) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[101299]: audit 2023-10-09T05:25:05.873952+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[101299]: audit 2023-10-09T05:25:05.877672+0000 mon.a (mon.0) 915 : 2023-10-09T05:25:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[101299]: audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[101299]: audit 2023-10-09T05:25: 2023-10-09T05:25:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[101299]: 06.236448+0000 mon.a (mon.0) 916 : audit 2023-10-09T05:25:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[101299]: [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[101299]: audit 2023-10-09T05:25: 2023-10-09T05:25:07.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[101299]: 06.240725+0000 mon.a (mon.0) 917 : audit 2023-10-09T05:25:07.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:06 smithi191 conmon[101299]: [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:07.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:07 smithi191 conmon[139071]: debug 2023-10-09T05:25:07.487+0000 7fc328644200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:25:07.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:07 smithi191 conmon[139071]: [09/Oct/2023:05:25:07] ENGINE Bus STARTING 2023-10-09T05:25:07.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:07 smithi191 conmon[139071]: CherryPy Checker: 2023-10-09T05:25:07.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:07 smithi191 conmon[139071]: The Application mounted at '' has an empty config. 2023-10-09T05:25:07.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:07 smithi191 conmon[139071]: 2023-10-09T05:25:07.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:07 smithi191 conmon[139071]: [09/Oct/2023:05:25:07] ENGINE Serving on http://:::9283 2023-10-09T05:25:07.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:07 smithi191 conmon[139071]: [09/Oct/2023:05:25:07] ENGINE Bus STARTED 2023-10-09T05:25:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:08 smithi191 conmon[101299]: cluster 2023-10-09T05:25:07.491586+0000 mon.a (mon.0) 918 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:25:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:08 smithi191 conmon[101299]: audit 2023-10-09T05:25:07.493062+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:25:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:08 smithi191 conmon[101299]: audit 2023-10-09T05:25:07.493899+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:08 smithi191 conmon[101299]: 2023-10-09T05:25:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:08 smithi191 conmon[101299]: audit 2023-10-09T05:25:07.496194+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:25:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:08 smithi191 conmon[101299]: audit 2023-10-09T05:25:07.496556+0000 mon.b (mon.1) 36 : audit 2023-10-09T05:25:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:08 smithi191 conmon[101299]: [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:25:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:08 smithi191 conmon[101299]: cluster 2023- 2023-10-09T05:25:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:08 smithi191 conmon[101299]: 10-09T05:25:07.689105+0000 mgr.y (mgr.15870) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:08 smithi191 conmon[101299]: audit 2023-10-09T05:25:07.812453+0000 mon.c (mon 2023-10-09T05:25:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:08 smithi191 conmon[101299]: .2) 142 : audit [DBG] from='client.? 172.21.15.8:0/2138440279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:08.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[101053]: cluster 2023-10-09T05:25:07.491586 2023-10-09T05:25:08.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[101053]: +0000 mon.a (mon.0) 918 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:25:08.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[101053]: audit 2023-10-09T05:25:07.493062+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:25:08.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[101053]: audit 2023-10-09T05:25:07.493899+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:08.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[101053]: audit 2023-10-09T05:25:07.496194+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:25:08.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[101053]: audit 2023-10-09T05:25:07.496556+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:25:08.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[101053]: cluster 2023-10-09T05:25:07.689105+0000 mgr.y (mgr.15870) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:08.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[101053]: audit 2023-10-09T05:25:07.812453+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.8:0/2138440279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:08.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[110782]: cluster 2023-10-09T05:25:07.491586+0000 2023-10-09T05:25:08.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[110782]: mon.a (mon.0) 918 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:25:08.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[110782]: audit 2023-10-09T05:25:07.493062+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:25:08.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[110782]: audit 2023-10-09T05:25:07.493899+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:08.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[110782]: audit 2023-10-09T05:25:07.496194+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:25:08.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[110782]: audit 2023-10-09T05:25:07.496556+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.191:0/2286721805' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:25:08.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[110782]: cluster 2023-10-09T05:25:07.689105+0000 mgr.y (mgr.15870) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:08.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:08 smithi008 conmon[110782]: audit 2023-10-09T05:25:07.812453+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.8:0/2138440279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:09.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[101053]: cluster 2023-10-09T05:25: 2023-10-09T05:25:09.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[101053]: 08.248406+0000 mon.a (mon.0) 919 : cluster [DBG] mgrmap e29: y(active, since 6s), standbys: x 2023-10-09T05:25:09.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[101053]: audit 2023-10-09T05:25:08.248553+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:25:09.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[101053]: audit 2023-10-09T05:25:09.102159+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:09.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[101053]: audit 2023-10-09T05:25:09.106350+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:09.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[101053]: audit 2023-10-09T05:25:09.107676+0000 mon.a (mon.0) 923 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:09.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[101053]: audit 2023-10-09T05:25:09.108436+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[110782]: cluster 2023-10-09T05:25:08 2023-10-09T05:25:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[110782]: .248406+0000 mon.a (mon.0) 919 : cluster [DBG] mgrmap e29: y(active, since 6s), standbys: x 2023-10-09T05:25:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[110782]: audit 2023-10-09T05:25:08.248553+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:25:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[110782]: audit 2023-10-09T05:25:09.102159+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[110782]: audit 2023-10-09T05:25:09.106350+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[110782]: audit 2023-10-09T05:25:09.107676+0000 mon.a (mon.0) 923 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:09 smithi008 conmon[110782]: audit 2023-10-09T05:25:09.108436+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:09.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:09 smithi191 conmon[101299]: cluster 2023-10-09T05:25:08.248406+0000 mon.a (mon.0) 919 : cluster [DBG] 2023-10-09T05:25:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:09 smithi191 conmon[101299]: mgrmap e29: y(active, since 6s), standbys: x 2023-10-09T05:25:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:09 smithi191 conmon[101299]: audit 2023-10-09T05:25:08.248553+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:25:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:09 smithi191 conmon[101299]: audit 2023-10-09T05:25:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:09 smithi191 conmon[101299]: 2023-10-09T05:25:09.102159+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:09 smithi191 conmon[101299]: audit 2023-10-09T05:25:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:09 smithi191 conmon[101299]: 2023-10-09T05:25:09.106350+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:09.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:09 smithi191 conmon[101299]: audit 2023-10-09T05:25:09 2023-10-09T05:25:09.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:09 smithi191 conmon[101299]: .107676+0000 mon.a (mon.0) 923 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:09.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:09 smithi191 conmon[101299]: audit 2023-10-09T05:25:09.108436+0000 2023-10-09T05:25:09.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:09 smithi191 conmon[101299]: mon.a (mon.0) 924 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:09.109557+0000 mgr.y (mgr.15870) 9 : cephadm [INF] 2023-10-09T05:25:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:25:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:09.109778+0000 mgr.y (mgr.15870) 10 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:25:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 2023-10-09T05:25:09.374530+0000 mgr.y (mgr.15870) 11 : cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:25:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: cephadm 2023-10-09 2023-10-09T05:25:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: T05:25:09.374674+0000 mgr.y (mgr. 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 15870) 12 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: cephadm 2023-10-09T05 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: :25:09.638931+0000 mgr.y (mgr.15870) 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 13 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: cephadm 2023-10-09T05: 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 25:09.639072+0000 mgr.y (mgr.15870) 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 14 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: cluster 2023-10-09T05 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: :25:09.689607+0000 mgr.y (mgr.15870) 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 15 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: cephadm 2023-10-09T05: 2023-10-09T05:25:10.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 25:09.903807+0000 mgr.y (mgr.15870) 16 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: : cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: cephadm 2023-10-09T05:25 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: :09.904005+0000 mgr.y (mgr.15870) 17 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 25:10.173222+0000 mon.a (mon.0) 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 925 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: :25:10.175958+0000 mon.a (mon.0 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: ) 926 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:25:10.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 25:10.179291+0000 mon.a (mon.0) 2023-10-09T05:25:10.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 927 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:25:10.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: :25:10.182333+0000 mon.a (mon.0 2023-10-09T05:25:10.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: ) 928 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:25:10.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: :25:10.186500+0000 mon.a (mon.0 2023-10-09T05:25:10.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: ) 929 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:25:10.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 05:25:10.243083+0000 mon.a (mon. 2023-10-09T05:25:10.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:10 smithi191 conmon[101299]: 0) 930 : audit [DBG] from='client.? 172.21.15.8:0/4143034402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:09.109557+0000 mgr.y 2023-10-09T05:25:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: (mgr.15870) 9 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:25:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:09.109778+0000 mgr.y (mgr.15870) 10 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:25:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:09.374530+0000 mgr.y (mgr.15870) 11 : cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:25:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:09.374674+0000 mgr.y (mgr.15870) 12 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:25:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:09.638931+0000 mgr.y (mgr.15870) 2023-10-09T05:25:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: 13 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:25:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: cephadm 2023 2023-10-09T05:25:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: -10-09T05:25:09 2023-10-09T05:25:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: .639072+0000 mgr.y (mgr. 2023-10-09T05:25:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: 15870) 14 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:25:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: cluster 2023 2023-10-09T05:25:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: -10-09T05:25:09.689607+0000 mgr.y 2023-10-09T05:25:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: (mgr.15870) 15 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: 2023-10-09T05:25:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:09 2023-10-09T05:25:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: .903807+0000 mgr.y (mgr.15870) 16 : cephadm 2023-10-09T05:25:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:25:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: cephadm 2023-10 2023-10-09T05:25:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: -09T05:25:09.904005+0000 mgr.y (mgr 2023-10-09T05:25:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: .15870) 17 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:25:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: audit 2023 2023-10-09T05:25:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: -10-09T05: 2023-10-09T05:25:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: 25:10.173222+0000 mon.a 2023-10-09T05:25:10.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: (mon.0) 925 : audit 2023-10-09T05:25:10.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:25:10.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: -09T05:25:10 2023-10-09T05:25:10.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: .175958+0000 mon.a (mon 2023-10-09T05:25:10.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: .0) 926 : audit [INF] 2023-10-09T05:25:10.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:25:10.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: -09T05:25:10. 2023-10-09T05:25:10.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: 179291+0000 mon.a (mon.0 2023-10-09T05:25:10.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: ) 927 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: 2023-10-09T05:25:10.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: audit 2023-10-09T05:25:10.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: 2023-10-09T05: 2023-10-09T05:25:10.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: 25:10.182333+0000 mon.a 2023-10-09T05:25:10.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: (mon.0) 928 : 2023-10-09T05:25:10.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: audit 2023-10-09T05:25:10.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: 2023-10-09T05:25 2023-10-09T05:25:10.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: :10.186500+0000 mon.a 2023-10-09T05:25:10.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: (mon.0) 929 : audit 2023-10-09T05:25:10.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:25:10.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: 10-09T05:25:10 2023-10-09T05:25:10.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: .243083+0000 mon.a (mon 2023-10-09T05:25:10.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: .0) 930 : audit [DBG] 2023-10-09T05:25:10.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[101053]: from='client.? 172.21.15.8:0/4143034402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:10.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: cephadm 2023-10-09T05:25 2023-10-09T05:25:10.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: :09.109557+0000 mgr.y (mgr.15870) 9 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:25:10.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:10.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 2023-10-09 2023-10-09T05:25:10.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: T05:25:09.109778+0000 mgr.y 2023-10-09T05:25:10.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: (mgr.15870) 10 : cephadm 2023-10-09T05:25:10.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:25:10.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: cephadm 2023-10- 2023-10-09T05:25:10.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 09T05:25:09.374530+0000 mgr.y 2023-10-09T05:25:10.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: (mgr.15870) 11 : 2023-10-09T05:25:10.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:25:10.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:10.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 2023-10-09T05:25:09. 2023-10-09T05:25:10.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 374674+0000 mgr.y (mgr.15870) 2023-10-09T05:25:10.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 12 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:25:10.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 2023-10-09T05:25:10.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:10.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 2023-10-09T05:25:09. 2023-10-09T05:25:10.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 638931+0000 mgr.y (mgr.15870) 2023-10-09T05:25:10.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 13 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:25:10.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: cephadm 2023- 2023-10-09T05:25:10.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 10-09T05:25:09.639072+0000 2023-10-09T05:25:10.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: mgr.y (mgr.15870) 14 : 2023-10-09T05:25:10.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: cephadm [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:25:10.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:25:10.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: -10-09T05:25:09.689607+0000 mgr.y (mgr 2023-10-09T05:25:10.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: .15870) 15 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:10.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:10.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 2023-10-09T05:25:09.903807 2023-10-09T05:25:10.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: +0000 mgr.y (mgr.15870) 16 : cephadm [INF] 2023-10-09T05:25:10.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:25:10.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:10.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 2023-10-09T05:25:09.904005 2023-10-09T05:25:10.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: +0000 mgr.y (mgr.15870) 17 2023-10-09T05:25:10.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:25:10.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: audit 2023 2023-10-09T05:25:10.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: -10-09T05:25:10.173222+0000 2023-10-09T05:25:10.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: mon.a (mon.0) 925 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:25:10.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 10-09T05:25:10.175958+0000 mon.a 2023-10-09T05:25:10.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: (mon.0) 926 : audit 2023-10-09T05:25:10.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: audit 2023 2023-10-09T05:25:10.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: -10-09T05:25:10.179291+0000 2023-10-09T05:25:10.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: mon.a (mon.0) 927 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:25:10.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: :25:10.182333+0000 mon.a (mon.0) 2023-10-09T05:25:10.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 928 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.725 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:25:10.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 25:10.186500+0000 mon.a (mon.0) 2023-10-09T05:25:10.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 929 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:10.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:25:10.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: 25:10.243083+0000 mon.a (mon.0) 930 : 2023-10-09T05:25:10.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:10 smithi008 conmon[110782]: audit [DBG] from='client.? 172.21.15.8:0/4143034402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:11 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:10.319422+0000 mgr.y (mgr.15870) 18 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:25:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:11 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:10.574064+0000 mgr.y (mgr.15870) 19 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:25:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:11 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:10.319422+0000 mgr.y (mgr. 2023-10-09T05:25:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:11 smithi008 conmon[101053]: 15870) 18 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:25:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:11 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:10.574064+0000 mgr.y (mgr.15870) 19 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:25:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:11 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:10.319422+0000 mgr.y (mgr. 2023-10-09T05:25:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:11 smithi008 conmon[110782]: 15870) 18 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:25:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:11 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:10.574064+0000 mgr.y (mgr.15870 2023-10-09T05:25:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:11 smithi008 conmon[110782]: ) 19 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:25:12.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:12 smithi191 conmon[101299]: cluster 2023-10-09T05:25:11.690254+0000 mgr.y (mgr.15870) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:12.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:12 smithi008 conmon[101053]: cluster 2023-10-09T05:25:11.690254+0000 mgr.y (mgr 2023-10-09T05:25:12.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:12 smithi008 conmon[101053]: .15870) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:12.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:12 smithi008 conmon[110782]: cluster 2023-10-09T05:25:11.690254+0000 2023-10-09T05:25:12.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:12 smithi008 conmon[110782]: mgr.y (mgr.15870) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:13.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[139071]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:25:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:25:13.550 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 systemd[1]: Stopping Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:25:13.550 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 bash[150545]: Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-prometheus.a found: no such container 2023-10-09T05:25:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[101299]: audit 2023-10-09T05:25:12.647634+0000 mon.a (mon.0) 931 : audit [DBG] from='client.? 172.21.15.8:0/509194751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:13.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:13 smithi008 conmon[110782]: audit 2023-10-09T05:25:12.647634+0000 mon.a 2023-10-09T05:25:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:13 smithi008 conmon[110782]: (mon.0) 931 : audit [DBG] from='client.? 172.21.15.8:0/509194751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:13.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:13 smithi008 conmon[101053]: audit 2023-10-09T05:25:12.647634+0000 mon.a (mon.0) 931 2023-10-09T05:25:13.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:13 smithi008 conmon[101053]: : audit [DBG] from='client.? 172.21.15.8:0/509194751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:14.049 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.690Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:25:14.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.690Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T05:25:14.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.690Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T05:25:14.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.690Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T05:25:14.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.690Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T05:25:14.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.690Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T05:25:14.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.690Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T05:25:14.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.690Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T05:25:14.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.690Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T05:25:14.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.691Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T05:25:14.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.691Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T05:25:14.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:13 smithi191 conmon[144968]: ts=2023-10-09T05:25:13.691Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T05:25:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[101299]: cluster 2023-10-09T05:25:13.690907+0000 mgr.y ( 2023-10-09T05:25:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[101299]: mgr.15870) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:14.550 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 bash[150661]: 7a37068c353e0d679a45833949067530a077aeee2cf0dde29526afe3d84ccb06 2023-10-09T05:25:14.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T05:25:14.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 systemd[1]: Stopped Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:25:14.551 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 systemd[1]: Starting Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:25:14.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:14 smithi008 conmon[101053]: cluster 2023-10-09T05:25:13.690907 2023-10-09T05:25:14.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:14 smithi008 conmon[101053]: +0000 mgr.y (mgr.15870) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:14.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:14 smithi008 conmon[110782]: cluster 2023-10-09T05:25: 2023-10-09T05:25:14.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:14 smithi008 conmon[110782]: 13.690907+0000 mgr.y (mgr.15870) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:15.299 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 bash[151000]: d24b4acc115690e5770b123a795bc2972c5ea233954d704d31fb6a38bb8563db 2023-10-09T05:25:15.300 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[151041]: ts=2023-10-09T05:25:14.984Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T05:25:15.300 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[151041]: ts=2023-10-09T05:25:14.984Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T05:25:15.300 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[151041]: ts=2023-10-09T05:25:14.984Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi191 (none))" 2023-10-09T05:25:15.300 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[151041]: ts=2023-10-09T05:25:14.984Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T05:25:15.300 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[151041]: ts=2023-10-09T05:25:14.984Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T05:25:15.300 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[151041]: ts=2023-10-09T05:25:14.986Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T05:25:15.300 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[151041]: ts=2023-10-09T05:25:14.987Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T05:25:15.300 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[151041]: ts=2023-10-09T05:25:14.988Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T05:25:15.301 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 systemd[1]: Started Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:25:15.301 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[151041]: ts=2023-10-09T05:25:14.990Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T05:25:15.301 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[151041]: ts=2023-10-09T05:25:14.990Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.599µs 2023-10-09T05:25:15.301 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:14 smithi191 conmon[151041]: ts=2023-10-09T05:25:14.990Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T05:25:15.301 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:15 smithi191 conmon[151041]: ts=2023-10-09T05:25:15.162Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T05:25:15.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:15 smithi008 conmon[177229]: [09/Oct/2023:05:25:15] ENGINE Bus STOPPING 2023-10-09T05:25:15.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:15 smithi008 conmon[177229]: [09/Oct/2023:05:25:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:25:15.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:15 smithi008 conmon[177229]: [09/Oct/2023:05:25:15] ENGINE Bus STOPPED 2023-10-09T05:25:15.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:15 smithi008 conmon[177229]: [09/Oct/2023:05:25:15] ENGINE Bus STARTING 2023-10-09T05:25:15.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:15 smithi008 conmon[177229]: [09/Oct/2023:05:25:15] ENGINE Serving on http://:::9283 2023-10-09T05:25:15.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:15 smithi008 conmon[177229]: [09/Oct/2023:05:25:15] ENGINE Bus STARTED 2023-10-09T05:25:15.640 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:15 smithi191 conmon[151041]: ts=2023-10-09T05:25:15.389Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T05:25:15.640 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:15 smithi191 conmon[151041]: ts=2023-10-09T05:25:15.430Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T05:25:15.640 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:15 smithi191 conmon[151041]: ts=2023-10-09T05:25:15.431Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T05:25:15.640 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:15 smithi191 conmon[151041]: ts=2023-10-09T05:25:15.431Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.079µs wal_replay_duration=440.160578ms total_replay_duration=440.209185ms 2023-10-09T05:25:15.640 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:15 smithi191 conmon[151041]: ts=2023-10-09T05:25:15.434Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T05:25:15.640 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:15 smithi191 conmon[151041]: ts=2023-10-09T05:25:15.434Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T05:25:15.641 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:15 smithi191 conmon[151041]: ts=2023-10-09T05:25:15.434Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T05:25:15.641 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:15 smithi191 conmon[151041]: ts=2023-10-09T05:25:15.452Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.274546ms db_storage=771ns remote_storage=1.5µs web_handler=366ns query_engine=625ns scrape=444.312µs scrape_sd=87.481µs notify=24.738µs notify_sd=14.789µs rules=17.251926ms 2023-10-09T05:25:15.641 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:15 smithi191 conmon[151041]: ts=2023-10-09T05:25:15.452Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T05:25:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:16 smithi191 conmon[101299]: audit 2023-10-09T05:25:15.014781+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:16 smithi191 conmon[101299]: audit 2023-10-09T05:25:15.019970+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:16.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:16 smithi191 conmon[101299]: audit 2023-10-09T05:25:15.055724+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.8:0/2265445833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:16.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:16 smithi191 conmon[101299]: audit 2023-10-09T05:25:15.058587+0000 mon.a (mon.0) 934 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:25:16.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:16 smithi191 conmon[101299]: audit 2023-10-09T05:25:15.059060+0000 mgr.y (mgr.15870) 22 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:25:16.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:16 smithi191 conmon[101299]: audit 2023-10-09T05:25:15.059977+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:25:16.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:16 smithi191 conmon[101299]: audit 2023-10-09T05:25:15.060332+0000 mgr.y (mgr.15870) 23 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:25:16.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:16 smithi191 conmon[101299]: audit 2023-10-09T05:25:15.064698+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:16.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:16 smithi191 conmon[101299]: audit 2023-10-09T05:25:15.081939+0000 mon.a (mon.0) 937 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:16.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit 2023-10-09T05:25:15.014781+0000 mon.a (mon. 2023-10-09T05:25:16.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: 0) 932 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:16.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit 2023-10-09T05:25:15.019970+0000 mon.a (mon.0) 933 : 2023-10-09T05:25:16.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:16.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit 2023 2023-10-09T05:25:16.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: -10-09T05:25:15.055724+0000 mon.c 2023-10-09T05:25:16.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: (mon.2) 143 : audit [DBG] from='client.? 172.21.15.8:0/2265445833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:16.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit 2023-10-09T05:25: 2023-10-09T05:25:16.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: 15.058587+0000 mon.a (mon.0) 934 : 2023-10-09T05:25:16.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:25:16.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: 2023-10-09T05:25:15.059060+0000 mgr.y 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: (mgr.15870) 22 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: 2023-10-09T05:25 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: :15.059977+0000 mon.a (mon. 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: 0) 935 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit 2023 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: -10-09T05:25:15 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: .060332+0000 mgr.y (mgr.15870) 23 : 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit 2023-10 2023-10-09T05:25:16.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: -09T05:25:15.064698 2023-10-09T05:25:16.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: +0000 mon.a (mon.0) 936 : audit [INF] 2023-10-09T05:25:16.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:16.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:25:16.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: T05:25:15.081939+0000 2023-10-09T05:25:16.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: mon.a (mon.0) 937 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:16.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[101053]: 2023-10-09T05:25:16.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: audit 2023-10-09T05:25:15 2023-10-09T05:25:16.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: .014781+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:16.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:25:16.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: :25:15.019970+0000 mon.a (mon.0) 933 : audit 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: audit 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: 2023-10-09T05:25:15.055724 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: +0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.8:0/2265445833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: audit 2023-10-09T05:25:15.058587+0000 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: mon.a (mon.0) 934 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: audit 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: 2023-10-09T05:25:15.059060+0000 mgr.y (mgr.15870) 22 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: 09T05:25:15.059977+0000 mon.a (mon.0) 935 : audit [INF] 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: audit 2023-10-09T05:25:15.060332 2023-10-09T05:25:16.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: +0000 mgr.y (mgr.15870) 23 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:25:16.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: audit 2023 2023-10-09T05:25:16.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: -10-09T05:25:15.064698+0000 mon.a (mon 2023-10-09T05:25:16.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: .0) 936 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:16.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: 2023-10-09T05:25:16.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: audit 2023 2023-10-09T05:25:16.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: -10-09T05:25:15.081939 2023-10-09T05:25:16.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: +0000 mon.a (mon.0) 937 : 2023-10-09T05:25:16.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:16 smithi008 conmon[110782]: audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:17.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:17 smithi191 conmon[101299]: cluster 2023-10-09T05:25:15.691324+0000 mgr.y (mgr.15870) 24 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:17.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:17 smithi191 conmon[101299]: audit 2023-10-09T05:25:16.652857+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:25:17.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:17 smithi191 conmon[101299]: audit 2023-10-09T05:25:16.653964+0000 mon.a (mon.0) 939 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:25:17.332 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:17 smithi191 conmon[101299]: audit 2023-10-09T05:25:16.654984+0000 mon.a (mon.0) 940 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:17.332 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:17 smithi191 conmon[101299]: audit 2023-10-09T05:25:16.905630+0000 mon.a (mon.0) 941 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:17.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[101053]: cluster 2023-10-09T05:25:15.691324+0000 mgr.y (mgr 2023-10-09T05:25:17.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[101053]: .15870) 24 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:17.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[101053]: audit 2023-10-09T05:25:16.652857+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:25:17.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[101053]: audit 2023-10-09T05:25:16.653964+0000 mon.a (mon.0) 939 : audit 2023-10-09T05:25:17.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[101053]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:25:17.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[101053]: audit 2023-10-09T05:25:16.654984 2023-10-09T05:25:17.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[101053]: +0000 mon.a (mon.0) 940 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:17.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[101053]: audit 2023-10-09T05:25:16.905630 2023-10-09T05:25:17.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[101053]: +0000 mon.a (mon.0) 941 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: cluster 2023-10-09T05:25:15.691324 2023-10-09T05:25:17.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: +0000 mgr.y (mgr.15870) 24 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:17.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: audit 2023-10-09T05:25:17.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: 2023-10-09T05:25:16.652857+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:25:17.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:25:17.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: -09T05:25:16.653964+0000 mon.a (mon. 2023-10-09T05:25:17.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: 0) 939 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:25:17.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:17.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: 09T05:25:16.654984+0000 mon.a (mon.0) 2023-10-09T05:25:17.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: 940 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:17.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: audit 2023-10-09T05:25:17.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: 2023-10-09T05:25:16.905630+0000 2023-10-09T05:25:17.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:17 smithi008 conmon[110782]: mon.a (mon.0) 941 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:18.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:18 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:16.652354+0000 mgr.y (mgr.15870) 25 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T05:25:18.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:18 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:16. 2023-10-09T05:25:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:18 smithi008 conmon[101053]: 656071+0000 mgr.y (mgr.15870) 26 : cephadm [INF] Deploying daemon mgr.x on smithi191 2023-10-09T05:25:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:18 smithi008 conmon[101053]: audit 2023-10-09T05:25:17.461204+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.8:0/4003209888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:18 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:16.652354+0000 mgr.y ( 2023-10-09T05:25:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:18 smithi008 conmon[110782]: mgr.15870) 25 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T05:25:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:18 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:18 smithi008 conmon[110782]: 2023-10-09T05:25:16.656071+0000 mgr.y (mgr.15870) 26 : cephadm [INF] Deploying daemon mgr.x on smithi191 2023-10-09T05:25:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:18 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:25:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:18 smithi008 conmon[110782]: 10-09T05:25:17.461204+0000 mon.c (mon.2) 144 : audit 2023-10-09T05:25:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:18 smithi008 conmon[110782]: [DBG] from='client.? 172.21.15.8:0/4003209888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:18.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:18 smithi191 systemd[1]: Stopping Ceph mgr.x for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:25:18.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:18 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:16.652354+0000 mgr.y (mgr. 2023-10-09T05:25:18.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:18 smithi191 conmon[101299]: 15870) 25 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T05:25:18.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:18 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:16.656071+0000 mgr.y (mgr.15870) 26 : cephadm [INF] Deploying daemon mgr.x on smithi191 2023-10-09T05:25:18.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:18 smithi191 conmon[101299]: audit 2023-10-09T05:25:17.461204+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.8:0/4003209888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:19.411 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:19 smithi191 conmon[101299]: cluster 2023-10-09T05:25 2023-10-09T05:25:19.412 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:19 smithi191 conmon[101299]: :17.691928+0000 mgr.y (mgr.15870) 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:19.412 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:19 smithi191 bash[151680]: 196b73760dba15a429271b13795cd8ae58babfa17ce98c38e245b7dd6728a780 2023-10-09T05:25:19.412 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:19 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:25:19.412 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:19 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T05:25:19.412 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:19 smithi191 systemd[1]: Stopped Ceph mgr.x for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:25:19.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:19 smithi008 conmon[101053]: cluster 2023-10-09T05:25:17.691928+0000 mgr.y (mgr. 2023-10-09T05:25:19.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:19 smithi008 conmon[101053]: 15870) 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:19.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:19 smithi008 conmon[110782]: cluster 2023-10-09T05:25:17.691928+0000 mgr.y (mgr.15870 2023-10-09T05:25:19.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:19 smithi008 conmon[110782]: ) 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:20.181 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:20 smithi191 conmon[101299]: audit 2023-10-09T05:25:19. 2023-10-09T05:25:20.181 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:20 smithi191 conmon[101299]: 863836+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.8:0/1631815727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:20.181 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:19 smithi191 systemd[1]: Starting Ceph mgr.x for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:25:20.181 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:20 smithi191 ceph-mgr[152105]: -- 172.21.15.191:0/1018381760 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5611879191e0 con 0x5611878f0c00 2023-10-09T05:25:20.182 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:20 smithi191 bash[152039]: e651b468b4c85bab6b52d169e84fb66c7cd0428b9c8519a0db545201b5d7f008 2023-10-09T05:25:20.182 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:20 smithi191 systemd[1]: Started Ceph mgr.x for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:25:20.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:20 smithi008 conmon[101053]: audit 2023-10-09T05:25:19 2023-10-09T05:25:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:20 smithi008 conmon[101053]: .863836+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.8:0/1631815727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:20 smithi008 conmon[110782]: audit 2023-10-09T05:25 2023-10-09T05:25:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:20 smithi008 conmon[110782]: :19.863836+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.8:0/1631815727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:20.474 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:20 smithi191 conmon[152080]: 2023-10-09T05:25:20.179+0000 7f9495db4200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:25:20.474 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:20 smithi191 conmon[152080]: 2023-10-09T05:25:20.263+0000 7f9495db4200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:25:20.474 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:20 smithi191 conmon[152080]: 2023-10-09T05:25:20.361+0000 7f9495db4200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:25:20.474 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:20 smithi191 conmon[152080]: 2023-10-09T05:25:20.472+0000 7f9495db4200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:25:20.799 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:20 smithi191 conmon[152080]: 2023-10-09T05:25:20.672+0000 7f9495db4200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:25:20.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:25:20] "GET /metrics HTTP/1.1" 200 33911 "" "Prometheus/2.33.4" 2023-10-09T05:25:21.416 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[152080]: 2023-10-09T05:25:21.063+0000 7f9495db4200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:25:21.416 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[152080]: 2023-10-09T05:25:21.212+0000 7f9495db4200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:25:21.416 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[152080]: 2023-10-09T05:25:21.302+0000 7f9495db4200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:25:21.416 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[152080]: 2023-10-09T05:25:21.414+0000 7f9495db4200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:25:21.416 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[101299]: cluster 2023-10-09T05:25:19.692404+0000 mgr.y (mgr. 2023-10-09T05:25:21.416 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[101299]: 15870) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:21.416 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[101299]: audit 2023-10-09T05:25:20.122215+0000 mon.a (mon.0) 2023-10-09T05:25:21.417 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[101299]: 942 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:21.417 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[101299]: audit 2023-10-09T05:25:21.417 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[101299]: 2023-10-09T05:25:20.126387+0000 mon.a (mon.0) 943 2023-10-09T05:25:21.417 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[101299]: : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:21.417 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:25:21.417 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[101299]: 05:25:20.128769+0000 mon.a (mon.0 2023-10-09T05:25:21.417 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[101299]: ) 944 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:21.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[101053]: cluster 2023-10-09T05:25:19.692404+0000 mgr.y 2023-10-09T05:25:21.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[101053]: (mgr.15870) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:21.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[101053]: audit 2023-10-09T05:25:20.122215+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:21.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[101053]: audit 2023-10-09T05:25:20.126387+0000 mon.a (mon.0) 943 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:21.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[101053]: audit 2023-10-09T05:25:20.128769 2023-10-09T05:25:21.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[101053]: +0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:21.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: cluster 2023-10-09T05:25:19.692404+0000 mgr.y 2023-10-09T05:25:21.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: (mgr.15870) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:21.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: audit 2023-10-09T05:25:21.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: 2023-10-09T05:25: 2023-10-09T05:25:21.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: 20.122215+0000 mon.a (mon.0) 942 2023-10-09T05:25:21.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:21.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:25:21.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: T05:25:20.126387+0000 mon.a (mon.0) 943 : audit [INF] 2023-10-09T05:25:21.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:21.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: audit 2023-10-09T05:25:21.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: 2023-10-09T05:25:20.128769+0000 mon.a (mon.0) 944 : audit [DBG] 2023-10-09T05:25:21.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:21 smithi008 conmon[110782]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:22.299 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:21 smithi191 conmon[152080]: 2023-10-09T05:25:21.901+0000 7f9495db4200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:25:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:22 smithi191 conmon[101299]: cluster 2023-10-09T05:25:21.692952+0000 mgr.y (mgr.15870) 29 : cluster 2023-10-09T05:25:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:22 smithi191 conmon[101299]: [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:22.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:22 smithi008 conmon[101053]: cluster 2023-10-09T05:25:21.692952+0000 2023-10-09T05:25:22.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:22 smithi008 conmon[101053]: mgr.y (mgr.15870) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:22.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:22 smithi008 conmon[110782]: cluster 2023-10-09T05:25:21.692952+0000 2023-10-09T05:25:22.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:22 smithi008 conmon[110782]: mgr.y (mgr.15870) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:23.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:22 smithi191 conmon[152080]: 2023-10-09T05:25:22.635+0000 7f9495db4200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:25:23.050 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:22 smithi191 conmon[152080]: 2023-10-09T05:25:22.726+0000 7f9495db4200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:25:23.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:23 smithi191 conmon[101299]: audit 2023-10-09T05:25:22.274032+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.8:0/3610850332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:23.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:23 smithi191 conmon[101299]: 2023-10-09T05:25:23.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:23 smithi008 conmon[101053]: audit 2023-10-09T05:25:22.274032+0000 2023-10-09T05:25:23.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:23 smithi008 conmon[101053]: mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.8:0/3610850332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:23.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:23 smithi008 conmon[110782]: audit 2023-10-09T05:25:22.274032+0000 mon.c (mon.2) 146 : audit 2023-10-09T05:25:23.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:23 smithi008 conmon[110782]: [DBG] from='client.? 172.21.15.8:0/3610850332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:23.863 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:23 smithi191 conmon[152080]: 2023-10-09T05:25:23.572+0000 7f9495db4200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:25:23.863 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:23 smithi191 conmon[152080]: 2023-10-09T05:25:23.670+0000 7f9495db4200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:25:24.168 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:23 smithi191 conmon[152080]: 2023-10-09T05:25:23.862+0000 7f9495db4200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:25:24.168 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[152080]: 2023-10-09T05:25:24.166+0000 7f9495db4200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:25:24.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[152080]: 2023-10-09T05:25:24.417+0000 7f9495db4200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:25:24.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[152080]: 2023-10-09T05:25:24.598+0000 7f9495db4200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:25:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: audit 2023-10-09T05:25:23. 2023-10-09T05:25:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: 510743+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:25:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: -09T05:25:23.515996+0000 mon.a (mon.0) 946 : audit [INF] 2023-10-09T05:25:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: cluster 2023-10-09T 2023-10-09T05:25:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: 05:25:23.693590+0000 mgr.y (mgr.15870 2023-10-09T05:25:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: ) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: audit 2023-10-09T05:25:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: 2023-10-09 2023-10-09T05:25:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: T05:25:24.203354+0000 mon.a (mon 2023-10-09T05:25:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: .0) 947 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: audit 2023-10-09T05:25:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: 2023-10-09T05:25 2023-10-09T05:25:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: :24.208451+0000 mon.a (mon.0) 948 2023-10-09T05:25:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[101299]: : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:24.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[101053]: audit 2023-10-09T05:25:23.510743+0000 mon.a ( 2023-10-09T05:25:24.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[101053]: mon.0) 945 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:24.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[101053]: audit 2023-10-09T05:25:23.515996+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:24.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[101053]: cluster 2023-10-09T05:25:24.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[101053]: 2023-10-09T05:25:23.693590+0000 mgr.y (mgr.15870) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:24.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:25:24.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[101053]: T05:25:24.203354+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:24.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[101053]: audit 2023-10-09T05:25: 2023-10-09T05:25:24.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[101053]: 24.208451+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: audit 2023-10-09T05:25:23 2023-10-09T05:25:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: .510743+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:25:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: :25:23.515996+0000 mon.a (mon.0 2023-10-09T05:25:24.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: ) 946 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:24.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: cluster 2023-10-09 2023-10-09T05:25:24.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: T05:25:23.693590+0000 mgr.y (mgr.15870) 2023-10-09T05:25:24.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:24.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: audit 2023-10-09T05:25:24 2023-10-09T05:25:24.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: .203354+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:24.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: audit 2023-10-09T05:25: 2023-10-09T05:25:24.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:24 smithi008 conmon[110782]: 24.208451+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:25.073 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:24 smithi191 conmon[152080]: 2023-10-09T05:25:24.899+0000 7f9495db4200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:25:25.782 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:25 smithi191 conmon[101299]: audit 2023-10-09T05:25:24.678732+0000 mon.a (mon.0) 949 : audit [DBG] from='client.? 172.21.15.8:0/203596795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:25.783 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:25 smithi191 conmon[152080]: 2023-10-09T05:25:25.642+0000 7f9495db4200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:25:25.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:25 smithi008 conmon[101053]: audit 2023-10-09T05:25:24 2023-10-09T05:25:25.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:25 smithi008 conmon[101053]: .678732+0000 mon.a (mon.0) 949 : audit [DBG] from='client.? 172.21.15.8:0/203596795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:25.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:25 smithi008 conmon[110782]: audit 2023-10-09T05:25:24.678732+0000 mon.a (mon.0) 949 : audit [DBG] 2023-10-09T05:25:25.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:25 smithi008 conmon[110782]: from='client.? 172.21.15.8:0/203596795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:26.035 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:25 smithi191 conmon[152080]: 2023-10-09T05:25:25.781+0000 7f9495db4200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:25:26.035 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:25 smithi191 conmon[152080]: 2023-10-09T05:25:25.991+0000 7f9495db4200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:25:26.300 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:26 smithi191 conmon[152080]: 2023-10-09T05:25:26.117+0000 7f9495db4200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:25:26.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:26 smithi191 conmon[152080]: 2023-10-09T05:25:26.351+0000 7f9495db4200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:25:26.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:26 smithi191 conmon[101299]: cluster 2023-10-09T05:25:25.694199+0000 mgr.y (mgr.15870) 31 : cluster [DBG] 2023-10-09T05:25:26.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:26 smithi191 conmon[101299]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:26.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:26 smithi008 conmon[101053]: cluster 2023-10-09T05:25:25.694199+0000 mgr.y (mgr 2023-10-09T05:25:26.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:26 smithi008 conmon[101053]: .15870) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:26.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:26 smithi008 conmon[110782]: cluster 2023-10-09T05:25:25.694199+0000 mgr.y (mgr.15870) 31 : cluster 2023-10-09T05:25:26.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:26 smithi008 conmon[110782]: [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:27.525 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[152080]: 2023-10-09T05:25:27.276+0000 7f9495db4200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:25:27.525 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[152080]: [09/Oct/2023:05:25:27] ENGINE Bus STARTING 2023-10-09T05:25:27.525 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[152080]: CherryPy Checker: 2023-10-09T05:25:27.525 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[152080]: The Application mounted at '' has an empty config. 2023-10-09T05:25:27.525 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[152080]: 2023-10-09T05:25:27.525 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[152080]: [09/Oct/2023:05:25:27] ENGINE Serving on http://:::9283 2023-10-09T05:25:27.525 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[152080]: [09/Oct/2023:05:25:27] ENGINE Bus STARTED 2023-10-09T05:25:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T05:25:27.087060+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.8:0/21927911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: cluster 2023-10-09T05:25:27.278968+0000 mon.a (mon.0) 951 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T05:25:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: cluster 2023-10-09T05:25:27. 2023-10-09T05:25:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 279090+0000 mon.a (mon.0) 952 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:25:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:25:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 09T05:25:27.280568+0000 2023-10-09T05:25:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: mon.b (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:25:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: -10-09T05:25:27.281076+0000 mon.b 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: T05:25:27.284020+0000 mon.a (mon. 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 0) 953 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 05:25:27.284237+0000 mon.b (mon.1 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: ) 39 : audit [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T05:25 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: :27.284600+0000 mon.b (mon.1) 40 : audit 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:25:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 25:27.288271+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 09T05:25:27.289149+0000 mon.a (mon.0) 955 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 25:27.289942+0000 mon.a (mon 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: .0) 956 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 2023-10-09T05:25:27.294189+0000 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: mon.a (mon.0) 957 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:25:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: :25:27.432840+0000 mon.a (mon.0 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: ) 958 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T05:25:27.435271+0000 mon.a ( 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: mon.0) 959 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 25:27.441171+0000 mon.a (mon.0) 960 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T05:25:27. 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 441908+0000 mon.a (mon.0) 961 : audit [INF] 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 05:25:27.446506+0000 mon.a (mon.0) 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 962 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T05:25:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023 2023-10-09T05:25:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: -10-09T05:25 2023-10-09T05:25:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: :27.447173+0000 mon.a (mon.0) 963 : audit 2023-10-09T05:25:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T05:25:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T05:25:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 2023-10-09T05:25:27.451340+0000 mon.a (mon.0 2023-10-09T05:25:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: ) 964 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T05:25:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:25:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 25:27.452530+0000 mon.a (mon.0) 965 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:25:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:25:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: 09T05:25:27.453168+0000 mon.a 2023-10-09T05:25:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:27 smithi191 conmon[101299]: (mon.0) 966 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T05:25:27.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05:25:27.087060+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.8:0/21927911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: cluster 2023-10-09T05:25:27.278968+0000 mon.a (mon.0) 951 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T05:25:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: cluster 2023-10-09T05:25:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 2023-10-09T05:25:27.279090+0000 mon.a (mon.0) 952 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:25:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05:25:27.280568+0000 mon.b (mon. 2023-10-09T05:25:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 1) 37 : audit [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:25:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05:25:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 2023-10-09T05:25:27.281076+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 2023-10-09T05:25:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 2023-10-09T05:25:27.284020+0000 mon.a (mon.0) 953 : audit 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10- 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 09T05:25:27.284237+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05:25:27. 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 284600+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05:25:27.288271 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: +0000 mon.a (mon.0) 954 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 2023-10-09T05:25:27.289149+0000 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: mon.a (mon.0) 955 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: -10-09T05:25:27.289942+0000 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: mon.a (mon.0) 956 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 25:27.294189+0000 mon.a (mon.0) 957 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05: 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 25:27.432840+0000 mon.a (mon.0) 958 : 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023- 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 10-09T05:25:27.435271+0000 mon.a (mon.0) 959 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:25:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: T05:25:27.441171+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05:25:27. 2023-10-09T05:25:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 441908+0000 mon.a (mon.0) 961 : audit [INF] 2023-10-09T05:25:27.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10-09T05:25:27.087060 2023-10-09T05:25:27.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: +0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.8:0/21927911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:27.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: cluster 2023-10-09T05:25:27.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 2023-10-09T05:25:27.278968+0000 mon.a (mon.0) 951 : cluster [DBG] 2023-10-09T05:25:27.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: Standby manager daemon x restarted 2023-10-09T05:25:27.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: cluster 2023-10-09T05 2023-10-09T05:25:27.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: :25:27.279090+0000 mon.a (mon.0) 2023-10-09T05:25:27.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 952 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023- 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 10-09T05:25:27.280568+0000 mon.b ( 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 09T05:25:27.281076+0000 mon.b (mon. 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 1) 38 : audit [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: -10-09T05:25:27.284020+0000 mon.a 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: (mon.0) 953 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 05:25:27.284237+0000 mon.b (mon.1) 39 : audit 2023-10-09T05:25:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:25:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10-09T05 2023-10-09T05:25:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: :25:27.284600+0000 mon.b (mon.1) 40 2023-10-09T05:25:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: : audit [DBG] from='mgr.? 172.21.15.191:0/3324874133' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:25:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 09T05:25:27.288271+0000 mon.a (mon 2023-10-09T05:25:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: .0) 954 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10-09T05:25:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 2023-10-09T05:25:27.289149+0000 2023-10-09T05:25:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: mon.a (mon.0) 955 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 2023-10-09T05:25:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10-09T05:25 2023-10-09T05:25:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: :27.289942+0000 mon.a (mon.0) 956 : 2023-10-09T05:25:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 09T05:25:27.294189+0000 mon.a (mon. 2023-10-09T05:25:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 0) 957 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10-09T05:25:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 2023-10-09T05:25:27 2023-10-09T05:25:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: .432840+0000 mon.a (mon.0) 958 : 2023-10-09T05:25:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 09T05:25:27.435271+0000 mon.a (mon 2023-10-09T05:25:27.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T05:25:27.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09 2023-10-09T05:25:27.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: T 2023-10-09T05:25:27.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 05:25: 2023-10-09T05:25:27.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 27.446506+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T05:25:27.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 2023-10-09T05:25:27.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05:25:27.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 2023-10-09T05:25:27.447173+0000 mon.a (mon.0 2023-10-09T05:25:27.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: ) 963 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T05:25:27.973 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05:25:27.973 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: 2023-10-09T05:25:27.451340+0000 mon.a (mon.0) 964 : 2023-10-09T05:25:27.973 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T05:25:27.973 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05:25:27.452530+0000 mon.a (mon.0) 965 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:25:27.973 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: audit 2023-10-09T05 2023-10-09T05:25:27.973 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: :25:27.453168+0000 mon.a (mon.0) 966 : audit [DBG] 2023-10-09T05:25:27.973 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[101053]: from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T05:25:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: .0) 959 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:25:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:25:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: -09T05:25:27.441171+0000 mon.a ( 2023-10-09T05:25:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: mon.0) 960 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:25:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: -09T05:25:27.441908+0000 mon.a ( 2023-10-09T05:25:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: mon.0) 961 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T05:25:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:25:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: -09T05:25:27.446506+0000 mon.a (mon. 2023-10-09T05:25:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 0) 962 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T05:25:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:25:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: T05:25:27.447173+0000 mon.a (mon. 2023-10-09T05:25:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 0) 963 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T05:25:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10-09T05: 2023-10-09T05:25:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 25:27.451340+0000 mon.a (mon.0) 2023-10-09T05:25:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 964 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T05:25:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10-09T05:25: 2023-10-09T05:25:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 27.452530+0000 mon.a (mon.0) 965 : audit 2023-10-09T05:25:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:25:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: audit 2023-10-09T05:25:27. 2023-10-09T05:25:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: 453168+0000 mon.a (mon.0) 966 : audit 2023-10-09T05:25:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:27 smithi008 conmon[110782]: [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T05:25:28.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:28 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:27.435903+0000 2023-10-09T05:25:28.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:28 smithi008 conmon[101053]: mgr.y (mgr.15870) 32 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T05:25:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:28 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:27.453651+0000 mgr.y (mgr.15870) 33 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T05:25:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:28 smithi008 conmon[101053]: cluster 2023-10-09T05:25:27.694623+0000 mgr.y (mgr.15870) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:28 smithi008 conmon[101053]: cluster 2023-10-09T05:25:28.249653+0000 mon.a (mon.0) 967 : cluster [DBG] mgrmap e30: y(active, since 26s), standbys: x 2023-10-09T05:25:28.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:28 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:27.435903 2023-10-09T05:25:28.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:28 smithi008 conmon[110782]: +0000 mgr.y (mgr.15870) 32 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T05:25:28.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:28 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:27.453651+0000 mgr.y (mgr.15870) 33 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T05:25:28.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:28 smithi008 conmon[110782]: cluster 2023-10-09T05:25:27.694623+0000 mgr.y (mgr.15870) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:28.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:28 smithi008 conmon[110782]: cluster 2023-10-09T05:25:28.249653+0000 mon.a (mon.0) 967 : cluster [DBG] mgrmap e30: y(active, since 26s), standbys: x 2023-10-09T05:25:28.800 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:28 smithi191 conmon[151041]: ts=2023-10-09T05:25:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:25:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:28 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:27.435903+0000 mgr.y (mgr.15870) 32 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T05:25:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:28 smithi191 conmon[101299]: 2023-10-09T05:25:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:28 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:27.453651+0000 mgr.y (mgr.15870) 33 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T05:25:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:28 smithi191 conmon[101299]: cluster 2023-10-09T05:25:27.694623+0000 mgr.y (mgr.15870) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:28 smithi191 conmon[101299]: cluster 2023-10-09T05:25:28.249653+0000 mon.a (mon.0) 967 : cluster [DBG] mgrmap e30: y(active, since 26s), standbys: x 2023-10-09T05:25:30.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:28.920657+0000 mgr.y (mgr.15870) 35 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T05:25:30.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[101053]: audit 2023-10-09T05:25:28.924980+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:30.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[101053]: audit 2023-10-09T05:25:28.925828+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:25:30.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[101053]: audit 2023-10-09T05:25:28.926441+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:25:30.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[101053]: audit 2023-10-09T05:25:28.927023+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:30.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[101053]: cephadm 2023-10-09T05:25:28.927821+0000 mgr.y (mgr.15870) 36 : cephadm [INF] Deploying daemon mon.a on smithi008 2023-10-09T05:25:30.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[101053]: audit 2023-10-09T05:25:29.503980+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.8:0/4129915813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:30.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:28.920657+0000 mgr.y (mgr.15870) 35 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T05:25:30.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[110782]: audit 2023-10-09T05:25:28.924980+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:30.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[110782]: audit 2023-10-09T05:25:28.925828+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:25:30.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[110782]: audit 2023-10-09T05:25:28.926441+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:25:30.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:25:30.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[110782]: T05:25:28.927023+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:30.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:28.927821+0000 mgr.y (mgr.15870) 36 : cephadm [INF] Deploying daemon mon.a on smithi008 2023-10-09T05:25:30.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:29 smithi008 conmon[110782]: audit 2023-10-09T05:25:29.503980+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.8:0/4129915813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:30.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:29 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:28.920657+0000 mgr.y (mgr.15870) 35 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T05:25:30.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:29 smithi191 conmon[101299]: audit 2023-10-09T05:25:28.924980+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' 2023-10-09T05:25:30.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:29 smithi191 conmon[101299]: audit 2023-10-09T05:25:28.925828+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:25:30.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:29 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:25:30.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:29 smithi191 conmon[101299]: 10-09T05:25:28.926441+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:25:30.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:29 smithi191 conmon[101299]: audit 2023-10-09T05:25:28.927023+0000 mon.a (mon 2023-10-09T05:25:30.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:29 smithi191 conmon[101299]: .0) 971 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:30.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:29 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:28.927821+0000 mgr.y ( 2023-10-09T05:25:30.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:29 smithi191 conmon[101299]: mgr.15870) 36 : cephadm [INF] Deploying daemon mon.a on smithi008 2023-10-09T05:25:30.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:29 smithi191 conmon[101299]: audit 2023-10-09T05:25: 2023-10-09T05:25:30.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:29 smithi191 conmon[101299]: 29.503980+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.8:0/4129915813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:30.504 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:30 smithi008 systemd[1]: Stopping Ceph mon.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:25:30.506 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:25:30] "GET /metrics HTTP/1.1" 200 33911 "" "Prometheus/2.33.4" 2023-10-09T05:25:30.769 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:30 smithi008 conmon[101053]: debug 2023-10-09T05:25:30.648+0000 7ff4aba13700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T05:25:30.770 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:30 smithi008 conmon[101053]: debug 2023-10-09T05:25:30.648+0000 7ff4aba13700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T05:25:30.770 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:30 smithi008 conmon[177229]: [09/Oct/2023:05:25:30] ENGINE Bus STOPPING 2023-10-09T05:25:30.770 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:30 smithi008 conmon[177229]: [09/Oct/2023:05:25:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:25:30.770 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:30 smithi008 conmon[177229]: [09/Oct/2023:05:25:30] ENGINE Bus STOPPED 2023-10-09T05:25:30.770 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:30 smithi008 conmon[177229]: [09/Oct/2023:05:25:30] ENGINE Bus STARTING 2023-10-09T05:25:30.771 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:30 smithi008 conmon[177229]: [09/Oct/2023:05:25:30] ENGINE Serving on http://:::9283 2023-10-09T05:25:30.771 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:30 smithi008 conmon[177229]: [09/Oct/2023:05:25:30] ENGINE Bus STARTED 2023-10-09T05:25:31.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:31 smithi008 bash[183955]: d155e1e1e35f7314228013264251d26590c9ba27beb998fb6950b033f07c090a 2023-10-09T05:25:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:31 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T05:25:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:31 smithi008 systemd[1]: Stopped Ceph mon.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:25:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:31 smithi008 systemd[1]: Starting Ceph mon.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:25:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:25:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T05:25:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: pidfile_write: ignore empty --pid-file 2023-10-09T05:25:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: load: jerasure load: lrc 2023-10-09T05:25:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: RocksDB version: 7.9.2 2023-10-09T05:25:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Git sha 0 2023-10-09T05:25:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T05:25:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: DB SUMMARY 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: DB Session ID: ZH8HXBQGGCWHIB9NGAS5 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: MANIFEST file: MANIFEST-000008 size: 780 Bytes 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000020.sst 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000018.log size: 3563304 ; 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:25:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.env: 0x55b482118720 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.info_log: 0x55b483ecf420 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.statistics: (nil) 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.use_fsync: 0 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:25:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:25:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:25:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:25:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:25:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:25:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:25:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.db_log_dir: 2023-10-09T05:25:32.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.wal_dir: 2023-10-09T05:25:32.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:25:32.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:25:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:25:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:25:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:25:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:25:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:25:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:25:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.write_buffer_manager: 0x55b48315e5a0 2023-10-09T05:25:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:25:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:25:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:25:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.unordered_write: 0 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.row_cache: None 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.wal_filter: None 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:25:32.721 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.wal_compression: 0 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:25:32.722 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:25:32.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:25:32.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:25:32.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T05:25:32.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:25:32.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:25:32.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:25:32.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:25:32.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:25:32.723 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_open_files: -1 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:25:32.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Compression algorithms supported: 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: kZSTD supported: 0 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T05:25:32.725 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.merge_operator: 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_filter: None 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b483ecf520) 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:25:32.726 INFO:journalctl@ceph.mon.a.smithi008.stdout: index_type: 0 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: data_block_index_type: 0 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: index_shortening: 1 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: checksum: 4 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: no_block_cache: 0 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: block_cache: 0x55b4831e1350 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: block_cache_options: 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: capacity : 536870912 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: num_shard_bits : 4 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: strict_capacity_limit : 0 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:25:32.727 INFO:journalctl@ceph.mon.a.smithi008.stdout: block_cache_compressed: (nil) 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: persistent_cache: (nil) 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: block_size: 4096 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: block_size_deviation: 10 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: block_restart_interval: 16 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: index_block_restart_interval: 1 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: metadata_block_size: 4096 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: partition_filters: 0 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: use_delta_encoding: 1 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: filter_policy: bloomfilter 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: whole_key_filtering: 1 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: verify_compression: 0 2023-10-09T05:25:32.728 INFO:journalctl@ceph.mon.a.smithi008.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout: format_version: 5 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout: enable_index_compression: 1 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout: block_align: 0 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout: max_auto_readahead_size: 262144 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout: prepopulate_block_cache: 0 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout: initial_auto_readahead_size: 8192 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compression: NoCompression 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:25:32.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.num_levels: 7 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:25:32.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:25:32.731 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:25:32.732 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:25:32.733 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:25:32.733 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T05:25:32.733 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T05:25:32.733 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:25:32.733 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:25:32.733 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:25:32.733 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:25:32.733 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:25:32.733 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:25:32.733 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:25:32.733 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:25:32.734 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:25:32.735 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:25:32.735 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:25:32.735 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:25:32.735 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:25:32.735 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:25:32.735 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.ttl: 2592000 2023-10-09T05:25:32.735 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:25:32.735 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T05:25:32.735 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T05:25:32.736 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:25:32.736 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 20.sst 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 22, last_sequence is 10175, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:25:32.737 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 18 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ec6c2e3c-4b5d-4804-84dc-402ff212e1e9 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829132353452, "job": 1, "event": "recovery_started", "wal_files": [18]} 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #18 mode 2 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829132362918, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 23, "file_size": 3425067, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10167, "largest_seqno": 10898, "table_properties": {"data_size": 3419966, "index_size": 3442, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 645, "raw_key_size": 5453, "raw_average_key_size": 21, "raw_value_size": 3412459, "raw_average_value_size": 13329, "num_data_blocks": 154, "num_entries": 256, "num_filter_entries": 256, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696829132, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ec6c2e3c-4b5d-4804-84dc-402ff212e1e9", "db_session_id": "ZH8HXBQGGCWHIB9NGAS5", "orig_file_number": 23, "seqno_to_time_mapping": "N/A"}} 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829132363011, "job": 1, "event": "recovery_finished"} 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/version_set.cc:5047] Creating manifest 25 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000018.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b48327c000 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: DB pointer 0x55b48326a000 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T05:25:32.738 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: ** DB Stats ** 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: ** Compaction Stats [default] ** 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:25:32.739 INFO:journalctl@ceph.mon.a.smithi008.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: L0 1/0 3.27 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 659.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: L6 1/0 23.59 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: Sum 2/0 26.86 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 659.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 659.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: ** Compaction Stats [default] ** 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 659.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T05:25:32.740 INFO:journalctl@ceph.mon.a.smithi008.stdout: 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: Cumulative compaction: 0.00 GB write, 272.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: Interval compaction: 0.00 GB write, 272.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: Block cache BinnedLRUCache@0x55b4831e1350#7 capacity: 512.00 MB usage: 4.80 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.72 KB,0.000137091%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-10-09T05:25:32.741 INFO:journalctl@ceph.mon.a.smithi008.stdout: 2023-10-09T05:25:32.742 INFO:journalctl@ceph.mon.a.smithi008.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:25:32.742 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: starting mon.a rank 0 at public addrs [v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0] at bind addrs [v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:25:32.742 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: mon.a@-1(???) e3 preinit fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:25:32.742 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: mon.a@-1(???).mds e6 new map 2023-10-09T05:25:32.742 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: mon.a@-1(???).mds e6 print_map 2023-10-09T05:25:32.742 INFO:journalctl@ceph.mon.a.smithi008.stdout: e6 2023-10-09T05:25:32.742 INFO:journalctl@ceph.mon.a.smithi008.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:25:32.742 INFO:journalctl@ceph.mon.a.smithi008.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:25:32.742 INFO:journalctl@ceph.mon.a.smithi008.stdout: legacy client fscid: 1 2023-10-09T05:25:32.742 INFO:journalctl@ceph.mon.a.smithi008.stdout: 2023-10-09T05:25:32.742 INFO:journalctl@ceph.mon.a.smithi008.stdout: Filesystem 'foo' (1) 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: fs_name foo 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: epoch 5 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: created 2023-10-09T05:21:16.789372+0000 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: modified 2023-10-09T05:21:26.247032+0000 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: tableserver 0 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: root 0 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: session_timeout 60 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: session_autoclose 300 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: max_file_size 1099511627776 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: required_client_features {} 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: last_failure 0 2023-10-09T05:25:32.743 INFO:journalctl@ceph.mon.a.smithi008.stdout: last_failure_osd_epoch 0 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: max_mds 1 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: in 0 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: up {0=24542} 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: failed 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: damaged 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: stopped 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: data_pools [4] 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: metadata_pool 3 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: inline_data disabled 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: balancer 2023-10-09T05:25:32.744 INFO:journalctl@ceph.mon.a.smithi008.stdout: bal_rank_mask -1 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout: standby_count_wanted 1 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout: [mds.foo.smithi008.zfejpl{0:24542} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.8:6834/560051770,v1:172.21.15.8:6835/560051770] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout: 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout: 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout: Standby daemons: 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout: 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout: [mds.foo.smithi191.walafj{-1:24556} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.191:6832/1976404494,v1:172.21.15.191:6833/1976404494] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: mon.a@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 ceph-mon[184418]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-10-09T05:25:32.745 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 bash[184353]: dbfd9e97488a61d7a86a687a3d4165e63ed21d6ca27ba60945fd121f6d2f2e4c 2023-10-09T05:25:32.746 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:25:32 smithi008 systemd[1]: Started Ceph mon.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:25:35.704 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[177229]: ignoring --setuser ceph since I am not root 2023-10-09T05:25:35.704 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[177229]: ignoring --setgroup ceph since I am not root 2023-10-09T05:25:35.704 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:35 smithi008 ceph-mgr[177273]: -- 172.21.15.8:0/899578970 <== mon.0 v2:172.21.15.8:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5631418631e0 con 0x56314183b000 2023-10-09T05:25:35.704 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[177229]: 2023-10-09T05:25:35.607+0000 7f0249135200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:25:35.704 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[177229]: 2023-10-09T05:25:35.703+0000 7f0249135200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:25:35.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: audit 2023-10-09T05:25:30.652382+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:25:35.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: audit 2023-10-09T05:25:30.653761+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:25:35.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: audit 2023-10-09T05:25:30.653996+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:25:35.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: cluster 2023-10-09T05:25:31.695704+0000 mgr.y (mgr.15870) 38 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:35.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: audit 2023-10-09T05:25:31.906116+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:35.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: audit 2023-10-09T05:25:31.942124+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.8:0/1754616984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:35.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: cluster 2023-10-09T05:25:33.696382+0000 mgr.y (mgr.15870) 39 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:35.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: cluster 2023-10-09T05:25:33.775284+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T05:25:35.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: audit 2023-10-09T05:25:34.352967+0000 mon.b (mon.1) 41 : audit [DBG] from='client.? 172.21.15.8:0/3661713319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:35.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: cluster 2023-10-09T05:25:35.379688+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:25:35.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: cluster 2023-10-09T05:25:35.384310+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: cluster 2023-10-09T05:25:35.384961+0000 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: cluster 2023-10-09T05:25:35.385017+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: cluster 2023-10-09T05:25:35.386125+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e30: y(active, since 33s), standbys: x 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: cluster 2023-10-09T05:25:35 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: .403492+0000 mon.a (mon.0) 7 : cluster [DBG] 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: mgrmap e31: y(active, since 33s), standbys: x 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: cluster 2023-10-09T05:25:35 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: .403543+0000 mon.a (mon.0) 8 : cluster [INF] 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: overall HEALTH_OK 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: audit 2023-10-09T05:25:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: 2023-10-09T05: 2023-10-09T05:25:35.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: 25:35.403649+0000 mon.a (mon.0) 2023-10-09T05:25:35.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[110782]: 9 : audit [INF] from='mgr.15870 ' entity='' 2023-10-09T05:25:35.738 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[152080]: ignoring --setuser ceph since I am not root 2023-10-09T05:25:35.738 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[152080]: ignoring --setgroup ceph since I am not root 2023-10-09T05:25:35.738 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:35 smithi191 ceph-mgr[152105]: -- 172.21.15.191:0/825972792 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556d35da31e0 con 0x556d35d7a800 2023-10-09T05:25:35.738 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[152080]: 2023-10-09T05:25:35.569+0000 7f6ce5ac8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:25:35.739 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[152080]: 2023-10-09T05:25:35.648+0000 7f6ce5ac8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:25:35.739 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[152080]: 2023-10-09T05:25:35.738+0000 7f6ce5ac8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:25:35.739 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: audit 2023-10-09T05:25:30.652382+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:25:35.739 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: audit 2023-10-09T05:25:30.653761+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:25:35.739 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: audit 2023-10-09T05:25:30.653996+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:25:35.739 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: cluster 2023-10-09T05:25:31.695704+0000 mgr.y (mgr.15870) 38 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:35.739 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: audit 2023-10-09T05:25:31.906116+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.15870 172.21.15.8:0/2287177756' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:35.739 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: audit 2023-10-09T05:25:35.739 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: 2023-10-09T05:25:31.942124+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.8:0/1754616984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: cluster 2023-10-09T05 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: :25:33.696382+0000 mgr.y (mgr.15870) 39 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: cluster 2023 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: -10-09T05:25:33.775284+0000 mon.a (mon. 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: 0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: audit 2023-10-09T05:25 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: :34.352967+0000 mon.b (mon.1) 41 : audit [DBG] from='client.? 172.21.15.8:0/3661713319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: cluster 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: 2023-10-09T05:25:35.379688+0000 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: cluster 2023- 2023-10-09T05:25:35.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: 10-09T05:25:35.384310+0000 mon.a (mon.0) 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: cluster 2023-10-09T05:25:35. 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: 384961+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: cluster 2023-10- 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: 09T05:25:35.385017+0000 mon.a (mon.0 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: ) 5 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: cluster 2023-10-09T05: 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: 25:35.386125+0000 mon.a (mon.0) 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: 6 : cluster [DBG] mgrmap e30: y(active, since 33s), standbys: x 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: cluster 2023-10-09T05 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: :25:35.403492+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e31: y(active, since 33s), standbys: x 2023-10-09T05:25:35.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: cluster 2023-10-09T05:25:35.403543+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-09T05:25:35.742 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[101299]: audit 2023-10-09T05:25:35.403649+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15870 ' entity='' 2023-10-09T05:25:35.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[177229]: 2023-10-09T05:25:35.814+0000 7f0249135200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:25:35.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:35 smithi008 conmon[177229]: 2023-10-09T05:25:35.926+0000 7f0249135200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:25:35.998 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[152080]: 2023-10-09T05:25:35.831+0000 7f6ce5ac8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:25:36.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:36 smithi008 conmon[177229]: 2023-10-09T05:25:36.134+0000 7f0249135200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:25:36.672 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:35 smithi191 conmon[152080]: 2023-10-09T05:25:35.997+0000 7f6ce5ac8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:25:36.672 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:36 smithi191 conmon[152080]: 2023-10-09T05:25:36.359+0000 7f6ce5ac8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:25:36.672 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:36 smithi191 conmon[152080]: 2023-10-09T05:25:36.491+0000 7f6ce5ac8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:25:36.672 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:36 smithi191 conmon[152080]: 2023-10-09T05:25:36.578+0000 7f6ce5ac8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:25:36.672 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:36 smithi191 conmon[152080]: 2023-10-09T05:25:36.671+0000 7f6ce5ac8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:25:36.765 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T05:25:36.886 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:36 smithi008 conmon[110782]: audit 2023-10-09T05:25:36.751519+0000 mon.a ( 2023-10-09T05:25:36.887 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:36 smithi008 conmon[110782]: mon.0) 10 : audit [DBG] from='client.? 172.21.15.8:0/2180942488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:36.887 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:36 smithi008 conmon[177229]: 2023-10-09T05:25:36.616+0000 7f0249135200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:25:36.887 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:36 smithi008 conmon[177229]: 2023-10-09T05:25:36.772+0000 7f0249135200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:25:36.887 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:36 smithi008 conmon[177229]: 2023-10-09T05:25:36.886+0000 7f0249135200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:25:37.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:36 smithi191 conmon[101299]: audit 2023-10-09T05:25:36.751519+0000 mon.a (mon. 2023-10-09T05:25:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:36 smithi191 conmon[101299]: 0) 10 : audit [DBG] from='client.? 172.21.15.8:0/2180942488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:25:37.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:37 smithi008 conmon[177229]: 2023-10-09T05:25:37.005+0000 7f0249135200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:25:37.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:37 smithi191 conmon[152080]: 2023-10-09T05:25:37.103+0000 7f6ce5ac8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:25:37.644 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:25:37.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:37 smithi008 conmon[177229]: 2023-10-09T05:25:37.525+0000 7f0249135200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:25:38.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:37 smithi191 conmon[152080]: 2023-10-09T05:25:37.766+0000 7f6ce5ac8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:25:38.049 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:37 smithi191 conmon[152080]: 2023-10-09T05:25:37.849+0000 7f6ce5ac8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:25:38.457 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:38 smithi008 conmon[177229]: 2023-10-09T05:25:38.348+0000 7f0249135200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:25:38.457 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:38 smithi008 conmon[177229]: 2023-10-09T05:25:38.457+0000 7f0249135200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:25:38.891 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:38 smithi191 conmon[152080]: 2023-10-09T05:25:38.622+0000 7f6ce5ac8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:25:38.891 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:38 smithi191 conmon[152080]: 2023-10-09T05:25:38.715+0000 7f6ce5ac8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:25:38.891 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:38 smithi191 conmon[151041]: ts=2023-10-09T05:25:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:25:39.549 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:38 smithi191 conmon[152080]: 2023-10-09T05:25:38.891+0000 7f6ce5ac8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:25:39.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:39 smithi191 conmon[152080]: 2023-10-09T05:25:39.174+0000 7f6ce5ac8200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:25:39.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:39 smithi191 conmon[152080]: 2023-10-09T05:25:39.402+0000 7f6ce5ac8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:25:39.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:39 smithi008 conmon[177229]: 2023-10-09T05:25:39.464+0000 7f0249135200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:25:39.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:39 smithi008 conmon[177229]: 2023-10-09T05:25:39.584+0000 7f0249135200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:25:39.822 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:39 smithi191 conmon[152080]: 2023-10-09T05:25:39.553+0000 7f6ce5ac8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:25:40.172 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:39 smithi008 conmon[177229]: 2023-10-09T05:25:39.814+0000 7f0249135200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:25:40.429 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:40 smithi008 conmon[177229]: 2023-10-09T05:25:40.172+0000 7f0249135200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:25:40.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:40 smithi008 conmon[177229]: 2023-10-09T05:25:40.429+0000 7f0249135200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:25:40.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:40 smithi008 conmon[177229]: 2023-10-09T05:25:40.615+0000 7f0249135200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:25:40.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:39 smithi191 conmon[152080]: 2023-10-09T05:25:39.821+0000 7f6ce5ac8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:25:40.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:40 smithi191 conmon[152080]: 2023-10-09T05:25:40.528+0000 7f6ce5ac8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:25:40.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:40 smithi191 conmon[152080]: 2023-10-09T05:25:40.661+0000 7f6ce5ac8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:25:41.202 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:40 smithi191 conmon[152080]: 2023-10-09T05:25:40.861+0000 7f6ce5ac8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:25:41.202 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:40 smithi191 conmon[152080]: 2023-10-09T05:25:40.977+0000 7f6ce5ac8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:25:41.216 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:40 smithi008 conmon[177229]: 2023-10-09T05:25:40.959+0000 7f0249135200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:25:41.549 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:41 smithi191 conmon[152080]: 2023-10-09T05:25:41.200+0000 7f6ce5ac8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:25:42.144 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: cluster 2023-10-09T05:25:42.085245+0000 mon.a (mon.0) 11 : cluster 2023-10-09T05:25:42.144 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: [DBG] Standby manager daemon x restarted 2023-10-09T05:25:42.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: cluster 2023-10-09T05:25:42.085315+0000 mon.a (mon.0) 12 : cluster 2023-10-09T05:25:42.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: [DBG] Standby manager daemon x started 2023-10-09T05:25:42.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:42.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: 09T05:25:42.087951+0000 mon.b (mon.1) 42 : 2023-10-09T05:25:42.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: audit [DBG] from='mgr.? 172.21.15.191:0/242053505' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:25:42.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: audit 2023-10-09T05:25:42.088746 2023-10-09T05:25:42.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: +0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.? 172.21.15.191:0/242053505' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:42.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: audit 2023-10-09T05:25:42.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: 2023-10-09T05:25:42.091097+0000 2023-10-09T05:25:42.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: mon.b (mon.1) 44 : audit [DBG] 2023-10-09T05:25:42.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: from='mgr.? 172.21.15.191:0/242053505' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:25:42.146 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: audit 2023-10-09T05:25:42. 2023-10-09T05:25:42.146 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: 091470+0000 mon.b (mon.1) 45 : audit 2023-10-09T05:25:42.146 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[110782]: [DBG] from='mgr.? 172.21.15.191:0/242053505' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:25:42.146 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:41 smithi008 conmon[177229]: 2023-10-09T05:25:41.801+0000 7f0249135200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:25:42.146 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:41 smithi008 conmon[177229]: 2023-10-09T05:25:41.963+0000 7f0249135200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:25:42.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[177229]: 2023-10-09T05:25:42.220+0000 7f0249135200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:25:42.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[177229]: 2023-10-09T05:25:42.361+0000 7f0249135200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:25:42.549 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[152080]: 2023-10-09T05:25:42.071+0000 7f6ce5ac8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:25:42.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[152080]: [09/Oct/2023:05:25:42] ENGINE Bus STARTING 2023-10-09T05:25:42.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[152080]: CherryPy Checker: 2023-10-09T05:25:42.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[152080]: The Application mounted at '' has an empty config. 2023-10-09T05:25:42.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[152080]: 2023-10-09T05:25:42.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[152080]: [09/Oct/2023:05:25:42] ENGINE Serving on http://:::9283 2023-10-09T05:25:42.550 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[152080]: [09/Oct/2023:05:25:42] ENGINE Bus STARTED 2023-10-09T05:25:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: cluster 2023-10-09T05:25:42.085245+0000 mon.a (mon.0 2023-10-09T05:25:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: ) 11 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: cluster 2023-10-09T05:25:42.085315+0000 mon.a (mon.0) 12 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: 25:42.087951+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.? 172.21.15.191:0/242053505' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: audit 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: 2023-10-09T05:25:42.088746+0000 mon.b ( 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: mon.1) 43 : audit [DBG] from='mgr.? 172.21.15.191:0/242053505' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: 05:25:42.091097+0000 mon.b (mon. 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: 1) 44 : audit [DBG] from='mgr.? 172.21.15.191:0/242053505' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: T05:25:42.091470+0000 mon.b (mon 2023-10-09T05:25:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:42 smithi191 conmon[101299]: .1) 45 : audit [DBG] from='mgr.? 172.21.15.191:0/242053505' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:25:42.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:42 smithi008 conmon[177229]: 2023-10-09T05:25:42.645+0000 7f0249135200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:25:43.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[110782]: cluster 2023-10-09T05:25:42.141308+0000 mon.a (mon.0) 13 : cluster [DBG] mgrmap e32: y(active, since 40s), standbys: x 2023-10-09T05:25:43.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:43 smithi191 conmon[101299]: cluster 2023-10-09T05:25:42.141308+0000 mon.a (mon.0) 13 : cluster [DBG] mgrmap e32: y(active, since 40s), standbys: x 2023-10-09T05:25:43.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[110782]: debug 2023-10-09T05:25:43.732+0000 7f089fa43700 -1 mon.c@2(peon).osd e84 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T05:25:43.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:43.714+0000 7f0249135200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:25:43.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: [09/Oct/2023:05:25:43] ENGINE Bus STARTING 2023-10-09T05:25:43.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:43.895+0000 7f01a6c2e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:43.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:43.895+0000 7f01a6c2e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:43.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:43.895+0000 7f01a6c2e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:43.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:43.895+0000 7f01a6c2e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:43.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:43.895+0000 7f01a6c2e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:43.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:43.895+0000 7f01a3c28700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:43.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:43.895+0000 7f01a3c28700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:43.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:43.895+0000 7f01a3c28700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:43.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:43.895+0000 7f01a3c28700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:43.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:43.895+0000 7f01a3c28700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:25:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:43 smithi191 conmon[101299]: debug 2023-10-09T05:25:43.732+0000 7f08d7f7b700 -1 mon.b@1(peon).osd e84 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T05:25:44.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: CherryPy Checker: 2023-10-09T05:25:44.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: The Application mounted at '' has an empty config. 2023-10-09T05:25:44.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: 2023-10-09T05:25:44.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: [09/Oct/2023:05:25:43] ENGINE Serving on http://:::9283 2023-10-09T05:25:44.465 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:43 smithi008 conmon[177229]: [09/Oct/2023:05:25:43] ENGINE Bus STARTED 2023-10-09T05:25:44.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: cluster 2023-10-09T05:25:43.728885+0000 mon.a (mon.0 2023-10-09T05:25:44.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: ) 14 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:25:44.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: cluster 2023-10-09T05:25:43.729230+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: cluster 2023-10-09T05:25:43.734687+0000 mon.a ( 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: mon.0) 16 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: cluster 2023 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: -10-09T05:25:43.734977+0000 mon.a (mon.0) 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 17 : cluster [DBG] mgrmap e33: y(active, starting, since 0.00584496s), standbys: x 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 2023-10-09T05:25:43. 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 742953+0000 mon.a (mon.0) 18 : audit 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: -10-09T05:25:43.743089 2023-10-09T05:25:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: +0000 mon.a (mon.0) 19 : audit [DBG] 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: -10-09T05:25:43.743286+0000 mon.a 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: (mon.0) 20 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: -09T05:25:43.743764+0000 mon.a ( 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: mon.0) 21 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi008.zfejpl"}]: dispatch 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25: 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 43.744460+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.walafj"}]: dispatch 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:43.745149+0000 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: mon.a (mon.0) 23 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:25:44.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:43.745388+0000 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: mon.a (mon.0) 24 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:43.745611+0000 mon.a 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: (mon.0) 25 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 2023-10-09T05:25:43.745885+0000 mon.a (mon.0) 26 : 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 09T05:25:43.746153+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:43.746421+0000 mon.a 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: (mon.0) 28 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:44.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 2023-10-09T05:25:43.746709+0000 mon.a (mon.0) 29 : 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 09T05:25:43.746961+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25: 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 43.747212+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:43.747479+0000 mon.a ( 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: mon.0) 32 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:43.747693 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: +0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:43. 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 747933+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:25:44.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:43. 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 748431+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: cluster 2023-10-09T05:25:43.768246 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: +0000 mon.a (mon.0) 36 : cluster [INF] Manager daemon y is now available 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:43. 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 794126+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:43. 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 796820+0000 mon.a (mon.0) 38 : 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10- 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 09T05:25:43.821831+0000 mon.a (mon.0) 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 39 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:44.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:44.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 2023-10-09T05:25:43.832851+0000 mon.a (mon. 2023-10-09T05:25:44.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 0) 40 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:25:44.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: audit 2023-10-09T05:25:43. 2023-10-09T05:25:44.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: 856752+0000 mon.a (mon.0) 41 : audit [INF] 2023-10-09T05:25:44.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[110782]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:25:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: cluster 2023-10-09T05:25:43.728885+0000 mon.a (mon. 2023-10-09T05:25:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: 0) 14 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:25:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: cluster 2023-10-09T05:25:43.729230+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-10-09T05:25:44.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: cluster 2023-10-09T05:25:43.734687+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:25:44.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: cluster 2023-10-09T05:25:43.734977+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e33: y(active, starting, since 0.00584496s), standbys: x 2023-10-09T05:25:44.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.742953+0000 mon.a (mon.0) 18 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:25:44.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.743089+0000 mon.a (mon.0) 19 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:25:44.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.743286+0000 mon.a (mon.0) 20 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:25:44.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.743764+0000 mon.a (mon.0) 21 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi008.zfejpl"}]: dispatch 2023-10-09T05:25:44.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.744460+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.walafj"}]: dispatch 2023-10-09T05:25:44.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.745149+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:25:44.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.745388+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:25:44.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.745611+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:25:44.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.745885+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:25:44.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.746153+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:25:44.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.746421+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:25:44.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.746709+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:25:44.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.746961+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:25:44.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:25:44.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: 25:43.747212+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:25:44.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:43.747479+0000 2023-10-09T05:25:44.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: mon.a (mon.0) 32 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:25:44.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:25:44.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: 10-09T05:25:43.747693+0000 2023-10-09T05:25:44.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: mon.a (mon.0) 33 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:25:44.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023 2023-10-09T05:25:44.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: -10-09T05:25:43.747933+0000 2023-10-09T05:25:44.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: mon.a (mon.0) 34 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:25:44.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: 2023-10-09T05:25:44.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25: 2023-10-09T05:25:44.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: 43.748431+0000 mon.a (mon.0) 35 : 2023-10-09T05:25:44.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:25:44.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: cluster 2023-10-09 2023-10-09T05:25:44.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: T05:25:43.768246+0000 mon.a (mon 2023-10-09T05:25:44.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: .0) 36 : cluster [INF] Manager daemon y is now available 2023-10-09T05:25:44.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:25:44.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: T05:25:43.794126+0000 mon.a (mon 2023-10-09T05:25:44.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: .0) 37 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:44.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023 2023-10-09T05:25:44.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: -10-09T05:25:43.796820+0000 2023-10-09T05:25:44.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: mon.a (mon.0) 38 : audit [DBG] 2023-10-09T05:25:44.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:44.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25:44.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: 2023-10-09T05:25:43.821831+0000 2023-10-09T05:25:44.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: mon.a (mon.0) 39 : audit [DBG] 2023-10-09T05:25:44.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:44.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T05:25 2023-10-09T05:25:44.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: :43.832851+0000 mon.a (mon.0) 40 2023-10-09T05:25:44.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:25:44.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:25:44.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: 05:25:43.856752+0000 mon.a (mon. 2023-10-09T05:25:44.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:44 smithi191 conmon[101299]: 0) 41 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:25:44.764 INFO:teuthology.orchestra.run.smithi008.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:25:44.764 INFO:teuthology.orchestra.run.smithi008.stdout:alertmanager.a smithi008 *:9093,9094 running (93s) - 5m 18.4M - 0.23.0 ba2b418f427c 172e511a984d 2023-10-09T05:25:44.764 INFO:teuthology.orchestra.run.smithi008.stdout:grafana.a smithi191 *:3000 running (57s) - 4m 48.1M - 8.3.5 dad864ee21e9 66b01c4fecc1 2023-10-09T05:25:44.764 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi008.zfejpl smithi008 running (4m) - 4m 15.7M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5d86a4e2d3b 2023-10-09T05:25:44.765 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi191.walafj smithi191 running (4m) - 4m 14.0M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 93130380f297 2023-10-09T05:25:44.765 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.x smithi191 *:8443,9283,8765 running (25s) - 9m 42.7M - 18.2.0-598-gb4fa663f 3060897fdf97 e651b468b4c8 2023-10-09T05:25:44.765 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.y smithi008 *:8443,9283,8765 running (52s) - 10m 475M - 18.2.0-598-gb4fa663f 3060897fdf97 b6adda8222c0 2023-10-09T05:25:44.765 INFO:teuthology.orchestra.run.smithi008.stdout:mon.a smithi008 running (10m) - 10m 114M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d155e1e1e35f 2023-10-09T05:25:44.765 INFO:teuthology.orchestra.run.smithi008.stdout:mon.b smithi191 running (9m) - 9m 88.9M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 2fa5874c9b2c 2023-10-09T05:25:44.765 INFO:teuthology.orchestra.run.smithi008.stdout:mon.c smithi008 running (9m) - 9m 86.4M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 bc3d5fa1338e 2023-10-09T05:25:44.765 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.a smithi008 *:9100 running (102s) - 5m 23.3M - 1.3.1 1dbe0e931976 0b0d19fe4991 2023-10-09T05:25:44.765 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.b smithi191 *:9100 running (78s) - 5m 10.6M - 1.3.1 1dbe0e931976 05f0cea760c5 2023-10-09T05:25:44.765 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0 smithi008 running (8m) - 8m 70.2M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 70d3660d55a1 2023-10-09T05:25:44.765 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 smithi008 running (8m) - 8m 85.7M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 312729dec19a 2023-10-09T05:25:44.766 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2 smithi008 running (7m) - 7m 64.1M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7c1912182283 2023-10-09T05:25:44.766 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 smithi008 running (7m) - 7m 60.5M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9c33044d2771 2023-10-09T05:25:44.766 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 smithi191 running (7m) - 7m 68.7M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 57a8d325e472 2023-10-09T05:25:44.766 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 smithi191 running (6m) - 6m 76.1M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 141ad4d0ecc5 2023-10-09T05:25:44.766 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6 smithi191 running (6m) - 6m 74.3M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0039e7e7a199 2023-10-09T05:25:44.766 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7 smithi191 running (6m) - 6m 92.7M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d71410a3a7bd 2023-10-09T05:25:44.766 INFO:teuthology.orchestra.run.smithi008.stdout:prometheus.a smithi191 *:9095 running (30s) - 5m 45.5M - 2.33.4 514e6a882f6e d24b4acc1156 2023-10-09T05:25:45.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:44 smithi008 conmon[177229]: 2023-10-09T05:25:44.747+0000 7f020285b700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T05:25:45.478 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T05:25:45.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:45 smithi008 conmon[110782]: cluster 2023-10-09T05:25:43.736567+0000 2023-10-09T05:25:45.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:45 smithi008 conmon[110782]: osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T05:25:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:45 smithi008 conmon[110782]: cluster 2023-10-09T05:25:43.736725+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T05:25:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:45 smithi008 conmon[110782]: cluster 2023-10-09T05:25:43.742504+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T05:25:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:45 smithi008 conmon[110782]: audit 2023-10-09T05:25:44.493002+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:45 smithi008 conmon[110782]: audit 2023-10-09T05:25:44.498590+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:45 smithi008 conmon[110782]: cluster 2023-10-09T05:25:44.740607+0000 mon.a (mon.0) 44 : cluster [DBG] mgrmap e34: y(active, since 1.01147s), standbys: x 2023-10-09T05:25:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:45 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:44.797194+0000 mgr.y (mgr.25913) 3 : cephadm [INF] [09/Oct/2023:05:25:44] ENGINE Bus STARTING 2023-10-09T05:25:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:45 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:44.899357+0000 mgr.y (mgr.25913) 4 : cephadm [INF] [09/Oct/2023:05:25:44] ENGINE Serving on http://172.21.15.8:8765 2023-10-09T05:25:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:45 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:45.016636+0000 mgr.y (mgr.25913) 5 : cephadm [INF] [09/Oct/2023:05:25:45] ENGINE Serving on https://172.21.15.8:7150 2023-10-09T05:25:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:45 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:45.016873+0000 mgr.y (mgr.25913) 6 : cephadm [INF] [09/Oct/2023:05:25:45] ENGINE Bus STARTED 2023-10-09T05:25:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: cluster 2023-10-09T05:25:43.736567+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T05:25:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: cluster 2023-10-09T05:25:43.736725+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T05:25:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: cluster 2023-10-09T05:25:43.742504+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T05:25:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: audit 2023-10-09T05:25:44.493002+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:25:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: T05:25:44.498590+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: cluster 2023-10-09T05:25:44.740607+0000 mon.a (mon.0) 44 : cluster [DBG] mgrmap e34: y(active, since 1.01147s), standbys: x 2023-10-09T05:25:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:44.797194+0000 mgr.y (mgr.25913) 3 : cephadm [INF] [09/Oct/2023:05:25:44] ENGINE Bus STARTING 2023-10-09T05:25:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:44.899357+0000 mgr.y (mgr.25913) 4 : cephadm [INF] [09/Oct/2023:05:25:44] ENGINE Serving on http://172.21.15.8:8765 2023-10-09T05:25:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:45.016636+0000 mgr.y (mgr.25913) 5 : cephadm [INF] [09/Oct/2023:05:25:45] ENGINE Serving on https://172.21.15.8:7150 2023-10-09T05:25:45.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:45. 2023-10-09T05:25:45.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: 016873+0000 mgr.y (mgr.25913) 2023-10-09T05:25:45.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:45 smithi191 conmon[101299]: 6 : cephadm [INF] [09/Oct/2023:05:25:45] ENGINE Bus STARTED 2023-10-09T05:25:46.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:46 smithi008 conmon[110782]: cluster 2023-10-09T05:25:45.746617+0000 mgr.y (mgr.25913) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:47.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:46 smithi191 conmon[101299]: cluster 2023-10-09T05:25:45. 2023-10-09T05:25:47.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:46 smithi191 conmon[101299]: 746617+0000 mgr.y (mgr.25913) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:48.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:47 smithi191 conmon[101299]: cluster 2023-10-09T05:25:46.746671+0000 mon.a (mon.0) 2023-10-09T05:25:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:47 smithi191 conmon[101299]: 45 : cluster [DBG] mgrmap e35: y(active, since 3s), standbys: x 2023-10-09T05:25:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:47 smithi191 conmon[101299]: audit 2023-10-09T05:25:47.601822+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:47 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:25:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:47 smithi191 conmon[101299]: T05:25:47.608012+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:48.082 INFO:teuthology.orchestra.run.smithi008.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:25:48.110 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:47 smithi008 conmon[110782]: cluster 2023-10-09T05:25:46.746671+0000 mon.a (mon.0 2023-10-09T05:25:48.111 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:47 smithi008 conmon[110782]: ) 45 : cluster [DBG] mgrmap e35: y(active, since 3s), standbys: x 2023-10-09T05:25:48.111 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:47 smithi008 conmon[110782]: audit 2023-10-09T05:25:47.601822+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:48.111 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:47 smithi008 conmon[110782]: audit 2023-10-09T05:25:47.608012+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:48.970 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T05:25:49.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:49 smithi008 conmon[110782]: cluster 2023-10-09T05:25:47.747006+0000 mgr.y (mgr.25913) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:49.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:49 smithi008 conmon[110782]: audit 2023-10-09T05:25:48.076798+0000 mgr.y (mgr.25913) 9 : audit [DBG] from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:25:49.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:49 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:48.077160+0000 mgr.y (mgr.25913) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:25:49.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:49 smithi008 conmon[110782]: audit 2023-10-09T05:25:48.081185+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:49.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:49 smithi008 conmon[110782]: audit 2023-10-09T05:25:48.293695+0000 mon.a (mon.0) 49 : audit 2023-10-09T05:25:49.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:49 smithi008 conmon[110782]: [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:49.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:49 smithi008 conmon[110782]: audit 2023-10-09T05:25:48.298211+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:49 smithi191 conmon[101299]: cluster 2023-10-09T05:25:47.747006+0000 2023-10-09T05:25:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:49 smithi191 conmon[101299]: mgr.y (mgr.25913) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:49 smithi191 conmon[101299]: audit 2023-10-09T05:25:48.076798+0000 mgr.y (mgr.25913) 9 : audit [DBG] from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:25:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:49 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:48.077160+0000 mgr.y (mgr.25913) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:25:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:49 smithi191 conmon[101299]: audit 2023-10-09T05:25:48.081185+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:49 smithi191 conmon[101299]: audit 2023-10-09T05:25:48.293695+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:49 smithi191 conmon[101299]: audit 2023-10-09T05:25:48.298211+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:50.920 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:50 smithi191 conmon[101299]: audit 2023-10-09T05:25:49.580183+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:50.920 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:50 smithi191 conmon[101299]: audit 2023-10-09T05:25:49.584610+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:50.921 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:50 smithi191 conmon[101299]: cluster 2023-10-09T05:25:49.747455+0000 mgr.y (mgr.25913) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:50.921 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:50 smithi191 conmon[101299]: audit 2023-10-09T05:25:50.473494+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:50.921 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:50 smithi191 conmon[101299]: audit 2023-10-09T05:25:50.478090+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:50.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[110782]: audit 2023-10-09T05:25:49.580183+0000 mon.a (mon. 2023-10-09T05:25:50.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[110782]: 0) 51 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:50.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[110782]: audit 2023-10-09T05:25:49.584610+0000 mon.a (mon.0) 52 : audit 2023-10-09T05:25:50.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[110782]: [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:50.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[110782]: cluster 2023-10-09T05:25:49 2023-10-09T05:25:50.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[110782]: .747455+0000 mgr.y (mgr.25913) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:50.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[110782]: audit 2023-10-09T05:25:50 2023-10-09T05:25:50.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[110782]: .473494+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:50.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[110782]: 2023-10-09T05:25:50.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[110782]: audit 2023-10-09 2023-10-09T05:25:50.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[110782]: T05:25:50.478090+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:50.936 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:25:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:25:50] "GET /metrics HTTP/1.1" 200 31214 "" "Prometheus/2.33.4" 2023-10-09T05:25:52.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: audit 2023-10-09T05:25:51.527531+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:25:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: 09T05:25:51.532712+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: audit 2023-10-09T05:25:51.534402+0000 mon.a (mon.0) 57 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: audit 2023-10-09T05:25:51.535393+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:51.536683+0000 mgr.y (mgr.25913) 12 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:25:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:51.536938+0000 mgr.y (mgr.25913) 13 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:25:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: cluster 2023-10-09T05:25:51.747884+0000 mgr.y (mgr.25913) 14 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:51.801875+0000 mgr.y (mgr.25913) 15 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:25:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:51.802085+0000 mgr.y (mgr.25913) 16 : cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:25:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:52.066377+0000 mgr.y (mgr.25913) 17 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:25:52.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:52.067017+0000 mgr.y (mgr.25913) 18 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:25:52.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:52.330422+0000 mgr.y (mgr.25913) 19 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:25:52.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:52 smithi191 conmon[101299]: cephadm 2023-10-09T05:25:52.331450+0000 mgr.y (mgr.25913) 20 : cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:25:52.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: audit 2023-10-09T05:25:51.527531+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: audit 2023-10-09T05:25:51.532712+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: audit 2023-10-09T05:25:51.534402+0000 mon.a (mon.0) 57 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: audit 2023-10-09T05:25:51.535393+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: cephadm 2023-10- 2023-10-09T05:25:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: 09T05:25:51.536683+0000 mgr.y (mgr.25913) 12 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:25:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:51.536938+0000 mgr.y (mgr.25913) 13 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:25:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: cluster 2023-10-09T05:25:51.747884+0000 mgr.y (mgr.25913) 14 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:51.801875+0000 mgr.y (mgr.25913) 15 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:25:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:51.802085+0000 mgr.y (mgr.25913) 16 : cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:25:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:52.066377+0000 mgr.y (mgr.25913) 17 : cephadm [INF] Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:25:52.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:52.067017+0000 mgr.y (mgr.25913) 18 : cephadm [INF] Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:25:52.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:52.330422+0000 mgr.y (mgr.25913) 19 : cephadm [INF] Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:25:52.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:52 smithi008 conmon[110782]: cephadm 2023-10-09T05:25:52.331450+0000 mgr.y (mgr.25913) 20 : cephadm [INF] Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:25:53.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit 2023-10-09T05:25:52.598755+0000 mon.a (mon.0) 59 : 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit 2023-10-09T05:25:52.601273+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit 2023-10-09T05:25:52.605062+0000 mon.a (mon.0 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: ) 61 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: 05:25:52.608118+0000 mon.a (mon.0) 62 : 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: 05:25:52.611885+0000 mon.a (mon.0) 63 : 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit 2023-10-09T 2023-10-09T05:25:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: 05:25:52.749291+0000 mon.a (mon.0) 64 2023-10-09T05:25:53.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:53.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit 2023-10-09T05:25:53.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: 2023-10-09T05:25:52.750730 2023-10-09T05:25:53.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: +0000 mon.a (mon.0) 65 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:53.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: 2023-10-09T05:25:53.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit 2023-10-09T05:25:53.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: 2023-10-09T05:25:52.751347+0000 mon.a (mon. 2023-10-09T05:25:53.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: 0) 66 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:53.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: audit 2023-10-09T05:25:53.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: 2023-10-09T05:25:52 2023-10-09T05:25:53.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: .756012+0000 mon.a (mon.0) 67 : audit [INF] 2023-10-09T05:25:53.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:53 smithi008 conmon[110782]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:54.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: audit 2023-10-09T05:25:52.598755+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:54.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: audit 2023-10-09T05:25:52.601273+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:54.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: audit 2023-10-09T05:25:52.605062+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:54.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: audit 2023-10-09T05:25:52 2023-10-09T05:25:54.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: .608118+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:54.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: audit 2023 2023-10-09T05:25:54.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: -10-09T05:25:52.611885+0000 mon.a (mon.0) 2023-10-09T05:25:54.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: 63 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:54.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: audit 2023-10-09T05:25:52.749291+0000 mon.a (mon 2023-10-09T05:25:54.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: .0) 64 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:25:54.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:25:54.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: 25:52.750730+0000 mon.a (mon.0) 65 2023-10-09T05:25:54.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:25:54.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: audit 2023-10-09T05 2023-10-09T05:25:54.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: :25:52.751347+0000 mon.a (mon.0) 66 2023-10-09T05:25:54.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:25:54.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:25:54.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: 25:52.756012+0000 mon.a (mon.0) 67 2023-10-09T05:25:54.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:53 smithi191 conmon[101299]: : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:25:54.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:54 smithi008 conmon[110782]: cluster 2023-10-09T05:25:53.748353+0000 mgr.y (mgr.25913) 21 : cluster [DBG] 2023-10-09T05:25:54.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:54 smithi008 conmon[110782]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:55.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:54 smithi191 conmon[101299]: cluster 2023-10-09T05:25:53.748353+0000 mgr.y (mgr.25913) 21 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:57.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:56 smithi008 conmon[110782]: cluster 2023-10-09T05:25:55.749104+0000 mgr.y (mgr.25913) 22 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:57.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:56 smithi191 conmon[101299]: cluster 2023-10-09T05:25:55.749104+0000 mgr.y (mgr.25913) 22 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:59.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:25:58 smithi191 conmon[151041]: ts=2023-10-09T05:25:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:25:59.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:59 smithi008 conmon[110782]: cluster 2023-10-09T05:25:57.749541+0000 mgr.y (mgr 2023-10-09T05:25:59.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:59 smithi008 conmon[110782]: .25913) 23 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:59.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:25:59 smithi008 conmon[110782]: audit 2023-10-09T05:25:58.822541+0000 mon.a (mon.0) 68 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:25:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:59 smithi191 conmon[101299]: cluster 2023-10-09T05:25:57.749541+0000 mgr.y (mgr.25913) 23 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:25:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:25:59 smithi191 conmon[101299]: audit 2023-10-09T05:25:58.822541+0000 mon.a (mon.0) 68 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:00.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:26:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:26:00] "GET /metrics HTTP/1.1" 200 33910 "" "Prometheus/2.33.4" 2023-10-09T05:26:01.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:01 smithi008 conmon[110782]: cluster 2023-10-09T05:25:59. 2023-10-09T05:26:01.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:01 smithi008 conmon[110782]: 750035+0000 mgr.y (mgr.25913) 24 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:01.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:01 smithi191 conmon[101299]: cluster 2023-10-09T05:25:59.750035+0000 mgr.y (mgr.25913) 24 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:03.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:03 smithi008 conmon[110782]: cluster 2023-10-09T05:26:01.750735+0000 mgr.y (mgr.25913) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:03.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:03 smithi191 conmon[101299]: cluster 2023-10-09T05:26:01.750735+0000 mgr.y (mgr.25913) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:05 smithi008 conmon[110782]: cluster 2023-10-09T05:26:03.751151+0000 mgr.y (mgr. 2023-10-09T05:26:05.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:05 smithi008 conmon[110782]: 25913) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:05.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:05 smithi191 conmon[101299]: cluster 2023-10-09T05:26:03.751151+0000 mgr.y (mgr.25913) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:07.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:07 smithi008 conmon[110782]: cluster 2023-10-09T05:26:05.751813+0000 mgr.y (mgr.25913) 2023-10-09T05:26:07.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:07 smithi008 conmon[110782]: 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:07.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:07 smithi191 conmon[101299]: cluster 2023-10-09T05:26:05.751813+0000 mgr.y (mgr.25913) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:07.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:07 smithi191 conmon[101299]: 2023-10-09T05:26:09.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:26:08 smithi191 conmon[151041]: ts=2023-10-09T05:26:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:26:09.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:09 smithi008 conmon[110782]: cluster 2023-10-09T05:26:07 2023-10-09T05:26:09.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:09 smithi008 conmon[110782]: .752338+0000 mgr.y (mgr.25913) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:09.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:09 smithi191 conmon[101299]: cluster 2023-10-09T05:26:07.752338+0000 mgr.y (mgr.25913) 28 : cluster [DBG] 2023-10-09T05:26:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:09 smithi191 conmon[101299]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:10.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:26:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:26:10] "GET /metrics HTTP/1.1" 200 33910 "" "Prometheus/2.33.4" 2023-10-09T05:26:11.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:11 smithi008 conmon[110782]: cluster 2023-10-09T05:26:09.753047+0000 mgr.y (mgr.25913) 2023-10-09T05:26:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:11 smithi008 conmon[110782]: 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:11 smithi191 conmon[101299]: cluster 2023-10-09T05:26:09.753047+0000 mgr.y (mgr.25913) 29 : cluster [DBG] 2023-10-09T05:26:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:11 smithi191 conmon[101299]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:13.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:13 smithi008 conmon[110782]: cluster 2023-10-09T05:26:11.753500+0000 mgr.y (mgr 2023-10-09T05:26:13.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:13 smithi008 conmon[110782]: .25913) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:13 smithi191 conmon[101299]: cluster 2023-10-09T05:26:11.753500+0000 mgr.y (mgr.25913) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:13 smithi191 conmon[101299]: 2023-10-09T05:26:14.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:14 smithi008 conmon[110782]: audit 2023-10-09T05:26:13.822865+0000 mon.a ( 2023-10-09T05:26:14.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:14 smithi008 conmon[110782]: mon.0) 69 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:14.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:14 smithi191 conmon[101299]: audit 2023-10-09T05:26:13.822865+0000 mon.a (mon.0) 69 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:14 smithi191 conmon[101299]: 2023-10-09T05:26:15.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:15 smithi008 conmon[110782]: cluster 2023-10-09T05:26:13.754107+0000 mgr.y (mgr 2023-10-09T05:26:15.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:15 smithi008 conmon[110782]: .25913) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:15.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:15 smithi191 conmon[101299]: cluster 2023-10-09T05:26:13.754107+0000 mgr.y (mgr.25913) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:17.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:17 smithi008 conmon[110782]: cluster 2023-10-09T05:26:15.754892+0000 mgr.y (mgr.25913) 32 : cluster 2023-10-09T05:26:17.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:17 smithi008 conmon[110782]: [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:17.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:17 smithi191 conmon[101299]: cluster 2023-10-09T05:26:15.754892+0000 mgr.y (mgr.25913) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:19.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:26:18 smithi191 conmon[151041]: ts=2023-10-09T05:26:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:26:19.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:19 smithi008 conmon[110782]: cluster 2023-10-09T05:26:17.755545+0000 mgr.y (mgr.25913) 33 : 2023-10-09T05:26:19.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:19 smithi008 conmon[110782]: cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:19.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:19 smithi191 conmon[101299]: cluster 2023-10-09T05:26:17.755545+0000 mgr.y (mgr.25913) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:20.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:26:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:26:20] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.33.4" 2023-10-09T05:26:21.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:21 smithi008 conmon[110782]: cluster 2023-10-09T05:26:19.756058+0000 mgr.y (mgr. 2023-10-09T05:26:21.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:21 smithi008 conmon[110782]: 25913) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:21 smithi191 conmon[101299]: cluster 2023-10-09T05:26:19.756058+0000 mgr.y (mgr.25913) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:23.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:23 smithi008 conmon[110782]: cluster 2023-10-09T05:26:21.756601+0000 mgr.y (mgr.25913) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:23 smithi191 conmon[101299]: cluster 2023-10-09T05:26:21.756601+0000 mgr.y (mgr.25913) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:25.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:25 smithi008 conmon[110782]: cluster 2023-10-09T05:26:23.757129+0000 mgr.y (mgr.25913 2023-10-09T05:26:25.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:25 smithi008 conmon[110782]: ) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:25.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:25 smithi191 conmon[101299]: cluster 2023-10-09T05:26:23.757129+0000 mgr.y (mgr.25913) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:27.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:27 smithi008 conmon[110782]: cluster 2023-10-09T05:26:25.757996+0000 mgr.y (mgr 2023-10-09T05:26:27.833 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:27 smithi008 conmon[110782]: .25913) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:27.833 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:27 smithi191 conmon[101299]: cluster 2023-10-09T05:26:25.757996+0000 mgr.y (mgr.25913) 37 : cluster 2023-10-09T05:26:27.833 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:27 smithi191 conmon[101299]: [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:29.049 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:26:28 smithi191 conmon[151041]: ts=2023-10-09T05:26:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:26:29.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:29 smithi008 conmon[110782]: cluster 2023-10-09T05:26:27.758466+0000 mgr.y (mgr. 2023-10-09T05:26:29.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:29 smithi008 conmon[110782]: 25913) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:29 smithi008 conmon[110782]: audit 2023-10-09T05:26:28.823044+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:29 smithi191 conmon[101299]: cluster 2023-10-09T05:26:27.758466+0000 mgr.y (mgr.25913) 38 : 2023-10-09T05:26:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:29 smithi191 conmon[101299]: cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:29 smithi191 conmon[101299]: audit 2023-10-09T05:26:28.823044+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:30.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:26:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:26:30] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.33.4" 2023-10-09T05:26:31.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:31 smithi008 conmon[110782]: cluster 2023-10-09T05:26:29.759030+0000 mgr.y (mgr. 2023-10-09T05:26:31.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:31 smithi008 conmon[110782]: 25913) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:31 smithi191 conmon[101299]: cluster 2023-10-09T05:26:29.759030+0000 mgr.y (mgr.25913) 39 : cluster [DBG] 2023-10-09T05:26:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:31 smithi191 conmon[101299]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:33.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:33 smithi008 conmon[110782]: cluster 2023-10-09T05:26:31.759520+0000 mgr.y (mgr.25913) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:33 smithi191 conmon[101299]: cluster 2023-10-09T05:26:31.759520+0000 mgr.y (mgr.25913) 40 : cluster [DBG] 2023-10-09T05:26:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:33 smithi191 conmon[101299]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:35.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:35 smithi008 conmon[110782]: cluster 2023-10-09T05:26:33.759887+0000 mgr.y (mgr 2023-10-09T05:26:35.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:35 smithi008 conmon[110782]: .25913) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:35.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:35 smithi191 conmon[101299]: cluster 2023-10-09T05:26:33.759887+0000 mgr.y (mgr.25913) 41 : 2023-10-09T05:26:35.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:35 smithi191 conmon[101299]: cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:37.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:37 smithi008 conmon[110782]: cluster 2023-10-09T05:26:35.760503+0000 mgr.y (mgr.25913) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:37.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:37 smithi191 conmon[101299]: cluster 2023-10-09T05:26:35.760503+0000 mgr.y (mgr.25913) 42 : cluster [DBG] 2023-10-09T05:26:37.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:37 smithi191 conmon[101299]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:39.049 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:26:38 smithi191 conmon[151041]: ts=2023-10-09T05:26:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:26:39.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:39 smithi008 conmon[110782]: cluster 2023-10-09T05:26:37.760959+0000 mgr.y (mgr 2023-10-09T05:26:39.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:39 smithi008 conmon[110782]: .25913) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:39.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:39 smithi191 conmon[101299]: cluster 2023-10-09T05:26:37.760959+0000 mgr.y (mgr.25913) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:40.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:26:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:26:40] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.33.4" 2023-10-09T05:26:41.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:41 smithi008 conmon[110782]: cluster 2023-10-09T05:26:39.761481+0000 mgr.y (mgr. 2023-10-09T05:26:41.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:41 smithi008 conmon[110782]: 25913) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:41 smithi191 conmon[101299]: cluster 2023-10-09T05:26:39.761481+0000 mgr.y (mgr.25913) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:43.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:43 smithi008 conmon[110782]: cluster 2023-10-09T05:26:41.761842+0000 mgr.y (mgr.25913) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:43.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:43 smithi191 conmon[101299]: cluster 2023-10-09T05:26:41.761842+0000 mgr.y (mgr.25913) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:44.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:44 smithi008 conmon[110782]: audit 2023-10-09T05:26:43.826344+0000 mon.a (mon.0) 2023-10-09T05:26:44.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:44 smithi008 conmon[110782]: 71 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:44.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:44 smithi191 conmon[101299]: audit 2023-10-09T05:26:43.826344+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:45.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:45 smithi008 conmon[110782]: cluster 2023-10-09T05:26:43.762415+0000 mgr.y (mgr.25913) 46 : cluster 2023-10-09T05:26:45.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:45 smithi008 conmon[110782]: [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:45.809 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:45 smithi191 conmon[101299]: cluster 2023-10-09T05:26:43.762415+0000 mgr.y (mgr.25913) 46 : cluster [DBG] 2023-10-09T05:26:45.809 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:45 smithi191 conmon[101299]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:47.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:47 smithi008 conmon[110782]: cluster 2023-10-09T05:26:45.763085+0000 mgr.y (mgr.25913) 47 : cluster [DBG] 2023-10-09T05:26:47.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:47 smithi008 conmon[110782]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:47 smithi191 conmon[101299]: cluster 2023-10-09T05:26:45.763085+0000 mgr.y (mgr.25913) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:49.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:26:48 smithi191 conmon[151041]: ts=2023-10-09T05:26:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:26:49.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:49 smithi008 conmon[110782]: cluster 2023-10-09T05:26:47.763574+0000 mgr.y (mgr.25913) 2023-10-09T05:26:49.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:49 smithi008 conmon[110782]: 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:49.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:49 smithi191 conmon[101299]: cluster 2023-10-09T05:26:47.763574+0000 mgr.y (mgr.25913) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:50.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:26:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:26:50] "GET /metrics HTTP/1.1" 200 33901 "" "Prometheus/2.33.4" 2023-10-09T05:26:51.677 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T05:26:51.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:51 smithi008 conmon[110782]: cluster 2023-10-09T05:26:49.764295+0000 mgr.y (mgr.25913) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:51 smithi191 conmon[101299]: cluster 2023-10-09T05:26:49.764295+0000 mgr.y (mgr.25913) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:53 smithi191 conmon[101299]: cluster 2023-10-09T05:26:51.764673+0000 mgr.y (mgr. 2023-10-09T05:26:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:53 smithi191 conmon[101299]: 25913) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:53 smithi191 conmon[101299]: audit 2023-10-09T05:26:52.861442+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:26:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:53 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:26:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:53 smithi191 conmon[101299]: 26:53.182037+0000 mon.a (mon.0 2023-10-09T05:26:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:53 smithi191 conmon[101299]: ) 73 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:53 smithi191 conmon[101299]: audit 2023-10-09T05:26:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:53 smithi191 conmon[101299]: 2023-10-09T05:26 2023-10-09T05:26:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:53 smithi191 conmon[101299]: :53.187943+0000 mon.a (mon. 2023-10-09T05:26:53.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:53 smithi191 conmon[101299]: 0) 74 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:53.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:53 smithi008 conmon[110782]: cluster 2023-10-09T05:26:51.764673+0000 mgr.y (mgr.25913) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:53.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:53 smithi008 conmon[110782]: audit 2023-10-09T05:26:52.861442+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:26:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:53 smithi008 conmon[110782]: audit 2023-10-09T05:26:53.182037+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:53 smithi008 conmon[110782]: audit 2023-10-09T05:26:53.187943+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:54.044 INFO:teuthology.orchestra.run.smithi008.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:26:54.873 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T05:26:55.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:55 smithi008 conmon[110782]: cluster 2023-10-09T05:26:53.765018+0000 mgr.y (mgr. 2023-10-09T05:26:55.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:55 smithi008 conmon[110782]: 25913) 51 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:55 smithi008 conmon[110782]: audit 2023-10-09T05:26:54.038978+0000 mgr.y (mgr.25913) 52 : audit [DBG] from='client.25961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:26:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:55 smithi008 conmon[110782]: cephadm 2023-10-09T05:26:54.039382+0000 mgr.y (mgr.25913) 53 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:26:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:55 smithi008 conmon[110782]: audit 2023-10-09T05:26:54.043867+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:55.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:55 smithi191 conmon[101299]: cluster 2023-10-09T05:26:53.765018+0000 mgr.y (mgr.25913) 51 : cluster 2023-10-09T05:26:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:55 smithi191 conmon[101299]: [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:55 smithi191 conmon[101299]: audit 2023-10-09T05:26:54.038978+0000 mgr.y (mgr.25913) 52 : audit [DBG] from='client.25961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:26:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:55 smithi191 conmon[101299]: cephadm 2023-10-09T05:26:54.039382+0000 mgr.y (mgr.25913) 53 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:26:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:55 smithi191 conmon[101299]: audit 2023-10-09T05:26:54.043867+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:56.774 INFO:teuthology.orchestra.run.smithi008.stdout:wait for majority of mons to upgrade 2023-10-09T05:26:57.671 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T05:26:57.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:57 smithi008 conmon[110782]: cluster 2023-10-09T05:26:55.765609+0000 mgr.y (mgr.25913) 54 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:57.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:57 smithi008 conmon[110782]: audit 2023-10-09T05:26:56.462911+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:57.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:57 smithi008 conmon[110782]: audit 2023-10-09T05:26:56.468115+0000 mon.a (mon.0) 77 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:57.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:57 smithi191 conmon[101299]: cluster 2023-10-09T05:26:55.765609+0000 mgr.y (mgr.25913) 54 : 2023-10-09T05:26:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:57 smithi191 conmon[101299]: cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:57 smithi191 conmon[101299]: audit 2023-10-09T05:26:56.462911+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:57 smithi191 conmon[101299]: audit 2023-10-09T05:26:56.468115+0000 mon.a (mon.0) 77 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:59.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:26:58 smithi191 conmon[151041]: ts=2023-10-09T05:26:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:26:59.385 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: cluster 2023-10-09T05:26:57.765954+0000 mgr.y (mgr 2023-10-09T05:26:59.386 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: .25913) 55 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:59.386 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: audit 2023-10-09T05:26:58.297484+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:59.386 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: audit 2023-10-09T05:26:58.303832+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:59.386 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: audit 2023-10-09T05:26:58.772914+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:26:59.386 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: audit 2023-10-09T05:26:58.773678+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:26:59.386 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: audit 2023-10-09T05:26:58.778813+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:59.386 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: audit 2023-10-09T05:26:58.826489+0000 mon.a (mon.0) 83 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:59.386 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: audit 2023-10-09T05:26:58.917429+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:26:59.386 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: audit 2023-10-09T05:26:58.919850+0000 mon.a (mon.0) 85 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:26:59.386 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: audit 2023-10-09T05:26:58.921008+0000 mon.a (mon.0) 86 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:26:59.386 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:26:59 smithi008 conmon[110782]: audit 2023-10-09T05:26:58.921621+0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T05:26:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:59 smithi191 conmon[101299]: cluster 2023-10-09T05:26:57.765954+0000 mgr.y (mgr.25913) 55 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:26:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:59 smithi191 conmon[101299]: audit 2023-10-09T05:26:58.297484+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:59 smithi191 conmon[101299]: audit 2023-10-09T05:26:58.303832+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:59 smithi191 conmon[101299]: audit 2023-10-09T05:26:58.772914+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:26:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:59 smithi191 conmon[101299]: audit 2023-10-09T05:26:58.773678+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:26:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:59 smithi191 conmon[101299]: audit 2023-10-09T05:26:58.778813+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:26:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:59 smithi191 conmon[101299]: audit 2023-10-09T05:26:58.826489+0000 mon.a (mon.0) 83 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:26:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:59 smithi191 conmon[101299]: audit 2023-10-09T05:26:58.917429+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:26:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:59 smithi191 conmon[101299]: audit 2023-10-09T05:26:58.919850+0000 mon.a (mon.0) 85 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:26:59.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:59 smithi191 conmon[101299]: audit 2023-10-09T05:26:58.921008+0000 mon.a (mon.0) 86 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:26:59.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:26:59 smithi191 conmon[101299]: audit 2023-10-09T05:26:58.921621+0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T05:27:00.620 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:27:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:27:00] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.33.4" 2023-10-09T05:27:00.621 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:00 smithi008 conmon[110782]: cephadm 2023-10-09T05:26:58.922041+0000 mgr.y (mgr.25913) 56 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T05:27:00.621 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:00 smithi008 conmon[110782]: audit 2023-10-09T05:26:59.907812+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.8:0/3809502107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:27:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:00 smithi191 conmon[101299]: cephadm 2023-10-09T05:26:58.922041+0000 mgr.y (mgr.25913) 56 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T05:27:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:00 smithi191 conmon[101299]: audit 2023-10-09T05:26:59.907812+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.8:0/3809502107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:27:01.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:01 smithi008 conmon[110782]: cluster 2023-10-09T05:26:59.766426+0000 mgr.y (mgr 2023-10-09T05:27:01.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:01 smithi008 conmon[110782]: .25913) 57 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:01.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:01 smithi008 conmon[110782]: audit 2023-10-09T05:27:00.693620+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:01 smithi008 conmon[110782]: audit 2023-10-09T05:27:00.694520+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:27:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:01 smithi008 conmon[110782]: audit 2023-10-09T05:27:00.695200+0000 mon.a (mon.0) 90 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:27:01.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:01 smithi008 conmon[110782]: audit 2023-10-09T05:27:00.695877+0000 mon.a (mon.0) 91 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:27:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:01 smithi191 conmon[101299]: cluster 2023-10-09T05:26:59.766426+0000 mgr.y (mgr.25913) 57 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:01 smithi191 conmon[101299]: audit 2023-10-09T05:27:00.693620+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:01 smithi191 conmon[101299]: audit 2023-10-09T05:27:00.694520+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:27:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:01 smithi191 conmon[101299]: audit 2023-10-09T05:27:00.695200+0000 mon.a (mon.0 2023-10-09T05:27:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:01 smithi191 conmon[101299]: ) 90 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:27:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:01 smithi191 conmon[101299]: audit 2023-10-09T05:27:00.695877+0000 mon.a (mon.0) 91 2023-10-09T05:27:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:01 smithi191 conmon[101299]: : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:27:02.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:02 smithi008 systemd[1]: Stopping Ceph mon.c for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:27:02.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:02 smithi008 conmon[110782]: cephadm 2023-10-09T05:27:00.688992 2023-10-09T05:27:02.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:02 smithi008 conmon[110782]: +0000 mgr.y (mgr.25913) 58 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T05:27:02.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:02 smithi008 conmon[110782]: cephadm 2023-10-09T05:27:00.696759+0000 mgr.y (mgr.25913) 59 : cephadm [INF] Deploying daemon mon.c on smithi008 2023-10-09T05:27:02.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:02 smithi191 conmon[101299]: cephadm 2023-10-09T05:27:00.688992+0000 mgr.y (mgr.25913) 58 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T05:27:02.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:02 smithi191 conmon[101299]: cephadm 2023-10-09T05:27:00.696759+0000 mgr.y (mgr.25913) 59 : cephadm [INF] Deploying daemon mon.c on smithi008 2023-10-09T05:27:02.849 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:02 smithi008 conmon[110782]: debug 2023-10-09T05:27:02.502+0000 7f08a6250700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T05:27:02.849 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:02 smithi008 conmon[110782]: debug 2023-10-09T05:27:02.503+0000 7f08a6250700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T05:27:03.600 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:03 smithi008 bash[190809]: bc3d5fa1338eeafeca9ac64bec49af7a55820b4d1016d95d0b8be0b43c45971e 2023-10-09T05:27:03.600 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:03 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T05:27:03.601 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:03 smithi008 systemd[1]: Stopped Ceph mon.c for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:27:03.601 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:03 smithi008 systemd[1]: Starting Ceph mon.c for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:27:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:27:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T05:27:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: pidfile_write: ignore empty --pid-file 2023-10-09T05:27:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: load: jerasure load: lrc 2023-10-09T05:27:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: RocksDB version: 7.9.2 2023-10-09T05:27:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Git sha 0 2023-10-09T05:27:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T05:27:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: DB SUMMARY 2023-10-09T05:27:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: DB Session ID: SN82136UAL8E2QGUYPPW 2023-10-09T05:27:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:27:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:27:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: MANIFEST file: MANIFEST-000005 size: 891 Bytes 2023-10-09T05:27:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000020.sst 2023-10-09T05:27:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000018.log size: 4671796 ; 2023-10-09T05:27:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:27:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:27:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:27:04.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T05:27:04.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:27:04.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T05:27:04.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.env: 0x563570668720 2023-10-09T05:27:04.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T05:27:04.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.info_log: 0x56357393b420 2023-10-09T05:27:04.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:27:04.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.statistics: (nil) 2023-10-09T05:27:04.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.use_fsync: 0 2023-10-09T05:27:04.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:27:04.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.db_log_dir: 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.wal_dir: 2023-10-09T05:27:04.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.write_buffer_manager: 0x563572bca5a0 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:27:04.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.unordered_write: 0 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.row_cache: None 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.wal_filter: None 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:27:04.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.wal_compression: 0 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:27:04.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:27:04.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:27:04.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:27:04.473 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T05:27:04.473 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:27:04.473 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:27:04.473 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:27:04.474 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:27:04.474 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:27:04.474 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:27:04.474 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:27:04.474 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:27:04.474 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:27:04.474 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:27:04.474 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:27:04.474 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_open_files: -1 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Compression algorithms supported: 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: kZSTD supported: 0 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:27:04.475 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:27:04.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:27:04.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:27:04.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T05:27:04.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-10-09T05:27:04.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T05:27:04.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:27:04.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.merge_operator: 2023-10-09T05:27:04.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_filter: None 2023-10-09T05:27:04.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:27:04.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:27:04.477 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:27:04.477 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:27:04.477 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56357393b520) 2023-10-09T05:27:04.477 INFO:journalctl@ceph.mon.c.smithi008.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:27:04.477 INFO:journalctl@ceph.mon.c.smithi008.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:27:04.477 INFO:journalctl@ceph.mon.c.smithi008.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:27:04.477 INFO:journalctl@ceph.mon.c.smithi008.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:27:04.477 INFO:journalctl@ceph.mon.c.smithi008.stdout: index_type: 0 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: data_block_index_type: 0 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: index_shortening: 1 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: checksum: 4 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: no_block_cache: 0 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: block_cache: 0x563572c4d1f0 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: block_cache_options: 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: capacity : 536870912 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: num_shard_bits : 4 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: strict_capacity_limit : 0 2023-10-09T05:27:04.478 INFO:journalctl@ceph.mon.c.smithi008.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:27:04.479 INFO:journalctl@ceph.mon.c.smithi008.stdout: block_cache_compressed: (nil) 2023-10-09T05:27:04.479 INFO:journalctl@ceph.mon.c.smithi008.stdout: persistent_cache: (nil) 2023-10-09T05:27:04.479 INFO:journalctl@ceph.mon.c.smithi008.stdout: block_size: 4096 2023-10-09T05:27:04.479 INFO:journalctl@ceph.mon.c.smithi008.stdout: block_size_deviation: 10 2023-10-09T05:27:04.479 INFO:journalctl@ceph.mon.c.smithi008.stdout: block_restart_interval: 16 2023-10-09T05:27:04.479 INFO:journalctl@ceph.mon.c.smithi008.stdout: index_block_restart_interval: 1 2023-10-09T05:27:04.479 INFO:journalctl@ceph.mon.c.smithi008.stdout: metadata_block_size: 4096 2023-10-09T05:27:04.479 INFO:journalctl@ceph.mon.c.smithi008.stdout: partition_filters: 0 2023-10-09T05:27:04.479 INFO:journalctl@ceph.mon.c.smithi008.stdout: use_delta_encoding: 1 2023-10-09T05:27:04.480 INFO:journalctl@ceph.mon.c.smithi008.stdout: filter_policy: bloomfilter 2023-10-09T05:27:04.480 INFO:journalctl@ceph.mon.c.smithi008.stdout: whole_key_filtering: 1 2023-10-09T05:27:04.480 INFO:journalctl@ceph.mon.c.smithi008.stdout: verify_compression: 0 2023-10-09T05:27:04.480 INFO:journalctl@ceph.mon.c.smithi008.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:27:04.480 INFO:journalctl@ceph.mon.c.smithi008.stdout: format_version: 5 2023-10-09T05:27:04.480 INFO:journalctl@ceph.mon.c.smithi008.stdout: enable_index_compression: 1 2023-10-09T05:27:04.480 INFO:journalctl@ceph.mon.c.smithi008.stdout: block_align: 0 2023-10-09T05:27:04.480 INFO:journalctl@ceph.mon.c.smithi008.stdout: max_auto_readahead_size: 262144 2023-10-09T05:27:04.481 INFO:journalctl@ceph.mon.c.smithi008.stdout: prepopulate_block_cache: 0 2023-10-09T05:27:04.481 INFO:journalctl@ceph.mon.c.smithi008.stdout: initial_auto_readahead_size: 8192 2023-10-09T05:27:04.481 INFO:journalctl@ceph.mon.c.smithi008.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T05:27:04.481 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:27:04.481 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:27:04.481 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compression: NoCompression 2023-10-09T05:27:04.481 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:27:04.481 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:27:04.481 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:27:04.481 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.num_levels: 7 2023-10-09T05:27:04.482 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:27:04.482 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:27:04.482 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:27:04.482 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:27:04.482 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:27:04.482 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:27:04.482 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:27:04.482 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:27:04.482 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:27:04.482 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:27:04.482 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:27:04.483 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:27:04.483 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:27:04.483 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:27:04.483 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:27:04.483 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:27:04.483 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:27:04.483 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:27:04.483 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:27:04.483 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:27:04.483 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:27:04.483 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:27:04.484 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:27:04.484 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:27:04.484 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:27:04.484 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:27:04.484 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:27:04.484 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:27:04.484 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:27:04.484 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:27:04.484 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:27:04.484 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:27:04.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:27:04.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:27:04.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:27:04.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:27:04.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:27:04.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:27:04.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T05:27:04.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T05:27:04.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:27:04.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:27:04.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:27:04.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:27:04.487 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:27:04.487 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:27:04.487 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:27:04.487 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:27:04.487 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:27:04.487 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:27:04.487 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:27:04.487 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:27:04.487 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:27:04.487 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:27:04.487 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.ttl: 2592000 2023-10-09T05:27:04.488 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:27:04.488 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T05:27:04.488 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T05:27:04.488 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:27:04.488 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:27:04.488 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:27:04.488 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:27:04.488 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:27:04.489 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:27:04.489 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T05:27:04.489 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T05:27:04.489 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T05:27:04.489 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T05:27:04.489 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 20.sst 2023-10-09T05:27:04.489 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T05:27:04.489 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 22, last_sequence is 12064, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:27:04.489 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 18 2023-10-09T05:27:04.489 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 90d296db-f4a5-45d0-8ffc-d4f41675bf06 2023-10-09T05:27:04.489 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829224141480, "job": 1, "event": "recovery_started", "wal_files": [18]} 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #18 mode 2 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829224151690, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 23, "file_size": 3055960, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12069, "largest_seqno": 13017, "table_properties": {"data_size": 3050191, "index_size": 4046, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 5301, "raw_average_key_size": 19, "raw_value_size": 3042145, "raw_average_value_size": 11267, "num_data_blocks": 178, "num_entries": 270, "num_filter_entries": 270, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696829224, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "90d296db-f4a5-45d0-8ffc-d4f41675bf06", "db_session_id": "SN82136UAL8E2QGUYPPW", "orig_file_number": 23, "seqno_to_time_mapping": "N/A"}} 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829224151779, "job": 1, "event": "recovery_finished"} 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/version_set.cc:5047] Creating manifest 25 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000018.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563572ce8000 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: DB pointer 0x563572cd6000 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout: ** DB Stats ** 2023-10-09T05:27:04.490 INFO:journalctl@ceph.mon.c.smithi008.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: ** Compaction Stats [default] ** 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: L0 1/0 2.91 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 643.5 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:27:04.491 INFO:journalctl@ceph.mon.c.smithi008.stdout: L6 1/0 24.72 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: Sum 2/0 27.64 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 643.5 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 643.5 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: ** Compaction Stats [default] ** 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 643.5 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:27:04.492 INFO:journalctl@ceph.mon.c.smithi008.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-10-09T05:27:04.493 INFO:journalctl@ceph.mon.c.smithi008.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:27:04.493 INFO:journalctl@ceph.mon.c.smithi008.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:27:04.493 INFO:journalctl@ceph.mon.c.smithi008.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:27:04.493 INFO:journalctl@ceph.mon.c.smithi008.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:27:04.493 INFO:journalctl@ceph.mon.c.smithi008.stdout: Cumulative compaction: 0.00 GB write, 227.49 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:27:04.493 INFO:journalctl@ceph.mon.c.smithi008.stdout: Interval compaction: 0.00 GB write, 227.49 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:27:04.493 INFO:journalctl@ceph.mon.c.smithi008.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:27:04.493 INFO:journalctl@ceph.mon.c.smithi008.stdout: Block cache BinnedLRUCache@0x563572c4d1f0#7 capacity: 512.00 MB usage: 4.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2023-10-09T05:27:04.493 INFO:journalctl@ceph.mon.c.smithi008.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.78 KB,0.000149012%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-10-09T05:27:04.493 INFO:journalctl@ceph.mon.c.smithi008.stdout: 2023-10-09T05:27:04.493 INFO:journalctl@ceph.mon.c.smithi008.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:27:04.494 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: starting mon.c rank 2 at public addrs [v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0] at bind addrs [v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:27:04.494 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: mon.c@-1(???) e3 preinit fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:27:04.494 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: mon.c@-1(???).mds e6 new map 2023-10-09T05:27:04.494 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: mon.c@-1(???).mds e6 print_map 2023-10-09T05:27:04.494 INFO:journalctl@ceph.mon.c.smithi008.stdout: e6 2023-10-09T05:27:04.494 INFO:journalctl@ceph.mon.c.smithi008.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:27:04.494 INFO:journalctl@ceph.mon.c.smithi008.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:27:04.494 INFO:journalctl@ceph.mon.c.smithi008.stdout: legacy client fscid: 1 2023-10-09T05:27:04.494 INFO:journalctl@ceph.mon.c.smithi008.stdout: 2023-10-09T05:27:04.494 INFO:journalctl@ceph.mon.c.smithi008.stdout: Filesystem 'foo' (1) 2023-10-09T05:27:04.494 INFO:journalctl@ceph.mon.c.smithi008.stdout: fs_name foo 2023-10-09T05:27:04.495 INFO:journalctl@ceph.mon.c.smithi008.stdout: epoch 5 2023-10-09T05:27:04.495 INFO:journalctl@ceph.mon.c.smithi008.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T05:27:04.495 INFO:journalctl@ceph.mon.c.smithi008.stdout: created 2023-10-09T05:21:16.789372+0000 2023-10-09T05:27:04.495 INFO:journalctl@ceph.mon.c.smithi008.stdout: modified 2023-10-09T05:21:26.247032+0000 2023-10-09T05:27:04.495 INFO:journalctl@ceph.mon.c.smithi008.stdout: tableserver 0 2023-10-09T05:27:04.495 INFO:journalctl@ceph.mon.c.smithi008.stdout: root 0 2023-10-09T05:27:04.495 INFO:journalctl@ceph.mon.c.smithi008.stdout: session_timeout 60 2023-10-09T05:27:04.495 INFO:journalctl@ceph.mon.c.smithi008.stdout: session_autoclose 300 2023-10-09T05:27:04.495 INFO:journalctl@ceph.mon.c.smithi008.stdout: max_file_size 1099511627776 2023-10-09T05:27:04.496 INFO:journalctl@ceph.mon.c.smithi008.stdout: required_client_features {} 2023-10-09T05:27:04.496 INFO:journalctl@ceph.mon.c.smithi008.stdout: last_failure 0 2023-10-09T05:27:04.496 INFO:journalctl@ceph.mon.c.smithi008.stdout: last_failure_osd_epoch 0 2023-10-09T05:27:04.496 INFO:journalctl@ceph.mon.c.smithi008.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:27:04.496 INFO:journalctl@ceph.mon.c.smithi008.stdout: max_mds 1 2023-10-09T05:27:04.496 INFO:journalctl@ceph.mon.c.smithi008.stdout: in 0 2023-10-09T05:27:04.496 INFO:journalctl@ceph.mon.c.smithi008.stdout: up {0=24542} 2023-10-09T05:27:04.496 INFO:journalctl@ceph.mon.c.smithi008.stdout: failed 2023-10-09T05:27:04.496 INFO:journalctl@ceph.mon.c.smithi008.stdout: damaged 2023-10-09T05:27:04.497 INFO:journalctl@ceph.mon.c.smithi008.stdout: stopped 2023-10-09T05:27:04.497 INFO:journalctl@ceph.mon.c.smithi008.stdout: data_pools [4] 2023-10-09T05:27:04.497 INFO:journalctl@ceph.mon.c.smithi008.stdout: metadata_pool 3 2023-10-09T05:27:04.497 INFO:journalctl@ceph.mon.c.smithi008.stdout: inline_data disabled 2023-10-09T05:27:04.497 INFO:journalctl@ceph.mon.c.smithi008.stdout: balancer 2023-10-09T05:27:04.497 INFO:journalctl@ceph.mon.c.smithi008.stdout: bal_rank_mask -1 2023-10-09T05:27:04.497 INFO:journalctl@ceph.mon.c.smithi008.stdout: standby_count_wanted 1 2023-10-09T05:27:04.497 INFO:journalctl@ceph.mon.c.smithi008.stdout: [mds.foo.smithi008.zfejpl{0:24542} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.8:6834/560051770,v1:172.21.15.8:6835/560051770] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:27:04.497 INFO:journalctl@ceph.mon.c.smithi008.stdout: 2023-10-09T05:27:04.497 INFO:journalctl@ceph.mon.c.smithi008.stdout: 2023-10-09T05:27:04.498 INFO:journalctl@ceph.mon.c.smithi008.stdout: Standby daemons: 2023-10-09T05:27:04.498 INFO:journalctl@ceph.mon.c.smithi008.stdout: 2023-10-09T05:27:04.498 INFO:journalctl@ceph.mon.c.smithi008.stdout: [mds.foo.smithi191.walafj{-1:24556} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.191:6832/1976404494,v1:172.21.15.191:6833/1976404494] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:27:04.498 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 bash[191162]: b504573c5a12362e1d77a42d2a3d88325112768e70599400b5b68aee10b7be71 2023-10-09T05:27:04.498 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: mon.c@-1(???).osd e84 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T05:27:04.498 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:27:04.498 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:27:04.498 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:27:04.498 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-10-09T05:27:04.498 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta expand map: {default=false} 2023-10-09T05:27:04.499 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta from 'false' to 'false' 2023-10-09T05:27:04.499 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta expanded map: {default=false} 2023-10-09T05:27:04.499 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta expand map: {default=info} 2023-10-09T05:27:04.499 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta from 'info' to 'info' 2023-10-09T05:27:04.499 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta expanded map: {default=info} 2023-10-09T05:27:04.499 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta expand map: {default=daemon} 2023-10-09T05:27:04.499 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T05:27:04.499 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T05:27:04.499 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta expand map: {default=debug} 2023-10-09T05:27:04.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T05:27:04.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 ceph-mon[191227]: expand_channel_meta expanded map: {default=debug} 2023-10-09T05:27:04.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:27:04 smithi008 systemd[1]: Started Ceph mon.c for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:27:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:04 smithi191 conmon[101299]: cluster 2023-10-09T05:27:01.766781+0000 mgr.y (mgr.25913) 60 : cluster 2023-10-09T05:27:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:04 smithi191 conmon[101299]: [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:04 smithi191 conmon[101299]: audit 2023-10-09T05:27:02.337636+0000 mon.a (mon.0) 92 : audit [DBG] from='client.? 172.21.15.8:0/1373059879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:27:04.785 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T05:27:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: cluster 2023-10-09T05:27:03.767089+0000 mgr.y (mgr.25913) 61 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: cluster 2023-10-09T05:27:04. 2023-10-09T05:27:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 193155+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T05:27:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: cluster 2023-10-09T05:27:04.193777+0000 mon.a (mon.0) 93 : cluster [INF] mon.a calling monitor election 2023-10-09T05:27:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: cluster 2023-10-09T05:27:04.208011+0000 mon.a (mon.0) 94 : 2023-10-09T05:27:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:27:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: cluster 2023-10-09T05:27:04. 2023-10-09T05:27:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 213226+0000 mon.a (mon.0) 95 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:27:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: cluster 2023-10-09T 2023-10-09T05:27:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 05:27:04.225547+0000 mon.a (mon. 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 0) 96 : cluster [DBG] fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: cluster 2023-10-09 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: T05:27:04.225571+0000 mon.a (mon. 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 0) 97 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: cluster 2023-10-09T 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 05:27:04.225845+0000 mon.a (mon.0) 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 98 : cluster [DBG] mgrmap e35: y(active, since 80s), standbys: x 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: cluster 2023-10-09 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: T05:27:04.229044+0000 mon.a (mon. 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 0) 99 : cluster [INF] overall HEALTH_OK 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: -09T05:27:04.231849+0000 mon.a ( 2023-10-09T05:27:05.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: mon.0) 100 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:05.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:27:05.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 09T05:27:04.236916+0000 mon.a ( 2023-10-09T05:27:05.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: mon.0) 101 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:05.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:27:05.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 09T05:27:04.266450+0000 mon.a (mon. 2023-10-09T05:27:05.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 0) 102 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:27:05.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: audit 2023-10-09T 2023-10-09T05:27:05.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: 05:27:04.770869+0000 mon.a (mon.0 2023-10-09T05:27:05.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:05 smithi191 conmon[101299]: ) 103 : audit [DBG] from='client.? 172.21.15.8:0/4180875543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:27:05.811 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:27:07.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:07 smithi191 conmon[101299]: cluster 2023-10-09T05:27:05.767689+0000 mgr.y (mgr.25913) 62 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:07 smithi191 conmon[101299]: 2023-10-09T05:27:08.429 INFO:teuthology.orchestra.run.smithi008.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:27:08.429 INFO:teuthology.orchestra.run.smithi008.stdout:alertmanager.a smithi008 *:9093,9094 running (2m) - 6m 25.6M - 0.23.0 ba2b418f427c 172e511a984d 2023-10-09T05:27:08.429 INFO:teuthology.orchestra.run.smithi008.stdout:grafana.a smithi191 *:3000 running (2m) - 6m 51.7M - 8.3.5 dad864ee21e9 66b01c4fecc1 2023-10-09T05:27:08.429 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi008.zfejpl smithi008 running (5m) - 5m 16.9M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5d86a4e2d3b 2023-10-09T05:27:08.429 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi191.walafj smithi191 running (5m) - 5m 14.6M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 93130380f297 2023-10-09T05:27:08.430 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.x smithi191 *:8443,9283,8765 running (108s) - 10m 447M - 18.2.0-598-gb4fa663f 3060897fdf97 e651b468b4c8 2023-10-09T05:27:08.430 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.y smithi008 *:8443,9283,8765 running (2m) - 12m 500M - 18.2.0-598-gb4fa663f 3060897fdf97 b6adda8222c0 2023-10-09T05:27:08.430 INFO:teuthology.orchestra.run.smithi008.stdout:mon.a smithi008 running (96s) - 12m 80.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 dbfd9e97488a 2023-10-09T05:27:08.430 INFO:teuthology.orchestra.run.smithi008.stdout:mon.b smithi191 running (10m) - 10m 95.5M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 2fa5874c9b2c 2023-10-09T05:27:08.430 INFO:teuthology.orchestra.run.smithi008.stdout:mon.c smithi008 running (4s) - 10m 27.3M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 b504573c5a12 2023-10-09T05:27:08.430 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.a smithi008 *:9100 running (3m) - 6m 24.1M - 1.3.1 1dbe0e931976 0b0d19fe4991 2023-10-09T05:27:08.430 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.b smithi191 *:9100 running (2m) - 6m 19.6M - 1.3.1 1dbe0e931976 05f0cea760c5 2023-10-09T05:27:08.430 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0 smithi008 running (10m) - 10m 74.8M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 70d3660d55a1 2023-10-09T05:27:08.430 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 smithi008 running (9m) - 9m 90.5M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 312729dec19a 2023-10-09T05:27:08.431 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2 smithi008 running (9m) - 9m 67.3M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7c1912182283 2023-10-09T05:27:08.431 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 smithi008 running (8m) - 8m 65.3M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9c33044d2771 2023-10-09T05:27:08.431 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 smithi191 running (8m) - 8m 72.0M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 57a8d325e472 2023-10-09T05:27:08.431 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 smithi191 running (8m) - 8m 78.9M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 141ad4d0ecc5 2023-10-09T05:27:08.431 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6 smithi191 running (7m) - 7m 77.7M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0039e7e7a199 2023-10-09T05:27:08.432 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7 smithi191 running (7m) - 7m 96.9M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d71410a3a7bd 2023-10-09T05:27:08.432 INFO:teuthology.orchestra.run.smithi008.stdout:prometheus.a smithi191 *:9095 running (113s) - 6m 57.2M - 2.33.4 514e6a882f6e d24b4acc1156 2023-10-09T05:27:09.042 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T05:27:09.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:27:08 smithi191 conmon[151041]: ts=2023-10-09T05:27:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:27:09.466 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:27:09 smithi008 conmon[177229]: 2023-10-09T05:27:09.162+0000 7f020285b700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T05:27:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:09 smithi191 conmon[101299]: cluster 2023-10-09T05:27:07.768026+0000 mgr.y (mgr.25913) 63 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:09 smithi191 conmon[101299]: audit 2023-10-09T05:27:08.344424+0000 mon.a ( 2023-10-09T05:27:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:09 smithi191 conmon[101299]: mon.0) 104 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:09 smithi191 conmon[101299]: audit 2023-10-09T05:27:08.349424+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:09 smithi191 conmon[101299]: audit 2023-10-09T05:27:08.416423+0000 mgr.y (mgr.25913) 64 : audit [DBG] from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:27:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:09 smithi191 conmon[101299]: audit 2023-10-09T05:27:09.286353 2023-10-09T05:27:09.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:09 smithi191 conmon[101299]: +0000 mon.a (mon.0) 106 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:09.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:09 smithi191 conmon[101299]: audit 2023-10 2023-10-09T05:27:09.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:09 smithi191 conmon[101299]: -09T05:27:09.291692+0000 mon.a 2023-10-09T05:27:09.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:09 smithi191 conmon[101299]: (mon.0) 107 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:09.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:09 smithi191 conmon[101299]: 2023-10-09T05:27:10.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:27:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:27:10] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.33.4" 2023-10-09T05:27:11.598 INFO:teuthology.orchestra.run.smithi008.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:27:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:11 smithi191 conmon[101299]: cluster 2023-10-09T05:27:09.768654+0000 mgr.y (mgr.25913) 65 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:12.222 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T05:27:13.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:12 smithi191 conmon[101299]: audit 2023-10-09T05:27:11.591445+0000 mgr.y (mgr.25913) 66 : audit [DBG] from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:27:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:12 smithi191 conmon[101299]: 2023-10-09T05:27:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:12 smithi191 conmon[101299]: cephadm 2023-10-09T05:27:11.591776+0000 mgr.y (mgr.25913) 67 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:27:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:12 smithi191 conmon[101299]: audit 2023-10-09T05:27:11.596490+0000 mon.a (mon. 2023-10-09T05:27:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:12 smithi191 conmon[101299]: 0) 108 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:12 smithi191 conmon[101299]: cluster 2023-10-09T05:27:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:12 smithi191 conmon[101299]: 2023-10-09T05:27:11.769019+0000 2023-10-09T05:27:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:12 smithi191 conmon[101299]: mgr.y (mgr.25913) 68 : cluster [DBG] 2023-10-09T05:27:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:12 smithi191 conmon[101299]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: cluster 2023-10-09T05:27:13.769556+0000 mgr.y (mgr.25913) 69 : cluster 2023-10-09T05:27:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: audit 2023-10-09T05:27:13.826991+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: audit 2023-10-09T05:27:14.019722+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:27:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: 09T05:27:14.025237+0000 mon.a (mon.0) 2023-10-09T05:27:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: 111 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: T05:27:14.026427+0000 mon.a (mon.0) 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: 112 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: T05:27:14.027335+0000 mon.a (mon.0 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: ) 113 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: audit 2023-10- 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: 09T05:27:14.032395+0000 mon.a (mon.0 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: ) 114 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: audit 2023 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: -10-09T05:27:14.169690+0000 mon.a ( 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: mon.0) 115 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:27:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: audit 2023 2023-10-09T05:27:15.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: -10-09T05:27:14.171064+0000 mon.a (mon 2023-10-09T05:27:15.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: .0) 116 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:27:15.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: audit 2023 2023-10-09T05:27:15.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: -10-09T05:27:14.171678+0000 mon.a ( 2023-10-09T05:27:15.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: mon.0) 117 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:27:15.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: audit 2023 2023-10-09T05:27:15.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: -10-09T05:27:14.176210+0000 mon.a (mon 2023-10-09T05:27:15.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:14 smithi191 conmon[101299]: .0) 118 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:27:16.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:16 smithi191 conmon[101299]: cluster 2023-10-09T05:27:15.040743+0000 mon.a (mon.0) 119 : cluster [DBG] 2023-10-09T05:27:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:16 smithi191 conmon[101299]: mgrmap e36: y(active, since 91s), standbys: x 2023-10-09T05:27:17.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:17 smithi191 conmon[101299]: cluster 2023-10-09T05:27:15.770181+0000 mgr.y (mgr.25913) 70 : cluster 2023-10-09T05:27:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:17 smithi191 conmon[101299]: [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:27:19.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:27:18 smithi191 conmon[151041]: ts=2023-10-09T05:27:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:27:19.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:19 smithi191 conmon[101299]: cluster 2023-10-09T05:27:17.770669+0000 mgr.y (mgr.25913) 71 2023-10-09T05:27:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:19 smithi191 conmon[101299]: : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:27:20.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:27:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:27:20] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.33.4" 2023-10-09T05:27:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:21 smithi191 conmon[101299]: cluster 2023-10-09T05:27:19.771282+0000 mgr.y (mgr.25913) 72 : cluster 2023-10-09T05:27:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:21 smithi191 conmon[101299]: [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:27:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:23 smithi191 conmon[101299]: cluster 2023-10-09T05:27:21.771704+0000 mgr.y (mgr.25913) 73 : cluster 2023-10-09T05:27:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:23 smithi191 conmon[101299]: [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:27:25.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:25 smithi191 conmon[101299]: cluster 2023-10-09T05:27:23.772274+0000 mgr.y (mgr.25913) 74 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:27:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:25 smithi191 conmon[101299]: 2023-10-09T05:27:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:27 smithi191 conmon[101299]: cluster 2023-10-09T05:27:25.772894+0000 mgr.y (mgr.25913) 75 : cluster 2023-10-09T05:27:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:27 smithi191 conmon[101299]: [DBG] pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:27:29.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:27:28 smithi191 conmon[151041]: ts=2023-10-09T05:27:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:27:29.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:29 smithi191 conmon[101299]: cluster 2023-10-09T05:27:27.773469+0000 mgr.y (mgr.25913) 2023-10-09T05:27:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:29 smithi191 conmon[101299]: 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:29 smithi191 conmon[101299]: audit 2023-10-09T05:27:28.827541+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:30.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:27:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:27:30] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.33.4" 2023-10-09T05:27:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:31 smithi191 conmon[101299]: cluster 2023-10-09T05:27:29.774228+0000 mgr.y (mgr.25913) 77 : cluster [DBG] 2023-10-09T05:27:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:31 smithi191 conmon[101299]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:33.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:33 smithi191 conmon[101299]: cluster 2023-10-09T05:27:31.774618+0000 mgr.y (mgr.25913) 78 : cluster 2023-10-09T05:27:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:33 smithi191 conmon[101299]: [DBG] pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:35.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:35 smithi191 conmon[101299]: cluster 2023-10-09T05:27:33.775131+0000 mgr.y (mgr.25913) 79 : cluster [DBG] 2023-10-09T05:27:35.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:35 smithi191 conmon[101299]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:37.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:37 smithi191 conmon[101299]: cluster 2023-10-09T05:27:35.775853+0000 mgr.y (mgr.25913) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:37.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:37 smithi191 conmon[101299]: 2023-10-09T05:27:39.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:27:38 smithi191 conmon[151041]: ts=2023-10-09T05:27:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:27:39.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:39 smithi191 conmon[101299]: cluster 2023-10-09T05:27:37.776315+0000 mgr.y (mgr.25913) 81 : cluster [DBG] 2023-10-09T05:27:39.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:39 smithi191 conmon[101299]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:40.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:27:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:27:40] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.33.4" 2023-10-09T05:27:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:41 smithi191 conmon[101299]: cluster 2023-10-09T05:27:39.777045+0000 mgr.y (mgr.25913) 82 : cluster [DBG] 2023-10-09T05:27:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:41 smithi191 conmon[101299]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:43.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:43 smithi191 conmon[101299]: cluster 2023-10-09T05:27:41.777483+0000 mgr.y (mgr.25913) 83 2023-10-09T05:27:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:43 smithi191 conmon[101299]: : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:44.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:44 smithi191 conmon[101299]: audit 2023-10-09T05:27:43.833460+0000 mon.a (mon.0) 121 : audit 2023-10-09T05:27:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:44 smithi191 conmon[101299]: [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:27:45.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:45 smithi191 conmon[101299]: cluster 2023-10-09T05:27:43.777971+0000 mgr.y (mgr.25913) 84 : cluster [DBG] 2023-10-09T05:27:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:45 smithi191 conmon[101299]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:47.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:47 smithi191 conmon[101299]: cluster 2023-10-09T05:27:45.778696+0000 mgr.y (mgr.25913) 85 : cluster 2023-10-09T05:27:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:47 smithi191 conmon[101299]: [DBG] pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:49.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:27:48 smithi191 conmon[151041]: ts=2023-10-09T05:27:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:27:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:49 smithi191 conmon[101299]: cluster 2023-10-09T05:27:47.779093+0000 mgr.y (mgr.25913) 86 : cluster [DBG] 2023-10-09T05:27:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:49 smithi191 conmon[101299]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:50.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:27:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:27:50] "GET /metrics HTTP/1.1" 200 33896 "" "Prometheus/2.33.4" 2023-10-09T05:27:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:51 smithi191 conmon[101299]: cluster 2023-10-09T05:27:49.779677+0000 mgr.y ( 2023-10-09T05:27:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:51 smithi191 conmon[101299]: mgr.25913) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:53.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:53 smithi191 conmon[101299]: cluster 2023-10-09T05:27:51.780212+0000 mgr.y (mgr.25913) 88 : cluster [DBG] 2023-10-09T05:27:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:53 smithi191 conmon[101299]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:55.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:55 smithi191 conmon[101299]: cluster 2023-10-09T05:27:53.780630+0000 mgr.y (mgr.25913) 89 : cluster [DBG] 2023-10-09T05:27:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:55 smithi191 conmon[101299]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:57 smithi191 conmon[101299]: cluster 2023-10-09T05:27:55.781284+0000 mgr.y (mgr.25913) 90 : cluster 2023-10-09T05:27:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:57 smithi191 conmon[101299]: [DBG] pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:59.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:27:58 smithi191 conmon[151041]: ts=2023-10-09T05:27:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:27:59.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:59 smithi191 conmon[101299]: cluster 2023-10-09T05:27:57.781696+0000 mgr.y (mgr.25913) 91 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:27:59.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:59 smithi191 conmon[101299]: audit 2023-10-09T05:27 2023-10-09T05:27:59.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:27:59 smithi191 conmon[101299]: :58.833980+0000 mon.a (mon.0) 122 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:00.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:28:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:28:00] "GET /metrics HTTP/1.1" 200 33906 "" "Prometheus/2.33.4" 2023-10-09T05:28:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:01 smithi191 conmon[101299]: cluster 2023-10-09T05:27:59.782304+0000 mgr.y (mgr.25913) 92 : 2023-10-09T05:28:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:01 smithi191 conmon[101299]: cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:03.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:03 smithi191 conmon[101299]: cluster 2023-10-09T05:28:01.782896+0000 mgr.y (mgr.25913) 93 : cluster [DBG] 2023-10-09T05:28:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:03 smithi191 conmon[101299]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:05.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:05 smithi191 conmon[101299]: cluster 2023-10-09T05:28:03.783439+0000 mgr.y (mgr.25913) 94 : cluster 2023-10-09T05:28:05.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:05 smithi191 conmon[101299]: [DBG] pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:07.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:07 smithi191 conmon[101299]: cluster 2023-10-09T05:28:05.784077+0000 mgr.y (mgr.25913) 95 : cluster [DBG] 2023-10-09T05:28:07.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:07 smithi191 conmon[101299]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:09.049 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:28:08 smithi191 conmon[151041]: ts=2023-10-09T05:28:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:28:09.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:09 smithi191 conmon[101299]: cluster 2023-10-09T05:28:07.784608+0000 mgr.y (mgr.25913) 96 : cluster 2023-10-09T05:28:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:09 smithi191 conmon[101299]: [DBG] pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:10.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:28:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:28:10] "GET /metrics HTTP/1.1" 200 33906 "" "Prometheus/2.33.4" 2023-10-09T05:28:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:11 smithi191 conmon[101299]: cluster 2023-10-09T05:28:09.785104+0000 mgr.y (mgr.25913) 97 : cluster [DBG] 2023-10-09T05:28:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:11 smithi191 conmon[101299]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:13.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:13 smithi191 conmon[101299]: cluster 2023-10-09T05:28:11.785723+0000 mgr.y (mgr.25913) 98 : cluster [DBG] 2023-10-09T05:28:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:13 smithi191 conmon[101299]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:14.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:14 smithi191 conmon[101299]: audit 2023-10-09T05:28:13.834412+0000 mon.a (mon.0) 123 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:14 smithi191 conmon[101299]: audit 2023-10-09T05:28:14.313843+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:14.841 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T05:28:15.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:15 smithi191 conmon[101299]: cluster 2023-10-09T05:28:13.786391+0000 mgr.y (mgr.25913 2023-10-09T05:28:15.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:15 smithi191 conmon[101299]: ) 99 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:17.295 INFO:teuthology.orchestra.run.smithi008.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:17.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:17 smithi191 conmon[101299]: cluster 2023-10-09T05:28:15.786861+0000 mgr.y ( 2023-10-09T05:28:17.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:17 smithi191 conmon[101299]: mgr.25913) 100 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:17 smithi191 conmon[101299]: audit 2023-10-09T05:28:17.295233+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:17.955 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T05:28:18.800 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:28:18 smithi191 conmon[151041]: ts=2023-10-09T05:28:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:28:18.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:18 smithi191 conmon[101299]: audit 2023-10-09T05:28:17.291499+0000 mgr.y (mgr.25913) 101 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:18.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:18 smithi191 conmon[101299]: cephadm 2023-10-09T05:28:17.291859+0000 mgr.y (mgr.25913) 102 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:18.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:18 smithi191 conmon[101299]: audit 2023-10-09T05:28:17.692984+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:18.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:18 smithi191 conmon[101299]: audit 2023-10-09T05:28:17.697526+0000 mon.a (mon.0) 127 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:19 smithi191 conmon[101299]: cluster 2023-10-09T05:28:17.787234+0000 mgr.y (mgr.25913) 103 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:19 smithi191 conmon[101299]: audit 2023-10-09T05:28:19.201771+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:19 smithi191 conmon[101299]: audit 2023-10-09T05:28:19.205986+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:20.178 INFO:teuthology.orchestra.run.smithi008.stdout:wait for all mons to upgrade 2023-10-09T05:28:20.653 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: audit 2023-10-09T05:28:19.725603+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:20.653 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:28:20.653 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: 10-09T05:28:19.726451+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:20.653 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: audit 2023-10-09T05:28:19.731100+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:20.653 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: audit 2023-10-09T05:28:19.869068+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:20.653 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:28:20.653 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: 10-09T05:28:19.871530+0000 mon.a (mon.0) 134 2023-10-09T05:28:20.653 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:28:20.653 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: audit 2023-10-09T05:28:20.653 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: 2023-10-09T05:28: 2023-10-09T05:28:20.654 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: 19.872758+0000 mon.a (mon.0) 135 : audit 2023-10-09T05:28:20.654 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:28:20.654 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: audit 2023-10-09T05: 2023-10-09T05:28:20.654 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: 28:19.873446+0000 mon.a (mon.0) 2023-10-09T05:28:20.654 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:20 smithi191 conmon[101299]: 136 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T05:28:20.712 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:28:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:28:20] "GET /metrics HTTP/1.1" 200 33912 "" "Prometheus/2.33.4" 2023-10-09T05:28:20.731 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T05:28:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: cluster 2023-10-09T05:28:19.787725+0000 mgr.y (mgr 2023-10-09T05:28:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: .25913) 104 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: cephadm 2023-10-09T05:28:19.873959+0000 mgr.y (mgr.25913) 105 : 2023-10-09T05:28:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-10-09T05:28:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: audit 2023- 2023-10-09T05:28:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: 10-09T05:28:21.445505+0000 mon.a (mon. 2023-10-09T05:28:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: 0) 137 : audit [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: audit 2023 2023-10-09T05:28:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: -10-09T05:28:21 2023-10-09T05:28:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: .446397+0000 mon.a (mon.0) 138 : audit 2023-10-09T05:28:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: [INF] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: audit 2023-10-09 2023-10-09T05:28:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: T05:28:21.447049+0000 mon.a ( 2023-10-09T05:28:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: mon.0) 139 : audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: audit 2023-10-09T05:28:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: 2023-10-09T05:28: 2023-10-09T05:28:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: 21.448217+0000 mon.a (mon.0) 140 : 2023-10-09T05:28:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:21 smithi191 conmon[101299]: audit [DBG] from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:22.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:22 smithi191 conmon[101299]: cephadm 2023-10-09T05:28:21.440489+0000 mgr.y (mgr.25913 2023-10-09T05:28:22.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:22 smithi191 conmon[101299]: ) 106 : cephadm [INF] Upgrade: Updating mon.b 2023-10-09T05:28:22.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:22 smithi191 conmon[101299]: cephadm 2023-10-09T05:28:21.449422+0000 mgr.y (mgr.25913) 107 : cephadm [INF] Deploying daemon mon.b on smithi191 2023-10-09T05:28:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:22 smithi191 systemd[1]: Stopping Ceph mon.b for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:28:23.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:23 smithi191 conmon[101299]: debug 2023-10-09T05:28:23.252+0000 7f08de788700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T05:28:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:23 smithi191 conmon[101299]: debug 2023-10-09T05:28:23.252+0000 7f08de788700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T05:28:24.413 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:24 smithi191 bash[161246]: 2fa5874c9b2ca9ba6b36d9f5a8c7c8de1eaeb99bf15bca7164d05f48542f0f4d 2023-10-09T05:28:24.413 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:24 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T05:28:24.413 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:24 smithi191 systemd[1]: Stopped Ceph mon.b for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:28:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:24 smithi191 systemd[1]: Starting Ceph mon.b for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:28:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:28:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T05:28:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: pidfile_write: ignore empty --pid-file 2023-10-09T05:28:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: load: jerasure load: lrc 2023-10-09T05:28:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: RocksDB version: 7.9.2 2023-10-09T05:28:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Git sha 0 2023-10-09T05:28:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: DB SUMMARY 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: DB Session ID: UKJHQG81FRUCNP44TVPR 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: MANIFEST file: MANIFEST-000005 size: 1078 Bytes 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000023.sst 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000021.log size: 4077288 ; 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T05:28:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.env: 0x55e50fe86720 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.info_log: 0x55e511f55420 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.statistics: (nil) 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.use_fsync: 0 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:28:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:28:25.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:28:25.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:28:25.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:28:25.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:28:25.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:28:25.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:28:25.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.db_log_dir: 2023-10-09T05:28:25.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.wal_dir: 2023-10-09T05:28:25.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:28:25.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:28:25.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:28:25.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:28:25.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:28:25.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:28:25.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:28:25.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:28:25.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.write_buffer_manager: 0x55e5111e45a0 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.unordered_write: 0 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:28:25.556 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.row_cache: None 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 bash[161602]: 8d871de2e613e825c03637ebc0e4eab2810d0346b87c44c794a8cecfc38cd0f8 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.wal_filter: None 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.wal_compression: 0 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:28:25.557 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:28:25.558 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_open_files: -1 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:28:25.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Compression algorithms supported: 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: kZSTD supported: 0 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T05:28:25.560 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.merge_operator: 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_filter: None 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e511f55520) 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:28:25.561 INFO:journalctl@ceph.mon.b.smithi191.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: index_type: 0 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: data_block_index_type: 0 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: index_shortening: 1 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: checksum: 4 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: no_block_cache: 0 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache: 0x55e511267350 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache_options: 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: capacity : 536870912 2023-10-09T05:28:25.562 INFO:journalctl@ceph.mon.b.smithi191.stdout: num_shard_bits : 4 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: strict_capacity_limit : 0 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache_compressed: (nil) 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: persistent_cache: (nil) 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_size: 4096 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_size_deviation: 10 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_restart_interval: 16 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: index_block_restart_interval: 1 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: metadata_block_size: 4096 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: partition_filters: 0 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: use_delta_encoding: 1 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: filter_policy: bloomfilter 2023-10-09T05:28:25.563 INFO:journalctl@ceph.mon.b.smithi191.stdout: whole_key_filtering: 1 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout: verify_compression: 0 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout: format_version: 5 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout: enable_index_compression: 1 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_align: 0 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout: max_auto_readahead_size: 262144 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout: prepopulate_block_cache: 0 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout: initial_auto_readahead_size: 8192 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:28:25.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compression: NoCompression 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.num_levels: 7 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:28:25.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:28:25.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:28:25.567 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:28:25.568 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:28:25.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.ttl: 2592000 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T05:28:25.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 25, last_sequence is 13639, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:28:25.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-10-09T05:28:25.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ad0932c5-3903-4f0e-a6ea-8d5ff09a8aca 2023-10-09T05:28:25.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829305090196, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-10-09T05:28:25.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-10-09T05:28:25.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 systemd[1]: Started Ceph mon.b for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:28:25.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829305099089, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 2707506, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13645, "largest_seqno": 14438, "table_properties": {"data_size": 2702432, "index_size": 3479, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 4563, "raw_average_key_size": 20, "raw_value_size": 2695537, "raw_average_value_size": 11822, "num_data_blocks": 152, "num_entries": 228, "num_filter_entries": 228, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696829305, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ad0932c5-3903-4f0e-a6ea-8d5ff09a8aca", "db_session_id": "UKJHQG81FRUCNP44TVPR", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-10-09T05:28:25.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829305099181, "job": 1, "event": "recovery_finished"} 2023-10-09T05:28:25.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-10-09T05:28:25.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:28:25.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e511302000 2023-10-09T05:28:25.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: DB pointer 0x55e5112f0000 2023-10-09T05:28:25.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T05:28:25.573 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T05:28:25.573 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** DB Stats ** 2023-10-09T05:28:25.573 INFO:journalctl@ceph.mon.b.smithi191.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:28:25.573 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:28:25.573 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:28:25.573 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:28:25.573 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:28:25.573 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:28:25.573 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:28:25.574 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2023-10-09T05:28:25.574 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** Compaction Stats [default] ** 2023-10-09T05:28:25.574 INFO:journalctl@ceph.mon.b.smithi191.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:28:25.574 INFO:journalctl@ceph.mon.b.smithi191.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T05:28:25.574 INFO:journalctl@ceph.mon.b.smithi191.stdout: L0 1/0 2.58 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 658.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T05:28:25.574 INFO:journalctl@ceph.mon.b.smithi191.stdout: L6 1/0 27.70 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T05:28:25.574 INFO:journalctl@ceph.mon.b.smithi191.stdout: Sum 2/0 30.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 658.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T05:28:25.574 INFO:journalctl@ceph.mon.b.smithi191.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 658.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T05:28:25.574 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2023-10-09T05:28:25.574 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** Compaction Stats [default] ** 2023-10-09T05:28:25.574 INFO:journalctl@ceph.mon.b.smithi191.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 658.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative compaction: 0.00 GB write, 187.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:28:25.575 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval compaction: 0.00 GB write, 187.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout: Block cache BinnedLRUCache@0x55e511267350#7 capacity: 512.00 MB usage: 4.73 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.66 KB,0.00012517%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: starting mon.b rank 1 at public addrs [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] at bind addrs [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: mon.b@-1(???) e3 preinit fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: mon.b@-1(???).mds e6 new map 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: mon.b@-1(???).mds e6 print_map 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout: e6 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:28:25.576 INFO:journalctl@ceph.mon.b.smithi191.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: legacy client fscid: 1 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: Filesystem 'foo' (1) 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: fs_name foo 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: epoch 5 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: created 2023-10-09T05:21:16.789372+0000 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: modified 2023-10-09T05:21:26.247032+0000 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: tableserver 0 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: root 0 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: session_timeout 60 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: session_autoclose 300 2023-10-09T05:28:25.577 INFO:journalctl@ceph.mon.b.smithi191.stdout: max_file_size 1099511627776 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: required_client_features {} 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: last_failure 0 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: last_failure_osd_epoch 0 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: max_mds 1 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: in 0 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: up {0=24542} 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: failed 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: damaged 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: stopped 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: data_pools [4] 2023-10-09T05:28:25.578 INFO:journalctl@ceph.mon.b.smithi191.stdout: metadata_pool 3 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout: inline_data disabled 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout: balancer 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout: bal_rank_mask -1 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout: standby_count_wanted 1 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout: [mds.foo.smithi008.zfejpl{0:24542} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.8:6834/560051770,v1:172.21.15.8:6835/560051770] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout: Standby daemons: 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout: [mds.foo.smithi191.walafj{-1:24556} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.191:6832/1976404494,v1:172.21.15.191:6833/1976404494] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: mon.b@-1(???).osd e84 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T05:28:25.579 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta expand map: {default=false} 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta from 'false' to 'false' 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta expanded map: {default=false} 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta expand map: {default=info} 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta from 'info' to 'info' 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta expanded map: {default=info} 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta expand map: {default=daemon} 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T05:28:25.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T05:28:25.581 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta expand map: {default=debug} 2023-10-09T05:28:25.581 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T05:28:25.581 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:25 smithi191 ceph-mon[161669]: expand_channel_meta expanded map: {default=debug} 2023-10-09T05:28:27.534 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T05:28:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: mkfs 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:28:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T05:28:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Reconfiguring daemon node-exporter.b on smithi191 2023-10-09T05:28:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T05:28:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Reconfiguring daemon grafana.a on smithi191 2023-10-09T05:28:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:28:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:28:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Updating mgr.y 2023-10-09T05:28:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Deploying daemon mgr.y on smithi008 2023-10-09T05:28:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: [09/Oct/2023:05:25:02] ENGINE Bus STARTING 2023-10-09T05:28:27.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: [09/Oct/2023:05:25:02] ENGINE Serving on https://172.21.15.8:7150 2023-10-09T05:28:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: [09/Oct/2023:05:25:03] ENGINE Serving on http://172.21.15.8:8765 2023-10-09T05:28:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: [09/Oct/2023:05:25:03] ENGINE Bus STARTED 2023-10-09T05:28:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:28:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:28:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Updating mgr.x 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Deploying daemon mgr.x on smithi191 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Setting container_image for all mgr 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: It appears safe to stop mon.a 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Updating mon.a 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Deploying daemon mon.a on smithi008 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: [09/Oct/2023:05:25:44] ENGINE Bus STARTING 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: [09/Oct/2023:05:25:44] ENGINE Serving on http://172.21.15.8:8765 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: [09/Oct/2023:05:25:45] ENGINE Serving on https://172.21.15.8:7150 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: [09/Oct/2023:05:25:45] ENGINE Bus STARTED 2023-10-09T05:28:27.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: It appears safe to stop mon.c 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3809502107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Updating mon.c 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Deploying daemon mon.c on smithi008 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1373059879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: mon.c calling monitor election 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: mon.a calling monitor election 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: monmap e3: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:28:27.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:28:27.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:28:27.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: mgrmap e35: y(active, since 80s), standbys: x 2023-10-09T05:28:27.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: overall HEALTH_OK 2023-10-09T05:28:27.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4180875543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: mgrmap e36: y(active, since 91s), standbys: x 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.808 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.809 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.809 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.809 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.809 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.809 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.809 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.809 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.809 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.809 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.810 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.810 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.810 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.810 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.810 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.810 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.810 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.810 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.810 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.810 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.811 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.812 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:28:27.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:28:27.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T05:28:27.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: It appears safe to stop mon.b 2023-10-09T05:28:27.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Upgrade: Updating mon.b 2023-10-09T05:28:27.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:27.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:27.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: Deploying daemon mon.b on smithi191 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3607667929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: overall HEALTH_OK 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: mon.b calling monitor election 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: mon.a calling monitor election 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: mon.c calling monitor election 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:28:27.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: monmap e4: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:28:27.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:28:27.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:28:27.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: mgrmap e36: y(active, since 2m), standbys: x 2023-10-09T05:28:27.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: overall HEALTH_OK 2023-10-09T05:28:27.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: mkfs 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:28:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T05:28:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Reconfiguring daemon node-exporter.b on smithi191 2023-10-09T05:28:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T05:28:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Reconfiguring daemon grafana.a on smithi191 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Updating mgr.y 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Deploying daemon mgr.y on smithi008 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: [09/Oct/2023:05:25:02] ENGINE Bus STARTING 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: [09/Oct/2023:05:25:02] ENGINE Serving on https://172.21.15.8:7150 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: [09/Oct/2023:05:25:03] ENGINE Serving on http://172.21.15.8:8765 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: [09/Oct/2023:05:25:03] ENGINE Bus STARTED 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:28:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:28:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Updating mgr.x 2023-10-09T05:28:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Deploying daemon mgr.x on smithi191 2023-10-09T05:28:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Setting container_image for all mgr 2023-10-09T05:28:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: It appears safe to stop mon.a 2023-10-09T05:28:27.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Updating mon.a 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Deploying daemon mon.a on smithi008 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: [09/Oct/2023:05:25:44] ENGINE Bus STARTING 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: [09/Oct/2023:05:25:44] ENGINE Serving on http://172.21.15.8:8765 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: [09/Oct/2023:05:25:45] ENGINE Serving on https://172.21.15.8:7150 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: [09/Oct/2023:05:25:45] ENGINE Bus STARTED 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: mkfs 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:28:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T05:28:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Reconfiguring daemon node-exporter.b on smithi191 2023-10-09T05:28:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T05:28:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Reconfiguring daemon grafana.a on smithi191 2023-10-09T05:28:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:28:27.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Updating mgr.y 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Deploying daemon mgr.y on smithi008 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: [09/Oct/2023:05:25:02] ENGINE Bus STARTING 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: [09/Oct/2023:05:25:02] ENGINE Serving on https://172.21.15.8:7150 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: [09/Oct/2023:05:25:03] ENGINE Serving on http://172.21.15.8:8765 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: [09/Oct/2023:05:25:03] ENGINE Bus STARTED 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Reconfiguring daemon prometheus.a on smithi191 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Updating mgr.x 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Deploying daemon mgr.x on smithi191 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Setting container_image for all mgr 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: It appears safe to stop mon.a 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Updating mon.a 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Deploying daemon mon.a on smithi008 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: [09/Oct/2023:05:25:44] ENGINE Bus STARTING 2023-10-09T05:28:27.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: [09/Oct/2023:05:25:44] ENGINE Serving on http://172.21.15.8:8765 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: [09/Oct/2023:05:25:45] ENGINE Serving on https://172.21.15.8:7150 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: [09/Oct/2023:05:25:45] ENGINE Bus STARTED 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi008:/etc/ceph/ceph.conf 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi191:/etc/ceph/ceph.conf 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi008:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: It appears safe to stop mon.c 2023-10-09T05:28:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3809502107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Updating mon.c 2023-10-09T05:28:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Deploying daemon mon.c on smithi008 2023-10-09T05:28:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1373059879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: mon.c calling monitor election 2023-10-09T05:28:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: mon.a calling monitor election 2023-10-09T05:28:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:28:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: monmap e3: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:28:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:28:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:28:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: mgrmap e35: y(active, since 80s), standbys: x 2023-10-09T05:28:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: overall HEALTH_OK 2023-10-09T05:28:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4180875543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Updating smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: It appears safe to stop mon.c 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3809502107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Updating mon.c 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Deploying daemon mon.c on smithi008 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1373059879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.976 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: mon.c calling monitor election 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: mon.a calling monitor election 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: monmap e3: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: mgrmap e35: y(active, since 80s), standbys: x 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: overall HEALTH_OK 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.977 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4180875543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:27.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.978 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.978 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.979 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: mgrmap e36: y(active, since 91s), standbys: x 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.980 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.981 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:27.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: mgrmap e36: y(active, since 91s), standbys: x 2023-10-09T05:28:27.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.984 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.985 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:27.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.986 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.987 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:27.988 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:27.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.989 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:27.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.989 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:28:27.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:28:27.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T05:28:27.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: It appears safe to stop mon.b 2023-10-09T05:28:27.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Upgrade: Updating mon.b 2023-10-09T05:28:27.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:27.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:27.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.990 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: Deploying daemon mon.b on smithi191 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3607667929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: overall HEALTH_OK 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: mon.b calling monitor election 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: mon.a calling monitor election 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: mon.c calling monitor election 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: monmap e4: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:28:27.991 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:28:27.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:28:27.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: mgrmap e36: y(active, since 2m), standbys: x 2023-10-09T05:28:27.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: overall HEALTH_OK 2023-10-09T05:28:27.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.992 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:27.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: It appears safe to stop mon.b 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Upgrade: Updating mon.b 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: Deploying daemon mon.b on smithi191 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3607667929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: overall HEALTH_OK 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: mon.b calling monitor election 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: mon.a calling monitor election 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: mon.c calling monitor election 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: monmap e4: 3 mons at {a=[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]} removed_ranks: {} 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: fsmap foo:1 {0=foo.smithi008.zfejpl=up:active} 1 up:standby 2023-10-09T05:28:27.994 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:28:27.995 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: mgrmap e36: y(active, since 2m), standbys: x 2023-10-09T05:28:27.995 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: overall HEALTH_OK 2023-10-09T05:28:27.995 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.995 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:27.995 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:28.161 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:28:28.764 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3727466799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:28.765 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3727466799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3727466799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:28:28.800 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:28:28 smithi191 conmon[151041]: ts=2023-10-09T05:28:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:28:29.631 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:29 smithi008 ceph-mon[184418]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:29.631 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:29.631 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:29 smithi008 ceph-mon[191227]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:29.631 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:29.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:29 smithi191 ceph-mon[161669]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:30.012 INFO:teuthology.orchestra.run.smithi008.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:28:30.012 INFO:teuthology.orchestra.run.smithi008.stdout:alertmanager.a smithi008 *:9093,9094 running (4m) - 7m 25.6M - 0.23.0 ba2b418f427c 172e511a984d 2023-10-09T05:28:30.012 INFO:teuthology.orchestra.run.smithi008.stdout:grafana.a smithi191 *:3000 running (3m) - 7m 59.1M - 8.3.5 dad864ee21e9 66b01c4fecc1 2023-10-09T05:28:30.012 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi008.zfejpl smithi008 running (7m) - 7m 17.2M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5d86a4e2d3b 2023-10-09T05:28:30.012 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi191.walafj smithi191 running (7m) - 7m 14.9M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 93130380f297 2023-10-09T05:28:30.012 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.x smithi191 *:8443,9283,8765 running (3m) - 11m 448M - 18.2.0-598-gb4fa663f 3060897fdf97 e651b468b4c8 2023-10-09T05:28:30.012 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.y smithi008 *:8443,9283,8765 running (3m) - 13m 502M - 18.2.0-598-gb4fa663f 3060897fdf97 b6adda8222c0 2023-10-09T05:28:30.012 INFO:teuthology.orchestra.run.smithi008.stdout:mon.a smithi008 running (2m) - 13m 85.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 dbfd9e97488a 2023-10-09T05:28:30.013 INFO:teuthology.orchestra.run.smithi008.stdout:mon.b smithi191 starting - - - 2048M 2023-10-09T05:28:30.013 INFO:teuthology.orchestra.run.smithi008.stdout:mon.c smithi008 running (86s) - 12m 73.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 b504573c5a12 2023-10-09T05:28:30.013 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.a smithi008 *:9100 running (4m) - 8m 23.9M - 1.3.1 1dbe0e931976 0b0d19fe4991 2023-10-09T05:28:30.013 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.b smithi191 *:9100 running (4m) - 8m 19.4M - 1.3.1 1dbe0e931976 05f0cea760c5 2023-10-09T05:28:30.013 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0 smithi008 running (11m) - 11m 77.4M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 70d3660d55a1 2023-10-09T05:28:30.013 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 smithi008 running (11m) - 11m 92.9M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 312729dec19a 2023-10-09T05:28:30.013 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2 smithi008 running (10m) - 10m 69.0M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7c1912182283 2023-10-09T05:28:30.013 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 smithi008 running (10m) - 10m 68.0M 2501M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9c33044d2771 2023-10-09T05:28:30.013 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 smithi191 running (9m) - 9m 75.1M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 57a8d325e472 2023-10-09T05:28:30.013 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 smithi191 running (9m) - 9m 81.3M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 141ad4d0ecc5 2023-10-09T05:28:30.013 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6 smithi191 running (9m) - 9m 80.2M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0039e7e7a199 2023-10-09T05:28:30.014 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7 smithi191 running (8m) - 8m 99.7M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d71410a3a7bd 2023-10-09T05:28:30.014 INFO:teuthology.orchestra.run.smithi008.stdout:prometheus.a smithi191 *:9095 running (3m) - 8m 65.7M - 2.33.4 514e6a882f6e d24b4acc1156 2023-10-09T05:28:30.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:28:30 smithi008 conmon[177229]: 2023-10-09T05:28:30.111+0000 7f020285b700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T05:28:30.621 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T05:28:30.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:28:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:28:30] "GET /metrics HTTP/1.1" 200 33835 "" "Prometheus/2.33.4" 2023-10-09T05:28:31.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:31 smithi191 ceph-mon[161669]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:31.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:31 smithi191 ceph-mon[161669]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:31.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:31.367 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:31.367 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:31.367 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:31.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[184418]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:31.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[184418]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:31.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:31.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:31.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:31.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:31.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[191227]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:31.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[191227]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:31.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:31.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:31.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:31.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:32.678 INFO:teuthology.orchestra.run.smithi008.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:33.295 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T05:28:33.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:33 smithi008 ceph-mon[184418]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:33.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:33.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:33 smithi008 ceph-mon[191227]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:33.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:33 smithi191 ceph-mon[161669]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:34.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:34 smithi191 ceph-mon[161669]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:34 smithi191 ceph-mon[161669]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:34 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:34 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:34 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:34 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:34 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:34.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[184418]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:34.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[184418]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:34.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:34.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:34.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:34.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:34.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:34.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[191227]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:28:34.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[191227]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:28:34.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:34.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:34.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:34.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:34.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:34 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[184418]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[184418]: Reconfiguring mon.a (monmap changed)... 2023-10-09T05:28:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[184418]: Reconfiguring daemon mon.a on smithi008 2023-10-09T05:28:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[191227]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[191227]: Reconfiguring mon.a (monmap changed)... 2023-10-09T05:28:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:35 smithi008 ceph-mon[191227]: Reconfiguring daemon mon.a on smithi008 2023-10-09T05:28:35.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:35 smithi191 ceph-mon[161669]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:35.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:35 smithi191 ceph-mon[161669]: Reconfiguring mon.a (monmap changed)... 2023-10-09T05:28:35.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:35.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:35.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:35.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:35 smithi191 ceph-mon[161669]: Reconfiguring daemon mon.a on smithi008 2023-10-09T05:28:37.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[191227]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:37.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[191227]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T05:28:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:28:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:28:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[191227]: Reconfiguring daemon mgr.y on smithi008 2023-10-09T05:28:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[184418]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[184418]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T05:28:37.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:28:37.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:28:37.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:37.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:37 smithi008 ceph-mon[184418]: Reconfiguring daemon mgr.y on smithi008 2023-10-09T05:28:37.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:37 smithi191 ceph-mon[161669]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:37 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:37 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:37 smithi191 ceph-mon[161669]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T05:28:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:37 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:28:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:37 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:28:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:37 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:37 smithi191 ceph-mon[161669]: Reconfiguring daemon mgr.y on smithi008 2023-10-09T05:28:38.661 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:38 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:38.661 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:38 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:38.662 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:38 smithi191 ceph-mon[161669]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:28:38.662 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:38 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:38.662 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:38 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:38.662 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:38 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:38.662 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:38 smithi191 ceph-mon[161669]: Reconfiguring daemon mon.c on smithi008 2023-10-09T05:28:38.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:38.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:38.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[184418]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:28:38.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:38.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:38.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:38.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[184418]: Reconfiguring daemon mon.c on smithi008 2023-10-09T05:28:38.666 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:38.666 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:38.666 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[191227]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:28:38.666 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:38.666 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:38.666 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:38.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:38 smithi008 ceph-mon[191227]: Reconfiguring daemon mon.c on smithi008 2023-10-09T05:28:39.049 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:28:38 smithi191 conmon[151041]: ts=2023-10-09T05:28:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:28:39.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:39 smithi008 ceph-mon[184418]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:39.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:39 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:39.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:39 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:39.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:39 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:28:39.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:39 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:39.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:39 smithi008 ceph-mon[191227]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:39.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:39 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:39.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:39 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:39.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:39 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:28:39.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:39 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:39.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:39 smithi191 ceph-mon[161669]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:39.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:39 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:39.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:39 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:39.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:39 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:28:39.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:39 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:40.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:28:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:28:40] "GET /metrics HTTP/1.1" 200 33835 "" "Prometheus/2.33.4" 2023-10-09T05:28:40.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[184418]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T05:28:40.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[184418]: Reconfiguring daemon osd.0 on smithi008 2023-10-09T05:28:40.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:40.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:40.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:28:40.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:40.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[191227]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T05:28:40.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[191227]: Reconfiguring daemon osd.0 on smithi008 2023-10-09T05:28:40.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:40.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:40.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:28:40.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:40 smithi191 ceph-mon[161669]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T05:28:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:40 smithi191 ceph-mon[161669]: Reconfiguring daemon osd.0 on smithi008 2023-10-09T05:28:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:28:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:41.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:41 smithi008 ceph-mon[184418]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:41.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:41 smithi008 ceph-mon[184418]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T05:28:41.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:41 smithi008 ceph-mon[184418]: Reconfiguring daemon osd.1 on smithi008 2023-10-09T05:28:41.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:41 smithi008 ceph-mon[191227]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:41.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:41 smithi008 ceph-mon[191227]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T05:28:41.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:41 smithi008 ceph-mon[191227]: Reconfiguring daemon osd.1 on smithi008 2023-10-09T05:28:41.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:41 smithi191 ceph-mon[161669]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:41 smithi191 ceph-mon[161669]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T05:28:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:41 smithi191 ceph-mon[161669]: Reconfiguring daemon osd.1 on smithi008 2023-10-09T05:28:42.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:42.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:42.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[184418]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T05:28:42.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:28:42.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:42.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[184418]: Reconfiguring daemon osd.2 on smithi008 2023-10-09T05:28:42.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:42.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:42.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[191227]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T05:28:42.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:28:42.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:42.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:42 smithi008 ceph-mon[191227]: Reconfiguring daemon osd.2 on smithi008 2023-10-09T05:28:42.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:42 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:42.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:42 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:42.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:42 smithi191 ceph-mon[161669]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T05:28:42.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:42 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:28:42.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:42 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:42.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:42 smithi191 ceph-mon[161669]: Reconfiguring daemon osd.2 on smithi008 2023-10-09T05:28:43.618 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:43 smithi008 ceph-mon[184418]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:43.618 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:43.619 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:43.619 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:28:43.619 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:43.619 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:43 smithi008 ceph-mon[191227]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:43.619 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:43.619 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:43.619 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:28:43.619 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:43.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:43 smithi191 ceph-mon[161669]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:28:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[184418]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T05:28:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[184418]: Reconfiguring daemon osd.3 on smithi008 2023-10-09T05:28:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:28:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[191227]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T05:28:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[191227]: Reconfiguring daemon osd.3 on smithi008 2023-10-09T05:28:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:28:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:44.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:44 smithi191 ceph-mon[161669]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T05:28:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:44 smithi191 ceph-mon[161669]: Reconfiguring daemon osd.3 on smithi008 2023-10-09T05:28:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi008.zfejpl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:28:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:45.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[184418]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[184418]: Reconfiguring mds.foo.smithi008.zfejpl (monmap changed)... 2023-10-09T05:28:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[184418]: Reconfiguring daemon mds.foo.smithi008.zfejpl on smithi008 2023-10-09T05:28:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[191227]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[191227]: Reconfiguring mds.foo.smithi008.zfejpl (monmap changed)... 2023-10-09T05:28:45.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[191227]: Reconfiguring daemon mds.foo.smithi008.zfejpl on smithi008 2023-10-09T05:28:45.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:45.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:45.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:45.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:45.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:45.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:45 smithi191 ceph-mon[161669]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:45 smithi191 ceph-mon[161669]: Reconfiguring mds.foo.smithi008.zfejpl (monmap changed)... 2023-10-09T05:28:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:45 smithi191 ceph-mon[161669]: Reconfiguring daemon mds.foo.smithi008.zfejpl on smithi008 2023-10-09T05:28:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:28:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:28:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:46.781 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:46 smithi191 ceph-mon[161669]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:28:46.781 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:46 smithi191 ceph-mon[161669]: Reconfiguring daemon mon.b on smithi191 2023-10-09T05:28:46.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:46 smithi008 ceph-mon[184418]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:28:46.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:46 smithi008 ceph-mon[184418]: Reconfiguring daemon mon.b on smithi191 2023-10-09T05:28:46.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:46 smithi008 ceph-mon[191227]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:28:46.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:46 smithi008 ceph-mon[191227]: Reconfiguring daemon mon.b on smithi191 2023-10-09T05:28:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:47 smithi191 ceph-mon[161669]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:28:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:28:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:47.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[184418]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:47.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:47.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:47.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:28:47.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:28:47.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:47.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[191227]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:47.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:47.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:47.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:28:47.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:28:47.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:48.472 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:48 smithi191 ceph-mon[161669]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T05:28:48.473 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:48 smithi191 ceph-mon[161669]: Reconfiguring daemon mgr.x on smithi191 2023-10-09T05:28:48.473 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:48.473 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:48.473 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:28:48.473 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:48.800 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:28:48 smithi191 conmon[151041]: ts=2023-10-09T05:28:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:28:48.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[184418]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T05:28:48.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[184418]: Reconfiguring daemon mgr.x on smithi191 2023-10-09T05:28:48.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:48.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:48.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:28:48.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:48.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[191227]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T05:28:48.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[191227]: Reconfiguring daemon mgr.x on smithi191 2023-10-09T05:28:48.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:48.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:48.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:28:48.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:49.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:49 smithi191 ceph-mon[161669]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:49 smithi191 ceph-mon[161669]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T05:28:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:49 smithi191 ceph-mon[161669]: Reconfiguring daemon osd.4 on smithi191 2023-10-09T05:28:49.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:49 smithi008 ceph-mon[184418]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:49.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:49 smithi008 ceph-mon[184418]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T05:28:49.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:49 smithi008 ceph-mon[184418]: Reconfiguring daemon osd.4 on smithi191 2023-10-09T05:28:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:49 smithi008 ceph-mon[191227]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:49 smithi008 ceph-mon[191227]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T05:28:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:49 smithi008 ceph-mon[191227]: Reconfiguring daemon osd.4 on smithi191 2023-10-09T05:28:50.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:28:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:28:50] "GET /metrics HTTP/1.1" 200 33905 "" "Prometheus/2.33.4" 2023-10-09T05:28:50.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:50.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:50.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[184418]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T05:28:50.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:28:50.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:50.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[184418]: Reconfiguring daemon osd.5 on smithi191 2023-10-09T05:28:50.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[184418]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:50.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:50.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:50.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[191227]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T05:28:50.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:28:50.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:50.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[191227]: Reconfiguring daemon osd.5 on smithi191 2023-10-09T05:28:50.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:50 smithi008 ceph-mon[191227]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:50 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:50 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:50 smithi191 ceph-mon[161669]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T05:28:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:50 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:28:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:50 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:50 smithi191 ceph-mon[161669]: Reconfiguring daemon osd.5 on smithi191 2023-10-09T05:28:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:50 smithi191 ceph-mon[161669]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:52.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:52.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:52.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[184418]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T05:28:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:28:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[184418]: Reconfiguring daemon osd.6 on smithi191 2023-10-09T05:28:52.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:52.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:52.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[191227]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T05:28:52.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:28:52.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:52.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:52 smithi008 ceph-mon[191227]: Reconfiguring daemon osd.6 on smithi191 2023-10-09T05:28:52.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:52 smithi191 ceph-mon[161669]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T05:28:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:28:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:52 smithi191 ceph-mon[161669]: Reconfiguring daemon osd.6 on smithi191 2023-10-09T05:28:53.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:53 smithi191 ceph-mon[161669]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:28:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:53.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:53 smithi008 ceph-mon[184418]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:53.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:53.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:53.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:28:53.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:53 smithi008 ceph-mon[191227]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:28:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:54.681 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:54 smithi191 ceph-mon[161669]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T05:28:54.681 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:54 smithi191 ceph-mon[161669]: Reconfiguring daemon osd.7 on smithi191 2023-10-09T05:28:54.682 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:54 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:54.682 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:54 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:54.682 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:54 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:28:54.682 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:54 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:54.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[184418]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T05:28:54.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[184418]: Reconfiguring daemon osd.7 on smithi191 2023-10-09T05:28:54.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:54.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:54.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:28:54.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[191227]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T05:28:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[191227]: Reconfiguring daemon osd.7 on smithi191 2023-10-09T05:28:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:54.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.walafj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:28:54.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:54 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:55 smithi008 ceph-mon[184418]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:55 smithi008 ceph-mon[184418]: Reconfiguring mds.foo.smithi191.walafj (monmap changed)... 2023-10-09T05:28:55.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:55 smithi008 ceph-mon[184418]: Reconfiguring daemon mds.foo.smithi191.walafj on smithi191 2023-10-09T05:28:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:55 smithi008 ceph-mon[191227]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:55 smithi008 ceph-mon[191227]: Reconfiguring mds.foo.smithi191.walafj (monmap changed)... 2023-10-09T05:28:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:55 smithi008 ceph-mon[191227]: Reconfiguring daemon mds.foo.smithi191.walafj on smithi191 2023-10-09T05:28:55.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:55 smithi191 ceph-mon[161669]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:55.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:55 smithi191 ceph-mon[161669]: Reconfiguring mds.foo.smithi191.walafj (monmap changed)... 2023-10-09T05:28:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:55 smithi191 ceph-mon[161669]: Reconfiguring daemon mds.foo.smithi191.walafj on smithi191 2023-10-09T05:28:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:56.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:56.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:56.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:28:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:28:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:28:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:28:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:57 smithi191 ceph-mon[161669]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:57.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:57 smithi008 ceph-mon[184418]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:57 smithi008 ceph-mon[191227]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:59.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:28:58 smithi191 conmon[151041]: ts=2023-10-09T05:28:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:28:59.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:59 smithi191 ceph-mon[161669]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:28:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:59 smithi008 ceph-mon[184418]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:28:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:28:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:59 smithi008 ceph-mon[191227]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:28:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:28:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:00.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:29:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:29:00] "GET /metrics HTTP/1.1" 200 33904 "" "Prometheus/2.33.4" 2023-10-09T05:29:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:01 smithi191 ceph-mon[161669]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:01.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:01 smithi008 ceph-mon[184418]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:01.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:01 smithi008 ceph-mon[191227]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:03.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:03 smithi008 ceph-mon[184418]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:03.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:03 smithi008 ceph-mon[191227]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:03.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:03 smithi191 ceph-mon[161669]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:05.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:05 smithi008 ceph-mon[184418]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:05.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:05 smithi008 ceph-mon[191227]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:05.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:05 smithi191 ceph-mon[161669]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:07.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:07 smithi008 ceph-mon[184418]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:07.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:07 smithi008 ceph-mon[191227]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:07.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:07 smithi191 ceph-mon[161669]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:09.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 05:29:08 smithi191 conmon[151041]: ts=2023-10-09T05:29:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi191.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.8\", device_class=\"ssd\", hostname=\"smithi008\", instance=\"smithi008:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.8\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:29:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:09 smithi008 ceph-mon[184418]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:09 smithi008 ceph-mon[191227]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:09 smithi191 ceph-mon[161669]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:10.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:29:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:29:10] "GET /metrics HTTP/1.1" 200 33904 "" "Prometheus/2.33.4" 2023-10-09T05:29:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:11 smithi008 ceph-mon[184418]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:11 smithi008 ceph-mon[191227]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:11 smithi191 ceph-mon[161669]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:13.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:13 smithi008 ceph-mon[184418]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:13 smithi008 ceph-mon[191227]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:13.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:13 smithi191 ceph-mon[161669]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:14.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:14.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:14.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:15.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:15 smithi008 ceph-mon[184418]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:15.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:15 smithi008 ceph-mon[191227]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:15.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:15 smithi191 ceph-mon[161669]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:17.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:17 smithi008 ceph-mon[184418]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:17.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:17 smithi008 ceph-mon[191227]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:17 smithi191 ceph-mon[161669]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:19 smithi008 ceph-mon[184418]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:19.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:19 smithi008 ceph-mon[191227]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:19.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:19 smithi191 ceph-mon[161669]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:20.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:29:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:29:20] "GET /metrics HTTP/1.1" 200 33904 "" "Prometheus/2.33.4" 2023-10-09T05:29:21.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:21 smithi008 ceph-mon[184418]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:21 smithi008 ceph-mon[191227]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:21 smithi191 ceph-mon[161669]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:23.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:23 smithi008 ceph-mon[184418]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:23.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:23 smithi008 ceph-mon[191227]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:23.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:23 smithi191 ceph-mon[161669]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:25.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:25 smithi008 ceph-mon[184418]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:25 smithi008 ceph-mon[191227]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:25.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:25 smithi191 ceph-mon[161669]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:27.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:27 smithi008 ceph-mon[184418]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:27.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:27 smithi008 ceph-mon[191227]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:27 smithi191 ceph-mon[161669]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:29 smithi008 ceph-mon[184418]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:29.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:29 smithi008 ceph-mon[191227]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:29.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:29 smithi191 ceph-mon[161669]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:30.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:29:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:29:30] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.33.4" 2023-10-09T05:29:31.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:31 smithi008 ceph-mon[184418]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:31 smithi008 ceph-mon[191227]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:31 smithi191 ceph-mon[161669]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:33.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:33 smithi008 ceph-mon[184418]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:33.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:33 smithi008 ceph-mon[191227]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:33 smithi191 ceph-mon[161669]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:35.533 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T05:29:35.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:35 smithi191 ceph-mon[161669]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:35.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:35 smithi008 ceph-mon[184418]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:35.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:35 smithi008 ceph-mon[191227]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:37.700 INFO:teuthology.orchestra.run.smithi008.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:29:37.726 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:37 smithi008 ceph-mon[184418]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:37.726 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:37 smithi008 ceph-mon[191227]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:37.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:37 smithi191 ceph-mon[161669]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:38.302 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T05:29:38.702 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:38 smithi008 ceph-mon[191227]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:29:38.702 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:38 smithi008 ceph-mon[191227]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:29:38.702 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:38 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:38.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:38 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:38.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:38 smithi008 ceph-mon[191227]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:38 smithi008 ceph-mon[184418]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:29:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:38 smithi008 ceph-mon[184418]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:29:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:38 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:38 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:38 smithi008 ceph-mon[184418]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:39.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:38 smithi191 ceph-mon[161669]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:29:39.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:38 smithi191 ceph-mon[161669]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:29:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:38 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:38 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:38 smithi191 ceph-mon[161669]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:40.501 INFO:teuthology.orchestra.run.smithi008.stdout:wait for half of osds to upgrade 2023-10-09T05:29:40.644 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:29:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:29:40] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.33.4" 2023-10-09T05:29:40.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:40 smithi008 ceph-mon[184418]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:40.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:40.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:40.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:40 smithi008 ceph-mon[191227]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:40.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:40.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:41.064 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T05:29:41.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:40 smithi191 ceph-mon[161669]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:29:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:29:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:29:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:29:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T05:29:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T05:29:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T05:29:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T05:29:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T05:29:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T05:29:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:29:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:29:43.585 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:43.585 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.585 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.585 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:29:43.585 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:29:43.585 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.585 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:43.585 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:29:43.585 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:29:43.585 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.585 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T05:29:43.586 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T05:29:43.586 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T05:29:43.586 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T05:29:43.586 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T05:29:43.586 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T05:29:43.586 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:29:43.586 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.586 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:29:43.586 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T05:29:43.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T05:29:43.588 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T05:29:43.588 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T05:29:43.588 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:29:43.588 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:43.588 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:29:44.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:44 smithi191 ceph-mon[161669]: Upgrade: Setting container_image for all mon 2023-10-09T05:29:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:44 smithi191 ceph-mon[161669]: Upgrade: Setting container_image for all crash 2023-10-09T05:29:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:44 smithi191 ceph-mon[161669]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:29:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:44 smithi191 ceph-mon[161669]: Upgrade: osd.0 is safe to restart 2023-10-09T05:29:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1400620927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[184418]: Upgrade: Setting container_image for all mon 2023-10-09T05:29:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[184418]: Upgrade: Setting container_image for all crash 2023-10-09T05:29:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[184418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:29:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[184418]: Upgrade: osd.0 is safe to restart 2023-10-09T05:29:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1400620927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[191227]: Upgrade: Setting container_image for all mon 2023-10-09T05:29:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[191227]: Upgrade: Setting container_image for all crash 2023-10-09T05:29:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[191227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:29:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[191227]: Upgrade: osd.0 is safe to restart 2023-10-09T05:29:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1400620927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:29:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:45 smithi191 ceph-mon[161669]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:29:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:29:45.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:45 smithi008 ceph-mon[184418]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:29:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:29:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:45 smithi008 ceph-mon[191227]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:29:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:29:46.446 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:46 smithi008 ceph-mon[184418]: Upgrade: Updating osd.0 2023-10-09T05:29:46.446 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:46 smithi008 ceph-mon[184418]: Deploying daemon osd.0 on smithi008 2023-10-09T05:29:46.447 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3952975974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:46.447 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:46 smithi008 ceph-mon[191227]: Upgrade: Updating osd.0 2023-10-09T05:29:46.447 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:46 smithi008 ceph-mon[191227]: Deploying daemon osd.0 on smithi008 2023-10-09T05:29:46.447 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3952975974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:46 smithi191 ceph-mon[161669]: Upgrade: Updating osd.0 2023-10-09T05:29:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:46 smithi191 ceph-mon[161669]: Deploying daemon osd.0 on smithi008 2023-10-09T05:29:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3952975974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:47 smithi191 ceph-mon[161669]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:47.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:47 smithi008 ceph-mon[184418]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:47.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:47 smithi008 ceph-mon[191227]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:47.715 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:47 smithi008 systemd[1]: Stopping Ceph osd.0 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:29:47.715 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:47 smithi008 conmon[115909]: debug 2023-10-09T05:29:47.514+0000 7fd2e9510700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:29:47.715 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:47 smithi008 conmon[115909]: debug 2023-10-09T05:29:47.514+0000 7fd2e9510700 -1 osd.0 84 *** Got signal Terminated *** 2023-10-09T05:29:47.715 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:47 smithi008 conmon[115909]: debug 2023-10-09T05:29:47.515+0000 7fd2e9510700 -1 osd.0 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:29:48.695 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:48 smithi008 bash[202668]: 70d3660d55a1456aa55cc4f05c9ce58d2e21aef5e4aa381990c46a09c4dbe8f5 2023-10-09T05:29:48.695 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:48 smithi008 ceph-mon[184418]: osd.0 marked itself down and dead 2023-10-09T05:29:48.696 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:48 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1872093032' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:48.696 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:48 smithi008 ceph-mon[191227]: osd.0 marked itself down and dead 2023-10-09T05:29:48.696 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:48 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1872093032' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:48.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:48 smithi191 ceph-mon[161669]: osd.0 marked itself down and dead 2023-10-09T05:29:48.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:48 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1872093032' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:49.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:49 smithi008 ceph-mon[184418]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:49.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:49 smithi008 ceph-mon[184418]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:29:49.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:49 smithi008 ceph-mon[184418]: failed to encode map e85 with expected crc 2023-10-09T05:29:49.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:49 smithi008 ceph-mon[184418]: failed to encode map e85 with expected crc 2023-10-09T05:29:49.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:49 smithi008 ceph-mon[184418]: osdmap e85: 8 total, 7 up, 8 in 2023-10-09T05:29:49.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:49 smithi008 ceph-mon[191227]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:49.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:49 smithi008 ceph-mon[191227]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:29:49.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:49 smithi008 ceph-mon[191227]: failed to encode map e85 with expected crc 2023-10-09T05:29:49.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:49 smithi008 ceph-mon[191227]: failed to encode map e85 with expected crc 2023-10-09T05:29:49.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:49 smithi008 ceph-mon[191227]: osdmap e85: 8 total, 7 up, 8 in 2023-10-09T05:29:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:49 smithi191 ceph-mon[161669]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:29:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:49 smithi191 ceph-mon[161669]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:29:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:49 smithi191 ceph-mon[161669]: failed to encode map e85 with expected crc 2023-10-09T05:29:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:49 smithi191 ceph-mon[161669]: failed to encode map e85 with expected crc 2023-10-09T05:29:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:49 smithi191 ceph-mon[161669]: osdmap e85: 8 total, 7 up, 8 in 2023-10-09T05:29:50.714 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:50 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T05:29:50.714 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:50 smithi008 systemd[1]: Stopped Ceph osd.0 for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:29:50.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:29:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:29:50] "GET /metrics HTTP/1.1" 200 33905 "" "Prometheus/2.33.4" 2023-10-09T05:29:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[184418]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[184418]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[184418]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T05:29:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[184418]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[184418]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[184418]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[184418]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[184418]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[184418]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[184418]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1954508507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:50.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[191227]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[191227]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[191227]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T05:29:50.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[191227]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[191227]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[191227]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[191227]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[191227]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[191227]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[191227]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1954508507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:50.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:50 smithi191 ceph-mon[161669]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:50 smithi191 ceph-mon[161669]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:50 smithi191 ceph-mon[161669]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T05:29:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:50 smithi191 ceph-mon[161669]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:50 smithi191 ceph-mon[161669]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:50 smithi191 ceph-mon[161669]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:50 smithi191 ceph-mon[161669]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:50 smithi191 ceph-mon[161669]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:50 smithi191 ceph-mon[161669]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:50 smithi191 ceph-mon[161669]: failed to encode map e86 with expected crc 2023-10-09T05:29:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1954508507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:51.199 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:50 smithi008 systemd[1]: Starting Ceph osd.0 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:29:51.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:51 smithi008 ceph-mon[184418]: pgmap v128: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:51.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:51 smithi008 ceph-mon[184418]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:29:51.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:51 smithi008 ceph-mon[191227]: pgmap v128: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:51.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:51 smithi008 ceph-mon[191227]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:29:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:51 smithi191 ceph-mon[161669]: pgmap v128: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:51 smithi191 ceph-mon[161669]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:29:51.964 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:51 smithi008 bash[203365]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:29:51.964 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:51 smithi008 bash[203365]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T05:29:51.964 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:51 smithi008 bash[203365]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T05:29:51.964 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:51 smithi008 bash[203365]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T05:29:51.965 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:51 smithi008 bash[203365]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T05:29:51.965 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:51 smithi008 bash[203365]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T05:29:51.965 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:51 smithi008 bash[203365]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T05:29:51.965 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:51 smithi008 bash[203365]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T05:29:53.215 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:52 smithi008 ceph-osd[203864]: -- 172.21.15.8:0/199301634 <== mon.0 v2:172.21.15.8:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56117cdba4e0 con 0x56117cd8d400 2023-10-09T05:29:53.215 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:52 smithi008 bash[203796]: 288560b20015cf290be5de477400f55b8ae6c80106ed4bb67ea16a85eba03bf8 2023-10-09T05:29:53.215 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:52 smithi008 systemd[1]: Started Ceph osd.0 for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:29:53.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:53 smithi008 ceph-mon[184418]: pgmap v129: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:53.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/31347726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:53.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:53.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:53.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:53 smithi008 ceph-mon[191227]: pgmap v129: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/31347726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:53.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:53.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:53.716 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:53 smithi008 conmon[203840]: 2023-10-09T05:29:53.407+0000 7fbbce0cc640 -1 Falling back to public interface 2023-10-09T05:29:53.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:53 smithi191 ceph-mon[161669]: pgmap v129: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/31347726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:29:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:55 smithi008 ceph-mon[184418]: pgmap v130: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:55 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4065793321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:55 smithi008 ceph-mon[191227]: pgmap v130: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:55 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4065793321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:55 smithi191 ceph-mon[161669]: pgmap v130: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:55 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4065793321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:56.964 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:56 smithi008 conmon[203840]: 2023-10-09T05:29:56.634+0000 7fbbce0cc640 -1 osd.0 84 log_to_monitors true 2023-10-09T05:29:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[184418]: pgmap v131: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:56.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:56.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[184418]: from='osd.0 [v2:172.21.15.8:6802/633445692,v1:172.21.15.8:6803/633445692]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:29:56.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[184418]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:29:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[191227]: pgmap v131: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:56.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:56.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:56.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[191227]: from='osd.0 [v2:172.21.15.8:6802/633445692,v1:172.21.15.8:6803/633445692]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:29:56.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:56 smithi008 ceph-mon[191227]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:29:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:56 smithi191 ceph-mon[161669]: pgmap v131: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:29:57.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:56 smithi191 ceph-mon[161669]: from='osd.0 [v2:172.21.15.8:6802/633445692,v1:172.21.15.8:6803/633445692]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:29:57.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:56 smithi191 ceph-mon[161669]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:29:57.714 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 05:29:57 smithi008 conmon[203840]: 2023-10-09T05:29:57.475+0000 7fbbbb530700 -1 osd.0 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:29:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:57 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1781156841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:57 smithi191 ceph-mon[161669]: failed to encode map e87 with expected crc 2023-10-09T05:29:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:57 smithi191 ceph-mon[161669]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:29:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:57 smithi191 ceph-mon[161669]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T05:29:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:57 smithi191 ceph-mon[161669]: from='osd.0 [v2:172.21.15.8:6802/633445692,v1:172.21.15.8:6803/633445692]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:29:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:57 smithi191 ceph-mon[161669]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:29:58.153 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1781156841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:58.153 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[184418]: failed to encode map e87 with expected crc 2023-10-09T05:29:58.153 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[184418]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:29:58.153 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[184418]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T05:29:58.153 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[184418]: from='osd.0 [v2:172.21.15.8:6802/633445692,v1:172.21.15.8:6803/633445692]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:29:58.153 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[184418]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:29:58.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1781156841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:29:58.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[191227]: failed to encode map e87 with expected crc 2023-10-09T05:29:58.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[191227]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:29:58.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[191227]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T05:29:58.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[191227]: from='osd.0 [v2:172.21.15.8:6802/633445692,v1:172.21.15.8:6803/633445692]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:29:58.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:57 smithi008 ceph-mon[191227]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:29:58.992 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[184418]: failed to encode map e87 with expected crc 2023-10-09T05:29:58.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[184418]: pgmap v133: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:58.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[184418]: failed to encode map e87 with expected crc 2023-10-09T05:29:58.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[184418]: failed to encode map e87 with expected crc 2023-10-09T05:29:58.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[184418]: failed to encode map e87 with expected crc 2023-10-09T05:29:58.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[184418]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:29:58.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[184418]: failed to encode map e88 with expected crc 2023-10-09T05:29:58.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[184418]: failed to encode map e88 with expected crc 2023-10-09T05:29:58.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[184418]: osd.0 [v2:172.21.15.8:6802/633445692,v1:172.21.15.8:6803/633445692] boot 2023-10-09T05:29:58.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[184418]: osdmap e88: 8 total, 8 up, 8 in 2023-10-09T05:29:58.993 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:29:58.994 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[191227]: failed to encode map e87 with expected crc 2023-10-09T05:29:58.994 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[191227]: pgmap v133: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:58.994 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[191227]: failed to encode map e87 with expected crc 2023-10-09T05:29:58.994 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[191227]: failed to encode map e87 with expected crc 2023-10-09T05:29:58.994 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[191227]: failed to encode map e87 with expected crc 2023-10-09T05:29:58.994 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[191227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:29:58.994 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[191227]: failed to encode map e88 with expected crc 2023-10-09T05:29:58.994 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[191227]: failed to encode map e88 with expected crc 2023-10-09T05:29:58.994 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[191227]: osd.0 [v2:172.21.15.8:6802/633445692,v1:172.21.15.8:6803/633445692] boot 2023-10-09T05:29:58.994 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[191227]: osdmap e88: 8 total, 8 up, 8 in 2023-10-09T05:29:58.994 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:58 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:29:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:58 smithi191 ceph-mon[161669]: failed to encode map e87 with expected crc 2023-10-09T05:29:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:58 smithi191 ceph-mon[161669]: pgmap v133: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-10-09T05:29:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:58 smithi191 ceph-mon[161669]: failed to encode map e87 with expected crc 2023-10-09T05:29:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:58 smithi191 ceph-mon[161669]: failed to encode map e87 with expected crc 2023-10-09T05:29:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:58 smithi191 ceph-mon[161669]: failed to encode map e87 with expected crc 2023-10-09T05:29:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:58 smithi191 ceph-mon[161669]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:29:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:58 smithi191 ceph-mon[161669]: failed to encode map e88 with expected crc 2023-10-09T05:29:59.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:58 smithi191 ceph-mon[161669]: failed to encode map e88 with expected crc 2023-10-09T05:29:59.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:58 smithi191 ceph-mon[161669]: osd.0 [v2:172.21.15.8:6802/633445692,v1:172.21.15.8:6803/633445692] boot 2023-10-09T05:29:59.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:58 smithi191 ceph-mon[161669]: osdmap e88: 8 total, 8 up, 8 in 2023-10-09T05:29:59.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:58 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:29:59.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:29:59 smithi008 conmon[177229]: 2023-10-09T05:29:59.641+0000 7f020285b700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:30:00.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T05:30:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:00.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:30:00.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:00.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:00.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:00.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:00.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:00.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/819596197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:30:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:00.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:29:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/819596197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:00.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T05:30:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: failed to encode map e89 with expected crc 2023-10-09T05:30:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:00.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:30:00.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:00.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:00.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:00.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:00.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:00.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:00.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:29:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/819596197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:00.849 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:30:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:30:00] "GET /metrics HTTP/1.1" 200 33823 "" "Prometheus/2.33.4" 2023-10-09T05:30:01.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: failed to encode map e89 with expected crc 2023-10-09T05:30:01.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: failed to encode map e89 with expected crc 2023-10-09T05:30:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: failed to encode map e89 with expected crc 2023-10-09T05:30:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:30:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:30:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: Health detail: HEALTH_WARN Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded 2023-10-09T05:30:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: [WRN] PG_DEGRADED: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded 2023-10-09T05:30:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: pg 1.0 is active+undersized+degraded, acting [7,6] 2023-10-09T05:30:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: pg 2.4 is active+undersized+degraded, acting [1,7] 2023-10-09T05:30:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: pg 3.6 is active+undersized+degraded, acting [1,4] 2023-10-09T05:30:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: pg 3.7 is active+undersized+degraded, acting [3,7] 2023-10-09T05:30:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: pg 3.b is active+undersized+degraded, acting [3,4] 2023-10-09T05:30:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: pg 3.f is active+undersized+degraded, acting [7,4] 2023-10-09T05:30:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: pg 3.10 is active+undersized+degraded, acting [6,5] 2023-10-09T05:30:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: pg 3.1f is active+undersized+degraded, acting [5,2] 2023-10-09T05:30:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:30:01.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: failed to encode map e89 with expected crc 2023-10-09T05:30:01.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: failed to encode map e89 with expected crc 2023-10-09T05:30:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: failed to encode map e89 with expected crc 2023-10-09T05:30:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:30:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:30:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: Health detail: HEALTH_WARN Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded 2023-10-09T05:30:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: [WRN] PG_DEGRADED: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded 2023-10-09T05:30:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: pg 1.0 is active+undersized+degraded, acting [7,6] 2023-10-09T05:30:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: pg 2.4 is active+undersized+degraded, acting [1,7] 2023-10-09T05:30:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: pg 3.6 is active+undersized+degraded, acting [1,4] 2023-10-09T05:30:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: pg 3.7 is active+undersized+degraded, acting [3,7] 2023-10-09T05:30:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: pg 3.b is active+undersized+degraded, acting [3,4] 2023-10-09T05:30:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: pg 3.f is active+undersized+degraded, acting [7,4] 2023-10-09T05:30:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: pg 3.10 is active+undersized+degraded, acting [6,5] 2023-10-09T05:30:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: pg 3.1f is active+undersized+degraded, acting [5,2] 2023-10-09T05:30:01.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:00 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:30:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: failed to encode map e89 with expected crc 2023-10-09T05:30:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: failed to encode map e89 with expected crc 2023-10-09T05:30:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: failed to encode map e89 with expected crc 2023-10-09T05:30:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:30:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:30:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: Health detail: HEALTH_WARN Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded 2023-10-09T05:30:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: [WRN] PG_DEGRADED: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded 2023-10-09T05:30:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: pg 1.0 is active+undersized+degraded, acting [7,6] 2023-10-09T05:30:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: pg 2.4 is active+undersized+degraded, acting [1,7] 2023-10-09T05:30:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: pg 3.6 is active+undersized+degraded, acting [1,4] 2023-10-09T05:30:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: pg 3.7 is active+undersized+degraded, acting [3,7] 2023-10-09T05:30:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: pg 3.b is active+undersized+degraded, acting [3,4] 2023-10-09T05:30:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: pg 3.f is active+undersized+degraded, acting [7,4] 2023-10-09T05:30:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: pg 3.10 is active+undersized+degraded, acting [6,5] 2023-10-09T05:30:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: pg 3.1f is active+undersized+degraded, acting [5,2] 2023-10-09T05:30:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:00 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:30:03.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:02 smithi008 ceph-mon[184418]: pgmap v137: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:30:03.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:02 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1708845230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:03.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:02 smithi008 ceph-mon[191227]: pgmap v137: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:30:03.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:02 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1708845230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:03.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:02 smithi191 ceph-mon[161669]: pgmap v137: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:30:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:02 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1708845230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:04.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:03 smithi008 ceph-mon[184418]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-09T05:30:04.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:03 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:30:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:03 smithi008 ceph-mon[191227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-09T05:30:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:03 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:30:04.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:03 smithi191 ceph-mon[161669]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-09T05:30:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:03 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:30:05.167 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:04 smithi191 ceph-mon[161669]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:05.168 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2580811173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:05.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:04 smithi008 ceph-mon[184418]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:05.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2580811173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:04 smithi008 ceph-mon[191227]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2580811173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:07.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:06 smithi008 ceph-mon[184418]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2354680294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:06 smithi008 ceph-mon[191227]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2354680294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:06 smithi191 ceph-mon[161669]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2354680294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:09 smithi008 ceph-mon[184418]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:09.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:09 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:09.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:09 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1629912700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:09 smithi008 ceph-mon[191227]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:09 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:09 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1629912700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:09 smithi191 ceph-mon[161669]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:09 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:09 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1629912700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:10.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:30:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:30:10] "GET /metrics HTTP/1.1" 200 33823 "" "Prometheus/2.33.4" 2023-10-09T05:30:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:11 smithi008 ceph-mon[184418]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1740333579' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:11 smithi008 ceph-mon[191227]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1740333579' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:11 smithi191 ceph-mon[161669]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1740333579' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:13.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:13 smithi008 ceph-mon[184418]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:13.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:13 smithi008 ceph-mon[191227]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:13.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:13 smithi191 ceph-mon[161669]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:14.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:14 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4161064949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:14.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:14.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:14.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:14 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4161064949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:14.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:14.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:14 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4161064949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:15.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:15 smithi008 ceph-mon[184418]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:15.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:15 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:15.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:15 smithi008 ceph-mon[191227]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:15.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:15 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:15 smithi191 ceph-mon[161669]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:15 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:16.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[184418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:16.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[184418]: Upgrade: osd.1 is safe to restart 2023-10-09T05:30:16.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1828252080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:16.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:16.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:30:16.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:16.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[191227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:16.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[191227]: Upgrade: osd.1 is safe to restart 2023-10-09T05:30:16.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1828252080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:16.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:16.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:30:16.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:16 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:16.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:16 smithi191 ceph-mon[161669]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:30:16.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:16 smithi191 ceph-mon[161669]: Upgrade: osd.1 is safe to restart 2023-10-09T05:30:16.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:16 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1828252080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:16.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:16 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:16.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:16 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:30:16.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:16 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:17.584 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:17 smithi008 ceph-mon[184418]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:17.584 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:17 smithi008 ceph-mon[184418]: Upgrade: Updating osd.1 2023-10-09T05:30:17.584 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:17 smithi008 ceph-mon[184418]: Deploying daemon osd.1 on smithi008 2023-10-09T05:30:17.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:17 smithi008 ceph-mon[191227]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:17.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:17 smithi008 ceph-mon[191227]: Upgrade: Updating osd.1 2023-10-09T05:30:17.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:17 smithi008 ceph-mon[191227]: Deploying daemon osd.1 on smithi008 2023-10-09T05:30:17.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:17 smithi191 ceph-mon[161669]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:17 smithi191 ceph-mon[161669]: Upgrade: Updating osd.1 2023-10-09T05:30:17.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:17 smithi191 ceph-mon[161669]: Deploying daemon osd.1 on smithi008 2023-10-09T05:30:18.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/599931695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/599931695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:18.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/599931695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:19.104 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:18 smithi008 systemd[1]: Stopping Ceph osd.1 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:30:19.457 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:19 smithi008 conmon[121813]: debug 2023-10-09T05:30:19.102+0000 7f04552d8700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:30:19.457 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:19 smithi008 conmon[121813]: debug 2023-10-09T05:30:19.102+0000 7f04552d8700 -1 osd.1 89 *** Got signal Terminated *** 2023-10-09T05:30:19.458 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:19 smithi008 conmon[121813]: debug 2023-10-09T05:30:19.102+0000 7f04552d8700 -1 osd.1 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:30:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:19 smithi008 ceph-mon[184418]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:19 smithi008 ceph-mon[184418]: osd.1 marked itself down and dead 2023-10-09T05:30:19.715 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:19 smithi008 bash[207573]: 312729dec19af10210143beedf49db2e3202a40aac1bfb93389492aa36d94353 2023-10-09T05:30:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:19 smithi008 ceph-mon[191227]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:19 smithi008 ceph-mon[191227]: osd.1 marked itself down and dead 2023-10-09T05:30:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:19 smithi191 ceph-mon[161669]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:19 smithi191 ceph-mon[161669]: osd.1 marked itself down and dead 2023-10-09T05:30:20.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:20 smithi008 ceph-mon[184418]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:30:20.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:20 smithi008 ceph-mon[184418]: failed to encode map e90 with expected crc 2023-10-09T05:30:20.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:20 smithi008 ceph-mon[184418]: failed to encode map e90 with expected crc 2023-10-09T05:30:20.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:20 smithi008 ceph-mon[184418]: osdmap e90: 8 total, 7 up, 8 in 2023-10-09T05:30:20.716 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:30:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:30:20] "GET /metrics HTTP/1.1" 200 33895 "" "Prometheus/2.33.4" 2023-10-09T05:30:20.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:20 smithi008 ceph-mon[191227]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:30:20.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:20 smithi008 ceph-mon[191227]: failed to encode map e90 with expected crc 2023-10-09T05:30:20.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:20 smithi008 ceph-mon[191227]: failed to encode map e90 with expected crc 2023-10-09T05:30:20.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:20 smithi008 ceph-mon[191227]: osdmap e90: 8 total, 7 up, 8 in 2023-10-09T05:30:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:20 smithi191 ceph-mon[161669]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:30:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:20 smithi191 ceph-mon[161669]: failed to encode map e90 with expected crc 2023-10-09T05:30:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:20 smithi191 ceph-mon[161669]: failed to encode map e90 with expected crc 2023-10-09T05:30:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:20 smithi191 ceph-mon[161669]: osdmap e90: 8 total, 7 up, 8 in 2023-10-09T05:30:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: pgmap v147: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T05:30:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T05:30:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2435377133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: pgmap v147: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T05:30:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T05:30:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:21 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2435377133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: pgmap v147: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 6 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T05:30:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T05:30:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: failed to encode map e91 with expected crc 2023-10-09T05:30:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:21 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2435377133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:22.216 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:21 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T05:30:22.216 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:21 smithi008 systemd[1]: Stopped Ceph osd.1 for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:30:22.216 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:21 smithi008 systemd[1]: Starting Ceph osd.1 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:30:23.214 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:22 smithi008 bash[208245]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:30:23.215 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:22 smithi008 bash[208245]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T05:30:23.215 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:22 smithi008 bash[208245]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T05:30:23.215 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:22 smithi008 bash[208245]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T05:30:23.215 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:22 smithi008 bash[208245]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T05:30:23.215 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:22 smithi008 bash[208245]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T05:30:23.215 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:22 smithi008 bash[208245]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T05:30:23.215 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:22 smithi008 bash[208245]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T05:30:23.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:23 smithi008 ceph-mon[191227]: pgmap v149: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:23.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1115191391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:23 smithi008 ceph-mon[184418]: pgmap v149: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1115191391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:23.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:23 smithi191 ceph-mon[161669]: pgmap v149: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1115191391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:24.464 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:24 smithi008 ceph-osd[208736]: -- 172.21.15.8:0/3666514289 <== mon.2 v2:172.21.15.8:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5575aa1644e0 con 0x5575aa137000 2023-10-09T05:30:24.464 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:24 smithi008 bash[208671]: 086a4a5d61a57bc68de83968d6d5a43d356dbd5d37239172fbf9a9bfb9f5e1e6 2023-10-09T05:30:24.464 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:24 smithi008 systemd[1]: Started Ceph osd.1 for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:30:24.964 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:24 smithi008 conmon[208712]: 2023-10-09T05:30:24.522+0000 7f05ca774640 -1 Falling back to public interface 2023-10-09T05:30:25.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:25 smithi008 ceph-mon[191227]: pgmap v150: 73 pgs: 2 active+undersized+degraded, 28 peering, 2 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:25.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:25 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:25.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:25 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:25.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:25 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:25.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:25 smithi008 ceph-mon[191227]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:30:25.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:25 smithi008 ceph-mon[184418]: pgmap v150: 73 pgs: 2 active+undersized+degraded, 28 peering, 2 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:25.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:25 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:25.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:25 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:25.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:25 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:25.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:25 smithi008 ceph-mon[184418]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:30:25.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:25 smithi191 ceph-mon[161669]: pgmap v150: 73 pgs: 2 active+undersized+degraded, 28 peering, 2 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:25 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:25 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:25 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:25 smithi191 ceph-mon[161669]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:30:26.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1899316748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:26.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1899316748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1899316748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:27.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:27 smithi008 ceph-mon[184418]: pgmap v151: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T05:30:27.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:27 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2023-10-09T05:30:27.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:27.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:27.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:27 smithi008 ceph-mon[191227]: pgmap v151: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T05:30:27.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:27 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2023-10-09T05:30:27.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:27.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:27 smithi191 ceph-mon[161669]: pgmap v151: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T05:30:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:27 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 7 pgs peering) 2023-10-09T05:30:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:27.964 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:27 smithi008 conmon[208712]: 2023-10-09T05:30:27.516+0000 7f05ca774640 -1 osd.1 89 log_to_monitors true 2023-10-09T05:30:28.715 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 05:30:28 smithi008 conmon[208712]: 2023-10-09T05:30:28.466+0000 7f05b9bdc700 -1 osd.1 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:30:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:28 smithi008 ceph-mon[184418]: from='osd.1 [v2:172.21.15.8:6810/417852095,v1:172.21.15.8:6811/417852095]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:30:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:28 smithi008 ceph-mon[184418]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:30:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:28 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:28 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2635289536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:28.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:28 smithi008 ceph-mon[191227]: from='osd.1 [v2:172.21.15.8:6810/417852095,v1:172.21.15.8:6811/417852095]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:30:28.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:28 smithi008 ceph-mon[191227]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:30:28.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:28 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:28.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:28 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:28.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2635289536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:28.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:28 smithi191 ceph-mon[161669]: from='osd.1 [v2:172.21.15.8:6810/417852095,v1:172.21.15.8:6811/417852095]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:30:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:28 smithi191 ceph-mon[161669]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:30:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:28 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:28 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2635289536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[184418]: pgmap v152: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T05:30:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[184418]: failed to encode map e92 with expected crc 2023-10-09T05:30:29.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[184418]: failed to encode map e92 with expected crc 2023-10-09T05:30:29.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[184418]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:30:29.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[184418]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T05:30:29.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[184418]: from='osd.1 [v2:172.21.15.8:6810/417852095,v1:172.21.15.8:6811/417852095]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:30:29.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[184418]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:30:29.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:29.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[191227]: pgmap v152: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T05:30:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[191227]: failed to encode map e92 with expected crc 2023-10-09T05:30:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[191227]: failed to encode map e92 with expected crc 2023-10-09T05:30:29.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[191227]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:30:29.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[191227]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T05:30:29.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[191227]: from='osd.1 [v2:172.21.15.8:6810/417852095,v1:172.21.15.8:6811/417852095]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:30:29.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[191227]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:30:29.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:29.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:29 smithi191 ceph-mon[161669]: pgmap v152: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T05:30:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:29 smithi191 ceph-mon[161669]: failed to encode map e92 with expected crc 2023-10-09T05:30:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:29 smithi191 ceph-mon[161669]: failed to encode map e92 with expected crc 2023-10-09T05:30:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:29 smithi191 ceph-mon[161669]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:30:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:29 smithi191 ceph-mon[161669]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T05:30:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:29 smithi191 ceph-mon[161669]: from='osd.1 [v2:172.21.15.8:6810/417852095,v1:172.21.15.8:6811/417852095]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:30:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:29 smithi191 ceph-mon[161669]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:30:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:30 smithi191 ceph-mon[161669]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:30:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:30 smithi191 ceph-mon[161669]: failed to encode map e93 with expected crc 2023-10-09T05:30:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:30 smithi191 ceph-mon[161669]: osd.1 [v2:172.21.15.8:6810/417852095,v1:172.21.15.8:6811/417852095] boot 2023-10-09T05:30:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:30 smithi191 ceph-mon[161669]: osdmap e93: 8 total, 8 up, 8 in 2023-10-09T05:30:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:30 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:30:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1145887055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:30 smithi191 ceph-mon[161669]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T05:30:30.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[191227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:30:30.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[191227]: failed to encode map e93 with expected crc 2023-10-09T05:30:30.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[191227]: osd.1 [v2:172.21.15.8:6810/417852095,v1:172.21.15.8:6811/417852095] boot 2023-10-09T05:30:30.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[191227]: osdmap e93: 8 total, 8 up, 8 in 2023-10-09T05:30:30.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:30:30.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1145887055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:30.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[191227]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T05:30:30.816 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[184418]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:30:30.816 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[184418]: failed to encode map e93 with expected crc 2023-10-09T05:30:30.816 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[184418]: osd.1 [v2:172.21.15.8:6810/417852095,v1:172.21.15.8:6811/417852095] boot 2023-10-09T05:30:30.816 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[184418]: osdmap e93: 8 total, 8 up, 8 in 2023-10-09T05:30:30.816 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:30:30.816 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1145887055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:30.816 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:30 smithi008 ceph-mon[184418]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T05:30:30.816 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:30:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:30:30] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.33.4" 2023-10-09T05:30:30.816 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:30:30 smithi008 conmon[177229]: 2023-10-09T05:30:30.814+0000 7f020285b700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:30:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: pgmap v155: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: failed to encode map e94 with expected crc 2023-10-09T05:30:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: failed to encode map e94 with expected crc 2023-10-09T05:30:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: failed to encode map e94 with expected crc 2023-10-09T05:30:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: failed to encode map e94 with expected crc 2023-10-09T05:30:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: failed to encode map e94 with expected crc 2023-10-09T05:30:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: failed to encode map e94 with expected crc 2023-10-09T05:30:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: failed to encode map e94 with expected crc 2023-10-09T05:30:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: failed to encode map e94 with expected crc 2023-10-09T05:30:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: failed to encode map e94 with expected crc 2023-10-09T05:30:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: failed to encode map e94 with expected crc 2023-10-09T05:30:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: failed to encode map e94 with expected crc 2023-10-09T05:30:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:30:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:30:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: pgmap v155: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:32.094 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:32.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:32.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: pgmap v155: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:32.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: failed to encode map e94 with expected crc 2023-10-09T05:30:32.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:30:32.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:32.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:32.099 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:32.099 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:30:32.099 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:32.099 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:32.099 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:32.100 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:32.100 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:30:32.100 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:32 smithi191 ceph-mon[161669]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:32 smithi191 ceph-mon[161669]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:30:32.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:32 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1149317809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:32.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:32 smithi008 ceph-mon[184418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:32.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:32 smithi008 ceph-mon[184418]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:30:32.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:32 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1149317809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:32.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:32 smithi008 ceph-mon[191227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:32.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:32 smithi008 ceph-mon[191227]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:30:32.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:32 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1149317809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:33 smithi191 ceph-mon[161669]: pgmap v157: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:33 smithi008 ceph-mon[184418]: pgmap v157: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:33 smithi008 ceph-mon[191227]: pgmap v157: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:34.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:34 smithi191 ceph-mon[161669]: pgmap v158: 73 pgs: 20 peering, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:34 smithi191 ceph-mon[161669]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-09T05:30:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:34 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:30:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:34 smithi008 ceph-mon[184418]: pgmap v158: 73 pgs: 20 peering, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:34 smithi008 ceph-mon[184418]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-09T05:30:34.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:34 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:30:34.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:34 smithi008 ceph-mon[191227]: pgmap v158: 73 pgs: 20 peering, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:34.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:34 smithi008 ceph-mon[191227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-09T05:30:34.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:34 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:30:35.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2413095308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:35.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2413095308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:35.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2413095308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:36.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:36 smithi191 ceph-mon[161669]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:36.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:36 smithi008 ceph-mon[184418]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:36.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:36 smithi008 ceph-mon[191227]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:37.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:37 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/206551730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:37.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:37 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/206551730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:37.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:37 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/206551730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:38.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:38 smithi191 ceph-mon[161669]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:38 smithi008 ceph-mon[184418]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:38 smithi008 ceph-mon[191227]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:39.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3634334317' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:39.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3634334317' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:39.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3634334317' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:40.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:40 smithi191 ceph-mon[161669]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:40.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:40 smithi008 ceph-mon[184418]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:40.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:40 smithi008 ceph-mon[191227]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:40.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:30:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:30:40] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.33.4" 2023-10-09T05:30:42.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:41 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1964662257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:42.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:41 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1964662257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:42.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:41 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1964662257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:43.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:43 smithi008 ceph-mon[184418]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:43.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:43 smithi008 ceph-mon[191227]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:43 smithi191 ceph-mon[161669]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:44 smithi008 ceph-mon[184418]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2065765065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:44 smithi008 ceph-mon[191227]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2065765065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:45.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:44 smithi191 ceph-mon[161669]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2065765065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:46.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:46.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:46.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:46.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:46 smithi008 ceph-mon[191227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:46.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:46 smithi008 ceph-mon[191227]: Upgrade: osd.2 is safe to restart 2023-10-09T05:30:46.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:46 smithi008 ceph-mon[191227]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:46.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1504496348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:46.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:46 smithi008 ceph-mon[184418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:46.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:46 smithi008 ceph-mon[184418]: Upgrade: osd.2 is safe to restart 2023-10-09T05:30:46.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:46 smithi008 ceph-mon[184418]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:46.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1504496348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:46 smithi191 ceph-mon[161669]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:30:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:46 smithi191 ceph-mon[161669]: Upgrade: osd.2 is safe to restart 2023-10-09T05:30:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:46 smithi191 ceph-mon[161669]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1504496348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:48.633 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:48 smithi008 ceph-mon[184418]: Upgrade: Updating osd.2 2023-10-09T05:30:48.633 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:48.633 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:30:48.633 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:48.633 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:48 smithi008 ceph-mon[184418]: Deploying daemon osd.2 on smithi008 2023-10-09T05:30:48.633 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:48 smithi008 ceph-mon[191227]: Upgrade: Updating osd.2 2023-10-09T05:30:48.633 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:48.634 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:30:48.634 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:48.634 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:48 smithi008 ceph-mon[191227]: Deploying daemon osd.2 on smithi008 2023-10-09T05:30:48.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:48 smithi191 ceph-mon[161669]: Upgrade: Updating osd.2 2023-10-09T05:30:48.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:48.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:30:48.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:30:48.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:48 smithi191 ceph-mon[161669]: Deploying daemon osd.2 on smithi008 2023-10-09T05:30:49.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:49 smithi008 ceph-mon[184418]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:49.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1838698779' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:49.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:49 smithi008 ceph-mon[191227]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:49.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1838698779' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:49.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:49 smithi191 ceph-mon[161669]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1838698779' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:50.545 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:30:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:30:50] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.33.4" 2023-10-09T05:30:50.546 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:50 smithi008 systemd[1]: Stopping Ceph osd.2 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:30:50.964 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:50 smithi008 conmon[127626]: debug 2023-10-09T05:30:50.544+0000 7fb64ab69700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:30:50.964 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:50 smithi008 conmon[127626]: debug 2023-10-09T05:30:50.544+0000 7fb64ab69700 -1 osd.2 94 *** Got signal Terminated *** 2023-10-09T05:30:50.965 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:50 smithi008 conmon[127626]: debug 2023-10-09T05:30:50.544+0000 7fb64ab69700 -1 osd.2 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:30:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:51 smithi008 ceph-mon[184418]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:51 smithi008 ceph-mon[184418]: osd.2 marked itself down and dead 2023-10-09T05:30:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:51 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3405588915' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:51.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:51 smithi008 ceph-mon[191227]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:51.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:51 smithi008 ceph-mon[191227]: osd.2 marked itself down and dead 2023-10-09T05:30:51.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:51 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3405588915' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:51.716 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:51 smithi008 bash[212551]: 7c191218228341e43883740331cafee0d4ce5529633039b967ac8d37a7a2ed6e 2023-10-09T05:30:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:51 smithi191 ceph-mon[161669]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:51 smithi191 ceph-mon[161669]: osd.2 marked itself down and dead 2023-10-09T05:30:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:51 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3405588915' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:52 smithi008 ceph-mon[184418]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:30:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:52 smithi008 ceph-mon[184418]: failed to encode map e95 with expected crc 2023-10-09T05:30:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:52 smithi008 ceph-mon[184418]: failed to encode map e95 with expected crc 2023-10-09T05:30:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:52 smithi008 ceph-mon[184418]: osdmap e95: 8 total, 7 up, 8 in 2023-10-09T05:30:52.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:52 smithi008 ceph-mon[191227]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:30:52.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:52 smithi008 ceph-mon[191227]: failed to encode map e95 with expected crc 2023-10-09T05:30:52.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:52 smithi008 ceph-mon[191227]: failed to encode map e95 with expected crc 2023-10-09T05:30:52.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:52 smithi008 ceph-mon[191227]: osdmap e95: 8 total, 7 up, 8 in 2023-10-09T05:30:52.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:52 smithi191 ceph-mon[161669]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:30:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:52 smithi191 ceph-mon[161669]: failed to encode map e95 with expected crc 2023-10-09T05:30:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:52 smithi191 ceph-mon[161669]: failed to encode map e95 with expected crc 2023-10-09T05:30:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:52 smithi191 ceph-mon[161669]: osdmap e95: 8 total, 7 up, 8 in 2023-10-09T05:30:53.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[184418]: pgmap v168: 73 pgs: 2 stale+active+clean, 71 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:53.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[184418]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[184418]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[184418]: osdmap e96: 8 total, 7 up, 8 in 2023-10-09T05:30:53.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[184418]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[184418]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[184418]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[184418]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[184418]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3203004747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:53.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[191227]: pgmap v168: 73 pgs: 2 stale+active+clean, 71 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:53.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[191227]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[191227]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[191227]: osdmap e96: 8 total, 7 up, 8 in 2023-10-09T05:30:53.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[191227]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[191227]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[191227]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[191227]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[191227]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3203004747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:53 smithi191 ceph-mon[161669]: pgmap v168: 73 pgs: 2 stale+active+clean, 71 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:30:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:53 smithi191 ceph-mon[161669]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:53 smithi191 ceph-mon[161669]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:53 smithi191 ceph-mon[161669]: osdmap e96: 8 total, 7 up, 8 in 2023-10-09T05:30:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:53 smithi191 ceph-mon[161669]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:53 smithi191 ceph-mon[161669]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:53 smithi191 ceph-mon[161669]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:53 smithi191 ceph-mon[161669]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:53 smithi191 ceph-mon[161669]: failed to encode map e96 with expected crc 2023-10-09T05:30:53.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3203004747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:54.215 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:53 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T05:30:54.215 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:53 smithi008 systemd[1]: Stopped Ceph osd.2 for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:30:54.215 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:53 smithi008 systemd[1]: Starting Ceph osd.2 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:30:54.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:54 smithi008 ceph-mon[184418]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:30:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:54 smithi008 ceph-mon[191227]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:30:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:54 smithi191 ceph-mon[161669]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:30:55.215 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:54 smithi008 bash[213256]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:30:55.215 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:54 smithi008 bash[213256]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T05:30:55.215 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:54 smithi008 bash[213256]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T05:30:55.215 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:54 smithi008 bash[213256]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T05:30:55.215 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:54 smithi008 bash[213256]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T05:30:55.215 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:54 smithi008 bash[213256]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T05:30:55.215 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:54 smithi008 bash[213256]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T05:30:55.215 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:54 smithi008 bash[213256]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T05:30:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:55 smithi191 ceph-mon[161669]: pgmap v170: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 5 active+undersized, 66 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:55.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:55 smithi008 ceph-mon[184418]: pgmap v170: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 5 active+undersized, 66 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:55.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:55 smithi008 ceph-mon[191227]: pgmap v170: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 5 active+undersized, 66 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:30:56.465 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:56 smithi008 ceph-osd[213747]: -- 172.21.15.8:0/3412649288 <== mon.2 v2:172.21.15.8:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562d121c24e0 con 0x562d12194c00 2023-10-09T05:30:56.465 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:56 smithi008 bash[213682]: 5a87289cd2f00cf82084594b83a07ccfd8d376bc5e03d2af3c09c1458df9ff14 2023-10-09T05:30:56.465 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:56 smithi008 systemd[1]: Started Ceph osd.2 for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:30:56.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:56 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3694450389' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:56.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:56.965 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:56 smithi008 conmon[213723]: 2023-10-09T05:30:56.687+0000 7f2a95417640 -1 Falling back to public interface 2023-10-09T05:30:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:56 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3694450389' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:56 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3694450389' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:30:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:57 smithi191 ceph-mon[161669]: pgmap v171: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:30:57.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:57 smithi008 ceph-mon[184418]: pgmap v171: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:30:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:57 smithi008 ceph-mon[191227]: pgmap v171: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:30:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2949421876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:58.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2949421876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:58.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2949421876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:30:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:59 smithi191 ceph-mon[161669]: pgmap v172: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:30:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:30:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.964 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:30:59 smithi008 conmon[213723]: 2023-10-09T05:30:59.637+0000 7f2a95417640 -1 osd.2 94 log_to_monitors true 2023-10-09T05:30:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[184418]: pgmap v172: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:30:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[191227]: pgmap v172: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:30:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:30:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:30:59.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:30:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:00.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:00 smithi191 ceph-mon[161669]: from='osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:31:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:00 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:00 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2354777145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:00.965 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 05:31:00 smithi008 conmon[213723]: 2023-10-09T05:31:00.544+0000 7f2a8487f700 -1 osd.2 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:31:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:00 smithi008 ceph-mon[184418]: from='osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:31:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:00 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:00 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2354777145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:00.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:31:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:31:00] "GET /metrics HTTP/1.1" 200 34013 "" "Prometheus/2.33.4" 2023-10-09T05:31:00.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:00 smithi008 ceph-mon[191227]: from='osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:31:00.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:00 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:00.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:00 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:00.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2354777145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:01.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[184418]: pgmap v173: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:31:01.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[184418]: failed to encode map e97 with expected crc 2023-10-09T05:31:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[184418]: failed to encode map e97 with expected crc 2023-10-09T05:31:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[184418]: from='osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:31:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[184418]: osdmap e97: 8 total, 7 up, 8 in 2023-10-09T05:31:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[184418]: from='osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:31:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[191227]: pgmap v173: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:31:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[191227]: failed to encode map e97 with expected crc 2023-10-09T05:31:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[191227]: failed to encode map e97 with expected crc 2023-10-09T05:31:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[191227]: from='osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:31:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[191227]: osdmap e97: 8 total, 7 up, 8 in 2023-10-09T05:31:01.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:01 smithi008 ceph-mon[191227]: from='osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:31:01.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:01 smithi191 ceph-mon[161669]: pgmap v173: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:31:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:01 smithi191 ceph-mon[161669]: failed to encode map e97 with expected crc 2023-10-09T05:31:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:01 smithi191 ceph-mon[161669]: failed to encode map e97 with expected crc 2023-10-09T05:31:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:01 smithi191 ceph-mon[161669]: from='osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:31:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:01 smithi191 ceph-mon[161669]: osdmap e97: 8 total, 7 up, 8 in 2023-10-09T05:31:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:01 smithi191 ceph-mon[161669]: from='osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:31:02.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:02 smithi191 ceph-mon[161669]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:31:02.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:02 smithi191 ceph-mon[161669]: failed to encode map e98 with expected crc 2023-10-09T05:31:02.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:02 smithi191 ceph-mon[161669]: failed to encode map e98 with expected crc 2023-10-09T05:31:02.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:02 smithi191 ceph-mon[161669]: osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054] boot 2023-10-09T05:31:02.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:02 smithi191 ceph-mon[161669]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T05:31:02.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:02 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:31:02.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:02 smithi191 ceph-mon[161669]: pgmap v176: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:31:02.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:02 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:02.838 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[184418]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:31:02.838 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[184418]: failed to encode map e98 with expected crc 2023-10-09T05:31:02.839 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[184418]: failed to encode map e98 with expected crc 2023-10-09T05:31:02.839 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[184418]: osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054] boot 2023-10-09T05:31:02.839 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[184418]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T05:31:02.839 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:31:02.839 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[184418]: pgmap v176: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:31:02.839 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:02.839 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[191227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:31:02.839 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[191227]: failed to encode map e98 with expected crc 2023-10-09T05:31:02.839 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[191227]: failed to encode map e98 with expected crc 2023-10-09T05:31:02.839 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[191227]: osd.2 [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054] boot 2023-10-09T05:31:02.840 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[191227]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T05:31:02.840 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:31:02.840 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[191227]: pgmap v176: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T05:31:02.840 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:02 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:03.214 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:31:02 smithi008 conmon[177229]: 2023-10-09T05:31:02.836+0000 7f020285b700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T05:31:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T05:31:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3120498808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:03.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:03 smithi191 ceph-mon[161669]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T05:31:03.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T05:31:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3120498808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:03.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:03.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:03.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:03.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[191227]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T05:31:03.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T05:31:03.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: failed to encode map e99 with expected crc 2023-10-09T05:31:03.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3120498808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:03.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:03.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:03.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:03.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:03.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:03.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:03.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:03.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:03.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:03.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:03.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:03.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:03 smithi008 ceph-mon[184418]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T05:31:05.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:04 smithi008 ceph-mon[184418]: pgmap v178: 73 pgs: 2 peering, 3 active+undersized+degraded, 11 active+undersized, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:05.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:04 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:04 smithi008 ceph-mon[191227]: pgmap v178: 73 pgs: 2 peering, 3 active+undersized+degraded, 11 active+undersized, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:04 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:05.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:04 smithi191 ceph-mon[161669]: pgmap v178: 73 pgs: 2 peering, 3 active+undersized+degraded, 11 active+undersized, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:05.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:04 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:06.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:05 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1737851790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:05 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1737851790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:05 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1737851790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:07.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:06 smithi008 ceph-mon[184418]: pgmap v179: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:06 smithi008 ceph-mon[184418]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2023-10-09T05:31:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:06 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:31:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:06 smithi008 ceph-mon[191227]: pgmap v179: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:06 smithi008 ceph-mon[191227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2023-10-09T05:31:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:06 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:31:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:06 smithi191 ceph-mon[161669]: pgmap v179: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:06 smithi191 ceph-mon[161669]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2023-10-09T05:31:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:06 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:31:08.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:07 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/606357158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:08.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:07 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/606357158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:08.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:07 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/606357158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:09 smithi008 ceph-mon[184418]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:09 smithi008 ceph-mon[191227]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:09 smithi191 ceph-mon[161669]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:10.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:10 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4074021214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:10.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:31:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:31:10] "GET /metrics HTTP/1.1" 200 34013 "" "Prometheus/2.33.4" 2023-10-09T05:31:10.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:10 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4074021214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:10.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:10 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4074021214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:11 smithi008 ceph-mon[184418]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:11 smithi008 ceph-mon[191227]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:11.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:11 smithi191 ceph-mon[161669]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:12.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2116288033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:12.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2116288033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2116288033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:13.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:13 smithi008 ceph-mon[184418]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:13 smithi008 ceph-mon[191227]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:13.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:13 smithi191 ceph-mon[161669]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:14 smithi008 ceph-mon[184418]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:15.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:15.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:15.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:15.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:14 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/904566196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:14 smithi008 ceph-mon[191227]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:14 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/904566196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:14 smithi191 ceph-mon[161669]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:14 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/904566196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:16 smithi008 ceph-mon[184418]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:16 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3519120293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:16 smithi008 ceph-mon[191227]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:16 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3519120293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:17.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:16 smithi191 ceph-mon[161669]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:16 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3519120293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:18.637 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:18 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:18.638 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:18 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:18.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:18 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[184418]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[184418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[184418]: Upgrade: osd.3 is safe to restart 2023-10-09T05:31:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1114970189' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:31:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[191227]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[191227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[191227]: Upgrade: osd.3 is safe to restart 2023-10-09T05:31:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1114970189' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:31:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:19 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:19.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:19 smithi191 ceph-mon[161669]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:19 smithi191 ceph-mon[161669]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:31:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:19 smithi191 ceph-mon[161669]: Upgrade: osd.3 is safe to restart 2023-10-09T05:31:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:19 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1114970189' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:19 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:19 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:31:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:19 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:20.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:20 smithi008 ceph-mon[184418]: Upgrade: Updating osd.3 2023-10-09T05:31:20.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:20 smithi008 ceph-mon[184418]: Deploying daemon osd.3 on smithi008 2023-10-09T05:31:20.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:31:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:31:20] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.33.4" 2023-10-09T05:31:20.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:20 smithi008 ceph-mon[191227]: Upgrade: Updating osd.3 2023-10-09T05:31:20.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:20 smithi008 ceph-mon[191227]: Deploying daemon osd.3 on smithi008 2023-10-09T05:31:20.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:20 smithi191 ceph-mon[161669]: Upgrade: Updating osd.3 2023-10-09T05:31:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:20 smithi191 ceph-mon[161669]: Deploying daemon osd.3 on smithi008 2023-10-09T05:31:21.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:21 smithi008 ceph-mon[184418]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:21.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:21 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3593541432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:21 smithi008 ceph-mon[191227]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:21.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:21 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3593541432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:21 smithi191 ceph-mon[161669]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:21 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3593541432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:22.460 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:22 smithi008 systemd[1]: Stopping Ceph osd.3 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:31:22.460 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:22 smithi008 conmon[133577]: debug 2023-10-09T05:31:22.302+0000 7fd585012700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:31:22.460 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:22 smithi008 conmon[133577]: debug 2023-10-09T05:31:22.302+0000 7fd585012700 -1 osd.3 99 *** Got signal Terminated *** 2023-10-09T05:31:22.460 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:22 smithi008 conmon[133577]: debug 2023-10-09T05:31:22.302+0000 7fd585012700 -1 osd.3 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:31:22.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:22 smithi008 ceph-mon[184418]: osd.3 marked itself down and dead 2023-10-09T05:31:22.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:22 smithi008 ceph-mon[191227]: osd.3 marked itself down and dead 2023-10-09T05:31:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:22 smithi191 ceph-mon[161669]: osd.3 marked itself down and dead 2023-10-09T05:31:23.465 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:23 smithi008 bash[217535]: 9c33044d2771fd3e8544f6ec56cba836761f280c7bf68007630c5e8650c8d6a0 2023-10-09T05:31:23.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[184418]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[184418]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:31:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[184418]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[184418]: osdmap e100: 8 total, 7 up, 8 in 2023-10-09T05:31:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[184418]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[184418]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[184418]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[184418]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[184418]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[191227]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:23.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[191227]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:31:23.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[191227]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[191227]: osdmap e100: 8 total, 7 up, 8 in 2023-10-09T05:31:23.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[191227]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[191227]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[191227]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[191227]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:23 smithi008 ceph-mon[191227]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:23 smithi191 ceph-mon[161669]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:23 smithi191 ceph-mon[161669]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:31:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:23 smithi191 ceph-mon[161669]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:23 smithi191 ceph-mon[161669]: osdmap e100: 8 total, 7 up, 8 in 2023-10-09T05:31:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:23 smithi191 ceph-mon[161669]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:23 smithi191 ceph-mon[161669]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:23 smithi191 ceph-mon[161669]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:23 smithi191 ceph-mon[161669]: failed to encode map e100 with expected crc 2023-10-09T05:31:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:23 smithi191 ceph-mon[161669]: failed to encode map e100 with expected crc 2023-10-09T05:31:24.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[184418]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[184418]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[184418]: osdmap e101: 8 total, 7 up, 8 in 2023-10-09T05:31:24.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[184418]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[184418]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[184418]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[184418]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[184418]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1755613233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:24.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[191227]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[191227]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[191227]: osdmap e101: 8 total, 7 up, 8 in 2023-10-09T05:31:24.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[191227]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[191227]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[191227]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[191227]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[191227]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:24 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1755613233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:24 smithi191 ceph-mon[161669]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:24 smithi191 ceph-mon[161669]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:24 smithi191 ceph-mon[161669]: osdmap e101: 8 total, 7 up, 8 in 2023-10-09T05:31:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:24 smithi191 ceph-mon[161669]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:24 smithi191 ceph-mon[161669]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:24 smithi191 ceph-mon[161669]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:24 smithi191 ceph-mon[161669]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:24 smithi191 ceph-mon[161669]: failed to encode map e101 with expected crc 2023-10-09T05:31:24.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:24 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1755613233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:25.214 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:24 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T05:31:25.214 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:24 smithi008 systemd[1]: Stopped Ceph osd.3 for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:31:25.215 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:24 smithi008 systemd[1]: Starting Ceph osd.3 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:31:25.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:25 smithi008 ceph-mon[184418]: pgmap v190: 73 pgs: 13 peering, 7 stale+active+clean, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:25 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:31:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:25 smithi008 ceph-mon[191227]: pgmap v190: 73 pgs: 13 peering, 7 stale+active+clean, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:25 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:31:25.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:25 smithi191 ceph-mon[161669]: pgmap v190: 73 pgs: 13 peering, 7 stale+active+clean, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:25 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:31:26.214 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:25 smithi008 bash[218192]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:31:26.215 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:25 smithi008 bash[218192]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T05:31:26.215 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:25 smithi008 bash[218192]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T05:31:26.215 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:25 smithi008 bash[218192]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T05:31:26.215 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:25 smithi008 bash[218192]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T05:31:26.215 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:25 smithi008 bash[218192]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T05:31:26.215 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:25 smithi008 bash[218192]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T05:31:26.215 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:25 smithi008 bash[218192]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T05:31:26.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1770087867' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:26.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1770087867' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:26.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1770087867' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:27.464 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:27 smithi008 ceph-osd[218715]: -- 172.21.15.8:0/630077282 <== mon.0 v2:172.21.15.8:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d73a4c44e0 con 0x55d73a497000 2023-10-09T05:31:27.464 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:27 smithi008 bash[218649]: 8625f8d5125ed07dd80332db22cb6d0f24261fe49c267ff682faa79d0b44009b 2023-10-09T05:31:27.465 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:27 smithi008 systemd[1]: Started Ceph osd.3 for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:31:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:27 smithi191 ceph-mon[161669]: pgmap v191: 73 pgs: 2 active+undersized+degraded, 13 peering, 16 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:31:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:27 smithi191 ceph-mon[161669]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:27.964 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:27 smithi008 conmon[218691]: 2023-10-09T05:31:27.556+0000 7fbdd945f640 -1 Falling back to public interface 2023-10-09T05:31:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:27 smithi008 ceph-mon[191227]: pgmap v191: 73 pgs: 2 active+undersized+degraded, 13 peering, 16 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:31:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:27 smithi008 ceph-mon[191227]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:27 smithi008 ceph-mon[184418]: pgmap v191: 73 pgs: 2 active+undersized+degraded, 13 peering, 16 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:31:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:27 smithi008 ceph-mon[184418]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:27.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:28.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2480160868' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:28.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2480160868' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:28.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2480160868' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:29.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:29 smithi191 ceph-mon[161669]: pgmap v192: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:29 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:31:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:29 smithi008 ceph-mon[184418]: pgmap v192: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:29 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:31:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:29.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:29 smithi008 ceph-mon[191227]: pgmap v192: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:29 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:31:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:30.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:31:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:31:30] "GET /metrics HTTP/1.1" 200 34025 "" "Prometheus/2.33.4" 2023-10-09T05:31:30.715 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:30 smithi008 conmon[218691]: 2023-10-09T05:31:30.417+0000 7fbdd945f640 -1 osd.3 99 log_to_monitors true 2023-10-09T05:31:31.138 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 05:31:30 smithi008 conmon[218691]: 2023-10-09T05:31:30.860+0000 7fbdca8cb700 -1 osd.3 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:31:31.139 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.139 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.139 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[184418]: pgmap v193: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:31.139 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[184418]: from='osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:31:31.139 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.139 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.139 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3507763853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:31.140 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.140 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.140 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[191227]: pgmap v193: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:31.140 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[191227]: from='osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:31:31.140 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.140 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.140 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3507763853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:30 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:30 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:30 smithi191 ceph-mon[161669]: pgmap v193: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:30 smithi191 ceph-mon[161669]: from='osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:31:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:30 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:30 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:31.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3507763853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:32.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[184418]: from='osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:31:32.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[184418]: osdmap e102: 8 total, 7 up, 8 in 2023-10-09T05:31:32.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[184418]: from='osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:31:32.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[184418]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[184418]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[184418]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[184418]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[191227]: from='osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:31:32.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[191227]: osdmap e102: 8 total, 7 up, 8 in 2023-10-09T05:31:32.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[191227]: from='osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:31:32.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[191227]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[191227]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[191227]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:31 smithi008 ceph-mon[191227]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:31 smithi191 ceph-mon[161669]: from='osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:31:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:31 smithi191 ceph-mon[161669]: osdmap e102: 8 total, 7 up, 8 in 2023-10-09T05:31:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:31 smithi191 ceph-mon[161669]: from='osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi008", "root=default"]}]: dispatch 2023-10-09T05:31:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:31 smithi191 ceph-mon[161669]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:31 smithi191 ceph-mon[161669]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:31 smithi191 ceph-mon[161669]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:31 smithi191 ceph-mon[161669]: failed to encode map e102 with expected crc 2023-10-09T05:31:32.982 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T05:31:33.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[191227]: failed to encode map e102 with expected crc 2023-10-09T05:31:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[191227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:31:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[191227]: pgmap v195: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[191227]: failed to encode map e103 with expected crc 2023-10-09T05:31:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[191227]: failed to encode map e103 with expected crc 2023-10-09T05:31:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[191227]: osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505] boot 2023-10-09T05:31:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[191227]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T05:31:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:31:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[184418]: failed to encode map e102 with expected crc 2023-10-09T05:31:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[184418]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:31:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[184418]: pgmap v195: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[184418]: failed to encode map e103 with expected crc 2023-10-09T05:31:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[184418]: failed to encode map e103 with expected crc 2023-10-09T05:31:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[184418]: osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505] boot 2023-10-09T05:31:33.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[184418]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T05:31:33.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:31:33.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:32 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:33.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:32 smithi191 ceph-mon[161669]: failed to encode map e102 with expected crc 2023-10-09T05:31:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:32 smithi191 ceph-mon[161669]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:31:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:32 smithi191 ceph-mon[161669]: pgmap v195: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:32 smithi191 ceph-mon[161669]: failed to encode map e103 with expected crc 2023-10-09T05:31:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:32 smithi191 ceph-mon[161669]: failed to encode map e103 with expected crc 2023-10-09T05:31:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:32 smithi191 ceph-mon[161669]: osd.3 [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505] boot 2023-10-09T05:31:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:32 smithi191 ceph-mon[161669]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T05:31:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:31:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:32 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:33.850 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:31:33 smithi008 conmon[177229]: 2023-10-09T05:31:33.808+0000 7f020285b700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T05:31:33.853 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: failed to encode map e104 with expected crc 2023-10-09T05:31:33.853 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: failed to encode map e104 with expected crc 2023-10-09T05:31:33.853 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T05:31:33.854 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: failed to encode map e104 with expected crc 2023-10-09T05:31:33.854 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: failed to encode map e104 with expected crc 2023-10-09T05:31:33.854 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: failed to encode map e104 with expected crc 2023-10-09T05:31:33.854 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: failed to encode map e104 with expected crc 2023-10-09T05:31:33.854 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T05:31:33.854 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: failed to encode map e104 with expected crc 2023-10-09T05:31:33.854 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: failed to encode map e104 with expected crc 2023-10-09T05:31:33.854 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: failed to encode map e104 with expected crc 2023-10-09T05:31:33.854 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: failed to encode map e104 with expected crc 2023-10-09T05:31:33.854 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: failed to encode map e104 with expected crc 2023-10-09T05:31:33.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2428313145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:33.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:33.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:33.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:33.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:33.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:33.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:33.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:33.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:33.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:33.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:31:34.128 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: failed to encode map e104 with expected crc 2023-10-09T05:31:34.128 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: failed to encode map e104 with expected crc 2023-10-09T05:31:34.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: failed to encode map e104 with expected crc 2023-10-09T05:31:34.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2428313145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:34.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:34.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:34.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:34.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:34.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:34.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:34.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:34.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:34.130 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:34.130 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:31:34.145 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T05:31:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: failed to encode map e104 with expected crc 2023-10-09T05:31:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: failed to encode map e104 with expected crc 2023-10-09T05:31:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T05:31:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: failed to encode map e104 with expected crc 2023-10-09T05:31:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: failed to encode map e104 with expected crc 2023-10-09T05:31:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: failed to encode map e104 with expected crc 2023-10-09T05:31:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: failed to encode map e104 with expected crc 2023-10-09T05:31:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: failed to encode map e104 with expected crc 2023-10-09T05:31:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2428313145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:31:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:34.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:31:35.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:34 smithi008 ceph-mon[184418]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:31:35.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:34 smithi008 ceph-mon[184418]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T05:31:35.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:34 smithi008 ceph-mon[184418]: pgmap v198: 73 pgs: 5 peering, 3 active+undersized+degraded, 21 active+undersized, 44 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:35.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:34 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:34 smithi008 ceph-mon[191227]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:31:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:34 smithi008 ceph-mon[191227]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T05:31:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:34 smithi008 ceph-mon[191227]: pgmap v198: 73 pgs: 5 peering, 3 active+undersized+degraded, 21 active+undersized, 44 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:34 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:34 smithi191 ceph-mon[161669]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:31:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:34 smithi191 ceph-mon[161669]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T05:31:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:34 smithi191 ceph-mon[161669]: pgmap v198: 73 pgs: 5 peering, 3 active+undersized+degraded, 21 active+undersized, 44 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-09T05:31:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:34 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:36.022 INFO:teuthology.orchestra.run.smithi008.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:31:36.653 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:31:37.286 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:37 smithi008 ceph-mon[184418]: pgmap v199: 73 pgs: 5 peering, 2 active+undersized+degraded, 11 active+undersized, 55 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:31:37.286 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:37 smithi008 ceph-mon[184418]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:31:37.286 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:37 smithi008 ceph-mon[184418]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:31:37.286 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:37 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:37.286 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:37 smithi008 ceph-mon[191227]: pgmap v199: 73 pgs: 5 peering, 2 active+undersized+degraded, 11 active+undersized, 55 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:31:37.286 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:37 smithi008 ceph-mon[191227]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:31:37.287 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:37 smithi008 ceph-mon[191227]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:31:37.287 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:37 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:37.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:37 smithi191 ceph-mon[161669]: pgmap v199: 73 pgs: 5 peering, 2 active+undersized+degraded, 11 active+undersized, 55 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-09T05:31:37.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:37 smithi191 ceph-mon[161669]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:31:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:37 smithi191 ceph-mon[161669]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:31:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:37 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:38.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:38.157 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:38.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:31:38.567 INFO:teuthology.orchestra.run.smithi008.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:31:38.568 INFO:teuthology.orchestra.run.smithi008.stdout:alertmanager.a smithi008 *:9093,9094 running (7m) - 10m 25.6M - 0.23.0 ba2b418f427c 172e511a984d 2023-10-09T05:31:38.568 INFO:teuthology.orchestra.run.smithi008.stdout:grafana.a smithi191 *:3000 running (6m) - 10m 60.1M - 8.3.5 dad864ee21e9 66b01c4fecc1 2023-10-09T05:31:38.568 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi008.zfejpl smithi008 running (10m) - 10m 18.2M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5d86a4e2d3b 2023-10-09T05:31:38.568 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi191.walafj smithi191 running (10m) - 10m 15.7M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 93130380f297 2023-10-09T05:31:38.568 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.x smithi191 *:8443,9283,8765 running (6m) - 14m 449M - 18.2.0-598-gb4fa663f 3060897fdf97 e651b468b4c8 2023-10-09T05:31:38.568 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.y smithi008 *:8443,9283,8765 running (6m) - 16m 516M - 18.2.0-598-gb4fa663f 3060897fdf97 b6adda8222c0 2023-10-09T05:31:38.568 INFO:teuthology.orchestra.run.smithi008.stdout:mon.a smithi008 running (6m) - 16m 104M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 dbfd9e97488a 2023-10-09T05:31:38.568 INFO:teuthology.orchestra.run.smithi008.stdout:mon.b smithi191 running (3m) - 15m 71.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8d871de2e613 2023-10-09T05:31:38.568 INFO:teuthology.orchestra.run.smithi008.stdout:mon.c smithi008 running (4m) - 15m 80.3M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 b504573c5a12 2023-10-09T05:31:38.568 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.a smithi008 *:9100 running (7m) - 11m 24.9M - 1.3.1 1dbe0e931976 0b0d19fe4991 2023-10-09T05:31:38.569 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.b smithi191 *:9100 running (7m) - 11m 19.6M - 1.3.1 1dbe0e931976 05f0cea760c5 2023-10-09T05:31:38.569 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0 smithi008 running (105s) - 14m 61.4M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 288560b20015 2023-10-09T05:31:38.569 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 smithi008 running (74s) - 14m 56.3M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 086a4a5d61a5 2023-10-09T05:31:38.569 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2 smithi008 running (42s) - 13m 50.2M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 5a87289cd2f0 2023-10-09T05:31:38.569 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 smithi008 running (11s) - 13m 11.9M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 8625f8d5125e 2023-10-09T05:31:38.569 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 smithi191 running (13m) - 13m 90.9M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 57a8d325e472 2023-10-09T05:31:38.569 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 smithi191 running (12m) - 12m 91.3M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 141ad4d0ecc5 2023-10-09T05:31:38.569 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6 smithi191 running (12m) - 12m 90.2M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0039e7e7a199 2023-10-09T05:31:38.569 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7 smithi191 running (12m) - 12m 112M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d71410a3a7bd 2023-10-09T05:31:38.569 INFO:teuthology.orchestra.run.smithi008.stdout:prometheus.a smithi191 *:9095 running (6m) - 11m 68.2M - 2.33.4 514e6a882f6e d24b4acc1156 2023-10-09T05:31:39.254 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:31:39.281 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:39 smithi008 ceph-mon[184418]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:39.281 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:39 smithi008 ceph-mon[184418]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-09T05:31:39.281 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:39 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:31:39.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:39 smithi008 ceph-mon[191227]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:39.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:39 smithi008 ceph-mon[191227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-09T05:31:39.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:39 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:31:39.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:39 smithi191 ceph-mon[161669]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:39.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:39 smithi191 ceph-mon[161669]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-09T05:31:39.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:39 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:31:40.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:40 smithi191 ceph-mon[161669]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:31:40.446 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:40 smithi008 ceph-mon[184418]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:31:40.447 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:40 smithi008 ceph-mon[191227]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:31:40.698 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:31:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:31:40] "GET /metrics HTTP/1.1" 200 34025 "" "Prometheus/2.33.4" 2023-10-09T05:31:41.103 INFO:teuthology.orchestra.run.smithi008.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:alertmanager.a smithi008 *:9093,9094 running (7m) - 11m 25.6M - 0.23.0 ba2b418f427c 172e511a984d 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:grafana.a smithi191 *:3000 running (6m) - 10m 60.1M - 8.3.5 dad864ee21e9 66b01c4fecc1 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi008.zfejpl smithi008 running (10m) - 10m 18.2M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 c5d86a4e2d3b 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:mds.foo.smithi191.walafj smithi191 running (10m) - 10m 15.7M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 93130380f297 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.x smithi191 *:8443,9283,8765 running (6m) - 14m 449M - 18.2.0-598-gb4fa663f 3060897fdf97 e651b468b4c8 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:mgr.y smithi008 *:8443,9283,8765 running (6m) - 16m 516M - 18.2.0-598-gb4fa663f 3060897fdf97 b6adda8222c0 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:mon.a smithi008 running (6m) - 16m 104M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 dbfd9e97488a 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:mon.b smithi191 running (3m) - 15m 71.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8d871de2e613 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:mon.c smithi008 running (4m) - 15m 80.3M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 b504573c5a12 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.a smithi008 *:9100 running (7m) - 11m 24.9M - 1.3.1 1dbe0e931976 0b0d19fe4991 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:node-exporter.b smithi191 *:9100 running (7m) - 11m 19.6M - 1.3.1 1dbe0e931976 05f0cea760c5 2023-10-09T05:31:41.104 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0 smithi008 running (108s) - 14m 61.4M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 288560b20015 2023-10-09T05:31:41.105 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 smithi008 running (77s) - 14m 56.3M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 086a4a5d61a5 2023-10-09T05:31:41.105 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2 smithi008 running (45s) - 13m 50.2M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 5a87289cd2f0 2023-10-09T05:31:41.105 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 smithi008 running (14s) - 13m 11.9M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 8625f8d5125e 2023-10-09T05:31:41.105 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 smithi191 running (13m) - 13m 90.9M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 57a8d325e472 2023-10-09T05:31:41.105 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 smithi191 running (12m) - 12m 91.3M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 141ad4d0ecc5 2023-10-09T05:31:41.105 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6 smithi191 running (12m) - 12m 90.2M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 0039e7e7a199 2023-10-09T05:31:41.105 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7 smithi191 running (12m) - 12m 112M 2500M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 d71410a3a7bd 2023-10-09T05:31:41.105 INFO:teuthology.orchestra.run.smithi008.stdout:prometheus.a smithi191 *:9095 running (6m) - 11m 68.2M - 2.33.4 514e6a882f6e d24b4acc1156 2023-10-09T05:31:41.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:41 smithi191 ceph-mon[161669]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:41.319 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:41 smithi008 ceph-mon[184418]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:41.319 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:41 smithi008 ceph-mon[191227]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:41.705 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T05:31:42.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:42 smithi191 ceph-mon[161669]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:31:42.374 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:42 smithi008 ceph-mon[184418]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:31:42.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:42 smithi008 ceph-mon[191227]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:31:43.645 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:31:43.645 INFO:teuthology.orchestra.run.smithi008.stdout: "mon": { 2023-10-09T05:31:43.645 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T05:31:43.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr": { 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd": { 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 4, 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi008.stdout: "mds": { 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 2 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:31:43.646 INFO:teuthology.orchestra.run.smithi008.stdout: "overall": { 2023-10-09T05:31:43.647 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 6, 2023-10-09T05:31:43.647 INFO:teuthology.orchestra.run.smithi008.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T05:31:43.647 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:31:43.647 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:31:43.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:43 smithi008 ceph-mon[184418]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:43.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:43 smithi008 ceph-mon[191227]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:43.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:43 smithi191 ceph-mon[161669]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:44.261 DEBUG:teuthology.parallel:result is None 2023-10-09T05:31:44.261 INFO:teuthology.run_tasks:Running task print... 2023-10-09T05:31:44.271 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T05:31:44.271 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T05:31:44.284 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T05:31:44.284 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T05:31:44.285 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T05:31:44.644 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:44.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1699627074' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1699627074' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1699627074' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:31:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:45.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:45 smithi008 ceph-mon[184418]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:45.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:45 smithi008 ceph-mon[191227]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:45.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:45 smithi191 ceph-mon[161669]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:46.323 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:31:46.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/931844052' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:31:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/931844052' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:31:46.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/931844052' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:31:46.953 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":36,"active_gid":25913,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6800","nonce":358867711},{"type":"v1","addr":"172.21.15.8:6801","nonce":358867711}]},"active_addr":"172.21.15.8:6801/358867711","active_change":"2023-10-09T05:25:43.729122+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25801,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.8:8443/","prometheus":"http://172.21.15.8:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":84,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.8:0","nonce":1415489421}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.8:0","nonce":3919595407}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.8:0","nonce":848605149}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.8:0","nonce":1545992134}]}]} 2023-10-09T05:31:46.958 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T05:31:46.958 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:31:46.958 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:31:47.322 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:47.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:47 smithi008 ceph-mon[184418]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:47.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:47 smithi008 ceph-mon[191227]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:47.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:47 smithi191 ceph-mon[161669]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:48.899 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:31:48.899 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":104,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:14:56.160913+0000","modified":"2023-10-09T05:31:32.844881+0000","last_up_change":"2023-10-09T05:31:31.839952+0000","last_in_change":"2023-10-09T05:19:26.801646+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:17:53.386952+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:20:22.166676+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"76","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":76,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:21:15.712971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:21:16.208554+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":103,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6802","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6803","nonce":633445692}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6804","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6805","nonce":633445692}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6808","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6809","nonce":633445692}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6806","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6807","nonce":633445692}]},"public_addr":"172.21.15.8:6803/633445692","cluster_addr":"172.21.15.8:6805/633445692","heartbeat_back_addr":"172.21.15.8:6809/633445692","heartbeat_front_addr":"172.21.15.8:6807/633445692","state":["exists","up"]},{"osd":1,"uuid":"55f3290e-b82d-4839-a28d-1123a20c3a4c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":103,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6810","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6811","nonce":417852095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6812","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6813","nonce":417852095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6816","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6817","nonce":417852095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6814","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6815","nonce":417852095}]},"public_addr":"172.21.15.8:6811/417852095","cluster_addr":"172.21.15.8:6813/417852095","heartbeat_back_addr":"172.21.15.8:6817/417852095","heartbeat_front_addr":"172.21.15.8:6815/417852095","state":["exists","up"]},{"osd":2,"uuid":"720b47df-d743-46cf-ad04-36e113c8fcb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":103,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6818","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6819","nonce":1092325054}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6820","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6821","nonce":1092325054}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6824","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6825","nonce":1092325054}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6822","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6823","nonce":1092325054}]},"public_addr":"172.21.15.8:6819/1092325054","cluster_addr":"172.21.15.8:6821/1092325054","heartbeat_back_addr":"172.21.15.8:6825/1092325054","heartbeat_front_addr":"172.21.15.8:6823/1092325054","state":["exists","up"]},{"osd":3,"uuid":"861bf05d-69f8-4000-999e-ab708531edb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":103,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6826","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6827","nonce":4118391505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6828","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6829","nonce":4118391505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6832","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6833","nonce":4118391505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6830","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6831","nonce":4118391505}]},"public_addr":"172.21.15.8:6827/4118391505","cluster_addr":"172.21.15.8:6829/4118391505","heartbeat_back_addr":"172.21.15.8:6833/4118391505","heartbeat_front_addr":"172.21.15.8:6831/4118391505","state":["exists","up"]},{"osd":4,"uuid":"b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6801","nonce":3473028939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6803","nonce":3473028939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6807","nonce":3473028939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6805","nonce":3473028939}]},"public_addr":"172.21.15.191:6801/3473028939","cluster_addr":"172.21.15.191:6803/3473028939","heartbeat_back_addr":"172.21.15.191:6807/3473028939","heartbeat_front_addr":"172.21.15.191:6805/3473028939","state":["exists","up"]},{"osd":5,"uuid":"01507478-481e-4646-a30d-542184240f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6809","nonce":1592222523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6811","nonce":1592222523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6815","nonce":1592222523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6813","nonce":1592222523}]},"public_addr":"172.21.15.191:6809/1592222523","cluster_addr":"172.21.15.191:6811/1592222523","heartbeat_back_addr":"172.21.15.191:6815/1592222523","heartbeat_front_addr":"172.21.15.191:6813/1592222523","state":["exists","up"]},{"osd":6,"uuid":"8bc75ec3-46ee-4d36-989b-4e65e330cfb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":100,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6817","nonce":4267099596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6819","nonce":4267099596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6823","nonce":4267099596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6821","nonce":4267099596}]},"public_addr":"172.21.15.191:6817/4267099596","cluster_addr":"172.21.15.191:6819/4267099596","heartbeat_back_addr":"172.21.15.191:6823/4267099596","heartbeat_front_addr":"172.21.15.191:6821/4267099596","state":["exists","up"]},{"osd":7,"uuid":"5ac2e382-7341-47dd-b6c9-0fbdba12829c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6825","nonce":1787611943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6827","nonce":1787611943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6831","nonce":1787611943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6829","nonce":1787611943}]},"public_addr":"172.21.15.191:6825/1787611943","cluster_addr":"172.21.15.191:6827/1787611943","heartbeat_back_addr":"172.21.15.191:6831/1787611943","heartbeat_front_addr":"172.21.15.191:6829/1787611943","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:06.542686+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:28.674405+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:51.044955+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:14.395353+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:36.486099+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:58.485442+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:20.015429+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:42.014771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.191:6835/1974805222":"2023-10-10T05:25:00.410091+0000","172.21.15.191:6834/1974805222":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6800/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/1497753039":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/1669399762":"2023-10-10T05:23:46.082746+0000","172.21.15.8:6800/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/357500347":"2023-10-10T05:23:46.082746+0000","172.21.15.8:0/1025807783":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/69531294":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/3932209052":"2023-10-10T05:25:43.728914+0000","172.21.15.191:0/3608336050":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6801/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/711490379":"2023-10-10T05:25:43.728914+0000","172.21.15.191:0/1074075163":"2023-10-10T05:25:00.410091+0000","172.21.15.191:0/3279324894":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/2854879000":"2023-10-10T05:23:46.082746+0000","172.21.15.191:0/1645031799":"2023-10-10T05:25:00.410091+0000","172.21.15.8:0/2096086544":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/819405330":"2023-10-10T05:25:43.728914+0000","172.21.15.8:6800/2853847379":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/2360193392":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6801/2853847379":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/3599326555":"2023-10-10T05:25:43.728914+0000","172.21.15.191:0/465298627":"2023-10-10T05:25:00.410091+0000","172.21.15.8:0/2178079664":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/2854879000":"2023-10-10T05:23:46.082746+0000","172.21.15.8:6800/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.191:0/2331494796":"2023-10-10T05:25:00.410091+0000","172.21.15.8:0/1936221066":"2023-10-10T05:20:01.878268+0000","172.21.15.8:0/1062877078":"2023-10-10T05:15:49.270008+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:31:49.565 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:31:49.565 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:31:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:49 smithi191 ceph-mon[161669]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1092432355' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:31:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:49.930 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:49.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[191227]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:49.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1092432355' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:31:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:49.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[184418]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:49.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:49.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:49.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:49.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:49.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1092432355' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:31:49.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:31:49.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:31:49.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:31:49.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:31:50.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:31:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:31:50] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.33.4" 2023-10-09T05:31:51.576 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:31:51.576 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":104,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:14:56.160913+0000","modified":"2023-10-09T05:31:32.844881+0000","last_up_change":"2023-10-09T05:31:31.839952+0000","last_in_change":"2023-10-09T05:19:26.801646+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:17:53.386952+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:20:22.166676+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"76","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":76,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:21:15.712971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:21:16.208554+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":103,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6802","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6803","nonce":633445692}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6804","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6805","nonce":633445692}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6808","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6809","nonce":633445692}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6806","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6807","nonce":633445692}]},"public_addr":"172.21.15.8:6803/633445692","cluster_addr":"172.21.15.8:6805/633445692","heartbeat_back_addr":"172.21.15.8:6809/633445692","heartbeat_front_addr":"172.21.15.8:6807/633445692","state":["exists","up"]},{"osd":1,"uuid":"55f3290e-b82d-4839-a28d-1123a20c3a4c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":103,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6810","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6811","nonce":417852095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6812","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6813","nonce":417852095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6816","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6817","nonce":417852095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6814","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6815","nonce":417852095}]},"public_addr":"172.21.15.8:6811/417852095","cluster_addr":"172.21.15.8:6813/417852095","heartbeat_back_addr":"172.21.15.8:6817/417852095","heartbeat_front_addr":"172.21.15.8:6815/417852095","state":["exists","up"]},{"osd":2,"uuid":"720b47df-d743-46cf-ad04-36e113c8fcb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":103,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6818","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6819","nonce":1092325054}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6820","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6821","nonce":1092325054}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6824","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6825","nonce":1092325054}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6822","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6823","nonce":1092325054}]},"public_addr":"172.21.15.8:6819/1092325054","cluster_addr":"172.21.15.8:6821/1092325054","heartbeat_back_addr":"172.21.15.8:6825/1092325054","heartbeat_front_addr":"172.21.15.8:6823/1092325054","state":["exists","up"]},{"osd":3,"uuid":"861bf05d-69f8-4000-999e-ab708531edb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":103,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6826","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6827","nonce":4118391505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6828","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6829","nonce":4118391505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6832","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6833","nonce":4118391505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6830","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6831","nonce":4118391505}]},"public_addr":"172.21.15.8:6827/4118391505","cluster_addr":"172.21.15.8:6829/4118391505","heartbeat_back_addr":"172.21.15.8:6833/4118391505","heartbeat_front_addr":"172.21.15.8:6831/4118391505","state":["exists","up"]},{"osd":4,"uuid":"b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6801","nonce":3473028939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6803","nonce":3473028939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6807","nonce":3473028939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6805","nonce":3473028939}]},"public_addr":"172.21.15.191:6801/3473028939","cluster_addr":"172.21.15.191:6803/3473028939","heartbeat_back_addr":"172.21.15.191:6807/3473028939","heartbeat_front_addr":"172.21.15.191:6805/3473028939","state":["exists","up"]},{"osd":5,"uuid":"01507478-481e-4646-a30d-542184240f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6809","nonce":1592222523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6811","nonce":1592222523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6815","nonce":1592222523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6813","nonce":1592222523}]},"public_addr":"172.21.15.191:6809/1592222523","cluster_addr":"172.21.15.191:6811/1592222523","heartbeat_back_addr":"172.21.15.191:6815/1592222523","heartbeat_front_addr":"172.21.15.191:6813/1592222523","state":["exists","up"]},{"osd":6,"uuid":"8bc75ec3-46ee-4d36-989b-4e65e330cfb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":100,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6817","nonce":4267099596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6819","nonce":4267099596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6823","nonce":4267099596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":4267099596},{"type":"v1","addr":"172.21.15.191:6821","nonce":4267099596}]},"public_addr":"172.21.15.191:6817/4267099596","cluster_addr":"172.21.15.191:6819/4267099596","heartbeat_back_addr":"172.21.15.191:6823/4267099596","heartbeat_front_addr":"172.21.15.191:6821/4267099596","state":["exists","up"]},{"osd":7,"uuid":"5ac2e382-7341-47dd-b6c9-0fbdba12829c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6825","nonce":1787611943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6827","nonce":1787611943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6831","nonce":1787611943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":1787611943},{"type":"v1","addr":"172.21.15.191:6829","nonce":1787611943}]},"public_addr":"172.21.15.191:6825/1787611943","cluster_addr":"172.21.15.191:6827/1787611943","heartbeat_back_addr":"172.21.15.191:6831/1787611943","heartbeat_front_addr":"172.21.15.191:6829/1787611943","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:06.542686+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:28.674405+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:51.044955+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:14.395353+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:36.486099+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:58.485442+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:20.015429+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:42.014771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.191:6835/1974805222":"2023-10-10T05:25:00.410091+0000","172.21.15.191:6834/1974805222":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6800/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/1497753039":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/1669399762":"2023-10-10T05:23:46.082746+0000","172.21.15.8:6800/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/357500347":"2023-10-10T05:23:46.082746+0000","172.21.15.8:0/1025807783":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/69531294":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/3932209052":"2023-10-10T05:25:43.728914+0000","172.21.15.191:0/3608336050":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6801/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/711490379":"2023-10-10T05:25:43.728914+0000","172.21.15.191:0/1074075163":"2023-10-10T05:25:00.410091+0000","172.21.15.191:0/3279324894":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/2854879000":"2023-10-10T05:23:46.082746+0000","172.21.15.191:0/1645031799":"2023-10-10T05:25:00.410091+0000","172.21.15.8:0/2096086544":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/819405330":"2023-10-10T05:25:43.728914+0000","172.21.15.8:6800/2853847379":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/2360193392":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6801/2853847379":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/3599326555":"2023-10-10T05:25:43.728914+0000","172.21.15.191:0/465298627":"2023-10-10T05:25:00.410091+0000","172.21.15.8:0/2178079664":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/2854879000":"2023-10-10T05:23:46.082746+0000","172.21.15.8:6800/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.191:0/2331494796":"2023-10-10T05:25:00.410091+0000","172.21.15.8:0/1936221066":"2023-10-10T05:20:01.878268+0000","172.21.15.8:0/1062877078":"2023-10-10T05:15:49.270008+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:31:51.777 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:51 smithi008 ceph-mon[184418]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:51.778 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:51 smithi008 ceph-mon[191227]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:51 smithi191 ceph-mon[161669]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:52.180 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:31:52.181 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:31:52.181 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:31:52.181 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:31:52.182 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:31:52.182 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:31:52.183 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:31:52.183 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:31:52.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/746130556' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:31:52.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/746130556' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:31:52.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/746130556' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:31:53.688 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:53.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:53 smithi191 ceph-mon[161669]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:53.855 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:53.855 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:53.881 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:53.953 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:53 smithi008 ceph-mon[184418]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:53.953 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:53 smithi008 ceph-mon[191227]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:54.031 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:54.092 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:55.652 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:55.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:55 smithi191 ceph-mon[161669]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:55.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:55 smithi008 ceph-mon[184418]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:55.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:55 smithi008 ceph-mon[191227]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:56.102 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:31:57.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:57 smithi191 ceph-mon[161669]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:57.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:57 smithi008 ceph-mon[184418]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:57.987 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:57 smithi008 ceph-mon[191227]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:59.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:59 smithi008 ceph-mon[184418]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:59.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:31:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:59.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:59 smithi008 ceph-mon[191227]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:59.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:31:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:31:59.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:59 smithi191 ceph-mon[161669]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:31:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:31:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:00.345 INFO:teuthology.orchestra.run.smithi008.stdout:128849019042 2023-10-09T05:32:00.346 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:32:00.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:32:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:32:00] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.33.4" 2023-10-09T05:32:01.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:01 smithi008 ceph-mon[184418]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:01.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:01 smithi008 ceph-mon[191227]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:01.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:01 smithi191 ceph-mon[161669]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:02.215 INFO:teuthology.orchestra.run.smithi008.stdout:420906795021 2023-10-09T05:32:02.215 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:32:02.709 INFO:teuthology.orchestra.run.smithi008.stdout:399431958548 2023-10-09T05:32:02.709 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:32:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:03 smithi191 ceph-mon[161669]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:03.858 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:03 smithi008 ceph-mon[184418]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:03.858 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:03 smithi008 ceph-mon[191227]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:04.763 INFO:teuthology.orchestra.run.smithi008.stdout:442381631496 2023-10-09T05:32:04.764 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:32:05.489 INFO:teuthology.orchestra.run.smithi008.stdout:180388626587 2023-10-09T05:32:05.489 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:32:05.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:05 smithi008 ceph-mon[184418]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:05.681 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:05 smithi008 ceph-mon[191227]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:05.682 INFO:teuthology.orchestra.run.smithi008.stdout:206158430359 2023-10-09T05:32:05.682 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:32:05.752 INFO:teuthology.orchestra.run.smithi008.stdout:377957122075 2023-10-09T05:32:05.752 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:32:05.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:05 smithi191 ceph-mon[161669]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:05.829 INFO:teuthology.orchestra.run.smithi008.stdout:154618822815 2023-10-09T05:32:05.829 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:32:05.982 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:05.998 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:06.021 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:06.071 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:06.159 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:06.461 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:07.602 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:07 smithi008 ceph-mon[184418]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:07.602 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:07 smithi008 ceph-mon[191227]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:07.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:07 smithi191 ceph-mon[161669]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:09.657 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:09 smithi008 ceph-mon[184418]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:09.657 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:09 smithi008 ceph-mon[191227]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:09.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:09 smithi191 ceph-mon[161669]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:09.883 INFO:teuthology.orchestra.run.smithi008.stdout:128849019044 2023-10-09T05:32:10.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:10 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4122676993' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:32:10.852 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:10 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4122676993' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:32:10.853 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:32:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:32:10] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.33.4" 2023-10-09T05:32:10.853 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:10 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4122676993' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:32:11.130 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019042 got 128849019044 for osd.4 2023-10-09T05:32:11.130 DEBUG:teuthology.parallel:result is None 2023-10-09T05:32:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:11 smithi191 ceph-mon[161669]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:11.863 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:11 smithi008 ceph-mon[184418]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:11.863 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:11 smithi008 ceph-mon[191227]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:11.881 INFO:teuthology.orchestra.run.smithi008.stdout:442381631497 2023-10-09T05:32:12.361 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:12.374 INFO:teuthology.orchestra.run.smithi008.stdout:180388626589 2023-10-09T05:32:12.674 INFO:teuthology.orchestra.run.smithi008.stdout:399431958550 2023-10-09T05:32:12.700 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2079859529' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:32:12.700 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:12 smithi008 ceph-mon[184418]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:12.700 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1024159938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:32:12.700 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2079859529' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:32:12.701 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:12 smithi008 ceph-mon[191227]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:12.701 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1024159938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:32:12.717 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:12.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2079859529' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:32:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:12 smithi191 ceph-mon[161669]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1024159938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:32:12.832 INFO:teuthology.orchestra.run.smithi008.stdout:206158430361 2023-10-09T05:32:12.929 INFO:teuthology.orchestra.run.smithi008.stdout:420906795024 2023-10-09T05:32:13.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1947082099' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:32:13.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/9671080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:32:13.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2136066090' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:32:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1947082099' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:32:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/9671080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:32:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2136066090' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:32:13.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1947082099' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:32:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/9671080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:32:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2136066090' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:32:14.082 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381631496 got 442381631497 for osd.3 2023-10-09T05:32:14.082 DEBUG:teuthology.parallel:result is None 2023-10-09T05:32:14.653 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958548 got 399431958550 for osd.1 2023-10-09T05:32:14.654 DEBUG:teuthology.parallel:result is None 2023-10-09T05:32:14.706 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795021 got 420906795024 for osd.2 2023-10-09T05:32:14.706 DEBUG:teuthology.parallel:result is None 2023-10-09T05:32:14.825 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:14.825 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:14 smithi008 ceph-mon[184418]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:14.825 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:14.825 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:14 smithi008 ceph-mon[191227]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:15.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:14 smithi191 ceph-mon[161669]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:15.594 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430359 got 206158430361 for osd.7 2023-10-09T05:32:15.594 DEBUG:teuthology.parallel:result is None 2023-10-09T05:32:15.623 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626587 got 180388626589 for osd.6 2023-10-09T05:32:15.623 DEBUG:teuthology.parallel:result is None 2023-10-09T05:32:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:16 smithi008 ceph-mon[184418]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:17.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:16 smithi008 ceph-mon[191227]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:17.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:16 smithi191 ceph-mon[161669]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:17.485 INFO:teuthology.orchestra.run.smithi008.stdout:154618822818 2023-10-09T05:32:17.804 INFO:teuthology.orchestra.run.smithi008.stdout:377957122077 2023-10-09T05:32:18.518 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822815 got 154618822818 for osd.5 2023-10-09T05:32:18.518 DEBUG:teuthology.parallel:result is None 2023-10-09T05:32:18.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1479588961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:32:18.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4141610470' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:32:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1479588961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:32:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4141610470' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:32:18.782 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122075 got 377957122077 for osd.0 2023-10-09T05:32:18.782 DEBUG:teuthology.parallel:result is None 2023-10-09T05:32:18.782 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:32:18.782 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:32:18.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1479588961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:32:18.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4141610470' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:32:19.147 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:19 smithi008 ceph-mon[184418]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:19 smithi008 ceph-mon[191227]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:19.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:19 smithi191 ceph-mon[161669]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:20.858 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:32:20.859 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:32:20.883 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:32:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:32:20] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.33.4" 2023-10-09T05:32:21.500 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":221,"stamp":"2023-10-09T05:32:19.854007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13724,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341060,"kb_used_data":22532,"kb_used_omap":31,"kb_used_meta":1318432,"kb_avail":748620156,"statfs":{"total":767960285184,"available":766587039744,"internally_reserved":0,"allocated":23072768,"data_stored":19030961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":1350074825},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002681"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848734+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:31:31.848734+0000","last_peered":"2023-10-09T05:31:31.848734+0000","last_clean":"2023-10-09T05:31:31.848734+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:31:31.848734+0000","last_undegraded":"2023-10-09T05:31:31.848734+0000","last_fullsized":"2023-10-09T05:31:31.848734+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854909+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:31:32.854909+0000","last_peered":"2023-10-09T05:31:32.854909+0000","last_clean":"2023-10-09T05:31:32.854909+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:31:32.854909+0000","last_undegraded":"2023-10-09T05:31:32.854909+0000","last_fullsized":"2023-10-09T05:31:32.854909+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.860364+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:31:32.860364+0000","last_peered":"2023-10-09T05:31:32.860364+0000","last_clean":"2023-10-09T05:31:32.860364+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:31:32.860364+0000","last_undegraded":"2023-10-09T05:31:32.860364+0000","last_fullsized":"2023-10-09T05:31:32.860364+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864546+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:31:32.864546+0000","last_peered":"2023-10-09T05:31:32.864546+0000","last_clean":"2023-10-09T05:31:32.864546+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:31:32.864546+0000","last_undegraded":"2023-10-09T05:31:32.864546+0000","last_fullsized":"2023-10-09T05:31:32.864546+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.867349+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:31:32.867349+0000","last_peered":"2023-10-09T05:31:32.867349+0000","last_clean":"2023-10-09T05:31:32.867349+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:31:32.867349+0000","last_undegraded":"2023-10-09T05:31:32.867349+0000","last_fullsized":"2023-10-09T05:31:32.867349+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":58,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850233+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:31:31.850233+0000","last_peered":"2023-10-09T05:31:31.850233+0000","last_clean":"2023-10-09T05:31:31.850233+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:31:31.850233+0000","last_undegraded":"2023-10-09T05:31:31.850233+0000","last_fullsized":"2023-10-09T05:31:31.850233+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869382+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:31:32.869382+0000","last_peered":"2023-10-09T05:31:32.869382+0000","last_clean":"2023-10-09T05:31:32.869382+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:31:32.869382+0000","last_undegraded":"2023-10-09T05:31:32.869382+0000","last_fullsized":"2023-10-09T05:31:32.869382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.849363+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:31:31.849363+0000","last_peered":"2023-10-09T05:31:31.849363+0000","last_clean":"2023-10-09T05:31:31.849363+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:31:31.849363+0000","last_undegraded":"2023-10-09T05:31:31.849363+0000","last_fullsized":"2023-10-09T05:31:31.849363+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870694+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:31:32.870694+0000","last_peered":"2023-10-09T05:31:32.870694+0000","last_clean":"2023-10-09T05:31:32.870694+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:31:32.870694+0000","last_undegraded":"2023-10-09T05:31:32.870694+0000","last_fullsized":"2023-10-09T05:31:32.870694+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":40,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.864012+0000","last_change":"2023-10-09T05:29:59.477246+0000","last_active":"2023-10-09T05:31:31.864012+0000","last_peered":"2023-10-09T05:31:31.864012+0000","last_clean":"2023-10-09T05:31:31.864012+0000","last_became_active":"2023-10-09T05:29:59.476390+0000","last_became_peered":"2023-10-09T05:29:59.476390+0000","last_unstale":"2023-10-09T05:31:31.864012+0000","last_undegraded":"2023-10-09T05:31:31.864012+0000","last_fullsized":"2023-10-09T05:31:31.864012+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:52:31.557279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.851843+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:31:31.851843+0000","last_peered":"2023-10-09T05:31:31.851843+0000","last_clean":"2023-10-09T05:31:31.851843+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:31:31.851843+0000","last_undegraded":"2023-10-09T05:31:31.851843+0000","last_fullsized":"2023-10-09T05:31:31.851843+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854621+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:31:32.854621+0000","last_peered":"2023-10-09T05:31:32.854621+0000","last_clean":"2023-10-09T05:31:32.854621+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:31:32.854621+0000","last_undegraded":"2023-10-09T05:31:32.854621+0000","last_fullsized":"2023-10-09T05:31:32.854621+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":552,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855814+0000","last_change":"2023-10-09T05:31:02.549304+0000","last_active":"2023-10-09T05:31:31.855814+0000","last_peered":"2023-10-09T05:31:31.855814+0000","last_clean":"2023-10-09T05:31:31.855814+0000","last_became_active":"2023-10-09T05:31:02.548085+0000","last_became_peered":"2023-10-09T05:31:02.548085+0000","last_unstale":"2023-10-09T05:31:31.855814+0000","last_undegraded":"2023-10-09T05:31:31.855814+0000","last_fullsized":"2023-10-09T05:31:31.855814+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862414+0000","last_change":"2023-10-09T05:31:32.862414+0000","last_active":"2023-10-09T05:31:32.862414+0000","last_peered":"2023-10-09T05:31:32.862414+0000","last_clean":"2023-10-09T05:31:32.862414+0000","last_became_active":"2023-10-09T05:31:32.860985+0000","last_became_peered":"2023-10-09T05:31:32.860985+0000","last_unstale":"2023-10-09T05:31:32.862414+0000","last_undegraded":"2023-10-09T05:31:32.862414+0000","last_fullsized":"2023-10-09T05:31:32.862414+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:04.492757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":488,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855778+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:31:31.855778+0000","last_peered":"2023-10-09T05:31:31.855778+0000","last_clean":"2023-10-09T05:31:31.855778+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:31:31.855778+0000","last_undegraded":"2023-10-09T05:31:31.855778+0000","last_fullsized":"2023-10-09T05:31:31.855778+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862179+0000","last_change":"2023-10-09T05:31:32.862179+0000","last_active":"2023-10-09T05:31:32.862179+0000","last_peered":"2023-10-09T05:31:32.862179+0000","last_clean":"2023-10-09T05:31:32.862179+0000","last_became_active":"2023-10-09T05:31:32.861777+0000","last_became_peered":"2023-10-09T05:31:32.861777+0000","last_unstale":"2023-10-09T05:31:32.862179+0000","last_undegraded":"2023-10-09T05:31:32.862179+0000","last_fullsized":"2023-10-09T05:31:32.862179+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:40.547661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.853486+0000","last_change":"2023-10-09T05:29:59.470093+0000","last_active":"2023-10-09T05:31:32.853486+0000","last_peered":"2023-10-09T05:31:32.853486+0000","last_clean":"2023-10-09T05:31:32.853486+0000","last_became_active":"2023-10-09T05:29:59.469627+0000","last_became_peered":"2023-10-09T05:29:59.469627+0000","last_unstale":"2023-10-09T05:31:32.853486+0000","last_undegraded":"2023-10-09T05:31:32.853486+0000","last_fullsized":"2023-10-09T05:31:32.853486+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:17.087268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.858901+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:31:31.858901+0000","last_peered":"2023-10-09T05:31:31.858901+0000","last_clean":"2023-10-09T05:31:31.858901+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:31:31.858901+0000","last_undegraded":"2023-10-09T05:31:31.858901+0000","last_fullsized":"2023-10-09T05:31:31.858901+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":333,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854396+0000","last_change":"2023-10-09T05:29:59.474688+0000","last_active":"2023-10-09T05:31:31.854396+0000","last_peered":"2023-10-09T05:31:31.854396+0000","last_clean":"2023-10-09T05:31:31.854396+0000","last_became_active":"2023-10-09T05:29:59.474080+0000","last_became_peered":"2023-10-09T05:29:59.474080+0000","last_unstale":"2023-10-09T05:31:31.854396+0000","last_undegraded":"2023-10-09T05:31:31.854396+0000","last_fullsized":"2023-10-09T05:31:31.854396+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854700+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:31:32.854700+0000","last_peered":"2023-10-09T05:31:32.854700+0000","last_clean":"2023-10-09T05:31:32.854700+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:31:32.854700+0000","last_undegraded":"2023-10-09T05:31:32.854700+0000","last_fullsized":"2023-10-09T05:31:32.854700+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866574+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:31:32.866574+0000","last_peered":"2023-10-09T05:31:32.866574+0000","last_clean":"2023-10-09T05:31:32.866574+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:31:32.866574+0000","last_undegraded":"2023-10-09T05:31:32.866574+0000","last_fullsized":"2023-10-09T05:31:32.866574+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":725,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854312+0000","last_change":"2023-10-09T05:30:30.456503+0000","last_active":"2023-10-09T05:31:31.854312+0000","last_peered":"2023-10-09T05:31:31.854312+0000","last_clean":"2023-10-09T05:31:31.854312+0000","last_became_active":"2023-10-09T05:30:30.455426+0000","last_became_peered":"2023-10-09T05:30:30.455426+0000","last_unstale":"2023-10-09T05:31:31.854312+0000","last_undegraded":"2023-10-09T05:31:31.854312+0000","last_fullsized":"2023-10-09T05:31:31.854312+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855635+0000","last_change":"2023-10-09T05:21:17.465038+0000","last_active":"2023-10-09T05:31:31.855635+0000","last_peered":"2023-10-09T05:31:31.855635+0000","last_clean":"2023-10-09T05:31:31.855635+0000","last_became_active":"2023-10-09T05:21:17.464695+0000","last_became_peered":"2023-10-09T05:21:17.464695+0000","last_unstale":"2023-10-09T05:31:31.855635+0000","last_undegraded":"2023-10-09T05:31:31.855635+0000","last_fullsized":"2023-10-09T05:31:31.855635+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864478+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:31:32.864478+0000","last_peered":"2023-10-09T05:31:32.864478+0000","last_clean":"2023-10-09T05:31:32.864478+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:31:32.864478+0000","last_undegraded":"2023-10-09T05:31:32.864478+0000","last_fullsized":"2023-10-09T05:31:32.864478+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":307,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854341+0000","last_change":"2023-10-09T05:29:59.475956+0000","last_active":"2023-10-09T05:31:31.854341+0000","last_peered":"2023-10-09T05:31:31.854341+0000","last_clean":"2023-10-09T05:31:31.854341+0000","last_became_active":"2023-10-09T05:29:59.475408+0000","last_became_peered":"2023-10-09T05:29:59.475408+0000","last_unstale":"2023-10-09T05:31:31.854341+0000","last_undegraded":"2023-10-09T05:31:31.854341+0000","last_fullsized":"2023-10-09T05:31:31.854341+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":665,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848463+0000","last_change":"2023-10-09T05:31:02.547804+0000","last_active":"2023-10-09T05:31:31.848463+0000","last_peered":"2023-10-09T05:31:31.848463+0000","last_clean":"2023-10-09T05:31:31.848463+0000","last_became_active":"2023-10-09T05:31:02.547237+0000","last_became_peered":"2023-10-09T05:31:02.547237+0000","last_unstale":"2023-10-09T05:31:31.848463+0000","last_undegraded":"2023-10-09T05:31:31.848463+0000","last_fullsized":"2023-10-09T05:31:31.848463+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.859446+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:31:31.859446+0000","last_peered":"2023-10-09T05:31:31.859446+0000","last_clean":"2023-10-09T05:31:31.859446+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:31:31.859446+0000","last_undegraded":"2023-10-09T05:31:31.859446+0000","last_fullsized":"2023-10-09T05:31:31.859446+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.859514+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:31:31.859514+0000","last_peered":"2023-10-09T05:31:31.859514+0000","last_clean":"2023-10-09T05:31:31.859514+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:31:31.859514+0000","last_undegraded":"2023-10-09T05:31:31.859514+0000","last_fullsized":"2023-10-09T05:31:31.859514+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":414,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.858194+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:31:32.858194+0000","last_peered":"2023-10-09T05:31:32.858194+0000","last_clean":"2023-10-09T05:31:32.858194+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:31:32.858194+0000","last_undegraded":"2023-10-09T05:31:32.858194+0000","last_fullsized":"2023-10-09T05:31:32.858194+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866540+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:31:32.866540+0000","last_peered":"2023-10-09T05:31:32.866540+0000","last_clean":"2023-10-09T05:31:32.866540+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:31:32.866540+0000","last_undegraded":"2023-10-09T05:31:32.866540+0000","last_fullsized":"2023-10-09T05:31:32.866540+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.849147+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:31:31.849147+0000","last_peered":"2023-10-09T05:31:31.849147+0000","last_clean":"2023-10-09T05:31:31.849147+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:31:31.849147+0000","last_undegraded":"2023-10-09T05:31:31.849147+0000","last_fullsized":"2023-10-09T05:31:31.849147+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":540,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848375+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:31:31.848375+0000","last_peered":"2023-10-09T05:31:31.848375+0000","last_clean":"2023-10-09T05:31:31.848375+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:31:31.848375+0000","last_undegraded":"2023-10-09T05:31:31.848375+0000","last_fullsized":"2023-10-09T05:31:31.848375+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855243+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:31:31.855243+0000","last_peered":"2023-10-09T05:31:31.855243+0000","last_clean":"2023-10-09T05:31:31.855243+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:31:31.855243+0000","last_undegraded":"2023-10-09T05:31:31.855243+0000","last_fullsized":"2023-10-09T05:31:31.855243+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866456+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:31:32.866456+0000","last_peered":"2023-10-09T05:31:32.866456+0000","last_clean":"2023-10-09T05:31:32.866456+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:31:32.866456+0000","last_undegraded":"2023-10-09T05:31:32.866456+0000","last_fullsized":"2023-10-09T05:31:32.866456+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":611,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.859682+0000","last_change":"2023-10-09T05:30:30.457812+0000","last_active":"2023-10-09T05:31:31.859682+0000","last_peered":"2023-10-09T05:31:31.859682+0000","last_clean":"2023-10-09T05:31:31.859682+0000","last_became_active":"2023-10-09T05:30:30.456811+0000","last_became_peered":"2023-10-09T05:30:30.456811+0000","last_unstale":"2023-10-09T05:31:31.859682+0000","last_undegraded":"2023-10-09T05:31:31.859682+0000","last_fullsized":"2023-10-09T05:31:31.859682+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:17.322315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869220+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:31:32.869220+0000","last_peered":"2023-10-09T05:31:32.869220+0000","last_clean":"2023-10-09T05:31:32.869220+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:31:32.869220+0000","last_undegraded":"2023-10-09T05:31:32.869220+0000","last_fullsized":"2023-10-09T05:31:32.869220+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870575+0000","last_change":"2023-10-09T05:31:32.870575+0000","last_active":"2023-10-09T05:31:32.870575+0000","last_peered":"2023-10-09T05:31:32.870575+0000","last_clean":"2023-10-09T05:31:32.870575+0000","last_became_active":"2023-10-09T05:31:32.870217+0000","last_became_peered":"2023-10-09T05:31:32.870217+0000","last_unstale":"2023-10-09T05:31:32.870575+0000","last_undegraded":"2023-10-09T05:31:32.870575+0000","last_fullsized":"2023-10-09T05:31:32.870575+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:08.078992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852652+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:31:31.852652+0000","last_peered":"2023-10-09T05:31:31.852652+0000","last_clean":"2023-10-09T05:31:31.852652+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:31:31.852652+0000","last_undegraded":"2023-10-09T05:31:31.852652+0000","last_fullsized":"2023-10-09T05:31:31.852652+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.851950+0000","last_change":"2023-10-09T05:31:02.552208+0000","last_active":"2023-10-09T05:31:31.851950+0000","last_peered":"2023-10-09T05:31:31.851950+0000","last_clean":"2023-10-09T05:31:31.851950+0000","last_became_active":"2023-10-09T05:31:02.551893+0000","last_became_peered":"2023-10-09T05:31:02.551893+0000","last_unstale":"2023-10-09T05:31:31.851950+0000","last_undegraded":"2023-10-09T05:31:31.851950+0000","last_fullsized":"2023-10-09T05:31:31.851950+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852458+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:31:31.852458+0000","last_peered":"2023-10-09T05:31:31.852458+0000","last_clean":"2023-10-09T05:31:31.852458+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:31:31.852458+0000","last_undegraded":"2023-10-09T05:31:31.852458+0000","last_fullsized":"2023-10-09T05:31:31.852458+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855645+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:31:31.855645+0000","last_peered":"2023-10-09T05:31:31.855645+0000","last_clean":"2023-10-09T05:31:31.855645+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:31:31.855645+0000","last_undegraded":"2023-10-09T05:31:31.855645+0000","last_fullsized":"2023-10-09T05:31:31.855645+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864605+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:31:32.864605+0000","last_peered":"2023-10-09T05:31:32.864605+0000","last_clean":"2023-10-09T05:31:32.864605+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:31:32.864605+0000","last_undegraded":"2023-10-09T05:31:32.864605+0000","last_fullsized":"2023-10-09T05:31:32.864605+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862044+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:31:32.862044+0000","last_peered":"2023-10-09T05:31:32.862044+0000","last_clean":"2023-10-09T05:31:32.862044+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:31:32.862044+0000","last_undegraded":"2023-10-09T05:31:32.862044+0000","last_fullsized":"2023-10-09T05:31:32.862044+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.853675+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:31:32.853675+0000","last_peered":"2023-10-09T05:31:32.853675+0000","last_clean":"2023-10-09T05:31:32.853675+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:31:32.853675+0000","last_undegraded":"2023-10-09T05:31:32.853675+0000","last_fullsized":"2023-10-09T05:31:32.853675+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866394+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:31:32.866394+0000","last_peered":"2023-10-09T05:31:32.866394+0000","last_clean":"2023-10-09T05:31:32.866394+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:31:32.866394+0000","last_undegraded":"2023-10-09T05:31:32.866394+0000","last_fullsized":"2023-10-09T05:31:32.866394+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855464+0000","last_change":"2023-10-09T05:31:02.548845+0000","last_active":"2023-10-09T05:31:31.855464+0000","last_peered":"2023-10-09T05:31:31.855464+0000","last_clean":"2023-10-09T05:31:31.855464+0000","last_became_active":"2023-10-09T05:31:02.548131+0000","last_became_peered":"2023-10-09T05:31:02.548131+0000","last_unstale":"2023-10-09T05:31:31.855464+0000","last_undegraded":"2023-10-09T05:31:31.855464+0000","last_fullsized":"2023-10-09T05:31:31.855464+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854508+0000","last_change":"2023-10-09T05:21:16.781147+0000","last_active":"2023-10-09T05:31:31.854508+0000","last_peered":"2023-10-09T05:31:31.854508+0000","last_clean":"2023-10-09T05:31:31.854508+0000","last_became_active":"2023-10-09T05:21:16.780755+0000","last_became_peered":"2023-10-09T05:21:16.780755+0000","last_unstale":"2023-10-09T05:31:31.854508+0000","last_undegraded":"2023-10-09T05:31:31.854508+0000","last_fullsized":"2023-10-09T05:31:31.854508+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869504+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:31:32.869504+0000","last_peered":"2023-10-09T05:31:32.869504+0000","last_clean":"2023-10-09T05:31:32.869504+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:31:32.869504+0000","last_undegraded":"2023-10-09T05:31:32.869504+0000","last_fullsized":"2023-10-09T05:31:32.869504+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.853555+0000","last_change":"2023-10-09T05:30:30.457635+0000","last_active":"2023-10-09T05:31:31.853555+0000","last_peered":"2023-10-09T05:31:31.853555+0000","last_clean":"2023-10-09T05:31:31.853555+0000","last_became_active":"2023-10-09T05:30:30.457276+0000","last_became_peered":"2023-10-09T05:30:30.457276+0000","last_unstale":"2023-10-09T05:31:31.853555+0000","last_undegraded":"2023-10-09T05:31:31.853555+0000","last_fullsized":"2023-10-09T05:31:31.853555+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848299+0000","last_change":"2023-10-09T05:21:17.464398+0000","last_active":"2023-10-09T05:31:31.848299+0000","last_peered":"2023-10-09T05:31:31.848299+0000","last_clean":"2023-10-09T05:31:31.848299+0000","last_became_active":"2023-10-09T05:21:17.464086+0000","last_became_peered":"2023-10-09T05:21:17.464086+0000","last_unstale":"2023-10-09T05:31:31.848299+0000","last_undegraded":"2023-10-09T05:31:31.848299+0000","last_fullsized":"2023-10-09T05:31:31.848299+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855156+0000","last_change":"2023-10-09T05:29:59.470664+0000","last_active":"2023-10-09T05:31:31.855156+0000","last_peered":"2023-10-09T05:31:31.855156+0000","last_clean":"2023-10-09T05:31:31.855156+0000","last_became_active":"2023-10-09T05:29:59.470276+0000","last_became_peered":"2023-10-09T05:29:59.470276+0000","last_unstale":"2023-10-09T05:31:31.855156+0000","last_undegraded":"2023-10-09T05:31:31.855156+0000","last_fullsized":"2023-10-09T05:31:31.855156+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.871093+0000","last_change":"2023-10-09T05:31:32.871093+0000","last_active":"2023-10-09T05:31:32.871093+0000","last_peered":"2023-10-09T05:31:32.871093+0000","last_clean":"2023-10-09T05:31:32.871093+0000","last_became_active":"2023-10-09T05:31:32.870702+0000","last_became_peered":"2023-10-09T05:31:32.870702+0000","last_unstale":"2023-10-09T05:31:32.871093+0000","last_undegraded":"2023-10-09T05:31:32.871093+0000","last_fullsized":"2023-10-09T05:31:32.871093+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:18.439344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.856510+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:31:31.856510+0000","last_peered":"2023-10-09T05:31:31.856510+0000","last_clean":"2023-10-09T05:31:31.856510+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:31:31.856510+0000","last_undegraded":"2023-10-09T05:31:31.856510+0000","last_fullsized":"2023-10-09T05:31:31.856510+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.861092+0000","last_change":"2023-10-09T05:31:32.861092+0000","last_active":"2023-10-09T05:31:32.861092+0000","last_peered":"2023-10-09T05:31:32.861092+0000","last_clean":"2023-10-09T05:31:32.861092+0000","last_became_active":"2023-10-09T05:31:32.860632+0000","last_became_peered":"2023-10-09T05:31:32.860632+0000","last_unstale":"2023-10-09T05:31:32.861092+0000","last_undegraded":"2023-10-09T05:31:32.861092+0000","last_fullsized":"2023-10-09T05:31:32.861092+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:54.308161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852402+0000","last_change":"2023-10-09T05:21:16.784983+0000","last_active":"2023-10-09T05:31:31.852402+0000","last_peered":"2023-10-09T05:31:31.852402+0000","last_clean":"2023-10-09T05:31:31.852402+0000","last_became_active":"2023-10-09T05:21:16.784699+0000","last_became_peered":"2023-10-09T05:21:16.784699+0000","last_unstale":"2023-10-09T05:31:31.852402+0000","last_undegraded":"2023-10-09T05:31:31.852402+0000","last_fullsized":"2023-10-09T05:31:31.852402+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866744+0000","last_change":"2023-10-09T05:31:32.866744+0000","last_active":"2023-10-09T05:31:32.866744+0000","last_peered":"2023-10-09T05:31:32.866744+0000","last_clean":"2023-10-09T05:31:32.866744+0000","last_became_active":"2023-10-09T05:31:32.866389+0000","last_became_peered":"2023-10-09T05:31:32.866389+0000","last_unstale":"2023-10-09T05:31:32.866744+0000","last_undegraded":"2023-10-09T05:31:32.866744+0000","last_fullsized":"2023-10-09T05:31:32.866744+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866600+0000","last_change":"2023-10-09T05:31:32.861963+0000","last_active":"2023-10-09T05:31:32.866600+0000","last_peered":"2023-10-09T05:31:32.866600+0000","last_clean":"2023-10-09T05:31:32.866600+0000","last_became_active":"2023-10-09T05:31:32.861679+0000","last_became_peered":"2023-10-09T05:31:32.861679+0000","last_unstale":"2023-10-09T05:31:32.866600+0000","last_undegraded":"2023-10-09T05:31:32.866600+0000","last_fullsized":"2023-10-09T05:31:32.866600+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:42.402077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.871695+0000","last_change":"2023-10-09T05:31:32.871695+0000","last_active":"2023-10-09T05:31:32.871695+0000","last_peered":"2023-10-09T05:31:32.871695+0000","last_clean":"2023-10-09T05:31:32.871695+0000","last_became_active":"2023-10-09T05:31:32.871449+0000","last_became_peered":"2023-10-09T05:31:32.871449+0000","last_unstale":"2023-10-09T05:31:32.871695+0000","last_undegraded":"2023-10-09T05:31:32.871695+0000","last_fullsized":"2023-10-09T05:31:32.871695+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:22.999187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852684+0000","last_change":"2023-10-09T05:31:02.550860+0000","last_active":"2023-10-09T05:31:31.852684+0000","last_peered":"2023-10-09T05:31:31.852684+0000","last_clean":"2023-10-09T05:31:31.852684+0000","last_became_active":"2023-10-09T05:31:02.550469+0000","last_became_peered":"2023-10-09T05:31:02.550469+0000","last_unstale":"2023-10-09T05:31:31.852684+0000","last_undegraded":"2023-10-09T05:31:31.852684+0000","last_fullsized":"2023-10-09T05:31:31.852684+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848987+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:31:31.848987+0000","last_peered":"2023-10-09T05:31:31.848987+0000","last_clean":"2023-10-09T05:31:31.848987+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:31:31.848987+0000","last_undegraded":"2023-10-09T05:31:31.848987+0000","last_fullsized":"2023-10-09T05:31:31.848987+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850669+0000","last_change":"2023-10-09T05:21:16.778811+0000","last_active":"2023-10-09T05:31:31.850669+0000","last_peered":"2023-10-09T05:31:31.850669+0000","last_clean":"2023-10-09T05:31:31.850669+0000","last_became_active":"2023-10-09T05:21:16.778515+0000","last_became_peered":"2023-10-09T05:21:16.778515+0000","last_unstale":"2023-10-09T05:31:31.850669+0000","last_undegraded":"2023-10-09T05:31:31.850669+0000","last_fullsized":"2023-10-09T05:31:31.850669+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855330+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:31:31.855330+0000","last_peered":"2023-10-09T05:31:31.855330+0000","last_clean":"2023-10-09T05:31:31.855330+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:31:31.855330+0000","last_undegraded":"2023-10-09T05:31:31.855330+0000","last_fullsized":"2023-10-09T05:31:31.855330+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.863294+0000","last_change":"2023-10-09T05:31:32.863294+0000","last_active":"2023-10-09T05:31:32.863294+0000","last_peered":"2023-10-09T05:31:32.863294+0000","last_clean":"2023-10-09T05:31:32.863294+0000","last_became_active":"2023-10-09T05:31:32.862914+0000","last_became_peered":"2023-10-09T05:31:32.862914+0000","last_unstale":"2023-10-09T05:31:32.863294+0000","last_undegraded":"2023-10-09T05:31:32.863294+0000","last_fullsized":"2023-10-09T05:31:32.863294+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.863335+0000","last_change":"2023-10-09T05:31:32.863335+0000","last_active":"2023-10-09T05:31:32.863335+0000","last_peered":"2023-10-09T05:31:32.863335+0000","last_clean":"2023-10-09T05:31:32.863335+0000","last_became_active":"2023-10-09T05:31:32.863036+0000","last_became_peered":"2023-10-09T05:31:32.863036+0000","last_unstale":"2023-10-09T05:31:32.863335+0000","last_undegraded":"2023-10-09T05:31:32.863335+0000","last_fullsized":"2023-10-09T05:31:32.863335+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:59.787997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848623+0000","last_change":"2023-10-09T05:30:30.457540+0000","last_active":"2023-10-09T05:31:31.848623+0000","last_peered":"2023-10-09T05:31:31.848623+0000","last_clean":"2023-10-09T05:31:31.848623+0000","last_became_active":"2023-10-09T05:30:30.457180+0000","last_became_peered":"2023-10-09T05:30:30.457180+0000","last_unstale":"2023-10-09T05:31:31.848623+0000","last_undegraded":"2023-10-09T05:31:31.848623+0000","last_fullsized":"2023-10-09T05:31:31.848623+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870178+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:31:32.870178+0000","last_peered":"2023-10-09T05:31:32.870178+0000","last_clean":"2023-10-09T05:31:32.870178+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:31:32.870178+0000","last_undegraded":"2023-10-09T05:31:32.870178+0000","last_fullsized":"2023-10-09T05:31:32.870178+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864410+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:31:32.864410+0000","last_peered":"2023-10-09T05:31:32.864410+0000","last_clean":"2023-10-09T05:31:32.864410+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:31:32.864410+0000","last_undegraded":"2023-10-09T05:31:32.864410+0000","last_fullsized":"2023-10-09T05:31:32.864410+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855720+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:31:31.855720+0000","last_peered":"2023-10-09T05:31:31.855720+0000","last_clean":"2023-10-09T05:31:31.855720+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:31:31.855720+0000","last_undegraded":"2023-10-09T05:31:31.855720+0000","last_fullsized":"2023-10-09T05:31:31.855720+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.867898+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:31:32.867898+0000","last_peered":"2023-10-09T05:31:32.867898+0000","last_clean":"2023-10-09T05:31:32.867898+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:31:32.867898+0000","last_undegraded":"2023-10-09T05:31:32.867898+0000","last_fullsized":"2023-10-09T05:31:32.867898+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866758+0000","last_change":"2023-10-09T05:31:32.860920+0000","last_active":"2023-10-09T05:31:32.866758+0000","last_peered":"2023-10-09T05:31:32.866758+0000","last_clean":"2023-10-09T05:31:32.866758+0000","last_became_active":"2023-10-09T05:31:32.860244+0000","last_became_peered":"2023-10-09T05:31:32.860244+0000","last_unstale":"2023-10-09T05:31:32.866758+0000","last_undegraded":"2023-10-09T05:31:32.866758+0000","last_fullsized":"2023-10-09T05:31:32.866758+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:37.134584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869382+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:31:32.869382+0000","last_peered":"2023-10-09T05:31:32.869382+0000","last_clean":"2023-10-09T05:31:32.869382+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:31:32.869382+0000","last_undegraded":"2023-10-09T05:31:32.869382+0000","last_fullsized":"2023-10-09T05:31:32.869382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.868031+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:31:32.868031+0000","last_peered":"2023-10-09T05:31:32.868031+0000","last_clean":"2023-10-09T05:31:32.868031+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:31:32.868031+0000","last_undegraded":"2023-10-09T05:31:32.868031+0000","last_fullsized":"2023-10-09T05:31:32.868031+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":87,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850461+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:31:31.850461+0000","last_peered":"2023-10-09T05:31:31.850461+0000","last_clean":"2023-10-09T05:31:31.850461+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:31:31.850461+0000","last_undegraded":"2023-10-09T05:31:31.850461+0000","last_fullsized":"2023-10-09T05:31:31.850461+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26751,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430363,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5368692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:31:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.64000000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.58099999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:32:05 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.72699999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:31:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.59499999999999997,"15min":0.60299999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.75700000000000001,"5min":1.153,"15min":2.6669999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.58199999999999996,"15min":0.58999999999999997},"min":{"1min":0.40000000000000002,"5min":0.39700000000000002,"15min":0.32000000000000001},"max":{"1min":0.76500000000000001,"5min":1.0309999999999999,"15min":2.1379999999999999},"last":0.61599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.49099999999999999,"15min":0.46500000000000002},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.249},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.497},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.52500000000000002,"15min":0.50800000000000001},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27300000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.59799999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.53000000000000003,"15min":0.50700000000000001},"min":{"1min":0.35799999999999998,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":1.462},"last":0.63},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.40200000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88700000000000001},"last":0.67500000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.56699999999999995,"15min":0.54000000000000004},"min":{"1min":0.42299999999999999,"5min":0.34799999999999998,"15min":0.30499999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.623,"5min":0.59599999999999997,"15min":0.58499999999999996},"min":{"1min":0.45000000000000001,"5min":0.372,"15min":0.33300000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.71199999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626591,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306916,"kb_used_data":5724,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438236,"statfs":{"total":95995035648,"available":95680753664,"internally_reserved":0,"allocated":5861376,"data_stored":5368300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.55500000000000005,"15min":0.54500000000000004},"min":{"1min":0.39300000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.014,"5min":0.63800000000000001,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.4569999999999999,"5min":7.4569999999999999,"15min":7.4569999999999999},"last":0.56499999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.65200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.075,"15min":1.075},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.5999999999999996,"15min":7.5999999999999996},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":7.6379999999999999,"5min":7.6379999999999999,"15min":7.6379999999999999},"last":0.61599999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:30:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.55700000000000005,"15min":0.55200000000000005},"min":{"1min":0.38600000000000001,"5min":0.38100000000000001,"15min":0.33300000000000002},"max":{"1min":0.73199999999999998,"5min":0.80700000000000005,"15min":3.9569999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59399999999999997,"15min":0.59499999999999997},"min":{"1min":0.436,"5min":0.34899999999999998,"15min":0.33800000000000002},"max":{"1min":0.79700000000000004,"5min":0.81999999999999995,"15min":2.9049999999999998},"last":0.67300000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.47399999999999998,"15min":0.46100000000000002},"min":{"1min":0.29099999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.86399999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.49399999999999999,"15min":0.47299999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":0.78800000000000003,"5min":0.83799999999999997,"15min":1.014},"last":0.36499999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.53700000000000003,"15min":0.504},"min":{"1min":0.33400000000000002,"5min":0.30499999999999999,"15min":0.27200000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.626},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.51200000000000001},"min":{"1min":0.35999999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.73299999999999998,"5min":0.82399999999999995,"15min":2.0579999999999998},"last":0.50600000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.54200000000000004,"15min":0.53300000000000003},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.60199999999999998,"15min":0.56999999999999995},"min":{"1min":0.498,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.999},"last":0.59399999999999997}]}]},{"osd":1,"up_from":93,"seq":399431958552,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32640,"kb_used_data":1080,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712512,"statfs":{"total":95995035648,"available":95961612288,"internally_reserved":0,"allocated":1105920,"data_stored":587447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":32301582},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.503},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.55500000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.45800000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47199999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.68899999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.67400000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.69999999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.63},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1,"5min":1,"15min":1},"last":0.62}]}]},{"osd":0,"up_from":88,"seq":377957122078,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33228,"kb_used_data":1668,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93711924,"statfs":{"total":95995035648,"available":95961010176,"internally_reserved":0,"allocated":1708032,"data_stored":1178579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":32300808},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.34399999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.32800000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:31:00 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.624}]},{"osd":4,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.65900000000000003,"15min":0.67300000000000004},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.83699999999999997,"5min":1.028,"15min":1.028},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.69999999999999996,"15min":0.71399999999999997},"min":{"1min":0.45300000000000001,"5min":0.434,"15min":0.434},"max":{"1min":0.81799999999999995,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.64300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.65400000000000003,"15min":0.67200000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.70399999999999996,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.65600000000000003,"15min":0.66900000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.79200000000000004,"5min":1.022,"15min":1.022},"last":0.66000000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.68400000000000005,"15min":0.69899999999999995},"min":{"1min":0.37,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80300000000000005,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.66300000000000003,"15min":0.67300000000000004},"min":{"1min":0.42899999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.85599999999999998,"5min":1.044,"15min":1.044},"last":0.49399999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.68999999999999995,"15min":0.70299999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.89600000000000002,"5min":1.052,"15min":1.052},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.621,"5min":0.71299999999999997,"15min":0.72799999999999998},"min":{"1min":0.38500000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77500000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.47099999999999997}]}]},{"osd":2,"up_from":98,"seq":420906795025,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36668,"kb_used_data":5172,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708484,"statfs":{"total":95995035648,"available":95957487616,"internally_reserved":0,"allocated":5296128,"data_stored":4780992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":32232021},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.34999999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.32200000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.56899999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.58199999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.5},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.64200000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.59599999999999997}]}]},{"osd":3,"up_from":103,"seq":442381631499,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28392,"kb_used_data":1056,"kb_used_omap":4,"kb_used_meta":27323,"kb_avail":93716760,"statfs":{"total":95995035648,"available":95965962240,"internally_reserved":0,"allocated":1081344,"data_stored":583603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":27979358},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34000000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.312}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.621}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66400000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76500000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019047,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":583950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52200000000000002,"15min":0.52100000000000002},"min":{"1min":0.373,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73299999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.63}]},{"osd":1,"last update":"Mon Oct 9 05:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.66800000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:32:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.56899999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.58299999999999996,"15min":0.56000000000000005},"min":{"1min":0.38500000000000001,"5min":0.374,"15min":0.28799999999999998},"max":{"1min":0.73899999999999999,"5min":0.84299999999999997,"15min":1.591},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56100000000000005,"15min":0.54700000000000004},"min":{"1min":0.434,"5min":0.373,"15min":0.33600000000000002},"max":{"1min":0.66700000000000004,"5min":0.877,"15min":0.877},"last":0.68000000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:31:27 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46800000000000003,"15min":0.441},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26700000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.85399999999999998},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56100000000000005,"15min":0.52900000000000003},"min":{"1min":0.44700000000000001,"5min":0.36599999999999999,"15min":0.311},"max":{"1min":0.70499999999999996,"5min":0.79600000000000004,"15min":1.54},"last":0.71299999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:31:40 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.48999999999999999,"15min":0.48899999999999999},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.27800000000000002},"max":{"1min":0.74399999999999999,"5min":0.80300000000000005,"15min":1.5660000000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.502,"15min":0.503},"min":{"1min":0.371,"5min":0.33100000000000002,"15min":0.28899999999999998},"max":{"1min":0.752,"5min":0.76300000000000001,"15min":0.94699999999999995},"last":0.72799999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:32:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53500000000000003,"15min":0.53600000000000003},"min":{"1min":0.433,"5min":0.35499999999999998,"15min":0.30499999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":1.6160000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53300000000000003,"15min":0.53000000000000003},"min":{"1min":0.46700000000000003,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":0.76700000000000002,"5min":0.79000000000000004,"15min":1.034},"last":0.56499999999999995}]}]},{"osd":5,"up_from":36,"seq":154618822819,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298136,"kb_used_data":1040,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447016,"statfs":{"total":95995035648,"available":95689744384,"internally_reserved":0,"allocated":1064960,"data_stored":579398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70499999999999996,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.39500000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.59299999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.51200000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.45500000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:30:24 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.60699999999999998,"15min":0.58499999999999996},"min":{"1min":0.47699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.94899999999999995},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.64800000000000002,"15min":0.627},"min":{"1min":0.501,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.85899999999999999,"5min":0.92800000000000005,"15min":0.93899999999999995},"last":0.61599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:31:24 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.48299999999999998,"15min":0.44900000000000001},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":0.69899999999999995,"5min":0.874,"15min":0.94499999999999995},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53600000000000003,"15min":0.48799999999999999},"min":{"1min":0.35499999999999998,"5min":0.33200000000000002,"15min":0.28699999999999998},"max":{"1min":0.70599999999999996,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.70299999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:31:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.55600000000000005,"15min":0.505},"min":{"1min":0.41199999999999998,"5min":0.34999999999999998,"15min":0.26000000000000001},"max":{"1min":0.78500000000000003,"5min":0.93500000000000005,"15min":1.1339999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.56100000000000005,"15min":0.53100000000000003},"min":{"1min":0.42999999999999999,"5min":0.36199999999999999,"15min":0.314},"max":{"1min":0.78800000000000003,"5min":0.92100000000000004,"15min":1.109},"last":0.61099999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:32:10 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.59399999999999997,"15min":0.57699999999999996},"min":{"1min":0.52400000000000002,"5min":0.40200000000000002,"15min":0.34399999999999997},"max":{"1min":0.71499999999999997,"5min":0.95199999999999996,"15min":1.143},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57399999999999995,"15min":0.55600000000000005},"min":{"1min":0.51100000000000001,"5min":0.35199999999999998,"15min":0.32400000000000001},"max":{"1min":0.83499999999999996,"5min":0.90900000000000003,"15min":1.1830000000000001},"last":0.68500000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:32:21.506 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:32:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:21 smithi191 ceph-mon[161669]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:21.902 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:21.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:21 smithi008 ceph-mon[184418]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:21 smithi008 ceph-mon[191227]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:22.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:22 smithi008 ceph-mon[184418]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:22 smithi008 ceph-mon[191227]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:22 smithi191 ceph-mon[161669]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:23.571 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:32:23.573 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:32:23.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:23 smithi191 ceph-mon[161669]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:23.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:23 smithi008 ceph-mon[184418]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:23.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:23 smithi008 ceph-mon[191227]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:24.256 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":222,"stamp":"2023-10-09T05:32:21.854428+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13724,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341060,"kb_used_data":22532,"kb_used_omap":31,"kb_used_meta":1318432,"kb_avail":748620156,"statfs":{"total":767960285184,"available":766587039744,"internally_reserved":0,"allocated":23072768,"data_stored":19030961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":1350074825},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002690"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848734+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:31:31.848734+0000","last_peered":"2023-10-09T05:31:31.848734+0000","last_clean":"2023-10-09T05:31:31.848734+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:31:31.848734+0000","last_undegraded":"2023-10-09T05:31:31.848734+0000","last_fullsized":"2023-10-09T05:31:31.848734+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854909+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:31:32.854909+0000","last_peered":"2023-10-09T05:31:32.854909+0000","last_clean":"2023-10-09T05:31:32.854909+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:31:32.854909+0000","last_undegraded":"2023-10-09T05:31:32.854909+0000","last_fullsized":"2023-10-09T05:31:32.854909+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.860364+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:31:32.860364+0000","last_peered":"2023-10-09T05:31:32.860364+0000","last_clean":"2023-10-09T05:31:32.860364+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:31:32.860364+0000","last_undegraded":"2023-10-09T05:31:32.860364+0000","last_fullsized":"2023-10-09T05:31:32.860364+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864546+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:31:32.864546+0000","last_peered":"2023-10-09T05:31:32.864546+0000","last_clean":"2023-10-09T05:31:32.864546+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:31:32.864546+0000","last_undegraded":"2023-10-09T05:31:32.864546+0000","last_fullsized":"2023-10-09T05:31:32.864546+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.867349+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:31:32.867349+0000","last_peered":"2023-10-09T05:31:32.867349+0000","last_clean":"2023-10-09T05:31:32.867349+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:31:32.867349+0000","last_undegraded":"2023-10-09T05:31:32.867349+0000","last_fullsized":"2023-10-09T05:31:32.867349+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":58,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850233+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:31:31.850233+0000","last_peered":"2023-10-09T05:31:31.850233+0000","last_clean":"2023-10-09T05:31:31.850233+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:31:31.850233+0000","last_undegraded":"2023-10-09T05:31:31.850233+0000","last_fullsized":"2023-10-09T05:31:31.850233+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869382+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:31:32.869382+0000","last_peered":"2023-10-09T05:31:32.869382+0000","last_clean":"2023-10-09T05:31:32.869382+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:31:32.869382+0000","last_undegraded":"2023-10-09T05:31:32.869382+0000","last_fullsized":"2023-10-09T05:31:32.869382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.849363+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:31:31.849363+0000","last_peered":"2023-10-09T05:31:31.849363+0000","last_clean":"2023-10-09T05:31:31.849363+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:31:31.849363+0000","last_undegraded":"2023-10-09T05:31:31.849363+0000","last_fullsized":"2023-10-09T05:31:31.849363+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870694+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:31:32.870694+0000","last_peered":"2023-10-09T05:31:32.870694+0000","last_clean":"2023-10-09T05:31:32.870694+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:31:32.870694+0000","last_undegraded":"2023-10-09T05:31:32.870694+0000","last_fullsized":"2023-10-09T05:31:32.870694+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":40,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.864012+0000","last_change":"2023-10-09T05:29:59.477246+0000","last_active":"2023-10-09T05:31:31.864012+0000","last_peered":"2023-10-09T05:31:31.864012+0000","last_clean":"2023-10-09T05:31:31.864012+0000","last_became_active":"2023-10-09T05:29:59.476390+0000","last_became_peered":"2023-10-09T05:29:59.476390+0000","last_unstale":"2023-10-09T05:31:31.864012+0000","last_undegraded":"2023-10-09T05:31:31.864012+0000","last_fullsized":"2023-10-09T05:31:31.864012+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:52:31.557279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.851843+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:31:31.851843+0000","last_peered":"2023-10-09T05:31:31.851843+0000","last_clean":"2023-10-09T05:31:31.851843+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:31:31.851843+0000","last_undegraded":"2023-10-09T05:31:31.851843+0000","last_fullsized":"2023-10-09T05:31:31.851843+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854621+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:31:32.854621+0000","last_peered":"2023-10-09T05:31:32.854621+0000","last_clean":"2023-10-09T05:31:32.854621+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:31:32.854621+0000","last_undegraded":"2023-10-09T05:31:32.854621+0000","last_fullsized":"2023-10-09T05:31:32.854621+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":552,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855814+0000","last_change":"2023-10-09T05:31:02.549304+0000","last_active":"2023-10-09T05:31:31.855814+0000","last_peered":"2023-10-09T05:31:31.855814+0000","last_clean":"2023-10-09T05:31:31.855814+0000","last_became_active":"2023-10-09T05:31:02.548085+0000","last_became_peered":"2023-10-09T05:31:02.548085+0000","last_unstale":"2023-10-09T05:31:31.855814+0000","last_undegraded":"2023-10-09T05:31:31.855814+0000","last_fullsized":"2023-10-09T05:31:31.855814+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862414+0000","last_change":"2023-10-09T05:31:32.862414+0000","last_active":"2023-10-09T05:31:32.862414+0000","last_peered":"2023-10-09T05:31:32.862414+0000","last_clean":"2023-10-09T05:31:32.862414+0000","last_became_active":"2023-10-09T05:31:32.860985+0000","last_became_peered":"2023-10-09T05:31:32.860985+0000","last_unstale":"2023-10-09T05:31:32.862414+0000","last_undegraded":"2023-10-09T05:31:32.862414+0000","last_fullsized":"2023-10-09T05:31:32.862414+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:04.492757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":488,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855778+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:31:31.855778+0000","last_peered":"2023-10-09T05:31:31.855778+0000","last_clean":"2023-10-09T05:31:31.855778+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:31:31.855778+0000","last_undegraded":"2023-10-09T05:31:31.855778+0000","last_fullsized":"2023-10-09T05:31:31.855778+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862179+0000","last_change":"2023-10-09T05:31:32.862179+0000","last_active":"2023-10-09T05:31:32.862179+0000","last_peered":"2023-10-09T05:31:32.862179+0000","last_clean":"2023-10-09T05:31:32.862179+0000","last_became_active":"2023-10-09T05:31:32.861777+0000","last_became_peered":"2023-10-09T05:31:32.861777+0000","last_unstale":"2023-10-09T05:31:32.862179+0000","last_undegraded":"2023-10-09T05:31:32.862179+0000","last_fullsized":"2023-10-09T05:31:32.862179+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:40.547661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.853486+0000","last_change":"2023-10-09T05:29:59.470093+0000","last_active":"2023-10-09T05:31:32.853486+0000","last_peered":"2023-10-09T05:31:32.853486+0000","last_clean":"2023-10-09T05:31:32.853486+0000","last_became_active":"2023-10-09T05:29:59.469627+0000","last_became_peered":"2023-10-09T05:29:59.469627+0000","last_unstale":"2023-10-09T05:31:32.853486+0000","last_undegraded":"2023-10-09T05:31:32.853486+0000","last_fullsized":"2023-10-09T05:31:32.853486+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:17.087268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.858901+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:31:31.858901+0000","last_peered":"2023-10-09T05:31:31.858901+0000","last_clean":"2023-10-09T05:31:31.858901+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:31:31.858901+0000","last_undegraded":"2023-10-09T05:31:31.858901+0000","last_fullsized":"2023-10-09T05:31:31.858901+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":333,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854396+0000","last_change":"2023-10-09T05:29:59.474688+0000","last_active":"2023-10-09T05:31:31.854396+0000","last_peered":"2023-10-09T05:31:31.854396+0000","last_clean":"2023-10-09T05:31:31.854396+0000","last_became_active":"2023-10-09T05:29:59.474080+0000","last_became_peered":"2023-10-09T05:29:59.474080+0000","last_unstale":"2023-10-09T05:31:31.854396+0000","last_undegraded":"2023-10-09T05:31:31.854396+0000","last_fullsized":"2023-10-09T05:31:31.854396+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854700+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:31:32.854700+0000","last_peered":"2023-10-09T05:31:32.854700+0000","last_clean":"2023-10-09T05:31:32.854700+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:31:32.854700+0000","last_undegraded":"2023-10-09T05:31:32.854700+0000","last_fullsized":"2023-10-09T05:31:32.854700+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866574+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:31:32.866574+0000","last_peered":"2023-10-09T05:31:32.866574+0000","last_clean":"2023-10-09T05:31:32.866574+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:31:32.866574+0000","last_undegraded":"2023-10-09T05:31:32.866574+0000","last_fullsized":"2023-10-09T05:31:32.866574+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":725,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854312+0000","last_change":"2023-10-09T05:30:30.456503+0000","last_active":"2023-10-09T05:31:31.854312+0000","last_peered":"2023-10-09T05:31:31.854312+0000","last_clean":"2023-10-09T05:31:31.854312+0000","last_became_active":"2023-10-09T05:30:30.455426+0000","last_became_peered":"2023-10-09T05:30:30.455426+0000","last_unstale":"2023-10-09T05:31:31.854312+0000","last_undegraded":"2023-10-09T05:31:31.854312+0000","last_fullsized":"2023-10-09T05:31:31.854312+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855635+0000","last_change":"2023-10-09T05:21:17.465038+0000","last_active":"2023-10-09T05:31:31.855635+0000","last_peered":"2023-10-09T05:31:31.855635+0000","last_clean":"2023-10-09T05:31:31.855635+0000","last_became_active":"2023-10-09T05:21:17.464695+0000","last_became_peered":"2023-10-09T05:21:17.464695+0000","last_unstale":"2023-10-09T05:31:31.855635+0000","last_undegraded":"2023-10-09T05:31:31.855635+0000","last_fullsized":"2023-10-09T05:31:31.855635+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864478+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:31:32.864478+0000","last_peered":"2023-10-09T05:31:32.864478+0000","last_clean":"2023-10-09T05:31:32.864478+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:31:32.864478+0000","last_undegraded":"2023-10-09T05:31:32.864478+0000","last_fullsized":"2023-10-09T05:31:32.864478+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":307,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854341+0000","last_change":"2023-10-09T05:29:59.475956+0000","last_active":"2023-10-09T05:31:31.854341+0000","last_peered":"2023-10-09T05:31:31.854341+0000","last_clean":"2023-10-09T05:31:31.854341+0000","last_became_active":"2023-10-09T05:29:59.475408+0000","last_became_peered":"2023-10-09T05:29:59.475408+0000","last_unstale":"2023-10-09T05:31:31.854341+0000","last_undegraded":"2023-10-09T05:31:31.854341+0000","last_fullsized":"2023-10-09T05:31:31.854341+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":665,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848463+0000","last_change":"2023-10-09T05:31:02.547804+0000","last_active":"2023-10-09T05:31:31.848463+0000","last_peered":"2023-10-09T05:31:31.848463+0000","last_clean":"2023-10-09T05:31:31.848463+0000","last_became_active":"2023-10-09T05:31:02.547237+0000","last_became_peered":"2023-10-09T05:31:02.547237+0000","last_unstale":"2023-10-09T05:31:31.848463+0000","last_undegraded":"2023-10-09T05:31:31.848463+0000","last_fullsized":"2023-10-09T05:31:31.848463+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.859446+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:31:31.859446+0000","last_peered":"2023-10-09T05:31:31.859446+0000","last_clean":"2023-10-09T05:31:31.859446+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:31:31.859446+0000","last_undegraded":"2023-10-09T05:31:31.859446+0000","last_fullsized":"2023-10-09T05:31:31.859446+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.859514+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:31:31.859514+0000","last_peered":"2023-10-09T05:31:31.859514+0000","last_clean":"2023-10-09T05:31:31.859514+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:31:31.859514+0000","last_undegraded":"2023-10-09T05:31:31.859514+0000","last_fullsized":"2023-10-09T05:31:31.859514+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":414,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.858194+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:31:32.858194+0000","last_peered":"2023-10-09T05:31:32.858194+0000","last_clean":"2023-10-09T05:31:32.858194+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:31:32.858194+0000","last_undegraded":"2023-10-09T05:31:32.858194+0000","last_fullsized":"2023-10-09T05:31:32.858194+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866540+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:31:32.866540+0000","last_peered":"2023-10-09T05:31:32.866540+0000","last_clean":"2023-10-09T05:31:32.866540+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:31:32.866540+0000","last_undegraded":"2023-10-09T05:31:32.866540+0000","last_fullsized":"2023-10-09T05:31:32.866540+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.849147+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:31:31.849147+0000","last_peered":"2023-10-09T05:31:31.849147+0000","last_clean":"2023-10-09T05:31:31.849147+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:31:31.849147+0000","last_undegraded":"2023-10-09T05:31:31.849147+0000","last_fullsized":"2023-10-09T05:31:31.849147+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":540,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848375+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:31:31.848375+0000","last_peered":"2023-10-09T05:31:31.848375+0000","last_clean":"2023-10-09T05:31:31.848375+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:31:31.848375+0000","last_undegraded":"2023-10-09T05:31:31.848375+0000","last_fullsized":"2023-10-09T05:31:31.848375+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855243+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:31:31.855243+0000","last_peered":"2023-10-09T05:31:31.855243+0000","last_clean":"2023-10-09T05:31:31.855243+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:31:31.855243+0000","last_undegraded":"2023-10-09T05:31:31.855243+0000","last_fullsized":"2023-10-09T05:31:31.855243+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866456+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:31:32.866456+0000","last_peered":"2023-10-09T05:31:32.866456+0000","last_clean":"2023-10-09T05:31:32.866456+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:31:32.866456+0000","last_undegraded":"2023-10-09T05:31:32.866456+0000","last_fullsized":"2023-10-09T05:31:32.866456+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":611,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.859682+0000","last_change":"2023-10-09T05:30:30.457812+0000","last_active":"2023-10-09T05:31:31.859682+0000","last_peered":"2023-10-09T05:31:31.859682+0000","last_clean":"2023-10-09T05:31:31.859682+0000","last_became_active":"2023-10-09T05:30:30.456811+0000","last_became_peered":"2023-10-09T05:30:30.456811+0000","last_unstale":"2023-10-09T05:31:31.859682+0000","last_undegraded":"2023-10-09T05:31:31.859682+0000","last_fullsized":"2023-10-09T05:31:31.859682+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:17.322315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869220+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:31:32.869220+0000","last_peered":"2023-10-09T05:31:32.869220+0000","last_clean":"2023-10-09T05:31:32.869220+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:31:32.869220+0000","last_undegraded":"2023-10-09T05:31:32.869220+0000","last_fullsized":"2023-10-09T05:31:32.869220+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870575+0000","last_change":"2023-10-09T05:31:32.870575+0000","last_active":"2023-10-09T05:31:32.870575+0000","last_peered":"2023-10-09T05:31:32.870575+0000","last_clean":"2023-10-09T05:31:32.870575+0000","last_became_active":"2023-10-09T05:31:32.870217+0000","last_became_peered":"2023-10-09T05:31:32.870217+0000","last_unstale":"2023-10-09T05:31:32.870575+0000","last_undegraded":"2023-10-09T05:31:32.870575+0000","last_fullsized":"2023-10-09T05:31:32.870575+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:08.078992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852652+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:31:31.852652+0000","last_peered":"2023-10-09T05:31:31.852652+0000","last_clean":"2023-10-09T05:31:31.852652+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:31:31.852652+0000","last_undegraded":"2023-10-09T05:31:31.852652+0000","last_fullsized":"2023-10-09T05:31:31.852652+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.851950+0000","last_change":"2023-10-09T05:31:02.552208+0000","last_active":"2023-10-09T05:31:31.851950+0000","last_peered":"2023-10-09T05:31:31.851950+0000","last_clean":"2023-10-09T05:31:31.851950+0000","last_became_active":"2023-10-09T05:31:02.551893+0000","last_became_peered":"2023-10-09T05:31:02.551893+0000","last_unstale":"2023-10-09T05:31:31.851950+0000","last_undegraded":"2023-10-09T05:31:31.851950+0000","last_fullsized":"2023-10-09T05:31:31.851950+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852458+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:31:31.852458+0000","last_peered":"2023-10-09T05:31:31.852458+0000","last_clean":"2023-10-09T05:31:31.852458+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:31:31.852458+0000","last_undegraded":"2023-10-09T05:31:31.852458+0000","last_fullsized":"2023-10-09T05:31:31.852458+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855645+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:31:31.855645+0000","last_peered":"2023-10-09T05:31:31.855645+0000","last_clean":"2023-10-09T05:31:31.855645+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:31:31.855645+0000","last_undegraded":"2023-10-09T05:31:31.855645+0000","last_fullsized":"2023-10-09T05:31:31.855645+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864605+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:31:32.864605+0000","last_peered":"2023-10-09T05:31:32.864605+0000","last_clean":"2023-10-09T05:31:32.864605+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:31:32.864605+0000","last_undegraded":"2023-10-09T05:31:32.864605+0000","last_fullsized":"2023-10-09T05:31:32.864605+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862044+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:31:32.862044+0000","last_peered":"2023-10-09T05:31:32.862044+0000","last_clean":"2023-10-09T05:31:32.862044+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:31:32.862044+0000","last_undegraded":"2023-10-09T05:31:32.862044+0000","last_fullsized":"2023-10-09T05:31:32.862044+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.853675+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:31:32.853675+0000","last_peered":"2023-10-09T05:31:32.853675+0000","last_clean":"2023-10-09T05:31:32.853675+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:31:32.853675+0000","last_undegraded":"2023-10-09T05:31:32.853675+0000","last_fullsized":"2023-10-09T05:31:32.853675+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866394+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:31:32.866394+0000","last_peered":"2023-10-09T05:31:32.866394+0000","last_clean":"2023-10-09T05:31:32.866394+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:31:32.866394+0000","last_undegraded":"2023-10-09T05:31:32.866394+0000","last_fullsized":"2023-10-09T05:31:32.866394+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855464+0000","last_change":"2023-10-09T05:31:02.548845+0000","last_active":"2023-10-09T05:31:31.855464+0000","last_peered":"2023-10-09T05:31:31.855464+0000","last_clean":"2023-10-09T05:31:31.855464+0000","last_became_active":"2023-10-09T05:31:02.548131+0000","last_became_peered":"2023-10-09T05:31:02.548131+0000","last_unstale":"2023-10-09T05:31:31.855464+0000","last_undegraded":"2023-10-09T05:31:31.855464+0000","last_fullsized":"2023-10-09T05:31:31.855464+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854508+0000","last_change":"2023-10-09T05:21:16.781147+0000","last_active":"2023-10-09T05:31:31.854508+0000","last_peered":"2023-10-09T05:31:31.854508+0000","last_clean":"2023-10-09T05:31:31.854508+0000","last_became_active":"2023-10-09T05:21:16.780755+0000","last_became_peered":"2023-10-09T05:21:16.780755+0000","last_unstale":"2023-10-09T05:31:31.854508+0000","last_undegraded":"2023-10-09T05:31:31.854508+0000","last_fullsized":"2023-10-09T05:31:31.854508+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869504+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:31:32.869504+0000","last_peered":"2023-10-09T05:31:32.869504+0000","last_clean":"2023-10-09T05:31:32.869504+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:31:32.869504+0000","last_undegraded":"2023-10-09T05:31:32.869504+0000","last_fullsized":"2023-10-09T05:31:32.869504+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.853555+0000","last_change":"2023-10-09T05:30:30.457635+0000","last_active":"2023-10-09T05:31:31.853555+0000","last_peered":"2023-10-09T05:31:31.853555+0000","last_clean":"2023-10-09T05:31:31.853555+0000","last_became_active":"2023-10-09T05:30:30.457276+0000","last_became_peered":"2023-10-09T05:30:30.457276+0000","last_unstale":"2023-10-09T05:31:31.853555+0000","last_undegraded":"2023-10-09T05:31:31.853555+0000","last_fullsized":"2023-10-09T05:31:31.853555+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848299+0000","last_change":"2023-10-09T05:21:17.464398+0000","last_active":"2023-10-09T05:31:31.848299+0000","last_peered":"2023-10-09T05:31:31.848299+0000","last_clean":"2023-10-09T05:31:31.848299+0000","last_became_active":"2023-10-09T05:21:17.464086+0000","last_became_peered":"2023-10-09T05:21:17.464086+0000","last_unstale":"2023-10-09T05:31:31.848299+0000","last_undegraded":"2023-10-09T05:31:31.848299+0000","last_fullsized":"2023-10-09T05:31:31.848299+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855156+0000","last_change":"2023-10-09T05:29:59.470664+0000","last_active":"2023-10-09T05:31:31.855156+0000","last_peered":"2023-10-09T05:31:31.855156+0000","last_clean":"2023-10-09T05:31:31.855156+0000","last_became_active":"2023-10-09T05:29:59.470276+0000","last_became_peered":"2023-10-09T05:29:59.470276+0000","last_unstale":"2023-10-09T05:31:31.855156+0000","last_undegraded":"2023-10-09T05:31:31.855156+0000","last_fullsized":"2023-10-09T05:31:31.855156+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.871093+0000","last_change":"2023-10-09T05:31:32.871093+0000","last_active":"2023-10-09T05:31:32.871093+0000","last_peered":"2023-10-09T05:31:32.871093+0000","last_clean":"2023-10-09T05:31:32.871093+0000","last_became_active":"2023-10-09T05:31:32.870702+0000","last_became_peered":"2023-10-09T05:31:32.870702+0000","last_unstale":"2023-10-09T05:31:32.871093+0000","last_undegraded":"2023-10-09T05:31:32.871093+0000","last_fullsized":"2023-10-09T05:31:32.871093+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:18.439344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.856510+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:31:31.856510+0000","last_peered":"2023-10-09T05:31:31.856510+0000","last_clean":"2023-10-09T05:31:31.856510+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:31:31.856510+0000","last_undegraded":"2023-10-09T05:31:31.856510+0000","last_fullsized":"2023-10-09T05:31:31.856510+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.861092+0000","last_change":"2023-10-09T05:31:32.861092+0000","last_active":"2023-10-09T05:31:32.861092+0000","last_peered":"2023-10-09T05:31:32.861092+0000","last_clean":"2023-10-09T05:31:32.861092+0000","last_became_active":"2023-10-09T05:31:32.860632+0000","last_became_peered":"2023-10-09T05:31:32.860632+0000","last_unstale":"2023-10-09T05:31:32.861092+0000","last_undegraded":"2023-10-09T05:31:32.861092+0000","last_fullsized":"2023-10-09T05:31:32.861092+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:54.308161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852402+0000","last_change":"2023-10-09T05:21:16.784983+0000","last_active":"2023-10-09T05:31:31.852402+0000","last_peered":"2023-10-09T05:31:31.852402+0000","last_clean":"2023-10-09T05:31:31.852402+0000","last_became_active":"2023-10-09T05:21:16.784699+0000","last_became_peered":"2023-10-09T05:21:16.784699+0000","last_unstale":"2023-10-09T05:31:31.852402+0000","last_undegraded":"2023-10-09T05:31:31.852402+0000","last_fullsized":"2023-10-09T05:31:31.852402+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866744+0000","last_change":"2023-10-09T05:31:32.866744+0000","last_active":"2023-10-09T05:31:32.866744+0000","last_peered":"2023-10-09T05:31:32.866744+0000","last_clean":"2023-10-09T05:31:32.866744+0000","last_became_active":"2023-10-09T05:31:32.866389+0000","last_became_peered":"2023-10-09T05:31:32.866389+0000","last_unstale":"2023-10-09T05:31:32.866744+0000","last_undegraded":"2023-10-09T05:31:32.866744+0000","last_fullsized":"2023-10-09T05:31:32.866744+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866600+0000","last_change":"2023-10-09T05:31:32.861963+0000","last_active":"2023-10-09T05:31:32.866600+0000","last_peered":"2023-10-09T05:31:32.866600+0000","last_clean":"2023-10-09T05:31:32.866600+0000","last_became_active":"2023-10-09T05:31:32.861679+0000","last_became_peered":"2023-10-09T05:31:32.861679+0000","last_unstale":"2023-10-09T05:31:32.866600+0000","last_undegraded":"2023-10-09T05:31:32.866600+0000","last_fullsized":"2023-10-09T05:31:32.866600+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:42.402077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.871695+0000","last_change":"2023-10-09T05:31:32.871695+0000","last_active":"2023-10-09T05:31:32.871695+0000","last_peered":"2023-10-09T05:31:32.871695+0000","last_clean":"2023-10-09T05:31:32.871695+0000","last_became_active":"2023-10-09T05:31:32.871449+0000","last_became_peered":"2023-10-09T05:31:32.871449+0000","last_unstale":"2023-10-09T05:31:32.871695+0000","last_undegraded":"2023-10-09T05:31:32.871695+0000","last_fullsized":"2023-10-09T05:31:32.871695+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:22.999187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852684+0000","last_change":"2023-10-09T05:31:02.550860+0000","last_active":"2023-10-09T05:31:31.852684+0000","last_peered":"2023-10-09T05:31:31.852684+0000","last_clean":"2023-10-09T05:31:31.852684+0000","last_became_active":"2023-10-09T05:31:02.550469+0000","last_became_peered":"2023-10-09T05:31:02.550469+0000","last_unstale":"2023-10-09T05:31:31.852684+0000","last_undegraded":"2023-10-09T05:31:31.852684+0000","last_fullsized":"2023-10-09T05:31:31.852684+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848987+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:31:31.848987+0000","last_peered":"2023-10-09T05:31:31.848987+0000","last_clean":"2023-10-09T05:31:31.848987+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:31:31.848987+0000","last_undegraded":"2023-10-09T05:31:31.848987+0000","last_fullsized":"2023-10-09T05:31:31.848987+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850669+0000","last_change":"2023-10-09T05:21:16.778811+0000","last_active":"2023-10-09T05:31:31.850669+0000","last_peered":"2023-10-09T05:31:31.850669+0000","last_clean":"2023-10-09T05:31:31.850669+0000","last_became_active":"2023-10-09T05:21:16.778515+0000","last_became_peered":"2023-10-09T05:21:16.778515+0000","last_unstale":"2023-10-09T05:31:31.850669+0000","last_undegraded":"2023-10-09T05:31:31.850669+0000","last_fullsized":"2023-10-09T05:31:31.850669+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855330+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:31:31.855330+0000","last_peered":"2023-10-09T05:31:31.855330+0000","last_clean":"2023-10-09T05:31:31.855330+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:31:31.855330+0000","last_undegraded":"2023-10-09T05:31:31.855330+0000","last_fullsized":"2023-10-09T05:31:31.855330+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.863294+0000","last_change":"2023-10-09T05:31:32.863294+0000","last_active":"2023-10-09T05:31:32.863294+0000","last_peered":"2023-10-09T05:31:32.863294+0000","last_clean":"2023-10-09T05:31:32.863294+0000","last_became_active":"2023-10-09T05:31:32.862914+0000","last_became_peered":"2023-10-09T05:31:32.862914+0000","last_unstale":"2023-10-09T05:31:32.863294+0000","last_undegraded":"2023-10-09T05:31:32.863294+0000","last_fullsized":"2023-10-09T05:31:32.863294+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.863335+0000","last_change":"2023-10-09T05:31:32.863335+0000","last_active":"2023-10-09T05:31:32.863335+0000","last_peered":"2023-10-09T05:31:32.863335+0000","last_clean":"2023-10-09T05:31:32.863335+0000","last_became_active":"2023-10-09T05:31:32.863036+0000","last_became_peered":"2023-10-09T05:31:32.863036+0000","last_unstale":"2023-10-09T05:31:32.863335+0000","last_undegraded":"2023-10-09T05:31:32.863335+0000","last_fullsized":"2023-10-09T05:31:32.863335+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:59.787997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848623+0000","last_change":"2023-10-09T05:30:30.457540+0000","last_active":"2023-10-09T05:31:31.848623+0000","last_peered":"2023-10-09T05:31:31.848623+0000","last_clean":"2023-10-09T05:31:31.848623+0000","last_became_active":"2023-10-09T05:30:30.457180+0000","last_became_peered":"2023-10-09T05:30:30.457180+0000","last_unstale":"2023-10-09T05:31:31.848623+0000","last_undegraded":"2023-10-09T05:31:31.848623+0000","last_fullsized":"2023-10-09T05:31:31.848623+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870178+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:31:32.870178+0000","last_peered":"2023-10-09T05:31:32.870178+0000","last_clean":"2023-10-09T05:31:32.870178+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:31:32.870178+0000","last_undegraded":"2023-10-09T05:31:32.870178+0000","last_fullsized":"2023-10-09T05:31:32.870178+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864410+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:31:32.864410+0000","last_peered":"2023-10-09T05:31:32.864410+0000","last_clean":"2023-10-09T05:31:32.864410+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:31:32.864410+0000","last_undegraded":"2023-10-09T05:31:32.864410+0000","last_fullsized":"2023-10-09T05:31:32.864410+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855720+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:31:31.855720+0000","last_peered":"2023-10-09T05:31:31.855720+0000","last_clean":"2023-10-09T05:31:31.855720+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:31:31.855720+0000","last_undegraded":"2023-10-09T05:31:31.855720+0000","last_fullsized":"2023-10-09T05:31:31.855720+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.867898+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:31:32.867898+0000","last_peered":"2023-10-09T05:31:32.867898+0000","last_clean":"2023-10-09T05:31:32.867898+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:31:32.867898+0000","last_undegraded":"2023-10-09T05:31:32.867898+0000","last_fullsized":"2023-10-09T05:31:32.867898+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866758+0000","last_change":"2023-10-09T05:31:32.860920+0000","last_active":"2023-10-09T05:31:32.866758+0000","last_peered":"2023-10-09T05:31:32.866758+0000","last_clean":"2023-10-09T05:31:32.866758+0000","last_became_active":"2023-10-09T05:31:32.860244+0000","last_became_peered":"2023-10-09T05:31:32.860244+0000","last_unstale":"2023-10-09T05:31:32.866758+0000","last_undegraded":"2023-10-09T05:31:32.866758+0000","last_fullsized":"2023-10-09T05:31:32.866758+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:37.134584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869382+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:31:32.869382+0000","last_peered":"2023-10-09T05:31:32.869382+0000","last_clean":"2023-10-09T05:31:32.869382+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:31:32.869382+0000","last_undegraded":"2023-10-09T05:31:32.869382+0000","last_fullsized":"2023-10-09T05:31:32.869382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.868031+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:31:32.868031+0000","last_peered":"2023-10-09T05:31:32.868031+0000","last_clean":"2023-10-09T05:31:32.868031+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:31:32.868031+0000","last_undegraded":"2023-10-09T05:31:32.868031+0000","last_fullsized":"2023-10-09T05:31:32.868031+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":87,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850461+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:31:31.850461+0000","last_peered":"2023-10-09T05:31:31.850461+0000","last_clean":"2023-10-09T05:31:31.850461+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:31:31.850461+0000","last_undegraded":"2023-10-09T05:31:31.850461+0000","last_fullsized":"2023-10-09T05:31:31.850461+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26751,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430363,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5368692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:31:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.64000000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.58099999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:32:05 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.72699999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:31:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.59499999999999997,"15min":0.60299999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.75700000000000001,"5min":1.153,"15min":2.6669999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.58199999999999996,"15min":0.58999999999999997},"min":{"1min":0.40000000000000002,"5min":0.39700000000000002,"15min":0.32000000000000001},"max":{"1min":0.76500000000000001,"5min":1.0309999999999999,"15min":2.1379999999999999},"last":0.61599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.49099999999999999,"15min":0.46500000000000002},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.249},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.497},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.52500000000000002,"15min":0.50800000000000001},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27300000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.59799999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.53000000000000003,"15min":0.50700000000000001},"min":{"1min":0.35799999999999998,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":1.462},"last":0.63},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.40200000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88700000000000001},"last":0.67500000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.56699999999999995,"15min":0.54000000000000004},"min":{"1min":0.42299999999999999,"5min":0.34799999999999998,"15min":0.30499999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.623,"5min":0.59599999999999997,"15min":0.58499999999999996},"min":{"1min":0.45000000000000001,"5min":0.372,"15min":0.33300000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.71199999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626591,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306916,"kb_used_data":5724,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438236,"statfs":{"total":95995035648,"available":95680753664,"internally_reserved":0,"allocated":5861376,"data_stored":5368300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.55500000000000005,"15min":0.54500000000000004},"min":{"1min":0.39300000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.014,"5min":0.63800000000000001,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.4569999999999999,"5min":7.4569999999999999,"15min":7.4569999999999999},"last":0.56499999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.65200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.075,"15min":1.075},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.5999999999999996,"15min":7.5999999999999996},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":7.6379999999999999,"5min":7.6379999999999999,"15min":7.6379999999999999},"last":0.61599999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:30:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.55700000000000005,"15min":0.55200000000000005},"min":{"1min":0.38600000000000001,"5min":0.38100000000000001,"15min":0.33300000000000002},"max":{"1min":0.73199999999999998,"5min":0.80700000000000005,"15min":3.9569999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59399999999999997,"15min":0.59499999999999997},"min":{"1min":0.436,"5min":0.34899999999999998,"15min":0.33800000000000002},"max":{"1min":0.79700000000000004,"5min":0.81999999999999995,"15min":2.9049999999999998},"last":0.67300000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.47399999999999998,"15min":0.46100000000000002},"min":{"1min":0.29099999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.86399999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.49399999999999999,"15min":0.47299999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":0.78800000000000003,"5min":0.83799999999999997,"15min":1.014},"last":0.36499999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.53700000000000003,"15min":0.504},"min":{"1min":0.33400000000000002,"5min":0.30499999999999999,"15min":0.27200000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.626},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.51200000000000001},"min":{"1min":0.35999999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.73299999999999998,"5min":0.82399999999999995,"15min":2.0579999999999998},"last":0.50600000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.54200000000000004,"15min":0.53300000000000003},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.60199999999999998,"15min":0.56999999999999995},"min":{"1min":0.498,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.999},"last":0.59399999999999997}]}]},{"osd":1,"up_from":93,"seq":399431958552,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32640,"kb_used_data":1080,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712512,"statfs":{"total":95995035648,"available":95961612288,"internally_reserved":0,"allocated":1105920,"data_stored":587447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":32301582},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.503},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.55500000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.45800000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47199999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.68899999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.67400000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.69999999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.63},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1,"5min":1,"15min":1},"last":0.62}]}]},{"osd":0,"up_from":88,"seq":377957122078,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33228,"kb_used_data":1668,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93711924,"statfs":{"total":95995035648,"available":95961010176,"internally_reserved":0,"allocated":1708032,"data_stored":1178579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":32300808},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.34399999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.32800000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:31:00 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.624}]},{"osd":4,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.65900000000000003,"15min":0.67300000000000004},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.83699999999999997,"5min":1.028,"15min":1.028},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.69999999999999996,"15min":0.71399999999999997},"min":{"1min":0.45300000000000001,"5min":0.434,"15min":0.434},"max":{"1min":0.81799999999999995,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.64300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.65400000000000003,"15min":0.67200000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.70399999999999996,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.65600000000000003,"15min":0.66900000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.79200000000000004,"5min":1.022,"15min":1.022},"last":0.66000000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.68400000000000005,"15min":0.69899999999999995},"min":{"1min":0.37,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80300000000000005,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.66300000000000003,"15min":0.67300000000000004},"min":{"1min":0.42899999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.85599999999999998,"5min":1.044,"15min":1.044},"last":0.49399999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.68999999999999995,"15min":0.70299999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.89600000000000002,"5min":1.052,"15min":1.052},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.621,"5min":0.71299999999999997,"15min":0.72799999999999998},"min":{"1min":0.38500000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77500000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.47099999999999997}]}]},{"osd":2,"up_from":98,"seq":420906795026,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36668,"kb_used_data":5172,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708484,"statfs":{"total":95995035648,"available":95957487616,"internally_reserved":0,"allocated":5296128,"data_stored":4780992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":32232021},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.246},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.35399999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.32000000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39500000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.60799999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.51700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.61699999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.64200000000000002}]}]},{"osd":3,"up_from":103,"seq":442381631500,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28392,"kb_used_data":1056,"kb_used_omap":4,"kb_used_meta":27323,"kb_avail":93716760,"statfs":{"total":95995035648,"available":95965962240,"internally_reserved":0,"allocated":1081344,"data_stored":583603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":27979358},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30099999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38100000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56100000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59099999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]}]},{"osd":4,"up_from":30,"seq":128849019047,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":583950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52200000000000002,"15min":0.52100000000000002},"min":{"1min":0.373,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73299999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.63}]},{"osd":1,"last update":"Mon Oct 9 05:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.66800000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:32:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.56899999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.58299999999999996,"15min":0.56000000000000005},"min":{"1min":0.38500000000000001,"5min":0.374,"15min":0.28799999999999998},"max":{"1min":0.73899999999999999,"5min":0.84299999999999997,"15min":1.591},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56100000000000005,"15min":0.54700000000000004},"min":{"1min":0.434,"5min":0.373,"15min":0.33600000000000002},"max":{"1min":0.66700000000000004,"5min":0.877,"15min":0.877},"last":0.68000000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:31:27 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46800000000000003,"15min":0.441},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26700000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.85399999999999998},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56100000000000005,"15min":0.52900000000000003},"min":{"1min":0.44700000000000001,"5min":0.36599999999999999,"15min":0.311},"max":{"1min":0.70499999999999996,"5min":0.79600000000000004,"15min":1.54},"last":0.71299999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:31:40 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.48999999999999999,"15min":0.48899999999999999},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.27800000000000002},"max":{"1min":0.74399999999999999,"5min":0.80300000000000005,"15min":1.5660000000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.502,"15min":0.503},"min":{"1min":0.371,"5min":0.33100000000000002,"15min":0.28899999999999998},"max":{"1min":0.752,"5min":0.76300000000000001,"15min":0.94699999999999995},"last":0.72799999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:32:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53500000000000003,"15min":0.53600000000000003},"min":{"1min":0.433,"5min":0.35499999999999998,"15min":0.30499999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":1.6160000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53300000000000003,"15min":0.53000000000000003},"min":{"1min":0.46700000000000003,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":0.76700000000000002,"5min":0.79000000000000004,"15min":1.034},"last":0.56499999999999995}]}]},{"osd":5,"up_from":36,"seq":154618822819,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298136,"kb_used_data":1040,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447016,"statfs":{"total":95995035648,"available":95689744384,"internally_reserved":0,"allocated":1064960,"data_stored":579398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70499999999999996,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.39500000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.59299999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.51200000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.45500000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:30:24 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.60699999999999998,"15min":0.58499999999999996},"min":{"1min":0.47699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.94899999999999995},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.64800000000000002,"15min":0.627},"min":{"1min":0.501,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.85899999999999999,"5min":0.92800000000000005,"15min":0.93899999999999995},"last":0.61599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:31:24 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.48299999999999998,"15min":0.44900000000000001},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":0.69899999999999995,"5min":0.874,"15min":0.94499999999999995},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53600000000000003,"15min":0.48799999999999999},"min":{"1min":0.35499999999999998,"5min":0.33200000000000002,"15min":0.28699999999999998},"max":{"1min":0.70599999999999996,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.70299999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:31:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.55600000000000005,"15min":0.505},"min":{"1min":0.41199999999999998,"5min":0.34999999999999998,"15min":0.26000000000000001},"max":{"1min":0.78500000000000003,"5min":0.93500000000000005,"15min":1.1339999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.56100000000000005,"15min":0.53100000000000003},"min":{"1min":0.42999999999999999,"5min":0.36199999999999999,"15min":0.314},"max":{"1min":0.78800000000000003,"5min":0.92100000000000004,"15min":1.109},"last":0.61099999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:32:10 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.59399999999999997,"15min":0.57699999999999996},"min":{"1min":0.52400000000000002,"5min":0.40200000000000002,"15min":0.34399999999999997},"max":{"1min":0.71499999999999997,"5min":0.95199999999999996,"15min":1.143},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57399999999999995,"15min":0.55600000000000005},"min":{"1min":0.51100000000000001,"5min":0.35199999999999998,"15min":0.32400000000000001},"max":{"1min":0.83499999999999996,"5min":0.90900000000000003,"15min":1.1830000000000001},"last":0.68500000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:32:24.261 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:32:24.261 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T05:32:24.261 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T05:32:24.262 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T05:32:24.622 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:25 smithi191 ceph-mon[161669]: from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:25 smithi191 ceph-mon[161669]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:25.810 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:25 smithi008 ceph-mon[184418]: from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:25.810 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:25 smithi008 ceph-mon[184418]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:25.810 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:25 smithi008 ceph-mon[191227]: from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:25.810 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:25 smithi008 ceph-mon[191227]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:26.269 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:32:26.269 INFO:teuthology.orchestra.run.smithi008.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T05:32:26.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3843777012' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:32:26.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3843777012' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:32:26.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3843777012' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:32:26.831 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T05:32:26.831 INFO:teuthology.run_tasks:Running task print... 2023-10-09T05:32:26.842 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T05:32:26.842 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T05:32:26.859 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T05:32:26.860 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T05:32:26.860 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T05:32:26.861 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T05:32:26.861 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T05:32:26.862 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T05:32:26.862 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T05:32:26.862 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:32:26.862 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:32:27.220 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:27.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:27 smithi008 ceph-mon[184418]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:27.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:27 smithi008 ceph-mon[191227]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:27 smithi191 ceph-mon[161669]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:28.751 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:32:28.753 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:32:29.330 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":225,"stamp":"2023-10-09T05:32:27.856201+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13724,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341060,"kb_used_data":22532,"kb_used_omap":31,"kb_used_meta":1318432,"kb_avail":748620156,"statfs":{"total":767960285184,"available":766587039744,"internally_reserved":0,"allocated":23072768,"data_stored":19030961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":1350074825},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003266"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848734+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:31:31.848734+0000","last_peered":"2023-10-09T05:31:31.848734+0000","last_clean":"2023-10-09T05:31:31.848734+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:31:31.848734+0000","last_undegraded":"2023-10-09T05:31:31.848734+0000","last_fullsized":"2023-10-09T05:31:31.848734+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854909+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:31:32.854909+0000","last_peered":"2023-10-09T05:31:32.854909+0000","last_clean":"2023-10-09T05:31:32.854909+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:31:32.854909+0000","last_undegraded":"2023-10-09T05:31:32.854909+0000","last_fullsized":"2023-10-09T05:31:32.854909+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.860364+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:31:32.860364+0000","last_peered":"2023-10-09T05:31:32.860364+0000","last_clean":"2023-10-09T05:31:32.860364+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:31:32.860364+0000","last_undegraded":"2023-10-09T05:31:32.860364+0000","last_fullsized":"2023-10-09T05:31:32.860364+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864546+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:31:32.864546+0000","last_peered":"2023-10-09T05:31:32.864546+0000","last_clean":"2023-10-09T05:31:32.864546+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:31:32.864546+0000","last_undegraded":"2023-10-09T05:31:32.864546+0000","last_fullsized":"2023-10-09T05:31:32.864546+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.867349+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:31:32.867349+0000","last_peered":"2023-10-09T05:31:32.867349+0000","last_clean":"2023-10-09T05:31:32.867349+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:31:32.867349+0000","last_undegraded":"2023-10-09T05:31:32.867349+0000","last_fullsized":"2023-10-09T05:31:32.867349+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":58,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850233+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:31:31.850233+0000","last_peered":"2023-10-09T05:31:31.850233+0000","last_clean":"2023-10-09T05:31:31.850233+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:31:31.850233+0000","last_undegraded":"2023-10-09T05:31:31.850233+0000","last_fullsized":"2023-10-09T05:31:31.850233+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869382+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:31:32.869382+0000","last_peered":"2023-10-09T05:31:32.869382+0000","last_clean":"2023-10-09T05:31:32.869382+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:31:32.869382+0000","last_undegraded":"2023-10-09T05:31:32.869382+0000","last_fullsized":"2023-10-09T05:31:32.869382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.849363+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:31:31.849363+0000","last_peered":"2023-10-09T05:31:31.849363+0000","last_clean":"2023-10-09T05:31:31.849363+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:31:31.849363+0000","last_undegraded":"2023-10-09T05:31:31.849363+0000","last_fullsized":"2023-10-09T05:31:31.849363+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870694+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:31:32.870694+0000","last_peered":"2023-10-09T05:31:32.870694+0000","last_clean":"2023-10-09T05:31:32.870694+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:31:32.870694+0000","last_undegraded":"2023-10-09T05:31:32.870694+0000","last_fullsized":"2023-10-09T05:31:32.870694+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":41,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.853762+0000","last_change":"2023-10-09T05:29:59.477246+0000","last_active":"2023-10-09T05:31:32.853762+0000","last_peered":"2023-10-09T05:31:32.853762+0000","last_clean":"2023-10-09T05:31:32.853762+0000","last_became_active":"2023-10-09T05:29:59.476390+0000","last_became_peered":"2023-10-09T05:29:59.476390+0000","last_unstale":"2023-10-09T05:31:32.853762+0000","last_undegraded":"2023-10-09T05:31:32.853762+0000","last_fullsized":"2023-10-09T05:31:32.853762+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:52:31.557279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.851843+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:31:31.851843+0000","last_peered":"2023-10-09T05:31:31.851843+0000","last_clean":"2023-10-09T05:31:31.851843+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:31:31.851843+0000","last_undegraded":"2023-10-09T05:31:31.851843+0000","last_fullsized":"2023-10-09T05:31:31.851843+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854621+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:31:32.854621+0000","last_peered":"2023-10-09T05:31:32.854621+0000","last_clean":"2023-10-09T05:31:32.854621+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:31:32.854621+0000","last_undegraded":"2023-10-09T05:31:32.854621+0000","last_fullsized":"2023-10-09T05:31:32.854621+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":552,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855814+0000","last_change":"2023-10-09T05:31:02.549304+0000","last_active":"2023-10-09T05:31:31.855814+0000","last_peered":"2023-10-09T05:31:31.855814+0000","last_clean":"2023-10-09T05:31:31.855814+0000","last_became_active":"2023-10-09T05:31:02.548085+0000","last_became_peered":"2023-10-09T05:31:02.548085+0000","last_unstale":"2023-10-09T05:31:31.855814+0000","last_undegraded":"2023-10-09T05:31:31.855814+0000","last_fullsized":"2023-10-09T05:31:31.855814+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862414+0000","last_change":"2023-10-09T05:31:32.862414+0000","last_active":"2023-10-09T05:31:32.862414+0000","last_peered":"2023-10-09T05:31:32.862414+0000","last_clean":"2023-10-09T05:31:32.862414+0000","last_became_active":"2023-10-09T05:31:32.860985+0000","last_became_peered":"2023-10-09T05:31:32.860985+0000","last_unstale":"2023-10-09T05:31:32.862414+0000","last_undegraded":"2023-10-09T05:31:32.862414+0000","last_fullsized":"2023-10-09T05:31:32.862414+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:04.492757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":488,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855778+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:31:31.855778+0000","last_peered":"2023-10-09T05:31:31.855778+0000","last_clean":"2023-10-09T05:31:31.855778+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:31:31.855778+0000","last_undegraded":"2023-10-09T05:31:31.855778+0000","last_fullsized":"2023-10-09T05:31:31.855778+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862179+0000","last_change":"2023-10-09T05:31:32.862179+0000","last_active":"2023-10-09T05:31:32.862179+0000","last_peered":"2023-10-09T05:31:32.862179+0000","last_clean":"2023-10-09T05:31:32.862179+0000","last_became_active":"2023-10-09T05:31:32.861777+0000","last_became_peered":"2023-10-09T05:31:32.861777+0000","last_unstale":"2023-10-09T05:31:32.862179+0000","last_undegraded":"2023-10-09T05:31:32.862179+0000","last_fullsized":"2023-10-09T05:31:32.862179+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:40.547661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.853486+0000","last_change":"2023-10-09T05:29:59.470093+0000","last_active":"2023-10-09T05:31:32.853486+0000","last_peered":"2023-10-09T05:31:32.853486+0000","last_clean":"2023-10-09T05:31:32.853486+0000","last_became_active":"2023-10-09T05:29:59.469627+0000","last_became_peered":"2023-10-09T05:29:59.469627+0000","last_unstale":"2023-10-09T05:31:32.853486+0000","last_undegraded":"2023-10-09T05:31:32.853486+0000","last_fullsized":"2023-10-09T05:31:32.853486+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:17.087268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.858901+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:31:31.858901+0000","last_peered":"2023-10-09T05:31:31.858901+0000","last_clean":"2023-10-09T05:31:31.858901+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:31:31.858901+0000","last_undegraded":"2023-10-09T05:31:31.858901+0000","last_fullsized":"2023-10-09T05:31:31.858901+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":333,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854396+0000","last_change":"2023-10-09T05:29:59.474688+0000","last_active":"2023-10-09T05:31:31.854396+0000","last_peered":"2023-10-09T05:31:31.854396+0000","last_clean":"2023-10-09T05:31:31.854396+0000","last_became_active":"2023-10-09T05:29:59.474080+0000","last_became_peered":"2023-10-09T05:29:59.474080+0000","last_unstale":"2023-10-09T05:31:31.854396+0000","last_undegraded":"2023-10-09T05:31:31.854396+0000","last_fullsized":"2023-10-09T05:31:31.854396+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854700+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:31:32.854700+0000","last_peered":"2023-10-09T05:31:32.854700+0000","last_clean":"2023-10-09T05:31:32.854700+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:31:32.854700+0000","last_undegraded":"2023-10-09T05:31:32.854700+0000","last_fullsized":"2023-10-09T05:31:32.854700+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866574+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:31:32.866574+0000","last_peered":"2023-10-09T05:31:32.866574+0000","last_clean":"2023-10-09T05:31:32.866574+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:31:32.866574+0000","last_undegraded":"2023-10-09T05:31:32.866574+0000","last_fullsized":"2023-10-09T05:31:32.866574+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":725,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854312+0000","last_change":"2023-10-09T05:30:30.456503+0000","last_active":"2023-10-09T05:31:31.854312+0000","last_peered":"2023-10-09T05:31:31.854312+0000","last_clean":"2023-10-09T05:31:31.854312+0000","last_became_active":"2023-10-09T05:30:30.455426+0000","last_became_peered":"2023-10-09T05:30:30.455426+0000","last_unstale":"2023-10-09T05:31:31.854312+0000","last_undegraded":"2023-10-09T05:31:31.854312+0000","last_fullsized":"2023-10-09T05:31:31.854312+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855635+0000","last_change":"2023-10-09T05:21:17.465038+0000","last_active":"2023-10-09T05:31:31.855635+0000","last_peered":"2023-10-09T05:31:31.855635+0000","last_clean":"2023-10-09T05:31:31.855635+0000","last_became_active":"2023-10-09T05:21:17.464695+0000","last_became_peered":"2023-10-09T05:21:17.464695+0000","last_unstale":"2023-10-09T05:31:31.855635+0000","last_undegraded":"2023-10-09T05:31:31.855635+0000","last_fullsized":"2023-10-09T05:31:31.855635+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864478+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:31:32.864478+0000","last_peered":"2023-10-09T05:31:32.864478+0000","last_clean":"2023-10-09T05:31:32.864478+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:31:32.864478+0000","last_undegraded":"2023-10-09T05:31:32.864478+0000","last_fullsized":"2023-10-09T05:31:32.864478+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":307,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854341+0000","last_change":"2023-10-09T05:29:59.475956+0000","last_active":"2023-10-09T05:31:31.854341+0000","last_peered":"2023-10-09T05:31:31.854341+0000","last_clean":"2023-10-09T05:31:31.854341+0000","last_became_active":"2023-10-09T05:29:59.475408+0000","last_became_peered":"2023-10-09T05:29:59.475408+0000","last_unstale":"2023-10-09T05:31:31.854341+0000","last_undegraded":"2023-10-09T05:31:31.854341+0000","last_fullsized":"2023-10-09T05:31:31.854341+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":665,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848463+0000","last_change":"2023-10-09T05:31:02.547804+0000","last_active":"2023-10-09T05:31:31.848463+0000","last_peered":"2023-10-09T05:31:31.848463+0000","last_clean":"2023-10-09T05:31:31.848463+0000","last_became_active":"2023-10-09T05:31:02.547237+0000","last_became_peered":"2023-10-09T05:31:02.547237+0000","last_unstale":"2023-10-09T05:31:31.848463+0000","last_undegraded":"2023-10-09T05:31:31.848463+0000","last_fullsized":"2023-10-09T05:31:31.848463+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.859446+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:31:31.859446+0000","last_peered":"2023-10-09T05:31:31.859446+0000","last_clean":"2023-10-09T05:31:31.859446+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:31:31.859446+0000","last_undegraded":"2023-10-09T05:31:31.859446+0000","last_fullsized":"2023-10-09T05:31:31.859446+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.859514+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:31:31.859514+0000","last_peered":"2023-10-09T05:31:31.859514+0000","last_clean":"2023-10-09T05:31:31.859514+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:31:31.859514+0000","last_undegraded":"2023-10-09T05:31:31.859514+0000","last_fullsized":"2023-10-09T05:31:31.859514+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":414,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.858194+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:31:32.858194+0000","last_peered":"2023-10-09T05:31:32.858194+0000","last_clean":"2023-10-09T05:31:32.858194+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:31:32.858194+0000","last_undegraded":"2023-10-09T05:31:32.858194+0000","last_fullsized":"2023-10-09T05:31:32.858194+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866540+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:31:32.866540+0000","last_peered":"2023-10-09T05:31:32.866540+0000","last_clean":"2023-10-09T05:31:32.866540+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:31:32.866540+0000","last_undegraded":"2023-10-09T05:31:32.866540+0000","last_fullsized":"2023-10-09T05:31:32.866540+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.849147+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:31:31.849147+0000","last_peered":"2023-10-09T05:31:31.849147+0000","last_clean":"2023-10-09T05:31:31.849147+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:31:31.849147+0000","last_undegraded":"2023-10-09T05:31:31.849147+0000","last_fullsized":"2023-10-09T05:31:31.849147+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":540,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848375+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:31:31.848375+0000","last_peered":"2023-10-09T05:31:31.848375+0000","last_clean":"2023-10-09T05:31:31.848375+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:31:31.848375+0000","last_undegraded":"2023-10-09T05:31:31.848375+0000","last_fullsized":"2023-10-09T05:31:31.848375+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855243+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:31:31.855243+0000","last_peered":"2023-10-09T05:31:31.855243+0000","last_clean":"2023-10-09T05:31:31.855243+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:31:31.855243+0000","last_undegraded":"2023-10-09T05:31:31.855243+0000","last_fullsized":"2023-10-09T05:31:31.855243+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866456+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:31:32.866456+0000","last_peered":"2023-10-09T05:31:32.866456+0000","last_clean":"2023-10-09T05:31:32.866456+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:31:32.866456+0000","last_undegraded":"2023-10-09T05:31:32.866456+0000","last_fullsized":"2023-10-09T05:31:32.866456+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":611,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.859682+0000","last_change":"2023-10-09T05:30:30.457812+0000","last_active":"2023-10-09T05:31:31.859682+0000","last_peered":"2023-10-09T05:31:31.859682+0000","last_clean":"2023-10-09T05:31:31.859682+0000","last_became_active":"2023-10-09T05:30:30.456811+0000","last_became_peered":"2023-10-09T05:30:30.456811+0000","last_unstale":"2023-10-09T05:31:31.859682+0000","last_undegraded":"2023-10-09T05:31:31.859682+0000","last_fullsized":"2023-10-09T05:31:31.859682+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:17.322315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869220+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:31:32.869220+0000","last_peered":"2023-10-09T05:31:32.869220+0000","last_clean":"2023-10-09T05:31:32.869220+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:31:32.869220+0000","last_undegraded":"2023-10-09T05:31:32.869220+0000","last_fullsized":"2023-10-09T05:31:32.869220+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870575+0000","last_change":"2023-10-09T05:31:32.870575+0000","last_active":"2023-10-09T05:31:32.870575+0000","last_peered":"2023-10-09T05:31:32.870575+0000","last_clean":"2023-10-09T05:31:32.870575+0000","last_became_active":"2023-10-09T05:31:32.870217+0000","last_became_peered":"2023-10-09T05:31:32.870217+0000","last_unstale":"2023-10-09T05:31:32.870575+0000","last_undegraded":"2023-10-09T05:31:32.870575+0000","last_fullsized":"2023-10-09T05:31:32.870575+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:08.078992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852652+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:31:31.852652+0000","last_peered":"2023-10-09T05:31:31.852652+0000","last_clean":"2023-10-09T05:31:31.852652+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:31:31.852652+0000","last_undegraded":"2023-10-09T05:31:31.852652+0000","last_fullsized":"2023-10-09T05:31:31.852652+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.851950+0000","last_change":"2023-10-09T05:31:02.552208+0000","last_active":"2023-10-09T05:31:31.851950+0000","last_peered":"2023-10-09T05:31:31.851950+0000","last_clean":"2023-10-09T05:31:31.851950+0000","last_became_active":"2023-10-09T05:31:02.551893+0000","last_became_peered":"2023-10-09T05:31:02.551893+0000","last_unstale":"2023-10-09T05:31:31.851950+0000","last_undegraded":"2023-10-09T05:31:31.851950+0000","last_fullsized":"2023-10-09T05:31:31.851950+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852458+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:31:31.852458+0000","last_peered":"2023-10-09T05:31:31.852458+0000","last_clean":"2023-10-09T05:31:31.852458+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:31:31.852458+0000","last_undegraded":"2023-10-09T05:31:31.852458+0000","last_fullsized":"2023-10-09T05:31:31.852458+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855645+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:31:31.855645+0000","last_peered":"2023-10-09T05:31:31.855645+0000","last_clean":"2023-10-09T05:31:31.855645+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:31:31.855645+0000","last_undegraded":"2023-10-09T05:31:31.855645+0000","last_fullsized":"2023-10-09T05:31:31.855645+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864605+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:31:32.864605+0000","last_peered":"2023-10-09T05:31:32.864605+0000","last_clean":"2023-10-09T05:31:32.864605+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:31:32.864605+0000","last_undegraded":"2023-10-09T05:31:32.864605+0000","last_fullsized":"2023-10-09T05:31:32.864605+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862044+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:31:32.862044+0000","last_peered":"2023-10-09T05:31:32.862044+0000","last_clean":"2023-10-09T05:31:32.862044+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:31:32.862044+0000","last_undegraded":"2023-10-09T05:31:32.862044+0000","last_fullsized":"2023-10-09T05:31:32.862044+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.853675+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:31:32.853675+0000","last_peered":"2023-10-09T05:31:32.853675+0000","last_clean":"2023-10-09T05:31:32.853675+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:31:32.853675+0000","last_undegraded":"2023-10-09T05:31:32.853675+0000","last_fullsized":"2023-10-09T05:31:32.853675+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866394+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:31:32.866394+0000","last_peered":"2023-10-09T05:31:32.866394+0000","last_clean":"2023-10-09T05:31:32.866394+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:31:32.866394+0000","last_undegraded":"2023-10-09T05:31:32.866394+0000","last_fullsized":"2023-10-09T05:31:32.866394+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855464+0000","last_change":"2023-10-09T05:31:02.548845+0000","last_active":"2023-10-09T05:31:31.855464+0000","last_peered":"2023-10-09T05:31:31.855464+0000","last_clean":"2023-10-09T05:31:31.855464+0000","last_became_active":"2023-10-09T05:31:02.548131+0000","last_became_peered":"2023-10-09T05:31:02.548131+0000","last_unstale":"2023-10-09T05:31:31.855464+0000","last_undegraded":"2023-10-09T05:31:31.855464+0000","last_fullsized":"2023-10-09T05:31:31.855464+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854508+0000","last_change":"2023-10-09T05:21:16.781147+0000","last_active":"2023-10-09T05:31:31.854508+0000","last_peered":"2023-10-09T05:31:31.854508+0000","last_clean":"2023-10-09T05:31:31.854508+0000","last_became_active":"2023-10-09T05:21:16.780755+0000","last_became_peered":"2023-10-09T05:21:16.780755+0000","last_unstale":"2023-10-09T05:31:31.854508+0000","last_undegraded":"2023-10-09T05:31:31.854508+0000","last_fullsized":"2023-10-09T05:31:31.854508+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869504+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:31:32.869504+0000","last_peered":"2023-10-09T05:31:32.869504+0000","last_clean":"2023-10-09T05:31:32.869504+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:31:32.869504+0000","last_undegraded":"2023-10-09T05:31:32.869504+0000","last_fullsized":"2023-10-09T05:31:32.869504+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.853555+0000","last_change":"2023-10-09T05:30:30.457635+0000","last_active":"2023-10-09T05:31:31.853555+0000","last_peered":"2023-10-09T05:31:31.853555+0000","last_clean":"2023-10-09T05:31:31.853555+0000","last_became_active":"2023-10-09T05:30:30.457276+0000","last_became_peered":"2023-10-09T05:30:30.457276+0000","last_unstale":"2023-10-09T05:31:31.853555+0000","last_undegraded":"2023-10-09T05:31:31.853555+0000","last_fullsized":"2023-10-09T05:31:31.853555+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848299+0000","last_change":"2023-10-09T05:21:17.464398+0000","last_active":"2023-10-09T05:31:31.848299+0000","last_peered":"2023-10-09T05:31:31.848299+0000","last_clean":"2023-10-09T05:31:31.848299+0000","last_became_active":"2023-10-09T05:21:17.464086+0000","last_became_peered":"2023-10-09T05:21:17.464086+0000","last_unstale":"2023-10-09T05:31:31.848299+0000","last_undegraded":"2023-10-09T05:31:31.848299+0000","last_fullsized":"2023-10-09T05:31:31.848299+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855156+0000","last_change":"2023-10-09T05:29:59.470664+0000","last_active":"2023-10-09T05:31:31.855156+0000","last_peered":"2023-10-09T05:31:31.855156+0000","last_clean":"2023-10-09T05:31:31.855156+0000","last_became_active":"2023-10-09T05:29:59.470276+0000","last_became_peered":"2023-10-09T05:29:59.470276+0000","last_unstale":"2023-10-09T05:31:31.855156+0000","last_undegraded":"2023-10-09T05:31:31.855156+0000","last_fullsized":"2023-10-09T05:31:31.855156+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.871093+0000","last_change":"2023-10-09T05:31:32.871093+0000","last_active":"2023-10-09T05:31:32.871093+0000","last_peered":"2023-10-09T05:31:32.871093+0000","last_clean":"2023-10-09T05:31:32.871093+0000","last_became_active":"2023-10-09T05:31:32.870702+0000","last_became_peered":"2023-10-09T05:31:32.870702+0000","last_unstale":"2023-10-09T05:31:32.871093+0000","last_undegraded":"2023-10-09T05:31:32.871093+0000","last_fullsized":"2023-10-09T05:31:32.871093+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:18.439344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.856510+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:31:31.856510+0000","last_peered":"2023-10-09T05:31:31.856510+0000","last_clean":"2023-10-09T05:31:31.856510+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:31:31.856510+0000","last_undegraded":"2023-10-09T05:31:31.856510+0000","last_fullsized":"2023-10-09T05:31:31.856510+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866775+0000","last_change":"2023-10-09T05:31:32.861092+0000","last_active":"2023-10-09T05:31:32.866775+0000","last_peered":"2023-10-09T05:31:32.866775+0000","last_clean":"2023-10-09T05:31:32.866775+0000","last_became_active":"2023-10-09T05:31:32.860632+0000","last_became_peered":"2023-10-09T05:31:32.860632+0000","last_unstale":"2023-10-09T05:31:32.866775+0000","last_undegraded":"2023-10-09T05:31:32.866775+0000","last_fullsized":"2023-10-09T05:31:32.866775+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:54.308161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852402+0000","last_change":"2023-10-09T05:21:16.784983+0000","last_active":"2023-10-09T05:31:31.852402+0000","last_peered":"2023-10-09T05:31:31.852402+0000","last_clean":"2023-10-09T05:31:31.852402+0000","last_became_active":"2023-10-09T05:21:16.784699+0000","last_became_peered":"2023-10-09T05:21:16.784699+0000","last_unstale":"2023-10-09T05:31:31.852402+0000","last_undegraded":"2023-10-09T05:31:31.852402+0000","last_fullsized":"2023-10-09T05:31:31.852402+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866744+0000","last_change":"2023-10-09T05:31:32.866744+0000","last_active":"2023-10-09T05:31:32.866744+0000","last_peered":"2023-10-09T05:31:32.866744+0000","last_clean":"2023-10-09T05:31:32.866744+0000","last_became_active":"2023-10-09T05:31:32.866389+0000","last_became_peered":"2023-10-09T05:31:32.866389+0000","last_unstale":"2023-10-09T05:31:32.866744+0000","last_undegraded":"2023-10-09T05:31:32.866744+0000","last_fullsized":"2023-10-09T05:31:32.866744+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866600+0000","last_change":"2023-10-09T05:31:32.861963+0000","last_active":"2023-10-09T05:31:32.866600+0000","last_peered":"2023-10-09T05:31:32.866600+0000","last_clean":"2023-10-09T05:31:32.866600+0000","last_became_active":"2023-10-09T05:31:32.861679+0000","last_became_peered":"2023-10-09T05:31:32.861679+0000","last_unstale":"2023-10-09T05:31:32.866600+0000","last_undegraded":"2023-10-09T05:31:32.866600+0000","last_fullsized":"2023-10-09T05:31:32.866600+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:42.402077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.871695+0000","last_change":"2023-10-09T05:31:32.871695+0000","last_active":"2023-10-09T05:31:32.871695+0000","last_peered":"2023-10-09T05:31:32.871695+0000","last_clean":"2023-10-09T05:31:32.871695+0000","last_became_active":"2023-10-09T05:31:32.871449+0000","last_became_peered":"2023-10-09T05:31:32.871449+0000","last_unstale":"2023-10-09T05:31:32.871695+0000","last_undegraded":"2023-10-09T05:31:32.871695+0000","last_fullsized":"2023-10-09T05:31:32.871695+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:22.999187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852684+0000","last_change":"2023-10-09T05:31:02.550860+0000","last_active":"2023-10-09T05:31:31.852684+0000","last_peered":"2023-10-09T05:31:31.852684+0000","last_clean":"2023-10-09T05:31:31.852684+0000","last_became_active":"2023-10-09T05:31:02.550469+0000","last_became_peered":"2023-10-09T05:31:02.550469+0000","last_unstale":"2023-10-09T05:31:31.852684+0000","last_undegraded":"2023-10-09T05:31:31.852684+0000","last_fullsized":"2023-10-09T05:31:31.852684+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848987+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:31:31.848987+0000","last_peered":"2023-10-09T05:31:31.848987+0000","last_clean":"2023-10-09T05:31:31.848987+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:31:31.848987+0000","last_undegraded":"2023-10-09T05:31:31.848987+0000","last_fullsized":"2023-10-09T05:31:31.848987+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850669+0000","last_change":"2023-10-09T05:21:16.778811+0000","last_active":"2023-10-09T05:31:31.850669+0000","last_peered":"2023-10-09T05:31:31.850669+0000","last_clean":"2023-10-09T05:31:31.850669+0000","last_became_active":"2023-10-09T05:21:16.778515+0000","last_became_peered":"2023-10-09T05:21:16.778515+0000","last_unstale":"2023-10-09T05:31:31.850669+0000","last_undegraded":"2023-10-09T05:31:31.850669+0000","last_fullsized":"2023-10-09T05:31:31.850669+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855330+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:31:31.855330+0000","last_peered":"2023-10-09T05:31:31.855330+0000","last_clean":"2023-10-09T05:31:31.855330+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:31:31.855330+0000","last_undegraded":"2023-10-09T05:31:31.855330+0000","last_fullsized":"2023-10-09T05:31:31.855330+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.863294+0000","last_change":"2023-10-09T05:31:32.863294+0000","last_active":"2023-10-09T05:31:32.863294+0000","last_peered":"2023-10-09T05:31:32.863294+0000","last_clean":"2023-10-09T05:31:32.863294+0000","last_became_active":"2023-10-09T05:31:32.862914+0000","last_became_peered":"2023-10-09T05:31:32.862914+0000","last_unstale":"2023-10-09T05:31:32.863294+0000","last_undegraded":"2023-10-09T05:31:32.863294+0000","last_fullsized":"2023-10-09T05:31:32.863294+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.863335+0000","last_change":"2023-10-09T05:31:32.863335+0000","last_active":"2023-10-09T05:31:32.863335+0000","last_peered":"2023-10-09T05:31:32.863335+0000","last_clean":"2023-10-09T05:31:32.863335+0000","last_became_active":"2023-10-09T05:31:32.863036+0000","last_became_peered":"2023-10-09T05:31:32.863036+0000","last_unstale":"2023-10-09T05:31:32.863335+0000","last_undegraded":"2023-10-09T05:31:32.863335+0000","last_fullsized":"2023-10-09T05:31:32.863335+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:59.787997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848623+0000","last_change":"2023-10-09T05:30:30.457540+0000","last_active":"2023-10-09T05:31:31.848623+0000","last_peered":"2023-10-09T05:31:31.848623+0000","last_clean":"2023-10-09T05:31:31.848623+0000","last_became_active":"2023-10-09T05:30:30.457180+0000","last_became_peered":"2023-10-09T05:30:30.457180+0000","last_unstale":"2023-10-09T05:31:31.848623+0000","last_undegraded":"2023-10-09T05:31:31.848623+0000","last_fullsized":"2023-10-09T05:31:31.848623+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870178+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:31:32.870178+0000","last_peered":"2023-10-09T05:31:32.870178+0000","last_clean":"2023-10-09T05:31:32.870178+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:31:32.870178+0000","last_undegraded":"2023-10-09T05:31:32.870178+0000","last_fullsized":"2023-10-09T05:31:32.870178+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866729+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:31:32.866729+0000","last_peered":"2023-10-09T05:31:32.866729+0000","last_clean":"2023-10-09T05:31:32.866729+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:31:32.866729+0000","last_undegraded":"2023-10-09T05:31:32.866729+0000","last_fullsized":"2023-10-09T05:31:32.866729+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855720+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:31:31.855720+0000","last_peered":"2023-10-09T05:31:31.855720+0000","last_clean":"2023-10-09T05:31:31.855720+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:31:31.855720+0000","last_undegraded":"2023-10-09T05:31:31.855720+0000","last_fullsized":"2023-10-09T05:31:31.855720+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.867898+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:31:32.867898+0000","last_peered":"2023-10-09T05:31:32.867898+0000","last_clean":"2023-10-09T05:31:32.867898+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:31:32.867898+0000","last_undegraded":"2023-10-09T05:31:32.867898+0000","last_fullsized":"2023-10-09T05:31:32.867898+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866758+0000","last_change":"2023-10-09T05:31:32.860920+0000","last_active":"2023-10-09T05:31:32.866758+0000","last_peered":"2023-10-09T05:31:32.866758+0000","last_clean":"2023-10-09T05:31:32.866758+0000","last_became_active":"2023-10-09T05:31:32.860244+0000","last_became_peered":"2023-10-09T05:31:32.860244+0000","last_unstale":"2023-10-09T05:31:32.866758+0000","last_undegraded":"2023-10-09T05:31:32.866758+0000","last_fullsized":"2023-10-09T05:31:32.866758+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:37.134584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869382+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:31:32.869382+0000","last_peered":"2023-10-09T05:31:32.869382+0000","last_clean":"2023-10-09T05:31:32.869382+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:31:32.869382+0000","last_undegraded":"2023-10-09T05:31:32.869382+0000","last_fullsized":"2023-10-09T05:31:32.869382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.868031+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:31:32.868031+0000","last_peered":"2023-10-09T05:31:32.868031+0000","last_clean":"2023-10-09T05:31:32.868031+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:31:32.868031+0000","last_undegraded":"2023-10-09T05:31:32.868031+0000","last_fullsized":"2023-10-09T05:31:32.868031+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":87,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850461+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:31:31.850461+0000","last_peered":"2023-10-09T05:31:31.850461+0000","last_clean":"2023-10-09T05:31:31.850461+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:31:31.850461+0000","last_undegraded":"2023-10-09T05:31:31.850461+0000","last_fullsized":"2023-10-09T05:31:31.850461+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26751,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430364,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5368692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:31:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.46300000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.60399999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:32:05 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.56000000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:31:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.59499999999999997,"15min":0.60299999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.75700000000000001,"5min":1.153,"15min":2.6669999999999998},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.58199999999999996,"15min":0.58999999999999997},"min":{"1min":0.40000000000000002,"5min":0.39700000000000002,"15min":0.32000000000000001},"max":{"1min":0.76500000000000001,"5min":1.0309999999999999,"15min":2.1379999999999999},"last":1.1100000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.49099999999999999,"15min":0.46500000000000002},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.249},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.52500000000000002,"15min":0.50800000000000001},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27300000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.70099999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.53000000000000003,"15min":0.50700000000000001},"min":{"1min":0.35799999999999998,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":1.462},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.40200000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88700000000000001},"last":0.77700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.56699999999999995,"15min":0.54000000000000004},"min":{"1min":0.42299999999999999,"5min":0.34799999999999998,"15min":0.30499999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.623,"5min":0.59599999999999997,"15min":0.58499999999999996},"min":{"1min":0.45000000000000001,"5min":0.372,"15min":0.33300000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.747}]}]},{"osd":6,"up_from":42,"seq":180388626592,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306916,"kb_used_data":5724,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438236,"statfs":{"total":95995035648,"available":95680753664,"internally_reserved":0,"allocated":5861376,"data_stored":5368300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.55500000000000005,"15min":0.54500000000000004},"min":{"1min":0.39300000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.014,"5min":0.63800000000000001,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.4569999999999999,"5min":7.4569999999999999,"15min":7.4569999999999999},"last":0.41199999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.434},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.57999999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.075,"15min":1.075},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.5999999999999996,"15min":7.5999999999999996},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":7.6379999999999999,"5min":7.6379999999999999,"15min":7.6379999999999999},"last":0.498}]},{"osd":3,"last update":"Mon Oct 9 05:30:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.55700000000000005,"15min":0.55200000000000005},"min":{"1min":0.38600000000000001,"5min":0.38100000000000001,"15min":0.33300000000000002},"max":{"1min":0.73199999999999998,"5min":0.80700000000000005,"15min":3.9569999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59399999999999997,"15min":0.59499999999999997},"min":{"1min":0.436,"5min":0.34899999999999998,"15min":0.33800000000000002},"max":{"1min":0.79700000000000004,"5min":0.81999999999999995,"15min":2.9049999999999998},"last":0.44700000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.47399999999999998,"15min":0.46100000000000002},"min":{"1min":0.29099999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.86399999999999999},"last":0.378},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.49399999999999999,"15min":0.47299999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":0.78800000000000003,"5min":0.83799999999999997,"15min":1.014},"last":0.50900000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.53700000000000003,"15min":0.504},"min":{"1min":0.33400000000000002,"5min":0.30499999999999999,"15min":0.27200000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.51200000000000001},"min":{"1min":0.35999999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.73299999999999998,"5min":0.82399999999999995,"15min":2.0579999999999998},"last":0.46100000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.54200000000000004,"15min":0.53300000000000003},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.60199999999999998,"15min":0.56999999999999995},"min":{"1min":0.498,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.999},"last":0.628}]}]},{"osd":1,"up_from":93,"seq":399431958553,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32640,"kb_used_data":1080,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712512,"statfs":{"total":95995035648,"available":95961612288,"internally_reserved":0,"allocated":1105920,"data_stored":587447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":32301582},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.34699999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.45600000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41299999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.621}]},{"osd":5,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.65000000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.59099999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1,"5min":1,"15min":1},"last":0.69299999999999995}]}]},{"osd":0,"up_from":88,"seq":377957122080,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33228,"kb_used_data":1668,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93711924,"statfs":{"total":95995035648,"available":95961010176,"internally_reserved":0,"allocated":1708032,"data_stored":1178579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":32300808},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.29999999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.247},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.316}]},{"osd":3,"last update":"Mon Oct 9 05:31:00 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.33300000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.65900000000000003,"15min":0.67300000000000004},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.83699999999999997,"5min":1.028,"15min":1.028},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.69999999999999996,"15min":0.71399999999999997},"min":{"1min":0.45300000000000001,"5min":0.434,"15min":0.434},"max":{"1min":0.81799999999999995,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.65400000000000003,"15min":0.67200000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.70399999999999996,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.65600000000000003,"15min":0.66900000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.79200000000000004,"5min":1.022,"15min":1.022},"last":0.54700000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.68400000000000005,"15min":0.69899999999999995},"min":{"1min":0.37,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80300000000000005,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.66300000000000003,"15min":0.67300000000000004},"min":{"1min":0.42899999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.85599999999999998,"5min":1.044,"15min":1.044},"last":0.59599999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.68999999999999995,"15min":0.70299999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.89600000000000002,"5min":1.052,"15min":1.052},"last":0.624},{"interface":"front","average":{"1min":0.621,"5min":0.71299999999999997,"15min":0.72799999999999998},"min":{"1min":0.38500000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77500000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.64100000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795027,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36668,"kb_used_data":5172,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708484,"statfs":{"total":95995035648,"available":95957487616,"internally_reserved":0,"allocated":5296128,"data_stored":4780992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":32232021},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.53600000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.56599999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61799999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.96299999999999997},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.86199999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.82999999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.94799999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.82299999999999995}]}]},{"osd":3,"up_from":103,"seq":442381631501,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28392,"kb_used_data":1056,"kb_used_omap":4,"kb_used_meta":27323,"kb_avail":93716760,"statfs":{"total":95995035648,"available":95965962240,"internally_reserved":0,"allocated":1081344,"data_stored":583603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":27979358},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34399999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.37}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66700000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64700000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61499999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019048,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":583950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52200000000000002,"15min":0.52100000000000002},"min":{"1min":0.373,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73299999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.5}]},{"osd":1,"last update":"Mon Oct 9 05:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.38100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:32:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.40600000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.58299999999999996,"15min":0.56000000000000005},"min":{"1min":0.38500000000000001,"5min":0.374,"15min":0.28799999999999998},"max":{"1min":0.73899999999999999,"5min":0.84299999999999997,"15min":1.591},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56100000000000005,"15min":0.54700000000000004},"min":{"1min":0.434,"5min":0.373,"15min":0.33600000000000002},"max":{"1min":0.66700000000000004,"5min":0.877,"15min":0.877},"last":0.53600000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:31:27 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46800000000000003,"15min":0.441},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26700000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.85399999999999998},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56100000000000005,"15min":0.52900000000000003},"min":{"1min":0.44700000000000001,"5min":0.36599999999999999,"15min":0.311},"max":{"1min":0.70499999999999996,"5min":0.79600000000000004,"15min":1.54},"last":0.58499999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:31:40 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.48999999999999999,"15min":0.48899999999999999},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.27800000000000002},"max":{"1min":0.74399999999999999,"5min":0.80300000000000005,"15min":1.5660000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.502,"15min":0.503},"min":{"1min":0.371,"5min":0.33100000000000002,"15min":0.28899999999999998},"max":{"1min":0.752,"5min":0.76300000000000001,"15min":0.94699999999999995},"last":0.55600000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:32:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53500000000000003,"15min":0.53600000000000003},"min":{"1min":0.433,"5min":0.35499999999999998,"15min":0.30499999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":1.6160000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53300000000000003,"15min":0.53000000000000003},"min":{"1min":0.46700000000000003,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":0.76700000000000002,"5min":0.79000000000000004,"15min":1.034},"last":0.60699999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822820,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298136,"kb_used_data":1040,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447016,"statfs":{"total":95995035648,"available":95689744384,"internally_reserved":0,"allocated":1064960,"data_stored":579398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70499999999999996,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.495},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.39500000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.44600000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.63800000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.55500000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:30:24 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.60699999999999998,"15min":0.58499999999999996},"min":{"1min":0.47699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.94899999999999995},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.64800000000000002,"15min":0.627},"min":{"1min":0.501,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.85899999999999999,"5min":0.92800000000000005,"15min":0.93899999999999995},"last":0.59399999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:31:24 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.48299999999999998,"15min":0.44900000000000001},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":0.69899999999999995,"5min":0.874,"15min":0.94499999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53600000000000003,"15min":0.48799999999999999},"min":{"1min":0.35499999999999998,"5min":0.33200000000000002,"15min":0.28699999999999998},"max":{"1min":0.70599999999999996,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.63100000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:31:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.55600000000000005,"15min":0.505},"min":{"1min":0.41199999999999998,"5min":0.34999999999999998,"15min":0.26000000000000001},"max":{"1min":0.78500000000000003,"5min":0.93500000000000005,"15min":1.1339999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.56100000000000005,"15min":0.53100000000000003},"min":{"1min":0.42999999999999999,"5min":0.36199999999999999,"15min":0.314},"max":{"1min":0.78800000000000003,"5min":0.92100000000000004,"15min":1.109},"last":0.58699999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:32:10 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.59399999999999997,"15min":0.57699999999999996},"min":{"1min":0.52400000000000002,"5min":0.40200000000000002,"15min":0.34399999999999997},"max":{"1min":0.71499999999999997,"5min":0.95199999999999996,"15min":1.143},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57399999999999995,"15min":0.55600000000000005},"min":{"1min":0.51100000000000001,"5min":0.35199999999999998,"15min":0.32400000000000001},"max":{"1min":0.83499999999999996,"5min":0.90900000000000003,"15min":1.1830000000000001},"last":0.73399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:32:29.336 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:32:29.698 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:29 smithi008 ceph-mon[184418]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:29 smithi008 ceph-mon[191227]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:29.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:29 smithi191 ceph-mon[161669]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:29.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:30.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:30 smithi008 ceph-mon[184418]: from='client.34710 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:30.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:32:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:32:30] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.33.4" 2023-10-09T05:32:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:30 smithi008 ceph-mon[191227]: from='client.34710 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:30.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:30 smithi191 ceph-mon[161669]: from='client.34710 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:31.373 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:32:31.375 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:32:31.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:31 smithi008 ceph-mon[184418]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:31 smithi008 ceph-mon[191227]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:31.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:31 smithi191 ceph-mon[161669]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:32.013 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":226,"stamp":"2023-10-09T05:32:29.856757+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13724,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341060,"kb_used_data":22532,"kb_used_omap":31,"kb_used_meta":1318432,"kb_avail":748620156,"statfs":{"total":767960285184,"available":766587039744,"internally_reserved":0,"allocated":23072768,"data_stored":19030961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":1350074825},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003318"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848734+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:31:31.848734+0000","last_peered":"2023-10-09T05:31:31.848734+0000","last_clean":"2023-10-09T05:31:31.848734+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:31:31.848734+0000","last_undegraded":"2023-10-09T05:31:31.848734+0000","last_fullsized":"2023-10-09T05:31:31.848734+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854909+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:31:32.854909+0000","last_peered":"2023-10-09T05:31:32.854909+0000","last_clean":"2023-10-09T05:31:32.854909+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:31:32.854909+0000","last_undegraded":"2023-10-09T05:31:32.854909+0000","last_fullsized":"2023-10-09T05:31:32.854909+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.860364+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:31:32.860364+0000","last_peered":"2023-10-09T05:31:32.860364+0000","last_clean":"2023-10-09T05:31:32.860364+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:31:32.860364+0000","last_undegraded":"2023-10-09T05:31:32.860364+0000","last_fullsized":"2023-10-09T05:31:32.860364+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864546+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:31:32.864546+0000","last_peered":"2023-10-09T05:31:32.864546+0000","last_clean":"2023-10-09T05:31:32.864546+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:31:32.864546+0000","last_undegraded":"2023-10-09T05:31:32.864546+0000","last_fullsized":"2023-10-09T05:31:32.864546+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.867349+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:31:32.867349+0000","last_peered":"2023-10-09T05:31:32.867349+0000","last_clean":"2023-10-09T05:31:32.867349+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:31:32.867349+0000","last_undegraded":"2023-10-09T05:31:32.867349+0000","last_fullsized":"2023-10-09T05:31:32.867349+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":58,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850233+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:31:31.850233+0000","last_peered":"2023-10-09T05:31:31.850233+0000","last_clean":"2023-10-09T05:31:31.850233+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:31:31.850233+0000","last_undegraded":"2023-10-09T05:31:31.850233+0000","last_fullsized":"2023-10-09T05:31:31.850233+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869382+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:31:32.869382+0000","last_peered":"2023-10-09T05:31:32.869382+0000","last_clean":"2023-10-09T05:31:32.869382+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:31:32.869382+0000","last_undegraded":"2023-10-09T05:31:32.869382+0000","last_fullsized":"2023-10-09T05:31:32.869382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.849363+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:31:31.849363+0000","last_peered":"2023-10-09T05:31:31.849363+0000","last_clean":"2023-10-09T05:31:31.849363+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:31:31.849363+0000","last_undegraded":"2023-10-09T05:31:31.849363+0000","last_fullsized":"2023-10-09T05:31:31.849363+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870694+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:31:32.870694+0000","last_peered":"2023-10-09T05:31:32.870694+0000","last_clean":"2023-10-09T05:31:32.870694+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:31:32.870694+0000","last_undegraded":"2023-10-09T05:31:32.870694+0000","last_fullsized":"2023-10-09T05:31:32.870694+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":41,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.853762+0000","last_change":"2023-10-09T05:29:59.477246+0000","last_active":"2023-10-09T05:31:32.853762+0000","last_peered":"2023-10-09T05:31:32.853762+0000","last_clean":"2023-10-09T05:31:32.853762+0000","last_became_active":"2023-10-09T05:29:59.476390+0000","last_became_peered":"2023-10-09T05:29:59.476390+0000","last_unstale":"2023-10-09T05:31:32.853762+0000","last_undegraded":"2023-10-09T05:31:32.853762+0000","last_fullsized":"2023-10-09T05:31:32.853762+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:52:31.557279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.851843+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:31:31.851843+0000","last_peered":"2023-10-09T05:31:31.851843+0000","last_clean":"2023-10-09T05:31:31.851843+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:31:31.851843+0000","last_undegraded":"2023-10-09T05:31:31.851843+0000","last_fullsized":"2023-10-09T05:31:31.851843+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854621+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:31:32.854621+0000","last_peered":"2023-10-09T05:31:32.854621+0000","last_clean":"2023-10-09T05:31:32.854621+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:31:32.854621+0000","last_undegraded":"2023-10-09T05:31:32.854621+0000","last_fullsized":"2023-10-09T05:31:32.854621+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":552,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855814+0000","last_change":"2023-10-09T05:31:02.549304+0000","last_active":"2023-10-09T05:31:31.855814+0000","last_peered":"2023-10-09T05:31:31.855814+0000","last_clean":"2023-10-09T05:31:31.855814+0000","last_became_active":"2023-10-09T05:31:02.548085+0000","last_became_peered":"2023-10-09T05:31:02.548085+0000","last_unstale":"2023-10-09T05:31:31.855814+0000","last_undegraded":"2023-10-09T05:31:31.855814+0000","last_fullsized":"2023-10-09T05:31:31.855814+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862414+0000","last_change":"2023-10-09T05:31:32.862414+0000","last_active":"2023-10-09T05:31:32.862414+0000","last_peered":"2023-10-09T05:31:32.862414+0000","last_clean":"2023-10-09T05:31:32.862414+0000","last_became_active":"2023-10-09T05:31:32.860985+0000","last_became_peered":"2023-10-09T05:31:32.860985+0000","last_unstale":"2023-10-09T05:31:32.862414+0000","last_undegraded":"2023-10-09T05:31:32.862414+0000","last_fullsized":"2023-10-09T05:31:32.862414+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:04.492757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":488,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855778+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:31:31.855778+0000","last_peered":"2023-10-09T05:31:31.855778+0000","last_clean":"2023-10-09T05:31:31.855778+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:31:31.855778+0000","last_undegraded":"2023-10-09T05:31:31.855778+0000","last_fullsized":"2023-10-09T05:31:31.855778+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862179+0000","last_change":"2023-10-09T05:31:32.862179+0000","last_active":"2023-10-09T05:31:32.862179+0000","last_peered":"2023-10-09T05:31:32.862179+0000","last_clean":"2023-10-09T05:31:32.862179+0000","last_became_active":"2023-10-09T05:31:32.861777+0000","last_became_peered":"2023-10-09T05:31:32.861777+0000","last_unstale":"2023-10-09T05:31:32.862179+0000","last_undegraded":"2023-10-09T05:31:32.862179+0000","last_fullsized":"2023-10-09T05:31:32.862179+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:40.547661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.853486+0000","last_change":"2023-10-09T05:29:59.470093+0000","last_active":"2023-10-09T05:31:32.853486+0000","last_peered":"2023-10-09T05:31:32.853486+0000","last_clean":"2023-10-09T05:31:32.853486+0000","last_became_active":"2023-10-09T05:29:59.469627+0000","last_became_peered":"2023-10-09T05:29:59.469627+0000","last_unstale":"2023-10-09T05:31:32.853486+0000","last_undegraded":"2023-10-09T05:31:32.853486+0000","last_fullsized":"2023-10-09T05:31:32.853486+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:17.087268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.858901+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:31:31.858901+0000","last_peered":"2023-10-09T05:31:31.858901+0000","last_clean":"2023-10-09T05:31:31.858901+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:31:31.858901+0000","last_undegraded":"2023-10-09T05:31:31.858901+0000","last_fullsized":"2023-10-09T05:31:31.858901+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":333,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854396+0000","last_change":"2023-10-09T05:29:59.474688+0000","last_active":"2023-10-09T05:31:31.854396+0000","last_peered":"2023-10-09T05:31:31.854396+0000","last_clean":"2023-10-09T05:31:31.854396+0000","last_became_active":"2023-10-09T05:29:59.474080+0000","last_became_peered":"2023-10-09T05:29:59.474080+0000","last_unstale":"2023-10-09T05:31:31.854396+0000","last_undegraded":"2023-10-09T05:31:31.854396+0000","last_fullsized":"2023-10-09T05:31:31.854396+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.854700+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:31:32.854700+0000","last_peered":"2023-10-09T05:31:32.854700+0000","last_clean":"2023-10-09T05:31:32.854700+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:31:32.854700+0000","last_undegraded":"2023-10-09T05:31:32.854700+0000","last_fullsized":"2023-10-09T05:31:32.854700+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866574+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:31:32.866574+0000","last_peered":"2023-10-09T05:31:32.866574+0000","last_clean":"2023-10-09T05:31:32.866574+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:31:32.866574+0000","last_undegraded":"2023-10-09T05:31:32.866574+0000","last_fullsized":"2023-10-09T05:31:32.866574+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":725,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854312+0000","last_change":"2023-10-09T05:30:30.456503+0000","last_active":"2023-10-09T05:31:31.854312+0000","last_peered":"2023-10-09T05:31:31.854312+0000","last_clean":"2023-10-09T05:31:31.854312+0000","last_became_active":"2023-10-09T05:30:30.455426+0000","last_became_peered":"2023-10-09T05:30:30.455426+0000","last_unstale":"2023-10-09T05:31:31.854312+0000","last_undegraded":"2023-10-09T05:31:31.854312+0000","last_fullsized":"2023-10-09T05:31:31.854312+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855635+0000","last_change":"2023-10-09T05:21:17.465038+0000","last_active":"2023-10-09T05:31:31.855635+0000","last_peered":"2023-10-09T05:31:31.855635+0000","last_clean":"2023-10-09T05:31:31.855635+0000","last_became_active":"2023-10-09T05:21:17.464695+0000","last_became_peered":"2023-10-09T05:21:17.464695+0000","last_unstale":"2023-10-09T05:31:31.855635+0000","last_undegraded":"2023-10-09T05:31:31.855635+0000","last_fullsized":"2023-10-09T05:31:31.855635+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864478+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:31:32.864478+0000","last_peered":"2023-10-09T05:31:32.864478+0000","last_clean":"2023-10-09T05:31:32.864478+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:31:32.864478+0000","last_undegraded":"2023-10-09T05:31:32.864478+0000","last_fullsized":"2023-10-09T05:31:32.864478+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":307,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854341+0000","last_change":"2023-10-09T05:29:59.475956+0000","last_active":"2023-10-09T05:31:31.854341+0000","last_peered":"2023-10-09T05:31:31.854341+0000","last_clean":"2023-10-09T05:31:31.854341+0000","last_became_active":"2023-10-09T05:29:59.475408+0000","last_became_peered":"2023-10-09T05:29:59.475408+0000","last_unstale":"2023-10-09T05:31:31.854341+0000","last_undegraded":"2023-10-09T05:31:31.854341+0000","last_fullsized":"2023-10-09T05:31:31.854341+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":665,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848463+0000","last_change":"2023-10-09T05:31:02.547804+0000","last_active":"2023-10-09T05:31:31.848463+0000","last_peered":"2023-10-09T05:31:31.848463+0000","last_clean":"2023-10-09T05:31:31.848463+0000","last_became_active":"2023-10-09T05:31:02.547237+0000","last_became_peered":"2023-10-09T05:31:02.547237+0000","last_unstale":"2023-10-09T05:31:31.848463+0000","last_undegraded":"2023-10-09T05:31:31.848463+0000","last_fullsized":"2023-10-09T05:31:31.848463+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.859446+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:31:31.859446+0000","last_peered":"2023-10-09T05:31:31.859446+0000","last_clean":"2023-10-09T05:31:31.859446+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:31:31.859446+0000","last_undegraded":"2023-10-09T05:31:31.859446+0000","last_fullsized":"2023-10-09T05:31:31.859446+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.863358+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:31:32.863358+0000","last_peered":"2023-10-09T05:31:32.863358+0000","last_clean":"2023-10-09T05:31:32.863358+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:31:32.863358+0000","last_undegraded":"2023-10-09T05:31:32.863358+0000","last_fullsized":"2023-10-09T05:31:32.863358+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":414,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.858194+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:31:32.858194+0000","last_peered":"2023-10-09T05:31:32.858194+0000","last_clean":"2023-10-09T05:31:32.858194+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:31:32.858194+0000","last_undegraded":"2023-10-09T05:31:32.858194+0000","last_fullsized":"2023-10-09T05:31:32.858194+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866540+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:31:32.866540+0000","last_peered":"2023-10-09T05:31:32.866540+0000","last_clean":"2023-10-09T05:31:32.866540+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:31:32.866540+0000","last_undegraded":"2023-10-09T05:31:32.866540+0000","last_fullsized":"2023-10-09T05:31:32.866540+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.849147+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:31:31.849147+0000","last_peered":"2023-10-09T05:31:31.849147+0000","last_clean":"2023-10-09T05:31:31.849147+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:31:31.849147+0000","last_undegraded":"2023-10-09T05:31:31.849147+0000","last_fullsized":"2023-10-09T05:31:31.849147+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":540,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848375+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:31:31.848375+0000","last_peered":"2023-10-09T05:31:31.848375+0000","last_clean":"2023-10-09T05:31:31.848375+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:31:31.848375+0000","last_undegraded":"2023-10-09T05:31:31.848375+0000","last_fullsized":"2023-10-09T05:31:31.848375+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855243+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:31:31.855243+0000","last_peered":"2023-10-09T05:31:31.855243+0000","last_clean":"2023-10-09T05:31:31.855243+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:31:31.855243+0000","last_undegraded":"2023-10-09T05:31:31.855243+0000","last_fullsized":"2023-10-09T05:31:31.855243+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866456+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:31:32.866456+0000","last_peered":"2023-10-09T05:31:32.866456+0000","last_clean":"2023-10-09T05:31:32.866456+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:31:32.866456+0000","last_undegraded":"2023-10-09T05:31:32.866456+0000","last_fullsized":"2023-10-09T05:31:32.866456+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":611,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.859682+0000","last_change":"2023-10-09T05:30:30.457812+0000","last_active":"2023-10-09T05:31:31.859682+0000","last_peered":"2023-10-09T05:31:31.859682+0000","last_clean":"2023-10-09T05:31:31.859682+0000","last_became_active":"2023-10-09T05:30:30.456811+0000","last_became_peered":"2023-10-09T05:30:30.456811+0000","last_unstale":"2023-10-09T05:31:31.859682+0000","last_undegraded":"2023-10-09T05:31:31.859682+0000","last_fullsized":"2023-10-09T05:31:31.859682+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:17.322315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869220+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:31:32.869220+0000","last_peered":"2023-10-09T05:31:32.869220+0000","last_clean":"2023-10-09T05:31:32.869220+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:31:32.869220+0000","last_undegraded":"2023-10-09T05:31:32.869220+0000","last_fullsized":"2023-10-09T05:31:32.869220+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870575+0000","last_change":"2023-10-09T05:31:32.870575+0000","last_active":"2023-10-09T05:31:32.870575+0000","last_peered":"2023-10-09T05:31:32.870575+0000","last_clean":"2023-10-09T05:31:32.870575+0000","last_became_active":"2023-10-09T05:31:32.870217+0000","last_became_peered":"2023-10-09T05:31:32.870217+0000","last_unstale":"2023-10-09T05:31:32.870575+0000","last_undegraded":"2023-10-09T05:31:32.870575+0000","last_fullsized":"2023-10-09T05:31:32.870575+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:08.078992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852652+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:31:31.852652+0000","last_peered":"2023-10-09T05:31:31.852652+0000","last_clean":"2023-10-09T05:31:31.852652+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:31:31.852652+0000","last_undegraded":"2023-10-09T05:31:31.852652+0000","last_fullsized":"2023-10-09T05:31:31.852652+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.851950+0000","last_change":"2023-10-09T05:31:02.552208+0000","last_active":"2023-10-09T05:31:31.851950+0000","last_peered":"2023-10-09T05:31:31.851950+0000","last_clean":"2023-10-09T05:31:31.851950+0000","last_became_active":"2023-10-09T05:31:02.551893+0000","last_became_peered":"2023-10-09T05:31:02.551893+0000","last_unstale":"2023-10-09T05:31:31.851950+0000","last_undegraded":"2023-10-09T05:31:31.851950+0000","last_fullsized":"2023-10-09T05:31:31.851950+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852458+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:31:31.852458+0000","last_peered":"2023-10-09T05:31:31.852458+0000","last_clean":"2023-10-09T05:31:31.852458+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:31:31.852458+0000","last_undegraded":"2023-10-09T05:31:31.852458+0000","last_fullsized":"2023-10-09T05:31:31.852458+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855645+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:31:31.855645+0000","last_peered":"2023-10-09T05:31:31.855645+0000","last_clean":"2023-10-09T05:31:31.855645+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:31:31.855645+0000","last_undegraded":"2023-10-09T05:31:31.855645+0000","last_fullsized":"2023-10-09T05:31:31.855645+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.864605+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:31:32.864605+0000","last_peered":"2023-10-09T05:31:32.864605+0000","last_clean":"2023-10-09T05:31:32.864605+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:31:32.864605+0000","last_undegraded":"2023-10-09T05:31:32.864605+0000","last_fullsized":"2023-10-09T05:31:32.864605+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.862044+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:31:32.862044+0000","last_peered":"2023-10-09T05:31:32.862044+0000","last_clean":"2023-10-09T05:31:32.862044+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:31:32.862044+0000","last_undegraded":"2023-10-09T05:31:32.862044+0000","last_fullsized":"2023-10-09T05:31:32.862044+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.853675+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:31:32.853675+0000","last_peered":"2023-10-09T05:31:32.853675+0000","last_clean":"2023-10-09T05:31:32.853675+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:31:32.853675+0000","last_undegraded":"2023-10-09T05:31:32.853675+0000","last_fullsized":"2023-10-09T05:31:32.853675+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866394+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:31:32.866394+0000","last_peered":"2023-10-09T05:31:32.866394+0000","last_clean":"2023-10-09T05:31:32.866394+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:31:32.866394+0000","last_undegraded":"2023-10-09T05:31:32.866394+0000","last_fullsized":"2023-10-09T05:31:32.866394+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855464+0000","last_change":"2023-10-09T05:31:02.548845+0000","last_active":"2023-10-09T05:31:31.855464+0000","last_peered":"2023-10-09T05:31:31.855464+0000","last_clean":"2023-10-09T05:31:31.855464+0000","last_became_active":"2023-10-09T05:31:02.548131+0000","last_became_peered":"2023-10-09T05:31:02.548131+0000","last_unstale":"2023-10-09T05:31:31.855464+0000","last_undegraded":"2023-10-09T05:31:31.855464+0000","last_fullsized":"2023-10-09T05:31:31.855464+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.854508+0000","last_change":"2023-10-09T05:21:16.781147+0000","last_active":"2023-10-09T05:31:31.854508+0000","last_peered":"2023-10-09T05:31:31.854508+0000","last_clean":"2023-10-09T05:31:31.854508+0000","last_became_active":"2023-10-09T05:21:16.780755+0000","last_became_peered":"2023-10-09T05:21:16.780755+0000","last_unstale":"2023-10-09T05:31:31.854508+0000","last_undegraded":"2023-10-09T05:31:31.854508+0000","last_fullsized":"2023-10-09T05:31:31.854508+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869504+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:31:32.869504+0000","last_peered":"2023-10-09T05:31:32.869504+0000","last_clean":"2023-10-09T05:31:32.869504+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:31:32.869504+0000","last_undegraded":"2023-10-09T05:31:32.869504+0000","last_fullsized":"2023-10-09T05:31:32.869504+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.853555+0000","last_change":"2023-10-09T05:30:30.457635+0000","last_active":"2023-10-09T05:31:31.853555+0000","last_peered":"2023-10-09T05:31:31.853555+0000","last_clean":"2023-10-09T05:31:31.853555+0000","last_became_active":"2023-10-09T05:30:30.457276+0000","last_became_peered":"2023-10-09T05:30:30.457276+0000","last_unstale":"2023-10-09T05:31:31.853555+0000","last_undegraded":"2023-10-09T05:31:31.853555+0000","last_fullsized":"2023-10-09T05:31:31.853555+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848299+0000","last_change":"2023-10-09T05:21:17.464398+0000","last_active":"2023-10-09T05:31:31.848299+0000","last_peered":"2023-10-09T05:31:31.848299+0000","last_clean":"2023-10-09T05:31:31.848299+0000","last_became_active":"2023-10-09T05:21:17.464086+0000","last_became_peered":"2023-10-09T05:21:17.464086+0000","last_unstale":"2023-10-09T05:31:31.848299+0000","last_undegraded":"2023-10-09T05:31:31.848299+0000","last_fullsized":"2023-10-09T05:31:31.848299+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855156+0000","last_change":"2023-10-09T05:29:59.470664+0000","last_active":"2023-10-09T05:31:31.855156+0000","last_peered":"2023-10-09T05:31:31.855156+0000","last_clean":"2023-10-09T05:31:31.855156+0000","last_became_active":"2023-10-09T05:29:59.470276+0000","last_became_peered":"2023-10-09T05:29:59.470276+0000","last_unstale":"2023-10-09T05:31:31.855156+0000","last_undegraded":"2023-10-09T05:31:31.855156+0000","last_fullsized":"2023-10-09T05:31:31.855156+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.871093+0000","last_change":"2023-10-09T05:31:32.871093+0000","last_active":"2023-10-09T05:31:32.871093+0000","last_peered":"2023-10-09T05:31:32.871093+0000","last_clean":"2023-10-09T05:31:32.871093+0000","last_became_active":"2023-10-09T05:31:32.870702+0000","last_became_peered":"2023-10-09T05:31:32.870702+0000","last_unstale":"2023-10-09T05:31:32.871093+0000","last_undegraded":"2023-10-09T05:31:32.871093+0000","last_fullsized":"2023-10-09T05:31:32.871093+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:18.439344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":79,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.856510+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:31:31.856510+0000","last_peered":"2023-10-09T05:31:31.856510+0000","last_clean":"2023-10-09T05:31:31.856510+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:31:31.856510+0000","last_undegraded":"2023-10-09T05:31:31.856510+0000","last_fullsized":"2023-10-09T05:31:31.856510+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866775+0000","last_change":"2023-10-09T05:31:32.861092+0000","last_active":"2023-10-09T05:31:32.866775+0000","last_peered":"2023-10-09T05:31:32.866775+0000","last_clean":"2023-10-09T05:31:32.866775+0000","last_became_active":"2023-10-09T05:31:32.860632+0000","last_became_peered":"2023-10-09T05:31:32.860632+0000","last_unstale":"2023-10-09T05:31:32.866775+0000","last_undegraded":"2023-10-09T05:31:32.866775+0000","last_fullsized":"2023-10-09T05:31:32.866775+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:54.308161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852402+0000","last_change":"2023-10-09T05:21:16.784983+0000","last_active":"2023-10-09T05:31:31.852402+0000","last_peered":"2023-10-09T05:31:31.852402+0000","last_clean":"2023-10-09T05:31:31.852402+0000","last_became_active":"2023-10-09T05:21:16.784699+0000","last_became_peered":"2023-10-09T05:21:16.784699+0000","last_unstale":"2023-10-09T05:31:31.852402+0000","last_undegraded":"2023-10-09T05:31:31.852402+0000","last_fullsized":"2023-10-09T05:31:31.852402+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866744+0000","last_change":"2023-10-09T05:31:32.866744+0000","last_active":"2023-10-09T05:31:32.866744+0000","last_peered":"2023-10-09T05:31:32.866744+0000","last_clean":"2023-10-09T05:31:32.866744+0000","last_became_active":"2023-10-09T05:31:32.866389+0000","last_became_peered":"2023-10-09T05:31:32.866389+0000","last_unstale":"2023-10-09T05:31:32.866744+0000","last_undegraded":"2023-10-09T05:31:32.866744+0000","last_fullsized":"2023-10-09T05:31:32.866744+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866600+0000","last_change":"2023-10-09T05:31:32.861963+0000","last_active":"2023-10-09T05:31:32.866600+0000","last_peered":"2023-10-09T05:31:32.866600+0000","last_clean":"2023-10-09T05:31:32.866600+0000","last_became_active":"2023-10-09T05:31:32.861679+0000","last_became_peered":"2023-10-09T05:31:32.861679+0000","last_unstale":"2023-10-09T05:31:32.866600+0000","last_undegraded":"2023-10-09T05:31:32.866600+0000","last_fullsized":"2023-10-09T05:31:32.866600+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:42.402077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.871695+0000","last_change":"2023-10-09T05:31:32.871695+0000","last_active":"2023-10-09T05:31:32.871695+0000","last_peered":"2023-10-09T05:31:32.871695+0000","last_clean":"2023-10-09T05:31:32.871695+0000","last_became_active":"2023-10-09T05:31:32.871449+0000","last_became_peered":"2023-10-09T05:31:32.871449+0000","last_unstale":"2023-10-09T05:31:32.871695+0000","last_undegraded":"2023-10-09T05:31:32.871695+0000","last_fullsized":"2023-10-09T05:31:32.871695+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:22.999187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.852684+0000","last_change":"2023-10-09T05:31:02.550860+0000","last_active":"2023-10-09T05:31:31.852684+0000","last_peered":"2023-10-09T05:31:31.852684+0000","last_clean":"2023-10-09T05:31:31.852684+0000","last_became_active":"2023-10-09T05:31:02.550469+0000","last_became_peered":"2023-10-09T05:31:02.550469+0000","last_unstale":"2023-10-09T05:31:31.852684+0000","last_undegraded":"2023-10-09T05:31:31.852684+0000","last_fullsized":"2023-10-09T05:31:31.852684+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848987+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:31:31.848987+0000","last_peered":"2023-10-09T05:31:31.848987+0000","last_clean":"2023-10-09T05:31:31.848987+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:31:31.848987+0000","last_undegraded":"2023-10-09T05:31:31.848987+0000","last_fullsized":"2023-10-09T05:31:31.848987+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850669+0000","last_change":"2023-10-09T05:21:16.778811+0000","last_active":"2023-10-09T05:31:31.850669+0000","last_peered":"2023-10-09T05:31:31.850669+0000","last_clean":"2023-10-09T05:31:31.850669+0000","last_became_active":"2023-10-09T05:21:16.778515+0000","last_became_peered":"2023-10-09T05:21:16.778515+0000","last_unstale":"2023-10-09T05:31:31.850669+0000","last_undegraded":"2023-10-09T05:31:31.850669+0000","last_fullsized":"2023-10-09T05:31:31.850669+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855330+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:31:31.855330+0000","last_peered":"2023-10-09T05:31:31.855330+0000","last_clean":"2023-10-09T05:31:31.855330+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:31:31.855330+0000","last_undegraded":"2023-10-09T05:31:31.855330+0000","last_fullsized":"2023-10-09T05:31:31.855330+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.863294+0000","last_change":"2023-10-09T05:31:32.863294+0000","last_active":"2023-10-09T05:31:32.863294+0000","last_peered":"2023-10-09T05:31:32.863294+0000","last_clean":"2023-10-09T05:31:32.863294+0000","last_became_active":"2023-10-09T05:31:32.862914+0000","last_became_peered":"2023-10-09T05:31:32.862914+0000","last_unstale":"2023-10-09T05:31:32.863294+0000","last_undegraded":"2023-10-09T05:31:32.863294+0000","last_fullsized":"2023-10-09T05:31:32.863294+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.863335+0000","last_change":"2023-10-09T05:31:32.863335+0000","last_active":"2023-10-09T05:31:32.863335+0000","last_peered":"2023-10-09T05:31:32.863335+0000","last_clean":"2023-10-09T05:31:32.863335+0000","last_became_active":"2023-10-09T05:31:32.863036+0000","last_became_peered":"2023-10-09T05:31:32.863036+0000","last_unstale":"2023-10-09T05:31:32.863335+0000","last_undegraded":"2023-10-09T05:31:32.863335+0000","last_fullsized":"2023-10-09T05:31:32.863335+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:59.787997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.848623+0000","last_change":"2023-10-09T05:30:30.457540+0000","last_active":"2023-10-09T05:31:31.848623+0000","last_peered":"2023-10-09T05:31:31.848623+0000","last_clean":"2023-10-09T05:31:31.848623+0000","last_became_active":"2023-10-09T05:30:30.457180+0000","last_became_peered":"2023-10-09T05:30:30.457180+0000","last_unstale":"2023-10-09T05:31:31.848623+0000","last_undegraded":"2023-10-09T05:31:31.848623+0000","last_fullsized":"2023-10-09T05:31:31.848623+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.870178+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:31:32.870178+0000","last_peered":"2023-10-09T05:31:32.870178+0000","last_clean":"2023-10-09T05:31:32.870178+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:31:32.870178+0000","last_undegraded":"2023-10-09T05:31:32.870178+0000","last_fullsized":"2023-10-09T05:31:32.870178+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866729+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:31:32.866729+0000","last_peered":"2023-10-09T05:31:32.866729+0000","last_clean":"2023-10-09T05:31:32.866729+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:31:32.866729+0000","last_undegraded":"2023-10-09T05:31:32.866729+0000","last_fullsized":"2023-10-09T05:31:32.866729+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.855720+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:31:31.855720+0000","last_peered":"2023-10-09T05:31:31.855720+0000","last_clean":"2023-10-09T05:31:31.855720+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:31:31.855720+0000","last_undegraded":"2023-10-09T05:31:31.855720+0000","last_fullsized":"2023-10-09T05:31:31.855720+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.867898+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:31:32.867898+0000","last_peered":"2023-10-09T05:31:32.867898+0000","last_clean":"2023-10-09T05:31:32.867898+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:31:32.867898+0000","last_undegraded":"2023-10-09T05:31:32.867898+0000","last_fullsized":"2023-10-09T05:31:32.867898+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.866758+0000","last_change":"2023-10-09T05:31:32.860920+0000","last_active":"2023-10-09T05:31:32.866758+0000","last_peered":"2023-10-09T05:31:32.866758+0000","last_clean":"2023-10-09T05:31:32.866758+0000","last_became_active":"2023-10-09T05:31:32.860244+0000","last_became_peered":"2023-10-09T05:31:32.860244+0000","last_unstale":"2023-10-09T05:31:32.866758+0000","last_undegraded":"2023-10-09T05:31:32.866758+0000","last_fullsized":"2023-10-09T05:31:32.866758+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:37.134584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.869382+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:31:32.869382+0000","last_peered":"2023-10-09T05:31:32.869382+0000","last_clean":"2023-10-09T05:31:32.869382+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:31:32.869382+0000","last_undegraded":"2023-10-09T05:31:32.869382+0000","last_fullsized":"2023-10-09T05:31:32.869382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T05:31:32.868031+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:31:32.868031+0000","last_peered":"2023-10-09T05:31:32.868031+0000","last_clean":"2023-10-09T05:31:32.868031+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:31:32.868031+0000","last_undegraded":"2023-10-09T05:31:32.868031+0000","last_fullsized":"2023-10-09T05:31:32.868031+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":87,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T05:31:31.850461+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:31:31.850461+0000","last_peered":"2023-10-09T05:31:31.850461+0000","last_clean":"2023-10-09T05:31:31.850461+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:31:31.850461+0000","last_undegraded":"2023-10-09T05:31:31.850461+0000","last_fullsized":"2023-10-09T05:31:31.850461+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26751,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430365,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306928,"kb_used_data":5736,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438224,"statfs":{"total":95995035648,"available":95680741376,"internally_reserved":0,"allocated":5873664,"data_stored":5368692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:31:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.374},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.44}]},{"osd":1,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.42299999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:32:05 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.46899999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:31:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.59499999999999997,"15min":0.60299999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.75700000000000001,"5min":1.153,"15min":2.6669999999999998},"last":0.504},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.58199999999999996,"15min":0.58999999999999997},"min":{"1min":0.40000000000000002,"5min":0.39700000000000002,"15min":0.32000000000000001},"max":{"1min":0.76500000000000001,"5min":1.0309999999999999,"15min":2.1379999999999999},"last":0.56799999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.49099999999999999,"15min":0.46500000000000002},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.249},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.52500000000000002,"15min":0.50800000000000001},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27300000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.61899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.53000000000000003,"15min":0.50700000000000001},"min":{"1min":0.35799999999999998,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":1.462},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.40200000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88700000000000001},"last":0.58199999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.56699999999999995,"15min":0.54000000000000004},"min":{"1min":0.42299999999999999,"5min":0.34799999999999998,"15min":0.30499999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.623,"5min":0.59599999999999997,"15min":0.58499999999999996},"min":{"1min":0.45000000000000001,"5min":0.372,"15min":0.33300000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.53700000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626593,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306916,"kb_used_data":5724,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438236,"statfs":{"total":95995035648,"available":95680753664,"internally_reserved":0,"allocated":5861376,"data_stored":5368300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.55500000000000005,"15min":0.54500000000000004},"min":{"1min":0.39300000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.014,"5min":0.63800000000000001,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.4569999999999999,"5min":7.4569999999999999,"15min":7.4569999999999999},"last":0.45500000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.46899999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.075,"15min":1.075},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.5999999999999996,"15min":7.5999999999999996},"last":0.505},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":7.6379999999999999,"5min":7.6379999999999999,"15min":7.6379999999999999},"last":0.54800000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:30:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.55700000000000005,"15min":0.55200000000000005},"min":{"1min":0.38600000000000001,"5min":0.38100000000000001,"15min":0.33300000000000002},"max":{"1min":0.73199999999999998,"5min":0.80700000000000005,"15min":3.9569999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59399999999999997,"15min":0.59499999999999997},"min":{"1min":0.436,"5min":0.34899999999999998,"15min":0.33800000000000002},"max":{"1min":0.79700000000000004,"5min":0.81999999999999995,"15min":2.9049999999999998},"last":0.65000000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.47399999999999998,"15min":0.46100000000000002},"min":{"1min":0.29099999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.86399999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.49399999999999999,"15min":0.47299999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":0.78800000000000003,"5min":0.83799999999999997,"15min":1.014},"last":0.40600000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.53700000000000003,"15min":0.504},"min":{"1min":0.33400000000000002,"5min":0.30499999999999999,"15min":0.27200000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.625},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.51200000000000001},"min":{"1min":0.35999999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.73299999999999998,"5min":0.82399999999999995,"15min":2.0579999999999998},"last":0.495}]},{"osd":7,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.54200000000000004,"15min":0.53300000000000003},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.60199999999999998,"15min":0.56999999999999995},"min":{"1min":0.498,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.999},"last":0.60299999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958554,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32640,"kb_used_data":1080,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712512,"statfs":{"total":95995035648,"available":95961612288,"internally_reserved":0,"allocated":1105920,"data_stored":587447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":32301582},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.372},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.313}]},{"osd":2,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.32900000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34799999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.72999999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.878}]},{"osd":6,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.90100000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1,"5min":1,"15min":1},"last":0.82299999999999995}]}]},{"osd":0,"up_from":88,"seq":377957122080,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33228,"kb_used_data":1668,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93711924,"statfs":{"total":95995035648,"available":95961010176,"internally_reserved":0,"allocated":1708032,"data_stored":1178579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":32300808},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.29999999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.247},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.316}]},{"osd":3,"last update":"Mon Oct 9 05:31:00 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.33300000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.65900000000000003,"15min":0.67300000000000004},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.83699999999999997,"5min":1.028,"15min":1.028},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.69999999999999996,"15min":0.71399999999999997},"min":{"1min":0.45300000000000001,"5min":0.434,"15min":0.434},"max":{"1min":0.81799999999999995,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.65400000000000003,"15min":0.67200000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.70399999999999996,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.65600000000000003,"15min":0.66900000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.79200000000000004,"5min":1.022,"15min":1.022},"last":0.54700000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.68400000000000005,"15min":0.69899999999999995},"min":{"1min":0.37,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80300000000000005,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.66300000000000003,"15min":0.67300000000000004},"min":{"1min":0.42899999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.85599999999999998,"5min":1.044,"15min":1.044},"last":0.59599999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.68999999999999995,"15min":0.70299999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.89600000000000002,"5min":1.052,"15min":1.052},"last":0.624},{"interface":"front","average":{"1min":0.621,"5min":0.71299999999999997,"15min":0.72799999999999998},"min":{"1min":0.38500000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77500000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.64100000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795027,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36668,"kb_used_data":5172,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708484,"statfs":{"total":95995035648,"available":95957487616,"internally_reserved":0,"allocated":5296128,"data_stored":4780992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":32232021},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.53600000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.56599999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61799999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.96299999999999997},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.86199999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.82999999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.94799999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.82299999999999995}]}]},{"osd":3,"up_from":103,"seq":442381631501,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28392,"kb_used_data":1056,"kb_used_omap":4,"kb_used_meta":27323,"kb_avail":93716760,"statfs":{"total":95995035648,"available":95965962240,"internally_reserved":0,"allocated":1081344,"data_stored":583603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":27979358},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34399999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.37}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66700000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64700000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61499999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019049,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":583950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52200000000000002,"15min":0.52100000000000002},"min":{"1min":0.373,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73299999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.442}]},{"osd":1,"last update":"Mon Oct 9 05:31:33 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.55500000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:32:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.58999999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.58299999999999996,"15min":0.56000000000000005},"min":{"1min":0.38500000000000001,"5min":0.374,"15min":0.28799999999999998},"max":{"1min":0.73899999999999999,"5min":0.84299999999999997,"15min":1.591},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56100000000000005,"15min":0.54700000000000004},"min":{"1min":0.434,"5min":0.373,"15min":0.33600000000000002},"max":{"1min":0.66700000000000004,"5min":0.877,"15min":0.877},"last":0.65900000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:31:27 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46800000000000003,"15min":0.441},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26700000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.85399999999999998},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56100000000000005,"15min":0.52900000000000003},"min":{"1min":0.44700000000000001,"5min":0.36599999999999999,"15min":0.311},"max":{"1min":0.70499999999999996,"5min":0.79600000000000004,"15min":1.54},"last":0.746}]},{"osd":6,"last update":"Mon Oct 9 05:31:40 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.48999999999999999,"15min":0.48899999999999999},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.27800000000000002},"max":{"1min":0.74399999999999999,"5min":0.80300000000000005,"15min":1.5660000000000001},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.502,"15min":0.503},"min":{"1min":0.371,"5min":0.33100000000000002,"15min":0.28899999999999998},"max":{"1min":0.752,"5min":0.76300000000000001,"15min":0.94699999999999995},"last":0.68999999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:32:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53500000000000003,"15min":0.53600000000000003},"min":{"1min":0.433,"5min":0.35499999999999998,"15min":0.30499999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":1.6160000000000001},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53300000000000003,"15min":0.53000000000000003},"min":{"1min":0.46700000000000003,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":0.76700000000000002,"5min":0.79000000000000004,"15min":1.034},"last":0.68000000000000005}]}]},{"osd":5,"up_from":36,"seq":154618822821,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298136,"kb_used_data":1040,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447016,"statfs":{"total":95995035648,"available":95689744384,"internally_reserved":0,"allocated":1064960,"data_stored":579398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70499999999999996,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.39500000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.56100000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.59299999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.53800000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:30:24 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.60699999999999998,"15min":0.58499999999999996},"min":{"1min":0.47699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.94899999999999995},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.64800000000000002,"15min":0.627},"min":{"1min":0.501,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.85899999999999999,"5min":0.92800000000000005,"15min":0.93899999999999995},"last":0.442}]},{"osd":4,"last update":"Mon Oct 9 05:32:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.49099999999999999,"15min":0.45900000000000002},"min":{"1min":0.38100000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":0.73999999999999999,"5min":0.874,"15min":0.94499999999999995},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.55000000000000004,"15min":0.501},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28699999999999998},"max":{"1min":0.749,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.622}]},{"osd":6,"last update":"Mon Oct 9 05:31:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.55600000000000005,"15min":0.505},"min":{"1min":0.41199999999999998,"5min":0.34999999999999998,"15min":0.26000000000000001},"max":{"1min":0.78500000000000003,"5min":0.93500000000000005,"15min":1.1339999999999999},"last":0.373},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.56100000000000005,"15min":0.53100000000000003},"min":{"1min":0.42999999999999999,"5min":0.36199999999999999,"15min":0.314},"max":{"1min":0.78800000000000003,"5min":0.92100000000000004,"15min":1.109},"last":0.70799999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:32:10 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.59399999999999997,"15min":0.57699999999999996},"min":{"1min":0.52400000000000002,"5min":0.40200000000000002,"15min":0.34399999999999997},"max":{"1min":0.71499999999999997,"5min":0.95199999999999996,"15min":1.143},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57399999999999995,"15min":0.55600000000000005},"min":{"1min":0.51100000000000001,"5min":0.35199999999999998,"15min":0.32400000000000001},"max":{"1min":0.83499999999999996,"5min":0.90900000000000003,"15min":1.1830000000000001},"last":0.63600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:32:32.019 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:32:32.019 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T05:32:32.380 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:32.784 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:32 smithi008 ceph-mon[184418]: from='client.44621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:32.784 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:32 smithi008 ceph-mon[191227]: from='client.44621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:32.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:32 smithi191 ceph-mon[161669]: from='client.44621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:32:33.623 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:33 smithi008 ceph-mon[184418]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:33.624 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:33 smithi008 ceph-mon[191227]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:33 smithi191 ceph-mon[161669]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:34.019 INFO:teuthology.orchestra.run.smithi008.stdout:epoch 104 2023-10-09T05:32:34.020 INFO:teuthology.orchestra.run.smithi008.stdout:fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 2023-10-09T05:32:34.020 INFO:teuthology.orchestra.run.smithi008.stdout:created 2023-10-09T05:14:56.160913+0000 2023-10-09T05:32:34.020 INFO:teuthology.orchestra.run.smithi008.stdout:modified 2023-10-09T05:31:32.844881+0000 2023-10-09T05:32:34.020 INFO:teuthology.orchestra.run.smithi008.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T05:32:34.020 INFO:teuthology.orchestra.run.smithi008.stdout:crush_version 22 2023-10-09T05:32:34.020 INFO:teuthology.orchestra.run.smithi008.stdout:full_ratio 0.95 2023-10-09T05:32:34.020 INFO:teuthology.orchestra.run.smithi008.stdout:backfillfull_ratio 0.9 2023-10-09T05:32:34.020 INFO:teuthology.orchestra.run.smithi008.stdout:nearfull_ratio 0.85 2023-10-09T05:32:34.020 INFO:teuthology.orchestra.run.smithi008.stdout:require_min_compat_client luminous 2023-10-09T05:32:34.020 INFO:teuthology.orchestra.run.smithi008.stdout:min_compat_client luminous 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:require_osd_release pacific 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:stretch_mode_enabled false 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 81 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 76 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:max_osd 8 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0 up in weight 1 up_from 88 up_thru 103 down_at 85 last_clean_interval [8,84) [v2:172.21.15.8:6802/633445692,v1:172.21.15.8:6803/633445692] [v2:172.21.15.8:6804/633445692,v1:172.21.15.8:6805/633445692] exists,up 457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 up in weight 1 up_from 93 up_thru 103 down_at 90 last_clean_interval [13,89) [v2:172.21.15.8:6810/417852095,v1:172.21.15.8:6811/417852095] [v2:172.21.15.8:6812/417852095,v1:172.21.15.8:6813/417852095] exists,up 55f3290e-b82d-4839-a28d-1123a20c3a4c 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2 up in weight 1 up_from 98 up_thru 103 down_at 95 last_clean_interval [18,94) [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054] [v2:172.21.15.8:6820/1092325054,v1:172.21.15.8:6821/1092325054] exists,up 720b47df-d743-46cf-ad04-36e113c8fcb3 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 up in weight 1 up_from 103 up_thru 103 down_at 100 last_clean_interval [25,99) [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505] [v2:172.21.15.8:6828/4118391505,v1:172.21.15.8:6829/4118391505] exists,up 861bf05d-69f8-4000-999e-ab708531edb0 2023-10-09T05:32:34.021 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 up in weight 1 up_from 30 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6800/3473028939,v1:172.21.15.191:6801/3473028939] [v2:172.21.15.191:6802/3473028939,v1:172.21.15.191:6803/3473028939] exists,up b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9 2023-10-09T05:32:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 up in weight 1 up_from 36 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6808/1592222523,v1:172.21.15.191:6809/1592222523] [v2:172.21.15.191:6810/1592222523,v1:172.21.15.191:6811/1592222523] exists,up 01507478-481e-4646-a30d-542184240f07 2023-10-09T05:32:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6 up in weight 1 up_from 42 up_thru 100 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6816/4267099596,v1:172.21.15.191:6817/4267099596] [v2:172.21.15.191:6818/4267099596,v1:172.21.15.191:6819/4267099596] exists,up 8bc75ec3-46ee-4d36-989b-4e65e330cfb9 2023-10-09T05:32:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7 up in weight 1 up_from 48 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6824/1787611943,v1:172.21.15.191:6825/1787611943] [v2:172.21.15.191:6826/1787611943,v1:172.21.15.191:6827/1787611943] exists,up 5ac2e382-7341-47dd-b6c9-0fbdba12829c 2023-10-09T05:32:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:pg_upmap_items 4.4 [1,5] 2023-10-09T05:32:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:pg_upmap_items 4.a [1,2] 2023-10-09T05:32:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:pg_upmap_items 4.e [1,2] 2023-10-09T05:32:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/1497753039 expires 2023-10-10T05:25:43.728914+0000 2023-10-09T05:32:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/1025807783 expires 2023-10-10T05:25:43.728914+0000 2023-10-09T05:32:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/3932209052 expires 2023-10-10T05:25:43.728914+0000 2023-10-09T05:32:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/3599326555 expires 2023-10-10T05:25:43.728914+0000 2023-10-09T05:32:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:6800/2853847379 expires 2023-10-10T05:25:43.728914+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.191:6835/1974805222 expires 2023-10-10T05:25:00.410091+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.191:6834/1974805222 expires 2023-10-10T05:25:00.410091+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:6801/2430836187 expires 2023-10-10T05:15:24.205194+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:6800/2430836187 expires 2023-10-10T05:15:24.205194+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/1669399762 expires 2023-10-10T05:23:46.082746+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:6800/310049223 expires 2023-10-10T05:15:49.270008+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/357500347 expires 2023-10-10T05:23:46.082746+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/711490379 expires 2023-10-10T05:25:43.728914+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:6801/310049223 expires 2023-10-10T05:15:49.270008+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.191:0/1074075163 expires 2023-10-10T05:25:00.410091+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.191:0/3279324894 expires 2023-10-10T05:25:00.410091+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:6801/2854879000 expires 2023-10-10T05:23:46.082746+0000 2023-10-09T05:32:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/819405330 expires 2023-10-10T05:25:43.728914+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/2096086544 expires 2023-10-10T05:15:49.270008+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.191:0/1645031799 expires 2023-10-10T05:25:00.410091+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:6801/2853847379 expires 2023-10-10T05:25:43.728914+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/2360193392 expires 2023-10-10T05:15:24.205194+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.191:0/2331494796 expires 2023-10-10T05:25:00.410091+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/1936221066 expires 2023-10-10T05:20:01.878268+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/1062877078 expires 2023-10-10T05:15:49.270008+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/69531294 expires 2023-10-10T05:15:24.205194+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:6800/1748522337 expires 2023-10-10T05:20:01.878268+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.191:0/465298627 expires 2023-10-10T05:25:00.410091+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:6800/2854879000 expires 2023-10-10T05:23:46.082746+0000 2023-10-09T05:32:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:0/2178079664 expires 2023-10-10T05:20:01.878268+0000 2023-10-09T05:32:34.025 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.8:6801/1748522337 expires 2023-10-10T05:20:01.878268+0000 2023-10-09T05:32:34.025 INFO:teuthology.orchestra.run.smithi008.stdout:blocklist 172.21.15.191:0/3608336050 expires 2023-10-10T05:25:00.410091+0000 2023-10-09T05:32:34.645 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 88 up_thru 103 down_at 85 last_clean_interval [8,84) [v2:172.21.15.8:6802/633445692,v1:172.21.15.8:6803/633445692] [v2:172.21.15.8:6804/633445692,v1:172.21.15.8:6805/633445692] exists,up 457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc', 'osd.1 up in weight 1 up_from 93 up_thru 103 down_at 90 last_clean_interval [13,89) [v2:172.21.15.8:6810/417852095,v1:172.21.15.8:6811/417852095] [v2:172.21.15.8:6812/417852095,v1:172.21.15.8:6813/417852095] exists,up 55f3290e-b82d-4839-a28d-1123a20c3a4c', 'osd.2 up in weight 1 up_from 98 up_thru 103 down_at 95 last_clean_interval [18,94) [v2:172.21.15.8:6818/1092325054,v1:172.21.15.8:6819/1092325054] [v2:172.21.15.8:6820/1092325054,v1:172.21.15.8:6821/1092325054] exists,up 720b47df-d743-46cf-ad04-36e113c8fcb3', 'osd.3 up in weight 1 up_from 103 up_thru 103 down_at 100 last_clean_interval [25,99) [v2:172.21.15.8:6826/4118391505,v1:172.21.15.8:6827/4118391505] [v2:172.21.15.8:6828/4118391505,v1:172.21.15.8:6829/4118391505] exists,up 861bf05d-69f8-4000-999e-ab708531edb0', 'osd.4 up in weight 1 up_from 30 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6800/3473028939,v1:172.21.15.191:6801/3473028939] [v2:172.21.15.191:6802/3473028939,v1:172.21.15.191:6803/3473028939] exists,up b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9', 'osd.5 up in weight 1 up_from 36 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6808/1592222523,v1:172.21.15.191:6809/1592222523] [v2:172.21.15.191:6810/1592222523,v1:172.21.15.191:6811/1592222523] exists,up 01507478-481e-4646-a30d-542184240f07', 'osd.6 up in weight 1 up_from 42 up_thru 100 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6816/4267099596,v1:172.21.15.191:6817/4267099596] [v2:172.21.15.191:6818/4267099596,v1:172.21.15.191:6819/4267099596] exists,up 8bc75ec3-46ee-4d36-989b-4e65e330cfb9', 'osd.7 up in weight 1 up_from 48 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6824/1787611943,v1:172.21.15.191:6825/1787611943] [v2:172.21.15.191:6826/1787611943,v1:172.21.15.191:6827/1787611943] exists,up 5ac2e382-7341-47dd-b6c9-0fbdba12829c'] 2023-10-09T05:32:34.646 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T05:32:34.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2635151643' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:32:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2635151643' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:32:34.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2635151643' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:32:35.009 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:35.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:35 smithi191 ceph-mon[161669]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:35.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:35 smithi008 ceph-mon[184418]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:35.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:35 smithi008 ceph-mon[191227]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:36.549 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:32:36.549 INFO:teuthology.orchestra.run.smithi008.stdout: "name": "mon.a", 2023-10-09T05:32:36.549 INFO:teuthology.orchestra.run.smithi008.stdout: "cluster": "ceph", 2023-10-09T05:32:36.549 INFO:teuthology.orchestra.run.smithi008.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T05:32:36.550 INFO:teuthology.orchestra.run.smithi008.stdout: "admin_socket_mode": "", 2023-10-09T05:32:36.550 INFO:teuthology.orchestra.run.smithi008.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T05:32:36.550 INFO:teuthology.orchestra.run.smithi008.stdout: "auth_client_required": "cephx, none", 2023-10-09T05:32:36.550 INFO:teuthology.orchestra.run.smithi008.stdout: "auth_cluster_required": "cephx", 2023-10-09T05:32:36.550 INFO:teuthology.orchestra.run.smithi008.stdout: "auth_debug": "true", 2023-10-09T05:32:36.550 INFO:teuthology.orchestra.run.smithi008.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T05:32:36.550 INFO:teuthology.orchestra.run.smithi008.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T05:32:36.550 INFO:teuthology.orchestra.run.smithi008.stdout: "auth_service_required": "cephx", 2023-10-09T05:32:36.550 INFO:teuthology.orchestra.run.smithi008.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T05:32:36.550 INFO:teuthology.orchestra.run.smithi008.stdout: "auth_supported": "", 2023-10-09T05:32:36.550 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_aio": "true", 2023-10-09T05:32:36.551 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T05:32:36.551 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T05:32:36.551 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_aio_reap_max": "16", 2023-10-09T05:32:36.551 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_async_discard": "false", 2023-10-09T05:32:36.551 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_block_size": "4096", 2023-10-09T05:32:36.551 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_debug_aio": "false", 2023-10-09T05:32:36.551 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T05:32:36.551 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T05:32:36.551 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T05:32:36.551 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_enable_discard": "false", 2023-10-09T05:32:36.551 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_flock_retry": "3", 2023-10-09T05:32:36.552 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T05:32:36.552 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_inject_crash": "0", 2023-10-09T05:32:36.552 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T05:32:36.552 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_ioring": "false", 2023-10-09T05:32:36.552 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_ioring_hipri": "false", 2023-10-09T05:32:36.552 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T05:32:36.552 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T05:32:36.552 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T05:32:36.552 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T05:32:36.552 INFO:teuthology.orchestra.run.smithi008.stdout: "bdev_type": "", 2023-10-09T05:32:36.553 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T05:32:36.553 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_allocator": "hybrid", 2023-10-09T05:32:36.553 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_buffered_io": "true", 2023-10-09T05:32:36.553 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T05:32:36.553 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T05:32:36.553 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T05:32:36.553 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T05:32:36.553 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T05:32:36.553 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T05:32:36.553 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T05:32:36.553 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_replay_recovery": "false", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluefs_sync_write": "false", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T05:32:36.554 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T05:32:36.555 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_allocator": "hybrid", 2023-10-09T05:32:36.555 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T05:32:36.555 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T05:32:36.555 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T05:32:36.555 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T05:32:36.555 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T05:32:36.555 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T05:32:36.555 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T05:32:36.555 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_block_create": "true", 2023-10-09T05:32:36.555 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_block_db_create": "false", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_block_db_path": "", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_block_db_size": "0", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_block_path": "", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_block_size": "107374182400", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_block_wal_create": "false", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_block_wal_path": "", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_bluefs": "true", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T05:32:36.556 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_autotune": "true", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_size": "0", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T05:32:36.557 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T05:32:36.558 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T05:32:36.558 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T05:32:36.558 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cache_type": "2q", 2023-10-09T05:32:36.558 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T05:32:36.558 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_clone_cow": "true", 2023-10-09T05:32:36.558 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T05:32:36.558 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T05:32:36.558 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T05:32:36.558 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T05:32:36.558 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T05:32:36.558 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_compression_mode": "none", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_freelist": "false", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T05:32:36.559 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T05:32:36.560 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fail_eio": "false", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T05:32:36.561 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T05:32:36.562 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T05:32:36.562 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T05:32:36.562 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T05:32:36.562 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T05:32:36.562 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T05:32:36.562 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T05:32:36.562 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T05:32:36.562 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T05:32:36.562 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T05:32:36.562 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T05:32:36.562 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_max_blob_size": "0", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T05:32:36.563 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T05:32:36.564 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T05:32:36.565 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T05:32:36.565 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T05:32:36.565 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T05:32:36.565 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T05:32:36.565 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T05:32:36.565 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T05:32:36.565 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_spdk_mem": "512", 2023-10-09T05:32:36.565 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T05:32:36.565 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T05:32:36.565 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T05:32:36.565 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T05:32:36.566 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T05:32:36.566 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T05:32:36.566 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T05:32:36.566 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_tracing": "false", 2023-10-09T05:32:36.566 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T05:32:36.566 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T05:32:36.566 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T05:32:36.566 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T05:32:36.566 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T05:32:36.566 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T05:32:36.566 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T05:32:36.567 INFO:teuthology.orchestra.run.smithi008.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "cephx_cluster_require_version": "2", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "cephx_require_signatures": "false", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "cephx_require_version": "2", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "cephx_service_require_signatures": "false", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "cephx_service_require_version": "2", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "cephx_sign_messages": "true", 2023-10-09T05:32:36.568 INFO:teuthology.orchestra.run.smithi008.stdout: "chdir": "", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_acl_type": "", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_asio_thread_count": "2", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_cache_mid": "0.750000", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_cache_size": "16384", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_caps_release_delay": "5", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_check_pool_perm": "true", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_debug_force_sync_read": "false", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_debug_getattr_caps": "false", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_die_on_failed_remount": "false", 2023-10-09T05:32:36.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_dirsize_rbytes": "true", 2023-10-09T05:32:36.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_force_lazyio": "false", 2023-10-09T05:32:36.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_fs": "", 2023-10-09T05:32:36.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T05:32:36.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_inject_release_failure": "false", 2023-10-09T05:32:36.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_max_inline_size": "4096", 2023-10-09T05:32:36.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T05:32:36.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_mds_namespace": "", 2023-10-09T05:32:36.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_metadata": "", 2023-10-09T05:32:36.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_mount_gid": "-1", 2023-10-09T05:32:36.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_mount_timeout": "300", 2023-10-09T05:32:36.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_mount_uid": "-1", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_mountpoint": "/", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_notify_timeout": "10", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_oc": "true", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_oc_max_objects": "1000", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_oc_size": "209715200", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_permissions": "true", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_quota": "true", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_quota_df": "true", 2023-10-09T05:32:36.571 INFO:teuthology.orchestra.run.smithi008.stdout: "client_readahead_max_bytes": "0", 2023-10-09T05:32:36.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_readahead_max_periods": "4", 2023-10-09T05:32:36.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_readahead_min": "131072", 2023-10-09T05:32:36.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_reconnect_stale": "false", 2023-10-09T05:32:36.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_shutdown_timeout": "30", 2023-10-09T05:32:36.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_snapdir": ".snap", 2023-10-09T05:32:36.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_tick_interval": "1", 2023-10-09T05:32:36.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_trace": "", 2023-10-09T05:32:36.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T05:32:36.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_use_faked_inos": "false", 2023-10-09T05:32:36.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_use_random_mds": "false", 2023-10-09T05:32:36.572 INFO:teuthology.orchestra.run.smithi008.stdout: "clog_to_graylog": "false", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "clog_to_graylog_port": "12201", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "clog_to_monitors": "default=true", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "clog_to_syslog": "false", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "clog_to_syslog_level": "info", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "cluster_addr": "-", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "cluster_network": "", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "cluster_network_interface": "", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "compressor_zlib_isal": "false", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "compressor_zlib_level": "5", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T05:32:36.573 INFO:teuthology.orchestra.run.smithi008.stdout: "compressor_zstd_level": "1", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "crimson_seastar_smp": "1", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "crush_location": "", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "crush_location_hook": "", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "crush_location_hook_timeout": "10", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "daemonize": "false", 2023-10-09T05:32:36.574 INFO:teuthology.orchestra.run.smithi008.stdout: "daos_pool": "tank", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_alienstore": "0/5", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_asok": "1/5", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_asok_assert_abort": "false", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_auth": "1/5", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_bdev": "1/3", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_bluefs": "1/5", 2023-10-09T05:32:36.575 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_bluestore": "1/5", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_buffer": "0/1", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_cephsqlite": "0/5", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_client": "0/5", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_compressor": "1/5", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_context": "0/1", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_crush": "1/1", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_crypto": "1/5", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_cyanstore": "0/5", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T05:32:36.576 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_dpdk": "1/5", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_eventtrace": "1/5", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_filer": "0/1", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_filestore": "1/3", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_finisher": "1/1", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_fuse": "1/5", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_javaclient": "1/5", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_journal": "1/3", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_journaler": "0/5", 2023-10-09T05:32:36.577 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_kstore": "1/5", 2023-10-09T05:32:36.578 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_leveldb": "4/5", 2023-10-09T05:32:36.578 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_lockdep": "0/1", 2023-10-09T05:32:36.578 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_mclock": "1/5", 2023-10-09T05:32:36.578 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_mds": "1/5", 2023-10-09T05:32:36.578 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_mds_balancer": "1/5", 2023-10-09T05:32:36.578 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_mds_locker": "1/5", 2023-10-09T05:32:36.578 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_mds_log": "1/5", 2023-10-09T05:32:36.578 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T05:32:36.578 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_mds_migrator": "1/5", 2023-10-09T05:32:36.578 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_memstore": "1/5", 2023-10-09T05:32:36.578 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_mgr": "2/5", 2023-10-09T05:32:36.579 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_mgrc": "1/5", 2023-10-09T05:32:36.579 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_mon": "20/20", 2023-10-09T05:32:36.579 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_monc": "0/10", 2023-10-09T05:32:36.579 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_ms": "1/1", 2023-10-09T05:32:36.579 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_none": "0/5", 2023-10-09T05:32:36.579 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_objclass": "0/5", 2023-10-09T05:32:36.579 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_objectcacher": "0/5", 2023-10-09T05:32:36.579 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_objecter": "0/1", 2023-10-09T05:32:36.579 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_optracker": "0/5", 2023-10-09T05:32:36.579 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_osd": "1/5", 2023-10-09T05:32:36.580 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_paxos": "20/20", 2023-10-09T05:32:36.580 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_perfcounter": "1/5", 2023-10-09T05:32:36.580 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_prioritycache": "1/5", 2023-10-09T05:32:36.580 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rados": "0/5", 2023-10-09T05:32:36.580 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rbd": "0/5", 2023-10-09T05:32:36.580 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T05:32:36.580 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T05:32:36.580 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rbd_replay": "0/5", 2023-10-09T05:32:36.580 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_refs": "0/0", 2023-10-09T05:32:36.580 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_reserver": "1/1", 2023-10-09T05:32:36.580 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rgw": "1/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rgw_access": "1/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rgw_flight": "1/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rgw_sync": "1/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_rocksdb": "4/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore": "0/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_backref": "0/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_cache": "0/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_device": "0/5", 2023-10-09T05:32:36.581 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_epm": "0/5", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_journal": "0/5", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_lba": "0/5", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_odata": "0/5", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_omap": "0/5", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_onode": "0/5", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_t": "0/5", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_seastore_tm": "0/5", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_striper": "0/1", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_test": "0/5", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_throttle": "1/1", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_timer": "0/1", 2023-10-09T05:32:36.582 INFO:teuthology.orchestra.run.smithi008.stdout: "debug_tp": "0/5", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "device_failure_prediction_mode": "none", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "err_to_graylog": "false", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "err_to_journald": "false", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "err_to_stderr": "true", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "err_to_syslog": "false", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event_tracing": "false", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "exporter_http_port": "9926", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "exporter_prio_limit": "5", 2023-10-09T05:32:36.583 INFO:teuthology.orchestra.run.smithi008.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "exporter_sort_metrics": "true", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "exporter_stats_period": "5", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "fake_statfs_for_testing": "0", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "fatal_signal_handlers": "true", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "filer_max_purge_ops": "10", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "filer_max_truncate_ops": "128", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_blackhole": "false", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_btrfs_snap": "true", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_caller_concurrency": "10", 2023-10-09T05:32:36.584 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_debug_omap_check": "false", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_debug_verify_split": "false", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_dump_file": "", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_fadvise": "true", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_fail_eio": "true", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T05:32:36.585 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_fd_cache_size": "128", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_fiemap": "false", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_inject_stall": "0", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_journal_parallel": "false", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_journal_trailing": "false", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_journal_writeahead": "false", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_kill_at": "0", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T05:32:36.586 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T05:32:36.587 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_merge_threshold": "-10", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_odsync_write": "false", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_omap_backend_path": "", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_op_threads": "2", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_punch_hole": "false", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T05:32:36.588 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_queue_max_ops": "50", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_seek_data_hole": "false", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_sloppy_crc": "false", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T05:32:36.589 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_splice": "false", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_split_multiple": "2", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_split_rand_factor": "20", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_update_to": "1000", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T05:32:36.590 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_xfs_extsize": "false", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "filestore_zfs_snap": "false", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "fio_dir": "/tmp/fio", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "fsid": "9e85b874-6662-11ee-8db5-212e2dc638e7", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_allow_other": "true", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_big_writes": "true", 2023-10-09T05:32:36.591 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_debug": "false", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_default_permissions": "false", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_disable_pagecache": "false", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_max_write": "0", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_multithreaded": "true", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_require_active_mds": "true", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_set_user_groups": "true", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_splice_move": "true", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_splice_read": "true", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_splice_write": "true", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T05:32:36.592 INFO:teuthology.orchestra.run.smithi008.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "gss_target_name": "ceph", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "heartbeat_file": "", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "heartbeat_inject_failure": "0", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "heartbeat_interval": "5", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "host": "smithi008", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T05:32:36.593 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "inject_early_sigterm": "false", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "jaeger_agent_port": "6799", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "jaeger_tracing_enable": "false", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_aio": "true", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_align_min_size": "65536", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_block_align": "true", 2023-10-09T05:32:36.594 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_block_size": "4096", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_dio": "true", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_discard": "false", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_force_aio": "false", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_ignore_corruption": "false", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_max_write_entries": "100", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_replay_from": "0", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T05:32:36.595 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_write_header_frequency": "0", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "journal_zero_on_create": "false", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "journaler_prefetch_periods": "10", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "journaler_prezero_periods": "5", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "journaler_write_head_interval": "15", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "key": "", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "keyfile": "", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "kstore_backend": "rocksdb", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T05:32:36.596 INFO:teuthology.orchestra.run.smithi008.stdout: "kstore_max_bytes": "67108864", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "kstore_max_ops": "512", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "kstore_onode_map_size": "1024", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "kstore_sync_transaction": "false", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "leveldb_block_size": "65536", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "leveldb_bloom_size": "0", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "leveldb_cache_size": "536870912", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "leveldb_compression": "false", 2023-10-09T05:32:36.597 INFO:teuthology.orchestra.run.smithi008.stdout: "leveldb_log": "", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "leveldb_max_open_files": "0", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "leveldb_paranoid": "false", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "librados_thread_count": "2", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "lockdep": "false", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "lockdep_force_backtrace": "false", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "log_coarse_timestamps": "true", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "log_flush_on_exit": "false", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T05:32:36.598 INFO:teuthology.orchestra.run.smithi008.stdout: "log_graylog_port": "12201", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "log_max_new": "1000", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "log_max_recent": "10000", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "log_stderr_prefix": "", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "log_to_file": "true", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "log_to_graylog": "false", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "log_to_journald": "false", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "log_to_stderr": "false", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "log_to_syslog": "false", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T05:32:36.599 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_action_on_write_error": "1", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_alternate_name_max": "8192", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_asio_thread_count": "2", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_export_pin": "true", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_interval": "10", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_max": "-1", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_max_until": "-1", 2023-10-09T05:32:36.600 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_merge_size": "50", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_mode": "0", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_split_bits": "3", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T05:32:36.601 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_split_size": "10000", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_cache_mid": "0.700000", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T05:32:36.602 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_cache_trim_interval": "1", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_debug_auth_pins": "false", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_debug_frag": "false", 2023-10-09T05:32:36.603 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_debug_scatterstat": "false", 2023-10-09T05:32:36.604 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_debug_subtrees": "false", 2023-10-09T05:32:36.604 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T05:32:36.604 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_default_dir_hash": "2", 2023-10-09T05:32:36.604 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_defer_session_stale": "true", 2023-10-09T05:32:36.604 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T05:32:36.604 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T05:32:36.604 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T05:32:36.604 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_dir_max_entries": "0", 2023-10-09T05:32:36.604 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_dir_prefetch": "true", 2023-10-09T05:32:36.604 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T05:32:36.604 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_early_reply": "true", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_enable_op_tracker": "true", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_enforce_unique_name": "true", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T05:32:36.605 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_inject_health_dummy": "false", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T05:32:36.606 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_join_fs": "", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_journal_format": "1", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_kill_create_at": "0", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_kill_export_at": "0", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_kill_import_at": "0", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_kill_journal_at": "0", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_kill_link_at": "0", 2023-10-09T05:32:36.607 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T05:32:36.608 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_kill_openc_at": "0", 2023-10-09T05:32:36.608 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_kill_rename_at": "0", 2023-10-09T05:32:36.608 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T05:32:36.608 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T05:32:36.608 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_log_max_events": "-1", 2023-10-09T05:32:36.608 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_log_max_segments": "128", 2023-10-09T05:32:36.608 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_log_pause": "false", 2023-10-09T05:32:36.608 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_log_segment_size": "0", 2023-10-09T05:32:36.608 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T05:32:36.608 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T05:32:36.608 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_max_completed_requests": "100000", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_max_export_size": "20971520", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_max_file_recover": "32", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_max_purge_files": "64", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_max_purge_ops": "8192", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_metrics_update_interval": "2", 2023-10-09T05:32:36.609 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_min_caps_per_client": "100", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_numa_node": "-1", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_op_history_duration": "600", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_op_history_size": "20", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_op_log_threshold": "5", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_ping_grace": "15", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_ping_interval": "5", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T05:32:36.610 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T05:32:36.611 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_recall_max_caps": "30000", 2023-10-09T05:32:36.611 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T05:32:36.611 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T05:32:36.611 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T05:32:36.611 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T05:32:36.611 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T05:32:36.611 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_replay_interval": "1.000000", 2023-10-09T05:32:36.611 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T05:32:36.611 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T05:32:36.611 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_root_ino_gid": "0", 2023-10-09T05:32:36.611 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_root_ino_uid": "0", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_shutdown_check": "0", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_skip_ino": "0", 2023-10-09T05:32:36.612 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T05:32:36.613 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T05:32:36.613 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_snap_min_uid": "0", 2023-10-09T05:32:36.613 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_snap_rstat": "false", 2023-10-09T05:32:36.613 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T05:32:36.613 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_symlink_recovery": "true", 2023-10-09T05:32:36.613 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T05:32:36.613 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_thrash_exports": "0", 2023-10-09T05:32:36.613 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_thrash_fragments": "0", 2023-10-09T05:32:36.613 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_tick_interval": "5.000000", 2023-10-09T05:32:36.613 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_valgrind_exit": "false", 2023-10-09T05:32:36.613 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_verify_backtrace": "1", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_verify_scatter": "false", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "mds_wipe_sessions": "false", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "mempool_debug": "false", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "memstore_page_set": "false", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "memstore_page_size": "65536", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_client_bytes": "134217728", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_client_messages": "512", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T05:32:36.614 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_disabled_modules": "", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_mds_messages": "128", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_mon_messages": "128", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T05:32:36.615 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_osd_messages": "8192", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_pool": "true", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_standby_modules": "true", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_stats_period": "5", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_stats_threshold": "5", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_tick_period": "2", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_allow_pool_delete": "true", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T05:32:36.616 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_client_bytes": "104857600", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T05:32:36.617 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cluster_log_to_journald": "true", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cluster_log_to_stderr": "false", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T05:32:36.618 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T05:32:36.619 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T05:32:36.619 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T05:32:36.619 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_compact_on_start": "false", 2023-10-09T05:32:36.619 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_compact_on_trim": "true", 2023-10-09T05:32:36.619 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T05:32:36.619 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T05:32:36.619 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T05:32:36.619 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_cpu_threads": "4", 2023-10-09T05:32:36.619 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T05:32:36.619 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T05:32:36.619 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_data_avail_crit": "5", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_data_avail_warn": "5", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_debug_dump_json": "false", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_debug_extra_checks": "false", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T05:32:36.620 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_election_default_strategy": "3", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_election_timeout": "5.000000", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_enable_op_tracker": "true", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_fake_pool_delete": "false", 2023-10-09T05:32:36.621 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_force_quorum_join": "false", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_health_log_update_period": "5", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_health_max_detail": "50", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_health_to_clog": "true", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_host": "[v2:172.21.15.8:3300/0,v1:172.21.15.8:6789/0] [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] [v2:172.21.15.8:3301/0,v1:172.21.15.8:6790/0]", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_host_override": "", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_initial_members": "", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T05:32:36.622 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_lease": "5.000000", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_log_full_interval": "50", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_log_max": "10000", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_log_max_summary": "50", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_max_log_epochs": "500", 2023-10-09T05:32:36.623 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_max_osd": "10000", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_memory_autotune": "true", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_memory_target": "2147483648", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T05:32:36.624 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_mgr_digest_period": "5", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_op_complaint_time": "30", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_op_history_duration": "600", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_op_history_size": "20", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_op_log_threshold": "5", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T05:32:36.625 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_cache_size": "500", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T05:32:36.626 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T05:32:36.627 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T05:32:36.628 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T05:32:36.628 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T05:32:36.628 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T05:32:36.628 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_report_timeout": "900", 2023-10-09T05:32:36.628 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T05:32:36.628 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T05:32:36.628 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T05:32:36.628 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T05:32:36.628 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T05:32:36.629 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_reweight_max_osds": "4", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_scrub_interval": "86400", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_scrub_max_keys": "100", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_scrub_timeout": "300", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_session_timeout": "300", 2023-10-09T05:32:36.630 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_smart_report_timeout": "5", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_stretch_pool_size": "4", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_sync_debug": "false", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T05:32:36.631 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_tick_interval": "5", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T05:32:36.632 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_older_version": "true", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T05:32:36.633 INFO:teuthology.orchestra.run.smithi008.stdout: "monmap": "", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "motr_tracing_enabled": "false", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_op_threads": "3", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_cm": "false", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_device_name": "", 2023-10-09T05:32:36.634 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_sl": "3", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T05:32:36.635 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_rdma_type": "ib", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_async_reap_threshold": "5", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_bind_before_connect": "false", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_bind_ipv4": "true", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_bind_ipv6": "false", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_bind_msgr1": "true", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_bind_msgr2": "true", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_bind_port_max": "7568", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_bind_port_min": "6800", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_bind_retry_count": "3", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_bind_retry_delay": "5", 2023-10-09T05:32:36.636 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_blackhole_client": "false", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_blackhole_mds": "false", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_blackhole_mgr": "false", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_blackhole_mon": "false", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_blackhole_osd": "false", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_client_mode": "crc secure", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_cluster_type": "", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_compress_secure": "false", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_crc_data": "true", 2023-10-09T05:32:36.637 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_crc_header": "true", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_die_on_bug": "true", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_die_on_old_message": "true", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T05:32:36.638 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_hugepages": "", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_lro": "true", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_pmd": "", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_port_id": "0", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_dump_on_send": "false", 2023-10-09T05:32:36.639 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_inject_delay_type": "", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_inject_network_congestion": "0", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_inject_socket_failures": "0", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_max_accept_failures": "4", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_max_backoff": "15.000000", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T05:32:36.640 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_osd_compress_mode": "none", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_pq_min_cost": "65536", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_public_type": "", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_service_mode": "crc secure", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_tcp_nodelay": "true", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "ms_type": "async+posix", 2023-10-09T05:32:36.641 INFO:teuthology.orchestra.run.smithi008.stdout: "no_config_file": "false", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "objecter_inflight_ops": "1024", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "objecter_timeout": "10.000000", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "objectstore_blackhole": "false", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "openssl_engine_opts": "", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T05:32:36.642 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_agent_max_ops": "4", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_agent_slop": "0.020000", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_asio_thread_count": "2", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T05:32:36.643 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_backfill_scan_max": "512", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_backfill_scan_min": "64", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_backoff_on_peering": "false", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_beacon_report_interval": "300", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_bench_duration": "30", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_blkin_trace_all": "false", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T05:32:36.644 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_class_update_on_start": "true", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_client_message_cap": "256", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_client_op_priority": "63", 2023-10-09T05:32:36.645 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_client_watch_timeout": "30", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_command_max_records": "256", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_compact_on_start": "false", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_crush_update_on_start": "true", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T05:32:36.646 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_op_order": "false", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T05:32:36.647 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_shutdown": "false", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T05:32:36.648 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T05:32:36.649 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T05:32:36.649 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T05:32:36.649 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T05:32:36.649 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_default_notify_timeout": "30", 2023-10-09T05:32:36.649 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T05:32:36.649 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T05:32:36.649 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T05:32:36.649 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T05:32:36.649 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T05:32:36.649 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_enable_op_tracker": "true", 2023-10-09T05:32:36.649 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_fast_info": "true", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_fast_shutdown": "true", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_function_tracing": "false", 2023-10-09T05:32:36.650 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_heartbeat_grace": "20", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_heartbeat_interval": "6", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_heartbeat_stale": "600", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T05:32:36.651 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T05:32:36.652 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T05:32:36.652 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_journal_size": "5120", 2023-10-09T05:32:36.652 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_kill_backfill_at": "0", 2023-10-09T05:32:36.652 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T05:32:36.652 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_map_cache_size": "50", 2023-10-09T05:32:36.652 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_map_dedup": "true", 2023-10-09T05:32:36.652 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_map_message_max": "40", 2023-10-09T05:32:36.652 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T05:32:36.652 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T05:32:36.652 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_attr_name_len": "100", 2023-10-09T05:32:36.652 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_attr_size": "0", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_backfills": "1", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_markdown_count": "5", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_markdown_period": "600", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_object_name_len": "2048", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_object_size": "134217728", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_pgls": "1024", 2023-10-09T05:32:36.653 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_push_cost": "8388608", 2023-10-09T05:32:36.654 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_push_objects": "10", 2023-10-09T05:32:36.654 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_scrubs": "1", 2023-10-09T05:32:36.654 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T05:32:36.654 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T05:32:36.654 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T05:32:36.654 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_max_write_size": "90", 2023-10-09T05:32:36.654 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T05:32:36.654 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T05:32:36.654 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T05:32:36.654 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T05:32:36.654 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_profile": "balanced", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T05:32:36.655 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_memory_base": "805306368", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_memory_target": "4294967296", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_memory_target_autotune": "false", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_min_recovery_priority": "0", 2023-10-09T05:32:36.656 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mon_report_interval": "5", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_num_cache_shards": "32", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_numa_node": "-1", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_objecter_finishers": "1", 2023-10-09T05:32:36.657 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_objectstore": "bluestore", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_objectstore_fuse": "false", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_objectstore_tracing": "false", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_history_duration": "600", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_history_size": "20", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_log_threshold": "5", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_num_shards": "0", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T05:32:36.658 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_op_thread_timeout": "15", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_open_classes_on_start": "true", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_os_flags": "0", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_peering_op_priority": "255", 2023-10-09T05:32:36.659 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pg_delete_priority": "5", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T05:32:36.660 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_crimson": "false", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T05:32:36.661 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_flags": "0", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_min_size": "0", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_size": "3", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_default_type": "replicated", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T05:32:36.662 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_cost": "20971520", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_max_active": "0", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_op_priority": "3", 2023-10-09T05:32:36.663 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_priority": "5", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_repair_during_recovery": "false", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T05:32:36.664 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_cost": "52428800", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_end_hour": "0", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T05:32:36.665 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_priority": "5", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T05:32:36.666 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_skip_data_digest": "false", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_smart_report_timeout": "5", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_snap_trim_priority": "5", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_target_transaction_size": "30", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T05:32:36.667 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_tracing": "false", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_use_stale_snap": "false", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T05:32:36.668 INFO:teuthology.orchestra.run.smithi008.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "paxos_kill_at": "0", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "paxos_max_join_drift": "10", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "paxos_min": "500", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "paxos_min_wait": "0.050000", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "paxos_service_trim_max": "500", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "paxos_service_trim_min": "250", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "paxos_stash_full_interval": "25", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "paxos_trim_max": "500", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "paxos_trim_min": "250", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "perf": "true", 2023-10-09T05:32:36.669 INFO:teuthology.orchestra.run.smithi008.stdout: "pid_file": "", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "public_addr": "-", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "public_addrv": "", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "public_bind_addr": "-", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "public_network": "172.21.0.0/20", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "public_network_interface": "", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "qat_compressor_enabled": "false", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "rados_mon_op_timeout": "0", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "rados_osd_op_timeout": "0", 2023-10-09T05:32:36.670 INFO:teuthology.orchestra.run.smithi008.stdout: "rados_tracing": "false", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_atime_update_interval": "60", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_cache": "true", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T05:32:36.671 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_cache_policy": "writearound", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_cache_size": "33554432", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_compression_hint": "none", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_default_clone_format": "auto", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_default_data_pool": "", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_default_features": "61", 2023-10-09T05:32:36.672 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_default_format": "2", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_default_map_options": "", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_default_order": "22", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_default_pool": "rbd", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_default_stripe_count": "0", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T05:32:36.673 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_io_scheduler": "simple", 2023-10-09T05:32:36.674 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T05:32:36.674 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T05:32:36.674 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T05:32:36.674 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T05:32:36.674 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T05:32:36.674 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T05:32:36.674 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T05:32:36.674 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T05:32:36.674 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T05:32:36.674 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_journal_order": "24", 2023-10-09T05:32:36.675 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_journal_pool": "", 2023-10-09T05:32:36.675 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_journal_splay_width": "4", 2023-10-09T05:32:36.675 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T05:32:36.675 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T05:32:36.675 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T05:32:36.675 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T05:32:36.675 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T05:32:36.675 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T05:32:36.675 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T05:32:36.675 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T05:32:36.675 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T05:32:36.676 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T05:32:36.676 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T05:32:36.676 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T05:32:36.676 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T05:32:36.676 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T05:32:36.676 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T05:32:36.676 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T05:32:36.676 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T05:32:36.676 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T05:32:36.676 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T05:32:36.677 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T05:32:36.677 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T05:32:36.677 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T05:32:36.677 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T05:32:36.677 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T05:32:36.677 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T05:32:36.677 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T05:32:36.677 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T05:32:36.677 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T05:32:36.677 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T05:32:36.677 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_op_threads": "1", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T05:32:36.678 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_plugins": "", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T05:32:36.679 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T05:32:36.680 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T05:32:36.681 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T05:32:36.681 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T05:32:36.681 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T05:32:36.681 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T05:32:36.681 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_tracing": "false", 2023-10-09T05:32:36.681 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_validate_names": "true", 2023-10-09T05:32:36.681 INFO:teuthology.orchestra.run.smithi008.stdout: "rbd_validate_pool": "true", 2023-10-09T05:32:36.681 INFO:teuthology.orchestra.run.smithi008.stdout: "restapi_base_url": "", 2023-10-09T05:32:36.681 INFO:teuthology.orchestra.run.smithi008.stdout: "restapi_log_level": "", 2023-10-09T05:32:36.681 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T05:32:36.681 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_admin_entry": "admin", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_backend_store": "rados", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_barbican_url": "", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_beast_enable_async": "true", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T05:32:36.682 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_cache_enabled": "true", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_config_store": "rados", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_content_length_compat": "false", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_cross_domain_policy": "", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T05:32:36.683 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T05:32:36.684 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T05:32:36.685 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T05:32:36.686 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T05:32:36.686 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T05:32:36.686 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T05:32:36.686 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T05:32:36.686 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T05:32:36.686 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T05:32:36.686 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T05:32:36.686 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T05:32:36.686 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T05:32:36.686 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T05:32:36.686 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T05:32:36.687 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_data_log_window": "30", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T05:32:36.688 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T05:32:36.689 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T05:32:36.690 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dns_name": "", 2023-10-09T05:32:36.690 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dns_s3website_name": "", 2023-10-09T05:32:36.690 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T05:32:36.690 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T05:32:36.690 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T05:32:36.690 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T05:32:36.690 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_enable_ops_log": "false", 2023-10-09T05:32:36.690 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T05:32:36.690 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_enable_static_website": "false", 2023-10-09T05:32:36.690 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_enable_usage_log": "false", 2023-10-09T05:32:36.690 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_expose_bucket": "false", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_extended_http_attrs": "", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_filter": "none", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_gc_max_objs": "32", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T05:32:36.691 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_init_timeout": "300", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T05:32:36.692 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T05:32:36.693 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T05:32:36.693 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_admin_password": "", 2023-10-09T05:32:36.693 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T05:32:36.693 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_admin_project": "", 2023-10-09T05:32:36.693 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T05:32:36.693 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_admin_token": "", 2023-10-09T05:32:36.693 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T05:32:36.693 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_admin_user": "", 2023-10-09T05:32:36.693 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_api_version": "2", 2023-10-09T05:32:36.693 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T05:32:36.693 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_url": "", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T05:32:36.694 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_lc_max_objs": "32", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_lc_max_worker": "3", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T05:32:36.695 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T05:32:36.696 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_log_http_headers": "", 2023-10-09T05:32:36.696 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T05:32:36.696 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T05:32:36.696 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T05:32:36.696 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T05:32:36.696 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T05:32:36.696 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_attr_size": "0", 2023-10-09T05:32:36.696 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T05:32:36.696 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T05:32:36.696 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T05:32:36.696 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_listing_results": "1000", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_notify_retries": "10", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T05:32:36.697 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T05:32:36.698 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_num_control_oids": "8", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_numa_node": "-1", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T05:32:36.699 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_op_tracing": "false", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_opa_token": "", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_opa_url": "", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_ops_log_rados": "false", 2023-10-09T05:32:36.700 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_print_continue": "true", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T05:32:36.701 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_rados_tracing": "false", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_realm": "", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_realm_id": "", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_region": "", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_request_uri": "", 2023-10-09T05:32:36.702 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_resolve_cname": "false", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_run_sync_thread": "true", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T05:32:36.703 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_script_uri": "", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_service_provider_name": "", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sts_client_id": "", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sts_client_secret": "", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sts_entry": "sts", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sts_key": "sts", 2023-10-09T05:32:36.704 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_swift_auth_url": "", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_swift_custom_header": "", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_swift_need_stats": "true", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_swift_tenant_name": "", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_swift_url": "", 2023-10-09T05:32:36.705 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_lease_period": "120", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T05:32:36.706 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_thread_pool_size": "512", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_torrent_comment": "", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_torrent_createby": "", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_torrent_encoding": "", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_torrent_flag": "false", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_torrent_origin": "", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_torrent_tracker": "", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T05:32:36.707 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_usage_max_shards": "32", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_use_opa_authz": "false", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T05:32:36.708 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_user_unique_email": "true", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_verify_ssl": "true", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_zone": "", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_zone_id": "", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_zonegroup": "", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_zonegroup_id": "", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_block_size": "4096", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T05:32:36.709 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T05:32:36.710 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T05:32:36.711 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T05:32:36.711 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T05:32:36.711 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T05:32:36.711 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_partition_filters": "false", 2023-10-09T05:32:36.711 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_perf": "false", 2023-10-09T05:32:36.711 INFO:teuthology.orchestra.run.smithi008.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T05:32:36.711 INFO:teuthology.orchestra.run.smithi008.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T05:32:36.711 INFO:teuthology.orchestra.run.smithi008.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T05:32:36.711 INFO:teuthology.orchestra.run.smithi008.stdout: "run_dir": "/var/run/ceph", 2023-10-09T05:32:36.711 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_block_create": "true", 2023-10-09T05:32:36.712 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T05:32:36.712 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T05:32:36.712 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T05:32:36.712 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T05:32:36.712 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_device_size": "53687091200", 2023-10-09T05:32:36.712 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T05:32:36.712 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T05:32:36.712 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T05:32:36.713 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T05:32:36.713 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_main_device_type": "SSD", 2023-10-09T05:32:36.713 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T05:32:36.713 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T05:32:36.713 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T05:32:36.713 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T05:32:36.713 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T05:32:36.713 INFO:teuthology.orchestra.run.smithi008.stdout: "seastore_segment_size": "67108864", 2023-10-09T05:32:36.713 INFO:teuthology.orchestra.run.smithi008.stdout: "set_keepcaps": "false", 2023-10-09T05:32:36.714 INFO:teuthology.orchestra.run.smithi008.stdout: "setgroup": "ceph", 2023-10-09T05:32:36.714 INFO:teuthology.orchestra.run.smithi008.stdout: "setuser": "ceph", 2023-10-09T05:32:36.714 INFO:teuthology.orchestra.run.smithi008.stdout: "setuser_match_path": "", 2023-10-09T05:32:36.714 INFO:teuthology.orchestra.run.smithi008.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T05:32:36.714 INFO:teuthology.orchestra.run.smithi008.stdout: "thp": "false", 2023-10-09T05:32:36.714 INFO:teuthology.orchestra.run.smithi008.stdout: "threadpool_default_timeout": "60", 2023-10-09T05:32:36.714 INFO:teuthology.orchestra.run.smithi008.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T05:32:36.714 INFO:teuthology.orchestra.run.smithi008.stdout: "throttler_perf_counter": "true" 2023-10-09T05:32:36.714 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:32:37.164 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T05:32:37.524 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:37.788 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:37 smithi008 ceph-mon[184418]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:37.788 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:37 smithi008 ceph-mon[191227]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:37.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:37 smithi191 ceph-mon[161669]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:39.015 INFO:teuthology.orchestra.run.smithi008.stdout:mon.a: {} 2023-10-09T05:32:39.025 INFO:teuthology.orchestra.run.smithi008.stdout:mon.b: {} 2023-10-09T05:32:39.034 INFO:teuthology.orchestra.run.smithi008.stdout:mon.c: {} 2023-10-09T05:32:39.036 INFO:teuthology.orchestra.run.smithi008.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T05:32:39.036 INFO:teuthology.orchestra.run.smithi008.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T05:32:39.036 INFO:teuthology.orchestra.run.smithi008.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T05:32:39.570 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T05:32:39.571 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T05:32:39.571 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:32:39.573 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:32:39.573 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T05:32:39.573 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T05:32:39.573 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T05:32:39.573 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:32:39.574 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:32:39.574 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T05:32:39.574 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:32:39.574 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:32:39.574 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T05:32:39.574 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T05:32:39.575 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:39.575 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T05:32:39.575 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:32:39.575 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T05:32:39.576 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:32:39.576 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T05:32:39.576 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:32:39.640 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:39.692 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:32:39.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:39.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:39.797 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:39.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:39 smithi191 ceph-mon[161669]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:39.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2003149200' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:32:39.826 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:39.873 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:32:39.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:39.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:39.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:39 smithi008 ceph-mon[184418]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:39.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2003149200' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:32:39.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:39 smithi008 ceph-mon[191227]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:39.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2003149200' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:32:39.968 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:32:39.979 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:40.011 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:40.059 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:32:40.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:40.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:40.094 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:40.162 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:40.189 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:40.283 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:32:40.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:40.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:40.346 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:40.347 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:40.387 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:40.421 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:40.506 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:32:40.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:40.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:40.556 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:40.610 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:40.636 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:32:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:32:40] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.33.4" 2023-10-09T05:32:40.639 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:40.685 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:32:40.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:40.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:40.789 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:40.818 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:40.997 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:32:40.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:41.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:41.101 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:41.127 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:41.168 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:32:41.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:41.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:41.272 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:41.300 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:41.393 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:32:41.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:41.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:41.469 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:32:41.469 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:32:41.469 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:32:41.497 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:41.525 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:41.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:41 smithi008 ceph-mon[184418]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:41.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:41 smithi008 ceph-mon[191227]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:41 smithi191 ceph-mon[161669]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:42.312 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:32:42.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:42.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:42.417 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:42.446 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:42.489 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:32:42.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:42.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:42.595 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:42.621 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:42.662 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:32:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:42.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:42.767 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:42.795 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:42.836 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:32:42.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:42.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:42.945 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:42.977 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:43.436 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:32:43.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:43.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:43.539 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:43.576 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:43.783 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:43 smithi008 ceph-mon[184418]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:43.783 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:43 smithi008 ceph-mon[191227]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:43 smithi191 ceph-mon[161669]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:43.835 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:32:43.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:43.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:43.913 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:32:43.931 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:32:43.940 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:43.952 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:32:43.973 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:43.975 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:32:44.006 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:32:44.020 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:32:44.021 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:32:44.026 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:32:44.034 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:32:44.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:44.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:44.047 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:32:44.066 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:32:44.068 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:32:44.068 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:32:44.068 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:32:44.068 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:32:44.068 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:32:44.068 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:32:44.068 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:32:44.068 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:32:44.141 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:44.173 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:44.320 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:32:44.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:44.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:44.424 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:44.451 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:44.917 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:32:44.918 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_0' created 2023-10-09T05:32:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/54104984' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:32:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4151550378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/54104984' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:32:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4151550378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:45.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/54104984' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:32:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4151550378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:45.354 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:32:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:45.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:45.458 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:45.486 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:45.947 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:32:46.013 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[184418]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:46.013 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/903642407' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:32:46.013 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:32:46.014 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[184418]: failed to encode map e105 with expected crc 2023-10-09T05:32:46.014 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/54104984' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:32:46.014 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/903642407' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T05:32:46.014 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[184418]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T05:32:46.014 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[191227]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:46.014 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/903642407' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:32:46.014 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:32:46.014 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[191227]: failed to encode map e105 with expected crc 2023-10-09T05:32:46.015 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/54104984' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:32:46.015 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/903642407' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T05:32:46.015 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:45 smithi008 ceph-mon[191227]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T05:32:46.247 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T05:32:46.286 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:32:46.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:46.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:45 smithi191 ceph-mon[161669]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/903642407' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:32:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:45 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:32:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:45 smithi191 ceph-mon[161669]: failed to encode map e105 with expected crc 2023-10-09T05:32:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/54104984' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:32:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/903642407' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T05:32:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:45 smithi191 ceph-mon[161669]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T05:32:46.392 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:46.423 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:46.461 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:46.473 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:32:46.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:46.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:46.581 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:46.609 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:46.711 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:32:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:46.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:46.814 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:46.843 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:46.889 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:32:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:46.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:46.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:32:47.025 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:47.073 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:32:47.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:47.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:32:47.102 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:32:47.102 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:32:47.180 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:47.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[184418]: pgmap v235: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:47.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[184418]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T05:32:47.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[184418]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[184418]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[184418]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[184418]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[191227]: pgmap v235: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:47.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[191227]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T05:32:47.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[191227]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[191227]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[191227]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:46 smithi008 ceph-mon[191227]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.208 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:47.252 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:32:47.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:47.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:47.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:46 smithi191 ceph-mon[161669]: pgmap v235: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:46 smithi191 ceph-mon[161669]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T05:32:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:46 smithi191 ceph-mon[161669]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:46 smithi191 ceph-mon[161669]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:46 smithi191 ceph-mon[161669]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:46 smithi191 ceph-mon[161669]: failed to encode map e106 with expected crc 2023-10-09T05:32:47.356 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:47.383 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:47.477 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:32:47.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:47.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:47.544 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:47.581 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:47.610 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:47.656 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:32:47.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:47.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:47.763 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:47.793 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:47.898 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:32:47.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:47.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:47.946 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:32:48.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:48.030 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:48.074 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:32:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:48.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:48.179 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:48.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:47 smithi008 ceph-mon[184418]: failed to encode map e107 with expected crc 2023-10-09T05:32:48.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:47 smithi008 ceph-mon[184418]: failed to encode map e107 with expected crc 2023-10-09T05:32:48.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:47 smithi008 ceph-mon[184418]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T05:32:48.209 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:47 smithi008 ceph-mon[191227]: failed to encode map e107 with expected crc 2023-10-09T05:32:48.209 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:47 smithi008 ceph-mon[191227]: failed to encode map e107 with expected crc 2023-10-09T05:32:48.209 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:47 smithi008 ceph-mon[191227]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T05:32:48.211 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:48.214 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:32:48.214 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:32:48.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:47 smithi191 ceph-mon[161669]: failed to encode map e107 with expected crc 2023-10-09T05:32:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:47 smithi191 ceph-mon[161669]: failed to encode map e107 with expected crc 2023-10-09T05:32:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:47 smithi191 ceph-mon[161669]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T05:32:48.300 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:32:48.361 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:32:48.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:48.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:48.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:48.496 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:48.544 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:32:48.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:48.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:48.651 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:48.678 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:49.403 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:32:49.432 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:32:49.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:49.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:49.508 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:49.536 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:49.572 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:49.579 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:49 smithi008 ceph-mon[184418]: pgmap v238: 89 pgs: 6 creating+peering, 8 unknown, 75 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:49.579 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:32:49.579 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:49 smithi008 ceph-mon[191227]: pgmap v238: 89 pgs: 6 creating+peering, 8 unknown, 75 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:49.579 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:32:49.701 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:49.733 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:32:49.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:49.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:49.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:49 smithi191 ceph-mon[161669]: pgmap v238: 89 pgs: 6 creating+peering, 8 unknown, 75 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:49.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:32:49.838 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:49.870 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:49.981 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:32:49.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:49.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:50.080 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:50.085 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:50.128 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:50.521 INFO:teuthology.orchestra.run.smithi008.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T05:32:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3275281112' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:32:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:50 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:32:50.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3275281112' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:32:50.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:50 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:32:50.804 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:32:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:32:50] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.33.4" 2023-10-09T05:32:50.805 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3275281112' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:32:50.805 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:50 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:32:51.330 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:32:51.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:51.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:51.434 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:51.462 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:51.504 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:32:51.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:51.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:51.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:51.634 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:51.680 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:32:51.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:51.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:51.704 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:51 smithi008 ceph-mon[184418]: pgmap v239: 89 pgs: 6 creating+peering, 83 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:51.704 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:51 smithi008 ceph-mon[184418]: failed to encode map e108 with expected crc 2023-10-09T05:32:51.704 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:51 smithi008 ceph-mon[184418]: failed to encode map e108 with expected crc 2023-10-09T05:32:51.705 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:51 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:32:51.705 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:51 smithi008 ceph-mon[184418]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T05:32:51.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:51 smithi008 ceph-mon[191227]: pgmap v239: 89 pgs: 6 creating+peering, 83 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:51.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:51 smithi008 ceph-mon[191227]: failed to encode map e108 with expected crc 2023-10-09T05:32:51.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:51 smithi008 ceph-mon[191227]: failed to encode map e108 with expected crc 2023-10-09T05:32:51.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:51 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:32:51.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:51 smithi008 ceph-mon[191227]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T05:32:51.788 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:51 smithi191 ceph-mon[161669]: pgmap v239: 89 pgs: 6 creating+peering, 83 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:51 smithi191 ceph-mon[161669]: failed to encode map e108 with expected crc 2023-10-09T05:32:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:51 smithi191 ceph-mon[161669]: failed to encode map e108 with expected crc 2023-10-09T05:32:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:51 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:32:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:51 smithi191 ceph-mon[161669]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T05:32:51.813 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:51.856 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:32:51.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:51.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:51.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:51.986 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:52.078 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:32:52.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:52.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:52.184 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:52.219 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:52.293 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:32:52.294 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:32:52.294 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:32:52.294 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:32:52.294 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:32:52.294 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:32:52.294 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:32:52.295 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:32:52.295 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:32:52.295 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:32:52.295 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:32:52.295 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:32:52.295 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:32:52.295 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:32:52.295 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:32:52.295 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:32:52.420 DEBUG:teuthology.orchestra.run.smithi008:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-09T05:32:52.421 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-10-09T05:32:52.421 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:32:52.424 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:32:52.424 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:32:52.474 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:hints = 1 2023-10-09T05:32:52.474 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:32:52.474 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object prefix: benchmark_data_smithi008_238708 2023-10-09T05:32:52.474 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:32:52.475 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:32:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1815143688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:32:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:32:52.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1815143688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:52.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:32:52.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:32:52.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1815143688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:32:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:32:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:32:53.477 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 1 15 2722 2707 169.081 169.188 0.00461343 0.00590174 2023-10-09T05:32:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:53 smithi191 ceph-mon[161669]: pgmap v241: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:53.908 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:53 smithi008 ceph-mon[184418]: pgmap v241: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:53.908 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:53 smithi008 ceph-mon[191227]: pgmap v241: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:32:54.096 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:32:54.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:54.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:54.206 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:32:54.263 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:54.476 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 2 16 5898 5882 183.725 198.438 0.00441521 0.00542928 2023-10-09T05:32:54.556 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:32:54.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:54.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:32:54.610 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:32:54.610 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:32:54.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:32:54.610 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:32:54.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.610 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:56 5.5 5:acd76884:::benchmark_data_smithi008_238708_object55:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:52.490869+0000", 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 2.1097605700000002, 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037633759000000003, 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490869+0000", 2023-10-09T05:32:54.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490869+0000", 2023-10-09T05:32:54.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490867+0000", 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490894+0000", 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7018000000000001e-05 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490907+0000", 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2852e-05 2023-10-09T05:32:54.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490921+0000", 2023-10-09T05:32:54.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3648999999999999e-05 2023-10-09T05:32:54.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490972+0000", 2023-10-09T05:32:54.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1598999999999998e-05 2023-10-09T05:32:54.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.525591+0000", 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.034619130999999997 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.525764+0000", 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000172265 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.526856+0000", 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010927580000000001 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.526968+0000", 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011192999999999999 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.528407+0000", 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014386990000000001 2023-10-09T05:32:54.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.528461+0000", 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3625999999999998e-05 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.528502+0000", 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1862e-05 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.617 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:57 5.5 5:a3a08917:::benchmark_data_smithi008_238708_object56:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:52.490939+0000", 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 2.1096900870000002, 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.038137259, 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.618 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 57 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490939+0000", 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490939+0000", 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490939+0000", 2023-10-09T05:32:54.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:32:54.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490958+0000", 2023-10-09T05:32:54.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9879999999999999e-05 2023-10-09T05:32:54.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490967+0000", 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1939999999999998e-06 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.490972+0000", 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7459999999999997e-06 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.526114+0000", 2023-10-09T05:32:54.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.035141613000000002 2023-10-09T05:32:54.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.526236+0000", 2023-10-09T05:32:54.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000121779 2023-10-09T05:32:54.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.526367+0000", 2023-10-09T05:32:54.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013165 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.527627+0000", 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001259678 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.528514+0000", 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00088705399999999997 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.528995+0000", 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000480464 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.529041+0000", 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6801000000000001e-05 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.529076+0000", 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4963999999999999e-05 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:1307 5.c e108/105 5:334d1c55:::benchmark_data_smithi008_238708_object1306:head v 108'81, mlcod=108'79)", 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:52.923778+0000", 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 1.6768509470000001, 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011610026000000001, 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.625 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.923778+0000", 2023-10-09T05:32:54.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.923778+0000", 2023-10-09T05:32:54.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.923777+0000", 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.923825+0000", 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7289000000000001e-05 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.923841+0000", 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6382999999999999e-05 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.923851+0000", 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0137e-05 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.925060+0000", 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00120879 2023-10-09T05:32:54.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.925088+0000", 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8144999999999999e-05 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.935343+0000", 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010255261 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.935388+0000", 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4796999999999998e-05 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:1444 5.5 5:aab9ef6e:::benchmark_data_smithi008_238708_object1443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:52.962392+0000", 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 1.6382376170000001, 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0099935100000000006, 2023-10-09T05:32:54.630 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 1444 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.962392+0000", 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.962392+0000", 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.962390+0000", 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.962421+0000", 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0602999999999998e-05 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.962429+0000", 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0050000000000004e-06 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.962457+0000", 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7744000000000001e-05 2023-10-09T05:32:54.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.962509+0000", 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2437e-05 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.962638+0000", 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012913900000000001 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.962772+0000", 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013397100000000001 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.963446+0000", 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00067442000000000001 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.964108+0000", 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00066163099999999998 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.972321+0000", 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0082130339999999993 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.972358+0000", 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7063000000000001e-05 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.972385+0000", 2023-10-09T05:32:54.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6879000000000001e-05 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:1447 5.5 5:a1d68570:::benchmark_data_smithi008_238708_object1446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:52.963455+0000", 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 1.637174101, 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0097483499999999994, 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.637 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 1447 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.963455+0000", 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.963455+0000", 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.963454+0000", 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.963473+0000", 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8306999999999999e-05 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.963481+0000", 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8429999999999999e-06 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.963487+0000", 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2209999999999996e-06 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.963529+0000", 2023-10-09T05:32:54.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2645999999999998e-05 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.963662+0000", 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013220699999999999 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.963808+0000", 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014683900000000001 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.964282+0000", 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00047321700000000001 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.964769+0000", 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00048757500000000002 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.973071+0000", 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0083020430000000003 2023-10-09T05:32:54.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.973140+0000", 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8408e-05 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:52.973203+0000", 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3835000000000001e-05 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.643 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:3976 5.5 5:a081ffc7:::benchmark_data_smithi008_238708_object3975:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:53.816476+0000", 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.78415328299999998, 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011297033, 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.644 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 3976 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.816476+0000", 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.816476+0000", 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.816475+0000", 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.816496+0000", 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0866999999999999e-05 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.816520+0000", 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4366999999999999e-05 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.816528+0000", 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5100000000000001e-06 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.816739+0000", 2023-10-09T05:32:54.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00021096600000000001 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.816831+0000", 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2633999999999994e-05 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.816910+0000", 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8825999999999994e-05 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.818806+0000", 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001895545 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.818989+0000", 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000183146 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.827682+0000", 2023-10-09T05:32:54.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0086934899999999999 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.827736+0000", 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3737999999999999e-05 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:53.827773+0000", 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6826000000000003e-05 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.650 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:5059 5.c e108/105 5:3b9d6d71:::benchmark_data_smithi008_238708_object5058:head v 108'303, mlcod=108'301)", 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.135755+0000", 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.46487467999999998, 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037899322999999999, 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.135755+0000", 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.135755+0000", 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.135754+0000", 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:32:54.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.135773+0000", 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9046e-05 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.135781+0000", 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6420000000000007e-06 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.135785+0000", 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5279999999999997e-06 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.135825+0000", 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9604000000000002e-05 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.135844+0000", 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8856999999999999e-05 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.173615+0000", 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.03777167 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.173654+0000", 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8661000000000001e-05 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:5061 5.2 e108/105 5:40c7a6ff:::benchmark_data_smithi008_238708_object5060:head v 108'319, mlcod=108'317)", 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.136048+0000", 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.46458145299999998, 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037437125000000002, 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.136048+0000", 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.136048+0000", 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.136047+0000", 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.136114+0000", 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7158999999999999e-05 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.170922+0000", 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.034808154000000001 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.170932+0000", 2023-10-09T05:32:54.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2779999999999999e-06 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.171887+0000", 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000955261 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.171910+0000", 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2589999999999999e-05 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.173407+0000", 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001497207 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.173485+0000", 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8149e-05 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:5203 5.5 5:aecca6ce:::benchmark_data_smithi008_238708_object5202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.213063+0000", 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.38756605399999999, 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010513217, 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 5203 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.661 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213063+0000", 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213063+0000", 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213063+0000", 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213081+0000", 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8689e-05 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213091+0000", 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3009999999999995e-06 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213096+0000", 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9070000000000004e-06 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213842+0000", 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00074548599999999996 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213940+0000", 2023-10-09T05:32:54.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8479000000000004e-05 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.214089+0000", 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000148536 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.215747+0000", 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016580080000000001 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.217836+0000", 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020891360000000001 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.223473+0000", 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0056373480000000004 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.223537+0000", 2023-10-09T05:32:54.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.368e-05 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.223576+0000", 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9258999999999998e-05 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:5204 5.5 5:aa957422:::benchmark_data_smithi008_238708_object5203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.213102+0000", 2023-10-09T05:32:54.667 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.38752694599999998, 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010568298, 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 5204 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213102+0000", 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213102+0000", 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213102+0000", 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213125+0000", 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3266999999999999e-05 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213132+0000", 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5320000000000001e-06 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.213138+0000", 2023-10-09T05:32:54.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1390000000000003e-06 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.214516+0000", 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013785259999999999 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.214647+0000", 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013091099999999999 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.214790+0000", 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000142738 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.215752+0000", 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000962505 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.217892+0000", 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021399399999999999 2023-10-09T05:32:54.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.223623+0000", 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0057310859999999998 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.223652+0000", 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8938999999999999e-05 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.223671+0000", 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8267e-05 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:5606 5.5 5:a6dd0d95:::benchmark_data_smithi008_238708_object5605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.361555+0000", 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.23907455, 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0099383570000000001, 2023-10-09T05:32:54.674 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 5606 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.361555+0000", 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.361555+0000", 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.361553+0000", 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.361577+0000", 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4071e-05 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.361596+0000", 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8694000000000001e-05 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.361605+0000", 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8589999999999994e-06 2023-10-09T05:32:54.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.361654+0000", 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9648000000000003e-05 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.361773+0000", 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011854999999999999 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.361909+0000", 2023-10-09T05:32:54.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013600600000000001 2023-10-09T05:32:54.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.363310+0000", 2023-10-09T05:32:54.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001401432 2023-10-09T05:32:54.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.370540+0000", 2023-10-09T05:32:54.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0072294170000000001 2023-10-09T05:32:54.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.371402+0000", 2023-10-09T05:32:54.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00086200299999999997 2023-10-09T05:32:54.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.371457+0000", 2023-10-09T05:32:54.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4948000000000003e-05 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.371493+0000", 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6372999999999999e-05 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:5658 5.5 5:ae8b438f:::benchmark_data_smithi008_238708_object5657:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.681 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.388088+0000", 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.21254079000000001, 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012375933, 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 5658 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388088+0000", 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388088+0000", 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388087+0000", 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:32:54.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388115+0000", 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8262000000000001e-05 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388134+0000", 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8916999999999999e-05 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388142+0000", 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.602e-06 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388247+0000", 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000104079 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388369+0000", 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012287100000000001 2023-10-09T05:32:54.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388501+0000", 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000131173 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.394647+0000", 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061467450000000003 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.398951+0000", 2023-10-09T05:32:54.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0043033180000000004 2023-10-09T05:32:54.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.400351+0000", 2023-10-09T05:32:54.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00139989 2023-10-09T05:32:54.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.400415+0000", 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4566999999999994e-05 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.400464+0000", 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9246999999999998e-05 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:5659 5.5 5:a1e63420:::benchmark_data_smithi008_238708_object5658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.688 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.388159+0000", 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.21247043800000001, 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012395826, 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 5659 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388159+0000", 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388159+0000", 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388158+0000", 2023-10-09T05:32:54.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388186+0000", 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7557000000000001e-05 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388199+0000", 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2853e-05 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388205+0000", 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2779999999999999e-06 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.388917+0000", 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00071190399999999997 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.389063+0000", 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014611299999999999 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.389213+0000", 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000149999 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.394686+0000", 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0054727650000000001 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.398980+0000", 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042945969999999998 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.400477+0000", 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014968990000000001 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.400522+0000", 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4669000000000002e-05 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.400555+0000", 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2753000000000003e-05 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:5663 5.5 5:ae5f6894:::benchmark_data_smithi008_238708_object5662:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.695 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.389601+0000", 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.21102789399999999, 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.01103262, 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 5663 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.389601+0000", 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.389601+0000", 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.389600+0000", 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.389625+0000", 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5261000000000001e-05 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.389635+0000", 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0438999999999999e-05 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.389643+0000", 2023-10-09T05:32:54.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5120000000000002e-06 2023-10-09T05:32:54.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.389699+0000", 2023-10-09T05:32:54.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5909999999999998e-05 2023-10-09T05:32:54.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.389815+0000", 2023-10-09T05:32:54.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011627499999999999 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.389952+0000", 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000137002 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.394730+0000", 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0047782800000000002 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.399008+0000", 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042775210000000003 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.400562+0000", 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001553926 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.400601+0000", 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8952999999999998e-05 2023-10-09T05:32:54.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.400634+0000", 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3249e-05 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:5670 5.5 5:a9cb1a83:::benchmark_data_smithi008_238708_object5669:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.394748+0000", 2023-10-09T05:32:54.704 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.205881327, 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011453990000000001, 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 5670 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.394748+0000", 2023-10-09T05:32:54.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.394748+0000", 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.394746+0000", 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.394771+0000", 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5333999999999998e-05 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.394783+0000", 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1392999999999999e-05 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.394788+0000", 2023-10-09T05:32:54.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9790000000000001e-06 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.399031+0000", 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042427150000000002 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.399177+0000", 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014689299999999999 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.399329+0000", 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015113 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.400669+0000", 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013399060000000001 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.402613+0000", 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019444969999999999 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.406091+0000", 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034777480000000001 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.406142+0000", 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0971999999999999e-05 2023-10-09T05:32:54.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.406202+0000", 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0130000000000002e-05 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:5680 5.5 5:a73b42e2:::benchmark_data_smithi008_238708_object5679:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.397590+0000", 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.20303916799999999, 2023-10-09T05:32:54.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011934333, 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 5680 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.397590+0000", 2023-10-09T05:32:54.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.397590+0000", 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.397589+0000", 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.397615+0000", 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6004999999999999e-05 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.397624+0000", 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2120000000000003e-06 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.397631+0000", 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1840000000000002e-06 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.399725+0000", 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002093807 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.399850+0000", 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000125 2023-10-09T05:32:54.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.400006+0000", 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015623899999999999 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.401076+0000", 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010701420000000001 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.402618+0000", 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001542353 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.409436+0000", 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068178589999999999 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.409486+0000", 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0019999999999999e-05 2023-10-09T05:32:54.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.409524+0000", 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8049000000000001e-05 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:6100 5.e e108/105 5:708e3bd0:::benchmark_data_smithi008_238708_object6099:head v 108'365, mlcod=108'362)", 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.544783+0000", 2023-10-09T05:32:54.718 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.055845978999999997, 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013760563999999999, 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.544783+0000", 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.544783+0000", 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.544783+0000", 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.544820+0000", 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7277000000000002e-05 2023-10-09T05:32:54.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.544835+0000", 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5627000000000001e-05 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.544843+0000", 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2810000000000003e-06 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.544942+0000", 2023-10-09T05:32:54.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9500999999999995e-05 2023-10-09T05:32:54.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.544962+0000", 2023-10-09T05:32:54.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0183e-05 2023-10-09T05:32:54.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.558506+0000", 2023-10-09T05:32:54.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013543881000000001 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.558544+0000", 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.752e-05 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:6117 5.8 e108/105 5:1eefa6c3:::benchmark_data_smithi008_238708_object6116:head v 108'406, mlcod=108'404)", 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.552156+0000", 2023-10-09T05:32:54.723 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.048473075999999997, 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011567518000000001, 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552156+0000", 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552156+0000", 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552155+0000", 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552294+0000", 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013925499999999999 2023-10-09T05:32:54.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552312+0000", 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7407000000000001e-05 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552320+0000", 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.371e-06 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552410+0000", 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0248999999999996e-05 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552446+0000", 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5607e-05 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.563684+0000", 2023-10-09T05:32:54.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011238454 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.563724+0000", 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9159000000000002e-05 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:6121 5.5 5:a9afd15b:::benchmark_data_smithi008_238708_object6120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.552264+0000", 2023-10-09T05:32:54.728 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.048365347000000003, 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013185798, 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 6121 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552264+0000", 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552264+0000", 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552263+0000", 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:32:54.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552287+0000", 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4552000000000001e-05 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552305+0000", 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7601999999999999e-05 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552313+0000", 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5629999999999992e-06 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552358+0000", 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4078e-05 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552501+0000", 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014319199999999999 2023-10-09T05:32:54.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.552634+0000", 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000133096 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.556091+0000", 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034576609999999999 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:32:54.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.563433+0000", 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0073413130000000004 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.565357+0000", 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019243210000000001 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.565410+0000", 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2916e-05 2023-10-09T05:32:54.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.565450+0000", 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9632999999999998e-05 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:6116 5.7 e108/105 5:ecc045e9:::benchmark_data_smithi008_238708_object6115:head v 108'374, mlcod=108'372)", 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:54.553380+0000", 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.047249418000000001, 2023-10-09T05:32:54.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0099207759999999992, 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.553380+0000", 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.553380+0000", 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:32:54.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.553380+0000", 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.553418+0000", 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8426000000000001e-05 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:32:54.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.553424+0000", 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3749999999999999e-06 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.553429+0000", 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6809999999999997e-06 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.555329+0000", 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018995119999999999 2023-10-09T05:32:54.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.555349+0000", 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0366000000000001e-05 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.563259+0000", 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0079099300000000008 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:32:54.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:54.563301+0000", 2023-10-09T05:32:54.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1726e-05 2023-10-09T05:32:54.740 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.740 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.740 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.740 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:32:54.740 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:32:54.740 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:32:54.740 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:54.741 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:54.800 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:55.476 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 3 16 9079 9063 188.746 198.812 0.00425975 0.00529067 2023-10-09T05:32:55.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:55 smithi191 ceph-mon[161669]: pgmap v242: 89 pgs: 89 active+clean; 15 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s wr, 20 op/s 2023-10-09T05:32:55.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:55 smithi008 ceph-mon[184418]: pgmap v242: 89 pgs: 89 active+clean; 15 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s wr, 20 op/s 2023-10-09T05:32:55.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:55 smithi008 ceph-mon[191227]: pgmap v242: 89 pgs: 89 active+clean; 15 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s wr, 20 op/s 2023-10-09T05:32:56.044 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:32:56.480 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 4 16 11920 11904 185.745 177.562 0.00630715 0.00537658 2023-10-09T05:32:56.552 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:32:56.578 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:32:56.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:56.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:56.683 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:32:56.723 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:56.784 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:32:56.784 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:56 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4035909025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:32:56.784 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:56 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4035909025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:32:56.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:56 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4035909025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:32:56.799 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:32:56.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:56.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:32:56.905 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:32:57.019 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:57.480 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 5 16 14587 14571 181.935 166.688 0.00343505 0.00549097 2023-10-09T05:32:57.492 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:32:57.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:57.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:32:57.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:32:57.653 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:57.741 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:32:57.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:57.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:32:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:57 smithi191 ceph-mon[161669]: pgmap v243: 89 pgs: 89 active+clean; 380 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s 2023-10-09T05:32:57.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:57 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:32:57.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:57 smithi191 ceph-mon[161669]: failed to encode map e109 with expected crc 2023-10-09T05:32:57.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:57 smithi191 ceph-mon[161669]: failed to encode map e109 with expected crc 2023-10-09T05:32:57.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:57 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4035909025' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:32:57.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:57 smithi191 ceph-mon[161669]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T05:32:57.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:57.897 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:57.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[184418]: pgmap v243: 89 pgs: 89 active+clean; 380 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s 2023-10-09T05:32:57.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:32:57.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[184418]: failed to encode map e109 with expected crc 2023-10-09T05:32:57.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[184418]: failed to encode map e109 with expected crc 2023-10-09T05:32:57.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4035909025' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:32:57.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[184418]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T05:32:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[191227]: pgmap v243: 89 pgs: 89 active+clean; 380 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s 2023-10-09T05:32:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:32:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[191227]: failed to encode map e109 with expected crc 2023-10-09T05:32:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[191227]: failed to encode map e109 with expected crc 2023-10-09T05:32:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4035909025' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:32:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:57 smithi008 ceph-mon[191227]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T05:32:58.006 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:32:58.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:58.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:58.115 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:32:58.171 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:58.480 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 6 16 17456 17440 181.494 179.312 0.00160663 0.00548627 2023-10-09T05:32:59.204 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:32:59.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:59.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:32:59.309 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:32:59.336 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:59.480 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 7 16 19464 19448 173.498 125.5 0.00598417 0.00575754 2023-10-09T05:32:59.545 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:32:59.686 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:32:59.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:59.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:32:59.795 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:32:59.836 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:59 smithi191 ceph-mon[161669]: pgmap v245: 89 pgs: 89 active+clean; 536 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T05:32:59.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:32:59.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:59.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:32:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:32:59.843 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:32:59.924 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:32:59.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:32:59.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:32:59.950 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:59 smithi008 ceph-mon[184418]: pgmap v245: 89 pgs: 89 active+clean; 536 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T05:32:59.950 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:32:59.950 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:59.950 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:32:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:32:59.951 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:59 smithi008 ceph-mon[191227]: pgmap v245: 89 pgs: 89 active+clean; 536 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T05:32:59.951 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:32:59.951 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:32:59.951 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:32:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:00.035 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:00.083 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:00.157 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:00.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:00.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:00.265 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:00.308 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:00.312 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:00.391 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:33:00.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:00.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:00.480 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 8 16 21770 21754 169.827 144.125 0.00520908 0.00588243 2023-10-09T05:33:00.498 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:00.558 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:33:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:33:00] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.33.4" 2023-10-09T05:33:00.574 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:00.908 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:00 smithi008 ceph-mon[184418]: pgmap v246: 89 pgs: 89 active+clean; 962 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-10-09T05:33:00.908 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:00 smithi008 ceph-mon[191227]: pgmap v246: 89 pgs: 89 active+clean; 962 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-10-09T05:33:00.959 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:33:00.965 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:33:01.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:00 smithi191 ceph-mon[161669]: pgmap v246: 89 pgs: 89 active+clean; 962 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-10-09T05:33:01.480 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 9 16 23688 23672 164.279 119.875 0.00212254 0.00607988 2023-10-09T05:33:01.593 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:33:01.851 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:01 smithi008 ceph-mon[191227]: from='client.54604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:33:01.851 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:01.851 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:01 smithi008 ceph-mon[184418]: from='client.54604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:33:01.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:01.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:01.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:01.997 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:01 smithi191 ceph-mon[161669]: from='client.54604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:33:01.999 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:02.085 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:02.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:02.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:02.193 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:02.218 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:02.270 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:02.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:02.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:02.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:02.432 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:02.480 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 10 16 25636 25620 160.028 121.75 0.00778993 0.00624324 2023-10-09T05:33:02.518 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:33:02.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:02.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:02.626 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:02.682 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":246,"stamp":"2023-10-09T05:32:59.864768+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1008670643,"num_objects":15346,"num_object_clones":0,"num_object_copies":46038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15346,"num_whiteouts":0,"num_read":6303,"num_read_kb":13724,"num_write":16424,"num_write_kb":995493,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":16488,"ondisk_log_size":16488,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4397132,"kb_used_data":3057356,"kb_used_omap":31,"kb_used_meta":1339680,"kb_avail":745564084,"statfs":{"total":767960285184,"available":763457622016,"internally_reserved":0,"allocated":3130732544,"data_stored":3126655427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":1371832777},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,6,2,0,2,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1003880448,"num_objects":15318,"num_object_clones":0,"num_object_copies":45954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15318,"num_write_kb":980352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.354612"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":92,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345352+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:32:51.345352+0000","last_peered":"2023-10-09T05:32:51.345352+0000","last_clean":"2023-10-09T05:32:51.345352+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:32:51.345352+0000","last_undegraded":"2023-10-09T05:32:51.345352+0000","last_fullsized":"2023-10-09T05:32:51.345352+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":77,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.178204+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:32:47.178204+0000","last_peered":"2023-10-09T05:32:47.178204+0000","last_clean":"2023-10-09T05:32:47.178204+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:32:47.178204+0000","last_undegraded":"2023-10-09T05:32:47.178204+0000","last_fullsized":"2023-10-09T05:32:47.178204+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:53.472050+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:32:53.472050+0000","last_peered":"2023-10-09T05:32:53.472050+0000","last_clean":"2023-10-09T05:32:53.472050+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:32:53.472050+0000","last_undegraded":"2023-10-09T05:32:53.472050+0000","last_fullsized":"2023-10-09T05:32:53.472050+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":58,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.262664+0000","last_change":"2023-10-09T05:31:32.860920+0000","last_active":"2023-10-09T05:32:51.262664+0000","last_peered":"2023-10-09T05:32:51.262664+0000","last_clean":"2023-10-09T05:32:51.262664+0000","last_became_active":"2023-10-09T05:31:32.860244+0000","last_became_peered":"2023-10-09T05:31:32.860244+0000","last_unstale":"2023-10-09T05:32:51.262664+0000","last_undegraded":"2023-10-09T05:32:51.262664+0000","last_fullsized":"2023-10-09T05:32:51.262664+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:37.134584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":93,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.178449+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:32:47.178449+0000","last_peered":"2023-10-09T05:32:47.178449+0000","last_clean":"2023-10-09T05:32:47.178449+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:32:47.178449+0000","last_undegraded":"2023-10-09T05:32:47.178449+0000","last_fullsized":"2023-10-09T05:32:47.178449+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346120+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:32:51.346120+0000","last_peered":"2023-10-09T05:32:51.346120+0000","last_clean":"2023-10-09T05:32:51.346120+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:32:51.346120+0000","last_undegraded":"2023-10-09T05:32:51.346120+0000","last_fullsized":"2023-10-09T05:32:51.346120+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":58,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.262748+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:32:51.262748+0000","last_peered":"2023-10-09T05:32:51.262748+0000","last_clean":"2023-10-09T05:32:51.262748+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:32:51.262748+0000","last_undegraded":"2023-10-09T05:32:51.262748+0000","last_fullsized":"2023-10-09T05:32:51.262748+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":77,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.178362+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:32:47.178362+0000","last_peered":"2023-10-09T05:32:47.178362+0000","last_clean":"2023-10-09T05:32:47.178362+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:32:47.178362+0000","last_undegraded":"2023-10-09T05:32:47.178362+0000","last_fullsized":"2023-10-09T05:32:47.178362+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":76,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.516316+0000","last_change":"2023-10-09T05:30:30.457540+0000","last_active":"2023-10-09T05:32:50.516316+0000","last_peered":"2023-10-09T05:32:50.516316+0000","last_clean":"2023-10-09T05:32:50.516316+0000","last_became_active":"2023-10-09T05:30:30.457180+0000","last_became_peered":"2023-10-09T05:30:30.457180+0000","last_unstale":"2023-10-09T05:32:50.516316+0000","last_undegraded":"2023-10-09T05:32:50.516316+0000","last_fullsized":"2023-10-09T05:32:50.516316+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":67,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.178004+0000","last_change":"2023-10-09T05:31:32.863335+0000","last_active":"2023-10-09T05:32:47.178004+0000","last_peered":"2023-10-09T05:32:47.178004+0000","last_clean":"2023-10-09T05:32:47.178004+0000","last_became_active":"2023-10-09T05:31:32.863036+0000","last_became_peered":"2023-10-09T05:31:32.863036+0000","last_unstale":"2023-10-09T05:32:47.178004+0000","last_undegraded":"2023-10-09T05:32:47.178004+0000","last_fullsized":"2023-10-09T05:32:47.178004+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:59.787997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":79,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.513914+0000","last_change":"2023-10-09T05:31:32.863294+0000","last_active":"2023-10-09T05:32:50.513914+0000","last_peered":"2023-10-09T05:32:50.513914+0000","last_clean":"2023-10-09T05:32:50.513914+0000","last_became_active":"2023-10-09T05:31:32.862914+0000","last_became_peered":"2023-10-09T05:31:32.862914+0000","last_unstale":"2023-10-09T05:32:50.513914+0000","last_undegraded":"2023-10-09T05:32:50.513914+0000","last_fullsized":"2023-10-09T05:32:50.513914+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346678+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:32:51.346678+0000","last_peered":"2023-10-09T05:32:51.346678+0000","last_clean":"2023-10-09T05:32:51.346678+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:32:51.346678+0000","last_undegraded":"2023-10-09T05:32:51.346678+0000","last_fullsized":"2023-10-09T05:32:51.346678+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":83,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345462+0000","last_change":"2023-10-09T05:21:16.778811+0000","last_active":"2023-10-09T05:32:51.345462+0000","last_peered":"2023-10-09T05:32:51.345462+0000","last_clean":"2023-10-09T05:32:51.345462+0000","last_became_active":"2023-10-09T05:21:16.778515+0000","last_became_peered":"2023-10-09T05:21:16.778515+0000","last_unstale":"2023-10-09T05:32:51.345462+0000","last_undegraded":"2023-10-09T05:32:51.345462+0000","last_fullsized":"2023-10-09T05:32:51.345462+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345378+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:32:51.345378+0000","last_peered":"2023-10-09T05:32:51.345378+0000","last_clean":"2023-10-09T05:32:51.345378+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:32:51.345378+0000","last_undegraded":"2023-10-09T05:32:51.345378+0000","last_fullsized":"2023-10-09T05:32:51.345378+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":78,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.513942+0000","last_change":"2023-10-09T05:31:02.550860+0000","last_active":"2023-10-09T05:32:50.513942+0000","last_peered":"2023-10-09T05:32:50.513942+0000","last_clean":"2023-10-09T05:32:50.513942+0000","last_became_active":"2023-10-09T05:31:02.550469+0000","last_became_peered":"2023-10-09T05:31:02.550469+0000","last_unstale":"2023-10-09T05:32:50.513942+0000","last_undegraded":"2023-10-09T05:32:50.513942+0000","last_fullsized":"2023-10-09T05:32:50.513942+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":77,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.178440+0000","last_change":"2023-10-09T05:31:32.871695+0000","last_active":"2023-10-09T05:32:47.178440+0000","last_peered":"2023-10-09T05:32:47.178440+0000","last_clean":"2023-10-09T05:32:47.178440+0000","last_became_active":"2023-10-09T05:31:32.871449+0000","last_became_peered":"2023-10-09T05:31:32.871449+0000","last_unstale":"2023-10-09T05:32:47.178440+0000","last_undegraded":"2023-10-09T05:32:47.178440+0000","last_fullsized":"2023-10-09T05:32:47.178440+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:22.999187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":58,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.262395+0000","last_change":"2023-10-09T05:31:32.861963+0000","last_active":"2023-10-09T05:32:51.262395+0000","last_peered":"2023-10-09T05:32:51.262395+0000","last_clean":"2023-10-09T05:32:51.262395+0000","last_became_active":"2023-10-09T05:31:32.861679+0000","last_became_peered":"2023-10-09T05:31:32.861679+0000","last_unstale":"2023-10-09T05:32:51.262395+0000","last_undegraded":"2023-10-09T05:32:51.262395+0000","last_fullsized":"2023-10-09T05:32:51.262395+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:42.402077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.516362+0000","last_change":"2023-10-09T05:31:32.866744+0000","last_active":"2023-10-09T05:32:50.516362+0000","last_peered":"2023-10-09T05:32:50.516362+0000","last_clean":"2023-10-09T05:32:50.516362+0000","last_became_active":"2023-10-09T05:31:32.866389+0000","last_became_peered":"2023-10-09T05:31:32.866389+0000","last_unstale":"2023-10-09T05:32:50.516362+0000","last_undegraded":"2023-10-09T05:32:50.516362+0000","last_fullsized":"2023-10-09T05:32:50.516362+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":61,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.514217+0000","last_change":"2023-10-09T05:21:16.784983+0000","last_active":"2023-10-09T05:32:50.514217+0000","last_peered":"2023-10-09T05:32:50.514217+0000","last_clean":"2023-10-09T05:32:50.514217+0000","last_became_active":"2023-10-09T05:21:16.784699+0000","last_became_peered":"2023-10-09T05:21:16.784699+0000","last_unstale":"2023-10-09T05:32:50.514217+0000","last_undegraded":"2023-10-09T05:32:50.514217+0000","last_fullsized":"2023-10-09T05:32:50.514217+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":58,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.262601+0000","last_change":"2023-10-09T05:31:32.861092+0000","last_active":"2023-10-09T05:32:51.262601+0000","last_peered":"2023-10-09T05:32:51.262601+0000","last_clean":"2023-10-09T05:32:51.262601+0000","last_became_active":"2023-10-09T05:31:32.860632+0000","last_became_peered":"2023-10-09T05:31:32.860632+0000","last_unstale":"2023-10-09T05:32:51.262601+0000","last_undegraded":"2023-10-09T05:32:51.262601+0000","last_fullsized":"2023-10-09T05:32:51.262601+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:54.308161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":84,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345871+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:32:51.345871+0000","last_peered":"2023-10-09T05:32:51.345871+0000","last_clean":"2023-10-09T05:32:51.345871+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:32:51.345871+0000","last_undegraded":"2023-10-09T05:32:51.345871+0000","last_fullsized":"2023-10-09T05:32:51.345871+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":77,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.178498+0000","last_change":"2023-10-09T05:31:32.871093+0000","last_active":"2023-10-09T05:32:47.178498+0000","last_peered":"2023-10-09T05:32:47.178498+0000","last_clean":"2023-10-09T05:32:47.178498+0000","last_became_active":"2023-10-09T05:31:32.870702+0000","last_became_peered":"2023-10-09T05:31:32.870702+0000","last_unstale":"2023-10-09T05:32:47.178498+0000","last_undegraded":"2023-10-09T05:32:47.178498+0000","last_fullsized":"2023-10-09T05:32:47.178498+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:18.439344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"109'1234","reported_seq":1246,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T05:32:59.774566+0000","last_change":"2023-10-09T05:32:45.936233+0000","last_active":"2023-10-09T05:32:59.774566+0000","last_peered":"2023-10-09T05:32:59.774566+0000","last_clean":"2023-10-09T05:32:59.774566+0000","last_became_active":"2023-10-09T05:32:45.935917+0000","last_became_peered":"2023-10-09T05:32:45.935917+0000","last_unstale":"2023-10-09T05:32:59.774566+0000","last_undegraded":"2023-10-09T05:32:59.774566+0000","last_fullsized":"2023-10-09T05:32:59.774566+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":1234,"log_dups_size":0,"ondisk_log_size":1234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80871424,"num_objects":1234,"num_object_clones":0,"num_object_copies":3702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1234,"num_write_kb":78976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":77,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.514128+0000","last_change":"2023-10-09T05:29:59.470664+0000","last_active":"2023-10-09T05:32:50.514128+0000","last_peered":"2023-10-09T05:32:50.514128+0000","last_clean":"2023-10-09T05:32:50.514128+0000","last_became_active":"2023-10-09T05:29:59.470276+0000","last_became_peered":"2023-10-09T05:29:59.470276+0000","last_unstale":"2023-10-09T05:32:50.514128+0000","last_undegraded":"2023-10-09T05:32:50.514128+0000","last_fullsized":"2023-10-09T05:32:50.514128+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":60,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.515985+0000","last_change":"2023-10-09T05:21:17.464398+0000","last_active":"2023-10-09T05:32:50.515985+0000","last_peered":"2023-10-09T05:32:50.515985+0000","last_clean":"2023-10-09T05:32:50.515985+0000","last_became_active":"2023-10-09T05:21:17.464086+0000","last_became_peered":"2023-10-09T05:21:17.464086+0000","last_unstale":"2023-10-09T05:32:50.515985+0000","last_undegraded":"2023-10-09T05:32:50.515985+0000","last_fullsized":"2023-10-09T05:32:50.515985+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"108'589","reported_seq":600,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:55.539613+0000","last_change":"2023-10-09T05:32:45.926858+0000","last_active":"2023-10-09T05:32:55.539613+0000","last_peered":"2023-10-09T05:32:55.539613+0000","last_clean":"2023-10-09T05:32:55.539613+0000","last_became_active":"2023-10-09T05:32:45.926538+0000","last_became_peered":"2023-10-09T05:32:45.926538+0000","last_unstale":"2023-10-09T05:32:55.539613+0000","last_undegraded":"2023-10-09T05:32:55.539613+0000","last_fullsized":"2023-10-09T05:32:55.539613+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":589,"log_dups_size":0,"ondisk_log_size":589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:44.854965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38600704,"num_objects":589,"num_object_clones":0,"num_object_copies":1767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":589,"num_write_kb":37696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":78,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.514193+0000","last_change":"2023-10-09T05:30:30.457635+0000","last_active":"2023-10-09T05:32:50.514193+0000","last_peered":"2023-10-09T05:32:50.514193+0000","last_clean":"2023-10-09T05:32:50.514193+0000","last_became_active":"2023-10-09T05:30:30.457276+0000","last_became_peered":"2023-10-09T05:30:30.457276+0000","last_unstale":"2023-10-09T05:32:50.514193+0000","last_undegraded":"2023-10-09T05:32:50.514193+0000","last_fullsized":"2023-10-09T05:32:50.514193+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":90,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345734+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:32:51.345734+0000","last_peered":"2023-10-09T05:32:51.345734+0000","last_clean":"2023-10-09T05:32:51.345734+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:32:51.345734+0000","last_undegraded":"2023-10-09T05:32:51.345734+0000","last_fullsized":"2023-10-09T05:32:51.345734+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"108'599","reported_seq":611,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:55.534834+0000","last_change":"2023-10-09T05:32:45.931141+0000","last_active":"2023-10-09T05:32:55.534834+0000","last_peered":"2023-10-09T05:32:55.534834+0000","last_clean":"2023-10-09T05:32:55.534834+0000","last_became_active":"2023-10-09T05:32:45.930819+0000","last_became_peered":"2023-10-09T05:32:45.930819+0000","last_unstale":"2023-10-09T05:32:55.534834+0000","last_undegraded":"2023-10-09T05:32:55.534834+0000","last_fullsized":"2023-10-09T05:32:55.534834+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":599,"log_dups_size":0,"ondisk_log_size":599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:25.657817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39256064,"num_objects":599,"num_object_clones":0,"num_object_copies":1797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":599,"num_write_kb":38336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":62,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.514017+0000","last_change":"2023-10-09T05:21:16.781147+0000","last_active":"2023-10-09T05:32:50.514017+0000","last_peered":"2023-10-09T05:32:50.514017+0000","last_clean":"2023-10-09T05:32:50.514017+0000","last_became_active":"2023-10-09T05:21:16.780755+0000","last_became_peered":"2023-10-09T05:21:16.780755+0000","last_unstale":"2023-10-09T05:32:50.514017+0000","last_undegraded":"2023-10-09T05:32:50.514017+0000","last_fullsized":"2023-10-09T05:32:50.514017+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":84,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346116+0000","last_change":"2023-10-09T05:31:02.548845+0000","last_active":"2023-10-09T05:32:51.346116+0000","last_peered":"2023-10-09T05:32:51.346116+0000","last_clean":"2023-10-09T05:32:51.346116+0000","last_became_active":"2023-10-09T05:31:02.548131+0000","last_became_peered":"2023-10-09T05:31:02.548131+0000","last_unstale":"2023-10-09T05:32:51.346116+0000","last_undegraded":"2023-10-09T05:32:51.346116+0000","last_fullsized":"2023-10-09T05:32:51.346116+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"108'566","reported_seq":578,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:55.534717+0000","last_change":"2023-10-09T05:32:45.929091+0000","last_active":"2023-10-09T05:32:55.534717+0000","last_peered":"2023-10-09T05:32:55.534717+0000","last_clean":"2023-10-09T05:32:55.534717+0000","last_became_active":"2023-10-09T05:32:45.928801+0000","last_became_peered":"2023-10-09T05:32:45.928801+0000","last_unstale":"2023-10-09T05:32:55.534717+0000","last_undegraded":"2023-10-09T05:32:55.534717+0000","last_fullsized":"2023-10-09T05:32:55.534717+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":566,"log_dups_size":0,"ondisk_log_size":566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:53.569691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37093376,"num_objects":566,"num_object_clones":0,"num_object_copies":1698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":566,"num_write_kb":36224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":76,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.516307+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:32:50.516307+0000","last_peered":"2023-10-09T05:32:50.516307+0000","last_clean":"2023-10-09T05:32:50.516307+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:32:50.516307+0000","last_undegraded":"2023-10-09T05:32:50.516307+0000","last_fullsized":"2023-10-09T05:32:50.516307+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":58,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.262427+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:32:51.262427+0000","last_peered":"2023-10-09T05:32:51.262427+0000","last_clean":"2023-10-09T05:32:51.262427+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:32:51.262427+0000","last_undegraded":"2023-10-09T05:32:51.262427+0000","last_fullsized":"2023-10-09T05:32:51.262427+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"108'578","reported_seq":590,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:55.536440+0000","last_change":"2023-10-09T05:32:45.931304+0000","last_active":"2023-10-09T05:32:55.536440+0000","last_peered":"2023-10-09T05:32:55.536440+0000","last_clean":"2023-10-09T05:32:55.536440+0000","last_became_active":"2023-10-09T05:32:45.930948+0000","last_became_peered":"2023-10-09T05:32:45.930948+0000","last_unstale":"2023-10-09T05:32:55.536440+0000","last_undegraded":"2023-10-09T05:32:55.536440+0000","last_fullsized":"2023-10-09T05:32:55.536440+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":578,"log_dups_size":0,"ondisk_log_size":578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:30.019591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37879808,"num_objects":578,"num_object_clones":0,"num_object_copies":1734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":578,"num_write_kb":36992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":75,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.178022+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:32:47.178022+0000","last_peered":"2023-10-09T05:32:47.178022+0000","last_clean":"2023-10-09T05:32:47.178022+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:32:47.178022+0000","last_undegraded":"2023-10-09T05:32:47.178022+0000","last_fullsized":"2023-10-09T05:32:47.178022+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345683+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:32:51.345683+0000","last_peered":"2023-10-09T05:32:51.345683+0000","last_clean":"2023-10-09T05:32:51.345683+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:32:51.345683+0000","last_undegraded":"2023-10-09T05:32:51.345683+0000","last_fullsized":"2023-10-09T05:32:51.345683+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"109'1121","reported_seq":1134,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T05:32:58.447745+0000","last_change":"2023-10-09T05:32:45.927644+0000","last_active":"2023-10-09T05:32:58.447745+0000","last_peered":"2023-10-09T05:32:58.447745+0000","last_clean":"2023-10-09T05:32:58.447745+0000","last_became_active":"2023-10-09T05:32:45.927388+0000","last_became_peered":"2023-10-09T05:32:45.927388+0000","last_unstale":"2023-10-09T05:32:58.447745+0000","last_undegraded":"2023-10-09T05:32:58.447745+0000","last_fullsized":"2023-10-09T05:32:58.447745+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":1121,"log_dups_size":0,"ondisk_log_size":1121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:42.111993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73465856,"num_objects":1121,"num_object_clones":0,"num_object_copies":3363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1121,"num_write_kb":71744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":76,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346167+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:32:51.346167+0000","last_peered":"2023-10-09T05:32:51.346167+0000","last_clean":"2023-10-09T05:32:51.346167+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:32:51.346167+0000","last_undegraded":"2023-10-09T05:32:51.346167+0000","last_fullsized":"2023-10-09T05:32:51.346167+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":90,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345597+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:32:51.345597+0000","last_peered":"2023-10-09T05:32:51.345597+0000","last_clean":"2023-10-09T05:32:51.345597+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:32:51.345597+0000","last_undegraded":"2023-10-09T05:32:51.345597+0000","last_fullsized":"2023-10-09T05:32:51.345597+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"109'1275","reported_seq":1287,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T05:32:59.753774+0000","last_change":"2023-10-09T05:32:45.936026+0000","last_active":"2023-10-09T05:32:59.753774+0000","last_peered":"2023-10-09T05:32:59.753774+0000","last_clean":"2023-10-09T05:32:59.753774+0000","last_became_active":"2023-10-09T05:32:45.935681+0000","last_became_peered":"2023-10-09T05:32:45.935681+0000","last_unstale":"2023-10-09T05:32:59.753774+0000","last_undegraded":"2023-10-09T05:32:59.753774+0000","last_fullsized":"2023-10-09T05:32:59.753774+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":1275,"log_dups_size":0,"ondisk_log_size":1275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83558400,"num_objects":1275,"num_object_clones":0,"num_object_copies":3825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1275,"num_write_kb":81600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":76,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345189+0000","last_change":"2023-10-09T05:31:02.552208+0000","last_active":"2023-10-09T05:32:51.345189+0000","last_peered":"2023-10-09T05:32:51.345189+0000","last_clean":"2023-10-09T05:32:51.345189+0000","last_became_active":"2023-10-09T05:31:02.551893+0000","last_became_peered":"2023-10-09T05:31:02.551893+0000","last_unstale":"2023-10-09T05:32:51.345189+0000","last_undegraded":"2023-10-09T05:32:51.345189+0000","last_fullsized":"2023-10-09T05:32:51.345189+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":84,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345626+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:32:51.345626+0000","last_peered":"2023-10-09T05:32:51.345626+0000","last_clean":"2023-10-09T05:32:51.345626+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:32:51.345626+0000","last_undegraded":"2023-10-09T05:32:51.345626+0000","last_fullsized":"2023-10-09T05:32:51.345626+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"109'1267","reported_seq":1279,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T05:32:59.748767+0000","last_change":"2023-10-09T05:32:45.929479+0000","last_active":"2023-10-09T05:32:59.748767+0000","last_peered":"2023-10-09T05:32:59.748767+0000","last_clean":"2023-10-09T05:32:59.748767+0000","last_became_active":"2023-10-09T05:32:45.929063+0000","last_became_peered":"2023-10-09T05:32:45.929063+0000","last_unstale":"2023-10-09T05:32:59.748767+0000","last_undegraded":"2023-10-09T05:32:59.748767+0000","last_fullsized":"2023-10-09T05:32:59.748767+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":1267,"log_dups_size":0,"ondisk_log_size":1267,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83034112,"num_objects":1267,"num_object_clones":0,"num_object_copies":3801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1267,"num_write_kb":81088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":78,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.178488+0000","last_change":"2023-10-09T05:31:32.870575+0000","last_active":"2023-10-09T05:32:47.178488+0000","last_peered":"2023-10-09T05:32:47.178488+0000","last_clean":"2023-10-09T05:32:47.178488+0000","last_became_active":"2023-10-09T05:31:32.870217+0000","last_became_peered":"2023-10-09T05:31:32.870217+0000","last_unstale":"2023-10-09T05:32:47.178488+0000","last_undegraded":"2023-10-09T05:32:47.178488+0000","last_fullsized":"2023-10-09T05:32:47.178488+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:08.078992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346979+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:32:51.346979+0000","last_peered":"2023-10-09T05:32:51.346979+0000","last_clean":"2023-10-09T05:32:51.346979+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:32:51.346979+0000","last_undegraded":"2023-10-09T05:32:51.346979+0000","last_fullsized":"2023-10-09T05:32:51.346979+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":617,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346739+0000","last_change":"2023-10-09T05:30:30.457812+0000","last_active":"2023-10-09T05:32:51.346739+0000","last_peered":"2023-10-09T05:32:51.346739+0000","last_clean":"2023-10-09T05:32:51.346739+0000","last_became_active":"2023-10-09T05:30:30.456811+0000","last_became_peered":"2023-10-09T05:30:30.456811+0000","last_unstale":"2023-10-09T05:32:51.346739+0000","last_undegraded":"2023-10-09T05:32:51.346739+0000","last_fullsized":"2023-10-09T05:32:51.346739+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:17.322315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":91,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.516450+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:32:50.516450+0000","last_peered":"2023-10-09T05:32:50.516450+0000","last_clean":"2023-10-09T05:32:50.516450+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:32:50.516450+0000","last_undegraded":"2023-10-09T05:32:50.516450+0000","last_fullsized":"2023-10-09T05:32:50.516450+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":38,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346599+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:32:51.346599+0000","last_peered":"2023-10-09T05:32:51.346599+0000","last_clean":"2023-10-09T05:32:51.346599+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:32:51.346599+0000","last_undegraded":"2023-10-09T05:32:51.346599+0000","last_fullsized":"2023-10-09T05:32:51.346599+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"109'890","reported_seq":903,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T05:32:57.524101+0000","last_change":"2023-10-09T05:32:45.936085+0000","last_active":"2023-10-09T05:32:57.524101+0000","last_peered":"2023-10-09T05:32:57.524101+0000","last_clean":"2023-10-09T05:32:57.524101+0000","last_became_active":"2023-10-09T05:32:45.935802+0000","last_became_peered":"2023-10-09T05:32:45.935802+0000","last_unstale":"2023-10-09T05:32:57.524101+0000","last_undegraded":"2023-10-09T05:32:57.524101+0000","last_fullsized":"2023-10-09T05:32:57.524101+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":890,"log_dups_size":0,"ondisk_log_size":890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:26.030229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58327040,"num_objects":890,"num_object_clones":0,"num_object_copies":2670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":890,"num_write_kb":56960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"109'939","reported_seq":951,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T05:32:57.525220+0000","last_change":"2023-10-09T05:32:45.927786+0000","last_active":"2023-10-09T05:32:57.525220+0000","last_peered":"2023-10-09T05:32:57.525220+0000","last_clean":"2023-10-09T05:32:57.525220+0000","last_became_active":"2023-10-09T05:32:45.927542+0000","last_became_peered":"2023-10-09T05:32:45.927542+0000","last_unstale":"2023-10-09T05:32:57.525220+0000","last_undegraded":"2023-10-09T05:32:57.525220+0000","last_fullsized":"2023-10-09T05:32:57.525220+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":939,"log_dups_size":0,"ondisk_log_size":939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:36.942422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61538304,"num_objects":939,"num_object_clones":0,"num_object_copies":2817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":939,"num_write_kb":60096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":545,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.516201+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:32:50.516201+0000","last_peered":"2023-10-09T05:32:50.516201+0000","last_clean":"2023-10-09T05:32:50.516201+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:32:50.516201+0000","last_undegraded":"2023-10-09T05:32:50.516201+0000","last_fullsized":"2023-10-09T05:32:50.516201+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":76,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345554+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:32:51.345554+0000","last_peered":"2023-10-09T05:32:51.345554+0000","last_clean":"2023-10-09T05:32:51.345554+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:32:51.345554+0000","last_undegraded":"2023-10-09T05:32:51.345554+0000","last_fullsized":"2023-10-09T05:32:51.345554+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":88,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.516569+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:32:50.516569+0000","last_peered":"2023-10-09T05:32:50.516569+0000","last_clean":"2023-10-09T05:32:50.516569+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:32:50.516569+0000","last_undegraded":"2023-10-09T05:32:50.516569+0000","last_fullsized":"2023-10-09T05:32:50.516569+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"108'554","reported_seq":566,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:55.521790+0000","last_change":"2023-10-09T05:32:45.930116+0000","last_active":"2023-10-09T05:32:55.521790+0000","last_peered":"2023-10-09T05:32:55.521790+0000","last_clean":"2023-10-09T05:32:55.521790+0000","last_became_active":"2023-10-09T05:32:45.929862+0000","last_became_peered":"2023-10-09T05:32:45.929862+0000","last_unstale":"2023-10-09T05:32:55.521790+0000","last_undegraded":"2023-10-09T05:32:55.521790+0000","last_fullsized":"2023-10-09T05:32:55.521790+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":554,"log_dups_size":0,"ondisk_log_size":554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:55.912123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36306944,"num_objects":554,"num_object_clones":0,"num_object_copies":1662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":554,"num_write_kb":35456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":418,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.176824+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:32:47.176824+0000","last_peered":"2023-10-09T05:32:47.176824+0000","last_clean":"2023-10-09T05:32:47.176824+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:32:47.176824+0000","last_undegraded":"2023-10-09T05:32:47.176824+0000","last_fullsized":"2023-10-09T05:32:47.176824+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346062+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:32:51.346062+0000","last_peered":"2023-10-09T05:32:51.346062+0000","last_clean":"2023-10-09T05:32:51.346062+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:32:51.346062+0000","last_undegraded":"2023-10-09T05:32:51.346062+0000","last_fullsized":"2023-10-09T05:32:51.346062+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":83,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346960+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:32:51.346960+0000","last_peered":"2023-10-09T05:32:51.346960+0000","last_clean":"2023-10-09T05:32:51.346960+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:32:51.346960+0000","last_undegraded":"2023-10-09T05:32:51.346960+0000","last_fullsized":"2023-10-09T05:32:51.346960+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":670,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.516092+0000","last_change":"2023-10-09T05:31:02.547804+0000","last_active":"2023-10-09T05:32:50.516092+0000","last_peered":"2023-10-09T05:32:50.516092+0000","last_clean":"2023-10-09T05:32:50.516092+0000","last_became_active":"2023-10-09T05:31:02.547237+0000","last_became_peered":"2023-10-09T05:31:02.547237+0000","last_unstale":"2023-10-09T05:32:50.516092+0000","last_undegraded":"2023-10-09T05:32:50.516092+0000","last_fullsized":"2023-10-09T05:32:50.516092+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":312,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.513681+0000","last_change":"2023-10-09T05:29:59.475956+0000","last_active":"2023-10-09T05:32:50.513681+0000","last_peered":"2023-10-09T05:32:50.513681+0000","last_clean":"2023-10-09T05:32:50.513681+0000","last_became_active":"2023-10-09T05:29:59.475408+0000","last_became_peered":"2023-10-09T05:29:59.475408+0000","last_unstale":"2023-10-09T05:32:50.513681+0000","last_undegraded":"2023-10-09T05:32:50.513681+0000","last_fullsized":"2023-10-09T05:32:50.513681+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":68,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.177959+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:32:47.177959+0000","last_peered":"2023-10-09T05:32:47.177959+0000","last_clean":"2023-10-09T05:32:47.177959+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:32:47.177959+0000","last_undegraded":"2023-10-09T05:32:47.177959+0000","last_fullsized":"2023-10-09T05:32:47.177959+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345869+0000","last_change":"2023-10-09T05:21:17.465038+0000","last_active":"2023-10-09T05:32:51.345869+0000","last_peered":"2023-10-09T05:32:51.345869+0000","last_clean":"2023-10-09T05:32:51.345869+0000","last_became_active":"2023-10-09T05:21:17.464695+0000","last_became_peered":"2023-10-09T05:21:17.464695+0000","last_unstale":"2023-10-09T05:32:51.345869+0000","last_undegraded":"2023-10-09T05:32:51.345869+0000","last_fullsized":"2023-10-09T05:32:51.345869+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"109'1261","reported_seq":1273,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T05:32:59.768255+0000","last_change":"2023-10-09T05:32:45.932566+0000","last_active":"2023-10-09T05:32:59.768255+0000","last_peered":"2023-10-09T05:32:59.768255+0000","last_clean":"2023-10-09T05:32:59.768255+0000","last_became_active":"2023-10-09T05:32:45.932288+0000","last_became_peered":"2023-10-09T05:32:45.932288+0000","last_unstale":"2023-10-09T05:32:59.768255+0000","last_undegraded":"2023-10-09T05:32:59.768255+0000","last_fullsized":"2023-10-09T05:32:59.768255+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":1261,"log_dups_size":0,"ondisk_log_size":1261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82640896,"num_objects":1261,"num_object_clones":0,"num_object_copies":3783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1261,"num_write_kb":80704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"109'1254","reported_seq":1265,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T05:32:59.770200+0000","last_change":"2023-10-09T05:32:45.934765+0000","last_active":"2023-10-09T05:32:59.770200+0000","last_peered":"2023-10-09T05:32:59.770200+0000","last_clean":"2023-10-09T05:32:59.770200+0000","last_became_active":"2023-10-09T05:32:45.934420+0000","last_became_peered":"2023-10-09T05:32:45.934420+0000","last_unstale":"2023-10-09T05:32:59.770200+0000","last_undegraded":"2023-10-09T05:32:59.770200+0000","last_fullsized":"2023-10-09T05:32:59.770200+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":1254,"log_dups_size":0,"ondisk_log_size":1254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82182144,"num_objects":1254,"num_object_clones":0,"num_object_copies":3762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1254,"num_write_kb":80256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":730,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.513731+0000","last_change":"2023-10-09T05:30:30.456503+0000","last_active":"2023-10-09T05:32:50.513731+0000","last_peered":"2023-10-09T05:32:50.513731+0000","last_clean":"2023-10-09T05:32:50.513731+0000","last_became_active":"2023-10-09T05:30:30.455426+0000","last_became_peered":"2023-10-09T05:30:30.455426+0000","last_unstale":"2023-10-09T05:32:50.513731+0000","last_undegraded":"2023-10-09T05:32:50.513731+0000","last_fullsized":"2023-10-09T05:32:50.513731+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":58,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.262276+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:32:51.262276+0000","last_peered":"2023-10-09T05:32:51.262276+0000","last_clean":"2023-10-09T05:32:51.262276+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:32:51.262276+0000","last_undegraded":"2023-10-09T05:32:51.262276+0000","last_fullsized":"2023-10-09T05:32:51.262276+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.262720+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:32:51.262720+0000","last_peered":"2023-10-09T05:32:51.262720+0000","last_clean":"2023-10-09T05:32:51.262720+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:32:51.262720+0000","last_undegraded":"2023-10-09T05:32:51.262720+0000","last_fullsized":"2023-10-09T05:32:51.262720+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":338,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.513952+0000","last_change":"2023-10-09T05:29:59.474688+0000","last_active":"2023-10-09T05:32:50.513952+0000","last_peered":"2023-10-09T05:32:50.513952+0000","last_clean":"2023-10-09T05:32:50.513952+0000","last_became_active":"2023-10-09T05:29:59.474080+0000","last_became_peered":"2023-10-09T05:29:59.474080+0000","last_unstale":"2023-10-09T05:32:50.513952+0000","last_undegraded":"2023-10-09T05:32:50.513952+0000","last_fullsized":"2023-10-09T05:32:50.513952+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346406+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:32:51.346406+0000","last_peered":"2023-10-09T05:32:51.346406+0000","last_clean":"2023-10-09T05:32:51.346406+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:32:51.346406+0000","last_undegraded":"2023-10-09T05:32:51.346406+0000","last_fullsized":"2023-10-09T05:32:51.346406+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":43,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.262534+0000","last_change":"2023-10-09T05:29:59.470093+0000","last_active":"2023-10-09T05:32:51.262534+0000","last_peered":"2023-10-09T05:32:51.262534+0000","last_clean":"2023-10-09T05:32:51.262534+0000","last_became_active":"2023-10-09T05:29:59.469627+0000","last_became_peered":"2023-10-09T05:29:59.469627+0000","last_unstale":"2023-10-09T05:32:51.262534+0000","last_undegraded":"2023-10-09T05:32:51.262534+0000","last_fullsized":"2023-10-09T05:32:51.262534+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:17.087268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"109'1276","reported_seq":1288,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T05:32:59.769839+0000","last_change":"2023-10-09T05:32:45.936284+0000","last_active":"2023-10-09T05:32:59.769839+0000","last_peered":"2023-10-09T05:32:59.769839+0000","last_clean":"2023-10-09T05:32:59.769839+0000","last_became_active":"2023-10-09T05:32:45.935947+0000","last_became_peered":"2023-10-09T05:32:45.935947+0000","last_unstale":"2023-10-09T05:32:59.769839+0000","last_undegraded":"2023-10-09T05:32:59.769839+0000","last_fullsized":"2023-10-09T05:32:59.769839+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":1276,"log_dups_size":0,"ondisk_log_size":1276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83623936,"num_objects":1276,"num_object_clones":0,"num_object_copies":3828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1276,"num_write_kb":81664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":75,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.178264+0000","last_change":"2023-10-09T05:31:32.862179+0000","last_active":"2023-10-09T05:32:47.178264+0000","last_peered":"2023-10-09T05:32:47.178264+0000","last_clean":"2023-10-09T05:32:47.178264+0000","last_became_active":"2023-10-09T05:31:32.861777+0000","last_became_peered":"2023-10-09T05:31:32.861777+0000","last_unstale":"2023-10-09T05:32:47.178264+0000","last_undegraded":"2023-10-09T05:32:47.178264+0000","last_fullsized":"2023-10-09T05:32:47.178264+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:40.547661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":494,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346970+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:32:51.346970+0000","last_peered":"2023-10-09T05:32:51.346970+0000","last_clean":"2023-10-09T05:32:51.346970+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:32:51.346970+0000","last_undegraded":"2023-10-09T05:32:51.346970+0000","last_fullsized":"2023-10-09T05:32:51.346970+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.178444+0000","last_change":"2023-10-09T05:31:32.862414+0000","last_active":"2023-10-09T05:32:47.178444+0000","last_peered":"2023-10-09T05:32:47.178444+0000","last_clean":"2023-10-09T05:32:47.178444+0000","last_became_active":"2023-10-09T05:31:32.860985+0000","last_became_peered":"2023-10-09T05:31:32.860985+0000","last_unstale":"2023-10-09T05:32:47.178444+0000","last_undegraded":"2023-10-09T05:32:47.178444+0000","last_fullsized":"2023-10-09T05:32:47.178444+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:04.492757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"109'1255","reported_seq":1266,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-10-09T05:32:59.763687+0000","last_change":"2023-10-09T05:32:45.927565+0000","last_active":"2023-10-09T05:32:59.763687+0000","last_peered":"2023-10-09T05:32:59.763687+0000","last_clean":"2023-10-09T05:32:59.763687+0000","last_became_active":"2023-10-09T05:32:45.927149+0000","last_became_peered":"2023-10-09T05:32:45.927149+0000","last_unstale":"2023-10-09T05:32:59.763687+0000","last_undegraded":"2023-10-09T05:32:59.763687+0000","last_fullsized":"2023-10-09T05:32:59.763687+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":1255,"log_dups_size":0,"ondisk_log_size":1255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82247680,"num_objects":1255,"num_object_clones":0,"num_object_copies":3765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1255,"num_write_kb":80320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":557,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.346084+0000","last_change":"2023-10-09T05:31:02.549304+0000","last_active":"2023-10-09T05:32:51.346084+0000","last_peered":"2023-10-09T05:32:51.346084+0000","last_clean":"2023-10-09T05:32:51.346084+0000","last_became_active":"2023-10-09T05:31:02.548085+0000","last_became_peered":"2023-10-09T05:31:02.548085+0000","last_unstale":"2023-10-09T05:32:51.346084+0000","last_undegraded":"2023-10-09T05:32:51.346084+0000","last_fullsized":"2023-10-09T05:32:51.346084+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":61,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.262700+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:32:51.262700+0000","last_peered":"2023-10-09T05:32:51.262700+0000","last_clean":"2023-10-09T05:32:51.262700+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:32:51.262700+0000","last_undegraded":"2023-10-09T05:32:51.262700+0000","last_fullsized":"2023-10-09T05:32:51.262700+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345246+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:32:51.345246+0000","last_peered":"2023-10-09T05:32:51.345246+0000","last_clean":"2023-10-09T05:32:51.345246+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:32:51.345246+0000","last_undegraded":"2023-10-09T05:32:51.345246+0000","last_fullsized":"2023-10-09T05:32:51.345246+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"108'660","reported_seq":671,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:55.857049+0000","last_change":"2023-10-09T05:32:45.930060+0000","last_active":"2023-10-09T05:32:55.857049+0000","last_peered":"2023-10-09T05:32:55.857049+0000","last_clean":"2023-10-09T05:32:55.857049+0000","last_became_active":"2023-10-09T05:32:45.929772+0000","last_became_peered":"2023-10-09T05:32:45.929772+0000","last_unstale":"2023-10-09T05:32:55.857049+0000","last_undegraded":"2023-10-09T05:32:55.857049+0000","last_fullsized":"2023-10-09T05:32:55.857049+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":660,"log_dups_size":0,"ondisk_log_size":660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:06.171642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43253760,"num_objects":660,"num_object_clones":0,"num_object_copies":1980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":660,"num_write_kb":42240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.262516+0000","last_change":"2023-10-09T05:29:59.477246+0000","last_active":"2023-10-09T05:32:51.262516+0000","last_peered":"2023-10-09T05:32:51.262516+0000","last_clean":"2023-10-09T05:32:51.262516+0000","last_became_active":"2023-10-09T05:29:59.476390+0000","last_became_peered":"2023-10-09T05:29:59.476390+0000","last_unstale":"2023-10-09T05:32:51.262516+0000","last_undegraded":"2023-10-09T05:32:51.262516+0000","last_fullsized":"2023-10-09T05:32:51.262516+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:52:31.557279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":96,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.177681+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:32:47.177681+0000","last_peered":"2023-10-09T05:32:47.177681+0000","last_clean":"2023-10-09T05:32:47.177681+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:32:47.177681+0000","last_undegraded":"2023-10-09T05:32:47.177681+0000","last_fullsized":"2023-10-09T05:32:47.177681+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":78,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.516391+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:32:50.516391+0000","last_peered":"2023-10-09T05:32:50.516391+0000","last_clean":"2023-10-09T05:32:50.516391+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:32:50.516391+0000","last_undegraded":"2023-10-09T05:32:50.516391+0000","last_fullsized":"2023-10-09T05:32:50.516391+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":90,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345655+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:32:51.345655+0000","last_peered":"2023-10-09T05:32:51.345655+0000","last_clean":"2023-10-09T05:32:51.345655+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:32:51.345655+0000","last_undegraded":"2023-10-09T05:32:51.345655+0000","last_fullsized":"2023-10-09T05:32:51.345655+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":63,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:50.516541+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:32:50.516541+0000","last_peered":"2023-10-09T05:32:50.516541+0000","last_clean":"2023-10-09T05:32:50.516541+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:32:50.516541+0000","last_undegraded":"2023-10-09T05:32:50.516541+0000","last_fullsized":"2023-10-09T05:32:50.516541+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":90,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345516+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:32:51.345516+0000","last_peered":"2023-10-09T05:32:51.345516+0000","last_clean":"2023-10-09T05:32:51.345516+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:32:51.345516+0000","last_undegraded":"2023-10-09T05:32:51.345516+0000","last_fullsized":"2023-10-09T05:32:51.345516+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":83,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.178113+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:32:47.178113+0000","last_peered":"2023-10-09T05:32:47.178113+0000","last_clean":"2023-10-09T05:32:47.178113+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:32:47.178113+0000","last_undegraded":"2023-10-09T05:32:47.178113+0000","last_fullsized":"2023-10-09T05:32:47.178113+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":96,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:32:47.177877+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:32:47.177877+0000","last_peered":"2023-10-09T05:32:47.177877+0000","last_clean":"2023-10-09T05:32:47.177877+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:32:47.177877+0000","last_undegraded":"2023-10-09T05:32:47.177877+0000","last_fullsized":"2023-10-09T05:32:47.177877+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":61,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.262718+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:32:51.262718+0000","last_peered":"2023-10-09T05:32:51.262718+0000","last_clean":"2023-10-09T05:32:51.262718+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:32:51.262718+0000","last_undegraded":"2023-10-09T05:32:51.262718+0000","last_fullsized":"2023-10-09T05:32:51.262718+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-10-09T05:32:51.345115+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:32:51.345115+0000","last_peered":"2023-10-09T05:32:51.345115+0000","last_clean":"2023-10-09T05:32:51.345115+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:32:51.345115+0000","last_undegraded":"2023-10-09T05:32:51.345115+0000","last_fullsized":"2023-10-09T05:32:51.345115+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1003880448,"num_objects":15318,"num_object_clones":0,"num_object_copies":45954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15318,"num_write_kb":980352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3346989056,"data_stored":3346989056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15318,"ondisk_log_size":15318,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26751,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430371,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":701528,"kb_used_data":396112,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93043624,"statfs":{"total":95995035648,"available":95276670976,"internally_reserved":0,"allocated":405618688,"data_stored":405111917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:31:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.879}]},{"osd":1,"last update":"Mon Oct 9 05:32:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.51700000000000002,"15min":0.51600000000000001},"min":{"1min":0.40500000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.68999999999999995,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.41799999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.73299999999999998,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.309}]},{"osd":2,"last update":"Mon Oct 9 05:32:05 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.33600000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:32:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.52100000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.49099999999999999,"15min":0.46500000000000002},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.249},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.52500000000000002,"15min":0.50800000000000001},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.27300000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.79600000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.53000000000000003,"15min":0.50700000000000001},"min":{"1min":0.35799999999999998,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":1.462},"last":1.1619999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.40200000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88700000000000001},"last":1.1879999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:32:09 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.56699999999999995,"15min":0.54000000000000004},"min":{"1min":0.42299999999999999,"5min":0.34799999999999998,"15min":0.30499999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.623,"5min":0.59599999999999997,"15min":0.58499999999999996},"min":{"1min":0.45000000000000001,"5min":0.372,"15min":0.33300000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.85999999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626599,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":628108,"kb_used_data":326788,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93117044,"statfs":{"total":95995035648,"available":95351853056,"internally_reserved":0,"allocated":334630912,"data_stored":334136037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.55500000000000005,"15min":0.54500000000000004},"min":{"1min":0.39300000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":0.378},{"interface":"front","average":{"1min":1.014,"5min":0.63800000000000001,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.4569999999999999,"5min":7.4569999999999999,"15min":7.4569999999999999},"last":1.298}]},{"osd":1,"last update":"Mon Oct 9 05:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":0.69799999999999995,"15min":0.60099999999999998},"min":{"1min":0.41899999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.7360000000000007,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.325,"5min":0.72599999999999998,"15min":0.626},"min":{"1min":0.46899999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.7189999999999994,"5min":8.7189999999999994,"15min":8.7189999999999994},"last":0.60499999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.075,"15min":1.075},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.5999999999999996,"15min":7.5999999999999996},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":7.6379999999999999,"5min":7.6379999999999999,"15min":7.6379999999999999},"last":0.73299999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":7.7439999999999998,"5min":7.7439999999999998,"15min":7.7439999999999998},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.123,"5min":1.123,"15min":1.123},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":7.6929999999999996,"5min":7.6929999999999996,"15min":7.6929999999999996},"last":0.46300000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:32:47 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.49299999999999999,"15min":0.46899999999999997},"min":{"1min":0.34100000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.74199999999999999,"5min":0.746,"15min":0.86399999999999999},"last":0.438},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.5,"15min":0.47899999999999998},"min":{"1min":0.36499999999999999,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":0.69399999999999995,"5min":0.83799999999999997,"15min":1.014},"last":0.47099999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:32:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.55400000000000005,"15min":0.51400000000000001},"min":{"1min":0.39300000000000002,"5min":0.30499999999999999,"15min":0.27200000000000002},"max":{"1min":0.749,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54400000000000004,"15min":0.52000000000000002},"min":{"1min":0.40100000000000002,"5min":0.313,"15min":0.313},"max":{"1min":0.73999999999999999,"5min":0.82399999999999995,"15min":2.0579999999999998},"last":1.2629999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.54200000000000004,"15min":0.53300000000000003},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.60199999999999998,"15min":0.56999999999999995},"min":{"1min":0.498,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.999},"last":0.94099999999999995}]}]},{"osd":1,"up_from":93,"seq":399431958560,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343600,"kb_used_data":311912,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93401552,"statfs":{"total":95995035648,"available":95643189248,"internally_reserved":0,"allocated":319397888,"data_stored":318870502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":32432654},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.39200000000000002,"15min":0.39900000000000002},"min":{"1min":0.23599999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.503,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":1.1419999999999999},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.39900000000000002,"15min":0.40500000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.55500000000000005,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":1.056}]},{"osd":2,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":1.1020000000000001},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":1.9119999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.2109999999999999,"5min":2.2109999999999999,"15min":2.2109999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":1.7090000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.63800000000000001,"15min":0.64100000000000001},"min":{"1min":0.46200000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.83999999999999997,"5min":0.872,"15min":0.872},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.65100000000000002,"15min":0.65700000000000003},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.84099999999999997,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":1.0780000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.63,"15min":0.63200000000000001},"min":{"1min":0.442,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.75700000000000001,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.63,"5min":0.66800000000000004,"15min":0.67400000000000004},"min":{"1min":0.48299999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.878,"5min":1.04,"15min":1.04},"last":0.72799999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.64700000000000002,"15min":0.65200000000000002},"min":{"1min":0.48599999999999999,"5min":0.441,"15min":0.441},"max":{"1min":0.80400000000000005,"5min":0.998,"15min":0.998},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.67500000000000004,"15min":0.68200000000000005},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.90100000000000002,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.69599999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.69699999999999995,"15min":0.70799999999999996},"min":{"1min":0.47299999999999998,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.77000000000000002,"5min":1.014,"15min":1.014},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.68400000000000005,"15min":0.69299999999999995},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.82799999999999996,"5min":1,"15min":1},"last":0.67400000000000004}]}]},{"osd":0,"up_from":88,"seq":377957122086,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":432312,"kb_used_data":400624,"kb_used_omap":8,"kb_used_meta":31671,"kb_avail":93312840,"statfs":{"total":95995035648,"available":95552348160,"internally_reserved":0,"allocated":410238976,"data_stored":409704023,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":32431880},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,4,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:32:37 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.38600000000000001,"15min":0.39200000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.46700000000000003,"5min":1.028,"15min":1.028},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.42499999999999999,"15min":0.435},"min":{"1min":0.28199999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.44600000000000001,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.65900000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.376},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.57199999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:32:33 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.80400000000000005}]},{"osd":4,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.65900000000000003,"15min":0.67300000000000004},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.83699999999999997,"5min":1.028,"15min":1.028},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.69999999999999996,"15min":0.71399999999999997},"min":{"1min":0.45300000000000001,"5min":0.434,"15min":0.434},"max":{"1min":0.81799999999999995,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.73999999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.65400000000000003,"15min":0.67200000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.70399999999999996,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.65600000000000003,"15min":0.66900000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.79200000000000004,"5min":1.022,"15min":1.022},"last":0.88}]},{"osd":6,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.68400000000000005,"15min":0.69899999999999995},"min":{"1min":0.37,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80300000000000005,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.66300000000000003,"15min":0.67300000000000004},"min":{"1min":0.42899999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.85599999999999998,"5min":1.044,"15min":1.044},"last":0.92800000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.68999999999999995,"15min":0.70299999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.89600000000000002,"5min":1.052,"15min":1.052},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.621,"5min":0.71299999999999997,"15min":0.72799999999999998},"min":{"1min":0.38500000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77500000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.84299999999999997}]}]},{"osd":2,"up_from":98,"seq":420906795033,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":209312,"kb_used_data":173720,"kb_used_omap":11,"kb_used_meta":35572,"kb_avail":93535840,"statfs":{"total":95995035648,"available":95780700160,"internally_reserved":0,"allocated":177889280,"data_stored":177369000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":36426325},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.313},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.23200000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.27300000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:32:32 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.245},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.45400000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.48999999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.35299999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.46400000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:32:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.439}]}]},{"osd":3,"up_from":103,"seq":442381631507,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":149328,"kb_used_data":113800,"kb_used_omap":4,"kb_used_meta":35515,"kb_avail":93595824,"statfs":{"total":95995035648,"available":95842123776,"internally_reserved":0,"allocated":116531200,"data_stored":116024902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":36367966},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.42599999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.33500000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.95099999999999996},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.39600000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":1.8520000000000001},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.97899999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":1.6539999999999999},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":1.206}]},{"osd":6,"last update":"Mon Oct 9 05:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":1.2549999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.52800000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.83799999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019055,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":918480,"kb_used_data":621256,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92826672,"statfs":{"total":95995035648,"available":95054512128,"internally_reserved":0,"allocated":636166144,"data_stored":635666951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52200000000000002,"15min":0.52100000000000002},"min":{"1min":0.373,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73299999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.28399999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.43099999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.68100000000000005,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.56299999999999994,"15min":0.56000000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.76300000000000001,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.53400000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:32:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.376}]},{"osd":3,"last update":"Mon Oct 9 05:32:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.42799999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:32:31 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.47999999999999998,"15min":0.44800000000000001},"min":{"1min":0.39100000000000001,"5min":0.32300000000000001,"15min":0.26700000000000002},"max":{"1min":0.621,"5min":0.80300000000000005,"15min":0.85399999999999998},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.56699999999999995,"15min":0.53800000000000003},"min":{"1min":0.42799999999999999,"5min":0.36599999999999999,"15min":0.311},"max":{"1min":0.746,"5min":0.77900000000000003,"15min":1.54},"last":0.40500000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:32:42 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.504,"15min":0.49399999999999999},"min":{"1min":0.33800000000000002,"5min":0.30599999999999999,"15min":0.27800000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":1.5660000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.51100000000000001,"15min":0.50600000000000001},"min":{"1min":0.34899999999999998,"5min":0.33100000000000002,"15min":0.28899999999999998},"max":{"1min":0.72799999999999998,"5min":0.752,"15min":0.94699999999999995},"last":0.45400000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:32:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53500000000000003,"15min":0.53600000000000003},"min":{"1min":0.433,"5min":0.35499999999999998,"15min":0.30499999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":1.6160000000000001},"last":0.5},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53300000000000003,"15min":0.53000000000000003},"min":{"1min":0.46700000000000003,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":0.76700000000000002,"5min":0.79000000000000004,"15min":1.034},"last":0.41899999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822827,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1014464,"kb_used_data":713144,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92730688,"statfs":{"total":95995035648,"available":94956224512,"internally_reserved":0,"allocated":730259456,"data_stored":729772095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:32:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70499999999999996,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.39500000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.93700000000000006}]},{"osd":1,"last update":"Mon Oct 9 05:32:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.71999999999999997,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.56699999999999995,"15min":0.56100000000000005},"min":{"1min":0.45700000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71499999999999997,"5min":0.754,"15min":0.754},"last":0.53600000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:32:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.37}]},{"osd":3,"last update":"Mon Oct 9 05:32:32 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.433},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:32:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.49099999999999999,"15min":0.45900000000000002},"min":{"1min":0.38100000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":0.73999999999999999,"5min":0.874,"15min":0.94499999999999995},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.55000000000000004,"15min":0.501},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28699999999999998},"max":{"1min":0.749,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.32200000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:32:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.55900000000000005,"15min":0.51500000000000001},"min":{"1min":0.373,"5min":0.34999999999999998,"15min":0.26000000000000001},"max":{"1min":0.72699999999999998,"5min":0.93500000000000005,"15min":1.1339999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.56499999999999995,"15min":0.53700000000000003},"min":{"1min":0.40899999999999997,"5min":0.36199999999999999,"15min":0.314},"max":{"1min":0.754,"5min":0.92100000000000004,"15min":1.109},"last":0.49399999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:32:10 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.59399999999999997,"15min":0.57699999999999996},"min":{"1min":0.52400000000000002,"5min":0.40200000000000002,"15min":0.34399999999999997},"max":{"1min":0.71499999999999997,"5min":0.95199999999999996,"15min":1.143},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57399999999999995,"15min":0.55600000000000005},"min":{"1min":0.51100000000000001,"5min":0.35199999999999998,"15min":0.32400000000000001},"max":{"1min":0.83499999999999996,"5min":0.90900000000000003,"15min":1.1830000000000001},"last":0.39500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":485490688,"data_stored":485490688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":362741760,"data_stored":362741760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":227737600,"data_stored":227737600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":128647168,"data_stored":128647168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":658046976,"data_stored":658046976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":744161280,"data_stored":744161280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":329318400,"data_stored":329318400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":410845184,"data_stored":410845184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:33:02.690 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 16 2023-10-09T05:33:02.691 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:02.891 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:33:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:02.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:02.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:03.043 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:03.113 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:33:03.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:03.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:03.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:03.260 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:03.335 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:33:03.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:03.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:03.360 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:33:03.396 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:33:03.430 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:33:03.442 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:03.449 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:33:03.480 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 11 15 27311 27296 155.004 104.75 0.00839936 0.00644677 2023-10-09T05:33:03.484 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:03.486 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:33:03.523 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:33:03.563 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:33:03.568 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:03.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:03.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:03.603 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:33:03.606 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:33:03.606 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:33:03.607 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:33:03.607 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:33:03.607 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:33:03.607 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:33:03.607 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:33:03.607 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:33:03.676 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:03.707 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:03 smithi191 ceph-mon[161669]: pgmap v247: 89 pgs: 89 active+clean; 1.2 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T05:33:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:03 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2431065495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:03.882 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:03.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:03 smithi008 ceph-mon[184418]: pgmap v247: 89 pgs: 89 active+clean; 1.2 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T05:33:03.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:03 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2431065495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:03 smithi008 ceph-mon[191227]: pgmap v247: 89 pgs: 89 active+clean; 1.2 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T05:33:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:03 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2431065495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:04.480 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 12 16 28853 28837 150.114 96.3125 0.00491132 0.0066561 2023-10-09T05:33:04.786 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:33:04.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:04.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:04.892 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:04.941 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:05.481 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 13 16 30756 30740 147.716 118.938 0.00610064 0.00676531 2023-10-09T05:33:05.716 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:05.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:05.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:05.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:05 smithi191 ceph-mon[161669]: pgmap v248: 89 pgs: 89 active+clean; 1.3 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T05:33:05.833 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:05.887 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:05 smithi008 ceph-mon[184418]: pgmap v248: 89 pgs: 89 active+clean; 1.3 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T05:33:05.887 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:05 smithi008 ceph-mon[191227]: pgmap v248: 89 pgs: 89 active+clean; 1.3 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T05:33:05.899 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:06.023 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:06.155 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:06.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:06.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:06.260 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:06.277 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:33:06.277 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:33:06.311 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:06.481 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 14 16 32621 32605 145.49 116.562 0.00642079 0.00686975 2023-10-09T05:33:07.481 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 15 16 34559 34543 143.865 121.125 0.00928434 0.00694587 2023-10-09T05:33:07.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:07 smithi008 ceph-mon[184418]: pgmap v249: 89 pgs: 89 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-10-09T05:33:07.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:07 smithi008 ceph-mon[191227]: pgmap v249: 89 pgs: 89 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-10-09T05:33:07.761 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:33:07.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:07 smithi191 ceph-mon[161669]: pgmap v249: 89 pgs: 89 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-10-09T05:33:08.303 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:08.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:08.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:08.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:08.458 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:08.481 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 16 16 36355 36339 141.889 112.25 0.00859164 0.00704292 2023-10-09T05:33:08.496 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:33:08.544 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:33:08.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:08.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:08.656 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:08.695 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:08.784 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:33:08.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:08.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:08.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:08.949 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:09.019 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:09.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:09.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:09.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:09.166 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:09.240 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:09.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:09.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:09.347 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:09.412 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:09.481 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 17 16 38038 38022 139.73 105.188 0.00724497 0.00715046 2023-10-09T05:33:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:09 smithi008 ceph-mon[184418]: pgmap v250: 89 pgs: 89 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T05:33:09.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:09 smithi008 ceph-mon[191227]: pgmap v250: 89 pgs: 89 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T05:33:09.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:09 smithi191 ceph-mon[161669]: pgmap v250: 89 pgs: 89 active+clean; 1.9 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T05:33:10.205 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:33:10.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:10.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:10.309 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:10.342 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:10.348 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:10.408 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:33:10.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:10.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:10.481 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 18 16 39767 39751 137.971 108.062 0.0127212 0.00724322 2023-10-09T05:33:10.516 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:10.578 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:33:10.616 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:10.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:10 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3810704177' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:33:10.825 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:10 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3810704177' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:33:10.825 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:10 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3810704177' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:33:10.826 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:33:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:33:10] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.33.4" 2023-10-09T05:33:10.899 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:11.086 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:11.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:11.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:11.194 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:11.247 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:11.335 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:11.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:11.442 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:11.482 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 19 16 41456 41440 136.259 105.562 0.00927284 0.00733407 2023-10-09T05:33:11.510 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:11.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[184418]: pgmap v251: 89 pgs: 89 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-10-09T05:33:11.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:11.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[184418]: failed to encode map e110 with expected crc 2023-10-09T05:33:11.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[184418]: failed to encode map e110 with expected crc 2023-10-09T05:33:11.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3810704177' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:33:11.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[184418]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T05:33:11.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[184418]: failed to encode map e110 with expected crc 2023-10-09T05:33:11.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[184418]: failed to encode map e110 with expected crc 2023-10-09T05:33:11.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[184418]: failed to encode map e110 with expected crc 2023-10-09T05:33:11.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[191227]: pgmap v251: 89 pgs: 89 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-10-09T05:33:11.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:11.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[191227]: failed to encode map e110 with expected crc 2023-10-09T05:33:11.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[191227]: failed to encode map e110 with expected crc 2023-10-09T05:33:11.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3810704177' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:33:11.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[191227]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T05:33:11.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[191227]: failed to encode map e110 with expected crc 2023-10-09T05:33:11.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[191227]: failed to encode map e110 with expected crc 2023-10-09T05:33:11.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:11 smithi008 ceph-mon[191227]: failed to encode map e110 with expected crc 2023-10-09T05:33:12.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:11 smithi191 ceph-mon[161669]: pgmap v251: 89 pgs: 89 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-10-09T05:33:12.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:11 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:12.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:11 smithi191 ceph-mon[161669]: failed to encode map e110 with expected crc 2023-10-09T05:33:12.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:11 smithi191 ceph-mon[161669]: failed to encode map e110 with expected crc 2023-10-09T05:33:12.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3810704177' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:33:12.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:11 smithi191 ceph-mon[161669]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T05:33:12.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:11 smithi191 ceph-mon[161669]: failed to encode map e110 with expected crc 2023-10-09T05:33:12.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:11 smithi191 ceph-mon[161669]: failed to encode map e110 with expected crc 2023-10-09T05:33:12.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:11 smithi191 ceph-mon[161669]: failed to encode map e110 with expected crc 2023-10-09T05:33:12.483 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:33:12.483259+0000 min lat: 0.00107405 max lat: 0.175517 avg lat: 0.00744165 2023-10-09T05:33:12.484 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:33:12.484 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 20 16 43005 42989 134.287 96.8125 0.0130427 0.00744165 2023-10-09T05:33:12.506 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:12.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:12.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:12.586 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/417518106' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:33:12.586 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:12 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:33:12.586 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:12 smithi008 ceph-mon[184418]: pgmap v253: 89 pgs: 89 active+clean; 2.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T05:33:12.587 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:12 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:12.587 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:12 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:12.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/417518106' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:33:12.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:12 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:33:12.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:12 smithi008 ceph-mon[191227]: pgmap v253: 89 pgs: 89 active+clean; 2.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T05:33:12.587 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:12 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:12.588 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:12 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:12.613 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 5 pgp_num to 16 2023-10-09T05:33:12.615 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:12.703 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:12.889 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:12.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:12.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:12.994 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:13.050 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/417518106' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:33:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:12 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:33:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:12 smithi191 ceph-mon[161669]: pgmap v253: 89 pgs: 89 active+clean; 2.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T05:33:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:12 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:12 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:13.482 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 21 16 44326 44310 131.824 82.5625 0.00148788 0.00754447 2023-10-09T05:33:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[184418]: failed to encode map e111 with expected crc 2023-10-09T05:33:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:33:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[184418]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T05:33:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[184418]: failed to encode map e111 with expected crc 2023-10-09T05:33:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:33:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:33:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:13.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[191227]: failed to encode map e111 with expected crc 2023-10-09T05:33:13.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:33:13.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[191227]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T05:33:13.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[191227]: failed to encode map e111 with expected crc 2023-10-09T05:33:13.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:33:13.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:33:13.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:13 smithi191 ceph-mon[161669]: failed to encode map e111 with expected crc 2023-10-09T05:33:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:13 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:33:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:13 smithi191 ceph-mon[161669]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T05:33:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:13 smithi191 ceph-mon[161669]: failed to encode map e111 with expected crc 2023-10-09T05:33:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:33:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:33:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:14.483 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 22 16 45796 45780 130.008 91.875 0.00918233 0.00768618 2023-10-09T05:33:14.501 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:14.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:14.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:14.590 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:33:14.605 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:14.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:14.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:14 smithi008 ceph-mon[191227]: pgmap v255: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:33:14.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:14.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:14 smithi008 ceph-mon[184418]: pgmap v255: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:33:14.699 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:14 smithi191 ceph-mon[161669]: pgmap v255: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:33:15.354 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:33:15.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:15.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:15.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:15.483 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 23 16 47243 47227 128.288 90.4375 0.00770687 0.00778981 2023-10-09T05:33:15.513 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:15.581 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:33:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:15.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:15.687 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:15.729 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:15.829 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:33:15.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:15.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:15.939 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:15.947 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:33:15.947 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:33:16.011 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:16.482 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:33:16.482 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:33:16.482 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:33:16.482 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:33:16.482 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:33:16.482 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:33:16.482 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:33:16.482 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:33:16.483 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:33:16.483 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:33:16.483 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:33:16.483 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:33:16.483 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:33:16.483 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:33:16.483 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:33:16.484 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:33:16.484 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 24 16 48795 48779 126.984 97 0.0105762 0.00786975 2023-10-09T05:33:16.522 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:16.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:16.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:16.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:16.695 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:16.792 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:33:16.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:16.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:16.898 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:16.926 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:17.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:16 smithi191 ceph-mon[161669]: pgmap v256: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T05:33:17.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:16 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4157076340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:16 smithi008 ceph-mon[184418]: pgmap v256: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T05:33:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:16 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4157076340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:17.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:16 smithi008 ceph-mon[191227]: pgmap v256: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T05:33:17.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:16 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4157076340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:17.325 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:17.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:17.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:17.437 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:17.483 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:17.483 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 25 16 50211 50195 125.445 88.5 0.00889782 0.00796358 2023-10-09T05:33:17.716 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:17.756 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:33:17.852 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:17.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:17.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:17.963 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:18.039 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:18.227 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:18.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:18.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:18.333 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:18.375 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:18.460 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:18.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:18.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:18.483 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 26 16 51443 51427 123.582 77 0.0063672 0.00808616 2023-10-09T05:33:18.571 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:18.621 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:18.735 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:18.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:18.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:18.843 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:18.910 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:18.951 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:18.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:19.059 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:19.089 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:19.134 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:33:19.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:19.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:19.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:33:19.291 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:19.385 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:33:19.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:19.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:33:19.483 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 27 16 52664 52648 121.831 76.3125 0.00916989 0.00820348 2023-10-09T05:33:19.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:19.494 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:33:19.494 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:33:19.495 INFO:tasks.thrashosds.thrasher:Killing osd 7, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:33:19.495 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T05:33:19.495 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T05:33:19.546 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:19 smithi191 ceph-mon[161669]: pgmap v257: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T05:33:19.553 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:19.662 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:19.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:19.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:19.678 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:19.775 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:19.799 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:33:19 smithi191 systemd[1]: Stopping Ceph osd.7 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:33:19.843 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:19.862 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:19 smithi008 ceph-mon[191227]: pgmap v257: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T05:33:19.863 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:19 smithi008 ceph-mon[184418]: pgmap v257: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T05:33:19.960 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:19.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:19.997 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:33:20.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:20.114 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:33:20 smithi191 conmon[125089]: debug 2023-10-09T05:33:20.023+0000 7fee8b092700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T05:33:20.114 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:33:20 smithi191 conmon[125089]: debug 2023-10-09T05:33:20.023+0000 7fee8b092700 -1 osd.7 111 *** Got signal Terminated *** 2023-10-09T05:33:20.114 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:33:20 smithi191 conmon[125089]: debug 2023-10-09T05:33:20.023+0000 7fee8b092700 -1 osd.7 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:33:20.118 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:20.190 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:20.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:20.295 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:20.396 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:20.483 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 28 16 53992 53976 120.444 83 0.0150141 0.00829768 2023-10-09T05:33:20.673 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:20.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:20.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:20.776 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:20.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:20 smithi191 ceph-mon[161669]: osd.7 marked itself down and dead 2023-10-09T05:33:20.800 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:20 smithi008 ceph-mon[184418]: osd.7 marked itself down and dead 2023-10-09T05:33:20.801 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:33:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:33:20] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.33.4" 2023-10-09T05:33:20.801 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:20 smithi008 ceph-mon[191227]: osd.7 marked itself down and dead 2023-10-09T05:33:20.804 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:20.848 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:20.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:20.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:20.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:20.978 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:21.020 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:21.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:21.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:21.124 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:21.154 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:21.452 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:21.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:21.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:21.483 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 29 16 54082 54066 116.486 5.625 0.00427314 0.00830625 2023-10-09T05:33:21.546 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:33:21.556 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:21.602 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:21.689 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:21.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:21.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:21.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[184418]: pgmap v258: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-10-09T05:33:21.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[184418]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:33:21.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[184418]: failed to encode map e112 with expected crc 2023-10-09T05:33:21.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[184418]: osdmap e112: 8 total, 7 up, 8 in 2023-10-09T05:33:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/934971622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:21.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[191227]: pgmap v258: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-10-09T05:33:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[191227]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:33:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[191227]: failed to encode map e112 with expected crc 2023-10-09T05:33:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[191227]: osdmap e112: 8 total, 7 up, 8 in 2023-10-09T05:33:21.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/934971622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:21.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:21 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:21.793 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:21 smithi191 ceph-mon[161669]: pgmap v258: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-10-09T05:33:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:21 smithi191 ceph-mon[161669]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:33:21.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:21 smithi191 ceph-mon[161669]: failed to encode map e112 with expected crc 2023-10-09T05:33:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:21 smithi191 ceph-mon[161669]: osdmap e112: 8 total, 7 up, 8 in 2023-10-09T05:33:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:21 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/934971622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:21 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:21.851 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:22.246 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:22.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:22.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:22.352 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:22.409 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:22.419 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:22.483 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 30 16 55314 55298 115.17 77 0.0116469 0.00867807 2023-10-09T05:33:22.522 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:33:22.522 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[184418]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.522 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[184418]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.522 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:33:22.523 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[184418]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.523 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[184418]: osdmap e113: 8 total, 7 up, 8 in 2023-10-09T05:33:22.523 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[184418]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.523 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[184418]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.523 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[184418]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.523 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[184418]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.563 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:22.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:22.676 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:22.733 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:22.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:22 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:33:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:22 smithi191 ceph-mon[161669]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:22 smithi191 ceph-mon[161669]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:22 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:33:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:22 smithi191 ceph-mon[161669]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:22 smithi191 ceph-mon[161669]: osdmap e113: 8 total, 7 up, 8 in 2023-10-09T05:33:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:22 smithi191 ceph-mon[161669]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:22 smithi191 ceph-mon[161669]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:22 smithi191 ceph-mon[161669]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:22 smithi191 ceph-mon[161669]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.930 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:33:22.930 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[191227]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.930 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[191227]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.930 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:33:22.930 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[191227]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.930 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[191227]: osdmap e113: 8 total, 7 up, 8 in 2023-10-09T05:33:22.931 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[191227]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.931 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[191227]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.931 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[191227]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.931 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:22 smithi008 ceph-mon[191227]: failed to encode map e113 with expected crc 2023-10-09T05:33:22.934 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:22.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:22.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:22.955 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:33:22.955 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:33:22.955 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:33:22.955 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:33:22.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:10991 5.5 e108/105 5:a03af33e:::benchmark_data_smithi008_238708_object10990:head v 108'691, mlcod=108'689)", 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:56.049176+0000", 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 26.903624131000001, 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.144458221, 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049176+0000", 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.956 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049176+0000", 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049175+0000", 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.957 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049199+0000", 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4014000000000001e-05 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049209+0000", 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9510000000000001e-06 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049218+0000", 2023-10-09T05:33:22.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1900000000000001e-06 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050199+0000", 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00098079999999999999 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050217+0000", 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8536e-05 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.959 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.051392+0000", 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001174598 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193634+0000", 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14224188300000001 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.960 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:11003 5.8 e108/105 5:1d0863ca:::benchmark_data_smithi008_238708_object11002:head v 108'694, mlcod=108'692)", 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:56.050613+0000", 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 26.902186886999999, 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14495682600000001, 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050613+0000", 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.961 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050613+0000", 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050612+0000", 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:22.962 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050637+0000", 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5170000000000001e-05 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050649+0000", 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1622e-05 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050656+0000", 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2540000000000003e-06 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.963 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050720+0000", 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3874999999999994e-05 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050740+0000", 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9783e-05 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:22.964 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195541+0000", 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14480122200000001 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195570+0000", 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8819999999999999e-05 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:11001 5.7 e108/105 5:ea85ce96:::benchmark_data_smithi008_238708_object11000:head v 108'703, mlcod=108'701)", 2023-10-09T05:33:22.965 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:56.050731+0000", 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 26.902068728, 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14540388300000001, 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050731+0000", 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:22.966 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050731+0000", 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050730+0000", 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.053271+0000", 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025400689999999998 2023-10-09T05:33:22.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193313+0000", 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14004271099999999 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193337+0000", 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3632e-05 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:22.968 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193495+0000", 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000157706 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193522+0000", 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7370999999999999e-05 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.196093+0000", 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025712450000000002 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.969 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.970 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:22.970 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.196135+0000", 2023-10-09T05:33:22.970 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1820999999999998e-05 2023-10-09T05:33:22.970 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.970 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:22.970 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.970 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.970 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.970 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:11006 5.8 e108/105 5:1ea71daf:::benchmark_data_smithi008_238708_object11005:head v 108'695, mlcod=108'692)", 2023-10-09T05:33:22.970 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:56.051675+0000", 2023-10-09T05:33:22.970 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 26.901125079, 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14391491300000001, 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.051675+0000", 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.051675+0000", 2023-10-09T05:33:22.971 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.051674+0000", 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.051699+0000", 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5198000000000002e-05 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.972 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193330+0000", 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14163066199999999 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193342+0000", 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1844999999999999e-05 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193894+0000", 2023-10-09T05:33:22.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00055221000000000003 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193928+0000", 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4434000000000003e-05 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195573+0000", 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016444949999999999 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195590+0000", 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6826000000000001e-05 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:10995 5.5 e108/105 5:aa6195bb:::benchmark_data_smithi008_238708_object10994:head v 108'692, mlcod=108'689)", 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:56.052562+0000", 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 26.900238336000001, 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14313236900000001, 2023-10-09T05:33:22.975 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.052562+0000", 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.052562+0000", 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.976 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.052561+0000", 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.052586+0000", 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5242e-05 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:22.977 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193387+0000", 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14080136900000001 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193399+0000", 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1807999999999999e-05 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193668+0000", 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00026856100000000001 2023-10-09T05:33:22.978 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193691+0000", 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.304e-05 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195657+0000", 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001965853 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:22.979 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195694+0000", 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7230000000000001e-05 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:18583 5.c 5:32e34d9e:::benchmark_data_smithi008_238708_object18582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:58.959866+0000", 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 23.992933894, 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.110356446, 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:22.980 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 18583 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.959866+0000", 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:22.981 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.959866+0000", 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.959865+0000", 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.959889+0000", 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4847000000000001e-05 2023-10-09T05:33:22.982 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.959902+0000", 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2118e-05 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.959909+0000", 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5619999999999998e-06 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:22.983 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.959970+0000", 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0713e-05 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.960097+0000", 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012743300000000001 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.960257+0000", 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000160156 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.961240+0000", 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0009828599999999999 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.962451+0000", 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00121123 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:22.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.962816+0000", 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00036480900000000001 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.070175+0000", 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.10735863599999999 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.070222+0000", 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7487999999999997e-05 2023-10-09T05:33:22.986 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:18590 5.8 e109/105 5:12b7ea09:::benchmark_data_smithi008_238708_object18589:head v 109'1217, mlcod=109'1215)", 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:58.963726+0000", 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 23.989073579999999, 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.109188814, 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.987 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.963726+0000", 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.963726+0000", 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.963726+0000", 2023-10-09T05:33:22.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.963746+0000", 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.054e-05 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.070147+0000", 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.106400921 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.070171+0000", 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3998e-05 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.070223+0000", 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2408999999999999e-05 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.070248+0000", 2023-10-09T05:33:22.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4890999999999999e-05 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.072876+0000", 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026278080000000001 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.072915+0000", 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8942000000000001e-05 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:22.991 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:18594 5.c 5:34636d0a:::benchmark_data_smithi008_238708_object18593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:58.964283+0000", 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 23.988517355999999, 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.110739173, 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:33:22.992 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 18594 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.964283+0000", 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.964283+0000", 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.993 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.964281+0000", 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.964303+0000", 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2141999999999999e-05 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:22.994 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.070182+0000", 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.105878369 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.070195+0000", 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3203e-05 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.070258+0000", 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2781999999999996e-05 2023-10-09T05:33:22.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.070394+0000", 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013654100000000001 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.070522+0000", 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012815999999999999 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:22.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.073077+0000", 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025551409999999999 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.073146+0000", 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8758999999999997e-05 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.074920+0000", 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001773602 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.074983+0000", 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3366000000000003e-05 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.075022+0000", 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8588999999999999e-05 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:22.998 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:19142 5.7 e109/105 5:ea7cadf4:::benchmark_data_smithi008_238708_object19141:head v 109'1174, mlcod=109'1172)", 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:59.248842+0000", 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 23.703958136000001, 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13454352999999999, 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248842+0000", 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:22.999 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248842+0000", 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248841+0000", 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.000 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248871+0000", 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0241000000000001e-05 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248881+0000", 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0115999999999999e-05 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248888+0000", 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7329999999999997e-06 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.001 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248947+0000", 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8977000000000001e-05 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248966+0000", 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8414e-05 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.250286+0000", 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013209739999999999 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383385+0000", 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.133098778 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:19144 5.0 e109/105 5:0930f39a:::benchmark_data_smithi008_238708_object19143:head v 109'1179, mlcod=109'1176)", 2023-10-09T05:33:23.003 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:59.249102+0000", 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 23.703697473999998, 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13596629299999999, 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249102+0000", 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.004 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249102+0000", 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249102+0000", 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249127+0000", 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5384e-05 2023-10-09T05:33:23.005 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249137+0000", 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0257e-05 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249144+0000", 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6549999999999998e-06 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.006 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249203+0000", 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8727000000000002e-05 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249220+0000", 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7802e-05 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.385031+0000", 2023-10-09T05:33:23.007 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.135810706 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.385069+0000", 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7483e-05 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:19149 5.8 e109/105 5:1908f647:::benchmark_data_smithi008_238708_object19148:head v 109'1252, mlcod=109'1249)", 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:59.249953+0000", 2023-10-09T05:33:23.008 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 23.702847002999999, 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13514635799999999, 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249953+0000", 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.009 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249953+0000", 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249952+0000", 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249978+0000", 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5910000000000001e-05 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.010 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249997+0000", 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8723e-05 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.250004+0000", 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3259999999999998e-06 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.250067+0000", 2023-10-09T05:33:23.011 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2829000000000004e-05 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.250087+0000", 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9823999999999999e-05 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.385073+0000", 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13498581800000001 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.012 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.385099+0000", 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6584000000000002e-05 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:19152 5.7 e109/105 5:e977ac15:::benchmark_data_smithi008_238708_object19151:head v 109'1175, mlcod=109'1172)", 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:59.251006+0000", 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 23.701793656, 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13563842400000001, 2023-10-09T05:33:23.013 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251006+0000", 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251006+0000", 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.014 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251006+0000", 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251069+0000", 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3309e-05 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383348+0000", 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.132279123 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383366+0000", 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7578999999999999e-05 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383422+0000", 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6631e-05 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383451+0000", 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8416e-05 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.386610+0000", 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003159111 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.386645+0000", 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4870000000000003e-05 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:19163 5.5 e109/105 5:ad4714f5:::benchmark_data_smithi008_238708_object19162:head v 109'1229, mlcod=109'1227)", 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:59.254054+0000", 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 23.698746259, 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13070844000000001, 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:23.018 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.254054+0000", 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.254054+0000", 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.019 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.254053+0000", 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.254079+0000", 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6111000000000001e-05 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383348+0000", 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.129269247 2023-10-09T05:33:23.020 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383366+0000", 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8306000000000001e-05 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383422+0000", 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5892e-05 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.021 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383451+0000", 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8478000000000001e-05 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.384718+0000", 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0012668569999999999 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.384762+0000", 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4478999999999999e-05 2023-10-09T05:33:23.022 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:25060 5.c 5:365c47a8:::benchmark_data_smithi008_238708_object25059:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:02.103412+0000", 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 20.849388009999998, 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.121611607, 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:33:23.023 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 25060 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.103412+0000", 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.103412+0000", 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.024 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.103411+0000", 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.103434+0000", 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3646999999999999e-05 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.025 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.103446+0000", 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1681e-05 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.103454+0000", 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0499999999999992e-06 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.103654+0000", 2023-10-09T05:33:23.026 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000200559 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.103732+0000", 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7947000000000004e-05 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.103830+0000", 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7931999999999997e-05 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.027 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.104670+0000", 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00083980199999999995 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.106104+0000", 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014338510000000001 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.224930+0000", 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.11882571 2023-10-09T05:33:23.028 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.224983+0000", 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2985999999999997e-05 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.225023+0000", 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0766999999999998e-05 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.029 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:25065 5.c 5:3231767d:::benchmark_data_smithi008_238708_object25064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:02.104716+0000", 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 20.848083583000001, 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.122009778, 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 25065 2023-10-09T05:33:23.030 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.104716+0000", 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.104716+0000", 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.031 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.104716+0000", 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.104735+0000", 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9279e-05 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.104744+0000", 2023-10-09T05:33:23.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6810000000000004e-06 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.104749+0000", 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1680000000000003e-06 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.104788+0000", 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8892e-05 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.104859+0000", 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1019000000000006e-05 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.104939+0000", 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0153000000000003e-05 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.106079+0000", 2023-10-09T05:33:23.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001139794 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.107584+0000", 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015054160000000001 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.226634+0000", 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.119049829 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.035 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.226685+0000", 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1439e-05 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.226726+0000", 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0812000000000002e-05 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.036 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:25066 5.c 5:3767c917:::benchmark_data_smithi008_238708_object25065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:02.105256+0000", 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 20.847543762000001, 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.121709805, 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 25066 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:23.037 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.105256+0000", 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.105256+0000", 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.038 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.105255+0000", 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.105274+0000", 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8542e-05 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.105281+0000", 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.221e-06 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.039 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.105286+0000", 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4039999999999997e-06 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.105429+0000", 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014380099999999999 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.105501+0000", 2023-10-09T05:33:23.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1651999999999998e-05 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.105579+0000", 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8325000000000006e-05 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.106191+0000", 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00061186900000000002 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.107676+0000", 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014843510000000001 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.226885+0000", 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.119209545 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.226931+0000", 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6295999999999999e-05 2023-10-09T05:33:23.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:02.226966+0000", 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4422999999999999e-05 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:49323 5.c 5:323e1adc:::benchmark_data_smithi008_238708_object49322:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:16.827970+0000", 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 6.1248295050000001, 2023-10-09T05:33:23.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.117739859, 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 49323 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.827970+0000", 2023-10-09T05:33:23.044 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.827970+0000", 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.827969+0000", 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.827994+0000", 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5561000000000001e-05 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.828008+0000", 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4047e-05 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.828017+0000", 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6440000000000006e-06 2023-10-09T05:33:23.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.828284+0000", 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00026761599999999998 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.828424+0000", 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000139479 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.828572+0000", 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014861 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.830446+0000", 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018737319999999999 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.837013+0000", 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0065663930000000002 2023-10-09T05:33:23.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.945613+0000", 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.108600453 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.945667+0000", 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3891000000000003e-05 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.049 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.945710+0000", 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3275000000000002e-05 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:49328 5.c 5:314d53fb:::benchmark_data_smithi008_238708_object49327:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:16.833291+0000", 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 6.1195085640000002, 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.11554600900000001, 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:23.050 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 49328 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833291+0000", 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.051 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833291+0000", 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833290+0000", 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833323+0000", 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3575999999999998e-05 2023-10-09T05:33:23.052 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833362+0000", 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8593999999999997e-05 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833393+0000", 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1044999999999998e-05 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.053 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833466+0000", 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3347000000000001e-05 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833648+0000", 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018139600000000001 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833802+0000", 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015411099999999999 2023-10-09T05:33:23.054 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.836967+0000", 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031651980000000001 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.839117+0000", 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021498229999999999 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.948743+0000", 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.109626101 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.948798+0000", 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5584000000000002e-05 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.948837+0000", 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8980000000000003e-05 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.056 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:49332 5.c 5:36fdd74c:::benchmark_data_smithi008_238708_object49331:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:16.833420+0000", 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 6.119380306, 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.11554342400000001, 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:33:23.057 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 49332 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833420+0000", 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833420+0000", 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.058 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833419+0000", 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833440+0000", 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1073e-05 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.059 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833449+0000", 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.619e-06 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.833455+0000", 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6050000000000003e-06 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.835497+0000", 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020416509999999998 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.060 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.835613+0000", 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000115903 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.835715+0000", 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010231800000000001 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:33:23.061 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.837186+0000", 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014708379999999999 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.839858+0000", 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026719560000000001 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.948877+0000", 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.109019423 2023-10-09T05:33:23.062 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.948926+0000", 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9157999999999999e-05 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:16.948963+0000", 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6501999999999997e-05 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.063 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:50205 5.c 5:3212117c:::benchmark_data_smithi008_238708_object50204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:17.469143+0000", 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 5.4836570629999999, 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.116312579, 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 50205 2023-10-09T05:33:23.064 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.469143+0000", 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.469143+0000", 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.065 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.469142+0000", 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.469192+0000", 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0794000000000002e-05 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.469205+0000", 2023-10-09T05:33:23.066 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2999e-05 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.469213+0000", 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.994e-06 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.469258+0000", 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5071999999999998e-05 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.067 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.469400+0000", 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000141815 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.469537+0000", 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000136429 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.471687+0000", 2023-10-09T05:33:23.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021504390000000001 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.471884+0000", 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00019640099999999999 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.585356+0000", 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.113472894 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.585411+0000", 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4129000000000003e-05 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:17.585455+0000", 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4808999999999997e-05 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:33:23.070 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:33:23.071 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:33:23.071 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:23.128 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:23.483 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 31 16 56819 56803 114.489 94.0625 0.011681 0.00873022 2023-10-09T05:33:23.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:23 smithi191 ceph-mon[161669]: pgmap v261: 89 pgs: 10 peering, 9 stale+active+clean, 70 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:33:23.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:23 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:33:23.844 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:33:23.849 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:23 smithi008 ceph-mon[184418]: pgmap v261: 89 pgs: 10 peering, 9 stale+active+clean, 70 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:33:23.849 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:23 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:33:23.849 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:23 smithi008 ceph-mon[191227]: pgmap v261: 89 pgs: 10 peering, 9 stale+active+clean, 70 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:33:23.849 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:23 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:33:23.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:23.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:23.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:23.989 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:24.066 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:24.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:24.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:24.171 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:24.239 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:24.343 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:33:24.483 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 32 16 58202 58186 113.612 86.4375 0.0025851 0.00878417 2023-10-09T05:33:24.576 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:33:24 smithi191 bash[172893]: d71410a3a7bd515fc201b72dc830802901ef58f4837d6148c34792e277773c03 2023-10-09T05:33:24.577 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:24 smithi008 ceph-mon[191227]: pgmap v262: 89 pgs: 2 active+undersized+degraded, 10 peering, 6 stale+active+clean, 7 active+undersized, 64 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 3/162174 objects degraded (0.002%) 2023-10-09T05:33:24.577 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:24 smithi008 ceph-mon[191227]: Health check failed: Degraded data redundancy: 3/162174 objects degraded (0.002%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:24.857 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:24.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:24.867 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:24 smithi008 ceph-mon[184418]: pgmap v262: 89 pgs: 2 active+undersized+degraded, 10 peering, 6 stale+active+clean, 7 active+undersized, 64 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 3/162174 objects degraded (0.002%) 2023-10-09T05:33:24.867 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:24 smithi008 ceph-mon[184418]: Health check failed: Degraded data redundancy: 3/162174 objects degraded (0.002%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:24.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:24.968 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:24.972 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:33:25.028 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:24 smithi191 ceph-mon[161669]: pgmap v262: 89 pgs: 2 active+undersized+degraded, 10 peering, 6 stale+active+clean, 7 active+undersized, 64 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 3/162174 objects degraded (0.002%) 2023-10-09T05:33:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:24 smithi191 ceph-mon[161669]: Health check failed: Degraded data redundancy: 3/162174 objects degraded (0.002%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:25.173 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:33:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:25.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:25.282 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:25.373 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:25.484 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 33 16 59476 59460 112.581 79.625 0.00664927 0.00887656 2023-10-09T05:33:25.631 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:25.643 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:25.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:25.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:25.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:25.802 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:25.814 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:33:25.852 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:25.857 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:33:25.886 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:25.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:25.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:25.897 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:33:25.940 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:33:25.983 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:33:25.994 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:26.023 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:33:26.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:25 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/528725093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:26.061 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:33:26.064 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:33:26.064 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:33:26.064 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:33:26.065 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:33:26.065 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:33:26.065 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:33:26.065 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:33:26.065 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T05:33:26.070 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:26.074 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:25 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/528725093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:26.074 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:25 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/528725093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:26.360 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:33:26.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:26.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:26.465 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:26.484 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 34 16 60546 60530 111.237 66.875 0.00889578 0.00898433 2023-10-09T05:33:26.523 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:26.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T05:33:26.762 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:33:26.762 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T05:33:26.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:26 smithi008 ceph-mon[184418]: pgmap v263: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 18463/175764 objects degraded (10.504%) 2023-10-09T05:33:26.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:26 smithi008 ceph-mon[191227]: pgmap v263: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 18463/175764 objects degraded (10.504%) 2023-10-09T05:33:27.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:26 smithi191 ceph-mon[161669]: pgmap v263: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 18463/175764 objects degraded (10.504%) 2023-10-09T05:33:27.489 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 35 16 61823 61807 110.323 79.8125 0.00760801 0.00905952 2023-10-09T05:33:27.573 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T05:33:27.574 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T05:33:27.836 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:27 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:33:28.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:27 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:33:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:27 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:33:28.489 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 36 16 63125 63109 109.519 81.375 0.00580881 0.00912645 2023-10-09T05:33:28.910 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:28.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:28.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:29.027 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:29.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:28 smithi008 ceph-mon[184418]: pgmap v264: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 18463/178242 objects degraded (10.358%) 2023-10-09T05:33:29.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:28 smithi008 ceph-mon[191227]: pgmap v264: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 18463/178242 objects degraded (10.358%) 2023-10-09T05:33:29.113 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:29.279 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:29.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:29.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:28 smithi191 ceph-mon[161669]: pgmap v264: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 18463/178242 objects degraded (10.358%) 2023-10-09T05:33:29.390 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:29.458 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:29.489 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 37 15 64462 64447 108.819 83.625 0.00926872 0.00918515 2023-10-09T05:33:29.557 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:29.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:29.575 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:33:29.669 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:29.821 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:29.960 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:29.974 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:29.974 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:29.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:29.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:30.027 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:33:30.027 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:33:30.076 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:30.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:30.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:30.115 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:30.195 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:30.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:30.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:30.301 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:30.370 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:30.470 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:30.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:30.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:30.493 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 38 16 65726 65710 108.027 78.9375 0.0102625 0.00925277 2023-10-09T05:33:30.580 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:30.617 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:30.698 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:30.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:30.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:30.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:33:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:33:30] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.33.4" 2023-10-09T05:33:30.808 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:30.857 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:30.906 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:33:30.948 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:33:30.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:30.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:31.055 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:31.108 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:31.205 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:31.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:30 smithi008 ceph-mon[184418]: pgmap v265: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 19197/186195 objects degraded (10.310%) 2023-10-09T05:33:31.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/875915733' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:33:31.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:30 smithi008 ceph-mon[191227]: pgmap v265: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 19197/186195 objects degraded (10.310%) 2023-10-09T05:33:31.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/875915733' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:33:31.215 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:30 smithi191 ceph-mon[161669]: pgmap v265: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 19197/186195 objects degraded (10.310%) 2023-10-09T05:33:31.216 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/875915733' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:33:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:31.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:31.317 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:31.391 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:31.465 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:33:31.491 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 39 16 67155 67139 107.547 89.3125 0.00467994 0.00929308 2023-10-09T05:33:31.499 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:31.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:31.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:31.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:31.648 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:31.717 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:31.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:31.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:31.763 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2023-10-09T05:33:31.763 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:33:31.764 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T05:33:31.764 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:33:31.764 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:33:31.764 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T05:33:31.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:31.865 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:31.930 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:31.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:31 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19197/186195 objects degraded (10.310%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:31.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:31 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:31.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:31 smithi191 ceph-mon[161669]: failed to encode map e114 with expected crc 2023-10-09T05:33:31.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:31 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/875915733' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:33:31.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:31 smithi191 ceph-mon[161669]: osdmap e114: 8 total, 7 up, 8 in 2023-10-09T05:33:31.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:31.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:32.038 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:32.129 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:31 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19197/186195 objects degraded (10.310%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:32.129 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:31 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:32.129 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:31 smithi008 ceph-mon[184418]: failed to encode map e114 with expected crc 2023-10-09T05:33:32.129 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:31 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/875915733' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:33:32.129 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:31 smithi008 ceph-mon[184418]: osdmap e114: 8 total, 7 up, 8 in 2023-10-09T05:33:32.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:31 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19197/186195 objects degraded (10.310%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:32.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:31 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:32.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:31 smithi008 ceph-mon[191227]: failed to encode map e114 with expected crc 2023-10-09T05:33:32.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:31 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/875915733' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:33:32.130 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:31 smithi008 ceph-mon[191227]: osdmap e114: 8 total, 7 up, 8 in 2023-10-09T05:33:32.137 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:32.158 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:32.323 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:33:32.324 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:32.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:32.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:32.438 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:32.492 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:33:32.492713+0000 min lat: 0.00107405 max lat: 1.03525 avg lat: 0.0093751 2023-10-09T05:33:32.492 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:33:32.492 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 40 15 68273 68258 106.607 69.9375 0.0171626 0.0093751 2023-10-09T05:33:32.500 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:32.791 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:32.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:32.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:32.895 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:32.966 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:32.984 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:33.492 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 41 16 69599 69583 106.026 82.8125 0.0123451 0.00942714 2023-10-09T05:33:33.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:33 smithi008 ceph-mon[184418]: pgmap v267: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 20443/194631 objects degraded (10.503%) 2023-10-09T05:33:33.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:33 smithi008 ceph-mon[191227]: pgmap v267: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 20443/194631 objects degraded (10.503%) 2023-10-09T05:33:33.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:33 smithi191 ceph-mon[161669]: pgmap v267: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 20443/194631 objects degraded (10.503%) 2023-10-09T05:33:33.894 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:33.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:33.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:33.999 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:34.047 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:34.129 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:34.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:34.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:34.177 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:33:34.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:34.276 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:34.339 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:34.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:34.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:34.446 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:34.473 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:34.492 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 42 16 70672 70656 105.099 67.0625 0.004153 0.00949763 2023-10-09T05:33:34.885 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:34.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:34.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:34.956 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:34 smithi008 ceph-mon[184418]: pgmap v268: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 20443/198183 objects degraded (10.315%) 2023-10-09T05:33:34.956 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:34 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:34.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:34 smithi008 ceph-mon[191227]: pgmap v268: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 20443/198183 objects degraded (10.315%) 2023-10-09T05:33:34.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:34 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:34.989 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:35.054 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:35.117 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:35.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:35.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:35.224 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:35.296 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:34 smithi191 ceph-mon[161669]: pgmap v268: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 20443/198183 objects degraded (10.315%) 2023-10-09T05:33:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:34 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:35.493 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 43 16 71780 71764 104.265 69.25 0.0138653 0.00958632 2023-10-09T05:33:35.659 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:35.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:35.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:35.767 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:35.807 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:35.807 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:33:35.843 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:33:35.870 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:35.876 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:33:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:35.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:35.918 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:33:35.961 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:33:35.977 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:36.004 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:33:36.045 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:33:36.045 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:33:36.045 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:33:36.045 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:33:36.046 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:33:36.046 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:33:36.046 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:33:36.046 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:33:36.046 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T05:33:36.059 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:36.171 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:33:36.171 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:33:36.171 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:33:36.171 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:33:36.171 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:33:36.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/767549398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/767549398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:36.288 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:36.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:36.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/767549398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:36.395 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:36.473 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:36.493 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 44 16 72953 72937 103.558 73.3125 0.00765876 0.00965082 2023-10-09T05:33:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:36 smithi008 ceph-mon[184418]: pgmap v269: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 22356/212946 objects degraded (10.498%) 2023-10-09T05:33:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:36 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 22356/212946 objects degraded (10.498%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:37.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:36 smithi008 ceph-mon[191227]: pgmap v269: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 22356/212946 objects degraded (10.498%) 2023-10-09T05:33:37.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:36 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 22356/212946 objects degraded (10.498%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:37.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:36 smithi191 ceph-mon[161669]: pgmap v269: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 22356/212946 objects degraded (10.498%) 2023-10-09T05:33:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:36 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 22356/212946 objects degraded (10.498%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:37.494 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 45 16 74100 74084 102.85 71.6875 0.0292818 0.00971151 2023-10-09T05:33:38.267 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T05:33:38.267 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T05:33:38.376 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:33:38.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:38.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:38.456 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:38.483 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:38.494 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 46 16 75260 75244 102.19 72.5 0.0108501 0.00978137 2023-10-09T05:33:38.500 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:33:38.577 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:38.846 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:33:38.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:38.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:38.958 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:38.998 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:39.041 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:33:39.041 INFO:teuthology.orchestra.run.smithi008.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:33:39.042 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:33:39.064 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:39.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:39.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:39.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:39.219 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:39.300 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:39.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:39.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:39.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:39.439 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:39.495 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 47 16 76388 76372 101.513 70.5 0.0108758 0.00984586 2023-10-09T05:33:39.508 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:39.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:39.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:39.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:39.642 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:39 smithi191 ceph-mon[161669]: pgmap v270: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 22356/215244 objects degraded (10.386%) 2023-10-09T05:33:39.644 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:39.686 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:39.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:39.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:39.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:39.829 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:39.888 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:39.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:39.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:39 smithi008 ceph-mon[184418]: pgmap v270: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 22356/215244 objects degraded (10.386%) 2023-10-09T05:33:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:39 smithi008 ceph-mon[191227]: pgmap v270: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 22356/215244 objects degraded (10.386%) 2023-10-09T05:33:39.995 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:40.071 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:40.268 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:33:40.495 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 48 16 77494 77478 100.839 69.125 0.0116903 0.009912 2023-10-09T05:33:40.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:33:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:33:40] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.33.4" 2023-10-09T05:33:40.932 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:40.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:40.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:41.051 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:41.084 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:41.162 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:41.169 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:41.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:41.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:41.273 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:41.311 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:41.396 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:41.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:41.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:41.467 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:41.495 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 49 16 78681 78665 100.295 74.1875 0.00993959 0.00996548 2023-10-09T05:33:41.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:41.537 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:41 smithi008 ceph-mon[184418]: pgmap v271: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 23084/222861 objects degraded (10.358%) 2023-10-09T05:33:41.547 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:41.633 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:41.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:41.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:41.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:41.772 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:41 smithi191 ceph-mon[161669]: pgmap v271: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 23084/222861 objects degraded (10.358%) 2023-10-09T05:33:41.775 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:41.850 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:41.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:41.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:41.869 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:41 smithi008 ceph-mon[191227]: pgmap v271: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 23084/222861 objects degraded (10.358%) 2023-10-09T05:33:41.959 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:42.058 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:42.495 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 50 16 79742 79726 99.6153 66.3125 0.0111247 0.0100338 2023-10-09T05:33:42.577 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:33:42.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1708331885' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:42.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:42 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23084/222861 objects degraded (10.358%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:42.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1708331885' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:42.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:42 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23084/222861 objects degraded (10.358%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:42.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1708331885' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:33:42.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:42 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23084/222861 objects degraded (10.358%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:43.380 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:43.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:43.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:43.488 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:43.495 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 51 16 80631 80615 98.7517 55.5625 0.0822692 0.0101116 2023-10-09T05:33:43.536 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:43.624 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:43.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:43.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[184418]: pgmap v272: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 24113/230235 objects degraded (10.473%) 2023-10-09T05:33:43.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:43.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[184418]: failed to encode map e115 with expected crc 2023-10-09T05:33:43.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[184418]: failed to encode map e115 with expected crc 2023-10-09T05:33:43.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1708331885' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:33:43.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[184418]: osdmap e115: 8 total, 7 up, 8 in 2023-10-09T05:33:43.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[191227]: pgmap v272: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 24113/230235 objects degraded (10.473%) 2023-10-09T05:33:43.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:43.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[191227]: failed to encode map e115 with expected crc 2023-10-09T05:33:43.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[191227]: failed to encode map e115 with expected crc 2023-10-09T05:33:43.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1708331885' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:33:43.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:43 smithi008 ceph-mon[191227]: osdmap e115: 8 total, 7 up, 8 in 2023-10-09T05:33:43.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:43.790 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:43 smithi191 ceph-mon[161669]: pgmap v272: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 24113/230235 objects degraded (10.473%) 2023-10-09T05:33:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:43 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:43 smithi191 ceph-mon[161669]: failed to encode map e115 with expected crc 2023-10-09T05:33:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:43 smithi191 ceph-mon[161669]: failed to encode map e115 with expected crc 2023-10-09T05:33:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:43 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1708331885' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:33:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:43 smithi191 ceph-mon[161669]: osdmap e115: 8 total, 7 up, 8 in 2023-10-09T05:33:43.865 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:43.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:43.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:43.970 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:44.009 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:44.087 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:44.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:44.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:44.196 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:44.224 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:44.499 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 52 16 81767 81751 98.2125 71 0.0134642 0.0101778 2023-10-09T05:33:44.540 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:44.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:44.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:44.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:44.702 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:44.761 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:44.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:44.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:44.872 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:44.903 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:45.004 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:45.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:45.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:45.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:45.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:45.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:44 smithi191 ceph-mon[161669]: pgmap v274: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 24113/233316 objects degraded (10.335%) 2023-10-09T05:33:45.088 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:45.088 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:45.088 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:44 smithi008 ceph-mon[184418]: pgmap v274: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 24113/233316 objects degraded (10.335%) 2023-10-09T05:33:45.088 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:33:45.088 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:45.088 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:44 smithi008 ceph-mon[191227]: pgmap v274: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 24113/233316 objects degraded (10.335%) 2023-10-09T05:33:45.115 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:45.199 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:45.284 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:33:45.324 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:33:45.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:45.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:45.342 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:33:45.385 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:33:45.395 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:33:45.395 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:33:45.426 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:33:45.431 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:33:45.458 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:33:45.500 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 53 16 82894 82878 97.6857 70.4375 0.00826293 0.0102317 2023-10-09T05:33:45.501 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:33:45.504 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:45.537 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:33:45.538 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:33:45.538 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:33:45.538 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:33:45.538 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:33:45.538 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:33:45.538 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:33:45.538 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:33:45.538 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T05:33:45.889 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2023-10-09T05:33:45.889 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:33:45.889 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-10-09T05:33:45.890 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:33:46.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2988265908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:46.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2988265908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:46.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2988265908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:46.500 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 54 16 84096 84080 97.268 75.125 0.0252491 0.0102762 2023-10-09T05:33:46.728 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:33:46.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:46 smithi008 ceph-mon[184418]: pgmap v275: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 25803/246441 objects degraded (10.470%) 2023-10-09T05:33:46.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:46 smithi008 ceph-mon[191227]: pgmap v275: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 25803/246441 objects degraded (10.470%) 2023-10-09T05:33:47.103 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:33:47.186 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:33:47.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:47.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:33:47.282 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T05:33:47.283 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T05:33:47.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:46 smithi191 ceph-mon[161669]: pgmap v275: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 25803/246441 objects degraded (10.470%) 2023-10-09T05:33:47.299 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:47.438 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:47.500 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 55 16 84990 84974 96.5156 55.875 0.027168 0.0103569 2023-10-09T05:33:47.553 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:47.584 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:47.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:47.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:47.599 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:47.698 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:47.732 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:47.797 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:47.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:47.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:47.908 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:47.954 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:48.030 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:48.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:48.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:48.039 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:47 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 25803/246441 objects degraded (10.470%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:48.137 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:47 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 25803/246441 objects degraded (10.470%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:48.137 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:47 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 25803/246441 objects degraded (10.470%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:48.140 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:48.200 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:48.283 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:48.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:48.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:48.350 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:48.394 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:48.429 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:48.501 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 56 16 86077 86061 96.0054 67.9375 0.0127823 0.0104113 2023-10-09T05:33:48.506 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:48.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:48.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:48.613 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:48.663 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:48.751 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:48.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:48.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:48.860 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:48.911 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:48 smithi191 ceph-mon[161669]: pgmap v276: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 25803/248364 objects degraded (10.389%) 2023-10-09T05:33:48.916 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:48.976 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:48.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:48.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:49.081 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:49.144 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:49.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:48 smithi008 ceph-mon[184418]: pgmap v276: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 25803/248364 objects degraded (10.389%) 2023-10-09T05:33:49.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:48 smithi008 ceph-mon[191227]: pgmap v276: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 25803/248364 objects degraded (10.389%) 2023-10-09T05:33:49.283 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:33:49.451 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:49.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:49.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:49.505 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 57 16 87072 87056 95.405 62.1875 0.013755 0.0104765 2023-10-09T05:33:49.566 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:49.599 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:49.660 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:49.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:49.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:49.766 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:49.809 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:50.184 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:50.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:50.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:50.290 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:50.353 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:50.505 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 58 16 88000 87984 94.7603 58 0.0151707 0.0105482 2023-10-09T05:33:50.569 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:33:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:33:50] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.33.4" 2023-10-09T05:33:50.634 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:50.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:50.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:50.743 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:50.813 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:50.907 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:50 smithi008 ceph-mon[184418]: pgmap v277: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 26427/255390 objects degraded (10.348%) 2023-10-09T05:33:51.178 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:51.185 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:50 smithi008 ceph-mon[191227]: pgmap v277: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 26427/255390 objects degraded (10.348%) 2023-10-09T05:33:51.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:51.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:51.200 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:33:51.216 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:33:51.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:51.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:50 smithi191 ceph-mon[161669]: pgmap v277: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 26427/255390 objects degraded (10.348%) 2023-10-09T05:33:51.340 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:51.424 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:51.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:51.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:51.505 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 59 16 89020 89004 94.2348 63.75 0.00415633 0.0106054 2023-10-09T05:33:51.531 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:51.561 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:52.034 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:52.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:52.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:52.140 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:52.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:51 smithi008 ceph-mon[184418]: from='client.54649 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:33:52.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:51 smithi008 ceph-mon[191227]: from='client.54649 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:33:52.224 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:52.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:51 smithi191 ceph-mon[161669]: from='client.54649 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:33:52.476 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:52.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:52.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:52.505 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:33:52.506131+0000 min lat: 0.00107405 max lat: 1.03525 avg lat: 0.0106708 2023-10-09T05:33:52.505 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:33:52.505 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 60 16 89991 89975 93.6758 60.6875 0.0151878 0.0106708 2023-10-09T05:33:52.590 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:52.644 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:52.681 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:52.687 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":277,"stamp":"2023-10-09T05:33:49.896509+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5582034867,"num_objects":85130,"num_object_clones":0,"num_object_copies":255390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26427,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85130,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":86208,"num_write_kb":5461669,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86272,"ondisk_log_size":86272,"up":232,"acting":232,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17244916,"kb_used_data":15777780,"kb_used_omap":31,"kb_used_meta":1467040,"kb_avail":732716300,"statfs":{"total":767960285184,"available":750301491200,"internally_reserved":0,"allocated":16156446720,"data_stored":16152303120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":1502249417},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":710606848,"num_objects":10843,"num_object_clones":0,"num_object_copies":32529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3343,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10843,"num_write_kb":693952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005662"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":98,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.900618+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:33:30.900618+0000","last_peered":"2023-10-09T05:33:30.900618+0000","last_clean":"2023-10-09T05:33:30.900618+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:33:30.900618+0000","last_undegraded":"2023-10-09T05:33:30.900618+0000","last_fullsized":"2023-10-09T05:33:30.900618+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":84,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.901816+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:33:30.901816+0000","last_peered":"2023-10-09T05:33:30.901816+0000","last_clean":"2023-10-09T05:33:30.901816+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:33:30.901816+0000","last_undegraded":"2023-10-09T05:33:30.901816+0000","last_fullsized":"2023-10-09T05:33:30.901816+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":59,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.898042+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:33:30.898042+0000","last_peered":"2023-10-09T05:33:30.898042+0000","last_clean":"2023-10-09T05:33:30.898042+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:33:30.898042+0000","last_undegraded":"2023-10-09T05:33:30.898042+0000","last_fullsized":"2023-10-09T05:33:30.898042+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":70,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.896125+0000","last_change":"2023-10-09T05:33:21.537914+0000","last_active":"2023-10-09T05:33:30.896125+0000","last_peered":"2023-10-09T05:33:30.896125+0000","last_clean":"2023-10-09T05:33:12.601246+0000","last_became_active":"2023-10-09T05:33:21.537914+0000","last_became_peered":"2023-10-09T05:33:21.537914+0000","last_unstale":"2023-10-09T05:33:30.896125+0000","last_undegraded":"2023-10-09T05:33:30.896125+0000","last_fullsized":"2023-10-09T05:33:21.532713+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:18.451448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":100,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.906968+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:33:30.906968+0000","last_peered":"2023-10-09T05:33:30.906968+0000","last_clean":"2023-10-09T05:33:30.906968+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:33:30.906968+0000","last_undegraded":"2023-10-09T05:33:30.906968+0000","last_fullsized":"2023-10-09T05:33:30.906968+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":81,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.898122+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:33:30.898122+0000","last_peered":"2023-10-09T05:33:30.898122+0000","last_clean":"2023-10-09T05:33:30.898122+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:33:30.898122+0000","last_undegraded":"2023-10-09T05:33:30.898122+0000","last_fullsized":"2023-10-09T05:33:30.898122+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":64,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.896427+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:33:30.896427+0000","last_peered":"2023-10-09T05:33:30.896427+0000","last_clean":"2023-10-09T05:33:30.896427+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:33:30.896427+0000","last_undegraded":"2023-10-09T05:33:30.896427+0000","last_fullsized":"2023-10-09T05:33:30.896427+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":84,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.906891+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:33:30.906891+0000","last_peered":"2023-10-09T05:33:30.906891+0000","last_clean":"2023-10-09T05:33:30.906891+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:33:30.906891+0000","last_undegraded":"2023-10-09T05:33:30.906891+0000","last_fullsized":"2023-10-09T05:33:30.906891+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":88,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.890435+0000","last_change":"2023-10-09T05:33:21.536321+0000","last_active":"2023-10-09T05:33:30.890435+0000","last_peered":"2023-10-09T05:33:30.890435+0000","last_clean":"2023-10-09T05:33:12.591820+0000","last_became_active":"2023-10-09T05:33:21.536321+0000","last_became_peered":"2023-10-09T05:33:21.536321+0000","last_unstale":"2023-10-09T05:33:30.890435+0000","last_undegraded":"2023-10-09T05:33:30.890435+0000","last_fullsized":"2023-10-09T05:33:21.531052+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":80,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.902861+0000","last_change":"2023-10-09T05:33:21.536924+0000","last_active":"2023-10-09T05:33:30.902861+0000","last_peered":"2023-10-09T05:33:30.902861+0000","last_clean":"2023-10-09T05:33:12.602231+0000","last_became_active":"2023-10-09T05:33:21.536924+0000","last_became_peered":"2023-10-09T05:33:21.536924+0000","last_unstale":"2023-10-09T05:33:30.902861+0000","last_undegraded":"2023-10-09T05:33:30.902861+0000","last_fullsized":"2023-10-09T05:33:21.531983+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.116098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":21,"reported_epoch":114,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:30.901700+0000","last_change":"2023-10-09T05:33:21.536951+0000","last_active":"2023-10-09T05:33:30.901700+0000","last_peered":"2023-10-09T05:33:30.901700+0000","last_clean":"2023-10-09T05:21:17.840041+0000","last_became_active":"2023-10-09T05:33:21.536951+0000","last_became_peered":"2023-10-09T05:33:21.536951+0000","last_unstale":"2023-10-09T05:33:30.901700+0000","last_undegraded":"2023-10-09T05:33:21.531336+0000","last_fullsized":"2023-10-09T05:33:21.531230+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:11.457304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":59,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.898816+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:33:30.898816+0000","last_peered":"2023-10-09T05:33:30.898816+0000","last_clean":"2023-10-09T05:33:30.898816+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:33:30.898816+0000","last_undegraded":"2023-10-09T05:33:30.898816+0000","last_fullsized":"2023-10-09T05:33:30.898816+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":95,"reported_epoch":114,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:30.900784+0000","last_change":"2023-10-09T05:33:21.537017+0000","last_active":"2023-10-09T05:33:30.900784+0000","last_peered":"2023-10-09T05:33:30.900784+0000","last_clean":"2023-10-09T05:33:12.599720+0000","last_became_active":"2023-10-09T05:33:21.537017+0000","last_became_peered":"2023-10-09T05:33:21.537017+0000","last_unstale":"2023-10-09T05:33:30.900784+0000","last_undegraded":"2023-10-09T05:33:21.534333+0000","last_fullsized":"2023-10-09T05:33:21.534169+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":81,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.900350+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:33:30.900350+0000","last_peered":"2023-10-09T05:33:30.900350+0000","last_clean":"2023-10-09T05:33:30.900350+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:33:30.900350+0000","last_undegraded":"2023-10-09T05:33:30.900350+0000","last_fullsized":"2023-10-09T05:33:30.900350+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":21,"reported_epoch":114,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:30.900440+0000","last_change":"2023-10-09T05:33:21.541382+0000","last_active":"2023-10-09T05:33:30.900440+0000","last_peered":"2023-10-09T05:33:30.900440+0000","last_clean":"2023-10-09T05:21:17.839552+0000","last_became_active":"2023-10-09T05:33:21.541382+0000","last_became_peered":"2023-10-09T05:33:21.541382+0000","last_unstale":"2023-10-09T05:33:30.900440+0000","last_undegraded":"2023-10-09T05:33:21.535121+0000","last_fullsized":"2023-10-09T05:33:21.534957+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":90,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.902790+0000","last_change":"2023-10-09T05:33:21.544048+0000","last_active":"2023-10-09T05:33:30.902790+0000","last_peered":"2023-10-09T05:33:30.902790+0000","last_clean":"2023-10-09T05:33:12.603569+0000","last_became_active":"2023-10-09T05:33:21.544048+0000","last_became_peered":"2023-10-09T05:33:21.544048+0000","last_unstale":"2023-10-09T05:33:30.902790+0000","last_undegraded":"2023-10-09T05:33:30.902790+0000","last_fullsized":"2023-10-09T05:33:21.530368+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:49.505710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":70,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.895734+0000","last_change":"2023-10-09T05:33:21.538555+0000","last_active":"2023-10-09T05:33:30.895734+0000","last_peered":"2023-10-09T05:33:30.895734+0000","last_clean":"2023-10-09T05:33:12.598696+0000","last_became_active":"2023-10-09T05:33:21.538555+0000","last_became_peered":"2023-10-09T05:33:21.538555+0000","last_unstale":"2023-10-09T05:33:30.895734+0000","last_undegraded":"2023-10-09T05:33:30.895734+0000","last_fullsized":"2023-10-09T05:33:21.531590+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:47.602581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":87,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.890933+0000","last_change":"2023-10-09T05:33:21.539311+0000","last_active":"2023-10-09T05:33:30.890933+0000","last_peered":"2023-10-09T05:33:30.890933+0000","last_clean":"2023-10-09T05:33:12.591928+0000","last_became_active":"2023-10-09T05:33:21.539311+0000","last_became_peered":"2023-10-09T05:33:21.539311+0000","last_unstale":"2023-10-09T05:33:30.890933+0000","last_undegraded":"2023-10-09T05:33:30.890933+0000","last_fullsized":"2023-10-09T05:33:21.532549+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.901147+0000","last_change":"2023-10-09T05:33:21.536611+0000","last_active":"2023-10-09T05:33:30.901147+0000","last_peered":"2023-10-09T05:33:30.901147+0000","last_clean":"2023-10-09T05:21:15.763192+0000","last_became_active":"2023-10-09T05:33:21.536611+0000","last_became_peered":"2023-10-09T05:33:21.536611+0000","last_unstale":"2023-10-09T05:33:30.901147+0000","last_undegraded":"2023-10-09T05:33:30.901147+0000","last_fullsized":"2023-10-09T05:33:21.534136+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":70,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.895898+0000","last_change":"2023-10-09T05:33:21.537621+0000","last_active":"2023-10-09T05:33:30.895898+0000","last_peered":"2023-10-09T05:33:30.895898+0000","last_clean":"2023-10-09T05:33:12.601147+0000","last_became_active":"2023-10-09T05:33:21.537621+0000","last_became_peered":"2023-10-09T05:33:21.537621+0000","last_unstale":"2023-10-09T05:33:30.895898+0000","last_undegraded":"2023-10-09T05:33:30.895898+0000","last_fullsized":"2023-10-09T05:33:21.532506+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:41.399098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":90,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.897005+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:33:30.897005+0000","last_peered":"2023-10-09T05:33:30.897005+0000","last_clean":"2023-10-09T05:33:30.897005+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:33:30.897005+0000","last_undegraded":"2023-10-09T05:33:30.897005+0000","last_fullsized":"2023-10-09T05:33:30.897005+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.902157+0000","last_change":"2023-10-09T05:33:21.536100+0000","last_active":"2023-10-09T05:33:30.902157+0000","last_peered":"2023-10-09T05:33:30.902157+0000","last_clean":"2023-10-09T05:33:12.603890+0000","last_became_active":"2023-10-09T05:33:21.536100+0000","last_became_peered":"2023-10-09T05:33:21.536100+0000","last_unstale":"2023-10-09T05:33:30.902157+0000","last_undegraded":"2023-10-09T05:33:30.902157+0000","last_fullsized":"2023-10-09T05:33:21.532317+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.588371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"115'5439","reported_seq":5461,"reported_epoch":115,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:49.791789+0000","last_change":"2023-10-09T05:33:21.537974+0000","last_active":"2023-10-09T05:33:49.791789+0000","last_peered":"2023-10-09T05:33:49.791789+0000","last_clean":"2023-10-09T05:33:20.515228+0000","last_became_active":"2023-10-09T05:33:21.537974+0000","last_became_peered":"2023-10-09T05:33:21.537974+0000","last_unstale":"2023-10-09T05:33:49.791789+0000","last_undegraded":"2023-10-09T05:33:21.533195+0000","last_fullsized":"2023-10-09T05:33:21.532956+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5439,"log_dups_size":0,"ondisk_log_size":5439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356450304,"num_objects":5439,"num_object_clones":0,"num_object_copies":16317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5439,"num_write_kb":348096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":5439}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":21,"reported_epoch":114,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:30.901344+0000","last_change":"2023-10-09T05:33:21.544135+0000","last_active":"2023-10-09T05:33:30.901344+0000","last_peered":"2023-10-09T05:33:30.901344+0000","last_clean":"2023-10-09T05:21:17.838784+0000","last_became_active":"2023-10-09T05:33:21.544135+0000","last_became_peered":"2023-10-09T05:33:21.544135+0000","last_unstale":"2023-10-09T05:33:30.901344+0000","last_undegraded":"2023-10-09T05:33:21.534556+0000","last_fullsized":"2023-10-09T05:33:21.534176+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":72,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.890197+0000","last_change":"2023-10-09T05:33:21.534526+0000","last_active":"2023-10-09T05:33:30.890197+0000","last_peered":"2023-10-09T05:33:30.890197+0000","last_clean":"2023-10-09T05:33:12.591527+0000","last_became_active":"2023-10-09T05:33:21.534526+0000","last_became_peered":"2023-10-09T05:33:21.534526+0000","last_unstale":"2023-10-09T05:33:30.890197+0000","last_undegraded":"2023-10-09T05:33:30.890197+0000","last_fullsized":"2023-10-09T05:33:21.531728+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"115'5219","reported_seq":5238,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:45.557008+0000","last_change":"2023-10-09T05:32:45.926858+0000","last_active":"2023-10-09T05:33:45.557008+0000","last_peered":"2023-10-09T05:33:45.557008+0000","last_clean":"2023-10-09T05:33:45.557008+0000","last_became_active":"2023-10-09T05:32:45.926538+0000","last_became_peered":"2023-10-09T05:32:45.926538+0000","last_unstale":"2023-10-09T05:33:45.557008+0000","last_undegraded":"2023-10-09T05:33:45.557008+0000","last_fullsized":"2023-10-09T05:33:45.557008+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5219,"log_dups_size":0,"ondisk_log_size":5219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:44.854965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342032384,"num_objects":5219,"num_object_clones":0,"num_object_copies":15657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5219,"num_write_kb":334016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":21,"reported_epoch":114,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:30.901291+0000","last_change":"2023-10-09T05:33:21.543381+0000","last_active":"2023-10-09T05:33:30.901291+0000","last_peered":"2023-10-09T05:33:30.901291+0000","last_clean":"2023-10-09T05:21:17.839217+0000","last_became_active":"2023-10-09T05:33:21.543381+0000","last_became_peered":"2023-10-09T05:33:21.543381+0000","last_unstale":"2023-10-09T05:33:30.901291+0000","last_undegraded":"2023-10-09T05:33:21.534386+0000","last_fullsized":"2023-10-09T05:33:21.534260+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":96,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.901070+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:33:30.901070+0000","last_peered":"2023-10-09T05:33:30.901070+0000","last_clean":"2023-10-09T05:33:30.901070+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:33:30.901070+0000","last_undegraded":"2023-10-09T05:33:30.901070+0000","last_fullsized":"2023-10-09T05:33:30.901070+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"115'5280","reported_seq":5299,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:45.518658+0000","last_change":"2023-10-09T05:32:45.931141+0000","last_active":"2023-10-09T05:33:45.518658+0000","last_peered":"2023-10-09T05:33:45.518658+0000","last_clean":"2023-10-09T05:33:45.518658+0000","last_became_active":"2023-10-09T05:32:45.930819+0000","last_became_peered":"2023-10-09T05:32:45.930819+0000","last_unstale":"2023-10-09T05:33:45.518658+0000","last_undegraded":"2023-10-09T05:33:45.518658+0000","last_fullsized":"2023-10-09T05:33:45.518658+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5280,"log_dups_size":0,"ondisk_log_size":5280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:25.657817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346030080,"num_objects":5280,"num_object_clones":0,"num_object_copies":15840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5280,"num_write_kb":337920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":21,"reported_epoch":114,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:30.890944+0000","last_change":"2023-10-09T05:33:21.537840+0000","last_active":"2023-10-09T05:33:30.890944+0000","last_peered":"2023-10-09T05:33:30.890944+0000","last_clean":"2023-10-09T05:21:17.840209+0000","last_became_active":"2023-10-09T05:33:21.537840+0000","last_became_peered":"2023-10-09T05:33:21.537840+0000","last_unstale":"2023-10-09T05:33:30.890944+0000","last_undegraded":"2023-10-09T05:33:21.532661+0000","last_fullsized":"2023-10-09T05:33:21.532525+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":96,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.897279+0000","last_change":"2023-10-09T05:33:21.541527+0000","last_active":"2023-10-09T05:33:30.897279+0000","last_peered":"2023-10-09T05:33:30.897279+0000","last_clean":"2023-10-09T05:33:12.592592+0000","last_became_active":"2023-10-09T05:33:21.541527+0000","last_became_peered":"2023-10-09T05:33:21.541527+0000","last_unstale":"2023-10-09T05:33:30.897279+0000","last_undegraded":"2023-10-09T05:33:30.897279+0000","last_fullsized":"2023-10-09T05:33:21.532177+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"115'5150","reported_seq":5169,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:45.574095+0000","last_change":"2023-10-09T05:32:45.929091+0000","last_active":"2023-10-09T05:33:45.574095+0000","last_peered":"2023-10-09T05:33:45.574095+0000","last_clean":"2023-10-09T05:33:45.574095+0000","last_became_active":"2023-10-09T05:32:45.928801+0000","last_became_peered":"2023-10-09T05:32:45.928801+0000","last_unstale":"2023-10-09T05:33:45.574095+0000","last_undegraded":"2023-10-09T05:33:45.574095+0000","last_fullsized":"2023-10-09T05:33:45.574095+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5150,"log_dups_size":0,"ondisk_log_size":5150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:53.569691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337510400,"num_objects":5150,"num_object_clones":0,"num_object_copies":15450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5150,"num_write_kb":329600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":82,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.890679+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:33:30.890679+0000","last_peered":"2023-10-09T05:33:30.890679+0000","last_clean":"2023-10-09T05:33:30.890679+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:33:30.890679+0000","last_undegraded":"2023-10-09T05:33:30.890679+0000","last_fullsized":"2023-10-09T05:33:30.890679+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":64,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.895255+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:33:30.895255+0000","last_peered":"2023-10-09T05:33:30.895255+0000","last_clean":"2023-10-09T05:33:30.895255+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:33:30.895255+0000","last_undegraded":"2023-10-09T05:33:30.895255+0000","last_fullsized":"2023-10-09T05:33:30.895255+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"115'5266","reported_seq":5291,"reported_epoch":115,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:45.537049+0000","last_change":"2023-10-09T05:33:21.536458+0000","last_active":"2023-10-09T05:33:45.537049+0000","last_peered":"2023-10-09T05:33:45.537049+0000","last_clean":"2023-10-09T05:33:20.502869+0000","last_became_active":"2023-10-09T05:33:21.536458+0000","last_became_peered":"2023-10-09T05:33:21.536458+0000","last_unstale":"2023-10-09T05:33:45.537049+0000","last_undegraded":"2023-10-09T05:33:21.526951+0000","last_fullsized":"2023-10-09T05:33:21.526843+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5266,"log_dups_size":0,"ondisk_log_size":5266,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:01.509884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345112576,"num_objects":5266,"num_object_clones":0,"num_object_copies":15798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5266,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5266,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":5266}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":83,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:42.565511+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:33:42.565511+0000","last_peered":"2023-10-09T05:33:42.565511+0000","last_clean":"2023-10-09T05:33:42.565511+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:33:42.565511+0000","last_undegraded":"2023-10-09T05:33:42.565511+0000","last_fullsized":"2023-10-09T05:33:42.565511+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":81,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.900829+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:33:30.900829+0000","last_peered":"2023-10-09T05:33:30.900829+0000","last_clean":"2023-10-09T05:33:30.900829+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:33:30.900829+0000","last_undegraded":"2023-10-09T05:33:30.900829+0000","last_fullsized":"2023-10-09T05:33:30.900829+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"115'5416","reported_seq":5434,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:48.458254+0000","last_change":"2023-10-09T05:32:45.927644+0000","last_active":"2023-10-09T05:33:48.458254+0000","last_peered":"2023-10-09T05:33:48.458254+0000","last_clean":"2023-10-09T05:33:48.458254+0000","last_became_active":"2023-10-09T05:32:45.927388+0000","last_became_peered":"2023-10-09T05:32:45.927388+0000","last_unstale":"2023-10-09T05:33:48.458254+0000","last_undegraded":"2023-10-09T05:33:48.458254+0000","last_fullsized":"2023-10-09T05:33:48.458254+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5416,"log_dups_size":0,"ondisk_log_size":5416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:42.111993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354942976,"num_objects":5416,"num_object_clones":0,"num_object_copies":16248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5416,"num_write_kb":346624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":82,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.897455+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:33:30.897455+0000","last_peered":"2023-10-09T05:33:30.897455+0000","last_clean":"2023-10-09T05:33:30.897455+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:33:30.897455+0000","last_undegraded":"2023-10-09T05:33:30.897455+0000","last_fullsized":"2023-10-09T05:33:30.897455+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":96,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.900896+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:33:30.900896+0000","last_peered":"2023-10-09T05:33:30.900896+0000","last_clean":"2023-10-09T05:33:30.900896+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:33:30.900896+0000","last_undegraded":"2023-10-09T05:33:30.900896+0000","last_fullsized":"2023-10-09T05:33:30.900896+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"115'5368","reported_seq":5386,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:49.787953+0000","last_change":"2023-10-09T05:32:45.936026+0000","last_active":"2023-10-09T05:33:49.787953+0000","last_peered":"2023-10-09T05:33:49.787953+0000","last_clean":"2023-10-09T05:33:49.787953+0000","last_became_active":"2023-10-09T05:32:45.935681+0000","last_became_peered":"2023-10-09T05:32:45.935681+0000","last_unstale":"2023-10-09T05:33:49.787953+0000","last_undegraded":"2023-10-09T05:33:49.787953+0000","last_fullsized":"2023-10-09T05:33:49.787953+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5368,"log_dups_size":0,"ondisk_log_size":5368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351797248,"num_objects":5368,"num_object_clones":0,"num_object_copies":16104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5368,"num_write_kb":343552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":88,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.900949+0000","last_change":"2023-10-09T05:33:21.541326+0000","last_active":"2023-10-09T05:33:30.900949+0000","last_peered":"2023-10-09T05:33:30.900949+0000","last_clean":"2023-10-09T05:33:12.599212+0000","last_became_active":"2023-10-09T05:33:21.541326+0000","last_became_peered":"2023-10-09T05:33:21.541326+0000","last_unstale":"2023-10-09T05:33:30.900949+0000","last_undegraded":"2023-10-09T05:33:30.900949+0000","last_fullsized":"2023-10-09T05:33:21.534788+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":90,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.900938+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:33:30.900938+0000","last_peered":"2023-10-09T05:33:30.900938+0000","last_clean":"2023-10-09T05:33:30.900938+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:33:30.900938+0000","last_undegraded":"2023-10-09T05:33:30.900938+0000","last_fullsized":"2023-10-09T05:33:30.900938+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"115'5580","reported_seq":5598,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:49.768568+0000","last_change":"2023-10-09T05:32:45.929479+0000","last_active":"2023-10-09T05:33:49.768568+0000","last_peered":"2023-10-09T05:33:49.768568+0000","last_clean":"2023-10-09T05:33:49.768568+0000","last_became_active":"2023-10-09T05:32:45.929063+0000","last_became_peered":"2023-10-09T05:32:45.929063+0000","last_unstale":"2023-10-09T05:33:49.768568+0000","last_undegraded":"2023-10-09T05:33:49.768568+0000","last_fullsized":"2023-10-09T05:33:49.768568+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5580,"log_dups_size":0,"ondisk_log_size":5580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365690880,"num_objects":5580,"num_object_clones":0,"num_object_copies":16740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5580,"num_write_kb":357120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:40.878665+0000","last_change":"2023-10-09T05:33:21.546397+0000","last_active":"2023-10-09T05:33:40.878665+0000","last_peered":"2023-10-09T05:33:40.878665+0000","last_clean":"2023-10-09T05:33:12.602017+0000","last_became_active":"2023-10-09T05:33:21.546397+0000","last_became_peered":"2023-10-09T05:33:21.546397+0000","last_unstale":"2023-10-09T05:33:40.878665+0000","last_undegraded":"2023-10-09T05:33:40.878665+0000","last_fullsized":"2023-10-09T05:33:21.530353+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:38:13.641630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":59,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.898986+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:33:30.898986+0000","last_peered":"2023-10-09T05:33:30.898986+0000","last_clean":"2023-10-09T05:33:30.898986+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:33:30.898986+0000","last_undegraded":"2023-10-09T05:33:30.898986+0000","last_fullsized":"2023-10-09T05:33:30.898986+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":629,"reported_epoch":114,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:30.898539+0000","last_change":"2023-10-09T05:33:21.534540+0000","last_active":"2023-10-09T05:33:30.898539+0000","last_peered":"2023-10-09T05:33:30.898539+0000","last_clean":"2023-10-09T05:33:12.603745+0000","last_became_active":"2023-10-09T05:33:21.534540+0000","last_became_peered":"2023-10-09T05:33:21.534540+0000","last_unstale":"2023-10-09T05:33:30.898539+0000","last_undegraded":"2023-10-09T05:33:21.527759+0000","last_fullsized":"2023-10-09T05:33:21.527646+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:47.310192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":97,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.891018+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:33:30.891018+0000","last_peered":"2023-10-09T05:33:30.891018+0000","last_clean":"2023-10-09T05:33:30.891018+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:33:30.891018+0000","last_undegraded":"2023-10-09T05:33:30.891018+0000","last_fullsized":"2023-10-09T05:33:30.891018+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.898748+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:33:30.898748+0000","last_peered":"2023-10-09T05:33:30.898748+0000","last_clean":"2023-10-09T05:33:30.898748+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:33:30.898748+0000","last_undegraded":"2023-10-09T05:33:30.898748+0000","last_fullsized":"2023-10-09T05:33:30.898748+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"115'5223","reported_seq":5242,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:47.605705+0000","last_change":"2023-10-09T05:32:45.936085+0000","last_active":"2023-10-09T05:33:47.605705+0000","last_peered":"2023-10-09T05:33:47.605705+0000","last_clean":"2023-10-09T05:33:47.605705+0000","last_became_active":"2023-10-09T05:32:45.935802+0000","last_became_peered":"2023-10-09T05:32:45.935802+0000","last_unstale":"2023-10-09T05:33:47.605705+0000","last_undegraded":"2023-10-09T05:33:47.605705+0000","last_fullsized":"2023-10-09T05:33:47.605705+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5223,"log_dups_size":0,"ondisk_log_size":5223,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:26.030229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342294528,"num_objects":5223,"num_object_clones":0,"num_object_copies":15669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5223,"num_write_kb":334272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"115'5439","reported_seq":5458,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:47.607340+0000","last_change":"2023-10-09T05:32:45.927786+0000","last_active":"2023-10-09T05:33:47.607340+0000","last_peered":"2023-10-09T05:33:47.607340+0000","last_clean":"2023-10-09T05:33:47.607340+0000","last_became_active":"2023-10-09T05:32:45.927542+0000","last_became_peered":"2023-10-09T05:32:45.927542+0000","last_unstale":"2023-10-09T05:33:47.607340+0000","last_undegraded":"2023-10-09T05:33:47.607340+0000","last_fullsized":"2023-10-09T05:33:47.607340+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5439,"log_dups_size":0,"ondisk_log_size":5439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:36.942422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356450304,"num_objects":5439,"num_object_clones":0,"num_object_copies":16317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5439,"num_write_kb":348096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":551,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.890335+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:33:30.890335+0000","last_peered":"2023-10-09T05:33:30.890335+0000","last_clean":"2023-10-09T05:33:30.890335+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:33:30.890335+0000","last_undegraded":"2023-10-09T05:33:30.890335+0000","last_fullsized":"2023-10-09T05:33:30.890335+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":82,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.900519+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:33:30.900519+0000","last_peered":"2023-10-09T05:33:30.900519+0000","last_clean":"2023-10-09T05:33:30.900519+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:33:30.900519+0000","last_undegraded":"2023-10-09T05:33:30.900519+0000","last_fullsized":"2023-10-09T05:33:30.900519+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":94,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.890428+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:33:30.890428+0000","last_peered":"2023-10-09T05:33:30.890428+0000","last_clean":"2023-10-09T05:33:30.890428+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:33:30.890428+0000","last_undegraded":"2023-10-09T05:33:30.890428+0000","last_fullsized":"2023-10-09T05:33:30.890428+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"115'5216","reported_seq":5239,"reported_epoch":115,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:45.548824+0000","last_change":"2023-10-09T05:33:21.536701+0000","last_active":"2023-10-09T05:33:45.548824+0000","last_peered":"2023-10-09T05:33:45.548824+0000","last_clean":"2023-10-09T05:33:20.529830+0000","last_became_active":"2023-10-09T05:33:21.536701+0000","last_became_peered":"2023-10-09T05:33:21.536701+0000","last_unstale":"2023-10-09T05:33:45.548824+0000","last_undegraded":"2023-10-09T05:33:21.527504+0000","last_fullsized":"2023-10-09T05:33:21.527410+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5216,"log_dups_size":0,"ondisk_log_size":5216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:58.672105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341835776,"num_objects":5216,"num_object_clones":0,"num_object_copies":15648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5216,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5216,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":5216}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":426,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.889965+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:33:30.889965+0000","last_peered":"2023-10-09T05:33:30.889965+0000","last_clean":"2023-10-09T05:33:30.889965+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:33:30.889965+0000","last_undegraded":"2023-10-09T05:33:30.889965+0000","last_fullsized":"2023-10-09T05:33:30.889965+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":59,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.898085+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:33:30.898085+0000","last_peered":"2023-10-09T05:33:30.898085+0000","last_clean":"2023-10-09T05:33:30.898085+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:33:30.898085+0000","last_undegraded":"2023-10-09T05:33:30.898085+0000","last_fullsized":"2023-10-09T05:33:30.898085+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.898922+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:33:30.898922+0000","last_peered":"2023-10-09T05:33:30.898922+0000","last_clean":"2023-10-09T05:33:30.898922+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:33:30.898922+0000","last_undegraded":"2023-10-09T05:33:30.898922+0000","last_fullsized":"2023-10-09T05:33:30.898922+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":682,"reported_epoch":114,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:30.890335+0000","last_change":"2023-10-09T05:33:21.539819+0000","last_active":"2023-10-09T05:33:30.890335+0000","last_peered":"2023-10-09T05:33:30.890335+0000","last_clean":"2023-10-09T05:33:12.591667+0000","last_became_active":"2023-10-09T05:33:21.539819+0000","last_became_peered":"2023-10-09T05:33:21.539819+0000","last_unstale":"2023-10-09T05:33:30.890335+0000","last_undegraded":"2023-10-09T05:33:21.531338+0000","last_fullsized":"2023-10-09T05:33:21.531040+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":281,"reported_epoch":114,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:30.895016+0000","last_change":"2023-10-09T05:33:21.534856+0000","last_active":"2023-10-09T05:33:30.895016+0000","last_peered":"2023-10-09T05:33:30.895016+0000","last_clean":"2023-10-09T05:25:43.873151+0000","last_became_active":"2023-10-09T05:33:21.534856+0000","last_became_peered":"2023-10-09T05:33:21.534856+0000","last_unstale":"2023-10-09T05:33:30.895016+0000","last_undegraded":"2023-10-09T05:33:21.531368+0000","last_fullsized":"2023-10-09T05:33:21.531257+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:36.567162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":76,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:35.877996+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:33:35.877996+0000","last_peered":"2023-10-09T05:33:35.877996+0000","last_clean":"2023-10-09T05:33:35.877996+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:33:35.877996+0000","last_undegraded":"2023-10-09T05:33:35.877996+0000","last_fullsized":"2023-10-09T05:33:35.877996+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":72,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.897353+0000","last_change":"2023-10-09T05:33:21.536840+0000","last_active":"2023-10-09T05:33:30.897353+0000","last_peered":"2023-10-09T05:33:30.897353+0000","last_clean":"2023-10-09T05:33:12.592541+0000","last_became_active":"2023-10-09T05:33:21.536840+0000","last_became_peered":"2023-10-09T05:33:21.536840+0000","last_unstale":"2023-10-09T05:33:30.897353+0000","last_undegraded":"2023-10-09T05:33:30.897353+0000","last_fullsized":"2023-10-09T05:33:21.532641+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"115'5115","reported_seq":5138,"reported_epoch":115,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:45.571064+0000","last_change":"2023-10-09T05:33:21.536493+0000","last_active":"2023-10-09T05:33:45.571064+0000","last_peered":"2023-10-09T05:33:45.571064+0000","last_clean":"2023-10-09T05:33:20.493128+0000","last_became_active":"2023-10-09T05:33:21.536493+0000","last_became_peered":"2023-10-09T05:33:21.536493+0000","last_unstale":"2023-10-09T05:33:45.571064+0000","last_undegraded":"2023-10-09T05:33:21.526661+0000","last_fullsized":"2023-10-09T05:33:21.526522+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5115,"log_dups_size":0,"ondisk_log_size":5115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:27.276135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335216640,"num_objects":5115,"num_object_clones":0,"num_object_copies":15345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5115,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5115,"num_write_kb":327360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":5115}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.7","version":"115'5437","reported_seq":5455,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:49.787794+0000","last_change":"2023-10-09T05:32:45.934765+0000","last_active":"2023-10-09T05:33:49.787794+0000","last_peered":"2023-10-09T05:33:49.787794+0000","last_clean":"2023-10-09T05:33:49.787794+0000","last_became_active":"2023-10-09T05:32:45.934420+0000","last_became_peered":"2023-10-09T05:32:45.934420+0000","last_unstale":"2023-10-09T05:33:49.787794+0000","last_undegraded":"2023-10-09T05:33:49.787794+0000","last_fullsized":"2023-10-09T05:33:49.787794+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5437,"log_dups_size":0,"ondisk_log_size":5437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356319232,"num_objects":5437,"num_object_clones":0,"num_object_copies":16311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5437,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":674,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.898224+0000","last_change":"2023-10-09T05:33:21.535548+0000","last_active":"2023-10-09T05:33:30.898224+0000","last_peered":"2023-10-09T05:33:30.898224+0000","last_clean":"2023-10-09T05:22:39.601058+0000","last_became_active":"2023-10-09T05:33:21.535548+0000","last_became_peered":"2023-10-09T05:33:21.535548+0000","last_unstale":"2023-10-09T05:33:30.898224+0000","last_undegraded":"2023-10-09T05:33:30.898224+0000","last_fullsized":"2023-10-09T05:33:21.527900+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:57.456065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":64,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.894866+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:33:30.894866+0000","last_peered":"2023-10-09T05:33:30.894866+0000","last_clean":"2023-10-09T05:33:30.894866+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:33:30.894866+0000","last_undegraded":"2023-10-09T05:33:30.894866+0000","last_fullsized":"2023-10-09T05:33:30.894866+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":79,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.896342+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:33:30.896342+0000","last_peered":"2023-10-09T05:33:30.896342+0000","last_clean":"2023-10-09T05:33:30.896342+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:33:30.896342+0000","last_undegraded":"2023-10-09T05:33:30.896342+0000","last_fullsized":"2023-10-09T05:33:30.896342+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":298,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.895466+0000","last_change":"2023-10-09T05:33:21.542131+0000","last_active":"2023-10-09T05:33:30.895466+0000","last_peered":"2023-10-09T05:33:30.895466+0000","last_clean":"2023-10-09T05:22:39.669502+0000","last_became_active":"2023-10-09T05:33:21.542131+0000","last_became_peered":"2023-10-09T05:33:21.542131+0000","last_unstale":"2023-10-09T05:33:30.895466+0000","last_undegraded":"2023-10-09T05:33:30.895466+0000","last_fullsized":"2023-10-09T05:33:21.531326+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:40.994915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":59,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.898134+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:33:30.898134+0000","last_peered":"2023-10-09T05:33:30.898134+0000","last_clean":"2023-10-09T05:33:30.898134+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:33:30.898134+0000","last_undegraded":"2023-10-09T05:33:30.898134+0000","last_fullsized":"2023-10-09T05:33:30.898134+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.895565+0000","last_change":"2023-10-09T05:33:21.541460+0000","last_active":"2023-10-09T05:33:30.895565+0000","last_peered":"2023-10-09T05:33:30.895565+0000","last_clean":"2023-10-09T05:33:12.598763+0000","last_became_active":"2023-10-09T05:33:21.541460+0000","last_became_peered":"2023-10-09T05:33:21.541460+0000","last_unstale":"2023-10-09T05:33:30.895565+0000","last_undegraded":"2023-10-09T05:33:30.895565+0000","last_fullsized":"2023-10-09T05:33:21.532650+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:49.788781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"115'5453","reported_seq":5471,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:49.761034+0000","last_change":"2023-10-09T05:32:45.936284+0000","last_active":"2023-10-09T05:33:49.761034+0000","last_peered":"2023-10-09T05:33:49.761034+0000","last_clean":"2023-10-09T05:33:49.761034+0000","last_became_active":"2023-10-09T05:32:45.935947+0000","last_became_peered":"2023-10-09T05:32:45.935947+0000","last_unstale":"2023-10-09T05:33:49.761034+0000","last_undegraded":"2023-10-09T05:33:49.761034+0000","last_fullsized":"2023-10-09T05:33:49.761034+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5453,"log_dups_size":0,"ondisk_log_size":5453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357367808,"num_objects":5453,"num_object_clones":0,"num_object_copies":16359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5453,"num_write_kb":348992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":89,"reported_epoch":114,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:40.878802+0000","last_change":"2023-10-09T05:33:21.537581+0000","last_active":"2023-10-09T05:33:40.878802+0000","last_peered":"2023-10-09T05:33:40.878802+0000","last_clean":"2023-10-09T05:33:12.603985+0000","last_became_active":"2023-10-09T05:33:21.537581+0000","last_became_peered":"2023-10-09T05:33:21.537581+0000","last_unstale":"2023-10-09T05:33:40.878802+0000","last_undegraded":"2023-10-09T05:33:21.532802+0000","last_fullsized":"2023-10-09T05:33:21.532691+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:25.676206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":500,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.898578+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:33:30.898578+0000","last_peered":"2023-10-09T05:33:30.898578+0000","last_clean":"2023-10-09T05:33:30.898578+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:33:30.898578+0000","last_undegraded":"2023-10-09T05:33:30.898578+0000","last_fullsized":"2023-10-09T05:33:30.898578+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":86,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.907062+0000","last_change":"2023-10-09T05:33:21.535940+0000","last_active":"2023-10-09T05:33:30.907062+0000","last_peered":"2023-10-09T05:33:30.907062+0000","last_clean":"2023-10-09T05:33:12.601950+0000","last_became_active":"2023-10-09T05:33:21.535940+0000","last_became_peered":"2023-10-09T05:33:21.535940+0000","last_unstale":"2023-10-09T05:33:30.907062+0000","last_undegraded":"2023-10-09T05:33:30.907062+0000","last_fullsized":"2023-10-09T05:33:21.531073+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:40.161605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"115'5374","reported_seq":5398,"reported_epoch":115,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:49.779927+0000","last_change":"2023-10-09T05:33:21.547110+0000","last_active":"2023-10-09T05:33:49.779927+0000","last_peered":"2023-10-09T05:33:49.779927+0000","last_clean":"2023-10-09T05:33:20.516527+0000","last_became_active":"2023-10-09T05:33:21.547110+0000","last_became_peered":"2023-10-09T05:33:21.547110+0000","last_unstale":"2023-10-09T05:33:49.779927+0000","last_undegraded":"2023-10-09T05:33:21.535488+0000","last_fullsized":"2023-10-09T05:33:21.535362+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5374,"log_dups_size":0,"ondisk_log_size":5374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352190464,"num_objects":5374,"num_object_clones":0,"num_object_copies":16122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5374,"num_write_kb":343936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":5374}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":569,"reported_epoch":114,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:33:30.898007+0000","last_change":"2023-10-09T05:33:21.539542+0000","last_active":"2023-10-09T05:33:30.898007+0000","last_peered":"2023-10-09T05:33:30.898007+0000","last_clean":"2023-10-09T05:33:12.592709+0000","last_became_active":"2023-10-09T05:33:21.539542+0000","last_became_peered":"2023-10-09T05:33:21.539542+0000","last_unstale":"2023-10-09T05:33:30.898007+0000","last_undegraded":"2023-10-09T05:33:21.532710+0000","last_fullsized":"2023-10-09T05:33:21.532559+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":67,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.896266+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:33:30.896266+0000","last_peered":"2023-10-09T05:33:30.896266+0000","last_clean":"2023-10-09T05:33:30.896266+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:33:30.896266+0000","last_undegraded":"2023-10-09T05:33:30.896266+0000","last_fullsized":"2023-10-09T05:33:30.896266+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":66,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.900938+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:33:30.900938+0000","last_peered":"2023-10-09T05:33:30.900938+0000","last_clean":"2023-10-09T05:33:30.900938+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:33:30.900938+0000","last_undegraded":"2023-10-09T05:33:30.900938+0000","last_fullsized":"2023-10-09T05:33:30.900938+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"115'5127","reported_seq":5146,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-10-09T05:33:45.842962+0000","last_change":"2023-10-09T05:32:45.930060+0000","last_active":"2023-10-09T05:33:45.842962+0000","last_peered":"2023-10-09T05:33:45.842962+0000","last_clean":"2023-10-09T05:33:45.842962+0000","last_became_active":"2023-10-09T05:32:45.929772+0000","last_became_peered":"2023-10-09T05:32:45.929772+0000","last_unstale":"2023-10-09T05:33:45.842962+0000","last_undegraded":"2023-10-09T05:33:45.842962+0000","last_fullsized":"2023-10-09T05:33:45.842962+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":5127,"log_dups_size":0,"ondisk_log_size":5127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:06.171642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336003072,"num_objects":5127,"num_object_clones":0,"num_object_copies":15381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5127,"num_write_kb":328128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":58,"reported_epoch":114,"state":"active+undersized","last_fresh":"2023-10-09T05:33:30.895428+0000","last_change":"2023-10-09T05:33:21.541180+0000","last_active":"2023-10-09T05:33:30.895428+0000","last_peered":"2023-10-09T05:33:30.895428+0000","last_clean":"2023-10-09T05:33:12.598740+0000","last_became_active":"2023-10-09T05:33:21.541180+0000","last_became_peered":"2023-10-09T05:33:21.541180+0000","last_unstale":"2023-10-09T05:33:30.895428+0000","last_undegraded":"2023-10-09T05:33:30.895428+0000","last_fullsized":"2023-10-09T05:33:21.532395+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:47.164978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":104,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.890462+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:33:30.890462+0000","last_peered":"2023-10-09T05:33:30.890462+0000","last_clean":"2023-10-09T05:33:30.890462+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:33:30.890462+0000","last_undegraded":"2023-10-09T05:33:30.890462+0000","last_fullsized":"2023-10-09T05:33:30.890462+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":84,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.890742+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:33:30.890742+0000","last_peered":"2023-10-09T05:33:30.890742+0000","last_clean":"2023-10-09T05:33:30.890742+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:33:30.890742+0000","last_undegraded":"2023-10-09T05:33:30.890742+0000","last_fullsized":"2023-10-09T05:33:30.890742+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":96,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.900562+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:33:30.900562+0000","last_peered":"2023-10-09T05:33:30.900562+0000","last_clean":"2023-10-09T05:33:30.900562+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:33:30.900562+0000","last_undegraded":"2023-10-09T05:33:30.900562+0000","last_fullsized":"2023-10-09T05:33:30.900562+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":69,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.891065+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:33:30.891065+0000","last_peered":"2023-10-09T05:33:30.891065+0000","last_clean":"2023-10-09T05:33:30.891065+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:33:30.891065+0000","last_undegraded":"2023-10-09T05:33:30.891065+0000","last_fullsized":"2023-10-09T05:33:30.891065+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":96,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.900760+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:33:30.900760+0000","last_peered":"2023-10-09T05:33:30.900760+0000","last_clean":"2023-10-09T05:33:30.900760+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:33:30.900760+0000","last_undegraded":"2023-10-09T05:33:30.900760+0000","last_fullsized":"2023-10-09T05:33:30.900760+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":91,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.901979+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:33:30.901979+0000","last_peered":"2023-10-09T05:33:30.901979+0000","last_clean":"2023-10-09T05:33:30.901979+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:33:30.901979+0000","last_undegraded":"2023-10-09T05:33:30.901979+0000","last_fullsized":"2023-10-09T05:33:30.901979+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":103,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.902724+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:33:30.902724+0000","last_peered":"2023-10-09T05:33:30.902724+0000","last_clean":"2023-10-09T05:33:30.902724+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:33:30.902724+0000","last_undegraded":"2023-10-09T05:33:30.902724+0000","last_fullsized":"2023-10-09T05:33:30.902724+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":67,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.896382+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:33:30.896382+0000","last_peered":"2023-10-09T05:33:30.896382+0000","last_clean":"2023-10-09T05:33:30.896382+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:33:30.896382+0000","last_undegraded":"2023-10-09T05:33:30.896382+0000","last_fullsized":"2023-10-09T05:33:30.896382+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":81,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-10-09T05:33:30.900141+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:33:30.900141+0000","last_peered":"2023-10-09T05:33:30.900141+0000","last_clean":"2023-10-09T05:33:30.900141+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:33:30.900141+0000","last_undegraded":"2023-10-09T05:33:30.900141+0000","last_fullsized":"2023-10-09T05:33:30.900141+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":5577244672,"num_objects":85102,"num_object_clones":0,"num_object_copies":255306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26410,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":85102,"num_write_kb":5446528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16216883200,"data_stored":16216883200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85102,"ondisk_log_size":85102,"up":43,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":85,"acting":85,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26751,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":83,"acting":83,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":2,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":19,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430375,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1375784,"kb_used_data":1066272,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92369368,"statfs":{"total":95995035648,"available":94586232832,"internally_reserved":0,"allocated":1091862528,"data_stored":1091355045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:33:02 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.56299999999999994,"15min":0.55000000000000004},"min":{"1min":0.374,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.6839999999999999,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.56000000000000005,"15min":0.55100000000000005},"min":{"1min":0.43099999999999999,"5min":0.315,"15min":0.315},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.47699999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:32:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.51700000000000002,"15min":0.51600000000000001},"min":{"1min":0.40500000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.68999999999999995,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.41799999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.73299999999999998,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.44}]},{"osd":2,"last update":"Mon Oct 9 05:33:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.52900000000000003,"15min":0.51700000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":81.849999999999994},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.60599999999999998,"15min":0.58399999999999996},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":2.7949999999999999,"5min":2.7949999999999999,"15min":2.7949999999999999},"last":81.861000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:32:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.46600000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:33:11 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.51800000000000002,"15min":0.47799999999999998},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.249},"max":{"1min":1.53,"5min":1.53,"15min":1.53},"last":0.498},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.55100000000000005,"15min":0.52100000000000002},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.27300000000000002},"max":{"1min":2.6909999999999998,"5min":2.6909999999999998,"15min":2.6909999999999998},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:33:11 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.57199999999999995,"15min":0.52500000000000002},"min":{"1min":0.36599999999999999,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":3.0289999999999999,"5min":3.0289999999999999,"15min":3.0289999999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.56599999999999995,"15min":0.55500000000000005},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.56499999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:33:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.58699999999999997,"15min":0.55000000000000004},"min":{"1min":0.40500000000000003,"5min":0.34799999999999998,"15min":0.30499999999999999},"max":{"1min":1.282,"5min":1.282,"15min":1.282},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.83499999999999996,"5min":0.63500000000000001,"15min":0.60299999999999998},"min":{"1min":0.42599999999999999,"5min":0.372,"15min":0.33300000000000002},"max":{"1min":3.4729999999999999,"5min":3.4729999999999999,"15min":3.4729999999999999},"last":0.63100000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626609,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1686844,"kb_used_data":1377332,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92058308,"statfs":{"total":95995035648,"available":94267707392,"internally_reserved":0,"allocated":1410387968,"data_stored":1409890957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:33:08 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.55500000000000005,"15min":0.54500000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.52,"5min":1.52,"15min":1.52},"last":1.8899999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.65300000000000002,"15min":0.58099999999999996},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.3009999999999999,"5min":7.4569999999999999,"15min":7.4569999999999999},"last":1.7470000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:33:33 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":0.91200000000000003,"15min":0.67200000000000004},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":6.7880000000000003,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":2.4550000000000001},{"interface":"front","average":{"1min":1.647,"5min":0.93999999999999995,"15min":0.69799999999999995},"min":{"1min":0.34799999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.9269999999999996,"5min":8.7189999999999994,"15min":8.7189999999999994},"last":2.4279999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:33:08 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.2430000000000001,"15min":1.131},"min":{"1min":0.45800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":25.937999999999999,"5min":25.937999999999999,"15min":25.937999999999999},"last":1.698},{"interface":"front","average":{"1min":1.9199999999999999,"5min":1.2589999999999999,"15min":1.149},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":25.969999999999999,"5min":25.969999999999999,"15min":25.969999999999999},"last":1.671}]},{"osd":3,"last update":"Mon Oct 9 05:33:33 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":1.304,"15min":1.1439999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":18.361000000000001,"5min":18.361000000000001,"15min":18.361000000000001},"last":2},{"interface":"front","average":{"1min":2.3250000000000002,"5min":1.363,"15min":1.2030000000000001},"min":{"1min":0.46300000000000002,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":18.536000000000001,"5min":18.536000000000001,"15min":18.536000000000001},"last":2.0960000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:33:49 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.60799999999999998,"15min":0.50800000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.1070000000000002,"5min":4.1070000000000002,"15min":4.1070000000000002},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.58499999999999996,"15min":0.50900000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27800000000000002},"max":{"1min":2.782,"5min":2.782,"15min":2.782},"last":0.28699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.63300000000000001,"15min":0.54200000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27200000000000002},"max":{"1min":2.3759999999999999,"5min":2.3759999999999999,"15min":2.3759999999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.0609999999999999,"5min":0.66000000000000003,"15min":0.56000000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":3.5049999999999999,"5min":3.5049999999999999,"15min":3.5049999999999999},"last":0.32100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:33:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.56999999999999995,"15min":0.54200000000000004},"min":{"1min":0.32900000000000001,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":1.9299999999999999,"5min":1.9299999999999999,"15min":1.9299999999999999},"last":3.831},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.63100000000000001,"15min":0.58199999999999996},"min":{"1min":0.48399999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4330000000000001,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":0.623}]}]},{"osd":1,"up_from":93,"seq":399431958570,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1784368,"kb_used_data":1726440,"kb_used_omap":7,"kb_used_meta":57912,"kb_avail":91960784,"statfs":{"total":95995035648,"available":94167842816,"internally_reserved":0,"allocated":1767874560,"data_stored":1767330827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":59302414},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.47199999999999998,"15min":0.42599999999999999},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":4.4980000000000002,"5min":4.4980000000000002,"15min":4.4980000000000002},"last":3.028},{"interface":"front","average":{"1min":1.123,"5min":0.54200000000000004,"15min":0.45200000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.0259999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":3.3260000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.38400000000000001,"15min":0.35599999999999998},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.7989999999999999,"5min":1.7989999999999999,"15min":1.7989999999999999},"last":2.5910000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.44900000000000001,"15min":0.42499999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.3140000000000001,"5min":2.3140000000000001,"15min":2.3140000000000001},"last":0.49299999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.59299999999999997,"15min":0.51700000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.806,"5min":5.806,"15min":5.806},"last":2.6349999999999998},{"interface":"front","average":{"1min":1.202,"5min":0.56299999999999994,"15min":0.45700000000000002},"min":{"1min":0.29999999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.4569999999999999,"5min":4.4569999999999999,"15min":4.4569999999999999},"last":0.40400000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":0.746,"15min":0.67700000000000005},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.7050000000000001,"5min":3.7050000000000001,"15min":3.7050000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.68600000000000005,"15min":0.66900000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.3999999999999999,"5min":3.3999999999999999,"15min":3.3999999999999999},"last":0.436}]},{"osd":5,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":0.71299999999999997,"15min":0.65900000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.3330000000000002,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":2.665},{"interface":"front","average":{"1min":2.0899999999999999,"5min":0.94999999999999996,"15min":0.76800000000000002},"min":{"1min":0.41099999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":25.68,"5min":25.68,"15min":25.68},"last":3.2090000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.75,"15min":0.68600000000000005},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":4.6449999999999996,"5min":4.6449999999999996,"15min":4.6449999999999996},"last":2.694},{"interface":"front","average":{"1min":1.03,"5min":0.74399999999999999,"15min":0.70499999999999996},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":3.4609999999999999,"5min":3.4609999999999999,"15min":3.4609999999999999},"last":0.67200000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.69699999999999995,"15min":0.70799999999999996},"min":{"1min":0.47299999999999998,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.77000000000000002,"5min":1.014,"15min":1.014},"last":2.3839999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.68400000000000005,"15min":0.69299999999999995},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.82799999999999996,"5min":1,"15min":1},"last":2.46}]}]},{"osd":0,"up_from":88,"seq":377957122096,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2793392,"kb_used_data":2725608,"kb_used_omap":8,"kb_used_meta":67767,"kb_avail":90951760,"statfs":{"total":95995035648,"available":93134602240,"internally_reserved":0,"allocated":2791022592,"data_stored":2790478029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":69394184},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:33:41 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.52200000000000002,"15min":0.438},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.4409999999999998,"5min":5.4409999999999998,"15min":5.4409999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.52000000000000002,"15min":0.46600000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":3.641,"5min":3.641,"15min":3.641},"last":0.39300000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:33:02 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.38600000000000001,"15min":0.36599999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.5699999999999998,"5min":2.5699999999999998,"15min":2.5699999999999998},"last":0.998},{"interface":"front","average":{"1min":0.623,"5min":0.42299999999999999,"15min":0.39000000000000001},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.371,"5min":2.371,"15min":2.371},"last":0.26800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.51300000000000001,"15min":0.438},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.2679999999999998,"5min":3.2679999999999998,"15min":3.2679999999999998},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.53,"5min":0.626,"15min":0.47599999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.760999999999999,"5min":10.760999999999999,"15min":10.760999999999999},"last":2.7029999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:33:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.66800000000000004,"15min":0.67600000000000005},"min":{"1min":0.41999999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.4079999999999999,"5min":2.4079999999999999,"15min":2.4079999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.69899999999999995,"15min":0.71399999999999997},"min":{"1min":0.46600000000000003,"5min":0.434,"15min":0.434},"max":{"1min":0.99099999999999999,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.42799999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:33:02 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.65900000000000003,"15min":0.67300000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.159,"5min":1.159,"15min":1.159},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.67400000000000004,"15min":0.67500000000000004},"min":{"1min":0.47599999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":2.79,"5min":2.79,"15min":2.79},"last":2.577}]},{"osd":6,"last update":"Mon Oct 9 05:33:02 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.68400000000000005,"15min":0.69899999999999995},"min":{"1min":0.47999999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.68100000000000005,"15min":0.67900000000000005},"min":{"1min":0.48399999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.8180000000000001,"5min":2.8180000000000001,"15min":2.8180000000000001},"last":2.6629999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:33:02 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.71199999999999997,"15min":0.70999999999999996},"min":{"1min":0.51300000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":2.8450000000000002,"5min":2.8450000000000002,"15min":2.8450000000000002},"last":2.6339999999999999},{"interface":"front","average":{"1min":0.755,"5min":0.71599999999999997,"15min":0.72899999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.448,"5min":2.448,"15min":2.448},"last":2.6739999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795043,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2062248,"kb_used_data":1982240,"kb_used_omap":11,"kb_used_meta":79988,"kb_avail":91682904,"statfs":{"total":95995035648,"available":93883293696,"internally_reserved":0,"allocated":2029813760,"data_stored":2029276777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":81908309},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.38800000000000001,"15min":0.36599999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":2.1320000000000001,"5min":2.1320000000000001,"15min":2.1320000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.40000000000000002,"15min":0.39600000000000002},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.1699999999999999,"5min":1.1699999999999999,"15min":1.1699999999999999},"last":0.93600000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.41399999999999998,"15min":0.39200000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.9390000000000001,"5min":1.9390000000000001,"15min":1.9390000000000001},"last":1.016},{"interface":"front","average":{"1min":0.44,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.4610000000000001,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":3.5619999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.45300000000000001,"15min":0.40100000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.1280000000000001,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":3.6579999999999999},{"interface":"front","average":{"1min":1.3660000000000001,"5min":0.59399999999999997,"15min":0.46500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.9539999999999997,"5min":5.9539999999999997,"15min":5.9539999999999997},"last":0.73099999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.59299999999999997,"15min":0.58099999999999996},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":3.6040000000000001},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.66800000000000004,"15min":0.65000000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.8929999999999998,"5min":2.8929999999999998,"15min":2.8929999999999998},"last":0.443}]},{"osd":5,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.64500000000000002,"15min":0.64300000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.995},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.621,"15min":0.61099999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.7030000000000001,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":3.6299999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.63900000000000001,"15min":0.629},"min":{"1min":0.47699999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":2.0990000000000002,"5min":2.0990000000000002,"15min":2.0990000000000002},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.68400000000000005,"15min":0.68500000000000005},"min":{"1min":0.44800000000000001,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.3819999999999999,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":0.97099999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.68000000000000005,"15min":0.66800000000000004},"min":{"1min":0.46100000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":2.2959999999999998,"5min":2.2959999999999998,"15min":2.2959999999999998},"last":1.5209999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.65400000000000003,"15min":0.65700000000000003},"min":{"1min":0.438,"5min":0.435,"15min":0.435},"max":{"1min":1.2190000000000001,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":0.52000000000000002}]}]},{"osd":3,"up_from":103,"seq":442381631517,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1012504,"kb_used_data":976848,"kb_used_omap":4,"kb_used_meta":35643,"kb_avail":92732648,"statfs":{"total":95995035648,"available":94958231552,"internally_reserved":0,"allocated":1000292352,"data_stored":999769351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":36499038},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.504,"15min":0.434},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.1310000000000002,"5min":5.1310000000000002,"15min":5.1310000000000002},"last":3.6360000000000001},{"interface":"front","average":{"1min":1.208,"5min":0.54200000000000004,"15min":0.43099999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":6.4089999999999998,"5min":6.4089999999999998,"15min":6.4089999999999998},"last":0.26800000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.443,"15min":0.41699999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":4.4619999999999997},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.39500000000000002,"15min":0.38200000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"last":1.4570000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":6.9960000000000004,"5min":1.7410000000000001,"15min":0.86499999999999999},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":117.23,"5min":117.23,"15min":117.23},"last":2.3140000000000001},{"interface":"front","average":{"1min":7.6550000000000002,"5min":1.883,"15min":0.92100000000000004},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":117.45999999999999,"5min":117.45999999999999,"15min":117.45999999999999},"last":0.78300000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.036,"15min":0.74399999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":32.987000000000002,"5min":32.987000000000002,"15min":32.987000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.044,"5min":0.72699999999999998,"15min":0.67400000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":3.8610000000000002,"5min":3.8610000000000002,"15min":3.8610000000000002},"last":1.381}]},{"osd":5,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.71499999999999997,"15min":0.66600000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.2919999999999998,"5min":4.2919999999999998,"15min":4.2919999999999998},"last":1.49},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.74099999999999999,"15min":0.69899999999999995},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.4809999999999999,"5min":4.4809999999999999,"15min":4.4809999999999999},"last":1.421}]},{"osd":6,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":0.80500000000000005,"15min":0.71599999999999997},"min":{"1min":0.51000000000000001,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":3.9020000000000001,"5min":3.9020000000000001,"15min":3.9020000000000001},"last":1.548},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.64400000000000002,"15min":0.629},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.978,"5min":1.978,"15min":1.978},"last":1.5209999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.79200000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019065,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3077120,"kb_used_data":2779896,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":90668032,"statfs":{"total":95995035648,"available":92844064768,"internally_reserved":0,"allocated":2846613504,"data_stored":2846112175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52300000000000002,"15min":0.52100000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.22,"5min":1.22,"15min":1.22},"last":3.9129999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":9.4830000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:33:37 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.57199999999999995,"15min":0.54800000000000004},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.702,"5min":4.702,"15min":4.702},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.01,"5min":0.65300000000000002,"15min":0.58999999999999997},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.7409999999999997,"5min":4.7409999999999997,"15min":4.7409999999999997},"last":9.5380000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.379,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.505},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.57999999999999996,"15min":0.58299999999999996},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.554,"5min":1.554,"15min":1.554},"last":3.7549999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.60699999999999998,"15min":0.57499999999999996},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.8460000000000001,"5min":3.8460000000000001,"15min":3.8460000000000001},"last":9.5909999999999993},{"interface":"front","average":{"1min":1.091,"5min":0.70799999999999996,"15min":0.64400000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":6.0970000000000004,"5min":6.0970000000000004,"15min":6.0970000000000004},"last":4.0190000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.53600000000000003,"15min":0.47199999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.653,"5min":2.653,"15min":2.653},"last":9.6470000000000002},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.59499999999999997,"15min":0.55200000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.311},"max":{"1min":2.5179999999999998,"5min":2.5179999999999998,"15min":2.5179999999999998},"last":3.7930000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:33:43 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.55100000000000005,"15min":0.50600000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":1.978,"5min":1.978,"15min":1.978},"last":3.8140000000000001},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.58999999999999997,"15min":0.53100000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":3.0510000000000002,"5min":3.0510000000000002,"15min":3.0510000000000002},"last":9.6820000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:33:09 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.55400000000000005,"15min":0.54200000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":1.7470000000000001,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.53600000000000003,"15min":0.53100000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.58199999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822837,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3452656,"kb_used_data":3143144,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":90292496,"statfs":{"total":95995035648,"available":92459515904,"internally_reserved":0,"allocated":3218579456,"data_stored":3218089959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":2.5009999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.56999999999999995,"15min":0.56000000000000005},"min":{"1min":0.38900000000000001,"5min":0.371,"15min":0.371},"max":{"1min":1.9730000000000001,"5min":1.9730000000000001,"15min":1.9730000000000001},"last":0.28999999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.59699999999999998,"15min":0.56899999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.0190000000000001,"5min":3.0190000000000001,"15min":3.0190000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.0800000000000001,"5min":0.67100000000000004,"15min":0.59499999999999997},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.1740000000000004,"5min":5.1740000000000004,"15min":5.1740000000000004},"last":0.84299999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:33:07 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.7709999999999999,"5min":1.7709999999999999,"15min":1.7709999999999999},"last":4.0229999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.69799999999999995,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.29899999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:33:35 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.67800000000000005,"15min":0.621},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.375,"5min":3.375,"15min":3.375},"last":2.0790000000000002},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.64600000000000002,"15min":0.60599999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.1709999999999998,"5min":3.1709999999999998,"15min":3.1709999999999998},"last":2.5299999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:33:28 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.501,"15min":0.46800000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26200000000000001},"max":{"1min":1.8340000000000001,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.54700000000000004,"15min":0.50800000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.9810000000000001,"5min":1.9810000000000001,"15min":1.9810000000000001},"last":2.9769999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:32:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.55900000000000005,"15min":0.51500000000000001},"min":{"1min":0.373,"5min":0.34999999999999998,"15min":0.26000000000000001},"max":{"1min":0.72699999999999998,"5min":0.93500000000000005,"15min":1.1339999999999999},"last":2.4590000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.56499999999999995,"15min":0.53700000000000003},"min":{"1min":0.40899999999999997,"5min":0.36199999999999999,"15min":0.314},"max":{"1min":0.754,"5min":0.92100000000000004,"15min":1.109},"last":0.38800000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:33:12 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.59999999999999998,"15min":0.58099999999999996},"min":{"1min":0.44400000000000001,"5min":0.40200000000000002,"15min":0.34399999999999997},"max":{"1min":1.5249999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.58799999999999997,"15min":0.56000000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32400000000000001},"max":{"1min":1.8600000000000001,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":0.42999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2804023296,"data_stored":2804023296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1767702528,"data_stored":1767702528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2048000000,"data_stored":2048000000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1012793344,"data_stored":1012793344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2857959424,"data_stored":2857959424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3220373504,"data_stored":3220373504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1418067968,"data_stored":1418067968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1087963136,"data_stored":1087963136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:33:52.693 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:33:52.693 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 26 2023-10-09T05:33:52.900 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:52.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:52.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:52.955 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:33:53.010 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:53.014 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:52 smithi008 ceph-mon[184418]: pgmap v278: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 27371/261963 objects degraded (10.448%) 2023-10-09T05:33:53.014 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4094969378' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:33:53.015 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:52 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 27371/261963 objects degraded (10.448%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:53.015 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:52 smithi008 ceph-mon[191227]: pgmap v278: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 27371/261963 objects degraded (10.448%) 2023-10-09T05:33:53.015 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4094969378' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:33:53.015 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:52 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 27371/261963 objects degraded (10.448%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:53.114 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:53.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:52 smithi191 ceph-mon[161669]: pgmap v278: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 27371/261963 objects degraded (10.448%) 2023-10-09T05:33:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4094969378' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:33:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:52 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 27371/261963 objects degraded (10.448%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:53.505 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 61 16 91002 90986 93.176 63.1875 0.00669025 0.0107274 2023-10-09T05:33:54.081 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:33:54.082 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:33:54.143 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:53 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:54.143 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4094969378' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:33:54.144 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:53 smithi008 ceph-mon[184418]: osdmap e116: 8 total, 7 up, 8 in 2023-10-09T05:33:54.144 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:53 smithi008 ceph-mon[184418]: failed to encode map e116 with expected crc 2023-10-09T05:33:54.144 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:53 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:54.144 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4094969378' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:33:54.144 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:53 smithi008 ceph-mon[191227]: osdmap e116: 8 total, 7 up, 8 in 2023-10-09T05:33:54.144 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:53 smithi008 ceph-mon[191227]: failed to encode map e116 with expected crc 2023-10-09T05:33:54.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:53 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:33:54.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4094969378' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:33:54.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:53 smithi191 ceph-mon[161669]: osdmap e116: 8 total, 7 up, 8 in 2023-10-09T05:33:54.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:53 smithi191 ceph-mon[161669]: failed to encode map e116 with expected crc 2023-10-09T05:33:54.505 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 62 16 92162 92146 92.8426 72.5 0.00900912 0.0107662 2023-10-09T05:33:54.737 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:33:54.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:54.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:54.845 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:54.892 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:54.980 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:54.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:54.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:55.025 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:54 smithi008 ceph-mon[184418]: pgmap v280: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 27371/264822 objects degraded (10.336%) 2023-10-09T05:33:55.028 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:54 smithi008 ceph-mon[191227]: pgmap v280: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 27371/264822 objects degraded (10.336%) 2023-10-09T05:33:55.089 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:33:55.125 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:55.171 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:33:55.175 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:54 smithi191 ceph-mon[161669]: pgmap v280: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 27371/264822 objects degraded (10.336%) 2023-10-09T05:33:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:55.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:33:55.277 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:55.367 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:55.512 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 63 16 93133 93117 92.3224 60.6875 0.0191693 0.0108273 2023-10-09T05:33:55.619 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:33:55.620 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:33:55.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:55.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:55.733 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:33:55.801 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:55.850 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:56.003 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:33:56.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:56.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:33:56.109 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:56.175 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:33:56.211 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:56.383 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:56.390 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:33:56.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:56.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:56.431 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:33:56.485 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:33:56.497 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:56.513 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 64 16 94024 94008 91.749 55.6875 0.0144294 0.0108941 2023-10-09T05:33:56.516 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:33:56.547 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:56.563 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:33:56.602 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:56.603 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:33:56.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:56.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:56.644 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:33:56.653 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:33:56.653 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:33:56.654 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:33:56.654 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:33:56.654 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:33:56.654 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:33:56.654 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:33:56.654 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T05:33:56.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:33:56.741 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:56.818 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:33:56.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:56.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:33:56.924 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:57.004 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:56 smithi008 ceph-mon[191227]: pgmap v281: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 28952/277221 objects degraded (10.444%) 2023-10-09T05:33:57.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:56 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2587424773' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:57.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:56 smithi008 ceph-mon[184418]: pgmap v281: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 28952/277221 objects degraded (10.444%) 2023-10-09T05:33:57.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:56 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2587424773' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:56 smithi191 ceph-mon[161669]: pgmap v281: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 28952/277221 objects degraded (10.444%) 2023-10-09T05:33:57.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:56 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2587424773' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:33:57.517 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 65 15 95063 95048 91.332 65 0.0144461 0.0109438 2023-10-09T05:33:57.771 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:57.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:57.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:57.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:33:57.908 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:57.949 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:33:57.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:57.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:33:58.054 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:33:58.115 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:57 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 28952/277221 objects degraded (10.444%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:58.115 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:57 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 28952/277221 objects degraded (10.444%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:58.130 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:57 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 28952/277221 objects degraded (10.444%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:33:58.383 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:58.390 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:33:58.404 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T05:33:58.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:58.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:33:58.405 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T05:33:58.506 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:58.517 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 66 16 96055 96039 90.8868 61.9375 0.0170441 0.0109979 2023-10-09T05:33:58.611 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:58.896 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:33:59.021 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:33:59.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:33:59.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:33:59.134 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:33:59.211 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:33:59.517 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 67 16 97015 96999 90.426 60 0.0158952 0.011054 2023-10-09T05:33:59.591 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 5 pg_num to 26 2023-10-09T05:33:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:59 smithi008 ceph-mon[184418]: pgmap v282: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 28952/279174 objects degraded (10.371%) 2023-10-09T05:33:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4133055694' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:33:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:33:59 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:33:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:59 smithi008 ceph-mon[191227]: pgmap v282: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 28952/279174 objects degraded (10.371%) 2023-10-09T05:33:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:33:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4133055694' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:33:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:33:59 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:34:00.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:59 smithi191 ceph-mon[161669]: pgmap v282: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 28952/279174 objects degraded (10.371%) 2023-10-09T05:34:00.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4133055694' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:34:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:33:59 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:34:00.407 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:34:00.517 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 68 16 98072 98056 90.0677 66.0625 0.0118383 0.0110978 2023-10-09T05:34:00.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:34:00.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[184418]: osdmap e117: 8 total, 7 up, 8 in 2023-10-09T05:34:00.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[184418]: failed to encode map e117 with expected crc 2023-10-09T05:34:00.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[184418]: failed to encode map e117 with expected crc 2023-10-09T05:34:00.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[184418]: failed to encode map e117 with expected crc 2023-10-09T05:34:00.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:34:00.718 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:34:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:34:00] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.33.4" 2023-10-09T05:34:00.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:34:00.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[191227]: osdmap e117: 8 total, 7 up, 8 in 2023-10-09T05:34:00.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[191227]: failed to encode map e117 with expected crc 2023-10-09T05:34:00.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[191227]: failed to encode map e117 with expected crc 2023-10-09T05:34:00.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[191227]: failed to encode map e117 with expected crc 2023-10-09T05:34:00.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:00 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:34:00.732 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:00.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:00.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:00.851 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:00.937 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:01.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:00 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:34:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:00 smithi191 ceph-mon[161669]: osdmap e117: 8 total, 7 up, 8 in 2023-10-09T05:34:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:00 smithi191 ceph-mon[161669]: failed to encode map e117 with expected crc 2023-10-09T05:34:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:00 smithi191 ceph-mon[161669]: failed to encode map e117 with expected crc 2023-10-09T05:34:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:00 smithi191 ceph-mon[161669]: failed to encode map e117 with expected crc 2023-10-09T05:34:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:00 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:34:01.518 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 69 16 98138 98122 88.8229 4.125 0.0285266 0.0111149 2023-10-09T05:34:01.609 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:01.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:01.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:01.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:01.783 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:01.868 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:01.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:01.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:01.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:01 smithi191 ceph-mon[161669]: pgmap v284: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 29563/285567 objects degraded (10.352%) 2023-10-09T05:34:01.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:01 smithi191 ceph-mon[161669]: failed to encode map e118 with expected crc 2023-10-09T05:34:01.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:01 smithi191 ceph-mon[161669]: failed to encode map e118 with expected crc 2023-10-09T05:34:01.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:01 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:34:01.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:01 smithi191 ceph-mon[161669]: osdmap e118: 8 total, 7 up, 8 in 2023-10-09T05:34:01.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:01 smithi008 ceph-mon[184418]: pgmap v284: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 29563/285567 objects degraded (10.352%) 2023-10-09T05:34:01.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:01 smithi008 ceph-mon[184418]: failed to encode map e118 with expected crc 2023-10-09T05:34:01.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:01 smithi008 ceph-mon[184418]: failed to encode map e118 with expected crc 2023-10-09T05:34:01.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:01 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:34:01.931 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:01 smithi008 ceph-mon[184418]: osdmap e118: 8 total, 7 up, 8 in 2023-10-09T05:34:01.931 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:01 smithi008 ceph-mon[191227]: pgmap v284: 89 pgs: 16 active+undersized+degraded, 19 active+undersized, 54 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 29563/285567 objects degraded (10.352%) 2023-10-09T05:34:01.931 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:01 smithi008 ceph-mon[191227]: failed to encode map e118 with expected crc 2023-10-09T05:34:01.932 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:01 smithi008 ceph-mon[191227]: failed to encode map e118 with expected crc 2023-10-09T05:34:01.932 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:01 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:34:01.932 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:01 smithi008 ceph-mon[191227]: osdmap e118: 8 total, 7 up, 8 in 2023-10-09T05:34:01.978 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:02.019 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:02.095 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:02.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:02.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:02.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:02.238 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:02.273 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:34:02.273 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:34:02.273 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:34:02.273 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:34:02.273 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:34:02.303 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:02.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:02.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:02.315 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:02.408 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:02.520 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:02.521 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 70 16 99217 99201 88.513 67.4375 0.0237576 0.0112937 2023-10-09T05:34:02.599 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[191227]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.599 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[191227]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.600 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[191227]: osdmap e119: 8 total, 7 up, 8 in 2023-10-09T05:34:02.600 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[191227]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.600 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[191227]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.600 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[191227]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.600 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[191227]: pgmap v287: 99 pgs: 10 unknown, 4 peering, 14 active+undersized+degraded, 19 active+undersized, 52 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 18281/292008 objects degraded (6.260%) 2023-10-09T05:34:02.600 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 29563/285567 objects degraded (10.352%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:02.736 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:02.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:02.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:02.842 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:02.882 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[184418]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.883 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[184418]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.883 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[184418]: osdmap e119: 8 total, 7 up, 8 in 2023-10-09T05:34:02.883 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[184418]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.883 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[184418]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.883 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[184418]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.883 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[184418]: pgmap v287: 99 pgs: 10 unknown, 4 peering, 14 active+undersized+degraded, 19 active+undersized, 52 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 18281/292008 objects degraded (6.260%) 2023-10-09T05:34:02.883 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:02 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 29563/285567 objects degraded (10.352%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:02.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:02 smithi191 ceph-mon[161669]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:02 smithi191 ceph-mon[161669]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:02 smithi191 ceph-mon[161669]: osdmap e119: 8 total, 7 up, 8 in 2023-10-09T05:34:02.884 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:02 smithi191 ceph-mon[161669]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.884 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:02 smithi191 ceph-mon[161669]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.884 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:02 smithi191 ceph-mon[161669]: failed to encode map e119 with expected crc 2023-10-09T05:34:02.884 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:02 smithi191 ceph-mon[161669]: pgmap v287: 99 pgs: 10 unknown, 4 peering, 14 active+undersized+degraded, 19 active+undersized, 52 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 18281/292008 objects degraded (6.260%) 2023-10-09T05:34:02.884 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:02 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 29563/285567 objects degraded (10.352%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:02.885 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:02.962 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:02.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:02.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:03.076 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:03.142 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:03.306 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:03.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:03.411 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:03.452 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:03.521 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 71 16 100500 100484 88.3957 80.1875 0.0188992 0.0113086 2023-10-09T05:34:03.539 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:03.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:03.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:03.604 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:34:03.648 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:03.676 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:03.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:03 smithi191 ceph-mon[161669]: 5.0 scrub starts 2023-10-09T05:34:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:03 smithi191 ceph-mon[161669]: 5.7 scrub starts 2023-10-09T05:34:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:03 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/418030987' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:03 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:03.871 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:03 smithi008 ceph-mon[191227]: 5.0 scrub starts 2023-10-09T05:34:03.871 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:03 smithi008 ceph-mon[191227]: 5.7 scrub starts 2023-10-09T05:34:03.871 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:03 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/418030987' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:03.871 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:03 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:03.872 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:03 smithi008 ceph-mon[184418]: 5.0 scrub starts 2023-10-09T05:34:03.872 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:03 smithi008 ceph-mon[184418]: 5.7 scrub starts 2023-10-09T05:34:03.872 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:03 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/418030987' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:03.873 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:03 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:04.171 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:04.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:04.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:04.281 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:04.316 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:04.404 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:04.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:04.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:04.464 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:34:04.516 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:04.521 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 72 16 101770 101754 88.2703 79.375 0.0120297 0.0113245 2023-10-09T05:34:04.595 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:04.632 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:04 smithi191 ceph-mon[161669]: 5.0 scrub ok 2023-10-09T05:34:04.633 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:04 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:34:04.633 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:04 smithi191 ceph-mon[161669]: failed to encode map e120 with expected crc 2023-10-09T05:34:04.633 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:04 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:04.633 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:04 smithi191 ceph-mon[161669]: osdmap e120: 8 total, 7 up, 8 in 2023-10-09T05:34:04.633 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:04 smithi191 ceph-mon[161669]: failed to encode map e120 with expected crc 2023-10-09T05:34:04.633 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:04 smithi191 ceph-mon[161669]: pgmap v289: 99 pgs: 8 unknown, 4 peering, 14 active+undersized+degraded, 19 active+undersized, 54 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 18281/294468 objects degraded (6.208%) 2023-10-09T05:34:04.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[184418]: 5.0 scrub ok 2023-10-09T05:34:04.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:34:04.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[184418]: failed to encode map e120 with expected crc 2023-10-09T05:34:04.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:04.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[184418]: osdmap e120: 8 total, 7 up, 8 in 2023-10-09T05:34:04.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[184418]: failed to encode map e120 with expected crc 2023-10-09T05:34:04.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[184418]: pgmap v289: 99 pgs: 8 unknown, 4 peering, 14 active+undersized+degraded, 19 active+undersized, 54 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 18281/294468 objects degraded (6.208%) 2023-10-09T05:34:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[191227]: 5.0 scrub ok 2023-10-09T05:34:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:34:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[191227]: failed to encode map e120 with expected crc 2023-10-09T05:34:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[191227]: osdmap e120: 8 total, 7 up, 8 in 2023-10-09T05:34:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[191227]: failed to encode map e120 with expected crc 2023-10-09T05:34:04.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:04 smithi008 ceph-mon[191227]: pgmap v289: 99 pgs: 8 unknown, 4 peering, 14 active+undersized+degraded, 19 active+undersized, 54 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 18281/294468 objects degraded (6.208%) 2023-10-09T05:34:04.807 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:04.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:04.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:04.918 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:04.959 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:05.058 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:05.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:05.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:05.173 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:05.199 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:05.521 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 73 16 102866 102850 87.9996 68.5 0.0174204 0.0113595 2023-10-09T05:34:05.730 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:05.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:05.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:05.842 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:05.903 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:05 smithi008 ceph-mon[191227]: 5.2 scrub starts 2023-10-09T05:34:05.904 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:05 smithi008 ceph-mon[184418]: 5.2 scrub starts 2023-10-09T05:34:05.913 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:06.029 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:05 smithi191 ceph-mon[161669]: 5.2 scrub starts 2023-10-09T05:34:06.129 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:06.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:06.235 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:06.311 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:06.483 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:06.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:06.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:06.522 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 74 16 103989 103973 87.7589 70.1875 0.00967074 0.0113887 2023-10-09T05:34:06.540 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:34:06.588 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:06.592 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:34:06.642 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:34:06.662 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:06.677 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2023-10-09T05:34:06.677 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:34:06.677 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-10-09T05:34:06.678 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:34:06.683 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:34:06.717 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:34:06.786 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:34:06.818 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:34:06.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:34:06.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:34:06.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:34:06.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:34:06.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:34:06.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:34:06.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:34:06.825 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T05:34:06.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:06 smithi008 ceph-mon[184418]: 5.2 scrub ok 2023-10-09T05:34:06.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:06 smithi008 ceph-mon[184418]: 5.12 scrub starts 2023-10-09T05:34:06.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:06 smithi008 ceph-mon[184418]: pgmap v290: 99 pgs: 1 active+clean+scrubbing, 1 unknown, 1 peering, 20 active+undersized+degraded, 19 active+undersized, 57 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 32028/305673 objects degraded (10.478%); 43 KiB/s, 0 objects/s recovering 2023-10-09T05:34:06.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:06 smithi008 ceph-mon[184418]: 5.8 scrub starts 2023-10-09T05:34:06.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/813388579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:06.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:06 smithi008 ceph-mon[191227]: 5.2 scrub ok 2023-10-09T05:34:06.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:06 smithi008 ceph-mon[191227]: 5.12 scrub starts 2023-10-09T05:34:06.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:06 smithi008 ceph-mon[191227]: pgmap v290: 99 pgs: 1 active+clean+scrubbing, 1 unknown, 1 peering, 20 active+undersized+degraded, 19 active+undersized, 57 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 32028/305673 objects degraded (10.478%); 43 KiB/s, 0 objects/s recovering 2023-10-09T05:34:06.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:06 smithi008 ceph-mon[191227]: 5.8 scrub starts 2023-10-09T05:34:06.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/813388579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:07.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:06 smithi191 ceph-mon[161669]: 5.2 scrub ok 2023-10-09T05:34:07.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:06 smithi191 ceph-mon[161669]: 5.12 scrub starts 2023-10-09T05:34:07.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:06 smithi191 ceph-mon[161669]: pgmap v290: 99 pgs: 1 active+clean+scrubbing, 1 unknown, 1 peering, 20 active+undersized+degraded, 19 active+undersized, 57 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 32028/305673 objects degraded (10.478%); 43 KiB/s, 0 objects/s recovering 2023-10-09T05:34:07.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:06 smithi191 ceph-mon[161669]: 5.8 scrub starts 2023-10-09T05:34:07.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/813388579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:07.128 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:07.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:07.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:07.236 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:07.266 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:07.307 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:07.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:07.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:07.413 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:07.490 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:07.522 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 75 16 104989 104973 87.4218 62.5 0.014316 0.011434 2023-10-09T05:34:07.735 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:34:07.780 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:07 smithi008 ceph-mon[184418]: 5.10 scrub starts 2023-10-09T05:34:07.780 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:07 smithi008 ceph-mon[184418]: 5.12 scrub ok 2023-10-09T05:34:07.780 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:07 smithi008 ceph-mon[184418]: 5.10 scrub ok 2023-10-09T05:34:07.780 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:07 smithi008 ceph-mon[184418]: 5.18 scrub starts 2023-10-09T05:34:07.780 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:07 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 32028/305673 objects degraded (10.478%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:07.783 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:07 smithi008 ceph-mon[191227]: 5.10 scrub starts 2023-10-09T05:34:07.783 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:07 smithi008 ceph-mon[191227]: 5.12 scrub ok 2023-10-09T05:34:07.783 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:07 smithi008 ceph-mon[191227]: 5.10 scrub ok 2023-10-09T05:34:07.783 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:07 smithi008 ceph-mon[191227]: 5.18 scrub starts 2023-10-09T05:34:07.784 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:07 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 32028/305673 objects degraded (10.478%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:08.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:07 smithi191 ceph-mon[161669]: 5.10 scrub starts 2023-10-09T05:34:08.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:07 smithi191 ceph-mon[161669]: 5.12 scrub ok 2023-10-09T05:34:08.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:07 smithi191 ceph-mon[161669]: 5.10 scrub ok 2023-10-09T05:34:08.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:07 smithi191 ceph-mon[161669]: 5.18 scrub starts 2023-10-09T05:34:08.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:07 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 32028/305673 objects degraded (10.478%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:08.522 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 76 16 106156 106140 87.2312 72.9375 0.00472601 0.0114539 2023-10-09T05:34:08.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:08 smithi008 ceph-mon[184418]: 5.18 scrub ok 2023-10-09T05:34:08.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:08 smithi008 ceph-mon[184418]: pgmap v291: 99 pgs: 1 active+clean+scrubbing, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s; 32028/309060 objects degraded (10.363%); 35 KiB/s, 0 objects/s recovering 2023-10-09T05:34:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:08 smithi008 ceph-mon[191227]: 5.18 scrub ok 2023-10-09T05:34:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:08 smithi008 ceph-mon[191227]: pgmap v291: 99 pgs: 1 active+clean+scrubbing, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s; 32028/309060 objects degraded (10.363%); 35 KiB/s, 0 objects/s recovering 2023-10-09T05:34:08.974 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:08.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:08.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:09.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:08 smithi191 ceph-mon[161669]: 5.18 scrub ok 2023-10-09T05:34:09.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:08 smithi191 ceph-mon[161669]: pgmap v291: 99 pgs: 1 active+clean+scrubbing, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s; 32028/309060 objects degraded (10.363%); 35 KiB/s, 0 objects/s recovering 2023-10-09T05:34:09.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:09.116 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:09.193 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:09.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:09.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:09.290 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T05:34:09.291 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T05:34:09.304 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:09.373 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:09.522 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 77 16 107158 107142 86.9118 62.625 0.00955429 0.0115021 2023-10-09T05:34:09.588 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:09.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:09.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:09.698 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:09.702 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:09.736 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:09.755 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:09.797 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:09.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:09.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:09.854 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:34:09.854 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:34:09.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:09 smithi008 ceph-mon[184418]: 5.3 scrub starts 2023-10-09T05:34:09.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:09 smithi008 ceph-mon[184418]: 5.3 scrub ok 2023-10-09T05:34:09.890 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:09 smithi008 ceph-mon[191227]: 5.3 scrub starts 2023-10-09T05:34:09.890 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:09 smithi008 ceph-mon[191227]: 5.3 scrub ok 2023-10-09T05:34:09.910 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:09.976 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:10.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:09 smithi191 ceph-mon[161669]: 5.3 scrub starts 2023-10-09T05:34:10.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:09 smithi191 ceph-mon[161669]: 5.3 scrub ok 2023-10-09T05:34:10.526 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 78 16 108362 108346 86.7602 75.25 0.012601 0.0115215 2023-10-09T05:34:10.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:10 smithi008 ceph-mon[184418]: pgmap v292: 99 pgs: 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32693/316083 objects degraded (10.343%); 31 KiB/s, 0 objects/s recovering 2023-10-09T05:34:10.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:10 smithi008 ceph-mon[191227]: pgmap v292: 99 pgs: 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32693/316083 objects degraded (10.343%); 31 KiB/s, 0 objects/s recovering 2023-10-09T05:34:10.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:34:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:34:10] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.33.4" 2023-10-09T05:34:11.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:10 smithi191 ceph-mon[161669]: pgmap v292: 99 pgs: 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32693/316083 objects degraded (10.343%); 31 KiB/s, 0 objects/s recovering 2023-10-09T05:34:11.292 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:34:11.525 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 79 16 109397 109381 86.4806 64.6875 0.0116213 0.0115591 2023-10-09T05:34:11.657 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:11.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:11.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:11.768 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:11.854 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:11.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:11 smithi008 ceph-mon[184418]: 5.5 scrub starts 2023-10-09T05:34:11.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:11 smithi008 ceph-mon[184418]: 5.5 scrub ok 2023-10-09T05:34:11.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:11 smithi008 ceph-mon[184418]: 5.8 deep-scrub starts 2023-10-09T05:34:11.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:11 smithi008 ceph-mon[184418]: 5.7 scrub starts 2023-10-09T05:34:11.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:11 smithi008 ceph-mon[191227]: 5.5 scrub starts 2023-10-09T05:34:11.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:11 smithi008 ceph-mon[191227]: 5.5 scrub ok 2023-10-09T05:34:11.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:11 smithi008 ceph-mon[191227]: 5.8 deep-scrub starts 2023-10-09T05:34:11.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:11 smithi008 ceph-mon[191227]: 5.7 scrub starts 2023-10-09T05:34:12.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:11 smithi191 ceph-mon[161669]: 5.5 scrub starts 2023-10-09T05:34:12.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:11 smithi191 ceph-mon[161669]: 5.5 scrub ok 2023-10-09T05:34:12.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:11 smithi191 ceph-mon[161669]: 5.8 deep-scrub starts 2023-10-09T05:34:12.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:11 smithi191 ceph-mon[161669]: 5.7 scrub starts 2023-10-09T05:34:12.447 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:34:12.530 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:12.530 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:34:12.530655+0000 min lat: 0.00107405 max lat: 1.08589 avg lat: 0.0116087 2023-10-09T05:34:12.530 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:34:12.530 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 80 16 110294 110278 86.0951 56.0625 0.00623048 0.0116087 2023-10-09T05:34:12.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:12.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:12.642 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:12.751 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:12.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:12 smithi191 ceph-mon[161669]: pgmap v293: 99 pgs: 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33707/323265 objects degraded (10.427%); 19 KiB/s, 0 objects/s recovering 2023-10-09T05:34:12.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:12 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 32693/316083 objects degraded (10.343%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:12 smithi191 ceph-mon[161669]: 5.7 scrub starts 2023-10-09T05:34:12.917 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:12.952 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:12.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:12 smithi008 ceph-mon[184418]: pgmap v293: 99 pgs: 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33707/323265 objects degraded (10.427%); 19 KiB/s, 0 objects/s recovering 2023-10-09T05:34:12.955 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:12 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 32693/316083 objects degraded (10.343%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:12.955 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:12 smithi008 ceph-mon[184418]: 5.7 scrub starts 2023-10-09T05:34:12.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:12 smithi008 ceph-mon[191227]: pgmap v293: 99 pgs: 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33707/323265 objects degraded (10.427%); 19 KiB/s, 0 objects/s recovering 2023-10-09T05:34:12.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:12 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 32693/316083 objects degraded (10.343%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:12.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:12 smithi008 ceph-mon[191227]: 5.7 scrub starts 2023-10-09T05:34:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:12.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:13.057 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:13.097 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:13.230 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:13.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:13.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:13.334 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:13.424 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:13.534 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 81 16 111250 111234 85.7657 59.75 0.00618975 0.0116552 2023-10-09T05:34:13.712 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:13.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:13.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:13.828 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:13.901 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:13.954 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:13.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:13 smithi008 ceph-mon[184418]: 5.7 scrub starts 2023-10-09T05:34:13.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:13.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:13 smithi008 ceph-mon[191227]: 5.7 scrub starts 2023-10-09T05:34:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:13 smithi191 ceph-mon[161669]: 5.7 scrub starts 2023-10-09T05:34:14.333 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:14.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:14.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:14.395 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:34:14.413 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:34:14.446 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:14.489 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:14.536 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 82 16 112072 112056 85.3441 51.375 0.0452842 0.0117114 2023-10-09T05:34:14.885 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:14.885 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:14.885 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:14 smithi008 ceph-mon[184418]: pgmap v294: 99 pgs: 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33707/326157 objects degraded (10.335%); 19 KiB/s, 0 objects/s recovering 2023-10-09T05:34:14.885 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:14 smithi008 ceph-mon[184418]: from='client.44762 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:34:15.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:15.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:15.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:14 smithi008 ceph-mon[191227]: pgmap v294: 99 pgs: 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33707/326157 objects degraded (10.335%); 19 KiB/s, 0 objects/s recovering 2023-10-09T05:34:15.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:14 smithi008 ceph-mon[191227]: from='client.44762 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:34:15.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:15.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:14 smithi191 ceph-mon[161669]: pgmap v294: 99 pgs: 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33707/326157 objects degraded (10.335%); 19 KiB/s, 0 objects/s recovering 2023-10-09T05:34:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:14 smithi191 ceph-mon[161669]: from='client.44762 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:34:15.536 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 83 16 113018 113002 85.0283 59.125 0.0173251 0.0117557 2023-10-09T05:34:15.916 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:34:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3999884980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:15 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:16.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3999884980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:16.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:15 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3999884980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:15 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:16.539 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 84 16 113810 113794 84.6035 49.5 0.00585879 0.0118155 2023-10-09T05:34:16.568 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:16.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:16.671 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:16.703 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:16.727 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":294,"stamp":"2023-10-09T05:34:13.910926+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7127996339,"num_objects":108719,"num_object_clones":0,"num_object_copies":326157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33707,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":108719,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":86972,"num_write_kb":5510597,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170856,"ondisk_log_size":170856,"up":258,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":293,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21256968,"kb_used_data":19692168,"kb_used_omap":31,"kb_used_meta":1564704,"kb_avail":728704248,"statfs":{"total":767960285184,"available":746193149952,"internally_reserved":0,"allocated":20164780032,"data_stored":20160569445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":1602257353},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":692289536,"num_objects":10563,"num_object_clones":0,"num_object_copies":31689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15426,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-12262,"num_write_kb":-784736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.312104"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":103,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.589854+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:34:01.589854+0000","last_peered":"2023-10-09T05:34:01.589854+0000","last_clean":"2023-10-09T05:34:01.589854+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:34:01.589854+0000","last_undegraded":"2023-10-09T05:34:01.589854+0000","last_fullsized":"2023-10-09T05:34:01.589854+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":90,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.584276+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:34:01.584276+0000","last_peered":"2023-10-09T05:34:01.584276+0000","last_clean":"2023-10-09T05:34:01.584276+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:34:01.584276+0000","last_undegraded":"2023-10-09T05:34:01.584276+0000","last_fullsized":"2023-10-09T05:34:01.584276+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":64,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.588961+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:34:01.588961+0000","last_peered":"2023-10-09T05:34:01.588961+0000","last_clean":"2023-10-09T05:34:01.588961+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:34:01.588961+0000","last_undegraded":"2023-10-09T05:34:01.588961+0000","last_fullsized":"2023-10-09T05:34:01.588961+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":76,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.585770+0000","last_change":"2023-10-09T05:33:21.537914+0000","last_active":"2023-10-09T05:34:01.585770+0000","last_peered":"2023-10-09T05:34:01.585770+0000","last_clean":"2023-10-09T05:33:12.601246+0000","last_became_active":"2023-10-09T05:33:21.537914+0000","last_became_peered":"2023-10-09T05:33:21.537914+0000","last_unstale":"2023-10-09T05:34:01.585770+0000","last_undegraded":"2023-10-09T05:34:01.585770+0000","last_fullsized":"2023-10-09T05:33:21.532713+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:18.451448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":106,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.583881+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:34:01.583881+0000","last_peered":"2023-10-09T05:34:01.583881+0000","last_clean":"2023-10-09T05:34:01.583881+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:34:01.583881+0000","last_undegraded":"2023-10-09T05:34:01.583881+0000","last_fullsized":"2023-10-09T05:34:01.583881+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":86,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.586126+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:34:01.586126+0000","last_peered":"2023-10-09T05:34:01.586126+0000","last_clean":"2023-10-09T05:34:01.586126+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:34:01.586126+0000","last_undegraded":"2023-10-09T05:34:01.586126+0000","last_fullsized":"2023-10-09T05:34:01.586126+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.11","version":"120'6328","reported_seq":6363,"reported_epoch":120,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:09.753849+0000","last_change":"2023-10-09T05:34:01.614652+0000","last_active":"2023-10-09T05:34:09.753849+0000","last_peered":"2023-10-09T05:34:09.753849+0000","last_clean":"2023-10-09T05:33:20.516527+0000","last_became_active":"2023-10-09T05:34:01.614652+0000","last_became_peered":"2023-10-09T05:34:01.614652+0000","last_unstale":"2023-10-09T05:34:09.753849+0000","last_undegraded":"2023-10-09T05:34:01.605619+0000","last_fullsized":"2023-10-09T05:34:01.587400+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6328,"log_dups_size":0,"ondisk_log_size":6328,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216956928,"num_objects":3311,"num_object_clones":0,"num_object_copies":9933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3311,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3311,"num_write_kb":211872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":3311}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":69,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.587467+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:34:01.587467+0000","last_peered":"2023-10-09T05:34:01.587467+0000","last_clean":"2023-10-09T05:34:01.587467+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:34:01.587467+0000","last_undegraded":"2023-10-09T05:34:01.587467+0000","last_fullsized":"2023-10-09T05:34:01.587467+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":90,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.583783+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:34:01.583783+0000","last_peered":"2023-10-09T05:34:01.583783+0000","last_clean":"2023-10-09T05:34:01.583783+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:34:01.583783+0000","last_undegraded":"2023-10-09T05:34:01.583783+0000","last_fullsized":"2023-10-09T05:34:01.583783+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.10","version":"120'6394","reported_seq":6502,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:10.898270+0000","last_change":"2023-10-09T05:34:06.833835+0000","last_active":"2023-10-09T05:34:10.898270+0000","last_peered":"2023-10-09T05:34:10.898270+0000","last_clean":"2023-10-09T05:34:10.898270+0000","last_became_active":"2023-10-09T05:34:01.635329+0000","last_became_peered":"2023-10-09T05:34:01.635329+0000","last_unstale":"2023-10-09T05:34:10.898270+0000","last_undegraded":"2023-10-09T05:34:10.898270+0000","last_fullsized":"2023-10-09T05:34:10.898270+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6258","last_scrub_stamp":"2023-10-09T05:34:06.833681+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:06.833681+0000","objects_scrubbed":3155,"log_size":6394,"log_dups_size":0,"ondisk_log_size":6394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:45.971615+0000","scrub_duration":0.69237216700000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216137728,"num_objects":3298,"num_object_clones":0,"num_object_copies":9894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":143,"num_write_kb":9152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":93,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.590401+0000","last_change":"2023-10-09T05:33:21.536321+0000","last_active":"2023-10-09T05:34:01.590401+0000","last_peered":"2023-10-09T05:34:01.590401+0000","last_clean":"2023-10-09T05:33:12.591820+0000","last_became_active":"2023-10-09T05:33:21.536321+0000","last_became_peered":"2023-10-09T05:33:21.536321+0000","last_unstale":"2023-10-09T05:34:01.590401+0000","last_undegraded":"2023-10-09T05:34:01.590401+0000","last_fullsized":"2023-10-09T05:33:21.531052+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":86,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.583850+0000","last_change":"2023-10-09T05:33:21.536924+0000","last_active":"2023-10-09T05:34:01.583850+0000","last_peered":"2023-10-09T05:34:01.583850+0000","last_clean":"2023-10-09T05:33:12.602231+0000","last_became_active":"2023-10-09T05:33:21.536924+0000","last_became_peered":"2023-10-09T05:33:21.536924+0000","last_unstale":"2023-10-09T05:34:01.583850+0000","last_undegraded":"2023-10-09T05:34:01.583850+0000","last_fullsized":"2023-10-09T05:33:21.531983+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.116098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.13","version":"120'6448","reported_seq":6482,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:12.625083+0000","last_change":"2023-10-09T05:34:01.629256+0000","last_active":"2023-10-09T05:34:12.625083+0000","last_peered":"2023-10-09T05:34:12.625083+0000","last_clean":"2023-10-09T05:34:12.625083+0000","last_became_active":"2023-10-09T05:34:01.625092+0000","last_became_peered":"2023-10-09T05:34:01.625092+0000","last_unstale":"2023-10-09T05:34:12.625083+0000","last_undegraded":"2023-10-09T05:34:12.625083+0000","last_fullsized":"2023-10-09T05:34:12.625083+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6448,"log_dups_size":0,"ondisk_log_size":6448,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224591872,"num_objects":3427,"num_object_clones":0,"num_object_copies":10281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3427,"num_write_kb":219328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":26,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:01.584116+0000","last_change":"2023-10-09T05:33:21.536951+0000","last_active":"2023-10-09T05:34:01.584116+0000","last_peered":"2023-10-09T05:34:01.584116+0000","last_clean":"2023-10-09T05:21:17.840041+0000","last_became_active":"2023-10-09T05:33:21.536951+0000","last_became_peered":"2023-10-09T05:33:21.536951+0000","last_unstale":"2023-10-09T05:34:01.584116+0000","last_undegraded":"2023-10-09T05:33:21.531336+0000","last_fullsized":"2023-10-09T05:33:21.531230+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:11.457304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":64,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.589310+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:34:01.589310+0000","last_peered":"2023-10-09T05:34:01.589310+0000","last_clean":"2023-10-09T05:34:01.589310+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:34:01.589310+0000","last_undegraded":"2023-10-09T05:34:01.589310+0000","last_fullsized":"2023-10-09T05:34:01.589310+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.12","version":"120'6447","reported_seq":6484,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:09.720540+0000","last_change":"2023-10-09T05:34:06.470865+0000","last_active":"2023-10-09T05:34:09.720540+0000","last_peered":"2023-10-09T05:34:09.720540+0000","last_clean":"2023-10-09T05:34:09.720540+0000","last_became_active":"2023-10-09T05:34:01.624493+0000","last_became_peered":"2023-10-09T05:34:01.624493+0000","last_unstale":"2023-10-09T05:34:09.720540+0000","last_undegraded":"2023-10-09T05:34:09.720540+0000","last_fullsized":"2023-10-09T05:34:09.720540+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6331","last_scrub_stamp":"2023-10-09T05:34:06.470790+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:06.470790+0000","objects_scrubbed":0,"log_size":6447,"log_dups_size":0,"ondisk_log_size":6447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222035968,"num_objects":3388,"num_object_clones":0,"num_object_copies":10164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":128,"num_write_kb":8192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":100,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:01.586202+0000","last_change":"2023-10-09T05:33:21.537017+0000","last_active":"2023-10-09T05:34:01.586202+0000","last_peered":"2023-10-09T05:34:01.586202+0000","last_clean":"2023-10-09T05:33:12.599720+0000","last_became_active":"2023-10-09T05:33:21.537017+0000","last_became_peered":"2023-10-09T05:33:21.537017+0000","last_unstale":"2023-10-09T05:34:01.586202+0000","last_undegraded":"2023-10-09T05:33:21.534333+0000","last_fullsized":"2023-10-09T05:33:21.534169+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":86,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.586520+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:34:01.586520+0000","last_peered":"2023-10-09T05:34:01.586520+0000","last_clean":"2023-10-09T05:34:01.586520+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:34:01.586520+0000","last_undegraded":"2023-10-09T05:34:01.586520+0000","last_fullsized":"2023-10-09T05:34:01.586520+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.15","version":"120'6620","reported_seq":6653,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:12.519724+0000","last_change":"2023-10-09T05:34:01.631033+0000","last_active":"2023-10-09T05:34:12.519724+0000","last_peered":"2023-10-09T05:34:12.519724+0000","last_clean":"2023-10-09T05:34:12.519724+0000","last_became_active":"2023-10-09T05:34:01.623738+0000","last_became_peered":"2023-10-09T05:34:01.623738+0000","last_unstale":"2023-10-09T05:34:12.519724+0000","last_undegraded":"2023-10-09T05:34:12.519724+0000","last_fullsized":"2023-10-09T05:34:12.519724+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6620,"log_dups_size":0,"ondisk_log_size":6620,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229048320,"num_objects":3495,"num_object_clones":0,"num_object_copies":10485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3495,"num_write_kb":223680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":26,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:01.586650+0000","last_change":"2023-10-09T05:33:21.541382+0000","last_active":"2023-10-09T05:34:01.586650+0000","last_peered":"2023-10-09T05:34:01.586650+0000","last_clean":"2023-10-09T05:21:17.839552+0000","last_became_active":"2023-10-09T05:33:21.541382+0000","last_became_peered":"2023-10-09T05:33:21.541382+0000","last_unstale":"2023-10-09T05:34:01.586650+0000","last_undegraded":"2023-10-09T05:33:21.535121+0000","last_fullsized":"2023-10-09T05:33:21.534957+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":96,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.584383+0000","last_change":"2023-10-09T05:33:21.544048+0000","last_active":"2023-10-09T05:34:01.584383+0000","last_peered":"2023-10-09T05:34:01.584383+0000","last_clean":"2023-10-09T05:33:12.603569+0000","last_became_active":"2023-10-09T05:33:21.544048+0000","last_became_peered":"2023-10-09T05:33:21.544048+0000","last_unstale":"2023-10-09T05:34:01.584383+0000","last_undegraded":"2023-10-09T05:34:01.584383+0000","last_fullsized":"2023-10-09T05:33:21.530368+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:49.505710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.14","version":"120'6408","reported_seq":6443,"reported_epoch":120,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:10.870354+0000","last_change":"2023-10-09T05:34:01.629736+0000","last_active":"2023-10-09T05:34:10.870354+0000","last_peered":"2023-10-09T05:34:10.870354+0000","last_clean":"2023-10-09T05:33:20.493128+0000","last_became_active":"2023-10-09T05:34:01.629736+0000","last_became_peered":"2023-10-09T05:34:01.629736+0000","last_unstale":"2023-10-09T05:34:10.870354+0000","last_undegraded":"2023-10-09T05:34:01.616544+0000","last_fullsized":"2023-10-09T05:34:01.593079+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221413376,"num_objects":3379,"num_object_clones":0,"num_object_copies":10137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3379,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3379,"num_write_kb":216224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":3379}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":75,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.586379+0000","last_change":"2023-10-09T05:33:21.538555+0000","last_active":"2023-10-09T05:34:01.586379+0000","last_peered":"2023-10-09T05:34:01.586379+0000","last_clean":"2023-10-09T05:33:12.598696+0000","last_became_active":"2023-10-09T05:33:21.538555+0000","last_became_peered":"2023-10-09T05:33:21.538555+0000","last_unstale":"2023-10-09T05:34:01.586379+0000","last_undegraded":"2023-10-09T05:34:01.586379+0000","last_fullsized":"2023-10-09T05:33:21.531590+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:47.602581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":92,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.586854+0000","last_change":"2023-10-09T05:33:21.539311+0000","last_active":"2023-10-09T05:34:01.586854+0000","last_peered":"2023-10-09T05:34:01.586854+0000","last_clean":"2023-10-09T05:33:12.591928+0000","last_became_active":"2023-10-09T05:33:21.539311+0000","last_became_peered":"2023-10-09T05:33:21.539311+0000","last_unstale":"2023-10-09T05:34:01.586854+0000","last_undegraded":"2023-10-09T05:34:01.586854+0000","last_fullsized":"2023-10-09T05:33:21.532549+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.17","version":"120'6398","reported_seq":6430,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:09.798645+0000","last_change":"2023-10-09T05:34:01.621450+0000","last_active":"2023-10-09T05:34:09.798645+0000","last_peered":"2023-10-09T05:34:09.798645+0000","last_clean":"2023-10-09T05:34:09.798645+0000","last_became_active":"2023-10-09T05:34:01.618906+0000","last_became_peered":"2023-10-09T05:34:01.618906+0000","last_unstale":"2023-10-09T05:34:09.798645+0000","last_undegraded":"2023-10-09T05:34:09.798645+0000","last_fullsized":"2023-10-09T05:34:09.798645+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218955776,"num_objects":3341,"num_object_clones":0,"num_object_copies":10023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3341,"num_write_kb":213824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":16,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.588208+0000","last_change":"2023-10-09T05:33:21.536611+0000","last_active":"2023-10-09T05:34:01.588208+0000","last_peered":"2023-10-09T05:34:01.588208+0000","last_clean":"2023-10-09T05:21:15.763192+0000","last_became_active":"2023-10-09T05:33:21.536611+0000","last_became_peered":"2023-10-09T05:33:21.536611+0000","last_unstale":"2023-10-09T05:34:01.588208+0000","last_undegraded":"2023-10-09T05:34:01.588208+0000","last_fullsized":"2023-10-09T05:33:21.534136+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":75,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.585632+0000","last_change":"2023-10-09T05:33:21.537621+0000","last_active":"2023-10-09T05:34:01.585632+0000","last_peered":"2023-10-09T05:34:01.585632+0000","last_clean":"2023-10-09T05:33:12.601147+0000","last_became_active":"2023-10-09T05:33:21.537621+0000","last_became_peered":"2023-10-09T05:33:21.537621+0000","last_unstale":"2023-10-09T05:34:01.585632+0000","last_undegraded":"2023-10-09T05:34:01.585632+0000","last_fullsized":"2023-10-09T05:33:21.532506+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:41.399098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.16","version":"120'6457","reported_seq":6491,"reported_epoch":120,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:10.830166+0000","last_change":"2023-10-09T05:34:01.638044+0000","last_active":"2023-10-09T05:34:10.830166+0000","last_peered":"2023-10-09T05:34:10.830166+0000","last_clean":"2023-10-09T05:33:20.529830+0000","last_became_active":"2023-10-09T05:34:01.638044+0000","last_became_peered":"2023-10-09T05:34:01.638044+0000","last_unstale":"2023-10-09T05:34:10.830166+0000","last_undegraded":"2023-10-09T05:34:01.625275+0000","last_fullsized":"2023-10-09T05:34:01.594445+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6457,"log_dups_size":0,"ondisk_log_size":6457,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222101504,"num_objects":3389,"num_object_clones":0,"num_object_copies":10167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3389,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3389,"num_write_kb":216896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":3389}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":95,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.586766+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:34:01.586766+0000","last_peered":"2023-10-09T05:34:01.586766+0000","last_clean":"2023-10-09T05:34:01.586766+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:34:01.586766+0000","last_undegraded":"2023-10-09T05:34:01.586766+0000","last_fullsized":"2023-10-09T05:34:01.586766+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":96,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.583943+0000","last_change":"2023-10-09T05:33:21.536100+0000","last_active":"2023-10-09T05:34:01.583943+0000","last_peered":"2023-10-09T05:34:01.583943+0000","last_clean":"2023-10-09T05:33:12.603890+0000","last_became_active":"2023-10-09T05:33:21.536100+0000","last_became_peered":"2023-10-09T05:33:21.536100+0000","last_unstale":"2023-10-09T05:34:01.583943+0000","last_undegraded":"2023-10-09T05:34:01.583943+0000","last_fullsized":"2023-10-09T05:33:21.532317+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.588371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"120'6386","reported_seq":6423,"reported_epoch":120,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:09.719723+0000","last_change":"2023-10-09T05:34:01.624027+0000","last_active":"2023-10-09T05:34:09.719723+0000","last_peered":"2023-10-09T05:34:09.719723+0000","last_clean":"2023-10-09T05:33:20.515228+0000","last_became_active":"2023-10-09T05:34:01.612377+0000","last_became_peered":"2023-10-09T05:34:01.612377+0000","last_unstale":"2023-10-09T05:34:09.719723+0000","last_undegraded":"2023-10-09T05:34:01.588765+0000","last_fullsized":"2023-10-09T05:34:01.585899+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6386,"log_dups_size":0,"ondisk_log_size":6386,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217972736,"num_objects":3326,"num_object_clones":0,"num_object_copies":9978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3326,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3326,"num_write_kb":212864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":3326}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":26,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:01.587180+0000","last_change":"2023-10-09T05:33:21.544135+0000","last_active":"2023-10-09T05:34:01.587180+0000","last_peered":"2023-10-09T05:34:01.587180+0000","last_clean":"2023-10-09T05:21:17.838784+0000","last_became_active":"2023-10-09T05:33:21.544135+0000","last_became_peered":"2023-10-09T05:33:21.544135+0000","last_unstale":"2023-10-09T05:34:01.587180+0000","last_undegraded":"2023-10-09T05:33:21.534556+0000","last_fullsized":"2023-10-09T05:33:21.534176+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.587434+0000","last_change":"2023-10-09T05:33:21.534526+0000","last_active":"2023-10-09T05:34:01.587434+0000","last_peered":"2023-10-09T05:34:01.587434+0000","last_clean":"2023-10-09T05:33:12.591527+0000","last_became_active":"2023-10-09T05:33:21.534526+0000","last_became_peered":"2023-10-09T05:33:21.534526+0000","last_unstale":"2023-10-09T05:34:01.587434+0000","last_undegraded":"2023-10-09T05:34:01.587434+0000","last_fullsized":"2023-10-09T05:33:21.531728+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"120'6508","reported_seq":6541,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:10.842370+0000","last_change":"2023-10-09T05:34:05.967276+0000","last_active":"2023-10-09T05:34:10.842370+0000","last_peered":"2023-10-09T05:34:10.842370+0000","last_clean":"2023-10-09T05:34:10.842370+0000","last_became_active":"2023-10-09T05:34:01.629769+0000","last_became_peered":"2023-10-09T05:34:01.629769+0000","last_unstale":"2023-10-09T05:34:10.842370+0000","last_undegraded":"2023-10-09T05:34:10.842370+0000","last_fullsized":"2023-10-09T05:34:10.842370+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6508,"log_dups_size":0,"ondisk_log_size":6508,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223903744,"num_objects":3416,"num_object_clones":0,"num_object_copies":10248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3416,"num_write_kb":218656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":26,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:01.586377+0000","last_change":"2023-10-09T05:33:21.543381+0000","last_active":"2023-10-09T05:34:01.586377+0000","last_peered":"2023-10-09T05:34:01.586377+0000","last_clean":"2023-10-09T05:21:17.839217+0000","last_became_active":"2023-10-09T05:33:21.543381+0000","last_became_peered":"2023-10-09T05:33:21.543381+0000","last_unstale":"2023-10-09T05:34:01.586377+0000","last_undegraded":"2023-10-09T05:33:21.534386+0000","last_fullsized":"2023-10-09T05:33:21.534260+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":101,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.588107+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:34:01.588107+0000","last_peered":"2023-10-09T05:34:01.588107+0000","last_clean":"2023-10-09T05:34:01.588107+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:34:01.588107+0000","last_undegraded":"2023-10-09T05:34:01.588107+0000","last_fullsized":"2023-10-09T05:34:01.588107+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"120'6850","reported_seq":6874,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:10.892101+0000","last_change":"2023-10-09T05:32:45.931141+0000","last_active":"2023-10-09T05:34:10.892101+0000","last_peered":"2023-10-09T05:34:10.892101+0000","last_clean":"2023-10-09T05:34:10.892101+0000","last_became_active":"2023-10-09T05:32:45.930819+0000","last_became_peered":"2023-10-09T05:32:45.930819+0000","last_unstale":"2023-10-09T05:34:10.892101+0000","last_undegraded":"2023-10-09T05:34:10.892101+0000","last_fullsized":"2023-10-09T05:34:10.892101+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6850,"log_dups_size":0,"ondisk_log_size":6850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:25.657817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":448921600,"num_objects":6850,"num_object_clones":0,"num_object_copies":20550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6850,"num_write_kb":438400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":26,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:01.586681+0000","last_change":"2023-10-09T05:33:21.537840+0000","last_active":"2023-10-09T05:34:01.586681+0000","last_peered":"2023-10-09T05:34:01.586681+0000","last_clean":"2023-10-09T05:21:17.840209+0000","last_became_active":"2023-10-09T05:33:21.537840+0000","last_became_peered":"2023-10-09T05:33:21.537840+0000","last_unstale":"2023-10-09T05:34:01.586681+0000","last_undegraded":"2023-10-09T05:33:21.532661+0000","last_fullsized":"2023-10-09T05:33:21.532525+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.587831+0000","last_change":"2023-10-09T05:33:21.541527+0000","last_active":"2023-10-09T05:34:01.587831+0000","last_peered":"2023-10-09T05:34:01.587831+0000","last_clean":"2023-10-09T05:33:12.592592+0000","last_became_active":"2023-10-09T05:33:21.541527+0000","last_became_peered":"2023-10-09T05:33:21.541527+0000","last_unstale":"2023-10-09T05:34:01.587831+0000","last_undegraded":"2023-10-09T05:34:01.587831+0000","last_fullsized":"2023-10-09T05:33:21.532177+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"120'6805","reported_seq":6829,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:10.865574+0000","last_change":"2023-10-09T05:32:45.929091+0000","last_active":"2023-10-09T05:34:10.865574+0000","last_peered":"2023-10-09T05:34:10.865574+0000","last_clean":"2023-10-09T05:34:10.865574+0000","last_became_active":"2023-10-09T05:32:45.928801+0000","last_became_peered":"2023-10-09T05:32:45.928801+0000","last_unstale":"2023-10-09T05:34:10.865574+0000","last_undegraded":"2023-10-09T05:34:10.865574+0000","last_fullsized":"2023-10-09T05:34:10.865574+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6805,"log_dups_size":0,"ondisk_log_size":6805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:53.569691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445972480,"num_objects":6805,"num_object_clones":0,"num_object_copies":20415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6805,"num_write_kb":435520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":87,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.587917+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:34:01.587917+0000","last_peered":"2023-10-09T05:34:01.587917+0000","last_clean":"2023-10-09T05:34:01.587917+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:34:01.587917+0000","last_undegraded":"2023-10-09T05:34:01.587917+0000","last_fullsized":"2023-10-09T05:34:01.587917+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":69,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.585797+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:34:01.585797+0000","last_peered":"2023-10-09T05:34:01.585797+0000","last_clean":"2023-10-09T05:34:01.585797+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:34:01.585797+0000","last_undegraded":"2023-10-09T05:34:01.585797+0000","last_fullsized":"2023-10-09T05:34:01.585797+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"120'6872","reported_seq":6901,"reported_epoch":120,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:10.851054+0000","last_change":"2023-10-09T05:33:21.536458+0000","last_active":"2023-10-09T05:34:10.851054+0000","last_peered":"2023-10-09T05:34:10.851054+0000","last_clean":"2023-10-09T05:33:20.502869+0000","last_became_active":"2023-10-09T05:33:21.536458+0000","last_became_peered":"2023-10-09T05:33:21.536458+0000","last_unstale":"2023-10-09T05:34:10.851054+0000","last_undegraded":"2023-10-09T05:33:21.526951+0000","last_fullsized":"2023-10-09T05:33:21.526843+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6872,"log_dups_size":0,"ondisk_log_size":6872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:01.509884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":450363392,"num_objects":6872,"num_object_clones":0,"num_object_copies":20616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6872,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6872,"num_write_kb":439808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":6872}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":88,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.583658+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:34:01.583658+0000","last_peered":"2023-10-09T05:34:01.583658+0000","last_clean":"2023-10-09T05:34:01.583658+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:34:01.583658+0000","last_undegraded":"2023-10-09T05:34:01.583658+0000","last_fullsized":"2023-10-09T05:34:01.583658+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":86,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.585992+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:34:01.585992+0000","last_peered":"2023-10-09T05:34:01.585992+0000","last_clean":"2023-10-09T05:34:01.585992+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:34:01.585992+0000","last_undegraded":"2023-10-09T05:34:01.585992+0000","last_fullsized":"2023-10-09T05:34:01.585992+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"120'7036","reported_seq":7060,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:13.453763+0000","last_change":"2023-10-09T05:32:45.927644+0000","last_active":"2023-10-09T05:34:13.453763+0000","last_peered":"2023-10-09T05:34:13.453763+0000","last_clean":"2023-10-09T05:34:13.453763+0000","last_became_active":"2023-10-09T05:32:45.927388+0000","last_became_peered":"2023-10-09T05:32:45.927388+0000","last_unstale":"2023-10-09T05:34:13.453763+0000","last_undegraded":"2023-10-09T05:34:13.453763+0000","last_fullsized":"2023-10-09T05:34:13.453763+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:42.111993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":461111296,"num_objects":7036,"num_object_clones":0,"num_object_copies":21108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7036,"num_write_kb":450304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":87,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.587654+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:34:01.587654+0000","last_peered":"2023-10-09T05:34:01.587654+0000","last_clean":"2023-10-09T05:34:01.587654+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:34:01.587654+0000","last_undegraded":"2023-10-09T05:34:01.587654+0000","last_fullsized":"2023-10-09T05:34:01.587654+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":101,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.587803+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:34:01.587803+0000","last_peered":"2023-10-09T05:34:01.587803+0000","last_clean":"2023-10-09T05:34:01.587803+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:34:01.587803+0000","last_undegraded":"2023-10-09T05:34:01.587803+0000","last_fullsized":"2023-10-09T05:34:01.587803+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"120'6686","reported_seq":6709,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:09.711280+0000","last_change":"2023-10-09T05:32:45.936026+0000","last_active":"2023-10-09T05:34:09.711280+0000","last_peered":"2023-10-09T05:34:09.711280+0000","last_clean":"2023-10-09T05:34:09.711280+0000","last_became_active":"2023-10-09T05:32:45.935681+0000","last_became_peered":"2023-10-09T05:32:45.935681+0000","last_unstale":"2023-10-09T05:34:09.711280+0000","last_undegraded":"2023-10-09T05:34:09.711280+0000","last_fullsized":"2023-10-09T05:34:09.711280+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6686,"log_dups_size":0,"ondisk_log_size":6686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438173696,"num_objects":6686,"num_object_clones":0,"num_object_copies":20058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6686,"num_write_kb":427904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":93,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.587829+0000","last_change":"2023-10-09T05:33:21.541326+0000","last_active":"2023-10-09T05:34:01.587829+0000","last_peered":"2023-10-09T05:34:01.587829+0000","last_clean":"2023-10-09T05:33:12.599212+0000","last_became_active":"2023-10-09T05:33:21.541326+0000","last_became_peered":"2023-10-09T05:33:21.541326+0000","last_unstale":"2023-10-09T05:34:01.587829+0000","last_undegraded":"2023-10-09T05:34:01.587829+0000","last_fullsized":"2023-10-09T05:33:21.534788+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":95,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.586525+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:34:01.586525+0000","last_peered":"2023-10-09T05:34:01.586525+0000","last_clean":"2023-10-09T05:34:01.586525+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:34:01.586525+0000","last_undegraded":"2023-10-09T05:34:01.586525+0000","last_fullsized":"2023-10-09T05:34:01.586525+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"120'6793","reported_seq":6815,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:09.800786+0000","last_change":"2023-10-09T05:32:45.929479+0000","last_active":"2023-10-09T05:34:09.800786+0000","last_peered":"2023-10-09T05:34:09.800786+0000","last_clean":"2023-10-09T05:34:09.800786+0000","last_became_active":"2023-10-09T05:32:45.929063+0000","last_became_peered":"2023-10-09T05:32:45.929063+0000","last_unstale":"2023-10-09T05:34:09.800786+0000","last_undegraded":"2023-10-09T05:34:09.800786+0000","last_fullsized":"2023-10-09T05:34:09.800786+0000","mapping_epoch":105,"log_start":"108'100","ondisk_log_start":"108'100","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6693,"log_dups_size":0,"ondisk_log_size":6693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445186048,"num_objects":6793,"num_object_clones":0,"num_object_copies":20379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6793,"num_write_kb":434752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":97,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.584307+0000","last_change":"2023-10-09T05:33:21.546397+0000","last_active":"2023-10-09T05:34:01.584307+0000","last_peered":"2023-10-09T05:34:01.584307+0000","last_clean":"2023-10-09T05:33:12.602017+0000","last_became_active":"2023-10-09T05:33:21.546397+0000","last_became_peered":"2023-10-09T05:33:21.546397+0000","last_unstale":"2023-10-09T05:34:01.584307+0000","last_undegraded":"2023-10-09T05:34:01.584307+0000","last_fullsized":"2023-10-09T05:33:21.530353+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:38:13.641630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":64,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.589561+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:34:01.589561+0000","last_peered":"2023-10-09T05:34:01.589561+0000","last_clean":"2023-10-09T05:34:01.589561+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:34:01.589561+0000","last_undegraded":"2023-10-09T05:34:01.589561+0000","last_fullsized":"2023-10-09T05:34:01.589561+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":634,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:01.589479+0000","last_change":"2023-10-09T05:33:21.534540+0000","last_active":"2023-10-09T05:34:01.589479+0000","last_peered":"2023-10-09T05:34:01.589479+0000","last_clean":"2023-10-09T05:33:12.603745+0000","last_became_active":"2023-10-09T05:33:21.534540+0000","last_became_peered":"2023-10-09T05:33:21.534540+0000","last_unstale":"2023-10-09T05:34:01.589479+0000","last_undegraded":"2023-10-09T05:33:21.527759+0000","last_fullsized":"2023-10-09T05:33:21.527646+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:47.310192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":102,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.586990+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:34:01.586990+0000","last_peered":"2023-10-09T05:34:01.586990+0000","last_clean":"2023-10-09T05:34:01.586990+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:34:01.586990+0000","last_undegraded":"2023-10-09T05:34:01.586990+0000","last_fullsized":"2023-10-09T05:34:01.586990+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":49,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.589227+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:34:01.589227+0000","last_peered":"2023-10-09T05:34:01.589227+0000","last_clean":"2023-10-09T05:34:01.589227+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:34:01.589227+0000","last_undegraded":"2023-10-09T05:34:01.589227+0000","last_fullsized":"2023-10-09T05:34:01.589227+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"120'6408","reported_seq":6520,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:12.493637+0000","last_change":"2023-10-09T05:34:08.749872+0000","last_active":"2023-10-09T05:34:12.493637+0000","last_peered":"2023-10-09T05:34:12.493637+0000","last_clean":"2023-10-09T05:34:12.493637+0000","last_became_active":"2023-10-09T05:34:01.617508+0000","last_became_peered":"2023-10-09T05:34:01.617508+0000","last_unstale":"2023-10-09T05:34:12.493637+0000","last_undegraded":"2023-10-09T05:34:12.493637+0000","last_fullsized":"2023-10-09T05:34:12.493637+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'6300","last_scrub_stamp":"2023-10-09T05:34:08.749711+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:08.749711+0000","objects_scrubbed":3317,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:54.425712+0000","scrub_duration":0.61798832400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224919552,"num_objects":3432,"num_object_clones":0,"num_object_copies":10296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":115,"num_write_kb":7360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"120'6658","reported_seq":6769,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:12.586119+0000","last_change":"2023-10-09T05:34:10.842783+0000","last_active":"2023-10-09T05:34:12.586119+0000","last_peered":"2023-10-09T05:34:12.586119+0000","last_clean":"2023-10-09T05:34:12.586119+0000","last_became_active":"2023-10-09T05:34:01.606336+0000","last_became_peered":"2023-10-09T05:34:01.606336+0000","last_unstale":"2023-10-09T05:34:12.586119+0000","last_undegraded":"2023-10-09T05:34:12.586119+0000","last_fullsized":"2023-10-09T05:34:12.586119+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'6606","last_scrub_stamp":"2023-10-09T05:34:10.842550+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:10.842550+0000","objects_scrubbed":3389,"log_size":6658,"log_dups_size":0,"ondisk_log_size":6658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:40.600228+0000","scrub_duration":0.74463188000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227213312,"num_objects":3467,"num_object_clones":0,"num_object_copies":10401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":78,"num_write_kb":4992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":556,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.586462+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:34:01.586462+0000","last_peered":"2023-10-09T05:34:01.586462+0000","last_clean":"2023-10-09T05:34:01.586462+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:34:01.586462+0000","last_undegraded":"2023-10-09T05:34:01.586462+0000","last_fullsized":"2023-10-09T05:34:01.586462+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":87,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.586815+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:34:01.586815+0000","last_peered":"2023-10-09T05:34:01.586815+0000","last_clean":"2023-10-09T05:34:01.586815+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:34:01.586815+0000","last_undegraded":"2023-10-09T05:34:01.586815+0000","last_fullsized":"2023-10-09T05:34:01.586815+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":99,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.587622+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:34:01.587622+0000","last_peered":"2023-10-09T05:34:01.587622+0000","last_clean":"2023-10-09T05:34:01.587622+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:34:01.587622+0000","last_undegraded":"2023-10-09T05:34:01.587622+0000","last_fullsized":"2023-10-09T05:34:01.587622+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"120'6470","reported_seq":6508,"reported_epoch":120,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:10.841693+0000","last_change":"2023-10-09T05:34:01.643489+0000","last_active":"2023-10-09T05:34:10.841693+0000","last_peered":"2023-10-09T05:34:10.841693+0000","last_clean":"2023-10-09T05:33:20.529830+0000","last_became_active":"2023-10-09T05:34:01.622986+0000","last_became_peered":"2023-10-09T05:34:01.622986+0000","last_unstale":"2023-10-09T05:34:10.841693+0000","last_undegraded":"2023-10-09T05:34:01.592558+0000","last_fullsized":"2023-10-09T05:34:01.584953+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6470,"log_dups_size":0,"ondisk_log_size":6470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222953472,"num_objects":3402,"num_object_clones":0,"num_object_copies":10206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3402,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3402,"num_write_kb":217728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":3402}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":431,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.584220+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:34:01.584220+0000","last_peered":"2023-10-09T05:34:01.584220+0000","last_clean":"2023-10-09T05:34:01.584220+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:34:01.584220+0000","last_undegraded":"2023-10-09T05:34:01.584220+0000","last_fullsized":"2023-10-09T05:34:01.584220+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":64,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.589269+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:34:01.589269+0000","last_peered":"2023-10-09T05:34:01.589269+0000","last_clean":"2023-10-09T05:34:01.589269+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:34:01.589269+0000","last_undegraded":"2023-10-09T05:34:01.589269+0000","last_fullsized":"2023-10-09T05:34:01.589269+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":94,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.589455+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:34:01.589455+0000","last_peered":"2023-10-09T05:34:01.589455+0000","last_clean":"2023-10-09T05:34:01.589455+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:34:01.589455+0000","last_undegraded":"2023-10-09T05:34:01.589455+0000","last_fullsized":"2023-10-09T05:34:01.589455+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":687,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:01.587534+0000","last_change":"2023-10-09T05:33:21.539819+0000","last_active":"2023-10-09T05:34:01.587534+0000","last_peered":"2023-10-09T05:34:01.587534+0000","last_clean":"2023-10-09T05:33:12.591667+0000","last_became_active":"2023-10-09T05:33:21.539819+0000","last_became_peered":"2023-10-09T05:33:21.539819+0000","last_unstale":"2023-10-09T05:34:01.587534+0000","last_undegraded":"2023-10-09T05:33:21.531338+0000","last_fullsized":"2023-10-09T05:33:21.531040+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":286,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:01.585801+0000","last_change":"2023-10-09T05:33:21.534856+0000","last_active":"2023-10-09T05:34:01.585801+0000","last_peered":"2023-10-09T05:34:01.585801+0000","last_clean":"2023-10-09T05:25:43.873151+0000","last_became_active":"2023-10-09T05:33:21.534856+0000","last_became_peered":"2023-10-09T05:33:21.534856+0000","last_unstale":"2023-10-09T05:34:01.585801+0000","last_undegraded":"2023-10-09T05:33:21.531368+0000","last_fullsized":"2023-10-09T05:33:21.531257+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:36.567162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":81,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.585392+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:34:01.585392+0000","last_peered":"2023-10-09T05:34:01.585392+0000","last_clean":"2023-10-09T05:34:01.585392+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:34:01.585392+0000","last_undegraded":"2023-10-09T05:34:01.585392+0000","last_fullsized":"2023-10-09T05:34:01.585392+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":77,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.586387+0000","last_change":"2023-10-09T05:33:21.536840+0000","last_active":"2023-10-09T05:34:01.586387+0000","last_peered":"2023-10-09T05:34:01.586387+0000","last_clean":"2023-10-09T05:33:12.592541+0000","last_became_active":"2023-10-09T05:33:21.536840+0000","last_became_peered":"2023-10-09T05:33:21.536840+0000","last_unstale":"2023-10-09T05:34:01.586387+0000","last_undegraded":"2023-10-09T05:34:01.586387+0000","last_fullsized":"2023-10-09T05:33:21.532641+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"120'6392","reported_seq":6427,"reported_epoch":120,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:10.864837+0000","last_change":"2023-10-09T05:34:01.601418+0000","last_active":"2023-10-09T05:34:10.864837+0000","last_peered":"2023-10-09T05:34:10.864837+0000","last_clean":"2023-10-09T05:33:20.493128+0000","last_became_active":"2023-10-09T05:34:01.601418+0000","last_became_peered":"2023-10-09T05:34:01.601418+0000","last_unstale":"2023-10-09T05:34:10.864837+0000","last_undegraded":"2023-10-09T05:34:01.584944+0000","last_fullsized":"2023-10-09T05:34:01.584819+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6392,"log_dups_size":0,"ondisk_log_size":6392,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220364800,"num_objects":3362,"num_object_clones":0,"num_object_copies":10086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3362,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3362,"num_write_kb":215200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":3362}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.7","version":"120'6390","reported_seq":6425,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:09.714757+0000","last_change":"2023-10-09T05:34:02.589391+0000","last_active":"2023-10-09T05:34:09.714757+0000","last_peered":"2023-10-09T05:34:09.714757+0000","last_clean":"2023-10-09T05:34:09.714757+0000","last_became_active":"2023-10-09T05:34:01.592987+0000","last_became_peered":"2023-10-09T05:34:01.592987+0000","last_unstale":"2023-10-09T05:34:09.714757+0000","last_undegraded":"2023-10-09T05:34:09.714757+0000","last_fullsized":"2023-10-09T05:34:09.714757+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6390,"log_dups_size":0,"ondisk_log_size":6390,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218431488,"num_objects":3333,"num_object_clones":0,"num_object_copies":9999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3333,"num_write_kb":213312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":679,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.589441+0000","last_change":"2023-10-09T05:33:21.535548+0000","last_active":"2023-10-09T05:34:01.589441+0000","last_peered":"2023-10-09T05:34:01.589441+0000","last_clean":"2023-10-09T05:22:39.601058+0000","last_became_active":"2023-10-09T05:33:21.535548+0000","last_became_peered":"2023-10-09T05:33:21.535548+0000","last_unstale":"2023-10-09T05:34:01.589441+0000","last_undegraded":"2023-10-09T05:34:01.589441+0000","last_fullsized":"2023-10-09T05:33:21.527900+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:57.456065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":70,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:03.698848+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:34:03.698848+0000","last_peered":"2023-10-09T05:34:03.698848+0000","last_clean":"2023-10-09T05:34:03.698848+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:34:03.698848+0000","last_undegraded":"2023-10-09T05:34:03.698848+0000","last_fullsized":"2023-10-09T05:34:03.698848+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":84,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.585916+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:34:01.585916+0000","last_peered":"2023-10-09T05:34:01.585916+0000","last_clean":"2023-10-09T05:34:01.585916+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:34:01.585916+0000","last_undegraded":"2023-10-09T05:34:01.585916+0000","last_fullsized":"2023-10-09T05:34:01.585916+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":303,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.586156+0000","last_change":"2023-10-09T05:33:21.542131+0000","last_active":"2023-10-09T05:34:01.586156+0000","last_peered":"2023-10-09T05:34:01.586156+0000","last_clean":"2023-10-09T05:22:39.669502+0000","last_became_active":"2023-10-09T05:33:21.542131+0000","last_became_peered":"2023-10-09T05:33:21.542131+0000","last_unstale":"2023-10-09T05:34:01.586156+0000","last_undegraded":"2023-10-09T05:34:01.586156+0000","last_fullsized":"2023-10-09T05:33:21.531326+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:40.994915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":64,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.589129+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:34:01.589129+0000","last_peered":"2023-10-09T05:34:01.589129+0000","last_clean":"2023-10-09T05:34:01.589129+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:34:01.589129+0000","last_undegraded":"2023-10-09T05:34:01.589129+0000","last_fullsized":"2023-10-09T05:34:01.589129+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":61,"reported_epoch":120,"state":"active+undersized","last_fresh":"2023-10-09T05:34:03.703850+0000","last_change":"2023-10-09T05:33:21.541460+0000","last_active":"2023-10-09T05:34:03.703850+0000","last_peered":"2023-10-09T05:34:03.703850+0000","last_clean":"2023-10-09T05:33:12.598763+0000","last_became_active":"2023-10-09T05:33:21.541460+0000","last_became_peered":"2023-10-09T05:33:21.541460+0000","last_unstale":"2023-10-09T05:34:03.703850+0000","last_undegraded":"2023-10-09T05:34:03.703850+0000","last_fullsized":"2023-10-09T05:33:21.532650+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:49.788781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"120'6380","reported_seq":6418,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:09.715710+0000","last_change":"2023-10-09T05:34:05.620097+0000","last_active":"2023-10-09T05:34:09.715710+0000","last_peered":"2023-10-09T05:34:09.715710+0000","last_clean":"2023-10-09T05:34:09.715710+0000","last_became_active":"2023-10-09T05:34:01.613938+0000","last_became_peered":"2023-10-09T05:34:01.613938+0000","last_unstale":"2023-10-09T05:34:09.715710+0000","last_undegraded":"2023-10-09T05:34:09.715710+0000","last_fullsized":"2023-10-09T05:34:09.715710+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'6267","last_scrub_stamp":"2023-10-09T05:34:05.620029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:05.620029+0000","objects_scrubbed":0,"log_size":6380,"log_dups_size":0,"ondisk_log_size":6380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217186304,"num_objects":3314,"num_object_clones":0,"num_object_copies":9942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":123,"num_write_kb":7872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":94,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:01.584095+0000","last_change":"2023-10-09T05:33:21.537581+0000","last_active":"2023-10-09T05:34:01.584095+0000","last_peered":"2023-10-09T05:34:01.584095+0000","last_clean":"2023-10-09T05:33:12.603985+0000","last_became_active":"2023-10-09T05:33:21.537581+0000","last_became_peered":"2023-10-09T05:33:21.537581+0000","last_unstale":"2023-10-09T05:34:01.584095+0000","last_undegraded":"2023-10-09T05:33:21.532802+0000","last_fullsized":"2023-10-09T05:33:21.532691+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:25.676206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":505,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.589639+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:34:01.589639+0000","last_peered":"2023-10-09T05:34:01.589639+0000","last_clean":"2023-10-09T05:34:01.589639+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:34:01.589639+0000","last_undegraded":"2023-10-09T05:34:01.589639+0000","last_fullsized":"2023-10-09T05:34:01.589639+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":91,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.583982+0000","last_change":"2023-10-09T05:33:21.535940+0000","last_active":"2023-10-09T05:34:01.583982+0000","last_peered":"2023-10-09T05:34:01.583982+0000","last_clean":"2023-10-09T05:33:12.601950+0000","last_became_active":"2023-10-09T05:33:21.535940+0000","last_became_peered":"2023-10-09T05:33:21.535940+0000","last_unstale":"2023-10-09T05:34:01.583982+0000","last_undegraded":"2023-10-09T05:34:01.583982+0000","last_fullsized":"2023-10-09T05:33:21.531073+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:40.161605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"120'6341","reported_seq":6377,"reported_epoch":120,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:09.770198+0000","last_change":"2023-10-09T05:34:01.606720+0000","last_active":"2023-10-09T05:34:09.770198+0000","last_peered":"2023-10-09T05:34:09.770198+0000","last_clean":"2023-10-09T05:33:20.516527+0000","last_became_active":"2023-10-09T05:34:01.606720+0000","last_became_peered":"2023-10-09T05:34:01.606720+0000","last_unstale":"2023-10-09T05:34:09.770198+0000","last_undegraded":"2023-10-09T05:34:01.586839+0000","last_fullsized":"2023-10-09T05:34:01.585627+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6341,"log_dups_size":0,"ondisk_log_size":6341,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217808896,"num_objects":3323,"num_object_clones":0,"num_object_copies":9969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3323,"num_write_kb":212704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":3323}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":574,"reported_epoch":119,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:01.585974+0000","last_change":"2023-10-09T05:33:21.539542+0000","last_active":"2023-10-09T05:34:01.585974+0000","last_peered":"2023-10-09T05:34:01.585974+0000","last_clean":"2023-10-09T05:33:12.592709+0000","last_became_active":"2023-10-09T05:33:21.539542+0000","last_became_peered":"2023-10-09T05:33:21.539542+0000","last_unstale":"2023-10-09T05:34:01.585974+0000","last_undegraded":"2023-10-09T05:33:21.532710+0000","last_fullsized":"2023-10-09T05:33:21.532559+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.585885+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:34:01.585885+0000","last_peered":"2023-10-09T05:34:01.585885+0000","last_clean":"2023-10-09T05:34:01.585885+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:34:01.585885+0000","last_undegraded":"2023-10-09T05:34:01.585885+0000","last_fullsized":"2023-10-09T05:34:01.585885+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":71,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.587943+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:34:01.587943+0000","last_peered":"2023-10-09T05:34:01.587943+0000","last_clean":"2023-10-09T05:34:01.587943+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:34:01.587943+0000","last_undegraded":"2023-10-09T05:34:01.587943+0000","last_fullsized":"2023-10-09T05:34:01.587943+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"120'6395","reported_seq":6502,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:10.877519+0000","last_change":"2023-10-09T05:34:02.926437+0000","last_active":"2023-10-09T05:34:10.877519+0000","last_peered":"2023-10-09T05:34:10.877519+0000","last_clean":"2023-10-09T05:34:10.877519+0000","last_became_active":"2023-10-09T05:34:01.591074+0000","last_became_peered":"2023-10-09T05:34:01.591074+0000","last_unstale":"2023-10-09T05:34:10.877519+0000","last_undegraded":"2023-10-09T05:34:10.877519+0000","last_fullsized":"2023-10-09T05:34:10.877519+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'6121","last_scrub_stamp":"2023-10-09T05:34:02.926205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:02.926205+0000","objects_scrubbed":3132,"log_size":6395,"log_dups_size":0,"ondisk_log_size":6395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:39.965618+0000","scrub_duration":0.800488054,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224264192,"num_objects":3422,"num_object_clones":0,"num_object_copies":10266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":290,"num_write_kb":18560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":63,"reported_epoch":119,"state":"active+undersized","last_fresh":"2023-10-09T05:34:01.585950+0000","last_change":"2023-10-09T05:33:21.541180+0000","last_active":"2023-10-09T05:34:01.585950+0000","last_peered":"2023-10-09T05:34:01.585950+0000","last_clean":"2023-10-09T05:33:12.598740+0000","last_became_active":"2023-10-09T05:33:21.541180+0000","last_became_peered":"2023-10-09T05:33:21.541180+0000","last_unstale":"2023-10-09T05:34:01.585950+0000","last_undegraded":"2023-10-09T05:34:01.585950+0000","last_fullsized":"2023-10-09T05:33:21.532395+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:47.164978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.585404+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:34:01.585404+0000","last_peered":"2023-10-09T05:34:01.585404+0000","last_clean":"2023-10-09T05:34:01.585404+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:34:01.585404+0000","last_undegraded":"2023-10-09T05:34:01.585404+0000","last_fullsized":"2023-10-09T05:34:01.585404+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":89,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.588155+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:34:01.588155+0000","last_peered":"2023-10-09T05:34:01.588155+0000","last_clean":"2023-10-09T05:34:01.588155+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:34:01.588155+0000","last_undegraded":"2023-10-09T05:34:01.588155+0000","last_fullsized":"2023-10-09T05:34:01.588155+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":101,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.586899+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:34:01.586899+0000","last_peered":"2023-10-09T05:34:01.586899+0000","last_clean":"2023-10-09T05:34:01.586899+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:34:01.586899+0000","last_undegraded":"2023-10-09T05:34:01.586899+0000","last_fullsized":"2023-10-09T05:34:01.586899+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":74,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.587087+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:34:01.587087+0000","last_peered":"2023-10-09T05:34:01.587087+0000","last_clean":"2023-10-09T05:34:01.587087+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:34:01.587087+0000","last_undegraded":"2023-10-09T05:34:01.587087+0000","last_fullsized":"2023-10-09T05:34:01.587087+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.590010+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:34:01.590010+0000","last_peered":"2023-10-09T05:34:01.590010+0000","last_clean":"2023-10-09T05:34:01.590010+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:34:01.590010+0000","last_undegraded":"2023-10-09T05:34:01.590010+0000","last_fullsized":"2023-10-09T05:34:01.590010+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"120'6530","reported_seq":6645,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:10.850591+0000","last_change":"2023-10-09T05:34:07.818690+0000","last_active":"2023-10-09T05:34:10.850591+0000","last_peered":"2023-10-09T05:34:10.850591+0000","last_clean":"2023-10-09T05:34:10.850591+0000","last_became_active":"2023-10-09T05:34:01.656619+0000","last_became_peered":"2023-10-09T05:34:01.656619+0000","last_unstale":"2023-10-09T05:34:10.850591+0000","last_undegraded":"2023-10-09T05:34:10.850591+0000","last_fullsized":"2023-10-09T05:34:10.850591+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6429","last_scrub_stamp":"2023-10-09T05:34:07.818377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:07.818377+0000","objects_scrubbed":3381,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:04.556045+0000","scrub_duration":0.90924191099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229244928,"num_objects":3498,"num_object_clones":0,"num_object_copies":10494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":117,"num_write_kb":7488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":96,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.583799+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:34:01.583799+0000","last_peered":"2023-10-09T05:34:01.583799+0000","last_clean":"2023-10-09T05:34:01.583799+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:34:01.583799+0000","last_undegraded":"2023-10-09T05:34:01.583799+0000","last_fullsized":"2023-10-09T05:34:01.583799+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":109,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.583638+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:34:01.583638+0000","last_peered":"2023-10-09T05:34:01.583638+0000","last_clean":"2023-10-09T05:34:01.583638+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:34:01.583638+0000","last_undegraded":"2023-10-09T05:34:01.583638+0000","last_fullsized":"2023-10-09T05:34:01.583638+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"120'6386","reported_seq":6421,"reported_epoch":120,"state":"active+undersized+degraded","last_fresh":"2023-10-09T05:34:09.783831+0000","last_change":"2023-10-09T05:34:01.633717+0000","last_active":"2023-10-09T05:34:09.783831+0000","last_peered":"2023-10-09T05:34:09.783831+0000","last_clean":"2023-10-09T05:33:20.515228+0000","last_became_active":"2023-10-09T05:34:01.624166+0000","last_became_peered":"2023-10-09T05:34:01.624166+0000","last_unstale":"2023-10-09T05:34:09.783831+0000","last_undegraded":"2023-10-09T05:34:01.616786+0000","last_fullsized":"2023-10-09T05:34:01.591902+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":6386,"log_dups_size":0,"ondisk_log_size":6386,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217972736,"num_objects":3326,"num_object_clones":0,"num_object_copies":9978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3326,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3326,"num_write_kb":212864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":3326}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":73,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T05:34:03.699791+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:34:03.699791+0000","last_peered":"2023-10-09T05:34:03.699791+0000","last_clean":"2023-10-09T05:34:03.699791+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:34:03.699791+0000","last_undegraded":"2023-10-09T05:34:03.699791+0000","last_fullsized":"2023-10-09T05:34:03.699791+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":86,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T05:34:01.586157+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:34:01.586157+0000","last_peered":"2023-10-09T05:34:01.586157+0000","last_clean":"2023-10-09T05:34:01.586157+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:34:01.586157+0000","last_undegraded":"2023-10-09T05:34:01.586157+0000","last_fullsized":"2023-10-09T05:34:01.586157+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":26,"stat_sum":{"num_bytes":7123206144,"num_objects":108691,"num_object_clones":0,"num_object_copies":326073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33690,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":108691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":85866,"num_write_kb":5495456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20249903104,"data_stored":20249903104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":169686,"ondisk_log_size":169686,"up":69,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":85,"acting":85,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26751,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":83,"acting":83,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":2,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5560,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":19,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430375,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1375784,"kb_used_data":1066272,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92369368,"statfs":{"total":95995035648,"available":94586232832,"internally_reserved":0,"allocated":1091862528,"data_stored":1091355045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:33:02 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.56299999999999994,"15min":0.55000000000000004},"min":{"1min":0.374,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.6839999999999999,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.56000000000000005,"15min":0.55100000000000005},"min":{"1min":0.43099999999999999,"5min":0.315,"15min":0.315},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.47699999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:32:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.51700000000000002,"15min":0.51600000000000001},"min":{"1min":0.40500000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.68999999999999995,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.41799999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.73299999999999998,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.44}]},{"osd":2,"last update":"Mon Oct 9 05:33:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.52900000000000003,"15min":0.51700000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":81.849999999999994},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.60599999999999998,"15min":0.58399999999999996},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":2.7949999999999999,"5min":2.7949999999999999,"15min":2.7949999999999999},"last":81.861000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:32:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.46600000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:33:11 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.51800000000000002,"15min":0.47799999999999998},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.249},"max":{"1min":1.53,"5min":1.53,"15min":1.53},"last":0.498},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.55100000000000005,"15min":0.52100000000000002},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.27300000000000002},"max":{"1min":2.6909999999999998,"5min":2.6909999999999998,"15min":2.6909999999999998},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:33:11 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.57199999999999995,"15min":0.52500000000000002},"min":{"1min":0.36599999999999999,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":3.0289999999999999,"5min":3.0289999999999999,"15min":3.0289999999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.56599999999999995,"15min":0.55500000000000005},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.56499999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:33:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.58699999999999997,"15min":0.55000000000000004},"min":{"1min":0.40500000000000003,"5min":0.34799999999999998,"15min":0.30499999999999999},"max":{"1min":1.282,"5min":1.282,"15min":1.282},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.83499999999999996,"5min":0.63500000000000001,"15min":0.60299999999999998},"min":{"1min":0.42599999999999999,"5min":0.372,"15min":0.33300000000000002},"max":{"1min":3.4729999999999999,"5min":3.4729999999999999,"15min":3.4729999999999999},"last":0.63100000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626613,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2006564,"kb_used_data":1709340,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91738588,"statfs":{"total":95995035648,"available":93940314112,"internally_reserved":0,"allocated":1750364160,"data_stored":1749865377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:34:09 2023","interfaces":[{"interface":"back","average":{"1min":4.9169999999999998,"5min":1.431,"15min":0.83699999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":53.771999999999998,"5min":53.771999999999998,"15min":53.771999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":4.4569999999999999,"5min":1.4350000000000001,"15min":0.84099999999999997},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":53.411999999999999,"5min":53.411999999999999,"15min":53.411999999999999},"last":0.30499999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:33:33 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":0.91200000000000003,"15min":0.67200000000000004},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":6.7880000000000003,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.647,"5min":0.93999999999999995,"15min":0.69799999999999995},"min":{"1min":0.34799999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.9269999999999996,"5min":8.7189999999999994,"15min":8.7189999999999994},"last":0.33200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:34:09 2023","interfaces":[{"interface":"back","average":{"1min":4.5190000000000001,"5min":1.9319999999999999,"15min":1.3600000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":34.393999999999998,"5min":34.393999999999998,"15min":34.393999999999998},"last":0.498},{"interface":"front","average":{"1min":4.6689999999999996,"5min":1.974,"15min":1.387},"min":{"1min":0.41499999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":34.381999999999998,"5min":34.381999999999998,"15min":34.381999999999998},"last":0.52100000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:33:33 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":1.304,"15min":1.1439999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":18.361000000000001,"5min":18.361000000000001,"15min":18.361000000000001},"last":0.83999999999999997},{"interface":"front","average":{"1min":2.3250000000000002,"5min":1.363,"15min":1.2030000000000001},"min":{"1min":0.46300000000000002,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":18.536000000000001,"5min":18.536000000000001,"15min":18.536000000000001},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:33:49 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.60799999999999998,"15min":0.50800000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.1070000000000002,"5min":4.1070000000000002,"15min":4.1070000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.58499999999999996,"15min":0.50900000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27800000000000002},"max":{"1min":2.782,"5min":2.782,"15min":2.782},"last":0.69699999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.63300000000000001,"15min":0.54200000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27200000000000002},"max":{"1min":2.3759999999999999,"5min":2.3759999999999999,"15min":2.3759999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.0609999999999999,"5min":0.66000000000000003,"15min":0.56000000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":3.5049999999999999,"5min":3.5049999999999999,"15min":3.5049999999999999},"last":0.35899999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:33:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.56999999999999995,"15min":0.54200000000000004},"min":{"1min":0.32900000000000001,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":1.9299999999999999,"5min":1.9299999999999999,"15min":1.9299999999999999},"last":3.831},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.63100000000000001,"15min":0.58199999999999996},"min":{"1min":0.48399999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4330000000000001,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":0.623}]}]},{"osd":1,"up_from":93,"seq":399431958575,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2284324,"kb_used_data":2216540,"kb_used_omap":7,"kb_used_meta":67768,"kb_avail":91460828,"statfs":{"total":95995035648,"available":93655887872,"internally_reserved":0,"allocated":2269736960,"data_stored":2269180548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":69394958},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.47199999999999998,"15min":0.42599999999999999},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":4.4980000000000002,"5min":4.4980000000000002,"15min":4.4980000000000002},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.123,"5min":0.54200000000000004,"15min":0.45200000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.0259999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":0.434}]},{"osd":2,"last update":"Mon Oct 9 05:34:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3660000000000001,"5min":0.78900000000000003,"15min":0.49099999999999999},"min":{"1min":0.28699999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.849,"5min":12.849,"15min":12.849},"last":4.915},{"interface":"front","average":{"1min":1.768,"5min":0.71899999999999997,"15min":0.51500000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.375,"5min":7.375,"15min":7.375},"last":0.57699999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.59299999999999997,"15min":0.51700000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.806,"5min":5.806,"15min":5.806},"last":4.9900000000000002},{"interface":"front","average":{"1min":1.202,"5min":0.56299999999999994,"15min":0.45700000000000002},"min":{"1min":0.29999999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.4569999999999999,"5min":4.4569999999999999,"15min":4.4569999999999999},"last":0.97799999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":0.746,"15min":0.67700000000000005},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.7050000000000001,"5min":3.7050000000000001,"15min":3.7050000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.68600000000000005,"15min":0.66900000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.3999999999999999,"5min":3.3999999999999999,"15min":3.3999999999999999},"last":0.54500000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":0.71299999999999997,"15min":0.65900000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.3330000000000002,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":4.8639999999999999},{"interface":"front","average":{"1min":2.0899999999999999,"5min":0.94999999999999996,"15min":0.76800000000000002},"min":{"1min":0.41099999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":25.68,"5min":25.68,"15min":25.68},"last":4.0499999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.75,"15min":0.68600000000000005},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":4.6449999999999996,"5min":4.6449999999999996,"15min":4.6449999999999996},"last":4.9539999999999997},{"interface":"front","average":{"1min":1.03,"5min":0.74399999999999999,"15min":0.70499999999999996},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":3.4609999999999999,"5min":3.4609999999999999,"15min":3.4609999999999999},"last":4.0890000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.69699999999999995,"15min":0.70799999999999996},"min":{"1min":0.47299999999999998,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.77000000000000002,"5min":1.014,"15min":1.014},"last":2.3839999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.68400000000000005,"15min":0.69299999999999995},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.82799999999999996,"5min":1,"15min":1},"last":2.46}]}]},{"osd":0,"up_from":88,"seq":377957122101,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3642980,"kb_used_data":3535004,"kb_used_omap":8,"kb_used_meta":107959,"kb_avail":90102172,"statfs":{"total":95995035648,"available":92264624128,"internally_reserved":0,"allocated":3619844096,"data_stored":3619286854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":110550792},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:33:41 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.52200000000000002,"15min":0.438},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.4409999999999998,"5min":5.4409999999999998,"15min":5.4409999999999998},"last":3.6930000000000001},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.52000000000000002,"15min":0.46600000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":3.641,"5min":3.641,"15min":3.641},"last":3.7690000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:34:03 2023","interfaces":[{"interface":"back","average":{"1min":5.9039999999999999,"5min":1.4950000000000001,"15min":0.73599999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":57.222999999999999,"5min":57.222999999999999,"15min":57.222999999999999},"last":4.7000000000000002},{"interface":"front","average":{"1min":5.3369999999999997,"5min":1.4159999999999999,"15min":0.72099999999999997},"min":{"1min":0.26800000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":57.408000000000001,"5min":57.408000000000001,"15min":57.408000000000001},"last":5.7380000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.51300000000000001,"15min":0.438},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.2679999999999998,"5min":3.2679999999999998,"15min":3.2679999999999998},"last":3.1930000000000001},{"interface":"front","average":{"1min":1.53,"5min":0.626,"15min":0.47599999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.760999999999999,"5min":10.760999999999999,"15min":10.760999999999999},"last":3.1259999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:34:03 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":0.754,"15min":0.70499999999999996},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.9870000000000001,"5min":3.9870000000000001,"15min":3.9870000000000001},"last":3.6459999999999999},{"interface":"front","average":{"1min":1.6439999999999999,"5min":0.88300000000000001,"15min":0.77500000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.3419999999999996,"5min":7.3419999999999996,"15min":7.3419999999999996},"last":3.5880000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:34:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":0.89200000000000002,"15min":0.751},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":7.3899999999999997,"5min":7.3899999999999997,"15min":7.3899999999999997},"last":3.6200000000000001},{"interface":"front","average":{"1min":1.6839999999999999,"5min":0.876,"15min":0.74199999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.8040000000000003,"5min":6.8040000000000003,"15min":6.8040000000000003},"last":2.8300000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:34:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":0.98399999999999999,"15min":0.79900000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":9.9809999999999999,"5min":9.9809999999999999,"15min":9.9809999999999999},"last":3.7200000000000002},{"interface":"front","average":{"1min":1.7490000000000001,"5min":0.89500000000000002,"15min":0.751},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.8460000000000001,"5min":6.8460000000000001,"15min":6.8460000000000001},"last":2.8660000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:33:02 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.71199999999999997,"15min":0.70999999999999996},"min":{"1min":0.51300000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":2.8450000000000002,"5min":2.8450000000000002,"15min":2.8450000000000002},"last":2.6339999999999999},{"interface":"front","average":{"1min":0.755,"5min":0.71599999999999997,"15min":0.72899999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.448,"5min":2.448,"15min":2.448},"last":2.6739999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795048,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2707872,"kb_used_data":2595864,"kb_used_omap":11,"kb_used_meta":111988,"kb_avail":91037280,"statfs":{"total":95995035648,"available":93222174720,"internally_reserved":0,"allocated":2658164736,"data_stored":2657611309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":114676309},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:34:03 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":0.64400000000000002,"15min":0.45200000000000001},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":5.2510000000000003,"5min":5.2510000000000003,"15min":5.2510000000000003},"last":8.9649999999999999},{"interface":"front","average":{"1min":1.7589999999999999,"5min":0.67300000000000004,"15min":0.48699999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":7.3390000000000004,"5min":7.3390000000000004,"15min":7.3390000000000004},"last":1.022}]},{"osd":1,"last update":"Mon Oct 9 05:34:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":0.65900000000000003,"15min":0.47399999999999998},"min":{"1min":0.27600000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.4160000000000004,"5min":7.4160000000000004,"15min":7.4160000000000004},"last":1.0489999999999999},{"interface":"front","average":{"1min":1.1240000000000001,"5min":0.54100000000000004,"15min":0.434},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.9140000000000001,"5min":3.9140000000000001,"15min":3.9140000000000001},"last":2.0649999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.45300000000000001,"15min":0.40100000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.1280000000000001,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":6.9779999999999998},{"interface":"front","average":{"1min":1.3660000000000001,"5min":0.59399999999999997,"15min":0.46500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.9539999999999997,"5min":5.9539999999999997,"15min":5.9539999999999997},"last":0.876}]},{"osd":4,"last update":"Mon Oct 9 05:34:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":0.82699999999999996,"15min":0.65900000000000003},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.4859999999999998,"5min":4.4859999999999998,"15min":4.4859999999999998},"last":1.9970000000000001},{"interface":"front","average":{"1min":1.696,"5min":0.879,"15min":0.72099999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.7789999999999999,"5min":4.7789999999999999,"15min":4.7789999999999999},"last":9.0060000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:34:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":0.92400000000000004,"15min":0.73599999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.3579999999999997,"5min":4.3579999999999997,"15min":4.3579999999999997},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.6990000000000001,"5min":0.83899999999999997,"15min":0.68400000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.5099999999999998,"5min":4.5099999999999998,"15min":4.5099999999999998},"last":2.032}]},{"osd":6,"last update":"Mon Oct 9 05:34:03 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":0.84399999999999997,"15min":0.69799999999999995},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":4.1760000000000002,"5min":4.1760000000000002,"15min":4.1760000000000002},"last":0.70799999999999996},{"interface":"front","average":{"1min":1.462,"5min":0.83899999999999997,"15min":0.73699999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.5369999999999999,"5min":5.5369999999999999,"15min":5.5369999999999999},"last":0.64500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.68000000000000005,"15min":0.66800000000000004},"min":{"1min":0.46100000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":2.2959999999999998,"5min":2.2959999999999998,"15min":2.2959999999999998},"last":1.5209999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.65400000000000003,"15min":0.65700000000000003},"min":{"1min":0.438,"5min":0.435,"15min":0.435},"max":{"1min":1.2190000000000001,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":0.52000000000000002}]}]},{"osd":3,"up_from":103,"seq":442381631522,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1357972,"kb_used_data":1286220,"kb_used_omap":4,"kb_used_meta":71739,"kb_avail":92387180,"statfs":{"total":95995035648,"available":94604472320,"internally_reserved":0,"allocated":1317089280,"data_stored":1316546410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":73461342},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.504,"15min":0.434},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.1310000000000002,"5min":5.1310000000000002,"15min":5.1310000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.208,"5min":0.54200000000000004,"15min":0.43099999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":6.4089999999999998,"5min":6.4089999999999998,"15min":6.4089999999999998},"last":6.024}]},{"osd":1,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.443,"15min":0.41699999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":2.069},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.39500000000000002,"15min":0.38200000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"last":6.8410000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":6.9960000000000004,"5min":1.7410000000000001,"15min":0.86499999999999999},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":117.23,"5min":117.23,"15min":117.23},"last":0.438},{"interface":"front","average":{"1min":7.6550000000000002,"5min":1.883,"15min":0.92100000000000004},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":117.45999999999999,"5min":117.45999999999999,"15min":117.45999999999999},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.036,"15min":0.74399999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":32.987000000000002,"5min":32.987000000000002,"15min":32.987000000000002},"last":2.7429999999999999},{"interface":"front","average":{"1min":1.044,"5min":0.72699999999999998,"15min":0.67400000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":3.8610000000000002,"5min":3.8610000000000002,"15min":3.8610000000000002},"last":2.0859999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.71499999999999997,"15min":0.66600000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.2919999999999998,"5min":4.2919999999999998,"15min":4.2919999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.74099999999999999,"15min":0.69899999999999995},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.4809999999999999,"5min":4.4809999999999999,"15min":4.4809999999999999},"last":2.04}]},{"osd":6,"last update":"Mon Oct 9 05:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":0.80500000000000005,"15min":0.71599999999999997},"min":{"1min":0.51000000000000001,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":3.9020000000000001,"5min":3.9020000000000001,"15min":3.9020000000000001},"last":2.6240000000000001},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.64400000000000002,"15min":0.629},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.978,"5min":1.978,"15min":1.978},"last":0.51400000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.79200000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019069,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3725352,"kb_used_data":3424032,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":90019800,"statfs":{"total":95995035648,"available":92180275200,"internally_reserved":0,"allocated":3506208768,"data_stored":3505705667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:34:07 2023","interfaces":[{"interface":"back","average":{"1min":2.641,"5min":0.94699999999999995,"15min":0.66300000000000003},"min":{"1min":0.307,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":19.041,"5min":19.041,"15min":19.041},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.9550000000000001,"5min":1.0269999999999999,"15min":0.71099999999999997},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":18.995999999999999,"5min":18.995999999999999,"15min":18.995999999999999},"last":3.496}]},{"osd":1,"last update":"Mon Oct 9 05:33:37 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.57199999999999995,"15min":0.54800000000000004},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.702,"5min":4.702,"15min":4.702},"last":1.788},{"interface":"front","average":{"1min":1.01,"5min":0.65300000000000002,"15min":0.58999999999999997},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.7409999999999997,"5min":4.7409999999999997,"15min":4.7409999999999997},"last":0.378}]},{"osd":2,"last update":"Mon Oct 9 05:34:07 2023","interfaces":[{"interface":"back","average":{"1min":4.0419999999999998,"5min":1.222,"15min":0.752},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":31.030999999999999,"5min":31.030999999999999,"15min":31.030999999999999},"last":1.726},{"interface":"front","average":{"1min":4.3029999999999999,"5min":1.3240000000000001,"15min":0.83099999999999996},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":35.377000000000002,"5min":35.377000000000002,"15min":35.377000000000002},"last":1.292}]},{"osd":3,"last update":"Mon Oct 9 05:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.60699999999999998,"15min":0.57499999999999996},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.8460000000000001,"5min":3.8460000000000001,"15min":3.8460000000000001},"last":1.3640000000000001},{"interface":"front","average":{"1min":1.091,"5min":0.70799999999999996,"15min":0.64400000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":6.0970000000000004,"5min":6.0970000000000004,"15min":6.0970000000000004},"last":0.46999999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.53600000000000003,"15min":0.47199999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.653,"5min":2.653,"15min":2.653},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.59499999999999997,"15min":0.55200000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.311},"max":{"1min":2.5179999999999998,"5min":2.5179999999999998,"15min":2.5179999999999998},"last":0.35799999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:33:43 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.55100000000000005,"15min":0.50600000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":1.978,"5min":1.978,"15min":1.978},"last":1.4610000000000001},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.58999999999999997,"15min":0.53100000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":3.0510000000000002,"5min":3.0510000000000002,"15min":3.0510000000000002},"last":1.399}]},{"osd":7,"last update":"Mon Oct 9 05:33:09 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.55400000000000005,"15min":0.54200000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":1.7470000000000001,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.53600000000000003,"15min":0.53100000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.58199999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822841,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4156120,"kb_used_data":3858896,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":89589032,"statfs":{"total":95995035648,"available":91739168768,"internally_reserved":0,"allocated":3951509504,"data_stored":3951018235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:34:05 2023","interfaces":[{"interface":"back","average":{"1min":6.048,"5min":1.6479999999999999,"15min":0.91700000000000004},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":93.563999999999993,"5min":93.563999999999993,"15min":93.563999999999993},"last":1.962},{"interface":"front","average":{"1min":7.0670000000000002,"5min":1.8720000000000001,"15min":0.99399999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":93.855999999999995,"5min":93.855999999999995,"15min":93.855999999999995},"last":0.61199999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.59699999999999998,"15min":0.56899999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.0190000000000001,"5min":3.0190000000000001,"15min":3.0190000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.0800000000000001,"5min":0.67100000000000004,"15min":0.59499999999999997},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.1740000000000004,"5min":5.1740000000000004,"15min":5.1740000000000004},"last":0.56399999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:33:07 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.7709999999999999,"5min":1.7709999999999999,"15min":1.7709999999999999},"last":1.9430000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.69799999999999995,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":2.8029999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:33:35 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.67800000000000005,"15min":0.621},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.375,"5min":3.375,"15min":3.375},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.64600000000000002,"15min":0.60599999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.1709999999999998,"5min":3.1709999999999998,"15min":3.1709999999999998},"last":1.986}]},{"osd":4,"last update":"Mon Oct 9 05:33:28 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.501,"15min":0.46800000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26200000000000001},"max":{"1min":1.8340000000000001,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":0.439},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.54700000000000004,"15min":0.50800000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.9810000000000001,"5min":1.9810000000000001,"15min":1.9810000000000001},"last":4.7770000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:33:55 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.63,"15min":0.54500000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.26000000000000001},"max":{"1min":4.3739999999999997,"5min":4.3739999999999997,"15min":4.3739999999999997},"last":1.909},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.59399999999999997,"15min":0.55000000000000004},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.4039999999999999,"5min":3.4039999999999999,"15min":3.4039999999999999},"last":0.313}]},{"osd":7,"last update":"Mon Oct 9 05:33:12 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.59999999999999998,"15min":0.58099999999999996},"min":{"1min":0.44400000000000001,"5min":0.40200000000000002,"15min":0.34399999999999997},"max":{"1min":1.5249999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.58799999999999997,"15min":0.56000000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32400000000000001},"max":{"1min":1.8600000000000001,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":0.42999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3634167808,"data_stored":3634167808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2287599616,"data_stored":2287599616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2681470976,"data_stored":2681470976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1327169536,"data_stored":1327169536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3523411968,"data_stored":3523411968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3959947264,"data_stored":3959947264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1748172800,"data_stored":1748172800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1087963136,"data_stored":1087963136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:34:16.734 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:34:16.734 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 36 2023-10-09T05:34:16.775 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:16.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:16.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:16.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:16.898 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:16.898 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:34:16.898 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[191227]: osdmap e121: 8 total, 7 up, 8 in 2023-10-09T05:34:16.899 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[191227]: failed to encode map e121 with expected crc 2023-10-09T05:34:16.899 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[191227]: failed to encode map e121 with expected crc 2023-10-09T05:34:16.899 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[191227]: pgmap v296: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35156/337527 objects degraded (10.416%) 2023-10-09T05:34:16.930 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:16.992 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:16.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:16.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:17.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:17.147 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:16 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:17.147 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:16 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:34:17.147 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:16 smithi191 ceph-mon[161669]: osdmap e121: 8 total, 7 up, 8 in 2023-10-09T05:34:17.147 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:16 smithi191 ceph-mon[161669]: failed to encode map e121 with expected crc 2023-10-09T05:34:17.147 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:16 smithi191 ceph-mon[161669]: failed to encode map e121 with expected crc 2023-10-09T05:34:17.147 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:16 smithi191 ceph-mon[161669]: pgmap v296: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35156/337527 objects degraded (10.416%) 2023-10-09T05:34:17.151 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:34:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[184418]: osdmap e121: 8 total, 7 up, 8 in 2023-10-09T05:34:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[184418]: failed to encode map e121 with expected crc 2023-10-09T05:34:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[184418]: failed to encode map e121 with expected crc 2023-10-09T05:34:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:16 smithi008 ceph-mon[184418]: pgmap v296: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35156/337527 objects degraded (10.416%) 2023-10-09T05:34:17.238 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:34:17.238 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:34:17.335 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:17.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:17.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:17.440 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:17.514 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:17.541 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 85 16 114504 114488 84.1187 43.375 0.00524016 0.0118818 2023-10-09T05:34:18.211 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:17 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 35156/337527 objects degraded (10.416%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:18.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:17 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 35156/337527 objects degraded (10.416%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:18.217 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:17 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 35156/337527 objects degraded (10.416%), 20 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:18.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:34:18.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:18.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:18.322 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:18.334 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:34:18.363 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:18.364 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:18.383 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:34:18.426 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:34:18.445 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:18.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:18.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:18.466 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:34:18.524 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:34:18.538 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 86 16 115244 115228 83.6786 46.25 0.0193725 0.0119449 2023-10-09T05:34:18.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:18.570 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:34:18.602 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:18.607 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:34:18.612 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:34:18.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:34:18.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:34:18.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:34:18.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:34:18.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:34:18.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:34:18.613 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T05:34:18.701 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:18.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:18.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:18.805 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:18.904 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:18.939 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:18 smithi191 ceph-mon[161669]: pgmap v297: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35156/339129 objects degraded (10.367%) 2023-10-09T05:34:19.010 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:19.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:18 smithi008 ceph-mon[184418]: pgmap v297: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35156/339129 objects degraded (10.367%) 2023-10-09T05:34:19.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/862883182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:19.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:18 smithi008 ceph-mon[191227]: pgmap v297: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35156/339129 objects degraded (10.367%) 2023-10-09T05:34:19.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/862883182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:19.263 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/862883182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:19.495 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:19.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:19.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:19.539 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 87 16 116056 116040 83.3003 50.75 0.00745324 0.0119926 2023-10-09T05:34:19.601 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:19.665 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:19.945 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:34:20.006 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:20.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:20.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:20.111 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:20.211 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:20.542 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 88 16 116723 116707 82.8243 41.6875 0.00737532 0.0120642 2023-10-09T05:34:20.628 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T05:34:20.629 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T05:34:20.726 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:20.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:20.833 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:34:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:34:20] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.33.4" 2023-10-09T05:34:20.837 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:20.920 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:20 smithi191 ceph-mon[161669]: pgmap v298: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35637/344349 objects degraded (10.349%) 2023-10-09T05:34:21.075 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:21.084 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:34:21.084 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:34:21.084 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:34:21.084 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:34:21.084 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:34:21.209 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:20 smithi008 ceph-mon[184418]: pgmap v298: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35637/344349 objects degraded (10.349%) 2023-10-09T05:34:21.209 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:20 smithi008 ceph-mon[191227]: pgmap v298: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35637/344349 objects degraded (10.349%) 2023-10-09T05:34:21.216 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:21.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:21.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:21.329 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:21.366 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:21.545 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 89 16 117463 117447 82.4136 46.25 0.0542089 0.0121275 2023-10-09T05:34:21.961 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:21 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2248190647' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:34:21.961 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:21 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:34:21.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:21 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2248190647' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:34:21.962 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:21 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:34:21.998 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 5 pg_num to 36 2023-10-09T05:34:22.134 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:22.136 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:34:22.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:22.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:22.226 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:21 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2248190647' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:34:22.226 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:21 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:34:22.242 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:22.325 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:22.543 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total time run: 90.0099 2023-10-09T05:34:22.544 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total writes made: 118133 2023-10-09T05:34:22.544 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Write size: 65536 2023-10-09T05:34:22.544 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object size: 65536 2023-10-09T05:34:22.544 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Bandwidth (MB/sec): 82.0278 2023-10-09T05:34:22.544 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Bandwidth: 35.3648 2023-10-09T05:34:22.545 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max bandwidth (MB/sec): 198.812 2023-10-09T05:34:22.545 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min bandwidth (MB/sec): 4.125 2023-10-09T05:34:22.545 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average IOPS: 1312 2023-10-09T05:34:22.545 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev IOPS: 565.837 2023-10-09T05:34:22.545 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max IOPS: 3181 2023-10-09T05:34:22.545 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min IOPS: 66 2023-10-09T05:34:22.546 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average Latency(s): 0.0121875 2023-10-09T05:34:22.546 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Latency(s): 0.0210281 2023-10-09T05:34:22.546 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max latency(s): 1.08589 2023-10-09T05:34:22.546 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min latency(s): 0.00107405 2023-10-09T05:34:22.555 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:34:22.630 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:34:22.793 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:23.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[184418]: pgmap v299: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 12 MiB/s wr, 192 op/s; 36333/349176 objects degraded (10.405%) 2023-10-09T05:34:23.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:34:23.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[184418]: osdmap e122: 8 total, 7 up, 8 in 2023-10-09T05:34:23.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[184418]: failed to encode map e122 with expected crc 2023-10-09T05:34:23.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[184418]: failed to encode map e122 with expected crc 2023-10-09T05:34:23.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 36333/349176 objects degraded (10.405%), 20 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:23.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:23.228 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:23.229 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[191227]: pgmap v299: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 12 MiB/s wr, 192 op/s; 36333/349176 objects degraded (10.405%) 2023-10-09T05:34:23.229 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:34:23.229 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[191227]: osdmap e122: 8 total, 7 up, 8 in 2023-10-09T05:34:23.229 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[191227]: failed to encode map e122 with expected crc 2023-10-09T05:34:23.229 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[191227]: failed to encode map e122 with expected crc 2023-10-09T05:34:23.230 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 36333/349176 objects degraded (10.405%), 20 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:23.230 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:23.230 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:22 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:22 smithi191 ceph-mon[161669]: pgmap v299: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 12 MiB/s wr, 192 op/s; 36333/349176 objects degraded (10.405%) 2023-10-09T05:34:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:22 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:34:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:22 smithi191 ceph-mon[161669]: osdmap e122: 8 total, 7 up, 8 in 2023-10-09T05:34:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:22 smithi191 ceph-mon[161669]: failed to encode map e122 with expected crc 2023-10-09T05:34:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:22 smithi191 ceph-mon[161669]: failed to encode map e122 with expected crc 2023-10-09T05:34:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:22 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 36333/349176 objects degraded (10.405%), 20 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:23.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:22 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:23.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:22 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:23.536 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:23.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:23.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:23.644 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:23.684 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:23.931 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:23.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:24.036 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:24.206 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:23 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:34:24.213 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:24.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:23 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:34:24.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:23 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:34:24.263 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:24.333 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:24.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:24.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:24.438 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:24.477 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:24.505 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:34:24.505 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:34:24.505 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:34:24.505 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:34:24.505 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.505 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:50206 5.e 5:75d05c3e:::benchmark_data_smithi008_238708_object50205:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:34:24.505 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:17.469218+0000", 2023-10-09T05:34:24.506 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 67.033618423999997, 2023-10-09T05:34:24.506 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.118221821, 2023-10-09T05:34:24.506 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.506 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.506 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.506 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.506 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.506 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 50206 2023-10-09T05:34:24.506 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.506 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.506 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.507 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.507 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.469218+0000", 2023-10-09T05:34:24.507 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.507 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.507 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.507 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.507 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.469218+0000", 2023-10-09T05:34:24.507 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:34:24.507 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.507 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.507 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.469216+0000", 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0580000000000004e-05 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.469307+0000", 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5187999999999999e-05 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.469322+0000", 2023-10-09T05:34:24.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3249999999999996e-06 2023-10-09T05:34:24.509 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.509 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.469328+0000", 2023-10-09T05:34:24.509 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3545999999999999e-05 2023-10-09T05:34:24.509 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.509 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.469382+0000", 2023-10-09T05:34:24.509 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000136265 2023-10-09T05:34:24.509 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.510 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.510 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.510 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.469518+0000", 2023-10-09T05:34:24.510 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018301 2023-10-09T05:34:24.510 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.510 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.510 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:24.510 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.469701+0000", 2023-10-09T05:34:24.510 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020100819999999998 2023-10-09T05:34:24.510 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.510 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.471711+0000", 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014977059999999999 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.473209+0000", 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.11415940199999999 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.511 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.587368+0000", 2023-10-09T05:34:24.512 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4941999999999998e-05 2023-10-09T05:34:24.512 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.512 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.512 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.512 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.587413+0000", 2023-10-09T05:34:24.512 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6455999999999999e-05 2023-10-09T05:34:24.512 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.512 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.512 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.512 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:17.587440+0000", 2023-10-09T05:34:24.512 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.118221821 2023-10-09T05:34:24.513 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.513 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.513 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.513 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.513 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.513 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54043 5.1 5:88ab6307:::benchmark_data_smithi008_238708_object54042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:34:24.513 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.530687+0000", 2023-10-09T05:34:24.513 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 63.972149094000002, 2023-10-09T05:34:24.513 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0331553470000001, 2023-10-09T05:34:24.513 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.513 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.514 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.514 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.514 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.514 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54043 2023-10-09T05:34:24.514 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.514 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.514 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.514 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.514 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530687+0000", 2023-10-09T05:34:24.514 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.514 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.515 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.515 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.515 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530687+0000", 2023-10-09T05:34:24.515 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:34:24.515 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.515 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.515 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.515 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530685+0000", 2023-10-09T05:34:24.515 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0238999999999999e-05 2023-10-09T05:34:24.516 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.516 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.516 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.516 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530726+0000", 2023-10-09T05:34:24.516 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5817000000000001e-05 2023-10-09T05:34:24.516 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.516 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.516 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.516 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530741+0000", 2023-10-09T05:34:24.516 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3200000000000002e-06 2023-10-09T05:34:24.517 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.517 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.517 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.517 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530749+0000", 2023-10-09T05:34:24.517 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0053712409999999997 2023-10-09T05:34:24.517 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.517 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.517 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.517 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536120+0000", 2023-10-09T05:34:24.518 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1669999999999999e-06 2023-10-09T05:34:24.518 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.518 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.518 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:24.518 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536128+0000", 2023-10-09T05:34:24.518 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0145035060000001 2023-10-09T05:34:24.518 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.518 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.518 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.519 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.550632+0000", 2023-10-09T05:34:24.519 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115936 2023-10-09T05:34:24.519 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.519 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.519 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.519 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.550747+0000", 2023-10-09T05:34:24.519 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014306200000000001 2023-10-09T05:34:24.519 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.519 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.519 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:34:24.520 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.550891+0000", 2023-10-09T05:34:24.520 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059699740000000003 2023-10-09T05:34:24.520 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.520 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.520 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.520 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.556861+0000", 2023-10-09T05:34:24.520 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046747400000000001 2023-10-09T05:34:24.520 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.520 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.520 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.521 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.561535+0000", 2023-10-09T05:34:24.521 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022096709999999999 2023-10-09T05:34:24.521 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.521 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.521 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.521 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563745+0000", 2023-10-09T05:34:24.521 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7768000000000003e-05 2023-10-09T05:34:24.521 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.521 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.522 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.522 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563843+0000", 2023-10-09T05:34:24.522 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0331553470000001 2023-10-09T05:34:24.522 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.522 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.522 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.522 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.522 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.522 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54047 5.1 5:8c453526:::benchmark_data_smithi008_238708_object54046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:34:24.522 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.534316+0000", 2023-10-09T05:34:24.523 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 63.968520628999997, 2023-10-09T05:34:24.523 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.029737098, 2023-10-09T05:34:24.523 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.523 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.523 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.523 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.523 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.523 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54047 2023-10-09T05:34:24.523 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.524 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.524 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.524 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.524 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534316+0000", 2023-10-09T05:34:24.524 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.524 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.524 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.524 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.524 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534316+0000", 2023-10-09T05:34:24.524 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:34:24.525 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.525 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.525 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.525 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534314+0000", 2023-10-09T05:34:24.525 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6623999999999999e-05 2023-10-09T05:34:24.525 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.525 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.525 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.525 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534360+0000", 2023-10-09T05:34:24.526 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0910000000000001e-05 2023-10-09T05:34:24.526 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.526 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.526 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.526 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534371+0000", 2023-10-09T05:34:24.526 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6819999999999997e-06 2023-10-09T05:34:24.526 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.526 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.526 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.527 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534378+0000", 2023-10-09T05:34:24.527 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001771051 2023-10-09T05:34:24.527 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.527 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.527 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.527 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536149+0000", 2023-10-09T05:34:24.527 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5040000000000002e-06 2023-10-09T05:34:24.527 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.527 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.527 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:24.527 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536155+0000", 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.018304179 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554459+0000", 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014521 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554604+0000", 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019115700000000001 2023-10-09T05:34:24.528 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554795+0000", 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0091080610000000006 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563903+0000", 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1428e-05 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563915+0000", 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3801000000000002e-05 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563968+0000", 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4501999999999998e-05 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564053+0000", 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.029737098 2023-10-09T05:34:24.530 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54057 5.1 5:851293d1:::benchmark_data_smithi008_238708_object54056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.550289+0000", 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 63.952547179, 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.021774086, 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.531 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54057 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550289+0000", 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550289+0000", 2023-10-09T05:34:24.532 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:34:24.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550287+0000", 2023-10-09T05:34:24.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2347e-05 2023-10-09T05:34:24.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550330+0000", 2023-10-09T05:34:24.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2855000000000001e-05 2023-10-09T05:34:24.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550353+0000", 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4370000000000001e-06 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550360+0000", 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5225e-05 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550425+0000", 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0944999999999999e-05 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550436+0000", 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.006471347 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.556908+0000", 2023-10-09T05:34:24.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014533999999999999 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.557053+0000", 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00025040400000000002 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.557303+0000", 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068016099999999996 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564105+0000", 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078524939999999998 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.571957+0000", 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6785e-05 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572004+0000", 2023-10-09T05:34:24.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9092000000000003e-05 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572063+0000", 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.021774086 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54062 5.1 5:890b0b94:::benchmark_data_smithi008_238708_object54061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:34:24.538 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.563895+0000", 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 63.938941151000002, 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082578959999999, 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54062 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.539 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563895+0000", 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563895+0000", 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563894+0000", 2023-10-09T05:34:24.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4879000000000001e-05 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563949+0000", 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6708999999999999e-05 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563966+0000", 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9090000000000003e-06 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563972+0000", 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7568000000000002e-05 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.564030+0000", 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0219e-05 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:24.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.564040+0000", 2023-10-09T05:34:24.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99530809600000003 2023-10-09T05:34:24.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.559348+0000", 2023-10-09T05:34:24.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011862999999999999 2023-10-09T05:34:24.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.559467+0000", 2023-10-09T05:34:24.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017885700000000001 2023-10-09T05:34:24.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:34:24.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.559646+0000", 2023-10-09T05:34:24.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0081993729999999994 2023-10-09T05:34:24.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.567845+0000", 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042279680000000004 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572073+0000", 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1065999999999997e-05 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572104+0000", 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8968000000000003e-05 2023-10-09T05:34:24.546 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572153+0000", 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082578959999999 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54068 5.1 5:8319935a:::benchmark_data_smithi008_238708_object54067:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.569507+0000", 2023-10-09T05:34:24.547 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 63.933329853000004, 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082394779999999, 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54068 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569507+0000", 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569507+0000", 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569505+0000", 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8206999999999999e-05 2023-10-09T05:34:24.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569533+0000", 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3752e-05 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569547+0000", 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0120000000000002e-06 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569553+0000", 2023-10-09T05:34:24.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3130999999999997e-05 2023-10-09T05:34:24.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.551 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.551 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569606+0000", 2023-10-09T05:34:24.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4290000000000001e-06 2023-10-09T05:34:24.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.551 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.551 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:24.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569614+0000", 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99451796599999998 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564132+0000", 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9108e-05 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564232+0000", 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115476 2023-10-09T05:34:24.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564347+0000", 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078146750000000001 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572162+0000", 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0053927940000000002 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.553 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.577554+0000", 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8337999999999995e-05 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.577623+0000", 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012327700000000001 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.577746+0000", 2023-10-09T05:34:24.554 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082394779999999 2023-10-09T05:34:24.555 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.555 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.555 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.555 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.555 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.555 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54070 5.1 5:819fd021:::benchmark_data_smithi008_238708_object54069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:34:24.555 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.571140+0000", 2023-10-09T05:34:24.555 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 63.931696713999997, 2023-10-09T05:34:24.555 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.009647422, 2023-10-09T05:34:24.555 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.555 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.556 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.556 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.556 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.556 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54070 2023-10-09T05:34:24.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.556 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.556 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.556 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571140+0000", 2023-10-09T05:34:24.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.557 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571140+0000", 2023-10-09T05:34:24.557 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:24.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.557 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571138+0000", 2023-10-09T05:34:24.557 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9555999999999998e-05 2023-10-09T05:34:24.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571188+0000", 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.417e-05 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571202+0000", 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7789999999999998e-06 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571209+0000", 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6211000000000002e-05 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571255+0000", 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2060000000000001e-06 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571262+0000", 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99464801999999997 2023-10-09T05:34:24.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.565910+0000", 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000141688 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.566052+0000", 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000160173 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:34:24.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.566212+0000", 2023-10-09T05:34:24.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059532430000000004 2023-10-09T05:34:24.561 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.561 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.561 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572165+0000", 2023-10-09T05:34:24.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0084587169999999993 2023-10-09T05:34:24.561 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.561 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.561 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.580624+0000", 2023-10-09T05:34:24.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9176999999999999e-05 2023-10-09T05:34:24.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.562 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.580683+0000", 2023-10-09T05:34:24.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010398 2023-10-09T05:34:24.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.562 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.580787+0000", 2023-10-09T05:34:24.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.009647422 2023-10-09T05:34:24.562 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54080 5.1 5:84267530:::benchmark_data_smithi008_238708_object54079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.584125+0000", 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 63.918711893000001, 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99768334199999997, 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.563 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54080 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584125+0000", 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584125+0000", 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584123+0000", 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5760000000000001e-05 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584149+0000", 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3940999999999999e-05 2023-10-09T05:34:24.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584163+0000", 2023-10-09T05:34:24.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6529999999999994e-06 2023-10-09T05:34:24.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584171+0000", 2023-10-09T05:34:24.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0436999999999998e-05 2023-10-09T05:34:24.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584221+0000", 2023-10-09T05:34:24.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.153e-06 2023-10-09T05:34:24.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:24.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584228+0000", 2023-10-09T05:34:24.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98363814599999999 2023-10-09T05:34:24.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.567866+0000", 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013288699999999999 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.567999+0000", 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015074399999999999 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.568150+0000", 2023-10-09T05:34:24.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040185330000000003 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572168+0000", 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0094922050000000001 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.581661+0000", 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6870000000000003e-05 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.581717+0000", 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0416999999999998e-05 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.581808+0000", 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99768334199999997 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54571 5.e 5:72c2dcea:::benchmark_data_smithi008_238708_object54570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e113)", 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:21.884258+0000", 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 62.618578296000003, 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.140310035, 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54571 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.884258+0000", 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.884258+0000", 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.884256+0000", 2023-10-09T05:34:24.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7058999999999997e-05 2023-10-09T05:34:24.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.884313+0000", 2023-10-09T05:34:24.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7467000000000001e-05 2023-10-09T05:34:24.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.884331+0000", 2023-10-09T05:34:24.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7009999999999996e-06 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.884339+0000", 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5458000000000003e-05 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.884394+0000", 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000146887 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.884541+0000", 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015276800000000001 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.884694+0000", 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029474589999999999 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.887641+0000", 2023-10-09T05:34:24.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00065271700000000003 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.888294+0000", 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.136177205 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:22.024471+0000", 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8260000000000001e-05 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:22.024529+0000", 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8853999999999998e-05 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:22.024568+0000", 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.140310035 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:98116 5.e 5:72fadb03:::benchmark_data_smithi008_238708_object98115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:00.575356+0000", 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.927479959999999, 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14863536699999999, 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 98116 2023-10-09T05:34:24.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.575356+0000", 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.575356+0000", 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.575355+0000", 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103125 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.575458+0000", 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7850000000000003e-06 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.575468+0000", 2023-10-09T05:34:24.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4179999999999996e-06 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.575473+0000", 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029655020000000001 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.578439+0000", 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000141249 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.578580+0000", 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015819599999999999 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.578738+0000", 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00314409 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.581882+0000", 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025408570000000001 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.584423+0000", 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13948085199999999 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.723904+0000", 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5986000000000003e-05 2023-10-09T05:34:24.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.723950+0000", 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1737999999999999e-05 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.723992+0000", 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14863536699999999 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.584 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:98118 5.5 e117/105 5:a3d60b4a:::benchmark_data_smithi008_238708_object98117:head v 117'6250, mlcod=117'6248)", 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:00.583194+0000", 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.91964269, 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.120601894, 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583194+0000", 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583194+0000", 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583193+0000", 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6005000000000003e-05 2023-10-09T05:34:24.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583249+0000", 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4127e-05 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583263+0000", 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5160000000000002e-06 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583269+0000", 2023-10-09T05:34:24.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7435000000000002e-05 2023-10-09T05:34:24.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583316+0000", 2023-10-09T05:34:24.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3207999999999999e-05 2023-10-09T05:34:24.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583339+0000", 2023-10-09T05:34:24.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12045644799999999 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.703796+0000", 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.120601894 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:116015 5.e 5:706fcd4c:::benchmark_data_smithi008_238708_object116014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:19.409350+0000", 2023-10-09T05:34:24.589 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 5.0934860220000004, 2023-10-09T05:34:24.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15295283000000001, 2023-10-09T05:34:24.590 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.590 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.590 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.590 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.590 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.590 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 116015 2023-10-09T05:34:24.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.590 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.409350+0000", 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.409350+0000", 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.409349+0000", 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5007999999999996e-05 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.409444+0000", 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3634e-05 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.409457+0000", 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7639999999999994e-06 2023-10-09T05:34:24.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.409466+0000", 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004993066 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.414459+0000", 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014029099999999999 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.414600+0000", 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016013600000000001 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.414760+0000", 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031836590000000001 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.417943+0000", 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048645269999999996 2023-10-09T05:34:24.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.422808+0000", 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13941573700000001 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.562224+0000", 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6788e-05 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.562270+0000", 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2843999999999999e-05 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.562303+0000", 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15295283000000001 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:116388 5.11 5:8cc2a231:::benchmark_data_smithi008_238708_object116387:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:19.941552+0000", 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 4.5612846539999996, 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13198232100000001, 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 116388 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.941552+0000", 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.941552+0000", 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.941551+0000", 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4042999999999997e-05 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.941605+0000", 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3789999999999995e-06 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.941614+0000", 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8239999999999999e-06 2023-10-09T05:34:24.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.941619+0000", 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2319e-05 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.941651+0000", 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119553 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.941771+0000", 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6810000000000003e-05 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.941867+0000", 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001114309 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.942982+0000", 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040858869999999999 2023-10-09T05:34:24.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.947068+0000", 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12636794500000001 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073436+0000", 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8484000000000001e-05 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073534+0000", 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13198232100000001 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:116387 5.c e121/105 5:32f306c3:::benchmark_data_smithi008_238708_object116386:head v 121'7345, mlcod=121'7343)", 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:19.945338+0000", 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 4.5574982989999997, 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12811072300000001, 2023-10-09T05:34:24.603 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.604 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.604 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.945338+0000", 2023-10-09T05:34:24.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.945338+0000", 2023-10-09T05:34:24.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.945337+0000", 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2880999999999996e-05 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.945400+0000", 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1357e-05 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.945411+0000", 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0349999999999998e-06 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.945418+0000", 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8195000000000003e-05 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.945456+0000", 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3349999999999999e-05 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.945469+0000", 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.127942104 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073411+0000", 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7707999999999999e-05 2023-10-09T05:34:24.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073449+0000", 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12811072300000001 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.34770.0:116388 5.11 e121/118 ondisk, result = 0)", 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:19.946998+0000", 2023-10-09T05:34:24.608 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 4.5558383019999997, 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.126546409, 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.946998+0000", 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.946998+0000", 2023-10-09T05:34:24.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.946997+0000", 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0120000000000005e-06 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.947005+0000", 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0599999999999997e-06 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.947014+0000", 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4379999999999997e-06 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.947019+0000", 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8519999999999997e-05 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.947057+0000", 2023-10-09T05:34:24.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.271e-06 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.947063+0000", 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12648188899999999 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073545+0000", 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.126546409 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.612 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:116392 5.11 5:8c82909e:::benchmark_data_smithi008_238708_object116391:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:19.949534+0000", 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 4.553301952, 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.130575782, 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.613 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 116392 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.949534+0000", 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.949534+0000", 2023-10-09T05:34:24.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:24.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.949533+0000", 2023-10-09T05:34:24.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3374999999999999e-05 2023-10-09T05:34:24.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.949556+0000", 2023-10-09T05:34:24.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.123822847 2023-10-09T05:34:24.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073379+0000", 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2756e-05 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073392+0000", 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00025645099999999999 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073648+0000", 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000101714 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073750+0000", 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015425599999999999 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073904+0000", 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042578679999999997 2023-10-09T05:34:24.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.078162+0000", 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018127849999999999 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.079975+0000", 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5163000000000001e-05 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.080020+0000", 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0224999999999998e-05 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.080110+0000", 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.130575782 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.620 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:116394 5.f e121/105 5:f9663045:::benchmark_data_smithi008_238708_object116393:head v 121'7245, mlcod=121'7243)", 2023-10-09T05:34:24.620 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:19.952920+0000", 2023-10-09T05:34:24.620 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 4.5499167160000002, 2023-10-09T05:34:24.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12869285899999999, 2023-10-09T05:34:24.620 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.620 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.620 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.952920+0000", 2023-10-09T05:34:24.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.952920+0000", 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.952919+0000", 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.743e-05 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:19.952946+0000", 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.120444048 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073390+0000", 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8709999999999997e-06 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073398+0000", 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000164606 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073563+0000", 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7665999999999999e-05 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.073590+0000", 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079747040000000009 2023-10-09T05:34:24.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.081565+0000", 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7428000000000001e-05 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:20.081613+0000", 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12869285899999999 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.624 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:118740 5.5 e122/118 5:a19fee25:::benchmark_data_smithi008_238708_object604:head v 122'6915, mlcod=122'6913)", 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:23.558111+0000", 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.94472498599999999, 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.137534827, 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.558111+0000", 2023-10-09T05:34:24.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.558111+0000", 2023-10-09T05:34:24.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:24.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.558110+0000", 2023-10-09T05:34:24.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2996999999999999e-05 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.558123+0000", 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7980000000000001e-05 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.558141+0000", 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.007e-05 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.558151+0000", 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13436724799999999 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.692518+0000", 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2625000000000001e-05 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.692541+0000", 2023-10-09T05:34:24.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030900939999999998 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.695631+0000", 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5196e-05 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.695646+0000", 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.137534827 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.629 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119020 5.e 5:7d0c0384:::benchmark_data_smithi008_238708_object884:head [delete] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:23.995511+0000", 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.50732530600000003, 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25974255699999999, 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.630 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119020 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995511+0000", 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995511+0000", 2023-10-09T05:34:24.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995510+0000", 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7929999999999999e-06 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995514+0000", 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5429999999999996e-06 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995522+0000", 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6040000000000003e-06 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995526+0000", 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029400580000000002 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.998466+0000", 2023-10-09T05:34:24.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000129059 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.998596+0000", 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3993000000000006e-05 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.998690+0000", 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0057475019999999998 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.004437+0000", 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25080364799999999 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255241+0000", 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3033e-05 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255254+0000", 2023-10-09T05:34:24.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25974255699999999 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119026 5.e 5:7513f17c:::benchmark_data_smithi008_238708_object890:head [delete] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:23.995587+0000", 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.50724972599999996, 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25971648200000003, 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:34:24.636 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119026 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995587+0000", 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:24.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995587+0000", 2023-10-09T05:34:24.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:24.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:24.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995586+0000", 2023-10-09T05:34:24.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6729999999999998e-06 2023-10-09T05:34:24.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995590+0000", 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1881e-05 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995602+0000", 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3959999999999999e-06 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.995606+0000", 2023-10-09T05:34:24.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0054507760000000001 2023-10-09T05:34:24.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001057+0000", 2023-10-09T05:34:24.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114603 2023-10-09T05:34:24.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:24.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001171+0000", 2023-10-09T05:34:24.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111282 2023-10-09T05:34:24.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001283+0000", 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033640229999999998 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.004647+0000", 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25064161899999998 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:24.642 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255288+0000", 2023-10-09T05:34:24.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.486e-05 2023-10-09T05:34:24.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:24.642 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:24.642 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:24.642 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255303+0000", 2023-10-09T05:34:24.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25971648200000003 2023-10-09T05:34:24.642 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.642 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.642 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.643 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:24.643 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:24.643 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:34:24.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:24.644 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:24.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:24.745 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:24.790 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:25.035 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:34:25.189 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:25.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:25.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:25.252 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:34:25.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:25 smithi191 ceph-mon[161669]: pgmap v301: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 602 op/s; 36333/350970 objects degraded (10.352%) 2023-10-09T05:34:25.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:25 smithi191 ceph-mon[161669]: failed to encode map e123 with expected crc 2023-10-09T05:34:25.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:25 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:34:25.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:25 smithi191 ceph-mon[161669]: osdmap e123: 8 total, 7 up, 8 in 2023-10-09T05:34:25.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:25 smithi191 ceph-mon[161669]: failed to encode map e123 with expected crc 2023-10-09T05:34:25.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:25 smithi191 ceph-mon[161669]: failed to encode map e123 with expected crc 2023-10-09T05:34:25.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:25 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1438330709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:25.299 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:25.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[184418]: pgmap v301: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 602 op/s; 36333/350970 objects degraded (10.352%) 2023-10-09T05:34:25.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[184418]: failed to encode map e123 with expected crc 2023-10-09T05:34:25.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:34:25.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[184418]: osdmap e123: 8 total, 7 up, 8 in 2023-10-09T05:34:25.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[184418]: failed to encode map e123 with expected crc 2023-10-09T05:34:25.327 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[184418]: failed to encode map e123 with expected crc 2023-10-09T05:34:25.327 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1438330709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:25.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[191227]: pgmap v301: 99 pgs: 1 active+clean+scrubbing+deep, 20 active+undersized+degraded, 19 active+undersized, 59 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 602 op/s; 36333/350970 objects degraded (10.352%) 2023-10-09T05:34:25.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[191227]: failed to encode map e123 with expected crc 2023-10-09T05:34:25.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:34:25.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[191227]: osdmap e123: 8 total, 7 up, 8 in 2023-10-09T05:34:25.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[191227]: failed to encode map e123 with expected crc 2023-10-09T05:34:25.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[191227]: failed to encode map e123 with expected crc 2023-10-09T05:34:25.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:25 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1438330709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:25.333 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:25.447 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:25.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:25.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:25.553 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:25.615 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:25.723 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:25.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:25.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:25.831 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:25.859 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:25.881 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:26.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:26.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[184418]: failed to encode map e124 with expected crc 2023-10-09T05:34:26.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1438330709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:26.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[184418]: osdmap e124: 8 total, 7 up, 8 in 2023-10-09T05:34:26.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[184418]: failed to encode map e124 with expected crc 2023-10-09T05:34:26.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[184418]: failed to encode map e124 with expected crc 2023-10-09T05:34:26.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:26.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[191227]: failed to encode map e124 with expected crc 2023-10-09T05:34:26.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1438330709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:26.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[191227]: osdmap e124: 8 total, 7 up, 8 in 2023-10-09T05:34:26.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[191227]: failed to encode map e124 with expected crc 2023-10-09T05:34:26.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:26 smithi008 ceph-mon[191227]: failed to encode map e124 with expected crc 2023-10-09T05:34:26.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:26 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:26.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:26 smithi191 ceph-mon[161669]: failed to encode map e124 with expected crc 2023-10-09T05:34:26.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1438330709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:26.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:26 smithi191 ceph-mon[161669]: osdmap e124: 8 total, 7 up, 8 in 2023-10-09T05:34:26.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:26 smithi191 ceph-mon[161669]: failed to encode map e124 with expected crc 2023-10-09T05:34:26.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:26 smithi191 ceph-mon[161669]: failed to encode map e124 with expected crc 2023-10-09T05:34:26.752 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:26.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:26.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:26.855 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:26.921 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:27.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:27 smithi008 ceph-mon[184418]: pgmap v304: 109 pgs: 6 unknown, 4 peering, 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 32578/317574 objects degraded (10.258%) 2023-10-09T05:34:27.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:27 smithi008 ceph-mon[191227]: pgmap v304: 109 pgs: 6 unknown, 4 peering, 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 32578/317574 objects degraded (10.258%) 2023-10-09T05:34:27.231 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:27.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:27.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:27.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:27 smithi191 ceph-mon[161669]: pgmap v304: 109 pgs: 6 unknown, 4 peering, 20 active+undersized+degraded, 19 active+undersized, 60 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 32578/317574 objects degraded (10.258%) 2023-10-09T05:34:27.309 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:34:27.337 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:27.346 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:34:27.387 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:34:27.405 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:27.428 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:34:27.464 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:34:27.505 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:34:27.508 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:27.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:27.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:27.551 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:34:27.552 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:34:27.552 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:34:27.552 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:34:27.552 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:34:27.552 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:34:27.552 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:34:27.552 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:34:27.552 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T05:34:27.618 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:27.658 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:34:27.659 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:34:27.683 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:27.850 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:27.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:27.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:27.969 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:27.997 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:28.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2533168504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:28.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:28 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:28.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:28 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 32578/317574 objects degraded (10.258%), 20 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2533168504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:28 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:28 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 32578/317574 objects degraded (10.258%), 20 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2533168504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:28 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:28 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 32578/317574 objects degraded (10.258%), 20 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:28.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:34:28.557 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:28.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:28.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:28.629 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:34:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T05:34:28.656 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T05:34:28.662 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:28.799 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:28.865 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2023-10-09T05:34:28.865 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:34:28.865 INFO:tasks.thrashosds.thrasher:Reviving osd 7 2023-10-09T05:34:28.866 INFO:tasks.cephadm.osd.7:Restarting osd.7 (starting--it wasn't running)... 2023-10-09T05:34:28.866 DEBUG:teuthology.orchestra.run.smithi191:osd.7> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T05:34:28.868 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl start ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T05:34:28.917 INFO:journalctl@ceph.osd.7.smithi191.stdout:-- Logs begin at Mon 2023-10-09 04:57:51 UTC. -- 2023-10-09T05:34:28.931 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:28.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:28.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:28.960 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:29.042 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:29.086 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:29.207 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:29.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:29.214 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:28 smithi191 systemd[1]: Starting Ceph osd.7 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T05:34:29.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:29.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:29.335 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:29.387 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:29.615 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:29.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:29.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:29.721 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:29.773 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:29.888 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[184418]: pgmap v305: 109 pgs: 5 unknown, 4 peering, 20 active+undersized+degraded, 19 active+undersized, 61 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 32578/316089 objects degraded (10.307%) 2023-10-09T05:34:29.888 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:34:29.888 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:34:29.888 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:29.888 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:29.888 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:29.889 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[191227]: pgmap v305: 109 pgs: 5 unknown, 4 peering, 20 active+undersized+degraded, 19 active+undersized, 61 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 32578/316089 objects degraded (10.307%) 2023-10-09T05:34:29.889 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:34:29.889 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:34:29.889 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:29.889 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:29.889 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:29 smithi191 ceph-mon[161669]: pgmap v305: 109 pgs: 5 unknown, 4 peering, 20 active+undersized+degraded, 19 active+undersized, 61 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 32578/316089 objects degraded (10.307%) 2023-10-09T05:34:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:34:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:34:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:30.438 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:30.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:30.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:30.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:30.575 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:30.628 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:30.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:30.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:30.656 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:34:30.732 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:30.757 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:34:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:34:30] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.33.4" 2023-10-09T05:34:30.759 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:30.808 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:30.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:30.911 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:30.965 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:31.205 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:31.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:31.229 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:34:31.230 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:34:31.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:31.371 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:31.422 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:31.454 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:31.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:31.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:31.513 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:31 smithi191 bash[180321]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-10-09T05:34:31.513 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:31 smithi191 bash[180321]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2023-10-09T05:34:31.513 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:31 smithi191 bash[180321]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2023-10-09T05:34:31.513 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:31 smithi191 bash[180321]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T05:34:31.513 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:31 smithi191 bash[180321]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2023-10-09T05:34:31.513 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:31 smithi191 bash[180321]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-10-09T05:34:31.513 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:31 smithi191 bash[180321]: --> ceph-volume raw activate successful for osd ID: 7 2023-10-09T05:34:31.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:31.587 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:31.658 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:31.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:31.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:31.767 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:31 smithi191 ceph-mon[161669]: pgmap v306: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 128 B/s rd, 14 MiB/s wr, 796 op/s; 35421/341016 objects degraded (10.387%) 2023-10-09T05:34:31.824 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:31 smithi008 ceph-mon[184418]: pgmap v306: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 128 B/s rd, 14 MiB/s wr, 796 op/s; 35421/341016 objects degraded (10.387%) 2023-10-09T05:34:31.825 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:31 smithi008 ceph-mon[191227]: pgmap v306: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 128 B/s rd, 14 MiB/s wr, 796 op/s; 35421/341016 objects degraded (10.387%) 2023-10-09T05:34:31.836 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:31.968 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:31.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:31.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:32.081 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:32.128 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:32.373 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:32.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:32.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:32.479 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:32.532 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:32.577 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:34:32.614 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:34:32.629 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:32.635 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:32 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1876817019' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:34:32.635 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:32 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:34:32.635 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:32 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 35421/341016 objects degraded (10.387%), 22 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:32.635 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:32 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1876817019' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:34:32.635 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:32 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:34:32.636 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:32 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 35421/341016 objects degraded (10.387%), 22 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:32.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:32.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:32.739 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:32.767 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:32.810 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:32.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:32.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:32.914 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:32.986 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:32 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1876817019' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:34:32.986 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:32 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:34:32.986 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:32 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 35421/341016 objects degraded (10.387%), 22 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:32.998 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:33.272 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:33.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:33.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:33.374 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:33.379 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:33.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:34:33.419 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:33.553 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:33 smithi191 bash[181119]: 6282488b13e674a60c927473c4d58e5b5597ad6c56ef3352eec88ff1e16bb2b4 2023-10-09T05:34:33.553 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:33 smithi191 conmon[181254]: debug 2023-10-09T05:34:33.367+0000 7efc03e19700 1 -- 172.21.15.191:0/3331755353 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55eb854c2680 con 0x55eb85489400 2023-10-09T05:34:33.553 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:33 smithi191 systemd[1]: Started Ceph osd.7 for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T05:34:33.902 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[184418]: pgmap v307: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 9.2 MiB/s wr, 1.05k op/s; 34292/332874 objects degraded (10.302%) 2023-10-09T05:34:33.902 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:33.903 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[184418]: failed to encode map e125 with expected crc 2023-10-09T05:34:33.903 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:34:33.903 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[184418]: osdmap e125: 8 total, 7 up, 8 in 2023-10-09T05:34:33.903 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[184418]: failed to encode map e125 with expected crc 2023-10-09T05:34:33.903 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[184418]: 5.2 scrub starts 2023-10-09T05:34:33.903 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[191227]: pgmap v307: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 9.2 MiB/s wr, 1.05k op/s; 34292/332874 objects degraded (10.302%) 2023-10-09T05:34:33.903 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:33.903 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[191227]: failed to encode map e125 with expected crc 2023-10-09T05:34:33.903 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:34:33.903 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[191227]: osdmap e125: 8 total, 7 up, 8 in 2023-10-09T05:34:33.904 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[191227]: failed to encode map e125 with expected crc 2023-10-09T05:34:33.904 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:33 smithi008 ceph-mon[191227]: 5.2 scrub starts 2023-10-09T05:34:33.904 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:33 smithi191 ceph-mon[161669]: pgmap v307: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 9.2 MiB/s wr, 1.05k op/s; 34292/332874 objects degraded (10.302%) 2023-10-09T05:34:33.904 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:33 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:33.904 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:33 smithi191 ceph-mon[161669]: failed to encode map e125 with expected crc 2023-10-09T05:34:33.904 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:33 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:34:33.904 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:33 smithi191 ceph-mon[161669]: osdmap e125: 8 total, 7 up, 8 in 2023-10-09T05:34:33.904 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:33 smithi191 ceph-mon[161669]: failed to encode map e125 with expected crc 2023-10-09T05:34:33.905 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:33 smithi191 ceph-mon[161669]: 5.2 scrub starts 2023-10-09T05:34:33.905 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:33 smithi191 conmon[181254]: debug 2023-10-09T05:34:33.900+0000 7efc0b894200 -1 Falling back to public interface 2023-10-09T05:34:33.990 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:34.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:34.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:34.104 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:34.157 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:34.248 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:34.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:34.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:34.412 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:34.501 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:34.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:34.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:34.592 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:34:34.606 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:34.621 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:34:34.643 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:34.646 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:34:34.679 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:34:34.717 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:34:34.748 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:34:34.777 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:34.779 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:34:34.780 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:34:34.780 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:34:34.780 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:34:34.780 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:34:34.780 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:34:34.780 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:34:34.780 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:34:34.780 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-10-09T05:34:34.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:34.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:34 smithi191 ceph-mon[161669]: 5.0 scrub starts 2023-10-09T05:34:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:34 smithi191 ceph-mon[161669]: 5.2 scrub ok 2023-10-09T05:34:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:34 smithi191 ceph-mon[161669]: 5.7 scrub starts 2023-10-09T05:34:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2699303564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:34.888 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:34.935 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:34.960 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:34 smithi008 ceph-mon[184418]: 5.0 scrub starts 2023-10-09T05:34:34.960 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:34 smithi008 ceph-mon[184418]: 5.2 scrub ok 2023-10-09T05:34:34.960 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:34 smithi008 ceph-mon[184418]: 5.7 scrub starts 2023-10-09T05:34:34.960 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2699303564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:34.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:34 smithi008 ceph-mon[191227]: 5.0 scrub starts 2023-10-09T05:34:34.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:34 smithi008 ceph-mon[191227]: 5.2 scrub ok 2023-10-09T05:34:34.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:34 smithi008 ceph-mon[191227]: 5.7 scrub starts 2023-10-09T05:34:34.961 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2699303564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:35.007 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:35.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:35.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:35.100 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:35.120 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:35.200 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:35.300 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:35.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:35.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:35.404 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:35.460 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:35.555 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:35.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:35.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:35.661 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:35.700 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:35 smithi191 ceph-mon[161669]: 5.0 scrub ok 2023-10-09T05:34:35.700 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:35 smithi191 ceph-mon[161669]: pgmap v309: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 114 B/s rd, 69 MiB/s wr, 2.03k op/s; 34292/329175 objects degraded (10.418%) 2023-10-09T05:34:35.700 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:35 smithi191 ceph-mon[161669]: 5.22 scrub starts 2023-10-09T05:34:35.700 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:35 smithi191 ceph-mon[161669]: 5.a scrub starts 2023-10-09T05:34:35.712 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:35.730 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:34:35.730 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:34:35.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:34:35.730 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:34:35.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:11001 5.7 5:ea85ce96:::benchmark_data_smithi008_238708_object11000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:32:56.049850+0000", 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 99.677612138000001, 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14689695, 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 11001 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.731 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.049850+0000", 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.049850+0000", 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.049848+0000", 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5807999999999998e-05 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.049904+0000", 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0387999999999999e-05 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.049915+0000", 2023-10-09T05:34:35.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7099999999999998e-06 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.049919+0000", 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7423999999999999e-05 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.049967+0000", 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012609399999999999 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.050093+0000", 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015343900000000001 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.050246+0000", 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00064992399999999997 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:35.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.050896+0000", 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002478606 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.053375+0000", 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14327674800000001 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.196652+0000", 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5515e-05 2023-10-09T05:34:35.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.196707+0000", 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9396e-05 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.196746+0000", 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14689695 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.737 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.738 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:11019 5.7 5:e28f22ab:::benchmark_data_smithi008_238708_object11018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:34:35.738 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:32:56.056541+0000", 2023-10-09T05:34:35.738 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 99.670921074999995, 2023-10-09T05:34:35.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14038457900000001, 2023-10-09T05:34:35.738 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.738 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.738 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:35.738 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:35.738 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 11019 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056541+0000", 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056541+0000", 2023-10-09T05:34:35.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056539+0000", 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8631000000000002e-05 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056598+0000", 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2805e-05 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056610+0000", 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8640000000000001e-06 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056616+0000", 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0272999999999998e-05 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056667+0000", 2023-10-09T05:34:35.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012323999999999999 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056790+0000", 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014696800000000001 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056937+0000", 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001309789 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.058247+0000", 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2310000000000001e-05 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.058279+0000", 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.138535037 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.196814+0000", 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4445999999999999e-05 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.196878+0000", 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6847000000000001e-05 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.196925+0000", 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14038457900000001 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.744 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:11020 5.7 5:e46ec1ba:::benchmark_data_smithi008_238708_object11019:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:32:56.056722+0000", 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 99.670740160999998, 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14030889499999999, 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:35.745 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:35.746 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 11020 2023-10-09T05:34:35.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.746 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.746 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056722+0000", 2023-10-09T05:34:35.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.746 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056722+0000", 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056721+0000", 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100955 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056821+0000", 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3298e-05 2023-10-09T05:34:35.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056835+0000", 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.956e-06 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.056842+0000", 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00067086000000000001 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.057513+0000", 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000123122 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.057636+0000", 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015995699999999999 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.057796+0000", 2023-10-09T05:34:35.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001088524 2023-10-09T05:34:35.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.058884+0000", 2023-10-09T05:34:35.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000188555 2023-10-09T05:34:35.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:35.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.059073+0000", 2023-10-09T05:34:35.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13790084 2023-10-09T05:34:35.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.196974+0000", 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0369999999999999e-05 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.197004+0000", 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6455e-05 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:56.197030+0000", 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14030889499999999 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:19142 5.7 5:ea7cadf4:::benchmark_data_smithi008_238708_object19141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:32:59.247533+0000", 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 96.479928498000007, 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13641583900000001, 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.752 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 19142 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.247533+0000", 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.247533+0000", 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.247532+0000", 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6922999999999999e-05 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.247569+0000", 2023-10-09T05:34:35.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2554000000000001e-05 2023-10-09T05:34:35.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.755 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.755 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.247591+0000", 2023-10-09T05:34:35.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5969999999999997e-06 2023-10-09T05:34:35.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.755 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.755 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.247598+0000", 2023-10-09T05:34:35.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8502000000000003e-05 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.247656+0000", 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013975500000000001 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.247796+0000", 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017001000000000001 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.247966+0000", 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001740033 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.249706+0000", 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002233699 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.251940+0000", 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13192416600000001 2023-10-09T05:34:35.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.383864+0000", 2023-10-09T05:34:35.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0878999999999999e-05 2023-10-09T05:34:35.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.383905+0000", 2023-10-09T05:34:35.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4134000000000001e-05 2023-10-09T05:34:35.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.383949+0000", 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13641583900000001 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:19152 5.7 5:e977ac15:::benchmark_data_smithi008_238708_object19151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:32:59.250006+0000", 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 96.477455575999997, 2023-10-09T05:34:35.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.137928998, 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 19152 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.250006+0000", 2023-10-09T05:34:35.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.250006+0000", 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.250005+0000", 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3822999999999998e-05 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.250039+0000", 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1880000000000001e-06 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.250044+0000", 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.269e-06 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.250048+0000", 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000548586 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.250596+0000", 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013101000000000001 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.250727+0000", 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000140486 2023-10-09T05:34:35.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.250868+0000", 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014043090000000001 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.252272+0000", 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000148756 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.252421+0000", 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13541082800000001 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.387832+0000", 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9360000000000002e-05 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.387881+0000", 2023-10-09T05:34:35.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4005000000000002e-05 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:32:59.387935+0000", 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.137928998 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:98116 5.e e117/105 5:72fadb03:::benchmark_data_smithi008_238708_object98115:head v 117'6238, mlcod=117'6234)", 2023-10-09T05:34:35.766 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:00.578842+0000", 2023-10-09T05:34:35.767 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 35.148619771, 2023-10-09T05:34:35.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14137978500000001, 2023-10-09T05:34:35.767 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.767 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.767 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.578842+0000", 2023-10-09T05:34:35.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.578842+0000", 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.578841+0000", 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3799e-05 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.578865+0000", 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2296999999999999e-05 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.578877+0000", 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4150000000000001e-06 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.578882+0000", 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024689759999999999 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.581351+0000", 2023-10-09T05:34:35.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1421e-05 2023-10-09T05:34:35.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.581372+0000", 2023-10-09T05:34:35.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13873474699999999 2023-10-09T05:34:35.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.720107+0000", 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114999 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.720222+0000", 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14137978500000001 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:118711 5.13 e122/118 5:c9ff11ec:::benchmark_data_smithi008_238708_object575:head v 122'6702, mlcod=122'6700)", 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:23.528613+0000", 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 12.198848396000001, 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.136013942, 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.528613+0000", 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.772 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.528613+0000", 2023-10-09T05:34:35.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:35.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.528613+0000", 2023-10-09T05:34:35.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8959999999999996e-06 2023-10-09T05:34:35.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.528617+0000", 2023-10-09T05:34:35.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0190000000000003e-06 2023-10-09T05:34:35.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.528623+0000", 2023-10-09T05:34:35.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7790000000000002e-06 2023-10-09T05:34:35.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.528627+0000", 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5936000000000003e-05 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.528663+0000", 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9259e-05 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.528682+0000", 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13591302699999999 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.664595+0000", 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2330999999999999e-05 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.664627+0000", 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.136013942 2023-10-09T05:34:35.776 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:118719 5.f 5:f561b4fc:::benchmark_data_smithi008_238708_object583:head [delete] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:23.529134+0000", 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 12.198327824, 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17491253800000001, 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:35.777 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:35.778 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:35.778 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 118719 2023-10-09T05:34:35.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.778 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.778 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.778 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.529134+0000", 2023-10-09T05:34:35.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.778 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.529134+0000", 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.529133+0000", 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2979999999999999e-06 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.529138+0000", 2023-10-09T05:34:35.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13545601600000001 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.664594+0000", 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0652999999999999e-05 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.664605+0000", 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5334999999999998e-05 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.664660+0000", 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012029200000000001 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.664780+0000", 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118245 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:34:35.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.664898+0000", 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015129935000000001 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.680028+0000", 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9750000000000007e-06 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.680036+0000", 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023904410000000001 2023-10-09T05:34:35.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.703941+0000", 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9045999999999997e-05 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.704000+0000", 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6647999999999998e-05 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.704046+0000", 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17491253800000001 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:118714 5.d e122/112 5:b36a7c3e:::benchmark_data_smithi008_238708_object578:head v 122'7470, mlcod=122'7468)", 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:23.529578+0000", 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 12.197883869, 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13892678999999999, 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.784 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.529578+0000", 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.529578+0000", 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.529577+0000", 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1109999999999996e-06 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.529583+0000", 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13500786400000001 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.664591+0000", 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2398000000000001e-05 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.664603+0000", 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4860000000000001e-05 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.664648+0000", 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3221e-05 2023-10-09T05:34:35.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.664671+0000", 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003799349 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.668471+0000", 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3874999999999997e-05 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:23.668505+0000", 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13892678999999999 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:119020 5.e e122/105 5:7d0c0384:::benchmark_data_smithi008_238708_object884:head v 122'7478, mlcod=122'7475)", 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.001139+0000", 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 11.726322447999999, 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.189668167, 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.789 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:34:35.790 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001139+0000", 2023-10-09T05:34:35.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001139+0000", 2023-10-09T05:34:35.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:35.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001138+0000", 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5030000000000003e-06 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001148+0000", 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7229999999999999e-05 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001165+0000", 2023-10-09T05:34:35.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3840000000000002e-06 2023-10-09T05:34:35.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001172+0000", 2023-10-09T05:34:35.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3034000000000002e-05 2023-10-09T05:34:35.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001215+0000", 2023-10-09T05:34:35.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3640000000000001e-05 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001238+0000", 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18956918 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.190807+0000", 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.189668167 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.793 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:119026 5.e e122/105 5:7513f17c:::benchmark_data_smithi008_238708_object890:head v 122'7479, mlcod=122'7475)", 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.001366+0000", 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 11.726096172, 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18949637499999999, 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001366+0000", 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001366+0000", 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001365+0000", 2023-10-09T05:34:35.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9510000000000003e-06 2023-10-09T05:34:35.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001371+0000", 2023-10-09T05:34:35.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1931e-05 2023-10-09T05:34:35.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001383+0000", 2023-10-09T05:34:35.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0930000000000002e-06 2023-10-09T05:34:35.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.001388+0000", 2023-10-09T05:34:35.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003476715 2023-10-09T05:34:35.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.004865+0000", 2023-10-09T05:34:35.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8771e-05 2023-10-09T05:34:35.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.004883+0000", 2023-10-09T05:34:35.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18597844299999999 2023-10-09T05:34:35.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.190862+0000", 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18949637499999999 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119031 5.f 5:f62bfa5d:::benchmark_data_smithi008_238708_object895:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.042073+0000", 2023-10-09T05:34:35.799 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 11.685388230999999, 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99245742100000001, 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119031 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042073+0000", 2023-10-09T05:34:35.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042073+0000", 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999962 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042070+0000", 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8694000000000001e-05 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042088+0000", 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3565e-05 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042132+0000", 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.365e-05 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042145+0000", 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14325328600000001 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185399+0000", 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8292e-05 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185417+0000", 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83638978799999997 2023-10-09T05:34:35.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.021807+0000", 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012924400000000001 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.021936+0000", 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014705600000000001 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.022083+0000", 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041951740000000003 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.026278+0000", 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0043576520000000001 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.030636+0000", 2023-10-09T05:34:35.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038215580000000001 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034457+0000", 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4993999999999997e-05 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034492+0000", 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8413e-05 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034531+0000", 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99245742100000001 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119032 5.1f 5:fba7d45d:::benchmark_data_smithi008_238708_object896:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 11.685297551, 2023-10-09T05:34:35.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0020802900000001, 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119032 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:34:35.808 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8399999999999997e-06 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042167+0000", 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7719999999999997e-06 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042174+0000", 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5770000000000004e-06 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042179+0000", 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14336328700000001 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185542+0000", 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4260000000000002e-06 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185547+0000", 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85482789199999998 2023-10-09T05:34:35.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.040375+0000", 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012875500000000001 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.040504+0000", 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017535599999999999 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.040679+0000", 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032237390000000002 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.043903+0000", 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021366899999999999 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044117+0000", 2023-10-09T05:34:35.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7777000000000001e-05 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044155+0000", 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9549999999999997e-05 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044214+0000", 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0266000000000001e-05 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044244+0000", 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0020802900000001 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119033 5.1f 5:fb715b35:::benchmark_data_smithi008_238708_object897:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.042184+0000", 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 11.685278061, 2023-10-09T05:34:35.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0031231839999999, 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119033 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042184+0000", 2023-10-09T05:34:35.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042184+0000", 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042183+0000", 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5550000000000001e-06 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042186+0000", 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4040000000000003e-06 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042191+0000", 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5389999999999999e-06 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042195+0000", 2023-10-09T05:34:35.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.143397471 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185592+0000", 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4479999999999996e-06 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185597+0000", 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85661535899999997 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.042212+0000", 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5107999999999998e-05 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.042307+0000", 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3749000000000001e-05 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.042391+0000", 2023-10-09T05:34:35.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001859688 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044251+0000", 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00037112400000000002 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044622+0000", 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060215500000000005 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.821 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.045224+0000", 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6007999999999997e-05 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.045270+0000", 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6947999999999999e-05 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.822 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.045307+0000", 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0031231839999999 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119034 5.f 5:f790e635:::benchmark_data_smithi008_238708_object898:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.066949+0000", 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 11.660512693999999, 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.967741929, 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.823 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119034 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066949+0000", 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066949+0000", 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999962 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066945+0000", 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3286e-05 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066959+0000", 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5304999999999999e-05 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066984+0000", 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2354999999999999e-05 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066996+0000", 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.118639008 2023-10-09T05:34:35.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185635+0000", 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0820000000000004e-06 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185641+0000", 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83823218700000002 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.023874+0000", 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011226 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.023986+0000", 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013421799999999999 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.024120+0000", 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003618137 2023-10-09T05:34:35.828 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.027738+0000", 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029752229999999999 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.030713+0000", 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039165859999999997 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.829 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:35.830 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034630+0000", 2023-10-09T05:34:35.830 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7750999999999999e-05 2023-10-09T05:34:35.830 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.830 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.830 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.830 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034658+0000", 2023-10-09T05:34:35.830 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3167000000000003e-05 2023-10-09T05:34:35.830 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.830 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.830 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.830 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034691+0000", 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.967741929 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:121192 5.14 e124/118 5:2ba45ed0:::benchmark_data_smithi008_238708_object3056:head v 124'6784, mlcod=124'6781)", 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:26.591107+0000", 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 9.1363548550000004, 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25660445700000001, 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.831 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591107+0000", 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591107+0000", 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591106+0000", 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1050000000000003e-06 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591111+0000", 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4117999999999998e-05 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591135+0000", 2023-10-09T05:34:35.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1389999999999997e-06 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591140+0000", 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2302000000000003e-05 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591182+0000", 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4080000000000001e-05 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591196+0000", 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25648668000000002 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847683+0000", 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8700999999999998e-05 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847711+0000", 2023-10-09T05:34:35.835 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25660445700000001 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:121196 5.b e124/123 5:d7e1199e:::benchmark_data_smithi008_238708_object3060:head v 124'7558, mlcod=124'7555)", 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:26.591149+0000", 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 9.1363129520000008, 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25505874699999997, 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.836 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591149+0000", 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591149+0000", 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967296 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.837 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591149+0000", 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.588e-06 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591151+0000", 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2380000000000002e-06 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591154+0000", 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7429999999999998e-06 2023-10-09T05:34:35.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591157+0000", 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0718e-05 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591188+0000", 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1955e-05 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.839 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591200+0000", 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25499218499999998 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.846192+0000", 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5536000000000002e-05 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.846207+0000", 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25505874699999997 2023-10-09T05:34:35.840 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:121199 5.12 e124/118 5:4c3a7433:::benchmark_data_smithi008_238708_object3063:head v 124'6890, mlcod=124'6888)", 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:26.591719+0000", 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 9.1357425320000001, 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25588323800000001, 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.841 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591719+0000", 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591719+0000", 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.842 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591718+0000", 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6850000000000001e-06 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591726+0000", 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.251386094 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843112+0000", 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1263000000000002e-05 2023-10-09T05:34:35.843 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843143+0000", 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0348e-05 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843194+0000", 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4278e-05 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.844 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843218+0000", 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0043434980000000003 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847561+0000", 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0933000000000003e-05 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847602+0000", 2023-10-09T05:34:35.845 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25588323800000001 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:121203 5.16 e124/118 5:6a4f9712:::benchmark_data_smithi008_238708_object3067:head v 124'6860, mlcod=124'6858)", 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:26.596474+0000", 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 9.1309878680000001, 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24983662000000001, 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.846 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.596474+0000", 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.596474+0000", 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.596473+0000", 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6680000000000004e-06 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.596480+0000", 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24663584399999999 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843116+0000", 2023-10-09T05:34:35.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1062999999999997e-05 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843147+0000", 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6440999999999995e-05 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843223+0000", 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0903999999999998e-05 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843244+0000", 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030374130000000001 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.846281+0000", 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9098000000000002e-05 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.846310+0000", 2023-10-09T05:34:35.850 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24983662000000001 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:121210 5.1e e124/123 5:78bc6dae:::benchmark_data_smithi008_238708_object3074:head v 124'7550, mlcod=124'7548)", 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:26.598123+0000", 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 9.1293384250000003, 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.247738244, 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:35.851 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.598123+0000", 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.598123+0000", 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:35.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.598123+0000", 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5310000000000004e-06 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.598130+0000", 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.244981332 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843111+0000", 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3433000000000003e-05 2023-10-09T05:34:35.853 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843145+0000", 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7419000000000001e-05 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843192+0000", 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5703000000000001e-05 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.843218+0000", 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026078210000000002 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.845826+0000", 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5722999999999997e-05 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.845862+0000", 2023-10-09T05:34:35.855 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.247738244 2023-10-09T05:34:35.856 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.856 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.856 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.856 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:35.856 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:35.856 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:34:35.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:35.857 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:35.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:35.958 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:35.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:35 smithi008 ceph-mon[184418]: 5.0 scrub ok 2023-10-09T05:34:35.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:35 smithi008 ceph-mon[184418]: pgmap v309: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 114 B/s rd, 69 MiB/s wr, 2.03k op/s; 34292/329175 objects degraded (10.418%) 2023-10-09T05:34:35.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:35 smithi008 ceph-mon[184418]: 5.22 scrub starts 2023-10-09T05:34:35.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:35 smithi008 ceph-mon[184418]: 5.a scrub starts 2023-10-09T05:34:35.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:35 smithi008 ceph-mon[191227]: 5.0 scrub ok 2023-10-09T05:34:35.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:35 smithi008 ceph-mon[191227]: pgmap v309: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 114 B/s rd, 69 MiB/s wr, 2.03k op/s; 34292/329175 objects degraded (10.418%) 2023-10-09T05:34:35.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:35 smithi008 ceph-mon[191227]: 5.22 scrub starts 2023-10-09T05:34:35.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:35 smithi008 ceph-mon[191227]: 5.a scrub starts 2023-10-09T05:34:36.005 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:34:36.018 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:36.070 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:36.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:36.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:36.173 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:36.259 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:36.472 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:36.477 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T05:34:36.477 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T05:34:36.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:36.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:36.579 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:36.620 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:36.633 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:36 smithi191 ceph-mon[161669]: 5.7 scrub ok 2023-10-09T05:34:36.633 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:36 smithi191 ceph-mon[161669]: 5.c scrub starts 2023-10-09T05:34:36.633 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:36 smithi191 ceph-mon[161669]: 5.a scrub ok 2023-10-09T05:34:36.633 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:36 smithi191 ceph-mon[161669]: 5.17 scrub starts 2023-10-09T05:34:36.635 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:36 smithi008 ceph-mon[184418]: 5.7 scrub ok 2023-10-09T05:34:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:36 smithi008 ceph-mon[184418]: 5.c scrub starts 2023-10-09T05:34:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:36 smithi008 ceph-mon[184418]: 5.a scrub ok 2023-10-09T05:34:36.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:36 smithi008 ceph-mon[184418]: 5.17 scrub starts 2023-10-09T05:34:36.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:36 smithi008 ceph-mon[191227]: 5.7 scrub ok 2023-10-09T05:34:36.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:36 smithi008 ceph-mon[191227]: 5.c scrub starts 2023-10-09T05:34:36.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:36 smithi008 ceph-mon[191227]: 5.a scrub ok 2023-10-09T05:34:36.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:36 smithi008 ceph-mon[191227]: 5.17 scrub starts 2023-10-09T05:34:37.293 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:37.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:37.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:37.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:34:37.399 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:37.462 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:37.556 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:37.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:37.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:37.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:37.695 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:37.744 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:37.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:37.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:37.839 INFO:teuthology.orchestra.run.smithi191.stderr:no valid command found; 10 closest matches: 2023-10-09T05:34:37.839 INFO:teuthology.orchestra.run.smithi191.stderr:0 2023-10-09T05:34:37.839 INFO:teuthology.orchestra.run.smithi191.stderr:1 2023-10-09T05:34:37.839 INFO:teuthology.orchestra.run.smithi191.stderr:2 2023-10-09T05:34:37.839 INFO:teuthology.orchestra.run.smithi191.stderr:abort 2023-10-09T05:34:37.839 INFO:teuthology.orchestra.run.smithi191.stderr:assert 2023-10-09T05:34:37.839 INFO:teuthology.orchestra.run.smithi191.stderr:bluefs debug_inject_read_zeros 2023-10-09T05:34:37.839 INFO:teuthology.orchestra.run.smithi191.stderr:bluefs files list 2023-10-09T05:34:37.840 INFO:teuthology.orchestra.run.smithi191.stderr:bluefs stats 2023-10-09T05:34:37.840 INFO:teuthology.orchestra.run.smithi191.stderr:bluestore allocator dump block 2023-10-09T05:34:37.840 INFO:teuthology.orchestra.run.smithi191.stderr:bluestore allocator fragmentation block 2023-10-09T05:34:37.840 INFO:teuthology.orchestra.run.smithi191.stderr:admin_socket: invalid command 2023-10-09T05:34:37.848 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:37.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:37 smithi191 ceph-mon[161669]: pgmap v310: 109 pgs: 1 active+clean+scrubbing, 22 active+undersized+degraded, 19 active+undersized, 67 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33 MiB/s wr, 1.69k op/s; 32272/313536 objects degraded (10.293%) 2023-10-09T05:34:37.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:37 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 32272/313536 objects degraded (10.293%), 22 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:37.929 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:37.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:37 smithi008 ceph-mon[184418]: pgmap v310: 109 pgs: 1 active+clean+scrubbing, 22 active+undersized+degraded, 19 active+undersized, 67 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33 MiB/s wr, 1.69k op/s; 32272/313536 objects degraded (10.293%) 2023-10-09T05:34:38.009 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:37 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 32272/313536 objects degraded (10.293%), 22 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:38.009 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:37 smithi008 ceph-mon[191227]: pgmap v310: 109 pgs: 1 active+clean+scrubbing, 22 active+undersized+degraded, 19 active+undersized, 67 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33 MiB/s wr, 1.69k op/s; 32272/313536 objects degraded (10.293%) 2023-10-09T05:34:38.009 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:37 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 32272/313536 objects degraded (10.293%), 22 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:38.080 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:38.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:38.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:38.187 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:38.236 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:38.326 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:38.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:38.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:38.433 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:38.478 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:34:38.494 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:38.606 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:38.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:38.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:38.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:38.760 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:38.780 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:38.806 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T05:34:38.806 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2023-10-09T05:34:38.866 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:38.869 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:38 smithi191 conmon[181254]: debug 2023-10-09T05:34:38.774+0000 7efc0b894200 -1 osd.7 111 log_to_monitors {default=true} 2023-10-09T05:34:38.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:38 smithi191 ceph-mon[161669]: 5.20 scrub starts 2023-10-09T05:34:38.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:38 smithi191 ceph-mon[161669]: 5.e scrub starts 2023-10-09T05:34:38.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:38 smithi191 ceph-mon[161669]: 5.17 scrub ok 2023-10-09T05:34:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:38.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:38 smithi008 ceph-mon[184418]: 5.20 scrub starts 2023-10-09T05:34:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:38 smithi008 ceph-mon[184418]: 5.e scrub starts 2023-10-09T05:34:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:38 smithi008 ceph-mon[184418]: 5.17 scrub ok 2023-10-09T05:34:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:38 smithi008 ceph-mon[191227]: 5.20 scrub starts 2023-10-09T05:34:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:38 smithi008 ceph-mon[191227]: 5.e scrub starts 2023-10-09T05:34:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:38 smithi008 ceph-mon[191227]: 5.17 scrub ok 2023-10-09T05:34:38.971 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:39.048 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:39.150 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:39.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:39.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:39.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:39.278 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:39.302 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:39.370 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:39.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:39.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:39.473 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:39.525 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:39.625 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:39.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:39.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:39.639 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:34:39.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:39.790 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:39.814 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[184418]: pgmap v311: 109 pgs: 1 active+clean+scrubbing, 22 active+undersized+degraded, 19 active+undersized, 67 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33 MiB/s wr, 1.72k op/s; 32272/311118 objects degraded (10.373%) 2023-10-09T05:34:39.814 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[184418]: 5.20 scrub starts 2023-10-09T05:34:39.815 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[184418]: 5.20 scrub ok 2023-10-09T05:34:39.815 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[184418]: from='osd.7 [v2:172.21.15.191:6824/859245059,v1:172.21.15.191:6825/859245059]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:34:39.815 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[184418]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:34:39.815 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1029486443' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:39.815 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:39.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[191227]: pgmap v311: 109 pgs: 1 active+clean+scrubbing, 22 active+undersized+degraded, 19 active+undersized, 67 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33 MiB/s wr, 1.72k op/s; 32272/311118 objects degraded (10.373%) 2023-10-09T05:34:39.816 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[191227]: 5.20 scrub starts 2023-10-09T05:34:39.816 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[191227]: 5.20 scrub ok 2023-10-09T05:34:39.816 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[191227]: from='osd.7 [v2:172.21.15.191:6824/859245059,v1:172.21.15.191:6825/859245059]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:34:39.816 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[191227]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:34:39.816 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1029486443' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:39.816 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:39 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:40.045 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:40.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:39 smithi191 ceph-mon[161669]: pgmap v311: 109 pgs: 1 active+clean+scrubbing, 22 active+undersized+degraded, 19 active+undersized, 67 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33 MiB/s wr, 1.72k op/s; 32272/311118 objects degraded (10.373%) 2023-10-09T05:34:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:39 smithi191 ceph-mon[161669]: 5.20 scrub starts 2023-10-09T05:34:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:39 smithi191 ceph-mon[161669]: 5.20 scrub ok 2023-10-09T05:34:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:39 smithi191 ceph-mon[161669]: from='osd.7 [v2:172.21.15.191:6824/859245059,v1:172.21.15.191:6825/859245059]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:34:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:39 smithi191 ceph-mon[161669]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:34:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1029486443' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:39 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:40.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 05:34:39 smithi191 conmon[181254]: debug 2023-10-09T05:34:39.644+0000 7efbfa589700 -1 osd.7 111 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:34:40.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:40.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:40.153 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:40.240 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:40.493 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:40.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:40.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:40.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:40.612 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:34:40.612 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:34:40.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.612 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138478 5.12 5:4e41c6da:::benchmark_data_smithi008_238708_object20342:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-10-09T05:34:40.612 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.607359+0000", 2023-10-09T05:34:40.612 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0047666059999999996, 2023-10-09T05:34:40.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047982449999999996, 2023-10-09T05:34:40.612 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:40.612 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:34:40.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138478 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.607359+0000", 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.607359+0000", 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.607357+0000", 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3149999999999999e-06 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.607362+0000", 2023-10-09T05:34:40.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0084e-05 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.607372+0000", 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.045e-06 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.607377+0000", 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8149999999999998e-05 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.607426+0000", 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015254799999999999 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.607578+0000", 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000151304 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:40.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.607729+0000", 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022255769999999999 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.609955+0000", 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00036078100000000001 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.610316+0000", 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029571509999999999 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:40.617 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:138482 5.23 e126/123 5:c61ba25a:::benchmark_data_smithi008_238708_object20346:head v 126'6980, mlcod=126'6978)", 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.610685+0000", 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0014399090000000001, 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001572731, 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.610685+0000", 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.610685+0000", 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.610684+0000", 2023-10-09T05:34:40.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2640000000000002e-06 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.610692+0000", 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.417e-05 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.610706+0000", 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8499999999999999e-06 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.610712+0000", 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6888000000000001e-05 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.610769+0000", 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8298999999999999e-05 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.610787+0000", 2023-10-09T05:34:40.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000101606 2023-10-09T05:34:40.622 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:40.622 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:40.622 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:40.622 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:40.622 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:34:40.622 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 2 2023-10-09T05:34:40.622 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:34:40.630 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: 5.1c scrub starts 2023-10-09T05:34:40.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: 5.1c scrub ok 2023-10-09T05:34:40.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:34:40.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:34:40.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:40.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: osdmap e126: 8 total, 7 up, 8 in 2023-10-09T05:34:40.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: from='osd.7 [v2:172.21.15.191:6824/859245059,v1:172.21.15.191:6825/859245059]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:34:40.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:34:40.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: 5.c scrub starts 2023-10-09T05:34:40.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: 5.22 scrub starts 2023-10-09T05:34:40.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.632 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.632 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.632 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.632 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: 5.f scrub starts 2023-10-09T05:34:40.632 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: pgmap v313: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31584/303645 objects degraded (10.402%) 2023-10-09T05:34:40.632 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:40 smithi191 ceph-mon[161669]: 5.c scrub ok 2023-10-09T05:34:40.663 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:40.825 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:40.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:40.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:40.928 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:40.953 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: 5.1c scrub starts 2023-10-09T05:34:40.953 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: 5.1c scrub ok 2023-10-09T05:34:40.953 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:34:40.953 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:34:40.953 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:40.953 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: osdmap e126: 8 total, 7 up, 8 in 2023-10-09T05:34:40.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: from='osd.7 [v2:172.21.15.191:6824/859245059,v1:172.21.15.191:6825/859245059]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:34:40.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:34:40.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: 5.c scrub starts 2023-10-09T05:34:40.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: 5.22 scrub starts 2023-10-09T05:34:40.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: 5.f scrub starts 2023-10-09T05:34:40.955 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: pgmap v313: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31584/303645 objects degraded (10.402%) 2023-10-09T05:34:40.955 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[184418]: 5.c scrub ok 2023-10-09T05:34:40.955 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:34:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:34:40] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.33.4" 2023-10-09T05:34:40.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: 5.1c scrub starts 2023-10-09T05:34:40.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: 5.1c scrub ok 2023-10-09T05:34:40.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:34:40.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:34:40.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:40.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: osdmap e126: 8 total, 7 up, 8 in 2023-10-09T05:34:40.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: from='osd.7 [v2:172.21.15.191:6824/859245059,v1:172.21.15.191:6825/859245059]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:34:40.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T05:34:40.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: 5.c scrub starts 2023-10-09T05:34:40.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: 5.22 scrub starts 2023-10-09T05:34:40.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.957 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.957 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: failed to encode map e126 with expected crc 2023-10-09T05:34:40.957 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: 5.f scrub starts 2023-10-09T05:34:40.957 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: pgmap v313: 109 pgs: 22 active+undersized+degraded, 19 active+undersized, 68 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31584/303645 objects degraded (10.402%) 2023-10-09T05:34:40.957 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:40 smithi008 ceph-mon[191227]: 5.c scrub ok 2023-10-09T05:34:40.957 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:40.999 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:41.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:41.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:41.103 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:41.130 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:41.172 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:41.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:41.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:41.276 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:41.304 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:41.348 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:41.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:41.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:41.451 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:41.480 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:41.528 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:41.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:41.571 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:34:41.631 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:41.720 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:41.770 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:41.776 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: failed to encode map e126 with expected crc 2023-10-09T05:34:41.776 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: osd.7 [v2:172.21.15.191:6824/859245059,v1:172.21.15.191:6825/859245059] boot 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: 5.22 scrub starts 2023-10-09T05:34:41.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:41 smithi191 ceph-mon[161669]: 5.22 scrub ok 2023-10-09T05:34:41.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:41.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:41.879 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:41.904 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: failed to encode map e126 with expected crc 2023-10-09T05:34:41.904 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:34:41.904 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: osd.7 [v2:172.21.15.191:6824/859245059,v1:172.21.15.191:6825/859245059] boot 2023-10-09T05:34:41.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T05:34:41.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:34:41.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: 5.22 scrub starts 2023-10-09T05:34:41.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[184418]: 5.22 scrub ok 2023-10-09T05:34:41.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: failed to encode map e126 with expected crc 2023-10-09T05:34:41.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:34:41.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: osd.7 [v2:172.21.15.191:6824/859245059,v1:172.21.15.191:6825/859245059] boot 2023-10-09T05:34:41.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T05:34:41.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:34:41.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: failed to encode map e127 with expected crc 2023-10-09T05:34:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: 5.22 scrub starts 2023-10-09T05:34:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:41 smithi008 ceph-mon[191227]: 5.22 scrub ok 2023-10-09T05:34:41.907 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:42.017 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:42.055 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:42.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:42.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:42.159 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:42.188 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:42.335 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:42.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:42.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:42.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:42.466 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:42.509 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:42.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:42.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:42.512 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:34:42.531 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:34:42.549 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:34:42.569 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:34:42.587 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:34:42.606 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:34:42.612 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:42.629 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:34:42.638 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:42.647 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:34:42.649 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:34:42.649 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:34:42.649 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:34:42.649 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:34:42.649 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:34:42.649 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:34:42.650 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:34:42.650 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:34:42.693 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:42.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:42.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:42.797 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:42.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e112 with expected crc 2023-10-09T05:34:42.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e127 with expected crc 2023-10-09T05:34:42.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e127 with expected crc 2023-10-09T05:34:42.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: 5.e scrub starts 2023-10-09T05:34:42.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T05:34:42.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: 5.b scrub starts 2023-10-09T05:34:42.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: pgmap v316: 109 pgs: 10 peering, 17 active+undersized+degraded, 14 active+undersized, 68 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18631/297636 objects degraded (6.260%) 2023-10-09T05:34:42.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3329301497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:42.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:42 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 31584/303645 objects degraded (10.402%), 22 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:42.894 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e112 with expected crc 2023-10-09T05:34:42.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e127 with expected crc 2023-10-09T05:34:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e127 with expected crc 2023-10-09T05:34:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: 5.e scrub starts 2023-10-09T05:34:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T05:34:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: 5.b scrub starts 2023-10-09T05:34:42.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: pgmap v316: 109 pgs: 10 peering, 17 active+undersized+degraded, 14 active+undersized, 68 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18631/297636 objects degraded (6.260%) 2023-10-09T05:34:42.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3329301497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:42.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 31584/303645 objects degraded (10.402%), 22 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:42.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e112 with expected crc 2023-10-09T05:34:42.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e127 with expected crc 2023-10-09T05:34:42.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e127 with expected crc 2023-10-09T05:34:42.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: 5.e scrub starts 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: failed to encode map e128 with expected crc 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: 5.b scrub starts 2023-10-09T05:34:42.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: pgmap v316: 109 pgs: 10 peering, 17 active+undersized+degraded, 14 active+undersized, 68 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18631/297636 objects degraded (6.260%) 2023-10-09T05:34:42.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3329301497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:42.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:42 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 31584/303645 objects degraded (10.402%), 22 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-10-09T05:34:43.191 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:43.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:43.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:43.297 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:43.365 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:43.499 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:43.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:43.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:43.608 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:43.655 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[191227]: 5.e scrub ok 2023-10-09T05:34:43.655 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[191227]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.655 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[191227]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.655 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[191227]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.655 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[191227]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T05:34:43.656 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[191227]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.656 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[191227]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.718 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:43.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:43 smithi191 ceph-mon[161669]: 5.e scrub ok 2023-10-09T05:34:43.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:43 smithi191 ceph-mon[161669]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:43 smithi191 ceph-mon[161669]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:43 smithi191 ceph-mon[161669]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:43 smithi191 ceph-mon[161669]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T05:34:43.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:43 smithi191 ceph-mon[161669]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:43 smithi191 ceph-mon[161669]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.807 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:34:43.846 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:43.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:43.858 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:34:43.858 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:34:43.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:43.959 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[184418]: 5.e scrub ok 2023-10-09T05:34:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[184418]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[184418]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[184418]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[184418]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T05:34:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[184418]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:43 smithi008 ceph-mon[184418]: failed to encode map e129 with expected crc 2023-10-09T05:34:43.967 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:34:44.045 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:44.164 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:44.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:44.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:44.268 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:44.326 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:44.538 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:45.152 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:45.163 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:45.163 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:45.163 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:44 smithi191 ceph-mon[161669]: pgmap v318: 109 pgs: 10 peering, 16 active+undersized+degraded, 9 active+undersized, 74 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18630/295779 objects degraded (6.299%) 2023-10-09T05:34:45.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:45.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:44 smithi008 ceph-mon[184418]: pgmap v318: 109 pgs: 10 peering, 16 active+undersized+degraded, 9 active+undersized, 74 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18630/295779 objects degraded (6.299%) 2023-10-09T05:34:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:44 smithi008 ceph-mon[191227]: pgmap v318: 109 pgs: 10 peering, 16 active+undersized+degraded, 9 active+undersized, 74 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18630/295779 objects degraded (6.299%) 2023-10-09T05:34:45.227 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:34:45.265 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:45.319 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:45.421 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:45.422 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:45.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:45.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:45.514 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:34:45.526 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:45.617 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:45.712 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:45.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:45.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:45.820 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:45.878 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:46.284 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:46.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:46.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:46.388 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:46.435 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:46.545 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:46.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:46.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:46.581 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:46.659 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:46.688 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:46.765 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:46.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:46.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:46.874 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:46.923 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:47.056 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:47.103 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:46 smithi008 ceph-mon[184418]: pgmap v319: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 25755/287709 objects degraded (8.952%); 1.2 MiB/s, 20 objects/s recovering 2023-10-09T05:34:47.103 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:46 smithi008 ceph-mon[191227]: pgmap v319: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 25755/287709 objects degraded (8.952%); 1.2 MiB/s, 20 objects/s recovering 2023-10-09T05:34:47.129 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:47.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:47.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:47.241 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:46 smithi191 ceph-mon[161669]: pgmap v319: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 25755/287709 objects degraded (8.952%); 1.2 MiB/s, 20 objects/s recovering 2023-10-09T05:34:47.304 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:47.503 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:47.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:47.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:47.611 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:47.651 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:47.705 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:47.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:47.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:47.710 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:34:47.710 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:34:47.809 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:47.901 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:48.004 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:34:48.009 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:48.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:48.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:48.027 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1612026183' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:48.027 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:47 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:48.027 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:47 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 25755/287709 objects degraded (8.952%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:48.028 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1612026183' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:48.028 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:47 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:48.028 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:47 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 25755/287709 objects degraded (8.952%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:48.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:48.158 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1612026183' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:48.158 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:47 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:34:48.158 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:47 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 25755/287709 objects degraded (8.952%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:48.159 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:49.246 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[184418]: pgmap v320: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25755/285264 objects degraded (9.028%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T05:34:49.246 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:49.246 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[184418]: failed to encode map e130 with expected crc 2023-10-09T05:34:49.246 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[184418]: failed to encode map e130 with expected crc 2023-10-09T05:34:49.247 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:34:49.247 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[184418]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T05:34:49.247 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[191227]: pgmap v320: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25755/285264 objects degraded (9.028%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T05:34:49.247 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:49.247 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[191227]: failed to encode map e130 with expected crc 2023-10-09T05:34:49.247 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[191227]: failed to encode map e130 with expected crc 2023-10-09T05:34:49.247 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:34:49.247 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:48 smithi008 ceph-mon[191227]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T05:34:49.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:48 smithi191 ceph-mon[161669]: pgmap v320: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25755/285264 objects degraded (9.028%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T05:34:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:48 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:48 smithi191 ceph-mon[161669]: failed to encode map e130 with expected crc 2023-10-09T05:34:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:48 smithi191 ceph-mon[161669]: failed to encode map e130 with expected crc 2023-10-09T05:34:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:48 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:34:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:48 smithi191 ceph-mon[161669]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T05:34:49.332 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:49.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:49.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:49.437 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:49.509 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:49.577 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:34:49.609 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:49.612 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:34:49.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:49.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:49.644 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:34:49.678 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:34:49.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:34:49.734 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:34:49.773 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:34:49.801 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:49.811 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:34:49.847 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:34:49.849 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:34:49.849 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:34:49.849 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:34:49.849 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:34:49.849 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:34:49.849 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:34:49.849 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:34:49.849 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:34:50.026 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:34:50.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:50.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:34:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3190886741' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:50.133 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:50.148 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:34:50.149 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:34:50.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:34:50.149 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:34:50.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.149 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54045 5.9 5:9b47d3b1:::benchmark_data_smithi008_238708_object54044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e112)", 2023-10-09T05:34:50.149 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.539603+0000", 2023-10-09T05:34:50.149 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 89.606224871999999, 2023-10-09T05:34:50.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0152003759999999, 2023-10-09T05:34:50.149 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.149 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.150 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.150 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.150 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.150 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54045 2023-10-09T05:34:50.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.150 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539603+0000", 2023-10-09T05:34:50.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539603+0000", 2023-10-09T05:34:50.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:50.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539601+0000", 2023-10-09T05:34:50.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3383999999999997e-05 2023-10-09T05:34:50.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539645+0000", 2023-10-09T05:34:50.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0492999999999999e-05 2023-10-09T05:34:50.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539655+0000", 2023-10-09T05:34:50.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3479999999999997e-06 2023-10-09T05:34:50.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539660+0000", 2023-10-09T05:34:50.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4764000000000001e-05 2023-10-09T05:34:50.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539694+0000", 2023-10-09T05:34:50.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9150000000000001e-06 2023-10-09T05:34:50.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:50.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539700+0000", 2023-10-09T05:34:50.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.002179355 2023-10-09T05:34:50.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.541880+0000", 2023-10-09T05:34:50.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010278150000000001 2023-10-09T05:34:50.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.542907+0000", 2023-10-09T05:34:50.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000208908 2023-10-09T05:34:50.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4", 2023-10-09T05:34:50.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.543116+0000", 2023-10-09T05:34:50.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052105160000000001 2023-10-09T05:34:50.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.548327+0000", 2023-10-09T05:34:50.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063079850000000003 2023-10-09T05:34:50.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554635+0000", 2023-10-09T05:34:50.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0263000000000002e-05 2023-10-09T05:34:50.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554695+0000", 2023-10-09T05:34:50.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010829500000000001 2023-10-09T05:34:50.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554803+0000", 2023-10-09T05:34:50.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0152003759999999 2023-10-09T05:34:50.157 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.157 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.158 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.158 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54567 5.2 5:46c255c2:::benchmark_data_smithi008_238708_object54566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e113)", 2023-10-09T05:34:50.158 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:21.883782+0000", 2023-10-09T05:34:50.158 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 88.262046170000005, 2023-10-09T05:34:50.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13964462599999999, 2023-10-09T05:34:50.158 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.158 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.158 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.159 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.159 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.159 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54567 2023-10-09T05:34:50.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.159 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.883782+0000", 2023-10-09T05:34:50.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.883782+0000", 2023-10-09T05:34:50.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:50.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.883780+0000", 2023-10-09T05:34:50.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4134999999999997e-05 2023-10-09T05:34:50.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.883824+0000", 2023-10-09T05:34:50.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7263000000000001e-05 2023-10-09T05:34:50.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.883842+0000", 2023-10-09T05:34:50.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4360000000000001e-06 2023-10-09T05:34:50.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.883847+0000", 2023-10-09T05:34:50.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3699999999999998e-05 2023-10-09T05:34:50.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.883891+0000", 2023-10-09T05:34:50.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011281399999999999 2023-10-09T05:34:50.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.884004+0000", 2023-10-09T05:34:50.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117872 2023-10-09T05:34:50.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.884122+0000", 2023-10-09T05:34:50.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035266009999999999 2023-10-09T05:34:50.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.887648+0000", 2023-10-09T05:34:50.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5465e-05 2023-10-09T05:34:50.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.887684+0000", 2023-10-09T05:34:50.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.135646189 2023-10-09T05:34:50.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:22.023330+0000", 2023-10-09T05:34:50.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9276000000000001e-05 2023-10-09T05:34:50.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:22.023379+0000", 2023-10-09T05:34:50.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7299000000000002e-05 2023-10-09T05:34:50.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:22.023426+0000", 2023-10-09T05:34:50.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13964462599999999 2023-10-09T05:34:50.167 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.167 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.167 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.167 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54572 5.2 5:44fbff62:::benchmark_data_smithi008_238708_object54571:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e113)", 2023-10-09T05:34:50.167 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:21.885941+0000", 2023-10-09T05:34:50.168 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 88.259886968000004, 2023-10-09T05:34:50.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.142685016, 2023-10-09T05:34:50.168 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.168 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.168 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.168 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.168 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.169 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54572 2023-10-09T05:34:50.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.169 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.885941+0000", 2023-10-09T05:34:50.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.885941+0000", 2023-10-09T05:34:50.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:50.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.885940+0000", 2023-10-09T05:34:50.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5495999999999998e-05 2023-10-09T05:34:50.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.886005+0000", 2023-10-09T05:34:50.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6364000000000001e-05 2023-10-09T05:34:50.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.886021+0000", 2023-10-09T05:34:50.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6990000000000003e-06 2023-10-09T05:34:50.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.886029+0000", 2023-10-09T05:34:50.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5377999999999997e-05 2023-10-09T05:34:50.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.886085+0000", 2023-10-09T05:34:50.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011106 2023-10-09T05:34:50.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.886196+0000", 2023-10-09T05:34:50.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117589 2023-10-09T05:34:50.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.886313+0000", 2023-10-09T05:34:50.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014921209999999999 2023-10-09T05:34:50.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.887805+0000", 2023-10-09T05:34:50.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019775000000000001 2023-10-09T05:34:50.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.888003+0000", 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.140514677 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:22.028518+0000", 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4930000000000003e-05 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:22.028583+0000", 2023-10-09T05:34:50.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3300000000000002e-05 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:22.028626+0000", 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.142685016 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:98121 5.9 5:930a82bc:::benchmark_data_smithi008_238708_object98120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:00.583011+0000", 2023-10-09T05:34:50.180 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 49.562816924000003, 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20323780999999999, 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 98121 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583011+0000", 2023-10-09T05:34:50.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583011+0000", 2023-10-09T05:34:50.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:34:50.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583009+0000", 2023-10-09T05:34:50.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4110999999999997e-05 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583073+0000", 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6398000000000001e-05 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583090+0000", 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3339999999999996e-06 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583097+0000", 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8928999999999998e-05 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583156+0000", 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000166971 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583323+0000", 2023-10-09T05:34:50.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017988100000000001 2023-10-09T05:34:50.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4", 2023-10-09T05:34:50.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583503+0000", 2023-10-09T05:34:50.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013026197 2023-10-09T05:34:50.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.596529+0000", 2023-10-09T05:34:50.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18970899299999999 2023-10-09T05:34:50.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.786238+0000", 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.099e-05 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.786249+0000", 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20323780999999999 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.186 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:98123 5.9 5:980279ad:::benchmark_data_smithi008_238708_object98122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:00.586202+0000", 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 49.559625756000003, 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20225238100000001, 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.187 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 98123 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586202+0000", 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586202+0000", 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:34:50.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586200+0000", 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7370000000000003e-05 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586237+0000", 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5962999999999999e-05 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586253+0000", 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4239999999999993e-06 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586261+0000", 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8481e-05 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586300+0000", 2023-10-09T05:34:50.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000164969 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586465+0000", 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000231108 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4", 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586696+0000", 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01166125 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.598357+0000", 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19008492199999999 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.788442+0000", 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2417e-05 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.788455+0000", 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20225238100000001 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:98121 5.9 5:930a82bc:::benchmark_data_smithi008_238708_object98120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:00.694607+0000", 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 49.451220599000003, 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92918373799999998, 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.193 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 98121 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694607+0000", 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694607+0000", 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999962 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694603+0000", 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6734999999999999e-05 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694650+0000", 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9122999999999999e-05 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694679+0000", 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0550999999999999e-05 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694690+0000", 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5582999999999996e-05 2023-10-09T05:34:50.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694785+0000", 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6899999999999992e-06 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694793+0000", 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.093691338999999998 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.788484+0000", 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2150000000000004e-06 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.788492+0000", 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.82731977400000001 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.615811+0000", 2023-10-09T05:34:50.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00076560599999999995 2023-10-09T05:34:50.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:34:50.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.616577+0000", 2023-10-09T05:34:50.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071166750000000003 2023-10-09T05:34:50.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.623694+0000", 2023-10-09T05:34:50.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7318999999999994e-05 2023-10-09T05:34:50.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.623791+0000", 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92918373799999998 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:98123 5.19 5:980279ad:::benchmark_data_smithi008_238708_object98122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:00.694705+0000", 2023-10-09T05:34:50.200 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 49.451122990999998, 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.938116383, 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 98123 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694705+0000", 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694705+0000", 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694704+0000", 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6398e-05 2023-10-09T05:34:50.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694731+0000", 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2520000000000005e-06 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694740+0000", 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3919999999999996e-06 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694744+0000", 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.091777336000000001 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.786522+0000", 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2589999999999997e-06 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.786528+0000", 2023-10-09T05:34:50.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83990997700000003 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.626438+0000", 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001340026 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.627778+0000", 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0049558120000000004 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.632734+0000", 2023-10-09T05:34:50.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7482e-05 2023-10-09T05:34:50.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.632821+0000", 2023-10-09T05:34:50.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.938116383 2023-10-09T05:34:50.206 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.206 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.206 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:100571 5.12 5:4b13c0f1:::benchmark_data_smithi008_238708_object100570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:03.570947+0000", 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 46.574881363000003, 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.133842867, 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.207 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 100571 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.570947+0000", 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.570947+0000", 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.570945+0000", 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5694e-05 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.570991+0000", 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.245e-05 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.571013+0000", 2023-10-09T05:34:50.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7360000000000001e-06 2023-10-09T05:34:50.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.571021+0000", 2023-10-09T05:34:50.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1728999999999999e-05 2023-10-09T05:34:50.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.571073+0000", 2023-10-09T05:34:50.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017829900000000001 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.571251+0000", 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000116617 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.571367+0000", 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001113136 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.572481+0000", 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001240959 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.573722+0000", 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13100657299999999 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.704728+0000", 2023-10-09T05:34:50.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6529000000000002e-05 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.704765+0000", 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4726999999999999e-05 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:03.704789+0000", 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.133842867 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.213 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.214 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.214 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119037 5.2 5:408f5192:::benchmark_data_smithi008_238708_object901:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:34:50.214 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.151111+0000", 2023-10-09T05:34:50.214 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 25.994717102999999, 2023-10-09T05:34:50.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88165001499999995, 2023-10-09T05:34:50.214 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.214 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.214 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.214 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119037 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151111+0000", 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151111+0000", 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151109+0000", 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1720000000000007e-06 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151117+0000", 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2285e-05 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151130+0000", 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6660000000000003e-06 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151139+0000", 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016795371999999999 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.167935+0000", 2023-10-09T05:34:50.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4433999999999999e-05 2023-10-09T05:34:50.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:34:50.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.167949+0000", 2023-10-09T05:34:50.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85092211299999998 2023-10-09T05:34:50.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.018871+0000", 2023-10-09T05:34:50.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3704999999999998e-05 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.018965+0000", 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5246000000000003e-05 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.019050+0000", 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052231589999999998 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.024273+0000", 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048098300000000002 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.029083+0000", 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035822990000000002 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.032666+0000", 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5321000000000001e-05 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.032721+0000", 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9962000000000001e-05 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.032761+0000", 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88165001499999995 2023-10-09T05:34:50.221 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:120525 5.2 5:42096c5b:::benchmark_data_smithi008_238708_object2389:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:25.927216+0000", 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 24.218611986999999, 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.141903696, 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.222 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 120525 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.927216+0000", 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.927216+0000", 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.927215+0000", 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0430000000000004e-06 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.927221+0000", 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1539000000000001e-05 2023-10-09T05:34:50.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.927232+0000", 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4028e-05 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.927246+0000", 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00037096600000000003 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.927617+0000", 2023-10-09T05:34:50.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010707799999999999 2023-10-09T05:34:50.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.927724+0000", 2023-10-09T05:34:50.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117667 2023-10-09T05:34:50.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.927842+0000", 2023-10-09T05:34:50.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026451119999999998 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.930487+0000", 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001809988 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.932297+0000", 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13672447500000001 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.069022+0000", 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3248000000000002e-05 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.069075+0000", 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4811000000000002e-05 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.069120+0000", 2023-10-09T05:34:50.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.141903696 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:120538 5.12 5:4fd03841:::benchmark_data_smithi008_238708_object2402:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:25.936593+0000", 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 24.209234537, 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13425999799999999, 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.229 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 120538 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936593+0000", 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936593+0000", 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936592+0000", 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0280000000000003e-06 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936599+0000", 2023-10-09T05:34:50.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5710999999999999e-05 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936615+0000", 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7469999999999999e-06 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936622+0000", 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0900000000000003e-05 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936682+0000", 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000154621 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936837+0000", 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013516900000000001 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936972+0000", 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021815820000000001 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.939154+0000", 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020646380000000002 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.941218+0000", 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12955705200000001 2023-10-09T05:34:50.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.070775+0000", 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0941e-05 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.070816+0000", 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6949999999999997e-05 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.070853+0000", 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13425999799999999 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:120539 5.2 5:431f36ca:::benchmark_data_smithi008_238708_object2403:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:25.936635+0000", 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 24.209193041999999, 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13422905199999999, 2023-10-09T05:34:50.236 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 120539 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936635+0000", 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936635+0000", 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936634+0000", 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0819999999999999e-06 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936637+0000", 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5522000000000001e-05 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936653+0000", 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6999999999999999e-06 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936658+0000", 2023-10-09T05:34:50.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8795999999999998e-05 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936706+0000", 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012648 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936833+0000", 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012232599999999999 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.936955+0000", 2023-10-09T05:34:50.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011248619999999999 2023-10-09T05:34:50.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.938080+0000", 2023-10-09T05:34:50.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029127150000000002 2023-10-09T05:34:50.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.940993+0000", 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12977159999999999 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.070764+0000", 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9138000000000003e-05 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.070814+0000", 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0340999999999998e-05 2023-10-09T05:34:50.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.070864+0000", 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13422905199999999 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:121199 5.12 5:4c3a7433:::benchmark_data_smithi008_238708_object3063:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:26.591400+0000", 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.554427715999999, 2023-10-09T05:34:50.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25645415500000002, 2023-10-09T05:34:50.244 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.244 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.244 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.244 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.244 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.244 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 121199 2023-10-09T05:34:50.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.244 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591400+0000", 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591400+0000", 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591399+0000", 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7750000000000004e-06 2023-10-09T05:34:50.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591406+0000", 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0115000000000001e-05 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591416+0000", 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9239999999999998e-06 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591421+0000", 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9996999999999997e-05 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591461+0000", 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7571999999999994e-05 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591548+0000", 2023-10-09T05:34:50.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6479999999999999e-05 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591635+0000", 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012101460000000001 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.592845+0000", 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031766419999999999 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.596022+0000", 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25174085699999998 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847763+0000", 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1802999999999999e-05 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847814+0000", 2023-10-09T05:34:50.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0043999999999998e-05 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847854+0000", 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25645415500000002 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:133402 5.22 5:45f05d84:::benchmark_data_smithi008_238708_object15266:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:34:50.250 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:36.035918+0000", 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 14.109909984, 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13519424699999999, 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 133402 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.035918+0000", 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.035918+0000", 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.035917+0000", 2023-10-09T05:34:50.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.938e-06 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.035922+0000", 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1144999999999999e-05 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.035933+0000", 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.195e-06 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.035939+0000", 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0148000000000002e-05 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.035989+0000", 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119213 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.036109+0000", 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013060900000000001 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.036239+0000", 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019439469999999999 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.038183+0000", 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00022302300000000001 2023-10-09T05:34:50.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.038406+0000", 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13261152600000001 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.171018+0000", 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3801999999999998e-05 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.171072+0000", 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0618000000000003e-05 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.171112+0000", 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13519424699999999 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.257 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:133404 5.2 5:43b477a7:::benchmark_data_smithi008_238708_object15268:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:36.036739+0000", 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 14.109089150999999, 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13438868300000001, 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 133404 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.036739+0000", 2023-10-09T05:34:50.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.036739+0000", 2023-10-09T05:34:50.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:50.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.036738+0000", 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9949999999999999e-06 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.036742+0000", 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9670000000000004e-06 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.036749+0000", 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.98e-06 2023-10-09T05:34:50.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.036753+0000", 2023-10-09T05:34:50.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00036999599999999999 2023-10-09T05:34:50.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.037123+0000", 2023-10-09T05:34:50.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2976000000000001e-05 2023-10-09T05:34:50.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.037196+0000", 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7078999999999998e-05 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.037283+0000", 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001129192 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.038412+0000", 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00031111399999999998 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.038723+0000", 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13233450999999999 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.171058+0000", 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7184000000000002e-05 2023-10-09T05:34:50.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.171095+0000", 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2435999999999999e-05 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:36.171127+0000", 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13438868300000001 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.264 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.265 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:135149 5.12 5:4a140978:::benchmark_data_smithi008_238708_object17013:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:34:50.265 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:37.646056+0000", 2023-10-09T05:34:50.265 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 12.499771442, 2023-10-09T05:34:50.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.171445025, 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135149 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646056+0000", 2023-10-09T05:34:50.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646056+0000", 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646056+0000", 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4149999999999999e-06 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646059+0000", 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3579999999999999e-06 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646066+0000", 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.369e-06 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646070+0000", 2023-10-09T05:34:50.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012852930000000001 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.647355+0000", 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118285 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.647474+0000", 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115395 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.647589+0000", 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033829889999999999 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.650972+0000", 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021853419999999998 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.653157+0000", 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16425049 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.817408+0000", 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6112000000000002e-05 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.817454+0000", 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7508e-05 2023-10-09T05:34:50.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.817501+0000", 2023-10-09T05:34:50.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.171445025 2023-10-09T05:34:50.272 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.272 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.272 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.272 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:135151 5.22 5:47224588:::benchmark_data_smithi008_238708_object17015:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:34:50.272 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:37.651091+0000", 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 12.494736571000001, 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.1747978, 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135151 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651091+0000", 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651091+0000", 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651090+0000", 2023-10-09T05:34:50.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1389999999999995e-06 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651098+0000", 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5180999999999999e-05 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651113+0000", 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7910000000000001e-06 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651119+0000", 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020545799999999999 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.653173+0000", 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6720999999999998e-05 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.653270+0000", 2023-10-09T05:34:50.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108272 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.653378+0000", 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027134020000000002 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.656092+0000", 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7579999999999998e-06 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.656098+0000", 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.169701516 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825799+0000", 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1576999999999998e-05 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825851+0000", 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8532000000000003e-05 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825889+0000", 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.1747978 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.279 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:135156 5.12 5:4dc0c42a:::benchmark_data_smithi008_238708_object17020:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:37.652947+0000", 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 12.492881226, 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17295703300000001, 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135156 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652947+0000", 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652947+0000", 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652945+0000", 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8889999999999996e-06 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652952+0000", 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.013e-05 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652962+0000", 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4469999999999997e-06 2023-10-09T05:34:50.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652967+0000", 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013374330000000001 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.654305+0000", 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2422000000000004e-05 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.654397+0000", 2023-10-09T05:34:50.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000101003 2023-10-09T05:34:50.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.654498+0000", 2023-10-09T05:34:50.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0043436280000000004 2023-10-09T05:34:50.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.658842+0000", 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000146315 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.658988+0000", 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16684702400000001 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825835+0000", 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6446000000000003e-05 2023-10-09T05:34:50.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825872+0000", 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2089999999999999e-05 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825904+0000", 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17295703300000001 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.286 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:137285 5.22 5:4438e8ac:::benchmark_data_smithi008_238708_object19149:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:39.617497+0000", 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 10.528331343, 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15020351300000001, 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.287 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 137285 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.617497+0000", 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.617497+0000", 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:34:50.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.617496+0000", 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0529999999999997e-06 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.617501+0000", 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9659999999999996e-06 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.617509+0000", 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8774e-05 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.617527+0000", 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6423000000000001e-05 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.617554+0000", 2023-10-09T05:34:50.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8734999999999997e-05 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.617633+0000", 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7879999999999994e-05 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.617700+0000", 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010817629999999999 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.618782+0000", 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00025691599999999999 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.619039+0000", 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.148558726 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.767598+0000", 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5619999999999999e-05 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.767653+0000", 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6635999999999999e-05 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.767700+0000", 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15020351300000001 2023-10-09T05:34:50.293 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:137292 5.12 5:4eff7b7b:::benchmark_data_smithi008_238708_object19156:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:39.625637+0000", 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 10.520191323000001, 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14360340299999999, 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:34:50.294 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 137292 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.625637+0000", 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:34:50.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.625637+0000", 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.625634+0000", 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3482999999999999e-05 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.625648+0000", 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2036000000000003e-05 2023-10-09T05:34:50.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.625680+0000", 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8620000000000001e-05 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.625698+0000", 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7972000000000003e-05 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:34:50.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.625796+0000", 2023-10-09T05:34:50.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106821 2023-10-09T05:34:50.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:34:50.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.625903+0000", 2023-10-09T05:34:50.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13591176699999999 2023-10-09T05:34:50.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:34:50.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.761815+0000", 2023-10-09T05:34:50.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026392989999999999 2023-10-09T05:34:50.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:34:50.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.764454+0000", 2023-10-09T05:34:50.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032844900000000002 2023-10-09T05:34:50.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.767739+0000", 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001415536 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.769154+0000", 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7747000000000002e-05 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.769202+0000", 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8012000000000001e-05 2023-10-09T05:34:50.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:34:50.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:34:50.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:34:50.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:39.769240+0000", 2023-10-09T05:34:50.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14360340299999999 2023-10-09T05:34:50.301 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.301 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.301 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.301 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:34:50.301 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:34:50.301 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:34:50.302 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3190886741' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:50.302 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3190886741' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:50.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:50.303 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:50.303 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:50.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:50.404 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:50.431 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:50.479 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:50.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:50.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:50.582 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:50.639 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:50.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:34:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:34:50] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.33.4" 2023-10-09T05:34:50.931 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:50.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:50.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:51.038 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:51.116 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:51.159 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:34:51.200 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:51 smithi008 ceph-mon[184418]: pgmap v322: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25296/278514 objects degraded (9.082%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:34:51.201 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:51 smithi008 ceph-mon[191227]: pgmap v322: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25296/278514 objects degraded (9.082%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:34:51.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:34:51.277 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:34:51.296 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:51 smithi191 ceph-mon[161669]: pgmap v322: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25296/278514 objects degraded (9.082%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:34:51.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:51.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:51.403 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:51.453 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:51.541 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:51.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:51.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:51.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:51.718 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:51.755 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:51.812 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:51.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:51.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:51.932 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:51.968 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:51.971 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:52.060 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:52.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:52.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:52.167 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:52.227 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:52.396 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:52.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:52.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:52.503 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:52.553 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:52.624 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:52.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:52.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:52.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:52.759 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:52.806 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:52.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:52.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:52.909 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:52.931 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:34:52.974 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:53.075 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:34:53.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:53.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:53.182 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:53.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:53 smithi008 ceph-mon[184418]: pgmap v323: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 25296/271842 objects degraded (9.305%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:34:53.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 25296/271842 objects degraded (9.305%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:53.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:53 smithi008 ceph-mon[191227]: pgmap v323: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 25296/271842 objects degraded (9.305%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:34:53.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 25296/271842 objects degraded (9.305%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:53.219 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:53.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:53 smithi191 ceph-mon[161669]: pgmap v323: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 25296/271842 objects degraded (9.305%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:34:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 25296/271842 objects degraded (9.305%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:53.327 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:53.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:53.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:53.434 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:53.494 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:53.513 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:53.795 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:53.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:53.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:53.914 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:53.976 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:54.316 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:54.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:54.417 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:34:54.417 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:34:54.417 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T05:34:54.417 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:34:54.420 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:54.465 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:54.523 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:34:54.524 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:34:54.524 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T05:34:54.524 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:34:54.524 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:34:54.525 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T05:34:54.541 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:54.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:54.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:54.646 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:54.696 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:54.797 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:54.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:54.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:54.903 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:34:54.947 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:55.091 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:34:55.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:55 smithi008 ceph-mon[184418]: pgmap v324: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 25296/268887 objects degraded (9.408%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T05:34:55.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:55 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2388498721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:55 smithi008 ceph-mon[191227]: pgmap v324: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 25296/268887 objects degraded (9.408%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T05:34:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:55 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2388498721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:55.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:55 smithi191 ceph-mon[161669]: pgmap v324: 109 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 25296/268887 objects degraded (9.408%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T05:34:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:55 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2388498721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:34:55.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:34:55.416 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:34:55.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:55.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:34:55.524 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:55.574 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:55.994 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:56.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:56.102 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:56.137 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:34:56.166 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:56.268 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:56 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:56.269 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:56 smithi008 ceph-mon[191227]: failed to encode map e131 with expected crc 2023-10-09T05:34:56.270 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:56 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2388498721' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:56.270 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:56 smithi008 ceph-mon[191227]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T05:34:56.270 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:56 smithi008 ceph-mon[191227]: failed to encode map e131 with expected crc 2023-10-09T05:34:56.271 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:56 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:56.271 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:56 smithi008 ceph-mon[184418]: failed to encode map e131 with expected crc 2023-10-09T05:34:56.271 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:56 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2388498721' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:56.271 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:56 smithi008 ceph-mon[184418]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T05:34:56.272 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:56 smithi008 ceph-mon[184418]: failed to encode map e131 with expected crc 2023-10-09T05:34:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:56 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:34:56.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:56 smithi191 ceph-mon[161669]: failed to encode map e131 with expected crc 2023-10-09T05:34:56.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:56 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2388498721' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:34:56.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:56 smithi191 ceph-mon[161669]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T05:34:56.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:56 smithi191 ceph-mon[161669]: failed to encode map e131 with expected crc 2023-10-09T05:34:56.324 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:56.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:56.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:56.431 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:34:56.433 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:56.505 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:56.530 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:34:56.570 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:34:56.596 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:34:56.626 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:34:56.627 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:34:56.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:56.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:34:56.654 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:34:56.686 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:34:56.720 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:34:56.731 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:56.765 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:34:56.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:34:56.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:34:56.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:34:56.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:34:56.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:34:56.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:34:56.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:34:56.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:34:56.779 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:57.347 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:57.351 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[184418]: pgmap v326: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 24869/257289 objects degraded (9.666%); 6.9 MiB/s, 131 objects/s recovering 2023-10-09T05:34:57.351 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[184418]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.351 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[184418]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T05:34:57.351 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[184418]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.351 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[184418]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.351 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[184418]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.351 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/285169612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:57.352 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[191227]: pgmap v326: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 24869/257289 objects degraded (9.666%); 6.9 MiB/s, 131 objects/s recovering 2023-10-09T05:34:57.352 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[191227]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.352 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[191227]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T05:34:57.352 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[191227]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.352 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[191227]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.352 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[191227]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.352 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:57 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/285169612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:57.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:57.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:57.454 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:57.494 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:57.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:57 smithi191 ceph-mon[161669]: pgmap v326: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 24869/257289 objects degraded (9.666%); 6.9 MiB/s, 131 objects/s recovering 2023-10-09T05:34:57.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:57 smithi191 ceph-mon[161669]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:57 smithi191 ceph-mon[161669]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T05:34:57.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:57 smithi191 ceph-mon[161669]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:57 smithi191 ceph-mon[161669]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:57 smithi191 ceph-mon[161669]: failed to encode map e132 with expected crc 2023-10-09T05:34:57.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:57 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/285169612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:34:57.762 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:57.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:57.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:57.868 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:34:57.946 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:58.105 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:34:58.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:58.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:34:58.128 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:34:58.215 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:58.256 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:58.372 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:58.376 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:58 smithi008 ceph-mon[184418]: failed to encode map e133 with expected crc 2023-10-09T05:34:58.376 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:58 smithi008 ceph-mon[184418]: failed to encode map e133 with expected crc 2023-10-09T05:34:58.376 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:58 smithi008 ceph-mon[184418]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T05:34:58.377 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:58 smithi008 ceph-mon[184418]: failed to encode map e133 with expected crc 2023-10-09T05:34:58.377 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24869/257289 objects degraded (9.666%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:58.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:58 smithi008 ceph-mon[191227]: failed to encode map e133 with expected crc 2023-10-09T05:34:58.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:58 smithi008 ceph-mon[191227]: failed to encode map e133 with expected crc 2023-10-09T05:34:58.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:58 smithi008 ceph-mon[191227]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T05:34:58.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:58 smithi008 ceph-mon[191227]: failed to encode map e133 with expected crc 2023-10-09T05:34:58.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24869/257289 objects degraded (9.666%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:58.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:58.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:58.412 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:34:58.412 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:34:58.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:58 smithi191 ceph-mon[161669]: failed to encode map e133 with expected crc 2023-10-09T05:34:58.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:58 smithi191 ceph-mon[161669]: failed to encode map e133 with expected crc 2023-10-09T05:34:58.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:58 smithi191 ceph-mon[161669]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T05:34:58.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:58 smithi191 ceph-mon[161669]: failed to encode map e133 with expected crc 2023-10-09T05:34:58.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24869/257289 objects degraded (9.666%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:34:58.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:34:58.562 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:58.709 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:34:58.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:58.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:34:58.816 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:58.898 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:58.996 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:59.000 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:34:59.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:59.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:59.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:34:59.147 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:59.189 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:34:59.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:59.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:34:59.293 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:34:59.360 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:59.447 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:34:59.447 INFO:teuthology.orchestra.run.smithi008.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:34:59.447 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:34:59.478 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:34:59.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:59.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:34:59.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:34:59.583 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:59.643 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:34:59.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:59 smithi008 ceph-mon[184418]: pgmap v329: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 964 op/s; 24869/254475 objects degraded (9.773%); 4.4 MiB/s, 99 objects/s recovering 2023-10-09T05:34:59.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:59.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:34:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:59.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:59 smithi008 ceph-mon[191227]: pgmap v329: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 964 op/s; 24869/254475 objects degraded (9.773%); 4.4 MiB/s, 99 objects/s recovering 2023-10-09T05:34:59.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:34:59.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:34:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:34:59.717 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:34:59.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:34:59.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:34:59.824 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:34:59.875 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:00.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:59 smithi191 ceph-mon[161669]: pgmap v329: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 964 op/s; 24869/254475 objects degraded (9.773%); 4.4 MiB/s, 99 objects/s recovering 2023-10-09T05:35:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:34:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:00.116 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:35:00.181 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:35:00.771 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:00.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:00.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:00.774 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:35:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:35:00] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T05:35:00.874 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:00.924 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:01.012 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:01.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:01.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:01.117 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:01.176 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:01.360 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:01.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:01.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:01.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:01.476 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:01.491 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:01.535 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:35:01.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:01.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:01.638 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:01.704 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:01 smithi191 ceph-mon[161669]: pgmap v330: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 24163/247959 objects degraded (9.745%); 8.5 MiB/s, 186 objects/s recovering 2023-10-09T05:35:01.892 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:01 smithi008 ceph-mon[184418]: pgmap v330: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 24163/247959 objects degraded (9.745%); 8.5 MiB/s, 186 objects/s recovering 2023-10-09T05:35:01.894 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:01 smithi008 ceph-mon[191227]: pgmap v330: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 24163/247959 objects degraded (9.745%); 8.5 MiB/s, 186 objects/s recovering 2023-10-09T05:35:01.904 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:01.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:01.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:02.012 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:02.064 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:02.378 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:02.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:02.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:02.482 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:02.531 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:02.611 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:35:02.611 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:35:02.620 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:35:02.630 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:02.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:02 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2008082547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:02.637 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:02 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:02.637 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:02 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24163/247959 objects degraded (9.745%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:02.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:02 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2008082547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:02.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:02 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:02.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:02 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24163/247959 objects degraded (9.745%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:02.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:02.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:02.739 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:02.816 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:02.944 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:35:02.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:02.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:02.956 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:02 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2008082547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:02.956 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:02 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:02.956 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:02 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24163/247959 objects degraded (9.745%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:03.057 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:03.139 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:03.203 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:03.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:03.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:03.309 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:03.344 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:03.609 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[191227]: pgmap v331: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 24163/241629 objects degraded (10.000%); 3.5 MiB/s, 75 objects/s recovering 2023-10-09T05:35:03.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:03.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:35:03.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[191227]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T05:35:03.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[191227]: failed to encode map e134 with expected crc 2023-10-09T05:35:03.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[191227]: failed to encode map e134 with expected crc 2023-10-09T05:35:03.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[191227]: failed to encode map e134 with expected crc 2023-10-09T05:35:03.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[191227]: failed to encode map e134 with expected crc 2023-10-09T05:35:03.658 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:35:03.816 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:03.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:03.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:03.933 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:03.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[184418]: pgmap v331: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 24163/241629 objects degraded (10.000%); 3.5 MiB/s, 75 objects/s recovering 2023-10-09T05:35:03.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:35:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[184418]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T05:35:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[184418]: failed to encode map e134 with expected crc 2023-10-09T05:35:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[184418]: failed to encode map e134 with expected crc 2023-10-09T05:35:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[184418]: failed to encode map e134 with expected crc 2023-10-09T05:35:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:03 smithi008 ceph-mon[184418]: failed to encode map e134 with expected crc 2023-10-09T05:35:03.999 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:03 smithi191 ceph-mon[161669]: pgmap v331: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 24163/241629 objects degraded (10.000%); 3.5 MiB/s, 75 objects/s recovering 2023-10-09T05:35:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:03 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:03 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:35:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:03 smithi191 ceph-mon[161669]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T05:35:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:03 smithi191 ceph-mon[161669]: failed to encode map e134 with expected crc 2023-10-09T05:35:04.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:03 smithi191 ceph-mon[161669]: failed to encode map e134 with expected crc 2023-10-09T05:35:04.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:03 smithi191 ceph-mon[161669]: failed to encode map e134 with expected crc 2023-10-09T05:35:04.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:03 smithi191 ceph-mon[161669]: failed to encode map e134 with expected crc 2023-10-09T05:35:04.118 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:04.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:04.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:04.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:04.297 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:04.383 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:04.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:04.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:04.445 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:35:04.460 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:04.488 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:35:04.497 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:04.527 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:35:04.559 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:04.576 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:35:04.618 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:35:04.664 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:35:04.686 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:04.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:04.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:04.705 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:35:04.745 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:35:04.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:35:04.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:35:04.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:35:04.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:35:04.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:35:04.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:35:04.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:35:04.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:35:04.794 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:04.820 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2616208585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:04.820 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2616208585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:04.856 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:05.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2616208585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:05.639 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:35:05.640 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:35:05.640 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T05:35:05.640 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:35:05.852 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:35:05.876 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:05 smithi008 ceph-mon[184418]: pgmap v333: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 24163/238818 objects degraded (10.118%); 3.1 MiB/s, 66 objects/s recovering 2023-10-09T05:35:05.876 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:05 smithi008 ceph-mon[191227]: pgmap v333: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 24163/238818 objects degraded (10.118%); 3.1 MiB/s, 66 objects/s recovering 2023-10-09T05:35:06.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:05 smithi191 ceph-mon[161669]: pgmap v333: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 24163/238818 objects degraded (10.118%); 3.1 MiB/s, 66 objects/s recovering 2023-10-09T05:35:06.745 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:06.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:06.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:06.924 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:06 smithi191 ceph-mon[161669]: pgmap v334: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 23620/233148 objects degraded (10.131%); 6.5 MiB/s, 120 objects/s recovering 2023-10-09T05:35:06.928 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:06.964 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:06.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:06 smithi008 ceph-mon[184418]: pgmap v334: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 23620/233148 objects degraded (10.131%); 6.5 MiB/s, 120 objects/s recovering 2023-10-09T05:35:06.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:06 smithi008 ceph-mon[191227]: pgmap v334: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 23620/233148 objects degraded (10.131%); 6.5 MiB/s, 120 objects/s recovering 2023-10-09T05:35:07.009 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:07.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:07.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:07.114 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:07.150 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:07.229 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:35:07.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:07.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:07.308 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:07.337 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:07.445 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:07.455 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:35:07.455 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:35:07.455 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:07.455 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:163534 5.5 5:a7b9787c:::benchmark_data_smithi008_238708_object45398:head [delete] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:35:07.449769+0000", 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0050399570000000003, 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050761310000000002, 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 163534 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:07.456 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:07.449769+0000", 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:07.449769+0000", 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:07.457 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:07.449767+0000", 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:07.449779+0000", 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2352e-05 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:07.449797+0000", 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7425999999999999e-05 2023-10-09T05:35:07.458 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:07.449809+0000", 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2150999999999999e-05 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:07.449877+0000", 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7512e-05 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:07.459 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:07.450044+0000", 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016731499999999999 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:07.450676+0000", 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00063232999999999996 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:07.460 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:35:07.461 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 1 2023-10-09T05:35:07.461 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:35:07.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:07 smithi008 ceph-mon[184418]: 5.15 scrub starts 2023-10-09T05:35:07.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:07 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23620/233148 objects degraded (10.131%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:07.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:07 smithi008 ceph-mon[184418]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T05:35:07.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:07 smithi008 ceph-mon[191227]: 5.15 scrub starts 2023-10-09T05:35:07.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:07 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23620/233148 objects degraded (10.131%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:07.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:07 smithi008 ceph-mon[191227]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T05:35:08.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:07 smithi191 ceph-mon[161669]: 5.15 scrub starts 2023-10-09T05:35:08.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:07 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23620/233148 objects degraded (10.131%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:08.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:07 smithi191 ceph-mon[161669]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T05:35:08.700 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:35:08.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[184418]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.012 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[184418]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.012 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[184418]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.012 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[184418]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.012 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[184418]: pgmap v336: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 973 op/s; 23620/224577 objects degraded (10.518%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:35:09.013 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[184418]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T05:35:09.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[191227]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[191227]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[191227]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[191227]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[191227]: pgmap v336: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 973 op/s; 23620/224577 objects degraded (10.518%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:35:09.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:08 smithi008 ceph-mon[191227]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T05:35:09.056 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:08 smithi191 ceph-mon[161669]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.057 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:08 smithi191 ceph-mon[161669]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.057 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:08 smithi191 ceph-mon[161669]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.057 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:08 smithi191 ceph-mon[161669]: failed to encode map e135 with expected crc 2023-10-09T05:35:09.057 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:08 smithi191 ceph-mon[161669]: pgmap v336: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 99 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 973 op/s; 23620/224577 objects degraded (10.518%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:35:09.057 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:08 smithi191 ceph-mon[161669]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T05:35:09.430 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:09.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:09.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:09.544 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:09.599 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:09.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[184418]: 5.13 scrub starts 2023-10-09T05:35:09.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[184418]: failed to encode map e136 with expected crc 2023-10-09T05:35:09.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[184418]: failed to encode map e136 with expected crc 2023-10-09T05:35:09.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[184418]: failed to encode map e136 with expected crc 2023-10-09T05:35:09.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[184418]: failed to encode map e136 with expected crc 2023-10-09T05:35:09.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[184418]: failed to encode map e136 with expected crc 2023-10-09T05:35:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[191227]: 5.13 scrub starts 2023-10-09T05:35:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[191227]: failed to encode map e136 with expected crc 2023-10-09T05:35:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[191227]: failed to encode map e136 with expected crc 2023-10-09T05:35:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[191227]: failed to encode map e136 with expected crc 2023-10-09T05:35:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[191227]: failed to encode map e136 with expected crc 2023-10-09T05:35:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:09 smithi008 ceph-mon[191227]: failed to encode map e136 with expected crc 2023-10-09T05:35:10.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:09 smithi191 ceph-mon[161669]: 5.13 scrub starts 2023-10-09T05:35:10.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:09 smithi191 ceph-mon[161669]: failed to encode map e136 with expected crc 2023-10-09T05:35:10.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:09 smithi191 ceph-mon[161669]: failed to encode map e136 with expected crc 2023-10-09T05:35:10.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:09 smithi191 ceph-mon[161669]: failed to encode map e136 with expected crc 2023-10-09T05:35:10.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:09 smithi191 ceph-mon[161669]: failed to encode map e136 with expected crc 2023-10-09T05:35:10.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:09 smithi191 ceph-mon[161669]: failed to encode map e136 with expected crc 2023-10-09T05:35:10.479 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:35:10.705 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:35:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:35:10] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T05:35:10.772 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:10.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:10.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:10.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:10.927 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:10.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:10 smithi008 ceph-mon[184418]: 5.3 scrub starts 2023-10-09T05:35:10.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:10 smithi008 ceph-mon[184418]: pgmap v338: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 23542/217863 objects degraded (10.806%); 5.3 MiB/s, 128 objects/s recovering 2023-10-09T05:35:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:10 smithi008 ceph-mon[191227]: 5.3 scrub starts 2023-10-09T05:35:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:10 smithi008 ceph-mon[191227]: pgmap v338: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 23542/217863 objects degraded (10.806%); 5.3 MiB/s, 128 objects/s recovering 2023-10-09T05:35:11.005 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:35:11.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:10 smithi191 ceph-mon[161669]: 5.3 scrub starts 2023-10-09T05:35:11.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:10 smithi191 ceph-mon[161669]: pgmap v338: 109 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 23542/217863 objects degraded (10.806%); 5.3 MiB/s, 128 objects/s recovering 2023-10-09T05:35:11.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:11.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:11.110 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:11.137 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:11.178 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:35:11.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:11.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:11.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:11.346 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:11.438 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:35:11.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:11.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:11.548 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:11.598 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:11.660 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:35:11.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:11.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:11.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:11.822 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:11.951 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:35:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:11.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:12.059 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:12.111 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:12.128 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:12.128 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:12.180 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:12.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:12.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:12.285 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:12.314 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:12.403 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:35:12.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:12.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:12.519 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:12.555 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:35:12.565 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:35:12.650 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:35:12.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:12.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:12.763 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:12.845 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:13.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:13 smithi191 ceph-mon[161669]: 5.23 scrub starts 2023-10-09T05:35:13.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:13 smithi191 ceph-mon[161669]: pgmap v339: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 22651/211062 objects degraded (10.732%); 6.9 MiB/s, 149 objects/s recovering 2023-10-09T05:35:13.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:13 smithi191 ceph-mon[161669]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:35:13.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 22651/211062 objects degraded (10.732%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1713996710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:13 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:13 smithi191 ceph-mon[161669]: 5.15 scrub starts 2023-10-09T05:35:13.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[184418]: 5.23 scrub starts 2023-10-09T05:35:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[184418]: pgmap v339: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 22651/211062 objects degraded (10.732%); 6.9 MiB/s, 149 objects/s recovering 2023-10-09T05:35:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[184418]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:35:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 22651/211062 objects degraded (10.732%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1713996710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[184418]: 5.15 scrub starts 2023-10-09T05:35:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[191227]: 5.23 scrub starts 2023-10-09T05:35:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[191227]: pgmap v339: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 22651/211062 objects degraded (10.732%); 6.9 MiB/s, 149 objects/s recovering 2023-10-09T05:35:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[191227]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:35:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 22651/211062 objects degraded (10.732%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1713996710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:13 smithi008 ceph-mon[191227]: 5.15 scrub starts 2023-10-09T05:35:13.597 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:35:14.080 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:14.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:14.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:14.188 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:14.260 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:14 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:35:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:14 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:14 smithi191 ceph-mon[161669]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T05:35:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:14 smithi191 ceph-mon[161669]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:14 smithi191 ceph-mon[161669]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:14 smithi191 ceph-mon[161669]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:14 smithi191 ceph-mon[161669]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:14 smithi191 ceph-mon[161669]: 5.13 scrub starts 2023-10-09T05:35:14.341 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:14.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:14.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:14.361 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:35:14.361 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:14.361 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[184418]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T05:35:14.361 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[184418]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[184418]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[184418]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[184418]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:14.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:14.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[184418]: 5.13 scrub starts 2023-10-09T05:35:14.362 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:35:14.362 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:14.362 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[191227]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T05:35:14.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[191227]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[191227]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[191227]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[191227]: failed to encode map e137 with expected crc 2023-10-09T05:35:14.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:14.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:14.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:14 smithi008 ceph-mon[191227]: 5.13 scrub starts 2023-10-09T05:35:14.449 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:14.521 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:15 smithi191 ceph-mon[161669]: failed to encode map e137 with expected crc 2023-10-09T05:35:15.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:15 smithi191 ceph-mon[161669]: pgmap v341: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 22651/208437 objects degraded (10.867%); 3.3 MiB/s, 102 objects/s recovering 2023-10-09T05:35:15.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:15 smithi008 ceph-mon[184418]: failed to encode map e137 with expected crc 2023-10-09T05:35:15.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:15 smithi008 ceph-mon[184418]: pgmap v341: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 22651/208437 objects degraded (10.867%); 3.3 MiB/s, 102 objects/s recovering 2023-10-09T05:35:15.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:15 smithi008 ceph-mon[191227]: failed to encode map e137 with expected crc 2023-10-09T05:35:15.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:15 smithi008 ceph-mon[191227]: pgmap v341: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 22651/208437 objects degraded (10.867%); 3.3 MiB/s, 102 objects/s recovering 2023-10-09T05:35:15.801 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":339,"stamp":"2023-10-09T05:35:11.941337+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4613576651,"num_objects":70354,"num_object_clones":0,"num_object_copies":211062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22651,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":70354,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":116860,"num_write_kb":4625030,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2903,"num_bytes_recovered":148897792,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":300601,"ondisk_log_size":300601,"up":327,"acting":318,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14612908,"kb_used_data":12926444,"kb_used_omap":45,"kb_used_meta":1685970,"kb_avail":735348308,"statfs":{"total":767960285184,"available":752996667392,"internally_reserved":0,"allocated":13236678656,"data_stored":13232283930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":1726433571},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,6,2,6,0,1,0,1,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-606339072,"num_objects":-9252,"num_object_clones":0,"num_object_copies":-27756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1512,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9252,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1199,"num_bytes_recovered":57540608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002702"},"pg_stats":[{"pgid":"5.23","version":"135'7383","reported_seq":7520,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.676374+0000","last_change":"2023-10-09T05:34:25.037728+0000","last_active":"2023-10-09T05:35:07.676374+0000","last_peered":"2023-10-09T05:35:07.676374+0000","last_clean":"2023-10-09T05:35:07.676374+0000","last_became_active":"2023-10-09T05:34:25.035930+0000","last_became_peered":"2023-10-09T05:34:25.035930+0000","last_unstale":"2023-10-09T05:35:07.676374+0000","last_undegraded":"2023-10-09T05:35:07.676374+0000","last_fullsized":"2023-10-09T05:35:07.676374+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"120'6300","last_scrub_stamp":"2023-10-09T05:34:08.749711+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:08.749711+0000","objects_scrubbed":3317,"log_size":7383,"log_dups_size":0,"ondisk_log_size":7383,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.61798832400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74022912,"num_objects":1130,"num_object_clones":0,"num_object_copies":3390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":895,"num_write_kb":11712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.22","version":"136'7477","reported_seq":7549,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:09.769673+0000","last_change":"2023-10-09T05:34:40.876515+0000","last_active":"2023-10-09T05:35:09.769673+0000","last_peered":"2023-10-09T05:35:09.769673+0000","last_clean":"2023-10-09T05:35:09.769673+0000","last_became_active":"2023-10-09T05:34:25.030851+0000","last_became_peered":"2023-10-09T05:34:25.030851+0000","last_unstale":"2023-10-09T05:35:09.769673+0000","last_undegraded":"2023-10-09T05:35:09.769673+0000","last_fullsized":"2023-10-09T05:35:09.769673+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"127'7054","last_scrub_stamp":"2023-10-09T05:34:40.876434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:40.876434+0000","objects_scrubbed":0,"log_size":7477,"log_dups_size":0,"ondisk_log_size":7477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69206016,"num_objects":1056,"num_object_clones":0,"num_object_copies":3168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":423,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.21","version":"136'7433","reported_seq":10165,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:09.810396+0000","last_change":"2023-10-09T05:35:08.601492+0000","last_active":"2023-10-09T05:35:09.810396+0000","last_peered":"2023-10-09T05:35:09.810396+0000","last_clean":"2023-10-09T05:35:09.810396+0000","last_became_active":"2023-10-09T05:35:08.595030+0000","last_became_peered":"2023-10-09T05:35:08.595030+0000","last_unstale":"2023-10-09T05:35:09.810396+0000","last_undegraded":"2023-10-09T05:35:09.810396+0000","last_fullsized":"2023-10-09T05:35:09.810396+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":7433,"log_dups_size":0,"ondisk_log_size":7433,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72531968,"num_objects":1107,"num_object_clones":0,"num_object_copies":3321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2564,"num_write_kb":117456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1327,"num_bytes_recovered":65929216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.20","version":"136'7423","reported_seq":7599,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:10.844528+0000","last_change":"2023-10-09T05:34:38.457569+0000","last_active":"2023-10-09T05:35:10.844528+0000","last_peered":"2023-10-09T05:35:10.844528+0000","last_clean":"2023-10-09T05:35:10.844528+0000","last_became_active":"2023-10-09T05:34:25.062440+0000","last_became_peered":"2023-10-09T05:34:25.062440+0000","last_unstale":"2023-10-09T05:35:10.844528+0000","last_undegraded":"2023-10-09T05:35:10.844528+0000","last_fullsized":"2023-10-09T05:35:10.844528+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"125'6962","last_scrub_stamp":"2023-10-09T05:34:38.457405+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:38.457405+0000","objects_scrubbed":1537,"log_size":7423,"log_dups_size":0,"ondisk_log_size":7423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:57.174556+0000","scrub_duration":0.224312019,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70451200,"num_objects":1075,"num_object_clones":0,"num_object_copies":3225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":462,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1c","version":"135'8976","reported_seq":9101,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:08.326703+0000","last_change":"2023-10-09T05:34:39.445074+0000","last_active":"2023-10-09T05:35:08.326703+0000","last_peered":"2023-10-09T05:35:08.326703+0000","last_clean":"2023-10-09T05:35:08.326703+0000","last_became_active":"2023-10-09T05:34:25.054943+0000","last_became_peered":"2023-10-09T05:34:25.054943+0000","last_unstale":"2023-10-09T05:35:08.326703+0000","last_undegraded":"2023-10-09T05:35:08.326703+0000","last_fullsized":"2023-10-09T05:35:08.326703+0000","mapping_epoch":123,"log_start":"108'200","ondisk_log_start":"108'200","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'8137","last_scrub_stamp":"2023-10-09T05:34:39.444955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:39.444955+0000","objects_scrubbed":3164,"log_size":8776,"log_dups_size":0,"ondisk_log_size":8776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:30.213649+0000","scrub_duration":0.76575304399999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151453696,"num_objects":2311,"num_object_clones":0,"num_object_copies":6933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":853,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":119,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.582760+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:35:07.582760+0000","last_peered":"2023-10-09T05:35:07.582760+0000","last_clean":"2023-10-09T05:35:07.582760+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:35:07.582760+0000","last_undegraded":"2023-10-09T05:35:07.582760+0000","last_fullsized":"2023-10-09T05:35:07.582760+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":106,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.573013+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:35:07.573013+0000","last_peered":"2023-10-09T05:35:07.573013+0000","last_clean":"2023-10-09T05:35:07.573013+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:35:07.573013+0000","last_undegraded":"2023-10-09T05:35:07.573013+0000","last_fullsized":"2023-10-09T05:35:07.573013+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1f","version":"136'8855","reported_seq":8903,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:09.822523+0000","last_change":"2023-10-09T05:34:25.044017+0000","last_active":"2023-10-09T05:35:09.822523+0000","last_peered":"2023-10-09T05:35:09.822523+0000","last_clean":"2023-10-09T05:35:09.822523+0000","last_became_active":"2023-10-09T05:34:25.037058+0000","last_became_peered":"2023-10-09T05:34:25.037058+0000","last_unstale":"2023-10-09T05:35:09.822523+0000","last_undegraded":"2023-10-09T05:35:09.822523+0000","last_fullsized":"2023-10-09T05:35:09.822523+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8855,"log_dups_size":0,"ondisk_log_size":8855,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143687680,"num_objects":2193,"num_object_clones":0,"num_object_copies":6579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5154,"num_write_kb":235072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":79,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.607744+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:35:02.607744+0000","last_peered":"2023-10-09T05:35:02.607744+0000","last_clean":"2023-10-09T05:35:02.607744+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:35:02.607744+0000","last_undegraded":"2023-10-09T05:35:02.607744+0000","last_fullsized":"2023-10-09T05:35:02.607744+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":100,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.624252+0000","last_change":"2023-10-09T05:34:41.662942+0000","last_active":"2023-10-09T05:35:02.624252+0000","last_peered":"2023-10-09T05:35:02.624252+0000","last_clean":"2023-10-09T05:35:02.624252+0000","last_became_active":"2023-10-09T05:34:41.662648+0000","last_became_peered":"2023-10-09T05:34:41.662648+0000","last_unstale":"2023-10-09T05:35:02.624252+0000","last_undegraded":"2023-10-09T05:35:02.624252+0000","last_fullsized":"2023-10-09T05:35:02.624252+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:08.691549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1e","version":"136'8961","reported_seq":9007,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:09.822393+0000","last_change":"2023-10-09T05:34:25.039990+0000","last_active":"2023-10-09T05:35:09.822393+0000","last_peered":"2023-10-09T05:35:09.822393+0000","last_clean":"2023-10-09T05:35:09.822393+0000","last_became_active":"2023-10-09T05:34:25.036888+0000","last_became_peered":"2023-10-09T05:34:25.036888+0000","last_unstale":"2023-10-09T05:35:09.822393+0000","last_undegraded":"2023-10-09T05:35:09.822393+0000","last_fullsized":"2023-10-09T05:35:09.822393+0000","mapping_epoch":123,"log_start":"109'800","ondisk_log_start":"109'800","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8161,"log_dups_size":0,"ondisk_log_size":8161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144736256,"num_objects":2209,"num_object_clones":0,"num_object_copies":6627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5222,"num_write_kb":237760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":122,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.572482+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:35:07.572482+0000","last_peered":"2023-10-09T05:35:07.572482+0000","last_clean":"2023-10-09T05:35:07.572482+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:35:07.572482+0000","last_undegraded":"2023-10-09T05:35:07.572482+0000","last_fullsized":"2023-10-09T05:35:07.572482+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.590875+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:35:07.590875+0000","last_peered":"2023-10-09T05:35:07.590875+0000","last_clean":"2023-10-09T05:35:07.590875+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:35:07.590875+0000","last_undegraded":"2023-10-09T05:35:07.590875+0000","last_fullsized":"2023-10-09T05:35:07.590875+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.11","version":"136'8162","reported_seq":8222,"reported_epoch":136,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:09.822758+0000","last_change":"2023-10-09T05:34:42.977328+0000","last_active":"2023-10-09T05:35:09.822758+0000","last_peered":"2023-10-09T05:35:09.822758+0000","last_clean":"2023-10-09T05:33:20.516527+0000","last_became_active":"2023-10-09T05:34:42.969107+0000","last_became_peered":"2023-10-09T05:34:42.969107+0000","last_unstale":"2023-10-09T05:35:09.822758+0000","last_undegraded":"2023-10-09T05:34:42.867828+0000","last_fullsized":"2023-10-09T05:34:42.853589+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8162,"log_dups_size":0,"ondisk_log_size":8162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141328384,"num_objects":2157,"num_object_clones":0,"num_object_copies":6471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2627,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5145,"num_write_kb":233632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":178}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":85,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.625121+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:35:02.625121+0000","last_peered":"2023-10-09T05:35:02.625121+0000","last_clean":"2023-10-09T05:35:02.625121+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:35:02.625121+0000","last_undegraded":"2023-10-09T05:35:02.625121+0000","last_fullsized":"2023-10-09T05:35:02.625121+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":106,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.572636+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:35:07.572636+0000","last_peered":"2023-10-09T05:35:07.572636+0000","last_clean":"2023-10-09T05:35:07.572636+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:35:07.572636+0000","last_undegraded":"2023-10-09T05:35:07.572636+0000","last_fullsized":"2023-10-09T05:35:07.572636+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.10","version":"136'8120","reported_seq":8245,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:10.912236+0000","last_change":"2023-10-09T05:34:06.833835+0000","last_active":"2023-10-09T05:35:10.912236+0000","last_peered":"2023-10-09T05:35:10.912236+0000","last_clean":"2023-10-09T05:35:10.912236+0000","last_became_active":"2023-10-09T05:34:01.635329+0000","last_became_peered":"2023-10-09T05:34:01.635329+0000","last_unstale":"2023-10-09T05:35:10.912236+0000","last_undegraded":"2023-10-09T05:35:10.912236+0000","last_fullsized":"2023-10-09T05:35:10.912236+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6258","last_scrub_stamp":"2023-10-09T05:34:06.833681+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:06.833681+0000","objects_scrubbed":3155,"log_size":8120,"log_dups_size":0,"ondisk_log_size":8120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:45.971615+0000","scrub_duration":0.69237216700000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139329536,"num_objects":2126,"num_object_clones":0,"num_object_copies":6378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1869,"num_write_kb":26880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":118,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.586567+0000","last_change":"2023-10-09T05:34:41.683934+0000","last_active":"2023-10-09T05:35:07.586567+0000","last_peered":"2023-10-09T05:35:07.586567+0000","last_clean":"2023-10-09T05:35:07.586567+0000","last_became_active":"2023-10-09T05:34:41.683618+0000","last_became_peered":"2023-10-09T05:34:41.683618+0000","last_unstale":"2023-10-09T05:35:07.586567+0000","last_undegraded":"2023-10-09T05:35:07.586567+0000","last_fullsized":"2023-10-09T05:35:07.586567+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":111,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.573903+0000","last_change":"2023-10-09T05:34:41.668251+0000","last_active":"2023-10-09T05:35:07.573903+0000","last_peered":"2023-10-09T05:35:07.573903+0000","last_clean":"2023-10-09T05:35:07.573903+0000","last_became_active":"2023-10-09T05:34:41.668017+0000","last_became_peered":"2023-10-09T05:34:41.668017+0000","last_unstale":"2023-10-09T05:35:07.573903+0000","last_undegraded":"2023-10-09T05:35:07.573903+0000","last_fullsized":"2023-10-09T05:35:07.573903+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:27.235656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.13","version":"135'8087","reported_seq":8136,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.659424+0000","last_change":"2023-10-09T05:34:01.629256+0000","last_active":"2023-10-09T05:35:07.659424+0000","last_peered":"2023-10-09T05:35:07.659424+0000","last_clean":"2023-10-09T05:35:07.659424+0000","last_became_active":"2023-10-09T05:34:01.625092+0000","last_became_peered":"2023-10-09T05:34:01.625092+0000","last_unstale":"2023-10-09T05:35:07.659424+0000","last_undegraded":"2023-10-09T05:35:07.659424+0000","last_fullsized":"2023-10-09T05:35:07.659424+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8087,"log_dups_size":0,"ondisk_log_size":8087,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148111360,"num_objects":2260,"num_object_clones":0,"num_object_copies":6780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5066,"num_write_kb":234432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":95,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.589346+0000","last_change":"2023-10-09T05:34:41.682874+0000","last_active":"2023-10-09T05:35:07.589346+0000","last_peered":"2023-10-09T05:35:07.589346+0000","last_clean":"2023-10-09T05:35:07.589346+0000","last_became_active":"2023-10-09T05:34:41.674789+0000","last_became_peered":"2023-10-09T05:34:41.674789+0000","last_unstale":"2023-10-09T05:35:07.589346+0000","last_undegraded":"2023-10-09T05:35:07.589346+0000","last_fullsized":"2023-10-09T05:35:07.589346+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":80,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.578084+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:35:07.578084+0000","last_peered":"2023-10-09T05:35:07.578084+0000","last_clean":"2023-10-09T05:35:07.578084+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:35:07.578084+0000","last_undegraded":"2023-10-09T05:35:07.578084+0000","last_fullsized":"2023-10-09T05:35:07.578084+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.12","version":"136'8330","reported_seq":8383,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:09.807749+0000","last_change":"2023-10-09T05:34:06.470865+0000","last_active":"2023-10-09T05:35:09.807749+0000","last_peered":"2023-10-09T05:35:09.807749+0000","last_clean":"2023-10-09T05:35:09.807749+0000","last_became_active":"2023-10-09T05:34:01.624493+0000","last_became_peered":"2023-10-09T05:34:01.624493+0000","last_unstale":"2023-10-09T05:35:09.807749+0000","last_undegraded":"2023-10-09T05:35:09.807749+0000","last_fullsized":"2023-10-09T05:35:09.807749+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6331","last_scrub_stamp":"2023-10-09T05:34:06.470790+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:06.470790+0000","objects_scrubbed":0,"log_size":8330,"log_dups_size":0,"ondisk_log_size":8330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145031168,"num_objects":2213,"num_object_clones":0,"num_object_copies":6639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2011,"num_write_kb":30848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":125,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.583277+0000","last_change":"2023-10-09T05:34:41.683953+0000","last_active":"2023-10-09T05:35:07.583277+0000","last_peered":"2023-10-09T05:35:07.583277+0000","last_clean":"2023-10-09T05:35:07.583277+0000","last_became_active":"2023-10-09T05:34:41.683519+0000","last_became_peered":"2023-10-09T05:34:41.683519+0000","last_unstale":"2023-10-09T05:35:07.583277+0000","last_undegraded":"2023-10-09T05:35:07.583277+0000","last_fullsized":"2023-10-09T05:35:07.583277+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.582285+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:35:07.582285+0000","last_peered":"2023-10-09T05:35:07.582285+0000","last_clean":"2023-10-09T05:35:07.582285+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:35:07.582285+0000","last_undegraded":"2023-10-09T05:35:07.582285+0000","last_fullsized":"2023-10-09T05:35:07.582285+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.15","version":"135'8347","reported_seq":8396,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.691237+0000","last_change":"2023-10-09T05:35:06.042748+0000","last_active":"2023-10-09T05:35:07.691237+0000","last_peered":"2023-10-09T05:35:07.691237+0000","last_clean":"2023-10-09T05:35:07.691237+0000","last_became_active":"2023-10-09T05:34:01.623738+0000","last_became_peered":"2023-10-09T05:34:01.623738+0000","last_unstale":"2023-10-09T05:35:07.691237+0000","last_undegraded":"2023-10-09T05:35:07.691237+0000","last_fullsized":"2023-10-09T05:35:07.691237+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8347,"log_dups_size":0,"ondisk_log_size":8347,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148635648,"num_objects":2268,"num_object_clones":0,"num_object_copies":6804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5222,"num_write_kb":239680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":89,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.588026+0000","last_change":"2023-10-09T05:34:41.673944+0000","last_active":"2023-10-09T05:35:07.588026+0000","last_peered":"2023-10-09T05:35:07.588026+0000","last_clean":"2023-10-09T05:35:07.588026+0000","last_became_active":"2023-10-09T05:34:41.673595+0000","last_became_peered":"2023-10-09T05:34:41.673595+0000","last_unstale":"2023-10-09T05:35:07.588026+0000","last_undegraded":"2023-10-09T05:35:07.588026+0000","last_fullsized":"2023-10-09T05:35:07.588026+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":121,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.572792+0000","last_change":"2023-10-09T05:34:41.667462+0000","last_active":"2023-10-09T05:35:07.572792+0000","last_peered":"2023-10-09T05:35:07.572792+0000","last_clean":"2023-10-09T05:35:07.572792+0000","last_became_active":"2023-10-09T05:34:41.667140+0000","last_became_peered":"2023-10-09T05:34:41.667140+0000","last_unstale":"2023-10-09T05:35:07.572792+0000","last_undegraded":"2023-10-09T05:35:07.572792+0000","last_fullsized":"2023-10-09T05:35:07.572792+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:52.888470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.14","version":"136'8160","reported_seq":8217,"reported_epoch":136,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:10.891541+0000","last_change":"2023-10-09T05:34:42.906108+0000","last_active":"2023-10-09T05:35:10.891541+0000","last_peered":"2023-10-09T05:35:10.891541+0000","last_clean":"2023-10-09T05:33:20.493128+0000","last_became_active":"2023-10-09T05:34:42.894931+0000","last_became_peered":"2023-10-09T05:34:42.894931+0000","last_unstale":"2023-10-09T05:35:10.891541+0000","last_undegraded":"2023-10-09T05:34:42.686528+0000","last_fullsized":"2023-10-09T05:34:42.658194+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8160,"log_dups_size":0,"ondisk_log_size":8160,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141721600,"num_objects":2163,"num_object_clones":0,"num_object_copies":6489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2585,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5131,"num_write_kb":233376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":186}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":100,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.624711+0000","last_change":"2023-10-09T05:34:41.662790+0000","last_active":"2023-10-09T05:35:02.624711+0000","last_peered":"2023-10-09T05:35:02.624711+0000","last_clean":"2023-10-09T05:35:02.624711+0000","last_became_active":"2023-10-09T05:34:41.662513+0000","last_became_peered":"2023-10-09T05:34:41.662513+0000","last_unstale":"2023-10-09T05:35:02.624711+0000","last_undegraded":"2023-10-09T05:35:02.624711+0000","last_fullsized":"2023-10-09T05:35:02.624711+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:17.781371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.586494+0000","last_change":"2023-10-09T05:34:41.694695+0000","last_active":"2023-10-09T05:35:07.586494+0000","last_peered":"2023-10-09T05:35:07.586494+0000","last_clean":"2023-10-09T05:35:07.586494+0000","last_became_active":"2023-10-09T05:34:41.694408+0000","last_became_peered":"2023-10-09T05:34:41.694408+0000","last_unstale":"2023-10-09T05:35:07.586494+0000","last_undegraded":"2023-10-09T05:35:07.586494+0000","last_fullsized":"2023-10-09T05:35:07.586494+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.17","version":"136'8263","reported_seq":8316,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:09.816132+0000","last_change":"2023-10-09T05:34:37.421122+0000","last_active":"2023-10-09T05:35:09.816132+0000","last_peered":"2023-10-09T05:35:09.816132+0000","last_clean":"2023-10-09T05:35:09.816132+0000","last_became_active":"2023-10-09T05:34:01.618906+0000","last_became_peered":"2023-10-09T05:34:01.618906+0000","last_unstale":"2023-10-09T05:35:09.816132+0000","last_undegraded":"2023-10-09T05:35:09.816132+0000","last_fullsized":"2023-10-09T05:35:09.816132+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'7322","last_scrub_stamp":"2023-10-09T05:34:37.421020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:37.421020+0000","objects_scrubbed":0,"log_size":8263,"log_dups_size":0,"ondisk_log_size":8263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146931712,"num_objects":2242,"num_object_clones":0,"num_object_copies":6726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":958,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":72,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.588388+0000","last_change":"2023-10-09T05:34:41.702205+0000","last_active":"2023-10-09T05:35:07.588388+0000","last_peered":"2023-10-09T05:35:07.588388+0000","last_clean":"2023-10-09T05:35:07.588388+0000","last_became_active":"2023-10-09T05:34:41.701849+0000","last_became_peered":"2023-10-09T05:34:41.701849+0000","last_unstale":"2023-10-09T05:35:07.588388+0000","last_undegraded":"2023-10-09T05:35:07.588388+0000","last_fullsized":"2023-10-09T05:35:07.588388+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":100,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.624137+0000","last_change":"2023-10-09T05:34:41.663111+0000","last_active":"2023-10-09T05:35:02.624137+0000","last_peered":"2023-10-09T05:35:02.624137+0000","last_clean":"2023-10-09T05:35:02.624137+0000","last_became_active":"2023-10-09T05:34:41.662816+0000","last_became_peered":"2023-10-09T05:34:41.662816+0000","last_unstale":"2023-10-09T05:35:02.624137+0000","last_undegraded":"2023-10-09T05:35:02.624137+0000","last_fullsized":"2023-10-09T05:35:02.624137+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:36.226743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.16","version":"136'8303","reported_seq":8362,"reported_epoch":136,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:10.891546+0000","last_change":"2023-10-09T05:34:42.899801+0000","last_active":"2023-10-09T05:35:10.891546+0000","last_peered":"2023-10-09T05:35:10.891546+0000","last_clean":"2023-10-09T05:33:20.529830+0000","last_became_active":"2023-10-09T05:34:42.891787+0000","last_became_peered":"2023-10-09T05:34:42.891787+0000","last_unstale":"2023-10-09T05:35:10.891546+0000","last_undegraded":"2023-10-09T05:34:42.686663+0000","last_fullsized":"2023-10-09T05:34:42.658538+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8303,"log_dups_size":0,"ondisk_log_size":8303,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141099008,"num_objects":2153,"num_object_clones":0,"num_object_copies":6459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2590,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5235,"num_write_kb":236416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":176}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":111,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.590186+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:35:07.590186+0000","last_peered":"2023-10-09T05:35:07.590186+0000","last_clean":"2023-10-09T05:35:07.590186+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:35:07.590186+0000","last_undegraded":"2023-10-09T05:35:07.590186+0000","last_fullsized":"2023-10-09T05:35:07.590186+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":121,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.573369+0000","last_change":"2023-10-09T05:34:41.664294+0000","last_active":"2023-10-09T05:35:07.573369+0000","last_peered":"2023-10-09T05:35:07.573369+0000","last_clean":"2023-10-09T05:35:07.573369+0000","last_became_active":"2023-10-09T05:34:41.663985+0000","last_became_peered":"2023-10-09T05:34:41.663985+0000","last_unstale":"2023-10-09T05:35:07.573369+0000","last_undegraded":"2023-10-09T05:35:07.573369+0000","last_fullsized":"2023-10-09T05:35:07.573369+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:58.417307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"136'8222","reported_seq":8278,"reported_epoch":136,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:09.843363+0000","last_change":"2023-10-09T05:34:42.977224+0000","last_active":"2023-10-09T05:35:09.843363+0000","last_peered":"2023-10-09T05:35:09.843363+0000","last_clean":"2023-10-09T05:33:20.515228+0000","last_became_active":"2023-10-09T05:34:42.967706+0000","last_became_peered":"2023-10-09T05:34:42.967706+0000","last_unstale":"2023-10-09T05:35:09.843363+0000","last_undegraded":"2023-10-09T05:34:42.852684+0000","last_fullsized":"2023-10-09T05:34:42.677882+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8222,"log_dups_size":0,"ondisk_log_size":8222,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141295616,"num_objects":2156,"num_object_clones":0,"num_object_copies":6468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2614,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5162,"num_write_kb":234176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":153}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":78,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.588201+0000","last_change":"2023-10-09T05:34:41.688229+0000","last_active":"2023-10-09T05:35:07.588201+0000","last_peered":"2023-10-09T05:35:07.588201+0000","last_clean":"2023-10-09T05:35:07.588201+0000","last_became_active":"2023-10-09T05:34:41.687795+0000","last_became_peered":"2023-10-09T05:34:41.687795+0000","last_unstale":"2023-10-09T05:35:07.588201+0000","last_undegraded":"2023-10-09T05:35:07.588201+0000","last_fullsized":"2023-10-09T05:35:07.588201+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.585692+0000","last_change":"2023-10-09T05:34:41.678569+0000","last_active":"2023-10-09T05:35:07.585692+0000","last_peered":"2023-10-09T05:35:07.585692+0000","last_clean":"2023-10-09T05:35:07.585692+0000","last_became_active":"2023-10-09T05:34:41.678271+0000","last_became_peered":"2023-10-09T05:34:41.678271+0000","last_unstale":"2023-10-09T05:35:07.585692+0000","last_undegraded":"2023-10-09T05:35:07.585692+0000","last_fullsized":"2023-10-09T05:35:07.585692+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"136'10945","reported_seq":11283,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:10.871669+0000","last_change":"2023-10-09T05:34:25.139244+0000","last_active":"2023-10-09T05:35:10.871669+0000","last_peered":"2023-10-09T05:35:10.871669+0000","last_clean":"2023-10-09T05:35:10.871669+0000","last_became_active":"2023-10-09T05:34:01.629769+0000","last_became_peered":"2023-10-09T05:34:01.629769+0000","last_unstale":"2023-10-09T05:35:10.871669+0000","last_undegraded":"2023-10-09T05:35:10.871669+0000","last_fullsized":"2023-10-09T05:35:10.871669+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":3497,"log_size":10945,"log_dups_size":0,"ondisk_log_size":10945,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141590528,"num_objects":2160,"num_object_clones":0,"num_object_copies":6480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5248,"num_write_kb":237088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":84,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.588244+0000","last_change":"2023-10-09T05:34:41.674752+0000","last_active":"2023-10-09T05:35:07.588244+0000","last_peered":"2023-10-09T05:35:07.588244+0000","last_clean":"2023-10-09T05:35:07.588244+0000","last_became_active":"2023-10-09T05:34:41.674253+0000","last_became_peered":"2023-10-09T05:34:41.674253+0000","last_unstale":"2023-10-09T05:35:07.588244+0000","last_undegraded":"2023-10-09T05:35:07.588244+0000","last_fullsized":"2023-10-09T05:35:07.588244+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":117,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.585882+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:35:07.585882+0000","last_peered":"2023-10-09T05:35:07.585882+0000","last_clean":"2023-10-09T05:35:07.585882+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:35:07.585882+0000","last_undegraded":"2023-10-09T05:35:07.585882+0000","last_fullsized":"2023-10-09T05:35:07.585882+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"136'9059","reported_seq":9111,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:10.906981+0000","last_change":"2023-10-09T05:34:42.657980+0000","last_active":"2023-10-09T05:35:10.906981+0000","last_peered":"2023-10-09T05:35:10.906981+0000","last_clean":"2023-10-09T05:35:10.906981+0000","last_became_active":"2023-10-09T05:34:25.039138+0000","last_became_peered":"2023-10-09T05:34:25.039138+0000","last_unstale":"2023-10-09T05:35:10.906981+0000","last_undegraded":"2023-10-09T05:35:10.906981+0000","last_fullsized":"2023-10-09T05:35:10.906981+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9059,"log_dups_size":0,"ondisk_log_size":9059,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137887744,"num_objects":2104,"num_object_clones":0,"num_object_copies":6312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5321,"num_write_kb":237600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":75,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.589627+0000","last_change":"2023-10-09T05:34:41.691449+0000","last_active":"2023-10-09T05:35:07.589627+0000","last_peered":"2023-10-09T05:35:07.589627+0000","last_clean":"2023-10-09T05:35:07.589627+0000","last_became_active":"2023-10-09T05:34:41.675227+0000","last_became_peered":"2023-10-09T05:34:41.675227+0000","last_unstale":"2023-10-09T05:35:07.589627+0000","last_undegraded":"2023-10-09T05:35:07.589627+0000","last_fullsized":"2023-10-09T05:35:07.589627+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":126,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.592175+0000","last_change":"2023-10-09T05:34:41.669908+0000","last_active":"2023-10-09T05:35:07.592175+0000","last_peered":"2023-10-09T05:35:07.592175+0000","last_clean":"2023-10-09T05:35:07.592175+0000","last_became_active":"2023-10-09T05:34:41.669596+0000","last_became_peered":"2023-10-09T05:34:41.669596+0000","last_unstale":"2023-10-09T05:35:07.592175+0000","last_undegraded":"2023-10-09T05:35:07.592175+0000","last_fullsized":"2023-10-09T05:35:07.592175+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"136'8857","reported_seq":8984,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:10.879214+0000","last_change":"2023-10-09T05:34:35.674328+0000","last_active":"2023-10-09T05:35:10.879214+0000","last_peered":"2023-10-09T05:35:10.879214+0000","last_clean":"2023-10-09T05:35:10.879214+0000","last_became_active":"2023-10-09T05:34:25.008155+0000","last_became_peered":"2023-10-09T05:34:25.008155+0000","last_unstale":"2023-10-09T05:35:10.879214+0000","last_undegraded":"2023-10-09T05:35:10.879214+0000","last_fullsized":"2023-10-09T05:35:10.879214+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7846","last_scrub_stamp":"2023-10-09T05:34:35.674093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:35.674093+0000","objects_scrubbed":3275,"log_size":8857,"log_dups_size":0,"ondisk_log_size":8857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:02.820818+0000","scrub_duration":0.75420257000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147652608,"num_objects":2253,"num_object_clones":0,"num_object_copies":6759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1022,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":103,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.585994+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:35:07.585994+0000","last_peered":"2023-10-09T05:35:07.585994+0000","last_clean":"2023-10-09T05:35:07.585994+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:35:07.585994+0000","last_undegraded":"2023-10-09T05:35:07.585994+0000","last_fullsized":"2023-10-09T05:35:07.585994+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":85,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.623398+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:35:02.623398+0000","last_peered":"2023-10-09T05:35:02.623398+0000","last_clean":"2023-10-09T05:35:02.623398+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:35:02.623398+0000","last_undegraded":"2023-10-09T05:35:02.623398+0000","last_fullsized":"2023-10-09T05:35:02.623398+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"136'8990","reported_seq":9566,"reported_epoch":136,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:10.936099+0000","last_change":"2023-10-09T05:35:07.068871+0000","last_active":"2023-10-09T05:35:10.936099+0000","last_peered":"2023-10-09T05:35:10.936099+0000","last_clean":"2023-10-09T05:33:20.502869+0000","last_became_active":"2023-10-09T05:34:42.879684+0000","last_became_peered":"2023-10-09T05:34:42.879684+0000","last_unstale":"2023-10-09T05:35:10.936099+0000","last_undegraded":"2023-10-09T05:34:42.683747+0000","last_fullsized":"2023-10-09T05:34:42.658139+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8990,"log_dups_size":0,"ondisk_log_size":8990,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143130636,"num_objects":2184,"num_object_clones":0,"num_object_copies":6552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1701,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5247,"num_write_kb":237792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":257,"num_bytes_recovered":16842752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":483}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":104,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.572405+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:35:07.572405+0000","last_peered":"2023-10-09T05:35:07.572405+0000","last_clean":"2023-10-09T05:35:07.572405+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:35:07.572405+0000","last_undegraded":"2023-10-09T05:35:07.572405+0000","last_fullsized":"2023-10-09T05:35:07.572405+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.583130+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:35:07.583130+0000","last_peered":"2023-10-09T05:35:07.583130+0000","last_clean":"2023-10-09T05:35:07.583130+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:35:07.583130+0000","last_undegraded":"2023-10-09T05:35:07.583130+0000","last_fullsized":"2023-10-09T05:35:07.583130+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"135'8945","reported_seq":9070,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:08.396086+0000","last_change":"2023-10-09T05:34:40.457466+0000","last_active":"2023-10-09T05:35:08.396086+0000","last_peered":"2023-10-09T05:35:08.396086+0000","last_clean":"2023-10-09T05:35:08.396086+0000","last_became_active":"2023-10-09T05:34:25.026725+0000","last_became_peered":"2023-10-09T05:34:25.026725+0000","last_unstale":"2023-10-09T05:35:08.396086+0000","last_undegraded":"2023-10-09T05:35:08.396086+0000","last_fullsized":"2023-10-09T05:35:08.396086+0000","mapping_epoch":105,"log_start":"108'200","ondisk_log_start":"108'200","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'8107","last_scrub_stamp":"2023-10-09T05:34:40.457321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:40.457321+0000","objects_scrubbed":3100,"log_size":8745,"log_dups_size":0,"ondisk_log_size":8745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:26.084493+0000","scrub_duration":0.74267715700000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147324928,"num_objects":2248,"num_object_clones":0,"num_object_copies":6744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":852,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":103,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.591954+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:35:07.591954+0000","last_peered":"2023-10-09T05:35:07.591954+0000","last_clean":"2023-10-09T05:35:07.591954+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:35:07.591954+0000","last_undegraded":"2023-10-09T05:35:07.591954+0000","last_fullsized":"2023-10-09T05:35:07.591954+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":117,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.584239+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:35:07.584239+0000","last_peered":"2023-10-09T05:35:07.584239+0000","last_clean":"2023-10-09T05:35:07.584239+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:35:07.584239+0000","last_undegraded":"2023-10-09T05:35:07.584239+0000","last_fullsized":"2023-10-09T05:35:07.584239+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"136'8814","reported_seq":8865,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:09.779983+0000","last_change":"2023-10-09T05:34:39.803946+0000","last_active":"2023-10-09T05:35:09.779983+0000","last_peered":"2023-10-09T05:35:09.779983+0000","last_clean":"2023-10-09T05:35:09.779983+0000","last_became_active":"2023-10-09T05:34:25.018175+0000","last_became_peered":"2023-10-09T05:34:25.018175+0000","last_unstale":"2023-10-09T05:35:09.779983+0000","last_undegraded":"2023-10-09T05:35:09.779983+0000","last_fullsized":"2023-10-09T05:35:09.779983+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8814,"log_dups_size":0,"ondisk_log_size":8814,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146374656,"num_objects":2233,"num_object_clones":0,"num_object_copies":6699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5112,"num_write_kb":235072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":118,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.585699+0000","last_change":"2023-10-09T05:34:41.706409+0000","last_active":"2023-10-09T05:35:07.585699+0000","last_peered":"2023-10-09T05:35:07.585699+0000","last_clean":"2023-10-09T05:35:07.585699+0000","last_became_active":"2023-10-09T05:34:41.706002+0000","last_became_peered":"2023-10-09T05:34:41.706002+0000","last_unstale":"2023-10-09T05:35:07.585699+0000","last_undegraded":"2023-10-09T05:35:07.585699+0000","last_fullsized":"2023-10-09T05:35:07.585699+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":111,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.584788+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:35:07.584788+0000","last_peered":"2023-10-09T05:35:07.584788+0000","last_clean":"2023-10-09T05:35:07.584788+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:35:07.584788+0000","last_undegraded":"2023-10-09T05:35:07.584788+0000","last_fullsized":"2023-10-09T05:35:07.584788+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"136'8985","reported_seq":9039,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:09.827237+0000","last_change":"2023-10-09T05:34:42.153027+0000","last_active":"2023-10-09T05:35:09.827237+0000","last_peered":"2023-10-09T05:35:09.827237+0000","last_clean":"2023-10-09T05:35:09.827237+0000","last_became_active":"2023-10-09T05:34:25.031927+0000","last_became_peered":"2023-10-09T05:34:25.031927+0000","last_unstale":"2023-10-09T05:35:09.827237+0000","last_undegraded":"2023-10-09T05:35:09.827237+0000","last_fullsized":"2023-10-09T05:35:09.827237+0000","mapping_epoch":105,"log_start":"109'800","ondisk_log_start":"109'800","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'8112","last_scrub_stamp":"2023-10-09T05:34:42.152946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:42.152946+0000","objects_scrubbed":0,"log_size":8185,"log_dups_size":0,"ondisk_log_size":8185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146407424,"num_objects":2234,"num_object_clones":0,"num_object_copies":6702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":873,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":122,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.572399+0000","last_change":"2023-10-09T05:34:41.666221+0000","last_active":"2023-10-09T05:35:07.572399+0000","last_peered":"2023-10-09T05:35:07.572399+0000","last_clean":"2023-10-09T05:35:07.572399+0000","last_became_active":"2023-10-09T05:34:41.665842+0000","last_became_peered":"2023-10-09T05:34:41.665842+0000","last_unstale":"2023-10-09T05:35:07.572399+0000","last_undegraded":"2023-10-09T05:35:07.572399+0000","last_fullsized":"2023-10-09T05:35:07.572399+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:28.435851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":80,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.608864+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:35:02.608864+0000","last_peered":"2023-10-09T05:35:02.608864+0000","last_clean":"2023-10-09T05:35:02.608864+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:35:02.608864+0000","last_undegraded":"2023-10-09T05:35:02.608864+0000","last_fullsized":"2023-10-09T05:35:02.608864+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":659,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.608130+0000","last_change":"2023-10-09T05:34:41.667358+0000","last_active":"2023-10-09T05:35:02.608130+0000","last_peered":"2023-10-09T05:35:02.608130+0000","last_clean":"2023-10-09T05:35:02.608130+0000","last_became_active":"2023-10-09T05:34:41.666757+0000","last_became_peered":"2023-10-09T05:34:41.666757+0000","last_unstale":"2023-10-09T05:35:02.608130+0000","last_undegraded":"2023-10-09T05:35:02.608130+0000","last_fullsized":"2023-10-09T05:35:02.608130+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:45:47.725111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":118,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.586568+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:35:07.586568+0000","last_peered":"2023-10-09T05:35:07.586568+0000","last_clean":"2023-10-09T05:35:07.586568+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:35:07.586568+0000","last_undegraded":"2023-10-09T05:35:07.586568+0000","last_fullsized":"2023-10-09T05:35:07.586568+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":65,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.608248+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:35:02.608248+0000","last_peered":"2023-10-09T05:35:02.608248+0000","last_clean":"2023-10-09T05:35:02.608248+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:35:02.608248+0000","last_undegraded":"2023-10-09T05:35:02.608248+0000","last_fullsized":"2023-10-09T05:35:02.608248+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"135'7384","reported_seq":7522,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.591819+0000","last_change":"2023-10-09T05:34:25.033028+0000","last_active":"2023-10-09T05:35:07.591819+0000","last_peered":"2023-10-09T05:35:07.591819+0000","last_clean":"2023-10-09T05:35:07.591819+0000","last_became_active":"2023-10-09T05:34:25.031183+0000","last_became_peered":"2023-10-09T05:34:25.031183+0000","last_unstale":"2023-10-09T05:35:07.591819+0000","last_undegraded":"2023-10-09T05:35:07.591819+0000","last_fullsized":"2023-10-09T05:35:07.591819+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'6300","last_scrub_stamp":"2023-10-09T05:34:08.749711+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:08.749711+0000","objects_scrubbed":3317,"log_size":7384,"log_dups_size":0,"ondisk_log_size":7384,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.61798832400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73957376,"num_objects":1128,"num_object_clones":0,"num_object_copies":3384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":896,"num_write_kb":11712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"135'8358","reported_seq":8485,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.675872+0000","last_change":"2023-10-09T05:34:10.842783+0000","last_active":"2023-10-09T05:35:07.675872+0000","last_peered":"2023-10-09T05:35:07.675872+0000","last_clean":"2023-10-09T05:35:07.675872+0000","last_became_active":"2023-10-09T05:34:01.606336+0000","last_became_peered":"2023-10-09T05:34:01.606336+0000","last_unstale":"2023-10-09T05:35:07.675872+0000","last_undegraded":"2023-10-09T05:35:07.675872+0000","last_fullsized":"2023-10-09T05:35:07.675872+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'6606","last_scrub_stamp":"2023-10-09T05:34:10.842550+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:10.842550+0000","objects_scrubbed":3389,"log_size":8358,"log_dups_size":0,"ondisk_log_size":8358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:40.600228+0000","scrub_duration":0.74463188000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146866176,"num_objects":2241,"num_object_clones":0,"num_object_copies":6723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1778,"num_write_kb":20160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":572,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.586071+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:35:07.586071+0000","last_peered":"2023-10-09T05:35:07.586071+0000","last_clean":"2023-10-09T05:35:07.586071+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:35:07.586071+0000","last_undegraded":"2023-10-09T05:35:07.586071+0000","last_fullsized":"2023-10-09T05:35:07.586071+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":103,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.582411+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:35:07.582411+0000","last_peered":"2023-10-09T05:35:07.582411+0000","last_clean":"2023-10-09T05:35:07.582411+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:35:07.582411+0000","last_undegraded":"2023-10-09T05:35:07.582411+0000","last_fullsized":"2023-10-09T05:35:07.582411+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":115,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.585791+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:35:07.585791+0000","last_peered":"2023-10-09T05:35:07.585791+0000","last_clean":"2023-10-09T05:35:07.585791+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:35:07.585791+0000","last_undegraded":"2023-10-09T05:35:07.585791+0000","last_fullsized":"2023-10-09T05:35:07.585791+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"136'8308","reported_seq":8383,"reported_epoch":136,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:10.841723+0000","last_change":"2023-10-09T05:34:42.959034+0000","last_active":"2023-10-09T05:35:10.841723+0000","last_peered":"2023-10-09T05:35:10.841723+0000","last_clean":"2023-10-09T05:33:20.529830+0000","last_became_active":"2023-10-09T05:34:42.879241+0000","last_became_peered":"2023-10-09T05:34:42.879241+0000","last_unstale":"2023-10-09T05:35:10.841723+0000","last_undegraded":"2023-10-09T05:34:42.688699+0000","last_fullsized":"2023-10-09T05:34:42.658753+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8308,"log_dups_size":0,"ondisk_log_size":8308,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140902400,"num_objects":2150,"num_object_clones":0,"num_object_copies":6450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5240,"num_write_kb":236480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":155}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":448,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:08.587597+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:35:08.587597+0000","last_peered":"2023-10-09T05:35:08.587597+0000","last_clean":"2023-10-09T05:35:08.587597+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:35:08.587597+0000","last_undegraded":"2023-10-09T05:35:08.587597+0000","last_fullsized":"2023-10-09T05:35:08.587597+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":80,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.609332+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:35:02.609332+0000","last_peered":"2023-10-09T05:35:02.609332+0000","last_clean":"2023-10-09T05:35:02.609332+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:35:02.609332+0000","last_undegraded":"2023-10-09T05:35:02.609332+0000","last_fullsized":"2023-10-09T05:35:02.609332+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.608742+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:35:02.608742+0000","last_peered":"2023-10-09T05:35:02.608742+0000","last_clean":"2023-10-09T05:35:02.608742+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:35:02.608742+0000","last_undegraded":"2023-10-09T05:35:02.608742+0000","last_fullsized":"2023-10-09T05:35:02.608742+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":712,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.585876+0000","last_change":"2023-10-09T05:34:41.677976+0000","last_active":"2023-10-09T05:35:07.585876+0000","last_peered":"2023-10-09T05:35:07.585876+0000","last_clean":"2023-10-09T05:35:07.585876+0000","last_became_active":"2023-10-09T05:34:41.677520+0000","last_became_peered":"2023-10-09T05:34:41.677520+0000","last_unstale":"2023-10-09T05:35:07.585876+0000","last_undegraded":"2023-10-09T05:35:07.585876+0000","last_fullsized":"2023-10-09T05:35:07.585876+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":313,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.587849+0000","last_change":"2023-10-09T05:34:41.686013+0000","last_active":"2023-10-09T05:35:07.587849+0000","last_peered":"2023-10-09T05:35:07.587849+0000","last_clean":"2023-10-09T05:35:07.587849+0000","last_became_active":"2023-10-09T05:34:41.680809+0000","last_became_peered":"2023-10-09T05:34:41.680809+0000","last_unstale":"2023-10-09T05:35:07.587849+0000","last_undegraded":"2023-10-09T05:35:07.587849+0000","last_fullsized":"2023-10-09T05:35:07.587849+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":97,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.572682+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:35:07.572682+0000","last_peered":"2023-10-09T05:35:07.572682+0000","last_clean":"2023-10-09T05:35:07.572682+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:35:07.572682+0000","last_undegraded":"2023-10-09T05:35:07.572682+0000","last_fullsized":"2023-10-09T05:35:07.572682+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.590311+0000","last_change":"2023-10-09T05:34:41.693650+0000","last_active":"2023-10-09T05:35:07.590311+0000","last_peered":"2023-10-09T05:35:07.590311+0000","last_clean":"2023-10-09T05:35:07.590311+0000","last_became_active":"2023-10-09T05:34:41.693305+0000","last_became_peered":"2023-10-09T05:34:41.693305+0000","last_unstale":"2023-10-09T05:35:07.590311+0000","last_undegraded":"2023-10-09T05:35:07.590311+0000","last_fullsized":"2023-10-09T05:35:07.590311+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"136'8230","reported_seq":8289,"reported_epoch":136,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:10.836992+0000","last_change":"2023-10-09T05:34:42.906080+0000","last_active":"2023-10-09T05:35:10.836992+0000","last_peered":"2023-10-09T05:35:10.836992+0000","last_clean":"2023-10-09T05:33:20.493128+0000","last_became_active":"2023-10-09T05:34:42.894560+0000","last_became_peered":"2023-10-09T05:34:42.894560+0000","last_unstale":"2023-10-09T05:35:10.836992+0000","last_undegraded":"2023-10-09T05:34:42.694084+0000","last_fullsized":"2023-10-09T05:34:42.658067+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8230,"log_dups_size":0,"ondisk_log_size":8230,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136740864,"num_objects":2086,"num_object_clones":0,"num_object_copies":6258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2624,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5200,"num_write_kb":233184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":128}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.7","version":"136'8206","reported_seq":8268,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:09.843007+0000","last_change":"2023-10-09T05:34:34.976158+0000","last_active":"2023-10-09T05:35:09.843007+0000","last_peered":"2023-10-09T05:35:09.843007+0000","last_clean":"2023-10-09T05:35:09.843007+0000","last_became_active":"2023-10-09T05:34:01.592987+0000","last_became_peered":"2023-10-09T05:34:01.592987+0000","last_unstale":"2023-10-09T05:35:09.843007+0000","last_undegraded":"2023-10-09T05:35:09.843007+0000","last_fullsized":"2023-10-09T05:35:09.843007+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7154","last_scrub_stamp":"2023-10-09T05:34:34.976067+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:34.976067+0000","objects_scrubbed":0,"log_size":8206,"log_dups_size":0,"ondisk_log_size":8206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146079744,"num_objects":2229,"num_object_clones":0,"num_object_copies":6687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1078,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":736,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.587729+0000","last_change":"2023-10-09T05:34:41.668526+0000","last_active":"2023-10-09T05:35:07.587729+0000","last_peered":"2023-10-09T05:35:07.587729+0000","last_clean":"2023-10-09T05:35:07.587729+0000","last_became_active":"2023-10-09T05:34:41.668010+0000","last_became_peered":"2023-10-09T05:34:41.668010+0000","last_unstale":"2023-10-09T05:35:07.587729+0000","last_undegraded":"2023-10-09T05:35:07.587729+0000","last_fullsized":"2023-10-09T05:35:07.587729+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":85,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.623806+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:35:02.623806+0000","last_peered":"2023-10-09T05:35:02.623806+0000","last_clean":"2023-10-09T05:35:02.623806+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:35:02.623806+0000","last_undegraded":"2023-10-09T05:35:02.623806+0000","last_fullsized":"2023-10-09T05:35:02.623806+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":100,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.624419+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:35:02.624419+0000","last_peered":"2023-10-09T05:35:02.624419+0000","last_clean":"2023-10-09T05:35:02.624419+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:35:02.624419+0000","last_undegraded":"2023-10-09T05:35:02.624419+0000","last_fullsized":"2023-10-09T05:35:02.624419+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":339,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.589517+0000","last_change":"2023-10-09T05:34:41.682539+0000","last_active":"2023-10-09T05:35:07.589517+0000","last_peered":"2023-10-09T05:35:07.589517+0000","last_clean":"2023-10-09T05:35:07.589517+0000","last_became_active":"2023-10-09T05:34:41.674099+0000","last_became_peered":"2023-10-09T05:34:41.674099+0000","last_unstale":"2023-10-09T05:35:07.589517+0000","last_undegraded":"2023-10-09T05:35:07.589517+0000","last_fullsized":"2023-10-09T05:35:07.589517+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":80,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.608473+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:35:02.608473+0000","last_peered":"2023-10-09T05:35:02.608473+0000","last_clean":"2023-10-09T05:35:02.608473+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:35:02.608473+0000","last_undegraded":"2023-10-09T05:35:02.608473+0000","last_fullsized":"2023-10-09T05:35:02.608473+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":85,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.626647+0000","last_change":"2023-10-09T05:34:41.671414+0000","last_active":"2023-10-09T05:35:02.626647+0000","last_peered":"2023-10-09T05:35:02.626647+0000","last_clean":"2023-10-09T05:35:02.626647+0000","last_became_active":"2023-10-09T05:34:41.671100+0000","last_became_peered":"2023-10-09T05:34:41.671100+0000","last_unstale":"2023-10-09T05:35:02.626647+0000","last_undegraded":"2023-10-09T05:35:02.626647+0000","last_fullsized":"2023-10-09T05:35:02.626647+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:19.832596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"136'7485","reported_seq":7554,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:09.763879+0000","last_change":"2023-10-09T05:34:33.321883+0000","last_active":"2023-10-09T05:35:09.763879+0000","last_peered":"2023-10-09T05:35:09.763879+0000","last_clean":"2023-10-09T05:35:09.763879+0000","last_became_active":"2023-10-09T05:34:25.017164+0000","last_became_peered":"2023-10-09T05:34:25.017164+0000","last_unstale":"2023-10-09T05:35:09.763879+0000","last_undegraded":"2023-10-09T05:35:09.763879+0000","last_fullsized":"2023-10-09T05:35:09.763879+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'6928","last_scrub_stamp":"2023-10-09T05:34:33.321793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:33.321793+0000","objects_scrubbed":0,"log_size":7485,"log_dups_size":0,"ondisk_log_size":7485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72024064,"num_objects":1099,"num_object_clones":0,"num_object_copies":3297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":568,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":119,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.573480+0000","last_change":"2023-10-09T05:34:41.664569+0000","last_active":"2023-10-09T05:35:07.573480+0000","last_peered":"2023-10-09T05:35:07.573480+0000","last_clean":"2023-10-09T05:35:07.573480+0000","last_became_active":"2023-10-09T05:34:41.664239+0000","last_became_peered":"2023-10-09T05:34:41.664239+0000","last_unstale":"2023-10-09T05:35:07.573480+0000","last_undegraded":"2023-10-09T05:35:07.573480+0000","last_fullsized":"2023-10-09T05:35:07.573480+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:26.511248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":521,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.608464+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:35:02.608464+0000","last_peered":"2023-10-09T05:35:02.608464+0000","last_clean":"2023-10-09T05:35:02.608464+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:35:02.608464+0000","last_undegraded":"2023-10-09T05:35:02.608464+0000","last_fullsized":"2023-10-09T05:35:02.608464+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.572787+0000","last_change":"2023-10-09T05:34:41.666200+0000","last_active":"2023-10-09T05:35:07.572787+0000","last_peered":"2023-10-09T05:35:07.572787+0000","last_clean":"2023-10-09T05:35:07.572787+0000","last_became_active":"2023-10-09T05:34:41.665841+0000","last_became_peered":"2023-10-09T05:34:41.665841+0000","last_unstale":"2023-10-09T05:35:07.572787+0000","last_undegraded":"2023-10-09T05:35:07.572787+0000","last_fullsized":"2023-10-09T05:35:07.572787+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:58.338181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"136'7442","reported_seq":10141,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:09.817277+0000","last_change":"2023-10-09T05:34:57.102099+0000","last_active":"2023-10-09T05:35:09.817277+0000","last_peered":"2023-10-09T05:35:09.817277+0000","last_clean":"2023-10-09T05:35:09.817277+0000","last_became_active":"2023-10-09T05:34:57.096500+0000","last_became_peered":"2023-10-09T05:34:57.096500+0000","last_unstale":"2023-10-09T05:35:09.817277+0000","last_undegraded":"2023-10-09T05:35:09.817277+0000","last_fullsized":"2023-10-09T05:35:09.817277+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":7442,"log_dups_size":0,"ondisk_log_size":7442,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71942144,"num_objects":1097,"num_object_clones":0,"num_object_copies":3291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2572,"num_write_kb":117456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1310,"num_bytes_recovered":65667072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":599,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.590800+0000","last_change":"2023-10-09T05:34:41.676842+0000","last_active":"2023-10-09T05:35:07.590800+0000","last_peered":"2023-10-09T05:35:07.590800+0000","last_clean":"2023-10-09T05:35:07.590800+0000","last_became_active":"2023-10-09T05:34:41.676428+0000","last_became_peered":"2023-10-09T05:34:41.676428+0000","last_unstale":"2023-10-09T05:35:07.590800+0000","last_undegraded":"2023-10-09T05:35:07.590800+0000","last_fullsized":"2023-10-09T05:35:07.590800+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":88,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.624339+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:35:02.624339+0000","last_peered":"2023-10-09T05:35:02.624339+0000","last_clean":"2023-10-09T05:35:02.624339+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:35:02.624339+0000","last_undegraded":"2023-10-09T05:35:02.624339+0000","last_fullsized":"2023-10-09T05:35:02.624339+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":87,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.585766+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:35:07.585766+0000","last_peered":"2023-10-09T05:35:07.585766+0000","last_clean":"2023-10-09T05:35:07.585766+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:35:07.585766+0000","last_undegraded":"2023-10-09T05:35:07.585766+0000","last_fullsized":"2023-10-09T05:35:07.585766+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"136'7480","reported_seq":7658,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:10.856827+0000","last_change":"2023-10-09T05:34:33.714772+0000","last_active":"2023-10-09T05:35:10.856827+0000","last_peered":"2023-10-09T05:35:10.856827+0000","last_clean":"2023-10-09T05:35:10.856827+0000","last_became_active":"2023-10-09T05:34:25.048537+0000","last_became_peered":"2023-10-09T05:34:25.048537+0000","last_unstale":"2023-10-09T05:35:10.856827+0000","last_undegraded":"2023-10-09T05:35:10.856827+0000","last_fullsized":"2023-10-09T05:35:10.856827+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'6879","last_scrub_stamp":"2023-10-09T05:34:33.714537+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:33.714537+0000","objects_scrubbed":1680,"log_size":7480,"log_dups_size":0,"ondisk_log_size":7480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:15.987881+0000","scrub_duration":0.49573348099999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70451200,"num_objects":1075,"num_object_clones":0,"num_object_copies":3225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":605,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1d","version":"136'9006","reported_seq":9068,"reported_epoch":136,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:10.812840+0000","last_change":"2023-10-09T05:34:42.908183+0000","last_active":"2023-10-09T05:35:10.812840+0000","last_peered":"2023-10-09T05:35:10.812840+0000","last_clean":"2023-10-09T05:33:20.502869+0000","last_became_active":"2023-10-09T05:34:42.896545+0000","last_became_peered":"2023-10-09T05:34:42.896545+0000","last_unstale":"2023-10-09T05:35:10.812840+0000","last_undegraded":"2023-10-09T05:34:42.687614+0000","last_fullsized":"2023-10-09T05:34:42.658151+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9006,"log_dups_size":0,"ondisk_log_size":9006,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142082060,"num_objects":2169,"num_object_clones":0,"num_object_copies":6507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2695,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2169,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5264,"num_write_kb":237793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":148}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":88,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.626575+0000","last_change":"2023-10-09T05:34:41.662989+0000","last_active":"2023-10-09T05:35:02.626575+0000","last_peered":"2023-10-09T05:35:02.626575+0000","last_clean":"2023-10-09T05:35:02.626575+0000","last_became_active":"2023-10-09T05:34:41.662621+0000","last_became_peered":"2023-10-09T05:34:41.662621+0000","last_unstale":"2023-10-09T05:35:02.626575+0000","last_undegraded":"2023-10-09T05:35:02.626575+0000","last_fullsized":"2023-10-09T05:35:02.626575+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:08.750044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":125,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.583048+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:35:07.583048+0000","last_peered":"2023-10-09T05:35:07.583048+0000","last_clean":"2023-10-09T05:35:07.583048+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:35:07.583048+0000","last_undegraded":"2023-10-09T05:35:07.583048+0000","last_fullsized":"2023-10-09T05:35:07.583048+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.1a","version":"136'8872","reported_seq":8922,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:10.743322+0000","last_change":"2023-10-09T05:34:25.055227+0000","last_active":"2023-10-09T05:35:10.743322+0000","last_peered":"2023-10-09T05:35:10.743322+0000","last_clean":"2023-10-09T05:35:10.743322+0000","last_became_active":"2023-10-09T05:34:25.051761+0000","last_became_peered":"2023-10-09T05:34:25.051761+0000","last_unstale":"2023-10-09T05:35:10.743322+0000","last_undegraded":"2023-10-09T05:35:10.743322+0000","last_fullsized":"2023-10-09T05:35:10.743322+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8872,"log_dups_size":0,"ondisk_log_size":8872,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146046976,"num_objects":2229,"num_object_clones":0,"num_object_copies":6687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5154,"num_write_kb":236224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":105,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.586282+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:35:07.586282+0000","last_peered":"2023-10-09T05:35:07.586282+0000","last_clean":"2023-10-09T05:35:07.586282+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:35:07.586282+0000","last_undegraded":"2023-10-09T05:35:07.586282+0000","last_fullsized":"2023-10-09T05:35:07.586282+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":117,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.582482+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:35:07.582482+0000","last_peered":"2023-10-09T05:35:07.582482+0000","last_clean":"2023-10-09T05:35:07.582482+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:35:07.582482+0000","last_undegraded":"2023-10-09T05:35:07.582482+0000","last_fullsized":"2023-10-09T05:35:07.582482+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1b","version":"136'8962","reported_seq":9012,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:10.906880+0000","last_change":"2023-10-09T05:34:25.065203+0000","last_active":"2023-10-09T05:35:10.906880+0000","last_peered":"2023-10-09T05:35:10.906880+0000","last_clean":"2023-10-09T05:35:10.906880+0000","last_became_active":"2023-10-09T05:34:25.059858+0000","last_became_peered":"2023-10-09T05:34:25.059858+0000","last_unstale":"2023-10-09T05:35:10.906880+0000","last_undegraded":"2023-10-09T05:35:10.906880+0000","last_fullsized":"2023-10-09T05:35:10.906880+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8962,"log_dups_size":0,"ondisk_log_size":8962,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144244736,"num_objects":2201,"num_object_clones":0,"num_object_copies":6603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5224,"num_write_kb":237600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":90,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.586656+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:35:07.586656+0000","last_peered":"2023-10-09T05:35:07.586656+0000","last_clean":"2023-10-09T05:35:07.586656+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:35:07.586656+0000","last_undegraded":"2023-10-09T05:35:07.586656+0000","last_fullsized":"2023-10-09T05:35:07.586656+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":117,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.582922+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:35:07.582922+0000","last_peered":"2023-10-09T05:35:07.582922+0000","last_clean":"2023-10-09T05:35:07.582922+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:35:07.582922+0000","last_undegraded":"2023-10-09T05:35:07.582922+0000","last_fullsized":"2023-10-09T05:35:07.582922+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"136'8407","reported_seq":8538,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:35:10.871550+0000","last_change":"2023-10-09T05:34:07.818690+0000","last_active":"2023-10-09T05:35:10.871550+0000","last_peered":"2023-10-09T05:35:10.871550+0000","last_clean":"2023-10-09T05:35:10.871550+0000","last_became_active":"2023-10-09T05:34:01.656619+0000","last_became_peered":"2023-10-09T05:34:01.656619+0000","last_unstale":"2023-10-09T05:35:10.871550+0000","last_undegraded":"2023-10-09T05:35:10.871550+0000","last_fullsized":"2023-10-09T05:35:10.871550+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6429","last_scrub_stamp":"2023-10-09T05:34:07.818377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:07.818377+0000","objects_scrubbed":3381,"log_size":8407,"log_dups_size":0,"ondisk_log_size":8407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:04.556045+0000","scrub_duration":0.90924191099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144113664,"num_objects":2199,"num_object_clones":0,"num_object_copies":6597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1994,"num_write_kb":25984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":112,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.573179+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:35:07.573179+0000","last_peered":"2023-10-09T05:35:07.573179+0000","last_clean":"2023-10-09T05:35:07.573179+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:35:07.573179+0000","last_undegraded":"2023-10-09T05:35:07.573179+0000","last_fullsized":"2023-10-09T05:35:07.573179+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":125,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.573763+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:35:07.573763+0000","last_peered":"2023-10-09T05:35:07.573763+0000","last_clean":"2023-10-09T05:35:07.573763+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:35:07.573763+0000","last_undegraded":"2023-10-09T05:35:07.573763+0000","last_fullsized":"2023-10-09T05:35:07.573763+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"136'8194","reported_seq":8252,"reported_epoch":136,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:09.835166+0000","last_change":"2023-10-09T05:34:42.888623+0000","last_active":"2023-10-09T05:35:09.835166+0000","last_peered":"2023-10-09T05:35:09.835166+0000","last_clean":"2023-10-09T05:33:20.515228+0000","last_became_active":"2023-10-09T05:34:42.874596+0000","last_became_peered":"2023-10-09T05:34:42.874596+0000","last_unstale":"2023-10-09T05:35:09.835166+0000","last_undegraded":"2023-10-09T05:34:42.676976+0000","last_fullsized":"2023-10-09T05:34:42.659894+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8194,"log_dups_size":0,"ondisk_log_size":8194,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143392768,"num_objects":2188,"num_object_clones":0,"num_object_copies":6564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2597,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5134,"num_write_kb":234304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":228}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":88,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T05:35:02.623443+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:35:02.623443+0000","last_peered":"2023-10-09T05:35:02.623443+0000","last_clean":"2023-10-09T05:35:02.623443+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:35:02.623443+0000","last_undegraded":"2023-10-09T05:35:02.623443+0000","last_fullsized":"2023-10-09T05:35:02.623443+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T05:35:07.582078+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:35:07.582078+0000","last_peered":"2023-10-09T05:35:07.582078+0000","last_clean":"2023-10-09T05:35:07.582078+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:35:07.582078+0000","last_undegraded":"2023-10-09T05:35:07.582078+0000","last_fullsized":"2023-10-09T05:35:07.582078+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":36,"stat_sum":{"num_bytes":4608786456,"num_objects":70326,"num_object_clones":0,"num_object_copies":210978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22651,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":70326,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":115754,"num_write_kb":4609889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2903,"num_bytes_recovered":148897792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13119004672,"data_stored":13118996528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":299431,"ondisk_log_size":299431,"up":108,"acting":99,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846599,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":962160,"kb_used_data":647848,"kb_used_omap":14,"kb_used_meta":314289,"kb_avail":92782992,"statfs":{"total":95995035648,"available":95009783808,"internally_reserved":0,"allocated":663396352,"data_stored":662882653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":321832794},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.7689999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":5.3090000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":6.0030000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.8500000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":9.6699999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.0699999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.1120000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626625,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1434916,"kb_used_data":1133596,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92310236,"statfs":{"total":95995035648,"available":94525681664,"internally_reserved":0,"allocated":1160802304,"data_stored":1160300501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:34:09 2023","interfaces":[{"interface":"back","average":{"1min":4.9169999999999998,"5min":1.431,"15min":0.83699999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":53.771999999999998,"5min":53.771999999999998,"15min":53.771999999999998},"last":0.22700000000000001},{"interface":"front","average":{"1min":4.4569999999999999,"5min":1.4350000000000001,"15min":0.84099999999999997},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":53.411999999999999,"5min":53.411999999999999,"15min":53.411999999999999},"last":1.7330000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.2250000000000001,"15min":0.77700000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.9539999999999997,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":0.74299999999999999},{"interface":"front","average":{"1min":2.097,"5min":1.244,"15min":0.79900000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.9690000000000003,"5min":8.7189999999999994,"15min":8.7189999999999994},"last":0.28000000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:34:09 2023","interfaces":[{"interface":"back","average":{"1min":4.5190000000000001,"5min":1.9319999999999999,"15min":1.3600000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":34.393999999999998,"5min":34.393999999999998,"15min":34.393999999999998},"last":0.88100000000000001},{"interface":"front","average":{"1min":4.6689999999999996,"5min":1.974,"15min":1.387},"min":{"1min":0.41499999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":34.381999999999998,"5min":34.381999999999998,"15min":34.381999999999998},"last":0.873}]},{"osd":3,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":1.4490000000000001,"15min":1.1930000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.0250000000000004,"5min":18.361000000000001,"15min":18.361000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.409,"5min":1.4199999999999999,"15min":1.222},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.5030000000000001,"5min":18.536000000000001,"15min":18.536000000000001},"last":0.32800000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:34:50 2023","interfaces":[{"interface":"back","average":{"1min":9.9870000000000001,"5min":2.512,"15min":1.145},"min":{"1min":0.27000000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":125.64,"5min":125.64,"15min":125.64},"last":0.30399999999999999},{"interface":"front","average":{"1min":10.308999999999999,"5min":2.5489999999999999,"15min":1.1679999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27800000000000002},"max":{"1min":125.625,"5min":125.625,"15min":125.625},"last":1.1539999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:34:50 2023","interfaces":[{"interface":"back","average":{"1min":8.0860000000000003,"5min":2.145,"15min":1.0509999999999999},"min":{"1min":0.29799999999999999,"5min":0.27900000000000003,"15min":0.27200000000000002},"max":{"1min":127.35899999999999,"5min":127.35899999999999,"15min":127.35899999999999},"last":0.84999999999999998},{"interface":"front","average":{"1min":8.4570000000000007,"5min":2.2480000000000002,"15min":1.093},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":128.93799999999999,"5min":128.93799999999999,"15min":128.93799999999999},"last":1.214}]},{"osd":7,"last update":"Mon Oct 9 05:33:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.56999999999999995,"15min":0.54200000000000004},"min":{"1min":0.32900000000000001,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":1.9299999999999999,"5min":1.9299999999999999,"15min":1.9299999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.63100000000000001,"15min":0.58199999999999996},"min":{"1min":0.48399999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4330000000000001,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.1890000000000001}]}]},{"osd":1,"up_from":93,"seq":399431958586,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1567372,"kb_used_data":1459396,"kb_used_omap":7,"kb_used_meta":107960,"kb_avail":92177780,"statfs":{"total":95995035648,"available":94390046720,"internally_reserved":0,"allocated":1494421504,"data_stored":1493820320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":110551566},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":0.77900000000000003,"15min":0.52800000000000002},"min":{"1min":0.191,"5min":0.185,"15min":0.185},"max":{"1min":17.117000000000001,"5min":17.117000000000001,"15min":17.117000000000001},"last":0.73999999999999999},{"interface":"front","average":{"1min":2.2189999999999999,"5min":0.90400000000000003,"15min":0.57299999999999995},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":17.097000000000001,"5min":17.097000000000001,"15min":17.097000000000001},"last":3.8570000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:35:07 2023","interfaces":[{"interface":"back","average":{"1min":7.3869999999999996,"5min":2.198,"15min":0.95999999999999996},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":98.483999999999995,"5min":98.483999999999995,"15min":98.483999999999995},"last":4.5709999999999997},{"interface":"front","average":{"1min":6.1189999999999998,"5min":1.8600000000000001,"15min":0.89600000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":98.409999999999997,"5min":98.409999999999997,"15min":98.409999999999997},"last":0.27100000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":0.89800000000000002,"15min":0.61799999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.0500000000000007,"5min":8.0500000000000007,"15min":8.0500000000000007},"last":3.5449999999999999},{"interface":"front","average":{"1min":1.2390000000000001,"5min":0.72999999999999998,"15min":0.51200000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.6879999999999997,"5min":4.6879999999999997,"15min":4.6879999999999997},"last":0.77100000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3010000000000002,"5min":1.0780000000000001,"15min":0.78800000000000003},"min":{"1min":0.40899999999999997,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":23.670999999999999,"5min":23.670999999999999,"15min":23.670999999999999},"last":3.7549999999999999},{"interface":"front","average":{"1min":2.202,"5min":0.995,"15min":0.77200000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":23.620000000000001,"5min":23.620000000000001,"15min":23.620000000000001},"last":0.33800000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2.9169999999999998,"5min":1.169,"15min":0.81100000000000005},"min":{"1min":0.38900000000000001,"5min":0.374,"15min":0.374},"max":{"1min":23.501999999999999,"5min":23.501999999999999,"15min":23.501999999999999},"last":3.3599999999999999},{"interface":"front","average":{"1min":3.048,"5min":1.4239999999999999,"15min":0.92600000000000005},"min":{"1min":0.434,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":23.965,"5min":25.68,"15min":25.68},"last":3.8239999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":1.0189999999999999,"15min":0.77600000000000002},"min":{"1min":0.45600000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":8.1270000000000007,"5min":8.1270000000000007,"15min":8.1270000000000007},"last":4.3819999999999997},{"interface":"front","average":{"1min":1.849,"5min":0.97599999999999998,"15min":0.78200000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":5.8140000000000001,"5min":5.8140000000000001,"15min":5.8140000000000001},"last":3.7930000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.69699999999999995,"15min":0.70799999999999996},"min":{"1min":0.47299999999999998,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.77000000000000002,"5min":1.014,"15min":1.014},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.68400000000000005,"15min":0.69299999999999995},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.82799999999999996,"5min":1,"15min":1},"last":0.67300000000000004}]}]},{"osd":0,"up_from":88,"seq":377957122112,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2493432,"kb_used_data":2349360,"kb_used_omap":8,"kb_used_meta":144055,"kb_avail":91251720,"statfs":{"total":95995035648,"available":93441761280,"internally_reserved":0,"allocated":2405744640,"data_stored":2405153979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":147513096},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:34:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2570000000000001,"5min":0.89400000000000002,"15min":0.56200000000000006},"min":{"1min":0.20899999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":13.919,"5min":13.919,"15min":13.919},"last":0.23400000000000001},{"interface":"front","average":{"1min":2.2719999999999998,"5min":0.88700000000000001,"15min":0.58899999999999997},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.9350000000000005,"5min":9.9350000000000005,"15min":9.9350000000000005},"last":0.32500000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:35:04 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.845,"15min":0.85299999999999998},"min":{"1min":0.22700000000000001,"5min":0.214,"15min":0.214},"max":{"1min":6.4640000000000004,"5min":57.222999999999999,"15min":57.222999999999999},"last":2.2029999999999998},{"interface":"front","average":{"1min":1.71,"5min":1.6830000000000001,"15min":0.81000000000000005},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":11.693,"5min":57.408000000000001,"15min":57.408000000000001},"last":0.53400000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":1.01,"15min":0.60399999999999998},"min":{"1min":0.32100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":12.429,"5min":12.429,"15min":12.429},"last":0.36199999999999999},{"interface":"front","average":{"1min":3.1659999999999999,"5min":1.179,"15min":0.66000000000000003},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":22.390999999999998,"5min":22.390999999999998,"15min":22.390999999999998},"last":0.30499999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:35:04 2023","interfaces":[{"interface":"back","average":{"1min":2.371,"5min":1.0920000000000001,"15min":0.81799999999999995},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":14.023,"5min":14.023,"15min":14.023},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.7130000000000001,"5min":1.282,"15min":0.90800000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":14.096,"5min":14.096,"15min":14.096},"last":0.437}]},{"osd":5,"last update":"Mon Oct 9 05:35:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5099999999999998,"5min":1.258,"15min":0.873},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":13.632,"5min":13.632,"15min":13.632},"last":0.99399999999999999},{"interface":"front","average":{"1min":2.8620000000000001,"5min":1.3129999999999999,"15min":0.88800000000000001},"min":{"1min":0.36699999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":13.676,"5min":13.676,"15min":13.676},"last":0.45800000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:35:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":1.2669999999999999,"15min":0.89300000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.0679999999999996,"5min":9.9809999999999999,"15min":9.9809999999999999},"last":1.8979999999999999},{"interface":"front","average":{"1min":2.73,"5min":1.3049999999999999,"15min":0.88700000000000001},"min":{"1min":0.42199999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.5259999999999998,"5min":6.8460000000000001,"15min":6.8460000000000001},"last":0.42199999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:33:02 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.71199999999999997,"15min":0.70999999999999996},"min":{"1min":0.51300000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":2.8450000000000002,"5min":2.8450000000000002,"15min":2.8450000000000002},"last":0.505},{"interface":"front","average":{"1min":0.755,"5min":0.71599999999999997,"15min":0.72899999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.448,"5min":2.448,"15min":2.448},"last":0.51700000000000002}]}]},{"osd":2,"up_from":98,"seq":420906795060,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1825364,"kb_used_data":1677132,"kb_used_omap":11,"kb_used_meta":148084,"kb_avail":91919788,"statfs":{"total":95995035648,"available":94125862912,"internally_reserved":0,"allocated":1717383168,"data_stored":1716780826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":151638613},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":2.9990000000000001,"5min":1.173,"15min":0.628},"min":{"1min":0.19,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":32.122,"5min":32.122,"15min":32.122},"last":2.7709999999999999},{"interface":"front","average":{"1min":2.577,"5min":1.109,"15min":0.63300000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":34.066000000000003,"5min":34.066000000000003,"15min":34.066000000000003},"last":4.2779999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":5.7160000000000002,"5min":1.7250000000000001,"15min":0.82899999999999996},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":98.459999999999994,"5min":98.459999999999994,"15min":98.459999999999994},"last":2.3119999999999998},{"interface":"front","average":{"1min":5.9119999999999999,"5min":1.647,"15min":0.80300000000000005},"min":{"1min":0.222,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":98.397000000000006,"5min":98.397000000000006,"15min":98.397000000000006},"last":0.27300000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":3.3450000000000002,"5min":1.0469999999999999,"15min":0.59899999999999998},"min":{"1min":0.24399999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":34.052999999999997,"5min":34.052999999999997,"15min":34.052999999999997},"last":0.249},{"interface":"front","average":{"1min":3.1960000000000002,"5min":1.153,"15min":0.65100000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":35.161999999999999,"5min":35.161999999999999,"15min":35.161999999999999},"last":2.8109999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":0.96699999999999997,"15min":0.70499999999999996},"min":{"1min":0.32100000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.1779999999999999,"5min":4.4859999999999998,"15min":4.4859999999999998},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.0209999999999999,"15min":0.76800000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.0060000000000002,"5min":9.0060000000000002,"15min":9.0060000000000002},"last":4.5529999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":1.161,"15min":0.81499999999999995},"min":{"1min":0.36699999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.8730000000000002,"5min":6.8730000000000002,"15min":6.8730000000000002},"last":0.376},{"interface":"front","average":{"1min":1.3420000000000001,"5min":0.98599999999999999,"15min":0.73299999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.2519999999999998,"5min":5.2519999999999998,"15min":5.2519999999999998},"last":0.49399999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.117,"15min":0.78900000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":11.496,"5min":11.496,"15min":11.496},"last":2.839},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.0680000000000001,"15min":0.81299999999999994},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.4049999999999994,"5min":9.4049999999999994,"15min":9.4049999999999994},"last":2.2730000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.68000000000000005,"15min":0.66800000000000004},"min":{"1min":0.46100000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":2.2959999999999998,"5min":2.2959999999999998,"15min":2.2959999999999998},"last":2.8620000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.65400000000000003,"15min":0.65700000000000003},"min":{"1min":0.438,"5min":0.435,"15min":0.435},"max":{"1min":1.2190000000000001,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":0.41599999999999998}]}]},{"osd":3,"up_from":103,"seq":442381631534,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905852,"kb_used_data":842292,"kb_used_omap":4,"kb_used_meta":63547,"kb_avail":92839300,"statfs":{"total":95995035648,"available":95067443200,"internally_reserved":0,"allocated":862507008,"data_stored":861926453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":65072734},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":4.9470000000000001,"5min":1.4139999999999999,"15min":0.73699999999999999},"min":{"1min":0.26000000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":56.094000000000001,"5min":56.094000000000001,"15min":56.094000000000001},"last":0.28299999999999997},{"interface":"front","average":{"1min":5.6289999999999996,"5min":1.593,"15min":0.78100000000000003},"min":{"1min":0.26800000000000002,"5min":0.214,"15min":0.214},"max":{"1min":57.255000000000003,"5min":57.255000000000003,"15min":57.255000000000003},"last":0.30399999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":1.5649999999999999,"5min":0.67500000000000004,"15min":0.49399999999999999},"min":{"1min":0.26000000000000001,"5min":0.255,"15min":0.255},"max":{"1min":4.4619999999999997,"5min":4.4619999999999997,"15min":4.4619999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.732,"5min":0.66700000000000004,"15min":0.47299999999999998},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":6.8769999999999998,"5min":6.8769999999999998,"15min":6.8769999999999998},"last":0.32600000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":5.5730000000000004,"5min":2.77,"15min":1.208},"min":{"1min":0.32400000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":50.411000000000001,"5min":117.23,"15min":117.23},"last":0.42499999999999999},{"interface":"front","average":{"1min":5.0229999999999997,"5min":2.7989999999999999,"15min":1.226},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":50.944000000000003,"5min":117.45999999999999,"15min":117.45999999999999},"last":0.48999999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":3.746,"5min":1.6659999999999999,"15min":0.95399999999999996},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":45.779000000000003,"5min":45.779000000000003,"15min":45.779000000000003},"last":0.53200000000000003},{"interface":"front","average":{"1min":4.3079999999999998,"5min":1.4590000000000001,"15min":0.91800000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":45.735999999999997,"5min":45.735999999999997,"15min":45.735999999999997},"last":0.54500000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":3.5379999999999998,"5min":1.294,"15min":0.85899999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":47.948999999999998,"5min":47.948999999999998,"15min":47.948999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":4.2000000000000002,"5min":1.4450000000000001,"15min":0.93400000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":47.968000000000004,"5min":47.968000000000004,"15min":47.968000000000004},"last":0.56899999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":1.079,"15min":0.80700000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.484,"5min":8.484,"15min":8.484},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.66700000000000004,"15min":0.63700000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.488,"5min":2.488,"15min":2.488},"last":0.44500000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.623},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.65700000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019081,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2567592,"kb_used_data":2266016,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91177560,"statfs":{"total":95995035648,"available":93365821440,"internally_reserved":0,"allocated":2320400384,"data_stored":2319894263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1,0,1,0,1,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:07 2023","interfaces":[{"interface":"back","average":{"1min":3.3109999999999999,"5min":1.5049999999999999,"15min":0.84899999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":33.488999999999997,"5min":33.488999999999997,"15min":33.488999999999997},"last":0.27900000000000003},{"interface":"front","average":{"1min":3.569,"5min":1.631,"15min":0.91200000000000003},"min":{"1min":0.27300000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":33.616999999999997,"5min":33.616999999999997,"15min":33.616999999999997},"last":3.2050000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:34:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0419999999999998,"5min":1.073,"15min":0.71499999999999997},"min":{"1min":0.313,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":31.073,"5min":31.073,"15min":31.073},"last":2.867},{"interface":"front","average":{"1min":3.5750000000000002,"5min":1.2569999999999999,"15min":0.79100000000000004},"min":{"1min":0.33600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":31.100000000000001,"5min":31.100000000000001,"15min":31.100000000000001},"last":3.2469999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:35:07 2023","interfaces":[{"interface":"back","average":{"1min":11.824,"5min":3.4830000000000001,"15min":1.506},"min":{"1min":0.35299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":164.26599999999999,"5min":164.26599999999999,"15min":164.26599999999999},"last":2.7909999999999999},{"interface":"front","average":{"1min":10.645,"5min":3.3359999999999999,"15min":1.502},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":162.57499999999999,"5min":162.57499999999999,"15min":162.57499999999999},"last":2.9060000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:34:34 2023","interfaces":[{"interface":"back","average":{"1min":2.96,"5min":1.087,"15min":0.73499999999999999},"min":{"1min":0.41299999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":9.5909999999999993,"5min":9.5909999999999993,"15min":9.5909999999999993},"last":3.2290000000000001},{"interface":"front","average":{"1min":2.0169999999999999,"5min":0.98899999999999999,"15min":0.73799999999999999},"min":{"1min":0.375,"5min":0.309,"15min":0.309},"max":{"1min":6.9459999999999997,"5min":6.9459999999999997,"15min":6.9459999999999997},"last":0.58999999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:34:34 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":0.73199999999999998,"15min":0.53800000000000003},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":9.6470000000000002,"5min":9.6470000000000002,"15min":9.6470000000000002},"last":3.1709999999999998},{"interface":"front","average":{"1min":1.03,"5min":0.69799999999999995,"15min":0.58899999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.7930000000000001,"5min":3.7930000000000001,"15min":3.7930000000000001},"last":0.35599999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:34:44 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":0.64000000000000001,"15min":0.53500000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27800000000000002},"max":{"1min":3.8140000000000001,"5min":3.8140000000000001,"15min":3.8140000000000001},"last":0.62},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.75,"15min":0.58199999999999996},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":9.6820000000000004,"5min":9.6820000000000004,"15min":9.6820000000000004},"last":3.3780000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:33:09 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.55400000000000005,"15min":0.54200000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":1.7470000000000001,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.53600000000000003,"15min":0.53100000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":2.8250000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822853,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2856220,"kb_used_data":2550804,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90888932,"statfs":{"total":95995035648,"available":93070266368,"internally_reserved":0,"allocated":2612023296,"data_stored":2611524935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":11.497999999999999,"5min":3.8370000000000002,"15min":1.647},"min":{"1min":0.224,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":150.13399999999999,"5min":150.13399999999999,"15min":150.13399999999999},"last":1.913},{"interface":"front","average":{"1min":12.132,"5min":4.1879999999999997,"15min":1.766},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":150.24700000000001,"5min":150.24700000000001,"15min":150.24700000000001},"last":1.7270000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:34:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1269999999999998,"5min":0.91100000000000003,"15min":0.67400000000000004},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.891999999999999,"5min":12.891999999999999,"15min":12.891999999999999},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.6080000000000001,"5min":0.88100000000000001,"15min":0.66500000000000004},"min":{"1min":0.26700000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":12.939,"5min":12.939,"15min":12.939},"last":1.752}]},{"osd":2,"last update":"Mon Oct 9 05:34:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5379999999999998,"5min":0.96499999999999997,"15min":0.70299999999999996},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.1719999999999997,"5min":6.1719999999999997,"15min":6.1719999999999997},"last":1.8220000000000001},{"interface":"front","average":{"1min":2.097,"5min":0.86899999999999999,"15min":0.66700000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.5809999999999995,"5min":8.5809999999999995,"15min":8.5809999999999995},"last":1.8009999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:34:35 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":0.86199999999999999,"15min":0.68200000000000005},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":6.4119999999999999,"5min":6.4119999999999999,"15min":6.4119999999999999},"last":1.8979999999999999},{"interface":"front","average":{"1min":2.2410000000000001,"5min":0.97699999999999998,"15min":0.71699999999999997},"min":{"1min":0.36599999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":12.779999999999999,"5min":12.779999999999999,"15min":12.779999999999999},"last":0.36699999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:34:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":0.64200000000000002,"15min":0.51800000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":9.3510000000000009,"5min":9.3510000000000009,"15min":9.3510000000000009},"last":1.6619999999999999},{"interface":"front","average":{"1min":1.698,"5min":0.78400000000000003,"15min":0.59399999999999997},"min":{"1min":0.28299999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.3010000000000002,"5min":9.3010000000000002,"15min":9.3010000000000002},"last":0.33600000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:34:57 2023","interfaces":[{"interface":"back","average":{"1min":5.8570000000000002,"5min":1.694,"15min":0.90700000000000003},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":95.010999999999996,"5min":95.010999999999996,"15min":95.010999999999996},"last":0.41299999999999998},{"interface":"front","average":{"1min":6.0250000000000004,"5min":1.6890000000000001,"15min":0.91900000000000004},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":94.194000000000003,"5min":94.194000000000003,"15min":94.194000000000003},"last":0.42399999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:33:12 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.59999999999999998,"15min":0.58099999999999996},"min":{"1min":0.44400000000000001,"5min":0.40200000000000002,"15min":0.34399999999999997},"max":{"1min":1.5249999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.7},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.58799999999999997,"15min":0.56000000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32400000000000001},"max":{"1min":1.8600000000000001,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":0.48699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2380005376,"data_stored":2380005376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1477574656,"data_stored":1477574656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1702170624,"data_stored":1702166552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":853737472,"data_stored":853737472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2313879552,"data_stored":2313879552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2589790208,"data_stored":2589786136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1146945536,"data_stored":1146945536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":654901248,"data_stored":654901248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:35:15.809 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 1 2023-10-09T05:35:15.863 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:15.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:15.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:15.981 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:16.097 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:16.219 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:16.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:16.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:16.332 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:16.395 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:16.461 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:35:16.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:16.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:16.568 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:16.603 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:16.880 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:16.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:16.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:16.990 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:17.031 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:17.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:17 smithi008 ceph-mon[184418]: pgmap v342: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 22651/203283 objects degraded (11.143%); 2.6 MiB/s, 81 objects/s recovering 2023-10-09T05:35:17.159 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:17 smithi008 ceph-mon[184418]: 5.3 scrub starts 2023-10-09T05:35:17.159 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:17 smithi008 ceph-mon[191227]: pgmap v342: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 22651/203283 objects degraded (11.143%); 2.6 MiB/s, 81 objects/s recovering 2023-10-09T05:35:17.159 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:17 smithi008 ceph-mon[191227]: 5.3 scrub starts 2023-10-09T05:35:17.176 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:17.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:17.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:17.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:17 smithi191 ceph-mon[161669]: pgmap v342: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 22651/203283 objects degraded (11.143%); 2.6 MiB/s, 81 objects/s recovering 2023-10-09T05:35:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:17 smithi191 ceph-mon[161669]: 5.3 scrub starts 2023-10-09T05:35:17.333 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:17.404 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:35:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:17.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:17.412 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:35:17.412 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:35:17.413 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:35:17.413 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:35:17.413 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:35:17.510 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:17.581 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:17.834 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:17.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:17.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:17.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:17.981 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:35:17.995 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:18.010 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:35:18.018 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:35:18.070 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:35:18.101 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:35:18.106 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:35:18.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:18.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:18.151 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:35:18.189 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:35:18.213 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:18.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 22651/203283 objects degraded (11.143%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:18.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4192466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:18.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 22651/203283 objects degraded (11.143%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:18.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4192466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:18.230 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:35:18.282 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:35:18.299 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:18.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 22651/203283 objects degraded (11.143%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:18.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4192466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:18.321 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:35:18.321 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:35:18.321 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:35:18.322 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:35:18.322 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:35:18.322 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:35:18.322 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:35:18.322 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:35:18.411 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:18.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:18.519 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:18.577 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:19.181 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:19.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:19.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:19.289 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:19.358 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:19.438 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:35:19.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:19.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:19.548 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:19.584 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:19.736 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:19.771 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:35:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:19 smithi191 ceph-mon[161669]: pgmap v343: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 22362/194979 objects degraded (11.469%); 4.2 MiB/s, 100 objects/s recovering 2023-10-09T05:35:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:19 smithi191 ceph-mon[161669]: 5.23 deep-scrub starts 2023-10-09T05:35:19.868 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:19 smithi008 ceph-mon[184418]: pgmap v343: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 22362/194979 objects degraded (11.469%); 4.2 MiB/s, 100 objects/s recovering 2023-10-09T05:35:19.868 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:19 smithi008 ceph-mon[184418]: 5.23 deep-scrub starts 2023-10-09T05:35:19.868 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:19 smithi008 ceph-mon[191227]: pgmap v343: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 22362/194979 objects degraded (11.469%); 4.2 MiB/s, 100 objects/s recovering 2023-10-09T05:35:19.869 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:19 smithi008 ceph-mon[191227]: 5.23 deep-scrub starts 2023-10-09T05:35:19.881 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:19.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:19.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:19.987 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:20.088 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:20.211 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:20.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:20.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:20.321 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:20.388 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:20.439 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:20.698 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:20.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:20.804 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:20.866 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:35:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:35:20] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.33.4" 2023-10-09T05:35:20.867 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:20.935 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:20.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:20.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:21.038 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:21.066 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:21.132 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:21.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:21.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:21.236 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:21.284 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:21.370 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:35:21.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:21.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:21.485 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:21.542 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:21.627 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:35:21.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:21.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:21.739 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:21.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:21 smithi191 ceph-mon[161669]: pgmap v344: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 22362/188553 objects degraded (11.860%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:35:21.803 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:21.934 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:21 smithi008 ceph-mon[184418]: pgmap v344: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 22362/188553 objects degraded (11.860%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:35:21.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:21 smithi008 ceph-mon[191227]: pgmap v344: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 22362/188553 objects degraded (11.860%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:35:21.993 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:22.000 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:35:22.219 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:22.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:22.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:22.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:22.374 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:22.474 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:35:22.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:22.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:22.594 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:22.627 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:22 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 22362/188553 objects degraded (11.860%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:22.635 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:22 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 22362/188553 objects degraded (11.860%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:22.658 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:22.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:22 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 22362/188553 objects degraded (11.860%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:23.161 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:35:23.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:23 smithi008 ceph-mon[184418]: pgmap v345: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 22050/182403 objects degraded (12.089%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:35:23.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2885691886' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:23.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:23 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:23 smithi008 ceph-mon[191227]: pgmap v345: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 22050/182403 objects degraded (12.089%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:35:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2885691886' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:23 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:24.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:23 smithi191 ceph-mon[161669]: pgmap v345: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 22050/182403 objects degraded (12.089%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:35:24.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2885691886' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:24.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:23 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:24.274 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:24.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:24.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:24.382 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:24.430 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:35:24.463 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:24.719 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:24.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:24.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:24.828 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:24.886 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:24 smithi008 ceph-mon[184418]: pgmap v346: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 22050/179772 objects degraded (12.266%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:35:24.887 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:24 smithi008 ceph-mon[191227]: pgmap v346: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 22050/179772 objects degraded (12.266%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:35:24.896 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:25.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:24 smithi191 ceph-mon[161669]: pgmap v346: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 22050/179772 objects degraded (12.266%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T05:35:25.054 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:25.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:25.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:25.167 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:25.246 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:25.336 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:25.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:25.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:25.445 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:25.506 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:25.589 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:35:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:25.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:25.660 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:25 smithi008 ceph-mon[191227]: 5.15 scrub starts 2023-10-09T05:35:25.660 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:25 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3349907145' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:35:25.674 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:35:25.708 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:25.772 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:25.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:25 smithi191 ceph-mon[161669]: 5.15 scrub starts 2023-10-09T05:35:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:25 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3349907145' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:35:25.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:25 smithi008 ceph-mon[184418]: 5.15 scrub starts 2023-10-09T05:35:25.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:25 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3349907145' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:35:26.723 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:26.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:26.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:26.824 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:26.828 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:26.861 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:26.861 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[184418]: failed to encode map e138 with expected crc 2023-10-09T05:35:26.861 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3349907145' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:35:26.861 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[184418]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T05:35:26.861 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[184418]: failed to encode map e138 with expected crc 2023-10-09T05:35:26.861 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[184418]: failed to encode map e138 with expected crc 2023-10-09T05:35:26.862 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[184418]: failed to encode map e138 with expected crc 2023-10-09T05:35:26.862 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[184418]: failed to encode map e138 with expected crc 2023-10-09T05:35:26.862 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[184418]: pgmap v348: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 22050/174858 objects degraded (12.610%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:35:26.862 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:26.862 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[191227]: failed to encode map e138 with expected crc 2023-10-09T05:35:26.863 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3349907145' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:35:26.863 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[191227]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T05:35:26.863 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[191227]: failed to encode map e138 with expected crc 2023-10-09T05:35:26.863 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[191227]: failed to encode map e138 with expected crc 2023-10-09T05:35:26.863 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[191227]: failed to encode map e138 with expected crc 2023-10-09T05:35:26.863 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[191227]: failed to encode map e138 with expected crc 2023-10-09T05:35:26.864 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:26 smithi008 ceph-mon[191227]: pgmap v348: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 22050/174858 objects degraded (12.610%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:35:26.919 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:27.037 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:27.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:27.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:27.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:26 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:27.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:26 smithi191 ceph-mon[161669]: failed to encode map e138 with expected crc 2023-10-09T05:35:27.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3349907145' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:35:27.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:26 smithi191 ceph-mon[161669]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T05:35:27.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:26 smithi191 ceph-mon[161669]: failed to encode map e138 with expected crc 2023-10-09T05:35:27.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:26 smithi191 ceph-mon[161669]: failed to encode map e138 with expected crc 2023-10-09T05:35:27.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:26 smithi191 ceph-mon[161669]: failed to encode map e138 with expected crc 2023-10-09T05:35:27.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:26 smithi191 ceph-mon[161669]: failed to encode map e138 with expected crc 2023-10-09T05:35:27.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:26 smithi191 ceph-mon[161669]: pgmap v348: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 22050/174858 objects degraded (12.610%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:35:27.149 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:27.234 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:27.361 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:27.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:27.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:27.469 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:27.493 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:35:27.493 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:35:27.533 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:27.745 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:27.746 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:27 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 22050/174858 objects degraded (12.610%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:27.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:27.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:27.749 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:27 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 22050/174858 objects degraded (12.610%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:27.850 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:27.911 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:28.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:27 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 22050/174858 objects degraded (12.610%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:28.862 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:35:28.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:28 smithi008 ceph-mon[184418]: pgmap v349: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 929 op/s; 21714/167073 objects degraded (12.997%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:35:28.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:28 smithi008 ceph-mon[191227]: pgmap v349: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 929 op/s; 21714/167073 objects degraded (12.997%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:35:29.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:28 smithi191 ceph-mon[161669]: pgmap v349: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 929 op/s; 21714/167073 objects degraded (12.997%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:35:29.432 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:35:29.432 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:35:29.432 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T05:35:29.432 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:35:29.444 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:29.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:29.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:29.456 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:35:29.556 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:29.698 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:29.905 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:29.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:29.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:29.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:29.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:29.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:29.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:30.013 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:30.094 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:30.215 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:30.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:30.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:30.266 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:35:30.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:30.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:30.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:30.312 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:35:30.324 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:30.356 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:35:30.388 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:30.398 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:35:30.399 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:30.400 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:30.406 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:30.443 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:35:30.492 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:35:30.542 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:35:30.585 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:35:30.594 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:35:30.594 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:35:30.595 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:35:30.595 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:35:30.595 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:35:30.595 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:35:30.595 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:35:30.595 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:35:30.623 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:35:30.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:30.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:30.732 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:30.786 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:30.789 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:35:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:35:30] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.33.4" 2023-10-09T05:35:30.888 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:35:30.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:30.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:30.993 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:31.047 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:30 smithi008 ceph-mon[184418]: pgmap v350: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 21714/161058 objects degraded (13.482%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:35:31.047 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/845255697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:31.047 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:30 smithi008 ceph-mon[191227]: pgmap v350: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 21714/161058 objects degraded (13.482%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:35:31.047 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/845255697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:30 smithi191 ceph-mon[161669]: pgmap v350: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 21714/161058 objects degraded (13.482%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:35:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/845255697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:31.054 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:32.146 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:32.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:32.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:32.252 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:32.326 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:32.421 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:35:32.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:32.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:32.530 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:32.590 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:33.276 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:33 smithi008 ceph-mon[184418]: pgmap v351: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 907 op/s; 21330/155175 objects degraded (13.746%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:35:33.276 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 21330/155175 objects degraded (13.746%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:33.276 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:33 smithi008 ceph-mon[191227]: pgmap v351: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 907 op/s; 21330/155175 objects degraded (13.746%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:35:33.276 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 21330/155175 objects degraded (13.746%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:33.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:33 smithi191 ceph-mon[161669]: pgmap v351: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 907 op/s; 21330/155175 objects degraded (13.746%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:35:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 21330/155175 objects degraded (13.746%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:33.750 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:33.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:33.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:33.862 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:33.944 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:35.297 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:35 smithi008 ceph-mon[184418]: pgmap v352: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 21330/152679 objects degraded (13.970%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:35:35.297 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:35.297 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:35.297 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:35 smithi008 ceph-mon[191227]: pgmap v352: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 21330/152679 objects degraded (13.970%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:35:35.297 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:35.298 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:35.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:35 smithi191 ceph-mon[161669]: pgmap v352: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 21330/152679 objects degraded (13.970%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:35:35.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:35.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:35.328 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:35.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:35.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:35.440 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:35.501 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:35.549 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:35:36.035 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:36.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:36.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:36.143 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:36.261 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:36.367 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:35:36.380 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:35:36.492 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:36.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:36.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:36.685 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:36.784 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:36.786 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:35:36.786 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:35:36.787 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T05:35:36.787 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:35:36.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:36.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:36.900 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:36.979 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:37.046 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:37 smithi191 ceph-mon[161669]: pgmap v353: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 21330/147975 objects degraded (14.415%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T05:35:37.047 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:37 smithi191 ceph-mon[161669]: from='client.34947 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:35:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:37 smithi008 ceph-mon[184418]: pgmap v353: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 21330/147975 objects degraded (14.415%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T05:35:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:37 smithi008 ceph-mon[184418]: from='client.34947 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:35:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:37 smithi008 ceph-mon[191227]: pgmap v353: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 21330/147975 objects degraded (14.415%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T05:35:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:37 smithi008 ceph-mon[191227]: from='client.34947 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:35:38.072 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:38.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:38.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:38.100 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:35:38.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:38.217 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:35:38.218 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:38.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:38 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/73327997' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:38.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 21330/147975 objects degraded (14.415%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:38.305 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:38 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/73327997' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:38.305 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 21330/147975 objects degraded (14.415%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:38.306 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:38 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/73327997' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:38.307 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 21330/147975 objects degraded (14.415%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:38.346 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:35:38.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:38.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:38.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:38.497 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:38.580 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:38.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:38.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:38.686 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:38.713 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:38.954 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:38.982 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":353,"stamp":"2023-10-09T05:35:35.972038+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3235420107,"num_objects":49325,"num_object_clones":0,"num_object_copies":147975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21330,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49325,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":137889,"num_write_kb":4625030,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4224,"num_bytes_recovered":235470848,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":321542,"ondisk_log_size":321542,"up":327,"acting":318,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10755688,"kb_used_data":9011304,"kb_used_omap":45,"kb_used_meta":1744210,"kb_avail":739205528,"statfs":{"total":767960285184,"available":756946460672,"internally_reserved":0,"allocated":9227575296,"data_stored":9223144917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":1786071331},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,6,3,3,5,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-587268096,"num_objects":-8961,"num_object_clones":0,"num_object_copies":-26883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-720,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8961,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":720,"num_bytes_recovered":47185920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.327691"},"pg_stats":[{"pgid":"5.23","version":"138'7719","reported_seq":7861,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:32.659426+0000","last_change":"2023-10-09T05:35:19.048870+0000","last_active":"2023-10-09T05:35:32.659426+0000","last_peered":"2023-10-09T05:35:32.659426+0000","last_clean":"2023-10-09T05:35:32.659426+0000","last_became_active":"2023-10-09T05:34:25.035930+0000","last_became_peered":"2023-10-09T05:34:25.035930+0000","last_unstale":"2023-10-09T05:35:32.659426+0000","last_undegraded":"2023-10-09T05:35:32.659426+0000","last_fullsized":"2023-10-09T05:35:32.659426+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"120'6300","last_scrub_stamp":"2023-10-09T05:34:08.749711+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:08.749711+0000","objects_scrubbed":3317,"log_size":7719,"log_dups_size":0,"ondisk_log_size":7719,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.61798832400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52002816,"num_objects":794,"num_object_clones":0,"num_object_copies":2382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1231,"num_write_kb":11712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.22","version":"138'7840","reported_seq":7914,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:34.742866+0000","last_change":"2023-10-09T05:34:40.876515+0000","last_active":"2023-10-09T05:35:34.742866+0000","last_peered":"2023-10-09T05:35:34.742866+0000","last_clean":"2023-10-09T05:35:34.742866+0000","last_became_active":"2023-10-09T05:34:25.030851+0000","last_became_peered":"2023-10-09T05:34:25.030851+0000","last_unstale":"2023-10-09T05:35:34.742866+0000","last_undegraded":"2023-10-09T05:35:34.742866+0000","last_fullsized":"2023-10-09T05:35:34.742866+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"127'7054","last_scrub_stamp":"2023-10-09T05:34:40.876434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:40.876434+0000","objects_scrubbed":0,"log_size":7840,"log_dups_size":0,"ondisk_log_size":7840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45416448,"num_objects":693,"num_object_clones":0,"num_object_copies":2079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":786,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.21","version":"138'7779","reported_seq":10513,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:34.597247+0000","last_change":"2023-10-09T05:35:08.601492+0000","last_active":"2023-10-09T05:35:34.597247+0000","last_peered":"2023-10-09T05:35:34.597247+0000","last_clean":"2023-10-09T05:35:34.597247+0000","last_became_active":"2023-10-09T05:35:08.595030+0000","last_became_peered":"2023-10-09T05:35:08.595030+0000","last_unstale":"2023-10-09T05:35:34.597247+0000","last_undegraded":"2023-10-09T05:35:34.597247+0000","last_fullsized":"2023-10-09T05:35:34.597247+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":7779,"log_dups_size":0,"ondisk_log_size":7779,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49856512,"num_objects":761,"num_object_clones":0,"num_object_copies":2283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2910,"num_write_kb":117456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1327,"num_bytes_recovered":65929216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.20","version":"138'7701","reported_seq":7879,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:30.752388+0000","last_change":"2023-10-09T05:34:38.457569+0000","last_active":"2023-10-09T05:35:30.752388+0000","last_peered":"2023-10-09T05:35:30.752388+0000","last_clean":"2023-10-09T05:35:30.752388+0000","last_became_active":"2023-10-09T05:34:25.062440+0000","last_became_peered":"2023-10-09T05:34:25.062440+0000","last_unstale":"2023-10-09T05:35:30.752388+0000","last_undegraded":"2023-10-09T05:35:30.752388+0000","last_fullsized":"2023-10-09T05:35:30.752388+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"125'6962","last_scrub_stamp":"2023-10-09T05:34:38.457405+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:38.457405+0000","objects_scrubbed":1537,"log_size":7701,"log_dups_size":0,"ondisk_log_size":7701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:57.174556+0000","scrub_duration":0.224312019,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52232192,"num_objects":797,"num_object_clones":0,"num_object_copies":2391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":740,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1c","version":"138'9706","reported_seq":9834,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:33.454245+0000","last_change":"2023-10-09T05:34:39.445074+0000","last_active":"2023-10-09T05:35:33.454245+0000","last_peered":"2023-10-09T05:35:33.454245+0000","last_clean":"2023-10-09T05:35:33.454245+0000","last_became_active":"2023-10-09T05:34:25.054943+0000","last_became_peered":"2023-10-09T05:34:25.054943+0000","last_unstale":"2023-10-09T05:35:33.454245+0000","last_undegraded":"2023-10-09T05:35:33.454245+0000","last_fullsized":"2023-10-09T05:35:33.454245+0000","mapping_epoch":123,"log_start":"108'200","ondisk_log_start":"108'200","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'8137","last_scrub_stamp":"2023-10-09T05:34:39.444955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:39.444955+0000","objects_scrubbed":3164,"log_size":9506,"log_dups_size":0,"ondisk_log_size":9506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:30.213649+0000","scrub_duration":0.76575304399999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103612416,"num_objects":1581,"num_object_clones":0,"num_object_copies":4743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1583,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.603223+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:35:13.603223+0000","last_peered":"2023-10-09T05:35:13.603223+0000","last_clean":"2023-10-09T05:35:13.603223+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:35:13.603223+0000","last_undegraded":"2023-10-09T05:35:13.603223+0000","last_fullsized":"2023-10-09T05:35:13.603223+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.599918+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:35:13.599918+0000","last_peered":"2023-10-09T05:35:13.599918+0000","last_clean":"2023-10-09T05:35:13.599918+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:35:13.599918+0000","last_undegraded":"2023-10-09T05:35:13.599918+0000","last_fullsized":"2023-10-09T05:35:13.599918+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1f","version":"138'9535","reported_seq":9585,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:34.758197+0000","last_change":"2023-10-09T05:34:25.044017+0000","last_active":"2023-10-09T05:35:34.758197+0000","last_peered":"2023-10-09T05:35:34.758197+0000","last_clean":"2023-10-09T05:35:34.758197+0000","last_became_active":"2023-10-09T05:34:25.037058+0000","last_became_peered":"2023-10-09T05:34:25.037058+0000","last_unstale":"2023-10-09T05:35:34.758197+0000","last_undegraded":"2023-10-09T05:35:34.758197+0000","last_fullsized":"2023-10-09T05:35:34.758197+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9535,"log_dups_size":0,"ondisk_log_size":9535,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99123200,"num_objects":1513,"num_object_clones":0,"num_object_copies":4539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5834,"num_write_kb":235072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:18.563955+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:35:18.563955+0000","last_peered":"2023-10-09T05:35:18.563955+0000","last_clean":"2023-10-09T05:35:18.563955+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:35:18.563955+0000","last_undegraded":"2023-10-09T05:35:18.563955+0000","last_fullsized":"2023-10-09T05:35:18.563955+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.597369+0000","last_change":"2023-10-09T05:34:41.662942+0000","last_active":"2023-10-09T05:35:13.597369+0000","last_peered":"2023-10-09T05:35:13.597369+0000","last_clean":"2023-10-09T05:35:13.597369+0000","last_became_active":"2023-10-09T05:34:41.662648+0000","last_became_peered":"2023-10-09T05:34:41.662648+0000","last_unstale":"2023-10-09T05:35:13.597369+0000","last_undegraded":"2023-10-09T05:35:13.597369+0000","last_fullsized":"2023-10-09T05:35:13.597369+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:08.691549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1e","version":"138'9656","reported_seq":9704,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:34.806440+0000","last_change":"2023-10-09T05:34:25.039990+0000","last_active":"2023-10-09T05:35:34.806440+0000","last_peered":"2023-10-09T05:35:34.806440+0000","last_clean":"2023-10-09T05:35:34.806440+0000","last_became_active":"2023-10-09T05:34:25.036888+0000","last_became_peered":"2023-10-09T05:34:25.036888+0000","last_unstale":"2023-10-09T05:35:34.806440+0000","last_undegraded":"2023-10-09T05:35:34.806440+0000","last_fullsized":"2023-10-09T05:35:34.806440+0000","mapping_epoch":123,"log_start":"109'800","ondisk_log_start":"109'800","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8856,"log_dups_size":0,"ondisk_log_size":8856,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99188736,"num_objects":1514,"num_object_clones":0,"num_object_copies":4542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5917,"num_write_kb":237760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":124,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.600626+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:35:13.600626+0000","last_peered":"2023-10-09T05:35:13.600626+0000","last_clean":"2023-10-09T05:35:13.600626+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:35:13.600626+0000","last_undegraded":"2023-10-09T05:35:13.600626+0000","last_fullsized":"2023-10-09T05:35:13.600626+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.599242+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:35:13.599242+0000","last_peered":"2023-10-09T05:35:13.599242+0000","last_clean":"2023-10-09T05:35:13.599242+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:35:13.599242+0000","last_undegraded":"2023-10-09T05:35:13.599242+0000","last_fullsized":"2023-10-09T05:35:13.599242+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.11","version":"138'8873","reported_seq":8935,"reported_epoch":138,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:34.464884+0000","last_change":"2023-10-09T05:34:42.977328+0000","last_active":"2023-10-09T05:35:34.464884+0000","last_peered":"2023-10-09T05:35:34.464884+0000","last_clean":"2023-10-09T05:33:20.516527+0000","last_became_active":"2023-10-09T05:34:42.969107+0000","last_became_peered":"2023-10-09T05:34:42.969107+0000","last_unstale":"2023-10-09T05:35:34.464884+0000","last_undegraded":"2023-10-09T05:34:42.867828+0000","last_fullsized":"2023-10-09T05:34:42.853589+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8873,"log_dups_size":0,"ondisk_log_size":8873,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94732288,"num_objects":1446,"num_object_clones":0,"num_object_copies":4338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2627,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5856,"num_write_kb":233632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":-533}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":88,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.590938+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:35:13.590938+0000","last_peered":"2023-10-09T05:35:13.590938+0000","last_clean":"2023-10-09T05:35:13.590938+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:35:13.590938+0000","last_undegraded":"2023-10-09T05:35:13.590938+0000","last_fullsized":"2023-10-09T05:35:13.590938+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.600772+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:35:13.600772+0000","last_peered":"2023-10-09T05:35:13.600772+0000","last_clean":"2023-10-09T05:35:13.600772+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:35:13.600772+0000","last_undegraded":"2023-10-09T05:35:13.600772+0000","last_fullsized":"2023-10-09T05:35:13.600772+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.10","version":"138'8700","reported_seq":8827,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:30.814714+0000","last_change":"2023-10-09T05:34:06.833835+0000","last_active":"2023-10-09T05:35:30.814714+0000","last_peered":"2023-10-09T05:35:30.814714+0000","last_clean":"2023-10-09T05:35:30.814714+0000","last_became_active":"2023-10-09T05:34:01.635329+0000","last_became_peered":"2023-10-09T05:34:01.635329+0000","last_unstale":"2023-10-09T05:35:30.814714+0000","last_undegraded":"2023-10-09T05:35:30.814714+0000","last_fullsized":"2023-10-09T05:35:30.814714+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6258","last_scrub_stamp":"2023-10-09T05:34:06.833681+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:06.833681+0000","objects_scrubbed":3155,"log_size":8700,"log_dups_size":0,"ondisk_log_size":8700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:45.971615+0000","scrub_duration":0.69237216700000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101318656,"num_objects":1546,"num_object_clones":0,"num_object_copies":4638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2449,"num_write_kb":26880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595556+0000","last_change":"2023-10-09T05:34:41.683934+0000","last_active":"2023-10-09T05:35:13.595556+0000","last_peered":"2023-10-09T05:35:13.595556+0000","last_clean":"2023-10-09T05:35:13.595556+0000","last_became_active":"2023-10-09T05:34:41.683618+0000","last_became_peered":"2023-10-09T05:34:41.683618+0000","last_unstale":"2023-10-09T05:35:13.595556+0000","last_undegraded":"2023-10-09T05:35:13.595556+0000","last_fullsized":"2023-10-09T05:35:13.595556+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":113,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.600242+0000","last_change":"2023-10-09T05:34:41.668251+0000","last_active":"2023-10-09T05:35:13.600242+0000","last_peered":"2023-10-09T05:35:13.600242+0000","last_clean":"2023-10-09T05:35:13.600242+0000","last_became_active":"2023-10-09T05:34:41.668017+0000","last_became_peered":"2023-10-09T05:34:41.668017+0000","last_unstale":"2023-10-09T05:35:13.600242+0000","last_undegraded":"2023-10-09T05:35:13.600242+0000","last_fullsized":"2023-10-09T05:35:13.600242+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:27.235656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.13","version":"138'8793","reported_seq":8847,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:32.659357+0000","last_change":"2023-10-09T05:35:14.038814+0000","last_active":"2023-10-09T05:35:32.659357+0000","last_peered":"2023-10-09T05:35:32.659357+0000","last_clean":"2023-10-09T05:35:32.659357+0000","last_became_active":"2023-10-09T05:34:01.625092+0000","last_became_peered":"2023-10-09T05:34:01.625092+0000","last_unstale":"2023-10-09T05:35:32.659357+0000","last_undegraded":"2023-10-09T05:35:32.659357+0000","last_fullsized":"2023-10-09T05:35:32.659357+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8793,"log_dups_size":0,"ondisk_log_size":8793,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101842944,"num_objects":1554,"num_object_clones":0,"num_object_copies":4662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5772,"num_write_kb":234432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":97,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.597600+0000","last_change":"2023-10-09T05:34:41.682874+0000","last_active":"2023-10-09T05:35:13.597600+0000","last_peered":"2023-10-09T05:35:13.597600+0000","last_clean":"2023-10-09T05:35:13.597600+0000","last_became_active":"2023-10-09T05:34:41.674789+0000","last_became_peered":"2023-10-09T05:34:41.674789+0000","last_unstale":"2023-10-09T05:35:13.597600+0000","last_undegraded":"2023-10-09T05:35:13.597600+0000","last_fullsized":"2023-10-09T05:35:13.597600+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":83,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.596826+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:35:13.596826+0000","last_peered":"2023-10-09T05:35:13.596826+0000","last_clean":"2023-10-09T05:35:13.596826+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:35:13.596826+0000","last_undegraded":"2023-10-09T05:35:13.596826+0000","last_fullsized":"2023-10-09T05:35:13.596826+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.12","version":"138'9019","reported_seq":9074,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:34.653713+0000","last_change":"2023-10-09T05:34:06.470865+0000","last_active":"2023-10-09T05:35:34.653713+0000","last_peered":"2023-10-09T05:35:34.653713+0000","last_clean":"2023-10-09T05:35:34.653713+0000","last_became_active":"2023-10-09T05:34:01.624493+0000","last_became_peered":"2023-10-09T05:34:01.624493+0000","last_unstale":"2023-10-09T05:35:34.653713+0000","last_undegraded":"2023-10-09T05:35:34.653713+0000","last_fullsized":"2023-10-09T05:35:34.653713+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6331","last_scrub_stamp":"2023-10-09T05:34:06.470790+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:06.470790+0000","objects_scrubbed":0,"log_size":9019,"log_dups_size":0,"ondisk_log_size":9019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99876864,"num_objects":1524,"num_object_clones":0,"num_object_copies":4572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2700,"num_write_kb":30848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":127,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.601787+0000","last_change":"2023-10-09T05:34:41.683953+0000","last_active":"2023-10-09T05:35:13.601787+0000","last_peered":"2023-10-09T05:35:13.601787+0000","last_clean":"2023-10-09T05:35:13.601787+0000","last_became_active":"2023-10-09T05:34:41.683519+0000","last_became_peered":"2023-10-09T05:34:41.683519+0000","last_unstale":"2023-10-09T05:35:13.601787+0000","last_undegraded":"2023-10-09T05:35:13.601787+0000","last_fullsized":"2023-10-09T05:35:13.601787+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.600834+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:35:13.600834+0000","last_peered":"2023-10-09T05:35:13.600834+0000","last_clean":"2023-10-09T05:35:13.600834+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:35:13.600834+0000","last_undegraded":"2023-10-09T05:35:13.600834+0000","last_fullsized":"2023-10-09T05:35:13.600834+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.15","version":"138'9110","reported_seq":9163,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:32.714278+0000","last_change":"2023-10-09T05:35:24.126409+0000","last_active":"2023-10-09T05:35:32.714278+0000","last_peered":"2023-10-09T05:35:32.714278+0000","last_clean":"2023-10-09T05:35:32.714278+0000","last_became_active":"2023-10-09T05:34:01.623738+0000","last_became_peered":"2023-10-09T05:34:01.623738+0000","last_unstale":"2023-10-09T05:35:32.714278+0000","last_undegraded":"2023-10-09T05:35:32.714278+0000","last_fullsized":"2023-10-09T05:35:32.714278+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9110,"log_dups_size":0,"ondisk_log_size":9110,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98631680,"num_objects":1505,"num_object_clones":0,"num_object_copies":4515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5985,"num_write_kb":239680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":91,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.598011+0000","last_change":"2023-10-09T05:34:41.673944+0000","last_active":"2023-10-09T05:35:13.598011+0000","last_peered":"2023-10-09T05:35:13.598011+0000","last_clean":"2023-10-09T05:35:13.598011+0000","last_became_active":"2023-10-09T05:34:41.673595+0000","last_became_peered":"2023-10-09T05:34:41.673595+0000","last_unstale":"2023-10-09T05:35:13.598011+0000","last_undegraded":"2023-10-09T05:35:13.598011+0000","last_fullsized":"2023-10-09T05:35:13.598011+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":123,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.599726+0000","last_change":"2023-10-09T05:34:41.667462+0000","last_active":"2023-10-09T05:35:13.599726+0000","last_peered":"2023-10-09T05:35:13.599726+0000","last_clean":"2023-10-09T05:35:13.599726+0000","last_became_active":"2023-10-09T05:34:41.667140+0000","last_became_peered":"2023-10-09T05:34:41.667140+0000","last_unstale":"2023-10-09T05:35:13.599726+0000","last_undegraded":"2023-10-09T05:35:13.599726+0000","last_fullsized":"2023-10-09T05:35:13.599726+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:52.888470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.14","version":"138'8703","reported_seq":8762,"reported_epoch":138,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:30.923386+0000","last_change":"2023-10-09T05:34:42.906108+0000","last_active":"2023-10-09T05:35:30.923386+0000","last_peered":"2023-10-09T05:35:30.923386+0000","last_clean":"2023-10-09T05:33:20.493128+0000","last_became_active":"2023-10-09T05:34:42.894931+0000","last_became_peered":"2023-10-09T05:34:42.894931+0000","last_unstale":"2023-10-09T05:35:30.923386+0000","last_undegraded":"2023-10-09T05:34:42.686528+0000","last_fullsized":"2023-10-09T05:34:42.658194+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8703,"log_dups_size":0,"ondisk_log_size":8703,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106135552,"num_objects":1620,"num_object_clones":0,"num_object_copies":4860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2585,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5674,"num_write_kb":233376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-357}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.592661+0000","last_change":"2023-10-09T05:34:41.662790+0000","last_active":"2023-10-09T05:35:13.592661+0000","last_peered":"2023-10-09T05:35:13.592661+0000","last_clean":"2023-10-09T05:35:13.592661+0000","last_became_active":"2023-10-09T05:34:41.662513+0000","last_became_peered":"2023-10-09T05:34:41.662513+0000","last_unstale":"2023-10-09T05:35:13.592661+0000","last_undegraded":"2023-10-09T05:35:13.592661+0000","last_fullsized":"2023-10-09T05:35:13.592661+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:17.781371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595823+0000","last_change":"2023-10-09T05:34:41.694695+0000","last_active":"2023-10-09T05:35:13.595823+0000","last_peered":"2023-10-09T05:35:13.595823+0000","last_clean":"2023-10-09T05:35:13.595823+0000","last_became_active":"2023-10-09T05:34:41.694408+0000","last_became_peered":"2023-10-09T05:34:41.694408+0000","last_unstale":"2023-10-09T05:35:13.595823+0000","last_undegraded":"2023-10-09T05:35:13.595823+0000","last_fullsized":"2023-10-09T05:35:13.595823+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.17","version":"138'8991","reported_seq":9046,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:34.742949+0000","last_change":"2023-10-09T05:34:37.421122+0000","last_active":"2023-10-09T05:35:34.742949+0000","last_peered":"2023-10-09T05:35:34.742949+0000","last_clean":"2023-10-09T05:35:34.742949+0000","last_became_active":"2023-10-09T05:34:01.618906+0000","last_became_peered":"2023-10-09T05:34:01.618906+0000","last_unstale":"2023-10-09T05:35:34.742949+0000","last_undegraded":"2023-10-09T05:35:34.742949+0000","last_fullsized":"2023-10-09T05:35:34.742949+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'7322","last_scrub_stamp":"2023-10-09T05:34:37.421020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:37.421020+0000","objects_scrubbed":0,"log_size":8991,"log_dups_size":0,"ondisk_log_size":8991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99221504,"num_objects":1514,"num_object_clones":0,"num_object_copies":4542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1686,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":74,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.597552+0000","last_change":"2023-10-09T05:34:41.702205+0000","last_active":"2023-10-09T05:35:13.597552+0000","last_peered":"2023-10-09T05:35:13.597552+0000","last_clean":"2023-10-09T05:35:13.597552+0000","last_became_active":"2023-10-09T05:34:41.701849+0000","last_became_peered":"2023-10-09T05:34:41.701849+0000","last_unstale":"2023-10-09T05:35:13.597552+0000","last_undegraded":"2023-10-09T05:35:13.597552+0000","last_fullsized":"2023-10-09T05:35:13.597552+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.591612+0000","last_change":"2023-10-09T05:34:41.663111+0000","last_active":"2023-10-09T05:35:13.591612+0000","last_peered":"2023-10-09T05:35:13.591612+0000","last_clean":"2023-10-09T05:35:13.591612+0000","last_became_active":"2023-10-09T05:34:41.662816+0000","last_became_peered":"2023-10-09T05:34:41.662816+0000","last_unstale":"2023-10-09T05:35:13.591612+0000","last_undegraded":"2023-10-09T05:35:13.591612+0000","last_fullsized":"2023-10-09T05:35:13.591612+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:36.226743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.16","version":"138'8898","reported_seq":8959,"reported_epoch":138,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:30.923434+0000","last_change":"2023-10-09T05:34:42.899801+0000","last_active":"2023-10-09T05:35:30.923434+0000","last_peered":"2023-10-09T05:35:30.923434+0000","last_clean":"2023-10-09T05:33:20.529830+0000","last_became_active":"2023-10-09T05:34:42.891787+0000","last_became_peered":"2023-10-09T05:34:42.891787+0000","last_unstale":"2023-10-09T05:35:30.923434+0000","last_undegraded":"2023-10-09T05:34:42.686663+0000","last_fullsized":"2023-10-09T05:34:42.658538+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8898,"log_dups_size":0,"ondisk_log_size":8898,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102105088,"num_objects":1558,"num_object_clones":0,"num_object_copies":4674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2590,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5830,"num_write_kb":236416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-419}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":113,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.598703+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:35:13.598703+0000","last_peered":"2023-10-09T05:35:13.598703+0000","last_clean":"2023-10-09T05:35:13.598703+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:35:13.598703+0000","last_undegraded":"2023-10-09T05:35:13.598703+0000","last_fullsized":"2023-10-09T05:35:13.598703+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":123,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.600273+0000","last_change":"2023-10-09T05:34:41.664294+0000","last_active":"2023-10-09T05:35:13.600273+0000","last_peered":"2023-10-09T05:35:13.600273+0000","last_clean":"2023-10-09T05:35:13.600273+0000","last_became_active":"2023-10-09T05:34:41.663985+0000","last_became_peered":"2023-10-09T05:34:41.663985+0000","last_unstale":"2023-10-09T05:35:13.600273+0000","last_undegraded":"2023-10-09T05:35:13.600273+0000","last_fullsized":"2023-10-09T05:35:13.600273+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:58.417307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"138'8942","reported_seq":9002,"reported_epoch":138,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:34.822751+0000","last_change":"2023-10-09T05:34:42.977224+0000","last_active":"2023-10-09T05:35:34.822751+0000","last_peered":"2023-10-09T05:35:34.822751+0000","last_clean":"2023-10-09T05:33:20.515228+0000","last_became_active":"2023-10-09T05:34:42.967706+0000","last_became_peered":"2023-10-09T05:34:42.967706+0000","last_unstale":"2023-10-09T05:35:34.822751+0000","last_undegraded":"2023-10-09T05:34:42.852684+0000","last_fullsized":"2023-10-09T05:34:42.677882+0000","mapping_epoch":128,"log_start":"108'88","ondisk_log_start":"108'88","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8854,"log_dups_size":0,"ondisk_log_size":8854,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94109696,"num_objects":1436,"num_object_clones":0,"num_object_copies":4308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2614,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5882,"num_write_kb":234176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":-567}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.598119+0000","last_change":"2023-10-09T05:34:41.688229+0000","last_active":"2023-10-09T05:35:13.598119+0000","last_peered":"2023-10-09T05:35:13.598119+0000","last_clean":"2023-10-09T05:35:13.598119+0000","last_became_active":"2023-10-09T05:34:41.687795+0000","last_became_peered":"2023-10-09T05:34:41.687795+0000","last_unstale":"2023-10-09T05:35:13.598119+0000","last_undegraded":"2023-10-09T05:35:13.598119+0000","last_fullsized":"2023-10-09T05:35:13.598119+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.594527+0000","last_change":"2023-10-09T05:34:41.678569+0000","last_active":"2023-10-09T05:35:13.594527+0000","last_peered":"2023-10-09T05:35:13.594527+0000","last_clean":"2023-10-09T05:35:13.594527+0000","last_became_active":"2023-10-09T05:34:41.678271+0000","last_became_peered":"2023-10-09T05:34:41.678271+0000","last_unstale":"2023-10-09T05:35:13.594527+0000","last_undegraded":"2023-10-09T05:35:13.594527+0000","last_fullsized":"2023-10-09T05:35:13.594527+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"138'11524","reported_seq":11864,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:30.866607+0000","last_change":"2023-10-09T05:34:25.139244+0000","last_active":"2023-10-09T05:35:30.866607+0000","last_peered":"2023-10-09T05:35:30.866607+0000","last_clean":"2023-10-09T05:35:30.866607+0000","last_became_active":"2023-10-09T05:34:01.629769+0000","last_became_peered":"2023-10-09T05:34:01.629769+0000","last_unstale":"2023-10-09T05:35:30.866607+0000","last_undegraded":"2023-10-09T05:35:30.866607+0000","last_fullsized":"2023-10-09T05:35:30.866607+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":3497,"log_size":11524,"log_dups_size":0,"ondisk_log_size":11524,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103645184,"num_objects":1581,"num_object_clones":0,"num_object_copies":4743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5827,"num_write_kb":237088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":86,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.597923+0000","last_change":"2023-10-09T05:34:41.674752+0000","last_active":"2023-10-09T05:35:13.597923+0000","last_peered":"2023-10-09T05:35:13.597923+0000","last_clean":"2023-10-09T05:35:13.597923+0000","last_became_active":"2023-10-09T05:34:41.674253+0000","last_became_peered":"2023-10-09T05:34:41.674253+0000","last_unstale":"2023-10-09T05:35:13.597923+0000","last_undegraded":"2023-10-09T05:35:13.597923+0000","last_fullsized":"2023-10-09T05:35:13.597923+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.603039+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:35:13.603039+0000","last_peered":"2023-10-09T05:35:13.603039+0000","last_clean":"2023-10-09T05:35:13.603039+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:35:13.603039+0000","last_undegraded":"2023-10-09T05:35:13.603039+0000","last_fullsized":"2023-10-09T05:35:13.603039+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"138'9675","reported_seq":9729,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:30.815012+0000","last_change":"2023-10-09T05:34:42.657980+0000","last_active":"2023-10-09T05:35:30.815012+0000","last_peered":"2023-10-09T05:35:30.815012+0000","last_clean":"2023-10-09T05:35:30.815012+0000","last_became_active":"2023-10-09T05:34:25.039138+0000","last_became_peered":"2023-10-09T05:34:25.039138+0000","last_unstale":"2023-10-09T05:35:30.815012+0000","last_undegraded":"2023-10-09T05:35:30.815012+0000","last_fullsized":"2023-10-09T05:35:30.815012+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9675,"log_dups_size":0,"ondisk_log_size":9675,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97517568,"num_objects":1488,"num_object_clones":0,"num_object_copies":4464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5937,"num_write_kb":237600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":77,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.597720+0000","last_change":"2023-10-09T05:34:41.691449+0000","last_active":"2023-10-09T05:35:13.597720+0000","last_peered":"2023-10-09T05:35:13.597720+0000","last_clean":"2023-10-09T05:35:13.597720+0000","last_became_active":"2023-10-09T05:34:41.675227+0000","last_became_peered":"2023-10-09T05:34:41.675227+0000","last_unstale":"2023-10-09T05:35:13.597720+0000","last_undegraded":"2023-10-09T05:35:13.597720+0000","last_fullsized":"2023-10-09T05:35:13.597720+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":128,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.599141+0000","last_change":"2023-10-09T05:34:41.669908+0000","last_active":"2023-10-09T05:35:13.599141+0000","last_peered":"2023-10-09T05:35:13.599141+0000","last_clean":"2023-10-09T05:35:13.599141+0000","last_became_active":"2023-10-09T05:34:41.669596+0000","last_became_peered":"2023-10-09T05:34:41.669596+0000","last_unstale":"2023-10-09T05:35:13.599141+0000","last_undegraded":"2023-10-09T05:35:13.599141+0000","last_fullsized":"2023-10-09T05:35:13.599141+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"138'9481","reported_seq":9610,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:30.909384+0000","last_change":"2023-10-09T05:34:35.674328+0000","last_active":"2023-10-09T05:35:30.909384+0000","last_peered":"2023-10-09T05:35:30.909384+0000","last_clean":"2023-10-09T05:35:30.909384+0000","last_became_active":"2023-10-09T05:34:25.008155+0000","last_became_peered":"2023-10-09T05:34:25.008155+0000","last_unstale":"2023-10-09T05:35:30.909384+0000","last_undegraded":"2023-10-09T05:35:30.909384+0000","last_fullsized":"2023-10-09T05:35:30.909384+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7846","last_scrub_stamp":"2023-10-09T05:34:35.674093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:35.674093+0000","objects_scrubbed":3275,"log_size":9481,"log_dups_size":0,"ondisk_log_size":9481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:02.820818+0000","scrub_duration":0.75420257000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106758144,"num_objects":1629,"num_object_clones":0,"num_object_copies":4887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1646,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":105,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595313+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:35:13.595313+0000","last_peered":"2023-10-09T05:35:13.595313+0000","last_clean":"2023-10-09T05:35:13.595313+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:35:13.595313+0000","last_undegraded":"2023-10-09T05:35:13.595313+0000","last_fullsized":"2023-10-09T05:35:13.595313+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":88,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.592815+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:35:13.592815+0000","last_peered":"2023-10-09T05:35:13.592815+0000","last_clean":"2023-10-09T05:35:13.592815+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:35:13.592815+0000","last_undegraded":"2023-10-09T05:35:13.592815+0000","last_fullsized":"2023-10-09T05:35:13.592815+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"138'9625","reported_seq":12847,"reported_epoch":138,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:30.966930+0000","last_change":"2023-10-09T05:35:07.068871+0000","last_active":"2023-10-09T05:35:30.966930+0000","last_peered":"2023-10-09T05:35:30.966930+0000","last_clean":"2023-10-09T05:33:20.502869+0000","last_became_active":"2023-10-09T05:34:42.879684+0000","last_became_peered":"2023-10-09T05:34:42.879684+0000","last_unstale":"2023-10-09T05:35:30.966930+0000","last_undegraded":"2023-10-09T05:34:42.683747+0000","last_fullsized":"2023-10-09T05:34:42.658139+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9625,"log_dups_size":0,"ondisk_log_size":9625,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101515276,"num_objects":1549,"num_object_clones":0,"num_object_copies":4647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":380,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5882,"num_write_kb":237792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1578,"num_bytes_recovered":103415808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":1169}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":106,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.603069+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:35:13.603069+0000","last_peered":"2023-10-09T05:35:13.603069+0000","last_clean":"2023-10-09T05:35:13.603069+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:35:13.603069+0000","last_undegraded":"2023-10-09T05:35:13.603069+0000","last_fullsized":"2023-10-09T05:35:13.603069+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.601708+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:35:13.601708+0000","last_peered":"2023-10-09T05:35:13.601708+0000","last_clean":"2023-10-09T05:35:13.601708+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:35:13.601708+0000","last_undegraded":"2023-10-09T05:35:13.601708+0000","last_fullsized":"2023-10-09T05:35:13.601708+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"138'9671","reported_seq":9799,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:33.464714+0000","last_change":"2023-10-09T05:34:40.457466+0000","last_active":"2023-10-09T05:35:33.464714+0000","last_peered":"2023-10-09T05:35:33.464714+0000","last_clean":"2023-10-09T05:35:33.464714+0000","last_became_active":"2023-10-09T05:34:25.026725+0000","last_became_peered":"2023-10-09T05:34:25.026725+0000","last_unstale":"2023-10-09T05:35:33.464714+0000","last_undegraded":"2023-10-09T05:35:33.464714+0000","last_fullsized":"2023-10-09T05:35:33.464714+0000","mapping_epoch":105,"log_start":"108'200","ondisk_log_start":"108'200","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'8107","last_scrub_stamp":"2023-10-09T05:34:40.457321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:40.457321+0000","objects_scrubbed":3100,"log_size":9471,"log_dups_size":0,"ondisk_log_size":9471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:26.084493+0000","scrub_duration":0.74267715700000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99745792,"num_objects":1522,"num_object_clones":0,"num_object_copies":4566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1578,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":105,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.599002+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:35:13.599002+0000","last_peered":"2023-10-09T05:35:13.599002+0000","last_clean":"2023-10-09T05:35:13.599002+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:35:13.599002+0000","last_undegraded":"2023-10-09T05:35:13.599002+0000","last_fullsized":"2023-10-09T05:35:13.599002+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.601152+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:35:13.601152+0000","last_peered":"2023-10-09T05:35:13.601152+0000","last_clean":"2023-10-09T05:35:13.601152+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:35:13.601152+0000","last_undegraded":"2023-10-09T05:35:13.601152+0000","last_fullsized":"2023-10-09T05:35:13.601152+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"138'9559","reported_seq":9612,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:34.755772+0000","last_change":"2023-10-09T05:34:39.803946+0000","last_active":"2023-10-09T05:35:34.755772+0000","last_peered":"2023-10-09T05:35:34.755772+0000","last_clean":"2023-10-09T05:35:34.755772+0000","last_became_active":"2023-10-09T05:34:25.018175+0000","last_became_peered":"2023-10-09T05:34:25.018175+0000","last_unstale":"2023-10-09T05:35:34.755772+0000","last_undegraded":"2023-10-09T05:35:34.755772+0000","last_fullsized":"2023-10-09T05:35:34.755772+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9559,"log_dups_size":0,"ondisk_log_size":9559,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97550336,"num_objects":1488,"num_object_clones":0,"num_object_copies":4464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5857,"num_write_kb":235072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.602857+0000","last_change":"2023-10-09T05:34:41.706409+0000","last_active":"2023-10-09T05:35:13.602857+0000","last_peered":"2023-10-09T05:35:13.602857+0000","last_clean":"2023-10-09T05:35:13.602857+0000","last_became_active":"2023-10-09T05:34:41.706002+0000","last_became_peered":"2023-10-09T05:34:41.706002+0000","last_unstale":"2023-10-09T05:35:13.602857+0000","last_undegraded":"2023-10-09T05:35:13.602857+0000","last_fullsized":"2023-10-09T05:35:13.602857+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":113,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.601661+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:35:13.601661+0000","last_peered":"2023-10-09T05:35:13.601661+0000","last_clean":"2023-10-09T05:35:13.601661+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:35:13.601661+0000","last_undegraded":"2023-10-09T05:35:13.601661+0000","last_fullsized":"2023-10-09T05:35:13.601661+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"138'9727","reported_seq":9783,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:34.807077+0000","last_change":"2023-10-09T05:34:42.153027+0000","last_active":"2023-10-09T05:35:34.807077+0000","last_peered":"2023-10-09T05:35:34.807077+0000","last_clean":"2023-10-09T05:35:34.807077+0000","last_became_active":"2023-10-09T05:34:25.031927+0000","last_became_peered":"2023-10-09T05:34:25.031927+0000","last_unstale":"2023-10-09T05:35:34.807077+0000","last_undegraded":"2023-10-09T05:35:34.807077+0000","last_fullsized":"2023-10-09T05:35:34.807077+0000","mapping_epoch":105,"log_start":"109'800","ondisk_log_start":"109'800","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'8112","last_scrub_stamp":"2023-10-09T05:34:42.152946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:42.152946+0000","objects_scrubbed":0,"log_size":8927,"log_dups_size":0,"ondisk_log_size":8927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97779712,"num_objects":1492,"num_object_clones":0,"num_object_copies":4476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1615,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":124,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.600557+0000","last_change":"2023-10-09T05:34:41.666221+0000","last_active":"2023-10-09T05:35:13.600557+0000","last_peered":"2023-10-09T05:35:13.600557+0000","last_clean":"2023-10-09T05:35:13.600557+0000","last_became_active":"2023-10-09T05:34:41.665842+0000","last_became_peered":"2023-10-09T05:34:41.665842+0000","last_unstale":"2023-10-09T05:35:13.600557+0000","last_undegraded":"2023-10-09T05:35:13.600557+0000","last_fullsized":"2023-10-09T05:35:13.600557+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:28.435851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":83,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.596057+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:35:13.596057+0000","last_peered":"2023-10-09T05:35:13.596057+0000","last_clean":"2023-10-09T05:35:13.596057+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:35:13.596057+0000","last_undegraded":"2023-10-09T05:35:13.596057+0000","last_fullsized":"2023-10-09T05:35:13.596057+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":662,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595629+0000","last_change":"2023-10-09T05:34:41.667358+0000","last_active":"2023-10-09T05:35:13.595629+0000","last_peered":"2023-10-09T05:35:13.595629+0000","last_clean":"2023-10-09T05:35:13.595629+0000","last_became_active":"2023-10-09T05:34:41.666757+0000","last_became_peered":"2023-10-09T05:34:41.666757+0000","last_unstale":"2023-10-09T05:35:13.595629+0000","last_undegraded":"2023-10-09T05:35:13.595629+0000","last_fullsized":"2023-10-09T05:35:13.595629+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:45:47.725111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595874+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:35:13.595874+0000","last_peered":"2023-10-09T05:35:13.595874+0000","last_clean":"2023-10-09T05:35:13.595874+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:35:13.595874+0000","last_undegraded":"2023-10-09T05:35:13.595874+0000","last_fullsized":"2023-10-09T05:35:13.595874+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":68,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.596761+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:35:13.596761+0000","last_peered":"2023-10-09T05:35:13.596761+0000","last_clean":"2023-10-09T05:35:13.596761+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:35:13.596761+0000","last_undegraded":"2023-10-09T05:35:13.596761+0000","last_fullsized":"2023-10-09T05:35:13.596761+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"138'7742","reported_seq":7885,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:32.634988+0000","last_change":"2023-10-09T05:35:16.964364+0000","last_active":"2023-10-09T05:35:32.634988+0000","last_peered":"2023-10-09T05:35:32.634988+0000","last_clean":"2023-10-09T05:35:32.634988+0000","last_became_active":"2023-10-09T05:34:25.031183+0000","last_became_peered":"2023-10-09T05:34:25.031183+0000","last_unstale":"2023-10-09T05:35:32.634988+0000","last_undegraded":"2023-10-09T05:35:32.634988+0000","last_fullsized":"2023-10-09T05:35:32.634988+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'6300","last_scrub_stamp":"2023-10-09T05:34:08.749711+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:08.749711+0000","objects_scrubbed":3317,"log_size":7742,"log_dups_size":0,"ondisk_log_size":7742,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.61798832400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50495488,"num_objects":770,"num_object_clones":0,"num_object_copies":2310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1254,"num_write_kb":11712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"138'9071","reported_seq":9201,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:32.652756+0000","last_change":"2023-10-09T05:34:10.842783+0000","last_active":"2023-10-09T05:35:32.652756+0000","last_peered":"2023-10-09T05:35:32.652756+0000","last_clean":"2023-10-09T05:35:32.652756+0000","last_became_active":"2023-10-09T05:34:01.606336+0000","last_became_peered":"2023-10-09T05:34:01.606336+0000","last_unstale":"2023-10-09T05:35:32.652756+0000","last_undegraded":"2023-10-09T05:35:32.652756+0000","last_fullsized":"2023-10-09T05:35:32.652756+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'6606","last_scrub_stamp":"2023-10-09T05:34:10.842550+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:10.842550+0000","objects_scrubbed":3389,"log_size":9071,"log_dups_size":0,"ondisk_log_size":9071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:40.600228+0000","scrub_duration":0.74463188000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100139008,"num_objects":1528,"num_object_clones":0,"num_object_copies":4584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2491,"num_write_kb":20160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":574,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595031+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:35:13.595031+0000","last_peered":"2023-10-09T05:35:13.595031+0000","last_clean":"2023-10-09T05:35:13.595031+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:35:13.595031+0000","last_undegraded":"2023-10-09T05:35:13.595031+0000","last_fullsized":"2023-10-09T05:35:13.595031+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":105,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.600981+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:35:13.600981+0000","last_peered":"2023-10-09T05:35:13.600981+0000","last_clean":"2023-10-09T05:35:13.600981+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:35:13.600981+0000","last_undegraded":"2023-10-09T05:35:13.600981+0000","last_fullsized":"2023-10-09T05:35:13.600981+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":117,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595117+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:35:13.595117+0000","last_peered":"2023-10-09T05:35:13.595117+0000","last_clean":"2023-10-09T05:35:13.595117+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:35:13.595117+0000","last_undegraded":"2023-10-09T05:35:13.595117+0000","last_fullsized":"2023-10-09T05:35:13.595117+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"138'8884","reported_seq":8961,"reported_epoch":138,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:30.869978+0000","last_change":"2023-10-09T05:34:42.959034+0000","last_active":"2023-10-09T05:35:30.869978+0000","last_peered":"2023-10-09T05:35:30.869978+0000","last_clean":"2023-10-09T05:33:20.529830+0000","last_became_active":"2023-10-09T05:34:42.879241+0000","last_became_peered":"2023-10-09T05:34:42.879241+0000","last_unstale":"2023-10-09T05:35:30.869978+0000","last_undegraded":"2023-10-09T05:34:42.688699+0000","last_fullsized":"2023-10-09T05:34:42.658753+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8884,"log_dups_size":0,"ondisk_log_size":8884,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103153664,"num_objects":1574,"num_object_clones":0,"num_object_copies":4722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5816,"num_write_kb":236480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-421}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":450,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.587547+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:35:13.587547+0000","last_peered":"2023-10-09T05:35:13.587547+0000","last_clean":"2023-10-09T05:35:13.587547+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:35:13.587547+0000","last_undegraded":"2023-10-09T05:35:13.587547+0000","last_fullsized":"2023-10-09T05:35:13.587547+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595785+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:35:13.595785+0000","last_peered":"2023-10-09T05:35:13.595785+0000","last_clean":"2023-10-09T05:35:13.595785+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:35:13.595785+0000","last_undegraded":"2023-10-09T05:35:13.595785+0000","last_fullsized":"2023-10-09T05:35:13.595785+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":113,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595923+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:35:13.595923+0000","last_peered":"2023-10-09T05:35:13.595923+0000","last_clean":"2023-10-09T05:35:13.595923+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:35:13.595923+0000","last_undegraded":"2023-10-09T05:35:13.595923+0000","last_fullsized":"2023-10-09T05:35:13.595923+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":714,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595920+0000","last_change":"2023-10-09T05:34:41.677976+0000","last_active":"2023-10-09T05:35:13.595920+0000","last_peered":"2023-10-09T05:35:13.595920+0000","last_clean":"2023-10-09T05:35:13.595920+0000","last_became_active":"2023-10-09T05:34:41.677520+0000","last_became_peered":"2023-10-09T05:34:41.677520+0000","last_unstale":"2023-10-09T05:35:13.595920+0000","last_undegraded":"2023-10-09T05:35:13.595920+0000","last_fullsized":"2023-10-09T05:35:13.595920+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":315,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.597078+0000","last_change":"2023-10-09T05:34:41.686013+0000","last_active":"2023-10-09T05:35:13.597078+0000","last_peered":"2023-10-09T05:35:13.597078+0000","last_clean":"2023-10-09T05:35:13.597078+0000","last_became_active":"2023-10-09T05:34:41.680809+0000","last_became_peered":"2023-10-09T05:34:41.680809+0000","last_unstale":"2023-10-09T05:35:13.597078+0000","last_undegraded":"2023-10-09T05:35:13.597078+0000","last_fullsized":"2023-10-09T05:35:13.597078+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.600671+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:35:13.600671+0000","last_peered":"2023-10-09T05:35:13.600671+0000","last_clean":"2023-10-09T05:35:13.600671+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:35:13.600671+0000","last_undegraded":"2023-10-09T05:35:13.600671+0000","last_fullsized":"2023-10-09T05:35:13.600671+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.598915+0000","last_change":"2023-10-09T05:34:41.693650+0000","last_active":"2023-10-09T05:35:13.598915+0000","last_peered":"2023-10-09T05:35:13.598915+0000","last_clean":"2023-10-09T05:35:13.598915+0000","last_became_active":"2023-10-09T05:34:41.693305+0000","last_became_peered":"2023-10-09T05:34:41.693305+0000","last_unstale":"2023-10-09T05:35:13.598915+0000","last_undegraded":"2023-10-09T05:35:13.598915+0000","last_fullsized":"2023-10-09T05:35:13.598915+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"138'8796","reported_seq":8858,"reported_epoch":138,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:30.928927+0000","last_change":"2023-10-09T05:34:42.906080+0000","last_active":"2023-10-09T05:35:30.928927+0000","last_peered":"2023-10-09T05:35:30.928927+0000","last_clean":"2023-10-09T05:33:20.493128+0000","last_became_active":"2023-10-09T05:34:42.894560+0000","last_became_peered":"2023-10-09T05:34:42.894560+0000","last_unstale":"2023-10-09T05:35:30.928927+0000","last_undegraded":"2023-10-09T05:34:42.694084+0000","last_fullsized":"2023-10-09T05:34:42.658067+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8796,"log_dups_size":0,"ondisk_log_size":8796,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99647488,"num_objects":1520,"num_object_clones":0,"num_object_copies":4560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2624,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5766,"num_write_kb":233184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-438}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.7","version":"138'8897","reported_seq":8961,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:34.743344+0000","last_change":"2023-10-09T05:34:34.976158+0000","last_active":"2023-10-09T05:35:34.743344+0000","last_peered":"2023-10-09T05:35:34.743344+0000","last_clean":"2023-10-09T05:35:34.743344+0000","last_became_active":"2023-10-09T05:34:01.592987+0000","last_became_peered":"2023-10-09T05:34:01.592987+0000","last_unstale":"2023-10-09T05:35:34.743344+0000","last_undegraded":"2023-10-09T05:35:34.743344+0000","last_fullsized":"2023-10-09T05:35:34.743344+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7154","last_scrub_stamp":"2023-10-09T05:34:34.976067+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:34.976067+0000","objects_scrubbed":0,"log_size":8897,"log_dups_size":0,"ondisk_log_size":8897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100794368,"num_objects":1538,"num_object_clones":0,"num_object_copies":4614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1769,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":738,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.596855+0000","last_change":"2023-10-09T05:34:41.668526+0000","last_active":"2023-10-09T05:35:13.596855+0000","last_peered":"2023-10-09T05:35:13.596855+0000","last_clean":"2023-10-09T05:35:13.596855+0000","last_became_active":"2023-10-09T05:34:41.668010+0000","last_became_peered":"2023-10-09T05:34:41.668010+0000","last_unstale":"2023-10-09T05:35:13.596855+0000","last_undegraded":"2023-10-09T05:35:13.596855+0000","last_fullsized":"2023-10-09T05:35:13.596855+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":88,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.591717+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:35:13.591717+0000","last_peered":"2023-10-09T05:35:13.591717+0000","last_clean":"2023-10-09T05:35:13.591717+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:35:13.591717+0000","last_undegraded":"2023-10-09T05:35:13.591717+0000","last_fullsized":"2023-10-09T05:35:13.591717+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.597541+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:35:13.597541+0000","last_peered":"2023-10-09T05:35:13.597541+0000","last_clean":"2023-10-09T05:35:13.597541+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:35:13.597541+0000","last_undegraded":"2023-10-09T05:35:13.597541+0000","last_fullsized":"2023-10-09T05:35:13.597541+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":341,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.597689+0000","last_change":"2023-10-09T05:34:41.682539+0000","last_active":"2023-10-09T05:35:13.597689+0000","last_peered":"2023-10-09T05:35:13.597689+0000","last_clean":"2023-10-09T05:35:13.597689+0000","last_became_active":"2023-10-09T05:34:41.674099+0000","last_became_peered":"2023-10-09T05:34:41.674099+0000","last_unstale":"2023-10-09T05:35:13.597689+0000","last_undegraded":"2023-10-09T05:35:13.597689+0000","last_fullsized":"2023-10-09T05:35:13.597689+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595983+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:35:13.595983+0000","last_peered":"2023-10-09T05:35:13.595983+0000","last_clean":"2023-10-09T05:35:13.595983+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:35:13.595983+0000","last_undegraded":"2023-10-09T05:35:13.595983+0000","last_fullsized":"2023-10-09T05:35:13.595983+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":88,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.593034+0000","last_change":"2023-10-09T05:34:41.671414+0000","last_active":"2023-10-09T05:35:13.593034+0000","last_peered":"2023-10-09T05:35:13.593034+0000","last_clean":"2023-10-09T05:35:13.593034+0000","last_became_active":"2023-10-09T05:34:41.671100+0000","last_became_peered":"2023-10-09T05:34:41.671100+0000","last_unstale":"2023-10-09T05:35:13.593034+0000","last_undegraded":"2023-10-09T05:35:13.593034+0000","last_fullsized":"2023-10-09T05:35:13.593034+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:19.832596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"138'7836","reported_seq":7907,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:34.623265+0000","last_change":"2023-10-09T05:34:33.321883+0000","last_active":"2023-10-09T05:35:34.623265+0000","last_peered":"2023-10-09T05:35:34.623265+0000","last_clean":"2023-10-09T05:35:34.623265+0000","last_became_active":"2023-10-09T05:34:25.017164+0000","last_became_peered":"2023-10-09T05:34:25.017164+0000","last_unstale":"2023-10-09T05:35:34.623265+0000","last_undegraded":"2023-10-09T05:35:34.623265+0000","last_fullsized":"2023-10-09T05:35:34.623265+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'6928","last_scrub_stamp":"2023-10-09T05:34:33.321793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:33.321793+0000","objects_scrubbed":0,"log_size":7836,"log_dups_size":0,"ondisk_log_size":7836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49020928,"num_objects":748,"num_object_clones":0,"num_object_copies":2244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":919,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.600355+0000","last_change":"2023-10-09T05:34:41.664569+0000","last_active":"2023-10-09T05:35:13.600355+0000","last_peered":"2023-10-09T05:35:13.600355+0000","last_clean":"2023-10-09T05:35:13.600355+0000","last_became_active":"2023-10-09T05:34:41.664239+0000","last_became_peered":"2023-10-09T05:34:41.664239+0000","last_unstale":"2023-10-09T05:35:13.600355+0000","last_undegraded":"2023-10-09T05:35:13.600355+0000","last_fullsized":"2023-10-09T05:35:13.600355+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:26.511248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":524,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595763+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:35:13.595763+0000","last_peered":"2023-10-09T05:35:13.595763+0000","last_clean":"2023-10-09T05:35:13.595763+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:35:13.595763+0000","last_undegraded":"2023-10-09T05:35:13.595763+0000","last_fullsized":"2023-10-09T05:35:13.595763+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":119,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:25.666949+0000","last_change":"2023-10-09T05:34:41.666200+0000","last_active":"2023-10-09T05:35:25.666949+0000","last_peered":"2023-10-09T05:35:25.666949+0000","last_clean":"2023-10-09T05:35:25.666949+0000","last_became_active":"2023-10-09T05:34:41.665841+0000","last_became_peered":"2023-10-09T05:34:41.665841+0000","last_unstale":"2023-10-09T05:35:25.666949+0000","last_undegraded":"2023-10-09T05:35:25.666949+0000","last_fullsized":"2023-10-09T05:35:25.666949+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:58.338181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"138'7782","reported_seq":10483,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:34.345379+0000","last_change":"2023-10-09T05:34:57.102099+0000","last_active":"2023-10-09T05:35:34.345379+0000","last_peered":"2023-10-09T05:35:34.345379+0000","last_clean":"2023-10-09T05:35:34.345379+0000","last_became_active":"2023-10-09T05:34:57.096500+0000","last_became_peered":"2023-10-09T05:34:57.096500+0000","last_unstale":"2023-10-09T05:35:34.345379+0000","last_undegraded":"2023-10-09T05:35:34.345379+0000","last_fullsized":"2023-10-09T05:35:34.345379+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":7782,"log_dups_size":0,"ondisk_log_size":7782,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49659904,"num_objects":757,"num_object_clones":0,"num_object_copies":2271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2912,"num_write_kb":117456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1310,"num_bytes_recovered":65667072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":601,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.599172+0000","last_change":"2023-10-09T05:34:41.676842+0000","last_active":"2023-10-09T05:35:13.599172+0000","last_peered":"2023-10-09T05:35:13.599172+0000","last_clean":"2023-10-09T05:35:13.599172+0000","last_became_active":"2023-10-09T05:34:41.676428+0000","last_became_peered":"2023-10-09T05:34:41.676428+0000","last_unstale":"2023-10-09T05:35:13.599172+0000","last_undegraded":"2023-10-09T05:35:13.599172+0000","last_fullsized":"2023-10-09T05:35:13.599172+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":91,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.597474+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:35:13.597474+0000","last_peered":"2023-10-09T05:35:13.597474+0000","last_clean":"2023-10-09T05:35:13.597474+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:35:13.597474+0000","last_undegraded":"2023-10-09T05:35:13.597474+0000","last_fullsized":"2023-10-09T05:35:13.597474+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":89,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.602924+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:35:13.602924+0000","last_peered":"2023-10-09T05:35:13.602924+0000","last_clean":"2023-10-09T05:35:13.602924+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:35:13.602924+0000","last_undegraded":"2023-10-09T05:35:13.602924+0000","last_fullsized":"2023-10-09T05:35:13.602924+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"138'7769","reported_seq":7949,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:30.903732+0000","last_change":"2023-10-09T05:34:33.714772+0000","last_active":"2023-10-09T05:35:30.903732+0000","last_peered":"2023-10-09T05:35:30.903732+0000","last_clean":"2023-10-09T05:35:30.903732+0000","last_became_active":"2023-10-09T05:34:25.048537+0000","last_became_peered":"2023-10-09T05:34:25.048537+0000","last_unstale":"2023-10-09T05:35:30.903732+0000","last_undegraded":"2023-10-09T05:35:30.903732+0000","last_fullsized":"2023-10-09T05:35:30.903732+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'6879","last_scrub_stamp":"2023-10-09T05:34:33.714537+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:33.714537+0000","objects_scrubbed":1680,"log_size":7769,"log_dups_size":0,"ondisk_log_size":7769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:15.987881+0000","scrub_duration":0.49573348099999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51511296,"num_objects":786,"num_object_clones":0,"num_object_copies":2358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":894,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1d","version":"138'9600","reported_seq":9664,"reported_epoch":138,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:30.927714+0000","last_change":"2023-10-09T05:34:42.908183+0000","last_active":"2023-10-09T05:35:30.927714+0000","last_peered":"2023-10-09T05:35:30.927714+0000","last_clean":"2023-10-09T05:33:20.502869+0000","last_became_active":"2023-10-09T05:34:42.896545+0000","last_became_peered":"2023-10-09T05:34:42.896545+0000","last_unstale":"2023-10-09T05:35:30.927714+0000","last_undegraded":"2023-10-09T05:34:42.687614+0000","last_fullsized":"2023-10-09T05:34:42.658151+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9600,"log_dups_size":0,"ondisk_log_size":9600,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103153676,"num_objects":1575,"num_object_clones":0,"num_object_copies":4725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2695,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1575,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5858,"num_write_kb":237793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-446}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":91,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.592962+0000","last_change":"2023-10-09T05:34:41.662989+0000","last_active":"2023-10-09T05:35:13.592962+0000","last_peered":"2023-10-09T05:35:13.592962+0000","last_clean":"2023-10-09T05:35:13.592962+0000","last_became_active":"2023-10-09T05:34:41.662621+0000","last_became_peered":"2023-10-09T05:34:41.662621+0000","last_unstale":"2023-10-09T05:35:13.592962+0000","last_undegraded":"2023-10-09T05:35:13.592962+0000","last_fullsized":"2023-10-09T05:35:13.592962+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:08.750044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":127,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.589712+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:35:13.589712+0000","last_peered":"2023-10-09T05:35:13.589712+0000","last_clean":"2023-10-09T05:35:13.589712+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:35:13.589712+0000","last_undegraded":"2023-10-09T05:35:13.589712+0000","last_fullsized":"2023-10-09T05:35:13.589712+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.1a","version":"138'9474","reported_seq":9526,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:30.882902+0000","last_change":"2023-10-09T05:34:25.055227+0000","last_active":"2023-10-09T05:35:30.882902+0000","last_peered":"2023-10-09T05:35:30.882902+0000","last_clean":"2023-10-09T05:35:30.882902+0000","last_became_active":"2023-10-09T05:34:25.051761+0000","last_became_peered":"2023-10-09T05:34:25.051761+0000","last_unstale":"2023-10-09T05:35:30.882902+0000","last_undegraded":"2023-10-09T05:35:30.882902+0000","last_fullsized":"2023-10-09T05:35:30.882902+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9474,"log_dups_size":0,"ondisk_log_size":9474,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106594304,"num_objects":1627,"num_object_clones":0,"num_object_copies":4881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5756,"num_write_kb":236224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":107,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595396+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:35:13.595396+0000","last_peered":"2023-10-09T05:35:13.595396+0000","last_clean":"2023-10-09T05:35:13.595396+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:35:13.595396+0000","last_undegraded":"2023-10-09T05:35:13.595396+0000","last_fullsized":"2023-10-09T05:35:13.595396+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.601057+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:35:13.601057+0000","last_peered":"2023-10-09T05:35:13.601057+0000","last_clean":"2023-10-09T05:35:13.601057+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:35:13.601057+0000","last_undegraded":"2023-10-09T05:35:13.601057+0000","last_fullsized":"2023-10-09T05:35:13.601057+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1b","version":"138'9505","reported_seq":9557,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:30.881070+0000","last_change":"2023-10-09T05:34:25.065203+0000","last_active":"2023-10-09T05:35:30.881070+0000","last_peered":"2023-10-09T05:35:30.881070+0000","last_clean":"2023-10-09T05:35:30.881070+0000","last_became_active":"2023-10-09T05:34:25.059858+0000","last_became_peered":"2023-10-09T05:34:25.059858+0000","last_unstale":"2023-10-09T05:35:30.881070+0000","last_undegraded":"2023-10-09T05:35:30.881070+0000","last_fullsized":"2023-10-09T05:35:30.881070+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9505,"log_dups_size":0,"ondisk_log_size":9505,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108658688,"num_objects":1658,"num_object_clones":0,"num_object_copies":4974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5767,"num_write_kb":237600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.595948+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:35:13.595948+0000","last_peered":"2023-10-09T05:35:13.595948+0000","last_clean":"2023-10-09T05:35:13.595948+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:35:13.595948+0000","last_undegraded":"2023-10-09T05:35:13.595948+0000","last_fullsized":"2023-10-09T05:35:13.595948+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.603324+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:35:13.603324+0000","last_peered":"2023-10-09T05:35:13.603324+0000","last_clean":"2023-10-09T05:35:13.603324+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:35:13.603324+0000","last_undegraded":"2023-10-09T05:35:13.603324+0000","last_fullsized":"2023-10-09T05:35:13.603324+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"138'8994","reported_seq":9127,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:30.922580+0000","last_change":"2023-10-09T05:34:07.818690+0000","last_active":"2023-10-09T05:35:30.922580+0000","last_peered":"2023-10-09T05:35:30.922580+0000","last_clean":"2023-10-09T05:35:30.922580+0000","last_became_active":"2023-10-09T05:34:01.656619+0000","last_became_peered":"2023-10-09T05:34:01.656619+0000","last_unstale":"2023-10-09T05:35:30.922580+0000","last_undegraded":"2023-10-09T05:35:30.922580+0000","last_fullsized":"2023-10-09T05:35:30.922580+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6429","last_scrub_stamp":"2023-10-09T05:34:07.818377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:07.818377+0000","objects_scrubbed":3381,"log_size":8994,"log_dups_size":0,"ondisk_log_size":8994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:04.556045+0000","scrub_duration":0.90924191099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105644032,"num_objects":1612,"num_object_clones":0,"num_object_copies":4836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2581,"num_write_kb":25984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T05:35:25.667535+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:35:25.667535+0000","last_peered":"2023-10-09T05:35:25.667535+0000","last_clean":"2023-10-09T05:35:25.667535+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:35:25.667535+0000","last_undegraded":"2023-10-09T05:35:25.667535+0000","last_fullsized":"2023-10-09T05:35:25.667535+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":127,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.600099+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:35:13.600099+0000","last_peered":"2023-10-09T05:35:13.600099+0000","last_clean":"2023-10-09T05:35:13.600099+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:35:13.600099+0000","last_undegraded":"2023-10-09T05:35:13.600099+0000","last_fullsized":"2023-10-09T05:35:13.600099+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"138'8883","reported_seq":8943,"reported_epoch":138,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:35:34.670955+0000","last_change":"2023-10-09T05:34:42.888623+0000","last_active":"2023-10-09T05:35:34.670955+0000","last_peered":"2023-10-09T05:35:34.670955+0000","last_clean":"2023-10-09T05:33:20.515228+0000","last_became_active":"2023-10-09T05:34:42.874596+0000","last_became_peered":"2023-10-09T05:34:42.874596+0000","last_unstale":"2023-10-09T05:35:34.670955+0000","last_undegraded":"2023-10-09T05:34:42.676976+0000","last_fullsized":"2023-10-09T05:34:42.659894+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8883,"log_dups_size":0,"ondisk_log_size":8883,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98238464,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2597,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5823,"num_write_kb":234304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":-461}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":91,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.590985+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:35:13.590985+0000","last_peered":"2023-10-09T05:35:13.590985+0000","last_clean":"2023-10-09T05:35:13.590985+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:35:13.590985+0000","last_undegraded":"2023-10-09T05:35:13.590985+0000","last_fullsized":"2023-10-09T05:35:13.590985+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:35:13.600655+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:35:13.600655+0000","last_peered":"2023-10-09T05:35:13.600655+0000","last_clean":"2023-10-09T05:35:13.600655+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:35:13.600655+0000","last_undegraded":"2023-10-09T05:35:13.600655+0000","last_fullsized":"2023-10-09T05:35:13.600655+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":36,"stat_sum":{"num_bytes":3230629912,"num_objects":49297,"num_object_clones":0,"num_object_copies":147891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21330,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49297,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136783,"num_write_kb":4609889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4224,"num_bytes_recovered":235470848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9119735808,"data_stored":9119727664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":320372,"ondisk_log_size":320372,"up":108,"acting":99,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846604,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":861888,"kb_used_data":551672,"kb_used_omap":14,"kb_used_meta":310193,"kb_avail":92883264,"statfs":{"total":95995035648,"available":95112462336,"internally_reserved":0,"allocated":564912128,"data_stored":564398239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":317638490},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.1110000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.7949999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41299999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38300000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.319}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.899}]}]},{"osd":6,"up_from":42,"seq":180388626630,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1073908,"kb_used_data":768492,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92671244,"statfs":{"total":95995035648,"available":94895353856,"internally_reserved":0,"allocated":786935808,"data_stored":786433815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:13 2023","interfaces":[{"interface":"back","average":{"1min":2.5859999999999999,"5min":1.8400000000000001,"15min":0.97399999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.8000000000000007,"5min":53.771999999999998,"15min":53.771999999999998},"last":2.1349999999999998},{"interface":"front","average":{"1min":2.399,"5min":1.806,"15min":0.96499999999999997},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.9149999999999991,"5min":53.411999999999999,"15min":53.411999999999999},"last":0.63500000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.2250000000000001,"15min":0.77700000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.9539999999999997,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.097,"5min":1.244,"15min":0.79900000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.9690000000000003,"5min":8.7189999999999994,"15min":8.7189999999999994},"last":2.0840000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:35:13 2023","interfaces":[{"interface":"back","average":{"1min":3.3109999999999999,"5min":2.379,"15min":1.5089999999999999},"min":{"1min":0.36599999999999999,"5min":0.314,"15min":0.314},"max":{"1min":20.084,"5min":34.393999999999998,"15min":34.393999999999998},"last":2.04},{"interface":"front","average":{"1min":2.7109999999999999,"5min":2.2970000000000002,"15min":1.4950000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":20.073,"5min":34.381999999999998,"15min":34.381999999999998},"last":0.47599999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":1.4490000000000001,"15min":1.1930000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.0250000000000004,"5min":18.361000000000001,"15min":18.361000000000001},"last":0.498},{"interface":"front","average":{"1min":1.409,"5min":1.4199999999999999,"15min":1.222},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.5030000000000001,"5min":18.536000000000001,"15min":18.536000000000001},"last":2.1120000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:34:50 2023","interfaces":[{"interface":"back","average":{"1min":9.9870000000000001,"5min":2.512,"15min":1.145},"min":{"1min":0.27000000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":125.64,"5min":125.64,"15min":125.64},"last":2.77},{"interface":"front","average":{"1min":10.308999999999999,"5min":2.5489999999999999,"15min":1.1679999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27800000000000002},"max":{"1min":125.625,"5min":125.625,"15min":125.625},"last":0.93100000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:34:50 2023","interfaces":[{"interface":"back","average":{"1min":8.0860000000000003,"5min":2.145,"15min":1.0509999999999999},"min":{"1min":0.29799999999999999,"5min":0.27900000000000003,"15min":0.27200000000000002},"max":{"1min":127.35899999999999,"5min":127.35899999999999,"15min":127.35899999999999},"last":2.1589999999999998},{"interface":"front","average":{"1min":8.4570000000000007,"5min":2.2480000000000002,"15min":1.093},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":128.93799999999999,"5min":128.93799999999999,"15min":128.93799999999999},"last":0.95699999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:33:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.56999999999999995,"15min":0.54200000000000004},"min":{"1min":0.32900000000000001,"5min":0.32300000000000001,"15min":0.29999999999999999},"max":{"1min":1.9299999999999999,"5min":1.9299999999999999,"15min":1.9299999999999999},"last":12.074999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.63100000000000001,"15min":0.58199999999999996},"min":{"1min":0.48399999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4330000000000001,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":0.40999999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958591,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1090476,"kb_used_data":986596,"kb_used_omap":7,"kb_used_meta":103864,"kb_avail":92654676,"statfs":{"total":95995035648,"available":94878388224,"internally_reserved":0,"allocated":1010274304,"data_stored":1009665305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":106357262},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":0.77900000000000003,"15min":0.52800000000000002},"min":{"1min":0.191,"5min":0.185,"15min":0.185},"max":{"1min":17.117000000000001,"5min":17.117000000000001,"15min":17.117000000000001},"last":124.82599999999999},{"interface":"front","average":{"1min":2.2189999999999999,"5min":0.90400000000000003,"15min":0.57299999999999995},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":17.097000000000001,"5min":17.097000000000001,"15min":17.097000000000001},"last":125.483}]},{"osd":2,"last update":"Mon Oct 9 05:35:07 2023","interfaces":[{"interface":"back","average":{"1min":7.3869999999999996,"5min":2.198,"15min":0.95999999999999996},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":98.483999999999995,"5min":98.483999999999995,"15min":98.483999999999995},"last":21.216999999999999},{"interface":"front","average":{"1min":6.1189999999999998,"5min":1.8600000000000001,"15min":0.89600000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":98.409999999999997,"5min":98.409999999999997,"15min":98.409999999999997},"last":21.161000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":0.89800000000000002,"15min":0.61799999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.0500000000000007,"5min":8.0500000000000007,"15min":8.0500000000000007},"last":0.223},{"interface":"front","average":{"1min":1.2390000000000001,"5min":0.72999999999999998,"15min":0.51200000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.6879999999999997,"5min":4.6879999999999997,"15min":4.6879999999999997},"last":0.26600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3010000000000002,"5min":1.0780000000000001,"15min":0.78800000000000003},"min":{"1min":0.40899999999999997,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":23.670999999999999,"5min":23.670999999999999,"15min":23.670999999999999},"last":0.377},{"interface":"front","average":{"1min":2.202,"5min":0.995,"15min":0.77200000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":23.620000000000001,"5min":23.620000000000001,"15min":23.620000000000001},"last":0.38800000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2.9169999999999998,"5min":1.169,"15min":0.81100000000000005},"min":{"1min":0.38900000000000001,"5min":0.374,"15min":0.374},"max":{"1min":23.501999999999999,"5min":23.501999999999999,"15min":23.501999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":3.048,"5min":1.4239999999999999,"15min":0.92600000000000005},"min":{"1min":0.434,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":23.965,"5min":25.68,"15min":25.68},"last":0.40699999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":1.0189999999999999,"15min":0.77600000000000002},"min":{"1min":0.45600000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":8.1270000000000007,"5min":8.1270000000000007,"15min":8.1270000000000007},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.849,"5min":0.97599999999999998,"15min":0.78200000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":5.8140000000000001,"5min":5.8140000000000001,"15min":5.8140000000000001},"last":0.43099999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.69699999999999995,"15min":0.70799999999999996},"min":{"1min":0.47299999999999998,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.77000000000000002,"5min":1.014,"15min":1.014},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.68400000000000005,"15min":0.69299999999999995},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.82799999999999996,"5min":1,"15min":1},"last":0.41799999999999998}]}]},{"osd":0,"up_from":88,"seq":377957122117,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1768680,"kb_used_data":1598304,"kb_used_omap":8,"kb_used_meta":170295,"kb_avail":91976472,"statfs":{"total":95995035648,"available":94183907328,"internally_reserved":0,"allocated":1636663296,"data_stored":1636057232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":174382856},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:34:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2570000000000001,"5min":0.89400000000000002,"15min":0.56200000000000006},"min":{"1min":0.20899999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":13.919,"5min":13.919,"15min":13.919},"last":0.504},{"interface":"front","average":{"1min":2.2719999999999998,"5min":0.88700000000000001,"15min":0.58899999999999997},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.9350000000000005,"5min":9.9350000000000005,"15min":9.9350000000000005},"last":0.22800000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:35:04 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.845,"15min":0.85299999999999998},"min":{"1min":0.22700000000000001,"5min":0.214,"15min":0.214},"max":{"1min":6.4640000000000004,"5min":57.222999999999999,"15min":57.222999999999999},"last":4.6369999999999996},{"interface":"front","average":{"1min":1.71,"5min":1.6830000000000001,"15min":0.81000000000000005},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":11.693,"5min":57.408000000000001,"15min":57.408000000000001},"last":0.36799999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":1.01,"15min":0.60399999999999998},"min":{"1min":0.32100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":12.429,"5min":12.429,"15min":12.429},"last":0.30099999999999999},{"interface":"front","average":{"1min":3.1659999999999999,"5min":1.179,"15min":0.66000000000000003},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":22.390999999999998,"5min":22.390999999999998,"15min":22.390999999999998},"last":0.33700000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:35:04 2023","interfaces":[{"interface":"back","average":{"1min":2.371,"5min":1.0920000000000001,"15min":0.81799999999999995},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":14.023,"5min":14.023,"15min":14.023},"last":1.657},{"interface":"front","average":{"1min":2.7130000000000001,"5min":1.282,"15min":0.90800000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":14.096,"5min":14.096,"15min":14.096},"last":1.643}]},{"osd":5,"last update":"Mon Oct 9 05:35:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5099999999999998,"5min":1.258,"15min":0.873},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":13.632,"5min":13.632,"15min":13.632},"last":0.437},{"interface":"front","average":{"1min":2.8620000000000001,"5min":1.3129999999999999,"15min":0.88800000000000001},"min":{"1min":0.36699999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":13.676,"5min":13.676,"15min":13.676},"last":1.397}]},{"osd":6,"last update":"Mon Oct 9 05:35:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":1.2669999999999999,"15min":0.89300000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.0679999999999996,"5min":9.9809999999999999,"15min":9.9809999999999999},"last":1.2150000000000001},{"interface":"front","average":{"1min":2.73,"5min":1.3049999999999999,"15min":0.88700000000000001},"min":{"1min":0.42199999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.5259999999999998,"5min":6.8460000000000001,"15min":6.8460000000000001},"last":0.38700000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:33:02 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.71199999999999997,"15min":0.70999999999999996},"min":{"1min":0.51300000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":2.8450000000000002,"5min":2.8450000000000002,"15min":2.8450000000000002},"last":1.8129999999999999},{"interface":"front","average":{"1min":0.755,"5min":0.71599999999999997,"15min":0.72899999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.448,"5min":2.448,"15min":2.448},"last":1.8300000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795064,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1381548,"kb_used_data":1223588,"kb_used_omap":11,"kb_used_meta":157940,"kb_avail":92363604,"statfs":{"total":95995035648,"available":94580330496,"internally_reserved":0,"allocated":1252954112,"data_stored":1252344050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":161731157},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":2.9990000000000001,"5min":1.173,"15min":0.628},"min":{"1min":0.19,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":32.122,"5min":32.122,"15min":32.122},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.577,"5min":1.109,"15min":0.63300000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":34.066000000000003,"5min":34.066000000000003,"15min":34.066000000000003},"last":0.25700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":5.7160000000000002,"5min":1.7250000000000001,"15min":0.82899999999999996},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":98.459999999999994,"5min":98.459999999999994,"15min":98.459999999999994},"last":0.29899999999999999},{"interface":"front","average":{"1min":5.9119999999999999,"5min":1.647,"15min":0.80300000000000005},"min":{"1min":0.222,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":98.397000000000006,"5min":98.397000000000006,"15min":98.397000000000006},"last":0.218}]},{"osd":3,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":3.3450000000000002,"5min":1.0469999999999999,"15min":0.59899999999999998},"min":{"1min":0.24399999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":34.052999999999997,"5min":34.052999999999997,"15min":34.052999999999997},"last":0.32100000000000001},{"interface":"front","average":{"1min":3.1960000000000002,"5min":1.153,"15min":0.65100000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":35.161999999999999,"5min":35.161999999999999,"15min":35.161999999999999},"last":0.39400000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":0.96699999999999997,"15min":0.70499999999999996},"min":{"1min":0.32100000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.1779999999999999,"5min":4.4859999999999998,"15min":4.4859999999999998},"last":3.0209999999999999},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.0209999999999999,"15min":0.76800000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.0060000000000002,"5min":9.0060000000000002,"15min":9.0060000000000002},"last":3.4409999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":1.161,"15min":0.81499999999999995},"min":{"1min":0.36699999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.8730000000000002,"5min":6.8730000000000002,"15min":6.8730000000000002},"last":1.248},{"interface":"front","average":{"1min":1.3420000000000001,"5min":0.98599999999999999,"15min":0.73299999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.2519999999999998,"5min":5.2519999999999998,"15min":5.2519999999999998},"last":5.407}]},{"osd":6,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.117,"15min":0.78900000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":11.496,"5min":11.496,"15min":11.496},"last":2.8650000000000002},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.0680000000000001,"15min":0.81299999999999994},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.4049999999999994,"5min":9.4049999999999994,"15min":9.4049999999999994},"last":3.8319999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.68000000000000005,"15min":0.66800000000000004},"min":{"1min":0.46100000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":2.2959999999999998,"5min":2.2959999999999998,"15min":2.2959999999999998},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.65400000000000003,"15min":0.65700000000000003},"min":{"1min":0.438,"5min":0.435,"15min":0.435},"max":{"1min":1.2190000000000001,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":0.45100000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631538,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":711952,"kb_used_data":618056,"kb_used_omap":4,"kb_used_meta":93883,"kb_avail":93033200,"statfs":{"total":95995035648,"available":95265996800,"internally_reserved":0,"allocated":632889344,"data_stored":632304834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":96136798},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":4.9470000000000001,"5min":1.4139999999999999,"15min":0.73699999999999999},"min":{"1min":0.26000000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":56.094000000000001,"5min":56.094000000000001,"15min":56.094000000000001},"last":2.4129999999999998},{"interface":"front","average":{"1min":5.6289999999999996,"5min":1.593,"15min":0.78100000000000003},"min":{"1min":0.26800000000000002,"5min":0.214,"15min":0.214},"max":{"1min":57.255000000000003,"5min":57.255000000000003,"15min":57.255000000000003},"last":3.3599999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":1.5649999999999999,"5min":0.67500000000000004,"15min":0.49399999999999999},"min":{"1min":0.26000000000000001,"5min":0.255,"15min":0.255},"max":{"1min":4.4619999999999997,"5min":4.4619999999999997,"15min":4.4619999999999997},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.732,"5min":0.66700000000000004,"15min":0.47299999999999998},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":6.8769999999999998,"5min":6.8769999999999998,"15min":6.8769999999999998},"last":4.3440000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":5.5730000000000004,"5min":2.77,"15min":1.208},"min":{"1min":0.32400000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":50.411000000000001,"5min":117.23,"15min":117.23},"last":1.054},{"interface":"front","average":{"1min":5.0229999999999997,"5min":2.7989999999999999,"15min":1.226},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":50.944000000000003,"5min":117.45999999999999,"15min":117.45999999999999},"last":2.3119999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":3.746,"5min":1.6659999999999999,"15min":0.95399999999999996},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":45.779000000000003,"5min":45.779000000000003,"15min":45.779000000000003},"last":0.70499999999999996},{"interface":"front","average":{"1min":4.3079999999999998,"5min":1.4590000000000001,"15min":0.91800000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":45.735999999999997,"5min":45.735999999999997,"15min":45.735999999999997},"last":1.5940000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":3.5379999999999998,"5min":1.294,"15min":0.85899999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":47.948999999999998,"5min":47.948999999999998,"15min":47.948999999999998},"last":2.0840000000000001},{"interface":"front","average":{"1min":4.2000000000000002,"5min":1.4450000000000001,"15min":0.93400000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":47.968000000000004,"5min":47.968000000000004,"15min":47.968000000000004},"last":0.84299999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:34:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":1.079,"15min":0.80700000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.484,"5min":8.484,"15min":8.484},"last":0.86299999999999999},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.66700000000000004,"15min":0.63700000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.488,"5min":2.488,"15min":2.488},"last":2.0139999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:32:35 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.74099999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019086,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1856568,"kb_used_data":1551152,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91888584,"statfs":{"total":95995035648,"available":94093910016,"internally_reserved":0,"allocated":1588379648,"data_stored":1587873337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:07 2023","interfaces":[{"interface":"back","average":{"1min":3.3109999999999999,"5min":1.5049999999999999,"15min":0.84899999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":33.488999999999997,"5min":33.488999999999997,"15min":33.488999999999997},"last":3.915},{"interface":"front","average":{"1min":3.569,"5min":1.631,"15min":0.91200000000000003},"min":{"1min":0.27300000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":33.616999999999997,"5min":33.616999999999997,"15min":33.616999999999997},"last":10.000999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:34:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0419999999999998,"5min":1.073,"15min":0.71499999999999997},"min":{"1min":0.313,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":31.073,"5min":31.073,"15min":31.073},"last":0.52800000000000002},{"interface":"front","average":{"1min":3.5750000000000002,"5min":1.2569999999999999,"15min":0.79100000000000004},"min":{"1min":0.33600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":31.100000000000001,"5min":31.100000000000001,"15min":31.100000000000001},"last":10.055}]},{"osd":2,"last update":"Mon Oct 9 05:35:07 2023","interfaces":[{"interface":"back","average":{"1min":11.824,"5min":3.4830000000000001,"15min":1.506},"min":{"1min":0.35299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":164.26599999999999,"5min":164.26599999999999,"15min":164.26599999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":10.645,"5min":3.3359999999999999,"15min":1.502},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":162.57499999999999,"5min":162.57499999999999,"15min":162.57499999999999},"last":3.8490000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.3899999999999999,"15min":0.83599999999999997},"min":{"1min":0.35399999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.037000000000001,"5min":10.037000000000001,"15min":10.037000000000001},"last":10.037000000000001},{"interface":"front","average":{"1min":1.429,"5min":1.1519999999999999,"15min":0.79200000000000004},"min":{"1min":0.34100000000000003,"5min":0.309,"15min":0.309},"max":{"1min":4.6219999999999999,"5min":6.9459999999999997,"15min":6.9459999999999997},"last":4.0270000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:35:34 2023","interfaces":[{"interface":"back","average":{"1min":7.8369999999999997,"5min":2.2010000000000001,"15min":1.034},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":113.35899999999999,"5min":113.35899999999999,"15min":113.35899999999999},"last":9.9570000000000007},{"interface":"front","average":{"1min":7.71,"5min":2.125,"15min":1.071},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":113.32299999999999,"5min":113.32299999999999,"15min":113.32299999999999},"last":3.972}]},{"osd":6,"last update":"Mon Oct 9 05:34:44 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":0.64000000000000001,"15min":0.53500000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27800000000000002},"max":{"1min":3.8140000000000001,"5min":3.8140000000000001,"15min":3.8140000000000001},"last":4.2519999999999998},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.75,"15min":0.58199999999999996},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":9.6820000000000004,"5min":9.6820000000000004,"15min":9.6820000000000004},"last":10.02}]},{"osd":7,"last update":"Mon Oct 9 05:33:09 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.55400000000000005,"15min":0.54200000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":1.7470000000000001,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":4.2220000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.53600000000000003,"15min":0.53100000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":2.7839999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822858,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2010668,"kb_used_data":1713444,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91734484,"statfs":{"total":95995035648,"available":93936111616,"internally_reserved":0,"allocated":1754566656,"data_stored":1754068105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:06 2023","interfaces":[{"interface":"back","average":{"1min":11.497999999999999,"5min":3.8370000000000002,"15min":1.647},"min":{"1min":0.224,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":150.13399999999999,"5min":150.13399999999999,"15min":150.13399999999999},"last":47.098999999999997},{"interface":"front","average":{"1min":12.132,"5min":4.1879999999999997,"15min":1.766},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":150.24700000000001,"5min":150.24700000000001,"15min":150.24700000000001},"last":50.683999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:34:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1269999999999998,"5min":0.91100000000000003,"15min":0.67400000000000004},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.891999999999999,"5min":12.891999999999999,"15min":12.891999999999999},"last":2.302},{"interface":"front","average":{"1min":1.6080000000000001,"5min":0.88100000000000001,"15min":0.66500000000000004},"min":{"1min":0.26700000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":12.939,"5min":12.939,"15min":12.939},"last":0.33000000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:35:13 2023","interfaces":[{"interface":"back","average":{"1min":19.474,"5min":4.7450000000000001,"15min":1.9630000000000001},"min":{"1min":0.28199999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":290.697,"5min":290.697,"15min":290.697},"last":0.28799999999999998},{"interface":"front","average":{"1min":20.004999999999999,"5min":4.7560000000000002,"15min":1.9630000000000001},"min":{"1min":0.32200000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":290.786,"5min":290.786,"15min":290.786},"last":2.3500000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:34:35 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":0.86199999999999999,"15min":0.68200000000000005},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":6.4119999999999999,"5min":6.4119999999999999,"15min":6.4119999999999999},"last":2.367},{"interface":"front","average":{"1min":2.2410000000000001,"5min":0.97699999999999998,"15min":0.71699999999999997},"min":{"1min":0.36599999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":12.779999999999999,"5min":12.779999999999999,"15min":12.779999999999999},"last":0.33900000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:35:32 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":0.877,"15min":0.60599999999999998},"min":{"1min":0.30499999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":9.2729999999999997,"5min":9.3510000000000009,"15min":9.3510000000000009},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.194,"5min":1.111,"15min":0.71299999999999997},"min":{"1min":0.32000000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":14.236000000000001,"5min":14.236000000000001,"15min":14.236000000000001},"last":0.47999999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:34:57 2023","interfaces":[{"interface":"back","average":{"1min":5.8570000000000002,"5min":1.694,"15min":0.90700000000000003},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":95.010999999999996,"5min":95.010999999999996,"15min":95.010999999999996},"last":0.57499999999999996},{"interface":"front","average":{"1min":6.0250000000000004,"5min":1.6890000000000001,"15min":0.91900000000000004},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":94.194000000000003,"5min":94.194000000000003,"15min":94.194000000000003},"last":2.3319999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:33:12 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.59999999999999998,"15min":0.58099999999999996},"min":{"1min":0.44400000000000001,"5min":0.40200000000000002,"15min":0.34399999999999997},"max":{"1min":1.5249999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.58799999999999997,"15min":0.56000000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32400000000000001},"max":{"1min":1.8600000000000001,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":2.3170000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1615921152,"data_stored":1615921152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":998244352,"data_stored":998244352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1239158784,"data_stored":1239154712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":625082368,"data_stored":625082368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1559887872,"data_stored":1559887872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1744179200,"data_stored":1744175128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":776667136,"data_stored":776667136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":560594944,"data_stored":560594944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:35:38.990 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 1 2023-10-09T05:35:39.084 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:39.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:39.195 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:39.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:39 smithi008 ceph-mon[191227]: pgmap v354: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 20950/140496 objects degraded (14.911%); 5.7 MiB/s, 95 objects/s recovering 2023-10-09T05:35:39.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:39 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:39.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:39 smithi008 ceph-mon[191227]: failed to encode map e139 with expected crc 2023-10-09T05:35:39.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/73327997' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:39.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:39 smithi008 ceph-mon[191227]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T05:35:39.296 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:39 smithi008 ceph-mon[184418]: pgmap v354: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 20950/140496 objects degraded (14.911%); 5.7 MiB/s, 95 objects/s recovering 2023-10-09T05:35:39.296 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:39 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:39.296 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:39 smithi008 ceph-mon[184418]: failed to encode map e139 with expected crc 2023-10-09T05:35:39.296 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/73327997' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:39.296 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:39 smithi008 ceph-mon[184418]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T05:35:39.307 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:39.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:39 smithi191 ceph-mon[161669]: pgmap v354: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 20950/140496 objects degraded (14.911%); 5.7 MiB/s, 95 objects/s recovering 2023-10-09T05:35:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:39 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:39 smithi191 ceph-mon[161669]: failed to encode map e139 with expected crc 2023-10-09T05:35:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/73327997' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:39 smithi191 ceph-mon[161669]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T05:35:39.564 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:39.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:39.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:39.672 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:39.720 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:39.904 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:40.144 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:40.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:40.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:40.251 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:40.310 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:40.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:35:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:35:40] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.33.4" 2023-10-09T05:35:40.950 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:40.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:40.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:41.056 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:41.077 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:41.157 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:41.205 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:41 smithi008 ceph-mon[184418]: pgmap v356: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 20950/134727 objects degraded (15.550%); 4.8 MiB/s, 80 objects/s recovering 2023-10-09T05:35:41.208 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:35:41.210 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:41 smithi008 ceph-mon[191227]: pgmap v356: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 20950/134727 objects degraded (15.550%); 4.8 MiB/s, 80 objects/s recovering 2023-10-09T05:35:41.273 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:41.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:41.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:41.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:41.415 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:41 smithi191 ceph-mon[161669]: pgmap v356: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 20950/134727 objects degraded (15.550%); 4.8 MiB/s, 80 objects/s recovering 2023-10-09T05:35:41.417 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:41.485 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:41.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:41.600 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:41.673 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:43.038 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:43.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:43.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:43.142 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:43.180 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:43.214 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:35:43.241 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:35:43.264 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:35:43.287 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:35:43.290 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[184418]: pgmap v357: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 20950/129159 objects degraded (16.220%); 2.4 MiB/s, 88 objects/s recovering 2023-10-09T05:35:43.290 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20950/129159 objects degraded (16.220%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:43.290 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[184418]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T05:35:43.290 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[184418]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.291 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[184418]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.291 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[184418]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.291 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[184418]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.291 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[184418]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.291 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[184418]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.291 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[191227]: pgmap v357: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 20950/129159 objects degraded (16.220%); 2.4 MiB/s, 88 objects/s recovering 2023-10-09T05:35:43.292 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20950/129159 objects degraded (16.220%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:43.292 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[191227]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T05:35:43.292 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[191227]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.292 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[191227]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.292 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[191227]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.292 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[191227]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.292 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[191227]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:43 smithi008 ceph-mon[191227]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.310 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:35:43.336 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:35:43.361 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:35:43.383 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:35:43.385 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:35:43.385 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:35:43.385 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:35:43.385 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:35:43.385 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:35:43.385 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:35:43.385 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:35:43.386 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:35:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:43 smithi191 ceph-mon[161669]: pgmap v357: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 20950/129159 objects degraded (16.220%); 2.4 MiB/s, 88 objects/s recovering 2023-10-09T05:35:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20950/129159 objects degraded (16.220%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:35:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:43 smithi191 ceph-mon[161669]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T05:35:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:43 smithi191 ceph-mon[161669]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:43 smithi191 ceph-mon[161669]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:43 smithi191 ceph-mon[161669]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:43 smithi191 ceph-mon[161669]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:43 smithi191 ceph-mon[161669]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:43 smithi191 ceph-mon[161669]: failed to encode map e140 with expected crc 2023-10-09T05:35:43.930 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:43.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:43.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:44.034 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:44.069 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:44.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[191227]: failed to encode map e140 with expected crc 2023-10-09T05:35:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/875467629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[191227]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[191227]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[191227]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[191227]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T05:35:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[191227]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:44.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[184418]: failed to encode map e140 with expected crc 2023-10-09T05:35:44.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/875467629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:44.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[184418]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[184418]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[184418]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[184418]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T05:35:44.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[184418]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.222 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:44 smithi191 ceph-mon[161669]: failed to encode map e140 with expected crc 2023-10-09T05:35:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/875467629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:44 smithi191 ceph-mon[161669]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:44 smithi191 ceph-mon[161669]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:44 smithi191 ceph-mon[161669]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:44 smithi191 ceph-mon[161669]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T05:35:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:44 smithi191 ceph-mon[161669]: failed to encode map e141 with expected crc 2023-10-09T05:35:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:44.818 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:44.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:44.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:44.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:44.999 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:45.103 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:45.111 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:45.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:45.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:45.217 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:45 smithi191 ceph-mon[161669]: failed to encode map e141 with expected crc 2023-10-09T05:35:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:45 smithi191 ceph-mon[161669]: pgmap v360: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 20950/126720 objects degraded (16.533%); 0 B/s, 75 objects/s recovering 2023-10-09T05:35:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:45.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:45.340 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:45.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:45 smithi008 ceph-mon[191227]: failed to encode map e141 with expected crc 2023-10-09T05:35:45.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:45 smithi008 ceph-mon[191227]: pgmap v360: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 20950/126720 objects degraded (16.533%); 0 B/s, 75 objects/s recovering 2023-10-09T05:35:45.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:45.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:45.353 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:45 smithi008 ceph-mon[184418]: failed to encode map e141 with expected crc 2023-10-09T05:35:45.353 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:45 smithi008 ceph-mon[184418]: pgmap v360: 109 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 20950/126720 objects degraded (16.533%); 0 B/s, 75 objects/s recovering 2023-10-09T05:35:45.353 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:45.354 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:45.560 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:35:45.568 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:45.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:45.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:45.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:45.674 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:35:45.674 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:35:45.729 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:45.856 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:35:45.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:45.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:45.971 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:46.019 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1527700208' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:46 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:46.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1527700208' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:46 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1527700208' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:46 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T05:35:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:46.748 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:35:47.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:47 smithi008 ceph-mon[184418]: pgmap v361: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 20076/122379 objects degraded (16.405%); 1.9 MiB/s, 87 objects/s recovering 2023-10-09T05:35:47.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:47 smithi008 ceph-mon[191227]: pgmap v361: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 20076/122379 objects degraded (16.405%); 1.9 MiB/s, 87 objects/s recovering 2023-10-09T05:35:47.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:47 smithi191 ceph-mon[161669]: pgmap v361: 109 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 20076/122379 objects degraded (16.405%); 1.9 MiB/s, 87 objects/s recovering 2023-10-09T05:35:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:35:47.650 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:47.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:47.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:47.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:47.808 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:47.943 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:35:47.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:47.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:48.056 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:48.140 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:48.264 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:48.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:48.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:48.316 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:35:48.373 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:48.425 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:48.508 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:35:48.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:48.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:48.597 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20076/122379 objects degraded (16.405%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:35:48.598 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20076/122379 objects degraded (16.405%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:35:48.617 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:48.660 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:48.703 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:48.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20076/122379 objects degraded (16.405%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:35:48.842 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:48.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:48.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:48.952 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:49.033 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:49.147 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:49.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:49.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:49.255 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:49.321 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:49.565 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:49.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:49.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:49.628 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:35:49.671 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:49.717 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:49.819 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:49.825 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:49 smithi008 ceph-mon[184418]: pgmap v362: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 20076/115839 objects degraded (17.331%); 1.9 MiB/s, 102 objects/s recovering 2023-10-09T05:35:49.826 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3091050725' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:49.826 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:49 smithi008 ceph-mon[191227]: pgmap v362: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 20076/115839 objects degraded (17.331%); 1.9 MiB/s, 102 objects/s recovering 2023-10-09T05:35:49.826 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3091050725' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:49.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:49.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:49.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:49.964 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:49 smithi191 ceph-mon[161669]: pgmap v362: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 20076/115839 objects degraded (17.331%); 1.9 MiB/s, 102 objects/s recovering 2023-10-09T05:35:49.965 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3091050725' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:35:49.966 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:50.014 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:50.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:50.121 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:50.162 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:50.807 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:50.811 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:50.811 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[184418]: failed to encode map e142 with expected crc 2023-10-09T05:35:50.811 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[184418]: failed to encode map e142 with expected crc 2023-10-09T05:35:50.811 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3091050725' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:35:50.811 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[184418]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T05:35:50.811 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[184418]: 5.15 scrub starts 2023-10-09T05:35:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:50.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:50.813 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:50.813 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[191227]: failed to encode map e142 with expected crc 2023-10-09T05:35:50.813 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[191227]: failed to encode map e142 with expected crc 2023-10-09T05:35:50.813 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3091050725' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:35:50.813 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[191227]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T05:35:50.813 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:50 smithi008 ceph-mon[191227]: 5.15 scrub starts 2023-10-09T05:35:50.814 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:35:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:35:50] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.33.4" 2023-10-09T05:35:50.913 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:50.980 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:51.007 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:51.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:50 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:35:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:50 smithi191 ceph-mon[161669]: failed to encode map e142 with expected crc 2023-10-09T05:35:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:50 smithi191 ceph-mon[161669]: failed to encode map e142 with expected crc 2023-10-09T05:35:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3091050725' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:35:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:50 smithi191 ceph-mon[161669]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T05:35:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:50 smithi191 ceph-mon[161669]: 5.15 scrub starts 2023-10-09T05:35:51.150 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:51.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:51.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:51.257 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:51.349 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:51.451 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:35:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:51.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:51.566 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:51.612 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:51.708 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:35:51.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:51 smithi008 ceph-mon[184418]: pgmap v364: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 19730/109830 objects degraded (17.964%); 4.9 MiB/s, 96 objects/s recovering 2023-10-09T05:35:51.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:51 smithi008 ceph-mon[191227]: pgmap v364: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 19730/109830 objects degraded (17.964%); 4.9 MiB/s, 96 objects/s recovering 2023-10-09T05:35:51.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:51.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:51.784 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:35:51.817 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:51.826 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:35:51.852 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:35:51.892 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:35:51.907 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:51.940 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:35:51.972 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:35:52.013 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:35:52.039 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:52.040 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:35:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:52.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:52.048 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:35:52.048 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:35:52.048 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:35:52.048 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:35:52.048 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:35:52.049 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:35:52.049 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:35:52.049 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:35:52.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:51 smithi191 ceph-mon[161669]: pgmap v364: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 19730/109830 objects degraded (17.964%); 4.9 MiB/s, 96 objects/s recovering 2023-10-09T05:35:52.143 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:52.189 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:52.268 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:52.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:52.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:52.373 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:52.420 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:52.647 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:35:52.647 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:35:52.648 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T05:35:52.648 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:35:52.648 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:35:52.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T05:35:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2473149708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:52 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19730/109830 objects degraded (17.964%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:35:52.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2473149708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:52.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:52 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19730/109830 objects degraded (17.964%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:35:52.720 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:52.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:52.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:52.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:35:52.880 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:52.968 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:35:52.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:52.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:35:52.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2473149708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:35:52.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:52 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19730/109830 objects degraded (17.964%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:35:53.075 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:53.107 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:35:53.127 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:53.205 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:35:53.226 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:35:53.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:53.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:53.333 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:53.407 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:53.658 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:53 smithi008 ceph-mon[184418]: pgmap v365: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 19730/104112 objects degraded (18.951%); 4.3 MiB/s, 85 objects/s recovering 2023-10-09T05:35:53.659 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:53 smithi008 ceph-mon[191227]: pgmap v365: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 19730/104112 objects degraded (18.951%); 4.3 MiB/s, 85 objects/s recovering 2023-10-09T05:35:53.908 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:53.926 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:35:53.926 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:35:53.926 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:35:53.926 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:35:53.926 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.926 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:10991 5.5 e108/105 5:a03af33e:::benchmark_data_smithi008_238708_object10990:head v 108'691, mlcod=108'689)", 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:56.049176+0000", 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 177.873759869, 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.144458221, 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049176+0000", 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.927 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049176+0000", 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049175+0000", 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049199+0000", 2023-10-09T05:35:53.928 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4014000000000001e-05 2023-10-09T05:35:53.929 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.929 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.929 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.929 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049209+0000", 2023-10-09T05:35:53.929 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9510000000000001e-06 2023-10-09T05:35:53.929 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.929 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.930 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.930 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.049218+0000", 2023-10-09T05:35:53.930 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1900000000000001e-06 2023-10-09T05:35:53.930 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.930 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.930 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.930 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050199+0000", 2023-10-09T05:35:53.930 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00098079999999999999 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050217+0000", 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8536e-05 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.051392+0000", 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001174598 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.931 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193634+0000", 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14224188300000001 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:11003 5.8 e108/105 5:1d0863ca:::benchmark_data_smithi008_238708_object11002:head v 108'694, mlcod=108'692)", 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:56.050613+0000", 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 177.87232262500001, 2023-10-09T05:35:53.932 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14495682600000001, 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050613+0000", 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050613+0000", 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.933 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050612+0000", 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050637+0000", 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5170000000000001e-05 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.934 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050649+0000", 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1622e-05 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050656+0000", 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2540000000000003e-06 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050720+0000", 2023-10-09T05:35:53.935 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3874999999999994e-05 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050740+0000", 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9783e-05 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195541+0000", 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14480122200000001 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.936 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195570+0000", 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8819999999999999e-05 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:11001 5.7 e108/105 5:ea85ce96:::benchmark_data_smithi008_238708_object11000:head v 108'703, mlcod=108'701)", 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:56.050731+0000", 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 177.872204466, 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14540388300000001, 2023-10-09T05:35:53.937 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050731+0000", 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050731+0000", 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.938 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.050730+0000", 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.053271+0000", 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025400689999999998 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.939 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193313+0000", 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14004271099999999 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193337+0000", 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3632e-05 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193495+0000", 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000157706 2023-10-09T05:35:53.940 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193522+0000", 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7370999999999999e-05 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.196093+0000", 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025712450000000002 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.196135+0000", 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1820999999999998e-05 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:11006 5.8 e108/105 5:1ea71daf:::benchmark_data_smithi008_238708_object11005:head v 108'695, mlcod=108'692)", 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:56.051675+0000", 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 177.87126081700001, 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14391491300000001, 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.942 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.051675+0000", 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.051675+0000", 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.943 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.051674+0000", 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.051699+0000", 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5198000000000002e-05 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193330+0000", 2023-10-09T05:35:53.944 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14163066199999999 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193342+0000", 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1844999999999999e-05 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193894+0000", 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00055221000000000003 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.945 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193928+0000", 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4434000000000003e-05 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195573+0000", 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016444949999999999 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195590+0000", 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6826000000000001e-05 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:10995 5.5 e108/105 5:aa6195bb:::benchmark_data_smithi008_238708_object10994:head v 108'692, mlcod=108'689)", 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:56.052562+0000", 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 177.87037407400001, 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14313236900000001, 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.947 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.052562+0000", 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.052562+0000", 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.052561+0000", 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.052586+0000", 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5242e-05 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193387+0000", 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14080136900000001 2023-10-09T05:35:53.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193399+0000", 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1807999999999999e-05 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193668+0000", 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00026856100000000001 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.950 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.193691+0000", 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.304e-05 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195657+0000", 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001965853 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.195694+0000", 2023-10-09T05:35:53.951 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7230000000000001e-05 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:19144 5.0 e109/105 5:0930f39a:::benchmark_data_smithi008_238708_object19143:head v 109'1179, mlcod=109'1176)", 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:59.249102+0000", 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 174.67383321200001, 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13596629299999999, 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.952 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249102+0000", 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249102+0000", 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.953 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249102+0000", 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249127+0000", 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5384e-05 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249137+0000", 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0257e-05 2023-10-09T05:35:53.954 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249144+0000", 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6549999999999998e-06 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249203+0000", 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8727000000000002e-05 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.955 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249220+0000", 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7802e-05 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.385031+0000", 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.135810706 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.385069+0000", 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7483e-05 2023-10-09T05:35:53.956 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:19149 5.8 e109/105 5:1908f647:::benchmark_data_smithi008_238708_object19148:head v 109'1252, mlcod=109'1249)", 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:59.249953+0000", 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 174.672982741, 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13514635799999999, 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.957 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249953+0000", 2023-10-09T05:35:53.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249953+0000", 2023-10-09T05:35:53.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249952+0000", 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249978+0000", 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5910000000000001e-05 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.249997+0000", 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8723e-05 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.250004+0000", 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3259999999999998e-06 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.250067+0000", 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2829000000000004e-05 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.250087+0000", 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9823999999999999e-05 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.385073+0000", 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13498581800000001 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.385099+0000", 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6584000000000002e-05 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.961 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:19152 5.7 e109/105 5:e977ac15:::benchmark_data_smithi008_238708_object19151:head v 109'1175, mlcod=109'1172)", 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:59.251006+0000", 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 174.67192939399999, 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13563842400000001, 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.962 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251006+0000", 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251006+0000", 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251006+0000", 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:35:53.963 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251069+0000", 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3309e-05 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383348+0000", 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.132279123 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.964 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383366+0000", 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7578999999999999e-05 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383422+0000", 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6631e-05 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.383451+0000", 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8416e-05 2023-10-09T05:35:53.965 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.386610+0000", 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003159111 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.386645+0000", 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4870000000000003e-05 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.966 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:98120 5.8 e117/105)", 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.675377+0000", 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 113.247558611, 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18163384699999999, 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.967 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675377+0000", 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675377+0000", 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675375+0000", 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675408+0000", 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2552999999999998e-05 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675426+0000", 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8842e-05 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.969 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675438+0000", 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1316e-05 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.856993+0000", 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.181555568 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.857011+0000", 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7574000000000001e-05 2023-10-09T05:35:53.970 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119029 5.c 5:33f591e4:::benchmark_data_smithi008_238708_object893:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.009231+0000", 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 89.913704566000007, 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0490911119999999, 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:53.971 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119029 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009231+0000", 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009231+0000", 2023-10-09T05:35:53.972 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009229+0000", 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009236+0000", 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2099999999999996e-06 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009250+0000", 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3377000000000001e-05 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009259+0000", 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4050000000000006e-06 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180684+0000", 2023-10-09T05:35:53.974 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.171425208 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180700+0000", 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5184999999999999e-05 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.030305+0000", 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.84960513900000001 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.975 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.030495+0000", 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000190605 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.030899+0000", 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00040406200000000001 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:35:53.976 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.040416+0000", 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0095167640000000005 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058217+0000", 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.01780087 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058246+0000", 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8473999999999999e-05 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.977 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058285+0000", 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9922999999999997e-05 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058322+0000", 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6687999999999998e-05 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.978 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119039 5.c 5:34e6951f:::benchmark_data_smithi008_238708_object903:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.151148+0000", 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 89.771787290000006, 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90729258899999998, 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119039 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.979 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151148+0000", 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151148+0000", 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.980 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151146+0000", 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151154+0000", 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2060000000000001e-06 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151177+0000", 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3495000000000001e-05 2023-10-09T05:35:53.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151189+0000", 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1959e-05 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180742+0000", 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.029553138 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180757+0000", 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4803999999999999e-05 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.032636+0000", 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85187944500000001 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.032759+0000", 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012297400000000001 2023-10-09T05:35:53.983 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.033078+0000", 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00031910800000000002 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.040420+0000", 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0073418340000000002 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.984 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058354+0000", 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017933893999999999 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058380+0000", 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6250000000000001e-05 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058412+0000", 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1640999999999997e-05 2023-10-09T05:35:53.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058441+0000", 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8843000000000002e-05 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:120532 5.1c 5:3b693aa3:::benchmark_data_smithi008_238708_object2396:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:25.931287+0000", 2023-10-09T05:35:53.986 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 87.991648956999995, 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13708150799999999, 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 120532 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.987 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.931287+0000", 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.931287+0000", 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.931285+0000", 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.931292+0000", 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4219999999999997e-06 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.931309+0000", 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7297999999999999e-05 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.931318+0000", 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5149999999999995e-06 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.931366+0000", 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7858999999999997e-05 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.931480+0000", 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000113804 2023-10-09T05:35:53.990 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.931723+0000", 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000243324 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.933205+0000", 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014816860000000001 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.991 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.935383+0000", 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021776120000000002 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:26.068278+0000", 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13289557199999999 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:26.068331+0000", 2023-10-09T05:35:53.992 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2747000000000002e-05 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:26.068368+0000", 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7208e-05 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:135157 5.1c 5:3e4fbf73:::benchmark_data_smithi008_238708_object17021:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:37.651204+0000", 2023-10-09T05:35:53.993 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 76.271731912000007, 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.16619640999999999, 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135157 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:53.994 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.651204+0000", 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.651204+0000", 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.651202+0000", 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:53.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.651208+0000", 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4310000000000002e-06 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.651218+0000", 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9180000000000006e-06 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:53.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.651224+0000", 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0469999999999999e-06 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.651260+0000", 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6693000000000003e-05 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.651335+0000", 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5067000000000005e-05 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.651588+0000", 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000252713 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.652762+0000", 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001173524 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:53.998 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.659212+0000", 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006450056 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.817280+0000", 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15806854300000001 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.817338+0000", 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7547999999999999e-05 2023-10-09T05:35:53.999 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.817400+0000", 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2401000000000001e-05 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:135163 5.c 5:31c88234:::benchmark_data_smithi008_238708_object17027:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:37.654823+0000", 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 76.268112259000006, 2023-10-09T05:35:54.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17046724199999999, 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135163 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.654823+0000", 2023-10-09T05:35:54.001 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.654823+0000", 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.654822+0000", 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.654827+0000", 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3190000000000002e-06 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.654838+0000", 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.084e-05 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.654845+0000", 2023-10-09T05:35:54.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8539999999999999e-06 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.654882+0000", 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6545999999999999e-05 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.654952+0000", 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0477999999999998e-05 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.004 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.655257+0000", 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00030448999999999998 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.656625+0000", 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001368789 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.005 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.659316+0000", 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026904950000000002 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.825201+0000", 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.165885435 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.825250+0000", 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8423e-05 2023-10-09T05:35:54.006 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:37.825291+0000", 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0930000000000003e-05 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:137273 5.c 5:353395ae:::benchmark_data_smithi008_238708_object19137:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:39.609292+0000", 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 74.313643571, 2023-10-09T05:35:54.007 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.159112947, 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 137273 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.609292+0000", 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.008 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.609292+0000", 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.609290+0000", 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.009 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.609296+0000", 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1800000000000001e-06 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.609309+0000", 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3059e-05 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.609323+0000", 2023-10-09T05:35:54.010 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3155e-05 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.609403+0000", 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0884999999999996e-05 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.609528+0000", 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012411799999999999 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.011 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.609930+0000", 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00040195299999999998 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.617205+0000", 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0072754500000000001 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.617233+0000", 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7971999999999999e-05 2023-10-09T05:35:54.012 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.768307+0000", 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15107415499999999 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.768354+0000", 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6705000000000001e-05 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.013 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.768405+0000", 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1146000000000001e-05 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:161807 5.1c 5:3af01b52:::benchmark_data_smithi008_238708_object43671:head [delete] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:35:05.702343+0000", 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 48.220592379999999, 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19827682399999999, 2023-10-09T05:35:54.014 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 161807 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702343+0000", 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.015 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702343+0000", 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702342+0000", 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702348+0000", 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9920000000000004e-06 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702360+0000", 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1688000000000001e-05 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702368+0000", 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1599999999999998e-06 2023-10-09T05:35:54.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702412+0000", 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3918999999999997e-05 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702534+0000", 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000121931 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.018 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702897+0000", 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00036327899999999999 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.707464+0000", 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045671349999999999 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.712601+0000", 2023-10-09T05:35:54.019 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051370840000000001 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900526+0000", 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18792462700000001 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900582+0000", 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6017999999999998e-05 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.020 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900620+0000", 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8303000000000002e-05 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:161808 5.1c 5:39ce818a:::benchmark_data_smithi008_238708_object43672:head [delete] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:35:05.702383+0000", 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 48.220552955999999, 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19837861000000001, 2023-10-09T05:35:54.021 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 161808 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702383+0000", 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.022 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702383+0000", 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702382+0000", 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:54.023 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702385+0000", 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3850000000000002e-06 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702393+0000", 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2930000000000003e-06 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702399+0000", 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6189999999999996e-06 2023-10-09T05:35:54.024 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.705223+0000", 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028235840000000001 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.705338+0000", 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011469800000000001 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:35:54.025 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.705630+0000", 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00029198199999999998 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.707621+0000", 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019917260000000001 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.714476+0000", 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068550089999999996 2023-10-09T05:35:54.026 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900672+0000", 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.186196107 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900719+0000", 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6526000000000001e-05 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:54.027 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900761+0000", 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2231999999999998e-05 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:170498 5.1c 5:3eed1d6c:::benchmark_data_smithi008_238708_object52362:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:35:14.632526+0000", 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 39.290409099000001, 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.153726053, 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:54.028 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 170498 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.632526+0000", 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.029 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.632526+0000", 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.632525+0000", 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.632531+0000", 2023-10-09T05:35:54.030 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6099999999999997e-06 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.632541+0000", 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9499999999999996e-06 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.632547+0000", 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7309999999999996e-06 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.031 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.632806+0000", 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00025861699999999999 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.635165+0000", 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002359107 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.635569+0000", 2023-10-09T05:35:54.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00040449000000000003 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.637755+0000", 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021860130000000001 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.640385+0000", 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026296010000000001 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.786149+0000", 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14576366399999999 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.786210+0000", 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1285999999999996e-05 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:14.786253+0000", 2023-10-09T05:35:54.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2515000000000002e-05 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:192182 5.c 5:32d1084d:::benchmark_data_smithi008_238708_object74046:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:35:38.634291+0000", 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 15.288644122999999, 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14501971, 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:54.035 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 192182 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.634291+0000", 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:54.036 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.634291+0000", 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.634290+0000", 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.634298+0000", 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2800000000000003e-06 2023-10-09T05:35:54.037 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.634311+0000", 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3283e-05 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.634322+0000", 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.03e-05 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.634358+0000", 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6888999999999999e-05 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.634449+0000", 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0626999999999998e-05 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.634874+0000", 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00042440400000000002 2023-10-09T05:35:54.039 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.637546+0000", 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026726720000000001 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.639190+0000", 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016437050000000001 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.040 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.779230+0000", 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.140040106 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.779275+0000", 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4910999999999997e-05 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:38.779311+0000", 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6278000000000001e-05 2023-10-09T05:35:54.041 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:204350 5.c 5:312bcca2:::benchmark_data_smithi008_238708_object86214:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:35:52.927123+0000", 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.99581253400000003, 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.16258330500000001, 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:35:54.042 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 204350 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:52.927123+0000", 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:35:54.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:52.927123+0000", 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:52.927121+0000", 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:52.927130+0000", 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4710000000000007e-06 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.044 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:52.927143+0000", 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3312000000000001e-05 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:52.927151+0000", 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2530000000000004e-06 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:35:54.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:52.927221+0000", 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9895999999999996e-05 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:52.927333+0000", 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011220100000000001 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:52.927850+0000", 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00051647499999999996 2023-10-09T05:35:54.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:52.930710+0000", 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002860769 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:52.942581+0000", 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011870095000000001 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:35:54.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:53.089616+0000", 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14703533599999999 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:53.089663+0000", 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7376999999999997e-05 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:53.089706+0000", 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3086e-05 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.048 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:54.049 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.049 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:35:54.049 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:35:54.049 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:35:54.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:53 smithi191 ceph-mon[161669]: pgmap v365: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 19730/104112 objects degraded (18.951%); 4.3 MiB/s, 85 objects/s recovering 2023-10-09T05:35:54.101 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:54.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:54.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:54.212 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:54.278 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:54.371 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:54.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:54.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:54.474 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:54.527 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:54.833 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:54.836 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:54 smithi008 ceph-mon[184418]: pgmap v366: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 19730/101583 objects degraded (19.423%); 3.6 MiB/s, 71 objects/s recovering 2023-10-09T05:35:54.836 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:54 smithi008 ceph-mon[191227]: pgmap v366: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 19730/101583 objects degraded (19.423%); 3.6 MiB/s, 71 objects/s recovering 2023-10-09T05:35:54.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:54.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:54.938 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:55.004 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:55.031 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:35:55.054 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:54 smithi191 ceph-mon[161669]: pgmap v366: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 19730/101583 objects degraded (19.423%); 3.6 MiB/s, 71 objects/s recovering 2023-10-09T05:35:55.137 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:55.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:55.258 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:55.373 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:55.433 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:35:55.473 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:55.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:55.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:55.579 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:55.624 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:55.723 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:35:55.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:55.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:35:55.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:35:55.879 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:55.984 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:35:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:56.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:35:56.102 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:35:56.154 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:56.351 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:35:56.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:56.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:35:56.400 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:56.467 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:56.484 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:56.542 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:56.684 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:56.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:56.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:56.791 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:56.859 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:57.076 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:57.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:57.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:57.112 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:57 smithi008 ceph-mon[184418]: 5.13 scrub starts 2023-10-09T05:35:57.112 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:57 smithi008 ceph-mon[184418]: pgmap v367: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 19397/96501 objects degraded (20.100%); 4.2 MiB/s, 81 objects/s recovering 2023-10-09T05:35:57.112 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:57 smithi008 ceph-mon[191227]: 5.13 scrub starts 2023-10-09T05:35:57.113 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:57 smithi008 ceph-mon[191227]: pgmap v367: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 19397/96501 objects degraded (20.100%); 4.2 MiB/s, 81 objects/s recovering 2023-10-09T05:35:57.183 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:35:57.277 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:57.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:57 smithi191 ceph-mon[161669]: 5.13 scrub starts 2023-10-09T05:35:57.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:57 smithi191 ceph-mon[161669]: pgmap v367: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 19397/96501 objects degraded (20.100%); 4.2 MiB/s, 81 objects/s recovering 2023-10-09T05:35:57.941 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:35:57.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:57.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:35:58.049 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:35:58.082 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:35:58.098 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:58.166 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:35:58.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:58.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:35:58.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2592441534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:58.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19397/96501 objects degraded (20.100%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:35:58.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2592441534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:58.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19397/96501 objects degraded (20.100%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:35:58.276 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:35:58.343 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2592441534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:35:58.343 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19397/96501 objects degraded (20.100%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:35:58.343 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:58.418 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:35:58.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:58.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:35:58.521 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:58.564 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:59.213 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:59.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[184418]: 5.3 scrub starts 2023-10-09T05:35:59.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[184418]: pgmap v368: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 19397/89418 objects degraded (21.693%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:35:59.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:35:59.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2592441534' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:59.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[184418]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T05:35:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[184418]: failed to encode map e143 with expected crc 2023-10-09T05:35:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[184418]: failed to encode map e143 with expected crc 2023-10-09T05:35:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[184418]: failed to encode map e143 with expected crc 2023-10-09T05:35:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:59.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[191227]: 5.3 scrub starts 2023-10-09T05:35:59.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[191227]: pgmap v368: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 19397/89418 objects degraded (21.693%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:35:59.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:35:59.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2592441534' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:59.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[191227]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T05:35:59.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[191227]: failed to encode map e143 with expected crc 2023-10-09T05:35:59.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[191227]: failed to encode map e143 with expected crc 2023-10-09T05:35:59.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[191227]: failed to encode map e143 with expected crc 2023-10-09T05:35:59.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:59.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:35:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:59.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:59.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:59.320 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:35:59.355 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:35:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:59 smithi191 ceph-mon[161669]: 5.3 scrub starts 2023-10-09T05:35:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:59 smithi191 ceph-mon[161669]: pgmap v368: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 19397/89418 objects degraded (21.693%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:35:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:59 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:35:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2592441534' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:35:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:59 smithi191 ceph-mon[161669]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T05:35:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:59 smithi191 ceph-mon[161669]: failed to encode map e143 with expected crc 2023-10-09T05:35:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:59 smithi191 ceph-mon[161669]: failed to encode map e143 with expected crc 2023-10-09T05:35:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:59 smithi191 ceph-mon[161669]: failed to encode map e143 with expected crc 2023-10-09T05:35:59.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:35:59.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:35:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:35:59.685 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:35:59.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:35:59.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:35:59.788 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:35:59.862 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:00.000 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:00.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:00.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:00.106 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:00.176 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:00.328 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:36:00.328 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:36:00.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.328 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:210468 5.16 5:6d2fccf5:::benchmark_data_smithi008_238708_object92332:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T05:36:00.328 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:36:00.315331+0000", 2023-10-09T05:36:00.328 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.01193338, 2023-10-09T05:36:00.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011966689000000001, 2023-10-09T05:36:00.328 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:00.328 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 210468 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315331+0000", 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315331+0000", 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315329+0000", 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:00.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315338+0000", 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0899999999999994e-06 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315354+0000", 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5668e-05 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315365+0000", 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0713999999999999e-05 2023-10-09T05:36:00.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315464+0000", 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9108e-05 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315635+0000", 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000170679 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.332 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:36:00.333 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315786+0000", 2023-10-09T05:36:00.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000150944 2023-10-09T05:36:00.333 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.333 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:00.333 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.333 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.333 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.333 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:210471 5.6 5:65c6c088:::benchmark_data_smithi008_238708_object92335:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T05:36:00.333 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:36:00.315384+0000", 2023-10-09T05:36:00.333 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.011880170000000001, 2023-10-09T05:36:00.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011999309999999999, 2023-10-09T05:36:00.334 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:00.334 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:36:00.334 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:00.334 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:00.334 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:00.334 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 210471 2023-10-09T05:36:00.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.334 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:00.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:00.334 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315384+0000", 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315384+0000", 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315384+0000", 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315387+0000", 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1970000000000001e-06 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315394+0000", 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7789999999999998e-06 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:00.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315400+0000", 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7060000000000001e-06 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.319024+0000", 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036244860000000001 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.319190+0000", 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016635099999999999 2023-10-09T05:36:00.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.338 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.338 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:36:00.338 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.319335+0000", 2023-10-09T05:36:00.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014446700000000001 2023-10-09T05:36:00.338 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.338 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:00.338 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.338 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.338 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:210475 5.16 5:69ab259a:::benchmark_data_smithi008_238708_object92339:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T05:36:00.338 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:36:00.315405+0000", 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.011859653, 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012055768, 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 210475 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315405+0000", 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315405+0000", 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315405+0000", 2023-10-09T05:36:00.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315407+0000", 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4600000000000002e-06 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315413+0000", 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7910000000000001e-06 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:00.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.315418+0000", 2023-10-09T05:36:00.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9749999999999998e-06 2023-10-09T05:36:00.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:00.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322610+0000", 2023-10-09T05:36:00.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0071922990000000001 2023-10-09T05:36:00.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:00.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322759+0000", 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014936699999999999 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322908+0000", 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000148818 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.343 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:210477 5.16 5.b96c596 (undecoded) ondisk+write+known_if_redirected e143)", 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:36:00.322402+0000", 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0048623989999999999, 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051303360000000001, 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "queued for pg", 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 210477 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:00.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322402+0000", 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322402+0000", 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322400+0000", 2023-10-09T05:36:00.345 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322412+0000", 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1973e-05 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322439+0000", 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7217e-05 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322451+0000", 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.173e-05 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:210478 5.1a 5:5dc6fff3:::benchmark_data_smithi008_238708_object92342:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:36:00.322462+0000", 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.004802387, 2023-10-09T05:36:00.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051284850000000003, 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 210478 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322462+0000", 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322462+0000", 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322462+0000", 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322466+0000", 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2459999999999997e-06 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322473+0000", 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.52e-06 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322479+0000", 2023-10-09T05:36:00.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8390000000000003e-06 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322532+0000", 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3195000000000001e-05 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.322670+0000", 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013764699999999999 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.323400+0000", 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00072999600000000001 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.34770.0:210478 5.1a e143/123)", 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:36:00.327203+0000", 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 6.1247999999999994e-05, 2023-10-09T05:36:00.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00045777200000000001, 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "queued for pg", 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.327203+0000", 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.327203+0000", 2023-10-09T05:36:00.353 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.327202+0000", 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.327209+0000", 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9809999999999997e-06 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.327222+0000", 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.295e-05 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:00.327231+0000", 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4509999999999992e-06 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.355 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:00.356 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:36:00.356 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 6 2023-10-09T05:36:00.356 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:36:00.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:00.357 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:36:00.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:00.415 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:36:00.459 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:00.476 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:36:00.522 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:00.525 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:36:00.561 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:36:00.599 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:36:00.627 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:36:00.668 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:36:00.670 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:36:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:36:00] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.33.4" 2023-10-09T05:36:00.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:00.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:00.772 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:00.799 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:00.853 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:36:00.890 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:36:00.908 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:00.910 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:36:00.910 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:36:00.910 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:36:00.910 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:36:00.910 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:36:00.910 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:36:00.910 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:36:00.910 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:36:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:00.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:01.014 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:01.069 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:01.166 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:01.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:01.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:01.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:01 smithi008 ceph-mon[191227]: 5.23 scrub starts 2023-10-09T05:36:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:01 smithi008 ceph-mon[191227]: pgmap v370: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 19062/83892 objects degraded (22.722%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T05:36:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:01 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2410138720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:01 smithi008 ceph-mon[184418]: 5.23 scrub starts 2023-10-09T05:36:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:01 smithi008 ceph-mon[184418]: pgmap v370: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 19062/83892 objects degraded (22.722%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T05:36:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:01 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2410138720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:01.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:01.324 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:01 smithi191 ceph-mon[161669]: 5.23 scrub starts 2023-10-09T05:36:01.324 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:01 smithi191 ceph-mon[161669]: pgmap v370: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 19062/83892 objects degraded (22.722%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T05:36:01.324 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:01 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2410138720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:01.331 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:01.446 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:36:01.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:01.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:01.554 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:01.582 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:01.806 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:36:01.888 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:36:02.071 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:02.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:02.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:02.202 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:02.251 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:02.397 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:02.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:02.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:02.506 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:02.529 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:02.590 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:02.606 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:02.680 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:36:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:36:02.680 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:36:02.711 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:02.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:02.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:02.897 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:02.998 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:36:03.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:03.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:03.109 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:03.182 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:03.305 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:03 smithi191 ceph-mon[161669]: pgmap v371: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 19062/78618 objects degraded (24.246%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T05:36:03.306 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19062/78618 objects degraded (24.246%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:03.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:03 smithi008 ceph-mon[184418]: pgmap v371: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 19062/78618 objects degraded (24.246%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T05:36:03.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19062/78618 objects degraded (24.246%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:03.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:03 smithi008 ceph-mon[191227]: pgmap v371: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 19062/78618 objects degraded (24.246%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T05:36:03.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19062/78618 objects degraded (24.246%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:03.818 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:03.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:03.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:03.924 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:04.006 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:04.072 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:36:04.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:04.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:04.085 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:36:04.181 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:04.248 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:04.962 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:04.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:04.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:05.075 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:05.132 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:05.262 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:05.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:05.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:05.413 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:05.424 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:05 smithi191 ceph-mon[161669]: 5.15 scrub starts 2023-10-09T05:36:05.424 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:05 smithi191 ceph-mon[161669]: pgmap v372: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 19062/76332 objects degraded (24.972%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T05:36:05.424 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:05.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:05 smithi008 ceph-mon[184418]: 5.15 scrub starts 2023-10-09T05:36:05.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:05 smithi008 ceph-mon[184418]: pgmap v372: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 19062/76332 objects degraded (24.972%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T05:36:05.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:05 smithi008 ceph-mon[191227]: 5.15 scrub starts 2023-10-09T05:36:05.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:05 smithi008 ceph-mon[191227]: pgmap v372: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 19062/76332 objects degraded (24.972%); 4.2 MiB/s, 78 objects/s recovering 2023-10-09T05:36:05.511 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:05.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:05.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:05.616 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:05.712 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:05.831 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:05.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:05.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:05.892 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:36:05.893 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:36:05.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:05.996 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:06.109 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:06.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:06.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:06.124 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:06 smithi008 ceph-mon[191227]: 5.13 scrub starts 2023-10-09T05:36:06.124 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1222258995' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:06.124 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:06 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:06.146 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:36:06.223 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:06.298 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:06.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:06 smithi191 ceph-mon[161669]: 5.13 scrub starts 2023-10-09T05:36:06.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1222258995' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:06 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:06 smithi008 ceph-mon[184418]: 5.13 scrub starts 2023-10-09T05:36:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1222258995' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:06 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:07.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[184418]: pgmap v373: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 19062/71907 objects degraded (26.509%); 2.1 MiB/s, 95 objects/s recovering 2023-10-09T05:36:07.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:07.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[184418]: failed to encode map e144 with expected crc 2023-10-09T05:36:07.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[184418]: failed to encode map e144 with expected crc 2023-10-09T05:36:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:36:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[184418]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T05:36:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[191227]: pgmap v373: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 19062/71907 objects degraded (26.509%); 2.1 MiB/s, 95 objects/s recovering 2023-10-09T05:36:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[191227]: failed to encode map e144 with expected crc 2023-10-09T05:36:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[191227]: failed to encode map e144 with expected crc 2023-10-09T05:36:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:36:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:07 smithi008 ceph-mon[191227]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T05:36:07.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:07 smithi191 ceph-mon[161669]: pgmap v373: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 19062/71907 objects degraded (26.509%); 2.1 MiB/s, 95 objects/s recovering 2023-10-09T05:36:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:07 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:07 smithi191 ceph-mon[161669]: failed to encode map e144 with expected crc 2023-10-09T05:36:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:07 smithi191 ceph-mon[161669]: failed to encode map e144 with expected crc 2023-10-09T05:36:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:07 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:36:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:07 smithi191 ceph-mon[161669]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T05:36:07.861 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:07.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:07.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:07.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:07.994 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:08.126 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19062/71907 objects degraded (26.509%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:08.137 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:08.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:08.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:08.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19062/71907 objects degraded (26.509%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:08.329 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:08.389 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19062/71907 objects degraded (26.509%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:08.419 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:36:08.438 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:08.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:08.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:08.544 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:08.633 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:08.765 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:08.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:08.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:08.871 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:08.916 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:08.985 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:36:08.986 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:36:08.986 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T05:36:08.986 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:36:09.023 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:09.036 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:36:09.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:09.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:09.077 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:36:09.111 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:36:09.141 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:09.170 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:36:09.197 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:09.214 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:36:09.250 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:36:09.278 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:09.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:09.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:09.289 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:36:09.327 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:36:09.337 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:09.341 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:36:09.341 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:36:09.341 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:36:09.341 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:36:09.341 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:36:09.341 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:36:09.342 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:36:09.342 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:36:09.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:09.454 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:09.576 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:09.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:09.687 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:09 smithi008 ceph-mon[184418]: pgmap v375: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 19062/64884 objects degraded (29.379%); 2.1 MiB/s, 96 objects/s recovering 2023-10-09T05:36:09.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:09 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2780527483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:09 smithi008 ceph-mon[191227]: pgmap v375: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 19062/64884 objects degraded (29.379%); 2.1 MiB/s, 96 objects/s recovering 2023-10-09T05:36:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:09 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2780527483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:09.730 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:09 smithi191 ceph-mon[161669]: pgmap v375: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 19062/64884 objects degraded (29.379%); 2.1 MiB/s, 96 objects/s recovering 2023-10-09T05:36:09.730 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:09 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2780527483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:09.741 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:09.846 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:09.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:09.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:09.959 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:10.030 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:10.122 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:10.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:10.232 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:10.298 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:10.375 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:36:10.381 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:10.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:10.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:10.495 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:10.660 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:10.701 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:36:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:36:10] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.33.4" 2023-10-09T05:36:10.815 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:10.942 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:10.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:10.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:11.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:11.082 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:11.136 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:11.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:11.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:11.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:11.270 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:11.316 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:11.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:11.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:11.422 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:11.453 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:11.677 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:36:11.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:11.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:11 smithi008 ceph-mon[184418]: pgmap v376: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 18394/59529 objects degraded (30.899%); 0 B/s, 118 objects/s recovering 2023-10-09T05:36:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:11 smithi008 ceph-mon[184418]: failed to encode map e145 with expected crc 2023-10-09T05:36:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:11 smithi008 ceph-mon[184418]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T05:36:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:11 smithi008 ceph-mon[184418]: failed to encode map e145 with expected crc 2023-10-09T05:36:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:11 smithi008 ceph-mon[184418]: failed to encode map e145 with expected crc 2023-10-09T05:36:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:11 smithi008 ceph-mon[191227]: pgmap v376: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 18394/59529 objects degraded (30.899%); 0 B/s, 118 objects/s recovering 2023-10-09T05:36:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:11 smithi008 ceph-mon[191227]: failed to encode map e145 with expected crc 2023-10-09T05:36:11.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:11 smithi008 ceph-mon[191227]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T05:36:11.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:11 smithi008 ceph-mon[191227]: failed to encode map e145 with expected crc 2023-10-09T05:36:11.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:11 smithi008 ceph-mon[191227]: failed to encode map e145 with expected crc 2023-10-09T05:36:11.781 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:11.833 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:11.923 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:11.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:11.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:11.934 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:11 smithi191 ceph-mon[161669]: pgmap v376: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 18394/59529 objects degraded (30.899%); 0 B/s, 118 objects/s recovering 2023-10-09T05:36:11.935 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:11 smithi191 ceph-mon[161669]: failed to encode map e145 with expected crc 2023-10-09T05:36:11.935 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:11 smithi191 ceph-mon[161669]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T05:36:11.935 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:11 smithi191 ceph-mon[161669]: failed to encode map e145 with expected crc 2023-10-09T05:36:11.935 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:11 smithi191 ceph-mon[161669]: failed to encode map e145 with expected crc 2023-10-09T05:36:12.035 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:12.118 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:12.213 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:12.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:12.287 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:12.328 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:12.430 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:12.535 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:36:12.550 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:12.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:12.657 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:12.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:12 smithi008 ceph-mon[184418]: failed to encode map e146 with expected crc 2023-10-09T05:36:12.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:12 smithi008 ceph-mon[184418]: failed to encode map e146 with expected crc 2023-10-09T05:36:12.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:12 smithi008 ceph-mon[184418]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T05:36:12.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:12 smithi008 ceph-mon[184418]: failed to encode map e146 with expected crc 2023-10-09T05:36:12.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:12 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 18394/59529 objects degraded (30.899%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:12.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:12 smithi008 ceph-mon[191227]: failed to encode map e146 with expected crc 2023-10-09T05:36:12.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:12 smithi008 ceph-mon[191227]: failed to encode map e146 with expected crc 2023-10-09T05:36:12.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:12 smithi008 ceph-mon[191227]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T05:36:12.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:12 smithi008 ceph-mon[191227]: failed to encode map e146 with expected crc 2023-10-09T05:36:12.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:12 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 18394/59529 objects degraded (30.899%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:12.750 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:12.863 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:36:12.864 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:36:12.864 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:36:12.864 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:36:12.864 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.864 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54034 5.6 5:6bcc9ee5:::benchmark_data_smithi008_238708_object54033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:36:12.864 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:36:12.864 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 172.014500739, 2023-10-09T05:36:12.864 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0234612670000001, 2023-10-09T05:36:12.864 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54034 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.865 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.866 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.866 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.866 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:36:12.866 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.866 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.866 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.866 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.866 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:36:12.866 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:12.866 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.866 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517446+0000", 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0636999999999999e-05 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517453+0000", 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5390000000000004e-06 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.867 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517458+0000", 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0839999999999999e-06 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.521341+0000", 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038830990000000001 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.521502+0000", 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016102400000000001 2023-10-09T05:36:12.868 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.869 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.869 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:36:12.869 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.521674+0000", 2023-10-09T05:36:12.869 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017155600000000001 2023-10-09T05:36:12.869 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.869 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.869 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.869 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.524020+0000", 2023-10-09T05:36:12.869 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002346295 2023-10-09T05:36:12.869 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.869 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.870 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.870 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.529910+0000", 2023-10-09T05:36:12.870 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058903289999999997 2023-10-09T05:36:12.870 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.870 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.870 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.870 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541133+0000", 2023-10-09T05:36:12.870 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011222492000000001 2023-10-09T05:36:12.870 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.870 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.870 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541145+0000", 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1871999999999999e-05 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540792+0000", 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99964734499999996 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540887+0000", 2023-10-09T05:36:12.871 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5445999999999994e-05 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54039 5.6 5:6a5b7d04:::benchmark_data_smithi008_238708_object54038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.527531+0000", 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 172.004396227, 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.017534768, 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:12.872 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54039 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527531+0000", 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.873 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.874 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527531+0000", 2023-10-09T05:36:12.874 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.874 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.874 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.874 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.874 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527529+0000", 2023-10-09T05:36:12.874 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:12.874 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.874 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.874 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.874 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527558+0000", 2023-10-09T05:36:12.875 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9082e-05 2023-10-09T05:36:12.875 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.875 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.875 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.875 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527573+0000", 2023-10-09T05:36:12.875 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5038000000000001e-05 2023-10-09T05:36:12.875 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.875 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.875 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.875 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527583+0000", 2023-10-09T05:36:12.875 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3249999999999997e-06 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527639+0000", 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6297e-05 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527793+0000", 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015416599999999999 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.876 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527952+0000", 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015940799999999999 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.530713+0000", 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002760878 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.877 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541193+0000", 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010480041000000001 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541203+0000", 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1980000000000007e-06 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.544999+0000", 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.00379627 2023-10-09T05:36:12.878 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545065+0000", 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6607999999999997e-05 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54041 5.4 5:2173e5b4:::benchmark_data_smithi008_238708_object54040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e112)", 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.535206+0000", 2023-10-09T05:36:12.879 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 171.996720665, 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.006119118, 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54041 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.880 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535206+0000", 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535206+0000", 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535205+0000", 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.881 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.882 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.882 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535227+0000", 2023-10-09T05:36:12.882 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2062999999999999e-05 2023-10-09T05:36:12.882 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.882 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.882 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.882 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535240+0000", 2023-10-09T05:36:12.882 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2991e-05 2023-10-09T05:36:12.882 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.882 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.882 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535248+0000", 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7789999999999996e-06 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535783+0000", 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000534484 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535798+0000", 2023-10-09T05:36:12.883 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5403000000000001e-05 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540201+0000", 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0044033379999999 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.541325+0000", 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001123906 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.884 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54046 5.4 5:2050c775:::benchmark_data_smithi008_238708_object54045:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e112)", 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 171.99666751699999, 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0145929970000001, 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.885 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54046 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.886 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535285+0000", 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6137999999999999e-05 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.887 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535294+0000", 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9299999999999992e-06 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535302+0000", 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0250000000000002e-06 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535822+0000", 2023-10-09T05:36:12.888 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052059000000000003 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535829+0000", 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5420000000000002e-06 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.541576+0000", 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.005747071 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.889 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.541934+0000", 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00035752700000000001 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.542079+0000", 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014546200000000001 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549695+0000", 2023-10-09T05:36:12.890 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0076156389999999996 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549765+0000", 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0362000000000002e-05 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549796+0000", 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1239999999999999e-05 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.891 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549852+0000", 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6103000000000001e-05 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54078 5.4 5:286f5610:::benchmark_data_smithi008_238708_object54077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 171.95059675499999, 2023-10-09T05:36:12.892 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96860710699999997, 2023-10-09T05:36:12.893 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.893 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:12.893 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.893 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.893 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.893 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54078 2023-10-09T05:36:12.893 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.893 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.893 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.893 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.893 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:12.894 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581351+0000", 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1622000000000001e-05 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581359+0000", 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5129999999999999e-06 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.895 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581366+0000", 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8000000000000001e-06 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581421+0000", 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5843e-05 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581430+0000", 2023-10-09T05:36:12.896 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7530000000000007e-06 2023-10-09T05:36:12.897 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.897 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.897 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.897 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543690+0000", 2023-10-09T05:36:12.897 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96226007599999996 2023-10-09T05:36:12.897 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.897 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.897 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.897 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543809+0000", 2023-10-09T05:36:12.897 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000118658 2023-10-09T05:36:12.897 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.898 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.898 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:36:12.898 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543945+0000", 2023-10-09T05:36:12.898 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013617699999999999 2023-10-09T05:36:12.898 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.898 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.898 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.898 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549727+0000", 2023-10-09T05:36:12.898 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0057819430000000003 2023-10-09T05:36:12.898 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.898 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.899 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.899 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549865+0000", 2023-10-09T05:36:12.899 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013792899999999999 2023-10-09T05:36:12.899 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.899 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.899 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:12.899 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549891+0000", 2023-10-09T05:36:12.899 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6105000000000002e-05 2023-10-09T05:36:12.899 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.899 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.899 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.900 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549937+0000", 2023-10-09T05:36:12.900 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6213e-05 2023-10-09T05:36:12.900 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.900 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.900 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.900 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.900 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.900 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54079 5.4 5:29366199:::benchmark_data_smithi008_238708_object54078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:36:12.900 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.582855+0000", 2023-10-09T05:36:12.900 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 171.94907186899999, 2023-10-09T05:36:12.900 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96716001200000001, 2023-10-09T05:36:12.901 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.901 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:12.901 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.901 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.901 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.901 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54079 2023-10-09T05:36:12.901 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.901 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.901 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.901 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.901 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582855+0000", 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582855+0000", 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582854+0000", 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:36:12.902 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.903 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.903 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.903 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582876+0000", 2023-10-09T05:36:12.903 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1637999999999998e-05 2023-10-09T05:36:12.903 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.903 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.903 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.903 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582891+0000", 2023-10-09T05:36:12.903 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5489999999999999e-05 2023-10-09T05:36:12.903 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.903 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582899+0000", 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5950000000000001e-06 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582953+0000", 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4355999999999999e-05 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.904 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582962+0000", 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9530000000000005e-06 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545663+0000", 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96270056100000001 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545753+0000", 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0152e-05 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.905 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.906 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:36:12.906 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545860+0000", 2023-10-09T05:36:12.906 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000107358 2023-10-09T05:36:12.906 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.906 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.906 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.906 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549748+0000", 2023-10-09T05:36:12.906 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038876010000000001 2023-10-09T05:36:12.906 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.906 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.906 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549948+0000", 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020022200000000001 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549973+0000", 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5219e-05 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550015+0000", 2023-10-09T05:36:12.907 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1847000000000001e-05 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54081 5.4 5:25d3ef94:::benchmark_data_smithi008_238708_object54080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 171.948562549, 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96692271200000002, 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:12.908 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54081 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.909 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.910 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:36:12.910 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.910 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.910 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.910 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.910 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:36:12.910 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:36:12.910 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.910 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.910 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.910 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583385+0000", 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1100000000000001e-05 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583399+0000", 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4654000000000001e-05 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583406+0000", 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1250000000000004e-06 2023-10-09T05:36:12.911 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583456+0000", 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9496000000000002e-05 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583464+0000", 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9300000000000003e-06 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.912 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547622+0000", 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96415766000000003 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547731+0000", 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000109104 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547847+0000", 2023-10-09T05:36:12.913 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000115983 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550186+0000", 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023388979999999998 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550198+0000", 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2286000000000001e-05 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.914 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550244+0000", 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6045999999999999e-05 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550287+0000", 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.32e-05 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.915 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54082 5.d 5:bd5f0afe:::benchmark_data_smithi008_238708_object54081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 171.94639643900001, 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96213848199999996, 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54082 2023-10-09T05:36:12.916 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.917 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.917 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.917 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.917 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:36:12.917 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.917 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.917 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.917 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.917 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:36:12.917 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.917 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585549+0000", 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8794999999999999e-05 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.918 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585559+0000", 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0594999999999999e-05 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585564+0000", 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1019999999999996e-06 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585609+0000", 2023-10-09T05:36:12.919 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4693000000000001e-05 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585619+0000", 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9839999999999996e-06 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540181+0000", 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.954562456 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.920 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540364+0000", 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000182428 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540531+0000", 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016716999999999999 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.921 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543639+0000", 2023-10-09T05:36:12.922 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003108235 2023-10-09T05:36:12.922 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.922 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.922 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.922 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547559+0000", 2023-10-09T05:36:12.922 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039201629999999999 2023-10-09T05:36:12.922 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.922 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.922 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:12.922 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547601+0000", 2023-10-09T05:36:12.922 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1631999999999997e-05 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547669+0000", 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7916000000000005e-05 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98105 5.8 5:13fcc3a0:::benchmark_data_smithi008_238708_object98104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:36:12.923 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695499+0000", 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 131.83642786799999, 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93824070100000001, 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98105 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.924 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.925 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695499+0000", 2023-10-09T05:36:12.925 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.925 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.925 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.925 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.925 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695499+0000", 2023-10-09T05:36:12.925 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.925 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.925 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.925 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.925 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695497+0000", 2023-10-09T05:36:12.926 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:12.926 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.926 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.926 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.926 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695527+0000", 2023-10-09T05:36:12.926 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9377999999999999e-05 2023-10-09T05:36:12.926 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.926 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.926 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.926 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695541+0000", 2023-10-09T05:36:12.926 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4511e-05 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695552+0000", 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0711e-05 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881129+0000", 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18557714 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.927 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.928 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.928 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881142+0000", 2023-10-09T05:36:12.928 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2874000000000001e-05 2023-10-09T05:36:12.928 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.928 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.928 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.928 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633642+0000", 2023-10-09T05:36:12.928 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75249991800000005 2023-10-09T05:36:12.928 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.928 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.928 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633740+0000", 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8040000000000003e-05 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98114 5.6 5:639cccb8:::benchmark_data_smithi008_238708_object98113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695561+0000", 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 131.836366274, 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94775633000000004, 2023-10-09T05:36:12.929 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98114 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695561+0000", 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.930 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695561+0000", 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695560+0000", 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.931 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695587+0000", 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6925000000000001e-05 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695596+0000", 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8750000000000006e-06 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695603+0000", 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.815e-06 2023-10-09T05:36:12.932 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.933 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.933 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.933 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887612+0000", 2023-10-09T05:36:12.933 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19200974600000001 2023-10-09T05:36:12.933 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.933 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.933 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.933 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887622+0000", 2023-10-09T05:36:12.933 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3079999999999997e-06 2023-10-09T05:36:12.933 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.933 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.934 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.934 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.626477+0000", 2023-10-09T05:36:12.934 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73885584199999998 2023-10-09T05:36:12.934 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.934 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.934 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:36:12.934 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627930+0000", 2023-10-09T05:36:12.934 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014526540000000001 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.643236+0000", 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015305859999999999 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.643317+0000", 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0939999999999994e-05 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.935 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.936 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.936 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.936 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98120 5.18 5:1c597501:::benchmark_data_smithi008_238708_object98119:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:36:12.936 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695612+0000", 2023-10-09T05:36:12.936 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 131.83631487400001, 2023-10-09T05:36:12.936 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96914215000000004, 2023-10-09T05:36:12.936 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.936 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:12.936 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.936 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.936 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98120 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695612+0000", 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695612+0000", 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.937 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.938 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.938 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.938 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695611+0000", 2023-10-09T05:36:12.938 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:12.938 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.938 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.938 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.938 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695637+0000", 2023-10-09T05:36:12.938 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5809999999999999e-05 2023-10-09T05:36:12.938 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.938 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695646+0000", 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.303e-06 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695652+0000", 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3300000000000004e-06 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881169+0000", 2023-10-09T05:36:12.939 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18551722700000001 2023-10-09T05:36:12.940 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.940 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.940 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.940 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881176+0000", 2023-10-09T05:36:12.940 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1720000000000001e-06 2023-10-09T05:36:12.940 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.940 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.658602+0000", 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77742560599999999 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659106+0000", 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000503821 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.664640+0000", 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0055338050000000001 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.664754+0000", 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114644 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.942 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98126 5.8 5:11d9570b:::benchmark_data_smithi008_238708_object98125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695658+0000", 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 131.836269053, 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96035853800000004, 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98126 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.943 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695658+0000", 2023-10-09T05:36:12.944 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.944 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.944 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.944 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695658+0000", 2023-10-09T05:36:12.944 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.944 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.944 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.944 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695657+0000", 2023-10-09T05:36:12.945 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:12.945 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.945 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.945 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.945 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695683+0000", 2023-10-09T05:36:12.945 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5728000000000002e-05 2023-10-09T05:36:12.945 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.945 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.945 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.945 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695691+0000", 2023-10-09T05:36:12.945 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9869999999999999e-06 2023-10-09T05:36:12.946 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.946 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.946 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695697+0000", 2023-10-09T05:36:12.946 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9120000000000003e-06 2023-10-09T05:36:12.946 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.946 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.946 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881200+0000", 2023-10-09T05:36:12.946 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.185503105 2023-10-09T05:36:12.946 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.947 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.947 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.947 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881210+0000", 2023-10-09T05:36:12.947 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4259999999999992e-06 2023-10-09T05:36:12.947 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.947 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.947 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633764+0000", 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75255434799999998 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633873+0000", 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010911700000000001 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.634034+0000", 2023-10-09T05:36:12.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000161149 2023-10-09T05:36:12.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.949 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.949 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.949 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.651360+0000", 2023-10-09T05:36:12.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017325393000000001 2023-10-09T05:36:12.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.949 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.949 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.949 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.654621+0000", 2023-10-09T05:36:12.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032617940000000002 2023-10-09T05:36:12.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.950 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.950 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.950 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.655917+0000", 2023-10-09T05:36:12.950 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001295678 2023-10-09T05:36:12.950 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.950 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.950 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:12.950 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.655974+0000", 2023-10-09T05:36:12.950 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6629999999999998e-05 2023-10-09T05:36:12.950 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.950 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.656016+0000", 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2697999999999997e-05 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98127 5.4 5:253a6397:::benchmark_data_smithi008_238708_object98126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695703+0000", 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 131.83622404299999, 2023-10-09T05:36:12.951 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.927888991, 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98127 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695703+0000", 2023-10-09T05:36:12.952 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.953 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.953 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.953 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.953 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695703+0000", 2023-10-09T05:36:12.953 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.953 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.953 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.953 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.953 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695702+0000", 2023-10-09T05:36:12.953 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:12.954 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.954 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.954 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.954 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695732+0000", 2023-10-09T05:36:12.954 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9261999999999999e-05 2023-10-09T05:36:12.954 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.954 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.954 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.954 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695740+0000", 2023-10-09T05:36:12.954 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1280000000000008e-06 2023-10-09T05:36:12.954 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.955 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.955 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695746+0000", 2023-10-09T05:36:12.955 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1310000000000003e-06 2023-10-09T05:36:12.955 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.955 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.955 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.866359+0000", 2023-10-09T05:36:12.955 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17061283099999999 2023-10-09T05:36:12.955 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.866369+0000", 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.043e-05 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.604226+0000", 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.737856874 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.604398+0000", 2023-10-09T05:36:12.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017228299999999999 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.604580+0000", 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000182213 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.616540+0000", 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011959505 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.623441+0000", 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0069005589999999997 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.623503+0000", 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2378000000000005e-05 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.623592+0000", 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8899000000000004e-05 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98129 5.18 5:1d3b051a:::benchmark_data_smithi008_238708_object98128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695754+0000", 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 131.836173003, 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97840130400000003, 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.959 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:12.960 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.960 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.960 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.960 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98129 2023-10-09T05:36:12.960 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.960 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.960 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.960 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695754+0000", 2023-10-09T05:36:12.960 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.961 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.961 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.961 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.961 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695754+0000", 2023-10-09T05:36:12.961 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.961 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.961 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.961 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.961 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695753+0000", 2023-10-09T05:36:12.961 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:12.962 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.962 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.962 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.962 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695781+0000", 2023-10-09T05:36:12.962 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7693000000000001e-05 2023-10-09T05:36:12.963 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.963 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.963 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.963 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695789+0000", 2023-10-09T05:36:12.963 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9589999999999995e-06 2023-10-09T05:36:12.963 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.963 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.963 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.963 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695795+0000", 2023-10-09T05:36:12.963 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4239999999999998e-06 2023-10-09T05:36:12.964 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.964 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.964 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.964 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881231+0000", 2023-10-09T05:36:12.964 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18543580200000001 2023-10-09T05:36:12.964 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.964 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.964 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.964 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881238+0000", 2023-10-09T05:36:12.964 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8290000000000002e-06 2023-10-09T05:36:12.964 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659365+0000", 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77812663800000004 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659507+0000", 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014198800000000001 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.965 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:36:12.966 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659616+0000", 2023-10-09T05:36:12.966 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010964 2023-10-09T05:36:12.966 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.966 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.966 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.966 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.663796+0000", 2023-10-09T05:36:12.966 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041793669999999998 2023-10-09T05:36:12.966 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.966 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.966 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.966 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.664218+0000", 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00042226699999999999 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.674041+0000", 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0098232590000000009 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.674109+0000", 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8119000000000003e-05 2023-10-09T05:36:12.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.968 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.968 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.674155+0000", 2023-10-09T05:36:12.968 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5831999999999998e-05 2023-10-09T05:36:12.968 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.968 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.968 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.968 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.968 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98131 5.6 5:62d535ca:::benchmark_data_smithi008_238708_object98130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-10-09T05:36:12.968 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.711769+0000", 2023-10-09T05:36:12.969 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 131.82015787700001, 2023-10-09T05:36:12.969 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92605176700000003, 2023-10-09T05:36:12.969 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.969 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:12.969 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.969 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.969 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.969 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98131 2023-10-09T05:36:12.969 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.969 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.969 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.970 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.970 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711769+0000", 2023-10-09T05:36:12.970 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.970 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.970 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.970 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.970 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711769+0000", 2023-10-09T05:36:12.970 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.970 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.970 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.970 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711766+0000", 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999967 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711806+0000", 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0120000000000002e-05 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711836+0000", 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0463e-05 2023-10-09T05:36:12.971 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.972 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.972 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.972 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711854+0000", 2023-10-09T05:36:12.972 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7082999999999999e-05 2023-10-09T05:36:12.972 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.972 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.972 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.972 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.890394+0000", 2023-10-09T05:36:12.972 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17854087599999999 2023-10-09T05:36:12.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.973 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.890407+0000", 2023-10-09T05:36:12.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2317000000000001e-05 2023-10-09T05:36:12.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.973 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.628026+0000", 2023-10-09T05:36:12.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73761953099999999 2023-10-09T05:36:12.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.974 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.628147+0000", 2023-10-09T05:36:12.974 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012068700000000001 2023-10-09T05:36:12.974 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.974 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:36:12.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.628304+0000", 2023-10-09T05:36:12.974 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015666500000000001 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633549+0000", 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0052457479999999997 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637688+0000", 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041385900000000002 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.975 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:12.976 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637744+0000", 2023-10-09T05:36:12.976 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6211000000000001e-05 2023-10-09T05:36:12.976 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.976 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.976 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.976 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637821+0000", 2023-10-09T05:36:12.976 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6623000000000001e-05 2023-10-09T05:36:12.976 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.976 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.977 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.977 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.977 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.977 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98136 5.16 5:6f5da42e:::benchmark_data_smithi008_238708_object98135:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-10-09T05:36:12.977 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.790198+0000", 2023-10-09T05:36:12.977 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 131.741729107, 2023-10-09T05:36:12.977 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85936307599999995, 2023-10-09T05:36:12.977 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.977 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:12.978 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.978 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.978 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.978 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98136 2023-10-09T05:36:12.978 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.978 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.978 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.979 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.979 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790198+0000", 2023-10-09T05:36:12.979 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.979 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.979 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.979 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.979 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790198+0000", 2023-10-09T05:36:12.979 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.979 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.980 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.980 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.980 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790195+0000", 2023-10-09T05:36:12.980 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:36:12.980 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.980 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.980 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.980 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790227+0000", 2023-10-09T05:36:12.981 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2187999999999997e-05 2023-10-09T05:36:12.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.981 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.981 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.981 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790252+0000", 2023-10-09T05:36:12.981 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4901000000000001e-05 2023-10-09T05:36:12.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.981 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.982 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790264+0000", 2023-10-09T05:36:12.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2323e-05 2023-10-09T05:36:12.982 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.982 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.982 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887643+0000", 2023-10-09T05:36:12.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.097379027000000007 2023-10-09T05:36:12.982 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.982 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887650+0000", 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4860000000000002e-06 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.645624+0000", 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75797419499999996 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.983 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.645742+0000", 2023-10-09T05:36:12.984 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000117934 2023-10-09T05:36:12.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.984 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:36:12.984 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.645860+0000", 2023-10-09T05:36:12.984 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000117945 2023-10-09T05:36:12.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.984 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.984 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.647879+0000", 2023-10-09T05:36:12.984 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020193310000000001 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.649421+0000", 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015419749999999999 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.649474+0000", 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3298999999999999e-05 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.985 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.649561+0000", 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6372000000000006e-05 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119015 5.1d 5:b91a094f:::benchmark_data_smithi008_238708_object879:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.007040+0000", 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 108.524887017, 2023-10-09T05:36:12.986 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.032601662, 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119015 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007040+0000", 2023-10-09T05:36:12.987 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007040+0000", 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007038+0000", 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007048+0000", 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6420000000000002e-06 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007071+0000", 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3663999999999999e-05 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007081+0000", 2023-10-09T05:36:12.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8069999999999994e-06 2023-10-09T05:36:12.990 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.990 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.990 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.990 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201929+0000", 2023-10-09T05:36:12.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19484790399999999 2023-10-09T05:36:12.990 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.990 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.990 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.990 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201935+0000", 2023-10-09T05:36:12.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8259999999999998e-06 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.039550+0000", 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83761566499999995 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.039642+0000", 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1113999999999995e-05 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:12.991 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:12.992 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.992 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.992 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119017 5.d 5:b096558c:::benchmark_data_smithi008_238708_object881:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:36:12.992 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:36:12.992 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 108.524835603, 2023-10-09T05:36:12.992 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.010292135, 2023-10-09T05:36:12.992 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:12.992 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:12.992 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:12.993 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:12.993 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:12.993 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119017 2023-10-09T05:36:12.993 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.993 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:12.993 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.993 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:12.993 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:36:12.993 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.993 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.994 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.994 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:12.994 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:36:12.994 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:12.994 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.994 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.994 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:12.994 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:36:12.994 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:12.994 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.994 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007094+0000", 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7e-06 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007099+0000", 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5899999999999998e-06 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007104+0000", 2023-10-09T05:36:12.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7180000000000004e-06 2023-10-09T05:36:12.996 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201969+0000", 2023-10-09T05:36:12.996 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19486541099999999 2023-10-09T05:36:12.996 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:12.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201973+0000", 2023-10-09T05:36:12.997 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1320000000000004e-06 2023-10-09T05:36:12.997 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.997 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:12.997 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011724+0000", 2023-10-09T05:36:12.997 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.80975027700000002 2023-10-09T05:36:12.997 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.997 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:12.997 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011860+0000", 2023-10-09T05:36:12.998 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013602500000000001 2023-10-09T05:36:12.998 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.998 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.998 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:36:12.998 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011985+0000", 2023-10-09T05:36:12.998 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012507600000000001 2023-10-09T05:36:12.998 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.998 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.998 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:12.998 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.016288+0000", 2023-10-09T05:36:12.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0043027300000000003 2023-10-09T05:36:12.999 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.999 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:12.999 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.017244+0000", 2023-10-09T05:36:12.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095609500000000001 2023-10-09T05:36:12.999 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:12.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:12.999 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:12.999 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.017296+0000", 2023-10-09T05:36:13.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2408999999999999e-05 2023-10-09T05:36:13.000 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.000 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.000 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:13.000 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.017383+0000", 2023-10-09T05:36:13.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7373999999999994e-05 2023-10-09T05:36:13.000 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:13.000 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:13.000 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:13.001 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.001 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.001 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119025 5.1b 5:dce538de:::benchmark_data_smithi008_238708_object889:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:36:13.001 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:36:13.001 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 108.52481859, 2023-10-09T05:36:13.001 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.063606593, 2023-10-09T05:36:13.001 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:13.001 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:13.001 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:13.002 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:13.002 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:13.002 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119025 2023-10-09T05:36:13.002 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.002 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:13.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:13.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:36:13.002 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:13.003 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.003 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:13.003 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:36:13.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:13.003 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.003 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:13.003 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:36:13.004 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:13.004 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.004 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.004 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:13.004 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007110+0000", 2023-10-09T05:36:13.004 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9549999999999999e-06 2023-10-09T05:36:13.004 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.004 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.004 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:13.004 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007117+0000", 2023-10-09T05:36:13.004 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5860000000000001e-06 2023-10-09T05:36:13.005 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.005 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.005 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:13.005 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007121+0000", 2023-10-09T05:36:13.005 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.425e-06 2023-10-09T05:36:13.005 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.005 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.005 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:13.006 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.226628+0000", 2023-10-09T05:36:13.006 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.21950726800000001 2023-10-09T05:36:13.006 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.006 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.006 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:13.006 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.226637+0000", 2023-10-09T05:36:13.006 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7229999999999993e-06 2023-10-09T05:36:13.006 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.006 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.006 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:13.006 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.063102+0000", 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83646540999999996 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.063229+0000", 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012684600000000001 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.063346+0000", 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011715899999999999 2023-10-09T05:36:13.007 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.008 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.008 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:13.008 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.065378+0000", 2023-10-09T05:36:13.008 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020315820000000001 2023-10-09T05:36:13.008 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.008 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.008 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:13.008 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070208+0000", 2023-10-09T05:36:13.008 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048300499999999998 2023-10-09T05:36:13.008 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.008 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070644+0000", 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00043635199999999999 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070683+0000", 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8967000000000001e-05 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:13.009 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070715+0000", 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1566000000000002e-05 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119036 5.a 5:5126f622:::benchmark_data_smithi008_238708_object900:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.063601+0000", 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 108.468325529, 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.98084244200000004, 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:13.010 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:13.011 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:13.011 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:13.011 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:13.011 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119036 2023-10-09T05:36:13.011 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.011 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:13.011 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.011 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:13.011 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063601+0000", 2023-10-09T05:36:13.011 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:13.011 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063601+0000", 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063599+0000", 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.012 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063609+0000", 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5370000000000003e-06 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063623+0000", 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4154e-05 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063633+0000", 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0023999999999999e-05 2023-10-09T05:36:13.013 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.162124+0000", 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.098490792999999993 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.162139+0000", 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4732000000000001e-05 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.014 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:13.015 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011770+0000", 2023-10-09T05:36:13.015 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.849631256 2023-10-09T05:36:13.015 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.015 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.015 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:13.015 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011877+0000", 2023-10-09T05:36:13.015 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010692999999999999 2023-10-09T05:36:13.015 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.015 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:36:13.016 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011982+0000", 2023-10-09T05:36:13.016 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010477 2023-10-09T05:36:13.016 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:13.016 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.016352+0000", 2023-10-09T05:36:13.016 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004370624 2023-10-09T05:36:13.016 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.022811+0000", 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064583419999999997 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.044346+0000", 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021535813000000001 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.044401+0000", 2023-10-09T05:36:13.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4834999999999999e-05 2023-10-09T05:36:13.018 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:13.018 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:13.018 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:13.018 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.044444+0000", 2023-10-09T05:36:13.018 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2522999999999999e-05 2023-10-09T05:36:13.018 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:13.018 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:13.018 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:13.018 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:13.018 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:13.019 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:36:13.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:12 smithi191 ceph-mon[161669]: failed to encode map e146 with expected crc 2023-10-09T05:36:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:12 smithi191 ceph-mon[161669]: failed to encode map e146 with expected crc 2023-10-09T05:36:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:12 smithi191 ceph-mon[161669]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T05:36:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:12 smithi191 ceph-mon[161669]: failed to encode map e146 with expected crc 2023-10-09T05:36:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:12 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 18394/59529 objects degraded (30.899%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:13.717 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:13.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:13.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:13.822 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:13.895 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:13.972 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:13 smithi008 ceph-mon[184418]: pgmap v379: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 989 op/s; 17736/54087 objects degraded (32.792%); 480 KiB/s, 120 objects/s recovering 2023-10-09T05:36:13.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:13 smithi008 ceph-mon[191227]: pgmap v379: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 989 op/s; 17736/54087 objects degraded (32.792%); 480 KiB/s, 120 objects/s recovering 2023-10-09T05:36:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:13 smithi191 ceph-mon[161669]: pgmap v379: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 989 op/s; 17736/54087 objects degraded (32.792%); 480 KiB/s, 120 objects/s recovering 2023-10-09T05:36:14.676 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:14.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:14.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:14.769 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:36:14.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:14.864 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:14.940 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:36:14.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:14.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:15.048 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:15.051 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:15.052 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:15.052 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:14 smithi008 ceph-mon[184418]: pgmap v380: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 17736/51999 objects degraded (34.108%); 366 KiB/s, 92 objects/s recovering 2023-10-09T05:36:15.052 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:15.052 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:15.052 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:14 smithi008 ceph-mon[191227]: pgmap v380: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 17736/51999 objects degraded (34.108%); 366 KiB/s, 92 objects/s recovering 2023-10-09T05:36:15.164 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:15.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:14 smithi191 ceph-mon[161669]: pgmap v380: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 17736/51999 objects degraded (34.108%); 366 KiB/s, 92 objects/s recovering 2023-10-09T05:36:16.099 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:36:16.114 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:36:16.751 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:16.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:16.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:16.927 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:17.052 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:17.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:17.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:17.158 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:17.224 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:17 smithi191 ceph-mon[161669]: from='client.44915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:36:17.224 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:17 smithi191 ceph-mon[161669]: pgmap v381: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1002 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 17695/47982 objects degraded (36.878%); 360 KiB/s, 95 objects/s recovering 2023-10-09T05:36:17.229 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:17.293 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":380,"stamp":"2023-10-09T05:36:13.991947+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1138792395,"num_objects":17333,"num_object_clones":0,"num_object_copies":51999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17736,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17333,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":169881,"num_write_kb":4625030,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7839,"num_bytes_recovered":345309184,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":345081,"ondisk_log_size":345081,"up":327,"acting":319,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5209704,"kb_used_data":3433064,"kb_used_omap":45,"kb_used_meta":1776210,"kb_avail":744751512,"statfs":{"total":767960285184,"available":762625548288,"internally_reserved":0,"allocated":3515457536,"data_stored":3510938035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":1818839331},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,3,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-315031552,"num_objects":-4807,"num_object_clones":0,"num_object_copies":-14421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1326,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4807,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":726,"num_bytes_recovered":2949120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.865552"},"pg_stats":[{"pgid":"5.23","version":"146'8255","reported_seq":8406,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T05:36:12.760369+0000","last_change":"2023-10-09T05:35:59.978123+0000","last_active":"2023-10-09T05:36:12.760369+0000","last_peered":"2023-10-09T05:36:12.760369+0000","last_clean":"2023-10-09T05:36:12.760369+0000","last_became_active":"2023-10-09T05:34:25.035930+0000","last_became_peered":"2023-10-09T05:34:25.035930+0000","last_unstale":"2023-10-09T05:36:12.760369+0000","last_undegraded":"2023-10-09T05:36:12.760369+0000","last_fullsized":"2023-10-09T05:36:12.760369+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"120'6300","last_scrub_stamp":"2023-10-09T05:34:08.749711+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:08.749711+0000","objects_scrubbed":3317,"log_size":8255,"log_dups_size":0,"ondisk_log_size":8255,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.61798832400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16875520,"num_objects":258,"num_object_clones":0,"num_object_copies":774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1767,"num_write_kb":11712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.22","version":"144'8279","reported_seq":8359,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:09.816059+0000","last_change":"2023-10-09T05:34:40.876515+0000","last_active":"2023-10-09T05:36:09.816059+0000","last_peered":"2023-10-09T05:36:09.816059+0000","last_clean":"2023-10-09T05:36:09.816059+0000","last_became_active":"2023-10-09T05:34:25.030851+0000","last_became_peered":"2023-10-09T05:34:25.030851+0000","last_unstale":"2023-10-09T05:36:09.816059+0000","last_undegraded":"2023-10-09T05:36:09.816059+0000","last_fullsized":"2023-10-09T05:36:09.816059+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"127'7054","last_scrub_stamp":"2023-10-09T05:34:40.876434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:40.876434+0000","objects_scrubbed":0,"log_size":8279,"log_dups_size":0,"ondisk_log_size":8279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16646144,"num_objects":254,"num_object_clones":0,"num_object_copies":762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1225,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.21","version":"144'8228","reported_seq":10968,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:09.810057+0000","last_change":"2023-10-09T05:35:08.601492+0000","last_active":"2023-10-09T05:36:09.810057+0000","last_peered":"2023-10-09T05:36:09.810057+0000","last_clean":"2023-10-09T05:36:09.810057+0000","last_became_active":"2023-10-09T05:35:08.595030+0000","last_became_peered":"2023-10-09T05:35:08.595030+0000","last_unstale":"2023-10-09T05:36:09.810057+0000","last_undegraded":"2023-10-09T05:36:09.810057+0000","last_fullsized":"2023-10-09T05:36:09.810057+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8228,"log_dups_size":0,"ondisk_log_size":8228,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20430848,"num_objects":312,"num_object_clones":0,"num_object_copies":936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3359,"num_write_kb":117456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1327,"num_bytes_recovered":65929216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.20","version":"145'8238","reported_seq":8423,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.654935+0000","last_change":"2023-10-09T05:34:38.457569+0000","last_active":"2023-10-09T05:36:10.654935+0000","last_peered":"2023-10-09T05:36:10.654935+0000","last_clean":"2023-10-09T05:36:10.654935+0000","last_became_active":"2023-10-09T05:34:25.062440+0000","last_became_peered":"2023-10-09T05:34:25.062440+0000","last_unstale":"2023-10-09T05:36:10.654935+0000","last_undegraded":"2023-10-09T05:36:10.654935+0000","last_fullsized":"2023-10-09T05:36:10.654935+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"125'6962","last_scrub_stamp":"2023-10-09T05:34:38.457405+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:38.457405+0000","objects_scrubbed":1537,"log_size":8238,"log_dups_size":0,"ondisk_log_size":8238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:57.174556+0000","scrub_duration":0.224312019,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17039360,"num_objects":260,"num_object_clones":0,"num_object_copies":780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1277,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1c","version":"146'10798","reported_seq":10934,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T05:36:13.571230+0000","last_change":"2023-10-09T05:34:39.445074+0000","last_active":"2023-10-09T05:36:13.571230+0000","last_peered":"2023-10-09T05:36:13.571230+0000","last_clean":"2023-10-09T05:36:13.571230+0000","last_became_active":"2023-10-09T05:34:25.054943+0000","last_became_peered":"2023-10-09T05:34:25.054943+0000","last_unstale":"2023-10-09T05:36:13.571230+0000","last_undegraded":"2023-10-09T05:36:13.571230+0000","last_fullsized":"2023-10-09T05:36:13.571230+0000","mapping_epoch":123,"log_start":"108'200","ondisk_log_start":"108'200","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'8137","last_scrub_stamp":"2023-10-09T05:34:39.444955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:39.444955+0000","objects_scrubbed":3164,"log_size":10598,"log_dups_size":0,"ondisk_log_size":10598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:30.213649+0000","scrub_duration":0.76575304399999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32047104,"num_objects":489,"num_object_clones":0,"num_object_copies":1467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2675,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":127,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.077229+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:35:58.077229+0000","last_peered":"2023-10-09T05:35:58.077229+0000","last_clean":"2023-10-09T05:35:58.077229+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:35:58.077229+0000","last_undegraded":"2023-10-09T05:35:58.077229+0000","last_fullsized":"2023-10-09T05:35:58.077229+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":115,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.133598+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:36:06.133598+0000","last_peered":"2023-10-09T05:36:06.133598+0000","last_clean":"2023-10-09T05:36:06.133598+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:36:06.133598+0000","last_undegraded":"2023-10-09T05:36:06.133598+0000","last_fullsized":"2023-10-09T05:36:06.133598+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1f","version":"144'10425","reported_seq":10481,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:09.902987+0000","last_change":"2023-10-09T05:34:25.044017+0000","last_active":"2023-10-09T05:36:09.902987+0000","last_peered":"2023-10-09T05:36:09.902987+0000","last_clean":"2023-10-09T05:36:09.902987+0000","last_became_active":"2023-10-09T05:34:25.037058+0000","last_became_peered":"2023-10-09T05:34:25.037058+0000","last_unstale":"2023-10-09T05:36:09.902987+0000","last_undegraded":"2023-10-09T05:36:09.902987+0000","last_fullsized":"2023-10-09T05:36:09.902987+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10425,"log_dups_size":0,"ondisk_log_size":10425,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40796160,"num_objects":623,"num_object_clones":0,"num_object_copies":1869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6724,"num_write_kb":235072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":91,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.640321+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:36:10.640321+0000","last_peered":"2023-10-09T05:36:10.640321+0000","last_clean":"2023-10-09T05:36:10.640321+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:36:10.640321+0000","last_undegraded":"2023-10-09T05:36:10.640321+0000","last_fullsized":"2023-10-09T05:36:10.640321+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.650379+0000","last_change":"2023-10-09T05:34:41.662942+0000","last_active":"2023-10-09T05:36:10.650379+0000","last_peered":"2023-10-09T05:36:10.650379+0000","last_clean":"2023-10-09T05:36:10.650379+0000","last_became_active":"2023-10-09T05:34:41.662648+0000","last_became_peered":"2023-10-09T05:34:41.662648+0000","last_unstale":"2023-10-09T05:36:10.650379+0000","last_undegraded":"2023-10-09T05:36:10.650379+0000","last_fullsized":"2023-10-09T05:36:10.650379+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:08.691549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1e","version":"144'10587","reported_seq":10641,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:09.880304+0000","last_change":"2023-10-09T05:34:25.039990+0000","last_active":"2023-10-09T05:36:09.880304+0000","last_peered":"2023-10-09T05:36:09.880304+0000","last_clean":"2023-10-09T05:36:09.880304+0000","last_became_active":"2023-10-09T05:34:25.036888+0000","last_became_peered":"2023-10-09T05:34:25.036888+0000","last_unstale":"2023-10-09T05:36:09.880304+0000","last_undegraded":"2023-10-09T05:36:09.880304+0000","last_fullsized":"2023-10-09T05:36:09.880304+0000","mapping_epoch":123,"log_start":"109'1844","ondisk_log_start":"109'1844","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8743,"log_dups_size":0,"ondisk_log_size":8743,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38174720,"num_objects":583,"num_object_clones":0,"num_object_copies":1749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6848,"num_write_kb":237760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":131,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.132840+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:36:06.132840+0000","last_peered":"2023-10-09T05:36:06.132840+0000","last_clean":"2023-10-09T05:36:06.132840+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:36:06.132840+0000","last_undegraded":"2023-10-09T05:36:06.132840+0000","last_fullsized":"2023-10-09T05:36:06.132840+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.098958+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:35:58.098958+0000","last_peered":"2023-10-09T05:35:58.098958+0000","last_clean":"2023-10-09T05:35:58.098958+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:35:58.098958+0000","last_undegraded":"2023-10-09T05:35:58.098958+0000","last_fullsized":"2023-10-09T05:35:58.098958+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.11","version":"144'9757","reported_seq":9825,"reported_epoch":144,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:09.809072+0000","last_change":"2023-10-09T05:34:42.977328+0000","last_active":"2023-10-09T05:36:09.809072+0000","last_peered":"2023-10-09T05:36:09.809072+0000","last_clean":"2023-10-09T05:33:20.516527+0000","last_became_active":"2023-10-09T05:34:42.969107+0000","last_became_peered":"2023-10-09T05:34:42.969107+0000","last_unstale":"2023-10-09T05:36:09.809072+0000","last_undegraded":"2023-10-09T05:34:42.867828+0000","last_fullsized":"2023-10-09T05:34:42.853589+0000","mapping_epoch":128,"log_start":"109'1704","ondisk_log_start":"109'1704","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8053,"log_dups_size":0,"ondisk_log_size":8053,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36798464,"num_objects":562,"num_object_clones":0,"num_object_copies":1686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2627,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6740,"num_write_kb":233632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":-1417}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":96,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.653140+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:36:10.653140+0000","last_peered":"2023-10-09T05:36:10.653140+0000","last_clean":"2023-10-09T05:36:10.653140+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:36:10.653140+0000","last_undegraded":"2023-10-09T05:36:10.653140+0000","last_fullsized":"2023-10-09T05:36:10.653140+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":116,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.637584+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:36:10.637584+0000","last_peered":"2023-10-09T05:36:10.637584+0000","last_clean":"2023-10-09T05:36:10.637584+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:36:10.637584+0000","last_undegraded":"2023-10-09T05:36:10.637584+0000","last_fullsized":"2023-10-09T05:36:10.637584+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.10","version":"145'9739","reported_seq":9873,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.990456+0000","last_change":"2023-10-09T05:34:06.833835+0000","last_active":"2023-10-09T05:36:10.990456+0000","last_peered":"2023-10-09T05:36:10.990456+0000","last_clean":"2023-10-09T05:36:10.990456+0000","last_became_active":"2023-10-09T05:34:01.635329+0000","last_became_peered":"2023-10-09T05:34:01.635329+0000","last_unstale":"2023-10-09T05:36:10.990456+0000","last_undegraded":"2023-10-09T05:36:10.990456+0000","last_fullsized":"2023-10-09T05:36:10.990456+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6258","last_scrub_stamp":"2023-10-09T05:34:06.833681+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:06.833681+0000","objects_scrubbed":3155,"log_size":9739,"log_dups_size":0,"ondisk_log_size":9739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:45.971615+0000","scrub_duration":0.69237216700000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33226752,"num_objects":507,"num_object_clones":0,"num_object_copies":1521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3488,"num_write_kb":26880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":126,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.089010+0000","last_change":"2023-10-09T05:34:41.683934+0000","last_active":"2023-10-09T05:35:58.089010+0000","last_peered":"2023-10-09T05:35:58.089010+0000","last_clean":"2023-10-09T05:35:58.089010+0000","last_became_active":"2023-10-09T05:34:41.683618+0000","last_became_peered":"2023-10-09T05:34:41.683618+0000","last_unstale":"2023-10-09T05:35:58.089010+0000","last_undegraded":"2023-10-09T05:35:58.089010+0000","last_fullsized":"2023-10-09T05:35:58.089010+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":121,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.639902+0000","last_change":"2023-10-09T05:34:41.668251+0000","last_active":"2023-10-09T05:36:10.639902+0000","last_peered":"2023-10-09T05:36:10.639902+0000","last_clean":"2023-10-09T05:36:10.639902+0000","last_became_active":"2023-10-09T05:34:41.668017+0000","last_became_peered":"2023-10-09T05:34:41.668017+0000","last_unstale":"2023-10-09T05:36:10.639902+0000","last_undegraded":"2023-10-09T05:36:10.639902+0000","last_fullsized":"2023-10-09T05:36:10.639902+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:27.235656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.13","version":"146'9805","reported_seq":9869,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T05:36:12.788436+0000","last_change":"2023-10-09T05:36:04.931964+0000","last_active":"2023-10-09T05:36:12.788436+0000","last_peered":"2023-10-09T05:36:12.788436+0000","last_clean":"2023-10-09T05:36:12.788436+0000","last_became_active":"2023-10-09T05:34:01.625092+0000","last_became_peered":"2023-10-09T05:34:01.625092+0000","last_unstale":"2023-10-09T05:36:12.788436+0000","last_undegraded":"2023-10-09T05:36:12.788436+0000","last_fullsized":"2023-10-09T05:36:12.788436+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9805,"log_dups_size":0,"ondisk_log_size":9805,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35520512,"num_objects":542,"num_object_clones":0,"num_object_copies":1626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6784,"num_write_kb":234432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.073602+0000","last_change":"2023-10-09T05:34:41.682874+0000","last_active":"2023-10-09T05:35:58.073602+0000","last_peered":"2023-10-09T05:35:58.073602+0000","last_clean":"2023-10-09T05:35:58.073602+0000","last_became_active":"2023-10-09T05:34:41.674789+0000","last_became_peered":"2023-10-09T05:34:41.674789+0000","last_unstale":"2023-10-09T05:35:58.073602+0000","last_undegraded":"2023-10-09T05:35:58.073602+0000","last_fullsized":"2023-10-09T05:35:58.073602+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":91,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.640691+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:36:10.640691+0000","last_peered":"2023-10-09T05:36:10.640691+0000","last_clean":"2023-10-09T05:36:10.640691+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:36:10.640691+0000","last_undegraded":"2023-10-09T05:36:10.640691+0000","last_fullsized":"2023-10-09T05:36:10.640691+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.12","version":"144'9934","reported_seq":9995,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:09.731146+0000","last_change":"2023-10-09T05:34:06.470865+0000","last_active":"2023-10-09T05:36:09.731146+0000","last_peered":"2023-10-09T05:36:09.731146+0000","last_clean":"2023-10-09T05:36:09.731146+0000","last_became_active":"2023-10-09T05:34:01.624493+0000","last_became_peered":"2023-10-09T05:34:01.624493+0000","last_unstale":"2023-10-09T05:36:09.731146+0000","last_undegraded":"2023-10-09T05:36:09.731146+0000","last_fullsized":"2023-10-09T05:36:09.731146+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6331","last_scrub_stamp":"2023-10-09T05:34:06.470790+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:06.470790+0000","objects_scrubbed":0,"log_size":9934,"log_dups_size":0,"ondisk_log_size":9934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39911424,"num_objects":609,"num_object_clones":0,"num_object_copies":1827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3615,"num_write_kb":30848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":133,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.077285+0000","last_change":"2023-10-09T05:34:41.683953+0000","last_active":"2023-10-09T05:35:58.077285+0000","last_peered":"2023-10-09T05:35:58.077285+0000","last_clean":"2023-10-09T05:35:58.077285+0000","last_became_active":"2023-10-09T05:34:41.683519+0000","last_became_peered":"2023-10-09T05:34:41.683519+0000","last_unstale":"2023-10-09T05:35:58.077285+0000","last_undegraded":"2023-10-09T05:35:58.077285+0000","last_fullsized":"2023-10-09T05:35:58.077285+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.077251+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:35:58.077251+0000","last_peered":"2023-10-09T05:35:58.077251+0000","last_clean":"2023-10-09T05:35:58.077251+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:35:58.077251+0000","last_undegraded":"2023-10-09T05:35:58.077251+0000","last_fullsized":"2023-10-09T05:35:58.077251+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.15","version":"146'10182","reported_seq":10246,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T05:36:12.834322+0000","last_change":"2023-10-09T05:36:03.970462+0000","last_active":"2023-10-09T05:36:12.834322+0000","last_peered":"2023-10-09T05:36:12.834322+0000","last_clean":"2023-10-09T05:36:12.834322+0000","last_became_active":"2023-10-09T05:34:01.623738+0000","last_became_peered":"2023-10-09T05:34:01.623738+0000","last_unstale":"2023-10-09T05:36:12.834322+0000","last_undegraded":"2023-10-09T05:36:12.834322+0000","last_fullsized":"2023-10-09T05:36:12.834322+0000","mapping_epoch":118,"log_start":"108'616","ondisk_log_start":"108'616","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9566,"log_dups_size":0,"ondisk_log_size":9566,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28377088,"num_objects":433,"num_object_clones":0,"num_object_copies":1299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7057,"num_write_kb":239680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":97,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.073346+0000","last_change":"2023-10-09T05:34:41.673944+0000","last_active":"2023-10-09T05:35:58.073346+0000","last_peered":"2023-10-09T05:35:58.073346+0000","last_clean":"2023-10-09T05:35:58.073346+0000","last_became_active":"2023-10-09T05:34:41.673595+0000","last_became_peered":"2023-10-09T05:34:41.673595+0000","last_unstale":"2023-10-09T05:35:58.073346+0000","last_undegraded":"2023-10-09T05:35:58.073346+0000","last_fullsized":"2023-10-09T05:35:58.073346+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":131,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.638293+0000","last_change":"2023-10-09T05:34:41.667462+0000","last_active":"2023-10-09T05:36:10.638293+0000","last_peered":"2023-10-09T05:36:10.638293+0000","last_clean":"2023-10-09T05:36:10.638293+0000","last_became_active":"2023-10-09T05:34:41.667140+0000","last_became_peered":"2023-10-09T05:34:41.667140+0000","last_unstale":"2023-10-09T05:36:10.638293+0000","last_undegraded":"2023-10-09T05:36:10.638293+0000","last_fullsized":"2023-10-09T05:36:10.638293+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:52.888470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.14","version":"145'9762","reported_seq":9828,"reported_epoch":145,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:10.854763+0000","last_change":"2023-10-09T05:34:42.906108+0000","last_active":"2023-10-09T05:36:10.854763+0000","last_peered":"2023-10-09T05:36:10.854763+0000","last_clean":"2023-10-09T05:33:20.493128+0000","last_became_active":"2023-10-09T05:34:42.894931+0000","last_became_peered":"2023-10-09T05:34:42.894931+0000","last_unstale":"2023-10-09T05:36:10.854763+0000","last_undegraded":"2023-10-09T05:34:42.686528+0000","last_fullsized":"2023-10-09T05:34:42.658194+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9762,"log_dups_size":0,"ondisk_log_size":9762,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36732928,"num_objects":561,"num_object_clones":0,"num_object_copies":1683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2585,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6733,"num_write_kb":233376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-1416}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":111,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.653301+0000","last_change":"2023-10-09T05:34:41.662790+0000","last_active":"2023-10-09T05:36:10.653301+0000","last_peered":"2023-10-09T05:36:10.653301+0000","last_clean":"2023-10-09T05:36:10.653301+0000","last_became_active":"2023-10-09T05:34:41.662513+0000","last_became_peered":"2023-10-09T05:34:41.662513+0000","last_unstale":"2023-10-09T05:36:10.653301+0000","last_undegraded":"2023-10-09T05:36:10.653301+0000","last_fullsized":"2023-10-09T05:36:10.653301+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:17.781371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":125,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.089693+0000","last_change":"2023-10-09T05:34:41.694695+0000","last_active":"2023-10-09T05:35:58.089693+0000","last_peered":"2023-10-09T05:35:58.089693+0000","last_clean":"2023-10-09T05:35:58.089693+0000","last_became_active":"2023-10-09T05:34:41.694408+0000","last_became_peered":"2023-10-09T05:34:41.694408+0000","last_unstale":"2023-10-09T05:35:58.089693+0000","last_undegraded":"2023-10-09T05:35:58.089693+0000","last_fullsized":"2023-10-09T05:35:58.089693+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.17","version":"144'9898","reported_seq":9959,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:09.886541+0000","last_change":"2023-10-09T05:34:37.421122+0000","last_active":"2023-10-09T05:36:09.886541+0000","last_peered":"2023-10-09T05:36:09.886541+0000","last_clean":"2023-10-09T05:36:09.886541+0000","last_became_active":"2023-10-09T05:34:01.618906+0000","last_became_peered":"2023-10-09T05:34:01.618906+0000","last_unstale":"2023-10-09T05:36:09.886541+0000","last_undegraded":"2023-10-09T05:36:09.886541+0000","last_fullsized":"2023-10-09T05:36:09.886541+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'7322","last_scrub_stamp":"2023-10-09T05:34:37.421020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:37.421020+0000","objects_scrubbed":0,"log_size":9898,"log_dups_size":0,"ondisk_log_size":9898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39780352,"num_objects":607,"num_object_clones":0,"num_object_copies":1821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2593,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":80,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.076148+0000","last_change":"2023-10-09T05:34:41.702205+0000","last_active":"2023-10-09T05:35:58.076148+0000","last_peered":"2023-10-09T05:35:58.076148+0000","last_clean":"2023-10-09T05:35:58.076148+0000","last_became_active":"2023-10-09T05:34:41.701849+0000","last_became_peered":"2023-10-09T05:34:41.701849+0000","last_unstale":"2023-10-09T05:35:58.076148+0000","last_undegraded":"2023-10-09T05:35:58.076148+0000","last_fullsized":"2023-10-09T05:35:58.076148+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.650305+0000","last_change":"2023-10-09T05:34:41.663111+0000","last_active":"2023-10-09T05:36:10.650305+0000","last_peered":"2023-10-09T05:36:10.650305+0000","last_clean":"2023-10-09T05:36:10.650305+0000","last_became_active":"2023-10-09T05:34:41.662816+0000","last_became_peered":"2023-10-09T05:34:41.662816+0000","last_unstale":"2023-10-09T05:36:10.650305+0000","last_undegraded":"2023-10-09T05:36:10.650305+0000","last_fullsized":"2023-10-09T05:36:10.650305+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:36.226743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.16","version":"145'9928","reported_seq":10087,"reported_epoch":145,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:11.004135+0000","last_change":"2023-10-09T05:36:10.374615+0000","last_active":"2023-10-09T05:36:11.004135+0000","last_peered":"2023-10-09T05:36:11.004135+0000","last_clean":"2023-10-09T05:33:20.529830+0000","last_became_active":"2023-10-09T05:34:42.891787+0000","last_became_peered":"2023-10-09T05:34:42.891787+0000","last_unstale":"2023-10-09T05:36:11.004135+0000","last_undegraded":"2023-10-09T05:34:42.686663+0000","last_fullsized":"2023-10-09T05:34:42.658538+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9928,"log_dups_size":0,"ondisk_log_size":9928,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34603008,"num_objects":528,"num_object_clones":0,"num_object_copies":1584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1932,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6860,"num_write_kb":236416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":2949120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-1404}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":119,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.092550+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:35:58.092550+0000","last_peered":"2023-10-09T05:35:58.092550+0000","last_clean":"2023-10-09T05:35:58.092550+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:35:58.092550+0000","last_undegraded":"2023-10-09T05:35:58.092550+0000","last_fullsized":"2023-10-09T05:35:58.092550+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":130,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.132958+0000","last_change":"2023-10-09T05:34:41.664294+0000","last_active":"2023-10-09T05:36:06.132958+0000","last_peered":"2023-10-09T05:36:06.132958+0000","last_clean":"2023-10-09T05:36:06.132958+0000","last_became_active":"2023-10-09T05:34:41.663985+0000","last_became_peered":"2023-10-09T05:34:41.663985+0000","last_unstale":"2023-10-09T05:36:06.132958+0000","last_undegraded":"2023-10-09T05:36:06.132958+0000","last_fullsized":"2023-10-09T05:36:06.132958+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:58.417307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"144'9880","reported_seq":9946,"reported_epoch":144,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:09.794659+0000","last_change":"2023-10-09T05:34:42.977224+0000","last_active":"2023-10-09T05:36:09.794659+0000","last_peered":"2023-10-09T05:36:09.794659+0000","last_clean":"2023-10-09T05:33:20.515228+0000","last_became_active":"2023-10-09T05:34:42.967706+0000","last_became_peered":"2023-10-09T05:34:42.967706+0000","last_unstale":"2023-10-09T05:36:09.794659+0000","last_undegraded":"2023-10-09T05:34:42.852684+0000","last_fullsized":"2023-10-09T05:34:42.677882+0000","mapping_epoch":128,"log_start":"109'1939","ondisk_log_start":"109'1939","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":7941,"log_dups_size":0,"ondisk_log_size":7941,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32636928,"num_objects":498,"num_object_clones":0,"num_object_copies":1494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2614,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6820,"num_write_kb":234176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":-1505}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":86,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.076795+0000","last_change":"2023-10-09T05:34:41.688229+0000","last_active":"2023-10-09T05:35:58.076795+0000","last_peered":"2023-10-09T05:35:58.076795+0000","last_clean":"2023-10-09T05:35:58.076795+0000","last_became_active":"2023-10-09T05:34:41.687795+0000","last_became_peered":"2023-10-09T05:34:41.687795+0000","last_unstale":"2023-10-09T05:35:58.076795+0000","last_undegraded":"2023-10-09T05:35:58.076795+0000","last_fullsized":"2023-10-09T05:35:58.076795+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.085296+0000","last_change":"2023-10-09T05:34:41.678569+0000","last_active":"2023-10-09T05:35:58.085296+0000","last_peered":"2023-10-09T05:35:58.085296+0000","last_clean":"2023-10-09T05:35:58.085296+0000","last_became_active":"2023-10-09T05:34:41.678271+0000","last_became_peered":"2023-10-09T05:34:41.678271+0000","last_unstale":"2023-10-09T05:35:58.085296+0000","last_undegraded":"2023-10-09T05:35:58.085296+0000","last_fullsized":"2023-10-09T05:35:58.085296+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"145'12519","reported_seq":12866,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.956276+0000","last_change":"2023-10-09T05:34:25.139244+0000","last_active":"2023-10-09T05:36:10.956276+0000","last_peered":"2023-10-09T05:36:10.956276+0000","last_clean":"2023-10-09T05:36:10.956276+0000","last_became_active":"2023-10-09T05:34:01.629769+0000","last_became_peered":"2023-10-09T05:34:01.629769+0000","last_unstale":"2023-10-09T05:36:10.956276+0000","last_undegraded":"2023-10-09T05:36:10.956276+0000","last_fullsized":"2023-10-09T05:36:10.956276+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":3497,"log_size":12519,"log_dups_size":0,"ondisk_log_size":12519,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38436864,"num_objects":586,"num_object_clones":0,"num_object_copies":1758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6822,"num_write_kb":237088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.076731+0000","last_change":"2023-10-09T05:34:41.674752+0000","last_active":"2023-10-09T05:35:58.076731+0000","last_peered":"2023-10-09T05:35:58.076731+0000","last_clean":"2023-10-09T05:35:58.076731+0000","last_became_active":"2023-10-09T05:34:41.674253+0000","last_became_peered":"2023-10-09T05:34:41.674253+0000","last_unstale":"2023-10-09T05:35:58.076731+0000","last_undegraded":"2023-10-09T05:35:58.076731+0000","last_fullsized":"2023-10-09T05:35:58.076731+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":125,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.086105+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:35:58.086105+0000","last_peered":"2023-10-09T05:35:58.086105+0000","last_clean":"2023-10-09T05:35:58.086105+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:35:58.086105+0000","last_undegraded":"2023-10-09T05:35:58.086105+0000","last_fullsized":"2023-10-09T05:35:58.086105+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"145'10703","reported_seq":10764,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.985095+0000","last_change":"2023-10-09T05:34:42.657980+0000","last_active":"2023-10-09T05:36:10.985095+0000","last_peered":"2023-10-09T05:36:10.985095+0000","last_clean":"2023-10-09T05:36:10.985095+0000","last_became_active":"2023-10-09T05:34:25.039138+0000","last_became_peered":"2023-10-09T05:34:25.039138+0000","last_unstale":"2023-10-09T05:36:10.985095+0000","last_undegraded":"2023-10-09T05:36:10.985095+0000","last_fullsized":"2023-10-09T05:36:10.985095+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10703,"log_dups_size":0,"ondisk_log_size":10703,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30146560,"num_objects":460,"num_object_clones":0,"num_object_copies":1380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6965,"num_write_kb":237600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":83,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.073854+0000","last_change":"2023-10-09T05:34:41.691449+0000","last_active":"2023-10-09T05:35:58.073854+0000","last_peered":"2023-10-09T05:35:58.073854+0000","last_clean":"2023-10-09T05:35:58.073854+0000","last_became_active":"2023-10-09T05:34:41.675227+0000","last_became_peered":"2023-10-09T05:34:41.675227+0000","last_unstale":"2023-10-09T05:35:58.073854+0000","last_undegraded":"2023-10-09T05:35:58.073854+0000","last_fullsized":"2023-10-09T05:35:58.073854+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":134,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.097396+0000","last_change":"2023-10-09T05:34:41.669908+0000","last_active":"2023-10-09T05:35:58.097396+0000","last_peered":"2023-10-09T05:35:58.097396+0000","last_clean":"2023-10-09T05:35:58.097396+0000","last_became_active":"2023-10-09T05:34:41.669596+0000","last_became_peered":"2023-10-09T05:34:41.669596+0000","last_unstale":"2023-10-09T05:35:58.097396+0000","last_undegraded":"2023-10-09T05:35:58.097396+0000","last_fullsized":"2023-10-09T05:35:58.097396+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"145'10566","reported_seq":10702,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.980331+0000","last_change":"2023-10-09T05:34:35.674328+0000","last_active":"2023-10-09T05:36:10.980331+0000","last_peered":"2023-10-09T05:36:10.980331+0000","last_clean":"2023-10-09T05:36:10.980331+0000","last_became_active":"2023-10-09T05:34:25.008155+0000","last_became_peered":"2023-10-09T05:34:25.008155+0000","last_unstale":"2023-10-09T05:36:10.980331+0000","last_undegraded":"2023-10-09T05:36:10.980331+0000","last_fullsized":"2023-10-09T05:36:10.980331+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7846","last_scrub_stamp":"2023-10-09T05:34:35.674093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:35.674093+0000","objects_scrubbed":3275,"log_size":10566,"log_dups_size":0,"ondisk_log_size":10566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:02.820818+0000","scrub_duration":0.75420257000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35651584,"num_objects":544,"num_object_clones":0,"num_object_copies":1632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2731,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.086637+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:35:58.086637+0000","last_peered":"2023-10-09T05:35:58.086637+0000","last_clean":"2023-10-09T05:35:58.086637+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:35:58.086637+0000","last_undegraded":"2023-10-09T05:35:58.086637+0000","last_fullsized":"2023-10-09T05:35:58.086637+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":96,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.649194+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:36:10.649194+0000","last_peered":"2023-10-09T05:36:10.649194+0000","last_clean":"2023-10-09T05:36:10.649194+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:36:10.649194+0000","last_undegraded":"2023-10-09T05:36:10.649194+0000","last_fullsized":"2023-10-09T05:36:10.649194+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"145'10638","reported_seq":15905,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.950353+0000","last_change":"2023-10-09T05:35:43.652202+0000","last_active":"2023-10-09T05:36:10.950353+0000","last_peered":"2023-10-09T05:36:10.950353+0000","last_clean":"2023-10-09T05:36:10.950353+0000","last_became_active":"2023-10-09T05:35:43.630210+0000","last_became_peered":"2023-10-09T05:35:43.630210+0000","last_unstale":"2023-10-09T05:36:10.950353+0000","last_undegraded":"2023-10-09T05:36:10.950353+0000","last_fullsized":"2023-10-09T05:36:10.950353+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10638,"log_dups_size":0,"ondisk_log_size":10638,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35127308,"num_objects":536,"num_object_clones":0,"num_object_copies":1608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6895,"num_write_kb":237792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2592,"num_bytes_recovered":128319488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":114,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.132985+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:36:06.132985+0000","last_peered":"2023-10-09T05:36:06.132985+0000","last_clean":"2023-10-09T05:36:06.132985+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:36:06.132985+0000","last_undegraded":"2023-10-09T05:36:06.132985+0000","last_fullsized":"2023-10-09T05:36:06.132985+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.077191+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:35:58.077191+0000","last_peered":"2023-10-09T05:35:58.077191+0000","last_clean":"2023-10-09T05:35:58.077191+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:35:58.077191+0000","last_undegraded":"2023-10-09T05:35:58.077191+0000","last_fullsized":"2023-10-09T05:35:58.077191+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"146'10689","reported_seq":10825,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T05:36:13.322387+0000","last_change":"2023-10-09T05:34:40.457466+0000","last_active":"2023-10-09T05:36:13.322387+0000","last_peered":"2023-10-09T05:36:13.322387+0000","last_clean":"2023-10-09T05:36:13.322387+0000","last_became_active":"2023-10-09T05:34:25.026725+0000","last_became_peered":"2023-10-09T05:34:25.026725+0000","last_unstale":"2023-10-09T05:36:13.322387+0000","last_undegraded":"2023-10-09T05:36:13.322387+0000","last_fullsized":"2023-10-09T05:36:13.322387+0000","mapping_epoch":105,"log_start":"108'200","ondisk_log_start":"108'200","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'8107","last_scrub_stamp":"2023-10-09T05:34:40.457321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:40.457321+0000","objects_scrubbed":3100,"log_size":10489,"log_dups_size":0,"ondisk_log_size":10489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:26.084493+0000","scrub_duration":0.74267715700000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33030144,"num_objects":504,"num_object_clones":0,"num_object_copies":1512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2596,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.093617+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:35:58.093617+0000","last_peered":"2023-10-09T05:35:58.093617+0000","last_clean":"2023-10-09T05:35:58.093617+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:35:58.093617+0000","last_undegraded":"2023-10-09T05:35:58.093617+0000","last_fullsized":"2023-10-09T05:35:58.093617+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":125,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.077602+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:35:58.077602+0000","last_peered":"2023-10-09T05:35:58.077602+0000","last_clean":"2023-10-09T05:35:58.077602+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:35:58.077602+0000","last_undegraded":"2023-10-09T05:35:58.077602+0000","last_fullsized":"2023-10-09T05:35:58.077602+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"144'10511","reported_seq":10570,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:09.902690+0000","last_change":"2023-10-09T05:34:39.803946+0000","last_active":"2023-10-09T05:36:09.902690+0000","last_peered":"2023-10-09T05:36:09.902690+0000","last_clean":"2023-10-09T05:36:09.902690+0000","last_became_active":"2023-10-09T05:34:25.018175+0000","last_became_peered":"2023-10-09T05:34:25.018175+0000","last_unstale":"2023-10-09T05:36:09.902690+0000","last_undegraded":"2023-10-09T05:36:09.902690+0000","last_fullsized":"2023-10-09T05:36:09.902690+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10511,"log_dups_size":0,"ondisk_log_size":10511,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35160064,"num_objects":536,"num_object_clones":0,"num_object_copies":1608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6809,"num_write_kb":235072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":126,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.085895+0000","last_change":"2023-10-09T05:34:41.706409+0000","last_active":"2023-10-09T05:35:58.085895+0000","last_peered":"2023-10-09T05:35:58.085895+0000","last_clean":"2023-10-09T05:35:58.085895+0000","last_became_active":"2023-10-09T05:34:41.706002+0000","last_became_peered":"2023-10-09T05:34:41.706002+0000","last_unstale":"2023-10-09T05:35:58.085895+0000","last_undegraded":"2023-10-09T05:35:58.085895+0000","last_fullsized":"2023-10-09T05:35:58.085895+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":119,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.077738+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:35:58.077738+0000","last_peered":"2023-10-09T05:35:58.077738+0000","last_clean":"2023-10-09T05:35:58.077738+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:35:58.077738+0000","last_undegraded":"2023-10-09T05:35:58.077738+0000","last_fullsized":"2023-10-09T05:35:58.077738+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"144'10685","reported_seq":10747,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:09.834037+0000","last_change":"2023-10-09T05:34:42.153027+0000","last_active":"2023-10-09T05:36:09.834037+0000","last_peered":"2023-10-09T05:36:09.834037+0000","last_clean":"2023-10-09T05:36:09.834037+0000","last_became_active":"2023-10-09T05:34:25.031927+0000","last_became_peered":"2023-10-09T05:34:25.031927+0000","last_unstale":"2023-10-09T05:36:09.834037+0000","last_undegraded":"2023-10-09T05:36:09.834037+0000","last_fullsized":"2023-10-09T05:36:09.834037+0000","mapping_epoch":105,"log_start":"109'2221","ondisk_log_start":"109'2221","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'8112","last_scrub_stamp":"2023-10-09T05:34:42.152946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:42.152946+0000","objects_scrubbed":0,"log_size":8464,"log_dups_size":0,"ondisk_log_size":8464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34996224,"num_objects":534,"num_object_clones":0,"num_object_copies":1602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2573,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":132,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.132755+0000","last_change":"2023-10-09T05:34:41.666221+0000","last_active":"2023-10-09T05:36:06.132755+0000","last_peered":"2023-10-09T05:36:06.132755+0000","last_clean":"2023-10-09T05:36:06.132755+0000","last_became_active":"2023-10-09T05:34:41.665842+0000","last_became_peered":"2023-10-09T05:34:41.665842+0000","last_unstale":"2023-10-09T05:36:06.132755+0000","last_undegraded":"2023-10-09T05:36:06.132755+0000","last_fullsized":"2023-10-09T05:36:06.132755+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:28.435851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":91,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.641034+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:36:10.641034+0000","last_peered":"2023-10-09T05:36:10.641034+0000","last_clean":"2023-10-09T05:36:10.641034+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:36:10.641034+0000","last_undegraded":"2023-10-09T05:36:10.641034+0000","last_fullsized":"2023-10-09T05:36:10.641034+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":670,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.640442+0000","last_change":"2023-10-09T05:34:41.667358+0000","last_active":"2023-10-09T05:36:10.640442+0000","last_peered":"2023-10-09T05:36:10.640442+0000","last_clean":"2023-10-09T05:36:10.640442+0000","last_became_active":"2023-10-09T05:34:41.666757+0000","last_became_peered":"2023-10-09T05:34:41.666757+0000","last_unstale":"2023-10-09T05:36:10.640442+0000","last_undegraded":"2023-10-09T05:36:10.640442+0000","last_fullsized":"2023-10-09T05:36:10.640442+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:45:47.725111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":126,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.089778+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:35:58.089778+0000","last_peered":"2023-10-09T05:35:58.089778+0000","last_clean":"2023-10-09T05:35:58.089778+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:35:58.089778+0000","last_undegraded":"2023-10-09T05:35:58.089778+0000","last_fullsized":"2023-10-09T05:35:58.089778+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":76,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.640590+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:36:10.640590+0000","last_peered":"2023-10-09T05:36:10.640590+0000","last_clean":"2023-10-09T05:36:10.640590+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:36:10.640590+0000","last_undegraded":"2023-10-09T05:36:10.640590+0000","last_fullsized":"2023-10-09T05:36:10.640590+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"146'8270","reported_seq":8422,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T05:36:12.727106+0000","last_change":"2023-10-09T05:35:58.126254+0000","last_active":"2023-10-09T05:36:12.727106+0000","last_peered":"2023-10-09T05:36:12.727106+0000","last_clean":"2023-10-09T05:36:12.727106+0000","last_became_active":"2023-10-09T05:34:25.031183+0000","last_became_peered":"2023-10-09T05:34:25.031183+0000","last_unstale":"2023-10-09T05:36:12.727106+0000","last_undegraded":"2023-10-09T05:36:12.727106+0000","last_fullsized":"2023-10-09T05:36:12.727106+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'6300","last_scrub_stamp":"2023-10-09T05:34:08.749711+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:08.749711+0000","objects_scrubbed":3317,"log_size":8270,"log_dups_size":0,"ondisk_log_size":8270,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.61798832400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15892480,"num_objects":242,"num_object_clones":0,"num_object_copies":726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1782,"num_write_kb":11712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"146'10102","reported_seq":10240,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T05:36:12.918913+0000","last_change":"2023-10-09T05:34:10.842783+0000","last_active":"2023-10-09T05:36:12.918913+0000","last_peered":"2023-10-09T05:36:12.918913+0000","last_clean":"2023-10-09T05:36:12.918913+0000","last_became_active":"2023-10-09T05:34:01.606336+0000","last_became_peered":"2023-10-09T05:34:01.606336+0000","last_unstale":"2023-10-09T05:36:12.918913+0000","last_undegraded":"2023-10-09T05:36:12.918913+0000","last_fullsized":"2023-10-09T05:36:12.918913+0000","mapping_epoch":105,"log_start":"108'193","ondisk_log_start":"108'193","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'6606","last_scrub_stamp":"2023-10-09T05:34:10.842550+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:10.842550+0000","objects_scrubbed":3389,"log_size":9909,"log_dups_size":0,"ondisk_log_size":9909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:40.600228+0000","scrub_duration":0.74463188000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32571392,"num_objects":497,"num_object_clones":0,"num_object_copies":1491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3522,"num_write_kb":20160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":580,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.086353+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:35:58.086353+0000","last_peered":"2023-10-09T05:35:58.086353+0000","last_clean":"2023-10-09T05:35:58.086353+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:35:58.086353+0000","last_undegraded":"2023-10-09T05:35:58.086353+0000","last_fullsized":"2023-10-09T05:35:58.086353+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.077374+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:35:58.077374+0000","last_peered":"2023-10-09T05:35:58.077374+0000","last_clean":"2023-10-09T05:35:58.077374+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:35:58.077374+0000","last_undegraded":"2023-10-09T05:35:58.077374+0000","last_fullsized":"2023-10-09T05:35:58.077374+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":123,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.086443+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:35:58.086443+0000","last_peered":"2023-10-09T05:35:58.086443+0000","last_clean":"2023-10-09T05:35:58.086443+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:35:58.086443+0000","last_undegraded":"2023-10-09T05:35:58.086443+0000","last_fullsized":"2023-10-09T05:35:58.086443+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"145'9919","reported_seq":10003,"reported_epoch":145,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:10.936270+0000","last_change":"2023-10-09T05:34:42.959034+0000","last_active":"2023-10-09T05:36:10.936270+0000","last_peered":"2023-10-09T05:36:10.936270+0000","last_clean":"2023-10-09T05:33:20.529830+0000","last_became_active":"2023-10-09T05:34:42.879241+0000","last_became_peered":"2023-10-09T05:34:42.879241+0000","last_unstale":"2023-10-09T05:36:10.936270+0000","last_undegraded":"2023-10-09T05:34:42.688699+0000","last_fullsized":"2023-10-09T05:34:42.658753+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9919,"log_dups_size":0,"ondisk_log_size":9919,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35323904,"num_objects":539,"num_object_clones":0,"num_object_copies":1617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6851,"num_write_kb":236480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-1456}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":457,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.130407+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:36:06.130407+0000","last_peered":"2023-10-09T05:36:06.130407+0000","last_clean":"2023-10-09T05:36:06.130407+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:36:06.130407+0000","last_undegraded":"2023-10-09T05:36:06.130407+0000","last_fullsized":"2023-10-09T05:36:06.130407+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":91,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.640714+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:36:10.640714+0000","last_peered":"2023-10-09T05:36:10.640714+0000","last_clean":"2023-10-09T05:36:10.640714+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:36:10.640714+0000","last_undegraded":"2023-10-09T05:36:10.640714+0000","last_fullsized":"2023-10-09T05:36:10.640714+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":121,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.640899+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:36:10.640899+0000","last_peered":"2023-10-09T05:36:10.640899+0000","last_clean":"2023-10-09T05:36:10.640899+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:36:10.640899+0000","last_undegraded":"2023-10-09T05:36:10.640899+0000","last_fullsized":"2023-10-09T05:36:10.640899+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":720,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.086301+0000","last_change":"2023-10-09T05:34:41.677976+0000","last_active":"2023-10-09T05:35:58.086301+0000","last_peered":"2023-10-09T05:35:58.086301+0000","last_clean":"2023-10-09T05:35:58.086301+0000","last_became_active":"2023-10-09T05:34:41.677520+0000","last_became_peered":"2023-10-09T05:34:41.677520+0000","last_unstale":"2023-10-09T05:35:58.086301+0000","last_undegraded":"2023-10-09T05:35:58.086301+0000","last_fullsized":"2023-10-09T05:35:58.086301+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":321,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.073031+0000","last_change":"2023-10-09T05:34:41.686013+0000","last_active":"2023-10-09T05:35:58.073031+0000","last_peered":"2023-10-09T05:35:58.073031+0000","last_clean":"2023-10-09T05:35:58.073031+0000","last_became_active":"2023-10-09T05:34:41.680809+0000","last_became_peered":"2023-10-09T05:34:41.680809+0000","last_unstale":"2023-10-09T05:35:58.073031+0000","last_undegraded":"2023-10-09T05:35:58.073031+0000","last_fullsized":"2023-10-09T05:35:58.073031+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":107,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.134014+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:36:06.134014+0000","last_peered":"2023-10-09T05:36:06.134014+0000","last_clean":"2023-10-09T05:36:06.134014+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:36:06.134014+0000","last_undegraded":"2023-10-09T05:36:06.134014+0000","last_fullsized":"2023-10-09T05:36:06.134014+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.098796+0000","last_change":"2023-10-09T05:34:41.693650+0000","last_active":"2023-10-09T05:35:58.098796+0000","last_peered":"2023-10-09T05:35:58.098796+0000","last_clean":"2023-10-09T05:35:58.098796+0000","last_became_active":"2023-10-09T05:34:41.693305+0000","last_became_peered":"2023-10-09T05:34:41.693305+0000","last_unstale":"2023-10-09T05:35:58.098796+0000","last_undegraded":"2023-10-09T05:35:58.098796+0000","last_fullsized":"2023-10-09T05:35:58.098796+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"145'9824","reported_seq":9893,"reported_epoch":145,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:10.972965+0000","last_change":"2023-10-09T05:34:42.906080+0000","last_active":"2023-10-09T05:36:10.972965+0000","last_peered":"2023-10-09T05:36:10.972965+0000","last_clean":"2023-10-09T05:33:20.493128+0000","last_became_active":"2023-10-09T05:34:42.894560+0000","last_became_peered":"2023-10-09T05:34:42.894560+0000","last_unstale":"2023-10-09T05:36:10.972965+0000","last_undegraded":"2023-10-09T05:34:42.694084+0000","last_fullsized":"2023-10-09T05:34:42.658067+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9824,"log_dups_size":0,"ondisk_log_size":9824,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32276480,"num_objects":492,"num_object_clones":0,"num_object_copies":1476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2624,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6794,"num_write_kb":233184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-1466}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.7","version":"144'9857","reported_seq":9927,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:09.816031+0000","last_change":"2023-10-09T05:34:34.976158+0000","last_active":"2023-10-09T05:36:09.816031+0000","last_peered":"2023-10-09T05:36:09.816031+0000","last_clean":"2023-10-09T05:36:09.816031+0000","last_became_active":"2023-10-09T05:34:01.592987+0000","last_became_peered":"2023-10-09T05:34:01.592987+0000","last_unstale":"2023-10-09T05:36:09.816031+0000","last_undegraded":"2023-10-09T05:36:09.816031+0000","last_fullsized":"2023-10-09T05:36:09.816031+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7154","last_scrub_stamp":"2023-10-09T05:34:34.976067+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:34.976067+0000","objects_scrubbed":0,"log_size":9857,"log_dups_size":0,"ondisk_log_size":9857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37879808,"num_objects":578,"num_object_clones":0,"num_object_copies":1734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2729,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":744,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.073031+0000","last_change":"2023-10-09T05:34:41.668526+0000","last_active":"2023-10-09T05:35:58.073031+0000","last_peered":"2023-10-09T05:35:58.073031+0000","last_clean":"2023-10-09T05:35:58.073031+0000","last_became_active":"2023-10-09T05:34:41.668010+0000","last_became_peered":"2023-10-09T05:34:41.668010+0000","last_unstale":"2023-10-09T05:35:58.073031+0000","last_undegraded":"2023-10-09T05:35:58.073031+0000","last_fullsized":"2023-10-09T05:35:58.073031+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T05:36:11.696581+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:36:11.696581+0000","last_peered":"2023-10-09T05:36:11.696581+0000","last_clean":"2023-10-09T05:36:11.696581+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:36:11.696581+0000","last_undegraded":"2023-10-09T05:36:11.696581+0000","last_fullsized":"2023-10-09T05:36:11.696581+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":111,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.650545+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:36:10.650545+0000","last_peered":"2023-10-09T05:36:10.650545+0000","last_clean":"2023-10-09T05:36:10.650545+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:36:10.650545+0000","last_undegraded":"2023-10-09T05:36:10.650545+0000","last_fullsized":"2023-10-09T05:36:10.650545+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":347,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.073751+0000","last_change":"2023-10-09T05:34:41.682539+0000","last_active":"2023-10-09T05:35:58.073751+0000","last_peered":"2023-10-09T05:35:58.073751+0000","last_clean":"2023-10-09T05:35:58.073751+0000","last_became_active":"2023-10-09T05:34:41.674099+0000","last_became_peered":"2023-10-09T05:34:41.674099+0000","last_unstale":"2023-10-09T05:35:58.073751+0000","last_undegraded":"2023-10-09T05:35:58.073751+0000","last_fullsized":"2023-10-09T05:35:58.073751+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":91,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.640864+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:36:10.640864+0000","last_peered":"2023-10-09T05:36:10.640864+0000","last_clean":"2023-10-09T05:36:10.640864+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:36:10.640864+0000","last_undegraded":"2023-10-09T05:36:10.640864+0000","last_fullsized":"2023-10-09T05:36:10.640864+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T05:36:11.698616+0000","last_change":"2023-10-09T05:34:41.671414+0000","last_active":"2023-10-09T05:36:11.698616+0000","last_peered":"2023-10-09T05:36:11.698616+0000","last_clean":"2023-10-09T05:36:11.698616+0000","last_became_active":"2023-10-09T05:34:41.671100+0000","last_became_peered":"2023-10-09T05:34:41.671100+0000","last_unstale":"2023-10-09T05:36:11.698616+0000","last_undegraded":"2023-10-09T05:36:11.698616+0000","last_fullsized":"2023-10-09T05:36:11.698616+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:19.832596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"144'8302","reported_seq":8379,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:09.630954+0000","last_change":"2023-10-09T05:34:33.321883+0000","last_active":"2023-10-09T05:36:09.630954+0000","last_peered":"2023-10-09T05:36:09.630954+0000","last_clean":"2023-10-09T05:36:09.630954+0000","last_became_active":"2023-10-09T05:34:25.017164+0000","last_became_peered":"2023-10-09T05:34:25.017164+0000","last_unstale":"2023-10-09T05:36:09.630954+0000","last_undegraded":"2023-10-09T05:36:09.630954+0000","last_fullsized":"2023-10-09T05:36:09.630954+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'6928","last_scrub_stamp":"2023-10-09T05:34:33.321793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:33.321793+0000","objects_scrubbed":0,"log_size":8302,"log_dups_size":0,"ondisk_log_size":8302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18481152,"num_objects":282,"num_object_clones":0,"num_object_copies":846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1385,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":129,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.133054+0000","last_change":"2023-10-09T05:34:41.664569+0000","last_active":"2023-10-09T05:36:06.133054+0000","last_peered":"2023-10-09T05:36:06.133054+0000","last_clean":"2023-10-09T05:36:06.133054+0000","last_became_active":"2023-10-09T05:34:41.664239+0000","last_became_peered":"2023-10-09T05:34:41.664239+0000","last_unstale":"2023-10-09T05:36:06.133054+0000","last_undegraded":"2023-10-09T05:36:06.133054+0000","last_fullsized":"2023-10-09T05:36:06.133054+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:26.511248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":532,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.640915+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:36:10.640915+0000","last_peered":"2023-10-09T05:36:10.640915+0000","last_clean":"2023-10-09T05:36:10.640915+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:36:10.640915+0000","last_undegraded":"2023-10-09T05:36:10.640915+0000","last_fullsized":"2023-10-09T05:36:10.640915+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":126,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.133147+0000","last_change":"2023-10-09T05:34:41.666200+0000","last_active":"2023-10-09T05:36:06.133147+0000","last_peered":"2023-10-09T05:36:06.133147+0000","last_clean":"2023-10-09T05:36:06.133147+0000","last_became_active":"2023-10-09T05:34:41.665841+0000","last_became_peered":"2023-10-09T05:34:41.665841+0000","last_unstale":"2023-10-09T05:36:06.133147+0000","last_undegraded":"2023-10-09T05:36:06.133147+0000","last_fullsized":"2023-10-09T05:36:06.133147+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:58.338181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"144'8240","reported_seq":10947,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:09.827013+0000","last_change":"2023-10-09T05:34:57.102099+0000","last_active":"2023-10-09T05:36:09.827013+0000","last_peered":"2023-10-09T05:36:09.827013+0000","last_clean":"2023-10-09T05:36:09.827013+0000","last_became_active":"2023-10-09T05:34:57.096500+0000","last_became_peered":"2023-10-09T05:34:57.096500+0000","last_unstale":"2023-10-09T05:36:09.827013+0000","last_undegraded":"2023-10-09T05:36:09.827013+0000","last_fullsized":"2023-10-09T05:36:09.827013+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8240,"log_dups_size":0,"ondisk_log_size":8240,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19644416,"num_objects":299,"num_object_clones":0,"num_object_copies":897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3370,"num_write_kb":117456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1310,"num_bytes_recovered":65667072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":607,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.098873+0000","last_change":"2023-10-09T05:34:41.676842+0000","last_active":"2023-10-09T05:35:58.098873+0000","last_peered":"2023-10-09T05:35:58.098873+0000","last_clean":"2023-10-09T05:35:58.098873+0000","last_became_active":"2023-10-09T05:34:41.676428+0000","last_became_peered":"2023-10-09T05:34:41.676428+0000","last_unstale":"2023-10-09T05:35:58.098873+0000","last_undegraded":"2023-10-09T05:35:58.098873+0000","last_fullsized":"2023-10-09T05:35:58.098873+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":99,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.650469+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:36:10.650469+0000","last_peered":"2023-10-09T05:36:10.650469+0000","last_clean":"2023-10-09T05:36:10.650469+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:36:10.650469+0000","last_undegraded":"2023-10-09T05:36:10.650469+0000","last_fullsized":"2023-10-09T05:36:10.650469+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":95,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.085968+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:35:58.085968+0000","last_peered":"2023-10-09T05:35:58.085968+0000","last_clean":"2023-10-09T05:35:58.085968+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:35:58.085968+0000","last_undegraded":"2023-10-09T05:35:58.085968+0000","last_fullsized":"2023-10-09T05:35:58.085968+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"145'8315","reported_seq":8502,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.956568+0000","last_change":"2023-10-09T05:34:33.714772+0000","last_active":"2023-10-09T05:36:10.956568+0000","last_peered":"2023-10-09T05:36:10.956568+0000","last_clean":"2023-10-09T05:36:10.956568+0000","last_became_active":"2023-10-09T05:34:25.048537+0000","last_became_peered":"2023-10-09T05:34:25.048537+0000","last_unstale":"2023-10-09T05:36:10.956568+0000","last_undegraded":"2023-10-09T05:36:10.956568+0000","last_fullsized":"2023-10-09T05:36:10.956568+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'6879","last_scrub_stamp":"2023-10-09T05:34:33.714537+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:33.714537+0000","objects_scrubbed":1680,"log_size":8315,"log_dups_size":0,"ondisk_log_size":8315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:15.987881+0000","scrub_duration":0.49573348099999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15728640,"num_objects":240,"num_object_clones":0,"num_object_copies":720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1440,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1d","version":"145'10668","reported_seq":10739,"reported_epoch":145,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:10.939926+0000","last_change":"2023-10-09T05:34:42.908183+0000","last_active":"2023-10-09T05:36:10.939926+0000","last_peered":"2023-10-09T05:36:10.939926+0000","last_clean":"2023-10-09T05:33:20.502869+0000","last_became_active":"2023-10-09T05:34:42.896545+0000","last_became_peered":"2023-10-09T05:34:42.896545+0000","last_unstale":"2023-10-09T05:36:10.939926+0000","last_undegraded":"2023-10-09T05:34:42.687614+0000","last_fullsized":"2023-10-09T05:34:42.658151+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10668,"log_dups_size":0,"ondisk_log_size":10668,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33161228,"num_objects":507,"num_object_clones":0,"num_object_copies":1521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2695,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":507,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6926,"num_write_kb":237793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-1514}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":99,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.649397+0000","last_change":"2023-10-09T05:34:41.662989+0000","last_active":"2023-10-09T05:36:10.649397+0000","last_peered":"2023-10-09T05:36:10.649397+0000","last_clean":"2023-10-09T05:36:10.649397+0000","last_became_active":"2023-10-09T05:34:41.662621+0000","last_became_peered":"2023-10-09T05:34:41.662621+0000","last_unstale":"2023-10-09T05:36:10.649397+0000","last_undegraded":"2023-10-09T05:36:10.649397+0000","last_fullsized":"2023-10-09T05:36:10.649397+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:08.750044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":135,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.128984+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:36:06.128984+0000","last_peered":"2023-10-09T05:36:06.128984+0000","last_clean":"2023-10-09T05:36:06.128984+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:36:06.128984+0000","last_undegraded":"2023-10-09T05:36:06.128984+0000","last_fullsized":"2023-10-09T05:36:06.128984+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.1a","version":"145'10557","reported_seq":10616,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.845536+0000","last_change":"2023-10-09T05:34:25.055227+0000","last_active":"2023-10-09T05:36:10.845536+0000","last_peered":"2023-10-09T05:36:10.845536+0000","last_clean":"2023-10-09T05:36:10.845536+0000","last_became_active":"2023-10-09T05:34:25.051761+0000","last_became_peered":"2023-10-09T05:34:25.051761+0000","last_unstale":"2023-10-09T05:36:10.845536+0000","last_undegraded":"2023-10-09T05:36:10.845536+0000","last_fullsized":"2023-10-09T05:36:10.845536+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10557,"log_dups_size":0,"ondisk_log_size":10557,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35618816,"num_objects":544,"num_object_clones":0,"num_object_copies":1632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6839,"num_write_kb":236224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":113,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.086779+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:35:58.086779+0000","last_peered":"2023-10-09T05:35:58.086779+0000","last_clean":"2023-10-09T05:35:58.086779+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:35:58.086779+0000","last_undegraded":"2023-10-09T05:35:58.086779+0000","last_fullsized":"2023-10-09T05:35:58.086779+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":125,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.077420+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:35:58.077420+0000","last_peered":"2023-10-09T05:35:58.077420+0000","last_clean":"2023-10-09T05:35:58.077420+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:35:58.077420+0000","last_undegraded":"2023-10-09T05:35:58.077420+0000","last_fullsized":"2023-10-09T05:35:58.077420+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1b","version":"145'10535","reported_seq":10594,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.989388+0000","last_change":"2023-10-09T05:34:25.065203+0000","last_active":"2023-10-09T05:36:10.989388+0000","last_peered":"2023-10-09T05:36:10.989388+0000","last_clean":"2023-10-09T05:36:10.989388+0000","last_became_active":"2023-10-09T05:34:25.059858+0000","last_became_peered":"2023-10-09T05:34:25.059858+0000","last_unstale":"2023-10-09T05:36:10.989388+0000","last_undegraded":"2023-10-09T05:36:10.989388+0000","last_fullsized":"2023-10-09T05:36:10.989388+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10535,"log_dups_size":0,"ondisk_log_size":10535,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41156608,"num_objects":628,"num_object_clones":0,"num_object_copies":1884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6797,"num_write_kb":237600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":98,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.089864+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:35:58.089864+0000","last_peered":"2023-10-09T05:35:58.089864+0000","last_clean":"2023-10-09T05:35:58.089864+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:35:58.089864+0000","last_undegraded":"2023-10-09T05:35:58.089864+0000","last_fullsized":"2023-10-09T05:35:58.089864+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.077365+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:35:58.077365+0000","last_peered":"2023-10-09T05:35:58.077365+0000","last_clean":"2023-10-09T05:35:58.077365+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:35:58.077365+0000","last_undegraded":"2023-10-09T05:35:58.077365+0000","last_fullsized":"2023-10-09T05:35:58.077365+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"145'10064","reported_seq":10204,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T05:36:10.853582+0000","last_change":"2023-10-09T05:34:07.818690+0000","last_active":"2023-10-09T05:36:10.853582+0000","last_peered":"2023-10-09T05:36:10.853582+0000","last_clean":"2023-10-09T05:36:10.853582+0000","last_became_active":"2023-10-09T05:34:01.656619+0000","last_became_peered":"2023-10-09T05:34:01.656619+0000","last_unstale":"2023-10-09T05:36:10.853582+0000","last_undegraded":"2023-10-09T05:36:10.853582+0000","last_fullsized":"2023-10-09T05:36:10.853582+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6429","last_scrub_stamp":"2023-10-09T05:34:07.818377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:07.818377+0000","objects_scrubbed":3381,"log_size":10064,"log_dups_size":0,"ondisk_log_size":10064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:04.556045+0000","scrub_duration":0.90924191099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35520512,"num_objects":542,"num_object_clones":0,"num_object_copies":1626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3651,"num_write_kb":25984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":122,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.133744+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:36:06.133744+0000","last_peered":"2023-10-09T05:36:06.133744+0000","last_clean":"2023-10-09T05:36:06.133744+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:36:06.133744+0000","last_undegraded":"2023-10-09T05:36:06.133744+0000","last_fullsized":"2023-10-09T05:36:06.133744+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":134,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T05:36:06.135614+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:36:06.135614+0000","last_peered":"2023-10-09T05:36:06.135614+0000","last_clean":"2023-10-09T05:36:06.135614+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:36:06.135614+0000","last_undegraded":"2023-10-09T05:36:06.135614+0000","last_fullsized":"2023-10-09T05:36:06.135614+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"144'9793","reported_seq":14972,"reported_epoch":144,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:09.895360+0000","last_change":"2023-10-09T05:35:42.217655+0000","last_active":"2023-10-09T05:36:09.895360+0000","last_peered":"2023-10-09T05:36:09.895360+0000","last_clean":"2023-10-09T05:33:20.515228+0000","last_became_active":"2023-10-09T05:34:42.874596+0000","last_became_peered":"2023-10-09T05:34:42.874596+0000","last_unstale":"2023-10-09T05:36:09.895360+0000","last_undegraded":"2023-10-09T05:34:42.676976+0000","last_fullsized":"2023-10-09T05:34:42.659894+0000","mapping_epoch":128,"log_start":"109'1624","ondisk_log_start":"109'1624","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8169,"log_dups_size":0,"ondisk_log_size":8169,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38600704,"num_objects":589,"num_object_clones":0,"num_object_copies":1767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":41,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6733,"num_write_kb":234304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2557,"num_bytes_recovered":82051072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":548}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":100,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T05:36:11.698318+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:36:11.698318+0000","last_peered":"2023-10-09T05:36:11.698318+0000","last_clean":"2023-10-09T05:36:11.698318+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:36:11.698318+0000","last_undegraded":"2023-10-09T05:36:11.698318+0000","last_fullsized":"2023-10-09T05:36:11.698318+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T05:35:58.076996+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:35:58.076996+0000","last_peered":"2023-10-09T05:35:58.076996+0000","last_clean":"2023-10-09T05:35:58.076996+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:35:58.076996+0000","last_undegraded":"2023-10-09T05:35:58.076996+0000","last_fullsized":"2023-10-09T05:35:58.076996+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":36,"stat_sum":{"num_bytes":1134002200,"num_objects":17305,"num_object_clones":0,"num_object_copies":51915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17736,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17305,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":168775,"num_write_kb":4609889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7839,"num_bytes_recovered":345309184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3421249536,"data_stored":3421241392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":343911,"ondisk_log_size":343911,"up":108,"acting":100,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846611,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":709744,"kb_used_data":399464,"kb_used_omap":14,"kb_used_meta":310193,"kb_avail":93035408,"statfs":{"total":95995035648,"available":95268257792,"internally_reserved":0,"allocated":409051136,"data_stored":408536532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":317638490},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":7.3810000000000002,"5min":7.3810000000000002,"15min":7.3810000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":60.323999999999998,"5min":60.323999999999998,"15min":60.323999999999998},"last":0.245},{"interface":"front","average":{"1min":8.548,"5min":8.548,"15min":8.548},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":60.334000000000003,"5min":60.334000000000003,"15min":60.334000000000003},"last":0.48499999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.6200000000000001,"15min":1.6200000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":2.6669999999999998},{"interface":"front","average":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.585000000000001,"5min":11.585000000000001,"15min":11.585000000000001},"last":0.59199999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":6.859,"5min":6.859,"15min":6.859},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":73.896000000000001,"5min":73.896000000000001,"15min":73.896000000000001},"last":2.1680000000000001},{"interface":"front","average":{"1min":6.798,"5min":6.798,"15min":6.798},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":73.881,"5min":73.881,"15min":73.881},"last":3.831}]},{"osd":3,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.0670000000000002,"5min":4.0670000000000002,"15min":4.0670000000000002},"last":1.121},{"interface":"front","average":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.0389999999999997,"5min":6.0389999999999997,"15min":6.0389999999999997},"last":1.6930000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.9930000000000001,"15min":1.9930000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":12.789,"5min":12.789,"15min":12.789},"last":0.28299999999999997},{"interface":"front","average":{"1min":2.2519999999999998,"5min":2.2519999999999998,"15min":2.2519999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":8.9009999999999998,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":0.38800000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":2.0649999999999999,"15min":2.0649999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":8.8230000000000004,"5min":8.8230000000000004,"15min":8.8230000000000004},"last":1.165},{"interface":"front","average":{"1min":2.7770000000000001,"5min":2.7770000000000001,"15min":2.7770000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":9.8049999999999997,"5min":9.8049999999999997,"15min":9.8049999999999997},"last":3.3340000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":2.2040000000000002,"15min":2.2040000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":8.8580000000000005,"5min":8.8580000000000005,"15min":8.8580000000000005},"last":1.056},{"interface":"front","average":{"1min":2.081,"5min":2.081,"15min":2.081},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.2680000000000007,"5min":8.2680000000000007,"15min":8.2680000000000007},"last":2.133}]}]},{"osd":6,"up_from":42,"seq":180388626637,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594212,"kb_used_data":296988,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93150940,"statfs":{"total":95995035648,"available":95386562560,"internally_reserved":0,"allocated":304115712,"data_stored":303613004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:13 2023","interfaces":[{"interface":"back","average":{"1min":2.5859999999999999,"5min":1.8400000000000001,"15min":0.97399999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.8000000000000007,"5min":53.771999999999998,"15min":53.771999999999998},"last":0.55900000000000005},{"interface":"front","average":{"1min":2.399,"5min":1.806,"15min":0.96499999999999997},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.9149999999999991,"5min":53.411999999999999,"15min":53.411999999999999},"last":0.40899999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:35:36 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":1.5429999999999999,"15min":0.88300000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.1840000000000002,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":0.28599999999999998},{"interface":"front","average":{"1min":3.1549999999999998,"5min":1.76,"15min":0.97099999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.9979999999999993,"5min":9.9979999999999993,"15min":9.9979999999999993},"last":0.61599999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:35:13 2023","interfaces":[{"interface":"back","average":{"1min":3.3109999999999999,"5min":2.379,"15min":1.5089999999999999},"min":{"1min":0.36599999999999999,"5min":0.314,"15min":0.314},"max":{"1min":20.084,"5min":34.393999999999998,"15min":34.393999999999998},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.7109999999999999,"5min":2.2970000000000002,"15min":1.4950000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":20.073,"5min":34.381999999999998,"15min":34.381999999999998},"last":0.32600000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":1.5660000000000001,"15min":1.232},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.0609999999999999,"5min":18.361000000000001,"15min":18.361000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.7729999999999999,"15min":1.339},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.042999999999999,"5min":18.536000000000001,"15min":18.536000000000001},"last":0.63700000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:35:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1800000000000002,"5min":2.8479999999999999,"15min":1.262},"min":{"1min":0.29599999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.8569999999999993,"5min":125.64,"15min":125.64},"last":1.6140000000000001},{"interface":"front","average":{"1min":1.6220000000000001,"5min":2.7669999999999999,"15min":1.248},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.27800000000000002},"max":{"1min":6.6109999999999998,"5min":125.625,"15min":125.625},"last":0.434}]},{"osd":5,"last update":"Mon Oct 9 05:35:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2749999999999999,"5min":2.488,"15min":1.1719999999999999},"min":{"1min":0.36699999999999999,"5min":0.27900000000000003,"15min":0.27200000000000002},"max":{"1min":9.7439999999999998,"5min":127.35899999999999,"15min":127.35899999999999},"last":0.872},{"interface":"front","average":{"1min":1.992,"5min":2.5310000000000001,"15min":1.1950000000000001},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.6379999999999999,"5min":128.93799999999999,"15min":128.93799999999999},"last":1.8500000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":3.1480000000000001,"5min":3.1480000000000001,"15min":3.1480000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.074999999999999,"5min":12.074999999999999,"15min":12.074999999999999},"last":4.96},{"interface":"front","average":{"1min":2.0920000000000001,"5min":2.0920000000000001,"15min":2.0920000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.083,"5min":10.083,"15min":10.083},"last":1.0209999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958599,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":463236,"kb_used_data":319164,"kb_used_omap":7,"kb_used_meta":144056,"kb_avail":93281916,"statfs":{"total":95995035648,"available":95520681984,"internally_reserved":0,"allocated":326823936,"data_stored":326191371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":147513870},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":7.8689999999999998,"5min":2.2719999999999998,"15min":1.026},"min":{"1min":0.19800000000000001,"5min":0.185,"15min":0.185},"max":{"1min":124.82599999999999,"5min":124.82599999999999,"15min":124.82599999999999},"last":4.5049999999999999},{"interface":"front","average":{"1min":7.7249999999999996,"5min":2.367,"15min":1.0609999999999999},"min":{"1min":0.24199999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":125.483,"5min":125.483,"15min":125.483},"last":6.0069999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":3.2440000000000002,"5min":2.778,"15min":1.1539999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":21.216999999999999,"5min":98.483999999999995,"15min":98.483999999999995},"last":4.1059999999999999},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.5649999999999999,"15min":1.131},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":21.161000000000001,"5min":98.409999999999997,"15min":98.409999999999997},"last":4.5679999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.107,"15min":0.68799999999999994},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":10.298999999999999,"5min":10.298999999999999,"15min":10.298999999999999},"last":3.9420000000000002},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.1859999999999999,"15min":0.66400000000000003},"min":{"1min":0.26600000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.8109999999999999,"5min":8.8109999999999999,"15min":8.8109999999999999},"last":4.5359999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.4390000000000001,"15min":0.90800000000000003},"min":{"1min":0.34399999999999997,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.1940000000000008,"5min":23.670999999999999,"15min":23.670999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.5800000000000001,"5min":1.1779999999999999,"15min":0.83299999999999996},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.8099999999999996,"5min":23.620000000000001,"15min":23.620000000000001},"last":0.51600000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.3720000000000001,"15min":0.879},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.1139999999999999,"5min":23.501999999999999,"15min":23.501999999999999},"last":2.9740000000000002},{"interface":"front","average":{"1min":1.75,"5min":1.639,"15min":0.998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":6.9100000000000001,"5min":25.68,"15min":25.68},"last":0.49099999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.1679999999999999,"15min":0.82599999999999996},"min":{"1min":0.44500000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":4.3819999999999997,"5min":8.1270000000000007,"15min":8.1270000000000007},"last":3.0030000000000001},{"interface":"front","average":{"1min":2.0110000000000001,"5min":1.2410000000000001,"15min":0.871},"min":{"1min":0.43099999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.0810000000000004,"5min":7.0810000000000004,"15min":7.0810000000000004},"last":0.46100000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9990000000000001,"5min":1.9990000000000001,"15min":1.9990000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":6.5919999999999996,"5min":6.5919999999999996,"15min":6.5919999999999996},"last":0.85099999999999998},{"interface":"front","average":{"1min":2.0790000000000002,"5min":2.0790000000000002,"15min":2.0790000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":6.5529999999999999,"5min":6.5529999999999999,"15min":6.5529999999999999},"last":0.81799999999999995}]}]},{"osd":0,"up_from":88,"seq":377957122125,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":677756,"kb_used_data":515636,"kb_used_omap":8,"kb_used_meta":162103,"kb_avail":93067396,"statfs":{"total":95995035648,"available":95301013504,"internally_reserved":0,"allocated":528011264,"data_stored":527385399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":165994248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:35:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":1.117,"15min":0.63600000000000001},"min":{"1min":0.22700000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":7.9089999999999998,"5min":13.919,"15min":13.919},"last":6.8769999999999998},{"interface":"front","average":{"1min":1.782,"5min":1.155,"15min":0.67800000000000005},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.0009999999999994,"5min":9.9350000000000005,"15min":9.9350000000000005},"last":4.5119999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":8.6180000000000003,"5min":3.4969999999999999,"15min":1.403},"min":{"1min":0.223,"5min":0.214,"15min":0.214},"max":{"1min":79.911000000000001,"5min":79.911000000000001,"15min":79.911000000000001},"last":2.1509999999999998},{"interface":"front","average":{"1min":8.1799999999999997,"5min":3.2440000000000002,"15min":1.3300000000000001},"min":{"1min":0.23200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":80.105000000000004,"5min":80.105000000000004,"15min":80.105000000000004},"last":1.736}]},{"osd":3,"last update":"Mon Oct 9 05:35:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4329999999999998,"5min":1.4159999999999999,"15min":0.73899999999999999},"min":{"1min":0.23999999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.9700000000000006,"5min":12.429,"15min":12.429},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.5169999999999999,"5min":1.603,"15min":0.80100000000000005},"min":{"1min":0.30499999999999999,"5min":0.222,"15min":0.222},"max":{"1min":7.5,"5min":22.390999999999998,"15min":22.390999999999998},"last":2.5819999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.1859999999999999,"15min":0.84899999999999998},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.6130000000000004,"5min":14.023,"15min":14.023},"last":1.7769999999999999},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.542,"15min":0.995},"min":{"1min":0.41099999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.75,"5min":14.096,"15min":14.096},"last":0.34799999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":1.5529999999999999,"15min":0.97099999999999997},"min":{"1min":0.34499999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.8019999999999996,"5min":13.632,"15min":13.632},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.5600000000000001,"15min":0.96999999999999997},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":8.1549999999999994,"5min":13.676,"15min":13.676},"last":1.9570000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.5289999999999999,"15min":0.98099999999999998},"min":{"1min":0.39200000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.8339999999999996,"5min":9.9809999999999999,"15min":9.9809999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.141,"5min":1.597,"15min":0.98499999999999999},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.4450000000000003,"5min":7.4450000000000003,"15min":7.4450000000000003},"last":1.9930000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:35:43 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.329,"15min":1.329},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":5.0899999999999999,"5min":5.0899999999999999,"15min":5.0899999999999999},"last":1.8149999999999999},{"interface":"front","average":{"1min":2.3900000000000001,"5min":2.3900000000000001,"15min":2.3900000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":11.068,"5min":11.068,"15min":11.068},"last":0.52900000000000003}]}]},{"osd":2,"up_from":98,"seq":420906795072,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":590144,"kb_used_data":419768,"kb_used_omap":11,"kb_used_meta":170228,"kb_avail":93155008,"statfs":{"total":95995035648,"available":95390728192,"internally_reserved":0,"allocated":429842432,"data_stored":429205213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":174314069},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2999999999999998,"5min":1.5620000000000001,"15min":0.75800000000000001},"min":{"1min":0.183,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":10.805999999999999,"5min":32.122,"15min":32.122},"last":2.7370000000000001},{"interface":"front","average":{"1min":1.5780000000000001,"5min":1.3460000000000001,"15min":0.71199999999999997},"min":{"1min":0.23599999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.3849999999999998,"5min":34.066000000000003,"15min":34.066000000000003},"last":0.59599999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":2.028,"15min":0.93000000000000005},"min":{"1min":0.218,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.3510000000000009,"5min":98.459999999999994,"15min":98.459999999999994},"last":6.0590000000000002},{"interface":"front","average":{"1min":1.9379999999999999,"5min":1.9590000000000001,"15min":0.90700000000000003},"min":{"1min":0.218,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":16.007999999999999,"5min":98.397000000000006,"15min":98.397000000000006},"last":3.6720000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:35:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4140000000000001,"5min":1.4550000000000001,"15min":0.73499999999999999},"min":{"1min":0.249,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":23.738,"5min":34.052999999999997,"15min":34.052999999999997},"last":5.7779999999999996},{"interface":"front","average":{"1min":2.9849999999999999,"5min":1.669,"15min":0.82299999999999995},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":23.901,"5min":35.161999999999999,"15min":35.161999999999999},"last":3.1419999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3159999999999998,"5min":1.3149999999999999,"15min":0.82099999999999995},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.8960000000000008,"5min":8.8960000000000008,"15min":8.8960000000000008},"last":3.7010000000000001},{"interface":"front","average":{"1min":3.427,"5min":1.5780000000000001,"15min":0.95399999999999996},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":16.869,"5min":16.869,"15min":16.869},"last":0.40300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":1.524,"15min":0.93600000000000005},"min":{"1min":0.376,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.2789999999999999,"5min":8.2789999999999999,"15min":8.2789999999999999},"last":6.1260000000000003},{"interface":"front","average":{"1min":3.1219999999999999,"5min":1.4890000000000001,"15min":0.90100000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":8.8399999999999999,"5min":8.8399999999999999,"15min":8.8399999999999999},"last":3.7330000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":1.554,"15min":0.93400000000000005},"min":{"1min":0.379,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":10.715,"5min":11.496,"15min":11.496},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.1459999999999999,"5min":1.3600000000000001,"15min":0.91000000000000003},"min":{"1min":0.40100000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.093,"5min":9.4049999999999994,"15min":9.4049999999999994},"last":6.0970000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:35:44 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":2.524,"15min":2.524},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":10.673,"5min":10.673,"15min":10.673},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.3190000000000008,"5min":8.3190000000000008,"15min":8.3190000000000008},"last":0.54400000000000004}]}]},{"osd":3,"up_from":103,"seq":442381631546,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297680,"kb_used_data":211912,"kb_used_omap":4,"kb_used_meta":85691,"kb_avail":93447472,"statfs":{"total":95995035648,"available":95690211328,"internally_reserved":0,"allocated":216997888,"data_stored":216397640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":87748190},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.6459999999999999,"15min":0.81399999999999995},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":5.3099999999999996,"5min":56.094000000000001,"15min":56.094000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.085,"5min":1.9339999999999999,"15min":0.89500000000000002},"min":{"1min":0.246,"5min":0.214,"15min":0.214},"max":{"1min":10.084,"5min":57.255000000000003,"15min":57.255000000000003},"last":9.2940000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":0.85599999999999998,"15min":0.55400000000000005},"min":{"1min":0.27000000000000002,"5min":0.255,"15min":0.255},"max":{"1min":5.04,"5min":5.04,"15min":5.04},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.617,"5min":0.91500000000000004,"15min":0.55500000000000005},"min":{"1min":0.32600000000000001,"5min":0.247,"15min":0.247},"max":{"1min":5.7080000000000002,"5min":6.8769999999999998,"15min":6.8769999999999998},"last":4.3639999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":3.0859999999999999,"15min":1.3140000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":6.3849999999999998,"5min":117.23,"15min":117.23},"last":0.71499999999999997},{"interface":"front","average":{"1min":2.2200000000000002,"5min":3.1549999999999998,"15min":1.345},"min":{"1min":0.313,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.227,"5min":117.45999999999999,"15min":117.45999999999999},"last":1.1639999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:35:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3359999999999999,"5min":2.0129999999999999,"15min":1.069},"min":{"1min":0.33200000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.1099999999999994,"5min":45.779000000000003,"15min":45.779000000000003},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.681,"15min":0.99199999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.2050000000000001,"5min":45.735999999999997,"15min":45.735999999999997},"last":0.66100000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":1.6950000000000001,"15min":0.99299999999999999},"min":{"1min":0.44500000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.4770000000000003,"5min":47.948999999999998,"15min":47.948999999999998},"last":4.4100000000000001},{"interface":"front","average":{"1min":2.5179999999999998,"5min":1.8129999999999999,"15min":1.0569999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":8.5009999999999994,"5min":47.968000000000004,"15min":47.968000000000004},"last":0.69299999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":2.5499999999999998,"5min":1.4550000000000001,"15min":0.93300000000000005},"min":{"1min":0.379,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.5990000000000002,"5min":8.5990000000000002,"15min":8.5990000000000002},"last":3.8050000000000002},{"interface":"front","average":{"1min":2.077,"5min":0.95799999999999996,"15min":0.73399999999999999},"min":{"1min":0.41199999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.6879999999999997,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":4.4459999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:35:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":2.2599999999999998,"15min":2.2599999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.8869999999999996,"5min":5.8869999999999996,"15min":5.8869999999999996},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.9630000000000001,"15min":1.9630000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.851,"5min":5.851,"15min":5.851},"last":1.714}]}]},{"osd":4,"up_from":30,"seq":128849019093,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905640,"kb_used_data":600160,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92839512,"statfs":{"total":95995035648,"available":95067660288,"internally_reserved":0,"allocated":614563840,"data_stored":614056814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:07 2023","interfaces":[{"interface":"back","average":{"1min":3.3109999999999999,"5min":1.5049999999999999,"15min":0.84899999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":33.488999999999997,"5min":33.488999999999997,"15min":33.488999999999997},"last":4.484},{"interface":"front","average":{"1min":3.569,"5min":1.631,"15min":0.91200000000000003},"min":{"1min":0.27300000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":33.616999999999997,"5min":33.616999999999997,"15min":33.616999999999997},"last":0.433}]},{"osd":1,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.2919999999999998,"5min":1.4239999999999999,"15min":0.83199999999999996},"min":{"1min":0.41599999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.073,"5min":31.073,"15min":31.073},"last":5.5069999999999997},{"interface":"front","average":{"1min":2.169,"5min":1.579,"15min":0.89900000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":10.055,"5min":31.100000000000001,"15min":31.100000000000001},"last":2.0920000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:35:07 2023","interfaces":[{"interface":"back","average":{"1min":11.824,"5min":3.4830000000000001,"15min":1.506},"min":{"1min":0.35299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":164.26599999999999,"5min":164.26599999999999,"15min":164.26599999999999},"last":7.6459999999999999},{"interface":"front","average":{"1min":10.645,"5min":3.3359999999999999,"15min":1.502},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":162.57499999999999,"5min":162.57499999999999,"15min":162.57499999999999},"last":4.5149999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.3899999999999999,"15min":0.83599999999999997},"min":{"1min":0.35399999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.037000000000001,"5min":10.037000000000001,"15min":10.037000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.429,"5min":1.1519999999999999,"15min":0.79200000000000004},"min":{"1min":0.34100000000000003,"5min":0.309,"15min":0.309},"max":{"1min":4.6219999999999999,"5min":6.9459999999999997,"15min":6.9459999999999997},"last":5.6909999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:35:34 2023","interfaces":[{"interface":"back","average":{"1min":7.8369999999999997,"5min":2.2010000000000001,"15min":1.034},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":113.35899999999999,"5min":113.35899999999999,"15min":113.35899999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":7.71,"5min":2.125,"15min":1.071},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":113.32299999999999,"5min":113.32299999999999,"15min":113.32299999999999},"last":4.4470000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:35:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8009999999999999,"5min":0.89800000000000002,"15min":0.622},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.27800000000000002},"max":{"1min":11.944000000000001,"5min":11.944000000000001,"15min":11.944000000000001},"last":4.5350000000000001},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.1539999999999999,"15min":0.71699999999999997},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":10.356,"5min":10.356,"15min":10.356},"last":2}]},{"osd":7,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":3.3700000000000001,"5min":3.3700000000000001,"15min":3.3700000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":30.393999999999998,"5min":30.393999999999998,"15min":30.393999999999998},"last":6.1900000000000004},{"interface":"front","average":{"1min":3.8300000000000001,"5min":3.8300000000000001,"15min":3.8300000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":30.334,"5min":30.334,"15min":30.334},"last":5.476}]}]},{"osd":5,"up_from":36,"seq":154618822865,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":971292,"kb_used_data":669972,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92773860,"statfs":{"total":95995035648,"available":95000432640,"internally_reserved":0,"allocated":686051328,"data_stored":685552062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:06 2023","interfaces":[{"interface":"back","average":{"1min":14.334,"5min":6.5940000000000003,"15min":2.5659999999999998},"min":{"1min":0.22700000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":165.90199999999999,"5min":165.90199999999999,"15min":165.90199999999999},"last":2.7810000000000001},{"interface":"front","average":{"1min":13.84,"5min":6.8440000000000003,"15min":2.6520000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":165.96700000000001,"5min":165.96700000000001,"15min":165.96700000000001},"last":0.36799999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:35:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7429999999999999,"5min":1.3480000000000001,"15min":0.81999999999999995},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.539999999999999,"5min":12.891999999999999,"15min":12.891999999999999},"last":0.98799999999999999},{"interface":"front","average":{"1min":1.522,"5min":1.0740000000000001,"15min":0.72999999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.2609999999999992,"5min":12.939,"15min":12.939},"last":0.29299999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:35:13 2023","interfaces":[{"interface":"back","average":{"1min":19.474,"5min":4.7450000000000001,"15min":1.9630000000000001},"min":{"1min":0.28199999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":290.697,"5min":290.697,"15min":290.697},"last":7.556},{"interface":"front","average":{"1min":20.004999999999999,"5min":4.7560000000000002,"15min":1.9630000000000001},"min":{"1min":0.32200000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":290.786,"5min":290.786,"15min":290.786},"last":1.0449999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:35:36 2023","interfaces":[{"interface":"back","average":{"1min":2.4300000000000002,"5min":1.23,"15min":0.80500000000000005},"min":{"1min":0.35599999999999998,"5min":0.314,"15min":0.314},"max":{"1min":7.8719999999999999,"5min":7.8719999999999999,"15min":7.8719999999999999},"last":3.0049999999999999},{"interface":"front","average":{"1min":2.173,"5min":1.294,"15min":0.82199999999999995},"min":{"1min":0.33900000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.0259999999999998,"5min":12.779999999999999,"15min":12.779999999999999},"last":2.819}]},{"osd":4,"last update":"Mon Oct 9 05:35:32 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":0.877,"15min":0.60599999999999998},"min":{"1min":0.30499999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":9.2729999999999997,"5min":9.3510000000000009,"15min":9.3510000000000009},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.194,"5min":1.111,"15min":0.71299999999999997},"min":{"1min":0.32000000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":14.236000000000001,"5min":14.236000000000001,"15min":14.236000000000001},"last":2.843}]},{"osd":6,"last update":"Mon Oct 9 05:36:01 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":2.3039999999999998,"15min":1.1220000000000001},"min":{"1min":0.41299999999999998,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":14.282999999999999,"5min":95.010999999999996,"15min":95.010999999999996},"last":3.4790000000000001},{"interface":"front","average":{"1min":2.649,"5min":2.1040000000000001,"15min":1.0629999999999999},"min":{"1min":0.34200000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.5979999999999999,"5min":94.194000000000003,"15min":94.194000000000003},"last":1.024}]},{"osd":7,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.2140000000000004,"5min":8.2140000000000004,"15min":8.2140000000000004},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.1230000000000002,"5min":2.1230000000000002,"15min":2.1230000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":14.432,"5min":14.432,"15min":14.432},"last":6.0490000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":525008896,"data_stored":525008896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":318308352,"data_stored":318308352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":419106816,"data_stored":419102744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":209911808,"data_stored":209911808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":585826304,"data_stored":585826304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":663883776,"data_stored":663879704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":296943616,"data_stored":296943616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":402259968,"data_stored":402259968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:36:17.301 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:36:17.301 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 11 2023-10-09T05:36:17.315 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:17.341 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:17.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:17.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:17.386 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:17 smithi008 ceph-mon[184418]: from='client.44915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:36:17.386 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:17 smithi008 ceph-mon[184418]: pgmap v381: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1002 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 17695/47982 objects degraded (36.878%); 360 KiB/s, 95 objects/s recovering 2023-10-09T05:36:17.387 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:17 smithi008 ceph-mon[191227]: from='client.44915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:36:17.387 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:17 smithi008 ceph-mon[191227]: pgmap v381: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1002 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 17695/47982 objects degraded (36.878%); 360 KiB/s, 95 objects/s recovering 2023-10-09T05:36:17.448 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:17.570 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:17.731 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:17.748 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:17.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:17.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:17.855 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:17.908 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:17.997 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:36:18.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:18.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:18.107 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:18.176 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:18.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 17695/47982 objects degraded (36.878%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:18.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 17695/47982 objects degraded (36.878%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 17695/47982 objects degraded (36.878%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:18.997 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:19.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:19.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:19.109 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:19.137 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:19.149 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:36:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:19 smithi008 ceph-mon[184418]: pgmap v382: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 866 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 17354/41448 objects degraded (41.869%); 3.0 MiB/s, 53 objects/s recovering 2023-10-09T05:36:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:19 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/736793422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:19 smithi008 ceph-mon[191227]: pgmap v382: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 866 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 17354/41448 objects degraded (41.869%); 3.0 MiB/s, 53 objects/s recovering 2023-10-09T05:36:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:19 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/736793422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:19.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:19 smithi191 ceph-mon[161669]: pgmap v382: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 866 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 17354/41448 objects degraded (41.869%); 3.0 MiB/s, 53 objects/s recovering 2023-10-09T05:36:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:19 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/736793422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[184418]: failed to encode map e147 with expected crc 2023-10-09T05:36:20.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[184418]: failed to encode map e147 with expected crc 2023-10-09T05:36:20.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:20.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[184418]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T05:36:20.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[184418]: failed to encode map e147 with expected crc 2023-10-09T05:36:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[191227]: failed to encode map e147 with expected crc 2023-10-09T05:36:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[191227]: failed to encode map e147 with expected crc 2023-10-09T05:36:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[191227]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T05:36:20.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:20 smithi008 ceph-mon[191227]: failed to encode map e147 with expected crc 2023-10-09T05:36:20.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:20 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:20.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:20 smithi191 ceph-mon[161669]: failed to encode map e147 with expected crc 2023-10-09T05:36:20.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:20 smithi191 ceph-mon[161669]: failed to encode map e147 with expected crc 2023-10-09T05:36:20.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:20 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:20.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:20 smithi191 ceph-mon[161669]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T05:36:20.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:20 smithi191 ceph-mon[161669]: failed to encode map e147 with expected crc 2023-10-09T05:36:20.784 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:20.786 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:36:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:36:20] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.33.4" 2023-10-09T05:36:20.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:20.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:20.890 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:20.950 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:21.416 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:21.422 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:21 smithi008 ceph-mon[191227]: pgmap v384: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 750 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 17354/35862 objects degraded (48.391%); 2.5 MiB/s, 45 objects/s recovering 2023-10-09T05:36:21.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:21.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:21.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:21 smithi008 ceph-mon[184418]: pgmap v384: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 750 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 17354/35862 objects degraded (48.391%); 2.5 MiB/s, 45 objects/s recovering 2023-10-09T05:36:21.524 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:21.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:21 smithi191 ceph-mon[161669]: pgmap v384: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 750 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 17354/35862 objects degraded (48.391%); 2.5 MiB/s, 45 objects/s recovering 2023-10-09T05:36:21.587 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:21.625 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:21.857 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:21.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:21.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:21.969 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:22.040 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:22.061 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:36:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:36:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:227835 5.0 5:03f2cbfa:::benchmark_data_smithi008_238708_object109699:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-09T05:36:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:36:22.051243+0000", 2023-10-09T05:36:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0090408420000000003, 2023-10-09T05:36:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0090788819999999999, 2023-10-09T05:36:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:36:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 227835 2023-10-09T05:36:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051243+0000", 2023-10-09T05:36:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051243+0000", 2023-10-09T05:36:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051240+0000", 2023-10-09T05:36:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:36:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051256+0000", 2023-10-09T05:36:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5327000000000001e-05 2023-10-09T05:36:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051283+0000", 2023-10-09T05:36:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7661000000000002e-05 2023-10-09T05:36:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051297+0000", 2023-10-09T05:36:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.347e-05 2023-10-09T05:36:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051373+0000", 2023-10-09T05:36:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6076999999999997e-05 2023-10-09T05:36:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051521+0000", 2023-10-09T05:36:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014786399999999999 2023-10-09T05:36:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051900+0000", 2023-10-09T05:36:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00037940100000000002 2023-10-09T05:36:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.055535+0000", 2023-10-09T05:36:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036345470000000001 2023-10-09T05:36:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.055698+0000", 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016316 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:227839 5.20 5:07fb6243:::benchmark_data_smithi008_238708_object109703:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:36:22.051313+0000", 2023-10-09T05:36:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0089706130000000005, 2023-10-09T05:36:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0091150859999999997, 2023-10-09T05:36:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:36:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 227839 2023-10-09T05:36:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051313+0000", 2023-10-09T05:36:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051313+0000", 2023-10-09T05:36:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051313+0000", 2023-10-09T05:36:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051316+0000", 2023-10-09T05:36:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2119999999999999e-06 2023-10-09T05:36:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051323+0000", 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8059999999999997e-06 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.051328+0000", 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5459999999999999e-06 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.053647+0000", 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023184070000000002 2023-10-09T05:36:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.053777+0000", 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013005800000000001 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.054092+0000", 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00031572399999999997 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.076 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.055706+0000", 2023-10-09T05:36:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016132399999999999 2023-10-09T05:36:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:36:22.056761+0000", 2023-10-09T05:36:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010552669999999999 2023-10-09T05:36:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:36:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 2 2023-10-09T05:36:22.078 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:36:22.248 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:36:22.278 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:36:22.332 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:36:22.376 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:36:22.527 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:36:22.528 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:36:22.528 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:36:22.528 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:36:22.528 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:36:22.528 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:36:22.528 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:36:22.529 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:36:22.529 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:36:22.529 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:36:22.529 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:36:22.529 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:36:22.612 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:22.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:22.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:22.719 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:22.753 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:23.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:23 smithi008 ceph-mon[191227]: pgmap v385: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 637 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 17207/30459 objects degraded (56.492%); 3.0 MiB/s, 66 objects/s recovering 2023-10-09T05:36:23.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1524818322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:23.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 17207/30459 objects degraded (56.492%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:23.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:23 smithi008 ceph-mon[184418]: pgmap v385: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 637 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 17207/30459 objects degraded (56.492%); 3.0 MiB/s, 66 objects/s recovering 2023-10-09T05:36:23.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1524818322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:23.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 17207/30459 objects degraded (56.492%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:23.532 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:36:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:23 smithi191 ceph-mon[161669]: pgmap v385: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 637 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 17207/30459 objects degraded (56.492%); 3.0 MiB/s, 66 objects/s recovering 2023-10-09T05:36:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1524818322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 17207/30459 objects degraded (56.492%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:23.700 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:36:23.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:23.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:23.816 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:23.886 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:23.929 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:23.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:23.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:24.034 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:24.060 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:24.172 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:36:24.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:24.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:24.287 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:24.332 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:24.526 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:36:24.868 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:36:24.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:24.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:24.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:25.038 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:25.141 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:25.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:25.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:25.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:25 smithi008 ceph-mon[184418]: pgmap v386: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 591 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 17207/28236 objects degraded (60.940%); 3.0 MiB/s, 66 objects/s recovering 2023-10-09T05:36:25.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:25 smithi008 ceph-mon[191227]: pgmap v386: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 591 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 17207/28236 objects degraded (60.940%); 3.0 MiB/s, 66 objects/s recovering 2023-10-09T05:36:25.255 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:25.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:25 smithi191 ceph-mon[161669]: pgmap v386: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 591 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 17207/28236 objects degraded (60.940%); 3.0 MiB/s, 66 objects/s recovering 2023-10-09T05:36:25.314 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:25.538 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:25.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:25.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:25.644 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:25.718 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:25.724 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:25.832 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:36:25.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:25.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:25.859 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:25.953 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:26.023 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:26.205 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:36:26.443 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:36:26.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:26.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:26.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:26.612 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:26.706 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:36:26.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:26.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:26.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:26.883 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:26.994 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:27.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:27.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:27.105 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:27.191 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:27.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:27 smithi191 ceph-mon[161669]: pgmap v387: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 493 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 17207/23547 objects degraded (73.075%); 3.0 MiB/s, 62 objects/s recovering 2023-10-09T05:36:27.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1547853442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:36:27.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:27 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:36:27.402 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:36:27.458 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:27 smithi008 ceph-mon[184418]: pgmap v387: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 493 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 17207/23547 objects degraded (73.075%); 3.0 MiB/s, 62 objects/s recovering 2023-10-09T05:36:27.458 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1547853442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:36:27.458 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:27 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:36:27.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:27 smithi008 ceph-mon[191227]: pgmap v387: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 493 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 17207/23547 objects degraded (73.075%); 3.0 MiB/s, 62 objects/s recovering 2023-10-09T05:36:27.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1547853442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:36:27.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:27 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:36:27.771 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:27.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:27.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:36:27.886 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:27.953 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:28.261 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:28.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:28.367 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:28.425 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 17207/23547 objects degraded (73.075%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:28.427 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 17207/23547 objects degraded (73.075%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:28.432 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:28.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 17207/23547 objects degraded (73.075%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:28.809 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:36:28.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:28.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:28.916 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:28.985 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:29.562 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:29.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:29.674 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:29.725 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:29.838 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:29.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:29.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:29.871 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:29 smithi008 ceph-mon[191227]: pgmap v388: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 17207/16791 objects degraded (102.478%); 940 KiB/s, 59 objects/s recovering 2023-10-09T05:36:29.871 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:29.871 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:29 smithi008 ceph-mon[184418]: pgmap v388: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 17207/16791 objects degraded (102.478%); 940 KiB/s, 59 objects/s recovering 2023-10-09T05:36:29.871 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:29.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:29.999 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:29 smithi191 ceph-mon[161669]: pgmap v388: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 17207/16791 objects degraded (102.478%); 940 KiB/s, 59 objects/s recovering 2023-10-09T05:36:29.999 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:30.009 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:30.028 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:30.094 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:30.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:30.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:30.204 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:30.247 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:30.369 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:30.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:30.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:30.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:30.518 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:30.529 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:36:30.529 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:36:30.613 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:30.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:30.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:30.697 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:36:30.729 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:30.763 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/919544679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:36:30.763 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:30 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:36:30.763 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/919544679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:36:30.763 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:30 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:36:30.764 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:36:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:36:30] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-09T05:36:30.803 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/919544679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:36:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:30 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:36:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[184418]: pgmap v389: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 239 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 17207/11355 objects degraded (151.537%); 860 KiB/s, 54 objects/s recovering 2023-10-09T05:36:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[184418]: failed to encode map e148 with expected crc 2023-10-09T05:36:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[184418]: failed to encode map e148 with expected crc 2023-10-09T05:36:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:36:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[184418]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T05:36:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[184418]: 5.15 scrub starts 2023-10-09T05:36:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[191227]: pgmap v389: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 239 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 17207/11355 objects degraded (151.537%); 860 KiB/s, 54 objects/s recovering 2023-10-09T05:36:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[191227]: failed to encode map e148 with expected crc 2023-10-09T05:36:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[191227]: failed to encode map e148 with expected crc 2023-10-09T05:36:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:36:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[191227]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T05:36:31.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:31 smithi008 ceph-mon[191227]: 5.15 scrub starts 2023-10-09T05:36:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:31 smithi191 ceph-mon[161669]: pgmap v389: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 239 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 17207/11355 objects degraded (151.537%); 860 KiB/s, 54 objects/s recovering 2023-10-09T05:36:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:31 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:32.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:31 smithi191 ceph-mon[161669]: failed to encode map e148 with expected crc 2023-10-09T05:36:32.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:31 smithi191 ceph-mon[161669]: failed to encode map e148 with expected crc 2023-10-09T05:36:32.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:31 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:36:32.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:31 smithi191 ceph-mon[161669]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T05:36:32.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:31 smithi191 ceph-mon[161669]: 5.15 scrub starts 2023-10-09T05:36:32.470 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:32.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:32.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:32.513 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Removed 118133 objects 2023-10-09T05:36:32.513 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Clean up completed and total clean up time :129.966 2023-10-09T05:36:32.569 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:36:32.583 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:32.641 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:32.695 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:32.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:32.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:32.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:32.809 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:32 smithi008 ceph-mon[184418]: 5.13 scrub starts 2023-10-09T05:36:32.809 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:32 smithi008 ceph-mon[184418]: pgmap v391: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 17035/6123 objects degraded (278.213%); 0 B/s, 64 objects/s recovering 2023-10-09T05:36:32.809 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:32 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 17207/11355 objects degraded (151.537%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:32.809 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:32 smithi008 ceph-mon[191227]: 5.13 scrub starts 2023-10-09T05:36:32.810 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:32 smithi008 ceph-mon[191227]: pgmap v391: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 17035/6123 objects degraded (278.213%); 0 B/s, 64 objects/s recovering 2023-10-09T05:36:32.810 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:32 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 17207/11355 objects degraded (151.537%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:32.824 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:36:32.824 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:36:32.824 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-10-09T05:36:32.825 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:36:32.825 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:32.868 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:32.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:32.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:32.976 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:33.005 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:33.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:32 smithi191 ceph-mon[161669]: 5.13 scrub starts 2023-10-09T05:36:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:32 smithi191 ceph-mon[161669]: pgmap v391: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 17035/6123 objects degraded (278.213%); 0 B/s, 64 objects/s recovering 2023-10-09T05:36:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:32 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 17207/11355 objects degraded (151.537%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:33.511 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:33.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:33.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:33.615 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:33.643 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:33.748 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:33.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:33.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:33.772 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:33.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:33.856 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:33.880 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:33.922 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:33.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:33.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:34.029 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:34.055 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:34.098 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:36:34.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:34.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:34.205 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:34.224 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:36:34.231 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:34.248 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:36:34.270 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:36:34.275 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:34.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:34.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:34.291 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:36:34.309 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:36:34.331 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:36:34.352 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:36:34.371 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:36:34.373 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:36:34.373 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:36:34.373 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:36:34.373 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:36:34.373 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:36:34.373 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:36:34.373 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:36:34.373 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:36:34.380 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:34.405 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:34.449 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:34.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:34.478 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:36:34.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:34.584 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:34.627 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:34.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:34.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:34.732 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:34.760 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:35.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:35 smithi008 ceph-mon[184418]: pgmap v392: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 95 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 17035/4419 objects degraded (385.494%); 0 B/s, 64 objects/s recovering 2023-10-09T05:36:35.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2894267053' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:35.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:35 smithi008 ceph-mon[191227]: pgmap v392: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 95 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 17035/4419 objects degraded (385.494%); 0 B/s, 64 objects/s recovering 2023-10-09T05:36:35.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2894267053' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:35.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:35 smithi191 ceph-mon[161669]: pgmap v392: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 95 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 17035/4419 objects degraded (385.494%); 0 B/s, 64 objects/s recovering 2023-10-09T05:36:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2894267053' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:36.463 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:36.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:36.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:36.567 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:36.593 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:36.638 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:36.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:36.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:36.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:36.841 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:36.893 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:36.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:36.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:36.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:37.026 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:37.057 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:37 smithi008 ceph-mon[191227]: pgmap v393: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 53 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 17035/2409 objects degraded (707.140%); 0 B/s, 64 objects/s recovering 2023-10-09T05:36:37.069 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:37.076 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:37.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:37.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:37.183 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:37.214 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:37.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:37 smithi191 ceph-mon[161669]: pgmap v393: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 53 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 17035/2409 objects degraded (707.140%); 0 B/s, 64 objects/s recovering 2023-10-09T05:36:37.311 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:37.313 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:37 smithi008 ceph-mon[184418]: pgmap v393: 109 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 53 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 17035/2409 objects degraded (707.140%); 0 B/s, 64 objects/s recovering 2023-10-09T05:36:37.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:37.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:37.377 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:36:37.377 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:36:37.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:36:37.377 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:36:37.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.377 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:11044 5.0 5:010bec57:::benchmark_data_smithi008_238708_object11043:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-10-09T05:36:37.378 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:56.067946+0000", 2023-10-09T05:36:37.378 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 221.30725162600001, 2023-10-09T05:36:37.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.128621494, 2023-10-09T05:36:37.378 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.378 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.378 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.378 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.378 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.378 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 11044 2023-10-09T05:36:37.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.378 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.067946+0000", 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.067946+0000", 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.067944+0000", 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.067970+0000", 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6021999999999999e-05 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.067986+0000", 2023-10-09T05:36:37.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5608999999999999e-05 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.067995+0000", 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8380000000000008e-06 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.068036+0000", 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1425999999999999e-05 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.068124+0000", 2023-10-09T05:36:37.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7610999999999999e-05 2023-10-09T05:36:37.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.068220+0000", 2023-10-09T05:36:37.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6181000000000006e-05 2023-10-09T05:36:37.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.383 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.068883+0000", 2023-10-09T05:36:37.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00066317700000000004 2023-10-09T05:36:37.383 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.383 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.383 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.383 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.069271+0000", 2023-10-09T05:36:37.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000387983 2023-10-09T05:36:37.383 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.383 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.383 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.383 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.196461+0000", 2023-10-09T05:36:37.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.12719016399999999 2023-10-09T05:36:37.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.384 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.384 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.196514+0000", 2023-10-09T05:36:37.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2918999999999999e-05 2023-10-09T05:36:37.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.384 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.384 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:56.196567+0000", 2023-10-09T05:36:37.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3173000000000001e-05 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:18596 5.0 5:026789a9:::benchmark_data_smithi008_238708_object18595:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:58.964866+0000", 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 218.41033137700001, 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.11017399, 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.385 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 18596 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.964866+0000", 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.964866+0000", 2023-10-09T05:36:37.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.964865+0000", 2023-10-09T05:36:37.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:37.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.964891+0000", 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6444e-05 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.964937+0000", 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5766999999999997e-05 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.964947+0000", 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0485999999999999e-05 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.965039+0000", 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1211000000000005e-05 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.965188+0000", 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014963699999999999 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.965312+0000", 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012416599999999999 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.966251+0000", 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000938735 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.966963+0000", 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00071211699999999996 2023-10-09T05:36:37.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.074952+0000", 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.107988443 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.075003+0000", 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1069000000000002e-05 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.075040+0000", 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7357999999999997e-05 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:18612 5.0 5:0b9738a3:::benchmark_data_smithi008_238708_object18611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:58.970156+0000", 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 218.40504149, 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.105013942, 2023-10-09T05:36:37.392 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.393 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.393 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.393 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.393 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.393 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 18612 2023-10-09T05:36:37.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.393 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.393 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.393 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.393 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.970156+0000", 2023-10-09T05:36:37.393 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.970156+0000", 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.970155+0000", 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.970183+0000", 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8770000000000001e-05 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.970202+0000", 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8403999999999999e-05 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.970209+0000", 2023-10-09T05:36:37.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.542e-06 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.970255+0000", 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6008999999999999e-05 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.970390+0000", 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013473599999999999 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.970532+0000", 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000141447 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.971529+0000", 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00099698499999999993 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:58.973122+0000", 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001593592 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.075079+0000", 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.101957193 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.075129+0000", 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0033e-05 2023-10-09T05:36:37.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.075170+0000", 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0571999999999997e-05 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:19144 5.0 5:0930f39a:::benchmark_data_smithi008_238708_object19143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e109)", 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:59.248301+0000", 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 218.12689628800001, 2023-10-09T05:36:37.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13715340100000001, 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 19144 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248301+0000", 2023-10-09T05:36:37.400 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248301+0000", 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248297+0000", 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999957 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.401 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.402 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248322+0000", 2023-10-09T05:36:37.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5077999999999999e-05 2023-10-09T05:36:37.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.402 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248340+0000", 2023-10-09T05:36:37.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8292999999999999e-05 2023-10-09T05:36:37.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248347+0000", 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5969999999999997e-06 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248715+0000", 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000368585 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248844+0000", 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012819400000000001 2023-10-09T05:36:37.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.248988+0000", 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014417799999999999 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.250102+0000", 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011147360000000001 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.250690+0000", 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00058742199999999999 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.385378+0000", 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13468839099999999 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.385415+0000", 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6758999999999998e-05 2023-10-09T05:36:37.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.385455+0000", 2023-10-09T05:36:37.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9598999999999998e-05 2023-10-09T05:36:37.406 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.406 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.406 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.406 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:19155 5.a e109/105 5:585de1ce:::benchmark_data_smithi008_238708_object19154:head v 109'1151, mlcod=109'1149)", 2023-10-09T05:36:37.406 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:32:59.251720+0000", 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 218.123477318, 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.10559572, 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251720+0000", 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251720+0000", 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251719+0000", 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251747+0000", 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7464e-05 2023-10-09T05:36:37.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.409 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251757+0000", 2023-10-09T05:36:37.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0821e-05 2023-10-09T05:36:37.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.409 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251764+0000", 2023-10-09T05:36:37.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9e-06 2023-10-09T05:36:37.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251813+0000", 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8296e-05 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.251832+0000", 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9151e-05 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.357273+0000", 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.105440879 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:32:59.357316+0000", 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3334999999999998e-05 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:79653 5.0 5:06f9585e:::benchmark_data_smithi008_238708_object79652:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-10-09T05:36:37.411 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:42.359202+0000", 2023-10-09T05:36:37.412 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 175.01599529500001, 2023-10-09T05:36:37.412 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.10721328500000001, 2023-10-09T05:36:37.412 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.412 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.412 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.412 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.412 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.412 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 79653 2023-10-09T05:36:37.412 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.412 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.412 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359202+0000", 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359202+0000", 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359200+0000", 2023-10-09T05:36:37.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359239+0000", 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8302e-05 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359260+0000", 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1676999999999999e-05 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359270+0000", 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0079e-05 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359362+0000", 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1650999999999994e-05 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359504+0000", 2023-10-09T05:36:37.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000142132 2023-10-09T05:36:37.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359634+0000", 2023-10-09T05:36:37.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013003600000000001 2023-10-09T05:36:37.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.364793+0000", 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051586640000000003 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.365096+0000", 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00030324399999999999 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.466325+0000", 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.101229284 2023-10-09T05:36:37.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.466378+0000", 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2092000000000002e-05 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.466415+0000", 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7892000000000002e-05 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.418 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.419 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:79655 5.0 5:0c61b8d6:::benchmark_data_smithi008_238708_object79654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-10-09T05:36:37.419 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:42.359286+0000", 2023-10-09T05:36:37.419 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 175.01591117999999, 2023-10-09T05:36:37.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.10726487899999999, 2023-10-09T05:36:37.419 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.419 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.419 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.419 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.419 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 79655 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359286+0000", 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359286+0000", 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359286+0000", 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359306+0000", 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0336e-05 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.421 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.422 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359317+0000", 2023-10-09T05:36:37.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0504e-05 2023-10-09T05:36:37.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.422 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.422 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.359323+0000", 2023-10-09T05:36:37.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1589999999999998e-06 2023-10-09T05:36:37.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.422 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.422 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.362044+0000", 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027207849999999999 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.362169+0000", 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000125363 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.362287+0000", 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000118374 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.365087+0000", 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027999869999999999 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.365102+0000", 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4222e-05 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.466471+0000", 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.10136972399999999 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.466517+0000", 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5766999999999997e-05 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:42.466551+0000", 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4183999999999997e-05 2023-10-09T05:36:37.425 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.426 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.426 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.426 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.426 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.426 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98110 5.0 5:08529fda:::benchmark_data_smithi008_238708_object98109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T05:36:37.426 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.572265+0000", 2023-10-09T05:36:37.426 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 156.80293270799999, 2023-10-09T05:36:37.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23367422900000001, 2023-10-09T05:36:37.426 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.426 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:37.426 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98110 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.572265+0000", 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.427 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.572265+0000", 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.572262+0000", 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.572296+0000", 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3288999999999999e-05 2023-10-09T05:36:37.428 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.572313+0000", 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7238999999999999e-05 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.572323+0000", 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7249999999999992e-06 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.572376+0000", 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3211000000000003e-05 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.572514+0000", 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000137632 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.572629+0000", 2023-10-09T05:36:37.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011556 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.576867+0000", 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042381800000000002 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.579187+0000", 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002319644 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.431 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.805919+0000", 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.22673221800000001 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.805939+0000", 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9823e-05 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.432 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.34770.0:98110 5.0 e117/105)", 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.582408+0000", 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 156.79278983, 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.22376075300000001, 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582408+0000", 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.433 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.434 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.434 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.434 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582408+0000", 2023-10-09T05:36:37.434 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.434 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.434 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.434 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.434 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582407+0000", 2023-10-09T05:36:37.434 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:36:37.434 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.434 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582412+0000", 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9030000000000003e-06 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.652073+0000", 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.069661445000000002 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.652099+0000", 2023-10-09T05:36:37.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5746e-05 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.806142+0000", 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15404362399999999 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.806168+0000", 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5987e-05 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.436 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98125 5.0 5.4b6441c0 (undecoded) ondisk+write+known_if_redirected e117)", 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.673092+0000", 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 156.70210587299999, 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13311485200000001, 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.437 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98125 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673092+0000", 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673092+0000", 2023-10-09T05:36:37.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673090+0000", 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673119+0000", 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9352e-05 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673146+0000", 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6639999999999999e-05 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673158+0000", 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2303000000000001e-05 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.806196+0000", 2023-10-09T05:36:37.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13303847899999999 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.806206+0000", 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0105000000000001e-05 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98130 5.0 5.a2473e80 (undecoded) ondisk+write+known_if_redirected e117)", 2023-10-09T05:36:37.441 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.673174+0000", 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 156.70202385499999, 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13308687899999999, 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98130 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.442 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673174+0000", 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673174+0000", 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673173+0000", 2023-10-09T05:36:37.443 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673198+0000", 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5429999999999999e-05 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673207+0000", 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6219999999999998e-06 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.444 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.673214+0000", 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6100000000000002e-06 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.806251+0000", 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13303746799999999 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.806261+0000", 2023-10-09T05:36:37.445 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4190000000000008e-06 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98110 5.10 5:08529fda:::benchmark_data_smithi008_238708_object98109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695249+0000", 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 156.679948688, 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94265819399999995, 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:36:37.446 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98110 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695249+0000", 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.447 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695249+0000", 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695246+0000", 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695282+0000", 2023-10-09T05:36:37.448 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5592000000000001e-05 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695308+0000", 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6058999999999999e-05 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695320+0000", 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.219e-05 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.449 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.808026+0000", 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.112705826 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.808037+0000", 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0699999999999999e-05 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.450 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637823+0000", 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.829785785 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637907+0000", 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4371999999999998e-05 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98125 5.0 5:038226d2:::benchmark_data_smithi008_238708_object98124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:36:37.451 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695331+0000", 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 156.67986655499999, 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92727647599999996, 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98125 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.452 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.453 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695331+0000", 2023-10-09T05:36:37.453 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.453 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.453 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.453 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.453 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695331+0000", 2023-10-09T05:36:37.453 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.453 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.453 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.453 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.453 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695330+0000", 2023-10-09T05:36:37.454 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:37.454 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.454 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.454 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.454 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695357+0000", 2023-10-09T05:36:37.454 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6591999999999999e-05 2023-10-09T05:36:37.454 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.454 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.454 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.454 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695366+0000", 2023-10-09T05:36:37.454 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7390000000000005e-06 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695372+0000", 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6039999999999998e-06 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.806288+0000", 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.11091574899999999 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.455 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.806299+0000", 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1238e-05 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.594236+0000", 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.78793663899999999 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.594386+0000", 2023-10-09T05:36:37.456 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000149903 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.594543+0000", 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015728899999999999 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.600593+0000", 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0060496919999999997 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.457 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.458 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.458 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622331+0000", 2023-10-09T05:36:37.458 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021738370999999999 2023-10-09T05:36:37.458 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.458 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.458 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.458 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622485+0000", 2023-10-09T05:36:37.458 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015411099999999999 2023-10-09T05:36:37.458 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.458 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.458 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622558+0000", 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2414000000000002e-05 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622607+0000", 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9796000000000002e-05 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.459 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98130 5.0 5:017ce245:::benchmark_data_smithi008_238708_object98129:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695378+0000", 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 156.67981945700001, 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92738032100000001, 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98130 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.460 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695378+0000", 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695378+0000", 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.461 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695377+0000", 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695404+0000", 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6525000000000001e-05 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695412+0000", 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4139999999999994e-06 2023-10-09T05:36:37.462 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695419+0000", 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3419999999999996e-06 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.807845+0000", 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.11242577099999999 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.463 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.807859+0000", 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4579000000000001e-05 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.596508+0000", 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.78864880500000001 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.596655+0000", 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014741 2023-10-09T05:36:37.464 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.596818+0000", 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000163022 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.600599+0000", 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037810140000000001 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.465 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622648+0000", 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022048531999999999 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622680+0000", 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1863999999999998e-05 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622722+0000", 2023-10-09T05:36:37.466 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2454999999999999e-05 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622758+0000", 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6159999999999999e-05 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98138 5.0 5:0007661a:::benchmark_data_smithi008_238708_object98137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-10-09T05:36:37.467 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.818516+0000", 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 156.55668139400001, 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.80438677199999997, 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98138 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.468 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818516+0000", 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818516+0000", 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818514+0000", 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:36:37.469 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.470 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.470 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.470 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818552+0000", 2023-10-09T05:36:37.470 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8036e-05 2023-10-09T05:36:37.470 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.470 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.470 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.470 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818582+0000", 2023-10-09T05:36:37.470 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9819e-05 2023-10-09T05:36:37.471 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.471 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.471 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.471 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818596+0000", 2023-10-09T05:36:37.471 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4635e-05 2023-10-09T05:36:37.471 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.471 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.471 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.471 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818664+0000", 2023-10-09T05:36:37.471 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7669999999999994e-05 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818679+0000", 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4914e-05 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.598641+0000", 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77996221300000002 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.472 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.598805+0000", 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016343600000000001 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.598978+0000", 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017355699999999999 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.600790+0000", 2023-10-09T05:36:37.473 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001812255 2023-10-09T05:36:37.474 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.474 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.474 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.474 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622797+0000", 2023-10-09T05:36:37.474 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022006438 2023-10-09T05:36:37.474 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.474 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.474 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.474 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622829+0000", 2023-10-09T05:36:37.474 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1943000000000002e-05 2023-10-09T05:36:37.474 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.475 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.475 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.475 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622871+0000", 2023-10-09T05:36:37.475 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1844000000000001e-05 2023-10-09T05:36:37.475 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.475 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.475 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.475 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622903+0000", 2023-10-09T05:36:37.475 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2283999999999998e-05 2023-10-09T05:36:37.475 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.475 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119035 5.0 5:0296cb6a:::benchmark_data_smithi008_238708_object899:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.063473+0000", 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 133.311724653, 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.003481047, 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.476 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119035 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063473+0000", 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063473+0000", 2023-10-09T05:36:37.477 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063470+0000", 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063484+0000", 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4049000000000001e-05 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.478 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.479 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.479 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063506+0000", 2023-10-09T05:36:37.479 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2082999999999998e-05 2023-10-09T05:36:37.479 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.479 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.479 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.479 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063520+0000", 2023-10-09T05:36:37.479 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4065e-05 2023-10-09T05:36:37.479 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.479 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.479 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.189233+0000", 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.125712348 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.189249+0000", 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5798999999999999e-05 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.050621+0000", 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86137281600000004 2023-10-09T05:36:37.480 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.050748+0000", 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012712199999999999 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.050845+0000", 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6850999999999998e-05 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.481 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.482 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.054218+0000", 2023-10-09T05:36:37.482 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033727660000000001 2023-10-09T05:36:37.482 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.482 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.482 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.482 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.055747+0000", 2023-10-09T05:36:37.482 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015285539999999999 2023-10-09T05:36:37.482 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.482 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.482 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.482 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.066866+0000", 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011119052000000001 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.066917+0000", 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1672999999999998e-05 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.066954+0000", 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6520999999999998e-05 2023-10-09T05:36:37.483 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119040 5.0 5:00ca488a:::benchmark_data_smithi008_238708_object904:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.262978+0000", 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 133.11221916900001, 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.80409076700000004, 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.484 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119040 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.262978+0000", 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.485 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.262978+0000", 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.262976+0000", 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.262987+0000", 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0406e-05 2023-10-09T05:36:37.486 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.263013+0000", 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6182000000000001e-05 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.263024+0000", 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1025e-05 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.487 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.263087+0000", 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3063000000000003e-05 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.263102+0000", 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4628e-05 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.051901+0000", 2023-10-09T05:36:37.488 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.78879942700000005 2023-10-09T05:36:37.489 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.489 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.489 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.489 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.052001+0000", 2023-10-09T05:36:37.489 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9431000000000004e-05 2023-10-09T05:36:37.489 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.489 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.489 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.489 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.052098+0000", 2023-10-09T05:36:37.489 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7127999999999996e-05 2023-10-09T05:36:37.489 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.054225+0000", 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002127004 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.055779+0000", 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001553912 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.490 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.066989+0000", 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011210036 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.067034+0000", 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5071999999999998e-05 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.067069+0000", 2023-10-09T05:36:37.491 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5422000000000001e-05 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:137279 5.10 5:0930f39a:::benchmark_data_smithi008_238708_object19143:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:39.608379+0000", 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 117.76681810300001, 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.155474957, 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.492 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 137279 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.608379+0000", 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.493 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.608379+0000", 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.608377+0000", 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.608385+0000", 2023-10-09T05:36:37.494 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6960000000000005e-06 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.608401+0000", 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5438e-05 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.608417+0000", 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6529e-05 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.495 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.608475+0000", 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8125999999999999e-05 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.608600+0000", 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012425199999999999 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.496 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.608724+0000", 2023-10-09T05:36:37.497 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000124635 2023-10-09T05:36:37.497 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.497 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.497 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.497 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.611801+0000", 2023-10-09T05:36:37.497 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030772040000000001 2023-10-09T05:36:37.497 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.497 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.497 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.497 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.617886+0000", 2023-10-09T05:36:37.497 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0060844829999999999 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.763756+0000", 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.145870151 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.763813+0000", 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6756999999999999e-05 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.498 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:39.763854+0000", 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1594999999999997e-05 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:169537 5.0 5:01a48eda:::benchmark_data_smithi008_238708_object51401:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:35:13.663366+0000", 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 83.711831488000001, 2023-10-09T05:36:37.499 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.11827024799999999, 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 169537 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.663366+0000", 2023-10-09T05:36:37.500 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.663366+0000", 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.663364+0000", 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.501 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.663373+0000", 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4519999999999997e-06 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.663387+0000", 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3665e-05 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.502 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.663395+0000", 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6999999999999997e-06 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.663443+0000", 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7574000000000002e-05 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.663556+0000", 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000113114 2023-10-09T05:36:37.503 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.663668+0000", 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000112008 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.665253+0000", 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001585349 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.504 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.505 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.666472+0000", 2023-10-09T05:36:37.505 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001218537 2023-10-09T05:36:37.505 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.505 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.505 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:37.505 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.781543+0000", 2023-10-09T05:36:37.505 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.115070954 2023-10-09T05:36:37.505 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.505 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.505 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.505 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.781596+0000", 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2917000000000002e-05 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.781636+0000", 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0540999999999997e-05 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.506 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:169557 5.1a e137/123 5:5e5acb32:::benchmark_data_smithi008_238708_object51421:head v 137'8967, mlcod=137'8965)", 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:35:13.681257+0000", 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 83.693940706999996, 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.108709347, 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.681257+0000", 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.507 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.681257+0000", 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.681256+0000", 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.681266+0000", 2023-10-09T05:36:37.508 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0129000000000001e-05 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.781138+0000", 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.099871759000000004 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.781148+0000", 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0352e-05 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.509 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.781523+0000", 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00037501499999999997 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.781546+0000", 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2583e-05 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:36:37.510 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.789934+0000", 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0083883429999999995 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:13.789966+0000", 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2277000000000003e-05 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:36:37.511 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:36:37.512 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:37.541 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:38.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:38 smithi008 ceph-mon[184418]: 5.3 scrub starts 2023-10-09T05:36:38.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:38 smithi008 ceph-mon[184418]: failed to encode map e149 with expected crc 2023-10-09T05:36:38.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:38 smithi008 ceph-mon[184418]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T05:36:38.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:38 smithi008 ceph-mon[184418]: failed to encode map e149 with expected crc 2023-10-09T05:36:38.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 17035/2409 objects degraded (707.140%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:38 smithi008 ceph-mon[191227]: 5.3 scrub starts 2023-10-09T05:36:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:38 smithi008 ceph-mon[191227]: failed to encode map e149 with expected crc 2023-10-09T05:36:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:38 smithi008 ceph-mon[191227]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T05:36:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:38 smithi008 ceph-mon[191227]: failed to encode map e149 with expected crc 2023-10-09T05:36:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 17035/2409 objects degraded (707.140%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:38.267 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:36:38.269 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:36:38.335 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:36:38.446 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:38.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:38.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:38 smithi191 ceph-mon[161669]: 5.3 scrub starts 2023-10-09T05:36:38.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:38 smithi191 ceph-mon[161669]: failed to encode map e149 with expected crc 2023-10-09T05:36:38.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:38 smithi191 ceph-mon[161669]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T05:36:38.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:38 smithi191 ceph-mon[161669]: failed to encode map e149 with expected crc 2023-10-09T05:36:38.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 17035/2409 objects degraded (707.140%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:38.550 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:38.581 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:38.726 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:38.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:38.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:38.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:38.857 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:38.906 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:38.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:38.977 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:36:39.012 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:39.039 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:39.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[191227]: failed to encode map e149 with expected crc 2023-10-09T05:36:39.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[191227]: 5.23 scrub starts 2023-10-09T05:36:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[191227]: pgmap v395: 109 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 31 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 15763/1359 objects degraded (1159.897%); 0 B/s, 200 objects/s recovering 2023-10-09T05:36:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[191227]: from='client.35010 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:36:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[191227]: failed to encode map e150 with expected crc 2023-10-09T05:36:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[191227]: failed to encode map e150 with expected crc 2023-10-09T05:36:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[191227]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T05:36:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[191227]: failed to encode map e150 with expected crc 2023-10-09T05:36:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[184418]: failed to encode map e149 with expected crc 2023-10-09T05:36:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[184418]: 5.23 scrub starts 2023-10-09T05:36:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[184418]: pgmap v395: 109 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 31 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 15763/1359 objects degraded (1159.897%); 0 B/s, 200 objects/s recovering 2023-10-09T05:36:39.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[184418]: from='client.35010 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:36:39.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[184418]: failed to encode map e150 with expected crc 2023-10-09T05:36:39.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[184418]: failed to encode map e150 with expected crc 2023-10-09T05:36:39.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[184418]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T05:36:39.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:39 smithi008 ceph-mon[184418]: failed to encode map e150 with expected crc 2023-10-09T05:36:39.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:39 smithi191 ceph-mon[161669]: failed to encode map e149 with expected crc 2023-10-09T05:36:39.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:39 smithi191 ceph-mon[161669]: 5.23 scrub starts 2023-10-09T05:36:39.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:39 smithi191 ceph-mon[161669]: pgmap v395: 109 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 31 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 15763/1359 objects degraded (1159.897%); 0 B/s, 200 objects/s recovering 2023-10-09T05:36:39.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:39 smithi191 ceph-mon[161669]: from='client.35010 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:36:39.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:39 smithi191 ceph-mon[161669]: failed to encode map e150 with expected crc 2023-10-09T05:36:39.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:39 smithi191 ceph-mon[161669]: failed to encode map e150 with expected crc 2023-10-09T05:36:39.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:39 smithi191 ceph-mon[161669]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T05:36:39.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:39 smithi191 ceph-mon[161669]: failed to encode map e150 with expected crc 2023-10-09T05:36:39.359 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:39.603 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":395,"stamp":"2023-10-09T05:36:38.003675+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":32544703,"num_objects":453,"num_object_clones":0,"num_object_copies":1359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15763,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":453,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":187088,"num_write_kb":4625030,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10426,"num_bytes_recovered":377290752,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":352566,"ondisk_log_size":352566,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2085012,"kb_used_data":270932,"kb_used_omap":45,"kb_used_meta":1813970,"kb_avail":747876204,"statfs":{"total":767960285184,"available":765825232896,"internally_reserved":0,"allocated":277434368,"data_stored":272899910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":1857505571},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-218365964,"num_objects":-3332,"num_object_clones":0,"num_object_copies":-9996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1444,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3659,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1607,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002720"},"pg_stats":[{"pgid":"5.23","version":"148'8499","reported_seq":8655,"reported_epoch":149,"state":"active+clean+scrubbing","last_fresh":"2023-10-09T05:36:37.510988+0000","last_change":"2023-10-09T05:36:37.871012+0000","last_active":"2023-10-09T05:36:37.510988+0000","last_peered":"2023-10-09T05:36:37.510988+0000","last_clean":"2023-10-09T05:36:37.510988+0000","last_became_active":"2023-10-09T05:34:25.035930+0000","last_became_peered":"2023-10-09T05:34:25.035930+0000","last_unstale":"2023-10-09T05:36:37.510988+0000","last_undegraded":"2023-10-09T05:36:37.510988+0000","last_fullsized":"2023-10-09T05:36:37.510988+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"120'6300","last_scrub_stamp":"2023-10-09T05:34:08.749711+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:08.749711+0000","objects_scrubbed":3317,"log_size":8499,"log_dups_size":0,"ondisk_log_size":8499,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.61798832400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":884736,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2011,"num_write_kb":11712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.22","version":"148'8533","reported_seq":8617,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.233758+0000","last_change":"2023-10-09T05:34:40.876515+0000","last_active":"2023-10-09T05:36:32.233758+0000","last_peered":"2023-10-09T05:36:32.233758+0000","last_clean":"2023-10-09T05:36:32.233758+0000","last_became_active":"2023-10-09T05:34:25.030851+0000","last_became_peered":"2023-10-09T05:34:25.030851+0000","last_unstale":"2023-10-09T05:36:32.233758+0000","last_undegraded":"2023-10-09T05:36:32.233758+0000","last_fullsized":"2023-10-09T05:36:32.233758+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"127'7054","last_scrub_stamp":"2023-10-09T05:34:40.876434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:40.876434+0000","objects_scrubbed":0,"log_size":8533,"log_dups_size":0,"ondisk_log_size":8533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1479,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.21","version":"148'8530","reported_seq":11274,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.447869+0000","last_change":"2023-10-09T05:35:08.601492+0000","last_active":"2023-10-09T05:36:32.447869+0000","last_peered":"2023-10-09T05:36:32.447869+0000","last_clean":"2023-10-09T05:36:32.447869+0000","last_became_active":"2023-10-09T05:35:08.595030+0000","last_became_peered":"2023-10-09T05:35:08.595030+0000","last_unstale":"2023-10-09T05:36:32.447869+0000","last_undegraded":"2023-10-09T05:36:32.447869+0000","last_fullsized":"2023-10-09T05:36:32.447869+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8530,"log_dups_size":0,"ondisk_log_size":8530,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":638976,"num_objects":10,"num_object_clones":0,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3661,"num_write_kb":117456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1327,"num_bytes_recovered":65929216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.20","version":"148'8498","reported_seq":8687,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.276703+0000","last_change":"2023-10-09T05:34:38.457569+0000","last_active":"2023-10-09T05:36:32.276703+0000","last_peered":"2023-10-09T05:36:32.276703+0000","last_clean":"2023-10-09T05:36:32.276703+0000","last_became_active":"2023-10-09T05:34:25.062440+0000","last_became_peered":"2023-10-09T05:34:25.062440+0000","last_unstale":"2023-10-09T05:36:32.276703+0000","last_undegraded":"2023-10-09T05:36:32.276703+0000","last_fullsized":"2023-10-09T05:36:32.276703+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":6,"last_scrub":"125'6962","last_scrub_stamp":"2023-10-09T05:34:38.457405+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:38.457405+0000","objects_scrubbed":1537,"log_size":8498,"log_dups_size":0,"ondisk_log_size":8498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:57.174556+0000","scrub_duration":0.224312019,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1537,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1c","version":"148'11287","reported_seq":11425,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.438910+0000","last_change":"2023-10-09T05:34:39.445074+0000","last_active":"2023-10-09T05:36:32.438910+0000","last_peered":"2023-10-09T05:36:32.438910+0000","last_clean":"2023-10-09T05:36:32.438910+0000","last_became_active":"2023-10-09T05:34:25.054943+0000","last_became_peered":"2023-10-09T05:34:25.054943+0000","last_unstale":"2023-10-09T05:36:32.438910+0000","last_undegraded":"2023-10-09T05:36:32.438910+0000","last_fullsized":"2023-10-09T05:36:32.438910+0000","mapping_epoch":123,"log_start":"109'1110","ondisk_log_start":"109'1110","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'8137","last_scrub_stamp":"2023-10-09T05:34:39.444955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:39.444955+0000","objects_scrubbed":3164,"log_size":10177,"log_dups_size":0,"ondisk_log_size":10177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:30.213649+0000","scrub_duration":0.76575304399999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3164,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":131,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072663+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:36:19.072663+0000","last_peered":"2023-10-09T05:36:19.072663+0000","last_clean":"2023-10-09T05:36:19.072663+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:36:19.072663+0000","last_undegraded":"2023-10-09T05:36:19.072663+0000","last_fullsized":"2023-10-09T05:36:19.072663+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":119,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.680814+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:36:30.680814+0000","last_peered":"2023-10-09T05:36:30.680814+0000","last_clean":"2023-10-09T05:36:30.680814+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:36:30.680814+0000","last_undegraded":"2023-10-09T05:36:30.680814+0000","last_fullsized":"2023-10-09T05:36:30.680814+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1f","version":"148'11015","reported_seq":11075,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.272266+0000","last_change":"2023-10-09T05:34:25.044017+0000","last_active":"2023-10-09T05:36:32.272266+0000","last_peered":"2023-10-09T05:36:32.272266+0000","last_clean":"2023-10-09T05:36:32.272266+0000","last_became_active":"2023-10-09T05:34:25.037058+0000","last_became_peered":"2023-10-09T05:34:25.037058+0000","last_unstale":"2023-10-09T05:36:32.272266+0000","last_undegraded":"2023-10-09T05:36:32.272266+0000","last_fullsized":"2023-10-09T05:36:32.272266+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":11015,"log_dups_size":0,"ondisk_log_size":11015,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2129920,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7314,"num_write_kb":235072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":93,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.073441+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:36:19.073441+0000","last_peered":"2023-10-09T05:36:19.073441+0000","last_clean":"2023-10-09T05:36:19.073441+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:36:19.073441+0000","last_undegraded":"2023-10-09T05:36:19.073441+0000","last_fullsized":"2023-10-09T05:36:19.073441+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":115,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.677356+0000","last_change":"2023-10-09T05:34:41.662942+0000","last_active":"2023-10-09T05:36:30.677356+0000","last_peered":"2023-10-09T05:36:30.677356+0000","last_clean":"2023-10-09T05:36:30.677356+0000","last_became_active":"2023-10-09T05:34:41.662648+0000","last_became_peered":"2023-10-09T05:34:41.662648+0000","last_unstale":"2023-10-09T05:36:30.677356+0000","last_undegraded":"2023-10-09T05:36:30.677356+0000","last_fullsized":"2023-10-09T05:36:30.677356+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:08.691549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1e","version":"148'11120","reported_seq":11178,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.512412+0000","last_change":"2023-10-09T05:34:25.039990+0000","last_active":"2023-10-09T05:36:32.512412+0000","last_peered":"2023-10-09T05:36:32.512412+0000","last_clean":"2023-10-09T05:36:32.512412+0000","last_became_active":"2023-10-09T05:34:25.036888+0000","last_became_peered":"2023-10-09T05:34:25.036888+0000","last_unstale":"2023-10-09T05:36:32.512412+0000","last_undegraded":"2023-10-09T05:36:32.512412+0000","last_fullsized":"2023-10-09T05:36:32.512412+0000","mapping_epoch":123,"log_start":"111'2929","ondisk_log_start":"111'2929","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8191,"log_dups_size":0,"ondisk_log_size":8191,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3244032,"num_objects":50,"num_object_clones":0,"num_object_copies":150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7381,"num_write_kb":237760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":135,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:21.010131+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:36:21.010131+0000","last_peered":"2023-10-09T05:36:21.010131+0000","last_clean":"2023-10-09T05:36:21.010131+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:36:21.010131+0000","last_undegraded":"2023-10-09T05:36:21.010131+0000","last_fullsized":"2023-10-09T05:36:21.010131+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":114,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072566+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:36:19.072566+0000","last_peered":"2023-10-09T05:36:19.072566+0000","last_clean":"2023-10-09T05:36:19.072566+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:36:19.072566+0000","last_undegraded":"2023-10-09T05:36:19.072566+0000","last_fullsized":"2023-10-09T05:36:19.072566+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.11","version":"148'10335","reported_seq":10407,"reported_epoch":148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:32.433916+0000","last_change":"2023-10-09T05:34:42.977328+0000","last_active":"2023-10-09T05:36:32.433916+0000","last_peered":"2023-10-09T05:36:32.433916+0000","last_clean":"2023-10-09T05:33:20.516527+0000","last_became_active":"2023-10-09T05:34:42.969107+0000","last_became_peered":"2023-10-09T05:34:42.969107+0000","last_unstale":"2023-10-09T05:36:32.433916+0000","last_undegraded":"2023-10-09T05:34:42.867828+0000","last_fullsized":"2023-10-09T05:34:42.853589+0000","mapping_epoch":128,"log_start":"111'2862","ondisk_log_start":"111'2862","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":7473,"log_dups_size":0,"ondisk_log_size":7473,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2627,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7318,"num_write_kb":233632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":-1979}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":99,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.677757+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:36:30.677757+0000","last_peered":"2023-10-09T05:36:30.677757+0000","last_clean":"2023-10-09T05:36:30.677757+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:36:30.677757+0000","last_undegraded":"2023-10-09T05:36:30.677757+0000","last_fullsized":"2023-10-09T05:36:30.677757+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":119,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.184359+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:36:19.184359+0000","last_peered":"2023-10-09T05:36:19.184359+0000","last_clean":"2023-10-09T05:36:19.184359+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:36:19.184359+0000","last_undegraded":"2023-10-09T05:36:19.184359+0000","last_fullsized":"2023-10-09T05:36:19.184359+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.10","version":"148'10246","reported_seq":10384,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.351696+0000","last_change":"2023-10-09T05:34:06.833835+0000","last_active":"2023-10-09T05:36:32.351696+0000","last_peered":"2023-10-09T05:36:32.351696+0000","last_clean":"2023-10-09T05:36:32.351696+0000","last_became_active":"2023-10-09T05:34:01.635329+0000","last_became_peered":"2023-10-09T05:34:01.635329+0000","last_unstale":"2023-10-09T05:36:32.351696+0000","last_undegraded":"2023-10-09T05:36:32.351696+0000","last_fullsized":"2023-10-09T05:36:32.351696+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6258","last_scrub_stamp":"2023-10-09T05:34:06.833681+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:06.833681+0000","objects_scrubbed":3155,"log_size":10246,"log_dups_size":0,"ondisk_log_size":10246,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:45.971615+0000","scrub_duration":0.69237216700000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3995,"num_write_kb":26880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":130,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.070700+0000","last_change":"2023-10-09T05:34:41.683934+0000","last_active":"2023-10-09T05:36:19.070700+0000","last_peered":"2023-10-09T05:36:19.070700+0000","last_clean":"2023-10-09T05:36:19.070700+0000","last_became_active":"2023-10-09T05:34:41.683618+0000","last_became_peered":"2023-10-09T05:34:41.683618+0000","last_unstale":"2023-10-09T05:36:19.070700+0000","last_undegraded":"2023-10-09T05:36:19.070700+0000","last_fullsized":"2023-10-09T05:36:19.070700+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":124,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.185348+0000","last_change":"2023-10-09T05:34:41.668251+0000","last_active":"2023-10-09T05:36:19.185348+0000","last_peered":"2023-10-09T05:36:19.185348+0000","last_clean":"2023-10-09T05:36:19.185348+0000","last_became_active":"2023-10-09T05:34:41.668017+0000","last_became_peered":"2023-10-09T05:34:41.668017+0000","last_unstale":"2023-10-09T05:36:19.185348+0000","last_undegraded":"2023-10-09T05:36:19.185348+0000","last_fullsized":"2023-10-09T05:36:19.185348+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:27.235656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.13","version":"148'10302","reported_seq":10371,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.480424+0000","last_change":"2023-10-09T05:36:32.043543+0000","last_active":"2023-10-09T05:36:32.480424+0000","last_peered":"2023-10-09T05:36:32.480424+0000","last_clean":"2023-10-09T05:36:32.480424+0000","last_became_active":"2023-10-09T05:34:01.625092+0000","last_became_peered":"2023-10-09T05:34:01.625092+0000","last_unstale":"2023-10-09T05:36:32.480424+0000","last_undegraded":"2023-10-09T05:36:32.480424+0000","last_fullsized":"2023-10-09T05:36:32.480424+0000","mapping_epoch":118,"log_start":"109'799","ondisk_log_start":"109'799","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9503,"log_dups_size":0,"ondisk_log_size":9503,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2949120,"num_objects":45,"num_object_clones":0,"num_object_copies":135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":45,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7281,"num_write_kb":234432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":107,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.074784+0000","last_change":"2023-10-09T05:34:41.682874+0000","last_active":"2023-10-09T05:36:19.074784+0000","last_peered":"2023-10-09T05:36:19.074784+0000","last_clean":"2023-10-09T05:36:19.074784+0000","last_became_active":"2023-10-09T05:34:41.674789+0000","last_became_peered":"2023-10-09T05:34:41.674789+0000","last_unstale":"2023-10-09T05:36:19.074784+0000","last_undegraded":"2023-10-09T05:36:19.074784+0000","last_fullsized":"2023-10-09T05:36:19.074784+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":93,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.147978+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:36:19.147978+0000","last_peered":"2023-10-09T05:36:19.147978+0000","last_clean":"2023-10-09T05:36:19.147978+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:36:19.147978+0000","last_undegraded":"2023-10-09T05:36:19.147978+0000","last_fullsized":"2023-10-09T05:36:19.147978+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.12","version":"148'10543","reported_seq":10608,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.411417+0000","last_change":"2023-10-09T05:34:06.470865+0000","last_active":"2023-10-09T05:36:32.411417+0000","last_peered":"2023-10-09T05:36:32.411417+0000","last_clean":"2023-10-09T05:36:32.411417+0000","last_became_active":"2023-10-09T05:34:01.624493+0000","last_became_peered":"2023-10-09T05:34:01.624493+0000","last_unstale":"2023-10-09T05:36:32.411417+0000","last_undegraded":"2023-10-09T05:36:32.411417+0000","last_fullsized":"2023-10-09T05:36:32.411417+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6331","last_scrub_stamp":"2023-10-09T05:34:06.470790+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:06.470790+0000","objects_scrubbed":0,"log_size":10543,"log_dups_size":0,"ondisk_log_size":10543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4224,"num_write_kb":30848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":137,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.073493+0000","last_change":"2023-10-09T05:34:41.683953+0000","last_active":"2023-10-09T05:36:19.073493+0000","last_peered":"2023-10-09T05:36:19.073493+0000","last_clean":"2023-10-09T05:36:19.073493+0000","last_became_active":"2023-10-09T05:34:41.683519+0000","last_became_peered":"2023-10-09T05:34:41.683519+0000","last_unstale":"2023-10-09T05:36:19.073493+0000","last_undegraded":"2023-10-09T05:36:19.073493+0000","last_fullsized":"2023-10-09T05:36:19.073493+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":114,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072794+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:36:19.072794+0000","last_peered":"2023-10-09T05:36:19.072794+0000","last_clean":"2023-10-09T05:36:19.072794+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:36:19.072794+0000","last_undegraded":"2023-10-09T05:36:19.072794+0000","last_fullsized":"2023-10-09T05:36:19.072794+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.15","version":"148'10681","reported_seq":10750,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.478728+0000","last_change":"2023-10-09T05:36:30.907357+0000","last_active":"2023-10-09T05:36:32.478728+0000","last_peered":"2023-10-09T05:36:32.478728+0000","last_clean":"2023-10-09T05:36:32.478728+0000","last_became_active":"2023-10-09T05:34:01.623738+0000","last_became_peered":"2023-10-09T05:34:01.623738+0000","last_unstale":"2023-10-09T05:36:32.478728+0000","last_undegraded":"2023-10-09T05:36:32.478728+0000","last_fullsized":"2023-10-09T05:36:32.478728+0000","mapping_epoch":118,"log_start":"109'1580","ondisk_log_start":"109'1580","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":9101,"log_dups_size":0,"ondisk_log_size":9101,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7556,"num_write_kb":239680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":101,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.074366+0000","last_change":"2023-10-09T05:34:41.673944+0000","last_active":"2023-10-09T05:36:19.074366+0000","last_peered":"2023-10-09T05:36:19.074366+0000","last_clean":"2023-10-09T05:36:19.074366+0000","last_became_active":"2023-10-09T05:34:41.673595+0000","last_became_peered":"2023-10-09T05:34:41.673595+0000","last_unstale":"2023-10-09T05:36:19.074366+0000","last_undegraded":"2023-10-09T05:36:19.074366+0000","last_fullsized":"2023-10-09T05:36:19.074366+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":134,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.184495+0000","last_change":"2023-10-09T05:34:41.667462+0000","last_active":"2023-10-09T05:36:19.184495+0000","last_peered":"2023-10-09T05:36:19.184495+0000","last_clean":"2023-10-09T05:36:19.184495+0000","last_became_active":"2023-10-09T05:34:41.667140+0000","last_became_peered":"2023-10-09T05:34:41.667140+0000","last_unstale":"2023-10-09T05:36:19.184495+0000","last_undegraded":"2023-10-09T05:36:19.184495+0000","last_fullsized":"2023-10-09T05:36:19.184495+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:52.888470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.14","version":"148'10293","reported_seq":10363,"reported_epoch":148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:32.513397+0000","last_change":"2023-10-09T05:34:42.906108+0000","last_active":"2023-10-09T05:36:32.513397+0000","last_peered":"2023-10-09T05:36:32.513397+0000","last_clean":"2023-10-09T05:33:20.493128+0000","last_became_active":"2023-10-09T05:34:42.894931+0000","last_became_peered":"2023-10-09T05:34:42.894931+0000","last_unstale":"2023-10-09T05:36:32.513397+0000","last_undegraded":"2023-10-09T05:34:42.686528+0000","last_fullsized":"2023-10-09T05:34:42.658194+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10293,"log_dups_size":0,"ondisk_log_size":10293,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1933312,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2585,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7264,"num_write_kb":233376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-1947}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":114,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.676772+0000","last_change":"2023-10-09T05:34:41.662790+0000","last_active":"2023-10-09T05:36:30.676772+0000","last_peered":"2023-10-09T05:36:30.676772+0000","last_clean":"2023-10-09T05:36:30.676772+0000","last_became_active":"2023-10-09T05:34:41.662513+0000","last_became_peered":"2023-10-09T05:34:41.662513+0000","last_unstale":"2023-10-09T05:36:30.676772+0000","last_undegraded":"2023-10-09T05:36:30.676772+0000","last_fullsized":"2023-10-09T05:36:30.676772+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:17.781371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":129,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.070019+0000","last_change":"2023-10-09T05:34:41.694695+0000","last_active":"2023-10-09T05:36:19.070019+0000","last_peered":"2023-10-09T05:36:19.070019+0000","last_clean":"2023-10-09T05:36:19.070019+0000","last_became_active":"2023-10-09T05:34:41.694408+0000","last_became_peered":"2023-10-09T05:34:41.694408+0000","last_unstale":"2023-10-09T05:36:19.070019+0000","last_undegraded":"2023-10-09T05:36:19.070019+0000","last_fullsized":"2023-10-09T05:36:19.070019+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.17","version":"148'10505","reported_seq":10570,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.442618+0000","last_change":"2023-10-09T05:34:37.421122+0000","last_active":"2023-10-09T05:36:32.442618+0000","last_peered":"2023-10-09T05:36:32.442618+0000","last_clean":"2023-10-09T05:36:32.442618+0000","last_became_active":"2023-10-09T05:34:01.618906+0000","last_became_peered":"2023-10-09T05:34:01.618906+0000","last_unstale":"2023-10-09T05:36:32.442618+0000","last_undegraded":"2023-10-09T05:36:32.442618+0000","last_fullsized":"2023-10-09T05:36:32.442618+0000","mapping_epoch":118,"log_start":"108'261","ondisk_log_start":"108'261","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"125'7322","last_scrub_stamp":"2023-10-09T05:34:37.421020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:37.421020+0000","objects_scrubbed":0,"log_size":10244,"log_dups_size":0,"ondisk_log_size":10244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3200,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":84,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.074301+0000","last_change":"2023-10-09T05:34:41.702205+0000","last_active":"2023-10-09T05:36:19.074301+0000","last_peered":"2023-10-09T05:36:19.074301+0000","last_clean":"2023-10-09T05:36:19.074301+0000","last_became_active":"2023-10-09T05:34:41.701849+0000","last_became_peered":"2023-10-09T05:34:41.701849+0000","last_unstale":"2023-10-09T05:36:19.074301+0000","last_undegraded":"2023-10-09T05:36:19.074301+0000","last_fullsized":"2023-10-09T05:36:19.074301+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":114,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.677281+0000","last_change":"2023-10-09T05:34:41.663111+0000","last_active":"2023-10-09T05:36:30.677281+0000","last_peered":"2023-10-09T05:36:30.677281+0000","last_clean":"2023-10-09T05:36:30.677281+0000","last_became_active":"2023-10-09T05:34:41.662816+0000","last_became_peered":"2023-10-09T05:34:41.662816+0000","last_unstale":"2023-10-09T05:36:30.677281+0000","last_undegraded":"2023-10-09T05:36:30.677281+0000","last_fullsized":"2023-10-09T05:36:30.677281+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:36.226743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.16","version":"148'10453","reported_seq":15707,"reported_epoch":148,"state":"active+undersized+remapped","last_fresh":"2023-10-09T05:36:36.087047+0000","last_change":"2023-10-09T05:36:36.087285+0000","last_active":"2023-10-09T05:36:36.087047+0000","last_peered":"2023-10-09T05:36:36.087047+0000","last_clean":"2023-10-09T05:33:20.529830+0000","last_became_active":"2023-10-09T05:34:42.891787+0000","last_became_peered":"2023-10-09T05:34:42.891787+0000","last_unstale":"2023-10-09T05:36:36.087047+0000","last_undegraded":"2023-10-09T05:34:42.686663+0000","last_fullsized":"2023-10-09T05:34:42.658538+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10453,"log_dups_size":0,"ondisk_log_size":10453,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7385,"num_write_kb":236416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2590,"num_bytes_recovered":34930688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5","objects":3}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":123,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.071711+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:36:19.071711+0000","last_peered":"2023-10-09T05:36:19.071711+0000","last_clean":"2023-10-09T05:36:19.071711+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:36:19.071711+0000","last_undegraded":"2023-10-09T05:36:19.071711+0000","last_fullsized":"2023-10-09T05:36:19.071711+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":134,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:21.010240+0000","last_change":"2023-10-09T05:34:41.664294+0000","last_active":"2023-10-09T05:36:21.010240+0000","last_peered":"2023-10-09T05:36:21.010240+0000","last_clean":"2023-10-09T05:36:21.010240+0000","last_became_active":"2023-10-09T05:34:41.663985+0000","last_became_peered":"2023-10-09T05:34:41.663985+0000","last_unstale":"2023-10-09T05:36:21.010240+0000","last_undegraded":"2023-10-09T05:36:21.010240+0000","last_fullsized":"2023-10-09T05:36:21.010240+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:58.417307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"148'10421","reported_seq":10491,"reported_epoch":148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:32.476126+0000","last_change":"2023-10-09T05:34:42.977224+0000","last_active":"2023-10-09T05:36:32.476126+0000","last_peered":"2023-10-09T05:36:32.476126+0000","last_clean":"2023-10-09T05:33:20.515228+0000","last_became_active":"2023-10-09T05:34:42.967706+0000","last_became_peered":"2023-10-09T05:34:42.967706+0000","last_unstale":"2023-10-09T05:36:32.476126+0000","last_undegraded":"2023-10-09T05:34:42.852684+0000","last_fullsized":"2023-10-09T05:34:42.677882+0000","mapping_epoch":128,"log_start":"111'3017","ondisk_log_start":"111'3017","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2614,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7361,"num_write_kb":234176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":-2003}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":90,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.074960+0000","last_change":"2023-10-09T05:34:41.688229+0000","last_active":"2023-10-09T05:36:19.074960+0000","last_peered":"2023-10-09T05:36:19.074960+0000","last_clean":"2023-10-09T05:36:19.074960+0000","last_became_active":"2023-10-09T05:34:41.687795+0000","last_became_peered":"2023-10-09T05:34:41.687795+0000","last_unstale":"2023-10-09T05:36:19.074960+0000","last_undegraded":"2023-10-09T05:36:19.074960+0000","last_fullsized":"2023-10-09T05:36:19.074960+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":114,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.069779+0000","last_change":"2023-10-09T05:34:41.678569+0000","last_active":"2023-10-09T05:36:19.069779+0000","last_peered":"2023-10-09T05:36:19.069779+0000","last_clean":"2023-10-09T05:36:19.069779+0000","last_became_active":"2023-10-09T05:34:41.678271+0000","last_became_peered":"2023-10-09T05:34:41.678271+0000","last_unstale":"2023-10-09T05:36:19.069779+0000","last_undegraded":"2023-10-09T05:36:19.069779+0000","last_fullsized":"2023-10-09T05:36:19.069779+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"148'13046","reported_seq":13397,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.473838+0000","last_change":"2023-10-09T05:34:25.139244+0000","last_active":"2023-10-09T05:36:32.473838+0000","last_peered":"2023-10-09T05:36:32.473838+0000","last_clean":"2023-10-09T05:36:32.473838+0000","last_became_active":"2023-10-09T05:34:01.629769+0000","last_became_peered":"2023-10-09T05:34:01.629769+0000","last_unstale":"2023-10-09T05:36:32.473838+0000","last_undegraded":"2023-10-09T05:36:32.473838+0000","last_fullsized":"2023-10-09T05:36:32.473838+0000","mapping_epoch":105,"log_start":"108'438","ondisk_log_start":"108'438","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":3497,"log_size":12608,"log_dups_size":0,"ondisk_log_size":12608,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3899392,"num_objects":59,"num_object_clones":0,"num_object_copies":177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7349,"num_write_kb":237088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":96,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.075029+0000","last_change":"2023-10-09T05:34:41.674752+0000","last_active":"2023-10-09T05:36:19.075029+0000","last_peered":"2023-10-09T05:36:19.075029+0000","last_clean":"2023-10-09T05:36:19.075029+0000","last_became_active":"2023-10-09T05:34:41.674253+0000","last_became_peered":"2023-10-09T05:34:41.674253+0000","last_unstale":"2023-10-09T05:36:19.075029+0000","last_undegraded":"2023-10-09T05:36:19.075029+0000","last_fullsized":"2023-10-09T05:36:19.075029+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":129,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072456+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:36:19.072456+0000","last_peered":"2023-10-09T05:36:19.072456+0000","last_clean":"2023-10-09T05:36:19.072456+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:36:19.072456+0000","last_undegraded":"2023-10-09T05:36:19.072456+0000","last_fullsized":"2023-10-09T05:36:19.072456+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"148'11250","reported_seq":11314,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.505465+0000","last_change":"2023-10-09T05:34:42.657980+0000","last_active":"2023-10-09T05:36:32.505465+0000","last_peered":"2023-10-09T05:36:32.505465+0000","last_clean":"2023-10-09T05:36:32.505465+0000","last_became_active":"2023-10-09T05:34:25.039138+0000","last_became_peered":"2023-10-09T05:34:25.039138+0000","last_unstale":"2023-10-09T05:36:32.505465+0000","last_undegraded":"2023-10-09T05:36:32.505465+0000","last_fullsized":"2023-10-09T05:36:32.505465+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":11250,"log_dups_size":0,"ondisk_log_size":11250,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7512,"num_write_kb":237600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":87,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.074940+0000","last_change":"2023-10-09T05:34:41.691449+0000","last_active":"2023-10-09T05:36:19.074940+0000","last_peered":"2023-10-09T05:36:19.074940+0000","last_clean":"2023-10-09T05:36:19.074940+0000","last_became_active":"2023-10-09T05:34:41.675227+0000","last_became_peered":"2023-10-09T05:34:41.675227+0000","last_unstale":"2023-10-09T05:36:19.074940+0000","last_undegraded":"2023-10-09T05:36:19.074940+0000","last_fullsized":"2023-10-09T05:36:19.074940+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":138,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072664+0000","last_change":"2023-10-09T05:34:41.669908+0000","last_active":"2023-10-09T05:36:19.072664+0000","last_peered":"2023-10-09T05:36:19.072664+0000","last_clean":"2023-10-09T05:36:19.072664+0000","last_became_active":"2023-10-09T05:34:41.669596+0000","last_became_peered":"2023-10-09T05:34:41.669596+0000","last_unstale":"2023-10-09T05:36:19.072664+0000","last_undegraded":"2023-10-09T05:36:19.072664+0000","last_fullsized":"2023-10-09T05:36:19.072664+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"148'11110","reported_seq":11249,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.440567+0000","last_change":"2023-10-09T05:34:35.674328+0000","last_active":"2023-10-09T05:36:32.440567+0000","last_peered":"2023-10-09T05:36:32.440567+0000","last_clean":"2023-10-09T05:36:32.440567+0000","last_became_active":"2023-10-09T05:34:25.008155+0000","last_became_peered":"2023-10-09T05:34:25.008155+0000","last_unstale":"2023-10-09T05:36:32.440567+0000","last_undegraded":"2023-10-09T05:36:32.440567+0000","last_fullsized":"2023-10-09T05:36:32.440567+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7846","last_scrub_stamp":"2023-10-09T05:34:35.674093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:35.674093+0000","objects_scrubbed":3275,"log_size":11110,"log_dups_size":0,"ondisk_log_size":11110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:02.820818+0000","scrub_duration":0.75420257000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3275,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":115,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.070320+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:36:19.070320+0000","last_peered":"2023-10-09T05:36:19.070320+0000","last_clean":"2023-10-09T05:36:19.070320+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:36:19.070320+0000","last_undegraded":"2023-10-09T05:36:19.070320+0000","last_fullsized":"2023-10-09T05:36:19.070320+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":99,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.677424+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:36:30.677424+0000","last_peered":"2023-10-09T05:36:30.677424+0000","last_clean":"2023-10-09T05:36:30.677424+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:36:30.677424+0000","last_undegraded":"2023-10-09T05:36:30.677424+0000","last_fullsized":"2023-10-09T05:36:30.677424+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"148'11139","reported_seq":16409,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.477693+0000","last_change":"2023-10-09T05:35:43.652202+0000","last_active":"2023-10-09T05:36:32.477693+0000","last_peered":"2023-10-09T05:36:32.477693+0000","last_clean":"2023-10-09T05:36:32.477693+0000","last_became_active":"2023-10-09T05:35:43.630210+0000","last_became_peered":"2023-10-09T05:35:43.630210+0000","last_unstale":"2023-10-09T05:36:32.477693+0000","last_undegraded":"2023-10-09T05:36:32.477693+0000","last_fullsized":"2023-10-09T05:36:32.477693+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":11139,"log_dups_size":0,"ondisk_log_size":11139,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2293772,"num_objects":35,"num_object_clones":0,"num_object_copies":105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7396,"num_write_kb":237792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2592,"num_bytes_recovered":128319488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":117,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.184105+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:36:19.184105+0000","last_peered":"2023-10-09T05:36:19.184105+0000","last_clean":"2023-10-09T05:36:19.184105+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:36:19.184105+0000","last_undegraded":"2023-10-09T05:36:19.184105+0000","last_fullsized":"2023-10-09T05:36:19.184105+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":114,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.073533+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:36:19.073533+0000","last_peered":"2023-10-09T05:36:19.073533+0000","last_clean":"2023-10-09T05:36:19.073533+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:36:19.073533+0000","last_undegraded":"2023-10-09T05:36:19.073533+0000","last_fullsized":"2023-10-09T05:36:19.073533+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"148'11193","reported_seq":11331,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.493432+0000","last_change":"2023-10-09T05:34:40.457466+0000","last_active":"2023-10-09T05:36:32.493432+0000","last_peered":"2023-10-09T05:36:32.493432+0000","last_clean":"2023-10-09T05:36:32.493432+0000","last_became_active":"2023-10-09T05:34:25.026725+0000","last_became_peered":"2023-10-09T05:34:25.026725+0000","last_unstale":"2023-10-09T05:36:32.493432+0000","last_undegraded":"2023-10-09T05:36:32.493432+0000","last_fullsized":"2023-10-09T05:36:32.493432+0000","mapping_epoch":105,"log_start":"109'865","ondisk_log_start":"109'865","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'8107","last_scrub_stamp":"2023-10-09T05:34:40.457321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:40.457321+0000","objects_scrubbed":3100,"log_size":10328,"log_dups_size":0,"ondisk_log_size":10328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:26.084493+0000","scrub_duration":0.74267715700000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3100,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":115,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.073418+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:36:19.073418+0000","last_peered":"2023-10-09T05:36:19.073418+0000","last_clean":"2023-10-09T05:36:19.073418+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:36:19.073418+0000","last_undegraded":"2023-10-09T05:36:19.073418+0000","last_fullsized":"2023-10-09T05:36:19.073418+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":129,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072976+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:36:19.072976+0000","last_peered":"2023-10-09T05:36:19.072976+0000","last_clean":"2023-10-09T05:36:19.072976+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:36:19.072976+0000","last_undegraded":"2023-10-09T05:36:19.072976+0000","last_fullsized":"2023-10-09T05:36:19.072976+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"148'11080","reported_seq":11143,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.433732+0000","last_change":"2023-10-09T05:34:39.803946+0000","last_active":"2023-10-09T05:36:32.433732+0000","last_peered":"2023-10-09T05:36:32.433732+0000","last_clean":"2023-10-09T05:36:32.433732+0000","last_became_active":"2023-10-09T05:34:25.018175+0000","last_became_peered":"2023-10-09T05:34:25.018175+0000","last_unstale":"2023-10-09T05:36:32.433732+0000","last_undegraded":"2023-10-09T05:36:32.433732+0000","last_fullsized":"2023-10-09T05:36:32.433732+0000","mapping_epoch":105,"log_start":"108'172","ondisk_log_start":"108'172","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10908,"log_dups_size":0,"ondisk_log_size":10908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7378,"num_write_kb":235072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":130,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072381+0000","last_change":"2023-10-09T05:34:41.706409+0000","last_active":"2023-10-09T05:36:19.072381+0000","last_peered":"2023-10-09T05:36:19.072381+0000","last_clean":"2023-10-09T05:36:19.072381+0000","last_became_active":"2023-10-09T05:34:41.706002+0000","last_became_peered":"2023-10-09T05:34:41.706002+0000","last_unstale":"2023-10-09T05:36:19.072381+0000","last_undegraded":"2023-10-09T05:36:19.072381+0000","last_fullsized":"2023-10-09T05:36:19.072381+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":123,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.073340+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:36:19.073340+0000","last_peered":"2023-10-09T05:36:19.073340+0000","last_clean":"2023-10-09T05:36:19.073340+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:36:19.073340+0000","last_undegraded":"2023-10-09T05:36:19.073340+0000","last_fullsized":"2023-10-09T05:36:19.073340+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"148'11219","reported_seq":11285,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.485659+0000","last_change":"2023-10-09T05:34:42.153027+0000","last_active":"2023-10-09T05:36:32.485659+0000","last_peered":"2023-10-09T05:36:32.485659+0000","last_clean":"2023-10-09T05:36:32.485659+0000","last_became_active":"2023-10-09T05:34:25.031927+0000","last_became_peered":"2023-10-09T05:34:25.031927+0000","last_unstale":"2023-10-09T05:36:32.485659+0000","last_undegraded":"2023-10-09T05:36:32.485659+0000","last_fullsized":"2023-10-09T05:36:32.485659+0000","mapping_epoch":105,"log_start":"111'3254","ondisk_log_start":"111'3254","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'8112","last_scrub_stamp":"2023-10-09T05:34:42.152946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:42.152946+0000","objects_scrubbed":0,"log_size":7965,"log_dups_size":0,"ondisk_log_size":7965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3107,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":135,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.184195+0000","last_change":"2023-10-09T05:34:41.666221+0000","last_active":"2023-10-09T05:36:19.184195+0000","last_peered":"2023-10-09T05:36:19.184195+0000","last_clean":"2023-10-09T05:36:19.184195+0000","last_became_active":"2023-10-09T05:34:41.665842+0000","last_became_peered":"2023-10-09T05:34:41.665842+0000","last_unstale":"2023-10-09T05:36:19.184195+0000","last_undegraded":"2023-10-09T05:36:19.184195+0000","last_fullsized":"2023-10-09T05:36:19.184195+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:28.435851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":93,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.154009+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:36:19.154009+0000","last_peered":"2023-10-09T05:36:19.154009+0000","last_clean":"2023-10-09T05:36:19.154009+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:36:19.154009+0000","last_undegraded":"2023-10-09T05:36:19.154009+0000","last_fullsized":"2023-10-09T05:36:19.154009+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":672,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.149980+0000","last_change":"2023-10-09T05:34:41.667358+0000","last_active":"2023-10-09T05:36:19.149980+0000","last_peered":"2023-10-09T05:36:19.149980+0000","last_clean":"2023-10-09T05:36:19.149980+0000","last_became_active":"2023-10-09T05:34:41.666757+0000","last_became_peered":"2023-10-09T05:34:41.666757+0000","last_unstale":"2023-10-09T05:36:19.149980+0000","last_undegraded":"2023-10-09T05:36:19.149980+0000","last_fullsized":"2023-10-09T05:36:19.149980+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:45:47.725111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":130,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.070192+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:36:19.070192+0000","last_peered":"2023-10-09T05:36:19.070192+0000","last_clean":"2023-10-09T05:36:19.070192+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:36:19.070192+0000","last_undegraded":"2023-10-09T05:36:19.070192+0000","last_fullsized":"2023-10-09T05:36:19.070192+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":78,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.149800+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:36:19.149800+0000","last_peered":"2023-10-09T05:36:19.149800+0000","last_clean":"2023-10-09T05:36:19.149800+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:36:19.149800+0000","last_undegraded":"2023-10-09T05:36:19.149800+0000","last_fullsized":"2023-10-09T05:36:19.149800+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"148'8526","reported_seq":8682,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:36.891241+0000","last_change":"2023-10-09T05:36:37.511452+0000","last_active":"2023-10-09T05:36:36.891241+0000","last_peered":"2023-10-09T05:36:36.891241+0000","last_clean":"2023-10-09T05:36:36.891241+0000","last_became_active":"2023-10-09T05:34:25.031183+0000","last_became_peered":"2023-10-09T05:34:25.031183+0000","last_unstale":"2023-10-09T05:36:36.891241+0000","last_undegraded":"2023-10-09T05:36:36.891241+0000","last_fullsized":"2023-10-09T05:36:36.891241+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'6300","last_scrub_stamp":"2023-10-09T05:34:08.749711+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:08.749711+0000","objects_scrubbed":3317,"log_size":8526,"log_dups_size":0,"ondisk_log_size":8526,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.61798832400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2038,"num_write_kb":11712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"148'10599","reported_seq":10740,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.510323+0000","last_change":"2023-10-09T05:34:10.842783+0000","last_active":"2023-10-09T05:36:32.510323+0000","last_peered":"2023-10-09T05:36:32.510323+0000","last_clean":"2023-10-09T05:36:32.510323+0000","last_became_active":"2023-10-09T05:34:01.606336+0000","last_became_peered":"2023-10-09T05:34:01.606336+0000","last_unstale":"2023-10-09T05:36:32.510323+0000","last_undegraded":"2023-10-09T05:36:32.510323+0000","last_fullsized":"2023-10-09T05:36:32.510323+0000","mapping_epoch":105,"log_start":"109'1202","ondisk_log_start":"109'1202","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'6606","last_scrub_stamp":"2023-10-09T05:34:10.842550+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:10.842550+0000","objects_scrubbed":3389,"log_size":9397,"log_dups_size":0,"ondisk_log_size":9397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:40.600228+0000","scrub_duration":0.74463188000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4019,"num_write_kb":20160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":584,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.070581+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:36:19.070581+0000","last_peered":"2023-10-09T05:36:19.070581+0000","last_clean":"2023-10-09T05:36:19.070581+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:36:19.070581+0000","last_undegraded":"2023-10-09T05:36:19.070581+0000","last_fullsized":"2023-10-09T05:36:19.070581+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":115,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072890+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:36:19.072890+0000","last_peered":"2023-10-09T05:36:19.072890+0000","last_clean":"2023-10-09T05:36:19.072890+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:36:19.072890+0000","last_undegraded":"2023-10-09T05:36:19.072890+0000","last_fullsized":"2023-10-09T05:36:19.072890+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":127,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.070104+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:36:19.070104+0000","last_peered":"2023-10-09T05:36:19.070104+0000","last_clean":"2023-10-09T05:36:19.070104+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:36:19.070104+0000","last_undegraded":"2023-10-09T05:36:19.070104+0000","last_fullsized":"2023-10-09T05:36:19.070104+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"148'10461","reported_seq":10548,"reported_epoch":148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:32.429663+0000","last_change":"2023-10-09T05:34:42.959034+0000","last_active":"2023-10-09T05:36:32.429663+0000","last_peered":"2023-10-09T05:36:32.429663+0000","last_clean":"2023-10-09T05:33:20.529830+0000","last_became_active":"2023-10-09T05:34:42.879241+0000","last_became_peered":"2023-10-09T05:34:42.879241+0000","last_unstale":"2023-10-09T05:36:32.429663+0000","last_undegraded":"2023-10-09T05:34:42.688699+0000","last_fullsized":"2023-10-09T05:34:42.658753+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10461,"log_dups_size":0,"ondisk_log_size":10461,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7393,"num_write_kb":236480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-1995}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":460,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.159758+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:36:19.159758+0000","last_peered":"2023-10-09T05:36:19.159758+0000","last_clean":"2023-10-09T05:36:19.159758+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:36:19.159758+0000","last_undegraded":"2023-10-09T05:36:19.159758+0000","last_fullsized":"2023-10-09T05:36:19.159758+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":94,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:28.625957+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:36:28.625957+0000","last_peered":"2023-10-09T05:36:28.625957+0000","last_clean":"2023-10-09T05:36:28.625957+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:36:28.625957+0000","last_undegraded":"2023-10-09T05:36:28.625957+0000","last_fullsized":"2023-10-09T05:36:28.625957+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":123,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.153890+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:36:19.153890+0000","last_peered":"2023-10-09T05:36:19.153890+0000","last_clean":"2023-10-09T05:36:19.153890+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:36:19.153890+0000","last_undegraded":"2023-10-09T05:36:19.153890+0000","last_fullsized":"2023-10-09T05:36:19.153890+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":724,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.070213+0000","last_change":"2023-10-09T05:34:41.677976+0000","last_active":"2023-10-09T05:36:19.070213+0000","last_peered":"2023-10-09T05:36:19.070213+0000","last_clean":"2023-10-09T05:36:19.070213+0000","last_became_active":"2023-10-09T05:34:41.677520+0000","last_became_peered":"2023-10-09T05:34:41.677520+0000","last_unstale":"2023-10-09T05:36:19.070213+0000","last_undegraded":"2023-10-09T05:36:19.070213+0000","last_fullsized":"2023-10-09T05:36:19.070213+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":325,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.074065+0000","last_change":"2023-10-09T05:34:41.686013+0000","last_active":"2023-10-09T05:36:19.074065+0000","last_peered":"2023-10-09T05:36:19.074065+0000","last_clean":"2023-10-09T05:36:19.074065+0000","last_became_active":"2023-10-09T05:34:41.680809+0000","last_became_peered":"2023-10-09T05:34:41.680809+0000","last_unstale":"2023-10-09T05:36:19.074065+0000","last_undegraded":"2023-10-09T05:36:19.074065+0000","last_fullsized":"2023-10-09T05:36:19.074065+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":110,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.186712+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:36:19.186712+0000","last_peered":"2023-10-09T05:36:19.186712+0000","last_clean":"2023-10-09T05:36:19.186712+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:36:19.186712+0000","last_undegraded":"2023-10-09T05:36:19.186712+0000","last_fullsized":"2023-10-09T05:36:19.186712+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":114,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072124+0000","last_change":"2023-10-09T05:34:41.693650+0000","last_active":"2023-10-09T05:36:19.072124+0000","last_peered":"2023-10-09T05:36:19.072124+0000","last_clean":"2023-10-09T05:36:19.072124+0000","last_became_active":"2023-10-09T05:34:41.693305+0000","last_became_peered":"2023-10-09T05:34:41.693305+0000","last_unstale":"2023-10-09T05:36:19.072124+0000","last_undegraded":"2023-10-09T05:36:19.072124+0000","last_fullsized":"2023-10-09T05:36:19.072124+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"148'10346","reported_seq":10418,"reported_epoch":148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:32.492401+0000","last_change":"2023-10-09T05:34:42.906080+0000","last_active":"2023-10-09T05:36:32.492401+0000","last_peered":"2023-10-09T05:36:32.492401+0000","last_clean":"2023-10-09T05:33:20.493128+0000","last_became_active":"2023-10-09T05:34:42.894560+0000","last_became_peered":"2023-10-09T05:34:42.894560+0000","last_unstale":"2023-10-09T05:36:32.492401+0000","last_undegraded":"2023-10-09T05:34:42.694084+0000","last_fullsized":"2023-10-09T05:34:42.658067+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":10346,"log_dups_size":0,"ondisk_log_size":10346,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2624,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7316,"num_write_kb":233184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-1958}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.7","version":"148'10435","reported_seq":10509,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.416877+0000","last_change":"2023-10-09T05:34:34.976158+0000","last_active":"2023-10-09T05:36:32.416877+0000","last_peered":"2023-10-09T05:36:32.416877+0000","last_clean":"2023-10-09T05:36:32.416877+0000","last_became_active":"2023-10-09T05:34:01.592987+0000","last_became_peered":"2023-10-09T05:34:01.592987+0000","last_unstale":"2023-10-09T05:36:32.416877+0000","last_undegraded":"2023-10-09T05:36:32.416877+0000","last_fullsized":"2023-10-09T05:36:32.416877+0000","mapping_epoch":105,"log_start":"108'107","ondisk_log_start":"108'107","created":105,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7154","last_scrub_stamp":"2023-10-09T05:34:34.976067+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:34.976067+0000","objects_scrubbed":0,"log_size":10328,"log_dups_size":0,"ondisk_log_size":10328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3307,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":748,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.074043+0000","last_change":"2023-10-09T05:34:41.668526+0000","last_active":"2023-10-09T05:36:19.074043+0000","last_peered":"2023-10-09T05:36:19.074043+0000","last_clean":"2023-10-09T05:36:19.074043+0000","last_became_active":"2023-10-09T05:34:41.668010+0000","last_became_peered":"2023-10-09T05:34:41.668010+0000","last_unstale":"2023-10-09T05:36:19.074043+0000","last_undegraded":"2023-10-09T05:36:19.074043+0000","last_fullsized":"2023-10-09T05:36:19.074043+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":100,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.676410+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:36:30.676410+0000","last_peered":"2023-10-09T05:36:30.676410+0000","last_clean":"2023-10-09T05:36:30.676410+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:36:30.676410+0000","last_undegraded":"2023-10-09T05:36:30.676410+0000","last_fullsized":"2023-10-09T05:36:30.676410+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":115,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.677518+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:36:30.677518+0000","last_peered":"2023-10-09T05:36:30.677518+0000","last_clean":"2023-10-09T05:36:30.677518+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:36:30.677518+0000","last_undegraded":"2023-10-09T05:36:30.677518+0000","last_fullsized":"2023-10-09T05:36:30.677518+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":351,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.074924+0000","last_change":"2023-10-09T05:34:41.682539+0000","last_active":"2023-10-09T05:36:19.074924+0000","last_peered":"2023-10-09T05:36:19.074924+0000","last_clean":"2023-10-09T05:36:19.074924+0000","last_became_active":"2023-10-09T05:34:41.674099+0000","last_became_peered":"2023-10-09T05:34:41.674099+0000","last_unstale":"2023-10-09T05:36:19.074924+0000","last_undegraded":"2023-10-09T05:36:19.074924+0000","last_fullsized":"2023-10-09T05:36:19.074924+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":93,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.150316+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:36:19.150316+0000","last_peered":"2023-10-09T05:36:19.150316+0000","last_clean":"2023-10-09T05:36:19.150316+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:36:19.150316+0000","last_undegraded":"2023-10-09T05:36:19.150316+0000","last_fullsized":"2023-10-09T05:36:19.150316+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":100,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.677739+0000","last_change":"2023-10-09T05:34:41.671414+0000","last_active":"2023-10-09T05:36:30.677739+0000","last_peered":"2023-10-09T05:36:30.677739+0000","last_clean":"2023-10-09T05:36:30.677739+0000","last_became_active":"2023-10-09T05:34:41.671100+0000","last_became_peered":"2023-10-09T05:34:41.671100+0000","last_unstale":"2023-10-09T05:36:30.677739+0000","last_undegraded":"2023-10-09T05:36:30.677739+0000","last_fullsized":"2023-10-09T05:36:30.677739+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:19.832596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"148'8584","reported_seq":8665,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.456110+0000","last_change":"2023-10-09T05:34:33.321883+0000","last_active":"2023-10-09T05:36:32.456110+0000","last_peered":"2023-10-09T05:36:32.456110+0000","last_clean":"2023-10-09T05:36:32.456110+0000","last_became_active":"2023-10-09T05:34:25.017164+0000","last_became_peered":"2023-10-09T05:34:25.017164+0000","last_unstale":"2023-10-09T05:36:32.456110+0000","last_undegraded":"2023-10-09T05:36:32.456110+0000","last_fullsized":"2023-10-09T05:36:32.456110+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'6928","last_scrub_stamp":"2023-10-09T05:34:33.321793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:33.321793+0000","objects_scrubbed":0,"log_size":8584,"log_dups_size":0,"ondisk_log_size":8584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1667,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":132,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.185229+0000","last_change":"2023-10-09T05:34:41.664569+0000","last_active":"2023-10-09T05:36:19.185229+0000","last_peered":"2023-10-09T05:36:19.185229+0000","last_clean":"2023-10-09T05:36:19.185229+0000","last_became_active":"2023-10-09T05:34:41.664239+0000","last_became_peered":"2023-10-09T05:34:41.664239+0000","last_unstale":"2023-10-09T05:36:19.185229+0000","last_undegraded":"2023-10-09T05:36:19.185229+0000","last_fullsized":"2023-10-09T05:36:19.185229+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:26.511248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":534,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.150607+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:36:19.150607+0000","last_peered":"2023-10-09T05:36:19.150607+0000","last_clean":"2023-10-09T05:36:19.150607+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:36:19.150607+0000","last_undegraded":"2023-10-09T05:36:19.150607+0000","last_fullsized":"2023-10-09T05:36:19.150607+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":129,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.184532+0000","last_change":"2023-10-09T05:34:41.666200+0000","last_active":"2023-10-09T05:36:19.184532+0000","last_peered":"2023-10-09T05:36:19.184532+0000","last_clean":"2023-10-09T05:36:19.184532+0000","last_became_active":"2023-10-09T05:34:41.665841+0000","last_became_peered":"2023-10-09T05:34:41.665841+0000","last_unstale":"2023-10-09T05:36:19.184532+0000","last_undegraded":"2023-10-09T05:36:19.184532+0000","last_fullsized":"2023-10-09T05:36:19.184532+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:58.338181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"148'8533","reported_seq":11244,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.334927+0000","last_change":"2023-10-09T05:34:57.102099+0000","last_active":"2023-10-09T05:36:32.334927+0000","last_peered":"2023-10-09T05:36:32.334927+0000","last_clean":"2023-10-09T05:36:32.334927+0000","last_became_active":"2023-10-09T05:34:57.096500+0000","last_became_peered":"2023-10-09T05:34:57.096500+0000","last_unstale":"2023-10-09T05:36:32.334927+0000","last_undegraded":"2023-10-09T05:36:32.334927+0000","last_fullsized":"2023-10-09T05:36:32.334927+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8533,"log_dups_size":0,"ondisk_log_size":8533,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":442368,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3663,"num_write_kb":117456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1310,"num_bytes_recovered":65667072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":611,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072487+0000","last_change":"2023-10-09T05:34:41.676842+0000","last_active":"2023-10-09T05:36:19.072487+0000","last_peered":"2023-10-09T05:36:19.072487+0000","last_clean":"2023-10-09T05:36:19.072487+0000","last_became_active":"2023-10-09T05:34:41.676428+0000","last_became_peered":"2023-10-09T05:34:41.676428+0000","last_unstale":"2023-10-09T05:36:19.072487+0000","last_undegraded":"2023-10-09T05:36:19.072487+0000","last_fullsized":"2023-10-09T05:36:19.072487+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":103,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.677449+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:36:30.677449+0000","last_peered":"2023-10-09T05:36:30.677449+0000","last_clean":"2023-10-09T05:36:30.677449+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:36:30.677449+0000","last_undegraded":"2023-10-09T05:36:30.677449+0000","last_fullsized":"2023-10-09T05:36:30.677449+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":99,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072388+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:36:19.072388+0000","last_peered":"2023-10-09T05:36:19.072388+0000","last_clean":"2023-10-09T05:36:19.072388+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:36:19.072388+0000","last_undegraded":"2023-10-09T05:36:19.072388+0000","last_fullsized":"2023-10-09T05:36:19.072388+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"148'8555","reported_seq":8745,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.269462+0000","last_change":"2023-10-09T05:34:33.714772+0000","last_active":"2023-10-09T05:36:32.269462+0000","last_peered":"2023-10-09T05:36:32.269462+0000","last_clean":"2023-10-09T05:36:32.269462+0000","last_became_active":"2023-10-09T05:34:25.048537+0000","last_became_peered":"2023-10-09T05:34:25.048537+0000","last_unstale":"2023-10-09T05:36:32.269462+0000","last_undegraded":"2023-10-09T05:36:32.269462+0000","last_fullsized":"2023-10-09T05:36:32.269462+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'6879","last_scrub_stamp":"2023-10-09T05:34:33.714537+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:33.714537+0000","objects_scrubbed":1680,"log_size":8555,"log_dups_size":0,"ondisk_log_size":8555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:15.987881+0000","scrub_duration":0.49573348099999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1680,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1d","version":"148'11210","reported_seq":11284,"reported_epoch":148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:36:32.476948+0000","last_change":"2023-10-09T05:34:42.908183+0000","last_active":"2023-10-09T05:36:32.476948+0000","last_peered":"2023-10-09T05:36:32.476948+0000","last_clean":"2023-10-09T05:33:20.502869+0000","last_became_active":"2023-10-09T05:34:42.896545+0000","last_became_peered":"2023-10-09T05:34:42.896545+0000","last_unstale":"2023-10-09T05:36:32.476948+0000","last_undegraded":"2023-10-09T05:34:42.687614+0000","last_fullsized":"2023-10-09T05:34:42.658151+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":11210,"log_dups_size":0,"ondisk_log_size":11210,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2695,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7468,"num_write_kb":237793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-2021}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":102,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.677666+0000","last_change":"2023-10-09T05:34:41.662989+0000","last_active":"2023-10-09T05:36:30.677666+0000","last_peered":"2023-10-09T05:36:30.677666+0000","last_clean":"2023-10-09T05:36:30.677666+0000","last_became_active":"2023-10-09T05:34:41.662621+0000","last_became_peered":"2023-10-09T05:34:41.662621+0000","last_unstale":"2023-10-09T05:36:30.677666+0000","last_undegraded":"2023-10-09T05:36:30.677666+0000","last_fullsized":"2023-10-09T05:36:30.677666+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:08.750044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":138,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.163502+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:36:19.163502+0000","last_peered":"2023-10-09T05:36:19.163502+0000","last_clean":"2023-10-09T05:36:19.163502+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:36:19.163502+0000","last_undegraded":"2023-10-09T05:36:19.163502+0000","last_fullsized":"2023-10-09T05:36:19.163502+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.1a","version":"148'11091","reported_seq":11153,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.440711+0000","last_change":"2023-10-09T05:34:25.055227+0000","last_active":"2023-10-09T05:36:32.440711+0000","last_peered":"2023-10-09T05:36:32.440711+0000","last_clean":"2023-10-09T05:36:32.440711+0000","last_became_active":"2023-10-09T05:34:25.051761+0000","last_became_peered":"2023-10-09T05:34:25.051761+0000","last_unstale":"2023-10-09T05:36:32.440711+0000","last_undegraded":"2023-10-09T05:36:32.440711+0000","last_fullsized":"2023-10-09T05:36:32.440711+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":11091,"log_dups_size":0,"ondisk_log_size":11091,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":622592,"num_objects":10,"num_object_clones":0,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7373,"num_write_kb":236224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":117,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.070463+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:36:19.070463+0000","last_peered":"2023-10-09T05:36:19.070463+0000","last_clean":"2023-10-09T05:36:19.070463+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:36:19.070463+0000","last_undegraded":"2023-10-09T05:36:19.070463+0000","last_fullsized":"2023-10-09T05:36:19.070463+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":129,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072964+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:36:19.072964+0000","last_peered":"2023-10-09T05:36:19.072964+0000","last_clean":"2023-10-09T05:36:19.072964+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:36:19.072964+0000","last_undegraded":"2023-10-09T05:36:19.072964+0000","last_fullsized":"2023-10-09T05:36:19.072964+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.1b","version":"148'11076","reported_seq":11138,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.482363+0000","last_change":"2023-10-09T05:34:25.065203+0000","last_active":"2023-10-09T05:36:32.482363+0000","last_peered":"2023-10-09T05:36:32.482363+0000","last_clean":"2023-10-09T05:36:32.482363+0000","last_became_active":"2023-10-09T05:34:25.059858+0000","last_became_peered":"2023-10-09T05:34:25.059858+0000","last_unstale":"2023-10-09T05:36:32.482363+0000","last_undegraded":"2023-10-09T05:36:32.482363+0000","last_fullsized":"2023-10-09T05:36:32.482363+0000","mapping_epoch":123,"log_start":"0'0","ondisk_log_start":"0'0","created":123,"last_epoch_clean":124,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":11076,"log_dups_size":0,"ondisk_log_size":11076,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5701632,"num_objects":87,"num_object_clones":0,"num_object_copies":261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7338,"num_write_kb":237600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":102,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.070111+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:36:19.070111+0000","last_peered":"2023-10-09T05:36:19.070111+0000","last_clean":"2023-10-09T05:36:19.070111+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:36:19.070111+0000","last_undegraded":"2023-10-09T05:36:19.070111+0000","last_fullsized":"2023-10-09T05:36:19.070111+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":129,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.072810+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:36:19.072810+0000","last_peered":"2023-10-09T05:36:19.072810+0000","last_clean":"2023-10-09T05:36:19.072810+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:36:19.072810+0000","last_undegraded":"2023-10-09T05:36:19.072810+0000","last_fullsized":"2023-10-09T05:36:19.072810+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"148'10606","reported_seq":10750,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:32.511159+0000","last_change":"2023-10-09T05:34:07.818690+0000","last_active":"2023-10-09T05:36:32.511159+0000","last_peered":"2023-10-09T05:36:32.511159+0000","last_clean":"2023-10-09T05:36:32.511159+0000","last_became_active":"2023-10-09T05:34:01.656619+0000","last_became_peered":"2023-10-09T05:34:01.656619+0000","last_unstale":"2023-10-09T05:36:32.511159+0000","last_undegraded":"2023-10-09T05:36:32.511159+0000","last_fullsized":"2023-10-09T05:36:32.511159+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'6429","last_scrub_stamp":"2023-10-09T05:34:07.818377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:34:07.818377+0000","objects_scrubbed":3381,"log_size":10606,"log_dups_size":0,"ondisk_log_size":10606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:04.556045+0000","scrub_duration":0.90924191099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4193,"num_write_kb":25984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":125,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.184879+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:36:19.184879+0000","last_peered":"2023-10-09T05:36:19.184879+0000","last_clean":"2023-10-09T05:36:19.184879+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:36:19.184879+0000","last_undegraded":"2023-10-09T05:36:19.184879+0000","last_fullsized":"2023-10-09T05:36:19.184879+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":138,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:21.010200+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:36:21.010200+0000","last_peered":"2023-10-09T05:36:21.010200+0000","last_clean":"2023-10-09T05:36:21.010200+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:36:21.010200+0000","last_undegraded":"2023-10-09T05:36:21.010200+0000","last_fullsized":"2023-10-09T05:36:21.010200+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"148'10339","reported_seq":15614,"reported_epoch":148,"state":"active+recovery_wait","last_fresh":"2023-10-09T05:36:32.455795+0000","last_change":"2023-10-09T05:36:11.687702+0000","last_active":"2023-10-09T05:36:32.455795+0000","last_peered":"2023-10-09T05:36:32.455795+0000","last_clean":"2023-10-09T05:33:20.515228+0000","last_became_active":"2023-10-09T05:36:11.631481+0000","last_became_peered":"2023-10-09T05:36:11.631481+0000","last_unstale":"2023-10-09T05:36:32.455795+0000","last_undegraded":"2023-10-09T05:36:32.455795+0000","last_fullsized":"2023-10-09T05:36:32.455795+0000","mapping_epoch":145,"log_start":"109'1667","ondisk_log_start":"109'1667","created":118,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:32:44.904058+0000","last_clean_scrub_stamp":"2023-10-09T05:32:44.904058+0000","objects_scrubbed":0,"log_size":8672,"log_dups_size":0,"ondisk_log_size":8672,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2818048,"num_objects":43,"num_object_clones":0,"num_object_copies":129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7279,"num_write_kb":234304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2599,"num_bytes_recovered":82051072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":["7","4","6"],"object_location_counts":[{"shards":"4,6,7","objects":43}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":103,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:36:30.677847+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:36:30.677847+0000","last_peered":"2023-10-09T05:36:30.677847+0000","last_clean":"2023-10-09T05:36:30.677847+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:36:30.677847+0000","last_undegraded":"2023-10-09T05:36:30.677847+0000","last_fullsized":"2023-10-09T05:36:30.677847+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":114,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:36:19.071185+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:36:19.071185+0000","last_peered":"2023-10-09T05:36:19.071185+0000","last_clean":"2023-10-09T05:36:19.071185+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:36:19.071185+0000","last_undegraded":"2023-10-09T05:36:19.071185+0000","last_fullsized":"2023-10-09T05:36:19.071185+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":36,"stat_sum":{"num_bytes":27754508,"num_objects":425,"num_object_clones":0,"num_object_copies":1275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15763,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":425,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":185982,"num_write_kb":4609889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10426,"num_bytes_recovered":377290752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":251002880,"data_stored":251002880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":351396,"ondisk_log_size":351396,"up":108,"acting":101,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846616,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":565520,"kb_used_data":251208,"kb_used_omap":14,"kb_used_meta":314289,"kb_avail":93179632,"statfs":{"total":95995035648,"available":95415943168,"internally_reserved":0,"allocated":257236992,"data_stored":256721776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":321832794},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":7.3810000000000002,"5min":7.3810000000000002,"15min":7.3810000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":60.323999999999998,"5min":60.323999999999998,"15min":60.323999999999998},"last":0.41099999999999998},{"interface":"front","average":{"1min":8.548,"5min":8.548,"15min":8.548},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":60.334000000000003,"5min":60.334000000000003,"15min":60.334000000000003},"last":0.45300000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.6200000000000001,"15min":1.6200000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":6.2370000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.585000000000001,"5min":11.585000000000001,"15min":11.585000000000001},"last":0.33100000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":6.859,"5min":6.859,"15min":6.859},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":73.896000000000001,"5min":73.896000000000001,"15min":73.896000000000001},"last":0.26900000000000002},{"interface":"front","average":{"1min":6.798,"5min":6.798,"15min":6.798},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":73.881,"5min":73.881,"15min":73.881},"last":0.38100000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.0670000000000002,"5min":4.0670000000000002,"15min":4.0670000000000002},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.0389999999999997,"5min":6.0389999999999997,"15min":6.0389999999999997},"last":0.47199999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.9930000000000001,"15min":1.9930000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":12.789,"5min":12.789,"15min":12.789},"last":0.27800000000000002},{"interface":"front","average":{"1min":2.2519999999999998,"5min":2.2519999999999998,"15min":2.2519999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":8.9009999999999998,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":2.0649999999999999,"15min":2.0649999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":8.8230000000000004,"5min":8.8230000000000004,"15min":8.8230000000000004},"last":0.34100000000000003},{"interface":"front","average":{"1min":2.7770000000000001,"5min":2.7770000000000001,"15min":2.7770000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":9.8049999999999997,"5min":9.8049999999999997,"15min":9.8049999999999997},"last":0.32200000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":2.2040000000000002,"15min":2.2040000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":8.8580000000000005,"5min":8.8580000000000005,"15min":8.8580000000000005},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.081,"5min":2.081,"15min":2.081},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.2680000000000007,"5min":8.2680000000000007,"15min":8.2680000000000007},"last":0.39400000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626642,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303300,"kb_used_data":6076,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93441852,"statfs":{"total":95995035648,"available":95684456448,"internally_reserved":0,"allocated":6221824,"data_stored":5718504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:14 2023","interfaces":[{"interface":"back","average":{"1min":8.5340000000000007,"5min":3.4380000000000002,"15min":1.506},"min":{"1min":0.27800000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":77.024000000000001,"5min":77.024000000000001,"15min":77.024000000000001},"last":0.27200000000000002},{"interface":"front","average":{"1min":8.1349999999999998,"5min":3.3239999999999998,"15min":1.4710000000000001},"min":{"1min":0.27600000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":79.305999999999997,"5min":79.305999999999997,"15min":79.305999999999997},"last":0.47599999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:35:36 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":1.5429999999999999,"15min":0.88300000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.1840000000000002,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":0.35599999999999998},{"interface":"front","average":{"1min":3.1549999999999998,"5min":1.76,"15min":0.97099999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.9979999999999993,"5min":9.9979999999999993,"15min":9.9979999999999993},"last":0.40899999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:36:14 2023","interfaces":[{"interface":"back","average":{"1min":4.516,"5min":3.0670000000000002,"15min":1.7390000000000001},"min":{"1min":0.33600000000000002,"5min":0.314,"15min":0.314},"max":{"1min":36.832000000000001,"5min":36.832000000000001,"15min":36.832000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":4.0019999999999998,"5min":2.879,"15min":1.6890000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":36.811999999999998,"5min":36.811999999999998,"15min":36.811999999999998},"last":0.32900000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":1.5660000000000001,"15min":1.232},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.0609999999999999,"5min":18.361000000000001,"15min":18.361000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.7729999999999999,"15min":1.339},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.042999999999999,"5min":18.536000000000001,"15min":18.536000000000001},"last":0.38300000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:35:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1800000000000002,"5min":2.8479999999999999,"15min":1.262},"min":{"1min":0.29599999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.8569999999999993,"5min":125.64,"15min":125.64},"last":0.314},{"interface":"front","average":{"1min":1.6220000000000001,"5min":2.7669999999999999,"15min":1.248},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.27800000000000002},"max":{"1min":6.6109999999999998,"5min":125.625,"15min":125.625},"last":0.28599999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:35:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2749999999999999,"5min":2.488,"15min":1.1719999999999999},"min":{"1min":0.36699999999999999,"5min":0.27900000000000003,"15min":0.27200000000000002},"max":{"1min":9.7439999999999998,"5min":127.35899999999999,"15min":127.35899999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.992,"5min":2.5310000000000001,"15min":1.1950000000000001},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.6379999999999999,"5min":128.93799999999999,"15min":128.93799999999999},"last":0.40000000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":3.1480000000000001,"5min":3.1480000000000001,"15min":3.1480000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.074999999999999,"5min":12.074999999999999,"15min":12.074999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.0920000000000001,"5min":2.0920000000000001,"15min":2.0920000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.083,"5min":10.083,"15min":10.083},"last":0.372}]}]},{"osd":1,"up_from":93,"seq":399431958603,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":145620,"kb_used_data":1548,"kb_used_omap":7,"kb_used_meta":144056,"kb_avail":93599532,"statfs":{"total":95995035648,"available":95845920768,"internally_reserved":0,"allocated":1585152,"data_stored":952281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":147513870},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":7.8689999999999998,"5min":2.2719999999999998,"15min":1.026},"min":{"1min":0.19800000000000001,"5min":0.185,"15min":0.185},"max":{"1min":124.82599999999999,"5min":124.82599999999999,"15min":124.82599999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":7.7249999999999996,"5min":2.367,"15min":1.0609999999999999},"min":{"1min":0.24199999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":125.483,"5min":125.483,"15min":125.483},"last":0.27100000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":3.2440000000000002,"5min":2.778,"15min":1.1539999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":21.216999999999999,"5min":98.483999999999995,"15min":98.483999999999995},"last":58.844000000000001},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.5649999999999999,"15min":1.131},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":21.161000000000001,"5min":98.409999999999997,"15min":98.409999999999997},"last":58.25}]},{"osd":3,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.107,"15min":0.68799999999999994},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":10.298999999999999,"5min":10.298999999999999,"15min":10.298999999999999},"last":0.23400000000000001},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.1859999999999999,"15min":0.66400000000000003},"min":{"1min":0.26600000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.8109999999999999,"5min":8.8109999999999999,"15min":8.8109999999999999},"last":0.28699999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.4390000000000001,"15min":0.90800000000000003},"min":{"1min":0.34399999999999997,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.1940000000000008,"5min":23.670999999999999,"15min":23.670999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.5800000000000001,"5min":1.1779999999999999,"15min":0.83299999999999996},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.8099999999999996,"5min":23.620000000000001,"15min":23.620000000000001},"last":0.40100000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.3720000000000001,"15min":0.879},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.1139999999999999,"5min":23.501999999999999,"15min":23.501999999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.75,"5min":1.639,"15min":0.998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":6.9100000000000001,"5min":25.68,"15min":25.68},"last":0.375}]},{"osd":6,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.1679999999999999,"15min":0.82599999999999996},"min":{"1min":0.44500000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":4.3819999999999997,"5min":8.1270000000000007,"15min":8.1270000000000007},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.0110000000000001,"5min":1.2410000000000001,"15min":0.871},"min":{"1min":0.43099999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.0810000000000004,"5min":7.0810000000000004,"15min":7.0810000000000004},"last":0.41299999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9990000000000001,"5min":1.9990000000000001,"15min":1.9990000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":6.5919999999999996,"5min":6.5919999999999996,"15min":6.5919999999999996},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.0790000000000002,"5min":2.0790000000000002,"15min":2.0790000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":6.5529999999999999,"5min":6.5529999999999999,"15min":6.5529999999999999},"last":0.34399999999999997}]}]},{"osd":0,"up_from":88,"seq":377957122130,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":196248,"kb_used_data":2128,"kb_used_omap":8,"kb_used_meta":194103,"kb_avail":93548904,"statfs":{"total":95995035648,"available":95794077696,"internally_reserved":0,"allocated":2179072,"data_stored":1548983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":198762248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:35:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":1.117,"15min":0.63600000000000001},"min":{"1min":0.22700000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":7.9089999999999998,"5min":13.919,"15min":13.919},"last":0.248},{"interface":"front","average":{"1min":1.782,"5min":1.155,"15min":0.67800000000000005},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.0009999999999994,"5min":9.9350000000000005,"15min":9.9350000000000005},"last":0.38700000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":8.6180000000000003,"5min":3.4969999999999999,"15min":1.403},"min":{"1min":0.223,"5min":0.214,"15min":0.214},"max":{"1min":79.911000000000001,"5min":79.911000000000001,"15min":79.911000000000001},"last":0.27500000000000002},{"interface":"front","average":{"1min":8.1799999999999997,"5min":3.2440000000000002,"15min":1.3300000000000001},"min":{"1min":0.23200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":80.105000000000004,"5min":80.105000000000004,"15min":80.105000000000004},"last":0.49299999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:35:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4329999999999998,"5min":1.4159999999999999,"15min":0.73899999999999999},"min":{"1min":0.23999999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.9700000000000006,"5min":12.429,"15min":12.429},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.5169999999999999,"5min":1.603,"15min":0.80100000000000005},"min":{"1min":0.30499999999999999,"5min":0.222,"15min":0.222},"max":{"1min":7.5,"5min":22.390999999999998,"15min":22.390999999999998},"last":0.32500000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.1859999999999999,"15min":0.84899999999999998},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.6130000000000004,"5min":14.023,"15min":14.023},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.542,"15min":0.995},"min":{"1min":0.41099999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.75,"5min":14.096,"15min":14.096},"last":0.60299999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":1.5529999999999999,"15min":0.97099999999999997},"min":{"1min":0.34499999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.8019999999999996,"5min":13.632,"15min":13.632},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.5600000000000001,"15min":0.96999999999999997},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":8.1549999999999994,"5min":13.676,"15min":13.676},"last":0.45800000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.5289999999999999,"15min":0.98099999999999998},"min":{"1min":0.39200000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.8339999999999996,"5min":9.9809999999999999,"15min":9.9809999999999999},"last":0.67200000000000004},{"interface":"front","average":{"1min":2.141,"5min":1.597,"15min":0.98499999999999999},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.4450000000000003,"5min":7.4450000000000003,"15min":7.4450000000000003},"last":0.62}]},{"osd":7,"last update":"Mon Oct 9 05:35:43 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.329,"15min":1.329},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":5.0899999999999999,"5min":5.0899999999999999,"15min":5.0899999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.3900000000000001,"5min":2.3900000000000001,"15min":2.3900000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":11.068,"5min":11.068,"15min":11.068},"last":0.72599999999999998}]}]},{"osd":2,"up_from":98,"seq":420906795077,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":181660,"kb_used_data":5652,"kb_used_omap":11,"kb_used_meta":175988,"kb_avail":93563492,"statfs":{"total":95995035648,"available":95809015808,"internally_reserved":0,"allocated":5787648,"data_stored":5146520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":180212309},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2999999999999998,"5min":1.5620000000000001,"15min":0.75800000000000001},"min":{"1min":0.183,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":10.805999999999999,"5min":32.122,"15min":32.122},"last":0.23000000000000001},{"interface":"front","average":{"1min":1.5780000000000001,"5min":1.3460000000000001,"15min":0.71199999999999997},"min":{"1min":0.23599999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.3849999999999998,"5min":34.066000000000003,"15min":34.066000000000003},"last":0.42599999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":2.028,"15min":0.93000000000000005},"min":{"1min":0.218,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.3510000000000009,"5min":98.459999999999994,"15min":98.459999999999994},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.9379999999999999,"5min":1.9590000000000001,"15min":0.90700000000000003},"min":{"1min":0.218,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":16.007999999999999,"5min":98.397000000000006,"15min":98.397000000000006},"last":0.317}]},{"osd":3,"last update":"Mon Oct 9 05:35:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4140000000000001,"5min":1.4550000000000001,"15min":0.73499999999999999},"min":{"1min":0.249,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":23.738,"5min":34.052999999999997,"15min":34.052999999999997},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.9849999999999999,"5min":1.669,"15min":0.82299999999999995},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":23.901,"5min":35.161999999999999,"15min":35.161999999999999},"last":0.32300000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3159999999999998,"5min":1.3149999999999999,"15min":0.82099999999999995},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.8960000000000008,"5min":8.8960000000000008,"15min":8.8960000000000008},"last":0.33700000000000002},{"interface":"front","average":{"1min":3.427,"5min":1.5780000000000001,"15min":0.95399999999999996},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":16.869,"5min":16.869,"15min":16.869},"last":0.34899999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":1.524,"15min":0.93600000000000005},"min":{"1min":0.376,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.2789999999999999,"5min":8.2789999999999999,"15min":8.2789999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":3.1219999999999999,"5min":1.4890000000000001,"15min":0.90100000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":8.8399999999999999,"5min":8.8399999999999999,"15min":8.8399999999999999},"last":0.36299999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":1.554,"15min":0.93400000000000005},"min":{"1min":0.379,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":10.715,"5min":11.496,"15min":11.496},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.1459999999999999,"5min":1.3600000000000001,"15min":0.91000000000000003},"min":{"1min":0.40100000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.093,"5min":9.4049999999999994,"15min":9.4049999999999994},"last":0.47199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:35:44 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":2.524,"15min":2.524},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":10.673,"5min":10.673,"15min":10.673},"last":0.438},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.3190000000000008,"5min":8.3190000000000008,"15min":8.3190000000000008},"last":0.61599999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631551,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":83128,"kb_used_data":1520,"kb_used_omap":4,"kb_used_meta":81595,"kb_avail":93662024,"statfs":{"total":95995035648,"available":95909912576,"internally_reserved":0,"allocated":1556480,"data_stored":948090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":83553886},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.6459999999999999,"15min":0.81399999999999995},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":5.3099999999999996,"5min":56.094000000000001,"15min":56.094000000000001},"last":0.25800000000000001},{"interface":"front","average":{"1min":2.085,"5min":1.9339999999999999,"15min":0.89500000000000002},"min":{"1min":0.246,"5min":0.214,"15min":0.214},"max":{"1min":10.084,"5min":57.255000000000003,"15min":57.255000000000003},"last":0.27800000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":0.85599999999999998,"15min":0.55400000000000005},"min":{"1min":0.27000000000000002,"5min":0.255,"15min":0.255},"max":{"1min":5.04,"5min":5.04,"15min":5.04},"last":0.316},{"interface":"front","average":{"1min":1.617,"5min":0.91500000000000004,"15min":0.55500000000000005},"min":{"1min":0.32600000000000001,"5min":0.247,"15min":0.247},"max":{"1min":5.7080000000000002,"5min":6.8769999999999998,"15min":6.8769999999999998},"last":0.29499999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":3.0859999999999999,"15min":1.3140000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":6.3849999999999998,"5min":117.23,"15min":117.23},"last":10.891999999999999},{"interface":"front","average":{"1min":2.2200000000000002,"5min":3.1549999999999998,"15min":1.345},"min":{"1min":0.313,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.227,"5min":117.45999999999999,"15min":117.45999999999999},"last":11.045999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:35:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3359999999999999,"5min":2.0129999999999999,"15min":1.069},"min":{"1min":0.33200000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.1099999999999994,"5min":45.779000000000003,"15min":45.779000000000003},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.681,"15min":0.99199999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.2050000000000001,"5min":45.735999999999997,"15min":45.735999999999997},"last":0.44700000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":1.6950000000000001,"15min":0.99299999999999999},"min":{"1min":0.44500000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.4770000000000003,"5min":47.948999999999998,"15min":47.948999999999998},"last":0.51300000000000001},{"interface":"front","average":{"1min":2.5179999999999998,"5min":1.8129999999999999,"15min":1.0569999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":8.5009999999999994,"5min":47.968000000000004,"15min":47.968000000000004},"last":0.52400000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:35:40 2023","interfaces":[{"interface":"back","average":{"1min":2.5499999999999998,"5min":1.4550000000000001,"15min":0.93300000000000005},"min":{"1min":0.379,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.5990000000000002,"5min":8.5990000000000002,"15min":8.5990000000000002},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.077,"5min":0.95799999999999996,"15min":0.73399999999999999},"min":{"1min":0.41199999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.6879999999999997,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.54500000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:35:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":2.2599999999999998,"15min":2.2599999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.8869999999999996,"5min":5.8869999999999996,"15min":5.8869999999999996},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.9630000000000001,"15min":1.9630000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.851,"5min":5.851,"15min":5.851},"last":0.53500000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019098,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306824,"kb_used_data":1408,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93438328,"statfs":{"total":95995035648,"available":95680847872,"internally_reserved":0,"allocated":1441792,"data_stored":934154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:11 2023","interfaces":[{"interface":"back","average":{"1min":4.1360000000000001,"5min":2.2280000000000002,"15min":1.0900000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":36.406999999999996,"5min":36.406999999999996,"15min":36.406999999999996},"last":0.318},{"interface":"front","average":{"1min":5.0629999999999997,"5min":2.5329999999999999,"15min":1.2130000000000001},"min":{"1min":0.26100000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":36.454999999999998,"5min":36.454999999999998,"15min":36.454999999999998},"last":0.40300000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.2919999999999998,"5min":1.4239999999999999,"15min":0.83199999999999996},"min":{"1min":0.41599999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.073,"5min":31.073,"15min":31.073},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.169,"5min":1.579,"15min":0.89900000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":10.055,"5min":31.100000000000001,"15min":31.100000000000001},"last":0.42999999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:36:11 2023","interfaces":[{"interface":"back","average":{"1min":12.515000000000001,"5min":5.8819999999999997,"15min":2.306},"min":{"1min":0.40100000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":128.84700000000001,"5min":164.26599999999999,"15min":164.26599999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":12.494,"5min":5.7169999999999996,"15min":2.2949999999999999},"min":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":128.911,"5min":162.57499999999999,"15min":162.57499999999999},"last":0.28599999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:35:34 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.3899999999999999,"15min":0.83599999999999997},"min":{"1min":0.35399999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.037000000000001,"5min":10.037000000000001,"15min":10.037000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.429,"5min":1.1519999999999999,"15min":0.79200000000000004},"min":{"1min":0.34100000000000003,"5min":0.309,"15min":0.309},"max":{"1min":4.6219999999999999,"5min":6.9459999999999997,"15min":6.9459999999999997},"last":0.42199999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:35:34 2023","interfaces":[{"interface":"back","average":{"1min":7.8369999999999997,"5min":2.2010000000000001,"15min":1.034},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":113.35899999999999,"5min":113.35899999999999,"15min":113.35899999999999},"last":0.37},{"interface":"front","average":{"1min":7.71,"5min":2.125,"15min":1.071},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":113.32299999999999,"5min":113.32299999999999,"15min":113.32299999999999},"last":0.442}]},{"osd":6,"last update":"Mon Oct 9 05:35:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8009999999999999,"5min":0.89800000000000002,"15min":0.622},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.27800000000000002},"max":{"1min":11.944000000000001,"5min":11.944000000000001,"15min":11.944000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.1539999999999999,"15min":0.71699999999999997},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":10.356,"5min":10.356,"15min":10.356},"last":0.32500000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":3.3700000000000001,"5min":3.3700000000000001,"15min":3.3700000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":30.393999999999998,"5min":30.393999999999998,"15min":30.393999999999998},"last":0.39100000000000001},{"interface":"front","average":{"1min":3.8300000000000001,"5min":3.8300000000000001,"15min":3.8300000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":30.334,"5min":30.334,"15min":30.334},"last":0.379}]}]},{"osd":5,"up_from":36,"seq":154618822870,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302712,"kb_used_data":1392,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93442440,"statfs":{"total":95995035648,"available":95685058560,"internally_reserved":0,"allocated":1425408,"data_stored":929602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:06 2023","interfaces":[{"interface":"back","average":{"1min":14.334,"5min":6.5940000000000003,"15min":2.5659999999999998},"min":{"1min":0.22700000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":165.90199999999999,"5min":165.90199999999999,"15min":165.90199999999999},"last":5.1459999999999999},{"interface":"front","average":{"1min":13.84,"5min":6.8440000000000003,"15min":2.6520000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":165.96700000000001,"5min":165.96700000000001,"15min":165.96700000000001},"last":0.56000000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:35:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7429999999999999,"5min":1.3480000000000001,"15min":0.81999999999999995},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.539999999999999,"5min":12.891999999999999,"15min":12.891999999999999},"last":5.7450000000000001},{"interface":"front","average":{"1min":1.522,"5min":1.0740000000000001,"15min":0.72999999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.2609999999999992,"5min":12.939,"15min":12.939},"last":2.8919999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:36:14 2023","interfaces":[{"interface":"back","average":{"1min":17.323,"5min":8.0950000000000006,"15min":3.0790000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":118.255,"5min":290.697,"15min":290.697},"last":2.8860000000000001},{"interface":"front","average":{"1min":16.324999999999999,"5min":7.9080000000000004,"15min":3.0139999999999998},"min":{"1min":0.38800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":118.438,"5min":290.786,"15min":290.786},"last":5.7800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:35:36 2023","interfaces":[{"interface":"back","average":{"1min":2.4300000000000002,"5min":1.23,"15min":0.80500000000000005},"min":{"1min":0.35599999999999998,"5min":0.314,"15min":0.314},"max":{"1min":7.8719999999999999,"5min":7.8719999999999999,"15min":7.8719999999999999},"last":5.8070000000000004},{"interface":"front","average":{"1min":2.173,"5min":1.294,"15min":0.82199999999999995},"min":{"1min":0.33900000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.0259999999999998,"5min":12.779999999999999,"15min":12.779999999999999},"last":2.9430000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:35:32 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":0.877,"15min":0.60599999999999998},"min":{"1min":0.30499999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":9.2729999999999997,"5min":9.3510000000000009,"15min":9.3510000000000009},"last":0.58199999999999996},{"interface":"front","average":{"1min":2.194,"5min":1.111,"15min":0.71299999999999997},"min":{"1min":0.32000000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":14.236000000000001,"5min":14.236000000000001,"15min":14.236000000000001},"last":5.1849999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:36:01 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":2.3039999999999998,"15min":1.1220000000000001},"min":{"1min":0.41299999999999998,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":14.282999999999999,"5min":95.010999999999996,"15min":95.010999999999996},"last":2.8519999999999999},{"interface":"front","average":{"1min":2.649,"5min":2.1040000000000001,"15min":1.0629999999999999},"min":{"1min":0.34200000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.5979999999999999,"5min":94.194000000000003,"15min":94.194000000000003},"last":5.8319999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:35:42 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.2140000000000004,"5min":8.2140000000000004,"15min":8.2140000000000004},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.1230000000000002,"5min":2.1230000000000002,"15min":2.1230000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":14.432,"5min":14.432,"15min":14.432},"last":2.9079999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":251002880,"data_stored":251002880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:36:39.613 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:36:39.613 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 46 2023-10-09T05:36:39.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:39.615 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:39.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:39.660 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:39.716 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:39.744 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:39.796 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:39.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:39.901 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:39.930 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:39.978 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:39.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:39.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:40.082 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:40.109 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:40.157 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:40.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:40.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:40.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:40.288 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:40.334 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:40.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:40.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:40.438 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:40.468 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:40.513 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:40.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:40.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:40.616 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:40.643 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:40.689 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:40.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:40.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:40.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:36:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:36:40] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-09T05:36:40.793 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:40.821 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:40.895 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:40.920 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:40.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:40.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:41.024 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:41.054 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:41.069 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:41 smithi008 ceph-mon[184418]: pgmap v397: 109 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 14583/1359 objects degraded (1073.068%); 0 B/s, 306 objects/s recovering 2023-10-09T05:36:41.069 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:41 smithi008 ceph-mon[191227]: pgmap v397: 109 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 14583/1359 objects degraded (1073.068%); 0 B/s, 306 objects/s recovering 2023-10-09T05:36:41.070 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:41 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1358870722' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:41.070 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:41 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:41.082 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:36:41.101 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:41.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:41.205 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:41.237 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:41 smithi191 ceph-mon[161669]: pgmap v397: 109 pgs: 1 active+clean+scrubbing, 1 active+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 14583/1359 objects degraded (1073.068%); 0 B/s, 306 objects/s recovering 2023-10-09T05:36:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:41 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1358870722' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:41 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:41 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1358870722' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:41 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:42.038 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:42.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:42.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:42.119 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:36:42.138 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:36:42.143 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:42.157 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:36:42.172 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:42.177 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:36:42.195 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:36:42.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:42 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:36:42.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:42 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:42.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:42 smithi008 ceph-mon[184418]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T05:36:42.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:42 smithi008 ceph-mon[184418]: failed to encode map e151 with expected crc 2023-10-09T05:36:42.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:42 smithi008 ceph-mon[184418]: failed to encode map e151 with expected crc 2023-10-09T05:36:42.208 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:42 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:36:42.208 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:42 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:42.208 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:42 smithi008 ceph-mon[191227]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T05:36:42.208 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:42 smithi008 ceph-mon[191227]: failed to encode map e151 with expected crc 2023-10-09T05:36:42.208 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:42 smithi008 ceph-mon[191227]: failed to encode map e151 with expected crc 2023-10-09T05:36:42.214 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:36:42.224 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:42.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:42.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:42.232 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:36:42.249 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:36:42.250 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:36:42.251 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:36:42.251 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:36:42.251 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:36:42.251 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:36:42.251 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:36:42.251 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:36:42.251 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:36:42.329 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:42.358 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:42.420 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:42.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:42.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:42.524 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:42 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:36:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:42 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:42 smithi191 ceph-mon[161669]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T05:36:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:42 smithi191 ceph-mon[161669]: failed to encode map e151 with expected crc 2023-10-09T05:36:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:42 smithi191 ceph-mon[161669]: failed to encode map e151 with expected crc 2023-10-09T05:36:42.553 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:42.597 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:42.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:42.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:42.700 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:42.728 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:43.179 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:43.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:43.263 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:43 smithi008 ceph-mon[184418]: pgmap v399: 109 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 14583/1359 objects degraded (1073.068%); 0 B/s, 408 objects/s recovering 2023-10-09T05:36:43.263 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:43 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3070638564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:43.264 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14583/1359 objects degraded (1073.068%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:43.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:43 smithi008 ceph-mon[191227]: pgmap v399: 109 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 14583/1359 objects degraded (1073.068%); 0 B/s, 408 objects/s recovering 2023-10-09T05:36:43.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:43 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3070638564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:43.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14583/1359 objects degraded (1073.068%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:43.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:43.310 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:43.352 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:43.354 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:43 smithi191 ceph-mon[161669]: pgmap v399: 109 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 14583/1359 objects degraded (1073.068%); 0 B/s, 408 objects/s recovering 2023-10-09T05:36:43.354 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:43 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3070638564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:43.354 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14583/1359 objects degraded (1073.068%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:43.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:43.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:43.456 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:43.482 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:43.651 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:43.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:43.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:43.755 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:43.787 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:43.840 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:43.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:43.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:43.940 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:36:43.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:43.972 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:44.024 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:44.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:44.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:44.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[191227]: 5.15 deep-scrub starts 2023-10-09T05:36:44.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[191227]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[191227]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[191227]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[191227]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T05:36:44.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[191227]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.082 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:44.082 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:44.082 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/185644145' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:36:44.082 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:36:44.101 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:36:44.101 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:36:44.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:44.157 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:44.260 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:44 smithi191 ceph-mon[161669]: 5.15 deep-scrub starts 2023-10-09T05:36:44.260 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:44 smithi191 ceph-mon[161669]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.260 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:44 smithi191 ceph-mon[161669]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.260 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:44 smithi191 ceph-mon[161669]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.260 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:44 smithi191 ceph-mon[161669]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T05:36:44.260 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:44 smithi191 ceph-mon[161669]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.260 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:44.260 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:44.261 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/185644145' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:36:44.261 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:44 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:36:44.321 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:44.355 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:36:44.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:44.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:44.459 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:44.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[184418]: 5.15 deep-scrub starts 2023-10-09T05:36:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[184418]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[184418]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[184418]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[184418]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T05:36:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[184418]: failed to encode map e152 with expected crc 2023-10-09T05:36:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/185644145' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:36:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:44 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:36:44.485 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:44.525 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:44.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:44.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:44.626 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 5 pg_num to 46 2023-10-09T05:36:44.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:44.657 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:44.839 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:44.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:44.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:44.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:36:44.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:44.970 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:45.014 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:45.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:45.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:45.119 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:45.145 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:45 smithi191 ceph-mon[161669]: 5.13 scrub starts 2023-10-09T05:36:45.146 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:45 smithi191 ceph-mon[161669]: pgmap v401: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14583/1359 objects degraded (1073.068%); 0 B/s, 196 objects/s recovering 2023-10-09T05:36:45.146 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:45 smithi191 ceph-mon[161669]: failed to encode map e153 with expected crc 2023-10-09T05:36:45.146 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:45 smithi191 ceph-mon[161669]: failed to encode map e153 with expected crc 2023-10-09T05:36:45.146 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:45 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]': finished 2023-10-09T05:36:45.146 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:45 smithi191 ceph-mon[161669]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T05:36:45.148 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:45.196 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:45.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:45.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[184418]: 5.13 scrub starts 2023-10-09T05:36:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[184418]: pgmap v401: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14583/1359 objects degraded (1073.068%); 0 B/s, 196 objects/s recovering 2023-10-09T05:36:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[184418]: failed to encode map e153 with expected crc 2023-10-09T05:36:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[184418]: failed to encode map e153 with expected crc 2023-10-09T05:36:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]': finished 2023-10-09T05:36:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[184418]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T05:36:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[191227]: 5.13 scrub starts 2023-10-09T05:36:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[191227]: pgmap v401: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14583/1359 objects degraded (1073.068%); 0 B/s, 196 objects/s recovering 2023-10-09T05:36:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[191227]: failed to encode map e153 with expected crc 2023-10-09T05:36:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[191227]: failed to encode map e153 with expected crc 2023-10-09T05:36:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]': finished 2023-10-09T05:36:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:45 smithi008 ceph-mon[191227]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T05:36:45.266 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:36:45.295 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:45.303 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:45.338 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:46.342 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:36:46.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:36:46.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:36:46.835 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:36:46.836 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:36:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:47 smithi191 ceph-mon[161669]: 5.3 scrub starts 2023-10-09T05:36:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:47 smithi191 ceph-mon[161669]: pgmap v403: 109 pgs: 1 active+clean+scrubbing+deep, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13149/1359 objects degraded (967.550%); 0 B/s, 238 objects/s recovering 2023-10-09T05:36:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:47 smithi191 ceph-mon[161669]: failed to encode map e154 with expected crc 2023-10-09T05:36:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]': finished 2023-10-09T05:36:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:47 smithi191 ceph-mon[161669]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T05:36:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:47 smithi191 ceph-mon[161669]: failed to encode map e154 with expected crc 2023-10-09T05:36:47.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[184418]: 5.3 scrub starts 2023-10-09T05:36:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[184418]: pgmap v403: 109 pgs: 1 active+clean+scrubbing+deep, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13149/1359 objects degraded (967.550%); 0 B/s, 238 objects/s recovering 2023-10-09T05:36:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[184418]: failed to encode map e154 with expected crc 2023-10-09T05:36:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]': finished 2023-10-09T05:36:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[184418]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T05:36:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[184418]: failed to encode map e154 with expected crc 2023-10-09T05:36:47.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[191227]: 5.3 scrub starts 2023-10-09T05:36:47.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[191227]: pgmap v403: 109 pgs: 1 active+clean+scrubbing+deep, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13149/1359 objects degraded (967.550%); 0 B/s, 238 objects/s recovering 2023-10-09T05:36:47.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[191227]: failed to encode map e154 with expected crc 2023-10-09T05:36:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]': finished 2023-10-09T05:36:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[191227]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T05:36:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:47 smithi008 ceph-mon[191227]: failed to encode map e154 with expected crc 2023-10-09T05:36:47.709 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:47.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:47.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:47.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:47.849 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:47.991 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T05:36:47.991 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T05:36:48.063 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:48.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:48.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: 5.23 scrub starts 2023-10-09T05:36:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T05:36:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: 5.c scrub starts 2023-10-09T05:36:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13149/1359 objects degraded (967.550%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T05:36:48.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:48.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:48.116 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:36:48.166 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:48.199 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:48.207 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:48.256 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:48.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:48.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:48.277 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: 5.23 scrub starts 2023-10-09T05:36:48.277 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T05:36:48.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: 5.c scrub starts 2023-10-09T05:36:48.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13149/1359 objects degraded (967.550%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:48.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.279 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.279 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T05:36:48.279 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:48.279 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:48.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: 5.23 scrub starts 2023-10-09T05:36:48.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T05:36:48.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: failed to encode map e155 with expected crc 2023-10-09T05:36:48.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: 5.c scrub starts 2023-10-09T05:36:48.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13149/1359 objects degraded (967.550%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:48.281 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.282 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.282 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.282 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: failed to encode map e156 with expected crc 2023-10-09T05:36:48.282 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T05:36:48.282 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:48.282 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:48.360 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:48.391 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:48.447 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:48.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:48.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:48.552 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:48.587 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:48.649 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:48.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:48.695 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:48.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:48.787 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:48.800 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:48.944 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:36:48.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:48.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:49.049 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:49.078 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:49.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[184418]: 5.5 deep-scrub starts 2023-10-09T05:36:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[184418]: pgmap v407: 119 pgs: 4 peering, 2 remapped+peering, 9 unknown, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 29 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7907/1269 objects degraded (623.089%); 0 B/s, 482 objects/s recovering 2023-10-09T05:36:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[184418]: failed to encode map e156 with expected crc 2023-10-09T05:36:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[184418]: 5.2c scrub starts 2023-10-09T05:36:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[184418]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[184418]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T05:36:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[184418]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[184418]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[184418]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3194702403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[191227]: 5.5 deep-scrub starts 2023-10-09T05:36:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[191227]: pgmap v407: 119 pgs: 4 peering, 2 remapped+peering, 9 unknown, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 29 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7907/1269 objects degraded (623.089%); 0 B/s, 482 objects/s recovering 2023-10-09T05:36:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[191227]: failed to encode map e156 with expected crc 2023-10-09T05:36:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[191227]: 5.2c scrub starts 2023-10-09T05:36:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[191227]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[191227]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T05:36:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[191227]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[191227]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[191227]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3194702403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:49.357 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:49 smithi191 ceph-mon[161669]: 5.5 deep-scrub starts 2023-10-09T05:36:49.357 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:49 smithi191 ceph-mon[161669]: pgmap v407: 119 pgs: 4 peering, 2 remapped+peering, 9 unknown, 3 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 29 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7907/1269 objects degraded (623.089%); 0 B/s, 482 objects/s recovering 2023-10-09T05:36:49.357 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:49 smithi191 ceph-mon[161669]: failed to encode map e156 with expected crc 2023-10-09T05:36:49.358 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:49 smithi191 ceph-mon[161669]: 5.2c scrub starts 2023-10-09T05:36:49.358 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:49 smithi191 ceph-mon[161669]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.358 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:49 smithi191 ceph-mon[161669]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T05:36:49.358 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:49 smithi191 ceph-mon[161669]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.358 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:49 smithi191 ceph-mon[161669]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.358 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:49 smithi191 ceph-mon[161669]: failed to encode map e157 with expected crc 2023-10-09T05:36:49.358 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3194702403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:36:49.377 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:49.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:49.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:49.480 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:49.512 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:49.576 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:49.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:49.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:49.631 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:36:49.681 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:49.713 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:50 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:50 smithi008 ceph-mon[184418]: failed to encode map e158 with expected crc 2023-10-09T05:36:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:50 smithi008 ceph-mon[184418]: failed to encode map e158 with expected crc 2023-10-09T05:36:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3194702403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:36:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:50 smithi008 ceph-mon[184418]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T05:36:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:50 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:50 smithi008 ceph-mon[191227]: failed to encode map e158 with expected crc 2023-10-09T05:36:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:50 smithi008 ceph-mon[191227]: failed to encode map e158 with expected crc 2023-10-09T05:36:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3194702403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:36:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:50 smithi008 ceph-mon[191227]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T05:36:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:50 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:50 smithi191 ceph-mon[161669]: failed to encode map e158 with expected crc 2023-10-09T05:36:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:50 smithi191 ceph-mon[161669]: failed to encode map e158 with expected crc 2023-10-09T05:36:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3194702403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:36:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:50 smithi191 ceph-mon[161669]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T05:36:50.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:36:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:36:50] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.33.4" 2023-10-09T05:36:51.325 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:36:51.325 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:36:51.326 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:36:51.331 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:36:51.351 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:36:51.370 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:36:51.378 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:51 smithi008 ceph-mon[184418]: failed to encode map e158 with expected crc 2023-10-09T05:36:51.378 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:51 smithi008 ceph-mon[184418]: pgmap v410: 119 pgs: 4 peering, 2 remapped+peering, 6 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 104 active+clean; 29 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6906/1269 objects degraded (544.208%); 0 B/s, 382 objects/s recovering 2023-10-09T05:36:51.378 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:51 smithi008 ceph-mon[191227]: failed to encode map e158 with expected crc 2023-10-09T05:36:51.378 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:51 smithi008 ceph-mon[191227]: pgmap v410: 119 pgs: 4 peering, 2 remapped+peering, 6 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 104 active+clean; 29 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6906/1269 objects degraded (544.208%); 0 B/s, 382 objects/s recovering 2023-10-09T05:36:51.389 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:36:51.410 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:36:51.412 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:36:51.412 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:36:51.412 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:36:51.412 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:36:51.412 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:36:51.412 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:36:51.412 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:36:51.412 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:36:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:51 smithi191 ceph-mon[161669]: failed to encode map e158 with expected crc 2023-10-09T05:36:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:51 smithi191 ceph-mon[161669]: pgmap v410: 119 pgs: 4 peering, 2 remapped+peering, 6 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 104 active+clean; 29 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6906/1269 objects degraded (544.208%); 0 B/s, 382 objects/s recovering 2023-10-09T05:36:52.161 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:52.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:52.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:52.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:52.299 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:52.464 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:52.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/912835856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:52.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/912835856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:52.466 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/912835856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:52.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:52.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:52.568 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:52.607 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:52.666 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:52.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:52.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:52.769 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:36:52.798 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:52.844 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:36:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:52.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:36:52.942 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:36:52.942 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:36:52.942 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:36:52.942 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:36:52.942 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.942 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54043 5.1 5:88ab6307:::benchmark_data_smithi008_238708_object54042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:36:52.942 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.530687+0000", 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 212.40544897300001, 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0331553470000001, 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54043 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.943 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530687+0000", 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530687+0000", 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530685+0000", 2023-10-09T05:36:52.944 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0238999999999999e-05 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530726+0000", 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5817000000000001e-05 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530741+0000", 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3200000000000002e-06 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.945 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.946 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:52.946 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530749+0000", 2023-10-09T05:36:52.946 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0053712409999999997 2023-10-09T05:36:52.946 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.946 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.946 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.946 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536120+0000", 2023-10-09T05:36:52.946 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1669999999999999e-06 2023-10-09T05:36:52.946 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.946 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.946 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536128+0000", 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0145035060000001 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.550632+0000", 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115936 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.550747+0000", 2023-10-09T05:36:52.947 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014306200000000001 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.550891+0000", 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059699740000000003 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.556861+0000", 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046747400000000001 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.561535+0000", 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022096709999999999 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563745+0000", 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7768000000000003e-05 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563843+0000", 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0331553470000001 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54047 5.1 5:8c453526:::benchmark_data_smithi008_238708_object54046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.534316+0000", 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 212.40182050799999, 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.029737098, 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54047 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534316+0000", 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534316+0000", 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534314+0000", 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6623999999999999e-05 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534360+0000", 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0910000000000001e-05 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534371+0000", 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6819999999999997e-06 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534378+0000", 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001771051 2023-10-09T05:36:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536149+0000", 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5040000000000002e-06 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536155+0000", 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.018304179 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554459+0000", 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014521 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554604+0000", 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019115700000000001 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554795+0000", 2023-10-09T05:36:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0091080610000000006 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563903+0000", 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1428e-05 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563915+0000", 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3801000000000002e-05 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563968+0000", 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4501999999999998e-05 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564053+0000", 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.029737098 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54057 5.1 5:851293d1:::benchmark_data_smithi008_238708_object54056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:36:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.550289+0000", 2023-10-09T05:36:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 212.385847058, 2023-10-09T05:36:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.021774086, 2023-10-09T05:36:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54057 2023-10-09T05:36:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550289+0000", 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550289+0000", 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550287+0000", 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2347e-05 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550330+0000", 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2855000000000001e-05 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550353+0000", 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4370000000000001e-06 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550360+0000", 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5225e-05 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550425+0000", 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0944999999999999e-05 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550436+0000", 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.006471347 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.556908+0000", 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014533999999999999 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.557053+0000", 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00025040400000000002 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.557303+0000", 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068016099999999996 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564105+0000", 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078524939999999998 2023-10-09T05:36:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.571957+0000", 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6785e-05 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572004+0000", 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9092000000000003e-05 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572063+0000", 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.021774086 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54062 5.1 5:890b0b94:::benchmark_data_smithi008_238708_object54061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.563895+0000", 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 212.37224103, 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082578959999999, 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54062 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563895+0000", 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563895+0000", 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563894+0000", 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4879000000000001e-05 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563949+0000", 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6708999999999999e-05 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563966+0000", 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9090000000000003e-06 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563972+0000", 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7568000000000002e-05 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.564030+0000", 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0219e-05 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.564040+0000", 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99530809600000003 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.559348+0000", 2023-10-09T05:36:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011862999999999999 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.559467+0000", 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017885700000000001 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.559646+0000", 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0081993729999999994 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.567845+0000", 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042279680000000004 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572073+0000", 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1065999999999997e-05 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572104+0000", 2023-10-09T05:36:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8968000000000003e-05 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572153+0000", 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082578959999999 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54068 5.1 5:8319935a:::benchmark_data_smithi008_238708_object54067:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:36:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.569507+0000", 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 212.36662973200001, 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082394779999999, 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54068 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569507+0000", 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569507+0000", 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569505+0000", 2023-10-09T05:36:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8206999999999999e-05 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569533+0000", 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3752e-05 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569547+0000", 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0120000000000002e-06 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569553+0000", 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3130999999999997e-05 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569606+0000", 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4290000000000001e-06 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569614+0000", 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99451796599999998 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564132+0000", 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9108e-05 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564232+0000", 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115476 2023-10-09T05:36:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564347+0000", 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078146750000000001 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572162+0000", 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0053927940000000002 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.577554+0000", 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8337999999999995e-05 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.577623+0000", 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012327700000000001 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.577746+0000", 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082394779999999 2023-10-09T05:36:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54070 5.1 5:819fd021:::benchmark_data_smithi008_238708_object54069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.571140+0000", 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 212.364996593, 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.009647422, 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54070 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571140+0000", 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571140+0000", 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571138+0000", 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9555999999999998e-05 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571188+0000", 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.417e-05 2023-10-09T05:36:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571202+0000", 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7789999999999998e-06 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571209+0000", 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6211000000000002e-05 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571255+0000", 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2060000000000001e-06 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571262+0000", 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99464801999999997 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.565910+0000", 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000141688 2023-10-09T05:36:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.566052+0000", 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000160173 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.566212+0000", 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059532430000000004 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572165+0000", 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0084587169999999993 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.580624+0000", 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9176999999999999e-05 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.580683+0000", 2023-10-09T05:36:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010398 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.580787+0000", 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.009647422 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54080 5.1 5:84267530:::benchmark_data_smithi008_238708_object54079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.584125+0000", 2023-10-09T05:36:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 212.352011772, 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99768334199999997, 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54080 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584125+0000", 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584125+0000", 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584123+0000", 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5760000000000001e-05 2023-10-09T05:36:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584149+0000", 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3940999999999999e-05 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584163+0000", 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6529999999999994e-06 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584171+0000", 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0436999999999998e-05 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584221+0000", 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.153e-06 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584228+0000", 2023-10-09T05:36:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98363814599999999 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.567866+0000", 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013288699999999999 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.567999+0000", 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015074399999999999 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.568150+0000", 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040185330000000003 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572168+0000", 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0094922050000000001 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.581661+0000", 2023-10-09T05:36:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6870000000000003e-05 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.581717+0000", 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0416999999999998e-05 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.581808+0000", 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99768334199999997 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119018 5.e 5:7572eb82:::benchmark_data_smithi008_238708_object882:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.007070+0000", 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 148.92906587900001, 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0282413180000001, 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119018 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007070+0000", 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007070+0000", 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007068+0000", 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2640000000000008e-06 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007077+0000", 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5773999999999998e-05 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007092+0000", 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9e-06 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007099+0000", 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5432000000000005e-05 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007185+0000", 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9950000000000005e-06 2023-10-09T05:36:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007193+0000", 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.248123126 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255316+0000", 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8770000000000004e-06 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255323+0000", 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.77988959099999999 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035212+0000", 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9400000000000004e-05 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035312+0000", 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0282413180000001 2023-10-09T05:36:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119020 5.1e 5:7d0c0384:::benchmark_data_smithi008_238708_object884:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.007113+0000", 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 148.92902344000001, 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0326345889999999, 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119020 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007113+0000", 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007113+0000", 2023-10-09T05:36:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007112+0000", 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.49e-06 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007116+0000", 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.498e-06 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007123+0000", 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.788e-06 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007127+0000", 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.248267246 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255395+0000", 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5020000000000001e-06 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255398+0000", 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78425046499999995 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.039648+0000", 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9006999999999996e-05 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.039747+0000", 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0326345889999999 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119026 5.e 5:7513f17c:::benchmark_data_smithi008_238708_object890:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.007132+0000", 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 148.92900466, 2023-10-09T05:36:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0283243520000001, 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119026 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007132+0000", 2023-10-09T05:36:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007132+0000", 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007131+0000", 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.322e-06 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007134+0000", 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5490000000000003e-06 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007140+0000", 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4379999999999999e-06 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007144+0000", 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00092436700000000005 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.008068+0000", 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.917e-06 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.008075+0000", 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.247270726 2023-10-09T05:36:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255346+0000", 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2480000000000002e-06 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255355+0000", 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78001380399999998 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035369+0000", 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7331999999999997e-05 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035456+0000", 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0283243520000001 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163684 5.21 5:84c14237:::benchmark_data_smithi008_238708_object45548:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.609434+0000", 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 105.326702048, 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99344012400000004, 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163684 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609434+0000", 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609434+0000", 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609433+0000", 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.511e-06 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609440+0000", 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2476999999999999e-05 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609452+0000", 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0360000000000001e-06 2023-10-09T05:36:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609459+0000", 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9319000000000004e-05 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609539+0000", 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0864e-05 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609549+0000", 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99053912899999996 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.600089+0000", 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103931 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.600193+0000", 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000164618 2023-10-09T05:36:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.600357+0000", 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023727349999999999 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602730+0000", 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1140000000000003e-05 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602771+0000", 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7978e-05 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602789+0000", 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5139000000000002e-05 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602844+0000", 2023-10-09T05:36:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0287e-05 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602874+0000", 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99344012400000004 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163704 5.21 5:87d25abf:::benchmark_data_smithi008_238708_object45568:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:36:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.629811+0000", 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 105.30632576399999, 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97411889600000001, 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163704 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629811+0000", 2023-10-09T05:36:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629811+0000", 2023-10-09T05:36:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629810+0000", 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.638e-06 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629813+0000", 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8369999999999996e-06 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629817+0000", 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4340000000000001e-06 2023-10-09T05:36:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629821+0000", 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9418999999999994e-05 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629890+0000", 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0139999999999998e-06 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629898+0000", 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97170687300000003 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.601605+0000", 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6179000000000002e-05 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.601701+0000", 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019179700000000001 2023-10-09T05:36:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.601893+0000", 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001016667 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602910+0000", 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6458999999999998e-05 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602946+0000", 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091263200000000003 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603859+0000", 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7240999999999998e-05 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603896+0000", 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3331000000000003e-05 2023-10-09T05:36:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603929+0000", 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97411889600000001 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163753 5.21 5:845d2bea:::benchmark_data_smithi008_238708_object45617:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.676299+0000", 2023-10-09T05:36:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 105.259837069, 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92971798100000003, 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163753 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676299+0000", 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676299+0000", 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676298+0000", 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4119999999999998e-06 2023-10-09T05:36:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676304+0000", 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4579999999999999e-05 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676319+0000", 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5139999999999998e-06 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676325+0000", 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0550999999999998e-05 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676376+0000", 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6689999999999995e-06 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676385+0000", 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92657416500000001 2023-10-09T05:36:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602959+0000", 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2798000000000003e-05 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603031+0000", 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013177499999999999 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603163+0000", 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027447750000000001 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605908+0000", 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7929999999999997e-05 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605956+0000", 2023-10-09T05:36:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4289999999999998e-06 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605962+0000", 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1736000000000002e-05 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605994+0000", 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3096999999999999e-05 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606017+0000", 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92971798100000003 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163923 5.21 5:843e313c:::benchmark_data_smithi008_238708_object45787:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.853413+0000", 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 105.082723485, 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.75769858499999998, 2023-10-09T05:36:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163923 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853413+0000", 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853413+0000", 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853412+0000", 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0589999999999999e-06 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853418+0000", 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2941e-05 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853431+0000", 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7400000000000001e-06 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853436+0000", 2023-10-09T05:36:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7589e-05 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853494+0000", 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2001000000000001e-05 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853506+0000", 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.75046967899999995 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603976+0000", 2023-10-09T05:36:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109487 2023-10-09T05:36:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.604085+0000", 2023-10-09T05:36:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019979399999999999 2023-10-09T05:36:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:36:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.604285+0000", 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001741006 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606026+0000", 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4724e-05 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606081+0000", 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004942559 2023-10-09T05:36:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611023+0000", 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8794000000000001e-05 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611072+0000", 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9496000000000003e-05 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611111+0000", 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.75769858499999998 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164030 5.21 5:866422a9:::benchmark_data_smithi008_238708_object45894:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 104.98758585900001, 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66267309900000004, 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164030 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7736000000000001e-05 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948567+0000", 2023-10-09T05:36:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8230999999999999e-05 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948585+0000", 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3960000000000003e-06 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948592+0000", 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.498e-05 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948637+0000", 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8820000000000003e-06 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948644+0000", 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65639657299999998 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605040+0000", 2023-10-09T05:36:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7799000000000003e-05 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605138+0000", 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013080500000000001 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605269+0000", 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017788770000000001 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607048+0000", 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012428719999999999 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608291+0000", 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002871307 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611162+0000", 2023-10-09T05:36:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2058999999999999e-05 2023-10-09T05:36:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611194+0000", 2023-10-09T05:36:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9536999999999999e-05 2023-10-09T05:36:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611224+0000", 2023-10-09T05:36:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66267309900000004 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164042 5.21 5:85682928:::benchmark_data_smithi008_238708_object45906:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.959315+0000", 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 104.97682156099999, 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65788200699999999, 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164042 2023-10-09T05:36:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959315+0000", 2023-10-09T05:36:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959315+0000", 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959314+0000", 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.07e-06 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959318+0000", 2023-10-09T05:36:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7539999999999995e-06 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959327+0000", 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.23e-06 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959330+0000", 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1047999999999998e-05 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959361+0000", 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0820000000000004e-06 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959367+0000", 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.64674576399999995 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606113+0000", 2023-10-09T05:36:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0732e-05 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606184+0000", 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013076400000000001 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606315+0000", 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048103449999999997 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611125+0000", 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014353599999999999 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611268+0000", 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0058299170000000004 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617098+0000", 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4318999999999999e-05 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617153+0000", 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4100000000000001e-05 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617197+0000", 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65788200699999999 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164098 5.21 5:85741b6f:::benchmark_data_smithi008_238708_object45962:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.014979+0000", 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 104.921157689, 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.60237710200000005, 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164098 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014979+0000", 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014979+0000", 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014977+0000", 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6270000000000002e-06 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014984+0000", 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2714000000000001e-05 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014996+0000", 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2259999999999996e-06 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.015002+0000", 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5050999999999999e-05 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.015057+0000", 2023-10-09T05:36:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0011e-05 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.015067+0000", 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.59200529499999999 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607072+0000", 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7960999999999994e-05 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607160+0000", 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017280900000000001 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607333+0000", 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037971039999999999 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611130+0000", 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000167384 2023-10-09T05:36:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611297+0000", 2023-10-09T05:36:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059718460000000003 2023-10-09T05:36:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617269+0000", 2023-10-09T05:36:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0393999999999999e-05 2023-10-09T05:36:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617320+0000", 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6199999999999999e-05 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617356+0000", 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.60237710200000005 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164190 5.21 5:85c1fc08:::benchmark_data_smithi008_238708_object46054:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.100063+0000", 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 104.836073277, 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.517522814, 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164190 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100063+0000", 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100063+0000", 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100062+0000", 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2819999999999996e-06 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100070+0000", 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5238000000000001e-05 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100095+0000", 2023-10-09T05:36:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2259999999999999e-06 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100102+0000", 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3122999999999999e-05 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100166+0000", 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1447000000000001e-05 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100177+0000", 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.50813475200000002 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608312+0000", 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102922 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608415+0000", 2023-10-09T05:36:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019721800000000001 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608612+0000", 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026200059999999998 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611232+0000", 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3609000000000004e-05 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611326+0000", 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0061575960000000004 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617483+0000", 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1966999999999999e-05 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617535+0000", 2023-10-09T05:36:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0797999999999997e-05 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617586+0000", 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.517522814 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164259 5.21 5:8505256a:::benchmark_data_smithi008_238708_object46123:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.167748+0000", 2023-10-09T05:36:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 104.768388512, 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45063465699999999, 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164259 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167748+0000", 2023-10-09T05:36:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167748+0000", 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167746+0000", 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4189999999999996e-06 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167753+0000", 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4399999999999999e-05 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167768+0000", 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6309999999999997e-06 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167774+0000", 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7620000000000006e-05 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167852+0000", 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0051000000000001e-05 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167862+0000", 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.441777105 2023-10-09T05:36:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.609639+0000", 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105538 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.609744+0000", 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018787400000000001 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:36:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.609932+0000", 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041432689999999998 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.614076+0000", 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033595819999999998 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617435+0000", 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00086552299999999999 2023-10-09T05:36:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618301+0000", 2023-10-09T05:36:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7574999999999999e-05 2023-10-09T05:36:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618338+0000", 2023-10-09T05:36:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4168000000000001e-05 2023-10-09T05:36:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618382+0000", 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45063465699999999 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164297 5.21 5:85a0851f:::benchmark_data_smithi008_238708_object46161:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.207114+0000", 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 104.729022779, 2023-10-09T05:36:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.41141557899999998, 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164297 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207114+0000", 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:36:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:36:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207114+0000", 2023-10-09T05:36:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:36:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:36:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207112+0000", 2023-10-09T05:36:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5089999999999999e-06 2023-10-09T05:36:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207118+0000", 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1423000000000001e-05 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207130+0000", 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7730000000000002e-06 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:36:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207137+0000", 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011262379999999999 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.208263+0000", 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1843e-05 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.208275+0000", 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.40307073700000001 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611345+0000", 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6700999999999995e-05 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611442+0000", 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00022087699999999999 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:36:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611663+0000", 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002453359 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.614116+0000", 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033269020000000001 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617443+0000", 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001025317 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618469+0000", 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4535000000000001e-05 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618493+0000", 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6001000000000003e-05 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618529+0000", 2023-10-09T05:36:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.41141557899999998 2023-10-09T05:36:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:36:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:36:53.091 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:36:53.091 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:53.092 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:36:53.092 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:36:53.145 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:53.194 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:53.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:53.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:53.298 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:36:53.326 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:53.392 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:53 smithi191 ceph-mon[161669]: pgmap v411: 119 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 112 active+clean; 31 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 47 MiB/s wr, 1.49k op/s; 11149/1359 objects degraded (820.383%); 0 B/s, 305 objects/s recovering 2023-10-09T05:36:53.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 11149/1359 objects degraded (820.383%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:53.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:53 smithi191 ceph-mon[161669]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:53 smithi191 ceph-mon[161669]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:53 smithi191 ceph-mon[161669]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:53 smithi191 ceph-mon[161669]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T05:36:53.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:53 smithi191 ceph-mon[161669]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.393 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3362996520' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:36:53.396 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:36:53.422 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[184418]: pgmap v411: 119 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 112 active+clean; 31 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 47 MiB/s wr, 1.49k op/s; 11149/1359 objects degraded (820.383%); 0 B/s, 305 objects/s recovering 2023-10-09T05:36:53.422 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 11149/1359 objects degraded (820.383%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:53.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[184418]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[184418]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[184418]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[184418]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T05:36:53.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[184418]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3362996520' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:36:53.423 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[191227]: pgmap v411: 119 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 112 active+clean; 31 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 47 MiB/s wr, 1.49k op/s; 11149/1359 objects degraded (820.383%); 0 B/s, 305 objects/s recovering 2023-10-09T05:36:53.423 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 11149/1359 objects degraded (820.383%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:36:53.423 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[191227]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.424 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[191227]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.424 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[191227]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.424 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[191227]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T05:36:53.424 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[191227]: failed to encode map e159 with expected crc 2023-10-09T05:36:53.424 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3362996520' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:36:53.424 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:36:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:53.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:36:53.527 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:53.558 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:53.615 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:53.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:53.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:53.639 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_0' removed 2023-10-09T05:36:53.719 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:53.826 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:54.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[191227]: failed to encode map e159 with expected crc 2023-10-09T05:36:54.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[191227]: failed to encode map e160 with expected crc 2023-10-09T05:36:54.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[191227]: failed to encode map e160 with expected crc 2023-10-09T05:36:54.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3362996520' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:36:54.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[191227]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T05:36:54.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[191227]: failed to encode map e160 with expected crc 2023-10-09T05:36:54.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[184418]: failed to encode map e159 with expected crc 2023-10-09T05:36:54.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[184418]: failed to encode map e160 with expected crc 2023-10-09T05:36:54.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[184418]: failed to encode map e160 with expected crc 2023-10-09T05:36:54.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3362996520' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:36:54.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[184418]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T05:36:54.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:54 smithi008 ceph-mon[184418]: failed to encode map e160 with expected crc 2023-10-09T05:36:54.262 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:36:54.449 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:54.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:54 smithi191 ceph-mon[161669]: failed to encode map e159 with expected crc 2023-10-09T05:36:54.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:54 smithi191 ceph-mon[161669]: failed to encode map e160 with expected crc 2023-10-09T05:36:54.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:54 smithi191 ceph-mon[161669]: failed to encode map e160 with expected crc 2023-10-09T05:36:54.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:54 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3362996520' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:36:54.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:54 smithi191 ceph-mon[161669]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T05:36:54.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:54 smithi191 ceph-mon[161669]: failed to encode map e160 with expected crc 2023-10-09T05:36:54.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:54.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:54.521 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:36:54.553 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:54.581 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:54.885 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:54.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:54.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:54.938 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:54.989 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:55.019 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:55.063 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:55.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:55.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:55.167 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:55.171 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:55 smithi008 ceph-mon[184418]: failed to encode map e160 with expected crc 2023-10-09T05:36:55.171 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:55 smithi008 ceph-mon[184418]: pgmap v414: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:36:55.171 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:55 smithi008 ceph-mon[184418]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11149/1359 objects degraded (820.383%), 7 pgs degraded, 3 pgs undersized) 2023-10-09T05:36:55.171 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:55 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:55.171 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:55 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:55.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:55 smithi008 ceph-mon[191227]: failed to encode map e160 with expected crc 2023-10-09T05:36:55.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:55 smithi008 ceph-mon[191227]: pgmap v414: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:36:55.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:55 smithi008 ceph-mon[191227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11149/1359 objects degraded (820.383%), 7 pgs degraded, 3 pgs undersized) 2023-10-09T05:36:55.172 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:55 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:55.172 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:55 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:55.172 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:36:55.173 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:36:55.173 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T05:36:55.173 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T05:36:55.202 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:55.217 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:55.263 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:55.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:55.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:55 smithi191 ceph-mon[161669]: failed to encode map e160 with expected crc 2023-10-09T05:36:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:55 smithi191 ceph-mon[161669]: pgmap v414: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:36:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:55 smithi191 ceph-mon[161669]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11149/1359 objects degraded (820.383%), 7 pgs degraded, 3 pgs undersized) 2023-10-09T05:36:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:55 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:55 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:55.351 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:55.372 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:55.401 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:55.547 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:55.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:55.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:55.651 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:55.683 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:55.837 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:55.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:55.941 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:55.969 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:56.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:56.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:56.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:56.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:56.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:56.348 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:36:56.349 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:36:56.349 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T05:36:56.349 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:36:56.415 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:56.416 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:56.416 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:56.416 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:56.416 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:56.416 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:56.416 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:56.416 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:56.416 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:56.417 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:56.870 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:56.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:56.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:56.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:36:56.974 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:57.009 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:57.069 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:57.160 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:36:57.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:57.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:57.264 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:57.293 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:57.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:57 smithi191 ceph-mon[161669]: pgmap v415: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:36:57.303 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:57.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:57 smithi008 ceph-mon[184418]: pgmap v415: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:36:57.424 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:57 smithi008 ceph-mon[191227]: pgmap v415: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:36:57.496 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:57.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:57.600 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:36:57.631 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:57.691 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:36:57.714 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:36:57.735 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:36:57.754 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:36:57.772 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:36:57.790 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:36:57.809 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:36:57.830 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:36:57.832 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:36:57.832 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:36:57.832 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:36:57.832 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:36:57.832 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:36:57.832 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:36:57.832 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:36:57.833 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:36:57.881 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:36:57.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:57.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:36:57.986 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:58.014 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:58.138 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:36:58.405 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4200658320' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.405 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:58 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.405 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3095962382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:58.406 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4200658320' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.406 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:58 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.406 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3095962382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:58.415 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:36:58.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:58.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:36:58.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4200658320' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:58 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:36:58.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3095962382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:36:58.519 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:36:58.547 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:58.668 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:36:58.668 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:36:59.130 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[191227]: pgmap v416: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:36:59.130 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:59.130 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[191227]: failed to encode map e161 with expected crc 2023-10-09T05:36:59.130 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[191227]: failed to encode map e161 with expected crc 2023-10-09T05:36:59.130 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:59.130 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[191227]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T05:36:59.130 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[191227]: failed to encode map e161 with expected crc 2023-10-09T05:36:59.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:59.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:59.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:59.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:36:59.278 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:36:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:59.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:36:59.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:36:59.406 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:59 smithi191 ceph-mon[161669]: pgmap v416: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:36:59.406 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:59 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:59.406 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:59 smithi191 ceph-mon[161669]: failed to encode map e161 with expected crc 2023-10-09T05:36:59.406 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:59 smithi191 ceph-mon[161669]: failed to encode map e161 with expected crc 2023-10-09T05:36:59.406 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:59 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:59.406 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:59 smithi191 ceph-mon[161669]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T05:36:59.407 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:59 smithi191 ceph-mon[161669]: failed to encode map e161 with expected crc 2023-10-09T05:36:59.407 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:59.407 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:59.407 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:36:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:59.409 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:59.449 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[184418]: pgmap v416: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:36:59.450 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:36:59.450 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[184418]: failed to encode map e161 with expected crc 2023-10-09T05:36:59.450 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[184418]: failed to encode map e161 with expected crc 2023-10-09T05:36:59.450 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:36:59.450 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[184418]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T05:36:59.451 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[184418]: failed to encode map e161 with expected crc 2023-10-09T05:36:59.451 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:59.451 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:36:59.451 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:36:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:36:59.453 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:36:59.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:59.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:36:59.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:36:59.586 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:59.619 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:36:59.633 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:36:59.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:59.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:36:59.739 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:36:59.769 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:59.816 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:36:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:59.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:36:59.920 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:36:59.948 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:36:59.994 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:36:59.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:36:59.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:00.097 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:00.124 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:00.169 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:00.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:00.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:00.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:00.302 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:00.344 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:00.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:00.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:00.429 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_1' created 2023-10-09T05:37:00.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:00.474 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:00.579 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:00.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:00.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:00.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:00.707 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1256324901' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:37:00.707 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:00 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:37:00.710 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:00.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1256324901' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:37:00.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:00 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:37:00.719 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:37:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:37:00] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.33.4" 2023-10-09T05:37:00.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1256324901' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:37:00.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:00 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:37:00.763 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:00.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:00.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:00.866 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:00.895 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:01.059 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:37:01.059 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:37:01.115 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:01.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:01.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:01.178 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:37:01.185 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T05:37:01.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:01.246 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:01.290 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:01.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:01.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:01.393 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:01.425 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:01.453 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:37:01.491 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:01.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:01.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:01.584 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:01.598 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:01.638 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:01.643 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:01.689 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:01.693 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[184418]: pgmap v418: 73 pgs: 73 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:01.693 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[184418]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.693 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[184418]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.693 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[184418]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.693 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T05:37:01.693 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[184418]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T05:37:01.693 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[184418]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.693 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[184418]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.693 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[184418]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T05:37:01.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[191227]: pgmap v418: 73 pgs: 73 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:01.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[191227]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[191227]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[191227]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T05:37:01.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[191227]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T05:37:01.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[191227]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[191227]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:01 smithi008 ceph-mon[191227]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T05:37:01.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:01.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:01.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:01 smithi191 ceph-mon[161669]: pgmap v418: 73 pgs: 73 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:01.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:01 smithi191 ceph-mon[161669]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:01 smithi191 ceph-mon[161669]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:01 smithi191 ceph-mon[161669]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:01 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T05:37:01.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:01 smithi191 ceph-mon[161669]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T05:37:01.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:01 smithi191 ceph-mon[161669]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:01 smithi191 ceph-mon[161669]: failed to encode map e162 with expected crc 2023-10-09T05:37:01.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:01 smithi191 ceph-mon[161669]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T05:37:01.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:37:01.796 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:01.828 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:02.023 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:02.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:02.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:02.125 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:02.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:02.156 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:02.251 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:02.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:02.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:02.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:02.382 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:02.425 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:02.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:02.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:02.529 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:02.558 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:02.703 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:02.704 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:02 smithi191 ceph-mon[161669]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.705 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:02 smithi191 ceph-mon[161669]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.705 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:02 smithi191 ceph-mon[161669]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.705 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:02 smithi191 ceph-mon[161669]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.705 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:02 smithi191 ceph-mon[161669]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T05:37:02.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:02.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:02.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:02 smithi008 ceph-mon[184418]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:02 smithi008 ceph-mon[184418]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:02 smithi008 ceph-mon[184418]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:02 smithi008 ceph-mon[184418]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:02 smithi008 ceph-mon[184418]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T05:37:02.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:02 smithi008 ceph-mon[191227]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:02 smithi008 ceph-mon[191227]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:02 smithi008 ceph-mon[191227]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:02 smithi008 ceph-mon[191227]: failed to encode map e163 with expected crc 2023-10-09T05:37:02.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:02 smithi008 ceph-mon[191227]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T05:37:02.807 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:02.833 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:02.875 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:37:02.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:02.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:02.978 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:03.005 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:03.097 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:03.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:03.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:03.228 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:03.271 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:37:03.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:03.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:03.374 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:03.406 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:03.525 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:03.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:03.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:03.544 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:03.562 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:37:03.562 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:37:03.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:03.657 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:03 smithi008 ceph-mon[184418]: pgmap v421: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:03.657 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:03 smithi008 ceph-mon[184418]: failed to encode map e164 with expected crc 2023-10-09T05:37:03.657 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:03 smithi008 ceph-mon[184418]: failed to encode map e164 with expected crc 2023-10-09T05:37:03.657 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:03 smithi008 ceph-mon[184418]: failed to encode map e164 with expected crc 2023-10-09T05:37:03.657 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:03 smithi008 ceph-mon[184418]: failed to encode map e164 with expected crc 2023-10-09T05:37:03.659 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:03.659 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:03 smithi008 ceph-mon[191227]: pgmap v421: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:03.659 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:03 smithi008 ceph-mon[191227]: failed to encode map e164 with expected crc 2023-10-09T05:37:03.659 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:03 smithi008 ceph-mon[191227]: failed to encode map e164 with expected crc 2023-10-09T05:37:03.660 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:03 smithi008 ceph-mon[191227]: failed to encode map e164 with expected crc 2023-10-09T05:37:03.660 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:03 smithi008 ceph-mon[191227]: failed to encode map e164 with expected crc 2023-10-09T05:37:03.805 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:03.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:03.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:03.909 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:03.940 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:03.994 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:03.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:03.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:03 smithi191 ceph-mon[161669]: pgmap v421: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:03 smithi191 ceph-mon[161669]: failed to encode map e164 with expected crc 2023-10-09T05:37:04.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:03 smithi191 ceph-mon[161669]: failed to encode map e164 with expected crc 2023-10-09T05:37:04.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:03 smithi191 ceph-mon[161669]: failed to encode map e164 with expected crc 2023-10-09T05:37:04.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:03 smithi191 ceph-mon[161669]: failed to encode map e164 with expected crc 2023-10-09T05:37:04.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:04.124 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:04.167 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:04.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:04.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:04.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:37:04.271 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:04.300 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:04.373 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:37:04.392 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:37:04.411 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:37:04.433 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:37:04.449 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:04.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:04.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:04.452 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:37:04.469 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:37:04.487 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:37:04.505 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:37:04.506 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:37:04.506 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:37:04.507 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:37:04.507 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:37:04.507 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:37:04.507 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:37:04.507 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:37:04.507 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:37:04.553 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:04.559 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:04.580 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:04.626 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:04.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:04.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:04.634 INFO:teuthology.orchestra.run.smithi008.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T05:37:04.730 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:04.763 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:04.941 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2731832437' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:37:04.941 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3718948222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:04.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2731832437' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:37:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3718948222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:04.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2731832437' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:37:04.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3718948222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:05.362 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:05.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:05.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:05.466 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:05.495 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:05.692 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:05.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:05.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:05.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:05 smithi191 ceph-mon[161669]: pgmap v423: 89 pgs: 4 creating+peering, 8 unknown, 77 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:05.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:05 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2731832437' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:37:05.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:05 smithi191 ceph-mon[161669]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T05:37:05.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:05 smithi191 ceph-mon[161669]: failed to encode map e165 with expected crc 2023-10-09T05:37:05.796 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:05.825 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:05.941 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:37:05.941 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:37:05.941 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:37:05.941 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:37:05.941 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:37:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:05 smithi008 ceph-mon[184418]: pgmap v423: 89 pgs: 4 creating+peering, 8 unknown, 77 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:05 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2731832437' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:37:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:05 smithi008 ceph-mon[184418]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T05:37:05.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:05 smithi008 ceph-mon[184418]: failed to encode map e165 with expected crc 2023-10-09T05:37:05.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:05 smithi008 ceph-mon[191227]: pgmap v423: 89 pgs: 4 creating+peering, 8 unknown, 77 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:05.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:05 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2731832437' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:37:05.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:05 smithi008 ceph-mon[191227]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T05:37:05.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:05 smithi008 ceph-mon[191227]: failed to encode map e165 with expected crc 2023-10-09T05:37:05.972 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:05.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:05.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:06.075 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:06.106 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:06.258 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:06.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:06.362 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:06.390 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:06.441 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:06.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:06.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:06.545 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:06.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:37:06.571 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:06.615 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:06.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:06.642 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:37:06.719 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:06.746 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:06.760 DEBUG:teuthology.orchestra.run.smithi008:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T05:37:06.760 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-10-09T05:37:06.761 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:37:06.763 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:37:06.763 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:37:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:06 smithi191 ceph-mon[161669]: failed to encode map e165 with expected crc 2023-10-09T05:37:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:06 smithi191 ceph-mon[161669]: failed to encode map e165 with expected crc 2023-10-09T05:37:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:06 smithi191 ceph-mon[161669]: failed to encode map e165 with expected crc 2023-10-09T05:37:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3820165733' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:06.814 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:hints = 1 2023-10-09T05:37:06.814 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:37:06.814 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object prefix: benchmark_data_smithi008_295510 2023-10-09T05:37:06.814 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:37:06.814 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:37:06.823 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:06 smithi008 ceph-mon[184418]: failed to encode map e165 with expected crc 2023-10-09T05:37:06.823 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:06 smithi008 ceph-mon[184418]: failed to encode map e165 with expected crc 2023-10-09T05:37:06.824 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:06 smithi008 ceph-mon[184418]: failed to encode map e165 with expected crc 2023-10-09T05:37:06.824 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3820165733' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:06.824 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:06 smithi008 ceph-mon[191227]: failed to encode map e165 with expected crc 2023-10-09T05:37:06.824 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:06 smithi008 ceph-mon[191227]: failed to encode map e165 with expected crc 2023-10-09T05:37:06.824 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:06 smithi008 ceph-mon[191227]: failed to encode map e165 with expected crc 2023-10-09T05:37:06.824 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3820165733' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:06.864 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:06.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:06.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:06.912 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:06.968 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:06.998 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:07.287 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:07.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:07.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:07.393 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:07.446 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:07.542 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:07.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:07.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:07.573 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:07.648 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:07.675 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:07.697 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:37:07.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:07 smithi191 ceph-mon[161669]: pgmap v425: 89 pgs: 4 creating+peering, 85 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:07.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:07 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:07.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:07 smithi191 ceph-mon[161669]: failed to encode map e166 with expected crc 2023-10-09T05:37:07.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:07 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3820165733' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:37:07.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:07 smithi191 ceph-mon[161669]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T05:37:07.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:07 smithi191 ceph-mon[161669]: failed to encode map e166 with expected crc 2023-10-09T05:37:07.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:07 smithi191 ceph-mon[161669]: failed to encode map e166 with expected crc 2023-10-09T05:37:07.814 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 1 16 1884 1868 116.743 116.75 0.0026497 0.00611454 2023-10-09T05:37:07.823 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[184418]: pgmap v425: 89 pgs: 4 creating+peering, 85 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:07.823 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:07.824 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[184418]: failed to encode map e166 with expected crc 2023-10-09T05:37:07.824 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3820165733' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:37:07.824 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[184418]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T05:37:07.824 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[184418]: failed to encode map e166 with expected crc 2023-10-09T05:37:07.824 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[184418]: failed to encode map e166 with expected crc 2023-10-09T05:37:07.827 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[191227]: pgmap v425: 89 pgs: 4 creating+peering, 85 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:37:07.827 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:07.827 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[191227]: failed to encode map e166 with expected crc 2023-10-09T05:37:07.827 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3820165733' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:37:07.827 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[191227]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T05:37:07.828 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[191227]: failed to encode map e166 with expected crc 2023-10-09T05:37:07.828 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:07 smithi008 ceph-mon[191227]: failed to encode map e166 with expected crc 2023-10-09T05:37:07.843 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:07.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:07.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:07.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:08.004 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:08.178 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:08.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:08.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:08.284 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:08.341 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:08.676 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:37:08.676 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:37:08.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:37:08.676 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:37:08.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.677 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54045 5.9 5:9b47d3b1:::benchmark_data_smithi008_238708_object54044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e112)", 2023-10-09T05:37:08.677 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.539603+0000", 2023-10-09T05:37:08.677 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 228.13373064500001, 2023-10-09T05:37:08.677 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0152003759999999, 2023-10-09T05:37:08.678 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.678 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.678 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.678 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.678 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.678 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54045 2023-10-09T05:37:08.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.679 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539603+0000", 2023-10-09T05:37:08.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539603+0000", 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539601+0000", 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3383999999999997e-05 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539645+0000", 2023-10-09T05:37:08.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0492999999999999e-05 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539655+0000", 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3479999999999997e-06 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539660+0000", 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4764000000000001e-05 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539694+0000", 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9150000000000001e-06 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.539700+0000", 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.002179355 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.541880+0000", 2023-10-09T05:37:08.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010278150000000001 2023-10-09T05:37:08.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.542907+0000", 2023-10-09T05:37:08.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000208908 2023-10-09T05:37:08.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4", 2023-10-09T05:37:08.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.543116+0000", 2023-10-09T05:37:08.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052105160000000001 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.548327+0000", 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063079850000000003 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554635+0000", 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0263000000000002e-05 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.684 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554695+0000", 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010829500000000001 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554803+0000", 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0152003759999999 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.685 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:98121 5.9 5:930a82bc:::benchmark_data_smithi008_238708_object98120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:00.583011+0000", 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 188.090322697, 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20323780999999999, 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 98121 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.686 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.687 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.687 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.687 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583011+0000", 2023-10-09T05:37:08.687 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583011+0000", 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583009+0000", 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4110999999999997e-05 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583073+0000", 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6398000000000001e-05 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583090+0000", 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3339999999999996e-06 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583097+0000", 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8928999999999998e-05 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583156+0000", 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000166971 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583323+0000", 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017988100000000001 2023-10-09T05:37:08.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.691 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.691 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4", 2023-10-09T05:37:08.691 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.583503+0000", 2023-10-09T05:37:08.691 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013026197 2023-10-09T05:37:08.691 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.691 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.691 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.691 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.596529+0000", 2023-10-09T05:37:08.692 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18970899299999999 2023-10-09T05:37:08.692 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.692 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.692 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.692 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.786238+0000", 2023-10-09T05:37:08.692 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.099e-05 2023-10-09T05:37:08.692 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.692 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.786249+0000", 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20323780999999999 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:98123 5.9 5:980279ad:::benchmark_data_smithi008_238708_object98122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:00.586202+0000", 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 188.087131529, 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20225238100000001, 2023-10-09T05:37:08.693 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.694 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:37:08.694 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.694 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.694 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.694 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 98123 2023-10-09T05:37:08.694 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.694 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.694 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.694 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586202+0000", 2023-10-09T05:37:08.694 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.695 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.695 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586202+0000", 2023-10-09T05:37:08.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:37:08.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586200+0000", 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7370000000000003e-05 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586237+0000", 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5962999999999999e-05 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586253+0000", 2023-10-09T05:37:08.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4239999999999993e-06 2023-10-09T05:37:08.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.697 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.697 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586261+0000", 2023-10-09T05:37:08.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8481e-05 2023-10-09T05:37:08.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.697 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.697 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586300+0000", 2023-10-09T05:37:08.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000164969 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586465+0000", 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000231108 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4", 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.586696+0000", 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01166125 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.598357+0000", 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19008492199999999 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.788442+0000", 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2417e-05 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.788455+0000", 2023-10-09T05:37:08.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20225238100000001 2023-10-09T05:37:08.700 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.700 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.700 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.700 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.700 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:98121 5.9 5:930a82bc:::benchmark_data_smithi008_238708_object98120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:37:08.700 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:00.694607+0000", 2023-10-09T05:37:08.701 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 187.97872637200001, 2023-10-09T05:37:08.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92918373799999998, 2023-10-09T05:37:08.701 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.701 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:37:08.701 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.701 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.701 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.701 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 98121 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694607+0000", 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694607+0000", 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999962 2023-10-09T05:37:08.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694603+0000", 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6734999999999999e-05 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694650+0000", 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9122999999999999e-05 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.703 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694679+0000", 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0550999999999999e-05 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694690+0000", 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5582999999999996e-05 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694785+0000", 2023-10-09T05:37:08.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6899999999999992e-06 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694793+0000", 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.093691338999999998 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.788484+0000", 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2150000000000004e-06 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.788492+0000", 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.82731977400000001 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.615811+0000", 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00076560599999999995 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:37:08.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.616577+0000", 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071166750000000003 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.623694+0000", 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7318999999999994e-05 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.623791+0000", 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92918373799999998 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.707 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.708 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.708 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.708 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.708 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:98123 5.19 5:980279ad:::benchmark_data_smithi008_238708_object98122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:37:08.708 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:00.694705+0000", 2023-10-09T05:37:08.708 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 187.97862876400001, 2023-10-09T05:37:08.708 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.938116383, 2023-10-09T05:37:08.708 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.708 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:37:08.708 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.708 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.709 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.709 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 98123 2023-10-09T05:37:08.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.709 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.709 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.709 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.709 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694705+0000", 2023-10-09T05:37:08.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.710 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.710 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.710 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.710 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694705+0000", 2023-10-09T05:37:08.710 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:08.710 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694704+0000", 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6398e-05 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694731+0000", 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2520000000000005e-06 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.711 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694740+0000", 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3919999999999996e-06 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.694744+0000", 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.091777336000000001 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.786522+0000", 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2589999999999997e-06 2023-10-09T05:37:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:00.786528+0000", 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83990997700000003 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.626438+0000", 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001340026 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:37:08.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.627778+0000", 2023-10-09T05:37:08.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0049558120000000004 2023-10-09T05:37:08.714 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.714 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.714 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.714 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.632734+0000", 2023-10-09T05:37:08.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7482e-05 2023-10-09T05:37:08.714 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.715 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.715 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.715 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:01.632821+0000", 2023-10-09T05:37:08.715 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.938116383 2023-10-09T05:37:08.715 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.715 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.715 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.716 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.716 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.716 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119037 5.2 5:408f5192:::benchmark_data_smithi008_238708_object901:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:37:08.716 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.151111+0000", 2023-10-09T05:37:08.716 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 164.522222876, 2023-10-09T05:37:08.716 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88165001499999995, 2023-10-09T05:37:08.717 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.717 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.717 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.717 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.717 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.717 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119037 2023-10-09T05:37:08.718 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.718 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.718 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.718 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.718 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151111+0000", 2023-10-09T05:37:08.718 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.719 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.719 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.719 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.719 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151111+0000", 2023-10-09T05:37:08.719 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:08.719 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.719 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.720 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.720 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151109+0000", 2023-10-09T05:37:08.720 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1720000000000007e-06 2023-10-09T05:37:08.720 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.720 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.720 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151117+0000", 2023-10-09T05:37:08.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2285e-05 2023-10-09T05:37:08.721 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.721 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.721 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151130+0000", 2023-10-09T05:37:08.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6660000000000003e-06 2023-10-09T05:37:08.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151139+0000", 2023-10-09T05:37:08.722 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016795371999999999 2023-10-09T05:37:08.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.167935+0000", 2023-10-09T05:37:08.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4433999999999999e-05 2023-10-09T05:37:08.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:37:08.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.167949+0000", 2023-10-09T05:37:08.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85092211299999998 2023-10-09T05:37:08.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.018871+0000", 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3704999999999998e-05 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.018965+0000", 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5246000000000003e-05 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.724 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:37:08.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.019050+0000", 2023-10-09T05:37:08.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052231589999999998 2023-10-09T05:37:08.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.024273+0000", 2023-10-09T05:37:08.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048098300000000002 2023-10-09T05:37:08.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.029083+0000", 2023-10-09T05:37:08.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035822990000000002 2023-10-09T05:37:08.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.032666+0000", 2023-10-09T05:37:08.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5321000000000001e-05 2023-10-09T05:37:08.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.032721+0000", 2023-10-09T05:37:08.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9962000000000001e-05 2023-10-09T05:37:08.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.032761+0000", 2023-10-09T05:37:08.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88165001499999995 2023-10-09T05:37:08.728 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.728 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.728 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:121199 5.12 5:4c3a7433:::benchmark_data_smithi008_238708_object3063:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:26.591400+0000", 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 162.08193348899999, 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25645415500000002, 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.729 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 121199 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591400+0000", 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591400+0000", 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:08.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591399+0000", 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7750000000000004e-06 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591406+0000", 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0115000000000001e-05 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591416+0000", 2023-10-09T05:37:08.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9239999999999998e-06 2023-10-09T05:37:08.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591421+0000", 2023-10-09T05:37:08.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9996999999999997e-05 2023-10-09T05:37:08.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591461+0000", 2023-10-09T05:37:08.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7571999999999994e-05 2023-10-09T05:37:08.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591548+0000", 2023-10-09T05:37:08.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6479999999999999e-05 2023-10-09T05:37:08.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:37:08.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591635+0000", 2023-10-09T05:37:08.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012101460000000001 2023-10-09T05:37:08.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.734 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.592845+0000", 2023-10-09T05:37:08.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031766419999999999 2023-10-09T05:37:08.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.596022+0000", 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25174085699999998 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847763+0000", 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1802999999999999e-05 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847814+0000", 2023-10-09T05:37:08.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0043999999999998e-05 2023-10-09T05:37:08.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847854+0000", 2023-10-09T05:37:08.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25645415500000002 2023-10-09T05:37:08.736 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.736 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.736 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:135149 5.12 5:4a140978:::benchmark_data_smithi008_238708_object17013:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:37.646056+0000", 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 151.027277215, 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.171445025, 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135149 2023-10-09T05:37:08.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.738 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.738 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.738 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646056+0000", 2023-10-09T05:37:08.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.738 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.738 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646056+0000", 2023-10-09T05:37:08.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:08.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646056+0000", 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4149999999999999e-06 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646059+0000", 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3579999999999999e-06 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646066+0000", 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.369e-06 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.646070+0000", 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012852930000000001 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.647355+0000", 2023-10-09T05:37:08.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118285 2023-10-09T05:37:08.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.647474+0000", 2023-10-09T05:37:08.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115395 2023-10-09T05:37:08.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:37:08.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.647589+0000", 2023-10-09T05:37:08.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033829889999999999 2023-10-09T05:37:08.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.650972+0000", 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021853419999999998 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.653157+0000", 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16425049 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.817408+0000", 2023-10-09T05:37:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6112000000000002e-05 2023-10-09T05:37:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.817454+0000", 2023-10-09T05:37:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7508e-05 2023-10-09T05:37:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.817501+0000", 2023-10-09T05:37:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.171445025 2023-10-09T05:37:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:135151 5.22 5:47224588:::benchmark_data_smithi008_238708_object17015:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:37:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:37.651091+0000", 2023-10-09T05:37:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 151.02224234400001, 2023-10-09T05:37:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.1747978, 2023-10-09T05:37:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135151 2023-10-09T05:37:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651091+0000", 2023-10-09T05:37:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651091+0000", 2023-10-09T05:37:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651090+0000", 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1389999999999995e-06 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651098+0000", 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5180999999999999e-05 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651113+0000", 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7910000000000001e-06 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.651119+0000", 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020545799999999999 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.653173+0000", 2023-10-09T05:37:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6720999999999998e-05 2023-10-09T05:37:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.653270+0000", 2023-10-09T05:37:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108272 2023-10-09T05:37:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:37:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.653378+0000", 2023-10-09T05:37:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027134020000000002 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.656092+0000", 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7579999999999998e-06 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.656098+0000", 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.169701516 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825799+0000", 2023-10-09T05:37:08.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1576999999999998e-05 2023-10-09T05:37:08.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825851+0000", 2023-10-09T05:37:08.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8532000000000003e-05 2023-10-09T05:37:08.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.753 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825889+0000", 2023-10-09T05:37:08.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.1747978 2023-10-09T05:37:08.753 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.753 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.753 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.754 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:135156 5.12 5:4dc0c42a:::benchmark_data_smithi008_238708_object17020:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-09T05:37:08.754 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:37.652947+0000", 2023-10-09T05:37:08.754 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 151.02038699900001, 2023-10-09T05:37:08.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17295703300000001, 2023-10-09T05:37:08.754 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.754 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.754 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.754 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.754 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.754 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135156 2023-10-09T05:37:08.754 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.755 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.755 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.755 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652947+0000", 2023-10-09T05:37:08.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.755 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.755 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652947+0000", 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652945+0000", 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8889999999999996e-06 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652952+0000", 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.013e-05 2023-10-09T05:37:08.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652962+0000", 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4469999999999997e-06 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.652967+0000", 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013374330000000001 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.654305+0000", 2023-10-09T05:37:08.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2422000000000004e-05 2023-10-09T05:37:08.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.654397+0000", 2023-10-09T05:37:08.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000101003 2023-10-09T05:37:08.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.654498+0000", 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0043436280000000004 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.658842+0000", 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000146315 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.658988+0000", 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16684702400000001 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825835+0000", 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6446000000000003e-05 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825872+0000", 2023-10-09T05:37:08.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2089999999999999e-05 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:37.825904+0000", 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17295703300000001 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:148743 5.12 5:4f7a7639:::benchmark_data_smithi008_238708_object30607:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:52.708913+0000", 2023-10-09T05:37:08.761 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 135.964420808, 2023-10-09T05:37:08.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17522900399999999, 2023-10-09T05:37:08.762 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.762 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.762 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.762 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.762 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.762 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 148743 2023-10-09T05:37:08.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.762 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.708913+0000", 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.708913+0000", 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.708910+0000", 2023-10-09T05:37:08.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7349999999999996e-06 2023-10-09T05:37:08.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.764 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.708918+0000", 2023-10-09T05:37:08.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2352e-05 2023-10-09T05:37:08.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.764 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.708930+0000", 2023-10-09T05:37:08.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1199999999999999e-06 2023-10-09T05:37:08.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.708936+0000", 2023-10-09T05:37:08.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9354999999999998e-05 2023-10-09T05:37:08.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.708986+0000", 2023-10-09T05:37:08.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000116344 2023-10-09T05:37:08.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.709102+0000", 2023-10-09T05:37:08.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000110482 2023-10-09T05:37:08.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:37:08.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.709213+0000", 2023-10-09T05:37:08.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004523801 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.713736+0000", 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011921000000000001 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.713856+0000", 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.170218541 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.884074+0000", 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2920000000000003e-05 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.884107+0000", 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4728999999999999e-05 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:52.884142+0000", 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17522900399999999 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:187395 5.22 5:44eb4796:::benchmark_data_smithi008_238708_object69259:head [delete] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:32.988274+0000", 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.685059459000001, 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17327574400000001, 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.769 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.770 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.770 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.770 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.770 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 187395 2023-10-09T05:37:08.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.770 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:32.988274+0000", 2023-10-09T05:37:08.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:32.988274+0000", 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:32.988273+0000", 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9630000000000004e-06 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:32.988282+0000", 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7598999999999999e-05 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:32.988299+0000", 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4679999999999997e-06 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:32.988306+0000", 2023-10-09T05:37:08.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9186e-05 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:32.988355+0000", 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108205 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:32.988463+0000", 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7934000000000001e-05 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:37:08.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:32.988561+0000", 2023-10-09T05:37:08.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001457701 2023-10-09T05:37:08.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:32.990019+0000", 2023-10-09T05:37:08.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010464200000000001 2023-10-09T05:37:08.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:32.990123+0000", 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.171320573 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:33.161444+0000", 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4205e-05 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:33.161498+0000", 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1870999999999999e-05 2023-10-09T05:37:08.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:33.161550+0000", 2023-10-09T05:37:08.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17327574400000001 2023-10-09T05:37:08.776 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.776 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.776 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.777 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:210706 5.22 5:46b57348:::benchmark_data_smithi008_238708_object92570:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T05:37:08.777 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:00.579470+0000", 2023-10-09T05:37:08.777 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 68.093864120999996, 2023-10-09T05:37:08.777 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22921691999999999, 2023-10-09T05:37:08.778 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.778 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.778 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.778 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.778 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.779 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 210706 2023-10-09T05:37:08.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.779 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.579470+0000", 2023-10-09T05:37:08.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.579470+0000", 2023-10-09T05:37:08.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:37:08.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.579468+0000", 2023-10-09T05:37:08.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1600000000000004e-06 2023-10-09T05:37:08.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.579477+0000", 2023-10-09T05:37:08.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4036000000000001e-05 2023-10-09T05:37:08.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.579491+0000", 2023-10-09T05:37:08.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4230000000000002e-06 2023-10-09T05:37:08.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.579498+0000", 2023-10-09T05:37:08.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4506999999999995e-05 2023-10-09T05:37:08.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.579592+0000", 2023-10-09T05:37:08.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013317 2023-10-09T05:37:08.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.579725+0000", 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012499399999999999 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.579850+0000", 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068125060000000003 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.784 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.586663+0000", 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22182085900000001 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808484+0000", 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102097 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808586+0000", 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3857999999999998e-05 2023-10-09T05:37:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808639+0000", 2023-10-09T05:37:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6983000000000001e-05 2023-10-09T05:37:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808686+0000", 2023-10-09T05:37:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22921691999999999 2023-10-09T05:37:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:210711 5.12 5:4f430670:::benchmark_data_smithi008_238708_object92575:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T05:37:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:00.581613+0000", 2023-10-09T05:37:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 68.091720821999999, 2023-10-09T05:37:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.227180244, 2023-10-09T05:37:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 210711 2023-10-09T05:37:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581613+0000", 2023-10-09T05:37:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581613+0000", 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581611+0000", 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1810000000000001e-06 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581619+0000", 2023-10-09T05:37:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3506000000000001e-05 2023-10-09T05:37:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581633+0000", 2023-10-09T05:37:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1729999999999998e-06 2023-10-09T05:37:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581640+0000", 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6005000000000003e-05 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581696+0000", 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114554 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581811+0000", 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00052909799999999998 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:37:08.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.582340+0000", 2023-10-09T05:37:08.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.226192913 2023-10-09T05:37:08.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808533+0000", 2023-10-09T05:37:08.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0481999999999994e-05 2023-10-09T05:37:08.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808603+0000", 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7306999999999995e-05 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808701+0000", 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2771e-05 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808753+0000", 2023-10-09T05:37:08.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9833000000000003e-05 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808793+0000", 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.227180244 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:210713 5.22 5:46fa28cd:::benchmark_data_smithi008_238708_object92577:head [delete] snapc 0=[] ondisk+write+known_if_redirected e143)", 2023-10-09T05:37:08.794 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:00.581652+0000", 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 68.091682075999998, 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22722336600000001, 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 210713 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581652+0000", 2023-10-09T05:37:08.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581652+0000", 2023-10-09T05:37:08.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:08.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581651+0000", 2023-10-09T05:37:08.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.191e-06 2023-10-09T05:37:08.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581654+0000", 2023-10-09T05:37:08.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.733e-06 2023-10-09T05:37:08.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581660+0000", 2023-10-09T05:37:08.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1679999999999997e-06 2023-10-09T05:37:08.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.581664+0000", 2023-10-09T05:37:08.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033844230000000001 2023-10-09T05:37:08.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.585049+0000", 2023-10-09T05:37:08.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4559000000000001e-05 2023-10-09T05:37:08.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.585133+0000", 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0070999999999999e-05 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.585223+0000", 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22333668100000001 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808560+0000", 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017767299999999999 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808738+0000", 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2697e-05 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808800+0000", 2023-10-09T05:37:08.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7672000000000001e-05 2023-10-09T05:37:08.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808838+0000", 2023-10-09T05:37:08.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6965999999999998e-05 2023-10-09T05:37:08.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808875+0000", 2023-10-09T05:37:08.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22722336600000001 2023-10-09T05:37:08.802 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.802 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.802 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.803 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.34770.0:210706 5.22 e143/123 ondisk, result = 0)", 2023-10-09T05:37:08.803 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:00.586983+0000", 2023-10-09T05:37:08.803 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 68.086350912, 2023-10-09T05:37:08.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22150499500000001, 2023-10-09T05:37:08.803 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.804 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:37:08.804 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.586983+0000", 2023-10-09T05:37:08.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.586983+0000", 2023-10-09T05:37:08.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:08.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.586982+0000", 2023-10-09T05:37:08.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0900000000000001e-06 2023-10-09T05:37:08.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.586988+0000", 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7059999999999999e-06 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.586998+0000", 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4410000000000001e-06 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.587003+0000", 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4499000000000002e-05 2023-10-09T05:37:08.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.807 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.807 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.587048+0000", 2023-10-09T05:37:08.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.221417735 2023-10-09T05:37:08.807 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.807 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.807 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808466+0000", 2023-10-09T05:37:08.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.215e-05 2023-10-09T05:37:08.807 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.808 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.808 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808488+0000", 2023-10-09T05:37:08.808 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22150499500000001 2023-10-09T05:37:08.808 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.808 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.808 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.808 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.808 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.808 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.34770.0:210711 5.12 e143/118 ondisk, result = 0)", 2023-10-09T05:37:08.808 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:00.587019+0000", 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 68.086314259999995, 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.221857475, 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.587019+0000", 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.587019+0000", 2023-10-09T05:37:08.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:08.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.587019+0000", 2023-10-09T05:37:08.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3120000000000001e-06 2023-10-09T05:37:08.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.587021+0000", 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0969999999999996e-06 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.587027+0000", 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3569999999999998e-06 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.587030+0000", 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22148515999999999 2023-10-09T05:37:08.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808515+0000", 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7210000000000006e-06 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808525+0000", 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000352097 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808877+0000", 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.221857475 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.34770.0:210713 5.22 e143/123 ondisk, result = 0)", 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:00.588658+0000", 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 68.084676117000001, 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21990780200000001, 2023-10-09T05:37:08.813 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.588658+0000", 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.588658+0000", 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:37:08.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.588657+0000", 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5319999999999997e-06 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.588662+0000", 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21981567699999999 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808478+0000", 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5690000000000005e-06 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808487+0000", 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0668999999999998e-05 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808547+0000", 2023-10-09T05:37:08.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5640000000000002e-06 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808554+0000", 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1533e-05 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808565+0000", 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21990780200000001 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.817 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.818 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.818 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.34770.0:210713 5.22 e143/123 ondisk, result = 0)", 2023-10-09T05:37:08.818 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:00.589051+0000", 2023-10-09T05:37:08.818 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 68.084283071000002, 2023-10-09T05:37:08.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21968996700000001, 2023-10-09T05:37:08.818 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.818 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:37:08.818 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.589051+0000", 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.589051+0000", 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.589050+0000", 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0360000000000003e-06 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.589056+0000", 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21940390900000001 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808460+0000", 2023-10-09T05:37:08.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2095000000000001e-05 2023-10-09T05:37:08.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.821 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.821 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.821 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808482+0000", 2023-10-09T05:37:08.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00023482900000000001 2023-10-09T05:37:08.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.821 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.821 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.821 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808717+0000", 2023-10-09T05:37:08.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4130000000000003e-06 2023-10-09T05:37:08.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808723+0000", 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7371e-05 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808741+0000", 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21968996700000001 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.822 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34770.0:210717 5.19 e143/128 5:9a0651b1:::benchmark_data_smithi008_238708_object92581:head v 143'9577, mlcod=124'6893)", 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:00.591535+0000", 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 68.081798527999993, 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.224932719, 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.591535+0000", 2023-10-09T05:37:08.823 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:08.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.824 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:08.824 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.591535+0000", 2023-10-09T05:37:08.824 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:37:08.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.824 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:08.824 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.591534+0000", 2023-10-09T05:37:08.824 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6730000000000004e-06 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.591544+0000", 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.216919842 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808464+0000", 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3169000000000001e-05 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808487+0000", 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4503999999999999e-05 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808531+0000", 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9682000000000001e-05 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:08.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.808561+0000", 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078716840000000003 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.816433+0000", 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5095000000000002e-05 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:00.816468+0000", 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.224932719 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.827 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.828 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.828 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:08.828 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:08.828 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:37:08.828 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 2 16 4097 4081 127.521 138.312 0.00131954 0.00654849 2023-10-09T05:37:09.212 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:09.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:09.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:09.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:09.345 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:09.388 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:09.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:09.447 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:09.492 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:09.506 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:37:09.555 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:09.674 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:09.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:09.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:09.779 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:09.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:37:09.826 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 3 16 5534 5518 114.564 89.8125 0.00318742 0.00871099 2023-10-09T05:37:09.852 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:09 smithi008 ceph-mon[184418]: pgmap v427: 89 pgs: 89 active+clean; 24 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 52 op/s 2023-10-09T05:37:09.852 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:09 smithi008 ceph-mon[191227]: pgmap v427: 89 pgs: 89 active+clean; 24 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 52 op/s 2023-10-09T05:37:09.860 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:09.986 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:09.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:09.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:10.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:09 smithi191 ceph-mon[161669]: pgmap v427: 89 pgs: 89 active+clean; 24 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 52 op/s 2023-10-09T05:37:10.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:10.133 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:10.206 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:10.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:10.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:10.312 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:10.318 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:10.358 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:10.405 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:37:10.412 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:37:10.501 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:10.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:10.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:10.608 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:10.661 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:10.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:37:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:37:10] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.33.4" 2023-10-09T05:37:10.825 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 4 16 8233 8217 128.057 168.688 0.00349916 0.00780244 2023-10-09T05:37:10.853 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:10.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:10.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:10.958 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:10.986 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:11.826 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 5 16 10727 10711 133.606 155.875 0.00361778 0.00747745 2023-10-09T05:37:11.859 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:11.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:11.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:11.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:11 smithi008 ceph-mon[184418]: pgmap v428: 89 pgs: 89 active+clean; 232 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s wr, 479 op/s 2023-10-09T05:37:11.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:11 smithi008 ceph-mon[184418]: from='client.45032 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:37:11.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:11 smithi008 ceph-mon[191227]: pgmap v428: 89 pgs: 89 active+clean; 232 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s wr, 479 op/s 2023-10-09T05:37:11.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:11 smithi008 ceph-mon[191227]: from='client.45032 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:37:11.965 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:12.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:11 smithi191 ceph-mon[161669]: pgmap v428: 89 pgs: 89 active+clean; 232 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s wr, 479 op/s 2023-10-09T05:37:12.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:11 smithi191 ceph-mon[161669]: from='client.45032 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:37:12.012 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:12.182 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:12.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:12.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:12.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:12.315 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:12.380 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:12.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:12.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:12.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:12.517 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:12.670 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:37:12.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:12.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:12.761 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:37:12.762 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:37:12.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.762 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:12518 6.5 6:aeda2976:::benchmark_data_smithi008_295510_object12517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:37:12.762 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:12.754221+0000", 2023-10-09T05:37:12.762 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0034064149999999999, 2023-10-09T05:37:12.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0034415560000000001, 2023-10-09T05:37:12.762 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:37:12.762 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:37:12.762 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:37:12.763 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:37:12.763 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:37:12.763 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 12518 2023-10-09T05:37:12.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:12.763 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:37:12.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.763 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:37:12.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:12.754221+0000", 2023-10-09T05:37:12.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:37:12.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:12.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:37:12.764 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:12.754221+0000", 2023-10-09T05:37:12.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:37:12.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:12.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:37:12.764 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:12.754220+0000", 2023-10-09T05:37:12.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103031 2023-10-09T05:37:12.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:12.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:37:12.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:12.754323+0000", 2023-10-09T05:37:12.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7960000000000007e-06 2023-10-09T05:37:12.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:12.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:37:12.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:12.754333+0000", 2023-10-09T05:37:12.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1189999999999999e-06 2023-10-09T05:37:12.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:12.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:37:12.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:12.754337+0000", 2023-10-09T05:37:12.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4653000000000002e-05 2023-10-09T05:37:12.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:12.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:37:12.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:12.754372+0000", 2023-10-09T05:37:12.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012397499999999999 2023-10-09T05:37:12.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:12.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:12.754495+0000", 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000170791 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:12.754666+0000", 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001701183 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:37:12.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:12.756367+0000", 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021194599999999999 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:12.756579+0000", 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023585559999999999 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 1 2023-10-09T05:37:12.768 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:37:12.775 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:12.822 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:12.825 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 6 16 12713 12697 132.007 124.125 0.00657085 0.00756692 2023-10-09T05:37:12.891 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":428,"stamp":"2023-10-09T05:37:10.013254+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":243341235,"num_objects":3668,"num_object_clones":0,"num_object_copies":11004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3668,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":4746,"num_write_kb":248101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4810,"ondisk_log_size":4810,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2492788,"kb_used_data":547636,"kb_used_omap":45,"kb_used_meta":1945042,"kb_avail":747468428,"statfs":{"total":767960285184,"available":765407670272,"internally_reserved":0,"allocated":560779264,"data_stored":556027958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":1991723299},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,4,0,1,0,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":238551040,"num_objects":3640,"num_object_clones":0,"num_object_copies":10920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3640,"num_write_kb":232960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.589913"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":149,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.617027+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:37:05.617027+0000","last_peered":"2023-10-09T05:37:05.617027+0000","last_clean":"2023-10-09T05:37:05.617027+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:37:05.617027+0000","last_undegraded":"2023-10-09T05:37:05.617027+0000","last_fullsized":"2023-10-09T05:37:05.617027+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.713047+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:37:02.713047+0000","last_peered":"2023-10-09T05:37:02.713047+0000","last_clean":"2023-10-09T05:37:02.713047+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:37:02.713047+0000","last_undegraded":"2023-10-09T05:37:02.713047+0000","last_fullsized":"2023-10-09T05:37:02.713047+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":111,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.614464+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:37:05.614464+0000","last_peered":"2023-10-09T05:37:05.614464+0000","last_clean":"2023-10-09T05:37:05.614464+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:37:05.614464+0000","last_undegraded":"2023-10-09T05:37:05.614464+0000","last_fullsized":"2023-10-09T05:37:05.614464+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":132,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.632291+0000","last_change":"2023-10-09T05:34:41.662942+0000","last_active":"2023-10-09T05:37:04.632291+0000","last_peered":"2023-10-09T05:37:04.632291+0000","last_clean":"2023-10-09T05:37:04.632291+0000","last_became_active":"2023-10-09T05:34:41.662648+0000","last_became_peered":"2023-10-09T05:34:41.662648+0000","last_unstale":"2023-10-09T05:37:04.632291+0000","last_undegraded":"2023-10-09T05:37:04.632291+0000","last_fullsized":"2023-10-09T05:37:04.632291+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:08.691549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":152,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.712321+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:37:02.712321+0000","last_peered":"2023-10-09T05:37:02.712321+0000","last_clean":"2023-10-09T05:37:02.712321+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:37:02.712321+0000","last_undegraded":"2023-10-09T05:37:02.712321+0000","last_fullsized":"2023-10-09T05:37:02.712321+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":132,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616471+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:37:05.616471+0000","last_peered":"2023-10-09T05:37:05.616471+0000","last_clean":"2023-10-09T05:37:05.616471+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:37:05.616471+0000","last_undegraded":"2023-10-09T05:37:05.616471+0000","last_fullsized":"2023-10-09T05:37:05.616471+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":116,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.632237+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:37:04.632237+0000","last_peered":"2023-10-09T05:37:04.632237+0000","last_clean":"2023-10-09T05:37:04.632237+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:37:04.632237+0000","last_undegraded":"2023-10-09T05:37:04.632237+0000","last_fullsized":"2023-10-09T05:37:04.632237+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":136,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.712432+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:37:02.712432+0000","last_peered":"2023-10-09T05:37:02.712432+0000","last_clean":"2023-10-09T05:37:02.712432+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:37:02.712432+0000","last_undegraded":"2023-10-09T05:37:02.712432+0000","last_fullsized":"2023-10-09T05:37:02.712432+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":148,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616290+0000","last_change":"2023-10-09T05:34:41.683934+0000","last_active":"2023-10-09T05:37:05.616290+0000","last_peered":"2023-10-09T05:37:05.616290+0000","last_clean":"2023-10-09T05:37:05.616290+0000","last_became_active":"2023-10-09T05:34:41.683618+0000","last_became_peered":"2023-10-09T05:34:41.683618+0000","last_unstale":"2023-10-09T05:37:05.616290+0000","last_undegraded":"2023-10-09T05:37:05.616290+0000","last_fullsized":"2023-10-09T05:37:05.616290+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":141,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.712343+0000","last_change":"2023-10-09T05:34:41.668251+0000","last_active":"2023-10-09T05:37:02.712343+0000","last_peered":"2023-10-09T05:37:02.712343+0000","last_clean":"2023-10-09T05:37:02.712343+0000","last_became_active":"2023-10-09T05:34:41.668017+0000","last_became_peered":"2023-10-09T05:34:41.668017+0000","last_unstale":"2023-10-09T05:37:02.712343+0000","last_undegraded":"2023-10-09T05:37:02.712343+0000","last_fullsized":"2023-10-09T05:37:02.712343+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:27.235656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":125,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.637408+0000","last_change":"2023-10-09T05:34:41.682874+0000","last_active":"2023-10-09T05:37:04.637408+0000","last_peered":"2023-10-09T05:37:04.637408+0000","last_clean":"2023-10-09T05:37:04.637408+0000","last_became_active":"2023-10-09T05:34:41.674789+0000","last_became_peered":"2023-10-09T05:34:41.674789+0000","last_unstale":"2023-10-09T05:37:04.637408+0000","last_undegraded":"2023-10-09T05:37:04.637408+0000","last_fullsized":"2023-10-09T05:37:04.637408+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":111,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.613849+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:37:05.613849+0000","last_peered":"2023-10-09T05:37:05.613849+0000","last_clean":"2023-10-09T05:37:05.613849+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:37:05.613849+0000","last_undegraded":"2023-10-09T05:37:05.613849+0000","last_fullsized":"2023-10-09T05:37:05.613849+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":155,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616329+0000","last_change":"2023-10-09T05:34:41.683953+0000","last_active":"2023-10-09T05:37:05.616329+0000","last_peered":"2023-10-09T05:37:05.616329+0000","last_clean":"2023-10-09T05:37:05.616329+0000","last_became_active":"2023-10-09T05:34:41.683519+0000","last_became_peered":"2023-10-09T05:34:41.683519+0000","last_unstale":"2023-10-09T05:37:05.616329+0000","last_undegraded":"2023-10-09T05:37:05.616329+0000","last_fullsized":"2023-10-09T05:37:05.616329+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":132,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.615979+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:37:05.615979+0000","last_peered":"2023-10-09T05:37:05.615979+0000","last_clean":"2023-10-09T05:37:05.615979+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:37:05.615979+0000","last_undegraded":"2023-10-09T05:37:05.615979+0000","last_fullsized":"2023-10-09T05:37:05.615979+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":119,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.637425+0000","last_change":"2023-10-09T05:34:41.673944+0000","last_active":"2023-10-09T05:37:04.637425+0000","last_peered":"2023-10-09T05:37:04.637425+0000","last_clean":"2023-10-09T05:37:04.637425+0000","last_became_active":"2023-10-09T05:34:41.673595+0000","last_became_peered":"2023-10-09T05:34:41.673595+0000","last_unstale":"2023-10-09T05:37:04.637425+0000","last_undegraded":"2023-10-09T05:37:04.637425+0000","last_fullsized":"2023-10-09T05:37:04.637425+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":151,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.712842+0000","last_change":"2023-10-09T05:34:41.667462+0000","last_active":"2023-10-09T05:37:02.712842+0000","last_peered":"2023-10-09T05:37:02.712842+0000","last_clean":"2023-10-09T05:37:02.712842+0000","last_became_active":"2023-10-09T05:34:41.667140+0000","last_became_peered":"2023-10-09T05:34:41.667140+0000","last_unstale":"2023-10-09T05:37:02.712842+0000","last_undegraded":"2023-10-09T05:37:02.712842+0000","last_fullsized":"2023-10-09T05:37:02.712842+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:52.888470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":132,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.632007+0000","last_change":"2023-10-09T05:34:41.662790+0000","last_active":"2023-10-09T05:37:04.632007+0000","last_peered":"2023-10-09T05:37:04.632007+0000","last_clean":"2023-10-09T05:37:04.632007+0000","last_became_active":"2023-10-09T05:34:41.662513+0000","last_became_peered":"2023-10-09T05:34:41.662513+0000","last_unstale":"2023-10-09T05:37:04.632007+0000","last_undegraded":"2023-10-09T05:37:04.632007+0000","last_fullsized":"2023-10-09T05:37:04.632007+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:17.781371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":147,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616426+0000","last_change":"2023-10-09T05:34:41.694695+0000","last_active":"2023-10-09T05:37:05.616426+0000","last_peered":"2023-10-09T05:37:05.616426+0000","last_clean":"2023-10-09T05:37:05.616426+0000","last_became_active":"2023-10-09T05:34:41.694408+0000","last_became_peered":"2023-10-09T05:34:41.694408+0000","last_unstale":"2023-10-09T05:37:05.616426+0000","last_undegraded":"2023-10-09T05:37:05.616426+0000","last_fullsized":"2023-10-09T05:37:05.616426+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":102,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.637182+0000","last_change":"2023-10-09T05:34:41.702205+0000","last_active":"2023-10-09T05:37:04.637182+0000","last_peered":"2023-10-09T05:37:04.637182+0000","last_clean":"2023-10-09T05:37:04.637182+0000","last_became_active":"2023-10-09T05:34:41.701849+0000","last_became_peered":"2023-10-09T05:34:41.701849+0000","last_unstale":"2023-10-09T05:37:04.637182+0000","last_undegraded":"2023-10-09T05:37:04.637182+0000","last_fullsized":"2023-10-09T05:37:04.637182+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":131,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.632235+0000","last_change":"2023-10-09T05:34:41.663111+0000","last_active":"2023-10-09T05:37:04.632235+0000","last_peered":"2023-10-09T05:37:04.632235+0000","last_clean":"2023-10-09T05:37:04.632235+0000","last_became_active":"2023-10-09T05:34:41.662816+0000","last_became_peered":"2023-10-09T05:34:41.662816+0000","last_unstale":"2023-10-09T05:37:04.632235+0000","last_undegraded":"2023-10-09T05:37:04.632235+0000","last_fullsized":"2023-10-09T05:37:04.632235+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:36.226743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":141,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.615348+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:37:05.615348+0000","last_peered":"2023-10-09T05:37:05.615348+0000","last_clean":"2023-10-09T05:37:05.615348+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:37:05.615348+0000","last_undegraded":"2023-10-09T05:37:05.615348+0000","last_fullsized":"2023-10-09T05:37:05.615348+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":151,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.719540+0000","last_change":"2023-10-09T05:34:41.664294+0000","last_active":"2023-10-09T05:37:02.719540+0000","last_peered":"2023-10-09T05:37:02.719540+0000","last_clean":"2023-10-09T05:37:02.719540+0000","last_became_active":"2023-10-09T05:34:41.663985+0000","last_became_peered":"2023-10-09T05:34:41.663985+0000","last_unstale":"2023-10-09T05:37:02.719540+0000","last_undegraded":"2023-10-09T05:37:02.719540+0000","last_fullsized":"2023-10-09T05:37:02.719540+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:58.417307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"166'357","reported_seq":369,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.894146+0000","last_change":"2023-10-09T05:37:01.636984+0000","last_active":"2023-10-09T05:37:09.894146+0000","last_peered":"2023-10-09T05:37:09.894146+0000","last_clean":"2023-10-09T05:37:09.894146+0000","last_became_active":"2023-10-09T05:37:01.636609+0000","last_became_peered":"2023-10-09T05:37:01.636609+0000","last_unstale":"2023-10-09T05:37:09.894146+0000","last_undegraded":"2023-10-09T05:37:09.894146+0000","last_fullsized":"2023-10-09T05:37:09.894146+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":357,"log_dups_size":0,"ondisk_log_size":357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23396352,"num_objects":357,"num_object_clones":0,"num_object_copies":1071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":357,"num_write_kb":22848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":108,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.637558+0000","last_change":"2023-10-09T05:34:41.688229+0000","last_active":"2023-10-09T05:37:04.637558+0000","last_peered":"2023-10-09T05:37:04.637558+0000","last_clean":"2023-10-09T05:37:04.637558+0000","last_became_active":"2023-10-09T05:34:41.687795+0000","last_became_peered":"2023-10-09T05:34:41.687795+0000","last_unstale":"2023-10-09T05:37:04.637558+0000","last_undegraded":"2023-10-09T05:37:04.637558+0000","last_fullsized":"2023-10-09T05:37:04.637558+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":132,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.615882+0000","last_change":"2023-10-09T05:34:41.678569+0000","last_active":"2023-10-09T05:37:05.615882+0000","last_peered":"2023-10-09T05:37:05.615882+0000","last_clean":"2023-10-09T05:37:05.615882+0000","last_became_active":"2023-10-09T05:34:41.678271+0000","last_became_peered":"2023-10-09T05:34:41.678271+0000","last_unstale":"2023-10-09T05:37:05.615882+0000","last_undegraded":"2023-10-09T05:37:05.615882+0000","last_fullsized":"2023-10-09T05:37:05.615882+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"0'0","reported_seq":12,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.712721+0000","last_change":"2023-10-09T05:37:01.437047+0000","last_active":"2023-10-09T05:37:02.712721+0000","last_peered":"2023-10-09T05:37:02.712721+0000","last_clean":"2023-10-09T05:37:02.712721+0000","last_became_active":"2023-10-09T05:37:01.436763+0000","last_became_peered":"2023-10-09T05:37:01.436763+0000","last_unstale":"2023-10-09T05:37:02.712721+0000","last_undegraded":"2023-10-09T05:37:02.712721+0000","last_fullsized":"2023-10-09T05:37:02.712721+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:17.862171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":114,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.637333+0000","last_change":"2023-10-09T05:34:41.674752+0000","last_active":"2023-10-09T05:37:04.637333+0000","last_peered":"2023-10-09T05:37:04.637333+0000","last_clean":"2023-10-09T05:37:04.637333+0000","last_became_active":"2023-10-09T05:34:41.674253+0000","last_became_peered":"2023-10-09T05:34:41.674253+0000","last_unstale":"2023-10-09T05:37:04.637333+0000","last_undegraded":"2023-10-09T05:37:04.637333+0000","last_fullsized":"2023-10-09T05:37:04.637333+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":147,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.615966+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:37:05.615966+0000","last_peered":"2023-10-09T05:37:05.615966+0000","last_clean":"2023-10-09T05:37:05.615966+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:37:05.615966+0000","last_undegraded":"2023-10-09T05:37:05.615966+0000","last_fullsized":"2023-10-09T05:37:05.615966+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"166'335","reported_seq":347,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.747504+0000","last_change":"2023-10-09T05:37:01.432987+0000","last_active":"2023-10-09T05:37:09.747504+0000","last_peered":"2023-10-09T05:37:09.747504+0000","last_clean":"2023-10-09T05:37:09.747504+0000","last_became_active":"2023-10-09T05:37:01.432746+0000","last_became_peered":"2023-10-09T05:37:01.432746+0000","last_unstale":"2023-10-09T05:37:09.747504+0000","last_undegraded":"2023-10-09T05:37:09.747504+0000","last_fullsized":"2023-10-09T05:37:09.747504+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":335,"log_dups_size":0,"ondisk_log_size":335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21954560,"num_objects":335,"num_object_clones":0,"num_object_copies":1005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":335,"num_write_kb":21440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":105,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.637608+0000","last_change":"2023-10-09T05:34:41.691449+0000","last_active":"2023-10-09T05:37:04.637608+0000","last_peered":"2023-10-09T05:37:04.637608+0000","last_clean":"2023-10-09T05:37:04.637608+0000","last_became_active":"2023-10-09T05:34:41.675227+0000","last_became_peered":"2023-10-09T05:34:41.675227+0000","last_unstale":"2023-10-09T05:37:04.637608+0000","last_undegraded":"2023-10-09T05:37:04.637608+0000","last_fullsized":"2023-10-09T05:37:04.637608+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":156,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.615442+0000","last_change":"2023-10-09T05:34:41.669908+0000","last_active":"2023-10-09T05:37:05.615442+0000","last_peered":"2023-10-09T05:37:05.615442+0000","last_clean":"2023-10-09T05:37:05.615442+0000","last_became_active":"2023-10-09T05:34:41.669596+0000","last_became_peered":"2023-10-09T05:34:41.669596+0000","last_unstale":"2023-10-09T05:37:05.615442+0000","last_undegraded":"2023-10-09T05:37:05.615442+0000","last_fullsized":"2023-10-09T05:37:05.615442+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"166'156","reported_seq":169,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:07.952762+0000","last_change":"2023-10-09T05:37:01.433668+0000","last_active":"2023-10-09T05:37:07.952762+0000","last_peered":"2023-10-09T05:37:07.952762+0000","last_clean":"2023-10-09T05:37:07.952762+0000","last_became_active":"2023-10-09T05:37:01.433332+0000","last_became_peered":"2023-10-09T05:37:01.433332+0000","last_unstale":"2023-10-09T05:37:07.952762+0000","last_undegraded":"2023-10-09T05:37:07.952762+0000","last_fullsized":"2023-10-09T05:37:07.952762+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":156,"log_dups_size":0,"ondisk_log_size":156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:06.789767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10223616,"num_objects":156,"num_object_clones":0,"num_object_copies":468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":156,"num_write_kb":9984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":133,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616810+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:37:05.616810+0000","last_peered":"2023-10-09T05:37:05.616810+0000","last_clean":"2023-10-09T05:37:05.616810+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:37:05.616810+0000","last_undegraded":"2023-10-09T05:37:05.616810+0000","last_fullsized":"2023-10-09T05:37:05.616810+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":117,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.631990+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:37:04.631990+0000","last_peered":"2023-10-09T05:37:04.631990+0000","last_clean":"2023-10-09T05:37:04.631990+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:37:04.631990+0000","last_undegraded":"2023-10-09T05:37:04.631990+0000","last_fullsized":"2023-10-09T05:37:04.631990+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"166'368","reported_seq":380,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.906700+0000","last_change":"2023-10-09T05:37:01.439878+0000","last_active":"2023-10-09T05:37:09.906700+0000","last_peered":"2023-10-09T05:37:09.906700+0000","last_clean":"2023-10-09T05:37:09.906700+0000","last_became_active":"2023-10-09T05:37:01.439481+0000","last_became_peered":"2023-10-09T05:37:01.439481+0000","last_unstale":"2023-10-09T05:37:09.906700+0000","last_undegraded":"2023-10-09T05:37:09.906700+0000","last_fullsized":"2023-10-09T05:37:09.906700+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":368,"log_dups_size":0,"ondisk_log_size":368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24117248,"num_objects":368,"num_object_clones":0,"num_object_copies":1104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":368,"num_write_kb":23552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":134,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.712818+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:37:02.712818+0000","last_peered":"2023-10-09T05:37:02.712818+0000","last_clean":"2023-10-09T05:37:02.712818+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:37:02.712818+0000","last_undegraded":"2023-10-09T05:37:02.712818+0000","last_fullsized":"2023-10-09T05:37:02.712818+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":132,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616241+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:37:05.616241+0000","last_peered":"2023-10-09T05:37:05.616241+0000","last_clean":"2023-10-09T05:37:05.616241+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:37:05.616241+0000","last_undegraded":"2023-10-09T05:37:05.616241+0000","last_fullsized":"2023-10-09T05:37:05.616241+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"0'0","reported_seq":12,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.425136+0000","last_change":"2023-10-09T05:37:01.440175+0000","last_active":"2023-10-09T05:37:02.425136+0000","last_peered":"2023-10-09T05:37:02.425136+0000","last_clean":"2023-10-09T05:37:02.425136+0000","last_became_active":"2023-10-09T05:37:01.439846+0000","last_became_peered":"2023-10-09T05:37:01.439846+0000","last_unstale":"2023-10-09T05:37:02.425136+0000","last_undegraded":"2023-10-09T05:37:02.425136+0000","last_fullsized":"2023-10-09T05:37:02.425136+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:27.883265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":133,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.615358+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:37:05.615358+0000","last_peered":"2023-10-09T05:37:05.615358+0000","last_clean":"2023-10-09T05:37:05.615358+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:37:05.615358+0000","last_undegraded":"2023-10-09T05:37:05.615358+0000","last_fullsized":"2023-10-09T05:37:05.615358+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":147,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616478+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:37:05.616478+0000","last_peered":"2023-10-09T05:37:05.616478+0000","last_clean":"2023-10-09T05:37:05.616478+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:37:05.616478+0000","last_undegraded":"2023-10-09T05:37:05.616478+0000","last_fullsized":"2023-10-09T05:37:05.616478+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":12,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.712925+0000","last_change":"2023-10-09T05:37:01.638178+0000","last_active":"2023-10-09T05:37:02.712925+0000","last_peered":"2023-10-09T05:37:02.712925+0000","last_clean":"2023-10-09T05:37:02.712925+0000","last_became_active":"2023-10-09T05:37:01.637822+0000","last_became_peered":"2023-10-09T05:37:01.637822+0000","last_unstale":"2023-10-09T05:37:02.712925+0000","last_undegraded":"2023-10-09T05:37:02.712925+0000","last_fullsized":"2023-10-09T05:37:02.712925+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:13.240069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":148,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.615736+0000","last_change":"2023-10-09T05:34:41.706409+0000","last_active":"2023-10-09T05:37:05.615736+0000","last_peered":"2023-10-09T05:37:05.615736+0000","last_clean":"2023-10-09T05:37:05.615736+0000","last_became_active":"2023-10-09T05:34:41.706002+0000","last_became_peered":"2023-10-09T05:34:41.706002+0000","last_unstale":"2023-10-09T05:37:05.615736+0000","last_undegraded":"2023-10-09T05:37:05.615736+0000","last_fullsized":"2023-10-09T05:37:05.615736+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":141,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616326+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:37:05.616326+0000","last_peered":"2023-10-09T05:37:05.616326+0000","last_clean":"2023-10-09T05:37:05.616326+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:37:05.616326+0000","last_undegraded":"2023-10-09T05:37:05.616326+0000","last_fullsized":"2023-10-09T05:37:05.616326+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"166'398","reported_seq":410,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.903270+0000","last_change":"2023-10-09T05:37:01.438917+0000","last_active":"2023-10-09T05:37:09.903270+0000","last_peered":"2023-10-09T05:37:09.903270+0000","last_clean":"2023-10-09T05:37:09.903270+0000","last_became_active":"2023-10-09T05:37:01.438658+0000","last_became_peered":"2023-10-09T05:37:01.438658+0000","last_unstale":"2023-10-09T05:37:09.903270+0000","last_undegraded":"2023-10-09T05:37:09.903270+0000","last_fullsized":"2023-10-09T05:37:09.903270+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":398,"log_dups_size":0,"ondisk_log_size":398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26083328,"num_objects":398,"num_object_clones":0,"num_object_copies":1194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":398,"num_write_kb":25472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":152,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.712284+0000","last_change":"2023-10-09T05:34:41.666221+0000","last_active":"2023-10-09T05:37:02.712284+0000","last_peered":"2023-10-09T05:37:02.712284+0000","last_clean":"2023-10-09T05:37:02.712284+0000","last_became_active":"2023-10-09T05:34:41.665842+0000","last_became_peered":"2023-10-09T05:34:41.665842+0000","last_unstale":"2023-10-09T05:37:02.712284+0000","last_undegraded":"2023-10-09T05:37:02.712284+0000","last_fullsized":"2023-10-09T05:37:02.712284+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:28.435851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":111,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.614512+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:37:05.614512+0000","last_peered":"2023-10-09T05:37:05.614512+0000","last_clean":"2023-10-09T05:37:05.614512+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:37:05.614512+0000","last_undegraded":"2023-10-09T05:37:05.614512+0000","last_fullsized":"2023-10-09T05:37:05.614512+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"166'160","reported_seq":173,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:07.946147+0000","last_change":"2023-10-09T05:37:01.440093+0000","last_active":"2023-10-09T05:37:07.946147+0000","last_peered":"2023-10-09T05:37:07.946147+0000","last_clean":"2023-10-09T05:37:07.946147+0000","last_became_active":"2023-10-09T05:37:01.439799+0000","last_became_peered":"2023-10-09T05:37:01.439799+0000","last_unstale":"2023-10-09T05:37:07.946147+0000","last_undegraded":"2023-10-09T05:37:07.946147+0000","last_fullsized":"2023-10-09T05:37:07.946147+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":160,"log_dups_size":0,"ondisk_log_size":160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:28.734471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10485760,"num_objects":160,"num_object_clones":0,"num_object_copies":480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":160,"num_write_kb":10240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":691,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.614256+0000","last_change":"2023-10-09T05:34:41.667358+0000","last_active":"2023-10-09T05:37:05.614256+0000","last_peered":"2023-10-09T05:37:05.614256+0000","last_clean":"2023-10-09T05:37:05.614256+0000","last_became_active":"2023-10-09T05:34:41.666757+0000","last_became_peered":"2023-10-09T05:34:41.666757+0000","last_unstale":"2023-10-09T05:37:05.614256+0000","last_undegraded":"2023-10-09T05:37:05.614256+0000","last_fullsized":"2023-10-09T05:37:05.614256+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:45:47.725111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":148,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616346+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:37:05.616346+0000","last_peered":"2023-10-09T05:37:05.616346+0000","last_clean":"2023-10-09T05:37:05.616346+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:37:05.616346+0000","last_undegraded":"2023-10-09T05:37:05.616346+0000","last_fullsized":"2023-10-09T05:37:05.616346+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":97,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.613773+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:37:05.613773+0000","last_peered":"2023-10-09T05:37:05.613773+0000","last_clean":"2023-10-09T05:37:05.613773+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:37:05.613773+0000","last_undegraded":"2023-10-09T05:37:05.613773+0000","last_fullsized":"2023-10-09T05:37:05.613773+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"0'0","reported_seq":12,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.713197+0000","last_change":"2023-10-09T05:37:01.439942+0000","last_active":"2023-10-09T05:37:02.713197+0000","last_peered":"2023-10-09T05:37:02.713197+0000","last_clean":"2023-10-09T05:37:02.713197+0000","last_became_active":"2023-10-09T05:37:01.439615+0000","last_became_peered":"2023-10-09T05:37:01.439615+0000","last_unstale":"2023-10-09T05:37:02.713197+0000","last_undegraded":"2023-10-09T05:37:02.713197+0000","last_fullsized":"2023-10-09T05:37:02.713197+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:27.915134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":602,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616414+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:37:05.616414+0000","last_peered":"2023-10-09T05:37:05.616414+0000","last_clean":"2023-10-09T05:37:05.616414+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:37:05.616414+0000","last_undegraded":"2023-10-09T05:37:05.616414+0000","last_fullsized":"2023-10-09T05:37:05.616414+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":133,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616128+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:37:05.616128+0000","last_peered":"2023-10-09T05:37:05.616128+0000","last_clean":"2023-10-09T05:37:05.616128+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:37:05.616128+0000","last_undegraded":"2023-10-09T05:37:05.616128+0000","last_fullsized":"2023-10-09T05:37:05.616128+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":145,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616938+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:37:05.616938+0000","last_peered":"2023-10-09T05:37:05.616938+0000","last_clean":"2023-10-09T05:37:05.616938+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:37:05.616938+0000","last_undegraded":"2023-10-09T05:37:05.616938+0000","last_fullsized":"2023-10-09T05:37:05.616938+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"166'359","reported_seq":370,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.759283+0000","last_change":"2023-10-09T05:37:01.636855+0000","last_active":"2023-10-09T05:37:09.759283+0000","last_peered":"2023-10-09T05:37:09.759283+0000","last_clean":"2023-10-09T05:37:09.759283+0000","last_became_active":"2023-10-09T05:37:01.636510+0000","last_became_peered":"2023-10-09T05:37:01.636510+0000","last_unstale":"2023-10-09T05:37:09.759283+0000","last_undegraded":"2023-10-09T05:37:09.759283+0000","last_fullsized":"2023-10-09T05:37:09.759283+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":359,"log_dups_size":0,"ondisk_log_size":359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23527424,"num_objects":359,"num_object_clones":0,"num_object_copies":1077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":359,"num_write_kb":22976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":477,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.424796+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:37:02.424796+0000","last_peered":"2023-10-09T05:37:02.424796+0000","last_clean":"2023-10-09T05:37:02.424796+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:37:02.424796+0000","last_undegraded":"2023-10-09T05:37:02.424796+0000","last_fullsized":"2023-10-09T05:37:02.424796+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":112,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.614148+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:37:05.614148+0000","last_peered":"2023-10-09T05:37:05.614148+0000","last_clean":"2023-10-09T05:37:05.614148+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:37:05.614148+0000","last_undegraded":"2023-10-09T05:37:05.614148+0000","last_fullsized":"2023-10-09T05:37:05.614148+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":142,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.614570+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:37:05.614570+0000","last_peered":"2023-10-09T05:37:05.614570+0000","last_clean":"2023-10-09T05:37:05.614570+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:37:05.614570+0000","last_undegraded":"2023-10-09T05:37:05.614570+0000","last_fullsized":"2023-10-09T05:37:05.614570+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"166'355","reported_seq":367,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.916520+0000","last_change":"2023-10-09T05:37:01.438236+0000","last_active":"2023-10-09T05:37:09.916520+0000","last_peered":"2023-10-09T05:37:09.916520+0000","last_clean":"2023-10-09T05:37:09.916520+0000","last_became_active":"2023-10-09T05:37:01.437917+0000","last_became_peered":"2023-10-09T05:37:01.437917+0000","last_unstale":"2023-10-09T05:37:09.916520+0000","last_undegraded":"2023-10-09T05:37:09.916520+0000","last_fullsized":"2023-10-09T05:37:09.916520+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":355,"log_dups_size":0,"ondisk_log_size":355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23265280,"num_objects":355,"num_object_clones":0,"num_object_copies":1065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":355,"num_write_kb":22720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":742,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616087+0000","last_change":"2023-10-09T05:34:41.677976+0000","last_active":"2023-10-09T05:37:05.616087+0000","last_peered":"2023-10-09T05:37:05.616087+0000","last_clean":"2023-10-09T05:37:05.616087+0000","last_became_active":"2023-10-09T05:34:41.677520+0000","last_became_peered":"2023-10-09T05:34:41.677520+0000","last_unstale":"2023-10-09T05:37:05.616087+0000","last_undegraded":"2023-10-09T05:37:05.616087+0000","last_fullsized":"2023-10-09T05:37:05.616087+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":343,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.637138+0000","last_change":"2023-10-09T05:34:41.686013+0000","last_active":"2023-10-09T05:37:04.637138+0000","last_peered":"2023-10-09T05:37:04.637138+0000","last_clean":"2023-10-09T05:37:04.637138+0000","last_became_active":"2023-10-09T05:34:41.680809+0000","last_became_peered":"2023-10-09T05:34:41.680809+0000","last_unstale":"2023-10-09T05:37:04.637138+0000","last_undegraded":"2023-10-09T05:37:04.637138+0000","last_fullsized":"2023-10-09T05:37:04.637138+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":127,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.712379+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:37:02.712379+0000","last_peered":"2023-10-09T05:37:02.712379+0000","last_clean":"2023-10-09T05:37:02.712379+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:37:02.712379+0000","last_undegraded":"2023-10-09T05:37:02.712379+0000","last_fullsized":"2023-10-09T05:37:02.712379+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":132,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.615764+0000","last_change":"2023-10-09T05:34:41.693650+0000","last_active":"2023-10-09T05:37:05.615764+0000","last_peered":"2023-10-09T05:37:05.615764+0000","last_clean":"2023-10-09T05:37:05.615764+0000","last_became_active":"2023-10-09T05:34:41.693305+0000","last_became_peered":"2023-10-09T05:34:41.693305+0000","last_unstale":"2023-10-09T05:37:05.615764+0000","last_undegraded":"2023-10-09T05:37:05.615764+0000","last_fullsized":"2023-10-09T05:37:05.615764+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"166'227","reported_seq":240,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:08.491294+0000","last_change":"2023-10-09T05:37:01.440172+0000","last_active":"2023-10-09T05:37:08.491294+0000","last_peered":"2023-10-09T05:37:08.491294+0000","last_clean":"2023-10-09T05:37:08.491294+0000","last_became_active":"2023-10-09T05:37:01.439932+0000","last_became_peered":"2023-10-09T05:37:01.439932+0000","last_unstale":"2023-10-09T05:37:08.491294+0000","last_undegraded":"2023-10-09T05:37:08.491294+0000","last_fullsized":"2023-10-09T05:37:08.491294+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":227,"log_dups_size":0,"ondisk_log_size":227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:57.826704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14876672,"num_objects":227,"num_object_clones":0,"num_object_copies":681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":227,"num_write_kb":14528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":766,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.637041+0000","last_change":"2023-10-09T05:34:41.668526+0000","last_active":"2023-10-09T05:37:04.637041+0000","last_peered":"2023-10-09T05:37:04.637041+0000","last_clean":"2023-10-09T05:37:04.637041+0000","last_became_active":"2023-10-09T05:34:41.668010+0000","last_became_peered":"2023-10-09T05:34:41.668010+0000","last_unstale":"2023-10-09T05:37:04.637041+0000","last_undegraded":"2023-10-09T05:37:04.637041+0000","last_fullsized":"2023-10-09T05:37:04.637041+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":117,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.631154+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:37:04.631154+0000","last_peered":"2023-10-09T05:37:04.631154+0000","last_clean":"2023-10-09T05:37:04.631154+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:37:04.631154+0000","last_undegraded":"2023-10-09T05:37:04.631154+0000","last_fullsized":"2023-10-09T05:37:04.631154+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":132,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.632371+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:37:04.632371+0000","last_peered":"2023-10-09T05:37:04.632371+0000","last_clean":"2023-10-09T05:37:04.632371+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:37:04.632371+0000","last_undegraded":"2023-10-09T05:37:04.632371+0000","last_fullsized":"2023-10-09T05:37:04.632371+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"166'253","reported_seq":266,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:08.484300+0000","last_change":"2023-10-09T05:37:01.636967+0000","last_active":"2023-10-09T05:37:08.484300+0000","last_peered":"2023-10-09T05:37:08.484300+0000","last_clean":"2023-10-09T05:37:08.484300+0000","last_became_active":"2023-10-09T05:37:01.636602+0000","last_became_peered":"2023-10-09T05:37:01.636602+0000","last_unstale":"2023-10-09T05:37:08.484300+0000","last_undegraded":"2023-10-09T05:37:08.484300+0000","last_fullsized":"2023-10-09T05:37:08.484300+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":253,"log_dups_size":0,"ondisk_log_size":253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:11.668143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16580608,"num_objects":253,"num_object_clones":0,"num_object_copies":759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":253,"num_write_kb":16192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":369,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.637493+0000","last_change":"2023-10-09T05:34:41.682539+0000","last_active":"2023-10-09T05:37:04.637493+0000","last_peered":"2023-10-09T05:37:04.637493+0000","last_clean":"2023-10-09T05:37:04.637493+0000","last_became_active":"2023-10-09T05:34:41.674099+0000","last_became_peered":"2023-10-09T05:34:41.674099+0000","last_unstale":"2023-10-09T05:37:04.637493+0000","last_undegraded":"2023-10-09T05:37:04.637493+0000","last_fullsized":"2023-10-09T05:37:04.637493+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":112,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.614518+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:37:05.614518+0000","last_peered":"2023-10-09T05:37:05.614518+0000","last_clean":"2023-10-09T05:37:05.614518+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:37:05.614518+0000","last_undegraded":"2023-10-09T05:37:05.614518+0000","last_fullsized":"2023-10-09T05:37:05.614518+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":117,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.632135+0000","last_change":"2023-10-09T05:34:41.671414+0000","last_active":"2023-10-09T05:37:04.632135+0000","last_peered":"2023-10-09T05:37:04.632135+0000","last_clean":"2023-10-09T05:37:04.632135+0000","last_became_active":"2023-10-09T05:34:41.671100+0000","last_became_peered":"2023-10-09T05:34:41.671100+0000","last_unstale":"2023-10-09T05:37:04.632135+0000","last_undegraded":"2023-10-09T05:37:04.632135+0000","last_fullsized":"2023-10-09T05:37:04.632135+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:19.832596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"166'355","reported_seq":367,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.909690+0000","last_change":"2023-10-09T05:37:01.440074+0000","last_active":"2023-10-09T05:37:09.909690+0000","last_peered":"2023-10-09T05:37:09.909690+0000","last_clean":"2023-10-09T05:37:09.909690+0000","last_became_active":"2023-10-09T05:37:01.439808+0000","last_became_peered":"2023-10-09T05:37:01.439808+0000","last_unstale":"2023-10-09T05:37:09.909690+0000","last_undegraded":"2023-10-09T05:37:09.909690+0000","last_fullsized":"2023-10-09T05:37:09.909690+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":355,"log_dups_size":0,"ondisk_log_size":355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23265280,"num_objects":355,"num_object_clones":0,"num_object_copies":1065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":355,"num_write_kb":22720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":149,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.719555+0000","last_change":"2023-10-09T05:34:41.664569+0000","last_active":"2023-10-09T05:37:02.719555+0000","last_peered":"2023-10-09T05:37:02.719555+0000","last_clean":"2023-10-09T05:37:02.719555+0000","last_became_active":"2023-10-09T05:34:41.664239+0000","last_became_peered":"2023-10-09T05:34:41.664239+0000","last_unstale":"2023-10-09T05:37:02.719555+0000","last_undegraded":"2023-10-09T05:37:02.719555+0000","last_fullsized":"2023-10-09T05:37:02.719555+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:26.511248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":553,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.614424+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:37:05.614424+0000","last_peered":"2023-10-09T05:37:05.614424+0000","last_clean":"2023-10-09T05:37:05.614424+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:37:05.614424+0000","last_undegraded":"2023-10-09T05:37:05.614424+0000","last_fullsized":"2023-10-09T05:37:05.614424+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":146,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.712415+0000","last_change":"2023-10-09T05:34:41.666200+0000","last_active":"2023-10-09T05:37:02.712415+0000","last_peered":"2023-10-09T05:37:02.712415+0000","last_clean":"2023-10-09T05:37:02.712415+0000","last_became_active":"2023-10-09T05:34:41.665841+0000","last_became_peered":"2023-10-09T05:34:41.665841+0000","last_unstale":"2023-10-09T05:37:02.712415+0000","last_undegraded":"2023-10-09T05:37:02.712415+0000","last_fullsized":"2023-10-09T05:37:02.712415+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:58.338181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"166'317","reported_seq":329,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-10-09T05:37:09.750578+0000","last_change":"2023-10-09T05:37:01.636899+0000","last_active":"2023-10-09T05:37:09.750578+0000","last_peered":"2023-10-09T05:37:09.750578+0000","last_clean":"2023-10-09T05:37:09.750578+0000","last_became_active":"2023-10-09T05:37:01.636550+0000","last_became_peered":"2023-10-09T05:37:01.636550+0000","last_unstale":"2023-10-09T05:37:09.750578+0000","last_undegraded":"2023-10-09T05:37:09.750578+0000","last_fullsized":"2023-10-09T05:37:09.750578+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":317,"log_dups_size":0,"ondisk_log_size":317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20774912,"num_objects":317,"num_object_clones":0,"num_object_copies":951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":317,"num_write_kb":20288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":629,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616390+0000","last_change":"2023-10-09T05:34:41.676842+0000","last_active":"2023-10-09T05:37:05.616390+0000","last_peered":"2023-10-09T05:37:05.616390+0000","last_clean":"2023-10-09T05:37:05.616390+0000","last_became_active":"2023-10-09T05:34:41.676428+0000","last_became_peered":"2023-10-09T05:34:41.676428+0000","last_unstale":"2023-10-09T05:37:05.616390+0000","last_undegraded":"2023-10-09T05:37:05.616390+0000","last_fullsized":"2023-10-09T05:37:05.616390+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":120,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.632303+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:37:04.632303+0000","last_peered":"2023-10-09T05:37:04.632303+0000","last_clean":"2023-10-09T05:37:04.632303+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:37:04.632303+0000","last_undegraded":"2023-10-09T05:37:04.632303+0000","last_fullsized":"2023-10-09T05:37:04.632303+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":117,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.615822+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:37:05.615822+0000","last_peered":"2023-10-09T05:37:05.615822+0000","last_clean":"2023-10-09T05:37:05.615822+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:37:05.615822+0000","last_undegraded":"2023-10-09T05:37:05.615822+0000","last_fullsized":"2023-10-09T05:37:05.615822+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":119,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.632086+0000","last_change":"2023-10-09T05:34:41.662989+0000","last_active":"2023-10-09T05:37:04.632086+0000","last_peered":"2023-10-09T05:37:04.632086+0000","last_clean":"2023-10-09T05:37:04.632086+0000","last_became_active":"2023-10-09T05:34:41.662621+0000","last_became_peered":"2023-10-09T05:34:41.662621+0000","last_unstale":"2023-10-09T05:37:04.632086+0000","last_undegraded":"2023-10-09T05:37:04.632086+0000","last_fullsized":"2023-10-09T05:37:04.632086+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:08.750044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":155,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.424950+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:37:02.424950+0000","last_peered":"2023-10-09T05:37:02.424950+0000","last_clean":"2023-10-09T05:37:02.424950+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:37:02.424950+0000","last_undegraded":"2023-10-09T05:37:02.424950+0000","last_fullsized":"2023-10-09T05:37:02.424950+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":135,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616733+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:37:05.616733+0000","last_peered":"2023-10-09T05:37:05.616733+0000","last_clean":"2023-10-09T05:37:05.616733+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:37:05.616733+0000","last_undegraded":"2023-10-09T05:37:05.616733+0000","last_fullsized":"2023-10-09T05:37:05.616733+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":147,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616207+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:37:05.616207+0000","last_peered":"2023-10-09T05:37:05.616207+0000","last_clean":"2023-10-09T05:37:05.616207+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:37:05.616207+0000","last_undegraded":"2023-10-09T05:37:05.616207+0000","last_fullsized":"2023-10-09T05:37:05.616207+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":120,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616229+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:37:05.616229+0000","last_peered":"2023-10-09T05:37:05.616229+0000","last_clean":"2023-10-09T05:37:05.616229+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:37:05.616229+0000","last_undegraded":"2023-10-09T05:37:05.616229+0000","last_fullsized":"2023-10-09T05:37:05.616229+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":147,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.616884+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:37:05.616884+0000","last_peered":"2023-10-09T05:37:05.616884+0000","last_clean":"2023-10-09T05:37:05.616884+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:37:05.616884+0000","last_undegraded":"2023-10-09T05:37:05.616884+0000","last_fullsized":"2023-10-09T05:37:05.616884+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":142,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.713265+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:37:02.713265+0000","last_peered":"2023-10-09T05:37:02.713265+0000","last_clean":"2023-10-09T05:37:02.713265+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:37:02.713265+0000","last_undegraded":"2023-10-09T05:37:02.713265+0000","last_fullsized":"2023-10-09T05:37:02.713265+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":155,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T05:37:02.712299+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:37:02.712299+0000","last_peered":"2023-10-09T05:37:02.712299+0000","last_clean":"2023-10-09T05:37:02.712299+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:37:02.712299+0000","last_undegraded":"2023-10-09T05:37:02.712299+0000","last_fullsized":"2023-10-09T05:37:02.712299+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":120,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:04.632400+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:37:04.632400+0000","last_peered":"2023-10-09T05:37:04.632400+0000","last_clean":"2023-10-09T05:37:04.632400+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:37:04.632400+0000","last_undegraded":"2023-10-09T05:37:04.632400+0000","last_fullsized":"2023-10-09T05:37:04.632400+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":132,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-10-09T05:37:05.615687+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:37:05.615687+0000","last_peered":"2023-10-09T05:37:05.615687+0000","last_clean":"2023-10-09T05:37:05.615687+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:37:05.615687+0000","last_undegraded":"2023-10-09T05:37:05.615687+0000","last_fullsized":"2023-10-09T05:37:05.615687+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":238551040,"num_objects":3640,"num_object_clones":0,"num_object_copies":10920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3640,"num_write_kb":232960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":603324416,"data_stored":603324416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3640,"ondisk_log_size":3640,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846623,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":441572,"kb_used_data":131356,"kb_used_omap":14,"kb_used_meta":310193,"kb_avail":93303580,"statfs":{"total":95995035648,"available":95542865920,"internally_reserved":0,"allocated":134508544,"data_stored":133984072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":317638490},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":6.2050000000000001,"15min":6.9889999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.1240000000000006,"5min":60.323999999999998,"15min":60.323999999999998},"last":0.504},{"interface":"front","average":{"1min":2.2349999999999999,"5min":7.2850000000000001,"15min":8.1270000000000007},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":9.0820000000000007,"5min":60.334000000000003,"15min":60.334000000000003},"last":0.48499999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:36:45 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.607,"15min":1.615},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.3789999999999996,"5min":8.3789999999999996,"15min":8.3789999999999996},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.2989999999999999,"5min":1.2709999999999999,"15min":1.2669999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.5449999999999999,"5min":11.585000000000001,"15min":11.585000000000001},"last":0.47399999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:36:45 2023","interfaces":[{"interface":"back","average":{"1min":6.5069999999999997,"5min":6.7880000000000003,"15min":6.835},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":78.716999999999999,"5min":78.716999999999999,"15min":78.716999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":5.8380000000000001,"5min":6.6059999999999999,"15min":6.734},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":78.768000000000001,"5min":78.768000000000001,"15min":78.768000000000001},"last":0.374}]},{"osd":3,"last update":"Mon Oct 9 05:36:45 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":0.96499999999999997,"15min":0.85299999999999998},"min":{"1min":0.35299999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.8949999999999996,"5min":5.8949999999999996,"15min":5.8949999999999996},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.909,"5min":2.0209999999999999,"15min":2.0390000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.9939999999999998,"5min":7.9939999999999998,"15min":7.9939999999999998},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:36:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.843,"15min":1.9430000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.8070000000000004,"5min":12.789,"15min":12.789},"last":0.443},{"interface":"front","average":{"1min":1.9670000000000001,"5min":2.1949999999999998,"15min":2.2330000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":9.0470000000000006,"5min":9.0470000000000006,"15min":9.0470000000000006},"last":0.33900000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:36:45 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":2.073,"15min":2.0670000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.813000000000001,"5min":10.813000000000001,"15min":10.813000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.2349999999999999,"5min":2.6680000000000001,"15min":2.7400000000000002},"min":{"1min":0.32200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":10.321,"5min":10.321,"15min":10.321},"last":0.42199999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:36:45 2023","interfaces":[{"interface":"back","average":{"1min":2.1030000000000002,"5min":2.1829999999999998,"15min":2.1970000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.2530000000000001,"5min":9.2530000000000001,"15min":9.2530000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.296,"5min":1.9239999999999999,"15min":2.028},"min":{"1min":0.378,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.4160000000000004,"5min":8.2680000000000007,"15min":8.2680000000000007},"last":0.45500000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626649,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":416600,"kb_used_data":111184,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93328552,"statfs":{"total":95995035648,"available":95568437248,"internally_reserved":0,"allocated":113852416,"data_stored":113339840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:14 2023","interfaces":[{"interface":"back","average":{"1min":8.5340000000000007,"5min":3.4380000000000002,"15min":1.506},"min":{"1min":0.27800000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":77.024000000000001,"5min":77.024000000000001,"15min":77.024000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":8.1349999999999998,"5min":3.3239999999999998,"15min":1.4710000000000001},"min":{"1min":0.27600000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":79.305999999999997,"5min":79.305999999999997,"15min":79.305999999999997},"last":0.35899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:36:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":1.931,"15min":1.012},"min":{"1min":0.28599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":10.1,"5min":10.1,"15min":10.1},"last":0.374},{"interface":"front","average":{"1min":2.819,"5min":2.2080000000000002,"15min":1.1200000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.8309999999999995,"5min":9.9979999999999993,"15min":9.9979999999999993},"last":0.38800000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:36:14 2023","interfaces":[{"interface":"back","average":{"1min":4.516,"5min":3.0670000000000002,"15min":1.7390000000000001},"min":{"1min":0.33600000000000002,"5min":0.314,"15min":0.314},"max":{"1min":36.832000000000001,"5min":36.832000000000001,"15min":36.832000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":4.0019999999999998,"5min":2.879,"15min":1.6890000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":36.811999999999998,"5min":36.811999999999998,"15min":36.811999999999998},"last":0.47399999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:36:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1429999999999998,"5min":1.7809999999999999,"15min":1.3029999999999999},"min":{"1min":0.35799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.135,"5min":18.361000000000001,"15min":18.361000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.3780000000000001,"5min":2.024,"15min":1.423},"min":{"1min":0.35799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":9.6630000000000003,"5min":18.536000000000001,"15min":18.536000000000001},"last":0.46700000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:36:52 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":3.274,"15min":1.4079999999999999},"min":{"1min":0.254,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":17.617000000000001,"5min":125.64,"15min":125.64},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":2.8530000000000002,"15min":1.2769999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.3959999999999999,"5min":125.625,"15min":125.625},"last":0.39600000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:36:52 2023","interfaces":[{"interface":"back","average":{"1min":2.754,"5min":2.9199999999999999,"15min":1.321},"min":{"1min":0.28899999999999998,"5min":0.27900000000000003,"15min":0.27200000000000002},"max":{"1min":16.091000000000001,"5min":127.35899999999999,"15min":127.35899999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.028,"5min":2.6219999999999999,"15min":1.2290000000000001},"min":{"1min":0.29599999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.9990000000000001,"5min":128.93799999999999,"15min":128.93799999999999},"last":0.45700000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:36:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7879999999999998,"5min":3.0760000000000001,"15min":3.1240000000000001},"min":{"1min":0.32200000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.734,"5min":12.074999999999999,"15min":12.074999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.9350000000000001,"15min":2.0390000000000001},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.0279999999999996,"5min":10.083,"15min":10.083},"last":0.57999999999999996}]}]},{"osd":1,"up_from":93,"seq":399431958610,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":227088,"kb_used_data":60872,"kb_used_omap":7,"kb_used_meta":166200,"kb_avail":93518064,"statfs":{"total":95995035648,"available":95762497536,"internally_reserved":0,"allocated":62332928,"data_stored":61654801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":170189326},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:44 2023","interfaces":[{"interface":"back","average":{"1min":2.294,"5min":2.6499999999999999,"15min":1.1519999999999999},"min":{"1min":0.19800000000000001,"5min":0.185,"15min":0.185},"max":{"1min":17.355,"5min":124.82599999999999,"15min":124.82599999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":3.3700000000000001,"5min":2.96,"15min":1.258},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":17.370999999999999,"5min":125.483,"15min":125.483},"last":1.3420000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:37:08 2023","interfaces":[{"interface":"back","average":{"1min":10.319000000000001,"5min":4.774,"15min":1.819},"min":{"1min":0.252,"5min":0.184,"15min":0.184},"max":{"1min":84.200999999999993,"5min":98.483999999999995,"15min":98.483999999999995},"last":2.2090000000000001},{"interface":"front","average":{"1min":9.6850000000000005,"5min":4.4199999999999999,"15min":1.7490000000000001},"min":{"1min":0.29999999999999999,"5min":0.251,"15min":0.251},"max":{"1min":84.153000000000006,"5min":98.409999999999997,"15min":98.409999999999997},"last":2.0750000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:36:44 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":1.387,"15min":0.78100000000000003},"min":{"1min":0.23400000000000001,"5min":0.223,"15min":0.223},"max":{"1min":10.314,"5min":10.314,"15min":10.314},"last":2.2509999999999999},{"interface":"front","average":{"1min":1.5509999999999999,"5min":1.4159999999999999,"15min":0.74099999999999999},"min":{"1min":0.28699999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.5339999999999998,"5min":8.8109999999999999,"15min":8.8109999999999999},"last":0.45500000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:36:44 2023","interfaces":[{"interface":"back","average":{"1min":2.5139999999999998,"5min":1.8129999999999999,"15min":1.0329999999999999},"min":{"1min":0.33200000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.8640000000000008,"5min":23.670999999999999,"15min":23.670999999999999},"last":1.294},{"interface":"front","average":{"1min":1.96,"5min":1.4379999999999999,"15min":0.92000000000000004},"min":{"1min":0.38900000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.444,"5min":23.620000000000001,"15min":23.620000000000001},"last":0.64600000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:36:44 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":1.7390000000000001,"15min":1.0009999999999999},"min":{"1min":0.376,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.276,"5min":23.501999999999999,"15min":23.501999999999999},"last":2.2810000000000001},{"interface":"front","average":{"1min":2.3039999999999998,"5min":1.964,"15min":1.1060000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":8.9380000000000006,"5min":25.68,"15min":25.68},"last":1.409}]},{"osd":6,"last update":"Mon Oct 9 05:36:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3530000000000002,"5min":1.5069999999999999,"15min":0.93899999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":10.333,"5min":10.333,"15min":10.333},"last":2.3069999999999999},{"interface":"front","average":{"1min":2.403,"5min":1.585,"15min":0.98499999999999999},"min":{"1min":0.41299999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":9.0749999999999993,"5min":9.0749999999999993,"15min":9.0749999999999993},"last":1.3779999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:36:44 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.9259999999999999,"15min":1.974},"min":{"1min":0.42699999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.6379999999999999,"5min":8.6379999999999999,"15min":8.6379999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.1509999999999998,"5min":2.093,"15min":2.0830000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.5980000000000008,"5min":8.5980000000000008,"15min":8.5980000000000008},"last":0.98099999999999998}]}]},{"osd":0,"up_from":88,"seq":377957122136,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":277828,"kb_used_data":34172,"kb_used_omap":8,"kb_used_meta":243639,"kb_avail":93467324,"statfs":{"total":95995035648,"available":95710539776,"internally_reserved":0,"allocated":34992128,"data_stored":34324421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":249487112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:36:44 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":1.5129999999999999,"15min":0.76800000000000002},"min":{"1min":0.23400000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":10.513,"5min":13.919,"15min":13.919},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.7709999999999999,"5min":1.621,"15min":0.83299999999999996},"min":{"1min":0.23100000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":10.563000000000001,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.31900000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":8.6180000000000003,"5min":3.4969999999999999,"15min":1.403},"min":{"1min":0.223,"5min":0.214,"15min":0.214},"max":{"1min":79.911000000000001,"5min":79.911000000000001,"15min":79.911000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":8.1799999999999997,"5min":3.2440000000000002,"15min":1.3300000000000001},"min":{"1min":0.23200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":80.105000000000004,"5min":80.105000000000004,"15min":80.105000000000004},"last":0.28199999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:36:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":1.8200000000000001,"15min":0.874},"min":{"1min":0.28699999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.744999999999999,"5min":12.429,"15min":12.429},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.7810000000000001,"5min":2.0790000000000002,"15min":0.95999999999999996},"min":{"1min":0.32300000000000001,"5min":0.222,"15min":0.222},"max":{"1min":17.135000000000002,"5min":22.390999999999998,"15min":22.390999999999998},"last":0.35499999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.1859999999999999,"15min":0.84899999999999998},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.6130000000000004,"5min":14.023,"15min":14.023},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.542,"15min":0.995},"min":{"1min":0.41099999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.75,"5min":14.096,"15min":14.096},"last":0.438}]},{"osd":5,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":1.5529999999999999,"15min":0.97099999999999997},"min":{"1min":0.34499999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.8019999999999996,"5min":13.632,"15min":13.632},"last":0.38},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.5600000000000001,"15min":0.96999999999999997},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":8.1549999999999994,"5min":13.676,"15min":13.676},"last":0.44900000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:36:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.5289999999999999,"15min":0.98099999999999998},"min":{"1min":0.39200000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.8339999999999996,"5min":9.9809999999999999,"15min":9.9809999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.141,"5min":1.597,"15min":0.98499999999999999},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.4450000000000003,"5min":7.4450000000000003,"15min":7.4450000000000003},"last":0.42199999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:36:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":1.476,"15min":1.3779999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":9.2400000000000002,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.67100000000000004},{"interface":"front","average":{"1min":2.6859999999999999,"5min":2.4489999999999998,"15min":2.4089999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.7159999999999993,"5min":11.068,"15min":11.068},"last":0.65800000000000003}]}]},{"osd":2,"up_from":98,"seq":420906795083,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":224860,"kb_used_data":5844,"kb_used_omap":11,"kb_used_meta":218996,"kb_avail":93520292,"statfs":{"total":95995035648,"available":95764779008,"internally_reserved":0,"allocated":5984256,"data_stored":5286949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":224252501},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2999999999999998,"5min":1.5620000000000001,"15min":0.75800000000000001},"min":{"1min":0.183,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":10.805999999999999,"5min":32.122,"15min":32.122},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.5780000000000001,"5min":1.3460000000000001,"15min":0.71199999999999997},"min":{"1min":0.23599999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.3849999999999998,"5min":34.066000000000003,"15min":34.066000000000003},"last":0.441}]},{"osd":1,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":2.028,"15min":0.93000000000000005},"min":{"1min":0.218,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.3510000000000009,"5min":98.459999999999994,"15min":98.459999999999994},"last":0.26400000000000001},{"interface":"front","average":{"1min":1.9379999999999999,"5min":1.9590000000000001,"15min":0.90700000000000003},"min":{"1min":0.218,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":16.007999999999999,"5min":98.397000000000006,"15min":98.397000000000006},"last":0.32900000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:36:41 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":1.853,"15min":0.86699999999999999},"min":{"1min":0.26900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.9420000000000002,"5min":34.052999999999997,"15min":34.052999999999997},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.472,"5min":2.0840000000000001,"15min":0.96199999999999997},"min":{"1min":0.32300000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":8.7840000000000007,"5min":35.161999999999999,"15min":35.161999999999999},"last":0.42099999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3159999999999998,"5min":1.3149999999999999,"15min":0.82099999999999995},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.8960000000000008,"5min":8.8960000000000008,"15min":8.8960000000000008},"last":0.48199999999999998},{"interface":"front","average":{"1min":3.427,"5min":1.5780000000000001,"15min":0.95399999999999996},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":16.869,"5min":16.869,"15min":16.869},"last":0.57599999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":1.524,"15min":0.93600000000000005},"min":{"1min":0.376,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.2789999999999999,"5min":8.2789999999999999,"15min":8.2789999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":3.1219999999999999,"5min":1.4890000000000001,"15min":0.90100000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":8.8399999999999999,"5min":8.8399999999999999,"15min":8.8399999999999999},"last":0.746}]},{"osd":6,"last update":"Mon Oct 9 05:36:07 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":1.554,"15min":0.93400000000000005},"min":{"1min":0.379,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":10.715,"5min":11.496,"15min":11.496},"last":0.68400000000000005},{"interface":"front","average":{"1min":2.1459999999999999,"5min":1.3600000000000001,"15min":0.91000000000000003},"min":{"1min":0.40100000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.093,"5min":9.4049999999999994,"15min":9.4049999999999994},"last":0.66000000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:36:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":2.702,"15min":2.5830000000000002},"min":{"1min":0.438,"5min":0.374,"15min":0.374},"max":{"1min":10.975,"5min":10.975,"15min":10.975},"last":0.65000000000000002},{"interface":"front","average":{"1min":3.218,"5min":2.0259999999999998,"15min":1.8280000000000001},"min":{"1min":0.51800000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":15.25,"5min":15.25,"15min":15.25},"last":0.627}]}]},{"osd":3,"up_from":103,"seq":442381631557,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":83296,"kb_used_data":1688,"kb_used_omap":4,"kb_used_meta":81595,"kb_avail":93661856,"statfs":{"total":95995035648,"available":95909740544,"internally_reserved":0,"allocated":1728512,"data_stored":1079351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":83553886},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:41 2023","interfaces":[{"interface":"back","average":{"1min":2.448,"5min":2.056,"15min":0.95099999999999996},"min":{"1min":0.224,"5min":0.183,"15min":0.183},"max":{"1min":8.9589999999999996,"5min":56.094000000000001,"15min":56.094000000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":2.3679999999999999,"5min":2.3330000000000002,"15min":1.028},"min":{"1min":0.24299999999999999,"5min":0.214,"15min":0.214},"max":{"1min":11.634,"5min":57.255000000000003,"15min":57.255000000000003},"last":0.29899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:36:41 2023","interfaces":[{"interface":"back","average":{"1min":5.4269999999999996,"5min":1.861,"15min":0.88900000000000001},"min":{"1min":0.29599999999999999,"5min":0.255,"15min":0.255},"max":{"1min":72.278999999999996,"5min":72.278999999999996,"15min":72.278999999999996},"last":0.40899999999999997},{"interface":"front","average":{"1min":5.774,"5min":1.994,"15min":0.91500000000000004},"min":{"1min":0.29499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":72.367999999999995,"5min":72.367999999999995,"15min":72.367999999999995},"last":0.32400000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:36:41 2023","interfaces":[{"interface":"back","average":{"1min":7.1449999999999996,"5min":4.4299999999999997,"15min":1.762},"min":{"1min":0.218,"5min":0.215,"15min":0.215},"max":{"1min":95.320999999999998,"5min":117.23,"15min":117.23},"last":0.36699999999999999},{"interface":"front","average":{"1min":7.6989999999999998,"5min":4.6070000000000002,"15min":1.829},"min":{"1min":0.313,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":94.170000000000002,"5min":117.45999999999999,"15min":117.45999999999999},"last":0.39000000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:36:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":2.3100000000000001,"15min":1.1679999999999999},"min":{"1min":0.33600000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":11.363,"5min":45.779000000000003,"15min":45.779000000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.7010000000000001,"5min":1.8919999999999999,"15min":1.0620000000000001},"min":{"1min":0.36199999999999999,"5min":0.313,"15min":0.313},"max":{"1min":7.8079999999999998,"5min":45.735999999999997,"15min":45.735999999999997},"last":0.46999999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:36:41 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.8979999999999999,"15min":1.0600000000000001},"min":{"1min":0.433,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0700000000000003,"5min":47.948999999999998,"15min":47.948999999999998},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.8999999999999999,"5min":2.0569999999999999,"15min":1.1379999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":7.8360000000000003,"5min":47.968000000000004,"15min":47.968000000000004},"last":0.53300000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:36:41 2023","interfaces":[{"interface":"back","average":{"1min":2.3159999999999998,"5min":1.7829999999999999,"15min":1.042},"min":{"1min":0.42699999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":7.8689999999999998,"5min":8.5990000000000002,"15min":8.5990000000000002},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.5529999999999999,"5min":1.345,"15min":0.86299999999999999},"min":{"1min":0.373,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.9660000000000002,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.58099999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:36:46 2023","interfaces":[{"interface":"back","average":{"1min":2.746,"5min":2.3570000000000002,"15min":2.2919999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":11.407,"5min":11.407,"15min":11.407},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.403,"5min":2.0510000000000002,"15min":1.992},"min":{"1min":0.48599999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":11.432,"5min":11.432,"15min":11.432},"last":0.60399999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019105,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":399644,"kb_used_data":90132,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93345508,"statfs":{"total":95995035648,"available":95585800192,"internally_reserved":0,"allocated":92295168,"data_stored":91778274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:36:11 2023","interfaces":[{"interface":"back","average":{"1min":4.1360000000000001,"5min":2.2280000000000002,"15min":1.0900000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":36.406999999999996,"5min":36.406999999999996,"15min":36.406999999999996},"last":0.41399999999999998},{"interface":"front","average":{"1min":5.0629999999999997,"5min":2.5329999999999999,"15min":1.2130000000000001},"min":{"1min":0.26100000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":36.454999999999998,"5min":36.454999999999998,"15min":36.454999999999998},"last":0.32700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:36:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":1.655,"15min":0.90900000000000003},"min":{"1min":0.378,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.4219999999999997,"5min":31.073,"15min":31.073},"last":0.505},{"interface":"front","average":{"1min":1.4910000000000001,"5min":1.7649999999999999,"15min":0.96099999999999997},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.8149999999999999,"5min":31.100000000000001,"15min":31.100000000000001},"last":0.36299999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:36:11 2023","interfaces":[{"interface":"back","average":{"1min":12.515000000000001,"5min":5.8819999999999997,"15min":2.306},"min":{"1min":0.40100000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":128.84700000000001,"5min":164.26599999999999,"15min":164.26599999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":12.494,"5min":5.7169999999999996,"15min":2.2949999999999999},"min":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":128.911,"5min":162.57499999999999,"15min":162.57499999999999},"last":0.51900000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:36:36 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":1.665,"15min":0.92800000000000005},"min":{"1min":0.39300000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.274,"5min":10.037000000000001,"15min":10.037000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.9610000000000001,"5min":1.4219999999999999,"15min":0.88200000000000001},"min":{"1min":0.311,"5min":0.309,"15min":0.309},"max":{"1min":6.218,"5min":6.9459999999999997,"15min":6.9459999999999997},"last":0.80300000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:36:36 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":2.464,"15min":1.123},"min":{"1min":0.27000000000000002,"5min":0.249,"15min":0.249},"max":{"1min":10.484999999999999,"5min":113.35899999999999,"15min":113.35899999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.421,"5min":2.2959999999999998,"15min":1.1319999999999999},"min":{"1min":0.39000000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.298,"5min":113.32299999999999,"15min":113.32299999999999},"last":0.35299999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:36:52 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":1.028,"15min":0.66900000000000004},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.27800000000000002},"max":{"1min":6.1779999999999999,"5min":11.944000000000001,"15min":11.944000000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.2509999999999999,"15min":0.753},"min":{"1min":0.32500000000000001,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":3.4220000000000002,"5min":10.356,"15min":10.356},"last":0.433}]},{"osd":7,"last update":"Mon Oct 9 05:36:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0369999999999999,"5min":3.1030000000000002,"15min":3.2810000000000001},"min":{"1min":0.371,"5min":0.312,"15min":0.312},"max":{"1min":11.845000000000001,"5min":30.393999999999998,"15min":30.393999999999998},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.861,"5min":3.4359999999999999,"15min":3.698},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.837,"5min":30.334,"15min":30.334},"last":0.53900000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822877,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":421900,"kb_used_data":112388,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93323252,"statfs":{"total":95995035648,"available":95563010048,"internally_reserved":0,"allocated":115085312,"data_stored":114580250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:37:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2410000000000001,"5min":6.9340000000000002,"15min":2.6779999999999999},"min":{"1min":0.28000000000000003,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":10.673,"5min":165.90199999999999,"15min":165.90199999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.48,"5min":7.032,"15min":2.7130000000000001},"min":{"1min":0.33100000000000002,"5min":0.215,"15min":0.215},"max":{"1min":8.1869999999999994,"5min":165.96700000000001,"15min":165.96700000000001},"last":0.33100000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:36:38 2023","interfaces":[{"interface":"back","average":{"1min":2.552,"5min":1.7470000000000001,"15min":0.95299999999999996},"min":{"1min":0.29699999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":8.1989999999999998,"5min":12.891999999999999,"15min":12.891999999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":3.0299999999999998,"5min":1.5680000000000001,"15min":0.89400000000000002},"min":{"1min":0.29299999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.5530000000000008,"5min":12.939,"15min":12.939},"last":0.41699999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:36:14 2023","interfaces":[{"interface":"back","average":{"1min":17.323,"5min":8.0950000000000006,"15min":3.0790000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":118.255,"5min":290.697,"15min":290.697},"last":0.36599999999999999},{"interface":"front","average":{"1min":16.324999999999999,"5min":7.9080000000000004,"15min":3.0139999999999998},"min":{"1min":0.38800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":118.438,"5min":290.786,"15min":290.786},"last":0.57499999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:36:38 2023","interfaces":[{"interface":"back","average":{"1min":3.468,"5min":1.8049999999999999,"15min":0.997},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.314},"max":{"1min":10.923,"5min":10.923,"15min":10.923},"last":2.605},{"interface":"front","average":{"1min":2.3450000000000002,"5min":1.645,"15min":0.93899999999999995},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.2910000000000004,"5min":12.779999999999999,"15min":12.779999999999999},"last":3.5600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:36:38 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.071,"15min":0.67200000000000004},"min":{"1min":0.27500000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.2060000000000004,"5min":9.3510000000000009,"15min":9.3510000000000009},"last":0.50700000000000001},{"interface":"front","average":{"1min":3.5270000000000001,"5min":1.708,"15min":0.91400000000000003},"min":{"1min":0.29899999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.913,"5min":14.236000000000001,"15min":14.236000000000001},"last":0.40000000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":2.5779999999999998,"15min":1.22},"min":{"1min":0.378,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":10.808,"5min":95.010999999999996,"15min":95.010999999999996},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.3140000000000001,"5min":2.2469999999999999,"15min":1.1160000000000001},"min":{"1min":0.35599999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.8319999999999999,"5min":94.194000000000003,"15min":94.194000000000003},"last":0.67000000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:36:45 2023","interfaces":[{"interface":"back","average":{"1min":2.5630000000000002,"5min":1.8919999999999999,"15min":1.78},"min":{"1min":0.371,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":9.9849999999999994,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":2.5230000000000001},{"interface":"front","average":{"1min":2.5499999999999998,"5min":2.2080000000000002,"15min":2.1509999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.779,"5min":14.432,"15min":14.432},"last":1.177}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40239104,"data_stored":40239104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":84148224,"data_stored":84148224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":117768192,"data_stored":117768192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":116981760,"data_stored":116981760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":112656384,"data_stored":112656384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":131530752,"data_stored":131530752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:37:12.899 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 16 2023-10-09T05:37:12.999 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:13.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:13.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:13.104 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:13.171 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:13.307 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:13.342 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:13.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:13.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:13.423 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:37:13.446 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:37:13.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:13.468 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:37:13.473 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:13.487 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:37:13.518 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:13.519 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:37:13.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:13.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:13.621 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:13.622 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:37:13.672 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:37:13.690 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:13.713 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:13.713 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:37:13.733 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:37:13.733 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:37:13.734 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:37:13.734 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:37:13.734 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:37:13.734 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:37:13.734 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:37:13.734 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:37:13.757 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:13 smithi008 ceph-mon[184418]: pgmap v429: 89 pgs: 89 active+clean; 367 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s 2023-10-09T05:37:13.757 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/878354320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:13.758 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:13 smithi008 ceph-mon[191227]: pgmap v429: 89 pgs: 89 active+clean; 367 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s 2023-10-09T05:37:13.758 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/878354320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:13 smithi191 ceph-mon[161669]: pgmap v429: 89 pgs: 89 active+clean; 367 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s 2023-10-09T05:37:13.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/878354320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:13.825 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 7 16 14252 14236 126.896 96.1875 0.00814206 0.00787314 2023-10-09T05:37:13.916 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:37:14.518 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:14.826 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 8 16 16504 16488 128.624 140.75 0.00260186 0.00769983 2023-10-09T05:37:14.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:14.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:14.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:14.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:14.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:14.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:15.578 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:15.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:15.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:15.681 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:15.711 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:15.826 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 9 16 17642 17626 122.242 71.125 0.00618986 0.008172 2023-10-09T05:37:15.834 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:15.839 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:15 smithi008 ceph-mon[184418]: pgmap v430: 89 pgs: 89 active+clean; 527 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2023-10-09T05:37:15.840 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:15 smithi008 ceph-mon[191227]: pgmap v430: 89 pgs: 89 active+clean; 527 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2023-10-09T05:37:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:15.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:15.941 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:15.980 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:15 smithi191 ceph-mon[161669]: pgmap v430: 89 pgs: 89 active+clean; 527 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2023-10-09T05:37:15.986 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:16.118 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:16.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:16.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:16.226 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:16.285 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:16.371 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:16.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:16.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:16.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:16.518 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:16.562 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:37:16.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:16.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:16.605 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:37:16.605 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:37:16.666 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:16.697 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:16.826 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 10 16 19472 19456 121.455 114.375 0.010109 0.00822731 2023-10-09T05:37:16.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:16 smithi008 ceph-mon[191227]: pgmap v431: 89 pgs: 89 active+clean; 867 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:37:16.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:16 smithi008 ceph-mon[184418]: pgmap v431: 89 pgs: 89 active+clean; 867 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:37:17.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:16 smithi191 ceph-mon[161669]: pgmap v431: 89 pgs: 89 active+clean; 867 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:37:17.056 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:17.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:17.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:17.162 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:17.190 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:17.236 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:17.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:17.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:17.342 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:17.389 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:17.621 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:37:17.623 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:37:17.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:17.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:17.730 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:17.776 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:37:17.807 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:17.826 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 11 15 21244 21229 120.484 110.812 0.00308125 0.00828914 2023-10-09T05:37:17.972 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:17.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:17.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:18.085 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:18.118 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:18.160 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:18.288 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:18.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:18.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:18.394 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:18.454 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:18.509 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:18.553 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:18.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:18.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:18.657 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:18.666 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:37:18.684 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 6 pgp_num to 16 2023-10-09T05:37:18.704 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2996379139' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:18.704 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:18.704 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1824980020' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:37:18.704 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:37:18.704 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2996379139' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:18.704 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:18.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1824980020' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:37:18.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:37:18.710 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:18.826 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 12 16 23129 23113 120.256 117.75 0.00959853 0.00830855 2023-10-09T05:37:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2996379139' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1824980020' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:37:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:37:19.826 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 13 16 24965 24949 119.832 114.75 0.00316557 0.00832418 2023-10-09T05:37:19.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:19 smithi191 ceph-mon[161669]: pgmap v432: 89 pgs: 89 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T05:37:19.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:19 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:37:19.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:19 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:37:19.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:19 smithi191 ceph-mon[161669]: failed to encode map e167 with expected crc 2023-10-09T05:37:19.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:19 smithi191 ceph-mon[161669]: failed to encode map e167 with expected crc 2023-10-09T05:37:19.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:37:19.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:37:19.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:19 smithi191 ceph-mon[161669]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T05:37:19.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[184418]: pgmap v432: 89 pgs: 89 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T05:37:19.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:37:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:37:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[184418]: failed to encode map e167 with expected crc 2023-10-09T05:37:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[184418]: failed to encode map e167 with expected crc 2023-10-09T05:37:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:37:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:37:19.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[184418]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T05:37:19.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[191227]: pgmap v432: 89 pgs: 89 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T05:37:19.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:37:19.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:37:19.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[191227]: failed to encode map e167 with expected crc 2023-10-09T05:37:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[191227]: failed to encode map e167 with expected crc 2023-10-09T05:37:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:37:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:37:19.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:19 smithi008 ceph-mon[191227]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T05:37:20.119 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:20.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:20.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:20.224 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:20.229 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:37:20.229 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:37:20.283 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:20.466 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:20.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:20.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:20.570 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:20.598 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:20.826 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 14 16 26582 26566 118.491 101.062 0.00180567 0.00842686 2023-10-09T05:37:20.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:37:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:37:20] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.33.4" 2023-10-09T05:37:21.156 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:21.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:21.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:21.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:21.325 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:21.328 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:37:21.396 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:21.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:21.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:21.505 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:21.564 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:21.702 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:21.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:21.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:21.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:21.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 15 16 27989 27973 116.455 87.9375 0.00173018 0.00854758 2023-10-09T05:37:21.832 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:21.874 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:21.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:21.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:21.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:21 smithi008 ceph-mon[184418]: pgmap v434: 89 pgs: 89 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T05:37:21.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:21 smithi008 ceph-mon[191227]: pgmap v434: 89 pgs: 89 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T05:37:21.979 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:22.044 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:22.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:21 smithi191 ceph-mon[161669]: pgmap v434: 89 pgs: 89 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T05:37:22.261 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:22.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:22.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:22.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:22.410 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:22.469 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:22.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:22.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:22.536 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:22.574 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:22.613 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:22.680 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:22.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:22.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:22.788 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:22.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 16 16 29672 29656 115.748 105.188 0.0091929 0.00863451 2023-10-09T05:37:22.831 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:22.902 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:22.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:22.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:23.012 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:23.048 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:37:23.051 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:23.089 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:37:23.118 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:23.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:23.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:23.132 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:37:23.176 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:37:23.215 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:37:23.225 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:23.252 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:37:23.258 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:37:23.302 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:37:23.307 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:23.352 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:37:23.390 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:37:23.391 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:37:23.391 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:37:23.391 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:37:23.391 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:37:23.391 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:37:23.391 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:37:23.391 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:37:23.829 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 17 16 31376 31360 115.189 106.5 0.00635639 0.00867571 2023-10-09T05:37:23.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:23 smithi008 ceph-mon[184418]: pgmap v435: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T05:37:23.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/958185086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:23 smithi008 ceph-mon[191227]: pgmap v435: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T05:37:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/958185086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:24.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:23 smithi191 ceph-mon[161669]: pgmap v435: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T05:37:24.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/958185086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:24.830 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 18 16 32776 32760 113.651 87.5 0.00761233 0.00874504 2023-10-09T05:37:25.430 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:25.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:25.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:25.539 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:25.641 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:25.689 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:25.830 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 19 16 34352 34336 112.852 98.5 0.00309304 0.0088559 2023-10-09T05:37:25.855 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:25.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:25.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:25.886 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:25 smithi008 ceph-mon[184418]: pgmap v436: 89 pgs: 89 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:37:25.889 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:25 smithi008 ceph-mon[191227]: pgmap v436: 89 pgs: 89 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:37:25.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:26.009 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:25 smithi191 ceph-mon[161669]: pgmap v436: 89 pgs: 89 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:37:26.012 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:26.079 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:26.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:26.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:26.179 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:37:26.179 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:37:26.179 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:37:26.179 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T05:37:26.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:26.210 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:37:26.211 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:37:26.211 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.211 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:34953 6.9 6:9f3a6857:::benchmark_data_smithi008_295510_object34952:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T05:37:26.211 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:26.200577+0000", 2023-10-09T05:37:26.211 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0088705529999999998, 2023-10-09T05:37:26.211 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0089023190000000006, 2023-10-09T05:37:26.211 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:26.211 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:37:26.211 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:26.211 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:37:26.211 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 34953 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.200577+0000", 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.200577+0000", 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:26.212 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.200576+0000", 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.200611+0000", 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4956000000000002e-05 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.200636+0000", 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5729e-05 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.200644+0000", 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0039999999999999e-06 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.200690+0000", 2023-10-09T05:37:26.214 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5938e-05 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.200839+0000", 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000148233 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.201001+0000", 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000162678 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.215 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.203745+0000", 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027438200000000001 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.207740+0000", 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039946529999999999 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.216 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:34964 6.9 6:92dc9473:::benchmark_data_smithi008_295510_object34963:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:26.201117+0000", 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0083305519999999997, 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0084394640000000007, 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 34964 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:26.217 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.201117+0000", 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.201117+0000", 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.201116+0000", 2023-10-09T05:37:26.218 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.201139+0000", 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2538999999999999e-05 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.201152+0000", 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3071000000000001e-05 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.219 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.201175+0000", 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3515000000000001e-05 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.202284+0000", 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011087810000000001 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.202400+0000", 2023-10-09T05:37:26.220 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011567 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.202526+0000", 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012584099999999999 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.203785+0000", 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001259404 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:26.221 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:26.222 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:26.222 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:26.207774+0000", 2023-10-09T05:37:26.222 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039889599999999997 2023-10-09T05:37:26.222 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:26.222 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:26.222 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:26.222 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:26.222 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:37:26.222 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 2 2023-10-09T05:37:26.222 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:37:26.223 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:26.284 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:26.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:26.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:26.389 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:26.464 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:26.832 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:37:26.831184+0000 min lat: 0.00103937 max lat: 0.48521 avg lat: 0.00890942 2023-10-09T05:37:26.832 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:37:26.832 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 20 16 35939 35923 112.168 99.1875 0.012769 0.00890942 2023-10-09T05:37:27.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:26 smithi191 ceph-mon[161669]: pgmap v437: 89 pgs: 89 active+clean; 1.9 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T05:37:27.058 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:27.061 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:26 smithi008 ceph-mon[184418]: pgmap v437: 89 pgs: 89 active+clean; 1.9 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T05:37:27.061 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:26 smithi008 ceph-mon[191227]: pgmap v437: 89 pgs: 89 active+clean; 1.9 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T05:37:27.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:27.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:27.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:27.205 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:27.250 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:27.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:27.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:27.356 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:27.383 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:27.447 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:37:27.830 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 21 16 37117 37101 110.333 73.625 0.010348 0.00905872 2023-10-09T05:37:28.170 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:28.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:28.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:28.273 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:28.301 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:28.476 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:37:28.527 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:28.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:28.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:28.631 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:28.790 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:28.833 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 22 16 38392 38376 108.927 79.6875 0.00255561 0.00917496 2023-10-09T05:37:28.856 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:28.873 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:28.930 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:28.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:28.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:29.034 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:29.071 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:29.133 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:29.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:29.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:29.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:29.277 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:29.344 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:29.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:29.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:29.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:29.489 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:29.555 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:29.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:29.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:29.643 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:29 smithi191 ceph-mon[161669]: pgmap v438: 89 pgs: 89 active+clean; 2.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:37:29.643 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:29.643 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:29.662 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:29.701 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:29 smithi008 ceph-mon[184418]: pgmap v438: 89 pgs: 89 active+clean; 2.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:37:29.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:29.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:29.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:29 smithi008 ceph-mon[191227]: pgmap v438: 89 pgs: 89 active+clean; 2.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:37:29.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:29.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:29.764 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:29.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:29.835 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 23 16 39776 39760 107.948 86.5 0.0133713 0.00925924 2023-10-09T05:37:29.874 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:29.963 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:30.654 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:37:30.834 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 24 16 41221 41205 107.213 90.3125 0.0143048 0.00932236 2023-10-09T05:37:30.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2084185763' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:37:30.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:30 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:37:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2084185763' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:37:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:30 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:37:30.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:37:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:37:30] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.33.4" 2023-10-09T05:37:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2084185763' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:37:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:30 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:37:31.836 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 25 16 42823 42807 106.924 100.125 0.00597749 0.00934725 2023-10-09T05:37:31.843 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:31.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:31.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:31.908 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[184418]: pgmap v439: 89 pgs: 89 active+clean; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:37:31.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:31.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[184418]: failed to encode map e168 with expected crc 2023-10-09T05:37:31.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:37:31.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[184418]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T05:37:31.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[184418]: failed to encode map e168 with expected crc 2023-10-09T05:37:31.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[184418]: failed to encode map e168 with expected crc 2023-10-09T05:37:31.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[191227]: pgmap v439: 89 pgs: 89 active+clean; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:37:31.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:31.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[191227]: failed to encode map e168 with expected crc 2023-10-09T05:37:31.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:37:31.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[191227]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T05:37:31.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[191227]: failed to encode map e168 with expected crc 2023-10-09T05:37:31.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:31 smithi008 ceph-mon[191227]: failed to encode map e168 with expected crc 2023-10-09T05:37:31.957 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:32.029 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:31 smithi191 ceph-mon[161669]: pgmap v439: 89 pgs: 89 active+clean; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:37:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:31 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:31 smithi191 ceph-mon[161669]: failed to encode map e168 with expected crc 2023-10-09T05:37:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:31 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:37:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:31 smithi191 ceph-mon[161669]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T05:37:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:31 smithi191 ceph-mon[161669]: failed to encode map e168 with expected crc 2023-10-09T05:37:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:31 smithi191 ceph-mon[161669]: failed to encode map e168 with expected crc 2023-10-09T05:37:32.463 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:32.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:32.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:32.572 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:32.643 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:32.753 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:32.837 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 26 16 44359 44343 106.504 96 0.00579507 0.00938282 2023-10-09T05:37:32.914 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:32.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:33.021 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:33.103 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:33.210 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:33.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:33.286 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:37:33.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:33.328 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:37:33.354 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:33.379 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:37:33.415 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:33.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:33.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:33.423 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:37:33.471 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:37:33.510 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:37:33.521 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:33.563 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:37:33.592 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:33.623 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:37:33.624 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:37:33.624 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:37:33.624 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:37:33.624 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:37:33.624 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:37:33.624 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:37:33.624 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:37:33.624 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:37:33.837 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 27 16 45634 45618 105.509 79.6875 0.0104666 0.009472 2023-10-09T05:37:33.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:33 smithi008 ceph-mon[184418]: pgmap v441: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T05:37:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:33 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2063231887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:33 smithi008 conmon[184394]: 2023-10-09T05:37:33.735+0000 7f8a66521700 -1 mon.a@0(leader).osd e168 definitely_dead 0 2023-10-09T05:37:33.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:33 smithi008 ceph-mon[191227]: pgmap v441: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T05:37:33.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:33 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2063231887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:34.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:33 smithi191 ceph-mon[161669]: pgmap v441: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T05:37:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:33 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2063231887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:34.704 INFO:teuthology.orchestra.run.smithi008.stderr:marked out osd.7. 2023-10-09T05:37:34.742 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:37:34.837 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 28 16 46707 46691 104.137 67.0625 0.00263126 0.00954919 2023-10-09T05:37:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3169413301' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:37:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:34 smithi008 ceph-mon[184418]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:37:34.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3169413301' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:37:34.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:34 smithi008 ceph-mon[191227]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:37:35.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3169413301' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:37:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:34 smithi191 ceph-mon[161669]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:37:35.266 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:35.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:35.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:35.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:35.399 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:35.444 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:35.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:35.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:35.551 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:35.580 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:35.694 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:35 smithi191 ceph-mon[161669]: pgmap v442: 89 pgs: 89 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T05:37:35.694 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:35 smithi191 ceph-mon[161669]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.694 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:35 smithi191 ceph-mon[161669]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.694 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3169413301' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:37:35.695 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:35 smithi191 ceph-mon[161669]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T05:37:35.695 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:35 smithi191 ceph-mon[161669]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.836 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 29 16 46707 46691 100.549 0 - 0.00954919 2023-10-09T05:37:35.846 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[184418]: pgmap v442: 89 pgs: 89 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T05:37:35.847 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[184418]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.847 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[184418]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.847 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3169413301' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:37:35.847 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[184418]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T05:37:35.847 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[184418]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.847 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[184418]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.847 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[184418]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.847 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[191227]: pgmap v442: 89 pgs: 89 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T05:37:35.847 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[191227]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.848 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[191227]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.848 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3169413301' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:37:35.848 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[191227]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T05:37:35.848 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[191227]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.848 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[191227]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.848 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:35 smithi008 ceph-mon[191227]: failed to encode map e169 with expected crc 2023-10-09T05:37:35.977 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:35.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:35.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:36.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:35 smithi191 ceph-mon[161669]: failed to encode map e169 with expected crc 2023-10-09T05:37:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:35 smithi191 ceph-mon[161669]: failed to encode map e169 with expected crc 2023-10-09T05:37:36.080 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:36.111 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:36.213 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:36.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:36.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:36.279 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:36.316 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:36.342 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:36.373 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:37:36.373 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:37:36.384 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:36.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:36.489 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:36.521 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:36.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[184418]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[184418]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[184418]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[184418]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T05:37:36.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[184418]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.709 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:36 smithi191 ceph-mon[161669]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.709 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:36 smithi191 ceph-mon[161669]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.709 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:36 smithi191 ceph-mon[161669]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.709 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:36 smithi191 ceph-mon[161669]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T05:37:36.709 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:36 smithi191 ceph-mon[161669]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.709 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:36 smithi191 ceph-mon[161669]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.709 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:36 smithi191 ceph-mon[161669]: pgmap v445: 89 pgs: 16 peering, 73 active+clean; 2.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T05:37:36.709 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:36 smithi191 ceph-mon[161669]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T05:37:36.837 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 30 16 46713 46697 97.2118 0.1875 0.00287763 0.00963899 2023-10-09T05:37:36.924 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:36.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:36.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:36.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[184418]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[184418]: pgmap v445: 89 pgs: 16 peering, 73 active+clean; 2.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T05:37:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[184418]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T05:37:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[191227]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[191227]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[191227]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[191227]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T05:37:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[191227]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[191227]: failed to encode map e170 with expected crc 2023-10-09T05:37:36.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[191227]: pgmap v445: 89 pgs: 16 peering, 73 active+clean; 2.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T05:37:36.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:36 smithi008 ceph-mon[191227]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T05:37:37.028 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:37.056 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:37.837 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 31 16 46933 46917 94.5211 13.75 0.0022719 0.010239 2023-10-09T05:37:37.871 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:37:37.898 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[184418]: failed to encode map e171 with expected crc 2023-10-09T05:37:37.898 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[184418]: failed to encode map e171 with expected crc 2023-10-09T05:37:37.898 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[184418]: failed to encode map e171 with expected crc 2023-10-09T05:37:37.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[184418]: failed to encode map e171 with expected crc 2023-10-09T05:37:37.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[184418]: failed to encode map e171 with expected crc 2023-10-09T05:37:37.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T05:37:37.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[184418]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T05:37:37.899 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[191227]: failed to encode map e171 with expected crc 2023-10-09T05:37:37.899 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[191227]: failed to encode map e171 with expected crc 2023-10-09T05:37:37.899 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[191227]: failed to encode map e171 with expected crc 2023-10-09T05:37:37.899 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[191227]: failed to encode map e171 with expected crc 2023-10-09T05:37:37.899 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[191227]: failed to encode map e171 with expected crc 2023-10-09T05:37:37.899 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T05:37:37.900 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:37 smithi008 ceph-mon[191227]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T05:37:38.015 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:38.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:38.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:38.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:37 smithi191 ceph-mon[161669]: failed to encode map e171 with expected crc 2023-10-09T05:37:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:37 smithi191 ceph-mon[161669]: failed to encode map e171 with expected crc 2023-10-09T05:37:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:37 smithi191 ceph-mon[161669]: failed to encode map e171 with expected crc 2023-10-09T05:37:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:37 smithi191 ceph-mon[161669]: failed to encode map e171 with expected crc 2023-10-09T05:37:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:37 smithi191 ceph-mon[161669]: failed to encode map e171 with expected crc 2023-10-09T05:37:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:37 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T05:37:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:37 smithi191 ceph-mon[161669]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T05:37:38.120 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:38.146 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:38.190 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:38.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:38.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:38.251 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:37:38.295 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:38.326 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:38.567 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:38.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:38.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:38.617 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:38.673 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:38.706 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:38.837 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 32 16 46933 46917 91.5692 0 - 0.010239 2023-10-09T05:37:38.964 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:38 smithi008 ceph-mon[184418]: failed to encode map e172 with expected crc 2023-10-09T05:37:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:38 smithi008 ceph-mon[184418]: failed to encode map e172 with expected crc 2023-10-09T05:37:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:38 smithi008 ceph-mon[184418]: failed to encode map e172 with expected crc 2023-10-09T05:37:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:38 smithi008 ceph-mon[184418]: pgmap v448: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 18 peering, 67 active+clean; 2.8 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 2897/139728 objects degraded (2.073%); 176 KiB/s, 5 objects/s recovering 2023-10-09T05:37:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:38 smithi008 ceph-mon[184418]: osdmap e173: 8 total, 8 up, 7 in 2023-10-09T05:37:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:38 smithi008 ceph-mon[191227]: failed to encode map e172 with expected crc 2023-10-09T05:37:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:38 smithi008 ceph-mon[191227]: failed to encode map e172 with expected crc 2023-10-09T05:37:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:38 smithi008 ceph-mon[191227]: failed to encode map e172 with expected crc 2023-10-09T05:37:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:38 smithi008 ceph-mon[191227]: pgmap v448: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 18 peering, 67 active+clean; 2.8 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 2897/139728 objects degraded (2.073%); 176 KiB/s, 5 objects/s recovering 2023-10-09T05:37:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:38 smithi008 ceph-mon[191227]: osdmap e173: 8 total, 8 up, 7 in 2023-10-09T05:37:38.967 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:38.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:38.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:39.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:38 smithi191 ceph-mon[161669]: failed to encode map e172 with expected crc 2023-10-09T05:37:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:38 smithi191 ceph-mon[161669]: failed to encode map e172 with expected crc 2023-10-09T05:37:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:38 smithi191 ceph-mon[161669]: failed to encode map e172 with expected crc 2023-10-09T05:37:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:38 smithi191 ceph-mon[161669]: pgmap v448: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 18 peering, 67 active+clean; 2.8 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 2897/139728 objects degraded (2.073%); 176 KiB/s, 5 objects/s recovering 2023-10-09T05:37:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:38 smithi191 ceph-mon[161669]: osdmap e173: 8 total, 8 up, 7 in 2023-10-09T05:37:39.071 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:39.097 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:39.144 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:39.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:39.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:39.249 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:39.276 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:39.320 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:39.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:39.425 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:39.451 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:39.500 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:39.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:39.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:39.606 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:39.637 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:39.723 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:37:39.837 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 33 16 46933 46917 88.796 0 - 0.010239 2023-10-09T05:37:39.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[191227]: failed to encode map e173 with expected crc 2023-10-09T05:37:39.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[191227]: failed to encode map e173 with expected crc 2023-10-09T05:37:39.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[191227]: failed to encode map e173 with expected crc 2023-10-09T05:37:39.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[191227]: Health check failed: Degraded data redundancy: 2897/139728 objects degraded (2.073%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:37:39.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3418631006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:39.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[184418]: failed to encode map e173 with expected crc 2023-10-09T05:37:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[184418]: failed to encode map e173 with expected crc 2023-10-09T05:37:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[184418]: failed to encode map e173 with expected crc 2023-10-09T05:37:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[184418]: Health check failed: Degraded data redundancy: 2897/139728 objects degraded (2.073%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:37:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3418631006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:39.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:39 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:40.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:39 smithi191 ceph-mon[161669]: failed to encode map e173 with expected crc 2023-10-09T05:37:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:39 smithi191 ceph-mon[161669]: failed to encode map e173 with expected crc 2023-10-09T05:37:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:39 smithi191 ceph-mon[161669]: failed to encode map e173 with expected crc 2023-10-09T05:37:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:39 smithi191 ceph-mon[161669]: Health check failed: Degraded data redundancy: 2897/139728 objects degraded (2.073%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:37:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3418631006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:39 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:37:40.838 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 34 16 47124 47108 86.5337 3.97917 0.010759 0.0115508 2023-10-09T05:37:40.939 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:37:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:37:40] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.33.4" 2023-10-09T05:37:40.940 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:40.940 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[184418]: failed to encode map e174 with expected crc 2023-10-09T05:37:40.940 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[184418]: failed to encode map e174 with expected crc 2023-10-09T05:37:40.940 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:37:40.940 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[184418]: osdmap e174: 8 total, 8 up, 7 in 2023-10-09T05:37:40.940 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[184418]: pgmap v451: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 7 peering, 75 active+clean; 2.9 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 36 MiB/s wr, 577 op/s; 2927/140538 objects degraded (2.083%); 177 KiB/s, 2 keys/s, 7 objects/s recovering 2023-10-09T05:37:40.944 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:40.944 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[191227]: failed to encode map e174 with expected crc 2023-10-09T05:37:40.944 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[191227]: failed to encode map e174 with expected crc 2023-10-09T05:37:40.945 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:37:40.945 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[191227]: osdmap e174: 8 total, 8 up, 7 in 2023-10-09T05:37:40.945 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:40 smithi008 ceph-mon[191227]: pgmap v451: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 7 peering, 75 active+clean; 2.9 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 36 MiB/s wr, 577 op/s; 2927/140538 objects degraded (2.083%); 177 KiB/s, 2 keys/s, 7 objects/s recovering 2023-10-09T05:37:41.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:40 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:41.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:40 smithi191 ceph-mon[161669]: failed to encode map e174 with expected crc 2023-10-09T05:37:41.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:40 smithi191 ceph-mon[161669]: failed to encode map e174 with expected crc 2023-10-09T05:37:41.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:40 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:37:41.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:40 smithi191 ceph-mon[161669]: osdmap e174: 8 total, 8 up, 7 in 2023-10-09T05:37:41.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:40 smithi191 ceph-mon[161669]: pgmap v451: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 7 peering, 75 active+clean; 2.9 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 36 MiB/s wr, 577 op/s; 2927/140538 objects degraded (2.083%); 177 KiB/s, 2 keys/s, 7 objects/s recovering 2023-10-09T05:37:41.089 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:37:41.090 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T05:37:41.792 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:41.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:41.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:41.840 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 35 16 48041 48025 85.6944 57.3125 0.0149179 0.0116638 2023-10-09T05:37:41.903 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:41.941 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:42.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:41 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 5 pgs peering) 2023-10-09T05:37:42.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:41 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 5 pgs peering) 2023-10-09T05:37:42.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:41 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 5 pgs peering) 2023-10-09T05:37:42.640 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:42.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:42.750 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:42.835 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:42.840 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 36 16 48881 48865 84.7726 52.5 0.00851218 0.0117893 2023-10-09T05:37:43.014 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:43.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:43.129 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:43.203 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:43.230 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:43.374 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:43.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:43.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:43.397 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:37:43.446 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:37:43.470 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:37:43.470 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:37:43.470 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:37:43.470 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98118 5.5 5:a3d60b4a:::benchmark_data_smithi008_238708_object98117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.582344+0000", 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 222.880584352, 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30221345300000002, 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98118 2023-10-09T05:37:43.471 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582344+0000", 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582344+0000", 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.472 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.473 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.473 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582342+0000", 2023-10-09T05:37:43.473 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:37:43.473 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.473 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.473 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.473 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582367+0000", 2023-10-09T05:37:43.473 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4702000000000002e-05 2023-10-09T05:37:43.473 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.473 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.473 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582381+0000", 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3407999999999999e-05 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582391+0000", 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9180000000000006e-06 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582482+0000", 2023-10-09T05:37:43.474 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1482000000000003e-05 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582673+0000", 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0001906 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582831+0000", 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000157904 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.475 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.879193+0000", 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29636228799999997 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.884558+0000", 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0053650299999999998 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.476 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.34770.0:98118 5.5 e117/105)", 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.586195+0000", 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 222.87673410599999, 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.292483255, 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.586195+0000", 2023-10-09T05:37:43.477 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.586195+0000", 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.586194+0000", 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.478 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.586201+0000", 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0140000000000001e-06 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.644224+0000", 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.058023650000000003 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.479 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.644240+0000", 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5147000000000001e-05 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.878662+0000", 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23442207500000001 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.878678+0000", 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6161e-05 2023-10-09T05:37:43.480 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:98120 5.8 e117/105)", 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.675582+0000", 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 222.787346601, 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.21251842900000001, 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.481 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.482 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.482 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675582+0000", 2023-10-09T05:37:43.482 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.482 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.482 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.482 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.482 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675582+0000", 2023-10-09T05:37:43.482 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.482 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.482 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.482 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675581+0000", 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675613+0000", 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2385999999999998e-05 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675631+0000", 2023-10-09T05:37:43.483 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7909000000000001e-05 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.675641+0000", 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5140000000000007e-06 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.888084+0000", 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.21244289999999999 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.484 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.888101+0000", 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6830999999999999e-05 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98118 5.5 5:a3d60b4a:::benchmark_data_smithi008_238708_object98117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.694337+0000", 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 222.76859216599999, 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.914715308, 2023-10-09T05:37:43.485 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98118 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694337+0000", 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.486 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694337+0000", 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694334+0000", 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.487 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694371+0000", 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7106e-05 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694394+0000", 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3269e-05 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694410+0000", 2023-10-09T05:37:43.488 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5454000000000001e-05 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.879229+0000", 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.184819179 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.879238+0000", 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6590000000000002e-06 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.489 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.608962+0000", 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72972381399999997 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.609052+0000", 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9624999999999997e-05 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.490 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98137 5.3 5:c7026034:::benchmark_data_smithi008_238708_object98136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.815708+0000", 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 222.647221167, 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.811726473, 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98137 2023-10-09T05:37:43.491 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815708+0000", 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815708+0000", 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.492 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815705+0000", 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815742+0000", 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6514999999999999e-05 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815763+0000", 2023-10-09T05:37:43.493 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1107999999999998e-05 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815778+0000", 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4718999999999999e-05 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.848379+0000", 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.032600899000000003 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.494 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.848390+0000", 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1735e-05 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.619939+0000", 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77154830200000002 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.495 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.620114+0000", 2023-10-09T05:37:43.496 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017572099999999999 2023-10-09T05:37:43.496 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.496 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.496 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:37:43.496 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.620276+0000", 2023-10-09T05:37:43.496 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016124600000000001 2023-10-09T05:37:43.496 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.496 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622357+0000", 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020815999999999999 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.625946+0000", 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035886339999999998 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.497 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627349+0000", 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014031130000000001 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627400+0000", 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1385999999999999e-05 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627434+0000", 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3640999999999999e-05 2023-10-09T05:37:43.498 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.499 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.499 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.499 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.499 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.499 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:119020 5.e e122/105 5:7d0c0384:::benchmark_data_smithi008_238708_object884:head v 122'7478, mlcod=122'7475)", 2023-10-09T05:37:43.499 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:23.998765+0000", 2023-10-09T05:37:43.499 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 199.464163818, 2023-10-09T05:37:43.499 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.22520707200000001, 2023-10-09T05:37:43.499 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.499 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:37:43.499 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:23.998765+0000", 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:23.998765+0000", 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.500 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.501 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:23.998764+0000", 2023-10-09T05:37:43.501 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:43.501 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.501 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.501 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.501 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:23.998776+0000", 2023-10-09T05:37:43.501 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.184e-05 2023-10-09T05:37:43.501 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.502 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.502 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.502 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:23.998792+0000", 2023-10-09T05:37:43.502 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6480000000000001e-05 2023-10-09T05:37:43.502 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.502 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.502 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.502 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:23.998801+0000", 2023-10-09T05:37:43.502 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2150000000000001e-06 2023-10-09T05:37:43.503 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.503 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.503 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.503 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.001026+0000", 2023-10-09T05:37:43.503 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022250379999999999 2023-10-09T05:37:43.503 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.503 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.503 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.503 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.001047+0000", 2023-10-09T05:37:43.503 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0843e-05 2023-10-09T05:37:43.503 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.504 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.504 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.504 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.223972+0000", 2023-10-09T05:37:43.504 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.22292541599999999 2023-10-09T05:37:43.504 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.504 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.504 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.504 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.505 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.505 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.34770.0:119026 5.e e122/105 5:7513f17c:::benchmark_data_smithi008_238708_object890:head v 122'7479, mlcod=122'7475)", 2023-10-09T05:37:43.505 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.003422+0000", 2023-10-09T05:37:43.505 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 199.45950633800001, 2023-10-09T05:37:43.505 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.220565179, 2023-10-09T05:37:43.505 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.505 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:37:43.505 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.505 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.505 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.505 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.003422+0000", 2023-10-09T05:37:43.506 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.506 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.506 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.506 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.506 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.003422+0000", 2023-10-09T05:37:43.506 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.506 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.506 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.506 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.506 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.003422+0000", 2023-10-09T05:37:43.506 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.003429+0000", 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6029999999999999e-06 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.003439+0000", 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9089999999999995e-06 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.507 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.508 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.003446+0000", 2023-10-09T05:37:43.508 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7469999999999999e-06 2023-10-09T05:37:43.508 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.508 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.508 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.508 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.004398+0000", 2023-10-09T05:37:43.508 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095160399999999999 2023-10-09T05:37:43.508 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.508 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.509 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.509 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.004416+0000", 2023-10-09T05:37:43.509 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8875000000000001e-05 2023-10-09T05:37:43.509 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.509 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.509 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.509 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.223988+0000", 2023-10-09T05:37:43.509 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.21957117100000001 2023-10-09T05:37:43.509 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.509 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.509 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:148727 5.15 5:a92a8353:::benchmark_data_smithi008_238708_object30591:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:52.692007+0000", 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 170.77092177099999, 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20566746799999999, 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:37:43.510 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 148727 2023-10-09T05:37:43.511 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.511 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.511 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.511 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.511 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.692007+0000", 2023-10-09T05:37:43.511 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.511 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.511 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.511 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.511 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.692007+0000", 2023-10-09T05:37:43.512 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.512 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.512 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.512 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.512 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.692007+0000", 2023-10-09T05:37:43.512 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:43.512 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.512 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.512 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.512 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.692009+0000", 2023-10-09T05:37:43.512 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7669999999999999e-06 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.692015+0000", 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7559999999999996e-06 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.692021+0000", 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3330000000000002e-06 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.513 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.886892+0000", 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19487014899999999 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.887008+0000", 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000116762 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.887385+0000", 2023-10-09T05:37:43.514 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00037671599999999998 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.892624+0000", 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0052392489999999996 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.895230+0000", 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002606106 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.515 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.897590+0000", 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023596580000000002 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.897636+0000", 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6162000000000003e-05 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.516 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:52.897674+0000", 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8166e-05 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4050 6.0 6:0be16091:::benchmark_data_smithi008_295510_object4049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.480314+0000", 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 34.982614749, 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48189201799999998, 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.517 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:43.518 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:43.518 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:37:43.518 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:37:43.518 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4050 2023-10-09T05:37:43.518 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.518 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.518 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.518 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.518 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480314+0000", 2023-10-09T05:37:43.518 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.518 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480314+0000", 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480313+0000", 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.519 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480347+0000", 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3819000000000003e-05 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480359+0000", 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2092999999999999e-05 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480374+0000", 2023-10-09T05:37:43.520 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5096000000000001e-05 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480421+0000", 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7611999999999997e-05 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480552+0000", 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013091099999999999 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.521 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480687+0000", 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013453799999999999 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959945+0000", 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47925849799999998 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:37:43.522 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959955+0000", 2023-10-09T05:37:43.523 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8859999999999999e-06 2023-10-09T05:37:43.523 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.523 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.523 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.523 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962107+0000", 2023-10-09T05:37:43.523 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002151975 2023-10-09T05:37:43.523 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.523 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.523 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:37:43.523 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962152+0000", 2023-10-09T05:37:43.523 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5049999999999997e-05 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962206+0000", 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3773999999999997e-05 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45029.0:4048 6.a e166/162 6:51c21166:::benchmark_data_smithi008_295510_object4047:head v 166'256, mlcod=166'251)", 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.481202+0000", 2023-10-09T05:37:43.524 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 34.981727092, 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47877170299999999, 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481202+0000", 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.525 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481202+0000", 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481201+0000", 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481234+0000", 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2854000000000001e-05 2023-10-09T05:37:43.526 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481243+0000", 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7019999999999996e-06 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481249+0000", 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7849999999999997e-06 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.527 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481289+0000", 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0154000000000002e-05 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481309+0000", 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9177e-05 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959928+0000", 2023-10-09T05:37:43.528 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47861928100000001 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959973+0000", 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5482000000000003e-05 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4053 6.9 6:9d44e78a:::benchmark_data_smithi008_295510_object4052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:37:43.529 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.481281+0000", 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 34.981647783, 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48114514400000002, 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4053 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.530 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481281+0000", 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481281+0000", 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481280+0000", 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:43.531 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481303+0000", 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3283999999999999e-05 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481314+0000", 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0095000000000001e-05 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.532 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481322+0000", 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5339999999999999e-06 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481583+0000", 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00026103399999999999 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481690+0000", 2023-10-09T05:37:43.533 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000107299 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481837+0000", 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000146656 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.961311+0000", 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.479473555 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.534 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962305+0000", 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00099412799999999994 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962340+0000", 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4835000000000001e-05 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:37:43.535 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962386+0000", 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6851000000000003e-05 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962426+0000", 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9617999999999999e-05 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.536 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4054 6.0 6:09150d51:::benchmark_data_smithi008_295510_object4053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.481336+0000", 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 34.981592329000001, 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.481099946, 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4054 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.537 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.538 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.538 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481336+0000", 2023-10-09T05:37:43.538 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.538 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.538 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.538 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.538 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481336+0000", 2023-10-09T05:37:43.538 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.538 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.538 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.538 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481336+0000", 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481360+0000", 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3734e-05 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481368+0000", 2023-10-09T05:37:43.539 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2109999999999998e-06 2023-10-09T05:37:43.540 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.540 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.540 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.540 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481374+0000", 2023-10-09T05:37:43.540 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6189999999999996e-06 2023-10-09T05:37:43.540 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.540 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.540 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.540 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481425+0000", 2023-10-09T05:37:43.540 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0209e-05 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481568+0000", 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014368900000000001 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481679+0000", 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011092299999999999 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.961347+0000", 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.479668066 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962257+0000", 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00090950199999999997 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962350+0000", 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3634000000000005e-05 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962398+0000", 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7218000000000001e-05 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962436+0000", 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8676e-05 2023-10-09T05:37:43.543 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4060 6.0 6:04bff711:::benchmark_data_smithi008_295510_object4059:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.482112+0000", 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 34.980817221999999, 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48355848200000001, 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:43.544 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4060 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482112+0000", 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.545 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482112+0000", 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482111+0000", 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482141+0000", 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0737000000000001e-05 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.546 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959856+0000", 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47771430100000001 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959871+0000", 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5611999999999999e-05 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.547 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959984+0000", 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000112897 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.960209+0000", 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00022461200000000001 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.960939+0000", 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000730622 2023-10-09T05:37:43.548 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.963896+0000", 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029571720000000001 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965560+0000", 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016638639999999999 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965601+0000", 2023-10-09T05:37:43.549 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0173999999999999e-05 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965632+0000", 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1591000000000003e-05 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965670+0000", 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7867000000000002e-05 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.550 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45029.0:4050 6.0 e166/162 ondisk, result = 0)", 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.482434+0000", 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 34.980494960000001, 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47752318500000002, 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.551 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482434+0000", 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482434+0000", 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482433+0000", 2023-10-09T05:37:43.552 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482437+0000", 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8990000000000003e-06 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959833+0000", 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47739634600000003 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.553 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959852+0000", 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8614000000000002e-05 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959925+0000", 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3526e-05 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.554 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959938+0000", 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2653999999999999e-05 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959957+0000", 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8861000000000001e-05 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.555 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:36511 6.0 6:0392d0f0:::benchmark_data_smithi008_295510_object36510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.167968+0000", 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 16.294960543999998, 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24803032899999999, 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 36511 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.556 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.167968+0000", 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.167968+0000", 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.167967+0000", 2023-10-09T05:37:43.557 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.167992+0000", 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5562999999999999e-05 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168005+0000", 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.271e-05 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.558 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168021+0000", 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6069000000000001e-05 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168084+0000", 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3437999999999998e-05 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.559 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168231+0000", 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014663799999999999 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168386+0000", 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015447900000000001 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171807+0000", 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034214509999999998 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.560 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415850+0000", 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.244042711 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415903+0000", 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3238000000000001e-05 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:37:43.561 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415954+0000", 2023-10-09T05:37:43.562 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0621999999999997e-05 2023-10-09T05:37:43.562 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.562 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.562 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.562 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415999+0000", 2023-10-09T05:37:43.562 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4891000000000001e-05 2023-10-09T05:37:43.562 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.562 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.562 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.562 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.562 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:36518 6.0 6:0f46dfdd:::benchmark_data_smithi008_295510_object36517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.168459+0000", 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 16.294470042, 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.25353774899999998, 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 36518 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.563 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168459+0000", 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168459+0000", 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.564 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168458+0000", 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168476+0000", 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7969999999999999e-05 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168501+0000", 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.565e-05 2023-10-09T05:37:43.565 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168526+0000", 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4383e-05 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.170220+0000", 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016944589999999999 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.566 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.170368+0000", 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000147241 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.170497+0000", 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012980199999999999 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.416044+0000", 2023-10-09T05:37:43.567 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.245546878 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.416108+0000", 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3987999999999998e-05 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.421919+0000", 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058104669999999997 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.568 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.421955+0000", 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6318e-05 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.421996+0000", 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1459999999999999e-05 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.569 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.570 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:36520 6.0 6:05b021a1:::benchmark_data_smithi008_295510_object36519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T05:37:43.570 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.171824+0000", 2023-10-09T05:37:43.570 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 16.291104886999999, 2023-10-09T05:37:43.570 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.250244829, 2023-10-09T05:37:43.570 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.570 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:43.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:43.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:37:43.570 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:37:43.570 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 36520 2023-10-09T05:37:43.570 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171824+0000", 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171824+0000", 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.571 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171822+0000", 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171846+0000", 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3102e-05 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.572 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171857+0000", 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1812e-05 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171866+0000", 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1599999999999998e-06 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171908+0000", 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2536000000000001e-05 2023-10-09T05:37:43.573 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172043+0000", 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013473099999999999 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172184+0000", 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000141395 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.574 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.416077+0000", 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24389317199999999 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.416142+0000", 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4652000000000004e-05 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.422004+0000", 2023-10-09T05:37:43.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058622209999999999 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.422031+0000", 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7056999999999999e-05 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.422069+0000", 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7394999999999997e-05 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.576 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45029.0:36511 6.0 e167/162 ondisk, result = 0)", 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.171991+0000", 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 16.290937978999999, 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24386655300000001, 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.577 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171991+0000", 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171991+0000", 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171990+0000", 2023-10-09T05:37:43.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171994+0000", 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.053e-06 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172002+0000", 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5390000000000002e-06 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172005+0000", 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8020000000000002e-06 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415835+0000", 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24382936399999999 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415843+0000", 2023-10-09T05:37:43.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4100000000000008e-06 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415857+0000", 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4168999999999999e-05 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45029.0:36519 6.a e167/162 6:58c94098:::benchmark_data_smithi008_295510_object36518:head v 167'2253, mlcod=167'2250)", 2023-10-09T05:37:43.581 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.172304+0000", 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 16.290624422, 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24684500100000001, 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172304+0000", 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.582 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172304+0000", 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172304+0000", 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172325+0000", 2023-10-09T05:37:43.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1075999999999999e-05 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414426+0000", 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24210095000000001 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414442+0000", 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6439999999999998e-05 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414499+0000", 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7275000000000003e-05 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414528+0000", 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8982000000000001e-05 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:37:43.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.419127+0000", 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045990479999999997 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.419149+0000", 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1875e-05 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45029.0:36511 6.0 e167/162 ondisk, result = 0)", 2023-10-09T05:37:43.586 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.173714+0000", 2023-10-09T05:37:43.587 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 16.289214928, 2023-10-09T05:37:43.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24229606300000001, 2023-10-09T05:37:43.587 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:43.587 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:37:43.587 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:43.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:43.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.173714+0000", 2023-10-09T05:37:43.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.589 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:43.589 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.173714+0000", 2023-10-09T05:37:43.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:43.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.173713+0000", 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.173719+0000", 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0260000000000004e-06 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:43.590 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414428+0000", 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24070894100000001 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414442+0000", 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4039999999999999e-05 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415890+0000", 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014484159999999999 2023-10-09T05:37:43.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415898+0000", 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4569999999999999e-06 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.416010+0000", 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011202 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.592 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.593 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:43.593 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:43.593 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:37:43.593 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:43.594 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:37:43.594 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:37:43.600 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:37:43.651 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:43 smithi008 ceph-mon[184418]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 peering, 77 active+clean; 2.9 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 9.5 MiB/s wr, 152 op/s; 14572/141342 objects degraded (10.310%); 2368/141342 objects misplaced (1.675%); 6.8 MiB/s, 1 keys/s, 112 objects/s recovering 2023-10-09T05:37:43.651 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:43 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1197181871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:43.659 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:37:43.677 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:43.708 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:37:43.758 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:37:43.767 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:37:43.767 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:37:43.767 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:37:43.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:37:43.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:37:43.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:37:43.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:37:43.768 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:37:43.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 37 16 49732 49716 83.9155 53.1875 0.0146283 0.0119097 2023-10-09T05:37:43.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:43 smithi008 ceph-mon[191227]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 peering, 77 active+clean; 2.9 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 9.5 MiB/s wr, 152 op/s; 14572/141342 objects degraded (10.310%); 2368/141342 objects misplaced (1.675%); 6.8 MiB/s, 1 keys/s, 112 objects/s recovering 2023-10-09T05:37:43.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:43 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1197181871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:44.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:43 smithi191 ceph-mon[161669]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 peering, 77 active+clean; 2.9 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 9.5 MiB/s wr, 152 op/s; 14572/141342 objects degraded (10.310%); 2368/141342 objects misplaced (1.675%); 6.8 MiB/s, 1 keys/s, 112 objects/s recovering 2023-10-09T05:37:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:43 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1197181871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:44.469 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:44.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:44.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:44.574 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:44.601 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:37:44.630 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:44.702 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:37:44.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:44.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:44.808 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:44.843 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 38 16 50554 50538 83.0596 51.375 0.00793442 0.0120186 2023-10-09T05:37:44.845 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:44 smithi191 ceph-mon[161669]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 79 active+clean; 2.9 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 14 MiB/s wr, 218 op/s; 14572/143088 objects degraded (10.184%); 2368/143088 objects misplaced (1.655%); 5.7 MiB/s, 1 keys/s, 93 objects/s recovering 2023-10-09T05:37:45.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:45.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:45.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:44 smithi008 ceph-mon[184418]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 79 active+clean; 2.9 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 14 MiB/s wr, 218 op/s; 14572/143088 objects degraded (10.184%); 2368/143088 objects misplaced (1.655%); 5.7 MiB/s, 1 keys/s, 93 objects/s recovering 2023-10-09T05:37:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:44 smithi008 ceph-mon[191227]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 79 active+clean; 2.9 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 14 MiB/s wr, 218 op/s; 14572/143088 objects degraded (10.184%); 2368/143088 objects misplaced (1.655%); 5.7 MiB/s, 1 keys/s, 93 objects/s recovering 2023-10-09T05:37:45.799 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:45.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:45.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:45.842 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 39 16 51341 51325 82.1914 49.1875 0.0209396 0.0121617 2023-10-09T05:37:45.907 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:45.948 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:46.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:45 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14572/143088 objects degraded (10.184%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:46.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:45 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14572/143088 objects degraded (10.184%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:46.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:45 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14572/143088 objects degraded (10.184%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:46.308 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:37:46.443 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:46.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:46.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:46.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:46.588 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:46.642 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:46.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:46.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:46.749 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:46.776 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:46.842 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:37:46.843482+0000 min lat: 0.00103937 max lat: 6.04263 avg lat: 0.0122519 2023-10-09T05:37:46.843 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:37:46.843 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 40 16 52128 52112 81.3668 49.1875 0.00291364 0.0122519 2023-10-09T05:37:47.103 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:46 smithi008 ceph-mon[184418]: pgmap v454: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 22 MiB/s wr, 348 op/s; 14569/148098 objects degraded (9.837%); 2368/148098 objects misplaced (1.599%); 4.4 MiB/s, 1 keys/s, 72 objects/s recovering 2023-10-09T05:37:47.103 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:46 smithi008 ceph-mon[191227]: pgmap v454: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 22 MiB/s wr, 348 op/s; 14569/148098 objects degraded (9.837%); 2368/148098 objects misplaced (1.599%); 4.4 MiB/s, 1 keys/s, 72 objects/s recovering 2023-10-09T05:37:47.243 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:47.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:47.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:46 smithi191 ceph-mon[161669]: pgmap v454: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 22 MiB/s wr, 348 op/s; 14569/148098 objects degraded (9.837%); 2368/148098 objects misplaced (1.599%); 4.4 MiB/s, 1 keys/s, 72 objects/s recovering 2023-10-09T05:37:47.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:47.381 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:47.440 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:47.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:47.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:47.547 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:47.670 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:47.794 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:47.827 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:47.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:47.844 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:47.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 41 16 52952 52936 80.6386 51.5 0.025209 0.0123941 2023-10-09T05:37:47.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:47.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:47.979 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:48.040 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:48.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:48.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:48.144 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:48.231 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:48.497 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:48.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:48.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:48.610 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:37:48.682 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:48 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2026048842' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:37:48.682 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:48 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:37:48.685 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T05:37:48.723 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:48.843 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 42 16 53790 53774 79.966 52.375 0.0247357 0.0124977 2023-10-09T05:37:48.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:48 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2026048842' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:37:48.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:48 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:37:49.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:48 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2026048842' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:37:49.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:48 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:37:49.384 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:37:49.843 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 43 16 54718 54702 79.4553 58 0.00570558 0.0125618 2023-10-09T05:37:49.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[184418]: pgmap v455: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 31 MiB/s wr, 490 op/s; 14569/153528 objects degraded (9.489%); 2024/153528 objects misplaced (1.318%); 6.1 MiB/s, 1 keys/s, 98 objects/s recovering 2023-10-09T05:37:49.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[184418]: failed to encode map e175 with expected crc 2023-10-09T05:37:49.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[184418]: failed to encode map e175 with expected crc 2023-10-09T05:37:49.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:37:49.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[184418]: osdmap e175: 8 total, 8 up, 7 in 2023-10-09T05:37:49.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[184418]: failed to encode map e175 with expected crc 2023-10-09T05:37:49.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:49.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[191227]: pgmap v455: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 31 MiB/s wr, 490 op/s; 14569/153528 objects degraded (9.489%); 2024/153528 objects misplaced (1.318%); 6.1 MiB/s, 1 keys/s, 98 objects/s recovering 2023-10-09T05:37:49.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[191227]: failed to encode map e175 with expected crc 2023-10-09T05:37:49.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[191227]: failed to encode map e175 with expected crc 2023-10-09T05:37:49.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:37:49.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[191227]: osdmap e175: 8 total, 8 up, 7 in 2023-10-09T05:37:49.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[191227]: failed to encode map e175 with expected crc 2023-10-09T05:37:49.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:49 smithi191 ceph-mon[161669]: pgmap v455: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 31 MiB/s wr, 490 op/s; 14569/153528 objects degraded (9.489%); 2024/153528 objects misplaced (1.318%); 6.1 MiB/s, 1 keys/s, 98 objects/s recovering 2023-10-09T05:37:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:49 smithi191 ceph-mon[161669]: failed to encode map e175 with expected crc 2023-10-09T05:37:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:49 smithi191 ceph-mon[161669]: failed to encode map e175 with expected crc 2023-10-09T05:37:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:49 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:37:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:49 smithi191 ceph-mon[161669]: osdmap e175: 8 total, 8 up, 7 in 2023-10-09T05:37:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:49 smithi191 ceph-mon[161669]: failed to encode map e175 with expected crc 2023-10-09T05:37:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:50.367 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:37:50.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:50.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:37:50.471 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:50.534 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:37:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:37:50] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.33.4" 2023-10-09T05:37:50.538 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:50.843 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 44 16 55356 55340 78.5562 39.875 0.00716786 0.0127233 2023-10-09T05:37:50.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:50 smithi008 ceph-mon[184418]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 38 MiB/s wr, 612 op/s; 14569/158910 objects degraded (9.168%); 2024/158910 objects misplaced (1.274%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T05:37:50.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:50 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14569/158910 objects degraded (9.168%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:50.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:50 smithi008 ceph-mon[191227]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 38 MiB/s wr, 612 op/s; 14569/158910 objects degraded (9.168%); 2024/158910 objects misplaced (1.274%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T05:37:50.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:50 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14569/158910 objects degraded (9.168%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:50 smithi191 ceph-mon[161669]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 38 MiB/s wr, 612 op/s; 14569/158910 objects degraded (9.168%); 2024/158910 objects misplaced (1.274%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T05:37:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:50 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14569/158910 objects degraded (9.168%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:51.313 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:51.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:51.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:51.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:51.459 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:51.519 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:51.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:51.624 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:51.725 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:51.831 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T05:37:51.843 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 45 16 56192 56176 77.9718 52.25 0.0264712 0.0128172 2023-10-09T05:37:51.860 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:51.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:51.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:51.988 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:52.029 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:52.091 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:52.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:52.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:52.200 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:52.278 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:52.388 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:52.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:52.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:52.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:52.539 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:52.589 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:37:52.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:52.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:52.625 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:52.697 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:52.776 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:37:52.776 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:37:52.776 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.776 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45029.0:56902 6.d e175/162 6:be2c3466:::benchmark_data_smithi008_295510_object56901:head v 175'3514, mlcod=175'3512)", 2023-10-09T05:37:52.776 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:52.764347+0000", 2023-10-09T05:37:52.776 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0060700349999999997, 2023-10-09T05:37:52.776 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006101174, 2023-10-09T05:37:52.777 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:52.777 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:37:52.777 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:52.777 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.777 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:52.777 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.764347+0000", 2023-10-09T05:37:52.777 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:52.777 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.777 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.777 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:52.778 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.764347+0000", 2023-10-09T05:37:52.778 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:52.778 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.778 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.778 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:52.778 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.764346+0000", 2023-10-09T05:37:52.778 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:37:52.778 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.778 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.778 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:52.778 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.764380+0000", 2023-10-09T05:37:52.779 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4078999999999997e-05 2023-10-09T05:37:52.779 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.779 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.779 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:52.779 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.764396+0000", 2023-10-09T05:37:52.779 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5523e-05 2023-10-09T05:37:52.779 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.779 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.779 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:52.779 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.764405+0000", 2023-10-09T05:37:52.780 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2350000000000005e-06 2023-10-09T05:37:52.780 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.780 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.780 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:52.780 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.764498+0000", 2023-10-09T05:37:52.780 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2816e-05 2023-10-09T05:37:52.780 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.780 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.780 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:52.780 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.764519+0000", 2023-10-09T05:37:52.781 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1296000000000001e-05 2023-10-09T05:37:52.781 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:52.781 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:52.781 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:52.781 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.781 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.781 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:56907 6.1 6:80efeb26:::benchmark_data_smithi008_295510_object56906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T05:37:52.781 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:52.766034+0000", 2023-10-09T05:37:52.781 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0043831419999999996, 2023-10-09T05:37:52.781 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0044901070000000001, 2023-10-09T05:37:52.781 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:37:52.782 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:37:52.782 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:37:52.782 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:37:52.782 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:37:52.782 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56907 2023-10-09T05:37:52.782 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.782 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:37:52.782 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.782 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:37:52.782 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.766034+0000", 2023-10-09T05:37:52.783 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:52.783 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.783 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.783 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:37:52.783 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.766034+0000", 2023-10-09T05:37:52.783 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:37:52.783 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.783 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.783 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:37:52.783 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.766032+0000", 2023-10-09T05:37:52.784 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:37:52.784 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.784 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.784 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:37:52.784 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.766069+0000", 2023-10-09T05:37:52.784 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6915999999999997e-05 2023-10-09T05:37:52.784 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.784 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.784 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:37:52.784 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.766084+0000", 2023-10-09T05:37:52.785 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4724e-05 2023-10-09T05:37:52.785 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.785 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.785 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:37:52.785 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.766097+0000", 2023-10-09T05:37:52.785 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3569e-05 2023-10-09T05:37:52.785 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.785 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.785 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:37:52.785 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.766180+0000", 2023-10-09T05:37:52.785 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2485999999999996e-05 2023-10-09T05:37:52.786 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.786 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.786 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:37:52.786 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.766343+0000", 2023-10-09T05:37:52.786 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016299199999999999 2023-10-09T05:37:52.786 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:37:52.786 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:37:52.786 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:37:52.786 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:52.766511+0000", 2023-10-09T05:37:52.786 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016875500000000001 2023-10-09T05:37:52.787 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:52.787 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:37:52.787 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:52.787 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:37:52.787 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:37:52.787 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 2 2023-10-09T05:37:52.787 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:37:52.809 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:52.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 46 16 56957 56941 77.3148 47.8125 0.0190713 0.0129279 2023-10-09T05:37:53.454 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:53.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:53.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:53.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:37:53.606 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:53.665 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:37:53.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:53.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:37:53.696 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:37:53.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:53 smithi008 ceph-mon[184418]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 14569/164241 objects degraded (8.871%); 1620/164241 objects misplaced (0.986%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:37:53.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/762351268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:53.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:53 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:53 smithi008 ceph-mon[191227]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 14569/164241 objects degraded (8.871%); 1620/164241 objects misplaced (0.986%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:37:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/762351268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:53 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:53.772 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:53.799 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:53.839 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:37:53.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:53.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:53.844 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 47 16 57827 57811 76.8267 54.375 0.00483529 0.0129795 2023-10-09T05:37:53.944 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:37:53.974 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:54.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:53 smithi191 ceph-mon[161669]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 14569/164241 objects degraded (8.871%); 1620/164241 objects misplaced (0.986%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:37:54.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/762351268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:54.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:53 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:37:54.336 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:37:54.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:54.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:37:54.456 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:54.478 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:37:54.577 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:54.688 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:54.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:37:54.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[184418]: osdmap e176: 8 total, 8 up, 7 in 2023-10-09T05:37:54.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[184418]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[184418]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[184418]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[184418]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[184418]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[184418]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[184418]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:54.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:37:54.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[191227]: osdmap e176: 8 total, 8 up, 7 in 2023-10-09T05:37:54.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[191227]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[191227]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[191227]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[191227]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[191227]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[191227]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.691 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:54 smithi008 ceph-mon[191227]: failed to encode map e176 with expected crc 2023-10-09T05:37:54.844 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 48 16 58597 58581 76.2288 48.125 0.0188031 0.0131116 2023-10-09T05:37:55.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:54 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:37:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:54 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:37:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:54 smithi191 ceph-mon[161669]: osdmap e176: 8 total, 8 up, 7 in 2023-10-09T05:37:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:54 smithi191 ceph-mon[161669]: failed to encode map e176 with expected crc 2023-10-09T05:37:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:54 smithi191 ceph-mon[161669]: failed to encode map e176 with expected crc 2023-10-09T05:37:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:54 smithi191 ceph-mon[161669]: failed to encode map e176 with expected crc 2023-10-09T05:37:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:54 smithi191 ceph-mon[161669]: failed to encode map e176 with expected crc 2023-10-09T05:37:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:54 smithi191 ceph-mon[161669]: failed to encode map e176 with expected crc 2023-10-09T05:37:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:54 smithi191 ceph-mon[161669]: failed to encode map e176 with expected crc 2023-10-09T05:37:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:54 smithi191 ceph-mon[161669]: failed to encode map e176 with expected crc 2023-10-09T05:37:55.844 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 49 16 59577 59561 75.9232 61.25 0.0027762 0.013156 2023-10-09T05:37:55.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:55 smithi008 ceph-mon[184418]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 50 MiB/s wr, 801 op/s; 14569/167343 objects degraded (8.706%); 1620/167343 objects misplaced (0.968%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T05:37:55.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:55 smithi008 ceph-mon[191227]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 50 MiB/s wr, 801 op/s; 14569/167343 objects degraded (8.706%); 1620/167343 objects misplaced (0.968%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T05:37:56.026 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:56.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:56.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:56.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:55 smithi191 ceph-mon[161669]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 50 MiB/s wr, 801 op/s; 14569/167343 objects degraded (8.706%); 1620/167343 objects misplaced (0.968%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T05:37:56.137 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:56.216 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:56.234 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:56.507 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:56.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:56.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:56.615 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:56.655 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:56.732 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:37:56.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:56.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:56.838 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:56.844 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 50 16 60313 60297 75.3249 46 0.0199081 0.0132697 2023-10-09T05:37:56.921 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:56.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:57.045 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:37:57.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:37:57.193 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:57.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:57.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:57.306 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:37:57.346 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:57.420 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:37:57.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:57.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:37:57.526 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:57.583 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:57.661 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:37:57.661 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:37:57.661 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:37:57.661 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:37:57.661 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:37:57.661 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:37:57.661 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:37:57.661 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:37:57.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:37:57.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:37:57.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:37:57.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:37:57.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:37:57.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:37:57.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:37:57.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:37:57.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:57 smithi191 ceph-mon[161669]: pgmap v461: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 14569/172497 objects degraded (8.446%); 1620/172497 objects misplaced (0.939%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:37:57.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:57 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14569/172497 objects degraded (8.446%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:57.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:57 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3691402617' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:57.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 51 16 61104 61088 74.8175 49.4375 0.00271291 0.0133387 2023-10-09T05:37:57.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:57 smithi008 ceph-mon[184418]: pgmap v461: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 14569/172497 objects degraded (8.446%); 1620/172497 objects misplaced (0.939%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:37:57.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:57 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14569/172497 objects degraded (8.446%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:57.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:57 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3691402617' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:57 smithi008 ceph-mon[191227]: pgmap v461: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 14569/172497 objects degraded (8.446%); 1620/172497 objects misplaced (0.939%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:37:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:57 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14569/172497 objects degraded (8.446%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:37:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:57 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3691402617' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:37:58.275 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:37:58.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:58.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:37:58.302 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:37:58.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:37:58.430 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:58.544 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:37:58.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:58.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:37:58.649 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:37:58.689 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:58.720 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:58 smithi191 ceph-mon[161669]: pgmap v462: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 14569/177873 objects degraded (8.191%); 1260/177873 objects misplaced (0.708%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T05:37:58.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 52 16 61774 61758 74.1843 41.875 0.00597898 0.0134625 2023-10-09T05:37:58.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:58 smithi008 ceph-mon[184418]: pgmap v462: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 14569/177873 objects degraded (8.191%); 1260/177873 objects misplaced (0.708%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T05:37:58.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:58 smithi008 ceph-mon[191227]: pgmap v462: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 14569/177873 objects degraded (8.191%); 1260/177873 objects misplaced (0.708%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T05:37:59.182 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:37:59.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:59.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:37:59.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:37:59.341 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:59.407 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:37:59.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:37:59.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:37:59.514 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:37:59.594 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:37:59.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:37:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:59.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 53 16 62601 62585 73.7599 51.6875 0.0269506 0.013551 2023-10-09T05:37:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:37:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:37:59.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:37:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:00.506 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:38:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:38:00] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T05:38:00.764 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:00.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:00.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:00.846 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 54 16 63500 63484 73.4336 56.1875 0.0115591 0.0136116 2023-10-09T05:38:00.871 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:00.961 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T05:38:00.979 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:00 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:00 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:00 smithi008 ceph-mon[191227]: pgmap v463: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s; 14569/183099 objects degraded (7.957%); 1260/183099 objects misplaced (0.688%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:38:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4119361011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:38:01.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:00 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:01.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:00 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:01.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:00 smithi008 ceph-mon[184418]: pgmap v463: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s; 14569/183099 objects degraded (7.957%); 1260/183099 objects misplaced (0.688%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:38:01.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4119361011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:38:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:00 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:00 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:00 smithi191 ceph-mon[161669]: pgmap v463: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 50 MiB/s wr, 805 op/s; 14569/183099 objects degraded (7.957%); 1260/183099 objects misplaced (0.688%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:38:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4119361011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:38:01.421 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:38:01.849 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 55 16 64288 64272 72.9921 49.25 0.0205374 0.0136951 2023-10-09T05:38:01.867 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:01.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:01.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:01.977 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:02.026 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[184418]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.026 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4119361011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:38:02.026 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[184418]: osdmap e177: 8 total, 8 up, 7 in 2023-10-09T05:38:02.026 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[184418]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.027 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[184418]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.027 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[184418]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.027 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[184418]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.027 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[184418]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.027 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[191227]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.027 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4119361011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:38:02.027 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[191227]: osdmap e177: 8 total, 8 up, 7 in 2023-10-09T05:38:02.027 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[191227]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.027 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[191227]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.027 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[191227]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.028 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[191227]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.028 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:01 smithi008 ceph-mon[191227]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.069 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:02.124 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T05:38:02.227 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:02.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:01 smithi191 ceph-mon[161669]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:01 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4119361011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:38:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:01 smithi191 ceph-mon[161669]: osdmap e177: 8 total, 8 up, 7 in 2023-10-09T05:38:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:01 smithi191 ceph-mon[161669]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:01 smithi191 ceph-mon[161669]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:01 smithi191 ceph-mon[161669]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:01 smithi191 ceph-mon[161669]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:01 smithi191 ceph-mon[161669]: failed to encode map e177 with expected crc 2023-10-09T05:38:02.344 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:02.473 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:02.491 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:02.614 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:38:02.614 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:38:02.614 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:02.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:02.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:02.620 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:02.721 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:02.792 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:02.847 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 56 16 64949 64933 72.4266 41.3125 0.0138955 0.0137998 2023-10-09T05:38:02.999 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:03.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:03.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:03.109 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:03.191 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:02 smithi008 ceph-mon[184418]: pgmap v465: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 14569/188367 objects degraded (7.734%); 821/188367 objects misplaced (0.436%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T05:38:03.191 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:02 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14569/183099 objects degraded (7.957%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:03.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:02 smithi008 ceph-mon[191227]: pgmap v465: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 14569/188367 objects degraded (7.734%); 821/188367 objects misplaced (0.436%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T05:38:03.192 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:02 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14569/183099 objects degraded (7.957%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:03.195 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:03.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:02 smithi191 ceph-mon[161669]: pgmap v465: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 14569/188367 objects degraded (7.734%); 821/188367 objects misplaced (0.436%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T05:38:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:02 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14569/183099 objects degraded (7.957%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:03.849 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 57 16 65613 65597 71.8819 41.5 0.018312 0.0139055 2023-10-09T05:38:04.037 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:04.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:04.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:04.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:04.185 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:04.251 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:38:04.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:04.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:04.267 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:38:04.361 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:04.430 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:04.850 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 58 16 66448 66432 71.5425 52.1875 0.0225809 0.0139715 2023-10-09T05:38:05.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:05 smithi008 ceph-mon[184418]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 14569/191229 objects degraded (7.619%); 821/191229 objects misplaced (0.429%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:38:05.367 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:05 smithi008 ceph-mon[191227]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 14569/191229 objects degraded (7.619%); 821/191229 objects misplaced (0.429%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:38:05.400 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:05.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:05.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:05.514 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:05 smithi191 ceph-mon[161669]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 14569/191229 objects degraded (7.619%); 821/191229 objects misplaced (0.429%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:38:05.596 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:05.850 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 59 16 67280 67264 71.2107 52 0.0134533 0.0140358 2023-10-09T05:38:05.951 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:05.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:05.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:06.066 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:06.097 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:06.154 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:06.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:06.235 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:06.236 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:06.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:06.292 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:06.347 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:06.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:06.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:06.452 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:06.554 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:06.859 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:38:06.857293+0000 min lat: 0.00103937 max lat: 6.04263 avg lat: 0.0141113 2023-10-09T05:38:06.859 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:38:06.859 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 60 16 68068 68052 70.8374 49.25 0.0111202 0.0141113 2023-10-09T05:38:07.099 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:07.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:07.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:07.173 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:38:07.222 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:07.259 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:07.271 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:38:07.314 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:07.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:07.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:07.322 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:38:07.352 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:07 smithi008 ceph-mon[191227]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 14569/196380 objects degraded (7.419%); 821/196380 objects misplaced (0.418%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:38:07.352 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:07 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3373340103' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:07.353 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:07 smithi008 ceph-mon[184418]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 14569/196380 objects degraded (7.419%); 821/196380 objects misplaced (0.418%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:38:07.353 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:07 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3373340103' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:07.396 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:38:07.420 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:07.440 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:38:07.512 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:07.519 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:38:07.522 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:38:07.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:07 smithi191 ceph-mon[161669]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 14569/196380 objects degraded (7.419%); 821/196380 objects misplaced (0.418%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:38:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:07 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3373340103' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:07.555 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:38:07.628 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:38:07.644 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:38:07.644 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:38:07.644 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:38:07.644 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:38:07.644 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:38:07.644 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:38:07.644 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:38:07.644 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:38:07.645 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:07.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:07.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:07.750 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:07.779 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:07.857 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 61 16 68828 68812 70.4547 47.5 0.0206256 0.0141875 2023-10-09T05:38:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:38:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[184418]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3373340103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:38:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[184418]: osdmap e178: 8 total, 8 up, 7 in 2023-10-09T05:38:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[184418]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[184418]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[184418]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2981047421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14569/196380 objects degraded (7.419%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:08.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:38:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[191227]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3373340103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:38:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[191227]: osdmap e178: 8 total, 8 up, 7 in 2023-10-09T05:38:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[191227]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[191227]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[191227]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2981047421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14569/196380 objects degraded (7.419%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:08 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:38:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:08 smithi191 ceph-mon[161669]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3373340103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:38:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:08 smithi191 ceph-mon[161669]: osdmap e178: 8 total, 8 up, 7 in 2023-10-09T05:38:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:08 smithi191 ceph-mon[161669]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:08 smithi191 ceph-mon[161669]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:08 smithi191 ceph-mon[161669]: failed to encode map e178 with expected crc 2023-10-09T05:38:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2981047421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14569/196380 objects degraded (7.419%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:08.857 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 62 16 69669 69653 70.1655 52.5625 0.019121 0.0142463 2023-10-09T05:38:09.621 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:09.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:09.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:09.726 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:09.754 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:09.858 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 63 16 70135 70119 69.5144 29.125 0.00302242 0.0142671 2023-10-09T05:38:09.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:09 smithi008 ceph-mon[184418]: pgmap v469: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s; 14569/201417 objects degraded (7.233%); 340/201417 objects misplaced (0.169%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:38:09.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:09 smithi008 ceph-mon[184418]: osdmap e179: 8 total, 8 up, 7 in 2023-10-09T05:38:09.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:09 smithi008 ceph-mon[184418]: failed to encode map e179 with expected crc 2023-10-09T05:38:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:09 smithi008 ceph-mon[191227]: pgmap v469: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s; 14569/201417 objects degraded (7.233%); 340/201417 objects misplaced (0.169%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:38:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:09 smithi008 ceph-mon[191227]: osdmap e179: 8 total, 8 up, 7 in 2023-10-09T05:38:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:09 smithi008 ceph-mon[191227]: failed to encode map e179 with expected crc 2023-10-09T05:38:10.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:09 smithi191 ceph-mon[161669]: pgmap v469: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s; 14569/201417 objects degraded (7.233%); 340/201417 objects misplaced (0.169%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:38:10.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:09 smithi191 ceph-mon[161669]: osdmap e179: 8 total, 8 up, 7 in 2023-10-09T05:38:10.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:09 smithi191 ceph-mon[161669]: failed to encode map e179 with expected crc 2023-10-09T05:38:10.709 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:38:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:38:10] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T05:38:10.802 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:10.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:10.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:10.858 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 64 16 70772 70756 69.0498 39.8125 0.00367648 0.0144727 2023-10-09T05:38:10.908 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:10.935 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:10.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:10 smithi008 ceph-mon[184418]: failed to encode map e179 with expected crc 2023-10-09T05:38:10.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:10 smithi008 ceph-mon[184418]: failed to encode map e179 with expected crc 2023-10-09T05:38:10.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:10 smithi008 ceph-mon[184418]: osdmap e180: 8 total, 8 up, 7 in 2023-10-09T05:38:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:10 smithi008 ceph-mon[184418]: failed to encode map e180 with expected crc 2023-10-09T05:38:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:10 smithi008 ceph-mon[191227]: failed to encode map e179 with expected crc 2023-10-09T05:38:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:10 smithi008 ceph-mon[191227]: failed to encode map e179 with expected crc 2023-10-09T05:38:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:10 smithi008 ceph-mon[191227]: osdmap e180: 8 total, 8 up, 7 in 2023-10-09T05:38:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:10 smithi008 ceph-mon[191227]: failed to encode map e180 with expected crc 2023-10-09T05:38:11.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:10 smithi191 ceph-mon[161669]: failed to encode map e179 with expected crc 2023-10-09T05:38:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:10 smithi191 ceph-mon[161669]: failed to encode map e179 with expected crc 2023-10-09T05:38:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:10 smithi191 ceph-mon[161669]: osdmap e180: 8 total, 8 up, 7 in 2023-10-09T05:38:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:10 smithi191 ceph-mon[161669]: failed to encode map e180 with expected crc 2023-10-09T05:38:11.289 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:11.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:11.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:11.430 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:11.485 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:38:11.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:11.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:11.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:11.629 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:11.686 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:11.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:11.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:11.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:11.826 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:11.859 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 65 16 71478 71462 68.6664 44.125 0.0257922 0.014558 2023-10-09T05:38:11.888 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:11.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:11.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:11.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:11 smithi008 ceph-mon[184418]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 14569/206235 objects degraded (7.064%); 340/206235 objects misplaced (0.165%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:38:11.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:11 smithi008 ceph-mon[184418]: failed to encode map e180 with expected crc 2023-10-09T05:38:11.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:11 smithi008 ceph-mon[184418]: failed to encode map e180 with expected crc 2023-10-09T05:38:11.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:11 smithi008 ceph-mon[184418]: failed to encode map e180 with expected crc 2023-10-09T05:38:11.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:11 smithi008 ceph-mon[184418]: failed to encode map e180 with expected crc 2023-10-09T05:38:11.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:11 smithi008 ceph-mon[191227]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 14569/206235 objects degraded (7.064%); 340/206235 objects misplaced (0.165%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:38:11.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:11 smithi008 ceph-mon[191227]: failed to encode map e180 with expected crc 2023-10-09T05:38:11.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:11 smithi008 ceph-mon[191227]: failed to encode map e180 with expected crc 2023-10-09T05:38:11.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:11 smithi008 ceph-mon[191227]: failed to encode map e180 with expected crc 2023-10-09T05:38:11.974 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:11 smithi008 ceph-mon[191227]: failed to encode map e180 with expected crc 2023-10-09T05:38:11.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:12.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:11 smithi191 ceph-mon[161669]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 14569/206235 objects degraded (7.064%); 340/206235 objects misplaced (0.165%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:38:12.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:11 smithi191 ceph-mon[161669]: failed to encode map e180 with expected crc 2023-10-09T05:38:12.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:11 smithi191 ceph-mon[161669]: failed to encode map e180 with expected crc 2023-10-09T05:38:12.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:11 smithi191 ceph-mon[161669]: failed to encode map e180 with expected crc 2023-10-09T05:38:12.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:11 smithi191 ceph-mon[161669]: failed to encode map e180 with expected crc 2023-10-09T05:38:12.027 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:12.086 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:12.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:12.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:12.193 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:12.307 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:12.417 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:38:12.431 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:38:12.433 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:12.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:12.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:12.513 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:38:12.513 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:38:12.513 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:38:12.513 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:38:12.513 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.513 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119029 5.c 5:33f591e4:::benchmark_data_smithi008_238708_object893:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:38:12.513 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.009231+0000", 2023-10-09T05:38:12.514 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 228.48726692299999, 2023-10-09T05:38:12.514 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0490911119999999, 2023-10-09T05:38:12.514 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.514 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.514 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.514 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:12.514 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:12.514 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119029 2023-10-09T05:38:12.515 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.515 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.515 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.515 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.515 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009231+0000", 2023-10-09T05:38:12.515 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.515 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.515 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.515 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.516 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009231+0000", 2023-10-09T05:38:12.516 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.516 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.516 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.516 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.516 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009229+0000", 2023-10-09T05:38:12.516 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:38:12.516 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.516 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.516 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.516 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009236+0000", 2023-10-09T05:38:12.517 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2099999999999996e-06 2023-10-09T05:38:12.517 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.517 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.517 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.517 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009250+0000", 2023-10-09T05:38:12.517 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3377000000000001e-05 2023-10-09T05:38:12.517 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.517 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.517 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.517 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009259+0000", 2023-10-09T05:38:12.518 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4050000000000006e-06 2023-10-09T05:38:12.518 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.518 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.518 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.518 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180684+0000", 2023-10-09T05:38:12.518 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.171425208 2023-10-09T05:38:12.518 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.518 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.518 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:12.518 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180700+0000", 2023-10-09T05:38:12.519 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5184999999999999e-05 2023-10-09T05:38:12.519 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.519 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.519 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.519 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.030305+0000", 2023-10-09T05:38:12.519 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.84960513900000001 2023-10-09T05:38:12.519 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.519 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.519 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.519 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.030495+0000", 2023-10-09T05:38:12.519 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000190605 2023-10-09T05:38:12.520 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.520 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.520 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:38:12.520 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.030899+0000", 2023-10-09T05:38:12.520 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00040406200000000001 2023-10-09T05:38:12.520 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.520 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.520 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.520 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.040416+0000", 2023-10-09T05:38:12.520 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0095167640000000005 2023-10-09T05:38:12.521 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.521 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.521 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.521 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058217+0000", 2023-10-09T05:38:12.521 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.01780087 2023-10-09T05:38:12.521 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.521 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.521 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.521 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058246+0000", 2023-10-09T05:38:12.521 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8473999999999999e-05 2023-10-09T05:38:12.522 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.522 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.522 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.522 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058285+0000", 2023-10-09T05:38:12.522 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9922999999999997e-05 2023-10-09T05:38:12.522 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.522 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.522 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.522 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058322+0000", 2023-10-09T05:38:12.522 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6687999999999998e-05 2023-10-09T05:38:12.522 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.523 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.523 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.523 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.523 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.523 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119039 5.c 5:34e6951f:::benchmark_data_smithi008_238708_object903:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:38:12.523 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.151148+0000", 2023-10-09T05:38:12.523 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 228.34534964700001, 2023-10-09T05:38:12.523 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90729258899999998, 2023-10-09T05:38:12.523 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.523 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.523 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.524 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:12.524 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:12.524 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119039 2023-10-09T05:38:12.524 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.524 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.524 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.524 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.524 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151148+0000", 2023-10-09T05:38:12.524 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.524 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.524 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.525 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.525 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151148+0000", 2023-10-09T05:38:12.525 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.525 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.525 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.525 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.525 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151146+0000", 2023-10-09T05:38:12.525 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:38:12.525 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.525 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.525 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151154+0000", 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2060000000000001e-06 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151177+0000", 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3495000000000001e-05 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151189+0000", 2023-10-09T05:38:12.526 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1959e-05 2023-10-09T05:38:12.527 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.527 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.527 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.527 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180742+0000", 2023-10-09T05:38:12.527 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.029553138 2023-10-09T05:38:12.527 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.527 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.527 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:12.527 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180757+0000", 2023-10-09T05:38:12.527 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4803999999999999e-05 2023-10-09T05:38:12.527 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.032636+0000", 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85187944500000001 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.032759+0000", 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012297400000000001 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.528 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:38:12.529 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.033078+0000", 2023-10-09T05:38:12.529 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00031910800000000002 2023-10-09T05:38:12.529 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.529 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.529 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.529 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.040420+0000", 2023-10-09T05:38:12.529 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0073418340000000002 2023-10-09T05:38:12.529 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.530 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.530 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.530 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058354+0000", 2023-10-09T05:38:12.530 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017933893999999999 2023-10-09T05:38:12.530 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.530 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.531 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.531 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058380+0000", 2023-10-09T05:38:12.531 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6250000000000001e-05 2023-10-09T05:38:12.531 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.531 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.531 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.531 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058412+0000", 2023-10-09T05:38:12.532 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1640999999999997e-05 2023-10-09T05:38:12.532 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.532 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.532 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.532 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058441+0000", 2023-10-09T05:38:12.532 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8843000000000002e-05 2023-10-09T05:38:12.532 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.532 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.532 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.532 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.532 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:161807 5.1c 5:3af01b52:::benchmark_data_smithi008_238708_object43671:head [delete] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:35:05.702343+0000", 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 186.79415473700001, 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19827682399999999, 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 161807 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.533 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.534 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.534 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.534 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702343+0000", 2023-10-09T05:38:12.534 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.534 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.534 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.534 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.534 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702343+0000", 2023-10-09T05:38:12.534 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702342+0000", 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702348+0000", 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9920000000000004e-06 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.535 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702360+0000", 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1688000000000001e-05 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702368+0000", 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1599999999999998e-06 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702412+0000", 2023-10-09T05:38:12.536 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3918999999999997e-05 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702534+0000", 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000121931 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702897+0000", 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00036327899999999999 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.537 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.538 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.538 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.707464+0000", 2023-10-09T05:38:12.538 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045671349999999999 2023-10-09T05:38:12.538 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.538 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.538 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.538 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.712601+0000", 2023-10-09T05:38:12.538 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051370840000000001 2023-10-09T05:38:12.538 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.538 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.538 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900526+0000", 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18792462700000001 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900582+0000", 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6017999999999998e-05 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900620+0000", 2023-10-09T05:38:12.539 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8303000000000002e-05 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:161808 5.1c 5:39ce818a:::benchmark_data_smithi008_238708_object43672:head [delete] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:35:05.702383+0000", 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 186.79411531299999, 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19837861000000001, 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.540 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 161808 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702383+0000", 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.541 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702383+0000", 2023-10-09T05:38:12.542 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.542 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.542 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702382+0000", 2023-10-09T05:38:12.542 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:12.542 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.542 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702385+0000", 2023-10-09T05:38:12.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3850000000000002e-06 2023-10-09T05:38:12.543 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.543 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.543 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.543 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702393+0000", 2023-10-09T05:38:12.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2930000000000003e-06 2023-10-09T05:38:12.543 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.543 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.543 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.543 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.702399+0000", 2023-10-09T05:38:12.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6189999999999996e-06 2023-10-09T05:38:12.544 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.544 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.544 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.544 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.705223+0000", 2023-10-09T05:38:12.544 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028235840000000001 2023-10-09T05:38:12.544 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.544 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.544 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.544 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.705338+0000", 2023-10-09T05:38:12.544 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011469800000000001 2023-10-09T05:38:12.545 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.545 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.545 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:38:12.545 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.705630+0000", 2023-10-09T05:38:12.545 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00029198199999999998 2023-10-09T05:38:12.545 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.545 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.545 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.545 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.707621+0000", 2023-10-09T05:38:12.545 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019917260000000001 2023-10-09T05:38:12.545 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.546 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.546 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.546 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.714476+0000", 2023-10-09T05:38:12.546 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068550089999999996 2023-10-09T05:38:12.546 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.546 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.546 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.546 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900672+0000", 2023-10-09T05:38:12.546 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.186196107 2023-10-09T05:38:12.546 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.546 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.547 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.547 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900719+0000", 2023-10-09T05:38:12.547 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6526000000000001e-05 2023-10-09T05:38:12.547 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.547 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.547 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.547 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:35:05.900761+0000", 2023-10-09T05:38:12.547 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2231999999999998e-05 2023-10-09T05:38:12.547 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.547 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.547 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.548 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.548 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.548 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4412 6.4 6:2c929489:::benchmark_data_smithi008_295510_object4411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:38:12.548 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:09.108561+0000", 2023-10-09T05:38:12.548 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 63.387937088000001, 2023-10-09T05:38:12.548 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30418963199999999, 2023-10-09T05:38:12.548 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.548 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.548 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.548 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.548 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.549 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4412 2023-10-09T05:38:12.549 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.549 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.549 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.549 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108561+0000", 2023-10-09T05:38:12.549 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.549 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.549 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.549 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108561+0000", 2023-10-09T05:38:12.550 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.550 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.550 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.550 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.550 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108559+0000", 2023-10-09T05:38:12.550 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:38:12.550 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.550 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.550 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.550 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108583+0000", 2023-10-09T05:38:12.550 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3278e-05 2023-10-09T05:38:12.551 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.551 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.551 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.551 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108598+0000", 2023-10-09T05:38:12.551 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4980999999999999e-05 2023-10-09T05:38:12.551 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.551 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.551 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.551 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108605+0000", 2023-10-09T05:38:12.551 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4880000000000001e-06 2023-10-09T05:38:12.551 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.552 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.552 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.552 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108653+0000", 2023-10-09T05:38:12.552 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8251000000000003e-05 2023-10-09T05:38:12.552 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.552 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.552 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.552 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108787+0000", 2023-10-09T05:38:12.553 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000133879 2023-10-09T05:38:12.553 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.553 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.553 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:38:12.553 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108927+0000", 2023-10-09T05:38:12.553 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000139949 2023-10-09T05:38:12.553 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.553 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.553 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.553 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111786+0000", 2023-10-09T05:38:12.554 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002858855 2023-10-09T05:38:12.554 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.554 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.554 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.554 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.114484+0000", 2023-10-09T05:38:12.554 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026978729999999999 2023-10-09T05:38:12.554 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.554 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.554 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412641+0000", 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29815728699999999 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412707+0000", 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5531999999999995e-05 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412750+0000", 2023-10-09T05:38:12.555 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3729000000000001e-05 2023-10-09T05:38:12.556 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.556 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.556 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.556 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.556 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.556 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4420 6.4 6:26d4e1d5:::benchmark_data_smithi008_295510_object4419:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:38:12.556 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:09.111673+0000", 2023-10-09T05:38:12.556 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 63.384825333000002, 2023-10-09T05:38:12.556 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30119500100000002, 2023-10-09T05:38:12.556 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.556 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4420 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111673+0000", 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.557 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.558 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.558 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111673+0000", 2023-10-09T05:38:12.558 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.558 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.558 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.558 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.558 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111671+0000", 2023-10-09T05:38:12.558 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:38:12.558 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.558 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.558 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111697+0000", 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5582999999999999e-05 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111712+0000", 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5546999999999999e-05 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111720+0000", 2023-10-09T05:38:12.559 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8970000000000008e-06 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111810+0000", 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0062000000000006e-05 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111951+0000", 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000140662 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.560 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.561 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:38:12.561 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.112099+0000", 2023-10-09T05:38:12.561 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014773400000000001 2023-10-09T05:38:12.561 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.561 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.561 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.561 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.113038+0000", 2023-10-09T05:38:12.561 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00093915700000000003 2023-10-09T05:38:12.561 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.561 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.562 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.562 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.114491+0000", 2023-10-09T05:38:12.562 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001453378 2023-10-09T05:38:12.562 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.562 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.562 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.562 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412787+0000", 2023-10-09T05:38:12.562 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29829549100000002 2023-10-09T05:38:12.562 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.562 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.562 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412833+0000", 2023-10-09T05:38:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5770999999999999e-05 2023-10-09T05:38:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412868+0000", 2023-10-09T05:38:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5008000000000001e-05 2023-10-09T05:38:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.563 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.564 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.564 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4428 6.4 6:22fa3692:::benchmark_data_smithi008_295510_object4427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:38:12.564 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:09.116232+0000", 2023-10-09T05:38:12.564 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 63.380265836, 2023-10-09T05:38:12.564 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29675727800000001, 2023-10-09T05:38:12.564 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.564 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.564 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.564 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.564 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.564 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4428 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116232+0000", 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116232+0000", 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.565 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.566 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.566 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.566 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116231+0000", 2023-10-09T05:38:12.566 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:38:12.566 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.566 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.566 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.566 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116251+0000", 2023-10-09T05:38:12.566 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0611999999999998e-05 2023-10-09T05:38:12.566 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.566 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.567 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.567 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116265+0000", 2023-10-09T05:38:12.567 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3195e-05 2023-10-09T05:38:12.567 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.567 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.567 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.567 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116271+0000", 2023-10-09T05:38:12.567 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5230000000000001e-06 2023-10-09T05:38:12.567 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.567 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.567 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116312+0000", 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0831000000000003e-05 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116415+0000", 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010352399999999999 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116514+0000", 2023-10-09T05:38:12.568 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8212000000000001e-05 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.117005+0000", 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000491516 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.117207+0000", 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000201659 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.569 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.570 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.570 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412905+0000", 2023-10-09T05:38:12.570 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29569794399999999 2023-10-09T05:38:12.570 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.570 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.570 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.570 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412954+0000", 2023-10-09T05:38:12.570 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9382000000000003e-05 2023-10-09T05:38:12.570 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.570 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.570 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.571 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412989+0000", 2023-10-09T05:38:12.571 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5240000000000001e-05 2023-10-09T05:38:12.571 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.571 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.571 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.571 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.571 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.571 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:32763 6.1 6:82107ccb:::benchmark_data_smithi008_295510_object32762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T05:38:12.571 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:24.727348+0000", 2023-10-09T05:38:12.571 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.769150003, 2023-10-09T05:38:12.571 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19561153000000001, 2023-10-09T05:38:12.572 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.572 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.572 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 32763 2023-10-09T05:38:12.572 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.573 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.573 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.573 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.573 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727348+0000", 2023-10-09T05:38:12.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.573 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.573 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.573 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.573 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727348+0000", 2023-10-09T05:38:12.574 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.574 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.574 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.574 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.574 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727347+0000", 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727372+0000", 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5570000000000001e-05 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727395+0000", 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2707000000000001e-05 2023-10-09T05:38:12.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727403+0000", 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8339999999999999e-06 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727459+0000", 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6317000000000003e-05 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.577 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727647+0000", 2023-10-09T05:38:12.577 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018767999999999999 2023-10-09T05:38:12.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.578 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:38:12.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727813+0000", 2023-10-09T05:38:12.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016620599999999999 2023-10-09T05:38:12.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.578 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.908773+0000", 2023-10-09T05:38:12.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18095999099999999 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.908779+0000", 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7089999999999999e-06 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.922851+0000", 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014072911 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.922921+0000", 2023-10-09T05:38:12.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9387000000000005e-05 2023-10-09T05:38:12.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.922959+0000", 2023-10-09T05:38:12.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8605000000000001e-05 2023-10-09T05:38:12.580 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.580 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.581 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.581 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.581 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.581 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:32774 6.1 6:88c611f5:::benchmark_data_smithi008_295510_object32773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T05:38:12.581 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:24.727954+0000", 2023-10-09T05:38:12.581 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.768544093000003, 2023-10-09T05:38:12.581 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19793961299999999, 2023-10-09T05:38:12.581 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.581 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.581 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.581 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.582 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.582 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 32774 2023-10-09T05:38:12.582 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.582 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.582 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.582 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.582 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727954+0000", 2023-10-09T05:38:12.582 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.582 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.582 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.582 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727954+0000", 2023-10-09T05:38:12.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727953+0000", 2023-10-09T05:38:12.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:38:12.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727986+0000", 2023-10-09T05:38:12.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3031000000000003e-05 2023-10-09T05:38:12.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.727996+0000", 2023-10-09T05:38:12.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6460000000000005e-06 2023-10-09T05:38:12.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.728001+0000", 2023-10-09T05:38:12.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4240000000000001e-06 2023-10-09T05:38:12.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.729119+0000", 2023-10-09T05:38:12.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011182950000000001 2023-10-09T05:38:12.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.906554+0000", 2023-10-09T05:38:12.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.177434745 2023-10-09T05:38:12.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:38:12.586 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.906678+0000", 2023-10-09T05:38:12.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012420399999999999 2023-10-09T05:38:12.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.908803+0000", 2023-10-09T05:38:12.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021245729999999998 2023-10-09T05:38:12.587 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.908806+0000", 2023-10-09T05:38:12.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.574e-06 2023-10-09T05:38:12.587 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.925800+0000", 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016993365 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.925851+0000", 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0810000000000003e-05 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.925893+0000", 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2849000000000003e-05 2023-10-09T05:38:12.588 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:32775 6.1 6:815a2317:::benchmark_data_smithi008_295510_object32774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:24.728014+0000", 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.768484135000001, 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19798786900000001, 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.589 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.590 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.590 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 32775 2023-10-09T05:38:12.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.590 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.590 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.590 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.728014+0000", 2023-10-09T05:38:12.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.728014+0000", 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.728013+0000", 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.728033+0000", 2023-10-09T05:38:12.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9874999999999999e-05 2023-10-09T05:38:12.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.728040+0000", 2023-10-09T05:38:12.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6610000000000003e-06 2023-10-09T05:38:12.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.728044+0000", 2023-10-09T05:38:12.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8089999999999999e-06 2023-10-09T05:38:12.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.907645+0000", 2023-10-09T05:38:12.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17960104599999999 2023-10-09T05:38:12.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.907776+0000", 2023-10-09T05:38:12.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000131144 2023-10-09T05:38:12.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:38:12.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.907919+0000", 2023-10-09T05:38:12.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014355600000000001 2023-10-09T05:38:12.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.909549+0000", 2023-10-09T05:38:12.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001629651 2023-10-09T05:38:12.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.910467+0000", 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00091797299999999999 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.925928+0000", 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015461101999999999 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.925969+0000", 2023-10-09T05:38:12.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0756000000000001e-05 2023-10-09T05:38:12.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.596 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.596 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:24.926002+0000", 2023-10-09T05:38:12.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2851000000000001e-05 2023-10-09T05:38:12.596 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.596 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.596 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.597 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:50568 6.4 6:21b69bd9:::benchmark_data_smithi008_295510_object50567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-10-09T05:38:12.597 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:44.899705+0000", 2023-10-09T05:38:12.597 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 27.596792921999999, 2023-10-09T05:38:12.597 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.210408966, 2023-10-09T05:38:12.597 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.597 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.597 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.597 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.597 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.597 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 50568 2023-10-09T05:38:12.597 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.598 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.598 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.899705+0000", 2023-10-09T05:38:12.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.598 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.899705+0000", 2023-10-09T05:38:12.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.599 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.599 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.899704+0000", 2023-10-09T05:38:12.599 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:38:12.599 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.599 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.599 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.899729+0000", 2023-10-09T05:38:12.599 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5335999999999999e-05 2023-10-09T05:38:12.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.600 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.600 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.899739+0000", 2023-10-09T05:38:12.600 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.009e-05 2023-10-09T05:38:12.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.601 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.899747+0000", 2023-10-09T05:38:12.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1380000000000001e-06 2023-10-09T05:38:12.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.602 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.901441+0000", 2023-10-09T05:38:12.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001694159 2023-10-09T05:38:12.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.602 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.602 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.901560+0000", 2023-10-09T05:38:12.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000119726 2023-10-09T05:38:12.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.602 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:38:12.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.901689+0000", 2023-10-09T05:38:12.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012855700000000001 2023-10-09T05:38:12.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.905788+0000", 2023-10-09T05:38:12.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040988530000000004 2023-10-09T05:38:12.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.905802+0000", 2023-10-09T05:38:12.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4683e-05 2023-10-09T05:38:12.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.110025+0000", 2023-10-09T05:38:12.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.204222078 2023-10-09T05:38:12.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.110076+0000", 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1535e-05 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.110114+0000", 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7895000000000002e-05 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.605 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:50573 6.4 6:2efdea99:::benchmark_data_smithi008_295510_object50572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-10-09T05:38:12.606 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:44.905654+0000", 2023-10-09T05:38:12.606 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 27.590843805999999, 2023-10-09T05:38:12.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20716092899999999, 2023-10-09T05:38:12.606 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.606 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.606 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.606 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.606 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.606 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 50573 2023-10-09T05:38:12.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.606 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.905654+0000", 2023-10-09T05:38:12.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.905654+0000", 2023-10-09T05:38:12.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.905652+0000", 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.905678+0000", 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6176000000000001e-05 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.905691+0000", 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3064000000000001e-05 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.905699+0000", 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7049999999999999e-06 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.905964+0000", 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00026438600000000003 2023-10-09T05:38:12.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.906114+0000", 2023-10-09T05:38:12.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014999999999999999 2023-10-09T05:38:12.610 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:38:12.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.906296+0000", 2023-10-09T05:38:12.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018232400000000001 2023-10-09T05:38:12.610 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.109992+0000", 2023-10-09T05:38:12.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.203695874 2023-10-09T05:38:12.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.110151+0000", 2023-10-09T05:38:12.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000159724 2023-10-09T05:38:12.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.112702+0000", 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025502419999999999 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.112769+0000", 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7759e-05 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.112815+0000", 2023-10-09T05:38:12.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5589e-05 2023-10-09T05:38:12.613 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.613 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.613 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.613 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45029.0:50568 6.4 e174/162 ondisk, result = 0)", 2023-10-09T05:38:12.613 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:44.906617+0000", 2023-10-09T05:38:12.613 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 27.589880811, 2023-10-09T05:38:12.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.203503083, 2023-10-09T05:38:12.613 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.614 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:38:12.614 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.906617+0000", 2023-10-09T05:38:12.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.906617+0000", 2023-10-09T05:38:12.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.906616+0000", 2023-10-09T05:38:12.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:12.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.906620+0000", 2023-10-09T05:38:12.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8750000000000002e-06 2023-10-09T05:38:12.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.108062+0000", 2023-10-09T05:38:12.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20144171699999999 2023-10-09T05:38:12.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.108078+0000", 2023-10-09T05:38:12.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5753e-05 2023-10-09T05:38:12.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.110014+0000", 2023-10-09T05:38:12.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001936192 2023-10-09T05:38:12.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.110020+0000", 2023-10-09T05:38:12.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6210000000000001e-06 2023-10-09T05:38:12.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.618 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.618 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.110120+0000", 2023-10-09T05:38:12.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010061899999999999 2023-10-09T05:38:12.618 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.618 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.618 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.618 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45029.0:50573 6.4 e174/162 ondisk, result = 0)", 2023-10-09T05:38:12.618 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:44.909709+0000", 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 27.586789156999998, 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20028559200000001, 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.909709+0000", 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.909709+0000", 2023-10-09T05:38:12.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.909708+0000", 2023-10-09T05:38:12.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:38:12.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.909718+0000", 2023-10-09T05:38:12.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6229999999999992e-06 2023-10-09T05:38:12.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.108066+0000", 2023-10-09T05:38:12.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19834865700000001 2023-10-09T05:38:12.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.108076+0000", 2023-10-09T05:38:12.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3659999999999998e-06 2023-10-09T05:38:12.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.109977+0000", 2023-10-09T05:38:12.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001901144 2023-10-09T05:38:12.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.109985+0000", 2023-10-09T05:38:12.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6389999999999998e-06 2023-10-09T05:38:12.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.109994+0000", 2023-10-09T05:38:12.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9760000000000001e-06 2023-10-09T05:38:12.623 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.623 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.623 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45029.0:50574 6.3 e174/170 6:c3b59747:::benchmark_data_smithi008_295510_object50573:head v 174'3112, mlcod=0'0)", 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:44.910640+0000", 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 27.585857695000001, 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.209510314, 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.910640+0000", 2023-10-09T05:38:12.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.910640+0000", 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.910639+0000", 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:44.910671+0000", 2023-10-09T05:38:12.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1467999999999998e-05 2023-10-09T05:38:12.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.108063+0000", 2023-10-09T05:38:12.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.197392497 2023-10-09T05:38:12.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.108078+0000", 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4297999999999999e-05 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.108134+0000", 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6202999999999997e-05 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.108175+0000", 2023-10-09T05:38:12.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1221999999999999e-05 2023-10-09T05:38:12.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.120123+0000", 2023-10-09T05:38:12.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011947998 2023-10-09T05:38:12.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:45.120151+0000", 2023-10-09T05:38:12.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7549999999999999e-05 2023-10-09T05:38:12.628 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:52108 6.4 6:233e017b:::benchmark_data_smithi008_295510_object52107:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:46.721577+0000", 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 25.774921363000001, 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.21384238899999999, 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.629 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.630 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.630 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 52108 2023-10-09T05:38:12.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.630 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.721577+0000", 2023-10-09T05:38:12.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.721577+0000", 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.721575+0000", 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.721602+0000", 2023-10-09T05:38:12.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7353000000000001e-05 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.721614+0000", 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1511e-05 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.721622+0000", 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6109999999999996e-06 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.721802+0000", 2023-10-09T05:38:12.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000180299 2023-10-09T05:38:12.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.721939+0000", 2023-10-09T05:38:12.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013668600000000001 2023-10-09T05:38:12.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:38:12.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.722082+0000", 2023-10-09T05:38:12.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014306599999999999 2023-10-09T05:38:12.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.724556+0000", 2023-10-09T05:38:12.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002473931 2023-10-09T05:38:12.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.724604+0000", 2023-10-09T05:38:12.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8860000000000003e-05 2023-10-09T05:38:12.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.935332+0000", 2023-10-09T05:38:12.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.210727419 2023-10-09T05:38:12.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.935381+0000", 2023-10-09T05:38:12.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8916999999999999e-05 2023-10-09T05:38:12.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.935419+0000", 2023-10-09T05:38:12.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8275999999999997e-05 2023-10-09T05:38:12.636 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.636 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.636 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.636 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:52111 6.4 6:2f500dea:::benchmark_data_smithi008_295510_object52110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:46.729360+0000", 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 25.767137633000001, 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20807588699999999, 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 52111 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.729360+0000", 2023-10-09T05:38:12.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.729360+0000", 2023-10-09T05:38:12.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.729358+0000", 2023-10-09T05:38:12.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:38:12.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.729386+0000", 2023-10-09T05:38:12.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7495999999999999e-05 2023-10-09T05:38:12.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.729403+0000", 2023-10-09T05:38:12.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7121000000000001e-05 2023-10-09T05:38:12.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.729413+0000", 2023-10-09T05:38:12.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0271999999999999e-05 2023-10-09T05:38:12.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.729475+0000", 2023-10-09T05:38:12.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1365000000000006e-05 2023-10-09T05:38:12.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.729617+0000", 2023-10-09T05:38:12.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014249600000000001 2023-10-09T05:38:12.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:38:12.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.729798+0000", 2023-10-09T05:38:12.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000180643 2023-10-09T05:38:12.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.732016+0000", 2023-10-09T05:38:12.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022186720000000001 2023-10-09T05:38:12.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.734177+0000", 2023-10-09T05:38:12.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021609590000000001 2023-10-09T05:38:12.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.937343+0000", 2023-10-09T05:38:12.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20316543400000001 2023-10-09T05:38:12.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.937397+0000", 2023-10-09T05:38:12.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3822000000000001e-05 2023-10-09T05:38:12.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:46.937436+0000", 2023-10-09T05:38:12.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9657000000000003e-05 2023-10-09T05:38:12.647 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.647 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.647 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.648 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:54729 6.1 6:8aa49213:::benchmark_data_smithi008_295510_object54728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T05:38:12.648 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:49.902724+0000", 2023-10-09T05:38:12.648 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 22.593774239999998, 2023-10-09T05:38:12.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.200914342, 2023-10-09T05:38:12.648 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.648 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.649 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.649 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.649 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.649 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54729 2023-10-09T05:38:12.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.649 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.902724+0000", 2023-10-09T05:38:12.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.902724+0000", 2023-10-09T05:38:12.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.902722+0000", 2023-10-09T05:38:12.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:38:12.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.902745+0000", 2023-10-09T05:38:12.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3085e-05 2023-10-09T05:38:12.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.902756+0000", 2023-10-09T05:38:12.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0750999999999999e-05 2023-10-09T05:38:12.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.902764+0000", 2023-10-09T05:38:12.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3080000000000002e-06 2023-10-09T05:38:12.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.904844+0000", 2023-10-09T05:38:12.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002080756 2023-10-09T05:38:12.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.904968+0000", 2023-10-09T05:38:12.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012351699999999999 2023-10-09T05:38:12.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:38:12.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.905102+0000", 2023-10-09T05:38:12.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013385200000000001 2023-10-09T05:38:12.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.908204+0000", 2023-10-09T05:38:12.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031025380000000002 2023-10-09T05:38:12.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.909734+0000", 2023-10-09T05:38:12.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015302390000000001 2023-10-09T05:38:12.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.103540+0000", 2023-10-09T05:38:12.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19380562600000001 2023-10-09T05:38:12.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.103594+0000", 2023-10-09T05:38:12.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4129000000000003e-05 2023-10-09T05:38:12.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.103638+0000", 2023-10-09T05:38:12.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3869999999999998e-05 2023-10-09T05:38:12.659 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.659 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.659 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.659 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:54740 6.1 6:8c065feb:::benchmark_data_smithi008_295510_object54739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T05:38:12.659 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:49.913991+0000", 2023-10-09T05:38:12.660 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 22.582507351, 2023-10-09T05:38:12.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.206862501, 2023-10-09T05:38:12.660 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.660 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.660 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.660 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.660 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.660 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54740 2023-10-09T05:38:12.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.661 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.913991+0000", 2023-10-09T05:38:12.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.913991+0000", 2023-10-09T05:38:12.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.913989+0000", 2023-10-09T05:38:12.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:38:12.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914019+0000", 2023-10-09T05:38:12.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.027e-05 2023-10-09T05:38:12.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:12.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914039+0000", 2023-10-09T05:38:12.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0064999999999998e-05 2023-10-09T05:38:12.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914047+0000", 2023-10-09T05:38:12.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5329999999999997e-06 2023-10-09T05:38:12.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914094+0000", 2023-10-09T05:38:12.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7420000000000003e-05 2023-10-09T05:38:12.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914278+0000", 2023-10-09T05:38:12.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018404700000000001 2023-10-09T05:38:12.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:38:12.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914460+0000", 2023-10-09T05:38:12.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018127799999999999 2023-10-09T05:38:12.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.918992+0000", 2023-10-09T05:38:12.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045325060000000004 2023-10-09T05:38:12.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.919046+0000", 2023-10-09T05:38:12.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3946000000000002e-05 2023-10-09T05:38:12.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.120781+0000", 2023-10-09T05:38:12.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.201734734 2023-10-09T05:38:12.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.120813+0000", 2023-10-09T05:38:12.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2282e-05 2023-10-09T05:38:12.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.120853+0000", 2023-10-09T05:38:12.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0061000000000002e-05 2023-10-09T05:38:12.673 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.673 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.673 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.673 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:54741 6.1 6:89360e60:::benchmark_data_smithi008_295510_object54740:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T05:38:12.673 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:49.914211+0000", 2023-10-09T05:38:12.674 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 22.582286878000001, 2023-10-09T05:38:12.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20673697399999999, 2023-10-09T05:38:12.674 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:12.674 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:12.674 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:12.674 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:38:12.674 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:38:12.675 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54741 2023-10-09T05:38:12.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.675 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:12.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:12.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914211+0000", 2023-10-09T05:38:12.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:12.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914211+0000", 2023-10-09T05:38:12.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:12.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:12.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914210+0000", 2023-10-09T05:38:12.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:38:12.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:12.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914246+0000", 2023-10-09T05:38:12.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5550999999999999e-05 2023-10-09T05:38:12.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:12.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914258+0000", 2023-10-09T05:38:12.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2853e-05 2023-10-09T05:38:12.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:12.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.914266+0000", 2023-10-09T05:38:12.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4660000000000002e-06 2023-10-09T05:38:12.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.916682+0000", 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0024156960000000002 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.916841+0000", 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015898400000000001 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.916985+0000", 2023-10-09T05:38:12.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000144289 2023-10-09T05:38:12.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.919135+0000", 2023-10-09T05:38:12.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021501699999999999 2023-10-09T05:38:12.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:12.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.919146+0000", 2023-10-09T05:38:12.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1307e-05 2023-10-09T05:38:12.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:12.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.120881+0000", 2023-10-09T05:38:12.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20173503000000001 2023-10-09T05:38:12.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:12.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.120914+0000", 2023-10-09T05:38:12.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2817000000000001e-05 2023-10-09T05:38:12.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:12.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:12.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:12.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.120948+0000", 2023-10-09T05:38:12.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.383e-05 2023-10-09T05:38:12.682 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.682 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.682 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.682 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:12.682 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:12.682 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:38:12.691 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:12.745 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:12.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:12.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:12.851 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:12.863 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 66 16 72157 72141 68.2643 42.4375 0.0267443 0.0146428 2023-10-09T05:38:12.887 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:12.945 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:12.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:12.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:12.955 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:12 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14569/206235 objects degraded (7.064%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:13.052 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:13.118 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:12 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14569/206235 objects degraded (7.064%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:13.118 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:12 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14569/206235 objects degraded (7.064%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:13.120 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:13.802 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T05:38:13.867 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 67 16 72789 72773 67.8326 39.5 0.017455 0.0147367 2023-10-09T05:38:14.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:13 smithi191 ceph-mon[161669]: pgmap v473: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s; 11547/210666 objects degraded (5.481%); 2811/210666 objects misplaced (1.334%); 11 MiB/s, 171 objects/s recovering 2023-10-09T05:38:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2800687002' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:38:14.081 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:13 smithi008 ceph-mon[184418]: pgmap v473: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s; 11547/210666 objects degraded (5.481%); 2811/210666 objects misplaced (1.334%); 11 MiB/s, 171 objects/s recovering 2023-10-09T05:38:14.081 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2800687002' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:38:14.084 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:13 smithi008 ceph-mon[191227]: pgmap v473: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s; 11547/210666 objects degraded (5.481%); 2811/210666 objects misplaced (1.334%); 11 MiB/s, 171 objects/s recovering 2023-10-09T05:38:14.084 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2800687002' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:38:14.660 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:38:14.819 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:14.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:14.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:14.866 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 68 16 73482 73466 67.4722 43.3125 0.0152831 0.0148134 2023-10-09T05:38:14.933 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:14.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[184418]: failed to encode map e181 with expected crc 2023-10-09T05:38:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2800687002' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:38:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[184418]: osdmap e181: 8 total, 8 up, 7 in 2023-10-09T05:38:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[184418]: failed to encode map e181 with expected crc 2023-10-09T05:38:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[184418]: failed to encode map e181 with expected crc 2023-10-09T05:38:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[184418]: failed to encode map e181 with expected crc 2023-10-09T05:38:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[184418]: pgmap v475: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 40 MiB/s wr, 647 op/s; 11547/213075 objects degraded (5.419%); 2811/213075 objects misplaced (1.319%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T05:38:14.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[191227]: failed to encode map e181 with expected crc 2023-10-09T05:38:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2800687002' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:38:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[191227]: osdmap e181: 8 total, 8 up, 7 in 2023-10-09T05:38:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[191227]: failed to encode map e181 with expected crc 2023-10-09T05:38:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[191227]: failed to encode map e181 with expected crc 2023-10-09T05:38:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[191227]: failed to encode map e181 with expected crc 2023-10-09T05:38:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:14 smithi008 ceph-mon[191227]: pgmap v475: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 40 MiB/s wr, 647 op/s; 11547/213075 objects degraded (5.419%); 2811/213075 objects misplaced (1.319%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T05:38:15.041 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:14 smithi191 ceph-mon[161669]: failed to encode map e181 with expected crc 2023-10-09T05:38:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:14 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2800687002' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:38:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:14 smithi191 ceph-mon[161669]: osdmap e181: 8 total, 8 up, 7 in 2023-10-09T05:38:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:14 smithi191 ceph-mon[161669]: failed to encode map e181 with expected crc 2023-10-09T05:38:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:14 smithi191 ceph-mon[161669]: failed to encode map e181 with expected crc 2023-10-09T05:38:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:14 smithi191 ceph-mon[161669]: failed to encode map e181 with expected crc 2023-10-09T05:38:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:15.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:15.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:14 smithi191 ceph-mon[161669]: pgmap v475: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 40 MiB/s wr, 647 op/s; 11547/213075 objects degraded (5.419%); 2811/213075 objects misplaced (1.319%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T05:38:15.073 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T05:38:15.213 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:15.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:15.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:15.277 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:15.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:15.356 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:15.374 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:15.417 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:15.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:15.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:15.531 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:15.625 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:15.814 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:15.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:15.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:15.868 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 69 16 74115 74099 67.0679 39.5625 0.0172886 0.0149043 2023-10-09T05:38:15.920 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:16.017 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:16.172 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:16.442 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:16.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:16.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:16.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:16 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:16.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:16 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:16.552 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:16.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:16 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:16.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:16 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:16.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:16 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:16.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:16 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:16.587 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:16.866 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 70 16 74825 74809 66.7439 44.375 0.0159449 0.0149771 2023-10-09T05:38:17.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:17 smithi191 ceph-mon[161669]: pgmap v476: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 11547/217587 objects degraded (5.307%); 2811/217587 objects misplaced (1.292%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:38:17.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:17 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:17.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:17 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:17.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:17 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:17.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:17 smithi008 ceph-mon[184418]: pgmap v476: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 11547/217587 objects degraded (5.307%); 2811/217587 objects misplaced (1.292%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:38:17.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:17 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:17 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:17.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:17 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:17.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:17 smithi008 ceph-mon[191227]: pgmap v476: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 11547/217587 objects degraded (5.307%); 2811/217587 objects misplaced (1.292%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T05:38:17.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:17 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:17.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:17 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:17.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:17 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:17.725 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:17.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:17.842 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:17.866 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 71 16 75508 75492 66.4052 42.6875 0.0279534 0.0150533 2023-10-09T05:38:17.908 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:18.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 11547/217587 objects degraded (5.307%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:18.650 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 11547/217587 objects degraded (5.307%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:18.651 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 11547/217587 objects degraded (5.307%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:18.687 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:18.866 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 72 16 76150 76134 66.0404 40.125 0.0143419 0.0151365 2023-10-09T05:38:18.885 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:18.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:18.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:19.003 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:19.097 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:19.734 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:19.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:19.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:19.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:19.866 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 73 16 76806 76790 65.6975 41 0.00613473 0.0151955 2023-10-09T05:38:19.873 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:19.917 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:38:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:19.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:19.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:19 smithi008 ceph-mon[191227]: pgmap v477: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 41 MiB/s wr, 655 op/s; 11547/221973 objects degraded (5.202%); 2604/221973 objects misplaced (1.173%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T05:38:19.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:19 smithi008 ceph-mon[184418]: pgmap v477: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 41 MiB/s wr, 655 op/s; 11547/221973 objects degraded (5.202%); 2604/221973 objects misplaced (1.173%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T05:38:20.023 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:20.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:19 smithi191 ceph-mon[161669]: pgmap v477: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 41 MiB/s wr, 655 op/s; 11547/221973 objects degraded (5.202%); 2604/221973 objects misplaced (1.173%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T05:38:20.120 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:20.567 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:20.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:20.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:20.687 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:20.701 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:38:20.710 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:38:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:38:20] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.33.4" 2023-10-09T05:38:20.750 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:38:20.750 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:38:20.772 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:20.823 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:38:20.867 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 74 16 77479 77463 65.3783 42.0625 0.0420223 0.0152883 2023-10-09T05:38:20.871 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:38:20.910 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:38:20.953 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:38:20.961 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:20 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3669296065' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:38:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:20 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:38:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:20 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/337782455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:20 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3669296065' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:38:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:20 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:38:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:20 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/337782455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:20.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:20.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:20.993 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:38:21.032 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:38:21.048 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:38:21.049 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:38:21.049 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:38:21.049 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:38:21.049 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:38:21.049 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:38:21.049 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:38:21.049 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:38:21.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:20 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3669296065' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:38:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:20 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:38:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:20 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/337782455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:21.078 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:21.151 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:21.867 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 75 16 78176 78160 65.0875 43.5625 0.00851448 0.0153581 2023-10-09T05:38:21.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[184418]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 11547/226233 objects degraded (5.104%); 2604/226233 objects misplaced (1.151%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:38:21.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:21.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:38:21.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[184418]: osdmap e182: 8 total, 8 up, 7 in 2023-10-09T05:38:21.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[184418]: failed to encode map e182 with expected crc 2023-10-09T05:38:21.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[184418]: failed to encode map e182 with expected crc 2023-10-09T05:38:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[191227]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 11547/226233 objects degraded (5.104%); 2604/226233 objects misplaced (1.151%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:38:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:38:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[191227]: osdmap e182: 8 total, 8 up, 7 in 2023-10-09T05:38:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[191227]: failed to encode map e182 with expected crc 2023-10-09T05:38:21.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:21 smithi008 ceph-mon[191227]: failed to encode map e182 with expected crc 2023-10-09T05:38:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:21 smithi191 ceph-mon[161669]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 11547/226233 objects degraded (5.104%); 2604/226233 objects misplaced (1.151%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:38:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:21 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:21 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:38:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:21 smithi191 ceph-mon[161669]: osdmap e182: 8 total, 8 up, 7 in 2023-10-09T05:38:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:21 smithi191 ceph-mon[161669]: failed to encode map e182 with expected crc 2023-10-09T05:38:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:21 smithi191 ceph-mon[161669]: failed to encode map e182 with expected crc 2023-10-09T05:38:22.246 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:22.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:22.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:22.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:22.387 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:22.447 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:38:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:22.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:22.552 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:22.615 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:22.867 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 76 16 78837 78821 64.7748 41.3125 0.0100411 0.0154301 2023-10-09T05:38:22.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:22 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 11547/226233 objects degraded (5.104%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:22.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:22 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 11547/226233 objects degraded (5.104%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:23.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:22 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 11547/226233 objects degraded (5.104%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:23.246 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:23.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:23.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:23.353 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:23.451 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:23.867 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 77 16 79567 79551 64.5262 45.625 0.00301318 0.0154828 2023-10-09T05:38:23.868 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:38:23.868 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:38:23.907 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:23.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:23.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:24.011 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:24.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:23 smithi191 ceph-mon[161669]: pgmap v480: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 45 MiB/s wr, 713 op/s; 11547/230808 objects degraded (5.003%); 2309/230808 objects misplaced (1.000%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:38:24.087 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:23 smithi008 ceph-mon[184418]: pgmap v480: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 45 MiB/s wr, 713 op/s; 11547/230808 objects degraded (5.003%); 2309/230808 objects misplaced (1.000%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:38:24.087 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:23 smithi008 ceph-mon[191227]: pgmap v480: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 45 MiB/s wr, 713 op/s; 11547/230808 objects degraded (5.003%); 2309/230808 objects misplaced (1.000%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:38:24.098 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:24.867 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 78 16 80311 80295 64.2952 46.5 0.0037243 0.0155462 2023-10-09T05:38:24.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:24 smithi008 ceph-mon[184418]: pgmap v481: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 42 MiB/s wr, 674 op/s; 11547/233313 objects degraded (4.949%); 2309/233313 objects misplaced (0.990%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:38:24.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:24 smithi008 ceph-mon[191227]: pgmap v481: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 42 MiB/s wr, 674 op/s; 11547/233313 objects degraded (4.949%); 2309/233313 objects misplaced (0.990%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:38:25.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:24 smithi191 ceph-mon[161669]: pgmap v481: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 42 MiB/s wr, 674 op/s; 11547/233313 objects degraded (4.949%); 2309/233313 objects misplaced (0.990%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:38:25.346 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:25.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:25.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:25.393 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:38:25.466 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:25.512 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:38:25.581 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:25.816 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:25.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:25.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:25.867 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 79 16 80907 80891 63.953 37.25 0.0317298 0.0156304 2023-10-09T05:38:25.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:25.965 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:26.020 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:26.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:26.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:26.157 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:26.210 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:38:26.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:26.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:26.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:26.337 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:26.345 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:26.397 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:38:26.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:26.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:26.454 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:26.504 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:26.583 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:26.762 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:26.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:26.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:26.867 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:38:26.868117+0000 min lat: 0.00103937 max lat: 6.04263 avg lat: 0.0156897 2023-10-09T05:38:26.867 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:38:26.868 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 80 16 81618 81602 63.7091 44.4375 0.0274289 0.0156897 2023-10-09T05:38:26.877 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:26.912 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:26.970 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:26.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:26.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:27.076 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:27.160 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:27.175 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T05:38:27.219 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:38:27.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:27 smithi191 ceph-mon[161669]: pgmap v482: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 674 op/s; 11547/237819 objects degraded (4.855%); 2309/237819 objects misplaced (0.971%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:38:27.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1669705570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:38:27.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:27 smithi008 ceph-mon[184418]: pgmap v482: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 674 op/s; 11547/237819 objects degraded (4.855%); 2309/237819 objects misplaced (0.971%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:38:27.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1669705570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:38:27.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:27 smithi008 ceph-mon[191227]: pgmap v482: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 674 op/s; 11547/237819 objects degraded (4.855%); 2309/237819 objects misplaced (0.971%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:38:27.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1669705570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:38:27.870 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 81 16 82299 82283 63.446 42.5625 0.0219706 0.0157558 2023-10-09T05:38:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[184418]: failed to encode map e183 with expected crc 2023-10-09T05:38:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[184418]: failed to encode map e183 with expected crc 2023-10-09T05:38:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1669705570' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:38:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[184418]: osdmap e183: 8 total, 8 up, 7 in 2023-10-09T05:38:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[184418]: failed to encode map e183 with expected crc 2023-10-09T05:38:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 11547/237819 objects degraded (4.855%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:28.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[191227]: failed to encode map e183 with expected crc 2023-10-09T05:38:28.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[191227]: failed to encode map e183 with expected crc 2023-10-09T05:38:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1669705570' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:38:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[191227]: osdmap e183: 8 total, 8 up, 7 in 2023-10-09T05:38:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[191227]: failed to encode map e183 with expected crc 2023-10-09T05:38:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 11547/237819 objects degraded (4.855%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:28.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:28 smithi191 ceph-mon[161669]: failed to encode map e183 with expected crc 2023-10-09T05:38:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:28 smithi191 ceph-mon[161669]: failed to encode map e183 with expected crc 2023-10-09T05:38:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1669705570' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:38:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:28 smithi191 ceph-mon[161669]: osdmap e183: 8 total, 8 up, 7 in 2023-10-09T05:38:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:28 smithi191 ceph-mon[161669]: failed to encode map e183 with expected crc 2023-10-09T05:38:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 11547/237819 objects degraded (4.855%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:28.636 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:28.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:28.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:28.744 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:28.772 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:28.870 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 82 16 82843 82827 63.0871 34 0.00524235 0.0158453 2023-10-09T05:38:29.406 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T05:38:29.666 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:29.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:29.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:29.804 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:29.869 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:29.870 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 83 16 83549 83533 62.8587 44.125 0.0253289 0.0159029 2023-10-09T05:38:29.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:29.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:29.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:29 smithi008 ceph-mon[184418]: pgmap v484: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 673 op/s; 11547/242403 objects degraded (4.764%); 2033/242403 objects misplaced (0.839%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:38:29.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:29.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:29 smithi008 ceph-mon[191227]: pgmap v484: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 673 op/s; 11547/242403 objects degraded (4.764%); 2033/242403 objects misplaced (0.839%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:38:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:29.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:29.975 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:30.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:29 smithi191 ceph-mon[161669]: pgmap v484: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 673 op/s; 11547/242403 objects degraded (4.764%); 2033/242403 objects misplaced (0.839%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:38:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:38:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:30.050 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:30.329 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:30.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:30.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:30.447 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:30.484 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:30.533 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:30.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:30.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:30.626 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:38:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:38:30] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T05:38:30.641 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:30.757 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:30.824 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:30.870 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 84 16 84288 84272 62.6604 46.1875 0.0306095 0.0159531 2023-10-09T05:38:30.886 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:30.933 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:38:30.974 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:38:31.020 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:38:31.043 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:31.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:31.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:31.080 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:38:31.113 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:38:31.161 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:38:31.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:31.192 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:38:31.213 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:31.236 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:38:31.265 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:38:31.265 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:38:31.265 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:38:31.265 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:38:31.265 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:38:31.265 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:38:31.265 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:38:31.266 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:38:31.268 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:38:31.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:31.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:31.373 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:31.411 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:31.464 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:31.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:31.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:31.495 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:38:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:38:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:38:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:38:31.496 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:38:31.569 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:31.661 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:31 smithi191 ceph-mon[161669]: pgmap v485: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 686 op/s; 11547/246756 objects degraded (4.680%); 2033/246756 objects misplaced (0.824%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:38:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:31 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/892559450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:31.873 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 85 16 84963 84947 62.4182 42.1875 0.0298246 0.0160153 2023-10-09T05:38:31.952 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:31 smithi008 ceph-mon[184418]: pgmap v485: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 686 op/s; 11547/246756 objects degraded (4.680%); 2033/246756 objects misplaced (0.824%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:38:31.953 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:31 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/892559450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:31.953 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:31 smithi008 ceph-mon[191227]: pgmap v485: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 686 op/s; 11547/246756 objects degraded (4.680%); 2033/246756 objects misplaced (0.824%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:38:31.953 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:31 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/892559450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:32.875 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 86 16 85753 85737 62.2645 49.375 0.0106785 0.0160543 2023-10-09T05:38:32.896 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:32.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:32.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:32.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:32 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 11547/246756 objects degraded (4.680%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:32.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:32 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 11547/246756 objects degraded (4.680%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:33.007 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:33.036 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:32 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 11547/246756 objects degraded (4.680%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:33.039 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:33.101 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:38:33.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:33.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:33.209 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:33.276 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:33.875 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 87 16 86408 86392 62.0194 40.9375 0.0101715 0.0161193 2023-10-09T05:38:33.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:33 smithi008 ceph-mon[184418]: pgmap v486: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s; 11547/251088 objects degraded (4.599%); 1721/251088 objects misplaced (0.685%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:38:33.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:33 smithi008 ceph-mon[191227]: pgmap v486: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s; 11547/251088 objects degraded (4.599%); 1721/251088 objects misplaced (0.685%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:38:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:33 smithi191 ceph-mon[161669]: pgmap v486: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s; 11547/251088 objects degraded (4.599%); 1721/251088 objects misplaced (0.685%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:38:34.123 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:34.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:34.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:34.222 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:38:34.234 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:34.265 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:34.331 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:38:34.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:34.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:34.439 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:34.479 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:34.535 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:38:34.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:34.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:34.643 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:34.688 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:34.875 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 88 16 87083 87067 61.7942 42.1875 0.00935629 0.0161694 2023-10-09T05:38:35.189 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:35.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:35.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:35.302 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:35.444 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:35.842 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:35.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:35.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:35.885 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 89 16 87810 87794 61.6037 45.4375 0.0299125 0.0162267 2023-10-09T05:38:35.954 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:36.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:35 smithi191 ceph-mon[161669]: pgmap v487: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 682 op/s; 11547/253794 objects degraded (4.550%); 1721/253794 objects misplaced (0.678%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:38:36.063 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:36.086 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:35 smithi008 ceph-mon[184418]: pgmap v487: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 682 op/s; 11547/253794 objects degraded (4.550%); 1721/253794 objects misplaced (0.678%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:38:36.086 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:35 smithi008 ceph-mon[191227]: pgmap v487: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 682 op/s; 11547/253794 objects degraded (4.550%); 1721/253794 objects misplaced (0.678%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:38:36.097 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:36.179 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:38:36.246 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:36.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:36.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:36.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:36.382 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:36.438 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:36.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:36.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:36.543 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:36.631 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:36.750 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1466823482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:36.750 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2891704556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:38:36.750 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:36 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:38:36.790 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:38:36.792 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T05:38:36.797 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:36.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:36.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:36.886 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total time run: 90.0088 2023-10-09T05:38:36.886 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total writes made: 88360 2023-10-09T05:38:36.886 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Write size: 65536 2023-10-09T05:38:36.886 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object size: 65536 2023-10-09T05:38:36.886 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Bandwidth (MB/sec): 61.3551 2023-10-09T05:38:36.886 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Bandwidth: 34.2281 2023-10-09T05:38:36.886 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max bandwidth (MB/sec): 168.688 2023-10-09T05:38:36.886 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min bandwidth (MB/sec): 0 2023-10-09T05:38:36.886 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average IOPS: 981 2023-10-09T05:38:36.887 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev IOPS: 547.662 2023-10-09T05:38:36.887 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max IOPS: 2699 2023-10-09T05:38:36.887 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min IOPS: 0 2023-10-09T05:38:36.887 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average Latency(s): 0.0162946 2023-10-09T05:38:36.887 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Latency(s): 0.0637659 2023-10-09T05:38:36.887 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max latency(s): 6.04263 2023-10-09T05:38:36.887 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min latency(s): 0.00103937 2023-10-09T05:38:36.893 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:38:36.907 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:36.999 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1466823482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:37.000 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2891704556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:38:37.000 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:36 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:38:37.000 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1466823482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:38:37.000 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2891704556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:38:37.000 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:36 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:38:37.006 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:37 smithi191 ceph-mon[161669]: pgmap v488: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 11547/258327 objects degraded (4.470%); 1721/258327 objects misplaced (0.666%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:38:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:37 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:37 smithi191 ceph-mon[161669]: failed to encode map e184 with expected crc 2023-10-09T05:38:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:37 smithi191 ceph-mon[161669]: failed to encode map e184 with expected crc 2023-10-09T05:38:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:37 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1466823482' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:38:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:37 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:38:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:37 smithi191 ceph-mon[161669]: osdmap e184: 8 total, 8 up, 7 in 2023-10-09T05:38:38.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:37 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 11547/258327 objects degraded (4.470%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:38.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[184418]: pgmap v488: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 11547/258327 objects degraded (4.470%); 1721/258327 objects misplaced (0.666%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:38:38.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:38.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[184418]: failed to encode map e184 with expected crc 2023-10-09T05:38:38.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[184418]: failed to encode map e184 with expected crc 2023-10-09T05:38:38.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1466823482' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:38:38.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:38:38.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[184418]: osdmap e184: 8 total, 8 up, 7 in 2023-10-09T05:38:38.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 11547/258327 objects degraded (4.470%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:38.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[191227]: pgmap v488: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 11547/258327 objects degraded (4.470%); 1721/258327 objects misplaced (0.666%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T05:38:38.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:38.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[191227]: failed to encode map e184 with expected crc 2023-10-09T05:38:38.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[191227]: failed to encode map e184 with expected crc 2023-10-09T05:38:38.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1466823482' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:38:38.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:38:38.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[191227]: osdmap e184: 8 total, 8 up, 7 in 2023-10-09T05:38:38.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:37 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 11547/258327 objects degraded (4.470%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:38:38.304 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:38.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:38.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:38.414 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:38.444 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:38.509 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:38.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:38.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:38.616 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:38.695 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:38 smithi008 ceph-mon[184418]: pgmap v490: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 682 op/s; 11547/262161 objects degraded (4.405%); 1395/262161 objects misplaced (0.532%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:38:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:38 smithi008 ceph-mon[191227]: pgmap v490: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 682 op/s; 11547/262161 objects degraded (4.405%); 1395/262161 objects misplaced (0.532%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:38:39.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:38 smithi191 ceph-mon[161669]: pgmap v490: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 682 op/s; 11547/262161 objects degraded (4.405%); 1395/262161 objects misplaced (0.532%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:38:40.013 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T05:38:40.016 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:40.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:40.126 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:40.268 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:40.433 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:40.510 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:40.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:40.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:40.573 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:38:40.573 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:38:40.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:38:40.574 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:38:40.574 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.574 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54034 5.6 5:6bcc9ee5:::benchmark_data_smithi008_238708_object54033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:38:40.574 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:38:40.574 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 320.04994056100003, 2023-10-09T05:38:40.574 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0234612670000001, 2023-10-09T05:38:40.574 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.574 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:38:40.574 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.574 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.574 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54034 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517446+0000", 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0636999999999999e-05 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517453+0000", 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5390000000000004e-06 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517458+0000", 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0839999999999999e-06 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.521341+0000", 2023-10-09T05:38:40.577 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038830990000000001 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.521502+0000", 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016102400000000001 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.521674+0000", 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017155600000000001 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.524020+0000", 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002346295 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.529910+0000", 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058903289999999997 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541133+0000", 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011222492000000001 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541145+0000", 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1871999999999999e-05 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540792+0000", 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99964734499999996 2023-10-09T05:38:40.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.581 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.581 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.581 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540887+0000", 2023-10-09T05:38:40.581 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5445999999999994e-05 2023-10-09T05:38:40.581 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.581 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.581 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.581 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.581 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.581 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54039 5.6 5:6a5b7d04:::benchmark_data_smithi008_238708_object54038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:38:40.581 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.527531+0000", 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 320.03983604899997, 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.017534768, 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54039 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.582 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527531+0000", 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527531+0000", 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527529+0000", 2023-10-09T05:38:40.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527558+0000", 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9082e-05 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527573+0000", 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5038000000000001e-05 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527583+0000", 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3249999999999997e-06 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527639+0000", 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6297e-05 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527793+0000", 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015416599999999999 2023-10-09T05:38:40.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527952+0000", 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015940799999999999 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.530713+0000", 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002760878 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541193+0000", 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010480041000000001 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541203+0000", 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1980000000000007e-06 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.544999+0000", 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.00379627 2023-10-09T05:38:40.587 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545065+0000", 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6607999999999997e-05 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54041 5.4 5:2173e5b4:::benchmark_data_smithi008_238708_object54040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e112)", 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.535206+0000", 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 320.032160487, 2023-10-09T05:38:40.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.006119118, 2023-10-09T05:38:40.589 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.589 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:38:40.589 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.589 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.589 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.589 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54041 2023-10-09T05:38:40.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.589 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.589 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.589 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535206+0000", 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535206+0000", 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535205+0000", 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535227+0000", 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2062999999999999e-05 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535240+0000", 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2991e-05 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535248+0000", 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7789999999999996e-06 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535783+0000", 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000534484 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535798+0000", 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5403000000000001e-05 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540201+0000", 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0044033379999999 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.541325+0000", 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001123906 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54046 5.4 5:2050c775:::benchmark_data_smithi008_238708_object54045:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e112)", 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 320.03210733899999, 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0145929970000001, 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54046 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.594 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:38:40.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:40.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.596 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.596 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535285+0000", 2023-10-09T05:38:40.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6137999999999999e-05 2023-10-09T05:38:40.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.596 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.596 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535294+0000", 2023-10-09T05:38:40.597 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9299999999999992e-06 2023-10-09T05:38:40.597 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.597 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.597 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.597 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535302+0000", 2023-10-09T05:38:40.597 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0250000000000002e-06 2023-10-09T05:38:40.597 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.598 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535822+0000", 2023-10-09T05:38:40.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052059000000000003 2023-10-09T05:38:40.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535829+0000", 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5420000000000002e-06 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.541576+0000", 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.005747071 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.599 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.541934+0000", 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00035752700000000001 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.542079+0000", 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014546200000000001 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549695+0000", 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0076156389999999996 2023-10-09T05:38:40.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549765+0000", 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0362000000000002e-05 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549796+0000", 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1239999999999999e-05 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.602 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549852+0000", 2023-10-09T05:38:40.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6103000000000001e-05 2023-10-09T05:38:40.602 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.602 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.602 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.602 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54078 5.4 5:286f5610:::benchmark_data_smithi008_238708_object54077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 319.98603657699999, 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96860710699999997, 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54078 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:40.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581351+0000", 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1622000000000001e-05 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581359+0000", 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5129999999999999e-06 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581366+0000", 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8000000000000001e-06 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581421+0000", 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5843e-05 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581430+0000", 2023-10-09T05:38:40.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7530000000000007e-06 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543690+0000", 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96226007599999996 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543809+0000", 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000118658 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543945+0000", 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013617699999999999 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549727+0000", 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0057819430000000003 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549865+0000", 2023-10-09T05:38:40.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013792899999999999 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549891+0000", 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6105000000000002e-05 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549937+0000", 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6213e-05 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.609 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.610 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.610 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54079 5.4 5:29366199:::benchmark_data_smithi008_238708_object54078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:38:40.610 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.582855+0000", 2023-10-09T05:38:40.610 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 319.98451169100002, 2023-10-09T05:38:40.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96716001200000001, 2023-10-09T05:38:40.610 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.610 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.610 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.610 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.610 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54079 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582855+0000", 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582855+0000", 2023-10-09T05:38:40.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582854+0000", 2023-10-09T05:38:40.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:38:40.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582876+0000", 2023-10-09T05:38:40.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1637999999999998e-05 2023-10-09T05:38:40.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582891+0000", 2023-10-09T05:38:40.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5489999999999999e-05 2023-10-09T05:38:40.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582899+0000", 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5950000000000001e-06 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582953+0000", 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4355999999999999e-05 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582962+0000", 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9530000000000005e-06 2023-10-09T05:38:40.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545663+0000", 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96270056100000001 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545753+0000", 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0152e-05 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:38:40.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545860+0000", 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000107358 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549748+0000", 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038876010000000001 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549948+0000", 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020022200000000001 2023-10-09T05:38:40.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549973+0000", 2023-10-09T05:38:40.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5219e-05 2023-10-09T05:38:40.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550015+0000", 2023-10-09T05:38:40.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1847000000000001e-05 2023-10-09T05:38:40.617 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.618 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.618 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.618 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54081 5.4 5:25d3ef94:::benchmark_data_smithi008_238708_object54080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:38:40.618 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:38:40.618 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 319.98400237099997, 2023-10-09T05:38:40.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96692271200000002, 2023-10-09T05:38:40.618 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.618 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54081 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583385+0000", 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1100000000000001e-05 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583399+0000", 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4654000000000001e-05 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583406+0000", 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1250000000000004e-06 2023-10-09T05:38:40.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583456+0000", 2023-10-09T05:38:40.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9496000000000002e-05 2023-10-09T05:38:40.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583464+0000", 2023-10-09T05:38:40.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9300000000000003e-06 2023-10-09T05:38:40.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547622+0000", 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96415766000000003 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547731+0000", 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000109104 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547847+0000", 2023-10-09T05:38:40.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000115983 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550186+0000", 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023388979999999998 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550198+0000", 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2286000000000001e-05 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550244+0000", 2023-10-09T05:38:40.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6045999999999999e-05 2023-10-09T05:38:40.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550287+0000", 2023-10-09T05:38:40.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.32e-05 2023-10-09T05:38:40.625 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.625 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.625 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54082 5.d 5:bd5f0afe:::benchmark_data_smithi008_238708_object54081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 319.98183626100001, 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96213848199999996, 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.626 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54082 2023-10-09T05:38:40.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.627 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:38:40.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:38:40.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585549+0000", 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8794999999999999e-05 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585559+0000", 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0594999999999999e-05 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585564+0000", 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1019999999999996e-06 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585609+0000", 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4693000000000001e-05 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585619+0000", 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9839999999999996e-06 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540181+0000", 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.954562456 2023-10-09T05:38:40.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540364+0000", 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000182428 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540531+0000", 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016716999999999999 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543639+0000", 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003108235 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547559+0000", 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039201629999999999 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547601+0000", 2023-10-09T05:38:40.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1631999999999997e-05 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547669+0000", 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7916000000000005e-05 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98105 5.8 5:13fcc3a0:::benchmark_data_smithi008_238708_object98104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695499+0000", 2023-10-09T05:38:40.634 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 279.87186768999999, 2023-10-09T05:38:40.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93824070100000001, 2023-10-09T05:38:40.634 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.634 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:38:40.634 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.634 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.634 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.634 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98105 2023-10-09T05:38:40.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695499+0000", 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695499+0000", 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695497+0000", 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695527+0000", 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9377999999999999e-05 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695541+0000", 2023-10-09T05:38:40.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4511e-05 2023-10-09T05:38:40.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695552+0000", 2023-10-09T05:38:40.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0711e-05 2023-10-09T05:38:40.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881129+0000", 2023-10-09T05:38:40.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18557714 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881142+0000", 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2874000000000001e-05 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633642+0000", 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75249991800000005 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633740+0000", 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8040000000000003e-05 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98114 5.6 5:639cccb8:::benchmark_data_smithi008_238708_object98113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695561+0000", 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 279.871806096, 2023-10-09T05:38:40.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94775633000000004, 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98114 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695561+0000", 2023-10-09T05:38:40.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695561+0000", 2023-10-09T05:38:40.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695560+0000", 2023-10-09T05:38:40.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:40.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695587+0000", 2023-10-09T05:38:40.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6925000000000001e-05 2023-10-09T05:38:40.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695596+0000", 2023-10-09T05:38:40.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8750000000000006e-06 2023-10-09T05:38:40.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695603+0000", 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.815e-06 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887612+0000", 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19200974600000001 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887622+0000", 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3079999999999997e-06 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.626477+0000", 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73885584199999998 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627930+0000", 2023-10-09T05:38:40.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014526540000000001 2023-10-09T05:38:40.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.643236+0000", 2023-10-09T05:38:40.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015305859999999999 2023-10-09T05:38:40.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.643317+0000", 2023-10-09T05:38:40.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0939999999999994e-05 2023-10-09T05:38:40.646 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.646 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.646 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.646 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98120 5.18 5:1c597501:::benchmark_data_smithi008_238708_object98119:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:38:40.646 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695612+0000", 2023-10-09T05:38:40.646 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 279.87175469599998, 2023-10-09T05:38:40.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96914215000000004, 2023-10-09T05:38:40.647 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.647 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:38:40.647 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.647 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.647 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.647 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98120 2023-10-09T05:38:40.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.647 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695612+0000", 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695612+0000", 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695611+0000", 2023-10-09T05:38:40.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695637+0000", 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5809999999999999e-05 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695646+0000", 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.303e-06 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695652+0000", 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3300000000000004e-06 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881169+0000", 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18551722700000001 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881176+0000", 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1720000000000001e-06 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.658602+0000", 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77742560599999999 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659106+0000", 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000503821 2023-10-09T05:38:40.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.664640+0000", 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0055338050000000001 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.664754+0000", 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114644 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.652 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.653 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98126 5.8 5:11d9570b:::benchmark_data_smithi008_238708_object98125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:38:40.653 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695658+0000", 2023-10-09T05:38:40.653 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 279.87170887500002, 2023-10-09T05:38:40.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96035853800000004, 2023-10-09T05:38:40.653 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.653 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.653 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.653 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.653 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98126 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695658+0000", 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695658+0000", 2023-10-09T05:38:40.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695657+0000", 2023-10-09T05:38:40.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:40.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695683+0000", 2023-10-09T05:38:40.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5728000000000002e-05 2023-10-09T05:38:40.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695691+0000", 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9869999999999999e-06 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695697+0000", 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9120000000000003e-06 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881200+0000", 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.185503105 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881210+0000", 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4259999999999992e-06 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633764+0000", 2023-10-09T05:38:40.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75255434799999998 2023-10-09T05:38:40.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633873+0000", 2023-10-09T05:38:40.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010911700000000001 2023-10-09T05:38:40.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:38:40.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.634034+0000", 2023-10-09T05:38:40.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000161149 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.651360+0000", 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017325393000000001 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.654621+0000", 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032617940000000002 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.655917+0000", 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001295678 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.655974+0000", 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6629999999999998e-05 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.656016+0000", 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2697999999999997e-05 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98127 5.4 5:253a6397:::benchmark_data_smithi008_238708_object98126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695703+0000", 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 279.87166386500002, 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.927888991, 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.661 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98127 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695703+0000", 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695703+0000", 2023-10-09T05:38:40.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695702+0000", 2023-10-09T05:38:40.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:40.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695732+0000", 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9261999999999999e-05 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695740+0000", 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1280000000000008e-06 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695746+0000", 2023-10-09T05:38:40.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1310000000000003e-06 2023-10-09T05:38:40.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.866359+0000", 2023-10-09T05:38:40.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17061283099999999 2023-10-09T05:38:40.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.866369+0000", 2023-10-09T05:38:40.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.043e-05 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.604226+0000", 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.737856874 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.604398+0000", 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017228299999999999 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:38:40.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.604580+0000", 2023-10-09T05:38:40.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000182213 2023-10-09T05:38:40.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.616540+0000", 2023-10-09T05:38:40.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011959505 2023-10-09T05:38:40.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.623441+0000", 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0069005589999999997 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.623503+0000", 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2378000000000005e-05 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.623592+0000", 2023-10-09T05:38:40.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8899000000000004e-05 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98129 5.18 5:1d3b051a:::benchmark_data_smithi008_238708_object98128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695754+0000", 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 279.871612825, 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97840130400000003, 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.669 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98129 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695754+0000", 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695754+0000", 2023-10-09T05:38:40.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695753+0000", 2023-10-09T05:38:40.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:40.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695781+0000", 2023-10-09T05:38:40.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7693000000000001e-05 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695789+0000", 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9589999999999995e-06 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695795+0000", 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4239999999999998e-06 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881231+0000", 2023-10-09T05:38:40.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18543580200000001 2023-10-09T05:38:40.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881238+0000", 2023-10-09T05:38:40.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8290000000000002e-06 2023-10-09T05:38:40.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659365+0000", 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77812663800000004 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659507+0000", 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014198800000000001 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659616+0000", 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010964 2023-10-09T05:38:40.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.663796+0000", 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041793669999999998 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.664218+0000", 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00042226699999999999 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.674041+0000", 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0098232590000000009 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.674109+0000", 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8119000000000003e-05 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.674155+0000", 2023-10-09T05:38:40.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5831999999999998e-05 2023-10-09T05:38:40.677 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.677 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.677 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.677 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98131 5.6 5:62d535ca:::benchmark_data_smithi008_238708_object98130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-10-09T05:38:40.677 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.711769+0000", 2023-10-09T05:38:40.677 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 279.85559769899999, 2023-10-09T05:38:40.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92605176700000003, 2023-10-09T05:38:40.677 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.677 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.678 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.678 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.678 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.678 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98131 2023-10-09T05:38:40.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.678 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711769+0000", 2023-10-09T05:38:40.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711769+0000", 2023-10-09T05:38:40.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711766+0000", 2023-10-09T05:38:40.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999967 2023-10-09T05:38:40.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711806+0000", 2023-10-09T05:38:40.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0120000000000002e-05 2023-10-09T05:38:40.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711836+0000", 2023-10-09T05:38:40.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0463e-05 2023-10-09T05:38:40.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711854+0000", 2023-10-09T05:38:40.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7082999999999999e-05 2023-10-09T05:38:40.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.890394+0000", 2023-10-09T05:38:40.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17854087599999999 2023-10-09T05:38:40.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.890407+0000", 2023-10-09T05:38:40.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2317000000000001e-05 2023-10-09T05:38:40.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.628026+0000", 2023-10-09T05:38:40.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73761953099999999 2023-10-09T05:38:40.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.628147+0000", 2023-10-09T05:38:40.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012068700000000001 2023-10-09T05:38:40.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:38:40.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.628304+0000", 2023-10-09T05:38:40.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015666500000000001 2023-10-09T05:38:40.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633549+0000", 2023-10-09T05:38:40.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0052457479999999997 2023-10-09T05:38:40.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637688+0000", 2023-10-09T05:38:40.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041385900000000002 2023-10-09T05:38:40.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637744+0000", 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6211000000000001e-05 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637821+0000", 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6623000000000001e-05 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.687 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98136 5.16 5:6f5da42e:::benchmark_data_smithi008_238708_object98135:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-10-09T05:38:40.687 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.790198+0000", 2023-10-09T05:38:40.687 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 279.77716892900003, 2023-10-09T05:38:40.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85936307599999995, 2023-10-09T05:38:40.687 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.687 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.687 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.687 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.687 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.687 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98136 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790198+0000", 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790198+0000", 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790195+0000", 2023-10-09T05:38:40.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:38:40.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790227+0000", 2023-10-09T05:38:40.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2187999999999997e-05 2023-10-09T05:38:40.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790252+0000", 2023-10-09T05:38:40.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4901000000000001e-05 2023-10-09T05:38:40.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790264+0000", 2023-10-09T05:38:40.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2323e-05 2023-10-09T05:38:40.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887643+0000", 2023-10-09T05:38:40.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.097379027000000007 2023-10-09T05:38:40.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887650+0000", 2023-10-09T05:38:40.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4860000000000002e-06 2023-10-09T05:38:40.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.645624+0000", 2023-10-09T05:38:40.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75797419499999996 2023-10-09T05:38:40.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.645742+0000", 2023-10-09T05:38:40.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000117934 2023-10-09T05:38:40.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.645860+0000", 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000117945 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.647879+0000", 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020193310000000001 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.649421+0000", 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015419749999999999 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.649474+0000", 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3298999999999999e-05 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.649561+0000", 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6372000000000006e-05 2023-10-09T05:38:40.694 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119015 5.1d 5:b91a094f:::benchmark_data_smithi008_238708_object879:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.007040+0000", 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 256.56032683900003, 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.032601662, 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.695 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.696 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.696 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119015 2023-10-09T05:38:40.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.696 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007040+0000", 2023-10-09T05:38:40.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007040+0000", 2023-10-09T05:38:40.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007038+0000", 2023-10-09T05:38:40.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:38:40.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007048+0000", 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6420000000000002e-06 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007071+0000", 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3663999999999999e-05 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007081+0000", 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8069999999999994e-06 2023-10-09T05:38:40.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201929+0000", 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19484790399999999 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201935+0000", 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8259999999999998e-06 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.039550+0000", 2023-10-09T05:38:40.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83761566499999995 2023-10-09T05:38:40.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.039642+0000", 2023-10-09T05:38:40.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1113999999999995e-05 2023-10-09T05:38:40.700 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.700 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.700 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119017 5.d 5:b096558c:::benchmark_data_smithi008_238708_object881:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 256.56027542499999, 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.010292135, 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.701 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119017 2023-10-09T05:38:40.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.702 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:38:40.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:38:40.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007094+0000", 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7e-06 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007099+0000", 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5899999999999998e-06 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007104+0000", 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7180000000000004e-06 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201969+0000", 2023-10-09T05:38:40.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19486541099999999 2023-10-09T05:38:40.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201973+0000", 2023-10-09T05:38:40.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1320000000000004e-06 2023-10-09T05:38:40.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011724+0000", 2023-10-09T05:38:40.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.80975027700000002 2023-10-09T05:38:40.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:40.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011860+0000", 2023-10-09T05:38:40.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013602500000000001 2023-10-09T05:38:40.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:38:40.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011985+0000", 2023-10-09T05:38:40.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012507600000000001 2023-10-09T05:38:40.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.016288+0000", 2023-10-09T05:38:40.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0043027300000000003 2023-10-09T05:38:40.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.017244+0000", 2023-10-09T05:38:40.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095609500000000001 2023-10-09T05:38:40.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.017296+0000", 2023-10-09T05:38:40.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2408999999999999e-05 2023-10-09T05:38:40.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.017383+0000", 2023-10-09T05:38:40.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7373999999999994e-05 2023-10-09T05:38:40.710 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.710 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.710 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.711 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119025 5.1b 5:dce538de:::benchmark_data_smithi008_238708_object889:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:38:40.711 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:38:40.711 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 256.560258412, 2023-10-09T05:38:40.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.063606593, 2023-10-09T05:38:40.711 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.711 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.711 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.711 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.711 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.711 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119025 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007110+0000", 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9549999999999999e-06 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007117+0000", 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5860000000000001e-06 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007121+0000", 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.425e-06 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.226628+0000", 2023-10-09T05:38:40.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.21950726800000001 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.226637+0000", 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7229999999999993e-06 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.063102+0000", 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83646540999999996 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.063229+0000", 2023-10-09T05:38:40.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012684600000000001 2023-10-09T05:38:40.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:38:40.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.063346+0000", 2023-10-09T05:38:40.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011715899999999999 2023-10-09T05:38:40.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.065378+0000", 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020315820000000001 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070208+0000", 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048300499999999998 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070644+0000", 2023-10-09T05:38:40.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00043635199999999999 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070683+0000", 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8967000000000001e-05 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070715+0000", 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1566000000000002e-05 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.718 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119036 5.a 5:5126f622:::benchmark_data_smithi008_238708_object900:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.063601+0000", 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 256.50376535100003, 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.98084244200000004, 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:38:40.719 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119036 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063601+0000", 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063601+0000", 2023-10-09T05:38:40.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063599+0000", 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063609+0000", 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5370000000000003e-06 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:38:40.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063623+0000", 2023-10-09T05:38:40.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4154e-05 2023-10-09T05:38:40.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:38:40.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063633+0000", 2023-10-09T05:38:40.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0023999999999999e-05 2023-10-09T05:38:40.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.162124+0000", 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.098490792999999993 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.162139+0000", 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4732000000000001e-05 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011770+0000", 2023-10-09T05:38:40.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.849631256 2023-10-09T05:38:40.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:38:40.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011877+0000", 2023-10-09T05:38:40.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010692999999999999 2023-10-09T05:38:40.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:38:40.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011982+0000", 2023-10-09T05:38:40.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010477 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.016352+0000", 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004370624 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.022811+0000", 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064583419999999997 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.044346+0000", 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021535813000000001 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.044401+0000", 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4834999999999999e-05 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:38:40.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.044444+0000", 2023-10-09T05:38:40.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2522999999999999e-05 2023-10-09T05:38:40.727 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.727 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.727 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.727 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:38:40.727 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:38:40.727 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:38:40.733 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:40.801 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:40.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:40.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:40.904 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:38:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:38:40] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T05:38:40.906 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:40.999 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:41.169 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:41.223 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:41 smithi008 ceph-mon[184418]: pgmap v491: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 37 MiB/s wr, 696 op/s; 11547/261432 objects degraded (4.417%); 1395/261432 objects misplaced (0.534%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:38:41.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:41 smithi008 ceph-mon[191227]: pgmap v491: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 37 MiB/s wr, 696 op/s; 11547/261432 objects degraded (4.417%); 1395/261432 objects misplaced (0.534%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:38:41.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:41.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:41.325 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:41.350 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:41.393 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:41.396 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:41 smithi191 ceph-mon[161669]: pgmap v491: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 37 MiB/s wr, 696 op/s; 11547/261432 objects degraded (4.417%); 1395/261432 objects misplaced (0.534%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T05:38:41.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:41.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:41.499 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:41.539 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:41.599 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:41.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:41.708 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:41.730 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:38:41.746 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:41.805 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:41.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:41.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:41.909 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:38:41.913 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:41.946 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:42.003 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:42.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:42.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:42.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:42.137 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:42.193 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:38:42.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:42.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:42.300 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:42.331 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:42.695 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:42.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:42.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:42.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:42.832 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:42.838 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:42.887 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:42.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:42.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:42.994 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:42.999 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:43.090 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:43.138 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:43 smithi191 ceph-mon[161669]: pgmap v492: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 29 MiB/s wr, 704 op/s; 11547/258117 objects degraded (4.474%); 1032/258117 objects misplaced (0.400%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:38:43.139 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 11547/258117 objects degraded (4.474%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:43.200 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:43.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:43.311 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:43.414 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:43.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:43 smithi008 ceph-mon[184418]: pgmap v492: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 29 MiB/s wr, 704 op/s; 11547/258117 objects degraded (4.474%); 1032/258117 objects misplaced (0.400%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:38:43.417 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 11547/258117 objects degraded (4.474%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:43.426 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:43 smithi008 ceph-mon[191227]: pgmap v492: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 29 MiB/s wr, 704 op/s; 11547/258117 objects degraded (4.474%); 1032/258117 objects misplaced (0.400%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:38:43.426 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 11547/258117 objects degraded (4.474%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:43.435 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:43.562 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:38:43.621 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:38:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:43.629 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:43.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:43.677 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:38:43.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:43.732 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:38:43.762 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:43.775 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:38:43.815 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:38:43.817 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:43.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:43.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:43.879 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:38:43.925 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:43.938 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:38:43.954 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:38:43.954 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:38:43.955 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:38:43.955 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:38:43.955 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:38:43.955 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:38:43.955 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:38:43.955 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:38:44.025 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:44.302 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:44.309 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1219921656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:44.309 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:44.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:44.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:44.312 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1219921656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:44.312 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:44.412 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:44.451 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1219921656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:38:45.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:45 smithi008 ceph-mon[184418]: pgmap v493: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 695 op/s; 11547/255657 objects degraded (4.517%); 1032/255657 objects misplaced (0.404%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:38:45.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:45 smithi008 ceph-mon[191227]: pgmap v493: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 695 op/s; 11547/255657 objects degraded (4.517%); 1032/255657 objects misplaced (0.404%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:38:45.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:45 smithi191 ceph-mon[161669]: pgmap v493: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 695 op/s; 11547/255657 objects degraded (4.517%); 1032/255657 objects misplaced (0.404%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:38:47.137 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:47.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:47.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:47.248 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:47.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:47 smithi008 ceph-mon[191227]: pgmap v494: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 690 op/s; 11547/251268 objects degraded (4.595%); 1032/251268 objects misplaced (0.411%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:38:47.357 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:47 smithi008 ceph-mon[184418]: pgmap v494: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 690 op/s; 11547/251268 objects degraded (4.595%); 1032/251268 objects misplaced (0.411%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:38:47.363 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:47.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:47 smithi191 ceph-mon[161669]: pgmap v494: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 690 op/s; 11547/251268 objects degraded (4.595%); 1032/251268 objects misplaced (0.411%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:38:48.092 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:38:48.093 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:38:48.132 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:48.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:48.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:48.241 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:48.245 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:38:48.349 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 11547/251268 objects degraded (4.595%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:48.350 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 11547/251268 objects degraded (4.595%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:48.360 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:48.502 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:48.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:48.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:48.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 11547/251268 objects degraded (4.595%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:48.615 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:48.689 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:48.740 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:48.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:48.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:48.844 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:48.870 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:49.217 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:49.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:49.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:49.250 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T05:38:49.287 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:38:49.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:49.356 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:49.385 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:38:49.416 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:38:49.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:49.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:49.485 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:49 smithi008 ceph-mon[184418]: pgmap v495: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 90 B/s rd, 6.5 MiB/s wr, 643 op/s; 11547/246906 objects degraded (4.677%); 610/246906 objects misplaced (0.247%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:38:49.485 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3004562249' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:38:49.485 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2426308273' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:38:49.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:49 smithi008 ceph-mon[191227]: pgmap v495: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 90 B/s rd, 6.5 MiB/s wr, 643 op/s; 11547/246906 objects degraded (4.677%); 610/246906 objects misplaced (0.247%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:38:49.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3004562249' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:38:49.486 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2426308273' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:38:49.524 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:49 smithi191 ceph-mon[161669]: pgmap v495: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 90 B/s rd, 6.5 MiB/s wr, 643 op/s; 11547/246906 objects degraded (4.677%); 610/246906 objects misplaced (0.247%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:38:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3004562249' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:38:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2426308273' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:38:49.565 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:49.623 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:38:49.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:49.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:49.730 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:49.766 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:50.201 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:50.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:50.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:50.307 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:50.383 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:50.437 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:50.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:50.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[184418]: failed to encode map e185 with expected crc 2023-10-09T05:38:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3004562249' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:38:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2426308273' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:38:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[184418]: osdmap e185: 8 total, 8 up, 7 in 2023-10-09T05:38:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[184418]: failed to encode map e185 with expected crc 2023-10-09T05:38:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[184418]: failed to encode map e185 with expected crc 2023-10-09T05:38:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[191227]: failed to encode map e185 with expected crc 2023-10-09T05:38:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3004562249' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:38:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2426308273' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:38:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[191227]: osdmap e185: 8 total, 8 up, 7 in 2023-10-09T05:38:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[191227]: failed to encode map e185 with expected crc 2023-10-09T05:38:50.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:50 smithi008 ceph-mon[191227]: failed to encode map e185 with expected crc 2023-10-09T05:38:50.541 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:50.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:50 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:38:50.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:50 smithi191 ceph-mon[161669]: failed to encode map e185 with expected crc 2023-10-09T05:38:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3004562249' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:38:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2426308273' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:38:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:50 smithi191 ceph-mon[161669]: osdmap e185: 8 total, 8 up, 7 in 2023-10-09T05:38:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:50 smithi191 ceph-mon[161669]: failed to encode map e185 with expected crc 2023-10-09T05:38:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:50 smithi191 ceph-mon[161669]: failed to encode map e185 with expected crc 2023-10-09T05:38:50.567 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:50.666 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:50.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:50.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:50.778 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:50.816 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:50.872 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:50.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:50.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:50.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:38:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:38:50] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T05:38:50.977 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:51.008 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:51.010 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T05:38:51.065 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:51.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:51.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:51.112 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:51.175 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:51.203 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:51.249 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:51.255 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:51.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:51.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:51.367 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:51.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:51 smithi008 ceph-mon[184418]: pgmap v497: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 1.3 MiB/s wr, 665 op/s; 11547/242703 objects degraded (4.758%); 610/242703 objects misplaced (0.251%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:38:51.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:51 smithi008 ceph-mon[191227]: pgmap v497: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 1.3 MiB/s wr, 665 op/s; 11547/242703 objects degraded (4.758%); 610/242703 objects misplaced (0.251%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:38:51.493 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:51 smithi191 ceph-mon[161669]: pgmap v497: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 1.3 MiB/s wr, 665 op/s; 11547/242703 objects degraded (4.758%); 610/242703 objects misplaced (0.251%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:38:51.934 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:51.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:52.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:52.073 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:52.146 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:52.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:52.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:52.253 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:52.308 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:38:52.358 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:53.515 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:53.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:53 smithi008 ceph-mon[191227]: pgmap v498: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 11547/238581 objects degraded (4.840%); 201/238581 objects misplaced (0.084%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:38:53.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 11547/238581 objects degraded (4.840%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:53.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:53.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:53.523 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:53 smithi008 ceph-mon[184418]: pgmap v498: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 11547/238581 objects degraded (4.840%); 201/238581 objects misplaced (0.084%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:38:53.523 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 11547/238581 objects degraded (4.840%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:53.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:53 smithi191 ceph-mon[161669]: pgmap v498: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 11547/238581 objects degraded (4.840%); 201/238581 objects misplaced (0.084%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:38:53.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 11547/238581 objects degraded (4.840%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:53.584 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:53.624 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:38:53.724 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:54.053 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:38:54.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:54.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:38:54.159 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:54.234 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:54.330 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:38:54.330 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:38:54.409 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:54.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:54.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:54.519 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:54.550 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:54.602 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:38:54.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:54.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:54.710 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:54.742 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:55.129 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:38:55.129 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:38:55.129 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:38:55.129 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:38:55.129 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:38:55.129 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:38:55.130 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:38:55.134 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:38:55.134 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:38:55.134 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:38:55.135 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:38:55.135 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:38:55.135 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:38:55.135 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:38:55.135 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:38:55.135 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:38:55.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:55.161 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:55.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:55.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:55.290 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:55.340 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:55.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:55.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:55.447 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:38:55.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:55 smithi008 ceph-mon[191227]: pgmap v499: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 11547/236037 objects degraded (4.892%); 201/236037 objects misplaced (0.085%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:38:55.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:55 smithi008 ceph-mon[191227]: failed to encode map e186 with expected crc 2023-10-09T05:38:55.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:55 smithi008 ceph-mon[191227]: osdmap e186: 8 total, 8 up, 7 in 2023-10-09T05:38:55.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:55 smithi008 ceph-mon[191227]: failed to encode map e186 with expected crc 2023-10-09T05:38:55.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:55 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1821359035' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:55.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:55 smithi008 ceph-mon[184418]: pgmap v499: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 11547/236037 objects degraded (4.892%); 201/236037 objects misplaced (0.085%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:38:55.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:55 smithi008 ceph-mon[184418]: failed to encode map e186 with expected crc 2023-10-09T05:38:55.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:55 smithi008 ceph-mon[184418]: osdmap e186: 8 total, 8 up, 7 in 2023-10-09T05:38:55.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:55 smithi008 ceph-mon[184418]: failed to encode map e186 with expected crc 2023-10-09T05:38:55.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:55 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1821359035' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:55.493 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:55.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:55 smithi191 ceph-mon[161669]: pgmap v499: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 11547/236037 objects degraded (4.892%); 201/236037 objects misplaced (0.085%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:38:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:55 smithi191 ceph-mon[161669]: failed to encode map e186 with expected crc 2023-10-09T05:38:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:55 smithi191 ceph-mon[161669]: osdmap e186: 8 total, 8 up, 7 in 2023-10-09T05:38:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:55 smithi191 ceph-mon[161669]: failed to encode map e186 with expected crc 2023-10-09T05:38:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:55 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1821359035' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:38:55.936 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:55.989 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:38:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:56 smithi191 ceph-mon[161669]: failed to encode map e186 with expected crc 2023-10-09T05:38:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:56 smithi191 ceph-mon[161669]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:56 smithi191 ceph-mon[161669]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:56 smithi191 ceph-mon[161669]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:56 smithi191 ceph-mon[161669]: osdmap e187: 8 total, 8 up, 7 in 2023-10-09T05:38:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:56 smithi191 ceph-mon[161669]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.670 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:38:56.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[191227]: failed to encode map e186 with expected crc 2023-10-09T05:38:56.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[191227]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[191227]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.678 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[191227]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[191227]: osdmap e187: 8 total, 8 up, 7 in 2023-10-09T05:38:56.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[191227]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:56.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:38:56.680 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[184418]: failed to encode map e186 with expected crc 2023-10-09T05:38:56.680 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[184418]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.680 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[184418]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.680 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[184418]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.680 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[184418]: osdmap e187: 8 total, 8 up, 7 in 2023-10-09T05:38:56.680 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:56 smithi008 ceph-mon[184418]: failed to encode map e187 with expected crc 2023-10-09T05:38:56.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:56.822 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:56.882 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:38:56.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:56.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:56.990 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:38:57.022 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:57.085 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:38:57.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:57.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:38:57.192 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:57.227 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:57.280 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:57.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:57.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:57.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:57.425 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:57 smithi191 ceph-mon[161669]: pgmap v502: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 11547/231927 objects degraded (4.979%); 201/231927 objects misplaced (0.087%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:38:57.427 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:57.480 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:38:57.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:57.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:57.590 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:38:57.618 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:57 smithi008 ceph-mon[184418]: pgmap v502: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 11547/231927 objects degraded (4.979%); 201/231927 objects misplaced (0.087%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:38:57.618 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:57 smithi008 ceph-mon[191227]: pgmap v502: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 11547/231927 objects degraded (4.979%); 201/231927 objects misplaced (0.087%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:38:57.669 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:57.921 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:38:57.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:57.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:38:58.031 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:38:58.062 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:58.122 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:38:58.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:58.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:38:58.227 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:58.317 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:58.423 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:58.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:58.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:58.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 11547/231927 objects degraded (4.979%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:58.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 11547/231927 objects degraded (4.979%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:58.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 11547/231927 objects degraded (4.979%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:38:58.565 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:58.621 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:38:58.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:58.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:58.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:38:58.763 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:58.819 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:38:58.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:58.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:38:58.925 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:58.998 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:59.144 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:59.224 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:59.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:59.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:59.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:38:59.372 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:59.431 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:38:59.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:59.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:38:59.475 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:38:59.537 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:38:59.635 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:38:59.792 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:38:59.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:38:59.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:38:59.814 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T05:38:59.913 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:00.007 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:59 smithi008 ceph-mon[184418]: pgmap v503: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 9399/227982 objects degraded (4.123%); 1936/227982 objects misplaced (0.849%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T05:39:00.007 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:00.007 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:38:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2602132866' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:39:00.008 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:59 smithi008 ceph-mon[191227]: pgmap v503: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 9399/227982 objects degraded (4.123%); 1936/227982 objects misplaced (0.849%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T05:39:00.008 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:00.008 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:38:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2602132866' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:39:00.020 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:00.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:59 smithi191 ceph-mon[161669]: pgmap v503: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 9399/227982 objects degraded (4.123%); 1936/227982 objects misplaced (0.849%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T05:39:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:38:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2602132866' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:39:00.611 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:00.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:00.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:00.717 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:00.748 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:39:00.748 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:00.774 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[184418]: failed to encode map e188 with expected crc 2023-10-09T05:39:00.774 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2602132866' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:39:00.774 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[184418]: osdmap e188: 8 total, 8 up, 7 in 2023-10-09T05:39:00.774 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[184418]: failed to encode map e188 with expected crc 2023-10-09T05:39:00.775 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[184418]: failed to encode map e188 with expected crc 2023-10-09T05:39:00.775 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[184418]: failed to encode map e188 with expected crc 2023-10-09T05:39:00.775 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[184418]: failed to encode map e188 with expected crc 2023-10-09T05:39:00.775 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2697494786' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:00.775 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:00.775 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[191227]: failed to encode map e188 with expected crc 2023-10-09T05:39:00.775 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2602132866' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:39:00.775 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[191227]: osdmap e188: 8 total, 8 up, 7 in 2023-10-09T05:39:00.775 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[191227]: failed to encode map e188 with expected crc 2023-10-09T05:39:00.775 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[191227]: failed to encode map e188 with expected crc 2023-10-09T05:39:00.776 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[191227]: failed to encode map e188 with expected crc 2023-10-09T05:39:00.776 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[191227]: failed to encode map e188 with expected crc 2023-10-09T05:39:00.776 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2697494786' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:00.776 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:00 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:00.776 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:39:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:39:00] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.33.4" 2023-10-09T05:39:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:00 smithi191 ceph-mon[161669]: failed to encode map e188 with expected crc 2023-10-09T05:39:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2602132866' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:39:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:00 smithi191 ceph-mon[161669]: osdmap e188: 8 total, 8 up, 7 in 2023-10-09T05:39:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:00 smithi191 ceph-mon[161669]: failed to encode map e188 with expected crc 2023-10-09T05:39:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:00 smithi191 ceph-mon[161669]: failed to encode map e188 with expected crc 2023-10-09T05:39:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:00 smithi191 ceph-mon[161669]: failed to encode map e188 with expected crc 2023-10-09T05:39:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:00 smithi191 ceph-mon[161669]: failed to encode map e188 with expected crc 2023-10-09T05:39:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2697494786' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:00 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:01.427 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:01.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:01.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:01.534 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:01.632 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:01.785 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T05:39:01.785 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:39:01.795 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:01.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:01.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:01.909 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:01.936 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:01 smithi008 ceph-mon[184418]: pgmap v505: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 9399/224157 objects degraded (4.193%); 1936/224157 objects misplaced (0.864%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:39:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:01 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:39:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:01 smithi008 ceph-mon[184418]: failed to encode map e189 with expected crc 2023-10-09T05:39:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:01 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:01 smithi008 ceph-mon[184418]: osdmap e189: 8 total, 8 up, 7 in 2023-10-09T05:39:01.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:01 smithi008 ceph-mon[191227]: pgmap v505: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 9399/224157 objects degraded (4.193%); 1936/224157 objects misplaced (0.864%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:39:01.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:01 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:39:01.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:01 smithi008 ceph-mon[191227]: failed to encode map e189 with expected crc 2023-10-09T05:39:01.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:01 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:01.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:01 smithi008 ceph-mon[191227]: osdmap e189: 8 total, 8 up, 7 in 2023-10-09T05:39:01.997 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:39:02.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:02.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:02.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:01 smithi191 ceph-mon[161669]: pgmap v505: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 9399/224157 objects degraded (4.193%); 1936/224157 objects misplaced (0.864%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:39:02.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:01 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:39:02.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:01 smithi191 ceph-mon[161669]: failed to encode map e189 with expected crc 2023-10-09T05:39:02.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:01 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:02.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:01 smithi191 ceph-mon[161669]: osdmap e189: 8 total, 8 up, 7 in 2023-10-09T05:39:02.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:02.136 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:02.198 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:02.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:02.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:02.304 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:02.386 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:02.693 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:02.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:02.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:02.812 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:02.876 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:02.952 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:02.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:02 smithi008 ceph-mon[184418]: pgmap v507: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 9399/220131 objects degraded (4.270%); 1936/220131 objects misplaced (0.879%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:39:02.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:02 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9399/224157 objects degraded (4.193%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:02 smithi008 ceph-mon[191227]: pgmap v507: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 9399/220131 objects degraded (4.270%); 1936/220131 objects misplaced (0.879%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:39:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:02 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9399/224157 objects degraded (4.193%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:03.000 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:03.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:03.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:03.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:02 smithi191 ceph-mon[161669]: pgmap v507: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 9399/220131 objects degraded (4.270%); 1936/220131 objects misplaced (0.879%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:39:03.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:02 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9399/224157 objects degraded (4.193%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:03.080 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:39:03.081 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:39:03.081 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:39:03.081 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:39:03.081 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.081 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98110 5.10 5:08529fda:::benchmark_data_smithi008_238708_object98109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:39:03.081 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695249+0000", 2023-10-09T05:39:03.082 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 302.37999091199998, 2023-10-09T05:39:03.082 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94265819399999995, 2023-10-09T05:39:03.082 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.082 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:39:03.082 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.082 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:39:03.082 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:03.082 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98110 2023-10-09T05:39:03.082 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.083 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.083 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.083 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.083 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695249+0000", 2023-10-09T05:39:03.083 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.083 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.084 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.084 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.084 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695249+0000", 2023-10-09T05:39:03.084 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.084 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.084 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.084 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.084 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695246+0000", 2023-10-09T05:39:03.084 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:39:03.084 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.085 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.085 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.085 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695282+0000", 2023-10-09T05:39:03.085 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5592000000000001e-05 2023-10-09T05:39:03.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.086 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.086 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.086 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695308+0000", 2023-10-09T05:39:03.086 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6058999999999999e-05 2023-10-09T05:39:03.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.086 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.086 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.086 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695320+0000", 2023-10-09T05:39:03.086 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.219e-05 2023-10-09T05:39:03.087 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.087 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.087 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.087 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.808026+0000", 2023-10-09T05:39:03.087 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.112705826 2023-10-09T05:39:03.087 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.087 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.087 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.087 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.808037+0000", 2023-10-09T05:39:03.088 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0699999999999999e-05 2023-10-09T05:39:03.088 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.088 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637823+0000", 2023-10-09T05:39:03.088 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.829785785 2023-10-09T05:39:03.088 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.089 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637907+0000", 2023-10-09T05:39:03.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4371999999999998e-05 2023-10-09T05:39:03.089 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.089 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.089 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.089 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.089 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.089 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98125 5.0 5:038226d2:::benchmark_data_smithi008_238708_object98124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:39:03.089 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695331+0000", 2023-10-09T05:39:03.089 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 302.379908779, 2023-10-09T05:39:03.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92727647599999996, 2023-10-09T05:39:03.090 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.090 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.090 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.090 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:39:03.090 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:03.090 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98125 2023-10-09T05:39:03.090 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.090 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.090 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.090 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.091 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695331+0000", 2023-10-09T05:39:03.091 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.091 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.091 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.091 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.091 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695331+0000", 2023-10-09T05:39:03.091 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.091 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.091 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.091 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.091 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695330+0000", 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695357+0000", 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6591999999999999e-05 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695366+0000", 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7390000000000005e-06 2023-10-09T05:39:03.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695372+0000", 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6039999999999998e-06 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.806288+0000", 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.11091574899999999 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.093 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.806299+0000", 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1238e-05 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.594236+0000", 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.78793663899999999 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.594386+0000", 2023-10-09T05:39:03.094 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000149903 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.594543+0000", 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015728899999999999 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.600593+0000", 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0060496919999999997 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.095 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622331+0000", 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021738370999999999 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622485+0000", 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015411099999999999 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622558+0000", 2023-10-09T05:39:03.096 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2414000000000002e-05 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622607+0000", 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9796000000000002e-05 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98130 5.0 5:017ce245:::benchmark_data_smithi008_238708_object98129:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695378+0000", 2023-10-09T05:39:03.097 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 302.37986168100002, 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92738032100000001, 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98130 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.098 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695378+0000", 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695378+0000", 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695377+0000", 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:03.099 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695404+0000", 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6525000000000001e-05 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695412+0000", 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4139999999999994e-06 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.100 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695419+0000", 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3419999999999996e-06 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.807845+0000", 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.11242577099999999 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.807859+0000", 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4579000000000001e-05 2023-10-09T05:39:03.101 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.596508+0000", 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.78864880500000001 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.596655+0000", 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014741 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.102 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.596818+0000", 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000163022 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.600599+0000", 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037810140000000001 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.103 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622648+0000", 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022048531999999999 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622680+0000", 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1863999999999998e-05 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622722+0000", 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2454999999999999e-05 2023-10-09T05:39:03.104 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622758+0000", 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6159999999999999e-05 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98138 5.0 5:0007661a:::benchmark_data_smithi008_238708_object98137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.818516+0000", 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 302.25672361800002, 2023-10-09T05:39:03.105 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.80438677199999997, 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98138 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818516+0000", 2023-10-09T05:39:03.106 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818516+0000", 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818514+0000", 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.107 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818552+0000", 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8036e-05 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818582+0000", 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9819e-05 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818596+0000", 2023-10-09T05:39:03.108 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4635e-05 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818664+0000", 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7669999999999994e-05 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.818679+0000", 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4914e-05 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.109 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.598641+0000", 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77996221300000002 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.598805+0000", 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016343600000000001 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.598978+0000", 2023-10-09T05:39:03.110 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017355699999999999 2023-10-09T05:39:03.111 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.111 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.111 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.111 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.600790+0000", 2023-10-09T05:39:03.111 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001812255 2023-10-09T05:39:03.111 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.111 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.111 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.111 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622797+0000", 2023-10-09T05:39:03.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022006438 2023-10-09T05:39:03.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.112 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.112 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622829+0000", 2023-10-09T05:39:03.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1943000000000002e-05 2023-10-09T05:39:03.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.112 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.112 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622871+0000", 2023-10-09T05:39:03.113 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1844000000000001e-05 2023-10-09T05:39:03.113 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.113 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.113 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.113 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622903+0000", 2023-10-09T05:39:03.113 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2283999999999998e-05 2023-10-09T05:39:03.113 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.113 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.113 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.113 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.113 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119035 5.0 5:0296cb6a:::benchmark_data_smithi008_238708_object899:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.063473+0000", 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 279.01176687700001, 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.003481047, 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119035 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.114 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063473+0000", 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063473+0000", 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063470+0000", 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063484+0000", 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4049000000000001e-05 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063506+0000", 2023-10-09T05:39:03.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2082999999999998e-05 2023-10-09T05:39:03.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.117 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.117 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.117 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063520+0000", 2023-10-09T05:39:03.117 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4065e-05 2023-10-09T05:39:03.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.117 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.117 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.117 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.189233+0000", 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.125712348 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.189249+0000", 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5798999999999999e-05 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.050621+0000", 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86137281600000004 2023-10-09T05:39:03.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.119 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.119 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.119 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.050748+0000", 2023-10-09T05:39:03.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012712199999999999 2023-10-09T05:39:03.119 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.119 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.119 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:39:03.119 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.050845+0000", 2023-10-09T05:39:03.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6850999999999998e-05 2023-10-09T05:39:03.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.054218+0000", 2023-10-09T05:39:03.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033727660000000001 2023-10-09T05:39:03.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.055747+0000", 2023-10-09T05:39:03.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015285539999999999 2023-10-09T05:39:03.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.121 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.066866+0000", 2023-10-09T05:39:03.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011119052000000001 2023-10-09T05:39:03.121 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.121 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.066917+0000", 2023-10-09T05:39:03.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1672999999999998e-05 2023-10-09T05:39:03.122 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.122 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.122 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.122 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.066954+0000", 2023-10-09T05:39:03.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6520999999999998e-05 2023-10-09T05:39:03.122 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.122 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.122 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.122 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.123 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.123 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119040 5.0 5:00ca488a:::benchmark_data_smithi008_238708_object904:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:39:03.123 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.262978+0000", 2023-10-09T05:39:03.123 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 278.81226139299997, 2023-10-09T05:39:03.123 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.80409076700000004, 2023-10-09T05:39:03.123 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.123 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.123 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.123 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:39:03.123 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:03.124 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119040 2023-10-09T05:39:03.124 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.124 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.124 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.124 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.124 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.262978+0000", 2023-10-09T05:39:03.124 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.124 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.124 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.125 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.125 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.262978+0000", 2023-10-09T05:39:03.125 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.125 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.125 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.125 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.125 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.262976+0000", 2023-10-09T05:39:03.125 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:39:03.125 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.126 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.126 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.126 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.262987+0000", 2023-10-09T05:39:03.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0406e-05 2023-10-09T05:39:03.126 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.126 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.126 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.126 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.263013+0000", 2023-10-09T05:39:03.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6182000000000001e-05 2023-10-09T05:39:03.127 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.127 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.127 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.127 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.263024+0000", 2023-10-09T05:39:03.127 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1025e-05 2023-10-09T05:39:03.127 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.127 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.127 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.127 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.263087+0000", 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3063000000000003e-05 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.263102+0000", 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4628e-05 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.051901+0000", 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.78879942700000005 2023-10-09T05:39:03.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.052001+0000", 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9431000000000004e-05 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.052098+0000", 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7127999999999996e-05 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.129 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.054225+0000", 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002127004 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.055779+0000", 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001553912 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.066989+0000", 2023-10-09T05:39:03.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011210036 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.067034+0000", 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5071999999999998e-05 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.067069+0000", 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5422000000000001e-05 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.131 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:69846 6.6 6:61b7a970:::benchmark_data_smithi008_295510_object69845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.190073+0000", 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.885166456999997, 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94522247000000004, 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.132 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 69846 2023-10-09T05:39:03.133 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.133 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.133 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.190073+0000", 2023-10-09T05:39:03.133 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.133 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.133 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.190073+0000", 2023-10-09T05:39:03.134 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.134 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.190072+0000", 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.190093+0000", 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1307000000000001e-05 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.190107+0000", 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3565e-05 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.190114+0000", 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8750000000000002e-06 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.190484+0000", 2023-10-09T05:39:03.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00037066399999999999 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.190496+0000", 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1695999999999999e-05 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.130045+0000", 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.939549364 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.137 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.130175+0000", 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000129896 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.130298+0000", 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000123282 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.135176+0000", 2023-10-09T05:39:03.138 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048780029999999997 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.135213+0000", 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6272000000000001e-05 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.135241+0000", 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7971e-05 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.135271+0000", 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0437999999999999e-05 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.135296+0000", 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4623000000000001e-05 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.140 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:69871 6.6 6:6c53b032:::benchmark_data_smithi008_295510_object69870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.213746+0000", 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.861493345, 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92507783099999996, 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 69871 2023-10-09T05:39:03.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.213746+0000", 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.213746+0000", 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.213745+0000", 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.213776+0000", 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0709e-05 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.213790+0000", 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4260000000000001e-05 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.213800+0000", 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0138e-05 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.213864+0000", 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4129999999999998e-05 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.213876+0000", 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2118e-05 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.132423+0000", 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.918546898 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.145 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.132515+0000", 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1576999999999994e-05 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.132606+0000", 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1151999999999997e-05 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.135355+0000", 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002749474 2023-10-09T05:39:03.146 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.135389+0000", 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3463999999999997e-05 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.138730+0000", 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003341091 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.147 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.138787+0000", 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6672000000000002e-05 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.138824+0000", 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7586000000000002e-05 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:69882 6.6 6:6666fa8a:::benchmark_data_smithi008_295510_object69881:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.148 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.224320+0000", 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.850919670000003, 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.91467100000000001, 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 69882 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.149 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.150 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224320+0000", 2023-10-09T05:39:03.150 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.150 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.150 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.150 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.150 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224320+0000", 2023-10-09T05:39:03.154 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.154 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.154 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.154 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.154 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224318+0000", 2023-10-09T05:39:03.154 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:39:03.155 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.155 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.155 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.155 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224341+0000", 2023-10-09T05:39:03.155 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2700999999999998e-05 2023-10-09T05:39:03.155 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.155 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.155 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.155 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224356+0000", 2023-10-09T05:39:03.155 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5108e-05 2023-10-09T05:39:03.155 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.156 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.156 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.156 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224363+0000", 2023-10-09T05:39:03.156 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4420000000000001e-06 2023-10-09T05:39:03.156 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.156 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.156 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.156 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.225130+0000", 2023-10-09T05:39:03.157 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00076627200000000003 2023-10-09T05:39:03.157 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.157 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.157 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.157 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.225140+0000", 2023-10-09T05:39:03.157 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0039e-05 2023-10-09T05:39:03.157 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.157 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.157 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.157 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.133983+0000", 2023-10-09T05:39:03.157 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90884282900000002 2023-10-09T05:39:03.158 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.158 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.158 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.158 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.134059+0000", 2023-10-09T05:39:03.158 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6229000000000005e-05 2023-10-09T05:39:03.158 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.158 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.158 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.158 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.134140+0000", 2023-10-09T05:39:03.159 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0834000000000006e-05 2023-10-09T05:39:03.159 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.159 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.159 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.159 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.136706+0000", 2023-10-09T05:39:03.159 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025666220000000002 2023-10-09T05:39:03.159 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.159 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.159 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.138870+0000", 2023-10-09T05:39:03.160 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021640919999999998 2023-10-09T05:39:03.160 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.160 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.160 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.138913+0000", 2023-10-09T05:39:03.160 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2144000000000002e-05 2023-10-09T05:39:03.160 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.160 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.160 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.138942+0000", 2023-10-09T05:39:03.161 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8994000000000001e-05 2023-10-09T05:39:03.161 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.161 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.161 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.161 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.138991+0000", 2023-10-09T05:39:03.161 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9484999999999998e-05 2023-10-09T05:39:03.161 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.161 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.162 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.162 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.162 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.162 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:69884 6.6 6:61fb505a:::benchmark_data_smithi008_295510_object69883:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.162 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.224694+0000", 2023-10-09T05:39:03.162 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.850545337, 2023-10-09T05:39:03.162 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.91801977400000001, 2023-10-09T05:39:03.162 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.163 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.163 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.163 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.163 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.163 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 69884 2023-10-09T05:39:03.163 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.163 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.164 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.164 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224694+0000", 2023-10-09T05:39:03.164 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.164 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.164 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.165 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224694+0000", 2023-10-09T05:39:03.165 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.165 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.165 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.165 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.165 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224694+0000", 2023-10-09T05:39:03.165 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:39:03.165 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.165 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.165 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.165 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224715+0000", 2023-10-09T05:39:03.166 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1620999999999998e-05 2023-10-09T05:39:03.166 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.166 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.166 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.166 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224728+0000", 2023-10-09T05:39:03.166 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3227999999999999e-05 2023-10-09T05:39:03.166 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.166 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.166 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.166 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.224735+0000", 2023-10-09T05:39:03.166 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5969999999999997e-06 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.225175+0000", 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00043984500000000001 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.225184+0000", 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6629999999999999e-06 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.167 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.168 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.135410+0000", 2023-10-09T05:39:03.168 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.91022611399999998 2023-10-09T05:39:03.168 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.168 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.168 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.168 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.135480+0000", 2023-10-09T05:39:03.168 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0432999999999995e-05 2023-10-09T05:39:03.168 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.168 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.168 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.168 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.135573+0000", 2023-10-09T05:39:03.169 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2918999999999996e-05 2023-10-09T05:39:03.169 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.169 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.169 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.138880+0000", 2023-10-09T05:39:03.169 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033070159999999999 2023-10-09T05:39:03.169 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.169 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.169 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142613+0000", 2023-10-09T05:39:03.169 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037333570000000001 2023-10-09T05:39:03.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.170 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.170 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.170 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142642+0000", 2023-10-09T05:39:03.170 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8819e-05 2023-10-09T05:39:03.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.170 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.170 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.170 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142673+0000", 2023-10-09T05:39:03.170 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0803999999999998e-05 2023-10-09T05:39:03.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142714+0000", 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1152000000000002e-05 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:69934 6.6 6:61fb6eca:::benchmark_data_smithi008_295510_object69933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.277253+0000", 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.797986860999998, 2023-10-09T05:39:03.171 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86563053499999998, 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 69934 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.277253+0000", 2023-10-09T05:39:03.172 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.277253+0000", 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.277251+0000", 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.173 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.277278+0000", 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6741999999999999e-05 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.277297+0000", 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9233999999999999e-05 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.277307+0000", 2023-10-09T05:39:03.174 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6490000000000003e-06 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.277367+0000", 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0424000000000003e-05 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.277379+0000", 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1292e-05 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.136742+0000", 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85936295699999998 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.136861+0000", 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000119709 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.136971+0000", 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000110059 2023-10-09T05:39:03.176 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.140917+0000", 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039456350000000003 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142775+0000", 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018584090000000001 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142806+0000", 2023-10-09T05:39:03.177 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0412999999999998e-05 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142852+0000", 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6678999999999998e-05 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142883+0000", 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0905000000000003e-05 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.178 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.179 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.179 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.179 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:69951 6.6 6:6ba1f1ec:::benchmark_data_smithi008_295510_object69950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.179 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.289985+0000", 2023-10-09T05:39:03.179 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.785254874000003, 2023-10-09T05:39:03.179 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85312180400000004, 2023-10-09T05:39:03.179 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.179 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.179 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.179 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.179 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 69951 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.289985+0000", 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.289985+0000", 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.181 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.181 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.181 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.289984+0000", 2023-10-09T05:39:03.181 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:03.181 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.181 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.181 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.181 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.290004+0000", 2023-10-09T05:39:03.181 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9417999999999999e-05 2023-10-09T05:39:03.181 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.181 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.290011+0000", 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8669999999999996e-06 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.290016+0000", 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1359999999999996e-06 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.290075+0000", 2023-10-09T05:39:03.182 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8736999999999997e-05 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.290086+0000", 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0903e-05 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.139024+0000", 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.84893813399999996 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.184 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.184 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.139135+0000", 2023-10-09T05:39:03.184 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000111017 2023-10-09T05:39:03.184 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.184 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.184 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.184 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.139261+0000", 2023-10-09T05:39:03.184 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012599399999999999 2023-10-09T05:39:03.184 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.184 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.184 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142729+0000", 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003468652 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142938+0000", 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020833399999999999 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142999+0000", 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1210000000000005e-05 2023-10-09T05:39:03.185 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.186 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.143048+0000", 2023-10-09T05:39:03.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9607999999999997e-05 2023-10-09T05:39:03.186 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.186 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.143107+0000", 2023-10-09T05:39:03.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8187999999999999e-05 2023-10-09T05:39:03.187 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.187 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.187 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.187 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:69954 6.6 6:65c0882e:::benchmark_data_smithi008_295510_object69953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.187 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.292223+0000", 2023-10-09T05:39:03.187 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.783016455999999, 2023-10-09T05:39:03.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85109091400000003, 2023-10-09T05:39:03.188 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.188 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.188 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.188 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.188 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.188 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 69954 2023-10-09T05:39:03.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.188 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.188 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.292223+0000", 2023-10-09T05:39:03.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.292223+0000", 2023-10-09T05:39:03.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.190 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.190 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.292221+0000", 2023-10-09T05:39:03.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:39:03.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.190 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.190 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.292255+0000", 2023-10-09T05:39:03.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3429999999999997e-05 2023-10-09T05:39:03.191 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.191 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.292276+0000", 2023-10-09T05:39:03.191 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0625e-05 2023-10-09T05:39:03.191 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.292286+0000", 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0159e-05 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.292347+0000", 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1526999999999996e-05 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.292359+0000", 2023-10-09T05:39:03.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1919000000000001e-05 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.140945+0000", 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.84858562000000004 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.141066+0000", 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000121666 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.193 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.141147+0000", 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0166999999999993e-05 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.142895+0000", 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001748547 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.194 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.143182+0000", 2023-10-09T05:39:03.195 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00028651700000000002 2023-10-09T05:39:03.195 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.195 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.195 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.195 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.143251+0000", 2023-10-09T05:39:03.195 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9292e-05 2023-10-09T05:39:03.195 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.195 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.195 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.195 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.143279+0000", 2023-10-09T05:39:03.196 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8441000000000001e-05 2023-10-09T05:39:03.196 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.196 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.196 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.196 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.143314+0000", 2023-10-09T05:39:03.196 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4786000000000002e-05 2023-10-09T05:39:03.196 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.196 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.196 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.197 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.197 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.197 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:69956 6.6 6:6694aac2:::benchmark_data_smithi008_295510_object69955:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.197 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.297765+0000", 2023-10-09T05:39:03.197 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.777474882999996, 2023-10-09T05:39:03.197 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85640996199999997, 2023-10-09T05:39:03.197 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.198 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.198 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.198 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.198 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.198 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 69956 2023-10-09T05:39:03.198 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.198 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.198 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.198 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.199 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.297765+0000", 2023-10-09T05:39:03.199 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.199 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.199 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.199 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.199 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.297765+0000", 2023-10-09T05:39:03.199 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.199 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.199 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.200 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.200 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.297764+0000", 2023-10-09T05:39:03.200 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:39:03.200 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.200 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.200 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.200 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.297783+0000", 2023-10-09T05:39:03.200 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9032999999999999e-05 2023-10-09T05:39:03.201 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.201 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.201 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.201 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.297793+0000", 2023-10-09T05:39:03.201 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8069999999999994e-06 2023-10-09T05:39:03.201 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.201 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.201 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.202 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.297801+0000", 2023-10-09T05:39:03.202 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0139999999999998e-06 2023-10-09T05:39:03.202 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.202 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.202 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.202 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.297935+0000", 2023-10-09T05:39:03.202 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0001348 2023-10-09T05:39:03.202 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.202 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.202 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.203 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.297947+0000", 2023-10-09T05:39:03.203 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1296e-05 2023-10-09T05:39:03.203 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.203 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.203 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.203 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.146436+0000", 2023-10-09T05:39:03.203 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.84848890799999999 2023-10-09T05:39:03.203 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.203 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.203 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.203 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.146571+0000", 2023-10-09T05:39:03.204 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013548100000000001 2023-10-09T05:39:03.204 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.204 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.204 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.204 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.146661+0000", 2023-10-09T05:39:03.204 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0060000000000002e-05 2023-10-09T05:39:03.204 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.204 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.204 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.204 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.154011+0000", 2023-10-09T05:39:03.204 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0073503409999999998 2023-10-09T05:39:03.205 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.205 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.205 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.205 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.154054+0000", 2023-10-09T05:39:03.205 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2883000000000003e-05 2023-10-09T05:39:03.205 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.205 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.205 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.205 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.154085+0000", 2023-10-09T05:39:03.205 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0414999999999999e-05 2023-10-09T05:39:03.205 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.154133+0000", 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8146000000000003e-05 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.154175+0000", 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1865e-05 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.206 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:70011 6.6 6:6b2798a2:::benchmark_data_smithi008_295510_object70010:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.356330+0000", 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.718909547000003, 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.803261577, 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.207 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 70011 2023-10-09T05:39:03.208 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.208 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.208 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.208 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.208 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.356330+0000", 2023-10-09T05:39:03.208 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.208 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.208 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.208 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.208 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.356330+0000", 2023-10-09T05:39:03.208 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.209 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.209 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.209 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.209 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.356329+0000", 2023-10-09T05:39:03.209 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:03.209 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.209 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.209 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.209 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.356356+0000", 2023-10-09T05:39:03.209 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7375000000000001e-05 2023-10-09T05:39:03.209 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.356375+0000", 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9074000000000001e-05 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.356384+0000", 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.251e-06 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.211 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.356454+0000", 2023-10-09T05:39:03.211 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9357999999999996e-05 2023-10-09T05:39:03.211 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.211 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.211 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.211 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.356466+0000", 2023-10-09T05:39:03.211 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1807000000000001e-05 2023-10-09T05:39:03.211 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.211 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.211 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.211 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.148640+0000", 2023-10-09T05:39:03.212 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.79217404899999999 2023-10-09T05:39:03.212 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.212 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.212 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.212 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.148746+0000", 2023-10-09T05:39:03.212 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000106393 2023-10-09T05:39:03.212 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.212 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.212 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.212 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.148842+0000", 2023-10-09T05:39:03.212 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6019000000000003e-05 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.154046+0000", 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0052036799999999996 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.159461+0000", 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0054151889999999999 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.214 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.214 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.159515+0000", 2023-10-09T05:39:03.214 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4549999999999998e-05 2023-10-09T05:39:03.214 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.214 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.214 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.214 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.159551+0000", 2023-10-09T05:39:03.214 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5329e-05 2023-10-09T05:39:03.214 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.214 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.214 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.159592+0000", 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0982000000000002e-05 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:70041 6.6 6:6014a99d:::benchmark_data_smithi008_295510_object70040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.399506+0000", 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.675734159000001, 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.760217692, 2023-10-09T05:39:03.215 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 70041 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.399506+0000", 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.216 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.399506+0000", 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.399504+0000", 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.217 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.399523+0000", 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8502999999999999e-05 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.399531+0000", 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9570000000000008e-06 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.399537+0000", 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.784e-06 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.218 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.399584+0000", 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6560999999999997e-05 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.399592+0000", 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6690000000000006e-06 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.219 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.151096+0000", 2023-10-09T05:39:03.220 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75150464400000005 2023-10-09T05:39:03.220 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.220 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.220 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.220 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.151274+0000", 2023-10-09T05:39:03.220 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017771900000000001 2023-10-09T05:39:03.220 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.220 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.220 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.220 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.151466+0000", 2023-10-09T05:39:03.221 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000191719 2023-10-09T05:39:03.221 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.221 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.221 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.221 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.154222+0000", 2023-10-09T05:39:03.221 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027567440000000002 2023-10-09T05:39:03.221 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.221 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.221 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.221 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.159471+0000", 2023-10-09T05:39:03.221 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0052486010000000003 2023-10-09T05:39:03.222 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.222 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.222 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.222 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.159639+0000", 2023-10-09T05:39:03.222 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016850799999999999 2023-10-09T05:39:03.222 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.222 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.222 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.222 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.159671+0000", 2023-10-09T05:39:03.222 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2042999999999998e-05 2023-10-09T05:39:03.222 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.159723+0000", 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1752999999999997e-05 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:70054 6.6 6:60b43f8b:::benchmark_data_smithi008_295510_object70053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.411341+0000", 2023-10-09T05:39:03.223 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.663899057000002, 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75395967500000005, 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 70054 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.224 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.411341+0000", 2023-10-09T05:39:03.225 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.225 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.225 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.225 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.225 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.411341+0000", 2023-10-09T05:39:03.225 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.225 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.225 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.411339+0000", 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.411368+0000", 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9359000000000001e-05 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.226 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.411383+0000", 2023-10-09T05:39:03.227 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4352000000000001e-05 2023-10-09T05:39:03.227 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.227 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.227 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.227 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.411392+0000", 2023-10-09T05:39:03.227 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6379999999999999e-06 2023-10-09T05:39:03.227 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.227 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.227 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.411458+0000", 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5714000000000001e-05 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.411471+0000", 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2955999999999999e-05 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.154253+0000", 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.74278151100000001 2023-10-09T05:39:03.228 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.229 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.229 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.229 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.154388+0000", 2023-10-09T05:39:03.229 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013584799999999999 2023-10-09T05:39:03.229 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.229 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.229 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.229 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.154537+0000", 2023-10-09T05:39:03.229 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014824400000000001 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.159765+0000", 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0052285580000000003 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.165169+0000", 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0054041599999999999 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.230 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.231 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.231 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.165219+0000", 2023-10-09T05:39:03.231 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9503999999999999e-05 2023-10-09T05:39:03.231 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.231 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.231 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.231 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.165264+0000", 2023-10-09T05:39:03.231 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5563999999999999e-05 2023-10-09T05:39:03.231 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.231 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.232 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.232 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.165300+0000", 2023-10-09T05:39:03.232 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5898000000000001e-05 2023-10-09T05:39:03.232 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.232 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.232 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.232 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.232 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.232 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:70064 6.6 6:68c12b07:::benchmark_data_smithi008_295510_object70063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.233 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.426904+0000", 2023-10-09T05:39:03.233 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.648335244999998, 2023-10-09T05:39:03.233 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73853470200000004, 2023-10-09T05:39:03.233 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.233 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.233 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.233 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.233 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.233 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 70064 2023-10-09T05:39:03.233 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.234 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.234 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.234 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.234 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.426904+0000", 2023-10-09T05:39:03.234 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.234 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.234 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.234 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.426904+0000", 2023-10-09T05:39:03.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.235 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.235 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.235 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.426904+0000", 2023-10-09T05:39:03.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:03.235 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.426928+0000", 2023-10-09T05:39:03.236 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4406e-05 2023-10-09T05:39:03.236 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.426937+0000", 2023-10-09T05:39:03.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4230000000000005e-06 2023-10-09T05:39:03.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.237 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.237 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.237 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.426943+0000", 2023-10-09T05:39:03.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.489e-06 2023-10-09T05:39:03.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.237 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.237 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.426994+0000", 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1020999999999998e-05 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.427003+0000", 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.85e-06 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.157088+0000", 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.730084603 2023-10-09T05:39:03.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.157245+0000", 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015725599999999999 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.157398+0000", 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015285299999999999 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.239 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.165210+0000", 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078117380000000004 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.165312+0000", 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000102957 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.165360+0000", 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7460000000000003e-05 2023-10-09T05:39:03.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.165405+0000", 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4651999999999999e-05 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.165439+0000", 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4588000000000002e-05 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.241 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.242 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:70075 6.6 6:61592782:::benchmark_data_smithi008_295510_object70074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.242 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.446495+0000", 2023-10-09T05:39:03.242 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.628745117999998, 2023-10-09T05:39:03.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72688271400000004, 2023-10-09T05:39:03.242 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.242 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.242 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.242 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 70075 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.446495+0000", 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.446495+0000", 2023-10-09T05:39:03.243 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.446493+0000", 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.446517+0000", 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4586000000000001e-05 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.244 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.446532+0000", 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4208e-05 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.446539+0000", 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.17e-06 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.446602+0000", 2023-10-09T05:39:03.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3208999999999997e-05 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.446614+0000", 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2031000000000001e-05 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.159780+0000", 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.71316566599999998 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.159894+0000", 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114349 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.160089+0000", 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000194754 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.165493+0000", 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0054036809999999996 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.168235+0000", 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027429199999999998 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.173288+0000", 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050521029999999996 2023-10-09T05:39:03.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.173336+0000", 2023-10-09T05:39:03.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8857000000000003e-05 2023-10-09T05:39:03.249 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.173377+0000", 2023-10-09T05:39:03.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0902000000000002e-05 2023-10-09T05:39:03.249 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.249 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.250 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.250 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:70083 6.6 6:6bd09698:::benchmark_data_smithi008_295510_object70082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-10-09T05:39:03.250 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:09.461051+0000", 2023-10-09T05:39:03.250 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.614188814999999, 2023-10-09T05:39:03.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.71246202800000002, 2023-10-09T05:39:03.250 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:03.250 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:03.250 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:03.250 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:03.251 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:03.251 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 70083 2023-10-09T05:39:03.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.251 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:03.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:03.251 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.461051+0000", 2023-10-09T05:39:03.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:03.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.461051+0000", 2023-10-09T05:39:03.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:03.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:03.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.461049+0000", 2023-10-09T05:39:03.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:03.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.461077+0000", 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8158e-05 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.461097+0000", 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9213000000000001e-05 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.461107+0000", 2023-10-09T05:39:03.253 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0616e-05 2023-10-09T05:39:03.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.461192+0000", 2023-10-09T05:39:03.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4438999999999996e-05 2023-10-09T05:39:03.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:39:03.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:09.461201+0000", 2023-10-09T05:39:03.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7140000000000005e-06 2023-10-09T05:39:03.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:03.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.162359+0000", 2023-10-09T05:39:03.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.70115726199999995 2023-10-09T05:39:03.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:03.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.162521+0000", 2023-10-09T05:39:03.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000162359 2023-10-09T05:39:03.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:03.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.162702+0000", 2023-10-09T05:39:03.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018052899999999999 2023-10-09T05:39:03.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.165533+0000", 2023-10-09T05:39:03.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028315839999999998 2023-10-09T05:39:03.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:03.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.168241+0000", 2023-10-09T05:39:03.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027081230000000002 2023-10-09T05:39:03.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:03.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.173443+0000", 2023-10-09T05:39:03.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0052018339999999998 2023-10-09T05:39:03.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:39:03.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.173481+0000", 2023-10-09T05:39:03.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8349999999999997e-05 2023-10-09T05:39:03.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:03.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:03.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:39:03.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:10.173513+0000", 2023-10-09T05:39:03.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1498999999999998e-05 2023-10-09T05:39:03.258 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.258 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.258 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.258 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:03.258 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:03.259 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:39:03.259 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:03.332 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:04.370 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:04.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:04.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:04.402 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:39:04.488 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:04.561 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:04.570 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:39:04.596 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:04.847 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:05.364 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:05.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:05.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:05.401 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:05 smithi191 ceph-mon[161669]: pgmap v508: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 9399/217689 objects degraded (4.318%); 1775/217689 objects misplaced (0.815%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:39:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:05 smithi008 ceph-mon[184418]: pgmap v508: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 9399/217689 objects degraded (4.318%); 1775/217689 objects misplaced (0.815%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:39:05.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:05 smithi008 ceph-mon[191227]: pgmap v508: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 9399/217689 objects degraded (4.318%); 1775/217689 objects misplaced (0.815%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:39:05.473 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:05.588 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:05.758 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:05.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:05.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:05.868 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:05.950 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:06.020 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:06.554 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:06.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:06.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:06.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:06.697 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:06.761 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:39:06.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:06.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:06.866 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:06.897 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:06.968 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:39:06.968 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:39:06.969 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:39:06.969 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:39:06.969 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:39:06.969 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:39:06.969 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:39:06.970 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:39:06.970 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:39:06.970 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:39:06.970 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:39:06.971 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:39:06.971 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:39:06.971 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:39:06.971 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:39:06.971 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:39:07.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:07 smithi008 ceph-mon[184418]: pgmap v509: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 9399/213624 objects degraded (4.400%); 1775/213624 objects misplaced (0.831%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:39:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:07 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2642104668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:07 smithi008 ceph-mon[191227]: pgmap v509: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 9399/213624 objects degraded (4.400%); 1775/213624 objects misplaced (0.831%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:39:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:07 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2642104668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:07.469 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:07.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:07.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:07 smithi191 ceph-mon[161669]: pgmap v509: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 9399/213624 objects degraded (4.400%); 1775/213624 objects misplaced (0.831%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T05:39:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:07 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2642104668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:07.575 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:07.670 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:07.820 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:39:07.821 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:39:08.142 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9399/213624 objects degraded (4.400%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:08.430 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:08.442 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9399/213624 objects degraded (4.400%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:08.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:08.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:08.545 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:08.546 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9399/213624 objects degraded (4.400%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:08.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:39:08.637 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:09.036 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:09.434 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:09.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:09.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:09.551 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:09.636 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:09.777 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:09.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:09.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:09.890 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:09 smithi008 ceph-mon[191227]: pgmap v510: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 9399/210540 objects degraded (4.464%); 1775/210540 objects misplaced (0.843%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:39:09.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:09 smithi008 ceph-mon[184418]: pgmap v510: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 9399/210540 objects degraded (4.464%); 1775/210540 objects misplaced (0.843%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:39:09.977 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:10.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:09 smithi191 ceph-mon[161669]: pgmap v510: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 9399/210540 objects degraded (4.464%); 1775/210540 objects misplaced (0.843%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:39:10.132 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:10.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:10.248 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:10.364 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:10.454 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:39:10.494 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:10.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:10.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:10.510 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:39:10.611 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:10.677 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:39:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:39:10] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.33.4" 2023-10-09T05:39:10.695 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:39:10.695 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:39:10.701 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:10.853 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:10.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:10.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:10.965 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:11.002 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:11.043 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:11.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:11.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:11.146 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:11.222 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:11.322 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:11.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:11.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:11.372 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:39:11.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:39:11.426 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:11.453 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:11.547 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:11.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:11.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:11.656 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:11.705 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:11.762 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:39:11.778 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:39:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:11.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:11.839 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:11.886 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:11.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:11 smithi008 ceph-mon[184418]: pgmap v511: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 9404/205089 objects degraded (4.585%); 1601/205089 objects misplaced (0.781%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T05:39:11.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:11 smithi008 ceph-mon[184418]: from='client.45149 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:11.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3843341833' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:39:11.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:11 smithi008 ceph-mon[191227]: pgmap v511: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 9404/205089 objects degraded (4.585%); 1601/205089 objects misplaced (0.781%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T05:39:11.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:11 smithi008 ceph-mon[191227]: from='client.45149 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:11.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3843341833' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:39:11.981 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:12.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:11 smithi191 ceph-mon[161669]: pgmap v511: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 9404/205089 objects degraded (4.585%); 1601/205089 objects misplaced (0.781%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T05:39:12.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:11 smithi191 ceph-mon[161669]: from='client.45149 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:12.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3843341833' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:39:12.104 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":511,"stamp":"2023-10-09T05:39:10.063873+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4483127243,"num_objects":68363,"num_object_clones":0,"num_object_copies":205089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9404,"num_objects_misplaced":1601,"num_objects_unfound":0,"num_objects_dirty":68363,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":109493,"num_write_kb":5670182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6536,"num_bytes_recovered":426775926,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108057,"ondisk_log_size":108057,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14817024,"kb_used_data":12814792,"kb_used_omap":31,"kb_used_meta":2001568,"kb_avail":641399040,"statfs":{"total":671965249536,"available":656792616960,"internally_reserved":0,"allocated":13122347008,"data_stored":13117851384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2049606089},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,2,3,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-416546816,"num_objects":-6356,"num_object_clones":0,"num_object_copies":-19068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":-335,"num_objects_unfound":0,"num_objects_dirty":-6356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6356,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":330,"num_bytes_recovered":21626880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003447"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":172,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.753954+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:38:59.753954+0000","last_peered":"2023-10-09T05:38:59.753954+0000","last_clean":"2023-10-09T05:38:59.753954+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:38:59.753954+0000","last_undegraded":"2023-10-09T05:38:59.753954+0000","last_fullsized":"2023-10-09T05:38:59.753954+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":161,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.828699+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:39:00.828699+0000","last_peered":"2023-10-09T05:39:00.828699+0000","last_clean":"2023-10-09T05:39:00.828699+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:39:00.828699+0000","last_undegraded":"2023-10-09T05:39:00.828699+0000","last_fullsized":"2023-10-09T05:39:00.828699+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":135,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.749074+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:38:59.749074+0000","last_peered":"2023-10-09T05:38:59.749074+0000","last_clean":"2023-10-09T05:38:59.749074+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:38:59.749074+0000","last_undegraded":"2023-10-09T05:38:59.749074+0000","last_fullsized":"2023-10-09T05:38:59.749074+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":165,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.739116+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:38:59.739116+0000","last_peered":"2023-10-09T05:38:59.739116+0000","last_clean":"2023-10-09T05:38:59.739116+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:38:59.739116+0000","last_undegraded":"2023-10-09T05:38:59.739116+0000","last_fullsized":"2023-10-09T05:38:59.739116+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":177,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.734597+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:38:59.734597+0000","last_peered":"2023-10-09T05:38:59.734597+0000","last_clean":"2023-10-09T05:38:59.734597+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:38:59.734597+0000","last_undegraded":"2023-10-09T05:38:59.734597+0000","last_fullsized":"2023-10-09T05:38:59.734597+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":155,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.753352+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:38:59.753352+0000","last_peered":"2023-10-09T05:38:59.753352+0000","last_clean":"2023-10-09T05:38:59.753352+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:38:59.753352+0000","last_undegraded":"2023-10-09T05:38:59.753352+0000","last_fullsized":"2023-10-09T05:38:59.753352+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":140,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.738764+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:38:59.738764+0000","last_peered":"2023-10-09T05:38:59.738764+0000","last_clean":"2023-10-09T05:38:59.738764+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:38:59.738764+0000","last_undegraded":"2023-10-09T05:38:59.738764+0000","last_fullsized":"2023-10-09T05:38:59.738764+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":161,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.734666+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:38:59.734666+0000","last_peered":"2023-10-09T05:38:59.734666+0000","last_clean":"2023-10-09T05:38:59.734666+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:38:59.734666+0000","last_undegraded":"2023-10-09T05:38:59.734666+0000","last_fullsized":"2023-10-09T05:38:59.734666+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":180,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.763397+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:38:59.763397+0000","last_peered":"2023-10-09T05:38:59.763397+0000","last_clean":"2023-10-09T05:38:59.763397+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:38:59.763397+0000","last_undegraded":"2023-10-09T05:38:59.763397+0000","last_fullsized":"2023-10-09T05:38:59.763397+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":175,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.741619+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:38:59.741619+0000","last_peered":"2023-10-09T05:38:59.741619+0000","last_clean":"2023-10-09T05:38:59.741619+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:38:59.741619+0000","last_undegraded":"2023-10-09T05:38:59.741619+0000","last_fullsized":"2023-10-09T05:38:59.741619+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":68,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.734680+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:38:59.734680+0000","last_peered":"2023-10-09T05:38:59.734680+0000","last_clean":"2023-10-09T05:38:59.734680+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:38:59.734680+0000","last_undegraded":"2023-10-09T05:38:59.734680+0000","last_fullsized":"2023-10-09T05:38:59.734680+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":135,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.749205+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:38:59.749205+0000","last_peered":"2023-10-09T05:38:59.749205+0000","last_clean":"2023-10-09T05:38:59.749205+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:38:59.749205+0000","last_undegraded":"2023-10-09T05:38:59.749205+0000","last_fullsized":"2023-10-09T05:38:59.749205+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":197,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.754284+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:38:59.754284+0000","last_peered":"2023-10-09T05:38:59.754284+0000","last_clean":"2023-10-09T05:38:59.754284+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:38:59.754284+0000","last_undegraded":"2023-10-09T05:38:59.754284+0000","last_fullsized":"2023-10-09T05:38:59.754284+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":155,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.753990+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:38:59.753990+0000","last_peered":"2023-10-09T05:38:59.753990+0000","last_clean":"2023-10-09T05:38:59.753990+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:38:59.753990+0000","last_undegraded":"2023-10-09T05:38:59.753990+0000","last_fullsized":"2023-10-09T05:38:59.753990+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":68,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.754039+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:38:59.754039+0000","last_peered":"2023-10-09T05:38:59.754039+0000","last_clean":"2023-10-09T05:38:59.754039+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:38:59.754039+0000","last_undegraded":"2023-10-09T05:38:59.754039+0000","last_fullsized":"2023-10-09T05:38:59.754039+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":185,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.737186+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:38:59.737186+0000","last_peered":"2023-10-09T05:38:59.737186+0000","last_clean":"2023-10-09T05:38:59.737186+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:38:59.737186+0000","last_undegraded":"2023-10-09T05:38:59.737186+0000","last_fullsized":"2023-10-09T05:38:59.737186+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.739557+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:38:59.739557+0000","last_peered":"2023-10-09T05:38:59.739557+0000","last_clean":"2023-10-09T05:38:59.739557+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:38:59.739557+0000","last_undegraded":"2023-10-09T05:38:59.739557+0000","last_fullsized":"2023-10-09T05:38:59.739557+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":179,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.763263+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:38:59.763263+0000","last_peered":"2023-10-09T05:38:59.763263+0000","last_clean":"2023-10-09T05:38:59.763263+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:38:59.763263+0000","last_undegraded":"2023-10-09T05:38:59.763263+0000","last_fullsized":"2023-10-09T05:38:59.763263+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":86,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.753796+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:38:59.753796+0000","last_peered":"2023-10-09T05:38:59.753796+0000","last_clean":"2023-10-09T05:38:59.753796+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:38:59.753796+0000","last_undegraded":"2023-10-09T05:38:59.753796+0000","last_fullsized":"2023-10-09T05:38:59.753796+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":164,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.739043+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:38:59.739043+0000","last_peered":"2023-10-09T05:38:59.739043+0000","last_clean":"2023-10-09T05:38:59.739043+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:38:59.739043+0000","last_undegraded":"2023-10-09T05:38:59.739043+0000","last_fullsized":"2023-10-09T05:38:59.739043+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":164,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.752784+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:38:59.752784+0000","last_peered":"2023-10-09T05:38:59.752784+0000","last_clean":"2023-10-09T05:38:59.752784+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:38:59.752784+0000","last_undegraded":"2023-10-09T05:38:59.752784+0000","last_fullsized":"2023-10-09T05:38:59.752784+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":185,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.735243+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:38:59.735243+0000","last_peered":"2023-10-09T05:38:59.735243+0000","last_clean":"2023-10-09T05:38:59.735243+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:38:59.735243+0000","last_undegraded":"2023-10-09T05:38:59.735243+0000","last_fullsized":"2023-10-09T05:38:59.735243+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"189'6823","reported_seq":6858,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:09.899696+0000","last_change":"2023-10-09T05:37:01.636984+0000","last_active":"2023-10-09T05:39:09.899696+0000","last_peered":"2023-10-09T05:39:09.899696+0000","last_clean":"2023-10-09T05:39:09.899696+0000","last_became_active":"2023-10-09T05:37:01.636609+0000","last_became_peered":"2023-10-09T05:37:01.636609+0000","last_unstale":"2023-10-09T05:39:09.899696+0000","last_undegraded":"2023-10-09T05:39:09.899696+0000","last_fullsized":"2023-10-09T05:39:09.899696+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6823,"log_dups_size":0,"ondisk_log_size":6823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277413888,"num_objects":4233,"num_object_clones":0,"num_object_copies":12699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6823,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":66,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.754616+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:38:59.754616+0000","last_peered":"2023-10-09T05:38:59.754616+0000","last_clean":"2023-10-09T05:38:59.754616+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:38:59.754616+0000","last_undegraded":"2023-10-09T05:38:59.754616+0000","last_fullsized":"2023-10-09T05:38:59.754616+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":164,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.762837+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:38:59.762837+0000","last_peered":"2023-10-09T05:38:59.762837+0000","last_clean":"2023-10-09T05:38:59.762837+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:38:59.762837+0000","last_undegraded":"2023-10-09T05:38:59.762837+0000","last_fullsized":"2023-10-09T05:38:59.762837+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"189'6640","reported_seq":6684,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:06.070370+0000","last_change":"2023-10-09T05:37:36.837344+0000","last_active":"2023-10-09T05:39:06.070370+0000","last_peered":"2023-10-09T05:39:06.070370+0000","last_clean":"2023-10-09T05:37:34.651883+0000","last_became_active":"2023-10-09T05:37:36.832502+0000","last_became_peered":"2023-10-09T05:37:36.832502+0000","last_unstale":"2023-10-09T05:39:06.070370+0000","last_undegraded":"2023-10-09T05:37:36.702797+0000","last_fullsized":"2023-10-09T05:37:36.678583+0000","mapping_epoch":170,"log_start":"166'200","ondisk_log_start":"166'200","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6440,"log_dups_size":0,"ondisk_log_size":6440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:57.419460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285212672,"num_objects":4352,"num_object_clones":0,"num_object_copies":13056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6640,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":1462},{"shards":"1,3,7","objects":1746}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":66,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.754607+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:38:59.754607+0000","last_peered":"2023-10-09T05:38:59.754607+0000","last_clean":"2023-10-09T05:38:59.754607+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:38:59.754607+0000","last_undegraded":"2023-10-09T05:38:59.754607+0000","last_fullsized":"2023-10-09T05:38:59.754607+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":170,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.753763+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:38:59.753763+0000","last_peered":"2023-10-09T05:38:59.753763+0000","last_clean":"2023-10-09T05:38:59.753763+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:38:59.753763+0000","last_undegraded":"2023-10-09T05:38:59.753763+0000","last_fullsized":"2023-10-09T05:38:59.753763+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"189'6839","reported_seq":12935,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:06.497967+0000","last_change":"2023-10-09T05:38:55.317808+0000","last_active":"2023-10-09T05:39:06.497967+0000","last_peered":"2023-10-09T05:39:06.497967+0000","last_clean":"2023-10-09T05:39:06.497967+0000","last_became_active":"2023-10-09T05:38:55.302739+0000","last_became_peered":"2023-10-09T05:38:55.302739+0000","last_unstale":"2023-10-09T05:39:06.497967+0000","last_undegraded":"2023-10-09T05:39:06.497967+0000","last_fullsized":"2023-10-09T05:39:06.497967+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6839,"log_dups_size":0,"ondisk_log_size":6839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:25.379818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286588928,"num_objects":4373,"num_object_clones":0,"num_object_copies":13119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6839,"num_write_kb":358784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":198049792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":66,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.763393+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:38:59.763393+0000","last_peered":"2023-10-09T05:38:59.763393+0000","last_clean":"2023-10-09T05:38:59.763393+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:38:59.763393+0000","last_undegraded":"2023-10-09T05:38:59.763393+0000","last_fullsized":"2023-10-09T05:38:59.763393+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":188,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.753054+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:38:59.753054+0000","last_peered":"2023-10-09T05:38:59.753054+0000","last_clean":"2023-10-09T05:38:59.753054+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:38:59.753054+0000","last_undegraded":"2023-10-09T05:38:59.753054+0000","last_fullsized":"2023-10-09T05:38:59.753054+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"189'6772","reported_seq":7906,"reported_epoch":189,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:08.043675+0000","last_change":"2023-10-09T05:38:53.467497+0000","last_active":"2023-10-09T05:39:08.043675+0000","last_peered":"2023-10-09T05:39:08.043675+0000","last_clean":"2023-10-09T05:37:34.677467+0000","last_became_active":"2023-10-09T05:37:36.803958+0000","last_became_peered":"2023-10-09T05:37:36.803958+0000","last_unstale":"2023-10-09T05:39:08.043675+0000","last_undegraded":"2023-10-09T05:37:36.800519+0000","last_fullsized":"2023-10-09T05:37:36.678690+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6772,"log_dups_size":0,"ondisk_log_size":6772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:40.690185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275120128,"num_objects":4198,"num_object_clones":0,"num_object_copies":12594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":754,"num_objects_misplaced":1601,"num_objects_unfound":0,"num_objects_dirty":4198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6772,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":544,"num_bytes_recovered":35651584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":1843},{"shards":"0,2,7","objects":1601}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":156,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.763111+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:38:59.763111+0000","last_peered":"2023-10-09T05:38:59.763111+0000","last_clean":"2023-10-09T05:38:59.763111+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:38:59.763111+0000","last_undegraded":"2023-10-09T05:38:59.763111+0000","last_fullsized":"2023-10-09T05:38:59.763111+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":141,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.738529+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:38:59.738529+0000","last_peered":"2023-10-09T05:38:59.738529+0000","last_clean":"2023-10-09T05:38:59.738529+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:38:59.738529+0000","last_undegraded":"2023-10-09T05:38:59.738529+0000","last_fullsized":"2023-10-09T05:38:59.738529+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"189'6833","reported_seq":6868,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:09.925450+0000","last_change":"2023-10-09T05:37:01.439878+0000","last_active":"2023-10-09T05:39:09.925450+0000","last_peered":"2023-10-09T05:39:09.925450+0000","last_clean":"2023-10-09T05:39:09.925450+0000","last_became_active":"2023-10-09T05:37:01.439481+0000","last_became_peered":"2023-10-09T05:37:01.439481+0000","last_unstale":"2023-10-09T05:39:09.925450+0000","last_undegraded":"2023-10-09T05:39:09.925450+0000","last_fullsized":"2023-10-09T05:39:09.925450+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6833,"log_dups_size":0,"ondisk_log_size":6833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272171008,"num_objects":4153,"num_object_clones":0,"num_object_copies":12459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6833,"num_write_kb":351552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":159,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.737043+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:38:59.737043+0000","last_peered":"2023-10-09T05:38:59.737043+0000","last_clean":"2023-10-09T05:38:59.737043+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:38:59.737043+0000","last_undegraded":"2023-10-09T05:38:59.737043+0000","last_fullsized":"2023-10-09T05:38:59.737043+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":155,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.754236+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:38:59.754236+0000","last_peered":"2023-10-09T05:38:59.754236+0000","last_clean":"2023-10-09T05:38:59.754236+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:38:59.754236+0000","last_undegraded":"2023-10-09T05:38:59.754236+0000","last_fullsized":"2023-10-09T05:38:59.754236+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"189'6739","reported_seq":6775,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:06.623868+0000","last_change":"2023-10-09T05:37:01.440175+0000","last_active":"2023-10-09T05:39:06.623868+0000","last_peered":"2023-10-09T05:39:06.623868+0000","last_clean":"2023-10-09T05:39:06.623868+0000","last_became_active":"2023-10-09T05:37:01.439846+0000","last_became_peered":"2023-10-09T05:37:01.439846+0000","last_unstale":"2023-10-09T05:39:06.623868+0000","last_undegraded":"2023-10-09T05:39:06.623868+0000","last_fullsized":"2023-10-09T05:39:06.623868+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6739,"log_dups_size":0,"ondisk_log_size":6739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:27.883265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282787840,"num_objects":4315,"num_object_clones":0,"num_object_copies":12945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6739,"num_write_kb":353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":156,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.755639+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:38:59.755639+0000","last_peered":"2023-10-09T05:38:59.755639+0000","last_clean":"2023-10-09T05:38:59.755639+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:38:59.755639+0000","last_undegraded":"2023-10-09T05:38:59.755639+0000","last_fullsized":"2023-10-09T05:38:59.755639+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":170,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.754353+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:38:59.754353+0000","last_peered":"2023-10-09T05:38:59.754353+0000","last_clean":"2023-10-09T05:38:59.754353+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:38:59.754353+0000","last_undegraded":"2023-10-09T05:38:59.754353+0000","last_fullsized":"2023-10-09T05:38:59.754353+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"189'6708","reported_seq":6744,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:06.071652+0000","last_change":"2023-10-09T05:37:01.638178+0000","last_active":"2023-10-09T05:39:06.071652+0000","last_peered":"2023-10-09T05:39:06.071652+0000","last_clean":"2023-10-09T05:39:06.071652+0000","last_became_active":"2023-10-09T05:37:01.637822+0000","last_became_peered":"2023-10-09T05:37:01.637822+0000","last_unstale":"2023-10-09T05:39:06.071652+0000","last_undegraded":"2023-10-09T05:39:06.071652+0000","last_fullsized":"2023-10-09T05:39:06.071652+0000","mapping_epoch":162,"log_start":"166'300","ondisk_log_start":"166'300","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:13.240069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288751616,"num_objects":4406,"num_object_clones":0,"num_object_copies":13218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6708,"num_write_kb":355648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":180,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.753666+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:38:59.753666+0000","last_peered":"2023-10-09T05:38:59.753666+0000","last_clean":"2023-10-09T05:38:59.753666+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:38:59.753666+0000","last_undegraded":"2023-10-09T05:38:59.753666+0000","last_fullsized":"2023-10-09T05:38:59.753666+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":164,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.754612+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:38:59.754612+0000","last_peered":"2023-10-09T05:38:59.754612+0000","last_clean":"2023-10-09T05:38:59.754612+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:38:59.754612+0000","last_undegraded":"2023-10-09T05:38:59.754612+0000","last_fullsized":"2023-10-09T05:38:59.754612+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"189'6926","reported_seq":6962,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:09.937605+0000","last_change":"2023-10-09T05:37:01.438917+0000","last_active":"2023-10-09T05:39:09.937605+0000","last_peered":"2023-10-09T05:39:09.937605+0000","last_clean":"2023-10-09T05:39:09.937605+0000","last_became_active":"2023-10-09T05:37:01.438658+0000","last_became_peered":"2023-10-09T05:37:01.438658+0000","last_unstale":"2023-10-09T05:39:09.937605+0000","last_undegraded":"2023-10-09T05:39:09.937605+0000","last_fullsized":"2023-10-09T05:39:09.937605+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6926,"log_dups_size":0,"ondisk_log_size":6926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270991384,"num_objects":4136,"num_object_clones":0,"num_object_copies":12408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4136,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6926,"num_write_kb":353921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":186,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.734520+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:38:59.734520+0000","last_peered":"2023-10-09T05:38:59.734520+0000","last_clean":"2023-10-09T05:38:59.734520+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:38:59.734520+0000","last_undegraded":"2023-10-09T05:38:59.734520+0000","last_fullsized":"2023-10-09T05:38:59.734520+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":135,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.770460+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:38:59.770460+0000","last_peered":"2023-10-09T05:38:59.770460+0000","last_clean":"2023-10-09T05:38:59.770460+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:38:59.770460+0000","last_undegraded":"2023-10-09T05:38:59.770460+0000","last_fullsized":"2023-10-09T05:38:59.770460+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"189'6821","reported_seq":6858,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:08.030373+0000","last_change":"2023-10-09T05:37:01.440093+0000","last_active":"2023-10-09T05:39:08.030373+0000","last_peered":"2023-10-09T05:39:08.030373+0000","last_clean":"2023-10-09T05:39:08.030373+0000","last_became_active":"2023-10-09T05:37:01.439799+0000","last_became_peered":"2023-10-09T05:37:01.439799+0000","last_unstale":"2023-10-09T05:39:08.030373+0000","last_undegraded":"2023-10-09T05:39:08.030373+0000","last_fullsized":"2023-10-09T05:39:08.030373+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6821,"log_dups_size":0,"ondisk_log_size":6821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:28.734471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280821760,"num_objects":4285,"num_object_clones":0,"num_object_copies":12855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6821,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":723,"reported_epoch":188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:38:59.751218+0000","last_change":"2023-10-09T05:37:36.736822+0000","last_active":"2023-10-09T05:38:59.751218+0000","last_peered":"2023-10-09T05:38:59.751218+0000","last_clean":"2023-10-09T05:37:30.792074+0000","last_became_active":"2023-10-09T05:37:36.736273+0000","last_became_peered":"2023-10-09T05:37:36.736273+0000","last_unstale":"2023-10-09T05:38:59.751218+0000","last_undegraded":"2023-10-09T05:37:36.682657+0000","last_fullsized":"2023-10-09T05:37:36.682189+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:23.350354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":171,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.763178+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:38:59.763178+0000","last_peered":"2023-10-09T05:38:59.763178+0000","last_clean":"2023-10-09T05:38:59.763178+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:38:59.763178+0000","last_undegraded":"2023-10-09T05:38:59.763178+0000","last_fullsized":"2023-10-09T05:38:59.763178+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":121,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.749132+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:38:59.749132+0000","last_peered":"2023-10-09T05:38:59.749132+0000","last_clean":"2023-10-09T05:38:59.749132+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:38:59.749132+0000","last_undegraded":"2023-10-09T05:38:59.749132+0000","last_fullsized":"2023-10-09T05:38:59.749132+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"189'6769","reported_seq":12695,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:06.104895+0000","last_change":"2023-10-09T05:38:10.132330+0000","last_active":"2023-10-09T05:39:06.104895+0000","last_peered":"2023-10-09T05:39:06.104895+0000","last_clean":"2023-10-09T05:39:06.104895+0000","last_became_active":"2023-10-09T05:38:10.127080+0000","last_became_peered":"2023-10-09T05:38:10.127080+0000","last_unstale":"2023-10-09T05:39:06.104895+0000","last_undegraded":"2023-10-09T05:39:06.104895+0000","last_fullsized":"2023-10-09T05:39:06.104895+0000","mapping_epoch":179,"log_start":"166'300","ondisk_log_start":"166'300","created":162,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6469,"log_dups_size":0,"ondisk_log_size":6469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:12.659227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286982144,"num_objects":4379,"num_object_clones":0,"num_object_copies":13137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6769,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":192413696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":625,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.763008+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:38:59.763008+0000","last_peered":"2023-10-09T05:38:59.763008+0000","last_clean":"2023-10-09T05:38:59.763008+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:38:59.763008+0000","last_undegraded":"2023-10-09T05:38:59.763008+0000","last_fullsized":"2023-10-09T05:38:59.763008+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":156,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.754082+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:38:59.754082+0000","last_peered":"2023-10-09T05:38:59.754082+0000","last_clean":"2023-10-09T05:38:59.754082+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:38:59.754082+0000","last_undegraded":"2023-10-09T05:38:59.754082+0000","last_fullsized":"2023-10-09T05:38:59.754082+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":168,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.763213+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:38:59.763213+0000","last_peered":"2023-10-09T05:38:59.763213+0000","last_clean":"2023-10-09T05:38:59.763213+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:38:59.763213+0000","last_undegraded":"2023-10-09T05:38:59.763213+0000","last_fullsized":"2023-10-09T05:38:59.763213+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"189'6638","reported_seq":6684,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:05.556309+0000","last_change":"2023-10-09T05:37:40.649912+0000","last_active":"2023-10-09T05:39:05.556309+0000","last_peered":"2023-10-09T05:39:05.556309+0000","last_clean":"2023-10-09T05:37:34.677100+0000","last_became_active":"2023-10-09T05:37:40.593033+0000","last_became_peered":"2023-10-09T05:37:40.593033+0000","last_unstale":"2023-10-09T05:39:05.556309+0000","last_undegraded":"2023-10-09T05:37:40.546647+0000","last_fullsized":"2023-10-09T05:37:40.543498+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6638,"log_dups_size":0,"ondisk_log_size":6638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285212672,"num_objects":4352,"num_object_clones":0,"num_object_copies":13056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6638,"num_write_kb":351680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":1502},{"shards":"3,6,7","objects":1707}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":502,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.777754+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:38:59.777754+0000","last_peered":"2023-10-09T05:38:59.777754+0000","last_clean":"2023-10-09T05:38:59.777754+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:38:59.777754+0000","last_undegraded":"2023-10-09T05:38:59.777754+0000","last_fullsized":"2023-10-09T05:38:59.777754+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":136,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.749708+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:38:59.749708+0000","last_peered":"2023-10-09T05:38:59.749708+0000","last_clean":"2023-10-09T05:38:59.749708+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:38:59.749708+0000","last_undegraded":"2023-10-09T05:38:59.749708+0000","last_fullsized":"2023-10-09T05:38:59.749708+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":166,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.892826+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:39:00.892826+0000","last_peered":"2023-10-09T05:39:00.892826+0000","last_clean":"2023-10-09T05:39:00.892826+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:39:00.892826+0000","last_undegraded":"2023-10-09T05:39:00.892826+0000","last_fullsized":"2023-10-09T05:39:00.892826+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"189'6854","reported_seq":6889,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:09.821829+0000","last_change":"2023-10-09T05:37:01.438236+0000","last_active":"2023-10-09T05:39:09.821829+0000","last_peered":"2023-10-09T05:39:09.821829+0000","last_clean":"2023-10-09T05:39:09.821829+0000","last_became_active":"2023-10-09T05:37:01.437917+0000","last_became_peered":"2023-10-09T05:37:01.437917+0000","last_unstale":"2023-10-09T05:39:09.821829+0000","last_undegraded":"2023-10-09T05:39:09.821829+0000","last_fullsized":"2023-10-09T05:39:09.821829+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6854,"log_dups_size":0,"ondisk_log_size":6854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276561920,"num_objects":4220,"num_object_clones":0,"num_object_copies":12660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6854,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":773,"reported_epoch":188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:38:59.763208+0000","last_change":"2023-10-09T05:37:36.709131+0000","last_active":"2023-10-09T05:38:59.763208+0000","last_peered":"2023-10-09T05:38:59.763208+0000","last_clean":"2023-10-09T05:37:30.688708+0000","last_became_active":"2023-10-09T05:37:36.708682+0000","last_became_peered":"2023-10-09T05:37:36.708682+0000","last_unstale":"2023-10-09T05:38:59.763208+0000","last_undegraded":"2023-10-09T05:37:36.683823+0000","last_fullsized":"2023-10-09T05:37:36.683109+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":336,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.742856+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:38:59.742856+0000","last_peered":"2023-10-09T05:38:59.742856+0000","last_clean":"2023-10-09T05:38:59.742856+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:38:59.742856+0000","last_undegraded":"2023-10-09T05:38:59.742856+0000","last_fullsized":"2023-10-09T05:38:59.742856+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":152,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.735428+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:38:59.735428+0000","last_peered":"2023-10-09T05:38:59.735428+0000","last_clean":"2023-10-09T05:38:59.735428+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:38:59.735428+0000","last_undegraded":"2023-10-09T05:38:59.735428+0000","last_fullsized":"2023-10-09T05:38:59.735428+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":162,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.753330+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:38:59.753330+0000","last_peered":"2023-10-09T05:38:59.753330+0000","last_clean":"2023-10-09T05:38:59.753330+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:38:59.753330+0000","last_undegraded":"2023-10-09T05:38:59.753330+0000","last_fullsized":"2023-10-09T05:38:59.753330+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"189'6781","reported_seq":6818,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:08.702384+0000","last_change":"2023-10-09T05:37:01.440172+0000","last_active":"2023-10-09T05:39:08.702384+0000","last_peered":"2023-10-09T05:39:08.702384+0000","last_clean":"2023-10-09T05:39:08.702384+0000","last_became_active":"2023-10-09T05:37:01.439932+0000","last_became_peered":"2023-10-09T05:37:01.439932+0000","last_unstale":"2023-10-09T05:39:08.702384+0000","last_undegraded":"2023-10-09T05:39:08.702384+0000","last_fullsized":"2023-10-09T05:39:08.702384+0000","mapping_epoch":162,"log_start":"166'300","ondisk_log_start":"166'300","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6481,"log_dups_size":0,"ondisk_log_size":6481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:57.826704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273874944,"num_objects":4179,"num_object_clones":0,"num_object_copies":12537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6781,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":714,"reported_epoch":188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:38:59.749324+0000","last_change":"2023-10-09T05:37:36.748736+0000","last_active":"2023-10-09T05:38:59.749324+0000","last_peered":"2023-10-09T05:38:59.749324+0000","last_clean":"2023-10-09T05:22:39.601058+0000","last_became_active":"2023-10-09T05:37:36.748304+0000","last_became_peered":"2023-10-09T05:37:36.748304+0000","last_unstale":"2023-10-09T05:38:59.749324+0000","last_undegraded":"2023-10-09T05:37:36.696424+0000","last_fullsized":"2023-10-09T05:37:36.695987+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:18.784578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":141,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.739000+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:38:59.739000+0000","last_peered":"2023-10-09T05:38:59.739000+0000","last_clean":"2023-10-09T05:38:59.739000+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:38:59.739000+0000","last_undegraded":"2023-10-09T05:38:59.739000+0000","last_fullsized":"2023-10-09T05:38:59.739000+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":156,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.739295+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:38:59.739295+0000","last_peered":"2023-10-09T05:38:59.739295+0000","last_clean":"2023-10-09T05:38:59.739295+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:38:59.739295+0000","last_undegraded":"2023-10-09T05:38:59.739295+0000","last_fullsized":"2023-10-09T05:38:59.739295+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"189'6811","reported_seq":6848,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:08.689288+0000","last_change":"2023-10-09T05:37:01.636967+0000","last_active":"2023-10-09T05:39:08.689288+0000","last_peered":"2023-10-09T05:39:08.689288+0000","last_clean":"2023-10-09T05:39:08.689288+0000","last_became_active":"2023-10-09T05:37:01.636602+0000","last_became_peered":"2023-10-09T05:37:01.636602+0000","last_unstale":"2023-10-09T05:39:08.689288+0000","last_undegraded":"2023-10-09T05:39:08.689288+0000","last_fullsized":"2023-10-09T05:39:08.689288+0000","mapping_epoch":162,"log_start":"166'400","ondisk_log_start":"166'400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6411,"log_dups_size":0,"ondisk_log_size":6411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:11.668143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280559616,"num_objects":4281,"num_object_clones":0,"num_object_copies":12843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6811,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":377,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.739655+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:38:59.739655+0000","last_peered":"2023-10-09T05:38:59.739655+0000","last_clean":"2023-10-09T05:38:59.739655+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:38:59.739655+0000","last_undegraded":"2023-10-09T05:38:59.739655+0000","last_fullsized":"2023-10-09T05:38:59.739655+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":136,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.751515+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:38:59.751515+0000","last_peered":"2023-10-09T05:38:59.751515+0000","last_clean":"2023-10-09T05:38:59.751515+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:38:59.751515+0000","last_undegraded":"2023-10-09T05:38:59.751515+0000","last_fullsized":"2023-10-09T05:38:59.751515+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":150,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.738780+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:38:59.738780+0000","last_peered":"2023-10-09T05:38:59.738780+0000","last_clean":"2023-10-09T05:38:59.738780+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:38:59.738780+0000","last_undegraded":"2023-10-09T05:38:59.738780+0000","last_fullsized":"2023-10-09T05:38:59.738780+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"189'6958","reported_seq":6993,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:09.882342+0000","last_change":"2023-10-09T05:37:01.440074+0000","last_active":"2023-10-09T05:39:09.882342+0000","last_peered":"2023-10-09T05:39:09.882342+0000","last_clean":"2023-10-09T05:39:09.882342+0000","last_became_active":"2023-10-09T05:37:01.439808+0000","last_became_peered":"2023-10-09T05:37:01.439808+0000","last_unstale":"2023-10-09T05:39:09.882342+0000","last_undegraded":"2023-10-09T05:39:09.882342+0000","last_fullsized":"2023-10-09T05:39:09.882342+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6958,"log_dups_size":0,"ondisk_log_size":6958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272629760,"num_objects":4160,"num_object_clones":0,"num_object_copies":12480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6958,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":187,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.735149+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:38:59.735149+0000","last_peered":"2023-10-09T05:38:59.735149+0000","last_clean":"2023-10-09T05:38:59.735149+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:38:59.735149+0000","last_undegraded":"2023-10-09T05:38:59.735149+0000","last_fullsized":"2023-10-09T05:38:59.735149+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":577,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.891999+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:39:00.891999+0000","last_peered":"2023-10-09T05:39:00.891999+0000","last_clean":"2023-10-09T05:39:00.891999+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:39:00.891999+0000","last_undegraded":"2023-10-09T05:39:00.891999+0000","last_fullsized":"2023-10-09T05:39:00.891999+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":180,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.734744+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:38:59.734744+0000","last_peered":"2023-10-09T05:38:59.734744+0000","last_clean":"2023-10-09T05:38:59.734744+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:38:59.734744+0000","last_undegraded":"2023-10-09T05:38:59.734744+0000","last_fullsized":"2023-10-09T05:38:59.734744+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"189'6475","reported_seq":6517,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:05.556259+0000","last_change":"2023-10-09T05:37:36.839957+0000","last_active":"2023-10-09T05:39:05.556259+0000","last_peered":"2023-10-09T05:39:05.556259+0000","last_clean":"2023-10-09T05:37:34.656587+0000","last_became_active":"2023-10-09T05:37:36.833510+0000","last_became_peered":"2023-10-09T05:37:36.833510+0000","last_unstale":"2023-10-09T05:39:05.556259+0000","last_undegraded":"2023-10-09T05:37:36.701190+0000","last_fullsized":"2023-10-09T05:37:36.681999+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6475,"log_dups_size":0,"ondisk_log_size":6475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282656768,"num_objects":4313,"num_object_clones":0,"num_object_copies":12939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2877,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6475,"num_write_kb":345216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":1436},{"shards":"2,6,7","objects":1796}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":673,"reported_epoch":188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:38:59.753206+0000","last_change":"2023-10-09T05:37:36.917512+0000","last_active":"2023-10-09T05:38:59.753206+0000","last_peered":"2023-10-09T05:38:59.753206+0000","last_clean":"2023-10-09T05:37:30.688967+0000","last_became_active":"2023-10-09T05:37:36.700218+0000","last_became_peered":"2023-10-09T05:37:36.700218+0000","last_unstale":"2023-10-09T05:38:59.753206+0000","last_undegraded":"2023-10-09T05:37:36.917335+0000","last_fullsized":"2023-10-09T05:37:36.682410+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":144,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.739217+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:38:59.739217+0000","last_peered":"2023-10-09T05:38:59.739217+0000","last_clean":"2023-10-09T05:38:59.739217+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:38:59.739217+0000","last_undegraded":"2023-10-09T05:38:59.739217+0000","last_fullsized":"2023-10-09T05:38:59.739217+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":140,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.753651+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:38:59.753651+0000","last_peered":"2023-10-09T05:38:59.753651+0000","last_clean":"2023-10-09T05:38:59.753651+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:38:59.753651+0000","last_undegraded":"2023-10-09T05:38:59.753651+0000","last_fullsized":"2023-10-09T05:38:59.753651+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":152,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.738704+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:38:59.738704+0000","last_peered":"2023-10-09T05:38:59.738704+0000","last_clean":"2023-10-09T05:38:59.738704+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:38:59.738704+0000","last_undegraded":"2023-10-09T05:38:59.738704+0000","last_fullsized":"2023-10-09T05:38:59.738704+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":180,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.775267+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:38:59.775267+0000","last_peered":"2023-10-09T05:38:59.775267+0000","last_clean":"2023-10-09T05:38:59.775267+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:38:59.775267+0000","last_undegraded":"2023-10-09T05:38:59.775267+0000","last_fullsized":"2023-10-09T05:38:59.775267+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":158,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.763092+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:38:59.763092+0000","last_peered":"2023-10-09T05:38:59.763092+0000","last_clean":"2023-10-09T05:38:59.763092+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:38:59.763092+0000","last_undegraded":"2023-10-09T05:38:59.763092+0000","last_fullsized":"2023-10-09T05:38:59.763092+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":170,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.754121+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:38:59.754121+0000","last_peered":"2023-10-09T05:38:59.754121+0000","last_clean":"2023-10-09T05:38:59.754121+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:38:59.754121+0000","last_undegraded":"2023-10-09T05:38:59.754121+0000","last_fullsized":"2023-10-09T05:38:59.754121+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":143,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.763118+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:38:59.763118+0000","last_peered":"2023-10-09T05:38:59.763118+0000","last_clean":"2023-10-09T05:38:59.763118+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:38:59.763118+0000","last_undegraded":"2023-10-09T05:38:59.763118+0000","last_fullsized":"2023-10-09T05:38:59.763118+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":170,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.753883+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:38:59.753883+0000","last_peered":"2023-10-09T05:38:59.753883+0000","last_clean":"2023-10-09T05:38:59.753883+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:38:59.753883+0000","last_undegraded":"2023-10-09T05:38:59.753883+0000","last_fullsized":"2023-10-09T05:38:59.753883+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":167,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.735839+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:38:59.735839+0000","last_peered":"2023-10-09T05:38:59.735839+0000","last_clean":"2023-10-09T05:38:59.735839+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:38:59.735839+0000","last_undegraded":"2023-10-09T05:38:59.735839+0000","last_fullsized":"2023-10-09T05:38:59.735839+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":180,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.741549+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:38:59.741549+0000","last_peered":"2023-10-09T05:38:59.741549+0000","last_clean":"2023-10-09T05:38:59.741549+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:38:59.741549+0000","last_undegraded":"2023-10-09T05:38:59.741549+0000","last_fullsized":"2023-10-09T05:38:59.741549+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":144,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.738853+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:38:59.738853+0000","last_peered":"2023-10-09T05:38:59.738853+0000","last_clean":"2023-10-09T05:38:59.738853+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:38:59.738853+0000","last_undegraded":"2023-10-09T05:38:59.738853+0000","last_fullsized":"2023-10-09T05:38:59.738853+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":155,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.753653+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:38:59.753653+0000","last_peered":"2023-10-09T05:38:59.753653+0000","last_clean":"2023-10-09T05:38:59.753653+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:38:59.753653+0000","last_undegraded":"2023-10-09T05:38:59.753653+0000","last_fullsized":"2023-10-09T05:38:59.753653+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4478337048,"num_objects":68335,"num_object_clones":0,"num_object_copies":205005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9371,"num_objects_misplaced":1601,"num_objects_unfound":0,"num_objects_dirty":68335,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":108387,"num_write_kb":5655041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6503,"num_bytes_recovered":426180608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14047719424,"data_stored":14047707208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106887,"ondisk_log_size":106887,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846647,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626672,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2451988,"kb_used_data":2142476,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91293164,"statfs":{"total":95995035648,"available":93484199936,"internally_reserved":0,"allocated":2193895424,"data_stored":2193376079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":4.3609999999999998,"15min":1.819},"min":{"1min":0.23599999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.8869999999999996,"5min":77.024000000000001,"15min":77.024000000000001},"last":6.2480000000000002},{"interface":"front","average":{"1min":0.82499999999999996,"5min":4.0880000000000001,"15min":1.762},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.3290000000000002,"5min":79.305999999999997,"15min":79.305999999999997},"last":1.4219999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":2.677,"15min":1.3799999999999999},"min":{"1min":0.30099999999999999,"5min":0.25,"15min":0.25},"max":{"1min":9.8670000000000009,"5min":69.667000000000002,"15min":69.667000000000002},"last":3},{"interface":"front","average":{"1min":1.972,"5min":2.8849999999999998,"15min":1.4670000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":10.624000000000001,"5min":69.697999999999993,"15min":69.697999999999993},"last":2.972}]},{"osd":2,"last update":"Mon Oct 9 05:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":3.8540000000000001,"15min":2.0569999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":12.99,"5min":70.219999999999999,"15min":70.219999999999999},"last":2.399},{"interface":"front","average":{"1min":1.9710000000000001,"5min":3.7040000000000002,"15min":2.0190000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.978,"5min":70.230000000000004,"15min":70.230000000000004},"last":2.4199999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":2.6160000000000001,"15min":1.661},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":29.885000000000002,"5min":68.920000000000002,"15min":68.920000000000002},"last":2.1389999999999998},{"interface":"front","average":{"1min":3.0230000000000001,"5min":2.794,"15min":1.76},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":29.771000000000001,"5min":68.846000000000004,"15min":68.846000000000004},"last":2.052}]},{"osd":4,"last update":"Mon Oct 9 05:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":3.1379999999999999,"15min":1.401},"min":{"1min":0.248,"5min":0.245,"15min":0.23400000000000001},"max":{"1min":0.58599999999999997,"5min":125.64,"15min":125.64},"last":0.308},{"interface":"front","average":{"1min":0.41499999999999998,"5min":2.7709999999999999,"15min":1.276},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":1.2709999999999999,"5min":125.625,"15min":125.625},"last":0.45700000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":2.871,"15min":1.3340000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.5510000000000002,"5min":127.35899999999999,"15min":127.35899999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.5110000000000001,"15min":1.23},"min":{"1min":0.29299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.8769999999999998,"5min":128.93799999999999,"15min":128.93799999999999},"last":0.40000000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":2.0800000000000001,"15min":2.7919999999999998},"min":{"1min":0.308,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.2789999999999999,"5min":12.074999999999999,"15min":12.074999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.349,"15min":1.8440000000000001},"min":{"1min":0.33800000000000002,"5min":0.317,"15min":0.317},"max":{"1min":1.343,"5min":10.083,"15min":10.083},"last":1.1419999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958634,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1728048,"kb_used_data":1515880,"kb_used_omap":7,"kb_used_meta":212152,"kb_avail":92017104,"statfs":{"total":95995035648,"available":94225514496,"internally_reserved":0,"allocated":1552261120,"data_stored":1551542701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":217244174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2120000000000002,"5min":5.5830000000000002,"15min":2.1520000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.185},"max":{"1min":9.9819999999999993,"5min":207.858,"15min":207.858},"last":0.46300000000000002},{"interface":"front","average":{"1min":6.5330000000000004,"5min":6.7000000000000002,"15min":2.5499999999999998},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":62.880000000000003,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.40799999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:38:08 2023","interfaces":[{"interface":"back","average":{"1min":10.404,"5min":6.7439999999999998,"15min":2.4900000000000002},"min":{"1min":0.23300000000000001,"5min":0.184,"15min":0.184},"max":{"1min":101.367,"5min":101.367,"15min":101.367},"last":0.438},{"interface":"front","average":{"1min":9.5120000000000005,"5min":6.2039999999999997,"15min":2.355},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":97.286000000000001,"5min":98.409999999999997,"15min":98.409999999999997},"last":0.42499999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":4.234,"5min":3.0209999999999999,"15min":1.3640000000000001},"min":{"1min":0.24199999999999999,"5min":0.223,"15min":0.223},"max":{"1min":33.146000000000001,"5min":83.885000000000005,"15min":83.885000000000005},"last":42.445999999999998},{"interface":"front","average":{"1min":8.5860000000000003,"5min":3.8889999999999998,"15min":1.619},"min":{"1min":0.26300000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":62.929000000000002,"5min":84.641999999999996,"15min":84.641999999999996},"last":42.243000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":5.9870000000000001,"5min":4.1109999999999998,"15min":1.833},"min":{"1min":0.432,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":63.116,"5min":141.32900000000001,"15min":141.32900000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.675,"5min":2.8999999999999999,"15min":1.415},"min":{"1min":0.34499999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.5549999999999997,"5min":141.15899999999999,"15min":141.15899999999999},"last":0.61399999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":4.0350000000000001,"15min":1.7929999999999999},"min":{"1min":0.39000000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.0009999999999994,"5min":235.18700000000001,"15min":235.18700000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":6.0140000000000002,"5min":4.9180000000000001,"15min":2.1819999999999999},"min":{"1min":0.46600000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":63.162999999999997,"5min":235.79300000000001,"15min":235.79300000000001},"last":0.68700000000000006}]},{"osd":6,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":1.8220000000000001,"15min":1.075},"min":{"1min":0.42599999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.0289999999999999,"5min":10.333,"15min":10.333},"last":0.60299999999999998},{"interface":"front","average":{"1min":6.2220000000000004,"5min":2.75,"15min":1.393},"min":{"1min":0.5,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":63.198,"5min":63.198,"15min":63.198},"last":0.58899999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":2.7949999999999999,"15min":2.2639999999999998},"min":{"1min":0.45700000000000002,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.6849999999999996,"5min":126.68300000000001,"15min":126.68300000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.6439999999999999,"5min":2.911,"15min":2.3559999999999999},"min":{"1min":0.48499999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.6180000000000003,"5min":126.738,"15min":126.738},"last":0.69799999999999995}]}]},{"osd":0,"up_from":88,"seq":377957122160,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1438664,"kb_used_data":1096576,"kb_used_omap":8,"kb_used_meta":342071,"kb_avail":92306488,"statfs":{"total":95995035648,"available":94521843712,"internally_reserved":0,"allocated":1122893824,"data_stored":1122152765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":350281480},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":1.8959999999999999,"15min":0.93799999999999994},"min":{"1min":0.20499999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":14.964,"5min":14.964,"15min":14.964},"last":0.499},{"interface":"front","average":{"1min":3.879,"5min":2.2850000000000001,"15min":1.0820000000000001},"min":{"1min":0.26100000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":11.315,"5min":11.315,"15min":11.315},"last":0.57399999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:38:13 2023","interfaces":[{"interface":"back","average":{"1min":3.0720000000000001,"5min":4.3079999999999998,"15min":1.6830000000000001},"min":{"1min":0.22500000000000001,"5min":0.214,"15min":0.214},"max":{"1min":18.792999999999999,"5min":79.911000000000001,"15min":79.911000000000001},"last":4.29},{"interface":"front","average":{"1min":1.7629999999999999,"5min":3.7599999999999998,"15min":1.5189999999999999},"min":{"1min":0.23300000000000001,"5min":0.218,"15min":0.218},"max":{"1min":9.2159999999999993,"5min":80.105000000000004,"15min":80.105000000000004},"last":4.1660000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":6.5289999999999999,"5min":3.0329999999999999,"15min":1.3149999999999999},"min":{"1min":0.48799999999999999,"5min":0.23999999999999999,"15min":0.22800000000000001},"max":{"1min":31.594000000000001,"5min":31.594000000000001,"15min":31.594000000000001},"last":0.316},{"interface":"front","average":{"1min":6.4569999999999999,"5min":3.1379999999999999,"15min":1.3879999999999999},"min":{"1min":0.38500000000000001,"5min":0.222,"15min":0.222},"max":{"1min":31.641999999999999,"5min":31.641999999999999,"15min":31.641999999999999},"last":4.5289999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:38:13 2023","interfaces":[{"interface":"back","average":{"1min":4.3879999999999999,"5min":2.0470000000000002,"15min":1.1319999999999999},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":69.082999999999998,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.55200000000000005},{"interface":"front","average":{"1min":4.8040000000000003,"5min":2.4980000000000002,"15min":1.306},"min":{"1min":0.38,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":69.102000000000004,"5min":69.102000000000004,"15min":69.102000000000004},"last":0.42099999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:38:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.8480000000000001,"15min":1.0629999999999999},"min":{"1min":0.33800000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.3150000000000004,"5min":13.632,"15min":13.632},"last":115.175},{"interface":"front","average":{"1min":1.4590000000000001,"5min":1.893,"15min":1.081},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":7.2729999999999997,"5min":13.676,"15min":13.676},"last":115.502}]},{"osd":6,"last update":"Mon Oct 9 05:38:13 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.7889999999999999,"15min":1.0600000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.1749999999999998,"5min":9.9809999999999999,"15min":9.9809999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.925,"5min":1.9390000000000001,"15min":1.099},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.0999999999999996,"5min":8.0999999999999996,"15min":8.0999999999999996},"last":4.3289999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":1.409,"15min":1.355},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.5499999999999998,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.59399999999999997},{"interface":"front","average":{"1min":2.649,"5min":2.206,"15min":2.3279999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":8.2490000000000006,"5min":11.068,"15min":11.068},"last":0.53600000000000003}]}]},{"osd":2,"up_from":98,"seq":420906795107,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2747248,"kb_used_data":2397416,"kb_used_omap":11,"kb_used_meta":349428,"kb_avail":90997904,"statfs":{"total":95995035648,"available":93181853696,"internally_reserved":0,"allocated":2454953984,"data_stored":2454180002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":357814869},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":2.4329999999999998,"15min":1.0589999999999999},"min":{"1min":0.25600000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":7.6479999999999997,"5min":32.122,"15min":32.122},"last":0.23400000000000001},{"interface":"front","average":{"1min":2.7080000000000002,"5min":2.0510000000000002,"15min":0.94799999999999995},"min":{"1min":0.23799999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":13.412000000000001,"5min":34.066000000000003,"15min":34.066000000000003},"last":0.25900000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:38:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7189999999999999,"5min":2.7989999999999999,"15min":1.198},"min":{"1min":0.26800000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":13.784000000000001,"5min":98.459999999999994,"15min":98.459999999999994},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.411,"5min":2.452,"15min":1.075},"min":{"1min":0.29699999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":4.9640000000000004,"5min":98.397000000000006,"15min":98.397000000000006},"last":0.28299999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":3.5640000000000001,"5min":3.0049999999999999,"15min":1.2769999999999999},"min":{"1min":0.41699999999999998,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":12.385999999999999,"5min":45.057000000000002,"15min":45.057000000000002},"last":150.65899999999999},{"interface":"front","average":{"1min":3.2970000000000002,"5min":3.3300000000000001,"15min":1.4410000000000001},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.942,"5min":45.813000000000002,"15min":45.813000000000002},"last":150.851}]},{"osd":4,"last update":"Mon Oct 9 05:38:10 2023","interfaces":[{"interface":"back","average":{"1min":6.4790000000000001,"5min":2.7989999999999999,"15min":1.3220000000000001},"min":{"1min":0.379,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":115.84999999999999,"5min":115.84999999999999,"15min":115.84999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":6.593,"5min":3.2480000000000002,"15min":1.5189999999999999},"min":{"1min":0.39000000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":116.149,"5min":116.149,"15min":116.149},"last":0.42699999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:38:10 2023","interfaces":[{"interface":"back","average":{"1min":3.367,"5min":2.367,"15min":1.218},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":24.597999999999999,"5min":24.597999999999999,"15min":24.597999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.3199999999999998,"5min":2.056,"15min":1.0940000000000001},"min":{"1min":0.36799999999999999,"5min":0.315,"15min":0.315},"max":{"1min":23.919,"5min":23.919,"15min":23.919},"last":0.374}]},{"osd":6,"last update":"Mon Oct 9 05:38:10 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":2.2440000000000002,"15min":1.169},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.1520000000000001,"5min":20.503,"15min":20.503},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.2269999999999999,"5min":1.9390000000000001,"15min":1.103},"min":{"1min":0.39600000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.2100000000000009,"5min":9.4049999999999994,"15min":9.4049999999999994},"last":0.52300000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:38:49 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":2.883,"15min":2.6429999999999998},"min":{"1min":0.42599999999999999,"5min":0.374,"15min":0.374},"max":{"1min":8.9819999999999993,"5min":24.030999999999999,"15min":24.030999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":3.3839999999999999,"5min":2.4950000000000001,"15min":1.984},"min":{"1min":0.47299999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.2569999999999997,"5min":24.178000000000001,"15min":24.178000000000001},"last":0.55500000000000005}]}]},{"osd":3,"up_from":103,"seq":442381631581,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3163112,"kb_used_data":2973024,"kb_used_omap":4,"kb_used_meta":189883,"kb_avail":90582040,"statfs":{"total":95995035648,"available":92756008960,"internally_reserved":0,"allocated":3044376576,"data_stored":3043665199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":194440798},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,2,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0840000000000001,"5min":2.6869999999999998,"15min":1.1970000000000001},"min":{"1min":0.26100000000000001,"5min":0.183,"15min":0.183},"max":{"1min":13.997,"5min":56.094000000000001,"15min":56.094000000000001},"last":3.7970000000000002},{"interface":"front","average":{"1min":3.7799999999999998,"5min":2.9700000000000002,"15min":1.296},"min":{"1min":0.221,"5min":0.221,"15min":0.214},"max":{"1min":15.593,"5min":57.255000000000003,"15min":57.255000000000003},"last":10.031000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0059999999999998,"5min":3.5619999999999998,"15min":1.4690000000000001},"min":{"1min":0.35399999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":13.932,"5min":83.921999999999997,"15min":83.921999999999997},"last":3.8420000000000001},{"interface":"front","average":{"1min":2.5019999999999998,"5min":3.4660000000000002,"15min":1.4119999999999999},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":7.1580000000000004,"5min":83.707999999999998,"15min":83.707999999999998},"last":6.8739999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":6.2699999999999996,"5min":4.5549999999999997,"15min":2.2410000000000001},"min":{"1min":0.25,"5min":0.215,"15min":0.215},"max":{"1min":60.722000000000001,"5min":95.320999999999998,"15min":117.23},"last":3.8660000000000001},{"interface":"front","average":{"1min":7.0670000000000002,"5min":4.6360000000000001,"15min":2.319},"min":{"1min":0.32200000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":60.469999999999999,"5min":94.170000000000002,"15min":117.45999999999999},"last":1.891}]},{"osd":4,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":2.3530000000000002,"15min":1.329},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.1709999999999994,"5min":45.779000000000003,"15min":45.779000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.9239999999999999,"5min":2.2570000000000001,"15min":1.21},"min":{"1min":0.35499999999999998,"5min":0.313,"15min":0.313},"max":{"1min":13.734,"5min":45.735999999999997,"15min":45.735999999999997},"last":3.8959999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":2.1400000000000001,"15min":1.165},"min":{"1min":0.433,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0800000000000001,"5min":47.948999999999998,"15min":47.948999999999998},"last":3.577},{"interface":"front","average":{"1min":1.9259999999999999,"5min":2.3730000000000002,"15min":1.264},"min":{"1min":0.38300000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":13.864000000000001,"5min":47.968000000000004,"15min":47.968000000000004},"last":3.9180000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":2.0499999999999998,"15min":1.175},"min":{"1min":0.41199999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":13.831,"5min":13.831,"15min":13.831},"last":3.9390000000000001},{"interface":"front","average":{"1min":1.944,"5min":1.6850000000000001,"15min":0.98299999999999998},"min":{"1min":0.53000000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.117,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":3.6160000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:38:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2869999999999999,"5min":2.3300000000000001,"15min":2.2829999999999999},"min":{"1min":0.46899999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":9.1319999999999997,"5min":11.407,"15min":11.407},"last":0.57699999999999996},{"interface":"front","average":{"1min":3.3050000000000002,"5min":2.2440000000000002,"15min":2.056},"min":{"1min":0.443,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.2949999999999999,"5min":11.432,"15min":11.432},"last":0.59899999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019129,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1646548,"kb_used_data":1345228,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92098604,"statfs":{"total":95995035648,"available":94308970496,"internally_reserved":0,"allocated":1377513472,"data_stored":1376993871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:17 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":2.3769999999999998,"15min":1.1399999999999999},"min":{"1min":0.25700000000000001,"5min":0.214,"15min":0.214},"max":{"1min":2.9180000000000001,"5min":36.406999999999996,"15min":36.406999999999996},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.0649999999999999,"5min":2.7480000000000002,"15min":1.282},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.3010000000000002,"5min":36.454999999999998,"15min":36.454999999999998},"last":0.28699999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":1.9430000000000001,"15min":1.0169999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.9550000000000001,"5min":31.073,"15min":31.073},"last":1.802},{"interface":"front","average":{"1min":1.421,"5min":1.968,"15min":1.0600000000000001},"min":{"1min":0.28699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.0109999999999992,"5min":31.100000000000001,"15min":31.100000000000001},"last":0.58499999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:38:13 2023","interfaces":[{"interface":"back","average":{"1min":11.664999999999999,"5min":8.2460000000000004,"15min":3.093},"min":{"1min":0.314,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":196.93700000000001,"5min":196.93700000000001,"15min":196.93700000000001},"last":1.778},{"interface":"front","average":{"1min":11.625999999999999,"5min":8.0389999999999997,"15min":3.0680000000000001},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":197.08099999999999,"5min":197.08099999999999,"15min":197.08099999999999},"last":0.42099999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":23.963000000000001,"5min":6.3440000000000003,"15min":2.504},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":211.48699999999999,"5min":211.48699999999999,"15min":211.48699999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":23.547000000000001,"5min":6.0309999999999997,"15min":2.4510000000000001},"min":{"1min":0.38600000000000001,"5min":0.311,"15min":0.309},"max":{"1min":211.31100000000001,"5min":211.31100000000001,"15min":211.31100000000001},"last":0.55100000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":3.6040000000000001,"15min":1.5289999999999999},"min":{"1min":0.25,"5min":0.249,"15min":0.249},"max":{"1min":2.6070000000000002,"5min":113.35899999999999,"15min":113.35899999999999},"last":0.434},{"interface":"front","average":{"1min":0.54500000000000004,"5min":3.3849999999999998,"15min":1.5109999999999999},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.0550000000000002,"5min":113.32299999999999,"15min":113.32299999999999},"last":0.501}]},{"osd":6,"last update":"Mon Oct 9 05:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.0569999999999999,"15min":0.69799999999999995},"min":{"1min":0.28999999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.0289999999999999,"5min":11.944000000000001,"15min":11.944000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.3169999999999999,"15min":0.80300000000000005},"min":{"1min":0.313,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.1899999999999999,"5min":10.356,"15min":10.356},"last":1.859}]},{"osd":7,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":2.0230000000000001,"15min":2.9209999999999998},"min":{"1min":0.38700000000000001,"5min":0.312,"15min":0.312},"max":{"1min":2.0750000000000002,"5min":30.393999999999998,"15min":30.393999999999998},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.876,"5min":2.1989999999999998,"15min":3.286},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.5859999999999999,"5min":30.334,"15min":30.334},"last":1.9350000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822901,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1641416,"kb_used_data":1344192,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92103736,"statfs":{"total":95995035648,"available":94314225664,"internally_reserved":0,"allocated":1376452608,"data_stored":1375940767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6319999999999999,"5min":7.1500000000000004,"15min":2.75},"min":{"1min":0.23400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":10.452,"5min":165.90199999999999,"15min":165.90199999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":2.306,"5min":7.3650000000000002,"15min":2.8300000000000001},"min":{"1min":0.30499999999999999,"5min":0.215,"15min":0.215},"max":{"1min":12.263999999999999,"5min":165.96700000000001,"15min":165.96700000000001},"last":2.028}]},{"osd":1,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":7.4249999999999998,"5min":3.0960000000000001,"15min":1.4159999999999999},"min":{"1min":0.30599999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":87.769000000000005,"5min":87.769000000000005,"15min":87.769000000000005},"last":0.23999999999999999},{"interface":"front","average":{"1min":6.9340000000000002,"5min":2.782,"15min":1.337},"min":{"1min":0.35599999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":87.864000000000004,"5min":87.864000000000004,"15min":87.864000000000004},"last":3.399}]},{"osd":2,"last update":"Mon Oct 9 05:38:19 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":8.4670000000000005,"15min":3.2029999999999998},"min":{"1min":0.28599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.9900000000000002,"5min":290.697,"15min":290.697},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.093,"5min":8.3529999999999998,"15min":3.161},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":9.3569999999999993,"5min":290.786,"15min":290.786},"last":0.96899999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":5.8250000000000002,"5min":2.7799999999999998,"15min":1.3500000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":52.560000000000002,"5min":52.560000000000002,"15min":52.560000000000002},"last":4.3170000000000002},{"interface":"front","average":{"1min":4.8470000000000004,"5min":2.5710000000000002,"15min":1.268},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":52.380000000000003,"5min":52.380000000000003,"15min":52.380000000000003},"last":4.266}]},{"osd":4,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":1.0940000000000001,"15min":0.68600000000000005},"min":{"1min":0.25700000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.2949999999999999,"5min":9.3510000000000009,"15min":9.3510000000000009},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.746,"15min":0.93200000000000005},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":2.669,"5min":14.236000000000001,"15min":14.236000000000001},"last":0.39100000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":2.5600000000000001,"15min":1.234},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":1.8069999999999999,"5min":95.010999999999996,"15min":95.010999999999996},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":2.2490000000000001,"15min":1.123},"min":{"1min":0.33500000000000002,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":2.4390000000000001,"5min":94.194000000000003,"15min":94.194000000000003},"last":0.34300000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.5149999999999999,"15min":1.655},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.4289999999999998,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.51600000000000001,"5min":1.6599999999999999,"15min":1.968},"min":{"1min":0.36199999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.7250000000000001,"5min":14.432,"15min":14.432},"last":0.57599999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1115230208,"data_stored":1115226136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1547309056,"data_stored":1547304984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2433941504,"data_stored":2433941504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3022848000,"data_stored":3022848000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1372454912,"data_stored":1372454912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1374162944,"data_stored":1374158872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2183987200,"data_stored":2183987200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":997785600,"data_stored":997785600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:39:12.111 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:39:12.217 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:12.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:12.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:12.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:12.370 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:12.480 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:12.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:12.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:12.603 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:12.703 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:12.887 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:12.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:12.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[184418]: failed to encode map e190 with expected crc 2023-10-09T05:39:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3843341833' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:39:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[184418]: osdmap e190: 8 total, 8 up, 7 in 2023-10-09T05:39:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[184418]: failed to encode map e190 with expected crc 2023-10-09T05:39:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[184418]: failed to encode map e190 with expected crc 2023-10-09T05:39:12.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9404/205089 objects degraded (4.585%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:12.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:12.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[191227]: failed to encode map e190 with expected crc 2023-10-09T05:39:12.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3843341833' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:39:12.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[191227]: osdmap e190: 8 total, 8 up, 7 in 2023-10-09T05:39:12.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[191227]: failed to encode map e190 with expected crc 2023-10-09T05:39:12.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[191227]: failed to encode map e190 with expected crc 2023-10-09T05:39:12.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:12 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9404/205089 objects degraded (4.585%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:13.002 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:13.020 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:13.045 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:13.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:12 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:13.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:12 smithi191 ceph-mon[161669]: failed to encode map e190 with expected crc 2023-10-09T05:39:13.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3843341833' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:39:13.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:12 smithi191 ceph-mon[161669]: osdmap e190: 8 total, 8 up, 7 in 2023-10-09T05:39:13.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:12 smithi191 ceph-mon[161669]: failed to encode map e190 with expected crc 2023-10-09T05:39:13.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:12 smithi191 ceph-mon[161669]: failed to encode map e190 with expected crc 2023-10-09T05:39:13.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:12 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9404/205089 objects degraded (4.585%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:13.123 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:13.415 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:13.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:13.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:13.466 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:39:13.467 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:39:13.530 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:13.622 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:13.919 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:13.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:13.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:14.025 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:14.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:13 smithi191 ceph-mon[161669]: pgmap v513: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 9404/200871 objects degraded (4.682%); 1601/200871 objects misplaced (0.797%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T05:39:14.063 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:14.120 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:14.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:14.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:14.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:13 smithi008 ceph-mon[184418]: pgmap v513: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 9404/200871 objects degraded (4.682%); 1601/200871 objects misplaced (0.797%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T05:39:14.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:13 smithi008 ceph-mon[191227]: pgmap v513: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 9404/200871 objects degraded (4.682%); 1601/200871 objects misplaced (0.797%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T05:39:14.224 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:14.310 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:14.348 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:39:14.762 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:14.762 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:14 smithi191 ceph-mon[161669]: pgmap v514: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 9414/198642 objects degraded (4.739%); 1397/198642 objects misplaced (0.703%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:14.769 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:14.861 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:39:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:14 smithi008 ceph-mon[184418]: pgmap v514: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 9414/198642 objects degraded (4.739%); 1397/198642 objects misplaced (0.703%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:14 smithi008 ceph-mon[191227]: pgmap v514: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 9414/198642 objects degraded (4.739%); 1397/198642 objects misplaced (0.703%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:15.604 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:15.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:15.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:15.707 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:15.828 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:16.219 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:16.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:16.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:16.295 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:16.328 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:16.405 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:39:16.405 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:39:16.434 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:16.600 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:39:16.633 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:39:16.708 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:39:16.713 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:16.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:16.755 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:39:16.801 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:39:16.804 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:39:16.834 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:16.853 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:39:16.898 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:39:16.934 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:39:16.969 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:39:16.989 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:16.997 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:39:16.997 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:39:16.997 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:39:16.997 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:39:16.997 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:39:16.997 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:39:16.997 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:39:16.997 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:39:17.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:39:17.232 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:17 smithi008 ceph-mon[184418]: pgmap v515: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 9414/194778 objects degraded (4.833%); 1397/194778 objects misplaced (0.717%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:17.232 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1776756498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:17.232 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:17 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:17.234 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:17 smithi008 ceph-mon[191227]: pgmap v515: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 9414/194778 objects degraded (4.833%); 1397/194778 objects misplaced (0.717%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:17.234 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1776756498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:17.234 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:17 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:17.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:17 smithi191 ceph-mon[161669]: pgmap v515: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 9414/194778 objects degraded (4.833%); 1397/194778 objects misplaced (0.717%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:17.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1776756498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:17.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:17 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:17.887 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:18 smithi008 ceph-mon[184418]: from='client.45161 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9414/194778 objects degraded (4.833%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:18 smithi008 ceph-mon[191227]: from='client.45161 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9414/194778 objects degraded (4.833%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:18.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:18 smithi191 ceph-mon[161669]: from='client.45161 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:18.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9414/194778 objects degraded (4.833%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:18.626 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":515,"stamp":"2023-10-09T05:39:16.065840+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4257880011,"num_objects":64926,"num_object_clones":0,"num_object_copies":194778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9414,"num_objects_misplaced":1397,"num_objects_unfound":0,"num_objects_dirty":64926,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":112930,"num_write_kb":5670182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6730,"num_bytes_recovered":439489910,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110494,"ondisk_log_size":110494,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14217768,"kb_used_data":12195632,"kb_used_omap":31,"kb_used_meta":2022048,"kb_avail":641998296,"statfs":{"total":671965249536,"available":657406255104,"internally_reserved":0,"allocated":12488327168,"data_stored":12483830924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2070577609},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-411697152,"num_objects":-6282,"num_object_clones":0,"num_object_copies":-18846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15,"num_objects_misplaced":-378,"num_objects_unfound":0,"num_objects_dirty":-6282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6282,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":363,"num_bytes_recovered":23789568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003252"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":173,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827978+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:39:00.827978+0000","last_peered":"2023-10-09T05:39:00.827978+0000","last_clean":"2023-10-09T05:39:00.827978+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:39:00.827978+0000","last_undegraded":"2023-10-09T05:39:00.827978+0000","last_fullsized":"2023-10-09T05:39:00.827978+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":161,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.828699+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:39:00.828699+0000","last_peered":"2023-10-09T05:39:00.828699+0000","last_clean":"2023-10-09T05:39:00.828699+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:39:00.828699+0000","last_undegraded":"2023-10-09T05:39:00.828699+0000","last_fullsized":"2023-10-09T05:39:00.828699+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":136,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.891525+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:39:00.891525+0000","last_peered":"2023-10-09T05:39:00.891525+0000","last_clean":"2023-10-09T05:39:00.891525+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:39:00.891525+0000","last_undegraded":"2023-10-09T05:39:00.891525+0000","last_fullsized":"2023-10-09T05:39:00.891525+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":166,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.854505+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:39:00.854505+0000","last_peered":"2023-10-09T05:39:00.854505+0000","last_clean":"2023-10-09T05:39:00.854505+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:39:00.854505+0000","last_undegraded":"2023-10-09T05:39:00.854505+0000","last_fullsized":"2023-10-09T05:39:00.854505+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":177,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.734597+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:38:59.734597+0000","last_peered":"2023-10-09T05:38:59.734597+0000","last_clean":"2023-10-09T05:38:59.734597+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:38:59.734597+0000","last_undegraded":"2023-10-09T05:38:59.734597+0000","last_fullsized":"2023-10-09T05:38:59.734597+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":156,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.754242+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:39:00.754242+0000","last_peered":"2023-10-09T05:39:00.754242+0000","last_clean":"2023-10-09T05:39:00.754242+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:39:00.754242+0000","last_undegraded":"2023-10-09T05:39:00.754242+0000","last_fullsized":"2023-10-09T05:39:00.754242+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":141,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.855297+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:39:00.855297+0000","last_peered":"2023-10-09T05:39:00.855297+0000","last_clean":"2023-10-09T05:39:00.855297+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:39:00.855297+0000","last_undegraded":"2023-10-09T05:39:00.855297+0000","last_fullsized":"2023-10-09T05:39:00.855297+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":161,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.734666+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:38:59.734666+0000","last_peered":"2023-10-09T05:38:59.734666+0000","last_clean":"2023-10-09T05:38:59.734666+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:38:59.734666+0000","last_undegraded":"2023-10-09T05:38:59.734666+0000","last_fullsized":"2023-10-09T05:38:59.734666+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":181,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.824665+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:39:00.824665+0000","last_peered":"2023-10-09T05:39:00.824665+0000","last_clean":"2023-10-09T05:39:00.824665+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:39:00.824665+0000","last_undegraded":"2023-10-09T05:39:00.824665+0000","last_fullsized":"2023-10-09T05:39:00.824665+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":175,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.741619+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:38:59.741619+0000","last_peered":"2023-10-09T05:38:59.741619+0000","last_clean":"2023-10-09T05:38:59.741619+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:38:59.741619+0000","last_undegraded":"2023-10-09T05:38:59.741619+0000","last_fullsized":"2023-10-09T05:38:59.741619+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":68,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.734680+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:38:59.734680+0000","last_peered":"2023-10-09T05:38:59.734680+0000","last_clean":"2023-10-09T05:38:59.734680+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:38:59.734680+0000","last_undegraded":"2023-10-09T05:38:59.734680+0000","last_fullsized":"2023-10-09T05:38:59.734680+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":136,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.891809+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:39:00.891809+0000","last_peered":"2023-10-09T05:39:00.891809+0000","last_clean":"2023-10-09T05:39:00.891809+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:39:00.891809+0000","last_undegraded":"2023-10-09T05:39:00.891809+0000","last_fullsized":"2023-10-09T05:39:00.891809+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":198,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.828295+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:39:00.828295+0000","last_peered":"2023-10-09T05:39:00.828295+0000","last_clean":"2023-10-09T05:39:00.828295+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:39:00.828295+0000","last_undegraded":"2023-10-09T05:39:00.828295+0000","last_fullsized":"2023-10-09T05:39:00.828295+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":156,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827742+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:39:00.827742+0000","last_peered":"2023-10-09T05:39:00.827742+0000","last_clean":"2023-10-09T05:39:00.827742+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:39:00.827742+0000","last_undegraded":"2023-10-09T05:39:00.827742+0000","last_fullsized":"2023-10-09T05:39:00.827742+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":69,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827827+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:39:00.827827+0000","last_peered":"2023-10-09T05:39:00.827827+0000","last_clean":"2023-10-09T05:39:00.827827+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:39:00.827827+0000","last_undegraded":"2023-10-09T05:39:00.827827+0000","last_fullsized":"2023-10-09T05:39:00.827827+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":185,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.737186+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:38:59.737186+0000","last_peered":"2023-10-09T05:38:59.737186+0000","last_clean":"2023-10-09T05:38:59.737186+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:38:59.737186+0000","last_undegraded":"2023-10-09T05:38:59.737186+0000","last_fullsized":"2023-10-09T05:38:59.737186+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":165,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.853349+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:39:00.853349+0000","last_peered":"2023-10-09T05:39:00.853349+0000","last_clean":"2023-10-09T05:39:00.853349+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:39:00.853349+0000","last_undegraded":"2023-10-09T05:39:00.853349+0000","last_fullsized":"2023-10-09T05:39:00.853349+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":180,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.825164+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:39:00.825164+0000","last_peered":"2023-10-09T05:39:00.825164+0000","last_clean":"2023-10-09T05:39:00.825164+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:39:00.825164+0000","last_undegraded":"2023-10-09T05:39:00.825164+0000","last_fullsized":"2023-10-09T05:39:00.825164+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":87,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827653+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:39:00.827653+0000","last_peered":"2023-10-09T05:39:00.827653+0000","last_clean":"2023-10-09T05:39:00.827653+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:39:00.827653+0000","last_undegraded":"2023-10-09T05:39:00.827653+0000","last_fullsized":"2023-10-09T05:39:00.827653+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":165,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.854432+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:39:00.854432+0000","last_peered":"2023-10-09T05:39:00.854432+0000","last_clean":"2023-10-09T05:39:00.854432+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:39:00.854432+0000","last_undegraded":"2023-10-09T05:39:00.854432+0000","last_fullsized":"2023-10-09T05:39:00.854432+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":165,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.753798+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:39:00.753798+0000","last_peered":"2023-10-09T05:39:00.753798+0000","last_clean":"2023-10-09T05:39:00.753798+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:39:00.753798+0000","last_undegraded":"2023-10-09T05:39:00.753798+0000","last_fullsized":"2023-10-09T05:39:00.753798+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":185,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.735243+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:38:59.735243+0000","last_peered":"2023-10-09T05:38:59.735243+0000","last_clean":"2023-10-09T05:38:59.735243+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:38:59.735243+0000","last_undegraded":"2023-10-09T05:38:59.735243+0000","last_fullsized":"2023-10-09T05:38:59.735243+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"190'7023","reported_seq":7059,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:14.932455+0000","last_change":"2023-10-09T05:37:01.636984+0000","last_active":"2023-10-09T05:39:14.932455+0000","last_peered":"2023-10-09T05:39:14.932455+0000","last_clean":"2023-10-09T05:39:14.932455+0000","last_became_active":"2023-10-09T05:37:01.636609+0000","last_became_peered":"2023-10-09T05:37:01.636609+0000","last_unstale":"2023-10-09T05:39:14.932455+0000","last_undegraded":"2023-10-09T05:39:14.932455+0000","last_fullsized":"2023-10-09T05:39:14.932455+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7023,"log_dups_size":0,"ondisk_log_size":7023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264306688,"num_objects":4033,"num_object_clones":0,"num_object_copies":12099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7023,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":67,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827868+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:39:00.827868+0000","last_peered":"2023-10-09T05:39:00.827868+0000","last_clean":"2023-10-09T05:39:00.827868+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:39:00.827868+0000","last_undegraded":"2023-10-09T05:39:00.827868+0000","last_fullsized":"2023-10-09T05:39:00.827868+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":165,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.824477+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:39:00.824477+0000","last_peered":"2023-10-09T05:39:00.824477+0000","last_clean":"2023-10-09T05:39:00.824477+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:39:00.824477+0000","last_undegraded":"2023-10-09T05:39:00.824477+0000","last_fullsized":"2023-10-09T05:39:00.824477+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"189'6853","reported_seq":6897,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:11.105025+0000","last_change":"2023-10-09T05:37:36.837344+0000","last_active":"2023-10-09T05:39:11.105025+0000","last_peered":"2023-10-09T05:39:11.105025+0000","last_clean":"2023-10-09T05:37:34.651883+0000","last_became_active":"2023-10-09T05:37:36.832502+0000","last_became_peered":"2023-10-09T05:37:36.832502+0000","last_unstale":"2023-10-09T05:39:11.105025+0000","last_undegraded":"2023-10-09T05:37:36.702797+0000","last_fullsized":"2023-10-09T05:37:36.678583+0000","mapping_epoch":170,"log_start":"166'400","ondisk_log_start":"166'400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6453,"log_dups_size":0,"ondisk_log_size":6453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:57.419460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271253504,"num_objects":4139,"num_object_clones":0,"num_object_copies":12417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6853,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":1249},{"shards":"1,3,7","objects":1533}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":67,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.828140+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:39:00.828140+0000","last_peered":"2023-10-09T05:39:00.828140+0000","last_clean":"2023-10-09T05:39:00.828140+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:39:00.828140+0000","last_undegraded":"2023-10-09T05:39:00.828140+0000","last_fullsized":"2023-10-09T05:39:00.828140+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":171,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827997+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:39:00.827997+0000","last_peered":"2023-10-09T05:39:00.827997+0000","last_clean":"2023-10-09T05:39:00.827997+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:39:00.827997+0000","last_undegraded":"2023-10-09T05:39:00.827997+0000","last_fullsized":"2023-10-09T05:39:00.827997+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"189'7053","reported_seq":13149,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.610545+0000","last_change":"2023-10-09T05:38:55.317808+0000","last_active":"2023-10-09T05:39:11.610545+0000","last_peered":"2023-10-09T05:39:11.610545+0000","last_clean":"2023-10-09T05:39:11.610545+0000","last_became_active":"2023-10-09T05:38:55.302739+0000","last_became_peered":"2023-10-09T05:38:55.302739+0000","last_unstale":"2023-10-09T05:39:11.610545+0000","last_undegraded":"2023-10-09T05:39:11.610545+0000","last_fullsized":"2023-10-09T05:39:11.610545+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7053,"log_dups_size":0,"ondisk_log_size":7053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:25.379818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272564224,"num_objects":4159,"num_object_clones":0,"num_object_copies":12477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7053,"num_write_kb":358784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":198049792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":67,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.825180+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:39:00.825180+0000","last_peered":"2023-10-09T05:39:00.825180+0000","last_clean":"2023-10-09T05:39:00.825180+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:39:00.825180+0000","last_undegraded":"2023-10-09T05:39:00.825180+0000","last_fullsized":"2023-10-09T05:39:00.825180+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":189,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.753990+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:39:00.753990+0000","last_peered":"2023-10-09T05:39:00.753990+0000","last_clean":"2023-10-09T05:39:00.753990+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:39:00.753990+0000","last_undegraded":"2023-10-09T05:39:00.753990+0000","last_fullsized":"2023-10-09T05:39:00.753990+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"190'6963","reported_seq":8485,"reported_epoch":190,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:13.053950+0000","last_change":"2023-10-09T05:38:53.467497+0000","last_active":"2023-10-09T05:39:13.053950+0000","last_peered":"2023-10-09T05:39:13.053950+0000","last_clean":"2023-10-09T05:37:34.677467+0000","last_became_active":"2023-10-09T05:37:36.803958+0000","last_became_peered":"2023-10-09T05:37:36.803958+0000","last_unstale":"2023-10-09T05:39:13.053950+0000","last_undegraded":"2023-10-09T05:37:36.800519+0000","last_fullsized":"2023-10-09T05:37:36.678690+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6963,"log_dups_size":0,"ondisk_log_size":6963,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:40.690185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262602752,"num_objects":4007,"num_object_clones":0,"num_object_copies":12021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":764,"num_objects_misplaced":1397,"num_objects_unfound":0,"num_objects_dirty":4007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6963,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":738,"num_bytes_recovered":48365568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":1846},{"shards":"0,2,7","objects":1397}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":157,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.828420+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:39:00.828420+0000","last_peered":"2023-10-09T05:39:00.828420+0000","last_clean":"2023-10-09T05:39:00.828420+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:39:00.828420+0000","last_undegraded":"2023-10-09T05:39:00.828420+0000","last_fullsized":"2023-10-09T05:39:00.828420+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":142,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.854824+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:39:00.854824+0000","last_peered":"2023-10-09T05:39:00.854824+0000","last_clean":"2023-10-09T05:39:00.854824+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:39:00.854824+0000","last_undegraded":"2023-10-09T05:39:00.854824+0000","last_fullsized":"2023-10-09T05:39:00.854824+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"190'7003","reported_seq":7039,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:14.843624+0000","last_change":"2023-10-09T05:37:01.439878+0000","last_active":"2023-10-09T05:39:14.843624+0000","last_peered":"2023-10-09T05:39:14.843624+0000","last_clean":"2023-10-09T05:39:14.843624+0000","last_became_active":"2023-10-09T05:37:01.439481+0000","last_became_peered":"2023-10-09T05:37:01.439481+0000","last_unstale":"2023-10-09T05:39:14.843624+0000","last_undegraded":"2023-10-09T05:39:14.843624+0000","last_fullsized":"2023-10-09T05:39:14.843624+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7003,"log_dups_size":0,"ondisk_log_size":7003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261029888,"num_objects":3983,"num_object_clones":0,"num_object_copies":11949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7003,"num_write_kb":351552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":159,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.737043+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:38:59.737043+0000","last_peered":"2023-10-09T05:38:59.737043+0000","last_clean":"2023-10-09T05:38:59.737043+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:38:59.737043+0000","last_undegraded":"2023-10-09T05:38:59.737043+0000","last_fullsized":"2023-10-09T05:38:59.737043+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":156,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.828207+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:39:00.828207+0000","last_peered":"2023-10-09T05:39:00.828207+0000","last_clean":"2023-10-09T05:39:00.828207+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:39:00.828207+0000","last_undegraded":"2023-10-09T05:39:00.828207+0000","last_fullsized":"2023-10-09T05:39:00.828207+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"189'6914","reported_seq":6950,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.571654+0000","last_change":"2023-10-09T05:37:01.440175+0000","last_active":"2023-10-09T05:39:11.571654+0000","last_peered":"2023-10-09T05:39:11.571654+0000","last_clean":"2023-10-09T05:39:11.571654+0000","last_became_active":"2023-10-09T05:37:01.439846+0000","last_became_peered":"2023-10-09T05:37:01.439846+0000","last_unstale":"2023-10-09T05:39:11.571654+0000","last_undegraded":"2023-10-09T05:39:11.571654+0000","last_fullsized":"2023-10-09T05:39:11.571654+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6914,"log_dups_size":0,"ondisk_log_size":6914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:27.883265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271319040,"num_objects":4140,"num_object_clones":0,"num_object_copies":12420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6914,"num_write_kb":353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":157,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.753900+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:39:00.753900+0000","last_peered":"2023-10-09T05:39:00.753900+0000","last_clean":"2023-10-09T05:39:00.753900+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:39:00.753900+0000","last_undegraded":"2023-10-09T05:39:00.753900+0000","last_fullsized":"2023-10-09T05:39:00.753900+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":171,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827726+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:39:00.827726+0000","last_peered":"2023-10-09T05:39:00.827726+0000","last_clean":"2023-10-09T05:39:00.827726+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:39:00.827726+0000","last_undegraded":"2023-10-09T05:39:00.827726+0000","last_fullsized":"2023-10-09T05:39:00.827726+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"189'6912","reported_seq":6948,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.086720+0000","last_change":"2023-10-09T05:37:01.638178+0000","last_active":"2023-10-09T05:39:11.086720+0000","last_peered":"2023-10-09T05:39:11.086720+0000","last_clean":"2023-10-09T05:39:11.086720+0000","last_became_active":"2023-10-09T05:37:01.637822+0000","last_became_peered":"2023-10-09T05:37:01.637822+0000","last_unstale":"2023-10-09T05:39:11.086720+0000","last_undegraded":"2023-10-09T05:39:11.086720+0000","last_fullsized":"2023-10-09T05:39:11.086720+0000","mapping_epoch":162,"log_start":"166'500","ondisk_log_start":"166'500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6412,"log_dups_size":0,"ondisk_log_size":6412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:13.240069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275382272,"num_objects":4202,"num_object_clones":0,"num_object_copies":12606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6912,"num_write_kb":355648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":181,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827501+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:39:00.827501+0000","last_peered":"2023-10-09T05:39:00.827501+0000","last_clean":"2023-10-09T05:39:00.827501+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:39:00.827501+0000","last_undegraded":"2023-10-09T05:39:00.827501+0000","last_fullsized":"2023-10-09T05:39:00.827501+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":165,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.828201+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:39:00.828201+0000","last_peered":"2023-10-09T05:39:00.828201+0000","last_clean":"2023-10-09T05:39:00.828201+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:39:00.828201+0000","last_undegraded":"2023-10-09T05:39:00.828201+0000","last_fullsized":"2023-10-09T05:39:00.828201+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"190'7101","reported_seq":7138,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:14.941305+0000","last_change":"2023-10-09T05:37:01.438917+0000","last_active":"2023-10-09T05:39:14.941305+0000","last_peered":"2023-10-09T05:39:14.941305+0000","last_clean":"2023-10-09T05:39:14.941305+0000","last_became_active":"2023-10-09T05:37:01.438658+0000","last_became_peered":"2023-10-09T05:37:01.438658+0000","last_unstale":"2023-10-09T05:39:14.941305+0000","last_undegraded":"2023-10-09T05:39:14.941305+0000","last_fullsized":"2023-10-09T05:39:14.941305+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7101,"log_dups_size":0,"ondisk_log_size":7101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259522584,"num_objects":3961,"num_object_clones":0,"num_object_copies":11883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3961,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7101,"num_write_kb":353921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":186,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.734520+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:38:59.734520+0000","last_peered":"2023-10-09T05:38:59.734520+0000","last_clean":"2023-10-09T05:38:59.734520+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:38:59.734520+0000","last_undegraded":"2023-10-09T05:38:59.734520+0000","last_fullsized":"2023-10-09T05:38:59.734520+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":136,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.892751+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:39:00.892751+0000","last_peered":"2023-10-09T05:39:00.892751+0000","last_clean":"2023-10-09T05:39:00.892751+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:39:00.892751+0000","last_undegraded":"2023-10-09T05:39:00.892751+0000","last_fullsized":"2023-10-09T05:39:00.892751+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"190'7018","reported_seq":7056,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:13.029216+0000","last_change":"2023-10-09T05:37:01.440093+0000","last_active":"2023-10-09T05:39:13.029216+0000","last_peered":"2023-10-09T05:39:13.029216+0000","last_clean":"2023-10-09T05:39:13.029216+0000","last_became_active":"2023-10-09T05:37:01.439799+0000","last_became_peered":"2023-10-09T05:37:01.439799+0000","last_unstale":"2023-10-09T05:39:13.029216+0000","last_undegraded":"2023-10-09T05:39:13.029216+0000","last_fullsized":"2023-10-09T05:39:13.029216+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:28.734471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267911168,"num_objects":4088,"num_object_clones":0,"num_object_copies":12264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7018,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":724,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:00.891818+0000","last_change":"2023-10-09T05:37:36.736822+0000","last_active":"2023-10-09T05:39:00.891818+0000","last_peered":"2023-10-09T05:39:00.891818+0000","last_clean":"2023-10-09T05:37:30.792074+0000","last_became_active":"2023-10-09T05:37:36.736273+0000","last_became_peered":"2023-10-09T05:37:36.736273+0000","last_unstale":"2023-10-09T05:39:00.891818+0000","last_undegraded":"2023-10-09T05:37:36.682657+0000","last_fullsized":"2023-10-09T05:37:36.682189+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:23.350354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":172,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.825080+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:39:00.825080+0000","last_peered":"2023-10-09T05:39:00.825080+0000","last_clean":"2023-10-09T05:39:00.825080+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:39:00.825080+0000","last_undegraded":"2023-10-09T05:39:00.825080+0000","last_fullsized":"2023-10-09T05:39:00.825080+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":122,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.891744+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:39:00.891744+0000","last_peered":"2023-10-09T05:39:00.891744+0000","last_clean":"2023-10-09T05:39:00.891744+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:39:00.891744+0000","last_undegraded":"2023-10-09T05:39:00.891744+0000","last_fullsized":"2023-10-09T05:39:00.891744+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"189'6969","reported_seq":12896,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.143145+0000","last_change":"2023-10-09T05:38:10.132330+0000","last_active":"2023-10-09T05:39:11.143145+0000","last_peered":"2023-10-09T05:39:11.143145+0000","last_clean":"2023-10-09T05:39:11.143145+0000","last_became_active":"2023-10-09T05:38:10.127080+0000","last_became_peered":"2023-10-09T05:38:10.127080+0000","last_unstale":"2023-10-09T05:39:11.143145+0000","last_undegraded":"2023-10-09T05:39:11.143145+0000","last_fullsized":"2023-10-09T05:39:11.143145+0000","mapping_epoch":179,"log_start":"166'500","ondisk_log_start":"166'500","created":162,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6469,"log_dups_size":0,"ondisk_log_size":6469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:12.659227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273874944,"num_objects":4179,"num_object_clones":0,"num_object_copies":12537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6969,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":192413696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":626,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.824919+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:39:00.824919+0000","last_peered":"2023-10-09T05:39:00.824919+0000","last_clean":"2023-10-09T05:39:00.824919+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:39:00.824919+0000","last_undegraded":"2023-10-09T05:39:00.824919+0000","last_fullsized":"2023-10-09T05:39:00.824919+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":157,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827906+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:39:00.827906+0000","last_peered":"2023-10-09T05:39:00.827906+0000","last_clean":"2023-10-09T05:39:00.827906+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:39:00.827906+0000","last_undegraded":"2023-10-09T05:39:00.827906+0000","last_fullsized":"2023-10-09T05:39:00.827906+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":169,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.828546+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:39:00.828546+0000","last_peered":"2023-10-09T05:39:00.828546+0000","last_clean":"2023-10-09T05:39:00.828546+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:39:00.828546+0000","last_undegraded":"2023-10-09T05:39:00.828546+0000","last_fullsized":"2023-10-09T05:39:00.828546+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"190'7035","reported_seq":7082,"reported_epoch":190,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:15.522867+0000","last_change":"2023-10-09T05:37:40.649912+0000","last_active":"2023-10-09T05:39:15.522867+0000","last_peered":"2023-10-09T05:39:15.522867+0000","last_clean":"2023-10-09T05:37:34.677100+0000","last_became_active":"2023-10-09T05:37:40.593033+0000","last_became_peered":"2023-10-09T05:37:40.593033+0000","last_unstale":"2023-10-09T05:39:15.522867+0000","last_undegraded":"2023-10-09T05:37:40.546647+0000","last_fullsized":"2023-10-09T05:37:40.543498+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7035,"log_dups_size":0,"ondisk_log_size":7035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259194880,"num_objects":3955,"num_object_clones":0,"num_object_copies":11865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7035,"num_write_kb":351680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":1105},{"shards":"3,6,7","objects":1310}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":502,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.777754+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:38:59.777754+0000","last_peered":"2023-10-09T05:38:59.777754+0000","last_clean":"2023-10-09T05:38:59.777754+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:38:59.777754+0000","last_undegraded":"2023-10-09T05:38:59.777754+0000","last_fullsized":"2023-10-09T05:38:59.777754+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":137,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.889518+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:39:00.889518+0000","last_peered":"2023-10-09T05:39:00.889518+0000","last_clean":"2023-10-09T05:39:00.889518+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:39:00.889518+0000","last_undegraded":"2023-10-09T05:39:00.889518+0000","last_fullsized":"2023-10-09T05:39:00.889518+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":167,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:08.702678+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:39:08.702678+0000","last_peered":"2023-10-09T05:39:08.702678+0000","last_clean":"2023-10-09T05:39:08.702678+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:39:08.702678+0000","last_undegraded":"2023-10-09T05:39:08.702678+0000","last_fullsized":"2023-10-09T05:39:08.702678+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"190'7022","reported_seq":7058,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:14.924481+0000","last_change":"2023-10-09T05:37:01.438236+0000","last_active":"2023-10-09T05:39:14.924481+0000","last_peered":"2023-10-09T05:39:14.924481+0000","last_clean":"2023-10-09T05:39:14.924481+0000","last_became_active":"2023-10-09T05:37:01.437917+0000","last_became_peered":"2023-10-09T05:37:01.437917+0000","last_unstale":"2023-10-09T05:39:14.924481+0000","last_undegraded":"2023-10-09T05:39:14.924481+0000","last_fullsized":"2023-10-09T05:39:14.924481+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265551872,"num_objects":4052,"num_object_clones":0,"num_object_copies":12156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7022,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":774,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:00.825054+0000","last_change":"2023-10-09T05:37:36.709131+0000","last_active":"2023-10-09T05:39:00.825054+0000","last_peered":"2023-10-09T05:39:00.825054+0000","last_clean":"2023-10-09T05:37:30.688708+0000","last_became_active":"2023-10-09T05:37:36.708682+0000","last_became_peered":"2023-10-09T05:37:36.708682+0000","last_unstale":"2023-10-09T05:39:00.825054+0000","last_undegraded":"2023-10-09T05:37:36.683823+0000","last_fullsized":"2023-10-09T05:37:36.683109+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":337,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.855410+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:39:00.855410+0000","last_peered":"2023-10-09T05:39:00.855410+0000","last_clean":"2023-10-09T05:39:00.855410+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:39:00.855410+0000","last_undegraded":"2023-10-09T05:39:00.855410+0000","last_fullsized":"2023-10-09T05:39:00.855410+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":152,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.735428+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:38:59.735428+0000","last_peered":"2023-10-09T05:38:59.735428+0000","last_clean":"2023-10-09T05:38:59.735428+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:38:59.735428+0000","last_undegraded":"2023-10-09T05:38:59.735428+0000","last_fullsized":"2023-10-09T05:38:59.735428+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":163,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.754350+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:39:00.754350+0000","last_peered":"2023-10-09T05:39:00.754350+0000","last_clean":"2023-10-09T05:39:00.754350+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:39:00.754350+0000","last_undegraded":"2023-10-09T05:39:00.754350+0000","last_fullsized":"2023-10-09T05:39:00.754350+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"190'6958","reported_seq":6996,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:13.694077+0000","last_change":"2023-10-09T05:37:01.440172+0000","last_active":"2023-10-09T05:39:13.694077+0000","last_peered":"2023-10-09T05:39:13.694077+0000","last_clean":"2023-10-09T05:39:13.694077+0000","last_became_active":"2023-10-09T05:37:01.439932+0000","last_became_peered":"2023-10-09T05:37:01.439932+0000","last_unstale":"2023-10-09T05:39:13.694077+0000","last_undegraded":"2023-10-09T05:39:13.694077+0000","last_fullsized":"2023-10-09T05:39:13.694077+0000","mapping_epoch":162,"log_start":"166'500","ondisk_log_start":"166'500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6458,"log_dups_size":0,"ondisk_log_size":6458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:57.826704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262275072,"num_objects":4002,"num_object_clones":0,"num_object_copies":12006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6958,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":715,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:00.889269+0000","last_change":"2023-10-09T05:37:36.748736+0000","last_active":"2023-10-09T05:39:00.889269+0000","last_peered":"2023-10-09T05:39:00.889269+0000","last_clean":"2023-10-09T05:22:39.601058+0000","last_became_active":"2023-10-09T05:37:36.748304+0000","last_became_peered":"2023-10-09T05:37:36.748304+0000","last_unstale":"2023-10-09T05:39:00.889269+0000","last_undegraded":"2023-10-09T05:37:36.696424+0000","last_fullsized":"2023-10-09T05:37:36.695987+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:18.784578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":142,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.856403+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:39:00.856403+0000","last_peered":"2023-10-09T05:39:00.856403+0000","last_clean":"2023-10-09T05:39:00.856403+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:39:00.856403+0000","last_undegraded":"2023-10-09T05:39:00.856403+0000","last_fullsized":"2023-10-09T05:39:00.856403+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":157,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.854661+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:39:00.854661+0000","last_peered":"2023-10-09T05:39:00.854661+0000","last_clean":"2023-10-09T05:39:00.854661+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:39:00.854661+0000","last_undegraded":"2023-10-09T05:39:00.854661+0000","last_fullsized":"2023-10-09T05:39:00.854661+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"190'6989","reported_seq":7027,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:13.571618+0000","last_change":"2023-10-09T05:37:01.636967+0000","last_active":"2023-10-09T05:39:13.571618+0000","last_peered":"2023-10-09T05:39:13.571618+0000","last_clean":"2023-10-09T05:39:13.571618+0000","last_became_active":"2023-10-09T05:37:01.636602+0000","last_became_peered":"2023-10-09T05:37:01.636602+0000","last_unstale":"2023-10-09T05:39:13.571618+0000","last_undegraded":"2023-10-09T05:39:13.571618+0000","last_fullsized":"2023-10-09T05:39:13.571618+0000","mapping_epoch":162,"log_start":"166'600","ondisk_log_start":"166'600","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6389,"log_dups_size":0,"ondisk_log_size":6389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:11.668143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268894208,"num_objects":4103,"num_object_clones":0,"num_object_copies":12309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6989,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":378,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.855740+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:39:00.855740+0000","last_peered":"2023-10-09T05:39:00.855740+0000","last_clean":"2023-10-09T05:39:00.855740+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:39:00.855740+0000","last_undegraded":"2023-10-09T05:39:00.855740+0000","last_fullsized":"2023-10-09T05:39:00.855740+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":137,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.892121+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:39:00.892121+0000","last_peered":"2023-10-09T05:39:00.892121+0000","last_clean":"2023-10-09T05:39:00.892121+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:39:00.892121+0000","last_undegraded":"2023-10-09T05:39:00.892121+0000","last_fullsized":"2023-10-09T05:39:00.892121+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":151,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.855039+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:39:00.855039+0000","last_peered":"2023-10-09T05:39:00.855039+0000","last_clean":"2023-10-09T05:39:00.855039+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:39:00.855039+0000","last_undegraded":"2023-10-09T05:39:00.855039+0000","last_fullsized":"2023-10-09T05:39:00.855039+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"190'7124","reported_seq":7160,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:14.892336+0000","last_change":"2023-10-09T05:37:01.440074+0000","last_active":"2023-10-09T05:39:14.892336+0000","last_peered":"2023-10-09T05:39:14.892336+0000","last_clean":"2023-10-09T05:39:14.892336+0000","last_became_active":"2023-10-09T05:37:01.439808+0000","last_became_peered":"2023-10-09T05:37:01.439808+0000","last_unstale":"2023-10-09T05:39:14.892336+0000","last_undegraded":"2023-10-09T05:39:14.892336+0000","last_fullsized":"2023-10-09T05:39:14.892336+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7124,"log_dups_size":0,"ondisk_log_size":7124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261750784,"num_objects":3994,"num_object_clones":0,"num_object_copies":11982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7124,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":187,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.735149+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:38:59.735149+0000","last_peered":"2023-10-09T05:38:59.735149+0000","last_clean":"2023-10-09T05:38:59.735149+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:38:59.735149+0000","last_undegraded":"2023-10-09T05:38:59.735149+0000","last_fullsized":"2023-10-09T05:38:59.735149+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":578,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:08.702662+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:39:08.702662+0000","last_peered":"2023-10-09T05:39:08.702662+0000","last_clean":"2023-10-09T05:39:08.702662+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:39:08.702662+0000","last_undegraded":"2023-10-09T05:39:08.702662+0000","last_fullsized":"2023-10-09T05:39:08.702662+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":180,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.734744+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:38:59.734744+0000","last_peered":"2023-10-09T05:38:59.734744+0000","last_clean":"2023-10-09T05:38:59.734744+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:38:59.734744+0000","last_undegraded":"2023-10-09T05:38:59.734744+0000","last_fullsized":"2023-10-09T05:38:59.734744+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"190'6887","reported_seq":6930,"reported_epoch":190,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:15.534346+0000","last_change":"2023-10-09T05:37:36.839957+0000","last_active":"2023-10-09T05:39:15.534346+0000","last_peered":"2023-10-09T05:39:15.534346+0000","last_clean":"2023-10-09T05:37:34.656587+0000","last_became_active":"2023-10-09T05:37:36.833510+0000","last_became_peered":"2023-10-09T05:37:36.833510+0000","last_unstale":"2023-10-09T05:39:15.534346+0000","last_undegraded":"2023-10-09T05:37:36.701190+0000","last_fullsized":"2023-10-09T05:37:36.681999+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6887,"log_dups_size":0,"ondisk_log_size":6887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255655936,"num_objects":3901,"num_object_clones":0,"num_object_copies":11703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2877,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6887,"num_write_kb":345216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":1024},{"shards":"2,6,7","objects":1384}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":674,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:00.754105+0000","last_change":"2023-10-09T05:37:36.917512+0000","last_active":"2023-10-09T05:39:00.754105+0000","last_peered":"2023-10-09T05:39:00.754105+0000","last_clean":"2023-10-09T05:37:30.688967+0000","last_became_active":"2023-10-09T05:37:36.700218+0000","last_became_peered":"2023-10-09T05:37:36.700218+0000","last_unstale":"2023-10-09T05:39:00.754105+0000","last_undegraded":"2023-10-09T05:37:36.917335+0000","last_fullsized":"2023-10-09T05:37:36.682410+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":145,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.854588+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:39:00.854588+0000","last_peered":"2023-10-09T05:39:00.854588+0000","last_clean":"2023-10-09T05:39:00.854588+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:39:00.854588+0000","last_undegraded":"2023-10-09T05:39:00.854588+0000","last_fullsized":"2023-10-09T05:39:00.854588+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":141,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827539+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:39:00.827539+0000","last_peered":"2023-10-09T05:39:00.827539+0000","last_clean":"2023-10-09T05:39:00.827539+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:39:00.827539+0000","last_undegraded":"2023-10-09T05:39:00.827539+0000","last_fullsized":"2023-10-09T05:39:00.827539+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":153,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.854966+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:39:00.854966+0000","last_peered":"2023-10-09T05:39:00.854966+0000","last_clean":"2023-10-09T05:39:00.854966+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:39:00.854966+0000","last_undegraded":"2023-10-09T05:39:00.854966+0000","last_fullsized":"2023-10-09T05:39:00.854966+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":180,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.775267+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:38:59.775267+0000","last_peered":"2023-10-09T05:38:59.775267+0000","last_clean":"2023-10-09T05:38:59.775267+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:38:59.775267+0000","last_undegraded":"2023-10-09T05:38:59.775267+0000","last_fullsized":"2023-10-09T05:38:59.775267+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":159,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.828361+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:39:00.828361+0000","last_peered":"2023-10-09T05:39:00.828361+0000","last_clean":"2023-10-09T05:39:00.828361+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:39:00.828361+0000","last_undegraded":"2023-10-09T05:39:00.828361+0000","last_fullsized":"2023-10-09T05:39:00.828361+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":171,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827980+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:39:00.827980+0000","last_peered":"2023-10-09T05:39:00.827980+0000","last_clean":"2023-10-09T05:39:00.827980+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:39:00.827980+0000","last_undegraded":"2023-10-09T05:39:00.827980+0000","last_fullsized":"2023-10-09T05:39:00.827980+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":144,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.825084+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:39:00.825084+0000","last_peered":"2023-10-09T05:39:00.825084+0000","last_clean":"2023-10-09T05:39:00.825084+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:39:00.825084+0000","last_undegraded":"2023-10-09T05:39:00.825084+0000","last_fullsized":"2023-10-09T05:39:00.825084+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":171,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827842+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:39:00.827842+0000","last_peered":"2023-10-09T05:39:00.827842+0000","last_clean":"2023-10-09T05:39:00.827842+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:39:00.827842+0000","last_undegraded":"2023-10-09T05:39:00.827842+0000","last_fullsized":"2023-10-09T05:39:00.827842+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":167,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.735839+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:38:59.735839+0000","last_peered":"2023-10-09T05:38:59.735839+0000","last_clean":"2023-10-09T05:38:59.735839+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:38:59.735839+0000","last_undegraded":"2023-10-09T05:38:59.735839+0000","last_fullsized":"2023-10-09T05:38:59.735839+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":180,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T05:38:59.741549+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:38:59.741549+0000","last_peered":"2023-10-09T05:38:59.741549+0000","last_clean":"2023-10-09T05:38:59.741549+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:38:59.741549+0000","last_undegraded":"2023-10-09T05:38:59.741549+0000","last_fullsized":"2023-10-09T05:38:59.741549+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":145,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.855326+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:39:00.855326+0000","last_peered":"2023-10-09T05:39:00.855326+0000","last_clean":"2023-10-09T05:39:00.855326+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:39:00.855326+0000","last_undegraded":"2023-10-09T05:39:00.855326+0000","last_fullsized":"2023-10-09T05:39:00.855326+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":156,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T05:39:00.827509+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:39:00.827509+0000","last_peered":"2023-10-09T05:39:00.827509+0000","last_clean":"2023-10-09T05:39:00.827509+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:39:00.827509+0000","last_undegraded":"2023-10-09T05:39:00.827509+0000","last_fullsized":"2023-10-09T05:39:00.827509+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4253089816,"num_objects":64898,"num_object_clones":0,"num_object_copies":194694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9381,"num_objects_misplaced":1397,"num_objects_unfound":0,"num_objects_dirty":64898,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":111824,"num_write_kb":5655041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6697,"num_bytes_recovered":438894592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13390327808,"data_stored":13390315592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109324,"ondisk_log_size":109324,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846648,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626674,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2269852,"kb_used_data":1964436,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91475300,"statfs":{"total":95995035648,"available":93670707200,"internally_reserved":0,"allocated":2011582464,"data_stored":2011062995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":4.3609999999999998,"15min":1.819},"min":{"1min":0.23599999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.8869999999999996,"5min":77.024000000000001,"15min":77.024000000000001},"last":7.1079999999999997},{"interface":"front","average":{"1min":0.82499999999999996,"5min":4.0880000000000001,"15min":1.762},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.3290000000000002,"5min":79.305999999999997,"15min":79.305999999999997},"last":0.28100000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":2.677,"15min":1.3799999999999999},"min":{"1min":0.30099999999999999,"5min":0.25,"15min":0.25},"max":{"1min":9.8670000000000009,"5min":69.667000000000002,"15min":69.667000000000002},"last":2.2469999999999999},{"interface":"front","average":{"1min":1.972,"5min":2.8849999999999998,"15min":1.4670000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":10.624000000000001,"5min":69.697999999999993,"15min":69.697999999999993},"last":2.2389999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":3.8540000000000001,"15min":2.0569999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":12.99,"5min":70.219999999999999,"15min":70.219999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.9710000000000001,"5min":3.7040000000000002,"15min":2.0190000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.978,"5min":70.230000000000004,"15min":70.230000000000004},"last":0.625}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":2.6160000000000001,"15min":1.661},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":29.885000000000002,"5min":68.920000000000002,"15min":68.920000000000002},"last":6.6230000000000002},{"interface":"front","average":{"1min":3.0230000000000001,"5min":2.794,"15min":1.76},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":29.771000000000001,"5min":68.846000000000004,"15min":68.846000000000004},"last":2.6219999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":3.1379999999999999,"15min":1.401},"min":{"1min":0.248,"5min":0.245,"15min":0.23400000000000001},"max":{"1min":0.58599999999999997,"5min":125.64,"15min":125.64},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":2.7709999999999999,"15min":1.276},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":1.2709999999999999,"5min":125.625,"15min":125.625},"last":0.34799999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":2.871,"15min":1.3340000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.5510000000000002,"5min":127.35899999999999,"15min":127.35899999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.5110000000000001,"15min":1.23},"min":{"1min":0.29299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.8769999999999998,"5min":128.93799999999999,"15min":128.93799999999999},"last":0.32200000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":2.0800000000000001,"15min":2.7919999999999998},"min":{"1min":0.308,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.2789999999999999,"5min":12.074999999999999,"15min":12.074999999999999},"last":0.432},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.349,"15min":1.8440000000000001},"min":{"1min":0.33800000000000002,"5min":0.317,"15min":0.317},"max":{"1min":1.343,"5min":10.083,"15min":10.083},"last":0.38700000000000001}]}]},{"osd":1,"up_from":93,"seq":399431958635,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1667960,"kb_used_data":1455792,"kb_used_omap":7,"kb_used_meta":212152,"kb_avail":92077192,"statfs":{"total":95995035648,"available":94287044608,"internally_reserved":0,"allocated":1490731008,"data_stored":1490012465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":217244174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2120000000000002,"5min":5.5830000000000002,"15min":2.1520000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.185},"max":{"1min":9.9819999999999993,"5min":207.858,"15min":207.858},"last":13.214},{"interface":"front","average":{"1min":6.5330000000000004,"5min":6.7000000000000002,"15min":2.5499999999999998},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":62.880000000000003,"5min":206.97200000000001,"15min":206.97200000000001},"last":10.849}]},{"osd":2,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":5.3289999999999997,"5min":7.3360000000000003,"15min":2.8220000000000001},"min":{"1min":0.22700000000000001,"5min":0.184,"15min":0.184},"max":{"1min":32.686999999999998,"5min":101.367,"15min":101.367},"last":3.4969999999999999},{"interface":"front","average":{"1min":4.468,"5min":6.7439999999999998,"15min":2.6259999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":19.975999999999999,"5min":98.409999999999997,"15min":98.409999999999997},"last":13.276}]},{"osd":3,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":4.234,"5min":3.0209999999999999,"15min":1.3640000000000001},"min":{"1min":0.24199999999999999,"5min":0.223,"15min":0.223},"max":{"1min":33.146000000000001,"5min":83.885000000000005,"15min":83.885000000000005},"last":3.9390000000000001},{"interface":"front","average":{"1min":8.5860000000000003,"5min":3.8889999999999998,"15min":1.619},"min":{"1min":0.26300000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":62.929000000000002,"5min":84.641999999999996,"15min":84.641999999999996},"last":6.9829999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":5.9870000000000001,"5min":4.1109999999999998,"15min":1.833},"min":{"1min":0.432,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":63.116,"5min":141.32900000000001,"15min":141.32900000000001},"last":10.813000000000001},{"interface":"front","average":{"1min":1.675,"5min":2.8999999999999999,"15min":1.415},"min":{"1min":0.34499999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.5549999999999997,"5min":141.15899999999999,"15min":141.15899999999999},"last":6.7409999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":4.0350000000000001,"15min":1.7929999999999999},"min":{"1min":0.39000000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.0009999999999994,"5min":235.18700000000001,"15min":235.18700000000001},"last":3.5329999999999999},{"interface":"front","average":{"1min":6.0140000000000002,"5min":4.9180000000000001,"15min":2.1819999999999999},"min":{"1min":0.46600000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":63.162999999999997,"5min":235.79300000000001,"15min":235.79300000000001},"last":10.779}]},{"osd":6,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":1.8220000000000001,"15min":1.075},"min":{"1min":0.42599999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.0289999999999999,"5min":10.333,"15min":10.333},"last":3.5619999999999998},{"interface":"front","average":{"1min":6.2220000000000004,"5min":2.75,"15min":1.393},"min":{"1min":0.5,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":63.198,"5min":63.198,"15min":63.198},"last":10.724}]},{"osd":7,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":2.7949999999999999,"15min":2.2639999999999998},"min":{"1min":0.45700000000000002,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.6849999999999996,"5min":126.68300000000001,"15min":126.68300000000001},"last":6.782},{"interface":"front","average":{"1min":1.6439999999999999,"5min":2.911,"15min":2.3559999999999999},"min":{"1min":0.48499999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.6180000000000003,"5min":126.738,"15min":126.738},"last":6.8109999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122161,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1392912,"kb_used_data":1046728,"kb_used_omap":8,"kb_used_meta":346167,"kb_avail":92352240,"statfs":{"total":95995035648,"available":94568693760,"internally_reserved":0,"allocated":1071849472,"data_stored":1071108289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":354475784},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":1.8959999999999999,"15min":0.93799999999999994},"min":{"1min":0.20499999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":14.964,"5min":14.964,"15min":14.964},"last":1.5049999999999999},{"interface":"front","average":{"1min":3.879,"5min":2.2850000000000001,"15min":1.0820000000000001},"min":{"1min":0.26100000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":11.315,"5min":11.315,"15min":11.315},"last":4.452}]},{"osd":2,"last update":"Mon Oct 9 05:38:13 2023","interfaces":[{"interface":"back","average":{"1min":3.0720000000000001,"5min":4.3079999999999998,"15min":1.6830000000000001},"min":{"1min":0.22500000000000001,"5min":0.214,"15min":0.214},"max":{"1min":18.792999999999999,"5min":79.911000000000001,"15min":79.911000000000001},"last":4.2599999999999998},{"interface":"front","average":{"1min":1.7629999999999999,"5min":3.7599999999999998,"15min":1.5189999999999999},"min":{"1min":0.23300000000000001,"5min":0.218,"15min":0.218},"max":{"1min":9.2159999999999993,"5min":80.105000000000004,"15min":80.105000000000004},"last":0.98299999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":6.5289999999999999,"5min":3.0329999999999999,"15min":1.3149999999999999},"min":{"1min":0.48799999999999999,"5min":0.23999999999999999,"15min":0.22800000000000001},"max":{"1min":31.594000000000001,"5min":31.594000000000001,"15min":31.594000000000001},"last":9.8350000000000009},{"interface":"front","average":{"1min":6.4569999999999999,"5min":3.1379999999999999,"15min":1.3879999999999999},"min":{"1min":0.38500000000000001,"5min":0.222,"15min":0.222},"max":{"1min":31.641999999999999,"5min":31.641999999999999,"15min":31.641999999999999},"last":16.937999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:38:13 2023","interfaces":[{"interface":"back","average":{"1min":4.3879999999999999,"5min":2.0470000000000002,"15min":1.1319999999999999},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":69.082999999999998,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.29699999999999999},{"interface":"front","average":{"1min":4.8040000000000003,"5min":2.4980000000000002,"15min":1.306},"min":{"1min":0.38,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":69.102000000000004,"5min":69.102000000000004,"15min":69.102000000000004},"last":4.327}]},{"osd":5,"last update":"Mon Oct 9 05:38:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.8480000000000001,"15min":1.0629999999999999},"min":{"1min":0.33800000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.3150000000000004,"5min":13.632,"15min":13.632},"last":4.3630000000000004},{"interface":"front","average":{"1min":1.4590000000000001,"5min":1.893,"15min":1.081},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":7.2729999999999997,"5min":13.676,"15min":13.676},"last":1.04}]},{"osd":6,"last update":"Mon Oct 9 05:38:13 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.7889999999999999,"15min":1.0600000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.1749999999999998,"5min":9.9809999999999999,"15min":9.9809999999999999},"last":4.423},{"interface":"front","average":{"1min":1.925,"5min":1.9390000000000001,"15min":1.099},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.0999999999999996,"5min":8.0999999999999996,"15min":8.0999999999999996},"last":1.0669999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":1.409,"15min":1.355},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.5499999999999998,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.438},{"interface":"front","average":{"1min":2.649,"5min":2.206,"15min":2.3279999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":8.2490000000000006,"5min":11.068,"15min":11.068},"last":4.3929999999999998}]}]},{"osd":2,"up_from":98,"seq":420906795108,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2649648,"kb_used_data":2296104,"kb_used_omap":11,"kb_used_meta":353524,"kb_avail":91095504,"statfs":{"total":95995035648,"available":93281796096,"internally_reserved":0,"allocated":2351210496,"data_stored":2350436514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":362009173},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":3.476,"5min":2.8010000000000002,"15min":1.2669999999999999},"min":{"1min":0.23400000000000001,"5min":0.183,"15min":0.17199999999999999},"max":{"1min":10.098000000000001,"5min":32.122,"15min":32.122},"last":0.27300000000000002},{"interface":"front","average":{"1min":3.27,"5min":2.3530000000000002,"15min":1.1399999999999999},"min":{"1min":0.25800000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.5850000000000009,"5min":34.066000000000003,"15min":34.066000000000003},"last":0.25800000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":14.41,"5min":5.3600000000000003,"15min":2.133},"min":{"1min":0.34499999999999997,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":98.575000000000003,"5min":98.575000000000003,"15min":98.575000000000003},"last":98.575000000000003},{"interface":"front","average":{"1min":12.257,"5min":4.6779999999999999,"15min":1.867},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":90.135000000000005,"5min":98.397000000000006,"15min":98.397000000000006},"last":89.087999999999994}]},{"osd":3,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":3.5640000000000001,"5min":3.0049999999999999,"15min":1.2769999999999999},"min":{"1min":0.41699999999999998,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":12.385999999999999,"5min":45.057000000000002,"15min":45.057000000000002},"last":91.527000000000001},{"interface":"front","average":{"1min":3.2970000000000002,"5min":3.3300000000000001,"15min":1.4410000000000001},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.942,"5min":45.813000000000002,"15min":45.813000000000002},"last":91.480999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":2.8050000000000002,"15min":1.4019999999999999},"min":{"1min":0.29099999999999998,"5min":0.28999999999999998,"15min":0.27700000000000002},"max":{"1min":11.125999999999999,"5min":115.84999999999999,"15min":115.84999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.976,"5min":3.504,"15min":1.675},"min":{"1min":0.34799999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.147,"5min":116.149,"15min":116.149},"last":0.5}]},{"osd":5,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":2.5409999999999999,"15min":1.369},"min":{"1min":0.39900000000000002,"5min":0.312,"15min":0.312},"max":{"1min":7.8220000000000001,"5min":24.597999999999999,"15min":24.597999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.867,"5min":2.089,"15min":1.1779999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.097,"5min":23.919,"15min":23.919},"last":0.44700000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":2.5129999999999999,"15min":1.327},"min":{"1min":0.39900000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.183,"5min":20.503,"15min":20.503},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.8519999999999999,"5min":2.2170000000000001,"15min":1.2470000000000001},"min":{"1min":0.45000000000000001,"5min":0.372,"15min":0.34499999999999997},"max":{"1min":7.8730000000000002,"5min":9.4049999999999994,"15min":9.4049999999999994},"last":0.51300000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:38:49 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":2.883,"15min":2.6429999999999998},"min":{"1min":0.42599999999999999,"5min":0.374,"15min":0.374},"max":{"1min":8.9819999999999993,"5min":24.030999999999999,"15min":24.030999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":3.3839999999999999,"5min":2.4950000000000001,"15min":1.984},"min":{"1min":0.47299999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.2569999999999997,"5min":24.178000000000001,"15min":24.178000000000001},"last":0.48899999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631582,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3042728,"kb_used_data":2844640,"kb_used_omap":4,"kb_used_meta":198075,"kb_avail":90702424,"statfs":{"total":95995035648,"available":92879282176,"internally_reserved":0,"allocated":2912911360,"data_stored":2912199983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":202829406},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0840000000000001,"5min":2.6869999999999998,"15min":1.1970000000000001},"min":{"1min":0.26100000000000001,"5min":0.183,"15min":0.183},"max":{"1min":13.997,"5min":56.094000000000001,"15min":56.094000000000001},"last":0.69099999999999995},{"interface":"front","average":{"1min":3.7799999999999998,"5min":2.9700000000000002,"15min":1.296},"min":{"1min":0.221,"5min":0.221,"15min":0.214},"max":{"1min":15.593,"5min":57.255000000000003,"15min":57.255000000000003},"last":4.1879999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0059999999999998,"5min":3.5619999999999998,"15min":1.4690000000000001},"min":{"1min":0.35399999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":13.932,"5min":83.921999999999997,"15min":83.921999999999997},"last":0.73199999999999998},{"interface":"front","average":{"1min":2.5019999999999998,"5min":3.4660000000000002,"15min":1.4119999999999999},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":7.1580000000000004,"5min":83.707999999999998,"15min":83.707999999999998},"last":6.6079999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":6.2699999999999996,"5min":4.5549999999999997,"15min":2.2410000000000001},"min":{"1min":0.25,"5min":0.215,"15min":0.215},"max":{"1min":60.722000000000001,"5min":95.320999999999998,"15min":117.23},"last":0.71199999999999997},{"interface":"front","average":{"1min":7.0670000000000002,"5min":4.6360000000000001,"15min":2.319},"min":{"1min":0.32200000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":60.469999999999999,"5min":94.170000000000002,"15min":117.45999999999999},"last":4.29}]},{"osd":4,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":2.3530000000000002,"15min":1.329},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.1709999999999994,"5min":45.779000000000003,"15min":45.779000000000003},"last":4.2519999999999998},{"interface":"front","average":{"1min":1.9239999999999999,"5min":2.2570000000000001,"15min":1.21},"min":{"1min":0.35499999999999998,"5min":0.313,"15min":0.313},"max":{"1min":13.734,"5min":45.735999999999997,"15min":45.735999999999997},"last":0.64000000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":2.1400000000000001,"15min":1.165},"min":{"1min":0.433,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0800000000000001,"5min":47.948999999999998,"15min":47.948999999999998},"last":6.5730000000000004},{"interface":"front","average":{"1min":1.9259999999999999,"5min":2.3730000000000002,"15min":1.264},"min":{"1min":0.38300000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":13.864000000000001,"5min":47.968000000000004,"15min":47.968000000000004},"last":0.60799999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":2.0499999999999998,"15min":1.175},"min":{"1min":0.41199999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":13.831,"5min":13.831,"15min":13.831},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.944,"5min":1.6850000000000001,"15min":0.98299999999999998},"min":{"1min":0.53000000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.117,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":6.5339999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:38:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2869999999999999,"5min":2.3300000000000001,"15min":2.2829999999999999},"min":{"1min":0.46899999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":9.1319999999999997,"5min":11.407,"15min":11.407},"last":4.327},{"interface":"front","average":{"1min":3.3050000000000002,"5min":2.2440000000000002,"15min":2.056},"min":{"1min":0.443,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.2949999999999999,"5min":11.432,"15min":11.432},"last":4.3659999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019130,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1594908,"kb_used_data":1289492,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92150244,"statfs":{"total":95995035648,"available":94361849856,"internally_reserved":0,"allocated":1320439808,"data_stored":1319920083,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:17 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":2.3769999999999998,"15min":1.1399999999999999},"min":{"1min":0.25700000000000001,"5min":0.214,"15min":0.214},"max":{"1min":2.9180000000000001,"5min":36.406999999999996,"15min":36.406999999999996},"last":0.442},{"interface":"front","average":{"1min":1.0649999999999999,"5min":2.7480000000000002,"15min":1.282},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.3010000000000002,"5min":36.454999999999998,"15min":36.454999999999998},"last":2.391}]},{"osd":1,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":1.9430000000000001,"15min":1.0169999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.9550000000000001,"5min":31.073,"15min":31.073},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.421,"5min":1.968,"15min":1.0600000000000001},"min":{"1min":0.28699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.0109999999999992,"5min":31.100000000000001,"15min":31.100000000000001},"last":5.2969999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:38:13 2023","interfaces":[{"interface":"back","average":{"1min":11.664999999999999,"5min":8.2460000000000004,"15min":3.093},"min":{"1min":0.314,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":196.93700000000001,"5min":196.93700000000001,"15min":196.93700000000001},"last":4.4690000000000003},{"interface":"front","average":{"1min":11.625999999999999,"5min":8.0389999999999997,"15min":3.0680000000000001},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":197.08099999999999,"5min":197.08099999999999,"15min":197.08099999999999},"last":0.48899999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":23.963000000000001,"5min":6.3440000000000003,"15min":2.504},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":211.48699999999999,"5min":211.48699999999999,"15min":211.48699999999999},"last":3.0670000000000002},{"interface":"front","average":{"1min":23.547000000000001,"5min":6.0309999999999997,"15min":2.4510000000000001},"min":{"1min":0.38600000000000001,"5min":0.311,"15min":0.309},"max":{"1min":211.31100000000001,"5min":211.31100000000001,"15min":211.31100000000001},"last":4.6870000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":3.6040000000000001,"15min":1.5289999999999999},"min":{"1min":0.25,"5min":0.249,"15min":0.249},"max":{"1min":2.6070000000000002,"5min":113.35899999999999,"15min":113.35899999999999},"last":2.4199999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":3.3849999999999998,"15min":1.5109999999999999},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.0550000000000002,"5min":113.32299999999999,"15min":113.32299999999999},"last":0.68899999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.0569999999999999,"15min":0.69799999999999995},"min":{"1min":0.28999999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.0289999999999999,"5min":11.944000000000001,"15min":11.944000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.3169999999999999,"15min":0.80300000000000005},"min":{"1min":0.313,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.1899999999999999,"5min":10.356,"15min":10.356},"last":2.3540000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":2.0230000000000001,"15min":2.9209999999999998},"min":{"1min":0.38700000000000001,"5min":0.312,"15min":0.312},"max":{"1min":2.0750000000000002,"5min":30.393999999999998,"15min":30.393999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.876,"5min":2.1989999999999998,"15min":3.286},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.5859999999999999,"5min":30.334,"15min":30.334},"last":1.3280000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822902,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1599760,"kb_used_data":1298440,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92145392,"statfs":{"total":95995035648,"available":94356881408,"internally_reserved":0,"allocated":1329602560,"data_stored":1329090595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":6.3940000000000001,"15min":2.8650000000000002},"min":{"1min":0.249,"5min":0.224,"15min":0.19400000000000001},"max":{"1min":10.712999999999999,"5min":165.90199999999999,"15min":165.90199999999999},"last":3.3119999999999998},{"interface":"front","average":{"1min":2.1419999999999999,"5min":6.3799999999999999,"15min":2.9359999999999999},"min":{"1min":0.28999999999999998,"5min":0.215,"15min":0.215},"max":{"1min":8.6099999999999994,"5min":165.96700000000001,"15min":165.96700000000001},"last":2.1070000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":7.4249999999999998,"5min":3.0960000000000001,"15min":1.4159999999999999},"min":{"1min":0.30599999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":87.769000000000005,"5min":87.769000000000005,"15min":87.769000000000005},"last":7.0979999999999999},{"interface":"front","average":{"1min":6.9340000000000002,"5min":2.782,"15min":1.337},"min":{"1min":0.35599999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":87.864000000000004,"5min":87.864000000000004,"15min":87.864000000000004},"last":7.1909999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:38:19 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":8.4670000000000005,"15min":3.2029999999999998},"min":{"1min":0.28599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.9900000000000002,"5min":290.697,"15min":290.697},"last":4.5119999999999996},{"interface":"front","average":{"1min":2.093,"5min":8.3529999999999998,"15min":3.161},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":9.3569999999999993,"5min":290.786,"15min":290.786},"last":3.548}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":5.8250000000000002,"5min":2.7799999999999998,"15min":1.3500000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":52.560000000000002,"5min":52.560000000000002,"15min":52.560000000000002},"last":1.8320000000000001},{"interface":"front","average":{"1min":4.8470000000000004,"5min":2.5710000000000002,"15min":1.268},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":52.380000000000003,"5min":52.380000000000003,"15min":52.380000000000003},"last":3.1970000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":1.0940000000000001,"15min":0.68600000000000005},"min":{"1min":0.25700000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.2949999999999999,"5min":9.3510000000000009,"15min":9.3510000000000009},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.746,"15min":0.93200000000000005},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":2.669,"5min":14.236000000000001,"15min":14.236000000000001},"last":0.43099999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":2.5600000000000001,"15min":1.234},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":1.8069999999999999,"5min":95.010999999999996,"15min":95.010999999999996},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":2.2490000000000001,"15min":1.123},"min":{"1min":0.33500000000000002,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":2.4390000000000001,"5min":94.194000000000003,"15min":94.194000000000003},"last":0.45300000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.5149999999999999,"15min":1.655},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.4289999999999998,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.504},{"interface":"front","average":{"1min":0.51600000000000001,"5min":1.6599999999999999,"15min":1.968},"min":{"1min":0.36199999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.7250000000000001,"5min":14.432,"15min":14.432},"last":0.57599999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1063849984,"data_stored":1063845912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1486950400,"data_stored":1486946328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2334720000,"data_stored":2334720000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2892038144,"data_stored":2892038144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1316814848,"data_stored":1316814848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1315442688,"data_stored":1315438616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2002386944,"data_stored":2002386944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":978124800,"data_stored":978124800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:39:18.636 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:39:18.648 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:18.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:18.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:18.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:18.789 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:19.337 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:39:19.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:19.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:19.446 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:19.549 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:19.634 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:19.666 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:19.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:19.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:19.785 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:19.791 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:39:19.791 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:39:19.791 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:39:19.791 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:39:19.791 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:39:19.813 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:19 smithi191 ceph-mon[161669]: pgmap v516: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 9414/191997 objects degraded (4.903%); 1397/191997 objects misplaced (0.728%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:19.895 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:20.178 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:19 smithi008 ceph-mon[184418]: pgmap v516: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 9414/191997 objects degraded (4.903%); 1397/191997 objects misplaced (0.728%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:20.179 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:19 smithi008 ceph-mon[191227]: pgmap v516: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 9414/191997 objects degraded (4.903%); 1397/191997 objects misplaced (0.728%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:20.471 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:20.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:20.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:20.585 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:20.588 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:39:20.623 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:20.747 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:20 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3006020749' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:20.747 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:20 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:20.747 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:39:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:39:20] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.33.4" 2023-10-09T05:39:20.747 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:20 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3006020749' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:20.747 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:20 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:20.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:39:20.794 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:39:20.796 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:20.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:20.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:20.900 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:20.993 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:20 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3006020749' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:20 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:21.435 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:21 smithi191 ceph-mon[161669]: pgmap v517: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 9414/187134 objects degraded (5.031%); 1207/187134 objects misplaced (0.645%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:21 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:21 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:21 smithi191 ceph-mon[161669]: osdmap e191: 8 total, 8 up, 7 in 2023-10-09T05:39:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:21 smithi191 ceph-mon[161669]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:21 smithi191 ceph-mon[161669]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:21 smithi191 ceph-mon[161669]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:21 smithi191 ceph-mon[161669]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.080 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[184418]: pgmap v517: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 9414/187134 objects degraded (5.031%); 1207/187134 objects misplaced (0.645%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:22.080 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:22.080 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:22.080 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[184418]: osdmap e191: 8 total, 8 up, 7 in 2023-10-09T05:39:22.081 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[184418]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.081 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[184418]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.081 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[184418]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.081 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[184418]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[191227]: pgmap v517: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 9414/187134 objects degraded (5.031%); 1207/187134 objects misplaced (0.645%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:22.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:22.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:22.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[191227]: osdmap e191: 8 total, 8 up, 7 in 2023-10-09T05:39:22.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[191227]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.081 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[191227]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.082 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[191227]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.082 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:21 smithi008 ceph-mon[191227]: failed to encode map e191 with expected crc 2023-10-09T05:39:22.414 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:22.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:22.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:22.524 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:22.614 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:22.830 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:22.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:22.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:22.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:22.984 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:23.057 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:23.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:23.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:23.159 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:23.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:23.213 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:23.214 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:39:23.214 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:39:23.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:39:23.214 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:39:23.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.214 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119031 5.f 5:f62bfa5d:::benchmark_data_smithi008_238708_object895:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:39:23.215 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.042073+0000", 2023-10-09T05:39:23.215 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.16948589600003, 2023-10-09T05:39:23.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99245742100000001, 2023-10-09T05:39:23.215 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.215 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.215 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.215 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:23.215 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:23.215 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119031 2023-10-09T05:39:23.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.216 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042073+0000", 2023-10-09T05:39:23.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042073+0000", 2023-10-09T05:39:23.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999962 2023-10-09T05:39:23.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042070+0000", 2023-10-09T05:39:23.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8694000000000001e-05 2023-10-09T05:39:23.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042088+0000", 2023-10-09T05:39:23.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3565e-05 2023-10-09T05:39:23.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042132+0000", 2023-10-09T05:39:23.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.365e-05 2023-10-09T05:39:23.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042145+0000", 2023-10-09T05:39:23.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14325328600000001 2023-10-09T05:39:23.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185399+0000", 2023-10-09T05:39:23.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8292e-05 2023-10-09T05:39:23.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:23.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185417+0000", 2023-10-09T05:39:23.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83638978799999997 2023-10-09T05:39:23.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.021807+0000", 2023-10-09T05:39:23.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012924400000000001 2023-10-09T05:39:23.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.021936+0000", 2023-10-09T05:39:23.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014705600000000001 2023-10-09T05:39:23.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:23.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.022083+0000", 2023-10-09T05:39:23.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041951740000000003 2023-10-09T05:39:23.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.026278+0000", 2023-10-09T05:39:23.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0043576520000000001 2023-10-09T05:39:23.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.030636+0000", 2023-10-09T05:39:23.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038215580000000001 2023-10-09T05:39:23.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034457+0000", 2023-10-09T05:39:23.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4993999999999997e-05 2023-10-09T05:39:23.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034492+0000", 2023-10-09T05:39:23.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8413e-05 2023-10-09T05:39:23.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034531+0000", 2023-10-09T05:39:23.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99245742100000001 2023-10-09T05:39:23.226 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.226 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.226 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.226 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119032 5.1f 5:fba7d45d:::benchmark_data_smithi008_238708_object896:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:39:23.226 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:39:23.226 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.169395216, 2023-10-09T05:39:23.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0020802900000001, 2023-10-09T05:39:23.228 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.228 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:23.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:23.229 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119032 2023-10-09T05:39:23.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.229 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:39:23.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:39:23.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:23.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:39:23.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8399999999999997e-06 2023-10-09T05:39:23.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042167+0000", 2023-10-09T05:39:23.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7719999999999997e-06 2023-10-09T05:39:23.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042174+0000", 2023-10-09T05:39:23.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5770000000000004e-06 2023-10-09T05:39:23.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042179+0000", 2023-10-09T05:39:23.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14336328700000001 2023-10-09T05:39:23.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185542+0000", 2023-10-09T05:39:23.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4260000000000002e-06 2023-10-09T05:39:23.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:23.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185547+0000", 2023-10-09T05:39:23.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85482789199999998 2023-10-09T05:39:23.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.040375+0000", 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012875500000000001 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.040504+0000", 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017535599999999999 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:23.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.040679+0000", 2023-10-09T05:39:23.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032237390000000002 2023-10-09T05:39:23.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.043903+0000", 2023-10-09T05:39:23.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021366899999999999 2023-10-09T05:39:23.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044117+0000", 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7777000000000001e-05 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044155+0000", 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9549999999999997e-05 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044214+0000", 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0266000000000001e-05 2023-10-09T05:39:23.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.236 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.237 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044244+0000", 2023-10-09T05:39:23.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0020802900000001 2023-10-09T05:39:23.237 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.238 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.238 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.238 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119033 5.1f 5:fb715b35:::benchmark_data_smithi008_238708_object897:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:39:23.238 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.042184+0000", 2023-10-09T05:39:23.239 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.169375726, 2023-10-09T05:39:23.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0031231839999999, 2023-10-09T05:39:23.239 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.239 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:23.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:23.240 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119033 2023-10-09T05:39:23.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.240 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042184+0000", 2023-10-09T05:39:23.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042184+0000", 2023-10-09T05:39:23.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:23.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042183+0000", 2023-10-09T05:39:23.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5550000000000001e-06 2023-10-09T05:39:23.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042186+0000", 2023-10-09T05:39:23.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4040000000000003e-06 2023-10-09T05:39:23.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042191+0000", 2023-10-09T05:39:23.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5389999999999999e-06 2023-10-09T05:39:23.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042195+0000", 2023-10-09T05:39:23.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.143397471 2023-10-09T05:39:23.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185592+0000", 2023-10-09T05:39:23.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4479999999999996e-06 2023-10-09T05:39:23.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:23.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185597+0000", 2023-10-09T05:39:23.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85661535899999997 2023-10-09T05:39:23.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.042212+0000", 2023-10-09T05:39:23.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5107999999999998e-05 2023-10-09T05:39:23.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.042307+0000", 2023-10-09T05:39:23.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3749000000000001e-05 2023-10-09T05:39:23.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:23.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.042391+0000", 2023-10-09T05:39:23.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001859688 2023-10-09T05:39:23.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044251+0000", 2023-10-09T05:39:23.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00037112400000000002 2023-10-09T05:39:23.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044622+0000", 2023-10-09T05:39:23.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060215500000000005 2023-10-09T05:39:23.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.045224+0000", 2023-10-09T05:39:23.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6007999999999997e-05 2023-10-09T05:39:23.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.045270+0000", 2023-10-09T05:39:23.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6947999999999999e-05 2023-10-09T05:39:23.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.045307+0000", 2023-10-09T05:39:23.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0031231839999999 2023-10-09T05:39:23.247 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.247 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.247 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.248 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119034 5.f 5:f790e635:::benchmark_data_smithi008_238708_object898:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:39:23.248 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.066949+0000", 2023-10-09T05:39:23.248 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.14461035900001, 2023-10-09T05:39:23.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.967741929, 2023-10-09T05:39:23.248 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.248 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.249 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.249 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:23.249 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:23.249 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119034 2023-10-09T05:39:23.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.249 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066949+0000", 2023-10-09T05:39:23.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066949+0000", 2023-10-09T05:39:23.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999962 2023-10-09T05:39:23.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066945+0000", 2023-10-09T05:39:23.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3286e-05 2023-10-09T05:39:23.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066959+0000", 2023-10-09T05:39:23.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5304999999999999e-05 2023-10-09T05:39:23.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066984+0000", 2023-10-09T05:39:23.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2354999999999999e-05 2023-10-09T05:39:23.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066996+0000", 2023-10-09T05:39:23.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.118639008 2023-10-09T05:39:23.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185635+0000", 2023-10-09T05:39:23.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0820000000000004e-06 2023-10-09T05:39:23.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:23.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185641+0000", 2023-10-09T05:39:23.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83823218700000002 2023-10-09T05:39:23.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.023874+0000", 2023-10-09T05:39:23.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011226 2023-10-09T05:39:23.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.023986+0000", 2023-10-09T05:39:23.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013421799999999999 2023-10-09T05:39:23.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:23.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.024120+0000", 2023-10-09T05:39:23.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003618137 2023-10-09T05:39:23.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.027738+0000", 2023-10-09T05:39:23.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029752229999999999 2023-10-09T05:39:23.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.030713+0000", 2023-10-09T05:39:23.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039165859999999997 2023-10-09T05:39:23.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034630+0000", 2023-10-09T05:39:23.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7750999999999999e-05 2023-10-09T05:39:23.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034658+0000", 2023-10-09T05:39:23.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3167000000000003e-05 2023-10-09T05:39:23.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034691+0000", 2023-10-09T05:39:23.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.967741929 2023-10-09T05:39:23.261 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.261 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.261 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.261 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4048 6.a 6:51c21166:::benchmark_data_smithi008_295510_object4047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.261 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.480144+0000", 2023-10-09T05:39:23.261 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 134.731415355, 2023-10-09T05:39:23.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48056587499999998, 2023-10-09T05:39:23.262 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.262 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.262 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.262 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.262 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.263 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4048 2023-10-09T05:39:23.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.263 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480144+0000", 2023-10-09T05:39:23.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480144+0000", 2023-10-09T05:39:23.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:39:23.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480143+0000", 2023-10-09T05:39:23.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7996e-05 2023-10-09T05:39:23.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480181+0000", 2023-10-09T05:39:23.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0966e-05 2023-10-09T05:39:23.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480192+0000", 2023-10-09T05:39:23.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6390000000000003e-06 2023-10-09T05:39:23.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.267 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480199+0000", 2023-10-09T05:39:23.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017538499999999999 2023-10-09T05:39:23.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480374+0000", 2023-10-09T05:39:23.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014747999999999999 2023-10-09T05:39:23.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480521+0000", 2023-10-09T05:39:23.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020043799999999999 2023-10-09T05:39:23.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.269 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:39:23.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480722+0000", 2023-10-09T05:39:23.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00076789800000000004 2023-10-09T05:39:23.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.269 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.481490+0000", 2023-10-09T05:39:23.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00070511100000000002 2023-10-09T05:39:23.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.482195+0000", 2023-10-09T05:39:23.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.478412911 2023-10-09T05:39:23.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.960608+0000", 2023-10-09T05:39:23.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5300000000000002e-05 2023-10-09T05:39:23.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.960663+0000", 2023-10-09T05:39:23.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6805999999999999e-05 2023-10-09T05:39:23.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.960710+0000", 2023-10-09T05:39:23.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48056587499999998 2023-10-09T05:39:23.273 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.273 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.273 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.273 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4066 6.a 6:51b3cfb1:::benchmark_data_smithi008_295510_object4065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.273 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.483733+0000", 2023-10-09T05:39:23.273 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 134.72782597400001, 2023-10-09T05:39:23.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47826635200000001, 2023-10-09T05:39:23.274 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.274 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.274 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.274 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.274 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.274 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4066 2023-10-09T05:39:23.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.274 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483733+0000", 2023-10-09T05:39:23.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483733+0000", 2023-10-09T05:39:23.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:23.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483732+0000", 2023-10-09T05:39:23.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3579999999999995e-05 2023-10-09T05:39:23.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483826+0000", 2023-10-09T05:39:23.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.503e-05 2023-10-09T05:39:23.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483841+0000", 2023-10-09T05:39:23.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4710000000000004e-06 2023-10-09T05:39:23.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483847+0000", 2023-10-09T05:39:23.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8745999999999997e-05 2023-10-09T05:39:23.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483896+0000", 2023-10-09T05:39:23.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013626099999999999 2023-10-09T05:39:23.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484032+0000", 2023-10-09T05:39:23.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016087400000000001 2023-10-09T05:39:23.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:39:23.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484193+0000", 2023-10-09T05:39:23.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060899000000000001 2023-10-09T05:39:23.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484802+0000", 2023-10-09T05:39:23.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001503536 2023-10-09T05:39:23.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486306+0000", 2023-10-09T05:39:23.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.475630736 2023-10-09T05:39:23.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.961936+0000", 2023-10-09T05:39:23.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1003000000000001e-05 2023-10-09T05:39:23.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.961967+0000", 2023-10-09T05:39:23.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2406000000000001e-05 2023-10-09T05:39:23.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962000+0000", 2023-10-09T05:39:23.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47826635200000001 2023-10-09T05:39:23.283 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.283 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.283 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.283 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4069 6.d 6:be699dd1:::benchmark_data_smithi008_295510_object4068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.283 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.484947+0000", 2023-10-09T05:39:23.283 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 134.72661252200001, 2023-10-09T05:39:23.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47790969900000002, 2023-10-09T05:39:23.283 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.284 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.284 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.284 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.284 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.284 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4069 2023-10-09T05:39:23.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.284 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484947+0000", 2023-10-09T05:39:23.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484947+0000", 2023-10-09T05:39:23.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:23.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484945+0000", 2023-10-09T05:39:23.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019272500000000001 2023-10-09T05:39:23.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485138+0000", 2023-10-09T05:39:23.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1980999999999999e-05 2023-10-09T05:39:23.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485150+0000", 2023-10-09T05:39:23.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2870000000000004e-06 2023-10-09T05:39:23.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485155+0000", 2023-10-09T05:39:23.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2120999999999998e-05 2023-10-09T05:39:23.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485207+0000", 2023-10-09T05:39:23.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117552 2023-10-09T05:39:23.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485325+0000", 2023-10-09T05:39:23.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000151662 2023-10-09T05:39:23.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:39:23.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485477+0000", 2023-10-09T05:39:23.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011680009999999999 2023-10-09T05:39:23.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486645+0000", 2023-10-09T05:39:23.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6749999999999999e-05 2023-10-09T05:39:23.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486681+0000", 2023-10-09T05:39:23.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47611776500000003 2023-10-09T05:39:23.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962799+0000", 2023-10-09T05:39:23.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4666999999999999e-05 2023-10-09T05:39:23.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962834+0000", 2023-10-09T05:39:23.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2668999999999999e-05 2023-10-09T05:39:23.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962857+0000", 2023-10-09T05:39:23.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47790969900000002 2023-10-09T05:39:23.293 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.293 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.293 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.293 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4072 6.d 6:b4da5f17:::benchmark_data_smithi008_295510_object4071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.293 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.485343+0000", 2023-10-09T05:39:23.294 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 134.72621607400001, 2023-10-09T05:39:23.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47815177399999997, 2023-10-09T05:39:23.294 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.294 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.294 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.294 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.295 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.295 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4072 2023-10-09T05:39:23.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.295 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485343+0000", 2023-10-09T05:39:23.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485343+0000", 2023-10-09T05:39:23.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:39:23.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485342+0000", 2023-10-09T05:39:23.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9259000000000006e-05 2023-10-09T05:39:23.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485442+0000", 2023-10-09T05:39:23.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0434999999999999e-05 2023-10-09T05:39:23.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485452+0000", 2023-10-09T05:39:23.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2230000000000004e-06 2023-10-09T05:39:23.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485458+0000", 2023-10-09T05:39:23.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000491495 2023-10-09T05:39:23.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485950+0000", 2023-10-09T05:39:23.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013497300000000001 2023-10-09T05:39:23.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.300 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.300 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486085+0000", 2023-10-09T05:39:23.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018227900000000001 2023-10-09T05:39:23.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:39:23.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486267+0000", 2023-10-09T05:39:23.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00061199699999999998 2023-10-09T05:39:23.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486879+0000", 2023-10-09T05:39:23.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00083038800000000005 2023-10-09T05:39:23.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.487709+0000", 2023-10-09T05:39:23.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47569748899999997 2023-10-09T05:39:23.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963407+0000", 2023-10-09T05:39:23.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8826000000000003e-05 2023-10-09T05:39:23.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963456+0000", 2023-10-09T05:39:23.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9415000000000001e-05 2023-10-09T05:39:23.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963495+0000", 2023-10-09T05:39:23.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47815177399999997 2023-10-09T05:39:23.304 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.305 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.305 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.306 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4084 6.d 6:b4676de6:::benchmark_data_smithi008_295510_object4083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.306 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.489711+0000", 2023-10-09T05:39:23.306 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 134.72184876099999, 2023-10-09T05:39:23.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47387225700000002, 2023-10-09T05:39:23.307 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.307 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.307 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.307 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.307 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.307 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4084 2023-10-09T05:39:23.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.308 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489711+0000", 2023-10-09T05:39:23.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489711+0000", 2023-10-09T05:39:23.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:39:23.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.310 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489710+0000", 2023-10-09T05:39:23.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7155000000000003e-05 2023-10-09T05:39:23.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489777+0000", 2023-10-09T05:39:23.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4321000000000001e-05 2023-10-09T05:39:23.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489791+0000", 2023-10-09T05:39:23.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6220000000000003e-06 2023-10-09T05:39:23.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489798+0000", 2023-10-09T05:39:23.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.515e-05 2023-10-09T05:39:23.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489843+0000", 2023-10-09T05:39:23.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6768000000000006e-05 2023-10-09T05:39:23.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489940+0000", 2023-10-09T05:39:23.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000134702 2023-10-09T05:39:23.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:39:23.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.490074+0000", 2023-10-09T05:39:23.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000556044 2023-10-09T05:39:23.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.490630+0000", 2023-10-09T05:39:23.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00032459399999999999 2023-10-09T05:39:23.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.490955+0000", 2023-10-09T05:39:23.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47257032300000001 2023-10-09T05:39:23.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963525+0000", 2023-10-09T05:39:23.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8180000000000001e-05 2023-10-09T05:39:23.317 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.317 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.317 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963554+0000", 2023-10-09T05:39:23.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9314000000000001e-05 2023-10-09T05:39:23.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.318 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.318 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963583+0000", 2023-10-09T05:39:23.318 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47387225700000002 2023-10-09T05:39:23.318 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.318 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.319 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.319 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4097 6.a 6:56ff3694:::benchmark_data_smithi008_295510_object4096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.319 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.498925+0000", 2023-10-09T05:39:23.319 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 134.71263450000001, 2023-10-09T05:39:23.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46416822400000002, 2023-10-09T05:39:23.320 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.320 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.320 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.320 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.320 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.320 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4097 2023-10-09T05:39:23.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.321 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498925+0000", 2023-10-09T05:39:23.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498925+0000", 2023-10-09T05:39:23.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:23.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498923+0000", 2023-10-09T05:39:23.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8837e-05 2023-10-09T05:39:23.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498972+0000", 2023-10-09T05:39:23.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7210999999999998e-05 2023-10-09T05:39:23.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.324 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.324 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498989+0000", 2023-10-09T05:39:23.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1210000000000003e-06 2023-10-09T05:39:23.324 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.324 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.324 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.324 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498995+0000", 2023-10-09T05:39:23.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0176000000000001e-05 2023-10-09T05:39:23.325 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.325 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.325 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.325 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499056+0000", 2023-10-09T05:39:23.325 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016777900000000001 2023-10-09T05:39:23.325 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.326 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.326 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.326 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499223+0000", 2023-10-09T05:39:23.326 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000203718 2023-10-09T05:39:23.326 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.326 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.326 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:39:23.327 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499427+0000", 2023-10-09T05:39:23.327 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00054037499999999995 2023-10-09T05:39:23.327 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.327 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.327 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.327 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499967+0000", 2023-10-09T05:39:23.327 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000180817 2023-10-09T05:39:23.328 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.328 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.328 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.328 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.500148+0000", 2023-10-09T05:39:23.328 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46288928400000001 2023-10-09T05:39:23.328 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.329 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.329 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.329 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963038+0000", 2023-10-09T05:39:23.329 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3631999999999999e-05 2023-10-09T05:39:23.329 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.329 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.329 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.329 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963071+0000", 2023-10-09T05:39:23.330 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1931999999999999e-05 2023-10-09T05:39:23.330 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.330 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.330 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.330 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963093+0000", 2023-10-09T05:39:23.330 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46416822400000002 2023-10-09T05:39:23.330 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.330 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.331 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.331 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.331 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.331 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4392 6.d 6:bd5207d1:::benchmark_data_smithi008_295510_object4391:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.331 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:09.102100+0000", 2023-10-09T05:39:23.331 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 134.10945953300001, 2023-10-09T05:39:23.331 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.302277982, 2023-10-09T05:39:23.332 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.332 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.332 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.332 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.332 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.332 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4392 2023-10-09T05:39:23.332 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.333 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.333 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.102100+0000", 2023-10-09T05:39:23.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.333 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.334 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.334 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.334 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.102100+0000", 2023-10-09T05:39:23.334 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:39:23.334 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.334 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.334 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.334 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.102098+0000", 2023-10-09T05:39:23.335 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3757999999999995e-05 2023-10-09T05:39:23.335 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.335 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.335 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.335 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.102162+0000", 2023-10-09T05:39:23.335 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3267e-05 2023-10-09T05:39:23.336 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.336 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.102175+0000", 2023-10-09T05:39:23.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.866e-06 2023-10-09T05:39:23.336 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.102182+0000", 2023-10-09T05:39:23.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3310000000000003e-05 2023-10-09T05:39:23.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.338 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.338 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.102235+0000", 2023-10-09T05:39:23.338 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012792899999999999 2023-10-09T05:39:23.338 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.338 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.338 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.338 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.102363+0000", 2023-10-09T05:39:23.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014733699999999999 2023-10-09T05:39:23.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:39:23.339 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.102511+0000", 2023-10-09T05:39:23.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00052944899999999998 2023-10-09T05:39:23.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.340 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.340 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103040+0000", 2023-10-09T05:39:23.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016374969999999999 2023-10-09T05:39:23.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104677+0000", 2023-10-09T05:39:23.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29960885900000001 2023-10-09T05:39:23.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.404286+0000", 2023-10-09T05:39:23.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9187999999999997e-05 2023-10-09T05:39:23.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.404336+0000", 2023-10-09T05:39:23.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2296999999999998e-05 2023-10-09T05:39:23.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.404378+0000", 2023-10-09T05:39:23.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.302277982 2023-10-09T05:39:23.343 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.343 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.343 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.343 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.343 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4396 6.a 6:57ef3331:::benchmark_data_smithi008_295510_object4395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.343 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:09.103785+0000", 2023-10-09T05:39:23.343 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 134.10777384900001, 2023-10-09T05:39:23.344 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30151979099999998, 2023-10-09T05:39:23.344 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.344 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.344 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.344 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.344 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.344 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4396 2023-10-09T05:39:23.344 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.344 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.345 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.345 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103785+0000", 2023-10-09T05:39:23.345 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.345 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.345 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.345 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.345 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103785+0000", 2023-10-09T05:39:23.345 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:39:23.345 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.346 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.346 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.346 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103784+0000", 2023-10-09T05:39:23.346 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7457000000000001e-05 2023-10-09T05:39:23.346 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.346 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.346 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.346 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103852+0000", 2023-10-09T05:39:23.346 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8090000000000004e-06 2023-10-09T05:39:23.347 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.347 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.347 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.347 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103862+0000", 2023-10-09T05:39:23.347 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6419999999999998e-06 2023-10-09T05:39:23.347 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.347 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103866+0000", 2023-10-09T05:39:23.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0012999999999998e-05 2023-10-09T05:39:23.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103906+0000", 2023-10-09T05:39:23.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5068000000000006e-05 2023-10-09T05:39:23.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103991+0000", 2023-10-09T05:39:23.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000127217 2023-10-09T05:39:23.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:39:23.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104119+0000", 2023-10-09T05:39:23.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30034979499999997 2023-10-09T05:39:23.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.351 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.404468+0000", 2023-10-09T05:39:23.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7435999999999995e-05 2023-10-09T05:39:23.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.404546+0000", 2023-10-09T05:39:23.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00067758100000000002 2023-10-09T05:39:23.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.405223+0000", 2023-10-09T05:39:23.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.265e-05 2023-10-09T05:39:23.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.405266+0000", 2023-10-09T05:39:23.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9165999999999997e-05 2023-10-09T05:39:23.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.405305+0000", 2023-10-09T05:39:23.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30151979099999998 2023-10-09T05:39:23.354 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.354 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.354 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.355 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4397 6.a 6:5ba7a206:::benchmark_data_smithi008_295510_object4396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.355 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:09.103876+0000", 2023-10-09T05:39:23.355 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 134.10768322499999, 2023-10-09T05:39:23.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.302216396, 2023-10-09T05:39:23.355 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.355 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.355 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.356 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.356 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.356 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4397 2023-10-09T05:39:23.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.356 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103876+0000", 2023-10-09T05:39:23.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103876+0000", 2023-10-09T05:39:23.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:23.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103876+0000", 2023-10-09T05:39:23.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4465999999999999e-05 2023-10-09T05:39:23.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103900+0000", 2023-10-09T05:39:23.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6330000000000004e-06 2023-10-09T05:39:23.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103905+0000", 2023-10-09T05:39:23.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8619999999999999e-06 2023-10-09T05:39:23.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.103908+0000", 2023-10-09T05:39:23.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00062188 2023-10-09T05:39:23.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104530+0000", 2023-10-09T05:39:23.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3424999999999994e-05 2023-10-09T05:39:23.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104623+0000", 2023-10-09T05:39:23.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012567200000000001 2023-10-09T05:39:23.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:39:23.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104749+0000", 2023-10-09T05:39:23.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.299760639 2023-10-09T05:39:23.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.404509+0000", 2023-10-09T05:39:23.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7634999999999998e-05 2023-10-09T05:39:23.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.404587+0000", 2023-10-09T05:39:23.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013824169999999999 2023-10-09T05:39:23.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.405969+0000", 2023-10-09T05:39:23.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5294000000000003e-05 2023-10-09T05:39:23.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.364 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.364 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.406025+0000", 2023-10-09T05:39:23.364 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7910000000000005e-05 2023-10-09T05:39:23.364 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.364 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.364 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.364 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.406093+0000", 2023-10-09T05:39:23.364 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.302216396 2023-10-09T05:39:23.364 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.365 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.365 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.365 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.365 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4400 6.7 6:e52dc0f4:::benchmark_data_smithi008_295510_object4399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.365 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:09.104039+0000", 2023-10-09T05:39:23.365 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 134.107520196, 2023-10-09T05:39:23.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30332403600000002, 2023-10-09T05:39:23.366 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.366 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.366 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.366 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.366 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.367 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4400 2023-10-09T05:39:23.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.367 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104039+0000", 2023-10-09T05:39:23.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104039+0000", 2023-10-09T05:39:23.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:23.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104039+0000", 2023-10-09T05:39:23.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5994000000000005e-05 2023-10-09T05:39:23.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104115+0000", 2023-10-09T05:39:23.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.245e-06 2023-10-09T05:39:23.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104119+0000", 2023-10-09T05:39:23.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5349999999999999e-06 2023-10-09T05:39:23.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104122+0000", 2023-10-09T05:39:23.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00045937199999999999 2023-10-09T05:39:23.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104581+0000", 2023-10-09T05:39:23.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112038 2023-10-09T05:39:23.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.104693+0000", 2023-10-09T05:39:23.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.299415873 2023-10-09T05:39:23.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:39:23.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.404109+0000", 2023-10-09T05:39:23.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013352850000000001 2023-10-09T05:39:23.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.405444+0000", 2023-10-09T05:39:23.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00067170700000000001 2023-10-09T05:39:23.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.406116+0000", 2023-10-09T05:39:23.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001163805 2023-10-09T05:39:23.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.407280+0000", 2023-10-09T05:39:23.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6820999999999998e-05 2023-10-09T05:39:23.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.407326+0000", 2023-10-09T05:39:23.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6764999999999998e-05 2023-10-09T05:39:23.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:09.407363+0000", 2023-10-09T05:39:23.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30332403600000002 2023-10-09T05:39:23.376 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.377 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.377 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.377 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:13694 6.7 6:e0b43690:::benchmark_data_smithi008_295510_object13693:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.377 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:13.266020+0000", 2023-10-09T05:39:23.378 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 129.94553983599999, 2023-10-09T05:39:23.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.317964937, 2023-10-09T05:39:23.378 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.378 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.378 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.378 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.379 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.379 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 13694 2023-10-09T05:39:23.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.379 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266020+0000", 2023-10-09T05:39:23.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266020+0000", 2023-10-09T05:39:23.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:39:23.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266018+0000", 2023-10-09T05:39:23.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015691500000000001 2023-10-09T05:39:23.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266175+0000", 2023-10-09T05:39:23.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9809999999999998e-06 2023-10-09T05:39:23.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266185+0000", 2023-10-09T05:39:23.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4829999999999999e-06 2023-10-09T05:39:23.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266191+0000", 2023-10-09T05:39:23.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6820000000000002e-05 2023-10-09T05:39:23.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.384 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266238+0000", 2023-10-09T05:39:23.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011752900000000001 2023-10-09T05:39:23.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.384 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.384 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266355+0000", 2023-10-09T05:39:23.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015299500000000001 2023-10-09T05:39:23.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:39:23.385 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266508+0000", 2023-10-09T05:39:23.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014897070000000001 2023-10-09T05:39:23.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.385 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.267998+0000", 2023-10-09T05:39:23.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00064413000000000003 2023-10-09T05:39:23.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.268642+0000", 2023-10-09T05:39:23.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31523500300000001 2023-10-09T05:39:23.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583877+0000", 2023-10-09T05:39:23.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0569000000000002e-05 2023-10-09T05:39:23.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583937+0000", 2023-10-09T05:39:23.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6968000000000002e-05 2023-10-09T05:39:23.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583984+0000", 2023-10-09T05:39:23.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.317964937 2023-10-09T05:39:23.388 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.389 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.389 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.389 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:13699 6.7 6:e2dc0f5f:::benchmark_data_smithi008_295510_object13698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.389 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:13.266204+0000", 2023-10-09T05:39:23.389 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 129.94535525000001, 2023-10-09T05:39:23.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31789818199999997, 2023-10-09T05:39:23.390 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.390 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.390 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.390 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.390 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.390 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 13699 2023-10-09T05:39:23.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.391 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.391 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.391 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266204+0000", 2023-10-09T05:39:23.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266204+0000", 2023-10-09T05:39:23.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:23.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266203+0000", 2023-10-09T05:39:23.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9275999999999999e-05 2023-10-09T05:39:23.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266233+0000", 2023-10-09T05:39:23.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3199999999999996e-06 2023-10-09T05:39:23.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266239+0000", 2023-10-09T05:39:23.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3409999999999999e-06 2023-10-09T05:39:23.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266242+0000", 2023-10-09T05:39:23.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00098051400000000004 2023-10-09T05:39:23.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.267223+0000", 2023-10-09T05:39:23.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108423 2023-10-09T05:39:23.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.267331+0000", 2023-10-09T05:39:23.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000150328 2023-10-09T05:39:23.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:39:23.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.267482+0000", 2023-10-09T05:39:23.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00078777399999999998 2023-10-09T05:39:23.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.268269+0000", 2023-10-09T05:39:23.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00047432399999999998 2023-10-09T05:39:23.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.268744+0000", 2023-10-09T05:39:23.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.315280326 2023-10-09T05:39:23.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584024+0000", 2023-10-09T05:39:23.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3627000000000001e-05 2023-10-09T05:39:23.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584068+0000", 2023-10-09T05:39:23.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.464e-05 2023-10-09T05:39:23.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584102+0000", 2023-10-09T05:39:23.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31789818199999997 2023-10-09T05:39:23.401 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.401 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.401 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.401 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:13705 6.d 6:b16d4aa8:::benchmark_data_smithi008_295510_object13704:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.401 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:13.269168+0000", 2023-10-09T05:39:23.402 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 129.94239159700001, 2023-10-09T05:39:23.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31545825999999999, 2023-10-09T05:39:23.402 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.402 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.402 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.402 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.403 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.403 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 13705 2023-10-09T05:39:23.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.403 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269168+0000", 2023-10-09T05:39:23.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.404 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269168+0000", 2023-10-09T05:39:23.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:39:23.404 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.404 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269167+0000", 2023-10-09T05:39:23.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7036e-05 2023-10-09T05:39:23.405 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.405 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.405 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.405 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269234+0000", 2023-10-09T05:39:23.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7920000000000004e-06 2023-10-09T05:39:23.405 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.405 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.406 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269241+0000", 2023-10-09T05:39:23.406 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8670000000000004e-06 2023-10-09T05:39:23.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.406 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269245+0000", 2023-10-09T05:39:23.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7756000000000002e-05 2023-10-09T05:39:23.407 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269292+0000", 2023-10-09T05:39:23.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000116999 2023-10-09T05:39:23.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.408 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.408 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269409+0000", 2023-10-09T05:39:23.408 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122028 2023-10-09T05:39:23.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.409 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:39:23.409 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269531+0000", 2023-10-09T05:39:23.409 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00063579900000000004 2023-10-09T05:39:23.409 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.409 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.409 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.409 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.270167+0000", 2023-10-09T05:39:23.410 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002216871 2023-10-09T05:39:23.410 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.410 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.410 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.410 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.272384+0000", 2023-10-09T05:39:23.410 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31213664800000002 2023-10-09T05:39:23.411 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.411 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.411 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.411 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584521+0000", 2023-10-09T05:39:23.411 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2773000000000004e-05 2023-10-09T05:39:23.411 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.412 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.412 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.412 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584583+0000", 2023-10-09T05:39:23.412 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2614000000000002e-05 2023-10-09T05:39:23.412 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.412 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.412 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.413 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584626+0000", 2023-10-09T05:39:23.413 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31545825999999999 2023-10-09T05:39:23.413 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.413 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.413 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.413 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.413 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.414 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:13709 6.d 6:b2644c35:::benchmark_data_smithi008_295510_object13708:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:39:23.414 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:13.273848+0000", 2023-10-09T05:39:23.414 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 129.937711656, 2023-10-09T05:39:23.414 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.312466246, 2023-10-09T05:39:23.414 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.414 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:23.414 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:23.415 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:39:23.415 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:23.415 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 13709 2023-10-09T05:39:23.415 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.415 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.415 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.415 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.415 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.273848+0000", 2023-10-09T05:39:23.415 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.416 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.416 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.416 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.416 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.273848+0000", 2023-10-09T05:39:23.416 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:39:23.416 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.416 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.416 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.417 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.273847+0000", 2023-10-09T05:39:23.417 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3615000000000001e-05 2023-10-09T05:39:23.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.417 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.417 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.417 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.273900+0000", 2023-10-09T05:39:23.417 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0903999999999999e-05 2023-10-09T05:39:23.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.418 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.418 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.273911+0000", 2023-10-09T05:39:23.418 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7439999999999998e-06 2023-10-09T05:39:23.418 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.418 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.419 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.273916+0000", 2023-10-09T05:39:23.419 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4001999999999997e-05 2023-10-09T05:39:23.419 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.419 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.419 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.419 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.273950+0000", 2023-10-09T05:39:23.419 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9932999999999995e-05 2023-10-09T05:39:23.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.420 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.420 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.420 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.274030+0000", 2023-10-09T05:39:23.420 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9518000000000006e-05 2023-10-09T05:39:23.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.421 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.421 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:39:23.421 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.274119+0000", 2023-10-09T05:39:23.421 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31037114900000001 2023-10-09T05:39:23.421 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.421 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.421 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.422 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584491+0000", 2023-10-09T05:39:23.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000178522 2023-10-09T05:39:23.422 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.422 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.422 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:23.422 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584669+0000", 2023-10-09T05:39:23.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001562106 2023-10-09T05:39:23.423 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.423 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.423 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:23.423 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.586231+0000", 2023-10-09T05:39:23.423 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5599999999999997e-05 2023-10-09T05:39:23.423 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.423 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.423 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:23.423 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.586277+0000", 2023-10-09T05:39:23.424 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7079000000000002e-05 2023-10-09T05:39:23.424 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.424 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.424 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.424 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.586314+0000", 2023-10-09T05:39:23.424 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.312466246 2023-10-09T05:39:23.424 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.424 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.425 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.425 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.425 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.45029.0:13694 6.7 e166/162 ondisk, result = 0)", 2023-10-09T05:39:23.425 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:13.274804+0000", 2023-10-09T05:39:23.425 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 129.936754882, 2023-10-09T05:39:23.425 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30918641899999999, 2023-10-09T05:39:23.425 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.425 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:39:23.425 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.426 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.426 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.274804+0000", 2023-10-09T05:39:23.426 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.426 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.426 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.426 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.274804+0000", 2023-10-09T05:39:23.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:39:23.427 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.427 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.427 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.427 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.274804+0000", 2023-10-09T05:39:23.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0369999999999996e-06 2023-10-09T05:39:23.427 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.427 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.428 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.428 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.274808+0000", 2023-10-09T05:39:23.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.308905284 2023-10-09T05:39:23.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.428 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.428 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.428 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583713+0000", 2023-10-09T05:39:23.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7307000000000001e-05 2023-10-09T05:39:23.429 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.429 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.429 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583740+0000", 2023-10-09T05:39:23.429 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115345 2023-10-09T05:39:23.429 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.429 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.429 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583856+0000", 2023-10-09T05:39:23.430 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1392999999999999e-05 2023-10-09T05:39:23.430 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.430 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.430 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.430 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583867+0000", 2023-10-09T05:39:23.430 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012384000000000001 2023-10-09T05:39:23.430 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.430 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.430 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.431 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583991+0000", 2023-10-09T05:39:23.431 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30918641899999999 2023-10-09T05:39:23.431 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.431 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.431 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.431 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.431 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.45029.0:13709 6.d e166/162 ondisk, result = 0)", 2023-10-09T05:39:23.431 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:13.275689+0000", 2023-10-09T05:39:23.432 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 129.93587029700001, 2023-10-09T05:39:23.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30880329200000001, 2023-10-09T05:39:23.432 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:23.432 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:39:23.432 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:23.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:23.432 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.275689+0000", 2023-10-09T05:39:23.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:23.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.433 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.433 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:23.433 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.275689+0000", 2023-10-09T05:39:23.433 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:23.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.433 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.433 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:23.433 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.275688+0000", 2023-10-09T05:39:23.434 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6640000000000001e-06 2023-10-09T05:39:23.434 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.434 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.434 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:23.434 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.275695+0000", 2023-10-09T05:39:23.434 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30802375300000001 2023-10-09T05:39:23.434 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.436 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.436 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:23.436 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583719+0000", 2023-10-09T05:39:23.436 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0290000000000001e-05 2023-10-09T05:39:23.436 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.437 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.437 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:23.437 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583739+0000", 2023-10-09T05:39:23.437 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00073784999999999996 2023-10-09T05:39:23.437 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.437 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.438 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:23.438 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584477+0000", 2023-10-09T05:39:23.438 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1789999999999996e-06 2023-10-09T05:39:23.438 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.438 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.438 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:23.438 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584483+0000", 2023-10-09T05:39:23.438 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1770000000000004e-06 2023-10-09T05:39:23.439 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:23.439 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:23.439 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:23.439 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584492+0000", 2023-10-09T05:39:23.439 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30880329200000001 2023-10-09T05:39:23.439 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.439 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.440 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.440 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:23.440 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:23.440 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:39:23.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:23.442 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:23.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:23.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:23.578 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:23.623 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:23.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:23.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:23.728 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:23.834 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:23 smithi008 ceph-mon[184418]: pgmap v519: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 9414/183330 objects degraded (5.135%); 1207/183330 objects misplaced (0.658%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:23.835 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:23 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:23.835 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9414/187134 objects degraded (5.031%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:23.835 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:23 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:23.835 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:23 smithi008 ceph-mon[191227]: pgmap v519: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 9414/183330 objects degraded (5.135%); 1207/183330 objects misplaced (0.658%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:23.835 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:23 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:23.836 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9414/187134 objects degraded (5.031%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:23.836 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:23 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:23.849 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:39:23.857 INFO:teuthology.orchestra.run.smithi008.stdout:{"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 9414/183330 objects degraded (5.135%), 8 pgs degraded, 8 pgs undersized","count":16},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":657,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":191,"num_osds":8,"num_up_osds":8,"osd_up_since":1696829680,"num_in_osds":7,"osd_in_since":1696829854,"num_remapped_pgs":8},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":81},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":61110,"data_bytes":4007794635,"bytes_used":13942398976,"bytes_avail":658022850560,"bytes_total":671965249536,"degraded_objects":9414,"degraded_total":183330,"degraded_ratio":0.051350024545900833,"misplaced_objects":1207,"misplaced_total":183330,"misplaced_ratio":0.0065837560682921511,"recovering_objects_per_sec":38,"recovering_bytes_per_sec":2515458,"recovering_keys_per_sec":0,"num_objects_recovered":384,"num_bytes_recovered":25165824,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":584},"fsmap":{"epoch":6,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi008.zfejpl","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.8:8443/","prometheus":"http://172.21.15.8:9283/"}},"servicemap":{"epoch":23,"modified":"2023-10-09T05:35:39.975308+0000","services":{}},"progress_events":{"a1c2a5cb-a243-4c8c-845f-d0215fa7ce6b":{"message":"Global Recovery Event (90s)\n [=========================...] (remaining: 8s)","progress":0.91011238098144531,"add_to_ceph_s":true},"e7a35707-3c6c-469d-ad94-cd0d5825f733":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T05:39:23.978 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:23 smithi191 ceph-mon[161669]: pgmap v519: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 9414/183330 objects degraded (5.135%); 1207/183330 objects misplaced (0.658%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:23.978 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:23 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:23.978 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9414/187134 objects degraded (5.031%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:23.978 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:23 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:23.988 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:39:24.120 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:24.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:24.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:24.244 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:24.286 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:24.397 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:24.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:24.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:24.467 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:24.507 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:24.589 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:24.766 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:39:24.955 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:24 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2655968938' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:39:24.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:24 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2655968938' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:39:24.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:24 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2655968938' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:39:25.462 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:25.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:25.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:25.568 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:25.596 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:25.738 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:25 smithi191 ceph-mon[161669]: pgmap v520: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 9414/181107 objects degraded (5.198%); 1007/181107 objects misplaced (0.556%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:25.865 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:25.889 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:25 smithi008 ceph-mon[191227]: pgmap v520: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 9414/181107 objects degraded (5.198%); 1007/181107 objects misplaced (0.556%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:25.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:25.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:25 smithi008 ceph-mon[184418]: pgmap v520: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 9414/181107 objects degraded (5.198%); 1007/181107 objects misplaced (0.556%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:25.949 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:39:25.950 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:39:25.950 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.950 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45029.0:118919 6.6 e191/179 6:63c97ed0:::benchmark_data_smithi008_295510_object30556:head v 191'7493, mlcod=191'7491)", 2023-10-09T05:39:25.950 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:39:25.947580+0000", 2023-10-09T05:39:25.950 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0021437219999999998, 2023-10-09T05:39:25.950 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021632349999999999, 2023-10-09T05:39:25.950 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:25.950 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:39:25.950 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:25.950 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.947580+0000", 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.947580+0000", 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.947579+0000", 2023-10-09T05:39:25.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1301000000000001e-05 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.947590+0000", 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3261000000000001e-05 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.947604+0000", 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0820000000000004e-06 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.947610+0000", 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5772999999999995e-05 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.947676+0000", 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.287e-05 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.947698+0000", 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118533 2023-10-09T05:39:25.954 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:25.954 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:25.954 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:25.954 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.954 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45029.0:118918 6.8 e191/186 6:1ec5663d:::benchmark_data_smithi008_295510_object30555:head v 191'7600, mlcod=191'7596)", 2023-10-09T05:39:25.954 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:39:25.948657+0000", 2023-10-09T05:39:25.954 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0010667649999999999, 2023-10-09T05:39:25.954 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011278390000000001, 2023-10-09T05:39:25.954 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:25.954 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.948657+0000", 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.948657+0000", 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.948656+0000", 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.049e-06 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.948662+0000", 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9750000000000002e-06 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.948672+0000", 2023-10-09T05:39:25.956 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9939999999999997e-06 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.948678+0000", 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1091999999999999e-05 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.948719+0000", 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1651e-05 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:25.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:25.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:39:25.948741+0000", 2023-10-09T05:39:25.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4092999999999996e-05 2023-10-09T05:39:25.958 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:25.958 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:25.958 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:25.958 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:25.958 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:39:25.958 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 2 2023-10-09T05:39:25.958 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:39:25.977 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T05:39:25.977 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:39:25.990 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:26.167 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:26.314 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:26.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:26.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:26.418 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:26.455 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:26.638 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:26.667 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:39:26.675 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:26.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:26.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:26.782 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:26.802 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:39:26.813 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:26.839 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:39:26.891 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:39:26.892 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:26.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:26.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:26.927 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:39:26.966 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:39:26.998 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:27.001 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:39:27.040 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:39:27.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/40698060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:27.061 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/40698060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:27.062 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/40698060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:27.095 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:39:27.103 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:39:27.104 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:39:27.104 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:39:27.104 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:39:27.104 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:39:27.104 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:39:27.104 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:39:27.104 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:39:27.107 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:27.129 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:27.652 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:27 smithi191 ceph-mon[161669]: pgmap v521: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 9414/177516 objects degraded (5.303%); 1007/177516 objects misplaced (0.567%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:27 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9414/177516 objects degraded (5.303%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:28.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:27 smithi008 ceph-mon[184418]: pgmap v521: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 9414/177516 objects degraded (5.303%); 1007/177516 objects misplaced (0.567%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:27 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9414/177516 objects degraded (5.303%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:27 smithi008 ceph-mon[191227]: pgmap v521: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 9414/177516 objects degraded (5.303%); 1007/177516 objects misplaced (0.567%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:27 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9414/177516 objects degraded (5.303%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:28.745 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:39:28.746 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:39:28.954 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:28.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:28.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:29.072 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:29.104 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:29.162 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:39:29.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:29.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:29.265 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:29.295 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:29.442 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:39:29.453 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:39:29.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:29.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:29.568 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:29.609 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:29.684 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:39:29.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:29.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:29.791 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:29.831 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:29.894 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:39:29.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:29.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:29.920 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:29 smithi008 ceph-mon[191227]: pgmap v522: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 9414/174816 objects degraded (5.385%); 1007/174816 objects misplaced (0.576%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:29.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:29 smithi008 ceph-mon[184418]: pgmap v522: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 9414/174816 objects degraded (5.385%); 1007/174816 objects misplaced (0.576%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:29.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:30.000 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:30.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:29 smithi191 ceph-mon[161669]: pgmap v522: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 9414/174816 objects degraded (5.385%); 1007/174816 objects misplaced (0.576%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:39:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:30.069 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:30.671 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:30.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:30.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:30.759 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:39:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:39:30] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.33.4" 2023-10-09T05:39:30.775 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:30.881 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:31.025 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:39:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:30 smithi191 ceph-mon[161669]: pgmap v523: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 9414/169983 objects degraded (5.538%); 796/169983 objects misplaced (0.468%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:39:31.118 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:30 smithi008 ceph-mon[184418]: pgmap v523: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 9414/169983 objects degraded (5.538%); 796/169983 objects misplaced (0.468%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:39:31.119 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:30 smithi008 ceph-mon[191227]: pgmap v523: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 9414/169983 objects degraded (5.538%); 796/169983 objects misplaced (0.468%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:39:31.284 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:31.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:31.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:31.408 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:31.492 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:39:31.493 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:39:31.568 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:31.726 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:31.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:31.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:31.835 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:32.011 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:32.064 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:39:32.113 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:32.132 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:39:32.190 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:32.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:32.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:32.236 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:39:32.303 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:32.354 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:32.424 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:32.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:32.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:32.534 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:32.558 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:39:32.598 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:32.714 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:32.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:32.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:32.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:32 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4294530702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:39:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:32 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4294530702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:39:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:32 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4294530702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:39:33.584 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:33.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:33.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:33.707 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:33.746 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: from='client.55078 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:33.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: pgmap v524: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 9414/166452 objects degraded (5.656%); 796/166452 objects misplaced (0.478%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:33.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:33.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: failed to encode map e192 with expected crc 2023-10-09T05:39:33.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: failed to encode map e192 with expected crc 2023-10-09T05:39:33.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: failed to encode map e192 with expected crc 2023-10-09T05:39:33.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4294530702' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:39:33.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: osdmap e192: 8 total, 8 up, 7 in 2023-10-09T05:39:33.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:39:33.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:39:33.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:33.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9414/166452 objects degraded (5.656%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: from='client.55078 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: pgmap v524: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 9414/166452 objects degraded (5.656%); 796/166452 objects misplaced (0.478%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: failed to encode map e192 with expected crc 2023-10-09T05:39:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: failed to encode map e192 with expected crc 2023-10-09T05:39:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: failed to encode map e192 with expected crc 2023-10-09T05:39:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4294530702' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:39:33.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: osdmap e192: 8 total, 8 up, 7 in 2023-10-09T05:39:33.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:39:33.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:39:33.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:33.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9414/166452 objects degraded (5.656%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:33.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: from='client.55078 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:33.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: pgmap v524: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 9414/166452 objects degraded (5.656%); 796/166452 objects misplaced (0.478%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:39:33.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:33.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: failed to encode map e192 with expected crc 2023-10-09T05:39:33.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: failed to encode map e192 with expected crc 2023-10-09T05:39:33.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: failed to encode map e192 with expected crc 2023-10-09T05:39:33.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4294530702' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:39:33.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: osdmap e192: 8 total, 8 up, 7 in 2023-10-09T05:39:33.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:39:33.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:39:33.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:33.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9414/166452 objects degraded (5.656%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:34.272 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:39:34.273 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:39:34.331 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":523,"stamp":"2023-10-09T05:39:30.075672+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3716224971,"num_objects":56661,"num_object_clones":0,"num_object_copies":169983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9414,"num_objects_misplaced":796,"num_objects_unfound":0,"num_objects_dirty":56661,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":121195,"num_write_kb":5670182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7331,"num_bytes_recovered":478877046,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115859,"ondisk_log_size":115859,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12904448,"kb_used_data":10776712,"kb_used_omap":31,"kb_used_meta":2127520,"kb_avail":643311616,"statfs":{"total":671965249536,"available":658751094784,"internally_reserved":0,"allocated":11035353088,"data_stored":11030840668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2178580937},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,3,5,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-374669312,"num_objects":-5717,"num_object_clones":0,"num_object_copies":-17151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-411,"num_objects_unfound":0,"num_objects_dirty":-5717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5717,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":411,"num_bytes_recovered":26935296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007318"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":174,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.765056+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:39:11.765056+0000","last_peered":"2023-10-09T05:39:11.765056+0000","last_clean":"2023-10-09T05:39:11.765056+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:39:11.765056+0000","last_undegraded":"2023-10-09T05:39:11.765056+0000","last_fullsized":"2023-10-09T05:39:11.765056+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":163,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.756652+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:39:11.756652+0000","last_peered":"2023-10-09T05:39:11.756652+0000","last_clean":"2023-10-09T05:39:11.756652+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:39:11.756652+0000","last_undegraded":"2023-10-09T05:39:11.756652+0000","last_fullsized":"2023-10-09T05:39:11.756652+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":137,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.744983+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:39:11.744983+0000","last_peered":"2023-10-09T05:39:11.744983+0000","last_clean":"2023-10-09T05:39:11.744983+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:39:11.744983+0000","last_undegraded":"2023-10-09T05:39:11.744983+0000","last_fullsized":"2023-10-09T05:39:11.744983+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":167,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.758310+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:39:11.758310+0000","last_peered":"2023-10-09T05:39:11.758310+0000","last_clean":"2023-10-09T05:39:11.758310+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:39:11.758310+0000","last_undegraded":"2023-10-09T05:39:11.758310+0000","last_fullsized":"2023-10-09T05:39:11.758310+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":179,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.753883+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:39:11.753883+0000","last_peered":"2023-10-09T05:39:11.753883+0000","last_clean":"2023-10-09T05:39:11.753883+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:39:11.753883+0000","last_undegraded":"2023-10-09T05:39:11.753883+0000","last_fullsized":"2023-10-09T05:39:11.753883+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":157,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.746822+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:39:11.746822+0000","last_peered":"2023-10-09T05:39:11.746822+0000","last_clean":"2023-10-09T05:39:11.746822+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:39:11.746822+0000","last_undegraded":"2023-10-09T05:39:11.746822+0000","last_fullsized":"2023-10-09T05:39:11.746822+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":142,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.758657+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:39:11.758657+0000","last_peered":"2023-10-09T05:39:11.758657+0000","last_clean":"2023-10-09T05:39:11.758657+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:39:11.758657+0000","last_undegraded":"2023-10-09T05:39:11.758657+0000","last_fullsized":"2023-10-09T05:39:11.758657+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":163,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.753966+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:39:11.753966+0000","last_peered":"2023-10-09T05:39:11.753966+0000","last_clean":"2023-10-09T05:39:11.753966+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:39:11.753966+0000","last_undegraded":"2023-10-09T05:39:11.753966+0000","last_fullsized":"2023-10-09T05:39:11.753966+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":182,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.751750+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:39:11.751750+0000","last_peered":"2023-10-09T05:39:11.751750+0000","last_clean":"2023-10-09T05:39:11.751750+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:39:11.751750+0000","last_undegraded":"2023-10-09T05:39:11.751750+0000","last_fullsized":"2023-10-09T05:39:11.751750+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":177,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.754520+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:39:11.754520+0000","last_peered":"2023-10-09T05:39:11.754520+0000","last_clean":"2023-10-09T05:39:11.754520+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:39:11.754520+0000","last_undegraded":"2023-10-09T05:39:11.754520+0000","last_fullsized":"2023-10-09T05:39:11.754520+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":70,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.756581+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:39:11.756581+0000","last_peered":"2023-10-09T05:39:11.756581+0000","last_clean":"2023-10-09T05:39:11.756581+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:39:11.756581+0000","last_undegraded":"2023-10-09T05:39:11.756581+0000","last_fullsized":"2023-10-09T05:39:11.756581+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":137,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.746353+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:39:11.746353+0000","last_peered":"2023-10-09T05:39:11.746353+0000","last_clean":"2023-10-09T05:39:11.746353+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:39:11.746353+0000","last_undegraded":"2023-10-09T05:39:11.746353+0000","last_fullsized":"2023-10-09T05:39:11.746353+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":199,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.760437+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:39:11.760437+0000","last_peered":"2023-10-09T05:39:11.760437+0000","last_clean":"2023-10-09T05:39:11.760437+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:39:11.760437+0000","last_undegraded":"2023-10-09T05:39:11.760437+0000","last_fullsized":"2023-10-09T05:39:11.760437+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":157,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.760071+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:39:11.760071+0000","last_peered":"2023-10-09T05:39:11.760071+0000","last_clean":"2023-10-09T05:39:11.760071+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:39:11.760071+0000","last_undegraded":"2023-10-09T05:39:11.760071+0000","last_fullsized":"2023-10-09T05:39:11.760071+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":70,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.760118+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:39:11.760118+0000","last_peered":"2023-10-09T05:39:11.760118+0000","last_clean":"2023-10-09T05:39:11.760118+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:39:11.760118+0000","last_undegraded":"2023-10-09T05:39:11.760118+0000","last_fullsized":"2023-10-09T05:39:11.760118+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":187,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.753980+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:39:11.753980+0000","last_peered":"2023-10-09T05:39:11.753980+0000","last_clean":"2023-10-09T05:39:11.753980+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:39:11.753980+0000","last_undegraded":"2023-10-09T05:39:11.753980+0000","last_fullsized":"2023-10-09T05:39:11.753980+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":167,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:18.074780+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:39:18.074780+0000","last_peered":"2023-10-09T05:39:18.074780+0000","last_clean":"2023-10-09T05:39:18.074780+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:39:18.074780+0000","last_undegraded":"2023-10-09T05:39:18.074780+0000","last_fullsized":"2023-10-09T05:39:18.074780+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":181,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.752451+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:39:11.752451+0000","last_peered":"2023-10-09T05:39:11.752451+0000","last_clean":"2023-10-09T05:39:11.752451+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:39:11.752451+0000","last_undegraded":"2023-10-09T05:39:11.752451+0000","last_fullsized":"2023-10-09T05:39:11.752451+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":88,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.759926+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:39:11.759926+0000","last_peered":"2023-10-09T05:39:11.759926+0000","last_clean":"2023-10-09T05:39:11.759926+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:39:11.759926+0000","last_undegraded":"2023-10-09T05:39:11.759926+0000","last_fullsized":"2023-10-09T05:39:11.759926+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":166,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.758240+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:39:11.758240+0000","last_peered":"2023-10-09T05:39:11.758240+0000","last_clean":"2023-10-09T05:39:11.758240+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:39:11.758240+0000","last_undegraded":"2023-10-09T05:39:11.758240+0000","last_fullsized":"2023-10-09T05:39:11.758240+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":166,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.747200+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:39:11.747200+0000","last_peered":"2023-10-09T05:39:11.747200+0000","last_clean":"2023-10-09T05:39:11.747200+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:39:11.747200+0000","last_undegraded":"2023-10-09T05:39:11.747200+0000","last_fullsized":"2023-10-09T05:39:11.747200+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":187,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.754550+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:39:11.754550+0000","last_peered":"2023-10-09T05:39:11.754550+0000","last_clean":"2023-10-09T05:39:11.754550+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:39:11.754550+0000","last_undegraded":"2023-10-09T05:39:11.754550+0000","last_fullsized":"2023-10-09T05:39:11.754550+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"191'7560","reported_seq":7597,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:29.900890+0000","last_change":"2023-10-09T05:37:01.636984+0000","last_active":"2023-10-09T05:39:29.900890+0000","last_peered":"2023-10-09T05:39:29.900890+0000","last_clean":"2023-10-09T05:39:29.900890+0000","last_became_active":"2023-10-09T05:37:01.636609+0000","last_became_peered":"2023-10-09T05:37:01.636609+0000","last_unstale":"2023-10-09T05:39:29.900890+0000","last_undegraded":"2023-10-09T05:39:29.900890+0000","last_fullsized":"2023-10-09T05:39:29.900890+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7560,"log_dups_size":0,"ondisk_log_size":7560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229113856,"num_objects":3496,"num_object_clones":0,"num_object_copies":10488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7560,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":68,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.761672+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:39:11.761672+0000","last_peered":"2023-10-09T05:39:11.761672+0000","last_clean":"2023-10-09T05:39:11.761672+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:39:11.761672+0000","last_undegraded":"2023-10-09T05:39:11.761672+0000","last_fullsized":"2023-10-09T05:39:11.761672+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":166,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.751553+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:39:11.751553+0000","last_peered":"2023-10-09T05:39:11.751553+0000","last_clean":"2023-10-09T05:39:11.751553+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:39:11.751553+0000","last_undegraded":"2023-10-09T05:39:11.751553+0000","last_fullsized":"2023-10-09T05:39:11.751553+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"191'7396","reported_seq":7443,"reported_epoch":191,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:26.107519+0000","last_change":"2023-10-09T05:37:36.837344+0000","last_active":"2023-10-09T05:39:26.107519+0000","last_peered":"2023-10-09T05:39:26.107519+0000","last_clean":"2023-10-09T05:37:34.651883+0000","last_became_active":"2023-10-09T05:37:36.832502+0000","last_became_peered":"2023-10-09T05:37:36.832502+0000","last_unstale":"2023-10-09T05:39:26.107519+0000","last_undegraded":"2023-10-09T05:37:36.702797+0000","last_fullsized":"2023-10-09T05:37:36.678583+0000","mapping_epoch":170,"log_start":"166'1000","ondisk_log_start":"166'1000","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6396,"log_dups_size":0,"ondisk_log_size":6396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:57.419460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235667456,"num_objects":3596,"num_object_clones":0,"num_object_copies":10788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7396,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":706},{"shards":"1,3,7","objects":990}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":68,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.761907+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:39:11.761907+0000","last_peered":"2023-10-09T05:39:11.761907+0000","last_clean":"2023-10-09T05:39:11.761907+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:39:11.761907+0000","last_undegraded":"2023-10-09T05:39:11.761907+0000","last_fullsized":"2023-10-09T05:39:11.761907+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":172,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.759885+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:39:11.759885+0000","last_peered":"2023-10-09T05:39:11.759885+0000","last_clean":"2023-10-09T05:39:11.759885+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:39:11.759885+0000","last_undegraded":"2023-10-09T05:39:11.759885+0000","last_fullsized":"2023-10-09T05:39:11.759885+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"191'7623","reported_seq":13721,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:26.592309+0000","last_change":"2023-10-09T05:38:55.317808+0000","last_active":"2023-10-09T05:39:26.592309+0000","last_peered":"2023-10-09T05:39:26.592309+0000","last_clean":"2023-10-09T05:39:26.592309+0000","last_became_active":"2023-10-09T05:38:55.302739+0000","last_became_peered":"2023-10-09T05:38:55.302739+0000","last_unstale":"2023-10-09T05:39:26.592309+0000","last_undegraded":"2023-10-09T05:39:26.592309+0000","last_fullsized":"2023-10-09T05:39:26.592309+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7623,"log_dups_size":0,"ondisk_log_size":7623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:25.379818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235208704,"num_objects":3589,"num_object_clones":0,"num_object_copies":10767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7623,"num_write_kb":358784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":198049792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":68,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.752535+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:39:11.752535+0000","last_peered":"2023-10-09T05:39:11.752535+0000","last_clean":"2023-10-09T05:39:11.752535+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:39:11.752535+0000","last_undegraded":"2023-10-09T05:39:11.752535+0000","last_fullsized":"2023-10-09T05:39:11.752535+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":190,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.746422+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:39:11.746422+0000","last_peered":"2023-10-09T05:39:11.746422+0000","last_clean":"2023-10-09T05:39:11.746422+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:39:11.746422+0000","last_undegraded":"2023-10-09T05:39:11.746422+0000","last_fullsized":"2023-10-09T05:39:11.746422+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"191'7491","reported_seq":10216,"reported_epoch":191,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:28.070888+0000","last_change":"2023-10-09T05:38:53.467497+0000","last_active":"2023-10-09T05:39:28.070888+0000","last_peered":"2023-10-09T05:39:28.070888+0000","last_clean":"2023-10-09T05:37:34.677467+0000","last_became_active":"2023-10-09T05:37:36.803958+0000","last_became_peered":"2023-10-09T05:37:36.803958+0000","last_unstale":"2023-10-09T05:39:28.070888+0000","last_undegraded":"2023-10-09T05:37:36.800519+0000","last_fullsized":"2023-10-09T05:37:36.678690+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7491,"log_dups_size":0,"ondisk_log_size":7491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:40.690185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227999744,"num_objects":3479,"num_object_clones":0,"num_object_copies":10437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":764,"num_objects_misplaced":796,"num_objects_unfound":0,"num_objects_dirty":3479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7491,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1339,"num_bytes_recovered":87752704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":1919},{"shards":"0,2,7","objects":796}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":158,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.751958+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:39:11.751958+0000","last_peered":"2023-10-09T05:39:11.751958+0000","last_clean":"2023-10-09T05:39:11.751958+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:39:11.751958+0000","last_undegraded":"2023-10-09T05:39:11.751958+0000","last_fullsized":"2023-10-09T05:39:11.751958+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":143,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.757715+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:39:11.757715+0000","last_peered":"2023-10-09T05:39:11.757715+0000","last_clean":"2023-10-09T05:39:11.757715+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:39:11.757715+0000","last_undegraded":"2023-10-09T05:39:11.757715+0000","last_fullsized":"2023-10-09T05:39:11.757715+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"191'7533","reported_seq":7570,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:29.944812+0000","last_change":"2023-10-09T05:37:01.439878+0000","last_active":"2023-10-09T05:39:29.944812+0000","last_peered":"2023-10-09T05:39:29.944812+0000","last_clean":"2023-10-09T05:39:29.944812+0000","last_became_active":"2023-10-09T05:37:01.439481+0000","last_became_peered":"2023-10-09T05:37:01.439481+0000","last_unstale":"2023-10-09T05:39:29.944812+0000","last_undegraded":"2023-10-09T05:39:29.944812+0000","last_fullsized":"2023-10-09T05:39:29.944812+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7533,"log_dups_size":0,"ondisk_log_size":7533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226295808,"num_objects":3453,"num_object_clones":0,"num_object_copies":10359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7533,"num_write_kb":351552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":161,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.759832+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:39:11.759832+0000","last_peered":"2023-10-09T05:39:11.759832+0000","last_clean":"2023-10-09T05:39:11.759832+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:39:11.759832+0000","last_undegraded":"2023-10-09T05:39:11.759832+0000","last_fullsized":"2023-10-09T05:39:11.759832+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":157,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.760341+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:39:11.760341+0000","last_peered":"2023-10-09T05:39:11.760341+0000","last_clean":"2023-10-09T05:39:11.760341+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:39:11.760341+0000","last_undegraded":"2023-10-09T05:39:11.760341+0000","last_fullsized":"2023-10-09T05:39:11.760341+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"191'7444","reported_seq":7483,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:26.525865+0000","last_change":"2023-10-09T05:37:01.440175+0000","last_active":"2023-10-09T05:39:26.525865+0000","last_peered":"2023-10-09T05:39:26.525865+0000","last_clean":"2023-10-09T05:39:26.525865+0000","last_became_active":"2023-10-09T05:37:01.439846+0000","last_became_peered":"2023-10-09T05:37:01.439846+0000","last_unstale":"2023-10-09T05:39:26.525865+0000","last_undegraded":"2023-10-09T05:39:26.525865+0000","last_fullsized":"2023-10-09T05:39:26.525865+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7444,"log_dups_size":0,"ondisk_log_size":7444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:27.883265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236584960,"num_objects":3610,"num_object_clones":0,"num_object_copies":10830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7444,"num_write_kb":353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":158,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.746290+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:39:11.746290+0000","last_peered":"2023-10-09T05:39:11.746290+0000","last_clean":"2023-10-09T05:39:11.746290+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:39:11.746290+0000","last_undegraded":"2023-10-09T05:39:11.746290+0000","last_fullsized":"2023-10-09T05:39:11.746290+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":172,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.760609+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:39:11.760609+0000","last_peered":"2023-10-09T05:39:11.760609+0000","last_clean":"2023-10-09T05:39:11.760609+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:39:11.760609+0000","last_undegraded":"2023-10-09T05:39:11.760609+0000","last_fullsized":"2023-10-09T05:39:11.760609+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"191'7450","reported_seq":7489,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:26.089595+0000","last_change":"2023-10-09T05:37:01.638178+0000","last_active":"2023-10-09T05:39:26.089595+0000","last_peered":"2023-10-09T05:39:26.089595+0000","last_clean":"2023-10-09T05:39:26.089595+0000","last_became_active":"2023-10-09T05:37:01.637822+0000","last_became_peered":"2023-10-09T05:37:01.637822+0000","last_unstale":"2023-10-09T05:39:26.089595+0000","last_undegraded":"2023-10-09T05:39:26.089595+0000","last_fullsized":"2023-10-09T05:39:26.089595+0000","mapping_epoch":162,"log_start":"166'1000","ondisk_log_start":"166'1000","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6450,"log_dups_size":0,"ondisk_log_size":6450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:13.240069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240123904,"num_objects":3664,"num_object_clones":0,"num_object_copies":10992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7450,"num_write_kb":355648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":182,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.759773+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:39:11.759773+0000","last_peered":"2023-10-09T05:39:11.759773+0000","last_clean":"2023-10-09T05:39:11.759773+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:39:11.759773+0000","last_undegraded":"2023-10-09T05:39:11.759773+0000","last_fullsized":"2023-10-09T05:39:11.759773+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":166,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.761991+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:39:11.761991+0000","last_peered":"2023-10-09T05:39:11.761991+0000","last_clean":"2023-10-09T05:39:11.761991+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:39:11.761991+0000","last_undegraded":"2023-10-09T05:39:11.761991+0000","last_fullsized":"2023-10-09T05:39:11.761991+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"191'7633","reported_seq":7671,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:29.849697+0000","last_change":"2023-10-09T05:37:01.438917+0000","last_active":"2023-10-09T05:39:29.849697+0000","last_peered":"2023-10-09T05:39:29.849697+0000","last_clean":"2023-10-09T05:39:29.849697+0000","last_became_active":"2023-10-09T05:37:01.438658+0000","last_became_peered":"2023-10-09T05:37:01.438658+0000","last_unstale":"2023-10-09T05:39:29.849697+0000","last_undegraded":"2023-10-09T05:39:29.849697+0000","last_fullsized":"2023-10-09T05:39:29.849697+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7633,"log_dups_size":0,"ondisk_log_size":7633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224657432,"num_objects":3429,"num_object_clones":0,"num_object_copies":10287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3429,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7633,"num_write_kb":353921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":188,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.753791+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:39:11.753791+0000","last_peered":"2023-10-09T05:39:11.753791+0000","last_clean":"2023-10-09T05:39:11.753791+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:39:11.753791+0000","last_undegraded":"2023-10-09T05:39:11.753791+0000","last_fullsized":"2023-10-09T05:39:11.753791+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":138,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.786971+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:39:20.786971+0000","last_peered":"2023-10-09T05:39:20.786971+0000","last_clean":"2023-10-09T05:39:20.786971+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:39:20.786971+0000","last_undegraded":"2023-10-09T05:39:20.786971+0000","last_fullsized":"2023-10-09T05:39:20.786971+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"191'7534","reported_seq":7572,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:28.083397+0000","last_change":"2023-10-09T05:37:01.440093+0000","last_active":"2023-10-09T05:39:28.083397+0000","last_peered":"2023-10-09T05:39:28.083397+0000","last_clean":"2023-10-09T05:39:28.083397+0000","last_became_active":"2023-10-09T05:37:01.439799+0000","last_became_peered":"2023-10-09T05:37:01.439799+0000","last_unstale":"2023-10-09T05:39:28.083397+0000","last_undegraded":"2023-10-09T05:39:28.083397+0000","last_fullsized":"2023-10-09T05:39:28.083397+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7534,"log_dups_size":0,"ondisk_log_size":7534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:28.734471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234094592,"num_objects":3572,"num_object_clones":0,"num_object_copies":10716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7534,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":725,"reported_epoch":190,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:11.742436+0000","last_change":"2023-10-09T05:37:36.736822+0000","last_active":"2023-10-09T05:39:11.742436+0000","last_peered":"2023-10-09T05:39:11.742436+0000","last_clean":"2023-10-09T05:37:30.792074+0000","last_became_active":"2023-10-09T05:37:36.736273+0000","last_became_peered":"2023-10-09T05:37:36.736273+0000","last_unstale":"2023-10-09T05:39:11.742436+0000","last_undegraded":"2023-10-09T05:37:36.682657+0000","last_fullsized":"2023-10-09T05:37:36.682189+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:23.350354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":173,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.752375+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:39:11.752375+0000","last_peered":"2023-10-09T05:39:11.752375+0000","last_clean":"2023-10-09T05:39:11.752375+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:39:11.752375+0000","last_undegraded":"2023-10-09T05:39:11.752375+0000","last_fullsized":"2023-10-09T05:39:11.752375+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":123,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.744997+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:39:11.744997+0000","last_peered":"2023-10-09T05:39:11.744997+0000","last_clean":"2023-10-09T05:39:11.744997+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:39:11.744997+0000","last_undegraded":"2023-10-09T05:39:11.744997+0000","last_fullsized":"2023-10-09T05:39:11.744997+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"191'7500","reported_seq":13429,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:26.040363+0000","last_change":"2023-10-09T05:38:10.132330+0000","last_active":"2023-10-09T05:39:26.040363+0000","last_peered":"2023-10-09T05:39:26.040363+0000","last_clean":"2023-10-09T05:39:26.040363+0000","last_became_active":"2023-10-09T05:38:10.127080+0000","last_became_peered":"2023-10-09T05:38:10.127080+0000","last_unstale":"2023-10-09T05:39:26.040363+0000","last_undegraded":"2023-10-09T05:39:26.040363+0000","last_fullsized":"2023-10-09T05:39:26.040363+0000","mapping_epoch":179,"log_start":"166'1100","ondisk_log_start":"166'1100","created":162,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6400,"log_dups_size":0,"ondisk_log_size":6400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:12.659227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239075328,"num_objects":3648,"num_object_clones":0,"num_object_copies":10944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7500,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":192413696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":627,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.752122+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:39:11.752122+0000","last_peered":"2023-10-09T05:39:11.752122+0000","last_clean":"2023-10-09T05:39:11.752122+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:39:11.752122+0000","last_undegraded":"2023-10-09T05:39:11.752122+0000","last_fullsized":"2023-10-09T05:39:11.752122+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":158,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.760161+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:39:11.760161+0000","last_peered":"2023-10-09T05:39:11.760161+0000","last_clean":"2023-10-09T05:39:11.760161+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:39:11.760161+0000","last_undegraded":"2023-10-09T05:39:11.760161+0000","last_fullsized":"2023-10-09T05:39:11.760161+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":170,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.752036+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:39:11.752036+0000","last_peered":"2023-10-09T05:39:11.752036+0000","last_clean":"2023-10-09T05:39:11.752036+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:39:11.752036+0000","last_undegraded":"2023-10-09T05:39:11.752036+0000","last_fullsized":"2023-10-09T05:39:11.752036+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"191'7385","reported_seq":7433,"reported_epoch":191,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:25.631005+0000","last_change":"2023-10-09T05:37:40.649912+0000","last_active":"2023-10-09T05:39:25.631005+0000","last_peered":"2023-10-09T05:39:25.631005+0000","last_clean":"2023-10-09T05:37:34.677100+0000","last_became_active":"2023-10-09T05:37:40.593033+0000","last_became_peered":"2023-10-09T05:37:40.593033+0000","last_unstale":"2023-10-09T05:39:25.631005+0000","last_undegraded":"2023-10-09T05:37:40.546647+0000","last_fullsized":"2023-10-09T05:37:40.543498+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7385,"log_dups_size":0,"ondisk_log_size":7385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236257280,"num_objects":3605,"num_object_clones":0,"num_object_copies":10815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7385,"num_write_kb":351680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":755},{"shards":"3,6,7","objects":960}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":505,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.764061+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:39:20.764061+0000","last_peered":"2023-10-09T05:39:20.764061+0000","last_clean":"2023-10-09T05:39:20.764061+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:39:20.764061+0000","last_undegraded":"2023-10-09T05:39:20.764061+0000","last_fullsized":"2023-10-09T05:39:20.764061+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":138,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.742344+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:39:11.742344+0000","last_peered":"2023-10-09T05:39:11.742344+0000","last_clean":"2023-10-09T05:39:11.742344+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:39:11.742344+0000","last_undegraded":"2023-10-09T05:39:11.742344+0000","last_fullsized":"2023-10-09T05:39:11.742344+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":168,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.742373+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:39:11.742373+0000","last_peered":"2023-10-09T05:39:11.742373+0000","last_clean":"2023-10-09T05:39:11.742373+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:39:11.742373+0000","last_undegraded":"2023-10-09T05:39:11.742373+0000","last_fullsized":"2023-10-09T05:39:11.742373+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"191'7557","reported_seq":7594,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:29.937562+0000","last_change":"2023-10-09T05:37:01.438236+0000","last_active":"2023-10-09T05:39:29.937562+0000","last_peered":"2023-10-09T05:39:29.937562+0000","last_clean":"2023-10-09T05:39:29.937562+0000","last_became_active":"2023-10-09T05:37:01.437917+0000","last_became_peered":"2023-10-09T05:37:01.437917+0000","last_unstale":"2023-10-09T05:39:29.937562+0000","last_undegraded":"2023-10-09T05:39:29.937562+0000","last_fullsized":"2023-10-09T05:39:29.937562+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7557,"log_dups_size":0,"ondisk_log_size":7557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230490112,"num_objects":3517,"num_object_clones":0,"num_object_copies":10551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7557,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":775,"reported_epoch":190,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:11.751751+0000","last_change":"2023-10-09T05:37:36.709131+0000","last_active":"2023-10-09T05:39:11.751751+0000","last_peered":"2023-10-09T05:39:11.751751+0000","last_clean":"2023-10-09T05:37:30.688708+0000","last_became_active":"2023-10-09T05:37:36.708682+0000","last_became_peered":"2023-10-09T05:37:36.708682+0000","last_unstale":"2023-10-09T05:39:11.751751+0000","last_undegraded":"2023-10-09T05:37:36.683823+0000","last_fullsized":"2023-10-09T05:37:36.683109+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":338,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.758904+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:39:11.758904+0000","last_peered":"2023-10-09T05:39:11.758904+0000","last_clean":"2023-10-09T05:39:11.758904+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:39:11.758904+0000","last_undegraded":"2023-10-09T05:39:11.758904+0000","last_fullsized":"2023-10-09T05:39:11.758904+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":154,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.754054+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:39:11.754054+0000","last_peered":"2023-10-09T05:39:11.754054+0000","last_clean":"2023-10-09T05:39:11.754054+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:39:11.754054+0000","last_undegraded":"2023-10-09T05:39:11.754054+0000","last_fullsized":"2023-10-09T05:39:11.754054+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":164,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.747836+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:39:11.747836+0000","last_peered":"2023-10-09T05:39:11.747836+0000","last_clean":"2023-10-09T05:39:11.747836+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:39:11.747836+0000","last_undegraded":"2023-10-09T05:39:11.747836+0000","last_fullsized":"2023-10-09T05:39:11.747836+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"191'7465","reported_seq":7503,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:28.788867+0000","last_change":"2023-10-09T05:37:01.440172+0000","last_active":"2023-10-09T05:39:28.788867+0000","last_peered":"2023-10-09T05:39:28.788867+0000","last_clean":"2023-10-09T05:39:28.788867+0000","last_became_active":"2023-10-09T05:37:01.439932+0000","last_became_peered":"2023-10-09T05:37:01.439932+0000","last_unstale":"2023-10-09T05:39:28.788867+0000","last_undegraded":"2023-10-09T05:39:28.788867+0000","last_fullsized":"2023-10-09T05:39:28.788867+0000","mapping_epoch":162,"log_start":"166'1000","ondisk_log_start":"166'1000","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6465,"log_dups_size":0,"ondisk_log_size":6465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:57.826704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229048320,"num_objects":3495,"num_object_clones":0,"num_object_copies":10485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7465,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":716,"reported_epoch":190,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:11.741944+0000","last_change":"2023-10-09T05:37:36.748736+0000","last_active":"2023-10-09T05:39:11.741944+0000","last_peered":"2023-10-09T05:39:11.741944+0000","last_clean":"2023-10-09T05:22:39.601058+0000","last_became_active":"2023-10-09T05:37:36.748304+0000","last_became_peered":"2023-10-09T05:37:36.748304+0000","last_unstale":"2023-10-09T05:39:11.741944+0000","last_undegraded":"2023-10-09T05:37:36.696424+0000","last_fullsized":"2023-10-09T05:37:36.695987+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:18.784578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":143,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.766457+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:39:11.766457+0000","last_peered":"2023-10-09T05:39:11.766457+0000","last_clean":"2023-10-09T05:39:11.766457+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:39:11.766457+0000","last_undegraded":"2023-10-09T05:39:11.766457+0000","last_fullsized":"2023-10-09T05:39:11.766457+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":158,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.758460+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:39:11.758460+0000","last_peered":"2023-10-09T05:39:11.758460+0000","last_clean":"2023-10-09T05:39:11.758460+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:39:11.758460+0000","last_undegraded":"2023-10-09T05:39:11.758460+0000","last_fullsized":"2023-10-09T05:39:11.758460+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"191'7545","reported_seq":7584,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:28.783313+0000","last_change":"2023-10-09T05:37:01.636967+0000","last_active":"2023-10-09T05:39:28.783313+0000","last_peered":"2023-10-09T05:39:28.783313+0000","last_clean":"2023-10-09T05:39:28.783313+0000","last_became_active":"2023-10-09T05:37:01.636602+0000","last_became_peered":"2023-10-09T05:37:01.636602+0000","last_unstale":"2023-10-09T05:39:28.783313+0000","last_undegraded":"2023-10-09T05:39:28.783313+0000","last_fullsized":"2023-10-09T05:39:28.783313+0000","mapping_epoch":162,"log_start":"166'1100","ondisk_log_start":"166'1100","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6445,"log_dups_size":0,"ondisk_log_size":6445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:11.668143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232456192,"num_objects":3547,"num_object_clones":0,"num_object_copies":10641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7545,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":379,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.759255+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:39:11.759255+0000","last_peered":"2023-10-09T05:39:11.759255+0000","last_clean":"2023-10-09T05:39:11.759255+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:39:11.759255+0000","last_undegraded":"2023-10-09T05:39:11.759255+0000","last_fullsized":"2023-10-09T05:39:11.759255+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":138,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.742749+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:39:11.742749+0000","last_peered":"2023-10-09T05:39:11.742749+0000","last_clean":"2023-10-09T05:39:11.742749+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:39:11.742749+0000","last_undegraded":"2023-10-09T05:39:11.742749+0000","last_fullsized":"2023-10-09T05:39:11.742749+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":152,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.757949+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:39:11.757949+0000","last_peered":"2023-10-09T05:39:11.757949+0000","last_clean":"2023-10-09T05:39:11.757949+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:39:11.757949+0000","last_undegraded":"2023-10-09T05:39:11.757949+0000","last_fullsized":"2023-10-09T05:39:11.757949+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"191'7720","reported_seq":7757,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:29.951736+0000","last_change":"2023-10-09T05:37:01.440074+0000","last_active":"2023-10-09T05:39:29.951736+0000","last_peered":"2023-10-09T05:39:29.951736+0000","last_clean":"2023-10-09T05:39:29.951736+0000","last_became_active":"2023-10-09T05:37:01.439808+0000","last_became_peered":"2023-10-09T05:37:01.439808+0000","last_unstale":"2023-10-09T05:39:29.951736+0000","last_undegraded":"2023-10-09T05:39:29.951736+0000","last_fullsized":"2023-10-09T05:39:29.951736+0000","mapping_epoch":162,"log_start":"166'200","ondisk_log_start":"166'200","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7520,"log_dups_size":0,"ondisk_log_size":7520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222691328,"num_objects":3398,"num_object_clones":0,"num_object_copies":10194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7720,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":189,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.754471+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:39:11.754471+0000","last_peered":"2023-10-09T05:39:11.754471+0000","last_clean":"2023-10-09T05:39:11.754471+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:39:11.754471+0000","last_undegraded":"2023-10-09T05:39:11.754471+0000","last_fullsized":"2023-10-09T05:39:11.754471+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":579,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.745452+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:39:11.745452+0000","last_peered":"2023-10-09T05:39:11.745452+0000","last_clean":"2023-10-09T05:39:11.745452+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:39:11.745452+0000","last_undegraded":"2023-10-09T05:39:11.745452+0000","last_fullsized":"2023-10-09T05:39:11.745452+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":182,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.754052+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:39:11.754052+0000","last_peered":"2023-10-09T05:39:11.754052+0000","last_clean":"2023-10-09T05:39:11.754052+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:39:11.754052+0000","last_undegraded":"2023-10-09T05:39:11.754052+0000","last_fullsized":"2023-10-09T05:39:11.754052+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"191'7253","reported_seq":7297,"reported_epoch":191,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:25.271607+0000","last_change":"2023-10-09T05:37:36.839957+0000","last_active":"2023-10-09T05:39:25.271607+0000","last_peered":"2023-10-09T05:39:25.271607+0000","last_clean":"2023-10-09T05:37:34.656587+0000","last_became_active":"2023-10-09T05:37:36.833510+0000","last_became_peered":"2023-10-09T05:37:36.833510+0000","last_unstale":"2023-10-09T05:39:25.271607+0000","last_undegraded":"2023-10-09T05:37:36.701190+0000","last_fullsized":"2023-10-09T05:37:36.681999+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7253,"log_dups_size":0,"ondisk_log_size":7253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231669760,"num_objects":3535,"num_object_clones":0,"num_object_copies":10605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2877,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7253,"num_write_kb":345216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":658},{"shards":"2,6,7","objects":1018}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":675,"reported_epoch":190,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:11.746650+0000","last_change":"2023-10-09T05:37:36.917512+0000","last_active":"2023-10-09T05:39:11.746650+0000","last_peered":"2023-10-09T05:39:11.746650+0000","last_clean":"2023-10-09T05:37:30.688967+0000","last_became_active":"2023-10-09T05:37:36.700218+0000","last_became_peered":"2023-10-09T05:37:36.700218+0000","last_unstale":"2023-10-09T05:39:11.746650+0000","last_undegraded":"2023-10-09T05:37:36.917335+0000","last_fullsized":"2023-10-09T05:37:36.682410+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":146,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.758390+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:39:11.758390+0000","last_peered":"2023-10-09T05:39:11.758390+0000","last_clean":"2023-10-09T05:39:11.758390+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:39:11.758390+0000","last_undegraded":"2023-10-09T05:39:11.758390+0000","last_fullsized":"2023-10-09T05:39:11.758390+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":142,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.759815+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:39:11.759815+0000","last_peered":"2023-10-09T05:39:11.759815+0000","last_clean":"2023-10-09T05:39:11.759815+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:39:11.759815+0000","last_undegraded":"2023-10-09T05:39:11.759815+0000","last_fullsized":"2023-10-09T05:39:11.759815+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":154,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.757875+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:39:11.757875+0000","last_peered":"2023-10-09T05:39:11.757875+0000","last_clean":"2023-10-09T05:39:11.757875+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:39:11.757875+0000","last_undegraded":"2023-10-09T05:39:11.757875+0000","last_fullsized":"2023-10-09T05:39:11.757875+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":182,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.745889+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:39:11.745889+0000","last_peered":"2023-10-09T05:39:11.745889+0000","last_clean":"2023-10-09T05:39:11.745889+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:39:11.745889+0000","last_undegraded":"2023-10-09T05:39:11.745889+0000","last_fullsized":"2023-10-09T05:39:11.745889+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":160,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.751907+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:39:11.751907+0000","last_peered":"2023-10-09T05:39:11.751907+0000","last_clean":"2023-10-09T05:39:11.751907+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:39:11.751907+0000","last_undegraded":"2023-10-09T05:39:11.751907+0000","last_fullsized":"2023-10-09T05:39:11.751907+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":172,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.760199+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:39:11.760199+0000","last_peered":"2023-10-09T05:39:11.760199+0000","last_clean":"2023-10-09T05:39:11.760199+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:39:11.760199+0000","last_undegraded":"2023-10-09T05:39:11.760199+0000","last_fullsized":"2023-10-09T05:39:11.760199+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":145,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.752296+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:39:11.752296+0000","last_peered":"2023-10-09T05:39:11.752296+0000","last_clean":"2023-10-09T05:39:11.752296+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:39:11.752296+0000","last_undegraded":"2023-10-09T05:39:11.752296+0000","last_fullsized":"2023-10-09T05:39:11.752296+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":172,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.764911+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:39:11.764911+0000","last_peered":"2023-10-09T05:39:11.764911+0000","last_clean":"2023-10-09T05:39:11.764911+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:39:11.764911+0000","last_undegraded":"2023-10-09T05:39:11.764911+0000","last_fullsized":"2023-10-09T05:39:11.764911+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":169,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.758775+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:39:11.758775+0000","last_peered":"2023-10-09T05:39:11.758775+0000","last_clean":"2023-10-09T05:39:11.758775+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:39:11.758775+0000","last_undegraded":"2023-10-09T05:39:11.758775+0000","last_fullsized":"2023-10-09T05:39:11.758775+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":182,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.754437+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:39:11.754437+0000","last_peered":"2023-10-09T05:39:11.754437+0000","last_clean":"2023-10-09T05:39:11.754437+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:39:11.754437+0000","last_undegraded":"2023-10-09T05:39:11.754437+0000","last_fullsized":"2023-10-09T05:39:11.754437+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":146,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.758734+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:39:11.758734+0000","last_peered":"2023-10-09T05:39:11.758734+0000","last_clean":"2023-10-09T05:39:11.758734+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:39:11.758734+0000","last_undegraded":"2023-10-09T05:39:11.758734+0000","last_fullsized":"2023-10-09T05:39:11.758734+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":157,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T05:39:11.759739+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:39:11.759739+0000","last_peered":"2023-10-09T05:39:11.759739+0000","last_clean":"2023-10-09T05:39:11.759739+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:39:11.759739+0000","last_undegraded":"2023-10-09T05:39:11.759739+0000","last_fullsized":"2023-10-09T05:39:11.759739+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3711434776,"num_objects":56633,"num_object_clones":0,"num_object_copies":169899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9381,"num_objects_misplaced":796,"num_objects_unfound":0,"num_objects_dirty":56633,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":120089,"num_write_kb":5655041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7298,"num_bytes_recovered":478281728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11866419200,"data_stored":11866406984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114689,"ondisk_log_size":114689,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846651,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626676,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2100964,"kb_used_data":1803740,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91644188,"statfs":{"total":95995035648,"available":93843648512,"internally_reserved":0,"allocated":1847029760,"data_stored":1846510167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:23 2023","interfaces":[{"interface":"back","average":{"1min":6.093,"5min":4.5960000000000001,"15min":2.1890000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":99.787999999999997,"5min":99.787999999999997,"15min":99.787999999999997},"last":115.377},{"interface":"front","average":{"1min":5.1890000000000001,"5min":4.234,"15min":2.0710000000000002},"min":{"1min":0.25600000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":99.844999999999999,"5min":99.844999999999999,"15min":99.844999999999999},"last":116.428}]},{"osd":1,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":2.677,"15min":1.3799999999999999},"min":{"1min":0.30099999999999999,"5min":0.25,"15min":0.25},"max":{"1min":9.8670000000000009,"5min":69.667000000000002,"15min":69.667000000000002},"last":115.416},{"interface":"front","average":{"1min":1.972,"5min":2.8849999999999998,"15min":1.4670000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":10.624000000000001,"5min":69.697999999999993,"15min":69.697999999999993},"last":115.45}]},{"osd":2,"last update":"Mon Oct 9 05:39:23 2023","interfaces":[{"interface":"back","average":{"1min":8.2159999999999993,"5min":4.593,"15min":2.5329999999999999},"min":{"1min":0.29499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":115.2,"5min":115.2,"15min":115.2},"last":115.40600000000001},{"interface":"front","average":{"1min":8.2409999999999997,"5min":4.4180000000000001,"15min":2.4950000000000001},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":115.297,"5min":115.297,"15min":115.297},"last":115.297}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":2.6160000000000001,"15min":1.661},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":29.885000000000002,"5min":68.920000000000002,"15min":68.920000000000002},"last":115.244},{"interface":"front","average":{"1min":3.0230000000000001,"5min":2.794,"15min":1.76},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":29.771000000000001,"5min":68.846000000000004,"15min":68.846000000000004},"last":115.34999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":3.1379999999999999,"15min":1.401},"min":{"1min":0.248,"5min":0.245,"15min":0.23400000000000001},"max":{"1min":0.58599999999999997,"5min":125.64,"15min":125.64},"last":115.214},{"interface":"front","average":{"1min":0.41499999999999998,"5min":2.7709999999999999,"15min":1.276},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":1.2709999999999999,"5min":125.625,"15min":125.625},"last":115.22799999999999}]},{"osd":5,"last update":"Mon Oct 9 05:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":2.871,"15min":1.3340000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.5510000000000002,"5min":127.35899999999999,"15min":127.35899999999999},"last":115.316},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.5110000000000001,"15min":1.23},"min":{"1min":0.29299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.8769999999999998,"5min":128.93799999999999,"15min":128.93799999999999},"last":115.30500000000001}]},{"osd":7,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":2.0800000000000001,"15min":2.7919999999999998},"min":{"1min":0.308,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.2789999999999999,"5min":12.074999999999999,"15min":12.074999999999999},"last":115.514},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.349,"15min":1.8440000000000001},"min":{"1min":0.33800000000000002,"5min":0.317,"15min":0.317},"max":{"1min":1.343,"5min":10.083,"15min":10.083},"last":116.53100000000001}]}]},{"osd":1,"up_from":93,"seq":399431958638,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1527552,"kb_used_data":1287800,"kb_used_omap":7,"kb_used_meta":239736,"kb_avail":92217600,"statfs":{"total":95995035648,"available":94430822400,"internally_reserved":0,"allocated":1318707200,"data_stored":1317988533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":245490190},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2120000000000002,"5min":5.5830000000000002,"15min":2.1520000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.185},"max":{"1min":9.9819999999999993,"5min":207.858,"15min":207.858},"last":3.8479999999999999},{"interface":"front","average":{"1min":6.5330000000000004,"5min":6.7000000000000002,"15min":2.5499999999999998},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":62.880000000000003,"5min":206.97200000000001,"15min":206.97200000000001},"last":4.5170000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":5.3289999999999997,"5min":7.3360000000000003,"15min":2.8220000000000001},"min":{"1min":0.22700000000000001,"5min":0.184,"15min":0.184},"max":{"1min":32.686999999999998,"5min":101.367,"15min":101.367},"last":4.6219999999999999},{"interface":"front","average":{"1min":4.468,"5min":6.7439999999999998,"15min":2.6259999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":19.975999999999999,"5min":98.409999999999997,"15min":98.409999999999997},"last":0.22800000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":4.234,"5min":3.0209999999999999,"15min":1.3640000000000001},"min":{"1min":0.24199999999999999,"5min":0.223,"15min":0.223},"max":{"1min":33.146000000000001,"5min":83.885000000000005,"15min":83.885000000000005},"last":0.40200000000000002},{"interface":"front","average":{"1min":8.5860000000000003,"5min":3.8889999999999998,"15min":1.619},"min":{"1min":0.26300000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":62.929000000000002,"5min":84.641999999999996,"15min":84.641999999999996},"last":0.65300000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":5.9870000000000001,"5min":4.1109999999999998,"15min":1.833},"min":{"1min":0.432,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":63.116,"5min":141.32900000000001,"15min":141.32900000000001},"last":4.4870000000000001},{"interface":"front","average":{"1min":1.675,"5min":2.8999999999999999,"15min":1.415},"min":{"1min":0.34499999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.5549999999999997,"5min":141.15899999999999,"15min":141.15899999999999},"last":0.42999999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":4.0350000000000001,"15min":1.7929999999999999},"min":{"1min":0.39000000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.0009999999999994,"5min":235.18700000000001,"15min":235.18700000000001},"last":0.443},{"interface":"front","average":{"1min":6.0140000000000002,"5min":4.9180000000000001,"15min":2.1819999999999999},"min":{"1min":0.46600000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":63.162999999999997,"5min":235.79300000000001,"15min":235.79300000000001},"last":4.4409999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":1.8220000000000001,"15min":1.075},"min":{"1min":0.42599999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.0289999999999999,"5min":10.333,"15min":10.333},"last":103.60899999999999},{"interface":"front","average":{"1min":6.2220000000000004,"5min":2.75,"15min":1.393},"min":{"1min":0.5,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":63.198,"5min":63.198,"15min":63.198},"last":103.679}]},{"osd":7,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":2.7949999999999999,"15min":2.2639999999999998},"min":{"1min":0.45700000000000002,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.6849999999999996,"5min":126.68300000000001,"15min":126.68300000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.6439999999999999,"5min":2.911,"15min":2.3559999999999999},"min":{"1min":0.48499999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.6180000000000003,"5min":126.738,"15min":126.738},"last":0.46600000000000003}]}]},{"osd":0,"up_from":88,"seq":377957122164,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1246876,"kb_used_data":912980,"kb_used_omap":8,"kb_used_meta":333879,"kb_avail":92498276,"statfs":{"total":95995035648,"available":94718234624,"internally_reserved":0,"allocated":934891520,"data_stored":934146448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":341892872},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":1.8959999999999999,"15min":0.93799999999999994},"min":{"1min":0.20499999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":14.964,"5min":14.964,"15min":14.964},"last":3.0379999999999998},{"interface":"front","average":{"1min":3.879,"5min":2.2850000000000001,"15min":1.0820000000000001},"min":{"1min":0.26100000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":11.315,"5min":11.315,"15min":11.315},"last":3.0659999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":3.4180000000000001,"5min":3.8109999999999999,"15min":1.8879999999999999},"min":{"1min":0.28999999999999998,"5min":0.223,"15min":0.214},"max":{"1min":12.759,"5min":79.911000000000001,"15min":79.911000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":4.085,"5min":3.5089999999999999,"15min":1.766},"min":{"1min":0.246,"5min":0.218,"15min":0.218},"max":{"1min":12.234999999999999,"5min":80.105000000000004,"15min":80.105000000000004},"last":32.344000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":6.5289999999999999,"5min":3.0329999999999999,"15min":1.3149999999999999},"min":{"1min":0.48799999999999999,"5min":0.23999999999999999,"15min":0.22800000000000001},"max":{"1min":31.594000000000001,"5min":31.594000000000001,"15min":31.594000000000001},"last":3.8639999999999999},{"interface":"front","average":{"1min":6.4569999999999999,"5min":3.1379999999999999,"15min":1.3879999999999999},"min":{"1min":0.38500000000000001,"5min":0.222,"15min":0.222},"max":{"1min":31.641999999999999,"5min":31.641999999999999,"15min":31.641999999999999},"last":2.093}]},{"osd":4,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":2.0539999999999998,"15min":1.163},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.1989999999999998,"5min":69.082999999999998,"15min":69.082999999999998},"last":2.669},{"interface":"front","average":{"1min":2.661,"5min":2.7010000000000001,"15min":1.4350000000000001},"min":{"1min":0.42099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.2149999999999999,"5min":69.102000000000004,"15min":69.102000000000004},"last":2.9350000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":8.0129999999999999,"5min":3.081,"15min":1.5509999999999999},"min":{"1min":0.45200000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":115.175,"5min":115.175,"15min":115.175},"last":2.9740000000000002},{"interface":"front","average":{"1min":7.0750000000000002,"5min":2.9710000000000001,"15min":1.5069999999999999},"min":{"1min":0.36499999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":115.502,"5min":115.502,"15min":115.502},"last":0.71399999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":1.9239999999999999,"15min":1.2050000000000001},"min":{"1min":0.41999999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.3409999999999993,"5min":9.3409999999999993,"15min":9.9809999999999999},"last":3.0219999999999998},{"interface":"front","average":{"1min":2.153,"5min":2.02,"15min":1.198},"min":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":7.266,"5min":8.0999999999999996,"15min":8.0999999999999996},"last":0.73999999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":1.409,"15min":1.355},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.5499999999999998,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":2.698},{"interface":"front","average":{"1min":2.649,"5min":2.206,"15min":2.3279999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":8.2490000000000006,"5min":11.068,"15min":11.068},"last":2.9990000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795111,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2406856,"kb_used_data":2017216,"kb_used_omap":11,"kb_used_meta":389620,"kb_avail":91338296,"statfs":{"total":95995035648,"available":93530415104,"internally_reserved":0,"allocated":2065629184,"data_stored":2064847424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":398971477},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":3.476,"5min":2.8010000000000002,"15min":1.2669999999999999},"min":{"1min":0.23400000000000001,"5min":0.183,"15min":0.17199999999999999},"max":{"1min":10.098000000000001,"5min":32.122,"15min":32.122},"last":2.4420000000000002},{"interface":"front","average":{"1min":3.27,"5min":2.3530000000000002,"15min":1.1399999999999999},"min":{"1min":0.25800000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.5850000000000009,"5min":34.066000000000003,"15min":34.066000000000003},"last":16.07}]},{"osd":1,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":14.41,"5min":5.3600000000000003,"15min":2.133},"min":{"1min":0.34499999999999997,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":98.575000000000003,"5min":98.575000000000003,"15min":98.575000000000003},"last":8.8499999999999996},{"interface":"front","average":{"1min":12.257,"5min":4.6779999999999999,"15min":1.867},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":90.135000000000005,"5min":98.397000000000006,"15min":98.397000000000006},"last":2.0920000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":3.5640000000000001,"5min":3.0049999999999999,"15min":1.2769999999999999},"min":{"1min":0.41699999999999998,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":12.385999999999999,"5min":45.057000000000002,"15min":45.057000000000002},"last":2.1299999999999999},{"interface":"front","average":{"1min":3.2970000000000002,"5min":3.3300000000000001,"15min":1.4410000000000001},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.942,"5min":45.813000000000002,"15min":45.813000000000002},"last":2.3999999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":2.8050000000000002,"15min":1.4019999999999999},"min":{"1min":0.29099999999999998,"5min":0.28999999999999998,"15min":0.27700000000000002},"max":{"1min":11.125999999999999,"5min":115.84999999999999,"15min":115.84999999999999},"last":2.194},{"interface":"front","average":{"1min":2.976,"5min":3.504,"15min":1.675},"min":{"1min":0.34799999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.147,"5min":116.149,"15min":116.149},"last":2.4729999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":2.5409999999999999,"15min":1.369},"min":{"1min":0.39900000000000002,"5min":0.312,"15min":0.312},"max":{"1min":7.8220000000000001,"5min":24.597999999999999,"15min":24.597999999999999},"last":8.7509999999999994},{"interface":"front","average":{"1min":1.867,"5min":2.089,"15min":1.1779999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.097,"5min":23.919,"15min":23.919},"last":2.1629999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":2.5129999999999999,"15min":1.327},"min":{"1min":0.39900000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.183,"5min":20.503,"15min":20.503},"last":2.5030000000000001},{"interface":"front","average":{"1min":2.8519999999999999,"5min":2.2170000000000001,"15min":1.2470000000000001},"min":{"1min":0.45000000000000001,"5min":0.372,"15min":0.34499999999999997},"max":{"1min":7.8730000000000002,"5min":9.4049999999999994,"15min":9.4049999999999994},"last":8.7949999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:38:49 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":2.883,"15min":2.6429999999999998},"min":{"1min":0.42599999999999999,"5min":0.374,"15min":0.374},"max":{"1min":8.9819999999999993,"5min":24.030999999999999,"15min":24.030999999999999},"last":4.4329999999999998},{"interface":"front","average":{"1min":3.3839999999999999,"5min":2.4950000000000001,"15min":1.984},"min":{"1min":0.47299999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.2569999999999997,"5min":24.178000000000001,"15min":24.178000000000001},"last":8.8230000000000004}]}]},{"osd":3,"up_from":103,"seq":442381631585,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2757884,"kb_used_data":2501492,"kb_used_omap":4,"kb_used_meta":256251,"kb_avail":90987268,"statfs":{"total":95995035648,"available":93170962432,"internally_reserved":0,"allocated":2561527808,"data_stored":2560812418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":262401630},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0840000000000001,"5min":2.6869999999999998,"15min":1.1970000000000001},"min":{"1min":0.26100000000000001,"5min":0.183,"15min":0.183},"max":{"1min":13.997,"5min":56.094000000000001,"15min":56.094000000000001},"last":9.1189999999999998},{"interface":"front","average":{"1min":3.7799999999999998,"5min":2.9700000000000002,"15min":1.296},"min":{"1min":0.221,"5min":0.221,"15min":0.214},"max":{"1min":15.593,"5min":57.255000000000003,"15min":57.255000000000003},"last":16.370000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0059999999999998,"5min":3.5619999999999998,"15min":1.4690000000000001},"min":{"1min":0.35399999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":13.932,"5min":83.921999999999997,"15min":83.921999999999997},"last":0.28199999999999997},{"interface":"front","average":{"1min":2.5019999999999998,"5min":3.4660000000000002,"15min":1.4119999999999999},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":7.1580000000000004,"5min":83.707999999999998,"15min":83.707999999999998},"last":4.6890000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":6.2699999999999996,"5min":4.5549999999999997,"15min":2.2410000000000001},"min":{"1min":0.25,"5min":0.215,"15min":0.215},"max":{"1min":60.722000000000001,"5min":95.320999999999998,"15min":117.23},"last":3.4969999999999999},{"interface":"front","average":{"1min":7.0670000000000002,"5min":4.6360000000000001,"15min":2.319},"min":{"1min":0.32200000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":60.469999999999999,"5min":94.170000000000002,"15min":117.45999999999999},"last":16.402999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":2.3530000000000002,"15min":1.329},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.1709999999999994,"5min":45.779000000000003,"15min":45.779000000000003},"last":15.590999999999999},{"interface":"front","average":{"1min":1.9239999999999999,"5min":2.2570000000000001,"15min":1.21},"min":{"1min":0.35499999999999998,"5min":0.313,"15min":0.313},"max":{"1min":13.734,"5min":45.735999999999997,"15min":45.735999999999997},"last":0.56200000000000006}]},{"osd":5,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":2.1400000000000001,"15min":1.165},"min":{"1min":0.433,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0800000000000001,"5min":47.948999999999998,"15min":47.948999999999998},"last":4.6219999999999999},{"interface":"front","average":{"1min":1.9259999999999999,"5min":2.3730000000000002,"15min":1.264},"min":{"1min":0.38300000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":13.864000000000001,"5min":47.968000000000004,"15min":47.968000000000004},"last":0.53300000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":2.0499999999999998,"15min":1.175},"min":{"1min":0.41199999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":13.831,"5min":13.831,"15min":13.831},"last":2.5139999999999998},{"interface":"front","average":{"1min":1.944,"5min":1.6850000000000001,"15min":0.98299999999999998},"min":{"1min":0.53000000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.117,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":4.6639999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:38:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2869999999999999,"5min":2.3300000000000001,"15min":2.2829999999999999},"min":{"1min":0.46899999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":9.1319999999999997,"5min":11.407,"15min":11.407},"last":15.632},{"interface":"front","average":{"1min":3.3050000000000002,"5min":2.2440000000000002,"15min":2.056},"min":{"1min":0.443,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.2949999999999999,"5min":11.432,"15min":11.432},"last":15.657}]}]},{"osd":4,"up_from":30,"seq":128849019133,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1426212,"kb_used_data":1124892,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92318940,"statfs":{"total":95995035648,"available":94534594560,"internally_reserved":0,"allocated":1151889408,"data_stored":1151369559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:20 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":2.1640000000000001,"15min":1.21},"min":{"1min":0.25600000000000001,"5min":0.214,"15min":0.214},"max":{"1min":7.4480000000000004,"5min":36.406999999999996,"15min":36.406999999999996},"last":0.26500000000000001},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.7810000000000001,"15min":1.454},"min":{"1min":0.28699999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.988,"5min":36.454999999999998,"15min":36.454999999999998},"last":7.7629999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":1.9430000000000001,"15min":1.0169999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.9550000000000001,"5min":31.073,"15min":31.073},"last":3.423},{"interface":"front","average":{"1min":1.421,"5min":1.968,"15min":1.0600000000000001},"min":{"1min":0.28699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.0109999999999992,"5min":31.100000000000001,"15min":31.100000000000001},"last":3.4060000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:39:15 2023","interfaces":[{"interface":"back","average":{"1min":3.4870000000000001,"5min":8.1349999999999998,"15min":3.2909999999999999},"min":{"1min":0.316,"5min":0.314,"15min":0.26800000000000002},"max":{"1min":20.684000000000001,"5min":196.93700000000001,"15min":196.93700000000001},"last":2.6850000000000001},{"interface":"front","average":{"1min":2.621,"5min":7.7030000000000003,"15min":3.2040000000000002},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":20.611000000000001,"5min":197.08099999999999,"15min":197.08099999999999},"last":3.73}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":23.963000000000001,"5min":6.3440000000000003,"15min":2.504},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":211.48699999999999,"5min":211.48699999999999,"15min":211.48699999999999},"last":2.5680000000000001},{"interface":"front","average":{"1min":23.547000000000001,"5min":6.0309999999999997,"15min":2.4510000000000001},"min":{"1min":0.38600000000000001,"5min":0.311,"15min":0.309},"max":{"1min":211.31100000000001,"5min":211.31100000000001,"15min":211.31100000000001},"last":9.7100000000000009}]},{"osd":5,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":3.6040000000000001,"15min":1.5289999999999999},"min":{"1min":0.25,"5min":0.249,"15min":0.249},"max":{"1min":2.6070000000000002,"5min":113.35899999999999,"15min":113.35899999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":3.3849999999999998,"15min":1.5109999999999999},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.0550000000000002,"5min":113.32299999999999,"15min":113.32299999999999},"last":0.443}]},{"osd":6,"last update":"Mon Oct 9 05:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.0569999999999999,"15min":0.69799999999999995},"min":{"1min":0.28999999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.0289999999999999,"5min":11.944000000000001,"15min":11.944000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.3169999999999999,"15min":0.80300000000000005},"min":{"1min":0.313,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.1899999999999999,"5min":10.356,"15min":10.356},"last":0.45500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":2.0230000000000001,"15min":2.9209999999999998},"min":{"1min":0.38700000000000001,"5min":0.312,"15min":0.312},"max":{"1min":2.0750000000000002,"5min":30.393999999999998,"15min":30.393999999999998},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.876,"5min":2.1989999999999998,"15min":3.286},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.5859999999999999,"5min":30.334,"15min":30.334},"last":0.42999999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822905,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1438104,"kb_used_data":1128592,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92307048,"statfs":{"total":95995035648,"available":94522417152,"internally_reserved":0,"allocated":1155678208,"data_stored":1155166119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":6.3940000000000001,"15min":2.8650000000000002},"min":{"1min":0.249,"5min":0.224,"15min":0.19400000000000001},"max":{"1min":10.712999999999999,"5min":165.90199999999999,"15min":165.90199999999999},"last":2.7000000000000002},{"interface":"front","average":{"1min":2.1419999999999999,"5min":6.3799999999999999,"15min":2.9359999999999999},"min":{"1min":0.28999999999999998,"5min":0.215,"15min":0.215},"max":{"1min":8.6099999999999994,"5min":165.96700000000001,"15min":165.96700000000001},"last":2.4540000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":7.4249999999999998,"5min":3.0960000000000001,"15min":1.4159999999999999},"min":{"1min":0.30599999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":87.769000000000005,"5min":87.769000000000005,"15min":87.769000000000005},"last":1.4530000000000001},{"interface":"front","average":{"1min":6.9340000000000002,"5min":2.782,"15min":1.337},"min":{"1min":0.35599999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":87.864000000000004,"5min":87.864000000000004,"15min":87.864000000000004},"last":6.6109999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:39:20 2023","interfaces":[{"interface":"back","average":{"1min":5.9400000000000004,"5min":9.1470000000000002,"15min":3.5609999999999999},"min":{"1min":0.33800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":42.055,"5min":290.697,"15min":290.697},"last":2.6429999999999998},{"interface":"front","average":{"1min":4.2720000000000002,"5min":8.7880000000000003,"15min":3.4079999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":41.953000000000003,"5min":290.786,"15min":290.786},"last":1.8380000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":5.8250000000000002,"5min":2.7799999999999998,"15min":1.3500000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":52.560000000000002,"5min":52.560000000000002,"15min":52.560000000000002},"last":1.5149999999999999},{"interface":"front","average":{"1min":4.8470000000000004,"5min":2.5710000000000002,"15min":1.268},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":52.380000000000003,"5min":52.380000000000003,"15min":52.380000000000003},"last":2.6749999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":1.0940000000000001,"15min":0.68600000000000005},"min":{"1min":0.25700000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.2949999999999999,"5min":9.3510000000000009,"15min":9.3510000000000009},"last":0.316},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.746,"15min":0.93200000000000005},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":2.669,"5min":14.236000000000001,"15min":14.236000000000001},"last":0.40799999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":2.5600000000000001,"15min":1.234},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":1.8069999999999999,"5min":95.010999999999996,"15min":95.010999999999996},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":2.2490000000000001,"15min":1.123},"min":{"1min":0.33500000000000002,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":2.4390000000000001,"5min":94.194000000000003,"15min":94.194000000000003},"last":0.76700000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.5149999999999999,"15min":1.655},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.4289999999999998,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.51600000000000001,"5min":1.6599999999999999,"15min":1.968},"min":{"1min":0.36199999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.7250000000000001,"5min":14.432,"15min":14.432},"last":0.49199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":926552064,"data_stored":926547992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1312296960,"data_stored":1312292888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2046492672,"data_stored":2046492672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2542796800,"data_stored":2542796800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1138229248,"data_stored":1138229248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1142099968,"data_stored":1142095896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1838809088,"data_stored":1838809088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":919142400,"data_stored":919142400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:39:34.398 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:34.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:34.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:34.505 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:34.590 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:34.749 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:34.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:34.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:34.903 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:35.051 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:39:35.066 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:35.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:35.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:35.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:35.216 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:35.297 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:39:35.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:35.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:35.404 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:35.495 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:35.499 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:35.537 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:35 smithi191 ceph-mon[161669]: pgmap v526: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 9414/164343 objects degraded (5.728%); 550/164343 objects misplaced (0.335%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T05:39:35.653 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:35.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:35.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:35.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:35.799 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:35.912 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:35 smithi008 ceph-mon[184418]: pgmap v526: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 9414/164343 objects degraded (5.728%); 550/164343 objects misplaced (0.335%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T05:39:35.913 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:35 smithi008 ceph-mon[191227]: pgmap v526: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 9414/164343 objects degraded (5.728%); 550/164343 objects misplaced (0.335%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T05:39:36.071 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:36.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:36.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:36.175 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:36.205 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:36.249 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:39:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:36.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:36.289 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:39:36.331 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:39:36.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:36.382 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:36.385 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:39:36.392 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:39:36.459 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:39:36.492 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:39:36.531 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:39:36.574 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:39:36.612 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:39:36.617 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:36.619 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:39:36.619 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:39:36.619 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:39:36.619 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:39:36.619 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:39:36.619 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:39:36.619 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:39:36.620 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:39:36.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:36.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:36.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3876343121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:36.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3876343121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:36.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:36.753 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3876343121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:36.755 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:36.863 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:39:36.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:36.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:36.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:37.006 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:37.080 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:39:37.081 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:39:37.081 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:39:37.081 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:39:37.081 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:39:37.094 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:37.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:37.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:37.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:37.227 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:37.270 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:37.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:37.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:37.342 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:39:37.373 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:37.488 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:37.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:37 smithi008 ceph-mon[191227]: pgmap v527: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 9414/160686 objects degraded (5.859%); 550/160686 objects misplaced (0.342%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T05:39:37.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:37 smithi008 ceph-mon[184418]: pgmap v527: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 9414/160686 objects degraded (5.859%); 550/160686 objects misplaced (0.342%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T05:39:37.817 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:37.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:37.826 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:37 smithi191 ceph-mon[161669]: pgmap v527: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 9414/160686 objects degraded (5.859%); 550/160686 objects misplaced (0.342%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T05:39:37.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:39:37.927 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:37.957 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:38.026 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:39:38.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:38.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:38.082 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:38.132 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:38.207 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:38.232 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:38.272 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:38.427 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:38.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:38.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:38.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:38.574 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:38.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9414/160686 objects degraded (5.859%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:38.836 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:39:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:38.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:38.946 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9414/160686 objects degraded (5.859%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9414/160686 objects degraded (5.859%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:38.991 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:39.839 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:39:39.883 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:39:39.883 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:39:39.883 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:39:39.883 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:39:39.883 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138523 5.4 5.84db7384 (undecoded) ondisk+retry+write+known_if_redirected e127)", 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.658381+0000", 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.22267757999998, 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99745561100000002, 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138523 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.884 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658381+0000", 2023-10-09T05:39:39.885 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.885 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.885 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658381+0000", 2023-10-09T05:39:39.885 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:39.885 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.885 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658380+0000", 2023-10-09T05:39:39.886 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7150000000000001e-06 2023-10-09T05:39:39.886 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.886 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.886 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.886 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658386+0000", 2023-10-09T05:39:39.886 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2750000000000001e-06 2023-10-09T05:39:39.886 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.886 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.886 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.886 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658396+0000", 2023-10-09T05:39:39.886 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7730000000000002e-06 2023-10-09T05:39:39.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.887 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.887 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658402+0000", 2023-10-09T05:39:39.887 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00063723800000000004 2023-10-09T05:39:39.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.887 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.887 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659040+0000", 2023-10-09T05:39:39.887 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2666999999999999e-05 2023-10-09T05:39:39.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.888 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.888 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.888 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659052+0000", 2023-10-09T05:39:39.888 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99677223800000003 2023-10-09T05:39:39.888 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.888 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.888 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.888 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655825+0000", 2023-10-09T05:39:39.888 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2245999999999999e-05 2023-10-09T05:39:39.890 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.890 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.890 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.890 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655837+0000", 2023-10-09T05:39:39.890 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99745561100000002 2023-10-09T05:39:39.890 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.890 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.890 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.890 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.890 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.890 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138528 5.4 5.bf234c24 (undecoded) ondisk+retry+write+known_if_redirected e127)", 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.658412+0000", 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.222646924, 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.997458705, 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138528 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.891 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658412+0000", 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658412+0000", 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658411+0000", 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6459999999999999e-06 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658415+0000", 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.49e-06 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658420+0000", 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6770000000000003e-06 2023-10-09T05:39:39.893 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.894 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.894 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.894 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658425+0000", 2023-10-09T05:39:39.894 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000652908 2023-10-09T05:39:39.894 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.894 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.894 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.894 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659078+0000", 2023-10-09T05:39:39.894 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7360000000000001e-06 2023-10-09T05:39:39.894 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.894 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659086+0000", 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99677464199999999 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655860+0000", 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0182e-05 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.895 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655871+0000", 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.997458705 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138537 5.14 5.de8f0eb4 (undecoded) ondisk+retry+write+known_if_redirected e127)", 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.658438+0000", 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.22262050199998, 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99594088300000005, 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:39:39.896 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.897 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.897 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.897 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138537 2023-10-09T05:39:39.897 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.897 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.897 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.897 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.897 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658438+0000", 2023-10-09T05:39:39.897 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658438+0000", 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658438+0000", 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0450000000000001e-06 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658441+0000", 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2435e-05 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658453+0000", 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7509999999999999e-06 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658458+0000", 2023-10-09T05:39:39.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00090573200000000002 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659364+0000", 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1409999999999999e-06 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659371+0000", 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99499399300000002 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.900 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.901 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.901 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654365+0000", 2023-10-09T05:39:39.901 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4296e-05 2023-10-09T05:39:39.901 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.901 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.901 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.901 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654379+0000", 2023-10-09T05:39:39.902 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99594088300000005 2023-10-09T05:39:39.902 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138539 5.14 5.df7ef574 (undecoded) ondisk+retry+write+known_if_redirected e127)", 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.658463+0000", 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.22259551899998, 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99595421100000003, 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.903 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138539 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658463+0000", 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.904 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658463+0000", 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658463+0000", 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3050000000000001e-06 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658466+0000", 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9939999999999997e-06 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.905 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.906 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.906 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658472+0000", 2023-10-09T05:39:39.906 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3320000000000002e-06 2023-10-09T05:39:39.906 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.906 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.906 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.906 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658476+0000", 2023-10-09T05:39:39.906 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001204175 2023-10-09T05:39:39.906 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.906 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.906 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659681+0000", 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6250000000000002e-06 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659690+0000", 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99471561100000006 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654406+0000", 2023-10-09T05:39:39.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1659e-05 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654418+0000", 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99595421100000003 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138541 5.9 5.c283ca09 (undecoded) ondisk+write+known_if_redirected e127)", 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.658481+0000", 2023-10-09T05:39:39.908 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.22257829300003, 2023-10-09T05:39:39.909 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0067079050000001, 2023-10-09T05:39:39.909 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.909 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:39:39.909 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.909 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.909 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.909 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138541 2023-10-09T05:39:39.909 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.910 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.910 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.910 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.910 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658481+0000", 2023-10-09T05:39:39.910 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.910 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.910 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.910 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.910 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658481+0000", 2023-10-09T05:39:39.910 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:39.911 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.911 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658480+0000", 2023-10-09T05:39:39.911 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3970000000000001e-06 2023-10-09T05:39:39.911 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.911 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658483+0000", 2023-10-09T05:39:39.911 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2420000000000002e-06 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658487+0000", 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4499999999999998e-06 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658489+0000", 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4831000000000001e-05 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.912 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658584+0000", 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7370000000000001e-06 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658594+0000", 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.006582222 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.665176+0000", 2023-10-09T05:39:39.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2413e-05 2023-10-09T05:39:39.914 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.914 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.914 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.914 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.665188+0000", 2023-10-09T05:39:39.914 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0067079050000001 2023-10-09T05:39:39.914 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.914 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.914 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.914 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.914 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.914 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138551 5.19 5.f6e6d179 (undecoded) ondisk+write+known_if_redirected e127)", 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.681781+0000", 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.19927746899998, 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97309186599999997, 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138551 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.915 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.916 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681781+0000", 2023-10-09T05:39:39.916 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.916 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.916 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.916 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.916 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681781+0000", 2023-10-09T05:39:39.916 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:39.916 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.916 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.916 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.916 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681780+0000", 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0450000000000001e-06 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681788+0000", 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0400999999999999e-05 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681798+0000", 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.439e-06 2023-10-09T05:39:39.917 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681807+0000", 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002494975 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684302+0000", 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0268999999999999e-05 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684312+0000", 2023-10-09T05:39:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97055105699999999 2023-10-09T05:39:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654863+0000", 2023-10-09T05:39:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0227999999999999e-05 2023-10-09T05:39:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654873+0000", 2023-10-09T05:39:39.920 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97309186599999997 2023-10-09T05:39:39.920 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.920 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.920 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.920 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.920 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.920 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138552 5.4 5.ef384104 (undecoded) ondisk+write+known_if_redirected e127)", 2023-10-09T05:39:39.920 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.684234+0000", 2023-10-09T05:39:39.920 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.19682435099998, 2023-10-09T05:39:39.920 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97166571400000001, 2023-10-09T05:39:39.920 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.921 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:39:39.921 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.921 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.921 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.921 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138552 2023-10-09T05:39:39.921 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.921 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.921 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.921 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.921 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684234+0000", 2023-10-09T05:39:39.922 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.922 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.922 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.922 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.922 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684234+0000", 2023-10-09T05:39:39.922 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:39:39.922 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.922 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.922 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.922 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684232+0000", 2023-10-09T05:39:39.923 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4124999999999999e-05 2023-10-09T05:39:39.923 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.923 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.923 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.923 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684247+0000", 2023-10-09T05:39:39.923 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3252999999999999e-05 2023-10-09T05:39:39.923 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.923 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.923 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.923 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684270+0000", 2023-10-09T05:39:39.923 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4730000000000006e-06 2023-10-09T05:39:39.924 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.924 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.924 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.924 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684279+0000", 2023-10-09T05:39:39.924 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028417399999999999 2023-10-09T05:39:39.924 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.924 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.924 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.924 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684563+0000", 2023-10-09T05:39:39.924 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.842e-05 2023-10-09T05:39:39.924 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684582+0000", 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97130895799999994 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655891+0000", 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3510000000000008e-06 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.925 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.926 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655900+0000", 2023-10-09T05:39:39.926 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97166571400000001 2023-10-09T05:39:39.926 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.926 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.926 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.926 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.926 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.926 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138555 5.14 5.8803b654 (undecoded) ondisk+write+known_if_redirected e127)", 2023-10-09T05:39:39.926 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.684293+0000", 2023-10-09T05:39:39.926 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.19676572200001, 2023-10-09T05:39:39.926 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97015670499999995, 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138555 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684293+0000", 2023-10-09T05:39:39.927 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.928 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.928 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.928 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.928 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684293+0000", 2023-10-09T05:39:39.928 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:39.928 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.928 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.928 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.928 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684292+0000", 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9009999999999996e-06 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684296+0000", 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6510000000000004e-06 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684303+0000", 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4569999999999998e-06 2023-10-09T05:39:39.929 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684307+0000", 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00047098700000000002 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684778+0000", 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3480000000000004e-06 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.930 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.931 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684787+0000", 2023-10-09T05:39:39.931 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96965241499999999 2023-10-09T05:39:39.931 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.931 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.931 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.931 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654439+0000", 2023-10-09T05:39:39.931 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0606e-05 2023-10-09T05:39:39.931 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.931 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.931 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654450+0000", 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97015670499999995 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138562 5.9 5.6835f569 (undecoded) ondisk+write+known_if_redirected e127)", 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.699110+0000", 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 299.18194846199998, 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96611003200000001, 2023-10-09T05:39:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138562 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699110+0000", 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.933 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.934 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.934 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.934 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699110+0000", 2023-10-09T05:39:39.934 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:39.934 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.934 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.934 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.934 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699109+0000", 2023-10-09T05:39:39.934 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1216999999999999e-05 2023-10-09T05:39:39.934 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.934 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699120+0000", 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7193e-05 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699137+0000", 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5379999999999999e-06 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699144+0000", 2023-10-09T05:39:39.935 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4489e-05 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699198+0000", 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9580000000000002e-06 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699208+0000", 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96600283600000003 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.936 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.665211+0000", 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2779999999999999e-06 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.665220+0000", 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96611003200000001 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.937 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:218946 5.19 5:9ed084b7:::benchmark_data_smithi008_238708_object100810:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e145)", 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.643711+0000", 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 209.23734761700001, 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.028762001, 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 218946 2023-10-09T05:39:39.938 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643711+0000", 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643711+0000", 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.939 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.940 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.940 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643709+0000", 2023-10-09T05:39:39.940 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5990000000000001e-05 2023-10-09T05:39:39.940 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.940 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.940 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.940 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643725+0000", 2023-10-09T05:39:39.940 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7798000000000001e-05 2023-10-09T05:39:39.940 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.940 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.940 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643743+0000", 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3549999999999999e-06 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643751+0000", 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4769000000000002e-05 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643825+0000", 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.361e-05 2023-10-09T05:39:39.941 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.942 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.942 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.942 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643839+0000", 2023-10-09T05:39:39.942 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.011016055 2023-10-09T05:39:39.942 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.942 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.942 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.942 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.654855+0000", 2023-10-09T05:39:39.942 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7491999999999994e-05 2023-10-09T05:39:39.942 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.942 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.654952+0000", 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017458551999999999 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672411+0000", 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2207999999999998e-05 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.943 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672473+0000", 2023-10-09T05:39:39.944 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.028762001 2023-10-09T05:39:39.944 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.944 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.944 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.944 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.944 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.944 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:218960 5.19 5:9c42dd17:::benchmark_data_smithi008_238708_object100824:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:39:39.944 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.653577+0000", 2023-10-09T05:39:39.944 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 209.22748149700001, 2023-10-09T05:39:39.944 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.034507125, 2023-10-09T05:39:39.944 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.945 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:39.945 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.945 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.945 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.945 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 218960 2023-10-09T05:39:39.945 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.945 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.945 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.945 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.945 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653577+0000", 2023-10-09T05:39:39.945 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.946 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.946 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.946 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.946 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653577+0000", 2023-10-09T05:39:39.946 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999919 2023-10-09T05:39:39.946 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.946 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.946 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.946 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653569+0000", 2023-10-09T05:39:39.946 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2618e-05 2023-10-09T05:39:39.946 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653582+0000", 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9889999999999998e-06 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653591+0000", 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7910000000000004e-06 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.947 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653596+0000", 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001950198 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.655546+0000", 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0429e-05 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.655556+0000", 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99943933200000001 2023-10-09T05:39:39.948 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.654996+0000", 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6644999999999994e-05 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.655092+0000", 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00037147599999999998 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:39.949 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.655464+0000", 2023-10-09T05:39:39.950 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0089923880000000005 2023-10-09T05:39:39.950 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.950 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.950 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:39.950 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.664456+0000", 2023-10-09T05:39:39.950 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023390363000000001 2023-10-09T05:39:39.950 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.950 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.950 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.950 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.687846+0000", 2023-10-09T05:39:39.950 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5685000000000005e-05 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.687922+0000", 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2720999999999998e-05 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.687985+0000", 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.959e-05 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688084+0000", 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.034507125 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:218961 5.19 5:9c1d5529:::benchmark_data_smithi008_238708_object100825:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.653606+0000", 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 209.227452881, 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0346523700000001, 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.952 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 218961 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653606+0000", 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653606+0000", 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653606+0000", 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.34e-06 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.954 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653608+0000", 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8399999999999995e-06 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653616+0000", 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9369999999999997e-06 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653620+0000", 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001972414 2023-10-09T05:39:39.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.655592+0000", 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.131e-06 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.655599+0000", 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.001532678 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.956 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.657132+0000", 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6515999999999997e-05 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.657208+0000", 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021616300000000001 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.657425+0000", 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010284753000000001 2023-10-09T05:39:39.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.667709+0000", 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020432143999999999 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688141+0000", 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4103000000000002e-05 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688176+0000", 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9569000000000001e-05 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688205+0000", 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3216e-05 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688258+0000", 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0346523700000001 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.959 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219000 5.19 5:9a536baf:::benchmark_data_smithi008_238708_object100864:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.708353+0000", 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 209.17270606400001, 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98004795200000006, 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.960 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219000 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708353+0000", 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708353+0000", 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:39:39.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708351+0000", 2023-10-09T05:39:39.962 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3520000000000007e-06 2023-10-09T05:39:39.962 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708359+0000", 2023-10-09T05:39:39.962 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2427e-05 2023-10-09T05:39:39.962 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708372+0000", 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3820000000000001e-06 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708378+0000", 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002761337 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.711139+0000", 2023-10-09T05:39:39.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0630000000000006e-06 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.711148+0000", 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94768007300000001 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.658828+0000", 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0051999999999998e-05 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.964 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.658918+0000", 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000231487 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.659149+0000", 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0085686330000000008 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.667718+0000", 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020580158000000001 2023-10-09T05:39:39.965 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688298+0000", 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1848000000000002e-05 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688320+0000", 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7568000000000001e-05 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:39.966 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688347+0000", 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3371000000000001e-05 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688401+0000", 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98004795200000006 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.967 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219006 5.19 5:9d7b33f6:::benchmark_data_smithi008_238708_object100870:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.718358+0000", 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 209.162700626, 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97025771199999999, 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219006 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.968 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718358+0000", 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718358+0000", 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718357+0000", 2023-10-09T05:39:39.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.238e-06 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718364+0000", 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5883999999999999e-05 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718380+0000", 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3749999999999997e-06 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718387+0000", 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5766999999999996e-05 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718453+0000", 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1304999999999999e-05 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718464+0000", 2023-10-09T05:39:39.971 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94278578400000002 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.661250+0000", 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119416 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.661369+0000", 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00037868800000000002 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.972 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.661748+0000", 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059772230000000003 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.667725+0000", 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020709274999999999 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688435+0000", 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7390999999999999e-05 2023-10-09T05:39:39.973 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.974 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.974 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.974 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688462+0000", 2023-10-09T05:39:39.974 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0025999999999998e-05 2023-10-09T05:39:39.974 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.974 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.974 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:39.974 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688512+0000", 2023-10-09T05:39:39.974 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103828 2023-10-09T05:39:39.974 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.974 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688616+0000", 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97025771199999999 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219040 5.19 5:9927ccc2:::benchmark_data_smithi008_238708_object100904:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.768643+0000", 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 209.11241625700001, 2023-10-09T05:39:39.975 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92022723900000003, 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219040 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768643+0000", 2023-10-09T05:39:39.976 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768643+0000", 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768641+0000", 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9090000000000006e-06 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.977 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768650+0000", 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5481999999999998e-05 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768665+0000", 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2269999999999998e-06 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768671+0000", 2023-10-09T05:39:39.978 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6886999999999999e-05 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768728+0000", 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0456e-05 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768739+0000", 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89575502600000001 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.664494+0000", 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9017000000000004e-05 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.664593+0000", 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00022533399999999999 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.664818+0000", 2023-10-09T05:39:39.980 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075507980000000001 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672369+0000", 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016304721000000001 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688673+0000", 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0516000000000001e-05 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.981 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688734+0000", 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4342000000000003e-05 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688778+0000", 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1475000000000001e-05 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688870+0000", 2023-10-09T05:39:39.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92022723900000003 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219067 5.19 5:9a00ab2f:::benchmark_data_smithi008_238708_object100931:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.806455+0000", 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 209.074604287, 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88268894200000003, 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:39.983 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219067 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806455+0000", 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.984 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806455+0000", 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806453+0000", 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7700000000000004e-06 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806460+0000", 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2864000000000001e-05 2023-10-09T05:39:39.985 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806473+0000", 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0069999999999997e-06 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806478+0000", 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1210999999999994e-05 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806539+0000", 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2235000000000001e-05 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806551+0000", 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85952352399999998 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.666075+0000", 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4686999999999998e-05 2023-10-09T05:39:39.987 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.666149+0000", 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021614200000000001 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.666365+0000", 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087013460000000004 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:39.988 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675067+0000", 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013862637000000001 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688929+0000", 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6715999999999999e-05 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688966+0000", 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9138000000000002e-05 2023-10-09T05:39:39.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689025+0000", 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118199 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689143+0000", 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88268894200000003 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.990 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219068 5.19 5:9c85096d:::benchmark_data_smithi008_238708_object100932:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.806490+0000", 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 209.07456929, 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88291725200000004, 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219068 2023-10-09T05:39:39.991 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806490+0000", 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806490+0000", 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.992 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806489+0000", 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2910000000000002e-06 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806491+0000", 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6740000000000002e-06 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806497+0000", 2023-10-09T05:39:39.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5790000000000002e-06 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806500+0000", 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00047362400000000002 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806973+0000", 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0248000000000001e-05 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.994 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806984+0000", 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86079349100000002 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.667777+0000", 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012534800000000001 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.667902+0000", 2023-10-09T05:39:39.995 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000286147 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.668189+0000", 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068859680000000001 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675075+0000", 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014158188 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.996 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689233+0000", 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6288e-05 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689289+0000", 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5240000000000001e-05 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689334+0000", 2023-10-09T05:39:39.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2529999999999998e-05 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689407+0000", 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88291725200000004 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219080 5.19 5:9b22456a:::benchmark_data_smithi008_238708_object100944:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:39:39.998 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.825998+0000", 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 209.055060895, 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86367064199999999, 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219080 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:39.999 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.825998+0000", 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.825998+0000", 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.825996+0000", 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6950000000000006e-06 2023-10-09T05:39:40.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.826005+0000", 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6087999999999999e-05 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.826021+0000", 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0870000000000001e-06 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:40.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.826028+0000", 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7341999999999993e-05 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.826096+0000", 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0266e-05 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.826106+0000", 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.84409459399999998 2023-10-09T05:39:40.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.670201+0000", 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4676e-05 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.670285+0000", 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000226407 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:40.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.670512+0000", 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045699679999999998 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675082+0000", 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01438443 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689466+0000", 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4876000000000003e-05 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689501+0000", 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5906999999999999e-05 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689557+0000", 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011178799999999999 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:40.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689669+0000", 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86367064199999999 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219108 5.19 5:9a26fec5:::benchmark_data_smithi008_238708_object100972:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.859141+0000", 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 209.021917828, 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83075073300000002, 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:40.006 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219108 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859141+0000", 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.007 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859141+0000", 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859140+0000", 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8099999999999997e-06 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859144+0000", 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0951000000000001e-05 2023-10-09T05:39:40.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859155+0000", 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.048e-06 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859159+0000", 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8704e-05 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859208+0000", 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6360000000000006e-06 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859218+0000", 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81329287100000003 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672511+0000", 2023-10-09T05:39:40.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000123986 2023-10-09T05:39:40.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:40.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672635+0000", 2023-10-09T05:39:40.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000329876 2023-10-09T05:39:40.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:40.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672964+0000", 2023-10-09T05:39:40.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010650517999999999 2023-10-09T05:39:40.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.683615+0000", 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060958770000000004 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689711+0000", 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1442999999999997e-05 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:40.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689742+0000", 2023-10-09T05:39:40.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6600000000000001e-05 2023-10-09T05:39:40.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:40.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689789+0000", 2023-10-09T05:39:40.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102797 2023-10-09T05:39:40.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:40.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689892+0000", 2023-10-09T05:39:40.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83075073300000002 2023-10-09T05:39:40.014 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:40.014 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:40.014 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:40.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.014 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219141 5.19 5:9c1c6174:::benchmark_data_smithi008_238708_object101005:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:39:40.014 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.913601+0000", 2023-10-09T05:39:40.014 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 208.96745811900001, 2023-10-09T05:39:40.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.77651063099999995, 2023-10-09T05:39:40.014 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:39:40.014 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219141 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913601+0000", 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913601+0000", 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913599+0000", 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7490000000000001e-06 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:39:40.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913606+0000", 2023-10-09T05:39:40.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2513e-05 2023-10-09T05:39:40.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:39:40.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913618+0000", 2023-10-09T05:39:40.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1380000000000001e-06 2023-10-09T05:39:40.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:39:40.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913626+0000", 2023-10-09T05:39:40.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8097999999999998e-05 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913694+0000", 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3261000000000001e-05 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913707+0000", 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.76142268800000001 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675130+0000", 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000116388 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675246+0000", 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000282212 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:39:40.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675528+0000", 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080939329999999993 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.683622+0000", 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063094969999999998 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689932+0000", 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1736000000000002e-05 2023-10-09T05:39:40.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689963+0000", 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4965000000000001e-05 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.690008+0000", 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102929 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:39:40.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:39:40.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.690111+0000", 2023-10-09T05:39:40.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.77651063099999995 2023-10-09T05:39:40.022 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:40.022 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:40.022 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:40.022 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:39:40.022 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:39:40.022 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:39:40.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:39 smithi191 ceph-mon[161669]: pgmap v528: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 9414/157851 objects degraded (5.964%); 550/157851 objects misplaced (0.348%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T05:39:40.034 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:39 smithi008 ceph-mon[184418]: pgmap v528: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 9414/157851 objects degraded (5.964%); 550/157851 objects misplaced (0.348%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T05:39:40.035 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:39 smithi008 ceph-mon[191227]: pgmap v528: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 9414/157851 objects degraded (5.964%); 550/157851 objects misplaced (0.348%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T05:39:40.420 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:40.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:40.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:40.528 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:40.626 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:40.631 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:39:40.718 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:39:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:39:40] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.33.4" 2023-10-09T05:39:40.916 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:40.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:40.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:41.034 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:41.128 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:41.473 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:41.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:41.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:41.541 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:41.585 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:41.628 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:41.690 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:41.774 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:41 smithi008 ceph-mon[184418]: pgmap v529: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 9414/152841 objects degraded (6.159%); 315/152841 objects misplaced (0.206%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:39:41.779 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:41 smithi008 ceph-mon[191227]: pgmap v529: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 9414/152841 objects degraded (6.159%); 315/152841 objects misplaced (0.206%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:39:41.998 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:39:42.009 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:42.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:42.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:42.027 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:39:42.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:41 smithi191 ceph-mon[161669]: pgmap v529: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 9414/152841 objects degraded (6.159%); 315/152841 objects misplaced (0.206%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:39:42.122 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:42.227 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:42.449 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:42.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:42.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:42.557 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:42.648 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:42.847 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:39:42.924 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/500087765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:42.924 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:42 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:42.924 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:42 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9414/152841 objects degraded (6.159%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:42.924 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/500087765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:42.924 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:42 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:42.925 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:42 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9414/152841 objects degraded (6.159%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:43.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/500087765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:43.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:42 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:39:43.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:42 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9414/152841 objects degraded (6.159%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:43.775 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:43.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:43.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:43.883 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:43.958 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:43 smithi191 ceph-mon[161669]: from='client.45194 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:43 smithi191 ceph-mon[161669]: pgmap v530: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 9414/149127 objects degraded (6.313%); 315/149127 objects misplaced (0.211%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:39:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:43 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:39:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:43 smithi191 ceph-mon[161669]: failed to encode map e193 with expected crc 2023-10-09T05:39:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:43 smithi191 ceph-mon[161669]: failed to encode map e193 with expected crc 2023-10-09T05:39:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:43 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:44.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:43 smithi191 ceph-mon[161669]: osdmap e193: 8 total, 8 up, 7 in 2023-10-09T05:39:44.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:43 smithi191 ceph-mon[161669]: failed to encode map e193 with expected crc 2023-10-09T05:39:44.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[191227]: from='client.45194 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[191227]: pgmap v530: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 9414/149127 objects degraded (6.313%); 315/149127 objects misplaced (0.211%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:39:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:39:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[191227]: failed to encode map e193 with expected crc 2023-10-09T05:39:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[191227]: failed to encode map e193 with expected crc 2023-10-09T05:39:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[191227]: osdmap e193: 8 total, 8 up, 7 in 2023-10-09T05:39:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[191227]: failed to encode map e193 with expected crc 2023-10-09T05:39:44.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[184418]: from='client.45194 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:39:44.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[184418]: pgmap v530: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 9414/149127 objects degraded (6.313%); 315/149127 objects misplaced (0.211%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:39:44.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:39:44.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[184418]: failed to encode map e193 with expected crc 2023-10-09T05:39:44.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[184418]: failed to encode map e193 with expected crc 2023-10-09T05:39:44.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:39:44.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[184418]: osdmap e193: 8 total, 8 up, 7 in 2023-10-09T05:39:44.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:43 smithi008 ceph-mon[184418]: failed to encode map e193 with expected crc 2023-10-09T05:39:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:44 smithi008 ceph-mon[184418]: pgmap v532: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 9414/146979 objects degraded (6.405%); 73/146979 objects misplaced (0.050%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:39:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:45.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:44 smithi008 ceph-mon[191227]: pgmap v532: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 9414/146979 objects degraded (6.405%); 73/146979 objects misplaced (0.050%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:39:45.298 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":529,"stamp":"2023-10-09T05:39:40.081114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3341752267,"num_objects":50947,"num_object_clones":0,"num_object_copies":152841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9414,"num_objects_misplaced":315,"num_objects_unfound":0,"num_objects_dirty":50947,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":126909,"num_write_kb":5670182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7812,"num_bytes_recovered":510399862,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119073,"ondisk_log_size":119073,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11914364,"kb_used_data":9769860,"kb_used_omap":31,"kb_used_meta":2143904,"kb_avail":644301700,"statfs":{"total":671965249536,"available":659764940800,"internally_reserved":0,"allocated":10004336640,"data_stored":9999819587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2195358153},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-374472704,"num_objects":-5714,"num_object_clones":0,"num_object_copies":-17142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-481,"num_objects_unfound":0,"num_objects_dirty":-5714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5714,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":481,"num_bytes_recovered":31522816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005442"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":175,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.783179+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:39:20.783179+0000","last_peered":"2023-10-09T05:39:20.783179+0000","last_clean":"2023-10-09T05:39:20.783179+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:39:20.783179+0000","last_undegraded":"2023-10-09T05:39:20.783179+0000","last_fullsized":"2023-10-09T05:39:20.783179+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":164,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.773781+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:39:20.773781+0000","last_peered":"2023-10-09T05:39:20.773781+0000","last_clean":"2023-10-09T05:39:20.773781+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:39:20.773781+0000","last_undegraded":"2023-10-09T05:39:20.773781+0000","last_fullsized":"2023-10-09T05:39:20.773781+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":138,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.784542+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:39:20.784542+0000","last_peered":"2023-10-09T05:39:20.784542+0000","last_clean":"2023-10-09T05:39:20.784542+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:39:20.784542+0000","last_undegraded":"2023-10-09T05:39:20.784542+0000","last_fullsized":"2023-10-09T05:39:20.784542+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":168,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.765185+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:39:20.765185+0000","last_peered":"2023-10-09T05:39:20.765185+0000","last_clean":"2023-10-09T05:39:20.765185+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:39:20.765185+0000","last_undegraded":"2023-10-09T05:39:20.765185+0000","last_fullsized":"2023-10-09T05:39:20.765185+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":180,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.776663+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:39:20.776663+0000","last_peered":"2023-10-09T05:39:20.776663+0000","last_clean":"2023-10-09T05:39:20.776663+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:39:20.776663+0000","last_undegraded":"2023-10-09T05:39:20.776663+0000","last_fullsized":"2023-10-09T05:39:20.776663+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":158,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.781049+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:39:20.781049+0000","last_peered":"2023-10-09T05:39:20.781049+0000","last_clean":"2023-10-09T05:39:20.781049+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:39:20.781049+0000","last_undegraded":"2023-10-09T05:39:20.781049+0000","last_fullsized":"2023-10-09T05:39:20.781049+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":144,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:32.568836+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:39:32.568836+0000","last_peered":"2023-10-09T05:39:32.568836+0000","last_clean":"2023-10-09T05:39:32.568836+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:39:32.568836+0000","last_undegraded":"2023-10-09T05:39:32.568836+0000","last_fullsized":"2023-10-09T05:39:32.568836+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":164,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.776730+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:39:20.776730+0000","last_peered":"2023-10-09T05:39:20.776730+0000","last_clean":"2023-10-09T05:39:20.776730+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:39:20.776730+0000","last_undegraded":"2023-10-09T05:39:20.776730+0000","last_fullsized":"2023-10-09T05:39:20.776730+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":183,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.786376+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:39:20.786376+0000","last_peered":"2023-10-09T05:39:20.786376+0000","last_clean":"2023-10-09T05:39:20.786376+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:39:20.786376+0000","last_undegraded":"2023-10-09T05:39:20.786376+0000","last_fullsized":"2023-10-09T05:39:20.786376+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":178,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.773627+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:39:20.773627+0000","last_peered":"2023-10-09T05:39:20.773627+0000","last_clean":"2023-10-09T05:39:20.773627+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:39:20.773627+0000","last_undegraded":"2023-10-09T05:39:20.773627+0000","last_fullsized":"2023-10-09T05:39:20.773627+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":71,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.773700+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:39:20.773700+0000","last_peered":"2023-10-09T05:39:20.773700+0000","last_clean":"2023-10-09T05:39:20.773700+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:39:20.773700+0000","last_undegraded":"2023-10-09T05:39:20.773700+0000","last_fullsized":"2023-10-09T05:39:20.773700+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":138,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.784696+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:39:20.784696+0000","last_peered":"2023-10-09T05:39:20.784696+0000","last_clean":"2023-10-09T05:39:20.784696+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:39:20.784696+0000","last_undegraded":"2023-10-09T05:39:20.784696+0000","last_fullsized":"2023-10-09T05:39:20.784696+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":200,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.783816+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:39:20.783816+0000","last_peered":"2023-10-09T05:39:20.783816+0000","last_clean":"2023-10-09T05:39:20.783816+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:39:20.783816+0000","last_undegraded":"2023-10-09T05:39:20.783816+0000","last_fullsized":"2023-10-09T05:39:20.783816+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":158,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.783355+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:39:20.783355+0000","last_peered":"2023-10-09T05:39:20.783355+0000","last_clean":"2023-10-09T05:39:20.783355+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:39:20.783355+0000","last_undegraded":"2023-10-09T05:39:20.783355+0000","last_fullsized":"2023-10-09T05:39:20.783355+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":71,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.783427+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:39:20.783427+0000","last_peered":"2023-10-09T05:39:20.783427+0000","last_clean":"2023-10-09T05:39:20.783427+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:39:20.783427+0000","last_undegraded":"2023-10-09T05:39:20.783427+0000","last_fullsized":"2023-10-09T05:39:20.783427+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":188,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.778026+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:39:20.778026+0000","last_peered":"2023-10-09T05:39:20.778026+0000","last_clean":"2023-10-09T05:39:20.778026+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:39:20.778026+0000","last_undegraded":"2023-10-09T05:39:20.778026+0000","last_fullsized":"2023-10-09T05:39:20.778026+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":168,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.768203+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:39:20.768203+0000","last_peered":"2023-10-09T05:39:20.768203+0000","last_clean":"2023-10-09T05:39:20.768203+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:39:20.768203+0000","last_undegraded":"2023-10-09T05:39:20.768203+0000","last_fullsized":"2023-10-09T05:39:20.768203+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":182,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.789941+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:39:20.789941+0000","last_peered":"2023-10-09T05:39:20.789941+0000","last_clean":"2023-10-09T05:39:20.789941+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:39:20.789941+0000","last_undegraded":"2023-10-09T05:39:20.789941+0000","last_fullsized":"2023-10-09T05:39:20.789941+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":89,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.783077+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:39:20.783077+0000","last_peered":"2023-10-09T05:39:20.783077+0000","last_clean":"2023-10-09T05:39:20.783077+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:39:20.783077+0000","last_undegraded":"2023-10-09T05:39:20.783077+0000","last_fullsized":"2023-10-09T05:39:20.783077+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":168,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:32.568403+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:39:32.568403+0000","last_peered":"2023-10-09T05:39:32.568403+0000","last_clean":"2023-10-09T05:39:32.568403+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:39:32.568403+0000","last_undegraded":"2023-10-09T05:39:32.568403+0000","last_fullsized":"2023-10-09T05:39:32.568403+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":167,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.780448+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:39:20.780448+0000","last_peered":"2023-10-09T05:39:20.780448+0000","last_clean":"2023-10-09T05:39:20.780448+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:39:20.780448+0000","last_undegraded":"2023-10-09T05:39:20.780448+0000","last_fullsized":"2023-10-09T05:39:20.780448+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":188,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.774104+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:39:20.774104+0000","last_peered":"2023-10-09T05:39:20.774104+0000","last_clean":"2023-10-09T05:39:20.774104+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:39:20.774104+0000","last_undegraded":"2023-10-09T05:39:20.774104+0000","last_fullsized":"2023-10-09T05:39:20.774104+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"192'7914","reported_seq":7952,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:39.915655+0000","last_change":"2023-10-09T05:37:01.636984+0000","last_active":"2023-10-09T05:39:39.915655+0000","last_peered":"2023-10-09T05:39:39.915655+0000","last_clean":"2023-10-09T05:39:39.915655+0000","last_became_active":"2023-10-09T05:37:01.636609+0000","last_became_peered":"2023-10-09T05:37:01.636609+0000","last_unstale":"2023-10-09T05:39:39.915655+0000","last_undegraded":"2023-10-09T05:39:39.915655+0000","last_fullsized":"2023-10-09T05:39:39.915655+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7914,"log_dups_size":0,"ondisk_log_size":7914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205914112,"num_objects":3142,"num_object_clones":0,"num_object_copies":9426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7914,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":69,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.787205+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:39:20.787205+0000","last_peered":"2023-10-09T05:39:20.787205+0000","last_clean":"2023-10-09T05:39:20.787205+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:39:20.787205+0000","last_undegraded":"2023-10-09T05:39:20.787205+0000","last_fullsized":"2023-10-09T05:39:20.787205+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":167,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.786019+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:39:20.786019+0000","last_peered":"2023-10-09T05:39:20.786019+0000","last_clean":"2023-10-09T05:39:20.786019+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:39:20.786019+0000","last_undegraded":"2023-10-09T05:39:20.786019+0000","last_fullsized":"2023-10-09T05:39:20.786019+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"192'7750","reported_seq":7798,"reported_epoch":192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:36.126815+0000","last_change":"2023-10-09T05:37:36.837344+0000","last_active":"2023-10-09T05:39:36.126815+0000","last_peered":"2023-10-09T05:39:36.126815+0000","last_clean":"2023-10-09T05:37:34.651883+0000","last_became_active":"2023-10-09T05:37:36.832502+0000","last_became_peered":"2023-10-09T05:37:36.832502+0000","last_unstale":"2023-10-09T05:39:36.126815+0000","last_undegraded":"2023-10-09T05:37:36.702797+0000","last_fullsized":"2023-10-09T05:37:36.678583+0000","mapping_epoch":170,"log_start":"166'1300","ondisk_log_start":"166'1300","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6450,"log_dups_size":0,"ondisk_log_size":6450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:57.419460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212467712,"num_objects":3242,"num_object_clones":0,"num_object_copies":9726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7750,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":352},{"shards":"1,3,7","objects":636}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":69,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.784008+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:39:20.784008+0000","last_peered":"2023-10-09T05:39:20.784008+0000","last_clean":"2023-10-09T05:39:20.784008+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:39:20.784008+0000","last_undegraded":"2023-10-09T05:39:20.784008+0000","last_fullsized":"2023-10-09T05:39:20.784008+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":173,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.782985+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:39:20.782985+0000","last_peered":"2023-10-09T05:39:20.782985+0000","last_clean":"2023-10-09T05:39:20.782985+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:39:20.782985+0000","last_undegraded":"2023-10-09T05:39:20.782985+0000","last_fullsized":"2023-10-09T05:39:20.782985+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"192'8002","reported_seq":14101,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:36.627942+0000","last_change":"2023-10-09T05:38:55.317808+0000","last_active":"2023-10-09T05:39:36.627942+0000","last_peered":"2023-10-09T05:39:36.627942+0000","last_clean":"2023-10-09T05:39:36.627942+0000","last_became_active":"2023-10-09T05:38:55.302739+0000","last_became_peered":"2023-10-09T05:38:55.302739+0000","last_unstale":"2023-10-09T05:39:36.627942+0000","last_undegraded":"2023-10-09T05:39:36.627942+0000","last_fullsized":"2023-10-09T05:39:36.627942+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8002,"log_dups_size":0,"ondisk_log_size":8002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:25.379818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210370560,"num_objects":3210,"num_object_clones":0,"num_object_copies":9630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8002,"num_write_kb":358784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":198049792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":69,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.790020+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:39:20.790020+0000","last_peered":"2023-10-09T05:39:20.790020+0000","last_clean":"2023-10-09T05:39:20.790020+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:39:20.790020+0000","last_undegraded":"2023-10-09T05:39:20.790020+0000","last_fullsized":"2023-10-09T05:39:20.790020+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":191,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.780777+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:39:20.780777+0000","last_peered":"2023-10-09T05:39:20.780777+0000","last_clean":"2023-10-09T05:39:20.780777+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:39:20.780777+0000","last_undegraded":"2023-10-09T05:39:20.780777+0000","last_fullsized":"2023-10-09T05:39:20.780777+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"192'7836","reported_seq":11526,"reported_epoch":192,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:38.110025+0000","last_change":"2023-10-09T05:38:53.467497+0000","last_active":"2023-10-09T05:39:38.110025+0000","last_peered":"2023-10-09T05:39:38.110025+0000","last_clean":"2023-10-09T05:37:34.677467+0000","last_became_active":"2023-10-09T05:37:36.803958+0000","last_became_peered":"2023-10-09T05:37:36.803958+0000","last_unstale":"2023-10-09T05:39:38.110025+0000","last_undegraded":"2023-10-09T05:37:36.800519+0000","last_fullsized":"2023-10-09T05:37:36.678690+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7836,"log_dups_size":0,"ondisk_log_size":7836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:40.690185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205389824,"num_objects":3134,"num_object_clones":0,"num_object_copies":9402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":764,"num_objects_misplaced":315,"num_objects_unfound":0,"num_objects_dirty":3134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7836,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1820,"num_bytes_recovered":119275520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":2055},{"shards":"0,2,7","objects":315}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":159,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.786488+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:39:20.786488+0000","last_peered":"2023-10-09T05:39:20.786488+0000","last_clean":"2023-10-09T05:39:20.786488+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:39:20.786488+0000","last_undegraded":"2023-10-09T05:39:20.786488+0000","last_fullsized":"2023-10-09T05:39:20.786488+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":144,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.764837+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:39:20.764837+0000","last_peered":"2023-10-09T05:39:20.764837+0000","last_clean":"2023-10-09T05:39:20.764837+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:39:20.764837+0000","last_undegraded":"2023-10-09T05:39:20.764837+0000","last_fullsized":"2023-10-09T05:39:20.764837+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"192'7903","reported_seq":7941,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:39.933618+0000","last_change":"2023-10-09T05:37:01.439878+0000","last_active":"2023-10-09T05:39:39.933618+0000","last_peered":"2023-10-09T05:39:39.933618+0000","last_clean":"2023-10-09T05:39:39.933618+0000","last_became_active":"2023-10-09T05:37:01.439481+0000","last_became_peered":"2023-10-09T05:37:01.439481+0000","last_unstale":"2023-10-09T05:39:39.933618+0000","last_undegraded":"2023-10-09T05:39:39.933618+0000","last_fullsized":"2023-10-09T05:39:39.933618+0000","mapping_epoch":162,"log_start":"166'400","ondisk_log_start":"166'400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7503,"log_dups_size":0,"ondisk_log_size":7503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202047488,"num_objects":3083,"num_object_clones":0,"num_object_copies":9249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7903,"num_write_kb":351552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":162,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.779740+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:39:20.779740+0000","last_peered":"2023-10-09T05:39:20.779740+0000","last_clean":"2023-10-09T05:39:20.779740+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:39:20.779740+0000","last_undegraded":"2023-10-09T05:39:20.779740+0000","last_fullsized":"2023-10-09T05:39:20.779740+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":158,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.783721+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:39:20.783721+0000","last_peered":"2023-10-09T05:39:20.783721+0000","last_clean":"2023-10-09T05:39:20.783721+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:39:20.783721+0000","last_undegraded":"2023-10-09T05:39:20.783721+0000","last_fullsized":"2023-10-09T05:39:20.783721+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"192'7803","reported_seq":7843,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:36.572693+0000","last_change":"2023-10-09T05:37:01.440175+0000","last_active":"2023-10-09T05:39:36.572693+0000","last_peered":"2023-10-09T05:39:36.572693+0000","last_clean":"2023-10-09T05:39:36.572693+0000","last_became_active":"2023-10-09T05:37:01.439846+0000","last_became_peered":"2023-10-09T05:37:01.439846+0000","last_unstale":"2023-10-09T05:39:36.572693+0000","last_undegraded":"2023-10-09T05:39:36.572693+0000","last_fullsized":"2023-10-09T05:39:36.572693+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7803,"log_dups_size":0,"ondisk_log_size":7803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:27.883265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213057536,"num_objects":3251,"num_object_clones":0,"num_object_copies":9753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7803,"num_write_kb":353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":159,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.780637+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:39:20.780637+0000","last_peered":"2023-10-09T05:39:20.780637+0000","last_clean":"2023-10-09T05:39:20.780637+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:39:20.780637+0000","last_undegraded":"2023-10-09T05:39:20.780637+0000","last_fullsized":"2023-10-09T05:39:20.780637+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":173,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.783552+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:39:20.783552+0000","last_peered":"2023-10-09T05:39:20.783552+0000","last_clean":"2023-10-09T05:39:20.783552+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:39:20.783552+0000","last_undegraded":"2023-10-09T05:39:20.783552+0000","last_fullsized":"2023-10-09T05:39:20.783552+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"192'7792","reported_seq":7832,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:36.097581+0000","last_change":"2023-10-09T05:37:01.638178+0000","last_active":"2023-10-09T05:39:36.097581+0000","last_peered":"2023-10-09T05:39:36.097581+0000","last_clean":"2023-10-09T05:39:36.097581+0000","last_became_active":"2023-10-09T05:37:01.637822+0000","last_became_peered":"2023-10-09T05:37:01.637822+0000","last_unstale":"2023-10-09T05:39:36.097581+0000","last_undegraded":"2023-10-09T05:39:36.097581+0000","last_fullsized":"2023-10-09T05:39:36.097581+0000","mapping_epoch":162,"log_start":"166'1400","ondisk_log_start":"166'1400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6392,"log_dups_size":0,"ondisk_log_size":6392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:13.240069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217710592,"num_objects":3322,"num_object_clones":0,"num_object_copies":9966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7792,"num_write_kb":355648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":183,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.782779+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:39:20.782779+0000","last_peered":"2023-10-09T05:39:20.782779+0000","last_clean":"2023-10-09T05:39:20.782779+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:39:20.782779+0000","last_undegraded":"2023-10-09T05:39:20.782779+0000","last_fullsized":"2023-10-09T05:39:20.782779+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":167,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.784009+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:39:20.784009+0000","last_peered":"2023-10-09T05:39:20.784009+0000","last_clean":"2023-10-09T05:39:20.784009+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:39:20.784009+0000","last_undegraded":"2023-10-09T05:39:20.784009+0000","last_fullsized":"2023-10-09T05:39:20.784009+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"192'7989","reported_seq":8028,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:39.930075+0000","last_change":"2023-10-09T05:37:01.438917+0000","last_active":"2023-10-09T05:39:39.930075+0000","last_peered":"2023-10-09T05:39:39.930075+0000","last_clean":"2023-10-09T05:39:39.930075+0000","last_became_active":"2023-10-09T05:37:01.438658+0000","last_became_peered":"2023-10-09T05:37:01.438658+0000","last_unstale":"2023-10-09T05:39:39.930075+0000","last_undegraded":"2023-10-09T05:39:39.930075+0000","last_fullsized":"2023-10-09T05:39:39.930075+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7989,"log_dups_size":0,"ondisk_log_size":7989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201326616,"num_objects":3073,"num_object_clones":0,"num_object_copies":9219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3073,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7989,"num_write_kb":353921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":189,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.776587+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:39:20.776587+0000","last_peered":"2023-10-09T05:39:20.776587+0000","last_clean":"2023-10-09T05:39:20.776587+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:39:20.776587+0000","last_undegraded":"2023-10-09T05:39:20.776587+0000","last_fullsized":"2023-10-09T05:39:20.776587+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":139,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:28.793198+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:39:28.793198+0000","last_peered":"2023-10-09T05:39:28.793198+0000","last_clean":"2023-10-09T05:39:28.793198+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:39:28.793198+0000","last_undegraded":"2023-10-09T05:39:28.793198+0000","last_fullsized":"2023-10-09T05:39:28.793198+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"192'7907","reported_seq":7947,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:37.989352+0000","last_change":"2023-10-09T05:37:01.440093+0000","last_active":"2023-10-09T05:39:37.989352+0000","last_peered":"2023-10-09T05:39:37.989352+0000","last_clean":"2023-10-09T05:39:37.989352+0000","last_became_active":"2023-10-09T05:37:01.439799+0000","last_became_peered":"2023-10-09T05:37:01.439799+0000","last_unstale":"2023-10-09T05:39:37.989352+0000","last_undegraded":"2023-10-09T05:39:37.989352+0000","last_fullsized":"2023-10-09T05:39:37.989352+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:28.734471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209649664,"num_objects":3199,"num_object_clones":0,"num_object_copies":9597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7907,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":726,"reported_epoch":191,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:20.775458+0000","last_change":"2023-10-09T05:37:36.736822+0000","last_active":"2023-10-09T05:39:20.775458+0000","last_peered":"2023-10-09T05:39:20.775458+0000","last_clean":"2023-10-09T05:37:30.792074+0000","last_became_active":"2023-10-09T05:37:36.736273+0000","last_became_peered":"2023-10-09T05:37:36.736273+0000","last_unstale":"2023-10-09T05:39:20.775458+0000","last_undegraded":"2023-10-09T05:37:36.682657+0000","last_fullsized":"2023-10-09T05:37:36.682189+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:23.350354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":174,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.789871+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:39:20.789871+0000","last_peered":"2023-10-09T05:39:20.789871+0000","last_clean":"2023-10-09T05:39:20.789871+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:39:20.789871+0000","last_undegraded":"2023-10-09T05:39:20.789871+0000","last_fullsized":"2023-10-09T05:39:20.789871+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":124,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.784864+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:39:20.784864+0000","last_peered":"2023-10-09T05:39:20.784864+0000","last_clean":"2023-10-09T05:39:20.784864+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:39:20.784864+0000","last_undegraded":"2023-10-09T05:39:20.784864+0000","last_fullsized":"2023-10-09T05:39:20.784864+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"192'7857","reported_seq":13787,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:36.084639+0000","last_change":"2023-10-09T05:38:10.132330+0000","last_active":"2023-10-09T05:39:36.084639+0000","last_peered":"2023-10-09T05:39:36.084639+0000","last_clean":"2023-10-09T05:39:36.084639+0000","last_became_active":"2023-10-09T05:38:10.127080+0000","last_became_peered":"2023-10-09T05:38:10.127080+0000","last_unstale":"2023-10-09T05:39:36.084639+0000","last_undegraded":"2023-10-09T05:39:36.084639+0000","last_fullsized":"2023-10-09T05:39:36.084639+0000","mapping_epoch":179,"log_start":"166'1400","ondisk_log_start":"166'1400","created":162,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6457,"log_dups_size":0,"ondisk_log_size":6457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:12.659227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215678976,"num_objects":3291,"num_object_clones":0,"num_object_copies":9873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7857,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":192413696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":628,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.786271+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:39:20.786271+0000","last_peered":"2023-10-09T05:39:20.786271+0000","last_clean":"2023-10-09T05:39:20.786271+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:39:20.786271+0000","last_undegraded":"2023-10-09T05:39:20.786271+0000","last_fullsized":"2023-10-09T05:39:20.786271+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":159,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.783443+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:39:20.783443+0000","last_peered":"2023-10-09T05:39:20.783443+0000","last_clean":"2023-10-09T05:39:20.783443+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:39:20.783443+0000","last_undegraded":"2023-10-09T05:39:20.783443+0000","last_fullsized":"2023-10-09T05:39:20.783443+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":171,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.786560+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:39:20.786560+0000","last_peered":"2023-10-09T05:39:20.786560+0000","last_clean":"2023-10-09T05:39:20.786560+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:39:20.786560+0000","last_undegraded":"2023-10-09T05:39:20.786560+0000","last_fullsized":"2023-10-09T05:39:20.786560+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"192'7714","reported_seq":7763,"reported_epoch":192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:35.718838+0000","last_change":"2023-10-09T05:37:40.649912+0000","last_active":"2023-10-09T05:39:35.718838+0000","last_peered":"2023-10-09T05:39:35.718838+0000","last_clean":"2023-10-09T05:37:34.677100+0000","last_became_active":"2023-10-09T05:37:40.593033+0000","last_became_peered":"2023-10-09T05:37:40.593033+0000","last_unstale":"2023-10-09T05:39:35.718838+0000","last_undegraded":"2023-10-09T05:37:40.546647+0000","last_fullsized":"2023-10-09T05:37:40.543498+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7714,"log_dups_size":0,"ondisk_log_size":7714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214695936,"num_objects":3276,"num_object_clones":0,"num_object_copies":9828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7714,"num_write_kb":351680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":426},{"shards":"3,6,7","objects":631}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":506,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:21.640831+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:39:21.640831+0000","last_peered":"2023-10-09T05:39:21.640831+0000","last_clean":"2023-10-09T05:39:21.640831+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:39:21.640831+0000","last_undegraded":"2023-10-09T05:39:21.640831+0000","last_fullsized":"2023-10-09T05:39:21.640831+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":139,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.784110+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:39:20.784110+0000","last_peered":"2023-10-09T05:39:20.784110+0000","last_clean":"2023-10-09T05:39:20.784110+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:39:20.784110+0000","last_undegraded":"2023-10-09T05:39:20.784110+0000","last_fullsized":"2023-10-09T05:39:20.784110+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":169,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.787042+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:39:20.787042+0000","last_peered":"2023-10-09T05:39:20.787042+0000","last_clean":"2023-10-09T05:39:20.787042+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:39:20.787042+0000","last_undegraded":"2023-10-09T05:39:20.787042+0000","last_fullsized":"2023-10-09T05:39:20.787042+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"192'7901","reported_seq":7939,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:39.883441+0000","last_change":"2023-10-09T05:37:01.438236+0000","last_active":"2023-10-09T05:39:39.883441+0000","last_peered":"2023-10-09T05:39:39.883441+0000","last_clean":"2023-10-09T05:39:39.883441+0000","last_became_active":"2023-10-09T05:37:01.437917+0000","last_became_peered":"2023-10-09T05:37:01.437917+0000","last_unstale":"2023-10-09T05:39:39.883441+0000","last_undegraded":"2023-10-09T05:39:39.883441+0000","last_fullsized":"2023-10-09T05:39:39.883441+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7901,"log_dups_size":0,"ondisk_log_size":7901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207945728,"num_objects":3173,"num_object_clones":0,"num_object_copies":9519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7901,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":776,"reported_epoch":191,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:20.786337+0000","last_change":"2023-10-09T05:37:36.709131+0000","last_active":"2023-10-09T05:39:20.786337+0000","last_peered":"2023-10-09T05:39:20.786337+0000","last_clean":"2023-10-09T05:37:30.688708+0000","last_became_active":"2023-10-09T05:37:36.708682+0000","last_became_peered":"2023-10-09T05:37:36.708682+0000","last_unstale":"2023-10-09T05:39:20.786337+0000","last_undegraded":"2023-10-09T05:37:36.683823+0000","last_fullsized":"2023-10-09T05:37:36.683109+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":340,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:32.566459+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:39:32.566459+0000","last_peered":"2023-10-09T05:39:32.566459+0000","last_clean":"2023-10-09T05:39:32.566459+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:39:32.566459+0000","last_undegraded":"2023-10-09T05:39:32.566459+0000","last_fullsized":"2023-10-09T05:39:32.566459+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":155,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.776539+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:39:20.776539+0000","last_peered":"2023-10-09T05:39:20.776539+0000","last_clean":"2023-10-09T05:39:20.776539+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:39:20.776539+0000","last_undegraded":"2023-10-09T05:39:20.776539+0000","last_fullsized":"2023-10-09T05:39:20.776539+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":165,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.781049+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:39:20.781049+0000","last_peered":"2023-10-09T05:39:20.781049+0000","last_clean":"2023-10-09T05:39:20.781049+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:39:20.781049+0000","last_undegraded":"2023-10-09T05:39:20.781049+0000","last_fullsized":"2023-10-09T05:39:20.781049+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"192'7839","reported_seq":7879,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:38.728731+0000","last_change":"2023-10-09T05:37:01.440172+0000","last_active":"2023-10-09T05:39:38.728731+0000","last_peered":"2023-10-09T05:39:38.728731+0000","last_clean":"2023-10-09T05:39:38.728731+0000","last_became_active":"2023-10-09T05:37:01.439932+0000","last_became_peered":"2023-10-09T05:37:01.439932+0000","last_unstale":"2023-10-09T05:39:38.728731+0000","last_undegraded":"2023-10-09T05:39:38.728731+0000","last_fullsized":"2023-10-09T05:39:38.728731+0000","mapping_epoch":162,"log_start":"166'1400","ondisk_log_start":"166'1400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6439,"log_dups_size":0,"ondisk_log_size":6439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:57.826704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204537856,"num_objects":3121,"num_object_clones":0,"num_object_copies":9363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7839,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":718,"reported_epoch":192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:32.546466+0000","last_change":"2023-10-09T05:37:36.748736+0000","last_active":"2023-10-09T05:39:32.546466+0000","last_peered":"2023-10-09T05:39:32.546466+0000","last_clean":"2023-10-09T05:22:39.601058+0000","last_became_active":"2023-10-09T05:37:36.748304+0000","last_became_peered":"2023-10-09T05:37:36.748304+0000","last_unstale":"2023-10-09T05:39:32.546466+0000","last_undegraded":"2023-10-09T05:37:36.696424+0000","last_fullsized":"2023-10-09T05:37:36.695987+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:18.784578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":144,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.770110+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:39:20.770110+0000","last_peered":"2023-10-09T05:39:20.770110+0000","last_clean":"2023-10-09T05:39:20.770110+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:39:20.770110+0000","last_undegraded":"2023-10-09T05:39:20.770110+0000","last_fullsized":"2023-10-09T05:39:20.770110+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":159,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.765267+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:39:20.765267+0000","last_peered":"2023-10-09T05:39:20.765267+0000","last_clean":"2023-10-09T05:39:20.765267+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:39:20.765267+0000","last_undegraded":"2023-10-09T05:39:20.765267+0000","last_fullsized":"2023-10-09T05:39:20.765267+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"192'7918","reported_seq":7958,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:38.756357+0000","last_change":"2023-10-09T05:37:01.636967+0000","last_active":"2023-10-09T05:39:38.756357+0000","last_peered":"2023-10-09T05:39:38.756357+0000","last_clean":"2023-10-09T05:39:38.756357+0000","last_became_active":"2023-10-09T05:37:01.636602+0000","last_became_peered":"2023-10-09T05:37:01.636602+0000","last_unstale":"2023-10-09T05:39:38.756357+0000","last_undegraded":"2023-10-09T05:39:38.756357+0000","last_fullsized":"2023-10-09T05:39:38.756357+0000","mapping_epoch":162,"log_start":"167'1500","ondisk_log_start":"167'1500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6418,"log_dups_size":0,"ondisk_log_size":6418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:11.668143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208011264,"num_objects":3174,"num_object_clones":0,"num_object_copies":9522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7918,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":380,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.768582+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:39:20.768582+0000","last_peered":"2023-10-09T05:39:20.768582+0000","last_clean":"2023-10-09T05:39:20.768582+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:39:20.768582+0000","last_undegraded":"2023-10-09T05:39:20.768582+0000","last_fullsized":"2023-10-09T05:39:20.768582+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":139,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.785584+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:39:20.785584+0000","last_peered":"2023-10-09T05:39:20.785584+0000","last_clean":"2023-10-09T05:39:20.785584+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:39:20.785584+0000","last_undegraded":"2023-10-09T05:39:20.785584+0000","last_fullsized":"2023-10-09T05:39:20.785584+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":153,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.765481+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:39:20.765481+0000","last_peered":"2023-10-09T05:39:20.765481+0000","last_clean":"2023-10-09T05:39:20.765481+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:39:20.765481+0000","last_undegraded":"2023-10-09T05:39:20.765481+0000","last_fullsized":"2023-10-09T05:39:20.765481+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"192'8061","reported_seq":8099,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:39.925703+0000","last_change":"2023-10-09T05:37:01.440074+0000","last_active":"2023-10-09T05:39:39.925703+0000","last_peered":"2023-10-09T05:39:39.925703+0000","last_clean":"2023-10-09T05:39:39.925703+0000","last_became_active":"2023-10-09T05:37:01.439808+0000","last_became_peered":"2023-10-09T05:37:01.439808+0000","last_unstale":"2023-10-09T05:39:39.925703+0000","last_undegraded":"2023-10-09T05:39:39.925703+0000","last_fullsized":"2023-10-09T05:39:39.925703+0000","mapping_epoch":162,"log_start":"166'500","ondisk_log_start":"166'500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7561,"log_dups_size":0,"ondisk_log_size":7561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200343552,"num_objects":3057,"num_object_clones":0,"num_object_copies":9171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8061,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":190,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.774116+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:39:20.774116+0000","last_peered":"2023-10-09T05:39:20.774116+0000","last_clean":"2023-10-09T05:39:20.774116+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:39:20.774116+0000","last_undegraded":"2023-10-09T05:39:20.774116+0000","last_fullsized":"2023-10-09T05:39:20.774116+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":580,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.785771+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:39:20.785771+0000","last_peered":"2023-10-09T05:39:20.785771+0000","last_clean":"2023-10-09T05:39:20.785771+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:39:20.785771+0000","last_undegraded":"2023-10-09T05:39:20.785771+0000","last_fullsized":"2023-10-09T05:39:20.785771+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":184,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:31.173260+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:39:31.173260+0000","last_peered":"2023-10-09T05:39:31.173260+0000","last_clean":"2023-10-09T05:39:31.173260+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:39:31.173260+0000","last_undegraded":"2023-10-09T05:39:31.173260+0000","last_fullsized":"2023-10-09T05:39:31.173260+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"192'7617","reported_seq":7662,"reported_epoch":192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:35.755339+0000","last_change":"2023-10-09T05:37:36.839957+0000","last_active":"2023-10-09T05:39:35.755339+0000","last_peered":"2023-10-09T05:39:35.755339+0000","last_clean":"2023-10-09T05:37:34.656587+0000","last_became_active":"2023-10-09T05:37:36.833510+0000","last_became_peered":"2023-10-09T05:37:36.833510+0000","last_unstale":"2023-10-09T05:39:35.755339+0000","last_undegraded":"2023-10-09T05:37:36.701190+0000","last_fullsized":"2023-10-09T05:37:36.681999+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7617,"log_dups_size":0,"ondisk_log_size":7617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207814656,"num_objects":3171,"num_object_clones":0,"num_object_copies":9513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2877,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7617,"num_write_kb":345216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":294},{"shards":"2,6,7","objects":654}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":676,"reported_epoch":191,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:20.780702+0000","last_change":"2023-10-09T05:37:36.917512+0000","last_active":"2023-10-09T05:39:20.780702+0000","last_peered":"2023-10-09T05:39:20.780702+0000","last_clean":"2023-10-09T05:37:30.688967+0000","last_became_active":"2023-10-09T05:37:36.700218+0000","last_became_peered":"2023-10-09T05:37:36.700218+0000","last_unstale":"2023-10-09T05:39:20.780702+0000","last_undegraded":"2023-10-09T05:37:36.917335+0000","last_fullsized":"2023-10-09T05:37:36.682410+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":147,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.765218+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:39:20.765218+0000","last_peered":"2023-10-09T05:39:20.765218+0000","last_clean":"2023-10-09T05:39:20.765218+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:39:20.765218+0000","last_undegraded":"2023-10-09T05:39:20.765218+0000","last_fullsized":"2023-10-09T05:39:20.765218+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":143,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.782863+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:39:20.782863+0000","last_peered":"2023-10-09T05:39:20.782863+0000","last_clean":"2023-10-09T05:39:20.782863+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:39:20.782863+0000","last_undegraded":"2023-10-09T05:39:20.782863+0000","last_fullsized":"2023-10-09T05:39:20.782863+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":155,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.764945+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:39:20.764945+0000","last_peered":"2023-10-09T05:39:20.764945+0000","last_clean":"2023-10-09T05:39:20.764945+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:39:20.764945+0000","last_undegraded":"2023-10-09T05:39:20.764945+0000","last_fullsized":"2023-10-09T05:39:20.764945+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":183,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.768541+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:39:20.768541+0000","last_peered":"2023-10-09T05:39:20.768541+0000","last_clean":"2023-10-09T05:39:20.768541+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:39:20.768541+0000","last_undegraded":"2023-10-09T05:39:20.768541+0000","last_fullsized":"2023-10-09T05:39:20.768541+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":161,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.786412+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:39:20.786412+0000","last_peered":"2023-10-09T05:39:20.786412+0000","last_clean":"2023-10-09T05:39:20.786412+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:39:20.786412+0000","last_undegraded":"2023-10-09T05:39:20.786412+0000","last_fullsized":"2023-10-09T05:39:20.786412+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":173,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.783519+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:39:20.783519+0000","last_peered":"2023-10-09T05:39:20.783519+0000","last_clean":"2023-10-09T05:39:20.783519+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:39:20.783519+0000","last_undegraded":"2023-10-09T05:39:20.783519+0000","last_fullsized":"2023-10-09T05:39:20.783519+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":146,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.789786+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:39:20.789786+0000","last_peered":"2023-10-09T05:39:20.789786+0000","last_clean":"2023-10-09T05:39:20.789786+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:39:20.789786+0000","last_undegraded":"2023-10-09T05:39:20.789786+0000","last_fullsized":"2023-10-09T05:39:20.789786+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":173,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.783091+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:39:20.783091+0000","last_peered":"2023-10-09T05:39:20.783091+0000","last_clean":"2023-10-09T05:39:20.783091+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:39:20.783091+0000","last_undegraded":"2023-10-09T05:39:20.783091+0000","last_fullsized":"2023-10-09T05:39:20.783091+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":171,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:31.173372+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:39:31.173372+0000","last_peered":"2023-10-09T05:39:31.173372+0000","last_clean":"2023-10-09T05:39:31.173372+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:39:31.173372+0000","last_undegraded":"2023-10-09T05:39:31.173372+0000","last_fullsized":"2023-10-09T05:39:31.173372+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":183,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.773541+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:39:20.773541+0000","last_peered":"2023-10-09T05:39:20.773541+0000","last_clean":"2023-10-09T05:39:20.773541+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:39:20.773541+0000","last_undegraded":"2023-10-09T05:39:20.773541+0000","last_fullsized":"2023-10-09T05:39:20.773541+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":147,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.765331+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:39:20.765331+0000","last_peered":"2023-10-09T05:39:20.765331+0000","last_clean":"2023-10-09T05:39:20.765331+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:39:20.765331+0000","last_undegraded":"2023-10-09T05:39:20.765331+0000","last_fullsized":"2023-10-09T05:39:20.765331+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":158,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.782731+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:39:20.782731+0000","last_peered":"2023-10-09T05:39:20.782731+0000","last_clean":"2023-10-09T05:39:20.782731+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:39:20.782731+0000","last_undegraded":"2023-10-09T05:39:20.782731+0000","last_fullsized":"2023-10-09T05:39:20.782731+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3336962072,"num_objects":50919,"num_object_clones":0,"num_object_copies":152757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9381,"num_objects_misplaced":315,"num_objects_unfound":0,"num_objects_dirty":50919,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":125803,"num_write_kb":5655041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7779,"num_bytes_recovered":509804544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10801459200,"data_stored":10801446984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117903,"ondisk_log_size":117903,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846653,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626678,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1958060,"kb_used_data":1648548,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91787092,"statfs":{"total":95995035648,"available":93989982208,"internally_reserved":0,"allocated":1688113152,"data_stored":1687593435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:23 2023","interfaces":[{"interface":"back","average":{"1min":6.093,"5min":4.5960000000000001,"15min":2.1890000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":99.787999999999997,"5min":99.787999999999997,"15min":99.787999999999997},"last":1.7010000000000001},{"interface":"front","average":{"1min":5.1890000000000001,"5min":4.234,"15min":2.0710000000000002},"min":{"1min":0.25600000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":99.844999999999999,"5min":99.844999999999999,"15min":99.844999999999999},"last":1.3069999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":2.677,"15min":1.3799999999999999},"min":{"1min":0.30099999999999999,"5min":0.25,"15min":0.25},"max":{"1min":9.8670000000000009,"5min":69.667000000000002,"15min":69.667000000000002},"last":4.9119999999999999},{"interface":"front","average":{"1min":1.972,"5min":2.8849999999999998,"15min":1.4670000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":10.624000000000001,"5min":69.697999999999993,"15min":69.697999999999993},"last":5.0640000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:39:23 2023","interfaces":[{"interface":"back","average":{"1min":8.2159999999999993,"5min":4.593,"15min":2.5329999999999999},"min":{"1min":0.29499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":115.2,"5min":115.2,"15min":115.2},"last":5.0380000000000003},{"interface":"front","average":{"1min":8.2409999999999997,"5min":4.4180000000000001,"15min":2.4950000000000001},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":115.297,"5min":115.297,"15min":115.297},"last":4.891}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":2.6160000000000001,"15min":1.661},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":29.885000000000002,"5min":68.920000000000002,"15min":68.920000000000002},"last":4.9960000000000004},{"interface":"front","average":{"1min":3.0230000000000001,"5min":2.794,"15min":1.76},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":29.771000000000001,"5min":68.846000000000004,"15min":68.846000000000004},"last":1.419}]},{"osd":4,"last update":"Mon Oct 9 05:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":3.1379999999999999,"15min":1.401},"min":{"1min":0.248,"5min":0.245,"15min":0.23400000000000001},"max":{"1min":0.58599999999999997,"5min":125.64,"15min":125.64},"last":0.312},{"interface":"front","average":{"1min":0.41499999999999998,"5min":2.7709999999999999,"15min":1.276},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":1.2709999999999999,"5min":125.625,"15min":125.625},"last":0.38400000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":2.871,"15min":1.3340000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.5510000000000002,"5min":127.35899999999999,"15min":127.35899999999999},"last":2.0529999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.5110000000000001,"15min":1.23},"min":{"1min":0.29299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.8769999999999998,"5min":128.93799999999999,"15min":128.93799999999999},"last":1.2709999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":2.0800000000000001,"15min":2.7919999999999998},"min":{"1min":0.308,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.2789999999999999,"5min":12.074999999999999,"15min":12.074999999999999},"last":1.952},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.349,"15min":1.8440000000000001},"min":{"1min":0.33800000000000002,"5min":0.317,"15min":0.317},"max":{"1min":1.343,"5min":10.083,"15min":10.083},"last":0.41899999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958640,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1420552,"kb_used_data":1172608,"kb_used_omap":7,"kb_used_meta":247928,"kb_avail":92324600,"statfs":{"total":95995035648,"available":94540390400,"internally_reserved":0,"allocated":1200750592,"data_stored":1200031801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":253878798},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2120000000000002,"5min":5.5830000000000002,"15min":2.1520000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.185},"max":{"1min":9.9819999999999993,"5min":207.858,"15min":207.858},"last":0.41299999999999998},{"interface":"front","average":{"1min":6.5330000000000004,"5min":6.7000000000000002,"15min":2.5499999999999998},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":62.880000000000003,"5min":206.97200000000001,"15min":206.97200000000001},"last":3.4390000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":5.3289999999999997,"5min":7.3360000000000003,"15min":2.8220000000000001},"min":{"1min":0.22700000000000001,"5min":0.184,"15min":0.184},"max":{"1min":32.686999999999998,"5min":101.367,"15min":101.367},"last":0.67300000000000004},{"interface":"front","average":{"1min":4.468,"5min":6.7439999999999998,"15min":2.6259999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":19.975999999999999,"5min":98.409999999999997,"15min":98.409999999999997},"last":3.1779999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":4.234,"5min":3.0209999999999999,"15min":1.3640000000000001},"min":{"1min":0.24199999999999999,"5min":0.223,"15min":0.223},"max":{"1min":33.146000000000001,"5min":83.885000000000005,"15min":83.885000000000005},"last":4.8280000000000003},{"interface":"front","average":{"1min":8.5860000000000003,"5min":3.8889999999999998,"15min":1.619},"min":{"1min":0.26300000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":62.929000000000002,"5min":84.641999999999996,"15min":84.641999999999996},"last":1.2310000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":5.9870000000000001,"5min":4.1109999999999998,"15min":1.833},"min":{"1min":0.432,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":63.116,"5min":141.32900000000001,"15min":141.32900000000001},"last":3.4809999999999999},{"interface":"front","average":{"1min":1.675,"5min":2.8999999999999999,"15min":1.415},"min":{"1min":0.34499999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.5549999999999997,"5min":141.15899999999999,"15min":141.15899999999999},"last":0.44700000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":4.0350000000000001,"15min":1.7929999999999999},"min":{"1min":0.39000000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.0009999999999994,"5min":235.18700000000001,"15min":235.18700000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":6.0140000000000002,"5min":4.9180000000000001,"15min":2.1819999999999999},"min":{"1min":0.46600000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":63.162999999999997,"5min":235.79300000000001,"15min":235.79300000000001},"last":3.5409999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":1.8220000000000001,"15min":1.075},"min":{"1min":0.42599999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.0289999999999999,"5min":10.333,"15min":10.333},"last":3.9609999999999999},{"interface":"front","average":{"1min":6.2220000000000004,"5min":2.75,"15min":1.393},"min":{"1min":0.5,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":63.198,"5min":63.198,"15min":63.198},"last":3.5129999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":2.7949999999999999,"15min":2.2639999999999998},"min":{"1min":0.45700000000000002,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.6849999999999996,"5min":126.68300000000001,"15min":126.68300000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.6439999999999999,"5min":2.911,"15min":2.3559999999999999},"min":{"1min":0.48499999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.6180000000000003,"5min":126.738,"15min":126.738},"last":0.61399999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122166,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1164068,"kb_used_data":821980,"kb_used_omap":8,"kb_used_meta":342071,"kb_avail":92581084,"statfs":{"total":95995035648,"available":94803030016,"internally_reserved":0,"allocated":841707520,"data_stored":840962324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":350281480},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":1.8959999999999999,"15min":0.93799999999999994},"min":{"1min":0.20499999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":14.964,"5min":14.964,"15min":14.964},"last":7.2089999999999996},{"interface":"front","average":{"1min":3.879,"5min":2.2850000000000001,"15min":1.0820000000000001},"min":{"1min":0.26100000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":11.315,"5min":11.315,"15min":11.315},"last":3.5129999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":3.4180000000000001,"5min":3.8109999999999999,"15min":1.8879999999999999},"min":{"1min":0.28999999999999998,"5min":0.223,"15min":0.214},"max":{"1min":12.759,"5min":79.911000000000001,"15min":79.911000000000001},"last":2.5619999999999998},{"interface":"front","average":{"1min":4.085,"5min":3.5089999999999999,"15min":1.766},"min":{"1min":0.246,"5min":0.218,"15min":0.218},"max":{"1min":12.234999999999999,"5min":80.105000000000004,"15min":80.105000000000004},"last":0.26000000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":6.5289999999999999,"5min":3.0329999999999999,"15min":1.3149999999999999},"min":{"1min":0.48799999999999999,"5min":0.23999999999999999,"15min":0.22800000000000001},"max":{"1min":31.594000000000001,"5min":31.594000000000001,"15min":31.594000000000001},"last":2.153},{"interface":"front","average":{"1min":6.4569999999999999,"5min":3.1379999999999999,"15min":1.3879999999999999},"min":{"1min":0.38500000000000001,"5min":0.222,"15min":0.222},"max":{"1min":31.641999999999999,"5min":31.641999999999999,"15min":31.641999999999999},"last":2.5910000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":2.0539999999999998,"15min":1.163},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.1989999999999998,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.52000000000000002},{"interface":"front","average":{"1min":2.661,"5min":2.7010000000000001,"15min":1.4350000000000001},"min":{"1min":0.42099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.2149999999999999,"5min":69.102000000000004,"15min":69.102000000000004},"last":0.45000000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":8.0129999999999999,"5min":3.081,"15min":1.5509999999999999},"min":{"1min":0.45200000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":115.175,"5min":115.175,"15min":115.175},"last":7.258},{"interface":"front","average":{"1min":7.0750000000000002,"5min":2.9710000000000001,"15min":1.5069999999999999},"min":{"1min":0.36499999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":115.502,"5min":115.502,"15min":115.502},"last":5.5869999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":1.9239999999999999,"15min":1.2050000000000001},"min":{"1min":0.41999999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.3409999999999993,"5min":9.3409999999999993,"15min":9.9809999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.153,"5min":2.02,"15min":1.198},"min":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":7.266,"5min":8.0999999999999996,"15min":8.0999999999999996},"last":2.673}]},{"osd":7,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":1.409,"15min":1.355},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.5499999999999998,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.649,"5min":2.206,"15min":2.3279999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":8.2490000000000006,"5min":11.068,"15min":11.068},"last":0.5}]}]},{"osd":2,"up_from":98,"seq":420906795113,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2213716,"kb_used_data":1828172,"kb_used_omap":11,"kb_used_meta":385524,"kb_avail":91531436,"statfs":{"total":95995035648,"available":93728190464,"internally_reserved":0,"allocated":1872048128,"data_stored":1871262479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":394777173},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":3.476,"5min":2.8010000000000002,"15min":1.2669999999999999},"min":{"1min":0.23400000000000001,"5min":0.183,"15min":0.17199999999999999},"max":{"1min":10.098000000000001,"5min":32.122,"15min":32.122},"last":6.1050000000000004},{"interface":"front","average":{"1min":3.27,"5min":2.3530000000000002,"15min":1.1399999999999999},"min":{"1min":0.25800000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.5850000000000009,"5min":34.066000000000003,"15min":34.066000000000003},"last":0.22500000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":14.41,"5min":5.3600000000000003,"15min":2.133},"min":{"1min":0.34499999999999997,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":98.575000000000003,"5min":98.575000000000003,"15min":98.575000000000003},"last":0.26000000000000001},{"interface":"front","average":{"1min":12.257,"5min":4.6779999999999999,"15min":1.867},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":90.135000000000005,"5min":98.397000000000006,"15min":98.397000000000006},"last":9.3260000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:38:47 2023","interfaces":[{"interface":"back","average":{"1min":3.5640000000000001,"5min":3.0049999999999999,"15min":1.2769999999999999},"min":{"1min":0.41699999999999998,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":12.385999999999999,"5min":45.057000000000002,"15min":45.057000000000002},"last":15.298},{"interface":"front","average":{"1min":3.2970000000000002,"5min":3.3300000000000001,"15min":1.4410000000000001},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.942,"5min":45.813000000000002,"15min":45.813000000000002},"last":0.98199999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":2.8050000000000002,"15min":1.4019999999999999},"min":{"1min":0.29099999999999998,"5min":0.28999999999999998,"15min":0.27700000000000002},"max":{"1min":11.125999999999999,"5min":115.84999999999999,"15min":115.84999999999999},"last":9.3000000000000007},{"interface":"front","average":{"1min":2.976,"5min":3.504,"15min":1.675},"min":{"1min":0.34799999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.147,"5min":116.149,"15min":116.149},"last":1.0329999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":2.5409999999999999,"15min":1.369},"min":{"1min":0.39900000000000002,"5min":0.312,"15min":0.312},"max":{"1min":7.8220000000000001,"5min":24.597999999999999,"15min":24.597999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.867,"5min":2.089,"15min":1.1779999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.097,"5min":23.919,"15min":23.919},"last":9.2590000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":2.5129999999999999,"15min":1.327},"min":{"1min":0.39900000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.183,"5min":20.503,"15min":20.503},"last":1.3460000000000001},{"interface":"front","average":{"1min":2.8519999999999999,"5min":2.2170000000000001,"15min":1.2470000000000001},"min":{"1min":0.45000000000000001,"5min":0.372,"15min":0.34499999999999997},"max":{"1min":7.8730000000000002,"5min":9.4049999999999994,"15min":9.4049999999999994},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:38:49 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":2.883,"15min":2.6429999999999998},"min":{"1min":0.42599999999999999,"5min":0.374,"15min":0.374},"max":{"1min":8.9819999999999993,"5min":24.030999999999999,"15min":24.030999999999999},"last":1.083},{"interface":"front","average":{"1min":3.3839999999999999,"5min":2.4950000000000001,"15min":1.984},"min":{"1min":0.47299999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.2569999999999997,"5min":24.178000000000001,"15min":24.178000000000001},"last":0.51600000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631587,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2536196,"kb_used_data":2283516,"kb_used_omap":4,"kb_used_meta":252155,"kb_avail":91208956,"statfs":{"total":95995035648,"available":93397970944,"internally_reserved":0,"allocated":2338320384,"data_stored":2337604870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":258207326},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0840000000000001,"5min":2.6869999999999998,"15min":1.1970000000000001},"min":{"1min":0.26100000000000001,"5min":0.183,"15min":0.183},"max":{"1min":13.997,"5min":56.094000000000001,"15min":56.094000000000001},"last":4.2590000000000003},{"interface":"front","average":{"1min":3.7799999999999998,"5min":2.9700000000000002,"15min":1.296},"min":{"1min":0.221,"5min":0.221,"15min":0.214},"max":{"1min":15.593,"5min":57.255000000000003,"15min":57.255000000000003},"last":6.3760000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0059999999999998,"5min":3.5619999999999998,"15min":1.4690000000000001},"min":{"1min":0.35399999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":13.932,"5min":83.921999999999997,"15min":83.921999999999997},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.5019999999999998,"5min":3.4660000000000002,"15min":1.4119999999999999},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":7.1580000000000004,"5min":83.707999999999998,"15min":83.707999999999998},"last":4.109}]},{"osd":2,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":6.2699999999999996,"5min":4.5549999999999997,"15min":2.2410000000000001},"min":{"1min":0.25,"5min":0.215,"15min":0.215},"max":{"1min":60.722000000000001,"5min":95.320999999999998,"15min":117.23},"last":4.2830000000000004},{"interface":"front","average":{"1min":7.0670000000000002,"5min":4.6360000000000001,"15min":2.319},"min":{"1min":0.32200000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":60.469999999999999,"5min":94.170000000000002,"15min":117.45999999999999},"last":10.206}]},{"osd":4,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":2.3530000000000002,"15min":1.329},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.1709999999999994,"5min":45.779000000000003,"15min":45.779000000000003},"last":6.117},{"interface":"front","average":{"1min":1.9239999999999999,"5min":2.2570000000000001,"15min":1.21},"min":{"1min":0.35499999999999998,"5min":0.313,"15min":0.313},"max":{"1min":13.734,"5min":45.735999999999997,"15min":45.735999999999997},"last":0.27900000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":2.1400000000000001,"15min":1.165},"min":{"1min":0.433,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0800000000000001,"5min":47.948999999999998,"15min":47.948999999999998},"last":4.1479999999999997},{"interface":"front","average":{"1min":1.9259999999999999,"5min":2.3730000000000002,"15min":1.264},"min":{"1min":0.38300000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":13.864000000000001,"5min":47.968000000000004,"15min":47.968000000000004},"last":0.37}]},{"osd":6,"last update":"Mon Oct 9 05:38:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":2.0499999999999998,"15min":1.175},"min":{"1min":0.41199999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":13.831,"5min":13.831,"15min":13.831},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.944,"5min":1.6850000000000001,"15min":0.98299999999999998},"min":{"1min":0.53000000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.117,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":4.1749999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:38:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2869999999999999,"5min":2.3300000000000001,"15min":2.2829999999999999},"min":{"1min":0.46899999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":9.1319999999999997,"5min":11.407,"15min":11.407},"last":6.181},{"interface":"front","average":{"1min":3.3050000000000002,"5min":2.2440000000000002,"15min":2.056},"min":{"1min":0.443,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.2949999999999999,"5min":11.432,"15min":11.432},"last":6.1559999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019135,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1316588,"kb_used_data":1011172,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92428564,"statfs":{"total":95995035648,"available":94646849536,"internally_reserved":0,"allocated":1035440128,"data_stored":1034920155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:20 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":2.1640000000000001,"15min":1.21},"min":{"1min":0.25600000000000001,"5min":0.214,"15min":0.214},"max":{"1min":7.4480000000000004,"5min":36.406999999999996,"15min":36.406999999999996},"last":1.8100000000000001},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.7810000000000001,"15min":1.454},"min":{"1min":0.28699999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.988,"5min":36.454999999999998,"15min":36.454999999999998},"last":0.53700000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":1.9430000000000001,"15min":1.0169999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.9550000000000001,"5min":31.073,"15min":31.073},"last":2.1110000000000002},{"interface":"front","average":{"1min":1.421,"5min":1.968,"15min":1.0600000000000001},"min":{"1min":0.28699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.0109999999999992,"5min":31.100000000000001,"15min":31.100000000000001},"last":2.1269999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:39:15 2023","interfaces":[{"interface":"back","average":{"1min":3.4870000000000001,"5min":8.1349999999999998,"15min":3.2909999999999999},"min":{"1min":0.316,"5min":0.314,"15min":0.26800000000000002},"max":{"1min":20.684000000000001,"5min":196.93700000000001,"15min":196.93700000000001},"last":1.0309999999999999},{"interface":"front","average":{"1min":2.621,"5min":7.7030000000000003,"15min":3.2040000000000002},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":20.611000000000001,"5min":197.08099999999999,"15min":197.08099999999999},"last":2.0419999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":23.963000000000001,"5min":6.3440000000000003,"15min":2.504},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":211.48699999999999,"5min":211.48699999999999,"15min":211.48699999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":23.547000000000001,"5min":6.0309999999999997,"15min":2.4510000000000001},"min":{"1min":0.38600000000000001,"5min":0.311,"15min":0.309},"max":{"1min":211.31100000000001,"5min":211.31100000000001,"15min":211.31100000000001},"last":6.9930000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":3.6040000000000001,"15min":1.5289999999999999},"min":{"1min":0.25,"5min":0.249,"15min":0.249},"max":{"1min":2.6070000000000002,"5min":113.35899999999999,"15min":113.35899999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":3.3849999999999998,"15min":1.5109999999999999},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.0550000000000002,"5min":113.32299999999999,"15min":113.32299999999999},"last":0.38900000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.0569999999999999,"15min":0.69799999999999995},"min":{"1min":0.28999999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.0289999999999999,"5min":11.944000000000001,"15min":11.944000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.3169999999999999,"15min":0.80300000000000005},"min":{"1min":0.313,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.1899999999999999,"5min":10.356,"15min":10.356},"last":0.51000000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:38:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":2.0230000000000001,"15min":2.9209999999999998},"min":{"1min":0.38700000000000001,"5min":0.312,"15min":0.312},"max":{"1min":2.0750000000000002,"5min":30.393999999999998,"15min":30.393999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.876,"5min":2.1989999999999998,"15min":3.286},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.5859999999999999,"5min":30.334,"15min":30.334},"last":0.36499999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822907,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1305184,"kb_used_data":1003864,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92439968,"statfs":{"total":95995035648,"available":94658527232,"internally_reserved":0,"allocated":1027956736,"data_stored":1027444523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":6.3940000000000001,"15min":2.8650000000000002},"min":{"1min":0.249,"5min":0.224,"15min":0.19400000000000001},"max":{"1min":10.712999999999999,"5min":165.90199999999999,"15min":165.90199999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":2.1419999999999999,"5min":6.3799999999999999,"15min":2.9359999999999999},"min":{"1min":0.28999999999999998,"5min":0.215,"15min":0.215},"max":{"1min":8.6099999999999994,"5min":165.96700000000001,"15min":165.96700000000001},"last":5.4660000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":7.4249999999999998,"5min":3.0960000000000001,"15min":1.4159999999999999},"min":{"1min":0.30599999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":87.769000000000005,"5min":87.769000000000005,"15min":87.769000000000005},"last":0.34799999999999998},{"interface":"front","average":{"1min":6.9340000000000002,"5min":2.782,"15min":1.337},"min":{"1min":0.35599999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":87.864000000000004,"5min":87.864000000000004,"15min":87.864000000000004},"last":0.374}]},{"osd":2,"last update":"Mon Oct 9 05:39:20 2023","interfaces":[{"interface":"back","average":{"1min":5.9400000000000004,"5min":9.1470000000000002,"15min":3.5609999999999999},"min":{"1min":0.33800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":42.055,"5min":290.697,"15min":290.697},"last":5.8449999999999998},{"interface":"front","average":{"1min":4.2720000000000002,"5min":8.7880000000000003,"15min":3.4079999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":41.953000000000003,"5min":290.786,"15min":290.786},"last":5.694}]},{"osd":3,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":5.8250000000000002,"5min":2.7799999999999998,"15min":1.3500000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":52.560000000000002,"5min":52.560000000000002,"15min":52.560000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":4.8470000000000004,"5min":2.5710000000000002,"15min":1.268},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":52.380000000000003,"5min":52.380000000000003,"15min":52.380000000000003},"last":0.45600000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":1.0940000000000001,"15min":0.68600000000000005},"min":{"1min":0.25700000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.2949999999999999,"5min":9.3510000000000009,"15min":9.3510000000000009},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.746,"15min":0.93200000000000005},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":2.669,"5min":14.236000000000001,"15min":14.236000000000001},"last":0.35899999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":2.5600000000000001,"15min":1.234},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":1.8069999999999999,"5min":95.010999999999996,"15min":95.010999999999996},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":2.2490000000000001,"15min":1.123},"min":{"1min":0.33500000000000002,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":2.4390000000000001,"5min":94.194000000000003,"15min":94.194000000000003},"last":0.29399999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.5149999999999999,"15min":1.655},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.4289999999999998,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":1.6599999999999999,"15min":1.968},"min":{"1min":0.36199999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.7250000000000001,"5min":14.432,"15min":14.432},"last":0.39600000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":831918080,"data_stored":831914008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1193283584,"data_stored":1193279512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1855520768,"data_stored":1855520768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2319056896,"data_stored":2319056896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1022361600,"data_stored":1022361600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1026101248,"data_stored":1026097176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1673396224,"data_stored":1673396224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":879820800,"data_stored":879820800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:39:45.304 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:39:45.305 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:45.305 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:45.305 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:44 smithi191 ceph-mon[161669]: pgmap v532: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 9414/146979 objects degraded (6.405%); 73/146979 objects misplaced (0.050%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:39:45.624 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:45.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:45.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:45.800 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:45.856 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:45.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:45.962 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:46.015 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:46.075 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:39:46.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:46.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:46.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:46.211 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:46.260 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:39:46.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:46.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:46.368 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:46.371 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:39:46.371 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:39:46.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.371 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:130669 6.9 6:97613e43:::benchmark_data_smithi008_295510_object42306:head [delete] snapc 0=[] ondisk+write+known_if_redirected e193)", 2023-10-09T05:39:46.371 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:39:46.345490+0000", 2023-10-09T05:39:46.371 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.024504214, 2023-10-09T05:39:46.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.02453841, 2023-10-09T05:39:46.371 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:46.371 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:39:46.372 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:46.372 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:46.372 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:46.372 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 130669 2023-10-09T05:39:46.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.372 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:46.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:46.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.345490+0000", 2023-10-09T05:39:46.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:46.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:46.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.345490+0000", 2023-10-09T05:39:46.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:46.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:46.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.345489+0000", 2023-10-09T05:39:46.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:46.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:46.374 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.345498+0000", 2023-10-09T05:39:46.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6910000000000003e-06 2023-10-09T05:39:46.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:46.374 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.345517+0000", 2023-10-09T05:39:46.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8893e-05 2023-10-09T05:39:46.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:46.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.345526+0000", 2023-10-09T05:39:46.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4949999999999997e-06 2023-10-09T05:39:46.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:46.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.351465+0000", 2023-10-09T05:39:46.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0059389789999999996 2023-10-09T05:39:46.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:46.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.351608+0000", 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000142764 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.351726+0000", 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000118025 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.360891+0000", 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0091650129999999996 2023-10-09T05:39:46.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.367109+0000", 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0062185729999999998 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:130681 6.9 6:9420572a:::benchmark_data_smithi008_295510_object42318:head [delete] snapc 0=[] ondisk+write+known_if_redirected e193)", 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:39:46.367298+0000", 2023-10-09T05:39:46.377 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0026962650000000002, 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028353559999999998, 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 130681 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:39:46.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.367298+0000", 2023-10-09T05:39:46.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:46.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:39:46.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.367298+0000", 2023-10-09T05:39:46.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:39:46.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:39:46.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.367297+0000", 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.367304+0000", 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1579999999999999e-06 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.367317+0000", 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3244999999999999e-05 2023-10-09T05:39:46.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:39:46.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.367323+0000", 2023-10-09T05:39:46.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.835e-06 2023-10-09T05:39:46.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:39:46.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.367378+0000", 2023-10-09T05:39:46.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4311999999999998e-05 2023-10-09T05:39:46.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:39:46.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.367565+0000", 2023-10-09T05:39:46.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018788299999999999 2023-10-09T05:39:46.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:39:46.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:39:46.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:39:46.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:46.367662+0000", 2023-10-09T05:39:46.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6466999999999996e-05 2023-10-09T05:39:46.382 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:46.382 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:39:46.382 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:46.383 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:39:46.383 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:39:46.383 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 2 2023-10-09T05:39:46.383 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:39:46.401 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:46.449 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:46.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:46.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:46.554 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:46.574 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:39:46.611 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:39:46.653 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:46.656 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:39:46.709 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:39:46.779 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:39:46.801 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:46.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:46.824 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:39:46.866 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:39:46.904 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:39:46.908 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:46.924 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:39:46.924 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:39:46.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:39:46.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:39:46.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:39:46.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:39:46.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:39:46.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:39:46.947 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:47.005 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:39:47.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:47.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:47.112 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:47.139 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:47.230 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:47.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:47.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:47.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:47 smithi191 ceph-mon[161669]: pgmap v533: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 9414/143298 objects degraded (6.570%); 73/143298 objects misplaced (0.051%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:39:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3360683933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:47.337 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:47.379 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:47.434 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:39:47.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:47.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:47.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:47 smithi008 ceph-mon[191227]: pgmap v533: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 9414/143298 objects degraded (6.570%); 73/143298 objects misplaced (0.051%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:39:47.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3360683933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:47.473 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:47 smithi008 ceph-mon[184418]: pgmap v533: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 9414/143298 objects degraded (6.570%); 73/143298 objects misplaced (0.051%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:39:47.473 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3360683933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:47.540 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:47.567 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:39:47.626 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:48.126 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:48.158 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:39:48.257 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:48.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:48.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:48.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:48.402 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:48.454 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:48.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:48.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:48.462 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9414/143298 objects degraded (6.570%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:48.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9414/143298 objects degraded (6.570%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:48.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9414/143298 objects degraded (6.570%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:48.563 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:48.593 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:48.658 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:48.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:48.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:48.764 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:48.800 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:48.860 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:39:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:48.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:48.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:48.998 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:49.054 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:49.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:49.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:49.165 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:49.204 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:49.266 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:49.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:49.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:49.374 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:49.489 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:49.554 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:49.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:49.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:49.656 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:49.749 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:49.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:49 smithi008 ceph-mon[184418]: pgmap v534: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 9414/140610 objects degraded (6.695%); 73/140610 objects misplaced (0.052%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:39:49.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:49 smithi008 ceph-mon[191227]: pgmap v534: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 9414/140610 objects degraded (6.695%); 73/140610 objects misplaced (0.052%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:39:50.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:49 smithi191 ceph-mon[161669]: pgmap v534: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 9414/140610 objects degraded (6.695%); 73/140610 objects misplaced (0.052%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T05:39:50.071 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:50.083 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:50.095 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:50.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:50.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:50.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:39:50.265 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:50.326 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:39:50.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:50.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:39:50.402 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:39:50.437 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:50.568 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:50.746 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:39:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:39:50] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.33.4" 2023-10-09T05:39:50.986 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:50.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:50.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:51.093 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:51.198 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:51.351 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:51.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:51.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:51.476 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:51.570 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:52.041 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:39:52.041 INFO:teuthology.orchestra.run.smithi008.stdout:{"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 9290/135576 objects degraded (6.852%), 8 pgs degraded, 8 pgs undersized","count":16},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":685,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":193,"num_osds":8,"num_up_osds":8,"osd_up_since":1696829680,"num_in_osds":7,"osd_in_since":1696829854,"num_remapped_pgs":8},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":81},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":45192,"data_bytes":2964592587,"bytes_used":11116871680,"bytes_avail":660848377856,"bytes_total":671965249536,"degraded_objects":9290,"degraded_total":135576,"degraded_ratio":0.068522452351448632,"recovering_objects_per_sec":43,"recovering_bytes_per_sec":2062898,"recovering_keys_per_sec":0,"num_objects_recovered":439,"num_bytes_recovered":20643840,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":575},"fsmap":{"epoch":6,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi008.zfejpl","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.8:8443/","prometheus":"http://172.21.15.8:9283/"}},"servicemap":{"epoch":23,"modified":"2023-10-09T05:35:39.975308+0000","services":{}},"progress_events":{"a1c2a5cb-a243-4c8c-845f-d0215fa7ce6b":{"message":"Global Recovery Event (2m)\n [=========================...] (remaining: 11s)","progress":0.91011238098144531,"add_to_ceph_s":true},"e7a35707-3c6c-469d-ad94-cd0d5825f733":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T05:39:52.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:51 smithi191 ceph-mon[161669]: pgmap v535: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 9290/135576 objects degraded (6.852%); 2.0 MiB/s, 43 objects/s recovering 2023-10-09T05:39:52.068 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:51 smithi008 ceph-mon[184418]: pgmap v535: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 9290/135576 objects degraded (6.852%); 2.0 MiB/s, 43 objects/s recovering 2023-10-09T05:39:52.068 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:51 smithi008 ceph-mon[191227]: pgmap v535: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 9290/135576 objects degraded (6.852%); 2.0 MiB/s, 43 objects/s recovering 2023-10-09T05:39:52.980 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:52.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:52.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:53.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/933939452' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:39:53.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:52 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9290/135576 objects degraded (6.852%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:53.090 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:53.118 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:53.179 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:53.180 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/933939452' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:39:53.180 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:52 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9290/135576 objects degraded (6.852%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:53.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:53.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:53.184 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/933939452' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:39:53.184 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:52 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9290/135576 objects degraded (6.852%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:53.284 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:53.338 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:53.782 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:53.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:53.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:53.900 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:53.940 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T05:39:53.941 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:39:54.036 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:54.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:53 smithi191 ceph-mon[161669]: pgmap v536: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 9290/131619 objects degraded (7.058%); 2.0 MiB/s, 43 objects/s recovering 2023-10-09T05:39:54.064 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:53 smithi008 ceph-mon[191227]: pgmap v536: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 9290/131619 objects degraded (7.058%); 2.0 MiB/s, 43 objects/s recovering 2023-10-09T05:39:54.065 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:53 smithi008 ceph-mon[184418]: pgmap v536: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 9290/131619 objects degraded (7.058%); 2.0 MiB/s, 43 objects/s recovering 2023-10-09T05:39:54.284 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:54.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:54.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:54.405 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:54.492 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:54.535 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:54.646 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:54.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:54.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:54.681 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:39:54.681 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:39:54.769 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:54.866 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:55.141 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:55.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:55.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:55.250 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:55.310 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:55.428 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:55.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:55.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:55.544 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:39:55.572 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:55.685 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:39:55.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:55.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:39:55.801 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:55.829 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:39:55.832 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:55.890 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:55.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:55.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:55.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:55 smithi008 ceph-mon[184418]: pgmap v537: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 9091/129510 objects degraded (7.020%); 1.7 MiB/s, 56 objects/s recovering 2023-10-09T05:39:55.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:55 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3562171686' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:39:55.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:55 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:39:55.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:55 smithi008 ceph-mon[191227]: pgmap v537: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 9091/129510 objects degraded (7.020%); 1.7 MiB/s, 56 objects/s recovering 2023-10-09T05:39:55.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:55 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3562171686' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:39:55.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:55 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:39:55.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:56.031 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:56.089 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:56.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:56.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:56.094 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:55 smithi191 ceph-mon[161669]: pgmap v537: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 9091/129510 objects degraded (7.020%); 1.7 MiB/s, 56 objects/s recovering 2023-10-09T05:39:56.095 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:55 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3562171686' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:39:56.095 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:55 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:39:56.195 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:56.229 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:56.282 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:56.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:56.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:56.389 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:39:56.421 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:56.477 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:39:56.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:56.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:39:56.582 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:56.607 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:56.649 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:39:56.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:56.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:56.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:56.782 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:56.847 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:56.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:56.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:39:56.871 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:39:56.958 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:39:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:56 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:56 smithi191 ceph-mon[161669]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:56 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:39:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:56 smithi191 ceph-mon[161669]: osdmap e194: 8 total, 8 up, 7 in 2023-10-09T05:39:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:56 smithi191 ceph-mon[161669]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:56 smithi191 ceph-mon[161669]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:56 smithi191 ceph-mon[161669]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:56 smithi191 ceph-mon[161669]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:56 smithi191 ceph-mon[161669]: pgmap v539: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 9091/125781 objects degraded (7.228%); 467 KiB/s, 39 objects/s recovering 2023-10-09T05:39:57.069 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:57.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:57.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[184418]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:39:57.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[184418]: osdmap e194: 8 total, 8 up, 7 in 2023-10-09T05:39:57.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[184418]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[184418]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[184418]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[184418]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[184418]: pgmap v539: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 9091/125781 objects degraded (7.228%); 467 KiB/s, 39 objects/s recovering 2023-10-09T05:39:57.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:39:57.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[191227]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:39:57.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[191227]: osdmap e194: 8 total, 8 up, 7 in 2023-10-09T05:39:57.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[191227]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[191227]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[191227]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[191227]: failed to encode map e194 with expected crc 2023-10-09T05:39:57.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:56 smithi008 ceph-mon[191227]: pgmap v539: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 9091/125781 objects degraded (7.228%); 467 KiB/s, 39 objects/s recovering 2023-10-09T05:39:57.441 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:57.467 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:39:57.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:57.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:39:57.589 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:39:57.700 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:57.880 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:39:57.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:57.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:39:57.990 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:58.015 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:58.084 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:58.201 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:39:58.266 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:39:58.314 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:39:58.355 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:58.365 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:39:58.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:58.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:58.396 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:39:58.447 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:39:58.467 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:39:58.496 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:58.497 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:39:58.535 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:39:58.540 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:39:58.540 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:39:58.540 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:39:58.540 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:39:58.540 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:39:58.540 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:39:58.541 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:39:58.541 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:39:58.546 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:39:58.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:58.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:39:58.651 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:39:58.715 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:58.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 9091/125781 objects degraded (7.228%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3782563440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:58.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 9091/125781 objects degraded (7.228%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:58.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3782563440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:58.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 9091/125781 objects degraded (7.228%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:39:58.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3782563440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:39:59.452 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:39:59.716 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:39:59.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:59.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:39:59.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:39:59.860 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:39:59.889 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:59 smithi008 ceph-mon[191227]: pgmap v540: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 9091/123075 objects degraded (7.387%); 467 KiB/s, 39 objects/s recovering 2023-10-09T05:39:59.889 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:59.889 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:39:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:59.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:59 smithi008 ceph-mon[184418]: pgmap v540: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 9091/123075 objects degraded (7.387%); 467 KiB/s, 39 objects/s recovering 2023-10-09T05:39:59.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:39:59.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:39:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:59.917 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:39:59.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:39:59.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:00.022 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:00.047 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:59 smithi191 ceph-mon[161669]: pgmap v540: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 9091/123075 objects degraded (7.387%); 467 KiB/s, 39 objects/s recovering 2023-10-09T05:40:00.047 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:00.047 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:39:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:00.054 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:00.111 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:00.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:00.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:00.217 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:00.257 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:00.319 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:00.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:00.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:00.424 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:00.453 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:00.754 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[184418]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 9091/123075 objects degraded (7.387%), 8 pgs degraded, 8 pgs undersized 2023-10-09T05:40:00.754 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[184418]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T05:40:00.755 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[184418]: [WRN] PG_DEGRADED: Degraded data redundancy: 9091/123075 objects degraded (7.387%), 8 pgs degraded, 8 pgs undersized 2023-10-09T05:40:00.755 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[184418]: pg 2.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-10-09T05:40:00.755 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[184418]: pg 2.3 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,2] 2023-10-09T05:40:00.755 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[184418]: pg 2.4 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-10-09T05:40:00.755 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[184418]: pg 2.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,2] 2023-10-09T05:40:00.755 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[184418]: pg 6.3 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,2] 2023-10-09T05:40:00.755 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[184418]: pg 6.5 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-10-09T05:40:00.755 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[184418]: pg 6.9 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [0,2] 2023-10-09T05:40:00.755 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[184418]: pg 6.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,1] 2023-10-09T05:40:00.756 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:40:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:40:00] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-09T05:40:00.756 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[191227]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 9091/123075 objects degraded (7.387%), 8 pgs degraded, 8 pgs undersized 2023-10-09T05:40:00.756 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[191227]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T05:40:00.756 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[191227]: [WRN] PG_DEGRADED: Degraded data redundancy: 9091/123075 objects degraded (7.387%), 8 pgs degraded, 8 pgs undersized 2023-10-09T05:40:00.756 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[191227]: pg 2.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-10-09T05:40:00.756 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[191227]: pg 2.3 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,2] 2023-10-09T05:40:00.756 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[191227]: pg 2.4 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-10-09T05:40:00.756 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[191227]: pg 2.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,2] 2023-10-09T05:40:00.756 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[191227]: pg 6.3 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,2] 2023-10-09T05:40:00.757 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[191227]: pg 6.5 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-10-09T05:40:00.757 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[191227]: pg 6.9 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [0,2] 2023-10-09T05:40:00.757 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:00 smithi008 ceph-mon[191227]: pg 6.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,1] 2023-10-09T05:40:00.923 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:00.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:00.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:01.031 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:00 smithi191 ceph-mon[161669]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 9091/123075 objects degraded (7.387%), 8 pgs degraded, 8 pgs undersized 2023-10-09T05:40:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:00 smithi191 ceph-mon[161669]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T05:40:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:00 smithi191 ceph-mon[161669]: [WRN] PG_DEGRADED: Degraded data redundancy: 9091/123075 objects degraded (7.387%), 8 pgs degraded, 8 pgs undersized 2023-10-09T05:40:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:00 smithi191 ceph-mon[161669]: pg 2.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-10-09T05:40:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:00 smithi191 ceph-mon[161669]: pg 2.3 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,2] 2023-10-09T05:40:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:00 smithi191 ceph-mon[161669]: pg 2.4 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-10-09T05:40:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:00 smithi191 ceph-mon[161669]: pg 2.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,2] 2023-10-09T05:40:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:00 smithi191 ceph-mon[161669]: pg 6.3 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,2] 2023-10-09T05:40:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:00 smithi191 ceph-mon[161669]: pg 6.5 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-10-09T05:40:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:00 smithi191 ceph-mon[161669]: pg 6.9 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [0,2] 2023-10-09T05:40:01.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:00 smithi191 ceph-mon[161669]: pg 6.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,1] 2023-10-09T05:40:01.058 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:01.111 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:01.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:01.217 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:01.252 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:01.308 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:40:01.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:01.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:01.418 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:01.447 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:01.499 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:01.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:01.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:01.607 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:01.686 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:01.727 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:01.755 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:01 smithi008 ceph-mon[184418]: pgmap v541: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 8887/118356 objects degraded (7.509%); 0 B/s, 40 objects/s recovering 2023-10-09T05:40:01.764 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:01 smithi008 ceph-mon[191227]: pgmap v541: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 8887/118356 objects degraded (7.509%); 0 B/s, 40 objects/s recovering 2023-10-09T05:40:01.838 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:01.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:01.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:01.878 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:40:01.878 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:40:01.878 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:40:01.878 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98118 5.5 5:a3d60b4a:::benchmark_data_smithi008_238708_object98117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.582344+0000", 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 361.29133944099999, 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30221345300000002, 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98118 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.879 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582344+0000", 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582344+0000", 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.880 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582342+0000", 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582367+0000", 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4702000000000002e-05 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582381+0000", 2023-10-09T05:40:01.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3407999999999999e-05 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582391+0000", 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9180000000000006e-06 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582482+0000", 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1482000000000003e-05 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.882 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582673+0000", 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0001906 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.582831+0000", 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000157904 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.879193+0000", 2023-10-09T05:40:01.883 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29636228799999997 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.884558+0000", 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0053650299999999998 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.34770.0:98118 5.5 e117/105)", 2023-10-09T05:40:01.884 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.586195+0000", 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 361.28748919499998, 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.292483255, 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.586195+0000", 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.885 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.586195+0000", 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.586194+0000", 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.586201+0000", 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0140000000000001e-06 2023-10-09T05:40:01.886 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.644224+0000", 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.058023650000000003 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.644240+0000", 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5147000000000001e-05 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.887 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.878662+0000", 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23442207500000001 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.878678+0000", 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6161e-05 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98118 5.5 5:a3d60b4a:::benchmark_data_smithi008_238708_object98117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:40:01.888 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.694337+0000", 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 361.17934725499998, 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.914715308, 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98118 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.889 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.890 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694337+0000", 2023-10-09T05:40:01.890 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.890 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.890 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.890 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.890 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694337+0000", 2023-10-09T05:40:01.890 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.890 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.890 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.890 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.890 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694334+0000", 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694371+0000", 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7106e-05 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694394+0000", 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3269e-05 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.891 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694410+0000", 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5454000000000001e-05 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.879229+0000", 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.184819179 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:01.892 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.879238+0000", 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6590000000000002e-06 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.608962+0000", 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72972381399999997 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.609052+0000", 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9624999999999997e-05 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.893 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98137 5.3 5:c7026034:::benchmark_data_smithi008_238708_object98136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.815708+0000", 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 361.05797625600002, 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.811726473, 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:01.894 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98137 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815708+0000", 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815708+0000", 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.895 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815705+0000", 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815742+0000", 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6514999999999999e-05 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.896 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815763+0000", 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1107999999999998e-05 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815778+0000", 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4718999999999999e-05 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.848379+0000", 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.032600899000000003 2023-10-09T05:40:01.897 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.898 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.898 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:01.898 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.848390+0000", 2023-10-09T05:40:01.898 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1735e-05 2023-10-09T05:40:01.898 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.898 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.898 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.898 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.619939+0000", 2023-10-09T05:40:01.898 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77154830200000002 2023-10-09T05:40:01.898 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.898 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.620114+0000", 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017572099999999999 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.620276+0000", 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016124600000000001 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:01.899 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622357+0000", 2023-10-09T05:40:01.900 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020815999999999999 2023-10-09T05:40:01.900 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.900 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.900 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.900 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.625946+0000", 2023-10-09T05:40:01.900 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035886339999999998 2023-10-09T05:40:01.900 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.900 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.900 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.900 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627349+0000", 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014031130000000001 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627400+0000", 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1385999999999999e-05 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627434+0000", 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3640999999999999e-05 2023-10-09T05:40:01.901 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4050 6.0 6:0be16091:::benchmark_data_smithi008_295510_object4049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.480314+0000", 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 173.39336983800001, 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48189201799999998, 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.902 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4050 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480314+0000", 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480314+0000", 2023-10-09T05:40:01.903 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480313+0000", 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480347+0000", 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3819000000000003e-05 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.904 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480359+0000", 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2092999999999999e-05 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480374+0000", 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5096000000000001e-05 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480421+0000", 2023-10-09T05:40:01.905 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7611999999999997e-05 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480552+0000", 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013091099999999999 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480687+0000", 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013453799999999999 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.906 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959945+0000", 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47925849799999998 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959955+0000", 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8859999999999999e-06 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.907 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962107+0000", 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002151975 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962152+0000", 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5049999999999997e-05 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962206+0000", 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3773999999999997e-05 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.908 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45029.0:4048 6.a e166/162 6:51c21166:::benchmark_data_smithi008_295510_object4047:head v 166'256, mlcod=166'251)", 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.481202+0000", 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 173.39248218099999, 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47877170299999999, 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.909 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481202+0000", 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481202+0000", 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481201+0000", 2023-10-09T05:40:01.910 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481234+0000", 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2854000000000001e-05 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481243+0000", 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7019999999999996e-06 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.911 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481249+0000", 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7849999999999997e-06 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481289+0000", 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0154000000000002e-05 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.912 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481309+0000", 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9177e-05 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959928+0000", 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47861928100000001 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959973+0000", 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5482000000000003e-05 2023-10-09T05:40:01.913 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4053 6.9 6:9d44e78a:::benchmark_data_smithi008_295510_object4052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.481281+0000", 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 173.39240287199999, 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48114514400000002, 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:01.914 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4053 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481281+0000", 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481281+0000", 2023-10-09T05:40:01.915 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481280+0000", 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481303+0000", 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3283999999999999e-05 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.916 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481314+0000", 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0095000000000001e-05 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481322+0000", 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5339999999999999e-06 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481583+0000", 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00026103399999999999 2023-10-09T05:40:01.917 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481690+0000", 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000107299 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481837+0000", 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000146656 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.918 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.961311+0000", 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.479473555 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962305+0000", 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00099412799999999994 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962340+0000", 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4835000000000001e-05 2023-10-09T05:40:01.919 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962386+0000", 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6851000000000003e-05 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962426+0000", 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9617999999999999e-05 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.920 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4054 6.0 6:09150d51:::benchmark_data_smithi008_295510_object4053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.481336+0000", 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 173.39234741800001, 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.481099946, 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4054 2023-10-09T05:40:01.921 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481336+0000", 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481336+0000", 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.922 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481336+0000", 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481360+0000", 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3734e-05 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481368+0000", 2023-10-09T05:40:01.923 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2109999999999998e-06 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481374+0000", 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6189999999999996e-06 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481425+0000", 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0209e-05 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.924 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481568+0000", 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014368900000000001 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.481679+0000", 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011092299999999999 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.961347+0000", 2023-10-09T05:40:01.925 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.479668066 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962257+0000", 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00090950199999999997 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962350+0000", 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3634000000000005e-05 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.926 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962398+0000", 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7218000000000001e-05 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962436+0000", 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8676e-05 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.927 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4060 6.0 6:04bff711:::benchmark_data_smithi008_295510_object4059:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.482112+0000", 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 173.391572311, 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48355848200000001, 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4060 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.928 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482112+0000", 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482112+0000", 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.929 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482111+0000", 2023-10-09T05:40:01.930 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:01.930 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.930 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.930 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.930 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482141+0000", 2023-10-09T05:40:01.930 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0737000000000001e-05 2023-10-09T05:40:01.930 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.930 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.930 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.930 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959856+0000", 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47771430100000001 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959871+0000", 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5611999999999999e-05 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959984+0000", 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000112897 2023-10-09T05:40:01.931 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.960209+0000", 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00022461200000000001 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.960939+0000", 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000730622 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.932 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.963896+0000", 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029571720000000001 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965560+0000", 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016638639999999999 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965601+0000", 2023-10-09T05:40:01.933 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0173999999999999e-05 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965632+0000", 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1591000000000003e-05 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965670+0000", 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7867000000000002e-05 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.934 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45029.0:4050 6.0 e166/162 ondisk, result = 0)", 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.482434+0000", 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 173.39125004900001, 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47752318500000002, 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.935 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482434+0000", 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482434+0000", 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482433+0000", 2023-10-09T05:40:01.936 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482437+0000", 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8990000000000003e-06 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959833+0000", 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47739634600000003 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.937 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959852+0000", 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8614000000000002e-05 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959925+0000", 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3526e-05 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.938 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959938+0000", 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2653999999999999e-05 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959957+0000", 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8861000000000001e-05 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.939 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:36511 6.0 6:0392d0f0:::benchmark_data_smithi008_295510_object36510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.167968+0000", 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 154.70571563300001, 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24803032899999999, 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 36511 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.940 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.167968+0000", 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.167968+0000", 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.167967+0000", 2023-10-09T05:40:01.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:01.942 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.942 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.942 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.942 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.167992+0000", 2023-10-09T05:40:01.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5562999999999999e-05 2023-10-09T05:40:01.942 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.942 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.942 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.942 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168005+0000", 2023-10-09T05:40:01.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.271e-05 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168021+0000", 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6069000000000001e-05 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168084+0000", 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3437999999999998e-05 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.943 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168231+0000", 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014663799999999999 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168386+0000", 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015447900000000001 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:01.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171807+0000", 2023-10-09T05:40:01.945 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034214509999999998 2023-10-09T05:40:01.945 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.945 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.945 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.945 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415850+0000", 2023-10-09T05:40:01.945 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.244042711 2023-10-09T05:40:01.945 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.945 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.945 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.945 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415903+0000", 2023-10-09T05:40:01.945 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3238000000000001e-05 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415954+0000", 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0621999999999997e-05 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415999+0000", 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4891000000000001e-05 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.946 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:36518 6.0 6:0f46dfdd:::benchmark_data_smithi008_295510_object36517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.168459+0000", 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 154.70522513099999, 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.25353774899999998, 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:01.947 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 36518 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168459+0000", 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168459+0000", 2023-10-09T05:40:01.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168458+0000", 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168476+0000", 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7969999999999999e-05 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.949 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168501+0000", 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.565e-05 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.168526+0000", 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4383e-05 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.950 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.170220+0000", 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016944589999999999 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.170368+0000", 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000147241 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.170497+0000", 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012980199999999999 2023-10-09T05:40:01.951 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.416044+0000", 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.245546878 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.416108+0000", 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3987999999999998e-05 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.952 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.421919+0000", 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058104669999999997 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.421955+0000", 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6318e-05 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.421996+0000", 2023-10-09T05:40:01.953 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1459999999999999e-05 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:36520 6.0 6:05b021a1:::benchmark_data_smithi008_295510_object36519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e167)", 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.171824+0000", 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 154.70185997600001, 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.250244829, 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.954 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 36520 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171824+0000", 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.955 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171824+0000", 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171822+0000", 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171846+0000", 2023-10-09T05:40:01.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3102e-05 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171857+0000", 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1812e-05 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171866+0000", 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1599999999999998e-06 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.957 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171908+0000", 2023-10-09T05:40:01.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2536000000000001e-05 2023-10-09T05:40:01.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172043+0000", 2023-10-09T05:40:01.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013473099999999999 2023-10-09T05:40:01.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:40:01.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172184+0000", 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000141395 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.416077+0000", 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24389317199999999 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.416142+0000", 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4652000000000004e-05 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.422004+0000", 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058622209999999999 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.422031+0000", 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7056999999999999e-05 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.422069+0000", 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7394999999999997e-05 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45029.0:36511 6.0 e167/162 ondisk, result = 0)", 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.171991+0000", 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 154.701693068, 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24386655300000001, 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.961 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171991+0000", 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171991+0000", 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.962 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171990+0000", 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.171994+0000", 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.053e-06 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.963 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172002+0000", 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5390000000000002e-06 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172005+0000", 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8020000000000002e-06 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415835+0000", 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24382936399999999 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.964 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415843+0000", 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4100000000000008e-06 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415857+0000", 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4168999999999999e-05 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.965 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.966 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.966 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45029.0:36519 6.a e167/162 6:58c94098:::benchmark_data_smithi008_295510_object36518:head v 167'2253, mlcod=167'2250)", 2023-10-09T05:40:01.966 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.172304+0000", 2023-10-09T05:40:01.966 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 154.701379511, 2023-10-09T05:40:01.966 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24684500100000001, 2023-10-09T05:40:01.966 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.966 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.966 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.966 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.966 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.966 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172304+0000", 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172304+0000", 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172304+0000", 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:01.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.968 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.968 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.172325+0000", 2023-10-09T05:40:01.968 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1075999999999999e-05 2023-10-09T05:40:01.968 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.968 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.968 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414426+0000", 2023-10-09T05:40:01.968 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24210095000000001 2023-10-09T05:40:01.968 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.969 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.969 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414442+0000", 2023-10-09T05:40:01.969 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6439999999999998e-05 2023-10-09T05:40:01.969 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.969 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.969 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.969 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414499+0000", 2023-10-09T05:40:01.969 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7275000000000003e-05 2023-10-09T05:40:01.969 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.969 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.969 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.970 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414528+0000", 2023-10-09T05:40:01.970 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8982000000000001e-05 2023-10-09T05:40:01.970 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.970 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.970 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.970 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.419127+0000", 2023-10-09T05:40:01.970 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045990479999999997 2023-10-09T05:40:01.970 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.970 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.419149+0000", 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1875e-05 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45029.0:36511 6.0 e167/162 ondisk, result = 0)", 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:27.173714+0000", 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 154.699970017, 2023-10-09T05:40:01.971 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24229606300000001, 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.173714+0000", 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.173714+0000", 2023-10-09T05:40:01.972 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.173713+0000", 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.173719+0000", 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0260000000000004e-06 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414428+0000", 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24070894100000001 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.414442+0000", 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4039999999999999e-05 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415890+0000", 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014484159999999999 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.415898+0000", 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4569999999999999e-06 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:27.416010+0000", 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011202 2023-10-09T05:40:01.975 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:54699 6.0 6:0d2a44bc:::benchmark_data_smithi008_295510_object54698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:49.768820+0000", 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.104863435, 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33636735600000001, 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.976 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54699 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.768820+0000", 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.977 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.768820+0000", 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.768819+0000", 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.768843+0000", 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3954000000000001e-05 2023-10-09T05:40:01.978 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.768856+0000", 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2777e-05 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.768864+0000", 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9820000000000008e-06 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.979 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.768912+0000", 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8760000000000001e-05 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.769065+0000", 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015226399999999999 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.769235+0000", 2023-10-09T05:40:01.980 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016987799999999999 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.771611+0000", 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023765169999999999 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.775677+0000", 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040656349999999997 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.981 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.105089+0000", 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.32941204299999999 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.105138+0000", 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9659e-05 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.105188+0000", 2023-10-09T05:40:01.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9378000000000001e-05 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:54714 6.0 6:0023c3ab:::benchmark_data_smithi008_295510_object54713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:49.783747+0000", 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.08993663800001, 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.32153881600000001, 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.983 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54714 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.783747+0000", 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.984 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.783747+0000", 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.783746+0000", 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.783771+0000", 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5295000000000001e-05 2023-10-09T05:40:01.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.783784+0000", 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3081000000000001e-05 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.783792+0000", 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.388e-06 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.986 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.783846+0000", 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3372000000000003e-05 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.783995+0000", 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000149598 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.784202+0000", 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020637800000000001 2023-10-09T05:40:01.987 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.786789+0000", 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025875239999999999 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:49.788104+0000", 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013143300000000001 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.105221+0000", 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.31711729399999999 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.105257+0000", 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5735999999999998e-05 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:50.105286+0000", 2023-10-09T05:40:01.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9349e-05 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:72555 6.9 6:93d42958:::benchmark_data_smithi008_295510_object72554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:13.303558+0000", 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 108.570125604, 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23518783500000001, 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.990 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 72555 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.303558+0000", 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.991 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.303558+0000", 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.303558+0000", 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.303579+0000", 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1624000000000002e-05 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.992 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.303588+0000", 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2609999999999993e-06 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.303594+0000", 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2899999999999999e-06 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:01.993 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.303643+0000", 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8980999999999997e-05 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.303785+0000", 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014263000000000001 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.303943+0000", 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000157028 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.994 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.307332+0000", 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033893830000000002 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.309134+0000", 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018023970000000001 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:01.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.536937+0000", 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.227802849 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.536987+0000", 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0203e-05 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.538746+0000", 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001758704 2023-10-09T05:40:01.996 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:129629 6.0 6:03ac2bbc:::benchmark_data_smithi008_295510_object41266:head [delete] snapc 0=[] ondisk+write+known_if_redirected e193)", 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:39:44.508355+0000", 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 17.365329263, 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.25118993699999997, 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:01.997 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 129629 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.508355+0000", 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.508355+0000", 2023-10-09T05:40:01.998 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.508354+0000", 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.508358+0000", 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9339999999999999e-06 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:01.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.508366+0000", 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7700000000000001e-06 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.508371+0000", 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.536e-06 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.512782+0000", 2023-10-09T05:40:02.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0044108050000000003 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.512896+0000", 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114072 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.675382+0000", 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.16248612900000001 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:02.001 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.687364+0000", 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011981769999999999 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.687432+0000", 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8458000000000001e-05 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:02.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.759458+0000", 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.072025275 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.759505+0000", 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7447999999999997e-05 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:39:44.759544+0000", 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9347000000000001e-05 2023-10-09T05:40:02.003 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:02.004 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:02.004 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:02.004 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:02.004 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:02.004 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:40:02.007 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:40:02.009 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:40:02.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:02.038 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:01 smithi191 ceph-mon[161669]: pgmap v541: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 8887/118356 objects degraded (7.509%); 0 B/s, 40 objects/s recovering 2023-10-09T05:40:02.040 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:02.093 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:02.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:02.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:02.203 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:02.273 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:02.639 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:40:03.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:02 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 8887/118356 objects degraded (7.509%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:03.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:02 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 8887/118356 objects degraded (7.509%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:03.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:02 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 8887/118356 objects degraded (7.509%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:03.500 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:03.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:03.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:03.588 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":541,"stamp":"2023-10-09T05:40:00.093425+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2588415947,"num_objects":39452,"num_object_clones":0,"num_object_copies":118356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8887,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39452,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":138404,"num_write_kb":5670182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8654,"num_bytes_recovered":531043702,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125168,"ondisk_log_size":125168,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9818160,"kb_used_data":7698552,"kb_used_omap":31,"kb_used_meta":2119328,"kb_avail":646397904,"statfs":{"total":671965249536,"available":661911453696,"internally_reserved":0,"allocated":7883317248,"data_stored":7878779882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2170192329},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-376176640,"num_objects":-5740,"num_object_clones":0,"num_object_copies":-17220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-403,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5740,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005111"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":177,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.815331+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:39:42.815331+0000","last_peered":"2023-10-09T05:39:42.815331+0000","last_clean":"2023-10-09T05:39:42.815331+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:39:42.815331+0000","last_undegraded":"2023-10-09T05:39:42.815331+0000","last_fullsized":"2023-10-09T05:39:42.815331+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":166,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.816296+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:39:42.816296+0000","last_peered":"2023-10-09T05:39:42.816296+0000","last_clean":"2023-10-09T05:39:42.816296+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:39:42.816296+0000","last_undegraded":"2023-10-09T05:39:42.816296+0000","last_fullsized":"2023-10-09T05:39:42.816296+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":140,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.792832+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:39:42.792832+0000","last_peered":"2023-10-09T05:39:42.792832+0000","last_clean":"2023-10-09T05:39:42.792832+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:39:42.792832+0000","last_undegraded":"2023-10-09T05:39:42.792832+0000","last_fullsized":"2023-10-09T05:39:42.792832+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":171,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:55.814026+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:39:55.814026+0000","last_peered":"2023-10-09T05:39:55.814026+0000","last_clean":"2023-10-09T05:39:55.814026+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:39:55.814026+0000","last_undegraded":"2023-10-09T05:39:55.814026+0000","last_fullsized":"2023-10-09T05:39:55.814026+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":182,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.823277+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:39:42.823277+0000","last_peered":"2023-10-09T05:39:42.823277+0000","last_clean":"2023-10-09T05:39:42.823277+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:39:42.823277+0000","last_undegraded":"2023-10-09T05:39:42.823277+0000","last_fullsized":"2023-10-09T05:39:42.823277+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":159,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:32.543794+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:39:32.543794+0000","last_peered":"2023-10-09T05:39:32.543794+0000","last_clean":"2023-10-09T05:39:32.543794+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:39:32.543794+0000","last_undegraded":"2023-10-09T05:39:32.543794+0000","last_fullsized":"2023-10-09T05:39:32.543794+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":146,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.809487+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:39:42.809487+0000","last_peered":"2023-10-09T05:39:42.809487+0000","last_clean":"2023-10-09T05:39:42.809487+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:39:42.809487+0000","last_undegraded":"2023-10-09T05:39:42.809487+0000","last_fullsized":"2023-10-09T05:39:42.809487+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":166,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.823332+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:39:42.823332+0000","last_peered":"2023-10-09T05:39:42.823332+0000","last_clean":"2023-10-09T05:39:42.823332+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:39:42.823332+0000","last_undegraded":"2023-10-09T05:39:42.823332+0000","last_fullsized":"2023-10-09T05:39:42.823332+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":185,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.798409+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:39:42.798409+0000","last_peered":"2023-10-09T05:39:42.798409+0000","last_clean":"2023-10-09T05:39:42.798409+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:39:42.798409+0000","last_undegraded":"2023-10-09T05:39:42.798409+0000","last_fullsized":"2023-10-09T05:39:42.798409+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":180,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.821088+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:39:42.821088+0000","last_peered":"2023-10-09T05:39:42.821088+0000","last_clean":"2023-10-09T05:39:42.821088+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:39:42.821088+0000","last_undegraded":"2023-10-09T05:39:42.821088+0000","last_fullsized":"2023-10-09T05:39:42.821088+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":73,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.816210+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:39:42.816210+0000","last_peered":"2023-10-09T05:39:42.816210+0000","last_clean":"2023-10-09T05:39:42.816210+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:39:42.816210+0000","last_undegraded":"2023-10-09T05:39:42.816210+0000","last_fullsized":"2023-10-09T05:39:42.816210+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":141,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:48.817913+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:39:48.817913+0000","last_peered":"2023-10-09T05:39:48.817913+0000","last_clean":"2023-10-09T05:39:48.817913+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:39:48.817913+0000","last_undegraded":"2023-10-09T05:39:48.817913+0000","last_fullsized":"2023-10-09T05:39:48.817913+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":202,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.814695+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:39:42.814695+0000","last_peered":"2023-10-09T05:39:42.814695+0000","last_clean":"2023-10-09T05:39:42.814695+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:39:42.814695+0000","last_undegraded":"2023-10-09T05:39:42.814695+0000","last_fullsized":"2023-10-09T05:39:42.814695+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":160,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.814196+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:39:42.814196+0000","last_peered":"2023-10-09T05:39:42.814196+0000","last_clean":"2023-10-09T05:39:42.814196+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:39:42.814196+0000","last_undegraded":"2023-10-09T05:39:42.814196+0000","last_fullsized":"2023-10-09T05:39:42.814196+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":73,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.814277+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:39:42.814277+0000","last_peered":"2023-10-09T05:39:42.814277+0000","last_clean":"2023-10-09T05:39:42.814277+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:39:42.814277+0000","last_undegraded":"2023-10-09T05:39:42.814277+0000","last_fullsized":"2023-10-09T05:39:42.814277+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":190,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.816550+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:39:42.816550+0000","last_peered":"2023-10-09T05:39:42.816550+0000","last_clean":"2023-10-09T05:39:42.816550+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:39:42.816550+0000","last_undegraded":"2023-10-09T05:39:42.816550+0000","last_fullsized":"2023-10-09T05:39:42.816550+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":170,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.808706+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:39:42.808706+0000","last_peered":"2023-10-09T05:39:42.808706+0000","last_clean":"2023-10-09T05:39:42.808706+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:39:42.808706+0000","last_undegraded":"2023-10-09T05:39:42.808706+0000","last_fullsized":"2023-10-09T05:39:42.808706+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":184,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.798415+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:39:42.798415+0000","last_peered":"2023-10-09T05:39:42.798415+0000","last_clean":"2023-10-09T05:39:42.798415+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:39:42.798415+0000","last_undegraded":"2023-10-09T05:39:42.798415+0000","last_fullsized":"2023-10-09T05:39:42.798415+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":91,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.813972+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:39:42.813972+0000","last_peered":"2023-10-09T05:39:42.813972+0000","last_clean":"2023-10-09T05:39:42.813972+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:39:42.813972+0000","last_undegraded":"2023-10-09T05:39:42.813972+0000","last_fullsized":"2023-10-09T05:39:42.813972+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":170,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.808742+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:39:42.808742+0000","last_peered":"2023-10-09T05:39:42.808742+0000","last_clean":"2023-10-09T05:39:42.808742+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:39:42.808742+0000","last_undegraded":"2023-10-09T05:39:42.808742+0000","last_fullsized":"2023-10-09T05:39:42.808742+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":168,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:32.543350+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:39:32.543350+0000","last_peered":"2023-10-09T05:39:32.543350+0000","last_clean":"2023-10-09T05:39:32.543350+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:39:32.543350+0000","last_undegraded":"2023-10-09T05:39:32.543350+0000","last_fullsized":"2023-10-09T05:39:32.543350+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":190,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.820973+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:39:42.820973+0000","last_peered":"2023-10-09T05:39:42.820973+0000","last_clean":"2023-10-09T05:39:42.820973+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:39:42.820973+0000","last_undegraded":"2023-10-09T05:39:42.820973+0000","last_fullsized":"2023-10-09T05:39:42.820973+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"194'8657","reported_seq":8697,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:59.894553+0000","last_change":"2023-10-09T05:37:01.636984+0000","last_active":"2023-10-09T05:39:59.894553+0000","last_peered":"2023-10-09T05:39:59.894553+0000","last_clean":"2023-10-09T05:39:59.894553+0000","last_became_active":"2023-10-09T05:37:01.636609+0000","last_became_peered":"2023-10-09T05:37:01.636609+0000","last_unstale":"2023-10-09T05:39:59.894553+0000","last_undegraded":"2023-10-09T05:39:59.894553+0000","last_fullsized":"2023-10-09T05:39:59.894553+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8657,"log_dups_size":0,"ondisk_log_size":8657,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157220864,"num_objects":2399,"num_object_clones":0,"num_object_copies":7197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8657,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":71,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.816754+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:39:42.816754+0000","last_peered":"2023-10-09T05:39:42.816754+0000","last_clean":"2023-10-09T05:39:42.816754+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:39:42.816754+0000","last_undegraded":"2023-10-09T05:39:42.816754+0000","last_fullsized":"2023-10-09T05:39:42.816754+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":169,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.797776+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:39:42.797776+0000","last_peered":"2023-10-09T05:39:42.797776+0000","last_clean":"2023-10-09T05:39:42.797776+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:39:42.797776+0000","last_undegraded":"2023-10-09T05:39:42.797776+0000","last_fullsized":"2023-10-09T05:39:42.797776+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"194'8469","reported_seq":8518,"reported_epoch":194,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:56.200553+0000","last_change":"2023-10-09T05:37:36.837344+0000","last_active":"2023-10-09T05:39:56.200553+0000","last_peered":"2023-10-09T05:39:56.200553+0000","last_clean":"2023-10-09T05:37:34.651883+0000","last_became_active":"2023-10-09T05:37:36.832502+0000","last_became_peered":"2023-10-09T05:37:36.832502+0000","last_unstale":"2023-10-09T05:39:56.200553+0000","last_undegraded":"2023-10-09T05:37:36.702797+0000","last_fullsized":"2023-10-09T05:37:36.678583+0000","mapping_epoch":170,"log_start":"167'2000","ondisk_log_start":"167'2000","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6469,"log_dups_size":0,"ondisk_log_size":6469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:57.419460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165347328,"num_objects":2523,"num_object_clones":0,"num_object_copies":7569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8469,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":-367}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":71,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.814262+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:39:42.814262+0000","last_peered":"2023-10-09T05:39:42.814262+0000","last_clean":"2023-10-09T05:39:42.814262+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:39:42.814262+0000","last_undegraded":"2023-10-09T05:39:42.814262+0000","last_fullsized":"2023-10-09T05:39:42.814262+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":175,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.813902+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:39:42.813902+0000","last_peered":"2023-10-09T05:39:42.813902+0000","last_clean":"2023-10-09T05:39:42.813902+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:39:42.813902+0000","last_undegraded":"2023-10-09T05:39:42.813902+0000","last_fullsized":"2023-10-09T05:39:42.813902+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"194'8741","reported_seq":14843,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:56.438180+0000","last_change":"2023-10-09T05:38:55.317808+0000","last_active":"2023-10-09T05:39:56.438180+0000","last_peered":"2023-10-09T05:39:56.438180+0000","last_clean":"2023-10-09T05:39:56.438180+0000","last_became_active":"2023-10-09T05:38:55.302739+0000","last_became_peered":"2023-10-09T05:38:55.302739+0000","last_unstale":"2023-10-09T05:39:56.438180+0000","last_undegraded":"2023-10-09T05:39:56.438180+0000","last_fullsized":"2023-10-09T05:39:56.438180+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8741,"log_dups_size":0,"ondisk_log_size":8741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:25.379818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161939456,"num_objects":2471,"num_object_clones":0,"num_object_copies":7413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8741,"num_write_kb":358784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":198049792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":71,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.798435+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:39:42.798435+0000","last_peered":"2023-10-09T05:39:42.798435+0000","last_clean":"2023-10-09T05:39:42.798435+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:39:42.798435+0000","last_undegraded":"2023-10-09T05:39:42.798435+0000","last_fullsized":"2023-10-09T05:39:42.798435+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":192,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:32.543646+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:39:32.543646+0000","last_peered":"2023-10-09T05:39:32.543646+0000","last_clean":"2023-10-09T05:39:32.543646+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:39:32.543646+0000","last_undegraded":"2023-10-09T05:39:32.543646+0000","last_fullsized":"2023-10-09T05:39:32.543646+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"194'8536","reported_seq":13911,"reported_epoch":194,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:58.110029+0000","last_change":"2023-10-09T05:38:53.467497+0000","last_active":"2023-10-09T05:39:58.110029+0000","last_peered":"2023-10-09T05:39:58.110029+0000","last_clean":"2023-10-09T05:37:34.677467+0000","last_became_active":"2023-10-09T05:37:36.803958+0000","last_became_peered":"2023-10-09T05:37:36.803958+0000","last_unstale":"2023-10-09T05:39:58.110029+0000","last_undegraded":"2023-10-09T05:37:36.800519+0000","last_fullsized":"2023-10-09T05:37:36.678690+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8536,"log_dups_size":0,"ondisk_log_size":8536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:40.690185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159514624,"num_objects":2434,"num_object_clones":0,"num_object_copies":7302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":237,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8536,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2662,"num_bytes_recovered":139919360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":2197}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":161,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.799537+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:39:42.799537+0000","last_peered":"2023-10-09T05:39:42.799537+0000","last_clean":"2023-10-09T05:39:42.799537+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:39:42.799537+0000","last_undegraded":"2023-10-09T05:39:42.799537+0000","last_fullsized":"2023-10-09T05:39:42.799537+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":146,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.804810+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:39:42.804810+0000","last_peered":"2023-10-09T05:39:42.804810+0000","last_clean":"2023-10-09T05:39:42.804810+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:39:42.804810+0000","last_undegraded":"2023-10-09T05:39:42.804810+0000","last_fullsized":"2023-10-09T05:39:42.804810+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"194'8614","reported_seq":8654,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:59.918930+0000","last_change":"2023-10-09T05:37:01.439878+0000","last_active":"2023-10-09T05:39:59.918930+0000","last_peered":"2023-10-09T05:39:59.918930+0000","last_clean":"2023-10-09T05:39:59.918930+0000","last_became_active":"2023-10-09T05:37:01.439481+0000","last_became_peered":"2023-10-09T05:37:01.439481+0000","last_unstale":"2023-10-09T05:39:59.918930+0000","last_undegraded":"2023-10-09T05:39:59.918930+0000","last_fullsized":"2023-10-09T05:39:59.918930+0000","mapping_epoch":162,"log_start":"166'1100","ondisk_log_start":"166'1100","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7514,"log_dups_size":0,"ondisk_log_size":7514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155451392,"num_objects":2372,"num_object_clones":0,"num_object_copies":7116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8614,"num_write_kb":351552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":164,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.816669+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:39:42.816669+0000","last_peered":"2023-10-09T05:39:42.816669+0000","last_clean":"2023-10-09T05:39:42.816669+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:39:42.816669+0000","last_undegraded":"2023-10-09T05:39:42.816669+0000","last_fullsized":"2023-10-09T05:39:42.816669+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":160,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.814616+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:39:42.814616+0000","last_peered":"2023-10-09T05:39:42.814616+0000","last_clean":"2023-10-09T05:39:42.814616+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:39:42.814616+0000","last_undegraded":"2023-10-09T05:39:42.814616+0000","last_fullsized":"2023-10-09T05:39:42.814616+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"194'8513","reported_seq":8555,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:56.442638+0000","last_change":"2023-10-09T05:37:01.440175+0000","last_active":"2023-10-09T05:39:56.442638+0000","last_peered":"2023-10-09T05:39:56.442638+0000","last_clean":"2023-10-09T05:39:56.442638+0000","last_became_active":"2023-10-09T05:37:01.439846+0000","last_became_peered":"2023-10-09T05:37:01.439846+0000","last_unstale":"2023-10-09T05:39:56.442638+0000","last_undegraded":"2023-10-09T05:39:56.442638+0000","last_fullsized":"2023-10-09T05:39:56.442638+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8513,"log_dups_size":0,"ondisk_log_size":8513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:27.883265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166526976,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8513,"num_write_kb":353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":160,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:32.543575+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:39:32.543575+0000","last_peered":"2023-10-09T05:39:32.543575+0000","last_clean":"2023-10-09T05:39:32.543575+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:39:32.543575+0000","last_undegraded":"2023-10-09T05:39:32.543575+0000","last_fullsized":"2023-10-09T05:39:32.543575+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":175,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.814849+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:39:42.814849+0000","last_peered":"2023-10-09T05:39:42.814849+0000","last_clean":"2023-10-09T05:39:42.814849+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:39:42.814849+0000","last_undegraded":"2023-10-09T05:39:42.814849+0000","last_fullsized":"2023-10-09T05:39:42.814849+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"194'8547","reported_seq":8589,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:56.117133+0000","last_change":"2023-10-09T05:37:01.638178+0000","last_active":"2023-10-09T05:39:56.117133+0000","last_peered":"2023-10-09T05:39:56.117133+0000","last_clean":"2023-10-09T05:39:56.117133+0000","last_became_active":"2023-10-09T05:37:01.637822+0000","last_became_peered":"2023-10-09T05:37:01.637822+0000","last_unstale":"2023-10-09T05:39:56.117133+0000","last_undegraded":"2023-10-09T05:39:56.117133+0000","last_fullsized":"2023-10-09T05:39:56.117133+0000","mapping_epoch":162,"log_start":"167'2100","ondisk_log_start":"167'2100","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6447,"log_dups_size":0,"ondisk_log_size":6447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:13.240069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168230912,"num_objects":2567,"num_object_clones":0,"num_object_copies":7701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8547,"num_write_kb":355648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":185,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.813708+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:39:42.813708+0000","last_peered":"2023-10-09T05:39:42.813708+0000","last_clean":"2023-10-09T05:39:42.813708+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:39:42.813708+0000","last_undegraded":"2023-10-09T05:39:42.813708+0000","last_fullsized":"2023-10-09T05:39:42.813708+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":169,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.814337+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:39:42.814337+0000","last_peered":"2023-10-09T05:39:42.814337+0000","last_clean":"2023-10-09T05:39:42.814337+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:39:42.814337+0000","last_undegraded":"2023-10-09T05:39:42.814337+0000","last_fullsized":"2023-10-09T05:39:42.814337+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"194'8656","reported_seq":8697,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:59.614202+0000","last_change":"2023-10-09T05:37:01.438917+0000","last_active":"2023-10-09T05:39:59.614202+0000","last_peered":"2023-10-09T05:39:59.614202+0000","last_clean":"2023-10-09T05:39:59.614202+0000","last_became_active":"2023-10-09T05:37:01.438658+0000","last_became_peered":"2023-10-09T05:37:01.438658+0000","last_unstale":"2023-10-09T05:39:59.614202+0000","last_undegraded":"2023-10-09T05:39:59.614202+0000","last_fullsized":"2023-10-09T05:39:59.614202+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8656,"log_dups_size":0,"ondisk_log_size":8656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157614104,"num_objects":2406,"num_object_clones":0,"num_object_copies":7218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2406,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8656,"num_write_kb":353921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":191,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.823256+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:39:42.823256+0000","last_peered":"2023-10-09T05:39:42.823256+0000","last_clean":"2023-10-09T05:39:42.823256+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:39:42.823256+0000","last_undegraded":"2023-10-09T05:39:42.823256+0000","last_fullsized":"2023-10-09T05:39:42.823256+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":141,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.790596+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:39:42.790596+0000","last_peered":"2023-10-09T05:39:42.790596+0000","last_clean":"2023-10-09T05:39:42.790596+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:39:42.790596+0000","last_undegraded":"2023-10-09T05:39:42.790596+0000","last_fullsized":"2023-10-09T05:39:42.790596+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"194'8653","reported_seq":8695,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:58.053585+0000","last_change":"2023-10-09T05:37:01.440093+0000","last_active":"2023-10-09T05:39:58.053585+0000","last_peered":"2023-10-09T05:39:58.053585+0000","last_clean":"2023-10-09T05:39:58.053585+0000","last_became_active":"2023-10-09T05:37:01.439799+0000","last_became_peered":"2023-10-09T05:37:01.439799+0000","last_unstale":"2023-10-09T05:39:58.053585+0000","last_undegraded":"2023-10-09T05:39:58.053585+0000","last_fullsized":"2023-10-09T05:39:58.053585+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8653,"log_dups_size":0,"ondisk_log_size":8653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:28.734471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160759808,"num_objects":2453,"num_object_clones":0,"num_object_copies":7359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8653,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":728,"reported_epoch":193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:42.791316+0000","last_change":"2023-10-09T05:37:36.736822+0000","last_active":"2023-10-09T05:39:42.791316+0000","last_peered":"2023-10-09T05:39:42.791316+0000","last_clean":"2023-10-09T05:37:30.792074+0000","last_became_active":"2023-10-09T05:37:36.736273+0000","last_became_peered":"2023-10-09T05:37:36.736273+0000","last_unstale":"2023-10-09T05:39:42.791316+0000","last_undegraded":"2023-10-09T05:37:36.682657+0000","last_fullsized":"2023-10-09T05:37:36.682189+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:23.350354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":176,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.798347+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:39:42.798347+0000","last_peered":"2023-10-09T05:39:42.798347+0000","last_clean":"2023-10-09T05:39:42.798347+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:39:42.798347+0000","last_undegraded":"2023-10-09T05:39:42.798347+0000","last_fullsized":"2023-10-09T05:39:42.798347+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":126,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.791021+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:39:42.791021+0000","last_peered":"2023-10-09T05:39:42.791021+0000","last_clean":"2023-10-09T05:39:42.791021+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:39:42.791021+0000","last_undegraded":"2023-10-09T05:39:42.791021+0000","last_fullsized":"2023-10-09T05:39:42.791021+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"194'8582","reported_seq":14514,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:56.131502+0000","last_change":"2023-10-09T05:38:10.132330+0000","last_active":"2023-10-09T05:39:56.131502+0000","last_peered":"2023-10-09T05:39:56.131502+0000","last_clean":"2023-10-09T05:39:56.131502+0000","last_became_active":"2023-10-09T05:38:10.127080+0000","last_became_peered":"2023-10-09T05:38:10.127080+0000","last_unstale":"2023-10-09T05:39:56.131502+0000","last_undegraded":"2023-10-09T05:39:56.131502+0000","last_fullsized":"2023-10-09T05:39:56.131502+0000","mapping_epoch":179,"log_start":"167'2100","ondisk_log_start":"167'2100","created":162,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6482,"log_dups_size":0,"ondisk_log_size":6482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:12.659227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168165376,"num_objects":2566,"num_object_clones":0,"num_object_copies":7698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8582,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":192413696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":630,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.804072+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:39:42.804072+0000","last_peered":"2023-10-09T05:39:42.804072+0000","last_clean":"2023-10-09T05:39:42.804072+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:39:42.804072+0000","last_undegraded":"2023-10-09T05:39:42.804072+0000","last_fullsized":"2023-10-09T05:39:42.804072+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":161,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.814349+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:39:42.814349+0000","last_peered":"2023-10-09T05:39:42.814349+0000","last_clean":"2023-10-09T05:39:42.814349+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:39:42.814349+0000","last_undegraded":"2023-10-09T05:39:42.814349+0000","last_fullsized":"2023-10-09T05:39:42.814349+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":173,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.799673+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:39:42.799673+0000","last_peered":"2023-10-09T05:39:42.799673+0000","last_clean":"2023-10-09T05:39:42.799673+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:39:42.799673+0000","last_undegraded":"2023-10-09T05:39:42.799673+0000","last_fullsized":"2023-10-09T05:39:42.799673+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"193'8411","reported_seq":8461,"reported_epoch":193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:55.744422+0000","last_change":"2023-10-09T05:37:40.649912+0000","last_active":"2023-10-09T05:39:55.744422+0000","last_peered":"2023-10-09T05:39:55.744422+0000","last_clean":"2023-10-09T05:37:34.677100+0000","last_became_active":"2023-10-09T05:37:40.593033+0000","last_became_peered":"2023-10-09T05:37:40.593033+0000","last_unstale":"2023-10-09T05:39:55.744422+0000","last_undegraded":"2023-10-09T05:37:40.546647+0000","last_fullsized":"2023-10-09T05:37:40.543498+0000","mapping_epoch":170,"log_start":"166'300","ondisk_log_start":"166'300","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8111,"log_dups_size":0,"ondisk_log_size":8111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169017344,"num_objects":2579,"num_object_clones":0,"num_object_copies":7737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8411,"num_write_kb":351680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":-271}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":508,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.805035+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:39:42.805035+0000","last_peered":"2023-10-09T05:39:42.805035+0000","last_clean":"2023-10-09T05:39:42.805035+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:39:42.805035+0000","last_undegraded":"2023-10-09T05:39:42.805035+0000","last_fullsized":"2023-10-09T05:39:42.805035+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":141,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.791050+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:39:42.791050+0000","last_peered":"2023-10-09T05:39:42.791050+0000","last_clean":"2023-10-09T05:39:42.791050+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:39:42.791050+0000","last_undegraded":"2023-10-09T05:39:42.791050+0000","last_fullsized":"2023-10-09T05:39:42.791050+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":171,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.790664+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:39:42.790664+0000","last_peered":"2023-10-09T05:39:42.790664+0000","last_clean":"2023-10-09T05:39:42.790664+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:39:42.790664+0000","last_undegraded":"2023-10-09T05:39:42.790664+0000","last_fullsized":"2023-10-09T05:39:42.790664+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"194'8650","reported_seq":8690,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:59.802826+0000","last_change":"2023-10-09T05:37:01.438236+0000","last_active":"2023-10-09T05:39:59.802826+0000","last_peered":"2023-10-09T05:39:59.802826+0000","last_clean":"2023-10-09T05:39:59.802826+0000","last_became_active":"2023-10-09T05:37:01.437917+0000","last_became_peered":"2023-10-09T05:37:01.437917+0000","last_unstale":"2023-10-09T05:39:59.802826+0000","last_undegraded":"2023-10-09T05:39:59.802826+0000","last_fullsized":"2023-10-09T05:39:59.802826+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8650,"log_dups_size":0,"ondisk_log_size":8650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158859264,"num_objects":2424,"num_object_clones":0,"num_object_copies":7272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8650,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":778,"reported_epoch":193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:42.798087+0000","last_change":"2023-10-09T05:37:36.709131+0000","last_active":"2023-10-09T05:39:42.798087+0000","last_peered":"2023-10-09T05:39:42.798087+0000","last_clean":"2023-10-09T05:37:30.688708+0000","last_became_active":"2023-10-09T05:37:36.708682+0000","last_became_peered":"2023-10-09T05:37:36.708682+0000","last_unstale":"2023-10-09T05:39:42.798087+0000","last_undegraded":"2023-10-09T05:37:36.683823+0000","last_fullsized":"2023-10-09T05:37:36.683109+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":342,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.803688+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:39:42.803688+0000","last_peered":"2023-10-09T05:39:42.803688+0000","last_clean":"2023-10-09T05:39:42.803688+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:39:42.803688+0000","last_undegraded":"2023-10-09T05:39:42.803688+0000","last_fullsized":"2023-10-09T05:39:42.803688+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":158,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:51.206792+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:39:51.206792+0000","last_peered":"2023-10-09T05:39:51.206792+0000","last_clean":"2023-10-09T05:39:51.206792+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:39:51.206792+0000","last_undegraded":"2023-10-09T05:39:51.206792+0000","last_fullsized":"2023-10-09T05:39:51.206792+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":166,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T05:39:32.543921+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:39:32.543921+0000","last_peered":"2023-10-09T05:39:32.543921+0000","last_clean":"2023-10-09T05:39:32.543921+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:39:32.543921+0000","last_undegraded":"2023-10-09T05:39:32.543921+0000","last_fullsized":"2023-10-09T05:39:32.543921+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"194'8537","reported_seq":8579,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:58.789821+0000","last_change":"2023-10-09T05:37:01.440172+0000","last_active":"2023-10-09T05:39:58.789821+0000","last_peered":"2023-10-09T05:39:58.789821+0000","last_clean":"2023-10-09T05:39:58.789821+0000","last_became_active":"2023-10-09T05:37:01.439932+0000","last_became_peered":"2023-10-09T05:37:01.439932+0000","last_unstale":"2023-10-09T05:39:58.789821+0000","last_undegraded":"2023-10-09T05:39:58.789821+0000","last_fullsized":"2023-10-09T05:39:58.789821+0000","mapping_epoch":162,"log_start":"167'2100","ondisk_log_start":"167'2100","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6437,"log_dups_size":0,"ondisk_log_size":6437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:57.826704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158793728,"num_objects":2423,"num_object_clones":0,"num_object_copies":7269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8537,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":720,"reported_epoch":193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:42.791226+0000","last_change":"2023-10-09T05:37:36.748736+0000","last_active":"2023-10-09T05:39:42.791226+0000","last_peered":"2023-10-09T05:39:42.791226+0000","last_clean":"2023-10-09T05:22:39.601058+0000","last_became_active":"2023-10-09T05:37:36.748304+0000","last_became_peered":"2023-10-09T05:37:36.748304+0000","last_unstale":"2023-10-09T05:39:42.791226+0000","last_undegraded":"2023-10-09T05:37:36.696424+0000","last_fullsized":"2023-10-09T05:37:36.695987+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:18.784578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":146,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.802139+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:39:42.802139+0000","last_peered":"2023-10-09T05:39:42.802139+0000","last_clean":"2023-10-09T05:39:42.802139+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:39:42.802139+0000","last_undegraded":"2023-10-09T05:39:42.802139+0000","last_fullsized":"2023-10-09T05:39:42.802139+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":161,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.808973+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:39:42.808973+0000","last_peered":"2023-10-09T05:39:42.808973+0000","last_clean":"2023-10-09T05:39:42.808973+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:39:42.808973+0000","last_undegraded":"2023-10-09T05:39:42.808973+0000","last_fullsized":"2023-10-09T05:39:42.808973+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"194'8635","reported_seq":8677,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:58.697479+0000","last_change":"2023-10-09T05:37:01.636967+0000","last_active":"2023-10-09T05:39:58.697479+0000","last_peered":"2023-10-09T05:39:58.697479+0000","last_clean":"2023-10-09T05:39:58.697479+0000","last_became_active":"2023-10-09T05:37:01.636602+0000","last_became_peered":"2023-10-09T05:37:01.636602+0000","last_unstale":"2023-10-09T05:39:58.697479+0000","last_undegraded":"2023-10-09T05:39:58.697479+0000","last_fullsized":"2023-10-09T05:39:58.697479+0000","mapping_epoch":162,"log_start":"167'2200","ondisk_log_start":"167'2200","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6435,"log_dups_size":0,"ondisk_log_size":6435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:11.668143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161021952,"num_objects":2457,"num_object_clones":0,"num_object_copies":7371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8635,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":383,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:43.117414+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:39:43.117414+0000","last_peered":"2023-10-09T05:39:43.117414+0000","last_clean":"2023-10-09T05:39:43.117414+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:39:43.117414+0000","last_undegraded":"2023-10-09T05:39:43.117414+0000","last_fullsized":"2023-10-09T05:39:43.117414+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":141,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.791675+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:39:42.791675+0000","last_peered":"2023-10-09T05:39:42.791675+0000","last_clean":"2023-10-09T05:39:42.791675+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:39:42.791675+0000","last_undegraded":"2023-10-09T05:39:42.791675+0000","last_fullsized":"2023-10-09T05:39:42.791675+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":155,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.805023+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:39:42.805023+0000","last_peered":"2023-10-09T05:39:42.805023+0000","last_clean":"2023-10-09T05:39:42.805023+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:39:42.805023+0000","last_undegraded":"2023-10-09T05:39:42.805023+0000","last_fullsized":"2023-10-09T05:39:42.805023+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"194'8761","reported_seq":8801,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:59.903742+0000","last_change":"2023-10-09T05:37:01.440074+0000","last_active":"2023-10-09T05:39:59.903742+0000","last_peered":"2023-10-09T05:39:59.903742+0000","last_clean":"2023-10-09T05:39:59.903742+0000","last_became_active":"2023-10-09T05:37:01.439808+0000","last_became_peered":"2023-10-09T05:37:01.439808+0000","last_unstale":"2023-10-09T05:39:59.903742+0000","last_undegraded":"2023-10-09T05:39:59.903742+0000","last_fullsized":"2023-10-09T05:39:59.903742+0000","mapping_epoch":162,"log_start":"166'1200","ondisk_log_start":"166'1200","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7561,"log_dups_size":0,"ondisk_log_size":7561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154468352,"num_objects":2357,"num_object_clones":0,"num_object_copies":7071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8761,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":192,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.820883+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:39:42.820883+0000","last_peered":"2023-10-09T05:39:42.820883+0000","last_clean":"2023-10-09T05:39:42.820883+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:39:42.820883+0000","last_undegraded":"2023-10-09T05:39:42.820883+0000","last_fullsized":"2023-10-09T05:39:42.820883+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":582,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.795401+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:39:42.795401+0000","last_peered":"2023-10-09T05:39:42.795401+0000","last_clean":"2023-10-09T05:39:42.795401+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:39:42.795401+0000","last_undegraded":"2023-10-09T05:39:42.795401+0000","last_fullsized":"2023-10-09T05:39:42.795401+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":186,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.823359+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:39:42.823359+0000","last_peered":"2023-10-09T05:39:42.823359+0000","last_clean":"2023-10-09T05:39:42.823359+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:39:42.823359+0000","last_undegraded":"2023-10-09T05:39:42.823359+0000","last_fullsized":"2023-10-09T05:39:42.823359+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"193'8336","reported_seq":8382,"reported_epoch":193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:55.731485+0000","last_change":"2023-10-09T05:37:36.839957+0000","last_active":"2023-10-09T05:39:55.731485+0000","last_peered":"2023-10-09T05:39:55.731485+0000","last_clean":"2023-10-09T05:37:34.656587+0000","last_became_active":"2023-10-09T05:37:36.833510+0000","last_became_peered":"2023-10-09T05:37:36.833510+0000","last_unstale":"2023-10-09T05:39:55.731485+0000","last_undegraded":"2023-10-09T05:37:36.701190+0000","last_fullsized":"2023-10-09T05:37:36.681999+0000","mapping_epoch":170,"log_start":"166'200","ondisk_log_start":"166'200","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8136,"log_dups_size":0,"ondisk_log_size":8136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160694272,"num_objects":2452,"num_object_clones":0,"num_object_copies":7356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2877,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8336,"num_write_kb":345216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-425}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":677,"reported_epoch":192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:39:32.543823+0000","last_change":"2023-10-09T05:37:36.917512+0000","last_active":"2023-10-09T05:39:32.543823+0000","last_peered":"2023-10-09T05:39:32.543823+0000","last_clean":"2023-10-09T05:37:30.688967+0000","last_became_active":"2023-10-09T05:37:36.700218+0000","last_became_peered":"2023-10-09T05:37:36.700218+0000","last_unstale":"2023-10-09T05:39:32.543823+0000","last_undegraded":"2023-10-09T05:37:36.917335+0000","last_fullsized":"2023-10-09T05:37:36.682410+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":149,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.808900+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:39:42.808900+0000","last_peered":"2023-10-09T05:39:42.808900+0000","last_clean":"2023-10-09T05:39:42.808900+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:39:42.808900+0000","last_undegraded":"2023-10-09T05:39:42.808900+0000","last_fullsized":"2023-10-09T05:39:42.808900+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":145,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.813785+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:39:42.813785+0000","last_peered":"2023-10-09T05:39:42.813785+0000","last_clean":"2023-10-09T05:39:42.813785+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:39:42.813785+0000","last_undegraded":"2023-10-09T05:39:42.813785+0000","last_fullsized":"2023-10-09T05:39:42.813785+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":158,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-10-09T05:39:55.816108+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:39:55.816108+0000","last_peered":"2023-10-09T05:39:55.816108+0000","last_clean":"2023-10-09T05:39:55.816108+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:39:55.816108+0000","last_undegraded":"2023-10-09T05:39:55.816108+0000","last_fullsized":"2023-10-09T05:39:55.816108+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":185,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.805356+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:39:42.805356+0000","last_peered":"2023-10-09T05:39:42.805356+0000","last_clean":"2023-10-09T05:39:42.805356+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:39:42.805356+0000","last_undegraded":"2023-10-09T05:39:42.805356+0000","last_fullsized":"2023-10-09T05:39:42.805356+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":163,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.799454+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:39:42.799454+0000","last_peered":"2023-10-09T05:39:42.799454+0000","last_clean":"2023-10-09T05:39:42.799454+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:39:42.799454+0000","last_undegraded":"2023-10-09T05:39:42.799454+0000","last_fullsized":"2023-10-09T05:39:42.799454+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":175,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.814417+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:39:42.814417+0000","last_peered":"2023-10-09T05:39:42.814417+0000","last_clean":"2023-10-09T05:39:42.814417+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:39:42.814417+0000","last_undegraded":"2023-10-09T05:39:42.814417+0000","last_fullsized":"2023-10-09T05:39:42.814417+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":148,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.798331+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:39:42.798331+0000","last_peered":"2023-10-09T05:39:42.798331+0000","last_clean":"2023-10-09T05:39:42.798331+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:39:42.798331+0000","last_undegraded":"2023-10-09T05:39:42.798331+0000","last_fullsized":"2023-10-09T05:39:42.798331+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":175,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.815190+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:39:42.815190+0000","last_peered":"2023-10-09T05:39:42.815190+0000","last_clean":"2023-10-09T05:39:42.815190+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:39:42.815190+0000","last_undegraded":"2023-10-09T05:39:42.815190+0000","last_fullsized":"2023-10-09T05:39:42.815190+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":173,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.819178+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:39:42.819178+0000","last_peered":"2023-10-09T05:39:42.819178+0000","last_clean":"2023-10-09T05:39:42.819178+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:39:42.819178+0000","last_undegraded":"2023-10-09T05:39:42.819178+0000","last_fullsized":"2023-10-09T05:39:42.819178+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":185,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.820992+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:39:42.820992+0000","last_peered":"2023-10-09T05:39:42.820992+0000","last_clean":"2023-10-09T05:39:42.820992+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:39:42.820992+0000","last_undegraded":"2023-10-09T05:39:42.820992+0000","last_fullsized":"2023-10-09T05:39:42.820992+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":149,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.809380+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:39:42.809380+0000","last_peered":"2023-10-09T05:39:42.809380+0000","last_clean":"2023-10-09T05:39:42.809380+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:39:42.809380+0000","last_undegraded":"2023-10-09T05:39:42.809380+0000","last_fullsized":"2023-10-09T05:39:42.809380+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":160,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-10-09T05:39:42.813768+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:39:42.813768+0000","last_peered":"2023-10-09T05:39:42.813768+0000","last_clean":"2023-10-09T05:39:42.813768+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:39:42.813768+0000","last_undegraded":"2023-10-09T05:39:42.813768+0000","last_fullsized":"2023-10-09T05:39:42.813768+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2583625752,"num_objects":39424,"num_object_clones":0,"num_object_copies":118272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8854,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39424,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137298,"num_write_kb":5655041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8621,"num_bytes_recovered":530448384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8622977024,"data_stored":8622964808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123998,"ondisk_log_size":123998,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846657,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626682,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1615732,"kb_used_data":1314284,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92129420,"statfs":{"total":95995035648,"available":94340526080,"internally_reserved":0,"allocated":1345826816,"data_stored":1345306975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:23 2023","interfaces":[{"interface":"back","average":{"1min":6.093,"5min":4.5960000000000001,"15min":2.1890000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":99.787999999999997,"5min":99.787999999999997,"15min":99.787999999999997},"last":5.9059999999999997},{"interface":"front","average":{"1min":5.1890000000000001,"5min":4.234,"15min":2.0710000000000002},"min":{"1min":0.25600000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":99.844999999999999,"5min":99.844999999999999,"15min":99.844999999999999},"last":0.76100000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:39:46 2023","interfaces":[{"interface":"back","average":{"1min":13.882,"5min":5.0300000000000002,"15min":2.2690000000000001},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.25},"max":{"1min":137.02199999999999,"5min":137.02199999999999,"15min":137.02199999999999},"last":1.3720000000000001},{"interface":"front","average":{"1min":13.885,"5min":5.2430000000000003,"15min":2.355},"min":{"1min":0.38100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":137.06700000000001,"5min":137.06700000000001,"15min":137.06700000000001},"last":0.318}]},{"osd":2,"last update":"Mon Oct 9 05:39:23 2023","interfaces":[{"interface":"back","average":{"1min":8.2159999999999993,"5min":4.593,"15min":2.5329999999999999},"min":{"1min":0.29499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":115.2,"5min":115.2,"15min":115.2},"last":0.51300000000000001},{"interface":"front","average":{"1min":8.2409999999999997,"5min":4.4180000000000001,"15min":2.4950000000000001},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":115.297,"5min":115.297,"15min":115.297},"last":10.023999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:46 2023","interfaces":[{"interface":"back","average":{"1min":12.359999999999999,"5min":4.7290000000000001,"15min":2.4140000000000001},"min":{"1min":0.307,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":115.244,"5min":115.244,"15min":115.244},"last":19.082000000000001},{"interface":"front","average":{"1min":12.371,"5min":4.9870000000000001,"15min":2.5099999999999998},"min":{"1min":0.36499999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":115.34999999999999,"5min":115.34999999999999,"15min":115.34999999999999},"last":0.84599999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":3.1379999999999999,"15min":1.401},"min":{"1min":0.248,"5min":0.245,"15min":0.23400000000000001},"max":{"1min":0.58599999999999997,"5min":125.64,"15min":125.64},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.41499999999999998,"5min":2.7709999999999999,"15min":1.276},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":1.2709999999999999,"5min":125.625,"15min":125.625},"last":0.45800000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":2.871,"15min":1.3340000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.5510000000000002,"5min":127.35899999999999,"15min":127.35899999999999},"last":0.37},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.5110000000000001,"15min":1.23},"min":{"1min":0.29299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.8769999999999998,"5min":128.93799999999999,"15min":128.93799999999999},"last":0.5}]},{"osd":7,"last update":"Mon Oct 9 05:39:47 2023","interfaces":[{"interface":"back","average":{"1min":5.5990000000000002,"5min":2.5710000000000002,"15min":2.9550000000000001},"min":{"1min":0.35899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":115.514,"5min":115.514,"15min":115.514},"last":0.41899999999999998},{"interface":"front","average":{"1min":5.6500000000000004,"5min":2.0600000000000001,"15min":2.081},"min":{"1min":0.35499999999999998,"5min":0.317,"15min":0.317},"max":{"1min":116.53100000000001,"5min":116.53100000000001,"15min":116.53100000000001},"last":1.1870000000000001}]}]},{"osd":1,"up_from":93,"seq":399431958644,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1178652,"kb_used_data":934740,"kb_used_omap":7,"kb_used_meta":243832,"kb_avail":92566500,"statfs":{"total":95995035648,"available":94788096000,"internally_reserved":0,"allocated":957173760,"data_stored":956450956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":249684494},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9460000000000002,"5min":5.7850000000000001,"15min":2.3220000000000001},"min":{"1min":0.20300000000000001,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":13.214,"5min":207.858,"15min":207.858},"last":2.5990000000000002},{"interface":"front","average":{"1min":2.4529999999999998,"5min":6.7469999999999999,"15min":2.6859999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.849,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.83399999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":5.3289999999999997,"5min":7.3360000000000003,"15min":2.8220000000000001},"min":{"1min":0.22700000000000001,"5min":0.184,"15min":0.184},"max":{"1min":32.686999999999998,"5min":101.367,"15min":101.367},"last":0.48399999999999999},{"interface":"front","average":{"1min":4.468,"5min":6.7439999999999998,"15min":2.6259999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":19.975999999999999,"5min":98.409999999999997,"15min":98.409999999999997},"last":2.9769999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":6.1749999999999998,"5min":3.8559999999999999,"15min":1.744},"min":{"1min":0.35399999999999998,"5min":0.223,"15min":0.223},"max":{"1min":42.445999999999998,"5min":83.885000000000005,"15min":83.885000000000005},"last":0.46200000000000002},{"interface":"front","average":{"1min":5.4379999999999997,"5min":4.7290000000000001,"15min":1.954},"min":{"1min":0.373,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":42.243000000000002,"5min":84.641999999999996,"15min":84.641999999999996},"last":2.6299999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":4.1799999999999997,"15min":1.9670000000000001},"min":{"1min":0.35599999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.813000000000001,"5min":141.32900000000001,"15min":141.32900000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":2.4129999999999998,"5min":2.9420000000000002,"15min":1.532},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.4509999999999996,"5min":141.15899999999999,"15min":141.15899999999999},"last":0.32500000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":3.9020000000000001,"15min":1.901},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.6769999999999996,"5min":235.18700000000001,"15min":235.18700000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.7330000000000001,"5min":4.8550000000000004,"15min":2.319},"min":{"1min":0.38300000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":10.779,"5min":235.79300000000001,"15min":235.79300000000001},"last":0.51400000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":7.46,"5min":2.9140000000000001,"15min":1.5289999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":103.60899999999999,"5min":103.60899999999999,"15min":103.60899999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":7.3520000000000003,"5min":3.8500000000000001,"15min":1.837},"min":{"1min":0.38,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":103.679,"5min":103.679,"15min":103.679},"last":0.57199999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7719999999999998,"5min":2.9489999999999998,"15min":2.3149999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.7390000000000008,"5min":126.68300000000001,"15min":126.68300000000001},"last":2.5110000000000001},{"interface":"front","average":{"1min":2.927,"5min":3.0800000000000001,"15min":2.4119999999999999},"min":{"1min":0.46600000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.7720000000000002,"5min":126.738,"15min":126.738},"last":2.5630000000000002}]}]},{"osd":0,"up_from":88,"seq":377957122170,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":969912,"kb_used_data":636016,"kb_used_omap":8,"kb_used_meta":333879,"kb_avail":92775240,"statfs":{"total":95995035648,"available":95001845760,"internally_reserved":0,"allocated":651280384,"data_stored":650531175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":341892872},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":1.9379999999999999,"15min":1.0760000000000001},"min":{"1min":0.219,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":12.005000000000001,"5min":14.964,"15min":14.964},"last":0.60599999999999998},{"interface":"front","average":{"1min":3.4350000000000001,"5min":2.5179999999999998,"15min":1.2809999999999999},"min":{"1min":0.22900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.218999999999999,"5min":15.218999999999999,"15min":15.218999999999999},"last":0.78500000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":3.4180000000000001,"5min":3.8109999999999999,"15min":1.8879999999999999},"min":{"1min":0.28999999999999998,"5min":0.223,"15min":0.214},"max":{"1min":12.759,"5min":79.911000000000001,"15min":79.911000000000001},"last":6.0919999999999996},{"interface":"front","average":{"1min":4.085,"5min":3.5089999999999999,"15min":1.766},"min":{"1min":0.246,"5min":0.218,"15min":0.218},"max":{"1min":12.234999999999999,"5min":80.105000000000004,"15min":80.105000000000004},"last":0.43099999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":4.0839999999999996,"5min":3.2730000000000001,"15min":1.5609999999999999},"min":{"1min":0.27000000000000002,"5min":0.23999999999999999,"15min":0.22800000000000001},"max":{"1min":10.519,"5min":31.594000000000001,"15min":31.594000000000001},"last":6.181},{"interface":"front","average":{"1min":4.3410000000000002,"5min":3.3730000000000002,"15min":1.651},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.222},"max":{"1min":16.937999999999999,"5min":31.641999999999999,"15min":31.641999999999999},"last":6.0510000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":2.0539999999999998,"15min":1.163},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.1989999999999998,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.661,"5min":2.7010000000000001,"15min":1.4350000000000001},"min":{"1min":0.42099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.2149999999999999,"5min":69.102000000000004,"15min":69.102000000000004},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":8.0129999999999999,"5min":3.081,"15min":1.5509999999999999},"min":{"1min":0.45200000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":115.175,"5min":115.175,"15min":115.175},"last":6.2649999999999997},{"interface":"front","average":{"1min":7.0750000000000002,"5min":2.9710000000000001,"15min":1.5069999999999999},"min":{"1min":0.36499999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":115.502,"5min":115.502,"15min":115.502},"last":0.67800000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":1.9239999999999999,"15min":1.2050000000000001},"min":{"1min":0.41999999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.3409999999999993,"5min":9.3409999999999993,"15min":9.9809999999999999},"last":6.3159999999999998},{"interface":"front","average":{"1min":2.153,"5min":2.02,"15min":1.198},"min":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":7.266,"5min":8.0999999999999996,"15min":8.0999999999999996},"last":6.1360000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":1.544,"15min":1.3999999999999999},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.9740000000000002,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.499},{"interface":"front","average":{"1min":3.2999999999999998,"5min":2.3879999999999999,"15min":2.3889999999999998},"min":{"1min":0.5,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.4100000000000001,"5min":11.068,"15min":11.068},"last":0.75700000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795117,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1841644,"kb_used_data":1456100,"kb_used_omap":11,"kb_used_meta":385524,"kb_avail":91903508,"statfs":{"total":95995035648,"available":94109192192,"internally_reserved":0,"allocated":1491046400,"data_stored":1490252973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":394777173},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":3.476,"5min":2.8010000000000002,"15min":1.2669999999999999},"min":{"1min":0.23400000000000001,"5min":0.183,"15min":0.17199999999999999},"max":{"1min":10.098000000000001,"5min":32.122,"15min":32.122},"last":5.8049999999999997},{"interface":"front","average":{"1min":3.27,"5min":2.3530000000000002,"15min":1.1399999999999999},"min":{"1min":0.25800000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.5850000000000009,"5min":34.066000000000003,"15min":34.066000000000003},"last":9.8480000000000008}]},{"osd":1,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":14.41,"5min":5.3600000000000003,"15min":2.133},"min":{"1min":0.34499999999999997,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":98.575000000000003,"5min":98.575000000000003,"15min":98.575000000000003},"last":2.9550000000000001},{"interface":"front","average":{"1min":12.257,"5min":4.6779999999999999,"15min":1.867},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":90.135000000000005,"5min":98.397000000000006,"15min":98.397000000000006},"last":0.30299999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":15.612,"5min":5.4580000000000002,"15min":2.2930000000000001},"min":{"1min":0.55200000000000005,"5min":0.249,"15min":0.24199999999999999},"max":{"1min":150.65899999999999,"5min":150.65899999999999,"15min":150.65899999999999},"last":5.5590000000000002},{"interface":"front","average":{"1min":17.797000000000001,"5min":6.25,"15min":2.601},"min":{"1min":0.54900000000000004,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":150.851,"5min":150.851,"15min":150.851},"last":5.6639999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":2.8050000000000002,"15min":1.4019999999999999},"min":{"1min":0.29099999999999998,"5min":0.28999999999999998,"15min":0.27700000000000002},"max":{"1min":11.125999999999999,"5min":115.84999999999999,"15min":115.84999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":2.976,"5min":3.504,"15min":1.675},"min":{"1min":0.34799999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.147,"5min":116.149,"15min":116.149},"last":5.7759999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":2.5409999999999999,"15min":1.369},"min":{"1min":0.39900000000000002,"5min":0.312,"15min":0.312},"max":{"1min":7.8220000000000001,"5min":24.597999999999999,"15min":24.597999999999999},"last":3.0270000000000001},{"interface":"front","average":{"1min":1.867,"5min":2.089,"15min":1.1779999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.097,"5min":23.919,"15min":23.919},"last":0.46000000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":2.5129999999999999,"15min":1.327},"min":{"1min":0.39900000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.183,"5min":20.503,"15min":20.503},"last":5.7110000000000003},{"interface":"front","average":{"1min":2.8519999999999999,"5min":2.2170000000000001,"15min":1.2470000000000001},"min":{"1min":0.45000000000000001,"5min":0.372,"15min":0.34499999999999997},"max":{"1min":7.8730000000000002,"5min":9.4049999999999994,"15min":9.4049999999999994},"last":3.0670000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:39:52 2023","interfaces":[{"interface":"back","average":{"1min":3.294,"5min":3.0369999999999999,"15min":2.6949999999999998},"min":{"1min":0.436,"5min":0.374,"15min":0.374},"max":{"1min":10.897,"5min":24.030999999999999,"15min":24.030999999999999},"last":5.7439999999999998},{"interface":"front","average":{"1min":3.0299999999999998,"5min":2.7549999999999999,"15min":2.0710000000000002},"min":{"1min":0.48899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.8230000000000004,"5min":24.178000000000001,"15min":24.178000000000001},"last":5.2590000000000003}]}]},{"osd":3,"up_from":103,"seq":442381631591,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2040976,"kb_used_data":1792904,"kb_used_omap":4,"kb_used_meta":248059,"kb_avail":91704176,"statfs":{"total":95995035648,"available":93905076224,"internally_reserved":0,"allocated":1835933696,"data_stored":1835214293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":254013022},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":14.382,"5min":4.5739999999999998,"15min":2.129},"min":{"1min":0.19800000000000001,"5min":0.183,"15min":0.183},"max":{"1min":167.82400000000001,"5min":167.82400000000001,"15min":167.82400000000001},"last":2.5670000000000002},{"interface":"front","average":{"1min":15.930999999999999,"5min":5.0309999999999997,"15min":2.3330000000000002},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.214},"max":{"1min":167.89599999999999,"5min":167.89599999999999,"15min":167.89599999999999},"last":0.86499999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":3.218,"5min":3.8919999999999999,"15min":1.657},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.663,"5min":83.921999999999997,"15min":83.921999999999997},"last":1.5469999999999999},{"interface":"front","average":{"1min":3.956,"5min":3.911,"15min":1.651},"min":{"1min":0.25,"5min":0.25,"15min":0.247},"max":{"1min":12.061999999999999,"5min":83.707999999999998,"15min":83.707999999999998},"last":4.46}]},{"osd":2,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":16.274000000000001,"5min":6.6950000000000003,"15min":3.298},"min":{"1min":0.26500000000000001,"5min":0.215,"15min":0.215},"max":{"1min":190.43299999999999,"5min":190.43299999999999,"15min":190.43299999999999},"last":0.96299999999999997},{"interface":"front","average":{"1min":21.088000000000001,"5min":7.8490000000000002,"15min":3.6960000000000002},"min":{"1min":0.317,"5min":0.28999999999999998,"15min":0.26000000000000001},"max":{"1min":190.267,"5min":190.267,"15min":190.267},"last":0.999}]},{"osd":4,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":6.7539999999999996,"5min":2.9540000000000002,"15min":1.7390000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.29699999999999999},"max":{"1min":51.591999999999999,"5min":51.591999999999999,"15min":51.591999999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.9209999999999998,"5min":1.9790000000000001,"15min":1.3620000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":11.271000000000001,"5min":13.734,"15min":45.735999999999997},"last":0.88100000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":2.0379999999999998,"15min":1.325},"min":{"1min":0.45100000000000001,"5min":0.33500000000000002,"15min":0.29999999999999999},"max":{"1min":8.3640000000000008,"5min":8.4770000000000003,"15min":47.948999999999998},"last":0.81000000000000005},{"interface":"front","average":{"1min":3.1640000000000001,"5min":2.1659999999999999,"15min":1.4299999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.339,"5min":13.864000000000001,"15min":47.968000000000004},"last":0.93999999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.2120000000000002,"15min":1.321},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.313000000000001,"5min":13.831,"15min":13.831},"last":0.91300000000000003},{"interface":"front","average":{"1min":2.8260000000000001,"5min":2.1019999999999999,"15min":1.1299999999999999},"min":{"1min":0.45700000000000002,"5min":0.373,"15min":0.33900000000000002},"max":{"1min":8.3979999999999997,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":0.67900000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:39:53 2023","interfaces":[{"interface":"back","average":{"1min":3.9590000000000001,"5min":2.6699999999999999,"15min":2.3959999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":20.065000000000001,"5min":20.065000000000001,"15min":20.065000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":4.0590000000000002,"5min":2.6629999999999998,"15min":2.1960000000000002},"min":{"1min":0.40899999999999997,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":20.138000000000002,"5min":20.138000000000002,"15min":20.138000000000002},"last":0.60099999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019139,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1087420,"kb_used_data":782004,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92657732,"statfs":{"total":95995035648,"available":94881517568,"internally_reserved":0,"allocated":800772096,"data_stored":800251875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:20 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":2.1640000000000001,"15min":1.21},"min":{"1min":0.25600000000000001,"5min":0.214,"15min":0.214},"max":{"1min":7.4480000000000004,"5min":36.406999999999996,"15min":36.406999999999996},"last":2.4769999999999999},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.7810000000000001,"15min":1.454},"min":{"1min":0.28699999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.988,"5min":36.454999999999998,"15min":36.454999999999998},"last":2.6299999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":1.7829999999999999,"15min":1.1299999999999999},"min":{"1min":0.28599999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.194,"5min":10.073,"15min":31.073},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.6709999999999998,"5min":1.7869999999999999,"15min":1.2010000000000001},"min":{"1min":0.33500000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.5279999999999996,"5min":10.055,"15min":31.100000000000001},"last":12.349}]},{"osd":2,"last update":"Mon Oct 9 05:39:15 2023","interfaces":[{"interface":"back","average":{"1min":3.4870000000000001,"5min":8.1349999999999998,"15min":3.2909999999999999},"min":{"1min":0.316,"5min":0.314,"15min":0.26800000000000002},"max":{"1min":20.684000000000001,"5min":196.93700000000001,"15min":196.93700000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":2.621,"5min":7.7030000000000003,"15min":3.2040000000000002},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":20.611000000000001,"5min":197.08099999999999,"15min":197.08099999999999},"last":11.534000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":5.6639999999999997,"5min":6.8849999999999998,"15min":2.8439999999999999},"min":{"1min":0.35399999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":34.615000000000002,"5min":211.48699999999999,"15min":211.48699999999999},"last":2.5910000000000002},{"interface":"front","average":{"1min":5.8650000000000002,"5min":6.8010000000000002,"15min":2.8010000000000002},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.309},"max":{"1min":34.57,"5min":211.31100000000001,"15min":211.31100000000001},"last":2.4399999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":3.4489999999999998,"15min":1.5429999999999999},"min":{"1min":0.26300000000000001,"5min":0.25,"15min":0.249},"max":{"1min":3.5259999999999998,"5min":113.35899999999999,"15min":113.35899999999999},"last":2.5190000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":3.302,"15min":1.52},"min":{"1min":0.35499999999999998,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":3.2160000000000002,"5min":113.32299999999999,"15min":113.32299999999999},"last":0.38400000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:39:57 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.999,"15min":0.70899999999999996},"min":{"1min":0.317,"5min":0.28999999999999998,"15min":0.28100000000000003},"max":{"1min":3.4390000000000001,"5min":11.944000000000001,"15min":11.944000000000001},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.3300000000000001,"15min":0.85899999999999999},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.6500000000000004,"5min":10.356,"15min":10.356},"last":2.5600000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":1.4810000000000001,"15min":2.7400000000000002},"min":{"1min":0.37,"5min":0.312,"15min":0.312},"max":{"1min":4.5819999999999999,"5min":30.393999999999998,"15min":30.393999999999998},"last":0.503},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.601,"15min":3.0870000000000002},"min":{"1min":0.36499999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5999999999999996,"5min":30.334,"15min":30.334},"last":0.47399999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822911,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1083824,"kb_used_data":782504,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92661328,"statfs":{"total":95995035648,"available":94885199872,"internally_reserved":0,"allocated":801284096,"data_stored":800771635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":6.3940000000000001,"15min":2.8650000000000002},"min":{"1min":0.249,"5min":0.224,"15min":0.19400000000000001},"max":{"1min":10.712999999999999,"5min":165.90199999999999,"15min":165.90199999999999},"last":0.22800000000000001},{"interface":"front","average":{"1min":2.1419999999999999,"5min":6.3799999999999999,"15min":2.9359999999999999},"min":{"1min":0.28999999999999998,"5min":0.215,"15min":0.215},"max":{"1min":8.6099999999999994,"5min":165.96700000000001,"15min":165.96700000000001},"last":0.436}]},{"osd":1,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":5.3810000000000002,"5min":3.7469999999999999,"15min":1.738},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":58.176000000000002,"5min":87.769000000000005,"15min":87.769000000000005},"last":3.3519999999999999},{"interface":"front","average":{"1min":7.1660000000000004,"5min":3.8940000000000001,"15min":1.7769999999999999},"min":{"1min":0.32200000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":54.591000000000001,"5min":87.864000000000004,"15min":87.864000000000004},"last":6.7599999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:39:20 2023","interfaces":[{"interface":"back","average":{"1min":5.9400000000000004,"5min":9.1470000000000002,"15min":3.5609999999999999},"min":{"1min":0.33800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":42.055,"5min":290.697,"15min":290.697},"last":4.6280000000000001},{"interface":"front","average":{"1min":4.2720000000000002,"5min":8.7880000000000003,"15min":3.4079999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":41.953000000000003,"5min":290.786,"15min":290.786},"last":0.34899999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":16.713999999999999,"5min":5.8200000000000003,"15min":2.4249999999999998},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":95.236999999999995,"5min":95.236999999999995,"15min":95.236999999999995},"last":0.67400000000000004},{"interface":"front","average":{"1min":16.225000000000001,"5min":5.3680000000000003,"15min":2.3100000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":93.471000000000004,"5min":93.471000000000004,"15min":93.471000000000004},"last":0.32900000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.94399999999999995,"15min":0.68100000000000005},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":0.91100000000000003,"5min":9.2729999999999997,"15min":9.3510000000000009},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.4990000000000001,"15min":0.92900000000000005},"min":{"1min":0.32700000000000001,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":1.4259999999999999,"5min":14.236000000000001,"15min":14.236000000000001},"last":0.28799999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":2.5600000000000001,"15min":1.234},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":1.8069999999999999,"5min":95.010999999999996,"15min":95.010999999999996},"last":0.311},{"interface":"front","average":{"1min":0.61199999999999999,"5min":2.2490000000000001,"15min":1.123},"min":{"1min":0.33500000000000002,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":2.4390000000000001,"5min":94.194000000000003,"15min":94.194000000000003},"last":0.371}]},{"osd":7,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.27,"15min":1.573},"min":{"1min":0.34399999999999997,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.94099999999999995,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.52100000000000002,"5min":1.3400000000000001,"15min":1.8620000000000001},"min":{"1min":0.35899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.5029999999999999,"5min":14.432,"15min":14.432},"last":0.38700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":641470464,"data_stored":641466392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":949555200,"data_stored":949551128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1472069632,"data_stored":1472069632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1821310976,"data_stored":1821310976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":789708800,"data_stored":789708800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":790499328,"data_stored":790495256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1333395456,"data_stored":1333395456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":824967168,"data_stored":824967168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:40:03.607 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:03.626 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:40:03.704 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:04.041 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:03 smithi008 ceph-mon[184418]: from='client.55108 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:04.041 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:03 smithi008 ceph-mon[184418]: pgmap v542: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 8887/114921 objects degraded (7.733%); 0 B/s, 40 objects/s recovering 2023-10-09T05:40:04.041 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:03 smithi008 ceph-mon[191227]: from='client.55108 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:04.042 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:03 smithi008 ceph-mon[191227]: pgmap v542: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 8887/114921 objects degraded (7.733%); 0 B/s, 40 objects/s recovering 2023-10-09T05:40:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:03 smithi191 ceph-mon[161669]: from='client.55108 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:03 smithi191 ceph-mon[161669]: pgmap v542: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 8887/114921 objects degraded (7.733%); 0 B/s, 40 objects/s recovering 2023-10-09T05:40:04.119 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:04.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:04.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:04.227 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:04.259 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:04.567 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:04.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:04.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:04.670 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:04.700 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:04.747 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:04.765 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:04.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:04.850 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:04.872 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:04.907 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:04.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:04 smithi008 ceph-mon[184418]: failed to encode map e195 with expected crc 2023-10-09T05:40:04.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:04 smithi008 ceph-mon[184418]: osdmap e195: 8 total, 8 up, 7 in 2023-10-09T05:40:04.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:04 smithi008 ceph-mon[184418]: failed to encode map e195 with expected crc 2023-10-09T05:40:04.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:04 smithi008 ceph-mon[184418]: failed to encode map e195 with expected crc 2023-10-09T05:40:04.969 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:04.972 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:04 smithi008 ceph-mon[191227]: failed to encode map e195 with expected crc 2023-10-09T05:40:04.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:04 smithi008 ceph-mon[191227]: osdmap e195: 8 total, 8 up, 7 in 2023-10-09T05:40:04.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:04 smithi008 ceph-mon[191227]: failed to encode map e195 with expected crc 2023-10-09T05:40:04.973 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:04 smithi008 ceph-mon[191227]: failed to encode map e195 with expected crc 2023-10-09T05:40:04.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:04.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:04 smithi191 ceph-mon[161669]: failed to encode map e195 with expected crc 2023-10-09T05:40:05.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:04 smithi191 ceph-mon[161669]: osdmap e195: 8 total, 8 up, 7 in 2023-10-09T05:40:05.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:04 smithi191 ceph-mon[161669]: failed to encode map e195 with expected crc 2023-10-09T05:40:05.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:04 smithi191 ceph-mon[161669]: failed to encode map e195 with expected crc 2023-10-09T05:40:05.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:05.110 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:05.171 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:40:05.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:05.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:05.279 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:05.404 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:05.732 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:05.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:05.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:05.801 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[184418]: pgmap v544: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 8638/112743 objects degraded (7.662%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:05.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[184418]: failed to encode map e196 with expected crc 2023-10-09T05:40:05.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[184418]: failed to encode map e196 with expected crc 2023-10-09T05:40:05.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[184418]: failed to encode map e196 with expected crc 2023-10-09T05:40:05.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[184418]: osdmap e196: 8 total, 8 up, 7 in 2023-10-09T05:40:05.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[184418]: failed to encode map e196 with expected crc 2023-10-09T05:40:05.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[184418]: failed to encode map e196 with expected crc 2023-10-09T05:40:05.802 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[191227]: pgmap v544: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 8638/112743 objects degraded (7.662%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:05.802 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[191227]: failed to encode map e196 with expected crc 2023-10-09T05:40:05.802 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[191227]: failed to encode map e196 with expected crc 2023-10-09T05:40:05.803 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[191227]: failed to encode map e196 with expected crc 2023-10-09T05:40:05.803 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[191227]: osdmap e196: 8 total, 8 up, 7 in 2023-10-09T05:40:05.803 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[191227]: failed to encode map e196 with expected crc 2023-10-09T05:40:05.803 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:05 smithi008 ceph-mon[191227]: failed to encode map e196 with expected crc 2023-10-09T05:40:05.855 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:05.885 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:05 smithi191 ceph-mon[161669]: pgmap v544: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 8638/112743 objects degraded (7.662%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:05 smithi191 ceph-mon[161669]: failed to encode map e196 with expected crc 2023-10-09T05:40:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:05 smithi191 ceph-mon[161669]: failed to encode map e196 with expected crc 2023-10-09T05:40:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:05 smithi191 ceph-mon[161669]: failed to encode map e196 with expected crc 2023-10-09T05:40:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:05 smithi191 ceph-mon[161669]: osdmap e196: 8 total, 8 up, 7 in 2023-10-09T05:40:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:05 smithi191 ceph-mon[161669]: failed to encode map e196 with expected crc 2023-10-09T05:40:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:05 smithi191 ceph-mon[161669]: failed to encode map e196 with expected crc 2023-10-09T05:40:06.481 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:06.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:06.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:06.596 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:40:06.596 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:06.627 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:06.683 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:06.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:06.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:06.789 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:06.823 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:06.850 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:40:06.887 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:06.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:06.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:06.896 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:06 smithi191 ceph-mon[161669]: failed to encode map e197 with expected crc 2023-10-09T05:40:06.896 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:06 smithi191 ceph-mon[161669]: osdmap e197: 8 total, 8 up, 7 in 2023-10-09T05:40:06.896 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:06 smithi191 ceph-mon[161669]: failed to encode map e197 with expected crc 2023-10-09T05:40:06.896 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:06 smithi191 ceph-mon[161669]: failed to encode map e197 with expected crc 2023-10-09T05:40:06.896 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4185001752' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:06.896 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:06 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:06.896 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:06 smithi191 ceph-mon[161669]: pgmap v547: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 8465/109407 objects degraded (7.737%); 0 B/s, 69 objects/s recovering 2023-10-09T05:40:06.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[184418]: failed to encode map e197 with expected crc 2023-10-09T05:40:06.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[184418]: osdmap e197: 8 total, 8 up, 7 in 2023-10-09T05:40:06.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[184418]: failed to encode map e197 with expected crc 2023-10-09T05:40:06.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[184418]: failed to encode map e197 with expected crc 2023-10-09T05:40:06.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4185001752' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:06.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:06.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[184418]: pgmap v547: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 8465/109407 objects degraded (7.737%); 0 B/s, 69 objects/s recovering 2023-10-09T05:40:06.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[191227]: failed to encode map e197 with expected crc 2023-10-09T05:40:06.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[191227]: osdmap e197: 8 total, 8 up, 7 in 2023-10-09T05:40:06.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[191227]: failed to encode map e197 with expected crc 2023-10-09T05:40:06.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[191227]: failed to encode map e197 with expected crc 2023-10-09T05:40:06.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4185001752' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:06.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:06.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:06 smithi008 ceph-mon[191227]: pgmap v547: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 8465/109407 objects degraded (7.737%); 0 B/s, 69 objects/s recovering 2023-10-09T05:40:06.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:07.028 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:07.074 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:07.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:07.180 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:07.257 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:07.595 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:07.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:07.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:07.698 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:07.725 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:07.768 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:40:07.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:07.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:07.873 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:07.914 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:07.979 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:07.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:07.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:07.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:07 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:07.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:07 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:07.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:07 smithi191 ceph-mon[161669]: osdmap e198: 8 total, 8 up, 7 in 2023-10-09T05:40:07.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:07 smithi191 ceph-mon[161669]: failed to encode map e198 with expected crc 2023-10-09T05:40:07.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:07 smithi191 ceph-mon[161669]: failed to encode map e198 with expected crc 2023-10-09T05:40:07.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:07 smithi191 ceph-mon[161669]: failed to encode map e198 with expected crc 2023-10-09T05:40:07.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:07 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 8465/109407 objects degraded (7.737%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:08.084 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:08.160 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:08.161 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:08.161 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[184418]: osdmap e198: 8 total, 8 up, 7 in 2023-10-09T05:40:08.161 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[184418]: failed to encode map e198 with expected crc 2023-10-09T05:40:08.161 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[184418]: failed to encode map e198 with expected crc 2023-10-09T05:40:08.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[184418]: failed to encode map e198 with expected crc 2023-10-09T05:40:08.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 8465/109407 objects degraded (7.737%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:08.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:08.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:08.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[191227]: osdmap e198: 8 total, 8 up, 7 in 2023-10-09T05:40:08.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[191227]: failed to encode map e198 with expected crc 2023-10-09T05:40:08.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[191227]: failed to encode map e198 with expected crc 2023-10-09T05:40:08.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[191227]: failed to encode map e198 with expected crc 2023-10-09T05:40:08.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:07 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 8465/109407 objects degraded (7.737%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:08.223 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:08.650 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:08.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:08.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:08.712 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:40:08.763 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:08.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:08.772 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:40:08.793 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:40:08.793 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:40:08.794 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:08.816 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:40:08.845 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:08.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:08.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:08.857 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:40:08.894 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:40:08.920 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:40:08.953 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:40:08.955 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:08.988 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:40:08.989 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:09.026 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:40:09.026 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:40:09.026 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:40:09.027 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:40:09.027 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:40:09.027 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:40:09.027 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:40:09.027 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:40:09.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:08 smithi191 ceph-mon[161669]: pgmap v549: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 8465/107187 objects degraded (7.897%); 0 B/s, 43 objects/s recovering 2023-10-09T05:40:09.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/914293636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:09.050 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:09.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:09.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:09.160 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:09.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:08 smithi008 ceph-mon[184418]: pgmap v549: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 8465/107187 objects degraded (7.897%); 0 B/s, 43 objects/s recovering 2023-10-09T05:40:09.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/914293636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:08 smithi008 ceph-mon[191227]: pgmap v549: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 8465/107187 objects degraded (7.897%); 0 B/s, 43 objects/s recovering 2023-10-09T05:40:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/914293636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:09.248 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:10.217 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:40:10.507 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:40:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:40:10] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-09T05:40:10.538 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:10.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:10.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:10.650 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:10.686 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:10.740 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:10.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:10.809 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:40:10.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:10.878 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:10.946 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:10.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:10.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:11.052 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:11.084 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:11.144 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:11.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:11.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:11.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:11.279 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:11 smithi191 ceph-mon[161669]: pgmap v550: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 8455/102912 objects degraded (8.216%); 0 B/s, 31 objects/s recovering 2023-10-09T05:40:11.286 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:11.343 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:11.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:11.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:11.452 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:11.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:11 smithi008 ceph-mon[184418]: pgmap v550: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 8455/102912 objects degraded (8.216%); 0 B/s, 31 objects/s recovering 2023-10-09T05:40:11.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:11 smithi008 ceph-mon[191227]: pgmap v550: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 8455/102912 objects degraded (8.216%); 0 B/s, 31 objects/s recovering 2023-10-09T05:40:11.575 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:11.839 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:11.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:11.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:11.955 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:11.995 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:12.031 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:12.081 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:12.270 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:12.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:12.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:12.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:12.412 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:12.475 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:12.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:12.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:12.587 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:12.686 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:12.723 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:40:13.107 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:40:13.152 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:40:13.411 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:13.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:13.428 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:13 smithi008 ceph-mon[184418]: pgmap v551: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 8153/99216 objects degraded (8.217%); 0 B/s, 67 objects/s recovering 2023-10-09T05:40:13.428 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 8153/99216 objects degraded (8.217%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:13.428 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:13 smithi008 ceph-mon[191227]: pgmap v551: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 8153/99216 objects degraded (8.217%); 0 B/s, 67 objects/s recovering 2023-10-09T05:40:13.428 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 8153/99216 objects degraded (8.217%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:13.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:13.529 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:13.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:13 smithi191 ceph-mon[161669]: pgmap v551: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 8153/99216 objects degraded (8.217%); 0 B/s, 67 objects/s recovering 2023-10-09T05:40:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 8153/99216 objects degraded (8.217%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:13.590 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:14.085 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:14.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:14.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:14.203 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:14.257 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:14.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:14 smithi008 ceph-mon[184418]: from='client.45227 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:14.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:14.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:14 smithi008 ceph-mon[191227]: from='client.45227 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:14.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:14 smithi191 ceph-mon[161669]: from='client.45227 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:15.319 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:15.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:15.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:15.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:15.455 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:15 smithi191 ceph-mon[161669]: pgmap v552: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 8153/96927 objects degraded (8.411%); 0 B/s, 38 objects/s recovering 2023-10-09T05:40:15.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:15 smithi008 ceph-mon[184418]: pgmap v552: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 8153/96927 objects degraded (8.411%); 0 B/s, 38 objects/s recovering 2023-10-09T05:40:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:15 smithi008 ceph-mon[191227]: pgmap v552: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 8153/96927 objects degraded (8.411%); 0 B/s, 38 objects/s recovering 2023-10-09T05:40:15.468 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:15.484 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":551,"stamp":"2023-10-09T05:40:12.104659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2170296267,"num_objects":33072,"num_object_clones":0,"num_object_copies":99216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33072,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":144784,"num_write_kb":5670182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9388,"num_bytes_recovered":531043702,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127648,"ondisk_log_size":127648,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8565844,"kb_used_data":6403228,"kb_used_omap":31,"kb_used_meta":2161952,"kb_avail":647650220,"statfs":{"total":671965249536,"available":663193825280,"internally_reserved":0,"allocated":6556905472,"data_stored":6552312869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2213839305},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-295501824,"num_objects":-4509,"num_object_clones":0,"num_object_copies":-13527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-485,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4509,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":492,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.306235"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":181,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823766+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:40:05.823766+0000","last_peered":"2023-10-09T05:40:05.823766+0000","last_clean":"2023-10-09T05:40:05.823766+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:40:05.823766+0000","last_undegraded":"2023-10-09T05:40:05.823766+0000","last_fullsized":"2023-10-09T05:40:05.823766+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":170,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.905904+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:40:05.905904+0000","last_peered":"2023-10-09T05:40:05.905904+0000","last_clean":"2023-10-09T05:40:05.905904+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:40:05.905904+0000","last_undegraded":"2023-10-09T05:40:05.905904+0000","last_fullsized":"2023-10-09T05:40:05.905904+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":145,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:06.875960+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:40:06.875960+0000","last_peered":"2023-10-09T05:40:06.875960+0000","last_clean":"2023-10-09T05:40:06.875960+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:40:06.875960+0000","last_undegraded":"2023-10-09T05:40:06.875960+0000","last_fullsized":"2023-10-09T05:40:06.875960+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":175,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.817669+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:40:05.817669+0000","last_peered":"2023-10-09T05:40:05.817669+0000","last_clean":"2023-10-09T05:40:05.817669+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:40:05.817669+0000","last_undegraded":"2023-10-09T05:40:05.817669+0000","last_fullsized":"2023-10-09T05:40:05.817669+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":186,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.905569+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:40:05.905569+0000","last_peered":"2023-10-09T05:40:05.905569+0000","last_clean":"2023-10-09T05:40:05.905569+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:40:05.905569+0000","last_undegraded":"2023-10-09T05:40:05.905569+0000","last_fullsized":"2023-10-09T05:40:05.905569+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":164,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.820671+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:40:05.820671+0000","last_peered":"2023-10-09T05:40:05.820671+0000","last_clean":"2023-10-09T05:40:05.820671+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:40:05.820671+0000","last_undegraded":"2023-10-09T05:40:05.820671+0000","last_fullsized":"2023-10-09T05:40:05.820671+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":150,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.819264+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:40:05.819264+0000","last_peered":"2023-10-09T05:40:05.819264+0000","last_clean":"2023-10-09T05:40:05.819264+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:40:05.819264+0000","last_undegraded":"2023-10-09T05:40:05.819264+0000","last_fullsized":"2023-10-09T05:40:05.819264+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":170,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.905658+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:40:05.905658+0000","last_peered":"2023-10-09T05:40:05.905658+0000","last_clean":"2023-10-09T05:40:05.905658+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:40:05.905658+0000","last_undegraded":"2023-10-09T05:40:05.905658+0000","last_fullsized":"2023-10-09T05:40:05.905658+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":189,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.811176+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:40:05.811176+0000","last_peered":"2023-10-09T05:40:05.811176+0000","last_clean":"2023-10-09T05:40:05.811176+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:40:05.811176+0000","last_undegraded":"2023-10-09T05:40:05.811176+0000","last_fullsized":"2023-10-09T05:40:05.811176+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":184,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.905080+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:40:05.905080+0000","last_peered":"2023-10-09T05:40:05.905080+0000","last_clean":"2023-10-09T05:40:05.905080+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:40:05.905080+0000","last_undegraded":"2023-10-09T05:40:05.905080+0000","last_fullsized":"2023-10-09T05:40:05.905080+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":78,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.905835+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:40:05.905835+0000","last_peered":"2023-10-09T05:40:05.905835+0000","last_clean":"2023-10-09T05:40:05.905835+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:40:05.905835+0000","last_undegraded":"2023-10-09T05:40:05.905835+0000","last_fullsized":"2023-10-09T05:40:05.905835+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":145,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.811868+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:40:05.811868+0000","last_peered":"2023-10-09T05:40:05.811868+0000","last_clean":"2023-10-09T05:40:05.811868+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:40:05.811868+0000","last_undegraded":"2023-10-09T05:40:05.811868+0000","last_fullsized":"2023-10-09T05:40:05.811868+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":206,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823597+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:40:05.823597+0000","last_peered":"2023-10-09T05:40:05.823597+0000","last_clean":"2023-10-09T05:40:05.823597+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:40:05.823597+0000","last_undegraded":"2023-10-09T05:40:05.823597+0000","last_fullsized":"2023-10-09T05:40:05.823597+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":164,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823095+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:40:05.823095+0000","last_peered":"2023-10-09T05:40:05.823095+0000","last_clean":"2023-10-09T05:40:05.823095+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:40:05.823095+0000","last_undegraded":"2023-10-09T05:40:05.823095+0000","last_fullsized":"2023-10-09T05:40:05.823095+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":77,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823191+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:40:05.823191+0000","last_peered":"2023-10-09T05:40:05.823191+0000","last_clean":"2023-10-09T05:40:05.823191+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:40:05.823191+0000","last_undegraded":"2023-10-09T05:40:05.823191+0000","last_fullsized":"2023-10-09T05:40:05.823191+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":194,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.915562+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:40:05.915562+0000","last_peered":"2023-10-09T05:40:05.915562+0000","last_clean":"2023-10-09T05:40:05.915562+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:40:05.915562+0000","last_undegraded":"2023-10-09T05:40:05.915562+0000","last_fullsized":"2023-10-09T05:40:05.915562+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":174,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.817556+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:40:05.817556+0000","last_peered":"2023-10-09T05:40:05.817556+0000","last_clean":"2023-10-09T05:40:05.817556+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:40:05.817556+0000","last_undegraded":"2023-10-09T05:40:05.817556+0000","last_fullsized":"2023-10-09T05:40:05.817556+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":188,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.812368+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:40:05.812368+0000","last_peered":"2023-10-09T05:40:05.812368+0000","last_clean":"2023-10-09T05:40:05.812368+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:40:05.812368+0000","last_undegraded":"2023-10-09T05:40:05.812368+0000","last_fullsized":"2023-10-09T05:40:05.812368+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":95,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823224+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:40:05.823224+0000","last_peered":"2023-10-09T05:40:05.823224+0000","last_clean":"2023-10-09T05:40:05.823224+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:40:05.823224+0000","last_undegraded":"2023-10-09T05:40:05.823224+0000","last_fullsized":"2023-10-09T05:40:05.823224+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":174,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.817602+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:40:05.817602+0000","last_peered":"2023-10-09T05:40:05.817602+0000","last_clean":"2023-10-09T05:40:05.817602+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:40:05.817602+0000","last_undegraded":"2023-10-09T05:40:05.817602+0000","last_fullsized":"2023-10-09T05:40:05.817602+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":173,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.819994+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:40:05.819994+0000","last_peered":"2023-10-09T05:40:05.819994+0000","last_clean":"2023-10-09T05:40:05.819994+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:40:05.819994+0000","last_undegraded":"2023-10-09T05:40:05.819994+0000","last_fullsized":"2023-10-09T05:40:05.819994+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":194,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.916783+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:40:05.916783+0000","last_peered":"2023-10-09T05:40:05.916783+0000","last_clean":"2023-10-09T05:40:05.916783+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:40:05.916783+0000","last_undegraded":"2023-10-09T05:40:05.916783+0000","last_fullsized":"2023-10-09T05:40:05.916783+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"198'8961","reported_seq":9005,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:09.871220+0000","last_change":"2023-10-09T05:37:01.636984+0000","last_active":"2023-10-09T05:40:09.871220+0000","last_peered":"2023-10-09T05:40:09.871220+0000","last_clean":"2023-10-09T05:40:09.871220+0000","last_became_active":"2023-10-09T05:37:01.636609+0000","last_became_peered":"2023-10-09T05:37:01.636609+0000","last_unstale":"2023-10-09T05:40:09.871220+0000","last_undegraded":"2023-10-09T05:40:09.871220+0000","last_fullsized":"2023-10-09T05:40:09.871220+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8961,"log_dups_size":0,"ondisk_log_size":8961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137297920,"num_objects":2095,"num_object_clones":0,"num_object_copies":6285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8961,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":75,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.826127+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:40:05.826127+0000","last_peered":"2023-10-09T05:40:05.826127+0000","last_clean":"2023-10-09T05:40:05.826127+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:40:05.826127+0000","last_undegraded":"2023-10-09T05:40:05.826127+0000","last_fullsized":"2023-10-09T05:40:05.826127+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":173,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.810763+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:40:05.810763+0000","last_peered":"2023-10-09T05:40:05.810763+0000","last_clean":"2023-10-09T05:40:05.810763+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:40:05.810763+0000","last_undegraded":"2023-10-09T05:40:05.810763+0000","last_fullsized":"2023-10-09T05:40:05.810763+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"198'8967","reported_seq":9021,"reported_epoch":198,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:11.201602+0000","last_change":"2023-10-09T05:37:36.837344+0000","last_active":"2023-10-09T05:40:11.201602+0000","last_peered":"2023-10-09T05:40:11.201602+0000","last_clean":"2023-10-09T05:37:34.651883+0000","last_became_active":"2023-10-09T05:37:36.832502+0000","last_became_peered":"2023-10-09T05:37:36.832502+0000","last_unstale":"2023-10-09T05:40:11.201602+0000","last_undegraded":"2023-10-09T05:37:36.702797+0000","last_fullsized":"2023-10-09T05:37:36.678583+0000","mapping_epoch":170,"log_start":"167'2500","ondisk_log_start":"167'2500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6467,"log_dups_size":0,"ondisk_log_size":6467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:57.419460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132710400,"num_objects":2025,"num_object_clones":0,"num_object_copies":6075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8967,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":-865}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":75,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.824097+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:40:05.824097+0000","last_peered":"2023-10-09T05:40:05.824097+0000","last_clean":"2023-10-09T05:40:05.824097+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:40:05.824097+0000","last_undegraded":"2023-10-09T05:40:05.824097+0000","last_fullsized":"2023-10-09T05:40:05.824097+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":179,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823154+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:40:05.823154+0000","last_peered":"2023-10-09T05:40:05.823154+0000","last_clean":"2023-10-09T05:40:05.823154+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:40:05.823154+0000","last_undegraded":"2023-10-09T05:40:05.823154+0000","last_fullsized":"2023-10-09T05:40:05.823154+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"198'9238","reported_seq":15344,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:11.707321+0000","last_change":"2023-10-09T05:38:55.317808+0000","last_active":"2023-10-09T05:40:11.707321+0000","last_peered":"2023-10-09T05:40:11.707321+0000","last_clean":"2023-10-09T05:40:11.707321+0000","last_became_active":"2023-10-09T05:38:55.302739+0000","last_became_peered":"2023-10-09T05:38:55.302739+0000","last_unstale":"2023-10-09T05:40:11.707321+0000","last_undegraded":"2023-10-09T05:40:11.707321+0000","last_fullsized":"2023-10-09T05:40:11.707321+0000","mapping_epoch":186,"log_start":"166'100","ondisk_log_start":"166'100","created":162,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9138,"log_dups_size":0,"ondisk_log_size":9138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:25.379818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129368064,"num_objects":1974,"num_object_clones":0,"num_object_copies":5922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9238,"num_write_kb":358784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":198049792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":75,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.812460+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:40:05.812460+0000","last_peered":"2023-10-09T05:40:05.812460+0000","last_clean":"2023-10-09T05:40:05.812460+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:40:05.812460+0000","last_undegraded":"2023-10-09T05:40:05.812460+0000","last_fullsized":"2023-10-09T05:40:05.812460+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":197,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.820978+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:40:05.820978+0000","last_peered":"2023-10-09T05:40:05.820978+0000","last_clean":"2023-10-09T05:40:05.820978+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:40:05.820978+0000","last_undegraded":"2023-10-09T05:40:05.820978+0000","last_fullsized":"2023-10-09T05:40:05.820978+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"198'8867","reported_seq":14731,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:08.061003+0000","last_change":"2023-10-09T05:40:04.830679+0000","last_active":"2023-10-09T05:40:08.061003+0000","last_peered":"2023-10-09T05:40:08.061003+0000","last_clean":"2023-10-09T05:40:08.061003+0000","last_became_active":"2023-10-09T05:40:04.807741+0000","last_became_peered":"2023-10-09T05:40:04.807741+0000","last_unstale":"2023-10-09T05:40:08.061003+0000","last_undegraded":"2023-10-09T05:40:08.061003+0000","last_fullsized":"2023-10-09T05:40:08.061003+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8867,"log_dups_size":0,"ondisk_log_size":8867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:51.289034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137822208,"num_objects":2103,"num_object_clones":0,"num_object_copies":6309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8867,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2899,"num_bytes_recovered":139919360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":165,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.811162+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:40:05.811162+0000","last_peered":"2023-10-09T05:40:05.811162+0000","last_clean":"2023-10-09T05:40:05.811162+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:40:05.811162+0000","last_undegraded":"2023-10-09T05:40:05.811162+0000","last_fullsized":"2023-10-09T05:40:05.811162+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":150,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.817046+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:40:05.817046+0000","last_peered":"2023-10-09T05:40:05.817046+0000","last_clean":"2023-10-09T05:40:05.817046+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:40:05.817046+0000","last_undegraded":"2023-10-09T05:40:05.817046+0000","last_fullsized":"2023-10-09T05:40:05.817046+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"198'8968","reported_seq":9012,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:09.874273+0000","last_change":"2023-10-09T05:37:01.439878+0000","last_active":"2023-10-09T05:40:09.874273+0000","last_peered":"2023-10-09T05:40:09.874273+0000","last_clean":"2023-10-09T05:40:09.874273+0000","last_became_active":"2023-10-09T05:37:01.439481+0000","last_became_peered":"2023-10-09T05:37:01.439481+0000","last_unstale":"2023-10-09T05:40:09.874273+0000","last_undegraded":"2023-10-09T05:40:09.874273+0000","last_fullsized":"2023-10-09T05:40:09.874273+0000","mapping_epoch":162,"log_start":"166'1400","ondisk_log_start":"166'1400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7568,"log_dups_size":0,"ondisk_log_size":7568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132251648,"num_objects":2018,"num_object_clones":0,"num_object_copies":6054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8968,"num_write_kb":351552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":168,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.907606+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:40:05.907606+0000","last_peered":"2023-10-09T05:40:05.907606+0000","last_clean":"2023-10-09T05:40:05.907606+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:40:05.907606+0000","last_undegraded":"2023-10-09T05:40:05.907606+0000","last_fullsized":"2023-10-09T05:40:05.907606+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":164,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823532+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:40:05.823532+0000","last_peered":"2023-10-09T05:40:05.823532+0000","last_clean":"2023-10-09T05:40:05.823532+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:40:05.823532+0000","last_undegraded":"2023-10-09T05:40:05.823532+0000","last_fullsized":"2023-10-09T05:40:05.823532+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"198'8994","reported_seq":9040,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:11.709407+0000","last_change":"2023-10-09T05:37:01.440175+0000","last_active":"2023-10-09T05:40:11.709407+0000","last_peered":"2023-10-09T05:40:11.709407+0000","last_clean":"2023-10-09T05:40:11.709407+0000","last_became_active":"2023-10-09T05:37:01.439846+0000","last_became_peered":"2023-10-09T05:37:01.439846+0000","last_unstale":"2023-10-09T05:40:11.709407+0000","last_undegraded":"2023-10-09T05:40:11.709407+0000","last_fullsized":"2023-10-09T05:40:11.709407+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8994,"log_dups_size":0,"ondisk_log_size":8994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:27.883265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135004160,"num_objects":2060,"num_object_clones":0,"num_object_copies":6180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8994,"num_write_kb":353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":165,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.820852+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:40:05.820852+0000","last_peered":"2023-10-09T05:40:05.820852+0000","last_clean":"2023-10-09T05:40:05.820852+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:40:05.820852+0000","last_undegraded":"2023-10-09T05:40:05.820852+0000","last_fullsized":"2023-10-09T05:40:05.820852+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":179,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823681+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:40:05.823681+0000","last_peered":"2023-10-09T05:40:05.823681+0000","last_clean":"2023-10-09T05:40:05.823681+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:40:05.823681+0000","last_undegraded":"2023-10-09T05:40:05.823681+0000","last_fullsized":"2023-10-09T05:40:05.823681+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"198'9068","reported_seq":9114,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:11.164839+0000","last_change":"2023-10-09T05:37:01.638178+0000","last_active":"2023-10-09T05:40:11.164839+0000","last_peered":"2023-10-09T05:40:11.164839+0000","last_clean":"2023-10-09T05:40:11.164839+0000","last_became_active":"2023-10-09T05:37:01.637822+0000","last_became_peered":"2023-10-09T05:37:01.637822+0000","last_unstale":"2023-10-09T05:40:11.164839+0000","last_undegraded":"2023-10-09T05:40:11.164839+0000","last_fullsized":"2023-10-09T05:40:11.164839+0000","mapping_epoch":162,"log_start":"168'2600","ondisk_log_start":"168'2600","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6468,"log_dups_size":0,"ondisk_log_size":6468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:13.240069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134086656,"num_objects":2046,"num_object_clones":0,"num_object_copies":6138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9068,"num_write_kb":355648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":189,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.822969+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:40:05.822969+0000","last_peered":"2023-10-09T05:40:05.822969+0000","last_clean":"2023-10-09T05:40:05.822969+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:40:05.822969+0000","last_undegraded":"2023-10-09T05:40:05.822969+0000","last_fullsized":"2023-10-09T05:40:05.822969+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":173,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.824165+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:40:05.824165+0000","last_peered":"2023-10-09T05:40:05.824165+0000","last_clean":"2023-10-09T05:40:05.824165+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:40:05.824165+0000","last_undegraded":"2023-10-09T05:40:05.824165+0000","last_fullsized":"2023-10-09T05:40:05.824165+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"198'8948","reported_seq":8993,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:09.894970+0000","last_change":"2023-10-09T05:37:01.438917+0000","last_active":"2023-10-09T05:40:09.894970+0000","last_peered":"2023-10-09T05:40:09.894970+0000","last_clean":"2023-10-09T05:40:09.894970+0000","last_became_active":"2023-10-09T05:37:01.438658+0000","last_became_peered":"2023-10-09T05:37:01.438658+0000","last_unstale":"2023-10-09T05:40:09.894970+0000","last_undegraded":"2023-10-09T05:40:09.894970+0000","last_fullsized":"2023-10-09T05:40:09.894970+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8948,"log_dups_size":0,"ondisk_log_size":8948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138477592,"num_objects":2114,"num_object_clones":0,"num_object_copies":6342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2114,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8948,"num_write_kb":353921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":196,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:06.872363+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:40:06.872363+0000","last_peered":"2023-10-09T05:40:06.872363+0000","last_clean":"2023-10-09T05:40:06.872363+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:40:06.872363+0000","last_undegraded":"2023-10-09T05:40:06.872363+0000","last_fullsized":"2023-10-09T05:40:06.872363+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":145,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.812045+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:40:05.812045+0000","last_peered":"2023-10-09T05:40:05.812045+0000","last_clean":"2023-10-09T05:40:05.812045+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:40:05.812045+0000","last_undegraded":"2023-10-09T05:40:05.812045+0000","last_fullsized":"2023-10-09T05:40:05.812045+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"198'8997","reported_seq":9043,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:08.096811+0000","last_change":"2023-10-09T05:37:01.440093+0000","last_active":"2023-10-09T05:40:08.096811+0000","last_peered":"2023-10-09T05:40:08.096811+0000","last_clean":"2023-10-09T05:40:08.096811+0000","last_became_active":"2023-10-09T05:37:01.439799+0000","last_became_peered":"2023-10-09T05:37:01.439799+0000","last_unstale":"2023-10-09T05:40:08.096811+0000","last_undegraded":"2023-10-09T05:40:08.096811+0000","last_fullsized":"2023-10-09T05:40:08.096811+0000","mapping_epoch":162,"log_start":"166'100","ondisk_log_start":"166'100","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8897,"log_dups_size":0,"ondisk_log_size":8897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:28.734471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138215424,"num_objects":2109,"num_object_clones":0,"num_object_copies":6327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8997,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":767,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.813320+0000","last_change":"2023-10-09T05:40:04.806908+0000","last_active":"2023-10-09T05:40:05.813320+0000","last_peered":"2023-10-09T05:40:05.813320+0000","last_clean":"2023-10-09T05:40:05.813320+0000","last_became_active":"2023-10-09T05:40:04.806232+0000","last_became_peered":"2023-10-09T05:40:04.806232+0000","last_unstale":"2023-10-09T05:40:05.813320+0000","last_undegraded":"2023-10-09T05:40:05.813320+0000","last_fullsized":"2023-10-09T05:40:05.813320+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:34.087553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":180,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.812359+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:40:05.812359+0000","last_peered":"2023-10-09T05:40:05.812359+0000","last_clean":"2023-10-09T05:40:05.812359+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:40:05.812359+0000","last_undegraded":"2023-10-09T05:40:05.812359+0000","last_fullsized":"2023-10-09T05:40:05.812359+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":130,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.811801+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:40:05.811801+0000","last_peered":"2023-10-09T05:40:05.811801+0000","last_clean":"2023-10-09T05:40:05.811801+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:40:05.811801+0000","last_undegraded":"2023-10-09T05:40:05.811801+0000","last_fullsized":"2023-10-09T05:40:05.811801+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"198'9036","reported_seq":14972,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:11.169292+0000","last_change":"2023-10-09T05:38:10.132330+0000","last_active":"2023-10-09T05:40:11.169292+0000","last_peered":"2023-10-09T05:40:11.169292+0000","last_clean":"2023-10-09T05:40:11.169292+0000","last_became_active":"2023-10-09T05:38:10.127080+0000","last_became_peered":"2023-10-09T05:38:10.127080+0000","last_unstale":"2023-10-09T05:40:11.169292+0000","last_undegraded":"2023-10-09T05:40:11.169292+0000","last_fullsized":"2023-10-09T05:40:11.169292+0000","mapping_epoch":179,"log_start":"168'2600","ondisk_log_start":"168'2600","created":162,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6436,"log_dups_size":0,"ondisk_log_size":6436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:12.659227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138412032,"num_objects":2112,"num_object_clones":0,"num_object_copies":6336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9036,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":192413696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":634,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.810939+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:40:05.810939+0000","last_peered":"2023-10-09T05:40:05.810939+0000","last_clean":"2023-10-09T05:40:05.810939+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:40:05.810939+0000","last_undegraded":"2023-10-09T05:40:05.810939+0000","last_fullsized":"2023-10-09T05:40:05.810939+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":165,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823270+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:40:05.823270+0000","last_peered":"2023-10-09T05:40:05.823270+0000","last_clean":"2023-10-09T05:40:05.823270+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:40:05.823270+0000","last_undegraded":"2023-10-09T05:40:05.823270+0000","last_fullsized":"2023-10-09T05:40:05.823270+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":177,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.811251+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:40:05.811251+0000","last_peered":"2023-10-09T05:40:05.811251+0000","last_clean":"2023-10-09T05:40:05.811251+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:40:05.811251+0000","last_undegraded":"2023-10-09T05:40:05.811251+0000","last_fullsized":"2023-10-09T05:40:05.811251+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"198'8936","reported_seq":8991,"reported_epoch":198,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:10.763374+0000","last_change":"2023-10-09T05:37:40.649912+0000","last_active":"2023-10-09T05:40:10.763374+0000","last_peered":"2023-10-09T05:40:10.763374+0000","last_clean":"2023-10-09T05:37:34.677100+0000","last_became_active":"2023-10-09T05:37:40.593033+0000","last_became_peered":"2023-10-09T05:37:40.593033+0000","last_unstale":"2023-10-09T05:40:10.763374+0000","last_undegraded":"2023-10-09T05:37:40.546647+0000","last_fullsized":"2023-10-09T05:37:40.543498+0000","mapping_epoch":170,"log_start":"166'800","ondisk_log_start":"166'800","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8136,"log_dups_size":0,"ondisk_log_size":8136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134610944,"num_objects":2054,"num_object_clones":0,"num_object_copies":6162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8936,"num_write_kb":351680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":-796}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":512,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.811946+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:40:05.811946+0000","last_peered":"2023-10-09T05:40:05.811946+0000","last_clean":"2023-10-09T05:40:05.811946+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:40:05.811946+0000","last_undegraded":"2023-10-09T05:40:05.811946+0000","last_fullsized":"2023-10-09T05:40:05.811946+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":145,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.815746+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:40:05.815746+0000","last_peered":"2023-10-09T05:40:05.815746+0000","last_clean":"2023-10-09T05:40:05.815746+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:40:05.815746+0000","last_undegraded":"2023-10-09T05:40:05.815746+0000","last_fullsized":"2023-10-09T05:40:05.815746+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":175,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.812107+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:40:05.812107+0000","last_peered":"2023-10-09T05:40:05.812107+0000","last_clean":"2023-10-09T05:40:05.812107+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:40:05.812107+0000","last_undegraded":"2023-10-09T05:40:05.812107+0000","last_fullsized":"2023-10-09T05:40:05.812107+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"198'8981","reported_seq":9025,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:09.887771+0000","last_change":"2023-10-09T05:37:01.438236+0000","last_active":"2023-10-09T05:40:09.887771+0000","last_peered":"2023-10-09T05:40:09.887771+0000","last_clean":"2023-10-09T05:40:09.887771+0000","last_became_active":"2023-10-09T05:37:01.437917+0000","last_became_peered":"2023-10-09T05:37:01.437917+0000","last_unstale":"2023-10-09T05:40:09.887771+0000","last_undegraded":"2023-10-09T05:40:09.887771+0000","last_fullsized":"2023-10-09T05:40:09.887771+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8981,"log_dups_size":0,"ondisk_log_size":8981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137166848,"num_objects":2093,"num_object_clones":0,"num_object_copies":6279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8981,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":809,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.811059+0000","last_change":"2023-10-09T05:40:04.805258+0000","last_active":"2023-10-09T05:40:05.811059+0000","last_peered":"2023-10-09T05:40:05.811059+0000","last_clean":"2023-10-09T05:40:05.811059+0000","last_became_active":"2023-10-09T05:40:04.804776+0000","last_became_peered":"2023-10-09T05:40:04.804776+0000","last_unstale":"2023-10-09T05:40:05.811059+0000","last_undegraded":"2023-10-09T05:40:05.811059+0000","last_fullsized":"2023-10-09T05:40:05.811059+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":346,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.816542+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:40:05.816542+0000","last_peered":"2023-10-09T05:40:05.816542+0000","last_clean":"2023-10-09T05:40:05.816542+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:40:05.816542+0000","last_undegraded":"2023-10-09T05:40:05.816542+0000","last_fullsized":"2023-10-09T05:40:05.816542+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":162,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.904812+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:40:05.904812+0000","last_peered":"2023-10-09T05:40:05.904812+0000","last_clean":"2023-10-09T05:40:05.904812+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:40:05.904812+0000","last_undegraded":"2023-10-09T05:40:05.904812+0000","last_fullsized":"2023-10-09T05:40:05.904812+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":171,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.821279+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:40:05.821279+0000","last_peered":"2023-10-09T05:40:05.821279+0000","last_clean":"2023-10-09T05:40:05.821279+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:40:05.821279+0000","last_undegraded":"2023-10-09T05:40:05.821279+0000","last_fullsized":"2023-10-09T05:40:05.821279+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"198'8838","reported_seq":8884,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:08.752451+0000","last_change":"2023-10-09T05:37:01.440172+0000","last_active":"2023-10-09T05:40:08.752451+0000","last_peered":"2023-10-09T05:40:08.752451+0000","last_clean":"2023-10-09T05:40:08.752451+0000","last_became_active":"2023-10-09T05:37:01.439932+0000","last_became_peered":"2023-10-09T05:37:01.439932+0000","last_unstale":"2023-10-09T05:40:08.752451+0000","last_undegraded":"2023-10-09T05:40:08.752451+0000","last_fullsized":"2023-10-09T05:40:08.752451+0000","mapping_epoch":162,"log_start":"167'2400","ondisk_log_start":"167'2400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6438,"log_dups_size":0,"ondisk_log_size":6438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:57.826704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139067392,"num_objects":2122,"num_object_clones":0,"num_object_copies":6366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8838,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":755,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.933320+0000","last_change":"2023-10-09T05:40:05.930112+0000","last_active":"2023-10-09T05:40:05.933320+0000","last_peered":"2023-10-09T05:40:05.933320+0000","last_clean":"2023-10-09T05:40:05.933320+0000","last_became_active":"2023-10-09T05:40:05.929574+0000","last_became_peered":"2023-10-09T05:40:05.929574+0000","last_unstale":"2023-10-09T05:40:05.933320+0000","last_undegraded":"2023-10-09T05:40:05.933320+0000","last_fullsized":"2023-10-09T05:40:05.933320+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:06.982326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":150,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.816845+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:40:05.816845+0000","last_peered":"2023-10-09T05:40:05.816845+0000","last_clean":"2023-10-09T05:40:05.816845+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:40:05.816845+0000","last_undegraded":"2023-10-09T05:40:05.816845+0000","last_fullsized":"2023-10-09T05:40:05.816845+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":165,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.817788+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:40:05.817788+0000","last_peered":"2023-10-09T05:40:05.817788+0000","last_clean":"2023-10-09T05:40:05.817788+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:40:05.817788+0000","last_undegraded":"2023-10-09T05:40:05.817788+0000","last_fullsized":"2023-10-09T05:40:05.817788+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"198'8978","reported_seq":9024,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:08.796364+0000","last_change":"2023-10-09T05:37:01.636967+0000","last_active":"2023-10-09T05:40:08.796364+0000","last_peered":"2023-10-09T05:40:08.796364+0000","last_clean":"2023-10-09T05:40:08.796364+0000","last_became_active":"2023-10-09T05:37:01.636602+0000","last_became_peered":"2023-10-09T05:37:01.636602+0000","last_unstale":"2023-10-09T05:40:08.796364+0000","last_undegraded":"2023-10-09T05:40:08.796364+0000","last_fullsized":"2023-10-09T05:40:08.796364+0000","mapping_epoch":162,"log_start":"167'2500","ondisk_log_start":"167'2500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6478,"log_dups_size":0,"ondisk_log_size":6478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:11.668143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138543104,"num_objects":2114,"num_object_clones":0,"num_object_copies":6342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8978,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":387,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.817477+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:40:05.817477+0000","last_peered":"2023-10-09T05:40:05.817477+0000","last_clean":"2023-10-09T05:40:05.817477+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:40:05.817477+0000","last_undegraded":"2023-10-09T05:40:05.817477+0000","last_fullsized":"2023-10-09T05:40:05.817477+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":145,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.813545+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:40:05.813545+0000","last_peered":"2023-10-09T05:40:05.813545+0000","last_clean":"2023-10-09T05:40:05.813545+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:40:05.813545+0000","last_undegraded":"2023-10-09T05:40:05.813545+0000","last_fullsized":"2023-10-09T05:40:05.813545+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":159,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.817259+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:40:05.817259+0000","last_peered":"2023-10-09T05:40:05.817259+0000","last_clean":"2023-10-09T05:40:05.817259+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:40:05.817259+0000","last_undegraded":"2023-10-09T05:40:05.817259+0000","last_fullsized":"2023-10-09T05:40:05.817259+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"198'9082","reported_seq":9126,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:09.943469+0000","last_change":"2023-10-09T05:37:01.440074+0000","last_active":"2023-10-09T05:40:09.943469+0000","last_peered":"2023-10-09T05:40:09.943469+0000","last_clean":"2023-10-09T05:40:09.943469+0000","last_became_active":"2023-10-09T05:37:01.439808+0000","last_became_peered":"2023-10-09T05:37:01.439808+0000","last_unstale":"2023-10-09T05:40:09.943469+0000","last_undegraded":"2023-10-09T05:40:09.943469+0000","last_fullsized":"2023-10-09T05:40:09.943469+0000","mapping_epoch":162,"log_start":"167'1500","ondisk_log_start":"167'1500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7582,"log_dups_size":0,"ondisk_log_size":7582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133431296,"num_objects":2036,"num_object_clones":0,"num_object_copies":6108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9082,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":197,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T05:40:06.877708+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:40:06.877708+0000","last_peered":"2023-10-09T05:40:06.877708+0000","last_clean":"2023-10-09T05:40:06.877708+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:40:06.877708+0000","last_undegraded":"2023-10-09T05:40:06.877708+0000","last_fullsized":"2023-10-09T05:40:06.877708+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":586,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.814076+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:40:05.814076+0000","last_peered":"2023-10-09T05:40:05.814076+0000","last_clean":"2023-10-09T05:40:05.814076+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:40:05.814076+0000","last_undegraded":"2023-10-09T05:40:05.814076+0000","last_fullsized":"2023-10-09T05:40:05.814076+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":190,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.905651+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:40:05.905651+0000","last_peered":"2023-10-09T05:40:05.905651+0000","last_clean":"2023-10-09T05:40:05.905651+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:40:05.905651+0000","last_undegraded":"2023-10-09T05:40:05.905651+0000","last_fullsized":"2023-10-09T05:40:05.905651+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"198'8819","reported_seq":9805,"reported_epoch":198,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:10.788006+0000","last_change":"2023-10-09T05:40:03.773505+0000","last_active":"2023-10-09T05:40:10.788006+0000","last_peered":"2023-10-09T05:40:10.788006+0000","last_clean":"2023-10-09T05:37:34.656587+0000","last_became_active":"2023-10-09T05:37:36.833510+0000","last_became_peered":"2023-10-09T05:37:36.833510+0000","last_unstale":"2023-10-09T05:40:10.788006+0000","last_undegraded":"2023-10-09T05:37:36.701190+0000","last_fullsized":"2023-10-09T05:37:36.681999+0000","mapping_epoch":170,"log_start":"166'700","ondisk_log_start":"166'700","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8119,"log_dups_size":0,"ondisk_log_size":8119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129040384,"num_objects":1969,"num_object_clones":0,"num_object_copies":5907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2410,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8819,"num_write_kb":345216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":467,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-441}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":682,"reported_epoch":197,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:05.820507+0000","last_change":"2023-10-09T05:37:36.917512+0000","last_active":"2023-10-09T05:40:05.820507+0000","last_peered":"2023-10-09T05:40:05.820507+0000","last_clean":"2023-10-09T05:37:30.688967+0000","last_became_active":"2023-10-09T05:37:36.700218+0000","last_became_peered":"2023-10-09T05:37:36.700218+0000","last_unstale":"2023-10-09T05:40:05.820507+0000","last_undegraded":"2023-10-09T05:37:36.917335+0000","last_fullsized":"2023-10-09T05:37:36.682410+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":153,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.817745+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:40:05.817745+0000","last_peered":"2023-10-09T05:40:05.817745+0000","last_clean":"2023-10-09T05:40:05.817745+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:40:05.817745+0000","last_undegraded":"2023-10-09T05:40:05.817745+0000","last_fullsized":"2023-10-09T05:40:05.817745+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":149,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823037+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:40:05.823037+0000","last_peered":"2023-10-09T05:40:05.823037+0000","last_clean":"2023-10-09T05:40:05.823037+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:40:05.823037+0000","last_undegraded":"2023-10-09T05:40:05.823037+0000","last_fullsized":"2023-10-09T05:40:05.823037+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":162,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.817225+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:40:05.817225+0000","last_peered":"2023-10-09T05:40:05.817225+0000","last_clean":"2023-10-09T05:40:05.817225+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:40:05.817225+0000","last_undegraded":"2023-10-09T05:40:05.817225+0000","last_fullsized":"2023-10-09T05:40:05.817225+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":189,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.811481+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:40:05.811481+0000","last_peered":"2023-10-09T05:40:05.811481+0000","last_clean":"2023-10-09T05:40:05.811481+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:40:05.811481+0000","last_undegraded":"2023-10-09T05:40:05.811481+0000","last_fullsized":"2023-10-09T05:40:05.811481+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":167,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.811166+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:40:05.811166+0000","last_peered":"2023-10-09T05:40:05.811166+0000","last_clean":"2023-10-09T05:40:05.811166+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:40:05.811166+0000","last_undegraded":"2023-10-09T05:40:05.811166+0000","last_fullsized":"2023-10-09T05:40:05.811166+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":179,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823338+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:40:05.823338+0000","last_peered":"2023-10-09T05:40:05.823338+0000","last_clean":"2023-10-09T05:40:05.823338+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:40:05.823338+0000","last_undegraded":"2023-10-09T05:40:05.823338+0000","last_fullsized":"2023-10-09T05:40:05.823338+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":152,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.812250+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:40:05.812250+0000","last_peered":"2023-10-09T05:40:05.812250+0000","last_clean":"2023-10-09T05:40:05.812250+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:40:05.812250+0000","last_undegraded":"2023-10-09T05:40:05.812250+0000","last_fullsized":"2023-10-09T05:40:05.812250+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":179,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.823632+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:40:05.823632+0000","last_peered":"2023-10-09T05:40:05.823632+0000","last_clean":"2023-10-09T05:40:05.823632+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:40:05.823632+0000","last_undegraded":"2023-10-09T05:40:05.823632+0000","last_fullsized":"2023-10-09T05:40:05.823632+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":177,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.919909+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:40:05.919909+0000","last_peered":"2023-10-09T05:40:05.919909+0000","last_clean":"2023-10-09T05:40:05.919909+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:40:05.919909+0000","last_undegraded":"2023-10-09T05:40:05.919909+0000","last_fullsized":"2023-10-09T05:40:05.919909+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":189,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.904995+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:40:05.904995+0000","last_peered":"2023-10-09T05:40:05.904995+0000","last_clean":"2023-10-09T05:40:05.904995+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:40:05.904995+0000","last_undegraded":"2023-10-09T05:40:05.904995+0000","last_fullsized":"2023-10-09T05:40:05.904995+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":153,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.819274+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:40:05.819274+0000","last_peered":"2023-10-09T05:40:05.819274+0000","last_clean":"2023-10-09T05:40:05.819274+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:40:05.819274+0000","last_undegraded":"2023-10-09T05:40:05.819274+0000","last_fullsized":"2023-10-09T05:40:05.819274+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":164,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T05:40:05.822871+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:40:05.822871+0000","last_peered":"2023-10-09T05:40:05.822871+0000","last_clean":"2023-10-09T05:40:05.822871+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:40:05.822871+0000","last_undegraded":"2023-10-09T05:40:05.822871+0000","last_fullsized":"2023-10-09T05:40:05.822871+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2165506072,"num_objects":33044,"num_object_clones":0,"num_object_copies":99132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33044,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":143678,"num_write_kb":5655041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9325,"num_bytes_recovered":530448384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7255633920,"data_stored":7255621704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126478,"ondisk_log_size":126478,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846659,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626685,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1357660,"kb_used_data":1056276,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92387492,"statfs":{"total":95995035648,"available":94604791808,"internally_reserved":0,"allocated":1081626624,"data_stored":1081105728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:23 2023","interfaces":[{"interface":"back","average":{"1min":6.093,"5min":4.5960000000000001,"15min":2.1890000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":99.787999999999997,"5min":99.787999999999997,"15min":99.787999999999997},"last":3.274},{"interface":"front","average":{"1min":5.1890000000000001,"5min":4.234,"15min":2.0710000000000002},"min":{"1min":0.25600000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":99.844999999999999,"5min":99.844999999999999,"15min":99.844999999999999},"last":4.2960000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:39:46 2023","interfaces":[{"interface":"back","average":{"1min":13.882,"5min":5.0300000000000002,"15min":2.2690000000000001},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.25},"max":{"1min":137.02199999999999,"5min":137.02199999999999,"15min":137.02199999999999},"last":3.1030000000000002},{"interface":"front","average":{"1min":13.885,"5min":5.2430000000000003,"15min":2.355},"min":{"1min":0.38100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":137.06700000000001,"5min":137.06700000000001,"15min":137.06700000000001},"last":3.0910000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:39:23 2023","interfaces":[{"interface":"back","average":{"1min":8.2159999999999993,"5min":4.593,"15min":2.5329999999999999},"min":{"1min":0.29499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":115.2,"5min":115.2,"15min":115.2},"last":3.6219999999999999},{"interface":"front","average":{"1min":8.2409999999999997,"5min":4.4180000000000001,"15min":2.4950000000000001},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":115.297,"5min":115.297,"15min":115.297},"last":3.6280000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:39:46 2023","interfaces":[{"interface":"back","average":{"1min":12.359999999999999,"5min":4.7290000000000001,"15min":2.4140000000000001},"min":{"1min":0.307,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":115.244,"5min":115.244,"15min":115.244},"last":2.7919999999999998},{"interface":"front","average":{"1min":12.371,"5min":4.9870000000000001,"15min":2.5099999999999998},"min":{"1min":0.36499999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":115.34999999999999,"5min":115.34999999999999,"15min":115.34999999999999},"last":3.0049999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":5.181,"5min":2.177,"15min":1.7150000000000001},"min":{"1min":0.249,"5min":0.245,"15min":0.23400000000000001},"max":{"1min":115.214,"5min":115.214,"15min":125.64},"last":0.27100000000000002},{"interface":"front","average":{"1min":5.282,"5min":1.766,"15min":1.597},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":115.22799999999999,"5min":115.22799999999999,"15min":125.625},"last":0.30599999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":7.7560000000000002,"5min":2.8050000000000002,"15min":1.819},"min":{"1min":0.31,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":115.316,"5min":115.316,"15min":127.35899999999999},"last":2.1309999999999998},{"interface":"front","average":{"1min":7.601,"5min":2.339,"15min":1.704},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29199999999999998},"max":{"1min":115.30500000000001,"5min":115.30500000000001,"15min":128.93799999999999},"last":0.33600000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:39:47 2023","interfaces":[{"interface":"back","average":{"1min":5.5990000000000002,"5min":2.5710000000000002,"15min":2.9550000000000001},"min":{"1min":0.35899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":115.514,"5min":115.514,"15min":115.514},"last":0.40300000000000002},{"interface":"front","average":{"1min":5.6500000000000004,"5min":2.0600000000000001,"15min":2.081},"min":{"1min":0.35499999999999998,"5min":0.317,"15min":0.317},"max":{"1min":116.53100000000001,"5min":116.53100000000001,"15min":116.53100000000001},"last":0.46999999999999997}]}]},{"osd":1,"up_from":93,"seq":399431958646,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1069192,"kb_used_data":812992,"kb_used_omap":7,"kb_used_meta":256120,"kb_avail":92675960,"statfs":{"total":95995035648,"available":94900183040,"internally_reserved":0,"allocated":832503808,"data_stored":831768918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":262267406},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9460000000000002,"5min":5.7850000000000001,"15min":2.3220000000000001},"min":{"1min":0.20300000000000001,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":13.214,"5min":207.858,"15min":207.858},"last":3.3010000000000002},{"interface":"front","average":{"1min":2.4529999999999998,"5min":6.7469999999999999,"15min":2.6859999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.849,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.46500000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":5.3289999999999997,"5min":7.3360000000000003,"15min":2.8220000000000001},"min":{"1min":0.22700000000000001,"5min":0.184,"15min":0.184},"max":{"1min":32.686999999999998,"5min":101.367,"15min":101.367},"last":0.26200000000000001},{"interface":"front","average":{"1min":4.468,"5min":6.7439999999999998,"15min":2.6259999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":19.975999999999999,"5min":98.409999999999997,"15min":98.409999999999997},"last":1.3109999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":6.1749999999999998,"5min":3.8559999999999999,"15min":1.744},"min":{"1min":0.35399999999999998,"5min":0.223,"15min":0.223},"max":{"1min":42.445999999999998,"5min":83.885000000000005,"15min":83.885000000000005},"last":0.28899999999999998},{"interface":"front","average":{"1min":5.4379999999999997,"5min":4.7290000000000001,"15min":1.954},"min":{"1min":0.373,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":42.243000000000002,"5min":84.641999999999996,"15min":84.641999999999996},"last":0.501}]},{"osd":4,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":4.1799999999999997,"15min":1.9670000000000001},"min":{"1min":0.35599999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.813000000000001,"5min":141.32900000000001,"15min":141.32900000000001},"last":0.68700000000000006},{"interface":"front","average":{"1min":2.4129999999999998,"5min":2.9420000000000002,"15min":1.532},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.4509999999999996,"5min":141.15899999999999,"15min":141.15899999999999},"last":0.56100000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":3.9020000000000001,"15min":1.901},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.6769999999999996,"5min":235.18700000000001,"15min":235.18700000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.7330000000000001,"5min":4.8550000000000004,"15min":2.319},"min":{"1min":0.38300000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":10.779,"5min":235.79300000000001,"15min":235.79300000000001},"last":0.72299999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":7.46,"5min":2.9140000000000001,"15min":1.5289999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":103.60899999999999,"5min":103.60899999999999,"15min":103.60899999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":7.3520000000000003,"5min":3.8500000000000001,"15min":1.837},"min":{"1min":0.38,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":103.679,"5min":103.679,"15min":103.679},"last":0.747}]},{"osd":7,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7719999999999998,"5min":2.9489999999999998,"15min":2.3149999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.7390000000000008,"5min":126.68300000000001,"15min":126.68300000000001},"last":0.58199999999999996},{"interface":"front","average":{"1min":2.927,"5min":3.0800000000000001,"15min":2.4119999999999999},"min":{"1min":0.46600000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.7720000000000002,"5min":126.738,"15min":126.738},"last":0.60999999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122172,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":892004,"kb_used_data":554012,"kb_used_omap":8,"kb_used_meta":337975,"kb_avail":92853148,"statfs":{"total":95995035648,"available":95081623552,"internally_reserved":0,"allocated":567308288,"data_stored":566546993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":346087176},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":1.9379999999999999,"15min":1.0760000000000001},"min":{"1min":0.219,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":12.005000000000001,"5min":14.964,"15min":14.964},"last":0.32800000000000001},{"interface":"front","average":{"1min":3.4350000000000001,"5min":2.5179999999999998,"15min":1.2809999999999999},"min":{"1min":0.22900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.218999999999999,"5min":15.218999999999999,"15min":15.218999999999999},"last":0.307}]},{"osd":2,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":3.4180000000000001,"5min":3.8109999999999999,"15min":1.8879999999999999},"min":{"1min":0.28999999999999998,"5min":0.223,"15min":0.214},"max":{"1min":12.759,"5min":79.911000000000001,"15min":79.911000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":4.085,"5min":3.5089999999999999,"15min":1.766},"min":{"1min":0.246,"5min":0.218,"15min":0.218},"max":{"1min":12.234999999999999,"5min":80.105000000000004,"15min":80.105000000000004},"last":0.40300000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":4.0839999999999996,"5min":3.2730000000000001,"15min":1.5609999999999999},"min":{"1min":0.27000000000000002,"5min":0.23999999999999999,"15min":0.22800000000000001},"max":{"1min":10.519,"5min":31.594000000000001,"15min":31.594000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":4.3410000000000002,"5min":3.3730000000000002,"15min":1.651},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.222},"max":{"1min":16.937999999999999,"5min":31.641999999999999,"15min":31.641999999999999},"last":0.35099999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":2.0539999999999998,"15min":1.163},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.1989999999999998,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.443},{"interface":"front","average":{"1min":2.661,"5min":2.7010000000000001,"15min":1.4350000000000001},"min":{"1min":0.42099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.2149999999999999,"5min":69.102000000000004,"15min":69.102000000000004},"last":0.45700000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":8.0129999999999999,"5min":3.081,"15min":1.5509999999999999},"min":{"1min":0.45200000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":115.175,"5min":115.175,"15min":115.175},"last":281.733},{"interface":"front","average":{"1min":7.0750000000000002,"5min":2.9710000000000001,"15min":1.5069999999999999},"min":{"1min":0.36499999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":115.502,"5min":115.502,"15min":115.502},"last":282.95400000000001}]},{"osd":6,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":1.9239999999999999,"15min":1.2050000000000001},"min":{"1min":0.41999999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.3409999999999993,"5min":9.3409999999999993,"15min":9.9809999999999999},"last":0.497},{"interface":"front","average":{"1min":2.153,"5min":2.02,"15min":1.198},"min":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":7.266,"5min":8.0999999999999996,"15min":8.0999999999999996},"last":0.46999999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":1.544,"15min":1.3999999999999999},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.9740000000000002,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.57499999999999996},{"interface":"front","average":{"1min":3.2999999999999998,"5min":2.3879999999999999,"15min":2.3889999999999998},"min":{"1min":0.5,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.4100000000000001,"5min":11.068,"15min":11.068},"last":0.52700000000000002}]}]},{"osd":2,"up_from":98,"seq":420906795120,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1552284,"kb_used_data":1166548,"kb_used_omap":11,"kb_used_meta":385524,"kb_avail":92192868,"statfs":{"total":95995035648,"available":94405496832,"internally_reserved":0,"allocated":1194545152,"data_stored":1193735874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":394777173},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":3.476,"5min":2.8010000000000002,"15min":1.2669999999999999},"min":{"1min":0.23400000000000001,"5min":0.183,"15min":0.17199999999999999},"max":{"1min":10.098000000000001,"5min":32.122,"15min":32.122},"last":3.5219999999999998},{"interface":"front","average":{"1min":3.27,"5min":2.3530000000000002,"15min":1.1399999999999999},"min":{"1min":0.25800000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.5850000000000009,"5min":34.066000000000003,"15min":34.066000000000003},"last":7.6210000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":14.41,"5min":5.3600000000000003,"15min":2.133},"min":{"1min":0.34499999999999997,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":98.575000000000003,"5min":98.575000000000003,"15min":98.575000000000003},"last":9.3469999999999995},{"interface":"front","average":{"1min":12.257,"5min":4.6779999999999999,"15min":1.867},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":90.135000000000005,"5min":98.397000000000006,"15min":98.397000000000006},"last":8.9809999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":15.612,"5min":5.4580000000000002,"15min":2.2930000000000001},"min":{"1min":0.55200000000000005,"5min":0.249,"15min":0.24199999999999999},"max":{"1min":150.65899999999999,"5min":150.65899999999999,"15min":150.65899999999999},"last":9.0109999999999992},{"interface":"front","average":{"1min":17.797000000000001,"5min":6.25,"15min":2.601},"min":{"1min":0.54900000000000004,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":150.851,"5min":150.851,"15min":150.851},"last":6.5259999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":2.8050000000000002,"15min":1.4019999999999999},"min":{"1min":0.29099999999999998,"5min":0.28999999999999998,"15min":0.27700000000000002},"max":{"1min":11.125999999999999,"5min":115.84999999999999,"15min":115.84999999999999},"last":8.5760000000000005},{"interface":"front","average":{"1min":2.976,"5min":3.504,"15min":1.675},"min":{"1min":0.34799999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.147,"5min":116.149,"15min":116.149},"last":3.589}]},{"osd":5,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":2.5409999999999999,"15min":1.369},"min":{"1min":0.39900000000000002,"5min":0.312,"15min":0.312},"max":{"1min":7.8220000000000001,"5min":24.597999999999999,"15min":24.597999999999999},"last":5.0389999999999997},{"interface":"front","average":{"1min":1.867,"5min":2.089,"15min":1.1779999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.097,"5min":23.919,"15min":23.919},"last":8.6099999999999994}]},{"osd":6,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":2.5129999999999999,"15min":1.327},"min":{"1min":0.39900000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.183,"5min":20.503,"15min":20.503},"last":3.5619999999999998},{"interface":"front","average":{"1min":2.8519999999999999,"5min":2.2170000000000001,"15min":1.2470000000000001},"min":{"1min":0.45000000000000001,"5min":0.372,"15min":0.34499999999999997},"max":{"1min":7.8730000000000002,"5min":9.4049999999999994,"15min":9.4049999999999994},"last":5.0810000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:39:52 2023","interfaces":[{"interface":"back","average":{"1min":3.294,"5min":3.0369999999999999,"15min":2.6949999999999998},"min":{"1min":0.436,"5min":0.374,"15min":0.374},"max":{"1min":10.897,"5min":24.030999999999999,"15min":24.030999999999999},"last":3.6139999999999999},{"interface":"front","average":{"1min":3.0299999999999998,"5min":2.7549999999999999,"15min":2.0710000000000002},"min":{"1min":0.48899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.8230000000000004,"5min":24.178000000000001,"15min":24.178000000000001},"last":5.1260000000000003}]}]},{"osd":3,"up_from":103,"seq":442381631594,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1714948,"kb_used_data":1448572,"kb_used_omap":4,"kb_used_meta":266107,"kb_avail":92030204,"statfs":{"total":95995035648,"available":94238928896,"internally_reserved":0,"allocated":1483337728,"data_stored":1482606028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":272494174},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":14.382,"5min":4.5739999999999998,"15min":2.129},"min":{"1min":0.19800000000000001,"5min":0.183,"15min":0.183},"max":{"1min":167.82400000000001,"5min":167.82400000000001,"15min":167.82400000000001},"last":1.4019999999999999},{"interface":"front","average":{"1min":15.930999999999999,"5min":5.0309999999999997,"15min":2.3330000000000002},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.214},"max":{"1min":167.89599999999999,"5min":167.89599999999999,"15min":167.89599999999999},"last":0.20899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":3.218,"5min":3.8919999999999999,"15min":1.657},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.663,"5min":83.921999999999997,"15min":83.921999999999997},"last":1.3480000000000001},{"interface":"front","average":{"1min":3.956,"5min":3.911,"15min":1.651},"min":{"1min":0.25,"5min":0.25,"15min":0.247},"max":{"1min":12.061999999999999,"5min":83.707999999999998,"15min":83.707999999999998},"last":0.27000000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":16.274000000000001,"5min":6.6950000000000003,"15min":3.298},"min":{"1min":0.26500000000000001,"5min":0.215,"15min":0.215},"max":{"1min":190.43299999999999,"5min":190.43299999999999,"15min":190.43299999999999},"last":1.9319999999999999},{"interface":"front","average":{"1min":21.088000000000001,"5min":7.8490000000000002,"15min":3.6960000000000002},"min":{"1min":0.317,"5min":0.28999999999999998,"15min":0.26000000000000001},"max":{"1min":190.267,"5min":190.267,"15min":190.267},"last":0.28499999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":6.7539999999999996,"5min":2.9540000000000002,"15min":1.7390000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.29699999999999999},"max":{"1min":51.591999999999999,"5min":51.591999999999999,"15min":51.591999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.9209999999999998,"5min":1.9790000000000001,"15min":1.3620000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":11.271000000000001,"5min":13.734,"15min":45.735999999999997},"last":1.4750000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":2.0379999999999998,"15min":1.325},"min":{"1min":0.45100000000000001,"5min":0.33500000000000002,"15min":0.29999999999999999},"max":{"1min":8.3640000000000008,"5min":8.4770000000000003,"15min":47.948999999999998},"last":0.32400000000000001},{"interface":"front","average":{"1min":3.1640000000000001,"5min":2.1659999999999999,"15min":1.4299999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.339,"5min":13.864000000000001,"15min":47.968000000000004},"last":1.5109999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.2120000000000002,"15min":1.321},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.313000000000001,"5min":13.831,"15min":13.831},"last":1.4430000000000001},{"interface":"front","average":{"1min":2.8260000000000001,"5min":2.1019999999999999,"15min":1.1299999999999999},"min":{"1min":0.45700000000000002,"5min":0.373,"15min":0.33900000000000002},"max":{"1min":8.3979999999999997,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":0.35999999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:39:53 2023","interfaces":[{"interface":"back","average":{"1min":3.9590000000000001,"5min":2.6699999999999999,"15min":2.3959999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":20.065000000000001,"5min":20.065000000000001,"15min":20.065000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":4.0590000000000002,"5min":2.6629999999999998,"15min":2.1960000000000002},"min":{"1min":0.40899999999999997,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":20.138000000000002,"5min":20.138000000000002,"15min":20.138000000000002},"last":0.42499999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019141,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":989916,"kb_used_data":680404,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92755236,"statfs":{"total":95995035648,"available":94981361664,"internally_reserved":0,"allocated":696733696,"data_stored":696212544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:20 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":2.1640000000000001,"15min":1.21},"min":{"1min":0.25600000000000001,"5min":0.214,"15min":0.214},"max":{"1min":7.4480000000000004,"5min":36.406999999999996,"15min":36.406999999999996},"last":0.26600000000000001},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.7810000000000001,"15min":1.454},"min":{"1min":0.28699999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.988,"5min":36.454999999999998,"15min":36.454999999999998},"last":1.2609999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":1.7829999999999999,"15min":1.1299999999999999},"min":{"1min":0.28599999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.194,"5min":10.073,"15min":31.073},"last":2.895},{"interface":"front","average":{"1min":2.6709999999999998,"5min":1.7869999999999999,"15min":1.2010000000000001},"min":{"1min":0.33500000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.5279999999999996,"5min":10.055,"15min":31.100000000000001},"last":2.9060000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:39:15 2023","interfaces":[{"interface":"back","average":{"1min":3.4870000000000001,"5min":8.1349999999999998,"15min":3.2909999999999999},"min":{"1min":0.316,"5min":0.314,"15min":0.26800000000000002},"max":{"1min":20.684000000000001,"5min":196.93700000000001,"15min":196.93700000000001},"last":2.956},{"interface":"front","average":{"1min":2.621,"5min":7.7030000000000003,"15min":3.2040000000000002},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":20.611000000000001,"5min":197.08099999999999,"15min":197.08099999999999},"last":1.1080000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":5.6639999999999997,"5min":6.8849999999999998,"15min":2.8439999999999999},"min":{"1min":0.35399999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":34.615000000000002,"5min":211.48699999999999,"15min":211.48699999999999},"last":2.5310000000000001},{"interface":"front","average":{"1min":5.8650000000000002,"5min":6.8010000000000002,"15min":2.8010000000000002},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.309},"max":{"1min":34.57,"5min":211.31100000000001,"15min":211.31100000000001},"last":1.9119999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":3.4489999999999998,"15min":1.5429999999999999},"min":{"1min":0.26300000000000001,"5min":0.25,"15min":0.249},"max":{"1min":3.5259999999999998,"5min":113.35899999999999,"15min":113.35899999999999},"last":0.434},{"interface":"front","average":{"1min":0.61399999999999999,"5min":3.302,"15min":1.52},"min":{"1min":0.35499999999999998,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":3.2160000000000002,"5min":113.32299999999999,"15min":113.32299999999999},"last":0.40000000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:39:57 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.999,"15min":0.70899999999999996},"min":{"1min":0.317,"5min":0.28999999999999998,"15min":0.28100000000000003},"max":{"1min":3.4390000000000001,"5min":11.944000000000001,"15min":11.944000000000001},"last":1.8740000000000001},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.3300000000000001,"15min":0.85899999999999999},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.6500000000000004,"5min":10.356,"15min":10.356},"last":0.48199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":1.4810000000000001,"15min":2.7400000000000002},"min":{"1min":0.37,"5min":0.312,"15min":0.312},"max":{"1min":4.5819999999999999,"5min":30.393999999999998,"15min":30.393999999999998},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.601,"15min":3.0870000000000002},"min":{"1min":0.36499999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5999999999999996,"5min":30.334,"15min":30.334},"last":1.534}]}]},{"osd":5,"up_from":36,"seq":154618822913,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":989840,"kb_used_data":684424,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92755312,"statfs":{"total":95995035648,"available":94981439488,"internally_reserved":0,"allocated":700850176,"data_stored":700336784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":6.3940000000000001,"15min":2.8650000000000002},"min":{"1min":0.249,"5min":0.224,"15min":0.19400000000000001},"max":{"1min":10.712999999999999,"5min":165.90199999999999,"15min":165.90199999999999},"last":8.3079999999999998},{"interface":"front","average":{"1min":2.1419999999999999,"5min":6.3799999999999999,"15min":2.9359999999999999},"min":{"1min":0.28999999999999998,"5min":0.215,"15min":0.215},"max":{"1min":8.6099999999999994,"5min":165.96700000000001,"15min":165.96700000000001},"last":1.2270000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":5.3810000000000002,"5min":3.7469999999999999,"15min":1.738},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":58.176000000000002,"5min":87.769000000000005,"15min":87.769000000000005},"last":7.6890000000000001},{"interface":"front","average":{"1min":7.1660000000000004,"5min":3.8940000000000001,"15min":1.7769999999999999},"min":{"1min":0.32200000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":54.591000000000001,"5min":87.864000000000004,"15min":87.864000000000004},"last":12.661}]},{"osd":2,"last update":"Mon Oct 9 05:39:20 2023","interfaces":[{"interface":"back","average":{"1min":5.9400000000000004,"5min":9.1470000000000002,"15min":3.5609999999999999},"min":{"1min":0.33800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":42.055,"5min":290.697,"15min":290.697},"last":8.3900000000000006},{"interface":"front","average":{"1min":4.2720000000000002,"5min":8.7880000000000003,"15min":3.4079999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":41.953000000000003,"5min":290.786,"15min":290.786},"last":6.7869999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":16.713999999999999,"5min":5.8200000000000003,"15min":2.4249999999999998},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":95.236999999999995,"5min":95.236999999999995,"15min":95.236999999999995},"last":2.165},{"interface":"front","average":{"1min":16.225000000000001,"5min":5.3680000000000003,"15min":2.3100000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":93.471000000000004,"5min":93.471000000000004,"15min":93.471000000000004},"last":7.9720000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.94399999999999995,"15min":0.68100000000000005},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":0.91100000000000003,"5min":9.2729999999999997,"15min":9.3510000000000009},"last":0.313},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.4990000000000001,"15min":0.92900000000000005},"min":{"1min":0.32700000000000001,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":1.4259999999999999,"5min":14.236000000000001,"15min":14.236000000000001},"last":0.33600000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:40:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1139999999999999,"5min":1.8120000000000001,"15min":1.3420000000000001},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.26000000000000001},"max":{"1min":27.146000000000001,"5min":27.146000000000001,"15min":95.010999999999996},"last":0.38400000000000001},{"interface":"front","average":{"1min":2.1269999999999998,"5min":1.4690000000000001,"15min":1.2290000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27700000000000002},"max":{"1min":27.215,"5min":27.215,"15min":94.194000000000003},"last":0.45800000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.27,"15min":1.573},"min":{"1min":0.34399999999999997,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.94099999999999995,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":1.3400000000000001,"15min":1.8620000000000001},"min":{"1min":0.35899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.5029999999999999,"5min":14.432,"15min":14.432},"last":2.2589999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":560205824,"data_stored":560201752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":825561088,"data_stored":825557016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1168572416,"data_stored":1168572416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1463943168,"data_stored":1463943168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":684261376,"data_stored":684261376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":685969408,"data_stored":685965336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1071579136,"data_stored":1071579136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":795541504,"data_stored":795541504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:40:15.490 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:40:15.525 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:40:15.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:15.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:15.635 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:15.663 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:15.720 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:15.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:15.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:15.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:15.857 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:15.918 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:40:15.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:15.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:16.027 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:16.065 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:16.068 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:40:16.068 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:40:16.186 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:16.500 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:16.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:16.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:16.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:16.636 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:16.696 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:16.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:16.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:16.808 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:16.841 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:16.889 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:16.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:16.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:16.994 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:17.044 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:17.205 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:17 smithi191 ceph-mon[161669]: pgmap v553: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 7925/93102 objects degraded (8.512%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:17.206 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1130395269' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:40:17.206 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:17 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:40:17.244 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:40:17.292 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:17 smithi008 ceph-mon[184418]: pgmap v553: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 7925/93102 objects degraded (8.512%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:17.292 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1130395269' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:40:17.293 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:17 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:40:17.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:17 smithi008 ceph-mon[191227]: pgmap v553: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 7925/93102 objects degraded (8.512%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:17.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1130395269' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:40:17.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:17 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:40:17.409 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:17.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:17.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:17.517 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:17.596 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:18.126 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:40:18.369 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:40:18.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:18.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:18.475 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:18.476 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[184418]: failed to encode map e199 with expected crc 2023-10-09T05:40:18.476 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:40:18.476 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[184418]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T05:40:18.476 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[184418]: failed to encode map e199 with expected crc 2023-10-09T05:40:18.476 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[184418]: failed to encode map e199 with expected crc 2023-10-09T05:40:18.476 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 7925/93102 objects degraded (8.512%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:18.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:18.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[191227]: failed to encode map e199 with expected crc 2023-10-09T05:40:18.476 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:40:18.477 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[191227]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T05:40:18.477 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[191227]: failed to encode map e199 with expected crc 2023-10-09T05:40:18.477 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[191227]: failed to encode map e199 with expected crc 2023-10-09T05:40:18.477 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 7925/93102 objects degraded (8.512%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:18.496 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:18.529 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:18 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:18.529 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:18 smithi191 ceph-mon[161669]: failed to encode map e199 with expected crc 2023-10-09T05:40:18.529 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:40:18.529 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:18 smithi191 ceph-mon[161669]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T05:40:18.529 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:18 smithi191 ceph-mon[161669]: failed to encode map e199 with expected crc 2023-10-09T05:40:18.530 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:18 smithi191 ceph-mon[161669]: failed to encode map e199 with expected crc 2023-10-09T05:40:18.530 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 7925/93102 objects degraded (8.512%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:18.534 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:18.583 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:18.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:18.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:18.690 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:18.727 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:19.009 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:19.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:19.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:19.115 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:19.156 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:19.178 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:19.215 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:19.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:19.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:19.322 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:19.350 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:19.442 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:19.726 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:19.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:19.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:19.845 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:19.878 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:19.936 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:19.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:19.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:20.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:20.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:19 smithi191 ceph-mon[161669]: pgmap v555: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 7925/90288 objects degraded (8.777%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:20.051 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:19 smithi008 ceph-mon[184418]: pgmap v555: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 7925/90288 objects degraded (8.777%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:20.051 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:19 smithi008 ceph-mon[191227]: pgmap v555: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 7925/90288 objects degraded (8.777%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:20.073 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:20.101 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:40:20.129 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:20.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:20.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:20.152 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:40:20.184 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:40:20.237 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:20.246 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:40:20.284 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:40:20.316 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:40:20.353 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:20.355 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:40:20.385 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:40:20.397 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:40:20.397 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:40:20.398 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:40:20.398 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:40:20.398 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:40:20.398 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:40:20.398 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:40:20.398 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:40:20.716 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:40:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:40:20] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.33.4" 2023-10-09T05:40:20.877 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:40:21.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:20 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2494645109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:21.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:20 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2494645109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:21.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:20 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2494645109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:21.395 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:21.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:21.508 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:21.566 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:21.887 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:40:21.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:21.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:21.913 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:21 smithi008 ceph-mon[184418]: pgmap v556: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 7925/85533 objects degraded (9.265%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:21.913 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:21 smithi008 ceph-mon[191227]: pgmap v556: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 7925/85533 objects degraded (9.265%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:22.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:22.040 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:21 smithi191 ceph-mon[161669]: pgmap v556: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 7925/85533 objects degraded (9.265%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:22.049 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:22.112 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:22.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:22.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:22.218 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:22.301 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:22.622 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:40:22.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:22.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:22.732 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:22.745 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:22.827 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:22 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 7925/85533 objects degraded (9.265%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:22.827 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:22 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 7925/85533 objects degraded (9.265%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:22.837 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:22.973 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:22.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:22.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:23.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:22 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 7925/85533 objects degraded (9.265%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:23.082 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:23.143 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:23.558 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:23.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:23.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:23.672 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:23.776 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:23.780 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:40:23.780 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:40:23.780 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:40:23.780 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:40:23.780 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.780 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119029 5.c 5:33f591e4:::benchmark_data_smithi008_238708_object893:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.009231+0000", 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 359.75606423699998, 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0490911119999999, 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119029 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.781 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009231+0000", 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009231+0000", 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009229+0000", 2023-10-09T05:40:23.782 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009236+0000", 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2099999999999996e-06 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009250+0000", 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3377000000000001e-05 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.783 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.009259+0000", 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4050000000000006e-06 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180684+0000", 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.171425208 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180700+0000", 2023-10-09T05:40:23.784 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5184999999999999e-05 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.030305+0000", 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.84960513900000001 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.030495+0000", 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000190605 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.785 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.030899+0000", 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00040406200000000001 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.040416+0000", 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0095167640000000005 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058217+0000", 2023-10-09T05:40:23.786 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.01780087 2023-10-09T05:40:23.787 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.787 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.787 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.787 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058246+0000", 2023-10-09T05:40:23.787 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8473999999999999e-05 2023-10-09T05:40:23.787 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.787 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.787 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.787 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058285+0000", 2023-10-09T05:40:23.787 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9922999999999997e-05 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058322+0000", 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6687999999999998e-05 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119039 5.c 5:34e6951f:::benchmark_data_smithi008_238708_object903:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.151148+0000", 2023-10-09T05:40:23.788 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 359.61414696100002, 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90729258899999998, 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119039 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151148+0000", 2023-10-09T05:40:23.789 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151148+0000", 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151146+0000", 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.790 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151154+0000", 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2060000000000001e-06 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151177+0000", 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3495000000000001e-05 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.151189+0000", 2023-10-09T05:40:23.791 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1959e-05 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180742+0000", 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.029553138 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.180757+0000", 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4803999999999999e-05 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.792 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.032636+0000", 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85187944500000001 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.032759+0000", 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012297400000000001 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.033078+0000", 2023-10-09T05:40:23.793 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00031910800000000002 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.040420+0000", 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0073418340000000002 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058354+0000", 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017933893999999999 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.794 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058380+0000", 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6250000000000001e-05 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058412+0000", 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1640999999999997e-05 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.058441+0000", 2023-10-09T05:40:23.795 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8843000000000002e-05 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4412 6.4 6:2c929489:::benchmark_data_smithi008_295510_object4411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:09.108561+0000", 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 194.65673440200001, 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30418963199999999, 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.796 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4412 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108561+0000", 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.797 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108561+0000", 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108559+0000", 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108583+0000", 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3278e-05 2023-10-09T05:40:23.798 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108598+0000", 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4980999999999999e-05 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108605+0000", 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4880000000000001e-06 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108653+0000", 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8251000000000003e-05 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108787+0000", 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000133879 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108927+0000", 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000139949 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.800 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111786+0000", 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002858855 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.114484+0000", 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026978729999999999 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412641+0000", 2023-10-09T05:40:23.801 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29815728699999999 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412707+0000", 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5531999999999995e-05 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412750+0000", 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3729000000000001e-05 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.802 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4420 6.4 6:26d4e1d5:::benchmark_data_smithi008_295510_object4419:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:09.111673+0000", 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 194.65362264699999, 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30119500100000002, 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.803 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4420 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111673+0000", 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111673+0000", 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.804 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111671+0000", 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111697+0000", 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5582999999999999e-05 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.805 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111712+0000", 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5546999999999999e-05 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111720+0000", 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8970000000000008e-06 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111810+0000", 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0062000000000006e-05 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.806 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111951+0000", 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000140662 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.112099+0000", 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014773400000000001 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.113038+0000", 2023-10-09T05:40:23.807 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00093915700000000003 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.114491+0000", 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001453378 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412787+0000", 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29829549100000002 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.808 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412833+0000", 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5770999999999999e-05 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412868+0000", 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5008000000000001e-05 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.809 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4428 6.4 6:22fa3692:::benchmark_data_smithi008_295510_object4427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:09.116232+0000", 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 194.64906314999999, 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29675727800000001, 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4428 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.810 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116232+0000", 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116232+0000", 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.811 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116231+0000", 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116251+0000", 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0611999999999998e-05 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116265+0000", 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3195e-05 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.812 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116271+0000", 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5230000000000001e-06 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116312+0000", 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0831000000000003e-05 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.813 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116415+0000", 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010352399999999999 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116514+0000", 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8212000000000001e-05 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.117005+0000", 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000491516 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.814 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.117207+0000", 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000201659 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412905+0000", 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29569794399999999 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.815 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412954+0000", 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9382000000000003e-05 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412989+0000", 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5240000000000001e-05 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:72545 6.4 6:23b10169:::benchmark_data_smithi008_295510_object72544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-10-09T05:40:23.816 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:13.301291+0000", 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 130.46400378300001, 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24652739700000001, 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 72545 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.817 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301291+0000", 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301291+0000", 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301290+0000", 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:40:23.818 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301324+0000", 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.455e-05 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301336+0000", 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.204e-05 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.819 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301344+0000", 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5959999999999998e-06 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301382+0000", 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8080000000000001e-05 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301527+0000", 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000144783 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.820 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301679+0000", 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015243400000000001 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.309834+0000", 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081545579999999993 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.821 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.312878+0000", 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003043844 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.547717+0000", 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.234839724 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.547775+0000", 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7395999999999997e-05 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.822 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.547819+0000", 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4252999999999998e-05 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:72546 6.4 6:2b9d56ff:::benchmark_data_smithi008_295510_object72545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:13.301357+0000", 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 130.46393849699999, 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.246598859, 2023-10-09T05:40:23.823 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 72546 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301357+0000", 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.824 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301357+0000", 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301356+0000", 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.825 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301384+0000", 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8062999999999999e-05 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301391+0000", 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0659999999999998e-06 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301397+0000", 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4489999999999998e-06 2023-10-09T05:40:23.826 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.304245+0000", 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028480659999999998 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.304361+0000", 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011643599999999999 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.827 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.304487+0000", 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012566399999999999 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.309839+0000", 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0053520579999999998 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.312900+0000", 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030611060000000001 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.828 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.547876+0000", 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23497620299999999 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.547914+0000", 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7707999999999999e-05 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.547956+0000", 2023-10-09T05:40:23.829 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1539000000000003e-05 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:72550 6.4 6:2a690193:::benchmark_data_smithi008_295510_object72549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:13.301402+0000", 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 130.46389348899999, 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24670176899999999, 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.830 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 72550 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301402+0000", 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.831 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301402+0000", 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301401+0000", 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301421+0000", 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9822000000000001e-05 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.832 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301427+0000", 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0079999999999999e-06 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.301440+0000", 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.26e-05 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.307140+0000", 2023-10-09T05:40:23.833 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0057002160000000001 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.307295+0000", 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015490100000000001 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.307440+0000", 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014552599999999999 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.834 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.309997+0000", 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025562879999999999 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.313119+0000", 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031219820000000001 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.548019+0000", 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.234899837 2023-10-09T05:40:23.835 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.548068+0000", 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9116999999999997e-05 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:13.548104+0000", 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5908999999999998e-05 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.836 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:74960 6.4 6:26e595e2:::benchmark_data_smithi008_295510_object74959:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e181)", 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:17.014249+0000", 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 126.751046243, 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26028721500000002, 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 74960 2023-10-09T05:40:23.837 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.014249+0000", 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.014249+0000", 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.838 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.014248+0000", 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.014272+0000", 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3946000000000001e-05 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.014284+0000", 2023-10-09T05:40:23.839 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1285e-05 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.014291+0000", 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4229999999999999e-06 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.014328+0000", 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6745000000000001e-05 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.840 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.014455+0000", 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000126959 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.014575+0000", 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000120704 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020150+0000", 2023-10-09T05:40:23.841 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0055749069999999996 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.021113+0000", 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00096264300000000001 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274428+0000", 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.25331520200000002 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.842 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274486+0000", 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8103000000000003e-05 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274536+0000", 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0043000000000002e-05 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.843 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:74963 6.4 6:2bb68b1c:::benchmark_data_smithi008_295510_object74962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e181)", 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:17.017205+0000", 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 126.748090729, 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.25745266100000003, 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 74963 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.844 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.017205+0000", 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.017205+0000", 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.845 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.846 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.017203+0000", 2023-10-09T05:40:23.846 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:23.846 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.846 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.846 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.846 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.017229+0000", 2023-10-09T05:40:23.846 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5530000000000001e-05 2023-10-09T05:40:23.846 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.846 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.846 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.846 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.017240+0000", 2023-10-09T05:40:23.847 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.183e-05 2023-10-09T05:40:23.847 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.847 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.847 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.847 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.017248+0000", 2023-10-09T05:40:23.847 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1599999999999998e-06 2023-10-09T05:40:23.847 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.847 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.847 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.847 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.017292+0000", 2023-10-09T05:40:23.847 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3937999999999998e-05 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.017440+0000", 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014708599999999999 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.017586+0000", 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000146203 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.848 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020645+0000", 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003059371 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.021145+0000", 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00049999300000000001 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274584+0000", 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.25343908199999998 2023-10-09T05:40:23.849 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274626+0000", 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1730000000000002e-05 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274657+0000", 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1300000000000002e-05 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.850 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45029.0:74957 6.3 e181/170 6:ca97a06f:::benchmark_data_smithi008_295510_object74956:head v 181'4590, mlcod=0'0)", 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:17.020578+0000", 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 126.74471692900001, 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24793385500000001, 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020578+0000", 2023-10-09T05:40:23.851 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020578+0000", 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020578+0000", 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967296 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.852 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020610+0000", 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1715999999999999e-05 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020616+0000", 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3790000000000003e-06 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020621+0000", 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4549999999999997e-06 2023-10-09T05:40:23.853 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.021457+0000", 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00083666000000000003 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.021474+0000", 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7031999999999999e-05 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.854 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.268473+0000", 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.246998258 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.268512+0000", 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9545999999999997e-05 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.855 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45029.0:74962 6.3 e181/170 6:c979a0e9:::benchmark_data_smithi008_295510_object74961:head v 181'4591, mlcod=0'0)", 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:17.020633+0000", 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 126.744661912, 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24791640000000001, 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020633+0000", 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.856 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020633+0000", 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020633+0000", 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967296 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020662+0000", 2023-10-09T05:40:23.857 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8785e-05 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020668+0000", 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2360000000000001e-06 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.020672+0000", 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3080000000000001e-06 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.858 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.025893+0000", 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0052204599999999997 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.025923+0000", 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9726e-05 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.859 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.268517+0000", 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24259388300000001 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.268550+0000", 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.324e-05 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.860 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45029.0:74969 6.d e181/162 6:bb94c61b:::benchmark_data_smithi008_295510_object74968:head v 181'4682, mlcod=181'4679)", 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:17.029911+0000", 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 126.735383977, 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24480763899999999, 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.029911+0000", 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.861 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.029911+0000", 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.029910+0000", 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.029947+0000", 2023-10-09T05:40:23.862 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6192000000000002e-05 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.029956+0000", 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2480000000000002e-06 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.029961+0000", 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4940000000000003e-06 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.863 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.030002+0000", 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0466000000000002e-05 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.030015+0000", 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3742e-05 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274672+0000", 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24465680400000001 2023-10-09T05:40:23.864 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274719+0000", 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6641000000000003e-05 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:74975 6.4 6:2b034feb:::benchmark_data_smithi008_295510_object74974:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e181)", 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:38:17.030456+0000", 2023-10-09T05:40:23.865 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 126.734838869, 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.244385782, 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 74975 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.030456+0000", 2023-10-09T05:40:23.866 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.030456+0000", 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.030455+0000", 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.867 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.030477+0000", 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2039999999999999e-05 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.268464+0000", 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23798686199999999 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.268479+0000", 2023-10-09T05:40:23.868 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5074e-05 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.268522+0000", 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3152000000000003e-05 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.268696+0000", 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017351000000000001 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.869 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.268861+0000", 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000165346 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274721+0000", 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058600370000000002 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274732+0000", 2023-10-09T05:40:23.870 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1342000000000001e-05 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274765+0000", 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2920000000000003e-05 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274809+0000", 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3718999999999999e-05 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.871 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:38:17.274842+0000", 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3092000000000001e-05 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:138621 6.4 6:2b4aa3f5:::benchmark_data_smithi008_295510_object50258:head [delete] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:00.294391+0000", 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 23.470904385000001, 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23318604100000001, 2023-10-09T05:40:23.872 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 138621 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.294391+0000", 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.873 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.294391+0000", 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.294389+0000", 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.874 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.294396+0000", 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8909999999999998e-06 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.294410+0000", 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3199e-05 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.294417+0000", 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4599999999999997e-06 2023-10-09T05:40:23.875 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.294454+0000", 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7379000000000002e-05 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.294577+0000", 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012243900000000001 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.876 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.294693+0000", 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000116016 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.298625+0000", 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003931665 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.309324+0000", 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010699488 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.877 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.527492+0000", 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.218167685 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.527540+0000", 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8402000000000002e-05 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.878 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.527577+0000", 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6804000000000002e-05 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:138626 6.4 6:22224c82:::benchmark_data_smithi008_295510_object50263:head [delete] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:00.301296+0000", 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 23.463999342000001, 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.22647421500000001, 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.879 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 138626 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.301296+0000", 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.880 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.301296+0000", 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.301296+0000", 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.301298+0000", 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8760000000000001e-06 2023-10-09T05:40:23.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.301305+0000", 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5309999999999998e-06 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.301311+0000", 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.562e-06 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.882 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.305812+0000", 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045005760000000001 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.305925+0000", 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000113311 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.306042+0000", 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000116347 2023-10-09T05:40:23.883 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.309261+0000", 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032195079999999998 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.309447+0000", 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018592000000000001 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.884 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.527708+0000", 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.218260813 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.527741+0000", 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2836999999999997e-05 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:00.527770+0000", 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9343e-05 2023-10-09T05:40:23.885 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:142240 6.4 6:2cab24fa:::benchmark_data_smithi008_295510_object53877:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:07.422273+0000", 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 16.343022183999999, 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.424140515, 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.886 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 142240 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422273+0000", 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.887 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422273+0000", 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422271+0000", 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422280+0000", 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8589999999999994e-06 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.888 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422300+0000", 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.982e-05 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422310+0000", 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7329999999999998e-06 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422354+0000", 2023-10-09T05:40:23.889 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4317999999999998e-05 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422530+0000", 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017590800000000001 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422682+0000", 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015153599999999999 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.890 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426323+0000", 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036415879999999999 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.436458+0000", 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010134364 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846310+0000", 2023-10-09T05:40:23.891 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.40985222199999999 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846362+0000", 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2284999999999998e-05 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846414+0000", 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1547e-05 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.892 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:142244 6.4 6:252372f6:::benchmark_data_smithi008_295510_object53881:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:07.426444+0000", 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 16.338851668, 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.42009002099999998, 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.893 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 142244 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426444+0000", 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426444+0000", 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.894 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426442+0000", 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426450+0000", 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8180000000000004e-06 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426467+0000", 2023-10-09T05:40:23.895 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7550999999999998e-05 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426478+0000", 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0356e-05 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426525+0000", 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7141000000000001e-05 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.896 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426671+0000", 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014611999999999999 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.430321+0000", 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036499219999999999 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.436469+0000", 2023-10-09T05:40:23.897 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061478349999999999 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.438906+0000", 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0024374570000000001 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846452+0000", 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.40754571000000001 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.898 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846494+0000", 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1936e-05 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846534+0000", 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.998e-05 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.899 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:142256 6.4 6:2549d186:::benchmark_data_smithi008_295510_object53893:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:07.449511+0000", 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 16.315784017999999, 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.39714472299999998, 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 142256 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.900 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.901 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.901 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449511+0000", 2023-10-09T05:40:23.901 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.901 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.901 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.901 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.901 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449511+0000", 2023-10-09T05:40:23.901 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.901 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.901 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.901 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.902 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449510+0000", 2023-10-09T05:40:23.902 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:23.902 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.902 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.902 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.902 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449517+0000", 2023-10-09T05:40:23.902 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2269999999999996e-06 2023-10-09T05:40:23.902 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449533+0000", 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6172e-05 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449544+0000", 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0431999999999999e-05 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.903 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449595+0000", 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0905000000000001e-05 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449734+0000", 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000139784 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449866+0000", 2023-10-09T05:40:23.904 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013137399999999999 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.453118+0000", 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032525430000000001 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.458251+0000", 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051326599999999998 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.905 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846581+0000", 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.38833016799999998 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846623+0000", 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2038e-05 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846656+0000", 2023-10-09T05:40:23.906 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2858000000000003e-05 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:142258 6.4 6:283fbab6:::benchmark_data_smithi008_295510_object53895:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:07.454512+0000", 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 16.310783307000001, 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.39222012299999998, 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:23.907 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 142258 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454512+0000", 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:23.908 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454512+0000", 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454511+0000", 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454517+0000", 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2840000000000003e-06 2023-10-09T05:40:23.909 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454525+0000", 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6299999999999998e-06 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454530+0000", 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0350000000000001e-06 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.910 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454570+0000", 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0723999999999998e-05 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454696+0000", 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012594599999999999 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454779+0000", 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2871e-05 2023-10-09T05:40:23.911 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.458263+0000", 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034837380000000001 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.459986+0000", 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017235970000000001 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.912 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846684+0000", 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.38669726799999998 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846709+0000", 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4811e-05 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846732+0000", 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3544e-05 2023-10-09T05:40:23.913 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.914 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.914 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.914 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:23.914 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:23.914 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:40:24.022 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:40:24.023 INFO:teuthology.orchestra.run.smithi008.stdout:{"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 7665/81639 objects degraded (9.389%), 4 pgs degraded, 4 pgs undersized","count":8},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":717,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":199,"num_osds":8,"num_up_osds":8,"osd_up_since":1696829680,"num_in_osds":7,"osd_in_since":1696829854,"num_remapped_pgs":4},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":85},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":27213,"data_bytes":1786320843,"bytes_used":7629856768,"bytes_avail":664335392768,"bytes_total":671965249536,"degraded_objects":7665,"degraded_total":81639,"degraded_ratio":0.093888950134127069,"recovering_objects_per_sec":48,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":488,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":585},"fsmap":{"epoch":6,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi008.zfejpl","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.8:8443/","prometheus":"http://172.21.15.8:9283/"}},"servicemap":{"epoch":23,"modified":"2023-10-09T05:35:39.975308+0000","services":{}},"progress_events":{"a1c2a5cb-a243-4c8c-845f-d0215fa7ce6b":{"message":"Global Recovery Event (2m)\n [==========================..] (remaining: 7s)","progress":0.95505619049072266,"add_to_ceph_s":true},"e7a35707-3c6c-469d-ad94-cd0d5825f733":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T05:40:24.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:23 smithi191 ceph-mon[161669]: pgmap v557: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 7665/81639 objects degraded (9.389%); 0 B/s, 48 objects/s recovering 2023-10-09T05:40:24.077 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:23 smithi008 ceph-mon[184418]: pgmap v557: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 7665/81639 objects degraded (9.389%); 0 B/s, 48 objects/s recovering 2023-10-09T05:40:24.078 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:23 smithi008 ceph-mon[191227]: pgmap v557: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 7665/81639 objects degraded (9.389%); 0 B/s, 48 objects/s recovering 2023-10-09T05:40:24.458 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:40:24.818 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:24 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3216736349' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:40:25.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:24 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3216736349' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:40:25.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:24 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3216736349' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:40:25.432 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:25.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:25.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:25.541 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:25.613 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:26.007 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:40:26.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:25 smithi008 ceph-mon[191227]: pgmap v558: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 7665/79476 objects degraded (9.644%); 0 B/s, 48 objects/s recovering 2023-10-09T05:40:26.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:25 smithi008 ceph-mon[184418]: pgmap v558: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 7665/79476 objects degraded (9.644%); 0 B/s, 48 objects/s recovering 2023-10-09T05:40:26.131 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:26.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:26.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:26.238 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T05:40:26.239 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:40:26.243 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:26.270 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:25 smithi191 ceph-mon[161669]: pgmap v558: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 7665/79476 objects degraded (9.644%); 0 B/s, 48 objects/s recovering 2023-10-09T05:40:26.273 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:26.324 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:26.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:26.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:26.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:26.455 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:26.499 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:26.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:26.604 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:26.633 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:26.658 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:26.688 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:26.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:26.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:26.792 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:26.887 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:26 smithi191 ceph-mon[161669]: pgmap v559: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 7385/75693 objects degraded (9.757%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:26.913 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:27.144 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:27.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:27.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:27.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:26 smithi008 ceph-mon[184418]: pgmap v559: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 7385/75693 objects degraded (9.757%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:27.152 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:26 smithi008 ceph-mon[191227]: pgmap v559: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 7385/75693 objects degraded (9.757%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:27.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:27.283 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:27.367 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:40:27.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:27.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:27.397 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:27.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:27.507 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:27.562 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:27.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:27.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:27.671 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:27.703 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:27.772 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:40:27.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:27.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:27.889 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3486993189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:27.889 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:27 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:27.889 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:27 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 7385/75693 objects degraded (9.757%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:27.889 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3486993189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:27.889 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:27 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:27.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:27 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 7385/75693 objects degraded (9.757%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:27.891 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:27.954 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:40:28.020 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:28.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3486993189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:28.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:27 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:28.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:27 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 7385/75693 objects degraded (9.757%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:28.423 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:40:28.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:28.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:28.536 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:28.576 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:28.642 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:28.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:28.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:28.783 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:28.829 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:28.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:28.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:28.935 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:28.969 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:29.020 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:29.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:29.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:29.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:29.166 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:28 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:40:29.166 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:28 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:29.166 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:28 smithi191 ceph-mon[161669]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T05:40:29.166 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:28 smithi191 ceph-mon[161669]: failed to encode map e200 with expected crc 2023-10-09T05:40:29.166 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:28 smithi191 ceph-mon[161669]: failed to encode map e200 with expected crc 2023-10-09T05:40:29.166 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:28 smithi191 ceph-mon[161669]: pgmap v561: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 7385/73146 objects degraded (10.096%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:29.168 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:29.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:40:29.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:29.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[191227]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T05:40:29.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[191227]: failed to encode map e200 with expected crc 2023-10-09T05:40:29.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[191227]: failed to encode map e200 with expected crc 2023-10-09T05:40:29.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[191227]: pgmap v561: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 7385/73146 objects degraded (10.096%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:29.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:40:29.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:29.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[184418]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T05:40:29.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[184418]: failed to encode map e200 with expected crc 2023-10-09T05:40:29.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[184418]: failed to encode map e200 with expected crc 2023-10-09T05:40:29.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:28 smithi008 ceph-mon[184418]: pgmap v561: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 7385/73146 objects degraded (10.096%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:29.231 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:40:29.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:29.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:29.343 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:29.358 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:29.424 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:29.814 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:29.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:29.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:29.923 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:30.036 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:30.036 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:30.037 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:30.037 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:30.040 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:30.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:30.695 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:40:30.707 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:40:30.736 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:40:30.781 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:40:30.801 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:40:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:40:30] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.33.4" 2023-10-09T05:40:30.816 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:40:30.871 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:40:30.881 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:30.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:30.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:30.919 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:40:30.955 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:40:30.969 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:40:30.969 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:40:30.969 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:40:30.969 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:40:30.969 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:40:30.969 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:40:30.969 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:40:30.969 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:40:31.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:31.038 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:31.091 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:31.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:31.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:31.101 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:30 smithi008 ceph-mon[184418]: pgmap v562: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 7385/68409 objects degraded (10.795%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:31.101 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1674539710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:31.102 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:30 smithi008 ceph-mon[191227]: pgmap v562: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 7385/68409 objects degraded (10.795%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:31.102 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1674539710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:31.197 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:31.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:30 smithi191 ceph-mon[161669]: pgmap v562: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 7385/68409 objects degraded (10.795%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1674539710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:31.316 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:31.350 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:40:32.131 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:32.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:32.239 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:32.280 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:32.350 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:40:32.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:32.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:32.456 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:32.483 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:32.953 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:32.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:32.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:33.062 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:33.093 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:33.163 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:33.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:33.205 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:33 smithi008 ceph-mon[184418]: pgmap v563: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 7136/64596 objects degraded (11.047%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:33.205 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:33.205 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 7136/64596 objects degraded (11.047%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:33.271 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:33.292 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:40:33.292 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:40:33.329 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:33.406 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:33.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:33 smithi008 ceph-mon[191227]: pgmap v563: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 7136/64596 objects degraded (11.047%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:33.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:33.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 7136/64596 objects degraded (11.047%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:33.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:33 smithi191 ceph-mon[161669]: pgmap v563: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 7136/64596 objects degraded (11.047%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:33.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:33.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 7136/64596 objects degraded (11.047%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:33.770 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:40:33.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:33.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:33.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:33.925 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:34.053 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:34.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:34.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:34.161 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:34.171 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:40:34.173 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:40:34.192 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:34.599 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:34.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:34.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:34.658 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:40:34.710 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:34.739 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:34.815 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:34.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:34.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:34.922 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:35.028 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:35.281 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":564,"stamp":"2023-10-09T05:40:34.123991+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1369577419,"num_objects":20854,"num_object_clones":0,"num_object_copies":62562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7136,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20854,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":157002,"num_write_kb":5670182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10405,"num_bytes_recovered":531043702,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129766,"ondisk_log_size":129766,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6269524,"kb_used_data":4089308,"kb_used_omap":31,"kb_used_meta":2180000,"kb_avail":649946540,"statfs":{"total":671965249536,"available":665545256960,"internally_reserved":0,"allocated":4187451392,"data_stored":4182840369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2232320457},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,2,8,6,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-369491968,"num_objects":-5638,"num_object_clones":0,"num_object_copies":-16914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-529,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5638,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":529,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008515"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":183,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233337+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:40:17.233337+0000","last_peered":"2023-10-09T05:40:17.233337+0000","last_clean":"2023-10-09T05:40:17.233337+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:40:17.233337+0000","last_undegraded":"2023-10-09T05:40:17.233337+0000","last_fullsized":"2023-10-09T05:40:17.233337+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":172,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.253006+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:40:17.253006+0000","last_peered":"2023-10-09T05:40:17.253006+0000","last_clean":"2023-10-09T05:40:17.253006+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:40:17.253006+0000","last_undegraded":"2023-10-09T05:40:17.253006+0000","last_fullsized":"2023-10-09T05:40:17.253006+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":147,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233030+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:40:17.233030+0000","last_peered":"2023-10-09T05:40:17.233030+0000","last_clean":"2023-10-09T05:40:17.233030+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:40:17.233030+0000","last_undegraded":"2023-10-09T05:40:17.233030+0000","last_fullsized":"2023-10-09T05:40:17.233030+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":177,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.247601+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:40:17.247601+0000","last_peered":"2023-10-09T05:40:17.247601+0000","last_clean":"2023-10-09T05:40:17.247601+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:40:17.247601+0000","last_undegraded":"2023-10-09T05:40:17.247601+0000","last_fullsized":"2023-10-09T05:40:17.247601+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":188,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.252544+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:40:17.252544+0000","last_peered":"2023-10-09T05:40:17.252544+0000","last_clean":"2023-10-09T05:40:17.252544+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:40:17.252544+0000","last_undegraded":"2023-10-09T05:40:17.252544+0000","last_fullsized":"2023-10-09T05:40:17.252544+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":166,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.222927+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:40:17.222927+0000","last_peered":"2023-10-09T05:40:17.222927+0000","last_clean":"2023-10-09T05:40:17.222927+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:40:17.222927+0000","last_undegraded":"2023-10-09T05:40:17.222927+0000","last_fullsized":"2023-10-09T05:40:17.222927+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":152,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.247134+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:40:17.247134+0000","last_peered":"2023-10-09T05:40:17.247134+0000","last_clean":"2023-10-09T05:40:17.247134+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:40:17.247134+0000","last_undegraded":"2023-10-09T05:40:17.247134+0000","last_fullsized":"2023-10-09T05:40:17.247134+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":172,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.252580+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:40:17.252580+0000","last_peered":"2023-10-09T05:40:17.252580+0000","last_clean":"2023-10-09T05:40:17.252580+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:40:17.252580+0000","last_undegraded":"2023-10-09T05:40:17.252580+0000","last_fullsized":"2023-10-09T05:40:17.252580+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":191,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.241807+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:40:17.241807+0000","last_peered":"2023-10-09T05:40:17.241807+0000","last_clean":"2023-10-09T05:40:17.241807+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:40:17.241807+0000","last_undegraded":"2023-10-09T05:40:17.241807+0000","last_fullsized":"2023-10-09T05:40:17.241807+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":186,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.252900+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:40:17.252900+0000","last_peered":"2023-10-09T05:40:17.252900+0000","last_clean":"2023-10-09T05:40:17.252900+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:40:17.252900+0000","last_undegraded":"2023-10-09T05:40:17.252900+0000","last_fullsized":"2023-10-09T05:40:17.252900+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":80,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.252919+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:40:17.252919+0000","last_peered":"2023-10-09T05:40:17.252919+0000","last_clean":"2023-10-09T05:40:17.252919+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:40:17.252919+0000","last_undegraded":"2023-10-09T05:40:17.252919+0000","last_fullsized":"2023-10-09T05:40:17.252919+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":147,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.232875+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:40:17.232875+0000","last_peered":"2023-10-09T05:40:17.232875+0000","last_clean":"2023-10-09T05:40:17.232875+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:40:17.232875+0000","last_undegraded":"2023-10-09T05:40:17.232875+0000","last_fullsized":"2023-10-09T05:40:17.232875+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":208,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233194+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:40:17.233194+0000","last_peered":"2023-10-09T05:40:17.233194+0000","last_clean":"2023-10-09T05:40:17.233194+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:40:17.233194+0000","last_undegraded":"2023-10-09T05:40:17.233194+0000","last_fullsized":"2023-10-09T05:40:17.233194+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":166,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233054+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:40:17.233054+0000","last_peered":"2023-10-09T05:40:17.233054+0000","last_clean":"2023-10-09T05:40:17.233054+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:40:17.233054+0000","last_undegraded":"2023-10-09T05:40:17.233054+0000","last_fullsized":"2023-10-09T05:40:17.233054+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":79,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233076+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:40:17.233076+0000","last_peered":"2023-10-09T05:40:17.233076+0000","last_clean":"2023-10-09T05:40:17.233076+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:40:17.233076+0000","last_undegraded":"2023-10-09T05:40:17.233076+0000","last_fullsized":"2023-10-09T05:40:17.233076+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":196,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.252826+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:40:17.252826+0000","last_peered":"2023-10-09T05:40:17.252826+0000","last_clean":"2023-10-09T05:40:17.252826+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:40:17.252826+0000","last_undegraded":"2023-10-09T05:40:17.252826+0000","last_fullsized":"2023-10-09T05:40:17.252826+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":176,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.250948+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:40:17.250948+0000","last_peered":"2023-10-09T05:40:17.250948+0000","last_clean":"2023-10-09T05:40:17.250948+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:40:17.250948+0000","last_undegraded":"2023-10-09T05:40:17.250948+0000","last_fullsized":"2023-10-09T05:40:17.250948+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":190,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.246010+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:40:17.246010+0000","last_peered":"2023-10-09T05:40:17.246010+0000","last_clean":"2023-10-09T05:40:17.246010+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:40:17.246010+0000","last_undegraded":"2023-10-09T05:40:17.246010+0000","last_fullsized":"2023-10-09T05:40:17.246010+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":97,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.232956+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:40:17.232956+0000","last_peered":"2023-10-09T05:40:17.232956+0000","last_clean":"2023-10-09T05:40:17.232956+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:40:17.232956+0000","last_undegraded":"2023-10-09T05:40:17.232956+0000","last_fullsized":"2023-10-09T05:40:17.232956+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":176,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.247531+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:40:17.247531+0000","last_peered":"2023-10-09T05:40:17.247531+0000","last_clean":"2023-10-09T05:40:17.247531+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:40:17.247531+0000","last_undegraded":"2023-10-09T05:40:17.247531+0000","last_fullsized":"2023-10-09T05:40:17.247531+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":175,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.222056+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:40:17.222056+0000","last_peered":"2023-10-09T05:40:17.222056+0000","last_clean":"2023-10-09T05:40:17.222056+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:40:17.222056+0000","last_undegraded":"2023-10-09T05:40:17.222056+0000","last_fullsized":"2023-10-09T05:40:17.222056+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":196,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.253717+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:40:17.253717+0000","last_peered":"2023-10-09T05:40:17.253717+0000","last_clean":"2023-10-09T05:40:17.253717+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:40:17.253717+0000","last_undegraded":"2023-10-09T05:40:17.253717+0000","last_fullsized":"2023-10-09T05:40:17.253717+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"200'9665","reported_seq":9711,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:29.951841+0000","last_change":"2023-10-09T05:37:01.636984+0000","last_active":"2023-10-09T05:40:29.951841+0000","last_peered":"2023-10-09T05:40:29.951841+0000","last_clean":"2023-10-09T05:40:29.951841+0000","last_became_active":"2023-10-09T05:37:01.636609+0000","last_became_peered":"2023-10-09T05:37:01.636609+0000","last_unstale":"2023-10-09T05:40:29.951841+0000","last_undegraded":"2023-10-09T05:40:29.951841+0000","last_fullsized":"2023-10-09T05:40:29.951841+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9665,"log_dups_size":0,"ondisk_log_size":9665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91160576,"num_objects":1391,"num_object_clones":0,"num_object_copies":4173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9665,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":77,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233361+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:40:17.233361+0000","last_peered":"2023-10-09T05:40:17.233361+0000","last_clean":"2023-10-09T05:40:17.233361+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:40:17.233361+0000","last_undegraded":"2023-10-09T05:40:17.233361+0000","last_fullsized":"2023-10-09T05:40:17.233361+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":175,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.241618+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:40:17.241618+0000","last_peered":"2023-10-09T05:40:17.241618+0000","last_clean":"2023-10-09T05:40:17.241618+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:40:17.241618+0000","last_undegraded":"2023-10-09T05:40:17.241618+0000","last_fullsized":"2023-10-09T05:40:17.241618+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"200'9677","reported_seq":9733,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:31.274824+0000","last_change":"2023-10-09T05:37:36.837344+0000","last_active":"2023-10-09T05:40:31.274824+0000","last_peered":"2023-10-09T05:40:31.274824+0000","last_clean":"2023-10-09T05:37:34.651883+0000","last_became_active":"2023-10-09T05:37:36.832502+0000","last_became_peered":"2023-10-09T05:37:36.832502+0000","last_unstale":"2023-10-09T05:40:31.274824+0000","last_undegraded":"2023-10-09T05:37:36.702797+0000","last_fullsized":"2023-10-09T05:37:36.678583+0000","mapping_epoch":170,"log_start":"174'3200","ondisk_log_start":"174'3200","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6477,"log_dups_size":0,"ondisk_log_size":6477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:57.419460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86179840,"num_objects":1315,"num_object_clones":0,"num_object_copies":3945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9677,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":-1575}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":77,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233249+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:40:17.233249+0000","last_peered":"2023-10-09T05:40:17.233249+0000","last_clean":"2023-10-09T05:40:17.233249+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:40:17.233249+0000","last_undegraded":"2023-10-09T05:40:17.233249+0000","last_fullsized":"2023-10-09T05:40:17.233249+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":181,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.232913+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:40:17.232913+0000","last_peered":"2023-10-09T05:40:17.232913+0000","last_clean":"2023-10-09T05:40:17.232913+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:40:17.232913+0000","last_undegraded":"2023-10-09T05:40:17.232913+0000","last_fullsized":"2023-10-09T05:40:17.232913+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"200'9934","reported_seq":16042,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:31.782800+0000","last_change":"2023-10-09T05:38:55.317808+0000","last_active":"2023-10-09T05:40:31.782800+0000","last_peered":"2023-10-09T05:40:31.782800+0000","last_clean":"2023-10-09T05:40:31.782800+0000","last_became_active":"2023-10-09T05:38:55.302739+0000","last_became_peered":"2023-10-09T05:38:55.302739+0000","last_unstale":"2023-10-09T05:40:31.782800+0000","last_undegraded":"2023-10-09T05:40:31.782800+0000","last_fullsized":"2023-10-09T05:40:31.782800+0000","mapping_epoch":186,"log_start":"166'800","ondisk_log_start":"166'800","created":162,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9134,"log_dups_size":0,"ondisk_log_size":9134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:25.379818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83755008,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9934,"num_write_kb":358784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":198049792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":77,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.246078+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:40:17.246078+0000","last_peered":"2023-10-09T05:40:17.246078+0000","last_clean":"2023-10-09T05:40:17.246078+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:40:17.246078+0000","last_undegraded":"2023-10-09T05:40:17.246078+0000","last_fullsized":"2023-10-09T05:40:17.246078+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":199,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.223377+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:40:17.223377+0000","last_peered":"2023-10-09T05:40:17.223377+0000","last_clean":"2023-10-09T05:40:17.223377+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:40:17.223377+0000","last_undegraded":"2023-10-09T05:40:17.223377+0000","last_fullsized":"2023-10-09T05:40:17.223377+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"200'9781","reported_seq":15647,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:33.217268+0000","last_change":"2023-10-09T05:40:04.830679+0000","last_active":"2023-10-09T05:40:33.217268+0000","last_peered":"2023-10-09T05:40:33.217268+0000","last_clean":"2023-10-09T05:40:33.217268+0000","last_became_active":"2023-10-09T05:40:04.807741+0000","last_became_peered":"2023-10-09T05:40:04.807741+0000","last_unstale":"2023-10-09T05:40:33.217268+0000","last_undegraded":"2023-10-09T05:40:33.217268+0000","last_fullsized":"2023-10-09T05:40:33.217268+0000","mapping_epoch":195,"log_start":"166'900","ondisk_log_start":"166'900","created":162,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8881,"log_dups_size":0,"ondisk_log_size":8881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:51.289034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77922304,"num_objects":1189,"num_object_clones":0,"num_object_copies":3567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9781,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2899,"num_bytes_recovered":139919360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":167,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.242034+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:40:17.242034+0000","last_peered":"2023-10-09T05:40:17.242034+0000","last_clean":"2023-10-09T05:40:17.242034+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:40:17.242034+0000","last_undegraded":"2023-10-09T05:40:17.242034+0000","last_fullsized":"2023-10-09T05:40:17.242034+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":152,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.246994+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:40:17.246994+0000","last_peered":"2023-10-09T05:40:17.246994+0000","last_clean":"2023-10-09T05:40:17.246994+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:40:17.246994+0000","last_undegraded":"2023-10-09T05:40:17.246994+0000","last_fullsized":"2023-10-09T05:40:17.246994+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"200'9675","reported_seq":9721,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:29.956833+0000","last_change":"2023-10-09T05:37:01.439878+0000","last_active":"2023-10-09T05:40:29.956833+0000","last_peered":"2023-10-09T05:40:29.956833+0000","last_clean":"2023-10-09T05:40:29.956833+0000","last_became_active":"2023-10-09T05:37:01.439481+0000","last_became_peered":"2023-10-09T05:37:01.439481+0000","last_unstale":"2023-10-09T05:40:29.956833+0000","last_undegraded":"2023-10-09T05:40:29.956833+0000","last_fullsized":"2023-10-09T05:40:29.956833+0000","mapping_epoch":162,"log_start":"167'2100","ondisk_log_start":"167'2100","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7575,"log_dups_size":0,"ondisk_log_size":7575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85917696,"num_objects":1311,"num_object_clones":0,"num_object_copies":3933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9675,"num_write_kb":351552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":171,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.924933+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:40:27.924933+0000","last_peered":"2023-10-09T05:40:27.924933+0000","last_clean":"2023-10-09T05:40:27.924933+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:40:27.924933+0000","last_undegraded":"2023-10-09T05:40:27.924933+0000","last_fullsized":"2023-10-09T05:40:27.924933+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":166,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233136+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:40:17.233136+0000","last_peered":"2023-10-09T05:40:17.233136+0000","last_clean":"2023-10-09T05:40:17.233136+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:40:17.233136+0000","last_undegraded":"2023-10-09T05:40:17.233136+0000","last_fullsized":"2023-10-09T05:40:17.233136+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"200'9742","reported_seq":9789,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:31.773179+0000","last_change":"2023-10-09T05:37:01.440175+0000","last_active":"2023-10-09T05:40:31.773179+0000","last_peered":"2023-10-09T05:40:31.773179+0000","last_clean":"2023-10-09T05:40:31.773179+0000","last_became_active":"2023-10-09T05:37:01.439846+0000","last_became_peered":"2023-10-09T05:37:01.439846+0000","last_unstale":"2023-10-09T05:40:31.773179+0000","last_undegraded":"2023-10-09T05:40:31.773179+0000","last_fullsized":"2023-10-09T05:40:31.773179+0000","mapping_epoch":162,"log_start":"166'600","ondisk_log_start":"166'600","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9142,"log_dups_size":0,"ondisk_log_size":9142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:27.883265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85983232,"num_objects":1312,"num_object_clones":0,"num_object_copies":3936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9742,"num_write_kb":353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":167,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.223241+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:40:17.223241+0000","last_peered":"2023-10-09T05:40:17.223241+0000","last_clean":"2023-10-09T05:40:17.223241+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:40:17.223241+0000","last_undegraded":"2023-10-09T05:40:17.223241+0000","last_fullsized":"2023-10-09T05:40:17.223241+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":181,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233088+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:40:17.233088+0000","last_peered":"2023-10-09T05:40:17.233088+0000","last_clean":"2023-10-09T05:40:17.233088+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:40:17.233088+0000","last_undegraded":"2023-10-09T05:40:17.233088+0000","last_fullsized":"2023-10-09T05:40:17.233088+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"200'9810","reported_seq":9858,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:31.320458+0000","last_change":"2023-10-09T05:37:01.638178+0000","last_active":"2023-10-09T05:40:31.320458+0000","last_peered":"2023-10-09T05:40:31.320458+0000","last_clean":"2023-10-09T05:40:31.320458+0000","last_became_active":"2023-10-09T05:37:01.637822+0000","last_became_peered":"2023-10-09T05:37:01.637822+0000","last_unstale":"2023-10-09T05:40:31.320458+0000","last_undegraded":"2023-10-09T05:40:31.320458+0000","last_fullsized":"2023-10-09T05:40:31.320458+0000","mapping_epoch":162,"log_start":"175'3400","ondisk_log_start":"175'3400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6410,"log_dups_size":0,"ondisk_log_size":6410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:13.240069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85458944,"num_objects":1304,"num_object_clones":0,"num_object_copies":3912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9810,"num_write_kb":355648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":191,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.232831+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:40:17.232831+0000","last_peered":"2023-10-09T05:40:17.232831+0000","last_clean":"2023-10-09T05:40:17.232831+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:40:17.232831+0000","last_undegraded":"2023-10-09T05:40:17.232831+0000","last_fullsized":"2023-10-09T05:40:17.232831+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":175,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233376+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:40:17.233376+0000","last_peered":"2023-10-09T05:40:17.233376+0000","last_clean":"2023-10-09T05:40:17.233376+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:40:17.233376+0000","last_undegraded":"2023-10-09T05:40:17.233376+0000","last_fullsized":"2023-10-09T05:40:17.233376+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"200'9683","reported_seq":9730,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:29.938094+0000","last_change":"2023-10-09T05:37:01.438917+0000","last_active":"2023-10-09T05:40:29.938094+0000","last_peered":"2023-10-09T05:40:29.938094+0000","last_clean":"2023-10-09T05:40:29.938094+0000","last_became_active":"2023-10-09T05:37:01.438658+0000","last_became_peered":"2023-10-09T05:37:01.438658+0000","last_unstale":"2023-10-09T05:40:29.938094+0000","last_undegraded":"2023-10-09T05:40:29.938094+0000","last_fullsized":"2023-10-09T05:40:29.938094+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9683,"log_dups_size":0,"ondisk_log_size":9683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90308632,"num_objects":1379,"num_object_clones":0,"num_object_copies":4137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1379,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9683,"num_write_kb":353921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":198,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.252487+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:40:17.252487+0000","last_peered":"2023-10-09T05:40:17.252487+0000","last_clean":"2023-10-09T05:40:17.252487+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:40:17.252487+0000","last_undegraded":"2023-10-09T05:40:17.252487+0000","last_fullsized":"2023-10-09T05:40:17.252487+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":147,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233371+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:40:17.233371+0000","last_peered":"2023-10-09T05:40:17.233371+0000","last_clean":"2023-10-09T05:40:17.233371+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:40:17.233371+0000","last_undegraded":"2023-10-09T05:40:17.233371+0000","last_fullsized":"2023-10-09T05:40:17.233371+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"200'9855","reported_seq":9903,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:33.182830+0000","last_change":"2023-10-09T05:37:01.440093+0000","last_active":"2023-10-09T05:40:33.182830+0000","last_peered":"2023-10-09T05:40:33.182830+0000","last_clean":"2023-10-09T05:40:33.182830+0000","last_became_active":"2023-10-09T05:37:01.439799+0000","last_became_peered":"2023-10-09T05:37:01.439799+0000","last_unstale":"2023-10-09T05:40:33.182830+0000","last_undegraded":"2023-10-09T05:40:33.182830+0000","last_fullsized":"2023-10-09T05:40:33.182830+0000","mapping_epoch":162,"log_start":"166'1000","ondisk_log_start":"166'1000","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8855,"log_dups_size":0,"ondisk_log_size":8855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:28.734471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81985536,"num_objects":1251,"num_object_clones":0,"num_object_copies":3753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9855,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":769,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.236589+0000","last_change":"2023-10-09T05:40:04.806908+0000","last_active":"2023-10-09T05:40:17.236589+0000","last_peered":"2023-10-09T05:40:17.236589+0000","last_clean":"2023-10-09T05:40:17.236589+0000","last_became_active":"2023-10-09T05:40:04.806232+0000","last_became_peered":"2023-10-09T05:40:04.806232+0000","last_unstale":"2023-10-09T05:40:17.236589+0000","last_undegraded":"2023-10-09T05:40:17.236589+0000","last_fullsized":"2023-10-09T05:40:17.236589+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:34.087553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":182,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.246019+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:40:17.246019+0000","last_peered":"2023-10-09T05:40:17.246019+0000","last_clean":"2023-10-09T05:40:17.246019+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:40:17.246019+0000","last_undegraded":"2023-10-09T05:40:17.246019+0000","last_fullsized":"2023-10-09T05:40:17.246019+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":132,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.232835+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:40:17.232835+0000","last_peered":"2023-10-09T05:40:17.232835+0000","last_clean":"2023-10-09T05:40:17.232835+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:40:17.232835+0000","last_undegraded":"2023-10-09T05:40:17.232835+0000","last_fullsized":"2023-10-09T05:40:17.232835+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"200'9803","reported_seq":15741,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:31.306367+0000","last_change":"2023-10-09T05:38:10.132330+0000","last_active":"2023-10-09T05:40:31.306367+0000","last_peered":"2023-10-09T05:40:31.306367+0000","last_clean":"2023-10-09T05:40:31.306367+0000","last_became_active":"2023-10-09T05:38:10.127080+0000","last_became_peered":"2023-10-09T05:38:10.127080+0000","last_unstale":"2023-10-09T05:40:31.306367+0000","last_undegraded":"2023-10-09T05:40:31.306367+0000","last_fullsized":"2023-10-09T05:40:31.306367+0000","mapping_epoch":179,"log_start":"175'3400","ondisk_log_start":"175'3400","created":162,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6403,"log_dups_size":0,"ondisk_log_size":6403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:12.659227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88145920,"num_objects":1345,"num_object_clones":0,"num_object_copies":4035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9803,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":192413696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":636,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.241884+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:40:17.241884+0000","last_peered":"2023-10-09T05:40:17.241884+0000","last_clean":"2023-10-09T05:40:17.241884+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:40:17.241884+0000","last_undegraded":"2023-10-09T05:40:17.241884+0000","last_fullsized":"2023-10-09T05:40:17.241884+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":167,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233195+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:40:17.233195+0000","last_peered":"2023-10-09T05:40:17.233195+0000","last_clean":"2023-10-09T05:40:17.233195+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:40:17.233195+0000","last_undegraded":"2023-10-09T05:40:17.233195+0000","last_fullsized":"2023-10-09T05:40:17.233195+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":179,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.242138+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:40:17.242138+0000","last_peered":"2023-10-09T05:40:17.242138+0000","last_clean":"2023-10-09T05:40:17.242138+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:40:17.242138+0000","last_undegraded":"2023-10-09T05:40:17.242138+0000","last_fullsized":"2023-10-09T05:40:17.242138+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"200'9640","reported_seq":9697,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:30.711328+0000","last_change":"2023-10-09T05:37:40.649912+0000","last_active":"2023-10-09T05:40:30.711328+0000","last_peered":"2023-10-09T05:40:30.711328+0000","last_clean":"2023-10-09T05:37:34.677100+0000","last_became_active":"2023-10-09T05:37:40.593033+0000","last_became_peered":"2023-10-09T05:37:40.593033+0000","last_unstale":"2023-10-09T05:40:30.711328+0000","last_undegraded":"2023-10-09T05:37:40.546647+0000","last_fullsized":"2023-10-09T05:37:40.543498+0000","mapping_epoch":170,"log_start":"167'1500","ondisk_log_start":"167'1500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8140,"log_dups_size":0,"ondisk_log_size":8140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88473600,"num_objects":1350,"num_object_clones":0,"num_object_copies":4050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9640,"num_write_kb":351680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":-1500}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":514,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.227071+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:40:17.227071+0000","last_peered":"2023-10-09T05:40:17.227071+0000","last_clean":"2023-10-09T05:40:17.227071+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:40:17.227071+0000","last_undegraded":"2023-10-09T05:40:17.227071+0000","last_fullsized":"2023-10-09T05:40:17.227071+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":148,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.929940+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:40:27.929940+0000","last_peered":"2023-10-09T05:40:27.929940+0000","last_clean":"2023-10-09T05:40:27.929940+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:40:27.929940+0000","last_undegraded":"2023-10-09T05:40:27.929940+0000","last_fullsized":"2023-10-09T05:40:27.929940+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":177,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233445+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:40:17.233445+0000","last_peered":"2023-10-09T05:40:17.233445+0000","last_clean":"2023-10-09T05:40:17.233445+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:40:17.233445+0000","last_undegraded":"2023-10-09T05:40:17.233445+0000","last_fullsized":"2023-10-09T05:40:17.233445+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"200'9727","reported_seq":9773,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:29.847441+0000","last_change":"2023-10-09T05:37:01.438236+0000","last_active":"2023-10-09T05:40:29.847441+0000","last_peered":"2023-10-09T05:40:29.847441+0000","last_clean":"2023-10-09T05:40:29.847441+0000","last_became_active":"2023-10-09T05:37:01.437917+0000","last_became_peered":"2023-10-09T05:37:01.437917+0000","last_unstale":"2023-10-09T05:40:29.847441+0000","last_undegraded":"2023-10-09T05:40:29.847441+0000","last_fullsized":"2023-10-09T05:40:29.847441+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9727,"log_dups_size":0,"ondisk_log_size":9727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88276992,"num_objects":1347,"num_object_clones":0,"num_object_copies":4041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9727,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":811,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.241799+0000","last_change":"2023-10-09T05:40:04.805258+0000","last_active":"2023-10-09T05:40:17.241799+0000","last_peered":"2023-10-09T05:40:17.241799+0000","last_clean":"2023-10-09T05:40:17.241799+0000","last_became_active":"2023-10-09T05:40:04.804776+0000","last_became_peered":"2023-10-09T05:40:04.804776+0000","last_unstale":"2023-10-09T05:40:17.241799+0000","last_undegraded":"2023-10-09T05:40:17.241799+0000","last_fullsized":"2023-10-09T05:40:17.241799+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":348,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.246395+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:40:17.246395+0000","last_peered":"2023-10-09T05:40:17.246395+0000","last_clean":"2023-10-09T05:40:17.246395+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:40:17.246395+0000","last_undegraded":"2023-10-09T05:40:17.246395+0000","last_fullsized":"2023-10-09T05:40:17.246395+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":164,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.252567+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:40:17.252567+0000","last_peered":"2023-10-09T05:40:17.252567+0000","last_clean":"2023-10-09T05:40:17.252567+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:40:17.252567+0000","last_undegraded":"2023-10-09T05:40:17.252567+0000","last_fullsized":"2023-10-09T05:40:17.252567+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":173,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.223454+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:40:17.223454+0000","last_peered":"2023-10-09T05:40:17.223454+0000","last_clean":"2023-10-09T05:40:17.223454+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:40:17.223454+0000","last_undegraded":"2023-10-09T05:40:17.223454+0000","last_fullsized":"2023-10-09T05:40:17.223454+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"200'9741","reported_seq":9789,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:33.886702+0000","last_change":"2023-10-09T05:37:01.440172+0000","last_active":"2023-10-09T05:40:33.886702+0000","last_peered":"2023-10-09T05:40:33.886702+0000","last_clean":"2023-10-09T05:40:33.886702+0000","last_became_active":"2023-10-09T05:37:01.439932+0000","last_became_peered":"2023-10-09T05:37:01.439932+0000","last_unstale":"2023-10-09T05:40:33.886702+0000","last_undegraded":"2023-10-09T05:40:33.886702+0000","last_fullsized":"2023-10-09T05:40:33.886702+0000","mapping_epoch":162,"log_start":"174'3300","ondisk_log_start":"174'3300","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6441,"log_dups_size":0,"ondisk_log_size":6441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:57.826704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79888384,"num_objects":1219,"num_object_clones":0,"num_object_copies":3657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9741,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":757,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.232741+0000","last_change":"2023-10-09T05:40:05.930112+0000","last_active":"2023-10-09T05:40:17.232741+0000","last_peered":"2023-10-09T05:40:17.232741+0000","last_clean":"2023-10-09T05:40:17.232741+0000","last_became_active":"2023-10-09T05:40:05.929574+0000","last_became_peered":"2023-10-09T05:40:05.929574+0000","last_unstale":"2023-10-09T05:40:17.232741+0000","last_undegraded":"2023-10-09T05:40:17.232741+0000","last_fullsized":"2023-10-09T05:40:17.232741+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:06.982326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":153,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:23.202531+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:40:23.202531+0000","last_peered":"2023-10-09T05:40:23.202531+0000","last_clean":"2023-10-09T05:40:23.202531+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:40:23.202531+0000","last_undegraded":"2023-10-09T05:40:23.202531+0000","last_fullsized":"2023-10-09T05:40:23.202531+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":167,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.247751+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:40:17.247751+0000","last_peered":"2023-10-09T05:40:17.247751+0000","last_clean":"2023-10-09T05:40:17.247751+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:40:17.247751+0000","last_undegraded":"2023-10-09T05:40:17.247751+0000","last_fullsized":"2023-10-09T05:40:17.247751+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"200'9835","reported_seq":9883,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:33.868518+0000","last_change":"2023-10-09T05:37:01.636967+0000","last_active":"2023-10-09T05:40:33.868518+0000","last_peered":"2023-10-09T05:40:33.868518+0000","last_clean":"2023-10-09T05:40:33.868518+0000","last_became_active":"2023-10-09T05:37:01.636602+0000","last_became_peered":"2023-10-09T05:37:01.636602+0000","last_unstale":"2023-10-09T05:40:33.868518+0000","last_undegraded":"2023-10-09T05:40:33.868518+0000","last_fullsized":"2023-10-09T05:40:33.868518+0000","mapping_epoch":162,"log_start":"175'3400","ondisk_log_start":"175'3400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6435,"log_dups_size":0,"ondisk_log_size":6435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:11.668143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82378752,"num_objects":1257,"num_object_clones":0,"num_object_copies":3771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9835,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":389,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.251259+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:40:17.251259+0000","last_peered":"2023-10-09T05:40:17.251259+0000","last_clean":"2023-10-09T05:40:17.251259+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:40:17.251259+0000","last_undegraded":"2023-10-09T05:40:17.251259+0000","last_fullsized":"2023-10-09T05:40:17.251259+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":147,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.236814+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:40:17.236814+0000","last_peered":"2023-10-09T05:40:17.236814+0000","last_clean":"2023-10-09T05:40:17.236814+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:40:17.236814+0000","last_undegraded":"2023-10-09T05:40:17.236814+0000","last_fullsized":"2023-10-09T05:40:17.236814+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":162,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:23.202586+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:40:23.202586+0000","last_peered":"2023-10-09T05:40:23.202586+0000","last_clean":"2023-10-09T05:40:23.202586+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:40:23.202586+0000","last_undegraded":"2023-10-09T05:40:23.202586+0000","last_fullsized":"2023-10-09T05:40:23.202586+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"200'9810","reported_seq":9856,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:29.941302+0000","last_change":"2023-10-09T05:37:01.440074+0000","last_active":"2023-10-09T05:40:29.941302+0000","last_peered":"2023-10-09T05:40:29.941302+0000","last_clean":"2023-10-09T05:40:29.941302+0000","last_became_active":"2023-10-09T05:37:01.439808+0000","last_became_peered":"2023-10-09T05:37:01.439808+0000","last_unstale":"2023-10-09T05:40:29.941302+0000","last_undegraded":"2023-10-09T05:40:29.941302+0000","last_fullsized":"2023-10-09T05:40:29.941302+0000","mapping_epoch":162,"log_start":"167'2300","ondisk_log_start":"167'2300","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7510,"log_dups_size":0,"ondisk_log_size":7510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85721088,"num_objects":1308,"num_object_clones":0,"num_object_copies":3924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9810,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":199,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.253727+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:40:17.253727+0000","last_peered":"2023-10-09T05:40:17.253727+0000","last_clean":"2023-10-09T05:40:17.253727+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:40:17.253727+0000","last_undegraded":"2023-10-09T05:40:17.253727+0000","last_fullsized":"2023-10-09T05:40:17.253727+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":588,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.234594+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:40:17.234594+0000","last_peered":"2023-10-09T05:40:17.234594+0000","last_clean":"2023-10-09T05:40:17.234594+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:40:17.234594+0000","last_undegraded":"2023-10-09T05:40:17.234594+0000","last_fullsized":"2023-10-09T05:40:17.234594+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":192,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.252631+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:40:17.252631+0000","last_peered":"2023-10-09T05:40:17.252631+0000","last_clean":"2023-10-09T05:40:17.252631+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:40:17.252631+0000","last_undegraded":"2023-10-09T05:40:17.252631+0000","last_fullsized":"2023-10-09T05:40:17.252631+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"200'9518","reported_seq":12540,"reported_epoch":200,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:30.805668+0000","last_change":"2023-10-09T05:40:03.773505+0000","last_active":"2023-10-09T05:40:30.805668+0000","last_peered":"2023-10-09T05:40:30.805668+0000","last_clean":"2023-10-09T05:37:34.656587+0000","last_became_active":"2023-10-09T05:37:36.833510+0000","last_became_peered":"2023-10-09T05:37:36.833510+0000","last_unstale":"2023-10-09T05:40:30.805668+0000","last_undegraded":"2023-10-09T05:37:36.701190+0000","last_fullsized":"2023-10-09T05:37:36.681999+0000","mapping_epoch":170,"log_start":"167'1400","ondisk_log_start":"167'1400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8118,"log_dups_size":0,"ondisk_log_size":8118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83230720,"num_objects":1270,"num_object_clones":0,"num_object_copies":3810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1393,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9518,"num_write_kb":345216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1484,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-123}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":684,"reported_epoch":199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:17.222930+0000","last_change":"2023-10-09T05:37:36.917512+0000","last_active":"2023-10-09T05:40:17.222930+0000","last_peered":"2023-10-09T05:40:17.222930+0000","last_clean":"2023-10-09T05:37:30.688967+0000","last_became_active":"2023-10-09T05:37:36.700218+0000","last_became_peered":"2023-10-09T05:37:36.700218+0000","last_unstale":"2023-10-09T05:40:17.222930+0000","last_undegraded":"2023-10-09T05:37:36.917335+0000","last_fullsized":"2023-10-09T05:37:36.682410+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":155,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.247681+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:40:17.247681+0000","last_peered":"2023-10-09T05:40:17.247681+0000","last_clean":"2023-10-09T05:40:17.247681+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:40:17.247681+0000","last_undegraded":"2023-10-09T05:40:17.247681+0000","last_fullsized":"2023-10-09T05:40:17.247681+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":151,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.232826+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:40:17.232826+0000","last_peered":"2023-10-09T05:40:17.232826+0000","last_clean":"2023-10-09T05:40:17.232826+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:40:17.232826+0000","last_undegraded":"2023-10-09T05:40:17.232826+0000","last_fullsized":"2023-10-09T05:40:17.232826+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":164,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.247159+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:40:17.247159+0000","last_peered":"2023-10-09T05:40:17.247159+0000","last_clean":"2023-10-09T05:40:17.247159+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:40:17.247159+0000","last_undegraded":"2023-10-09T05:40:17.247159+0000","last_fullsized":"2023-10-09T05:40:17.247159+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":191,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.227180+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:40:17.227180+0000","last_peered":"2023-10-09T05:40:17.227180+0000","last_clean":"2023-10-09T05:40:17.227180+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:40:17.227180+0000","last_undegraded":"2023-10-09T05:40:17.227180+0000","last_fullsized":"2023-10-09T05:40:17.227180+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":169,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.241975+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:40:17.241975+0000","last_peered":"2023-10-09T05:40:17.241975+0000","last_clean":"2023-10-09T05:40:17.241975+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:40:17.241975+0000","last_undegraded":"2023-10-09T05:40:17.241975+0000","last_fullsized":"2023-10-09T05:40:17.241975+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":181,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.233239+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:40:17.233239+0000","last_peered":"2023-10-09T05:40:17.233239+0000","last_clean":"2023-10-09T05:40:17.233239+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:40:17.233239+0000","last_undegraded":"2023-10-09T05:40:17.233239+0000","last_fullsized":"2023-10-09T05:40:17.233239+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":154,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.245907+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:40:17.245907+0000","last_peered":"2023-10-09T05:40:17.245907+0000","last_clean":"2023-10-09T05:40:17.245907+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:40:17.245907+0000","last_undegraded":"2023-10-09T05:40:17.245907+0000","last_fullsized":"2023-10-09T05:40:17.245907+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":181,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.232951+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:40:17.232951+0000","last_peered":"2023-10-09T05:40:17.232951+0000","last_clean":"2023-10-09T05:40:17.232951+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:40:17.232951+0000","last_undegraded":"2023-10-09T05:40:17.232951+0000","last_fullsized":"2023-10-09T05:40:17.232951+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":179,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.252815+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:40:17.252815+0000","last_peered":"2023-10-09T05:40:17.252815+0000","last_clean":"2023-10-09T05:40:17.252815+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:40:17.252815+0000","last_undegraded":"2023-10-09T05:40:17.252815+0000","last_fullsized":"2023-10-09T05:40:17.252815+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":191,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.252814+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:40:17.252814+0000","last_peered":"2023-10-09T05:40:17.252814+0000","last_clean":"2023-10-09T05:40:17.252814+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:40:17.252814+0000","last_undegraded":"2023-10-09T05:40:17.252814+0000","last_fullsized":"2023-10-09T05:40:17.252814+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":156,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:23.202651+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:40:23.202651+0000","last_peered":"2023-10-09T05:40:23.202651+0000","last_clean":"2023-10-09T05:40:23.202651+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:40:23.202651+0000","last_undegraded":"2023-10-09T05:40:23.202651+0000","last_fullsized":"2023-10-09T05:40:23.202651+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":166,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.232825+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:40:17.232825+0000","last_peered":"2023-10-09T05:40:17.232825+0000","last_clean":"2023-10-09T05:40:17.232825+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:40:17.232825+0000","last_undegraded":"2023-10-09T05:40:17.232825+0000","last_fullsized":"2023-10-09T05:40:17.232825+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1364787224,"num_objects":20826,"num_object_clones":0,"num_object_copies":62478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7133,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20826,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":155896,"num_write_kb":5655041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10342,"num_bytes_recovered":530448384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4799803392,"data_stored":4799791176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128596,"ondisk_log_size":128596,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846663,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626689,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1010732,"kb_used_data":705316,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92734420,"statfs":{"total":95995035648,"available":94960046080,"internally_reserved":0,"allocated":722243584,"data_stored":721722208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:25 2023","interfaces":[{"interface":"back","average":{"1min":14.522,"5min":6.984,"15min":3.121},"min":{"1min":0.24199999999999999,"5min":0.214,"15min":0.214},"max":{"1min":115.377,"5min":115.377,"15min":115.377},"last":0.34599999999999997},{"interface":"front","average":{"1min":14.285,"5min":6.6109999999999998,"15min":2.9870000000000001},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.23999999999999999},"max":{"1min":116.428,"5min":116.428,"15min":116.428},"last":2.23}]},{"osd":1,"last update":"Mon Oct 9 05:39:46 2023","interfaces":[{"interface":"back","average":{"1min":13.882,"5min":5.0300000000000002,"15min":2.2690000000000001},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.25},"max":{"1min":137.02199999999999,"5min":137.02199999999999,"15min":137.02199999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":13.885,"5min":5.2430000000000003,"15min":2.355},"min":{"1min":0.38100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":137.06700000000001,"5min":137.06700000000001,"15min":137.06700000000001},"last":0.44500000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:40:25 2023","interfaces":[{"interface":"back","average":{"1min":8.1809999999999992,"5min":5.5670000000000002,"15min":3.0070000000000001},"min":{"1min":0.443,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":115.40600000000001,"5min":115.40600000000001,"15min":115.40600000000001},"last":4.9569999999999999},{"interface":"front","average":{"1min":8.7759999999999998,"5min":5.6310000000000002,"15min":3.0070000000000001},"min":{"1min":0.434,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":115.297,"5min":115.297,"15min":115.297},"last":4.9729999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:46 2023","interfaces":[{"interface":"back","average":{"1min":12.359999999999999,"5min":4.7290000000000001,"15min":2.4140000000000001},"min":{"1min":0.307,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":115.244,"5min":115.244,"15min":115.244},"last":0.35799999999999998},{"interface":"front","average":{"1min":12.371,"5min":4.9870000000000001,"15min":2.5099999999999998},"min":{"1min":0.36499999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":115.34999999999999,"5min":115.34999999999999,"15min":115.34999999999999},"last":3.3279999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":5.181,"5min":2.177,"15min":1.7150000000000001},"min":{"1min":0.249,"5min":0.245,"15min":0.23400000000000001},"max":{"1min":115.214,"5min":115.214,"15min":125.64},"last":0.38},{"interface":"front","average":{"1min":5.282,"5min":1.766,"15min":1.597},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":115.22799999999999,"5min":115.22799999999999,"15min":125.625},"last":1.2}]},{"osd":5,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":7.7560000000000002,"5min":2.8050000000000002,"15min":1.819},"min":{"1min":0.31,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":115.316,"5min":115.316,"15min":127.35899999999999},"last":0.59499999999999997},{"interface":"front","average":{"1min":7.601,"5min":2.339,"15min":1.704},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29199999999999998},"max":{"1min":115.30500000000001,"5min":115.30500000000001,"15min":128.93799999999999},"last":1.2410000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:39:47 2023","interfaces":[{"interface":"back","average":{"1min":5.5990000000000002,"5min":2.5710000000000002,"15min":2.9550000000000001},"min":{"1min":0.35899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":115.514,"5min":115.514,"15min":115.514},"last":0.56499999999999995},{"interface":"front","average":{"1min":5.6500000000000004,"5min":2.0600000000000001,"15min":2.081},"min":{"1min":0.35499999999999998,"5min":0.317,"15min":0.317},"max":{"1min":116.53100000000001,"5min":116.53100000000001,"15min":116.53100000000001},"last":1.2709999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958651,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":720152,"kb_used_data":472144,"kb_used_omap":7,"kb_used_meta":247928,"kb_avail":93025000,"statfs":{"total":95995035648,"available":95257600000,"internally_reserved":0,"allocated":483475456,"data_stored":482740086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":253878798},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9460000000000002,"5min":5.7850000000000001,"15min":2.3220000000000001},"min":{"1min":0.20300000000000001,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":13.214,"5min":207.858,"15min":207.858},"last":0.23899999999999999},{"interface":"front","average":{"1min":2.4529999999999998,"5min":6.7469999999999999,"15min":2.6859999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.849,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.25800000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:40:11 2023","interfaces":[{"interface":"back","average":{"1min":4.6580000000000004,"5min":6.79,"15min":3.1099999999999999},"min":{"1min":0.22,"5min":0.184,"15min":0.184},"max":{"1min":31.029,"5min":101.367,"15min":101.367},"last":0.499},{"interface":"front","average":{"1min":5.5090000000000003,"5min":6.6219999999999999,"15min":2.9649999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":38.622999999999998,"5min":97.286000000000001,"15min":98.409999999999997},"last":0.28100000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":6.1749999999999998,"5min":3.8559999999999999,"15min":1.744},"min":{"1min":0.35399999999999998,"5min":0.223,"15min":0.223},"max":{"1min":42.445999999999998,"5min":83.885000000000005,"15min":83.885000000000005},"last":0.53500000000000003},{"interface":"front","average":{"1min":5.4379999999999997,"5min":4.7290000000000001,"15min":1.954},"min":{"1min":0.373,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":42.243000000000002,"5min":84.641999999999996,"15min":84.641999999999996},"last":0.30599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":4.1799999999999997,"15min":1.9670000000000001},"min":{"1min":0.35599999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.813000000000001,"5min":141.32900000000001,"15min":141.32900000000001},"last":0.71799999999999997},{"interface":"front","average":{"1min":2.4129999999999998,"5min":2.9420000000000002,"15min":1.532},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.4509999999999996,"5min":141.15899999999999,"15min":141.15899999999999},"last":4.282}]},{"osd":5,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":3.9020000000000001,"15min":1.901},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.6769999999999996,"5min":235.18700000000001,"15min":235.18700000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.7330000000000001,"5min":4.8550000000000004,"15min":2.319},"min":{"1min":0.38300000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":10.779,"5min":235.79300000000001,"15min":235.79300000000001},"last":0.68899999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":7.46,"5min":2.9140000000000001,"15min":1.5289999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":103.60899999999999,"5min":103.60899999999999,"15min":103.60899999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":7.3520000000000003,"5min":3.8500000000000001,"15min":1.837},"min":{"1min":0.38,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":103.679,"5min":103.679,"15min":103.679},"last":0.73399999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7719999999999998,"5min":2.9489999999999998,"15min":2.3149999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.7390000000000008,"5min":126.68300000000001,"15min":126.68300000000001},"last":4.3470000000000004},{"interface":"front","average":{"1min":2.927,"5min":3.0800000000000001,"15min":2.4119999999999999},"min":{"1min":0.46600000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.7720000000000002,"5min":126.738,"15min":126.738},"last":4.3220000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122177,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":663416,"kb_used_data":329520,"kb_used_omap":8,"kb_used_meta":333879,"kb_avail":93081736,"statfs":{"total":95995035648,"available":95315697664,"internally_reserved":0,"allocated":337428480,"data_stored":336662940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":341892872},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":1.9379999999999999,"15min":1.0760000000000001},"min":{"1min":0.219,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":12.005000000000001,"5min":14.964,"15min":14.964},"last":0.249},{"interface":"front","average":{"1min":3.4350000000000001,"5min":2.5179999999999998,"15min":1.2809999999999999},"min":{"1min":0.22900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.218999999999999,"5min":15.218999999999999,"15min":15.218999999999999},"last":0.39500000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":5.0410000000000004,"5min":4.3979999999999997,"15min":2.2000000000000002},"min":{"1min":0.26800000000000002,"5min":0.223,"15min":0.214},"max":{"1min":21.648,"5min":79.911000000000001,"15min":79.911000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":5.4370000000000003,"5min":4.2549999999999999,"15min":2.1040000000000001},"min":{"1min":0.26000000000000001,"5min":0.221,"15min":0.218},"max":{"1min":32.344000000000001,"5min":80.105000000000004,"15min":80.105000000000004},"last":0.94299999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":4.0839999999999996,"5min":3.2730000000000001,"15min":1.5609999999999999},"min":{"1min":0.27000000000000002,"5min":0.23999999999999999,"15min":0.22800000000000001},"max":{"1min":10.519,"5min":31.594000000000001,"15min":31.594000000000001},"last":4.8710000000000004},{"interface":"front","average":{"1min":4.3410000000000002,"5min":3.3730000000000002,"15min":1.651},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.222},"max":{"1min":16.937999999999999,"5min":31.641999999999999,"15min":31.641999999999999},"last":4.7210000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.8420000000000001,"15min":1.2050000000000001},"min":{"1min":0.39500000000000002,"5min":0.27500000000000002,"15min":0.27200000000000002},"max":{"1min":5.157,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.9119999999999999,"5min":2.5409999999999999,"15min":1.5149999999999999},"min":{"1min":0.34300000000000003,"5min":0.33000000000000002,"15min":0.30399999999999999},"max":{"1min":9.5190000000000001,"5min":69.102000000000004,"15min":69.102000000000004},"last":0.44500000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":18.456,"5min":6.2709999999999999,"15min":2.7360000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":281.733,"5min":281.733,"15min":281.733},"last":0.48499999999999999},{"interface":"front","average":{"1min":19.111999999999998,"5min":6.2210000000000001,"15min":2.7370000000000001},"min":{"1min":0.40899999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":282.95400000000001,"5min":282.95400000000001,"15min":282.95400000000001},"last":0.35499999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5529999999999999,"5min":2.0110000000000001,"15min":1.3280000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":0.51500000000000001},{"interface":"front","average":{"1min":3.5190000000000001,"5min":2.1779999999999999,"15min":1.387},"min":{"1min":0.46200000000000002,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":10.83,"5min":10.83,"15min":10.83},"last":0.435}]},{"osd":7,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":1.544,"15min":1.3999999999999999},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.9740000000000002,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.41899999999999998},{"interface":"front","average":{"1min":3.2999999999999998,"5min":2.3879999999999999,"15min":2.3889999999999998},"min":{"1min":0.5,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.4100000000000001,"5min":11.068,"15min":11.068},"last":0.54900000000000004}]}]},{"osd":2,"up_from":98,"seq":420906795124,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1167028,"kb_used_data":759340,"kb_used_omap":11,"kb_used_meta":407668,"kb_avail":92578124,"statfs":{"total":95995035648,"available":94799998976,"internally_reserved":0,"allocated":777564160,"data_stored":776746872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":417452629},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":4.3079999999999998,"5min":3.0630000000000002,"15min":1.53},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17199999999999999},"max":{"1min":20.771000000000001,"5min":20.771000000000001,"15min":32.122},"last":0.35199999999999998},{"interface":"front","average":{"1min":6.133,"5min":3.0640000000000001,"15min":1.5229999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":21.997,"5min":21.997,"15min":34.066000000000003},"last":8.5960000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":4.8250000000000002,"15min":2.3109999999999999},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.20699999999999999},"max":{"1min":9.3469999999999995,"5min":98.575000000000003,"15min":98.575000000000003},"last":8.7579999999999991},{"interface":"front","average":{"1min":2.431,"5min":3.9820000000000002,"15min":2.0030000000000001},"min":{"1min":0.23899999999999999,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":9.3260000000000005,"5min":90.135000000000005,"15min":98.397000000000006},"last":4.7430000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":15.612,"5min":5.4580000000000002,"15min":2.2930000000000001},"min":{"1min":0.55200000000000005,"5min":0.249,"15min":0.24199999999999999},"max":{"1min":150.65899999999999,"5min":150.65899999999999,"15min":150.65899999999999},"last":4.8310000000000004},{"interface":"front","average":{"1min":17.797000000000001,"5min":6.25,"15min":2.601},"min":{"1min":0.54900000000000004,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":150.851,"5min":150.851,"15min":150.851},"last":22.631}]},{"osd":4,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2629999999999999,"5min":3.0019999999999998,"15min":1.5149999999999999},"min":{"1min":0.29799999999999999,"5min":0.28999999999999998,"15min":0.27700000000000002},"max":{"1min":9.3000000000000007,"5min":115.84999999999999,"15min":115.84999999999999},"last":4.7729999999999997},{"interface":"front","average":{"1min":3.2669999999999999,"5min":3.887,"15min":1.8500000000000001},"min":{"1min":0.48399999999999999,"5min":0.312,"15min":0.29799999999999999},"max":{"1min":10.818,"5min":116.149,"15min":116.149},"last":0.53500000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":2.964,"5min":2.7690000000000001,"15min":1.524},"min":{"1min":0.44,"5min":0.312,"15min":0.312},"max":{"1min":8.7509999999999994,"5min":24.597999999999999,"15min":24.597999999999999},"last":8.6780000000000008},{"interface":"front","average":{"1min":2.3039999999999998,"5min":2.282,"15min":1.292},"min":{"1min":0.32700000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.2590000000000003,"5min":23.919,"15min":23.919},"last":4.8090000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":3.2869999999999999,"5min":2.7719999999999998,"15min":1.5049999999999999},"min":{"1min":0.41199999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.864000000000001,"5min":20.503,"15min":20.503},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.9940000000000002,"5min":2.4489999999999998,"15min":1.401},"min":{"1min":0.46800000000000003,"5min":0.39600000000000002,"15min":0.34499999999999997},"max":{"1min":8.7949999999999999,"5min":9.093,"15min":9.4049999999999994},"last":8.7070000000000007}]},{"osd":7,"last update":"Mon Oct 9 05:39:52 2023","interfaces":[{"interface":"back","average":{"1min":3.294,"5min":3.0369999999999999,"15min":2.6949999999999998},"min":{"1min":0.436,"5min":0.374,"15min":0.374},"max":{"1min":10.897,"5min":24.030999999999999,"15min":24.030999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":3.0299999999999998,"5min":2.7549999999999999,"15min":2.0710000000000002},"min":{"1min":0.48899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.8230000000000004,"5min":24.178000000000001,"15min":24.178000000000001},"last":8.7319999999999993}]}]},{"osd":3,"up_from":103,"seq":442381631598,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1206808,"kb_used_data":932432,"kb_used_omap":4,"kb_used_meta":274299,"kb_avail":92538344,"statfs":{"total":95995035648,"available":94759264256,"internally_reserved":0,"allocated":954810368,"data_stored":954074423,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":280882782},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":14.382,"5min":4.5739999999999998,"15min":2.129},"min":{"1min":0.19800000000000001,"5min":0.183,"15min":0.183},"max":{"1min":167.82400000000001,"5min":167.82400000000001,"15min":167.82400000000001},"last":0.318},{"interface":"front","average":{"1min":15.930999999999999,"5min":5.0309999999999997,"15min":2.3330000000000002},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.214},"max":{"1min":167.89599999999999,"5min":167.89599999999999,"15min":167.89599999999999},"last":26.140000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":3.218,"5min":3.8919999999999999,"15min":1.657},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.663,"5min":83.921999999999997,"15min":83.921999999999997},"last":5.468},{"interface":"front","average":{"1min":3.956,"5min":3.911,"15min":1.651},"min":{"1min":0.25,"5min":0.25,"15min":0.247},"max":{"1min":12.061999999999999,"5min":83.707999999999998,"15min":83.707999999999998},"last":0.56100000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":16.274000000000001,"5min":6.6950000000000003,"15min":3.298},"min":{"1min":0.26500000000000001,"5min":0.215,"15min":0.215},"max":{"1min":190.43299999999999,"5min":190.43299999999999,"15min":190.43299999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":21.088000000000001,"5min":7.8490000000000002,"15min":3.6960000000000002},"min":{"1min":0.317,"5min":0.28999999999999998,"15min":0.26000000000000001},"max":{"1min":190.267,"5min":190.267,"15min":190.267},"last":26.097999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":6.7539999999999996,"5min":2.9540000000000002,"15min":1.7390000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.29699999999999999},"max":{"1min":51.591999999999999,"5min":51.591999999999999,"15min":51.591999999999999},"last":25.946999999999999},{"interface":"front","average":{"1min":2.9209999999999998,"5min":1.9790000000000001,"15min":1.3620000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":11.271000000000001,"5min":13.734,"15min":45.735999999999997},"last":0.54500000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":2.0379999999999998,"15min":1.325},"min":{"1min":0.45100000000000001,"5min":0.33500000000000002,"15min":0.29999999999999999},"max":{"1min":8.3640000000000008,"5min":8.4770000000000003,"15min":47.948999999999998},"last":0.53300000000000003},{"interface":"front","average":{"1min":3.1640000000000001,"5min":2.1659999999999999,"15min":1.4299999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.339,"5min":13.864000000000001,"15min":47.968000000000004},"last":0.57599999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.2120000000000002,"15min":1.321},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.313000000000001,"5min":13.831,"15min":13.831},"last":5.5129999999999999},{"interface":"front","average":{"1min":2.8260000000000001,"5min":2.1019999999999999,"15min":1.1299999999999999},"min":{"1min":0.45700000000000002,"5min":0.373,"15min":0.33900000000000002},"max":{"1min":8.3979999999999997,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":0.495}]},{"osd":7,"last update":"Mon Oct 9 05:39:53 2023","interfaces":[{"interface":"back","average":{"1min":3.9590000000000001,"5min":2.6699999999999999,"15min":2.3959999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":20.065000000000001,"5min":20.065000000000001,"15min":20.065000000000001},"last":26.024999999999999},{"interface":"front","average":{"1min":4.0590000000000002,"5min":2.6629999999999998,"15min":2.1960000000000002},"min":{"1min":0.40899999999999997,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":20.138000000000002,"5min":20.138000000000002,"15min":20.138000000000002},"last":25.995999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019145,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":753196,"kb_used_data":443684,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92991956,"statfs":{"total":95995035648,"available":95223762944,"internally_reserved":0,"allocated":454332416,"data_stored":453810784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:22 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":1.929,"15min":1.3180000000000001},"min":{"1min":0.248,"5min":0.214,"15min":0.214},"max":{"1min":6.2590000000000003,"5min":36.406999999999996,"15min":36.406999999999996},"last":0.28000000000000003},{"interface":"front","average":{"1min":2.4460000000000002,"5min":2.556,"15min":1.5800000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.7629999999999999,"5min":36.454999999999998,"15min":36.454999999999998},"last":7.077}]},{"osd":1,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":1.7829999999999999,"15min":1.1299999999999999},"min":{"1min":0.28599999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.194,"5min":10.073,"15min":31.073},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.6709999999999998,"5min":1.7869999999999999,"15min":1.2010000000000001},"min":{"1min":0.33500000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.5279999999999996,"5min":10.055,"15min":31.100000000000001},"last":0.36499999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":6.4269999999999996,"15min":3.476},"min":{"1min":0.31,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":10.099,"5min":196.93700000000001,"15min":196.93700000000001},"last":0.315},{"interface":"front","average":{"1min":3.048,"5min":6.1829999999999998,"15min":3.3679999999999999},"min":{"1min":0.34399999999999997,"5min":0.28299999999999997,"15min":0.28100000000000003},"max":{"1min":11.534000000000001,"5min":197.08099999999999,"15min":197.08099999999999},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":5.6639999999999997,"5min":6.8849999999999998,"15min":2.8439999999999999},"min":{"1min":0.35399999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":34.615000000000002,"5min":211.48699999999999,"15min":211.48699999999999},"last":7.2809999999999997},{"interface":"front","average":{"1min":5.8650000000000002,"5min":6.8010000000000002,"15min":2.8010000000000002},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.309},"max":{"1min":34.57,"5min":211.31100000000001,"15min":211.31100000000001},"last":1.0640000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":3.4489999999999998,"15min":1.5429999999999999},"min":{"1min":0.26300000000000001,"5min":0.25,"15min":0.249},"max":{"1min":3.5259999999999998,"5min":113.35899999999999,"15min":113.35899999999999},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":3.302,"15min":1.52},"min":{"1min":0.35499999999999998,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":3.2160000000000002,"5min":113.32299999999999,"15min":113.32299999999999},"last":0.41899999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:39:57 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.999,"15min":0.70899999999999996},"min":{"1min":0.317,"5min":0.28999999999999998,"15min":0.28100000000000003},"max":{"1min":3.4390000000000001,"5min":11.944000000000001,"15min":11.944000000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.3300000000000001,"15min":0.85899999999999999},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.6500000000000004,"5min":10.356,"15min":10.356},"last":0.33700000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":1.4810000000000001,"15min":2.7400000000000002},"min":{"1min":0.37,"5min":0.312,"15min":0.312},"max":{"1min":4.5819999999999999,"5min":30.393999999999998,"15min":30.393999999999998},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.601,"15min":3.0870000000000002},"min":{"1min":0.36499999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5999999999999996,"5min":30.334,"15min":30.334},"last":0.379}]}]},{"osd":5,"up_from":36,"seq":154618822917,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":748192,"kb_used_data":446872,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92996960,"statfs":{"total":95995035648,"available":95228887040,"internally_reserved":0,"allocated":457596928,"data_stored":457083056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":4.4219999999999997,"15min":2.9369999999999998},"min":{"1min":0.22800000000000001,"5min":0.22700000000000001,"15min":0.19400000000000001},"max":{"1min":8.3079999999999998,"5min":165.90199999999999,"15min":165.90199999999999},"last":0.318},{"interface":"front","average":{"1min":1.377,"5min":4.2290000000000001,"15min":2.9910000000000001},"min":{"1min":0.28399999999999997,"5min":0.215,"15min":0.215},"max":{"1min":5.4660000000000002,"5min":165.96700000000001,"15min":165.96700000000001},"last":7.4050000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":5.3810000000000002,"5min":3.7469999999999999,"15min":1.738},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":58.176000000000002,"5min":87.769000000000005,"15min":87.769000000000005},"last":4.9119999999999999},{"interface":"front","average":{"1min":7.1660000000000004,"5min":3.8940000000000001,"15min":1.7769999999999999},"min":{"1min":0.32200000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":54.591000000000001,"5min":87.864000000000004,"15min":87.864000000000004},"last":2.5470000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:40:20 2023","interfaces":[{"interface":"back","average":{"1min":7.2709999999999999,"5min":6.7069999999999999,"15min":4.008},"min":{"1min":0.33600000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":73.311999999999998,"5min":118.255,"15min":290.697},"last":4.3419999999999996},{"interface":"front","average":{"1min":6.1639999999999997,"5min":6.0199999999999996,"15min":3.7810000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":73.280000000000001,"5min":118.438,"15min":290.786},"last":4.6269999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":16.713999999999999,"5min":5.8200000000000003,"15min":2.4249999999999998},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":95.236999999999995,"5min":95.236999999999995,"15min":95.236999999999995},"last":10.231},{"interface":"front","average":{"1min":16.225000000000001,"5min":5.3680000000000003,"15min":2.3100000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":93.471000000000004,"5min":93.471000000000004,"15min":93.471000000000004},"last":4.0739999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.94399999999999995,"15min":0.68100000000000005},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":0.91100000000000003,"5min":9.2729999999999997,"15min":9.3510000000000009},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.4990000000000001,"15min":0.92900000000000005},"min":{"1min":0.32700000000000001,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":1.4259999999999999,"5min":14.236000000000001,"15min":14.236000000000001},"last":0.68100000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:40:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1139999999999999,"5min":1.8120000000000001,"15min":1.3420000000000001},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.26000000000000001},"max":{"1min":27.146000000000001,"5min":27.146000000000001,"15min":95.010999999999996},"last":0.63900000000000001},{"interface":"front","average":{"1min":2.1269999999999998,"5min":1.4690000000000001,"15min":1.2290000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27700000000000002},"max":{"1min":27.215,"5min":27.215,"15min":94.194000000000003},"last":1.056}]},{"osd":7,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.27,"15min":1.573},"min":{"1min":0.34399999999999997,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.94099999999999995,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":1.3400000000000001,"15min":1.8620000000000001},"min":{"1min":0.35899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.5029999999999999,"5min":14.432,"15min":14.432},"last":0.66000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":326701056,"data_stored":326696984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":478285824,"data_stored":478281752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":755367936,"data_stored":755367936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":937623552,"data_stored":937623552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":440991744,"data_stored":440991744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":445386752,"data_stored":445382680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":698548224,"data_stored":698548224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":716898304,"data_stored":716898304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:40:35.336 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:40:35.369 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:35.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:35.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:35.482 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:35.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:35 smithi191 ceph-mon[161669]: pgmap v564: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 7136/62562 objects degraded (11.406%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:35 smithi191 ceph-mon[161669]: from='client.35292 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:35.597 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:35 smithi008 ceph-mon[184418]: pgmap v564: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 7136/62562 objects degraded (11.406%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:35.597 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:35 smithi008 ceph-mon[184418]: from='client.35292 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:35.599 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:35 smithi008 ceph-mon[191227]: pgmap v564: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 7136/62562 objects degraded (11.406%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:35.599 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:35 smithi008 ceph-mon[191227]: from='client.35292 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:35.606 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:35.790 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:40:35.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:35.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:35.897 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:35.924 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:36.539 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:36.666 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:36.777 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:36.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:36.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:36.884 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:36.985 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:37.295 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:37.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:37.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:37.405 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:37.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:37 smithi008 ceph-mon[184418]: pgmap v565: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 6849/59136 objects degraded (11.582%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:37.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:37 smithi008 ceph-mon[191227]: pgmap v565: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 6849/59136 objects degraded (11.582%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:37.543 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:37 smithi191 ceph-mon[161669]: pgmap v565: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 6849/59136 objects degraded (11.582%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:38.241 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:38 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2554274870' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:40:38.241 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 6849/59136 objects degraded (11.582%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:38.281 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:40:38.288 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:40:38.608 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:38 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2554274870' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:40:38.608 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 6849/59136 objects degraded (11.582%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:38.611 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:38 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2554274870' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:40:38.611 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 6849/59136 objects degraded (11.582%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:38.785 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:38.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:38.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:38.890 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:38.982 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:39.250 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[184418]: pgmap v566: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 6849/56535 objects degraded (12.115%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:39.250 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:39.250 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[184418]: failed to encode map e201 with expected crc 2023-10-09T05:40:39.251 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2554274870' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:40:39.251 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[184418]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T05:40:39.251 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[184418]: failed to encode map e201 with expected crc 2023-10-09T05:40:39.251 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[184418]: failed to encode map e201 with expected crc 2023-10-09T05:40:39.251 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:39.251 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:39.535 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[191227]: pgmap v566: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 6849/56535 objects degraded (12.115%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:39.536 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:39.536 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[191227]: failed to encode map e201 with expected crc 2023-10-09T05:40:39.536 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2554274870' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:40:39.536 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[191227]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T05:40:39.536 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[191227]: failed to encode map e201 with expected crc 2023-10-09T05:40:39.536 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[191227]: failed to encode map e201 with expected crc 2023-10-09T05:40:39.536 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:39.536 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:39 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:39 smithi191 ceph-mon[161669]: pgmap v566: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 6849/56535 objects degraded (12.115%); 0 B/s, 52 objects/s recovering 2023-10-09T05:40:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:39 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:39 smithi191 ceph-mon[161669]: failed to encode map e201 with expected crc 2023-10-09T05:40:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2554274870' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:40:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:39 smithi191 ceph-mon[161669]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T05:40:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:39 smithi191 ceph-mon[161669]: failed to encode map e201 with expected crc 2023-10-09T05:40:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:39 smithi191 ceph-mon[161669]: failed to encode map e201 with expected crc 2023-10-09T05:40:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:39 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:39 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:39.811 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:39.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:39.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:39.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:39.958 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:40.025 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:40.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:40.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:40.140 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:40.174 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:40.230 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:40.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:40.336 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:40.449 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:40.478 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:40.572 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:40.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:40.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:40.684 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:40.705 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:40:40.712 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:40.719 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:40:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:40:40] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.33.4" 2023-10-09T05:40:40.748 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:40:40.766 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:40.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:40.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:40.793 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:40:40.855 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:40:40.874 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:40.890 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:40:40.923 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:40:40.947 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:40:40.948 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:40:40.948 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:40:40.948 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:40:40.948 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:40:40.972 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:40:40.997 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:41.020 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:40:41.061 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:40:41.062 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:40:41.062 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:40:41.062 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:40:41.062 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:40:41.062 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:40:41.062 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:40:41.062 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:40:41.270 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:41 smithi008 ceph-mon[184418]: pgmap v568: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 6849/51570 objects degraded (13.281%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:41.273 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:41 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3690695359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:41.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:41 smithi191 ceph-mon[161669]: pgmap v568: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 6849/51570 objects degraded (13.281%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:41.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:41 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3690695359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:41.644 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:40:41.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:41 smithi008 ceph-mon[191227]: pgmap v568: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 6849/51570 objects degraded (13.281%); 0 B/s, 53 objects/s recovering 2023-10-09T05:40:41.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:41 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3690695359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:42.181 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:42.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:42.298 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:42.341 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:42.412 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:42.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:42.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:42.523 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:42.605 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:43.286 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:40:43.545 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:43 smithi008 ceph-mon[191227]: pgmap v569: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1001 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 6590/47931 objects degraded (13.749%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:43.545 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 6590/47931 objects degraded (13.749%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:43.548 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:43 smithi008 ceph-mon[184418]: pgmap v569: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1001 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 6590/47931 objects degraded (13.749%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:43.548 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 6590/47931 objects degraded (13.749%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:43.553 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:43.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:43.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:43.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:43.691 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:43 smithi191 ceph-mon[161669]: pgmap v569: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1001 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 6590/47931 objects degraded (13.749%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:43.691 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 6590/47931 objects degraded (13.749%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:43.694 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:43.750 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:43.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:43.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:43.861 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:43.894 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:43.901 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:43.946 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:43.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:44.055 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:44.097 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:44.160 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:44.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:44.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:44.265 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:44.354 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:44.527 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:44.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:44.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:44.644 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:44.672 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:44.723 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:44.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:44.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:44.838 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:44.873 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:44.906 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:40:44.922 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:44.922 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:44.922 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:44 smithi191 ceph-mon[161669]: pgmap v570: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 959 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 6590/45900 objects degraded (14.357%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:44.922 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:44.923 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:44.923 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:44.923 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:44 smithi008 ceph-mon[191227]: pgmap v570: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 959 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 6590/45900 objects degraded (14.357%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:44.926 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:40:44.933 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:45.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:45.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:44 smithi008 ceph-mon[184418]: pgmap v570: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 959 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 6590/45900 objects degraded (14.357%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:45.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:45.393 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:45.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:45.494 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:45.523 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:45.585 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:45.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:45.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:45.592 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:40:45.690 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:45.802 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:45.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:45 smithi008 ceph-mon[184418]: from='client.55150 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:45.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:45 smithi008 ceph-mon[191227]: from='client.55150 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:46.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:45 smithi191 ceph-mon[161669]: from='client.55150 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:46.584 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":570,"stamp":"2023-10-09T05:40:44.131666+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1005590475,"num_objects":15300,"num_object_clones":0,"num_object_copies":45900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6590,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15300,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":162556,"num_write_kb":5670182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10951,"num_bytes_recovered":531043702,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131020,"ondisk_log_size":131020,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5237908,"kb_used_data":3019676,"kb_used_omap":31,"kb_used_meta":2217760,"kb_avail":650978156,"statfs":{"total":671965249536,"available":666601631744,"internally_reserved":0,"allocated":3092148224,"data_stored":3087527991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2270986697},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,8,2,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-363986944,"num_objects":-5554,"num_object_clones":0,"num_object_copies":-16662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-546,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5554,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":546,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007674"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":184,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927567+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:40:27.927567+0000","last_peered":"2023-10-09T05:40:27.927567+0000","last_clean":"2023-10-09T05:40:27.927567+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:40:27.927567+0000","last_undegraded":"2023-10-09T05:40:27.927567+0000","last_fullsized":"2023-10-09T05:40:27.927567+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":173,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.923826+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:40:27.923826+0000","last_peered":"2023-10-09T05:40:27.923826+0000","last_clean":"2023-10-09T05:40:27.923826+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:40:27.923826+0000","last_undegraded":"2023-10-09T05:40:27.923826+0000","last_fullsized":"2023-10-09T05:40:27.923826+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":148,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.928321+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:40:27.928321+0000","last_peered":"2023-10-09T05:40:27.928321+0000","last_clean":"2023-10-09T05:40:27.928321+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:40:27.928321+0000","last_undegraded":"2023-10-09T05:40:27.928321+0000","last_fullsized":"2023-10-09T05:40:27.928321+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":178,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.931126+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:40:27.931126+0000","last_peered":"2023-10-09T05:40:27.931126+0000","last_clean":"2023-10-09T05:40:27.931126+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:40:27.931126+0000","last_undegraded":"2023-10-09T05:40:27.931126+0000","last_fullsized":"2023-10-09T05:40:27.931126+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":189,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927543+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:40:27.927543+0000","last_peered":"2023-10-09T05:40:27.927543+0000","last_clean":"2023-10-09T05:40:27.927543+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:40:27.927543+0000","last_undegraded":"2023-10-09T05:40:27.927543+0000","last_fullsized":"2023-10-09T05:40:27.927543+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":167,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.917327+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:40:27.917327+0000","last_peered":"2023-10-09T05:40:27.917327+0000","last_clean":"2023-10-09T05:40:27.917327+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:40:27.917327+0000","last_undegraded":"2023-10-09T05:40:27.917327+0000","last_fullsized":"2023-10-09T05:40:27.917327+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":153,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.931207+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:40:27.931207+0000","last_peered":"2023-10-09T05:40:27.931207+0000","last_clean":"2023-10-09T05:40:27.931207+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:40:27.931207+0000","last_undegraded":"2023-10-09T05:40:27.931207+0000","last_fullsized":"2023-10-09T05:40:27.931207+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":173,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927619+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:40:27.927619+0000","last_peered":"2023-10-09T05:40:27.927619+0000","last_clean":"2023-10-09T05:40:27.927619+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:40:27.927619+0000","last_undegraded":"2023-10-09T05:40:27.927619+0000","last_fullsized":"2023-10-09T05:40:27.927619+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":192,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.920362+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:40:27.920362+0000","last_peered":"2023-10-09T05:40:27.920362+0000","last_clean":"2023-10-09T05:40:27.920362+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:40:27.920362+0000","last_undegraded":"2023-10-09T05:40:27.920362+0000","last_fullsized":"2023-10-09T05:40:27.920362+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":187,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.923605+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:40:27.923605+0000","last_peered":"2023-10-09T05:40:27.923605+0000","last_clean":"2023-10-09T05:40:27.923605+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:40:27.923605+0000","last_undegraded":"2023-10-09T05:40:27.923605+0000","last_fullsized":"2023-10-09T05:40:27.923605+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":81,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.923746+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:40:27.923746+0000","last_peered":"2023-10-09T05:40:27.923746+0000","last_clean":"2023-10-09T05:40:27.923746+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:40:27.923746+0000","last_undegraded":"2023-10-09T05:40:27.923746+0000","last_fullsized":"2023-10-09T05:40:27.923746+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":148,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.928645+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:40:27.928645+0000","last_peered":"2023-10-09T05:40:27.928645+0000","last_clean":"2023-10-09T05:40:27.928645+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:40:27.928645+0000","last_undegraded":"2023-10-09T05:40:27.928645+0000","last_fullsized":"2023-10-09T05:40:27.928645+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":209,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927521+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:40:27.927521+0000","last_peered":"2023-10-09T05:40:27.927521+0000","last_clean":"2023-10-09T05:40:27.927521+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:40:27.927521+0000","last_undegraded":"2023-10-09T05:40:27.927521+0000","last_fullsized":"2023-10-09T05:40:27.927521+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":167,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927741+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:40:27.927741+0000","last_peered":"2023-10-09T05:40:27.927741+0000","last_clean":"2023-10-09T05:40:27.927741+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:40:27.927741+0000","last_undegraded":"2023-10-09T05:40:27.927741+0000","last_fullsized":"2023-10-09T05:40:27.927741+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":80,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927827+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:40:27.927827+0000","last_peered":"2023-10-09T05:40:27.927827+0000","last_clean":"2023-10-09T05:40:27.927827+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:40:27.927827+0000","last_undegraded":"2023-10-09T05:40:27.927827+0000","last_fullsized":"2023-10-09T05:40:27.927827+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":197,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.923484+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:40:27.923484+0000","last_peered":"2023-10-09T05:40:27.923484+0000","last_clean":"2023-10-09T05:40:27.923484+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:40:27.923484+0000","last_undegraded":"2023-10-09T05:40:27.923484+0000","last_fullsized":"2023-10-09T05:40:27.923484+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":177,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.935623+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:40:27.935623+0000","last_peered":"2023-10-09T05:40:27.935623+0000","last_clean":"2023-10-09T05:40:27.935623+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:40:27.935623+0000","last_undegraded":"2023-10-09T05:40:27.935623+0000","last_fullsized":"2023-10-09T05:40:27.935623+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":191,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.920361+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:40:27.920361+0000","last_peered":"2023-10-09T05:40:27.920361+0000","last_clean":"2023-10-09T05:40:27.920361+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:40:27.920361+0000","last_undegraded":"2023-10-09T05:40:27.920361+0000","last_fullsized":"2023-10-09T05:40:27.920361+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":98,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927617+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:40:27.927617+0000","last_peered":"2023-10-09T05:40:27.927617+0000","last_clean":"2023-10-09T05:40:27.927617+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:40:27.927617+0000","last_undegraded":"2023-10-09T05:40:27.927617+0000","last_fullsized":"2023-10-09T05:40:27.927617+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":177,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.931048+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:40:27.931048+0000","last_peered":"2023-10-09T05:40:27.931048+0000","last_clean":"2023-10-09T05:40:27.931048+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:40:27.931048+0000","last_undegraded":"2023-10-09T05:40:27.931048+0000","last_fullsized":"2023-10-09T05:40:27.931048+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":176,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.916834+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:40:27.916834+0000","last_peered":"2023-10-09T05:40:27.916834+0000","last_clean":"2023-10-09T05:40:27.916834+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:40:27.916834+0000","last_undegraded":"2023-10-09T05:40:27.916834+0000","last_fullsized":"2023-10-09T05:40:27.916834+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":197,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.924429+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:40:27.924429+0000","last_peered":"2023-10-09T05:40:27.924429+0000","last_clean":"2023-10-09T05:40:27.924429+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:40:27.924429+0000","last_undegraded":"2023-10-09T05:40:27.924429+0000","last_fullsized":"2023-10-09T05:40:27.924429+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"201'10035","reported_seq":10082,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:39.865636+0000","last_change":"2023-10-09T05:37:01.636984+0000","last_active":"2023-10-09T05:40:39.865636+0000","last_peered":"2023-10-09T05:40:39.865636+0000","last_clean":"2023-10-09T05:40:39.865636+0000","last_became_active":"2023-10-09T05:37:01.636609+0000","last_became_peered":"2023-10-09T05:37:01.636609+0000","last_unstale":"2023-10-09T05:40:39.865636+0000","last_undegraded":"2023-10-09T05:40:39.865636+0000","last_fullsized":"2023-10-09T05:40:39.865636+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":10035,"log_dups_size":0,"ondisk_log_size":10035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66912256,"num_objects":1021,"num_object_clones":0,"num_object_copies":3063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10035,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":78,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.928103+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:40:27.928103+0000","last_peered":"2023-10-09T05:40:27.928103+0000","last_clean":"2023-10-09T05:40:27.928103+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:40:27.928103+0000","last_undegraded":"2023-10-09T05:40:27.928103+0000","last_fullsized":"2023-10-09T05:40:27.928103+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":176,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.919944+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:40:27.919944+0000","last_peered":"2023-10-09T05:40:27.919944+0000","last_clean":"2023-10-09T05:40:27.919944+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:40:27.919944+0000","last_undegraded":"2023-10-09T05:40:27.919944+0000","last_fullsized":"2023-10-09T05:40:27.919944+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"201'10037","reported_seq":10093,"reported_epoch":201,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:41.355946+0000","last_change":"2023-10-09T05:37:36.837344+0000","last_active":"2023-10-09T05:40:41.355946+0000","last_peered":"2023-10-09T05:40:41.355946+0000","last_clean":"2023-10-09T05:37:34.651883+0000","last_became_active":"2023-10-09T05:37:36.832502+0000","last_became_peered":"2023-10-09T05:37:36.832502+0000","last_unstale":"2023-10-09T05:40:41.355946+0000","last_undegraded":"2023-10-09T05:37:36.702797+0000","last_fullsized":"2023-10-09T05:37:36.678583+0000","mapping_epoch":170,"log_start":"176'3600","ondisk_log_start":"176'3600","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6437,"log_dups_size":0,"ondisk_log_size":6437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:57.419460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62586880,"num_objects":955,"num_object_clones":0,"num_object_copies":2865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10037,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":-1935}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":78,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927608+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:40:27.927608+0000","last_peered":"2023-10-09T05:40:27.927608+0000","last_clean":"2023-10-09T05:40:27.927608+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:40:27.927608+0000","last_undegraded":"2023-10-09T05:40:27.927608+0000","last_fullsized":"2023-10-09T05:40:27.927608+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":182,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927354+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:40:27.927354+0000","last_peered":"2023-10-09T05:40:27.927354+0000","last_clean":"2023-10-09T05:40:27.927354+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:40:27.927354+0000","last_undegraded":"2023-10-09T05:40:27.927354+0000","last_fullsized":"2023-10-09T05:40:27.927354+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"201'10253","reported_seq":16362,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:41.723269+0000","last_change":"2023-10-09T05:38:55.317808+0000","last_active":"2023-10-09T05:40:41.723269+0000","last_peered":"2023-10-09T05:40:41.723269+0000","last_clean":"2023-10-09T05:40:41.723269+0000","last_became_active":"2023-10-09T05:38:55.302739+0000","last_became_peered":"2023-10-09T05:38:55.302739+0000","last_unstale":"2023-10-09T05:40:41.723269+0000","last_undegraded":"2023-10-09T05:40:41.723269+0000","last_fullsized":"2023-10-09T05:40:41.723269+0000","mapping_epoch":186,"log_start":"166'1100","ondisk_log_start":"166'1100","created":162,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9153,"log_dups_size":0,"ondisk_log_size":9153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:25.379818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62849024,"num_objects":959,"num_object_clones":0,"num_object_copies":2877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10253,"num_write_kb":358784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":198049792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":78,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.920414+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:40:27.920414+0000","last_peered":"2023-10-09T05:40:27.920414+0000","last_clean":"2023-10-09T05:40:27.920414+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:40:27.920414+0000","last_undegraded":"2023-10-09T05:40:27.920414+0000","last_fullsized":"2023-10-09T05:40:27.920414+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":200,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.919041+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:40:27.919041+0000","last_peered":"2023-10-09T05:40:27.919041+0000","last_clean":"2023-10-09T05:40:27.919041+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:40:27.919041+0000","last_undegraded":"2023-10-09T05:40:27.919041+0000","last_fullsized":"2023-10-09T05:40:27.919041+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"201'10105","reported_seq":15972,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:43.220106+0000","last_change":"2023-10-09T05:40:04.830679+0000","last_active":"2023-10-09T05:40:43.220106+0000","last_peered":"2023-10-09T05:40:43.220106+0000","last_clean":"2023-10-09T05:40:43.220106+0000","last_became_active":"2023-10-09T05:40:04.807741+0000","last_became_peered":"2023-10-09T05:40:04.807741+0000","last_unstale":"2023-10-09T05:40:43.220106+0000","last_undegraded":"2023-10-09T05:40:43.220106+0000","last_fullsized":"2023-10-09T05:40:43.220106+0000","mapping_epoch":195,"log_start":"166'1200","ondisk_log_start":"166'1200","created":162,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8905,"log_dups_size":0,"ondisk_log_size":8905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:51.289034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56688640,"num_objects":865,"num_object_clones":0,"num_object_copies":2595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10105,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2899,"num_bytes_recovered":139919360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":168,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.920261+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:40:27.920261+0000","last_peered":"2023-10-09T05:40:27.920261+0000","last_clean":"2023-10-09T05:40:27.920261+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:40:27.920261+0000","last_undegraded":"2023-10-09T05:40:27.920261+0000","last_fullsized":"2023-10-09T05:40:27.920261+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":153,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.930649+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:40:27.930649+0000","last_peered":"2023-10-09T05:40:27.930649+0000","last_clean":"2023-10-09T05:40:27.930649+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:40:27.930649+0000","last_undegraded":"2023-10-09T05:40:27.930649+0000","last_fullsized":"2023-10-09T05:40:27.930649+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"201'10023","reported_seq":10070,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:39.935226+0000","last_change":"2023-10-09T05:37:01.439878+0000","last_active":"2023-10-09T05:40:39.935226+0000","last_peered":"2023-10-09T05:40:39.935226+0000","last_clean":"2023-10-09T05:40:39.935226+0000","last_became_active":"2023-10-09T05:37:01.439481+0000","last_became_peered":"2023-10-09T05:37:01.439481+0000","last_unstale":"2023-10-09T05:40:39.935226+0000","last_undegraded":"2023-10-09T05:40:39.935226+0000","last_fullsized":"2023-10-09T05:40:39.935226+0000","mapping_epoch":162,"log_start":"167'2500","ondisk_log_start":"167'2500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7523,"log_dups_size":0,"ondisk_log_size":7523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63111168,"num_objects":963,"num_object_clones":0,"num_object_copies":2889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10023,"num_write_kb":351552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":172,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:31.354251+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:40:31.354251+0000","last_peered":"2023-10-09T05:40:31.354251+0000","last_clean":"2023-10-09T05:40:31.354251+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:40:31.354251+0000","last_undegraded":"2023-10-09T05:40:31.354251+0000","last_fullsized":"2023-10-09T05:40:31.354251+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":167,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927904+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:40:27.927904+0000","last_peered":"2023-10-09T05:40:27.927904+0000","last_clean":"2023-10-09T05:40:27.927904+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:40:27.927904+0000","last_undegraded":"2023-10-09T05:40:27.927904+0000","last_fullsized":"2023-10-09T05:40:27.927904+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"201'10128","reported_seq":10177,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:41.705043+0000","last_change":"2023-10-09T05:37:01.440175+0000","last_active":"2023-10-09T05:40:41.705043+0000","last_peered":"2023-10-09T05:40:41.705043+0000","last_clean":"2023-10-09T05:40:41.705043+0000","last_became_active":"2023-10-09T05:37:01.439846+0000","last_became_peered":"2023-10-09T05:37:01.439846+0000","last_unstale":"2023-10-09T05:40:41.705043+0000","last_undegraded":"2023-10-09T05:40:41.705043+0000","last_fullsized":"2023-10-09T05:40:41.705043+0000","mapping_epoch":162,"log_start":"166'1000","ondisk_log_start":"166'1000","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9128,"log_dups_size":0,"ondisk_log_size":9128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:27.883265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60686336,"num_objects":926,"num_object_clones":0,"num_object_copies":2778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10128,"num_write_kb":353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":168,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.918949+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:40:27.918949+0000","last_peered":"2023-10-09T05:40:27.918949+0000","last_clean":"2023-10-09T05:40:27.918949+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:40:27.918949+0000","last_undegraded":"2023-10-09T05:40:27.918949+0000","last_fullsized":"2023-10-09T05:40:27.918949+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":182,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927519+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:40:27.927519+0000","last_peered":"2023-10-09T05:40:27.927519+0000","last_clean":"2023-10-09T05:40:27.927519+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:40:27.927519+0000","last_undegraded":"2023-10-09T05:40:27.927519+0000","last_fullsized":"2023-10-09T05:40:27.927519+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"201'10136","reported_seq":10185,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:41.359459+0000","last_change":"2023-10-09T05:37:01.638178+0000","last_active":"2023-10-09T05:40:41.359459+0000","last_peered":"2023-10-09T05:40:41.359459+0000","last_clean":"2023-10-09T05:40:41.359459+0000","last_became_active":"2023-10-09T05:37:01.637822+0000","last_became_peered":"2023-10-09T05:37:01.637822+0000","last_unstale":"2023-10-09T05:40:41.359459+0000","last_undegraded":"2023-10-09T05:40:41.359459+0000","last_fullsized":"2023-10-09T05:40:41.359459+0000","mapping_epoch":162,"log_start":"176'3700","ondisk_log_start":"176'3700","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6436,"log_dups_size":0,"ondisk_log_size":6436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:13.240069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64094208,"num_objects":978,"num_object_clones":0,"num_object_copies":2934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10136,"num_write_kb":355648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":192,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927163+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:40:27.927163+0000","last_peered":"2023-10-09T05:40:27.927163+0000","last_clean":"2023-10-09T05:40:27.927163+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:40:27.927163+0000","last_undegraded":"2023-10-09T05:40:27.927163+0000","last_fullsized":"2023-10-09T05:40:27.927163+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":176,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927690+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:40:27.927690+0000","last_peered":"2023-10-09T05:40:27.927690+0000","last_clean":"2023-10-09T05:40:27.927690+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:40:27.927690+0000","last_undegraded":"2023-10-09T05:40:27.927690+0000","last_fullsized":"2023-10-09T05:40:27.927690+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"201'10046","reported_seq":10094,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:39.890508+0000","last_change":"2023-10-09T05:37:01.438917+0000","last_active":"2023-10-09T05:40:39.890508+0000","last_peered":"2023-10-09T05:40:39.890508+0000","last_clean":"2023-10-09T05:40:39.890508+0000","last_became_active":"2023-10-09T05:37:01.438658+0000","last_became_peered":"2023-10-09T05:37:01.438658+0000","last_unstale":"2023-10-09T05:40:39.890508+0000","last_undegraded":"2023-10-09T05:40:39.890508+0000","last_fullsized":"2023-10-09T05:40:39.890508+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":10046,"log_dups_size":0,"ondisk_log_size":10046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66519064,"num_objects":1016,"num_object_clones":0,"num_object_copies":3048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1016,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10046,"num_write_kb":353921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":199,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927462+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:40:27.927462+0000","last_peered":"2023-10-09T05:40:27.927462+0000","last_clean":"2023-10-09T05:40:27.927462+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:40:27.927462+0000","last_undegraded":"2023-10-09T05:40:27.927462+0000","last_fullsized":"2023-10-09T05:40:27.927462+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":148,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.928825+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:40:27.928825+0000","last_peered":"2023-10-09T05:40:27.928825+0000","last_clean":"2023-10-09T05:40:27.928825+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:40:27.928825+0000","last_undegraded":"2023-10-09T05:40:27.928825+0000","last_fullsized":"2023-10-09T05:40:27.928825+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"201'10216","reported_seq":10264,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:43.257031+0000","last_change":"2023-10-09T05:37:01.440093+0000","last_active":"2023-10-09T05:40:43.257031+0000","last_peered":"2023-10-09T05:40:43.257031+0000","last_clean":"2023-10-09T05:40:43.257031+0000","last_became_active":"2023-10-09T05:37:01.439799+0000","last_became_peered":"2023-10-09T05:37:01.439799+0000","last_unstale":"2023-10-09T05:40:43.257031+0000","last_undegraded":"2023-10-09T05:40:43.257031+0000","last_fullsized":"2023-10-09T05:40:43.257031+0000","mapping_epoch":162,"log_start":"166'1300","ondisk_log_start":"166'1300","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8916,"log_dups_size":0,"ondisk_log_size":8916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:28.734471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58327040,"num_objects":890,"num_object_clones":0,"num_object_copies":2670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10216,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":770,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.929008+0000","last_change":"2023-10-09T05:40:04.806908+0000","last_active":"2023-10-09T05:40:27.929008+0000","last_peered":"2023-10-09T05:40:27.929008+0000","last_clean":"2023-10-09T05:40:27.929008+0000","last_became_active":"2023-10-09T05:40:04.806232+0000","last_became_peered":"2023-10-09T05:40:04.806232+0000","last_unstale":"2023-10-09T05:40:27.929008+0000","last_undegraded":"2023-10-09T05:40:27.929008+0000","last_fullsized":"2023-10-09T05:40:27.929008+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:34.087553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":183,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.920363+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:40:27.920363+0000","last_peered":"2023-10-09T05:40:27.920363+0000","last_clean":"2023-10-09T05:40:27.920363+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:40:27.920363+0000","last_undegraded":"2023-10-09T05:40:27.920363+0000","last_fullsized":"2023-10-09T05:40:27.920363+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":133,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.928567+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:40:27.928567+0000","last_peered":"2023-10-09T05:40:27.928567+0000","last_clean":"2023-10-09T05:40:27.928567+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:40:27.928567+0000","last_undegraded":"2023-10-09T05:40:27.928567+0000","last_fullsized":"2023-10-09T05:40:27.928567+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"201'10154","reported_seq":16093,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:41.359317+0000","last_change":"2023-10-09T05:38:10.132330+0000","last_active":"2023-10-09T05:40:41.359317+0000","last_peered":"2023-10-09T05:40:41.359317+0000","last_clean":"2023-10-09T05:40:41.359317+0000","last_became_active":"2023-10-09T05:38:10.127080+0000","last_became_peered":"2023-10-09T05:38:10.127080+0000","last_unstale":"2023-10-09T05:40:41.359317+0000","last_undegraded":"2023-10-09T05:40:41.359317+0000","last_fullsized":"2023-10-09T05:40:41.359317+0000","mapping_epoch":179,"log_start":"176'3700","ondisk_log_start":"176'3700","created":162,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6454,"log_dups_size":0,"ondisk_log_size":6454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:12.659227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65142784,"num_objects":994,"num_object_clones":0,"num_object_copies":2982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10154,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":192413696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":637,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.920107+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:40:27.920107+0000","last_peered":"2023-10-09T05:40:27.920107+0000","last_clean":"2023-10-09T05:40:27.920107+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:40:27.920107+0000","last_undegraded":"2023-10-09T05:40:27.920107+0000","last_fullsized":"2023-10-09T05:40:27.920107+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":168,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927847+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:40:27.927847+0000","last_peered":"2023-10-09T05:40:27.927847+0000","last_clean":"2023-10-09T05:40:27.927847+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:40:27.927847+0000","last_undegraded":"2023-10-09T05:40:27.927847+0000","last_fullsized":"2023-10-09T05:40:27.927847+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":180,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.920338+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:40:27.920338+0000","last_peered":"2023-10-09T05:40:27.920338+0000","last_clean":"2023-10-09T05:40:27.920338+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:40:27.920338+0000","last_undegraded":"2023-10-09T05:40:27.920338+0000","last_fullsized":"2023-10-09T05:40:27.920338+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"201'9979","reported_seq":10037,"reported_epoch":201,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:40.758025+0000","last_change":"2023-10-09T05:37:40.649912+0000","last_active":"2023-10-09T05:40:40.758025+0000","last_peered":"2023-10-09T05:40:40.758025+0000","last_clean":"2023-10-09T05:37:34.677100+0000","last_became_active":"2023-10-09T05:37:40.593033+0000","last_became_peered":"2023-10-09T05:37:40.593033+0000","last_unstale":"2023-10-09T05:40:40.758025+0000","last_undegraded":"2023-10-09T05:37:40.546647+0000","last_fullsized":"2023-10-09T05:37:40.543498+0000","mapping_epoch":170,"log_start":"167'1800","ondisk_log_start":"167'1800","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8179,"log_dups_size":0,"ondisk_log_size":8179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66256896,"num_objects":1011,"num_object_clones":0,"num_object_copies":3033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9979,"num_write_kb":351680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":-1839}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":515,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.933225+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:40:27.933225+0000","last_peered":"2023-10-09T05:40:27.933225+0000","last_clean":"2023-10-09T05:40:27.933225+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:40:27.933225+0000","last_undegraded":"2023-10-09T05:40:27.933225+0000","last_fullsized":"2023-10-09T05:40:27.933225+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":149,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:28.894931+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:40:28.894931+0000","last_peered":"2023-10-09T05:40:28.894931+0000","last_clean":"2023-10-09T05:40:28.894931+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:40:28.894931+0000","last_undegraded":"2023-10-09T05:40:28.894931+0000","last_fullsized":"2023-10-09T05:40:28.894931+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":178,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.928864+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:40:27.928864+0000","last_peered":"2023-10-09T05:40:27.928864+0000","last_clean":"2023-10-09T05:40:27.928864+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:40:27.928864+0000","last_undegraded":"2023-10-09T05:40:27.928864+0000","last_fullsized":"2023-10-09T05:40:27.928864+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"201'10070","reported_seq":10117,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:39.950321+0000","last_change":"2023-10-09T05:37:01.438236+0000","last_active":"2023-10-09T05:40:39.950321+0000","last_peered":"2023-10-09T05:40:39.950321+0000","last_clean":"2023-10-09T05:40:39.950321+0000","last_became_active":"2023-10-09T05:37:01.437917+0000","last_became_peered":"2023-10-09T05:37:01.437917+0000","last_unstale":"2023-10-09T05:40:39.950321+0000","last_undegraded":"2023-10-09T05:40:39.950321+0000","last_fullsized":"2023-10-09T05:40:39.950321+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":10070,"log_dups_size":0,"ondisk_log_size":10070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65798144,"num_objects":1004,"num_object_clones":0,"num_object_copies":3012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10070,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":812,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.920124+0000","last_change":"2023-10-09T05:40:04.805258+0000","last_active":"2023-10-09T05:40:27.920124+0000","last_peered":"2023-10-09T05:40:27.920124+0000","last_clean":"2023-10-09T05:40:27.920124+0000","last_became_active":"2023-10-09T05:40:04.804776+0000","last_became_peered":"2023-10-09T05:40:04.804776+0000","last_unstale":"2023-10-09T05:40:27.920124+0000","last_undegraded":"2023-10-09T05:40:27.920124+0000","last_fullsized":"2023-10-09T05:40:27.920124+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":349,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.930554+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:40:27.930554+0000","last_peered":"2023-10-09T05:40:27.930554+0000","last_clean":"2023-10-09T05:40:27.930554+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:40:27.930554+0000","last_undegraded":"2023-10-09T05:40:27.930554+0000","last_fullsized":"2023-10-09T05:40:27.930554+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":165,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.925862+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:40:27.925862+0000","last_peered":"2023-10-09T05:40:27.925862+0000","last_clean":"2023-10-09T05:40:27.925862+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:40:27.925862+0000","last_undegraded":"2023-10-09T05:40:27.925862+0000","last_fullsized":"2023-10-09T05:40:27.925862+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":174,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.918307+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:40:27.918307+0000","last_peered":"2023-10-09T05:40:27.918307+0000","last_clean":"2023-10-09T05:40:27.918307+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:40:27.918307+0000","last_undegraded":"2023-10-09T05:40:27.918307+0000","last_fullsized":"2023-10-09T05:40:27.918307+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"201'10095","reported_seq":10144,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:43.909746+0000","last_change":"2023-10-09T05:37:01.440172+0000","last_active":"2023-10-09T05:40:43.909746+0000","last_peered":"2023-10-09T05:40:43.909746+0000","last_clean":"2023-10-09T05:40:43.909746+0000","last_became_active":"2023-10-09T05:37:01.439932+0000","last_became_peered":"2023-10-09T05:37:01.439932+0000","last_unstale":"2023-10-09T05:40:43.909746+0000","last_undegraded":"2023-10-09T05:40:43.909746+0000","last_fullsized":"2023-10-09T05:40:43.909746+0000","mapping_epoch":162,"log_start":"176'3700","ondisk_log_start":"176'3700","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6395,"log_dups_size":0,"ondisk_log_size":6395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:57.826704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56688640,"num_objects":865,"num_object_clones":0,"num_object_copies":2595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10095,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":758,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.930018+0000","last_change":"2023-10-09T05:40:05.930112+0000","last_active":"2023-10-09T05:40:27.930018+0000","last_peered":"2023-10-09T05:40:27.930018+0000","last_clean":"2023-10-09T05:40:27.930018+0000","last_became_active":"2023-10-09T05:40:05.929574+0000","last_became_peered":"2023-10-09T05:40:05.929574+0000","last_unstale":"2023-10-09T05:40:27.930018+0000","last_undegraded":"2023-10-09T05:40:27.930018+0000","last_fullsized":"2023-10-09T05:40:27.930018+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:06.982326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":154,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.930599+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:40:27.930599+0000","last_peered":"2023-10-09T05:40:27.930599+0000","last_clean":"2023-10-09T05:40:27.930599+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:40:27.930599+0000","last_undegraded":"2023-10-09T05:40:27.930599+0000","last_fullsized":"2023-10-09T05:40:27.930599+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":168,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.931299+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:40:27.931299+0000","last_peered":"2023-10-09T05:40:27.931299+0000","last_clean":"2023-10-09T05:40:27.931299+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:40:27.931299+0000","last_undegraded":"2023-10-09T05:40:27.931299+0000","last_fullsized":"2023-10-09T05:40:27.931299+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"201'10181","reported_seq":10230,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:43.893134+0000","last_change":"2023-10-09T05:37:01.636967+0000","last_active":"2023-10-09T05:40:43.893134+0000","last_peered":"2023-10-09T05:40:43.893134+0000","last_clean":"2023-10-09T05:40:43.893134+0000","last_became_active":"2023-10-09T05:37:01.636602+0000","last_became_peered":"2023-10-09T05:37:01.636602+0000","last_unstale":"2023-10-09T05:40:43.893134+0000","last_undegraded":"2023-10-09T05:40:43.893134+0000","last_fullsized":"2023-10-09T05:40:43.893134+0000","mapping_epoch":162,"log_start":"176'3700","ondisk_log_start":"176'3700","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6481,"log_dups_size":0,"ondisk_log_size":6481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:11.668143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59703296,"num_objects":911,"num_object_clones":0,"num_object_copies":2733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10181,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":390,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.931182+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:40:27.931182+0000","last_peered":"2023-10-09T05:40:27.931182+0000","last_clean":"2023-10-09T05:40:27.931182+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:40:27.931182+0000","last_undegraded":"2023-10-09T05:40:27.931182+0000","last_fullsized":"2023-10-09T05:40:27.931182+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":148,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.929254+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:40:27.929254+0000","last_peered":"2023-10-09T05:40:27.929254+0000","last_clean":"2023-10-09T05:40:27.929254+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:40:27.929254+0000","last_undegraded":"2023-10-09T05:40:27.929254+0000","last_fullsized":"2023-10-09T05:40:27.929254+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":163,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.930799+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:40:27.930799+0000","last_peered":"2023-10-09T05:40:27.930799+0000","last_clean":"2023-10-09T05:40:27.930799+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:40:27.930799+0000","last_undegraded":"2023-10-09T05:40:27.930799+0000","last_fullsized":"2023-10-09T05:40:27.930799+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"201'10147","reported_seq":10194,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T05:40:39.906733+0000","last_change":"2023-10-09T05:37:01.440074+0000","last_active":"2023-10-09T05:40:39.906733+0000","last_peered":"2023-10-09T05:40:39.906733+0000","last_clean":"2023-10-09T05:40:39.906733+0000","last_became_active":"2023-10-09T05:37:01.439808+0000","last_became_peered":"2023-10-09T05:37:01.439808+0000","last_unstale":"2023-10-09T05:40:39.906733+0000","last_undegraded":"2023-10-09T05:40:39.906733+0000","last_fullsized":"2023-10-09T05:40:39.906733+0000","mapping_epoch":162,"log_start":"167'2600","ondisk_log_start":"167'2600","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7547,"log_dups_size":0,"ondisk_log_size":7547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63635456,"num_objects":971,"num_object_clones":0,"num_object_copies":2913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10147,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":200,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.924357+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:40:27.924357+0000","last_peered":"2023-10-09T05:40:27.924357+0000","last_clean":"2023-10-09T05:40:27.924357+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:40:27.924357+0000","last_undegraded":"2023-10-09T05:40:27.924357+0000","last_fullsized":"2023-10-09T05:40:27.924357+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":589,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.928514+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:40:27.928514+0000","last_peered":"2023-10-09T05:40:27.928514+0000","last_clean":"2023-10-09T05:40:27.928514+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:40:27.928514+0000","last_undegraded":"2023-10-09T05:40:27.928514+0000","last_fullsized":"2023-10-09T05:40:27.928514+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":193,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927701+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:40:27.927701+0000","last_peered":"2023-10-09T05:40:27.927701+0000","last_clean":"2023-10-09T05:40:27.927701+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:40:27.927701+0000","last_undegraded":"2023-10-09T05:40:27.927701+0000","last_fullsized":"2023-10-09T05:40:27.927701+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"201'9845","reported_seq":13960,"reported_epoch":201,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:40.806838+0000","last_change":"2023-10-09T05:40:03.773505+0000","last_active":"2023-10-09T05:40:40.806838+0000","last_peered":"2023-10-09T05:40:40.806838+0000","last_clean":"2023-10-09T05:37:34.656587+0000","last_became_active":"2023-10-09T05:37:36.833510+0000","last_became_peered":"2023-10-09T05:37:36.833510+0000","last_unstale":"2023-10-09T05:40:40.806838+0000","last_undegraded":"2023-10-09T05:37:36.701190+0000","last_fullsized":"2023-10-09T05:37:36.681999+0000","mapping_epoch":170,"log_start":"167'1700","ondisk_log_start":"167'1700","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8145,"log_dups_size":0,"ondisk_log_size":8145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61800448,"num_objects":943,"num_object_clones":0,"num_object_copies":2829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":847,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9845,"num_write_kb":345216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2030,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":96}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":685,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:27.917176+0000","last_change":"2023-10-09T05:37:36.917512+0000","last_active":"2023-10-09T05:40:27.917176+0000","last_peered":"2023-10-09T05:40:27.917176+0000","last_clean":"2023-10-09T05:37:30.688967+0000","last_became_active":"2023-10-09T05:37:36.700218+0000","last_became_peered":"2023-10-09T05:37:36.700218+0000","last_unstale":"2023-10-09T05:40:27.917176+0000","last_undegraded":"2023-10-09T05:37:36.917335+0000","last_fullsized":"2023-10-09T05:37:36.682410+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":156,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.931223+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:40:27.931223+0000","last_peered":"2023-10-09T05:40:27.931223+0000","last_clean":"2023-10-09T05:40:27.931223+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:40:27.931223+0000","last_undegraded":"2023-10-09T05:40:27.931223+0000","last_fullsized":"2023-10-09T05:40:27.931223+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":152,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927243+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:40:27.927243+0000","last_peered":"2023-10-09T05:40:27.927243+0000","last_clean":"2023-10-09T05:40:27.927243+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:40:27.927243+0000","last_undegraded":"2023-10-09T05:40:27.927243+0000","last_fullsized":"2023-10-09T05:40:27.927243+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":165,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.930747+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:40:27.930747+0000","last_peered":"2023-10-09T05:40:27.930747+0000","last_clean":"2023-10-09T05:40:27.930747+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:40:27.930747+0000","last_undegraded":"2023-10-09T05:40:27.930747+0000","last_fullsized":"2023-10-09T05:40:27.930747+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":192,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.933329+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:40:27.933329+0000","last_peered":"2023-10-09T05:40:27.933329+0000","last_clean":"2023-10-09T05:40:27.933329+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:40:27.933329+0000","last_undegraded":"2023-10-09T05:40:27.933329+0000","last_fullsized":"2023-10-09T05:40:27.933329+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":170,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.920208+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:40:27.920208+0000","last_peered":"2023-10-09T05:40:27.920208+0000","last_clean":"2023-10-09T05:40:27.920208+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:40:27.920208+0000","last_undegraded":"2023-10-09T05:40:27.920208+0000","last_fullsized":"2023-10-09T05:40:27.920208+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":182,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927908+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:40:27.927908+0000","last_peered":"2023-10-09T05:40:27.927908+0000","last_clean":"2023-10-09T05:40:27.927908+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:40:27.927908+0000","last_undegraded":"2023-10-09T05:40:27.927908+0000","last_fullsized":"2023-10-09T05:40:27.927908+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":155,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.920271+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:40:27.920271+0000","last_peered":"2023-10-09T05:40:27.920271+0000","last_clean":"2023-10-09T05:40:27.920271+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:40:27.920271+0000","last_undegraded":"2023-10-09T05:40:27.920271+0000","last_fullsized":"2023-10-09T05:40:27.920271+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":182,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927417+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:40:27.927417+0000","last_peered":"2023-10-09T05:40:27.927417+0000","last_clean":"2023-10-09T05:40:27.927417+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:40:27.927417+0000","last_undegraded":"2023-10-09T05:40:27.927417+0000","last_fullsized":"2023-10-09T05:40:27.927417+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":180,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.923999+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:40:27.923999+0000","last_peered":"2023-10-09T05:40:27.923999+0000","last_clean":"2023-10-09T05:40:27.923999+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:40:27.923999+0000","last_undegraded":"2023-10-09T05:40:27.923999+0000","last_fullsized":"2023-10-09T05:40:27.923999+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":192,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.923528+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:40:27.923528+0000","last_peered":"2023-10-09T05:40:27.923528+0000","last_clean":"2023-10-09T05:40:27.923528+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:40:27.923528+0000","last_undegraded":"2023-10-09T05:40:27.923528+0000","last_fullsized":"2023-10-09T05:40:27.923528+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":157,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.931327+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:40:27.931327+0000","last_peered":"2023-10-09T05:40:27.931327+0000","last_clean":"2023-10-09T05:40:27.931327+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:40:27.931327+0000","last_undegraded":"2023-10-09T05:40:27.931327+0000","last_fullsized":"2023-10-09T05:40:27.931327+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":167,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-10-09T05:40:27.927133+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:40:27.927133+0000","last_peered":"2023-10-09T05:40:27.927133+0000","last_clean":"2023-10-09T05:40:27.927133+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:40:27.927133+0000","last_undegraded":"2023-10-09T05:40:27.927133+0000","last_fullsized":"2023-10-09T05:40:27.927133+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1000800280,"num_objects":15272,"num_object_clones":0,"num_object_copies":45816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6587,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15272,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":161450,"num_write_kb":5655041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10888,"num_bytes_recovered":530448384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3678416896,"data_stored":3678404680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129850,"ondisk_log_size":129850,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846665,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626691,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":828020,"kb_used_data":530796,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92917132,"statfs":{"total":95995035648,"available":95147143168,"internally_reserved":0,"allocated":543535104,"data_stored":543013488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:25 2023","interfaces":[{"interface":"back","average":{"1min":14.522,"5min":6.984,"15min":3.121},"min":{"1min":0.24199999999999999,"5min":0.214,"15min":0.214},"max":{"1min":115.377,"5min":115.377,"15min":115.377},"last":5.5430000000000001},{"interface":"front","average":{"1min":14.285,"5min":6.6109999999999998,"15min":2.9870000000000001},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.23999999999999999},"max":{"1min":116.428,"5min":116.428,"15min":116.428},"last":0.44700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:39:46 2023","interfaces":[{"interface":"back","average":{"1min":13.882,"5min":5.0300000000000002,"15min":2.2690000000000001},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.25},"max":{"1min":137.02199999999999,"5min":137.02199999999999,"15min":137.02199999999999},"last":2.6120000000000001},{"interface":"front","average":{"1min":13.885,"5min":5.2430000000000003,"15min":2.355},"min":{"1min":0.38100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":137.06700000000001,"5min":137.06700000000001,"15min":137.06700000000001},"last":2.601}]},{"osd":2,"last update":"Mon Oct 9 05:40:25 2023","interfaces":[{"interface":"back","average":{"1min":8.1809999999999992,"5min":5.5670000000000002,"15min":3.0070000000000001},"min":{"1min":0.443,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":115.40600000000001,"5min":115.40600000000001,"15min":115.40600000000001},"last":13.836},{"interface":"front","average":{"1min":8.7759999999999998,"5min":5.6310000000000002,"15min":3.0070000000000001},"min":{"1min":0.434,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":115.297,"5min":115.297,"15min":115.297},"last":13.865}]},{"osd":3,"last update":"Mon Oct 9 05:39:46 2023","interfaces":[{"interface":"back","average":{"1min":12.359999999999999,"5min":4.7290000000000001,"15min":2.4140000000000001},"min":{"1min":0.307,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":115.244,"5min":115.244,"15min":115.244},"last":1.7290000000000001},{"interface":"front","average":{"1min":12.371,"5min":4.9870000000000001,"15min":2.5099999999999998},"min":{"1min":0.36499999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":115.34999999999999,"5min":115.34999999999999,"15min":115.34999999999999},"last":2.823}]},{"osd":4,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":5.181,"5min":2.177,"15min":1.7150000000000001},"min":{"1min":0.249,"5min":0.245,"15min":0.23400000000000001},"max":{"1min":115.214,"5min":115.214,"15min":125.64},"last":0.317},{"interface":"front","average":{"1min":5.282,"5min":1.766,"15min":1.597},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":115.22799999999999,"5min":115.22799999999999,"15min":125.625},"last":0.51100000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":7.7560000000000002,"5min":2.8050000000000002,"15min":1.819},"min":{"1min":0.31,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":115.316,"5min":115.316,"15min":127.35899999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":7.601,"5min":2.339,"15min":1.704},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29199999999999998},"max":{"1min":115.30500000000001,"5min":115.30500000000001,"15min":128.93799999999999},"last":0.47599999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:39:47 2023","interfaces":[{"interface":"back","average":{"1min":5.5990000000000002,"5min":2.5710000000000002,"15min":2.9550000000000001},"min":{"1min":0.35899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":115.514,"5min":115.514,"15min":115.514},"last":0.49399999999999999},{"interface":"front","average":{"1min":5.6500000000000004,"5min":2.0600000000000001,"15min":2.081},"min":{"1min":0.35499999999999998,"5min":0.317,"15min":0.317},"max":{"1min":116.53100000000001,"5min":116.53100000000001,"15min":116.53100000000001},"last":0.64300000000000002}]}]},{"osd":1,"up_from":93,"seq":399431958653,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":598624,"kb_used_data":336728,"kb_used_omap":7,"kb_used_meta":261880,"kb_avail":93146528,"statfs":{"total":95995035648,"available":95382044672,"internally_reserved":0,"allocated":344809472,"data_stored":344073862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":268165646},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,5,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9460000000000002,"5min":5.7850000000000001,"15min":2.3220000000000001},"min":{"1min":0.20300000000000001,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":13.214,"5min":207.858,"15min":207.858},"last":3.0070000000000001},{"interface":"front","average":{"1min":2.4529999999999998,"5min":6.7469999999999999,"15min":2.6859999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.849,"5min":206.97200000000001,"15min":206.97200000000001},"last":3.0259999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:40:11 2023","interfaces":[{"interface":"back","average":{"1min":4.6580000000000004,"5min":6.79,"15min":3.1099999999999999},"min":{"1min":0.22,"5min":0.184,"15min":0.184},"max":{"1min":31.029,"5min":101.367,"15min":101.367},"last":2.1880000000000002},{"interface":"front","average":{"1min":5.5090000000000003,"5min":6.6219999999999999,"15min":2.9649999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":38.622999999999998,"5min":97.286000000000001,"15min":98.409999999999997},"last":0.28599999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":6.1749999999999998,"5min":3.8559999999999999,"15min":1.744},"min":{"1min":0.35399999999999998,"5min":0.223,"15min":0.223},"max":{"1min":42.445999999999998,"5min":83.885000000000005,"15min":83.885000000000005},"last":2.222},{"interface":"front","average":{"1min":5.4379999999999997,"5min":4.7290000000000001,"15min":1.954},"min":{"1min":0.373,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":42.243000000000002,"5min":84.641999999999996,"15min":84.641999999999996},"last":2.8900000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":4.1799999999999997,"15min":1.9670000000000001},"min":{"1min":0.35599999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.813000000000001,"5min":141.32900000000001,"15min":141.32900000000001},"last":1.915},{"interface":"front","average":{"1min":2.4129999999999998,"5min":2.9420000000000002,"15min":1.532},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.4509999999999996,"5min":141.15899999999999,"15min":141.15899999999999},"last":0.34399999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":3.9020000000000001,"15min":1.901},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.6769999999999996,"5min":235.18700000000001,"15min":235.18700000000001},"last":2.2450000000000001},{"interface":"front","average":{"1min":2.7330000000000001,"5min":4.8550000000000004,"15min":2.319},"min":{"1min":0.38300000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":10.779,"5min":235.79300000000001,"15min":235.79300000000001},"last":1.9399999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":7.46,"5min":2.9140000000000001,"15min":1.5289999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":103.60899999999999,"5min":103.60899999999999,"15min":103.60899999999999},"last":2.27},{"interface":"front","average":{"1min":7.3520000000000003,"5min":3.8500000000000001,"15min":1.837},"min":{"1min":0.38,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":103.679,"5min":103.679,"15min":103.679},"last":1.9590000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7719999999999998,"5min":2.9489999999999998,"15min":2.3149999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.7390000000000008,"5min":126.68300000000001,"15min":126.68300000000001},"last":0.501},{"interface":"front","average":{"1min":2.927,"5min":3.0800000000000001,"15min":2.4119999999999999},"min":{"1min":0.46600000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.7720000000000002,"5min":126.738,"15min":126.738},"last":0.54000000000000004}]}]},{"osd":0,"up_from":88,"seq":377957122179,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":578436,"kb_used_data":236348,"kb_used_omap":8,"kb_used_meta":342071,"kb_avail":93166716,"statfs":{"total":95995035648,"available":95402717184,"internally_reserved":0,"allocated":242020352,"data_stored":241250807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":350281480},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":1.9379999999999999,"15min":1.0760000000000001},"min":{"1min":0.219,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":12.005000000000001,"5min":14.964,"15min":14.964},"last":0.52100000000000002},{"interface":"front","average":{"1min":3.4350000000000001,"5min":2.5179999999999998,"15min":1.2809999999999999},"min":{"1min":0.22900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.218999999999999,"5min":15.218999999999999,"15min":15.218999999999999},"last":3.7160000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":5.0410000000000004,"5min":4.3979999999999997,"15min":2.2000000000000002},"min":{"1min":0.26800000000000002,"5min":0.223,"15min":0.214},"max":{"1min":21.648,"5min":79.911000000000001,"15min":79.911000000000001},"last":3.5670000000000002},{"interface":"front","average":{"1min":5.4370000000000003,"5min":4.2549999999999999,"15min":2.1040000000000001},"min":{"1min":0.26000000000000001,"5min":0.221,"15min":0.218},"max":{"1min":32.344000000000001,"5min":80.105000000000004,"15min":80.105000000000004},"last":0.60699999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":4.0839999999999996,"5min":3.2730000000000001,"15min":1.5609999999999999},"min":{"1min":0.27000000000000002,"5min":0.23999999999999999,"15min":0.22800000000000001},"max":{"1min":10.519,"5min":31.594000000000001,"15min":31.594000000000001},"last":11.304},{"interface":"front","average":{"1min":4.3410000000000002,"5min":3.3730000000000002,"15min":1.651},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.222},"max":{"1min":16.937999999999999,"5min":31.641999999999999,"15min":31.641999999999999},"last":2.2069999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.8420000000000001,"15min":1.2050000000000001},"min":{"1min":0.39500000000000002,"5min":0.27500000000000002,"15min":0.27200000000000002},"max":{"1min":5.157,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.9119999999999999,"5min":2.5409999999999999,"15min":1.5149999999999999},"min":{"1min":0.34300000000000003,"5min":0.33000000000000002,"15min":0.30399999999999999},"max":{"1min":9.5190000000000001,"5min":69.102000000000004,"15min":69.102000000000004},"last":3.7490000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":18.456,"5min":6.2709999999999999,"15min":2.7360000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":281.733,"5min":281.733,"15min":281.733},"last":3.7749999999999999},{"interface":"front","average":{"1min":19.111999999999998,"5min":6.2210000000000001,"15min":2.7370000000000001},"min":{"1min":0.40899999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":282.95400000000001,"5min":282.95400000000001,"15min":282.95400000000001},"last":1.0049999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5529999999999999,"5min":2.0110000000000001,"15min":1.3280000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":3.8279999999999998},{"interface":"front","average":{"1min":3.5190000000000001,"5min":2.1779999999999999,"15min":1.387},"min":{"1min":0.46200000000000002,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":10.83,"5min":10.83,"15min":10.83},"last":1.036}]},{"osd":7,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":1.544,"15min":1.3999999999999999},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.9740000000000002,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":3.2999999999999998,"5min":2.3879999999999999,"15min":2.3889999999999998},"min":{"1min":0.5,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.4100000000000001,"5min":11.068,"15min":11.068},"last":3.8010000000000002}]}]},{"osd":2,"up_from":98,"seq":420906795126,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":960896,"kb_used_data":561144,"kb_used_omap":11,"kb_used_meta":399476,"kb_avail":92784256,"statfs":{"total":95995035648,"available":95011078144,"internally_reserved":0,"allocated":574611456,"data_stored":573790163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":409064021},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,3,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":4.3079999999999998,"5min":3.0630000000000002,"15min":1.53},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17199999999999999},"max":{"1min":20.771000000000001,"5min":20.771000000000001,"15min":32.122},"last":5.7599999999999998},{"interface":"front","average":{"1min":6.133,"5min":3.0640000000000001,"15min":1.5229999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":21.997,"5min":21.997,"15min":34.066000000000003},"last":0.22700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":4.8250000000000002,"15min":2.3109999999999999},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.20699999999999999},"max":{"1min":9.3469999999999995,"5min":98.575000000000003,"15min":98.575000000000003},"last":7.0060000000000002},{"interface":"front","average":{"1min":2.431,"5min":3.9820000000000002,"15min":2.0030000000000001},"min":{"1min":0.23899999999999999,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":9.3260000000000005,"5min":90.135000000000005,"15min":98.397000000000006},"last":18.882999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":15.612,"5min":5.4580000000000002,"15min":2.2930000000000001},"min":{"1min":0.55200000000000005,"5min":0.249,"15min":0.24199999999999999},"max":{"1min":150.65899999999999,"5min":150.65899999999999,"15min":150.65899999999999},"last":24.471},{"interface":"front","average":{"1min":17.797000000000001,"5min":6.25,"15min":2.601},"min":{"1min":0.54900000000000004,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":150.851,"5min":150.851,"15min":150.851},"last":8.5109999999999992}]},{"osd":4,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2629999999999999,"5min":3.0019999999999998,"15min":1.5149999999999999},"min":{"1min":0.29799999999999999,"5min":0.28999999999999998,"15min":0.27700000000000002},"max":{"1min":9.3000000000000007,"5min":115.84999999999999,"15min":115.84999999999999},"last":14.382},{"interface":"front","average":{"1min":3.2669999999999999,"5min":3.887,"15min":1.8500000000000001},"min":{"1min":0.48399999999999999,"5min":0.312,"15min":0.29799999999999999},"max":{"1min":10.818,"5min":116.149,"15min":116.149},"last":0.34000000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":2.964,"5min":2.7690000000000001,"15min":1.524},"min":{"1min":0.44,"5min":0.312,"15min":0.312},"max":{"1min":8.7509999999999994,"5min":24.597999999999999,"15min":24.597999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.3039999999999998,"5min":2.282,"15min":1.292},"min":{"1min":0.32700000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.2590000000000003,"5min":23.919,"15min":23.919},"last":14.426}]},{"osd":6,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":3.2869999999999999,"5min":2.7719999999999998,"15min":1.5049999999999999},"min":{"1min":0.41199999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.864000000000001,"5min":20.503,"15min":20.503},"last":1.1200000000000001},{"interface":"front","average":{"1min":2.9940000000000002,"5min":2.4489999999999998,"15min":1.401},"min":{"1min":0.46800000000000003,"5min":0.39600000000000002,"15min":0.34499999999999997},"max":{"1min":8.7949999999999999,"5min":9.093,"15min":9.4049999999999994},"last":0.48699999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:39:52 2023","interfaces":[{"interface":"back","average":{"1min":3.294,"5min":3.0369999999999999,"15min":2.6949999999999998},"min":{"1min":0.436,"5min":0.374,"15min":0.374},"max":{"1min":10.897,"5min":24.030999999999999,"15min":24.030999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":3.0299999999999998,"5min":2.7549999999999999,"15min":2.0710000000000002},"min":{"1min":0.48899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":8.8230000000000004,"5min":24.178000000000001,"15min":24.178000000000001},"last":1.417}]}]},{"osd":3,"up_from":103,"seq":442381631600,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":994528,"kb_used_data":688088,"kb_used_omap":4,"kb_used_meta":306299,"kb_avail":92750624,"statfs":{"total":95995035648,"available":94976638976,"internally_reserved":0,"allocated":704602112,"data_stored":703865927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":313650782},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":14.382,"5min":4.5739999999999998,"15min":2.129},"min":{"1min":0.19800000000000001,"5min":0.183,"15min":0.183},"max":{"1min":167.82400000000001,"5min":167.82400000000001,"15min":167.82400000000001},"last":5.1020000000000003},{"interface":"front","average":{"1min":15.930999999999999,"5min":5.0309999999999997,"15min":2.3330000000000002},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.214},"max":{"1min":167.89599999999999,"5min":167.89599999999999,"15min":167.89599999999999},"last":3.0760000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":3.218,"5min":3.8919999999999999,"15min":1.657},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.663,"5min":83.921999999999997,"15min":83.921999999999997},"last":13.884},{"interface":"front","average":{"1min":3.956,"5min":3.911,"15min":1.651},"min":{"1min":0.25,"5min":0.25,"15min":0.247},"max":{"1min":12.061999999999999,"5min":83.707999999999998,"15min":83.707999999999998},"last":2.7109999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":16.274000000000001,"5min":6.6950000000000003,"15min":3.298},"min":{"1min":0.26500000000000001,"5min":0.215,"15min":0.215},"max":{"1min":190.43299999999999,"5min":190.43299999999999,"15min":190.43299999999999},"last":7.532},{"interface":"front","average":{"1min":21.088000000000001,"5min":7.8490000000000002,"15min":3.6960000000000002},"min":{"1min":0.317,"5min":0.28999999999999998,"15min":0.26000000000000001},"max":{"1min":190.267,"5min":190.267,"15min":190.267},"last":3.0409999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":6.7539999999999996,"5min":2.9540000000000002,"15min":1.7390000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.29699999999999999},"max":{"1min":51.591999999999999,"5min":51.591999999999999,"15min":51.591999999999999},"last":7.9870000000000001},{"interface":"front","average":{"1min":2.9209999999999998,"5min":1.9790000000000001,"15min":1.3620000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":11.271000000000001,"5min":13.734,"15min":45.735999999999997},"last":5.1710000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":2.0379999999999998,"15min":1.325},"min":{"1min":0.45100000000000001,"5min":0.33500000000000002,"15min":0.29999999999999999},"max":{"1min":8.3640000000000008,"5min":8.4770000000000003,"15min":47.948999999999998},"last":0.65900000000000003},{"interface":"front","average":{"1min":3.1640000000000001,"5min":2.1659999999999999,"15min":1.4299999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.339,"5min":13.864000000000001,"15min":47.968000000000004},"last":5.2110000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.2120000000000002,"15min":1.321},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.313000000000001,"5min":13.831,"15min":13.831},"last":5.2409999999999997},{"interface":"front","average":{"1min":2.8260000000000001,"5min":2.1019999999999999,"15min":1.1299999999999999},"min":{"1min":0.45700000000000002,"5min":0.373,"15min":0.33900000000000002},"max":{"1min":8.3979999999999997,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":1.8600000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:39:53 2023","interfaces":[{"interface":"back","average":{"1min":3.9590000000000001,"5min":2.6699999999999999,"15min":2.3959999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":20.065000000000001,"5min":20.065000000000001,"15min":20.065000000000001},"last":3.1379999999999999},{"interface":"front","average":{"1min":4.0590000000000002,"5min":2.6629999999999998,"15min":2.1960000000000002},"min":{"1min":0.40899999999999997,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":20.138000000000002,"5min":20.138000000000002,"15min":20.138000000000002},"last":3.1059999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019147,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":631988,"kb_used_data":330668,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93113164,"statfs":{"total":95995035648,"available":95347879936,"internally_reserved":0,"allocated":338604032,"data_stored":338082160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:22 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":1.929,"15min":1.3180000000000001},"min":{"1min":0.248,"5min":0.214,"15min":0.214},"max":{"1min":6.2590000000000003,"5min":36.406999999999996,"15min":36.406999999999996},"last":0.34200000000000003},{"interface":"front","average":{"1min":2.4460000000000002,"5min":2.556,"15min":1.5800000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.7629999999999999,"5min":36.454999999999998,"15min":36.454999999999998},"last":5.6150000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":1.7829999999999999,"15min":1.1299999999999999},"min":{"1min":0.28599999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.194,"5min":10.073,"15min":31.073},"last":5.5419999999999998},{"interface":"front","average":{"1min":2.6709999999999998,"5min":1.7869999999999999,"15min":1.2010000000000001},"min":{"1min":0.33500000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.5279999999999996,"5min":10.055,"15min":31.100000000000001},"last":5.5350000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":6.4269999999999996,"15min":3.476},"min":{"1min":0.31,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":10.099,"5min":196.93700000000001,"15min":196.93700000000001},"last":0.998},{"interface":"front","average":{"1min":3.048,"5min":6.1829999999999998,"15min":3.3679999999999999},"min":{"1min":0.34399999999999997,"5min":0.28299999999999997,"15min":0.28100000000000003},"max":{"1min":11.534000000000001,"5min":197.08099999999999,"15min":197.08099999999999},"last":0.78200000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":5.6639999999999997,"5min":6.8849999999999998,"15min":2.8439999999999999},"min":{"1min":0.35399999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":34.615000000000002,"5min":211.48699999999999,"15min":211.48699999999999},"last":5.3780000000000001},{"interface":"front","average":{"1min":5.8650000000000002,"5min":6.8010000000000002,"15min":2.8010000000000002},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.309},"max":{"1min":34.57,"5min":211.31100000000001,"15min":211.31100000000001},"last":1.919}]},{"osd":5,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":3.4489999999999998,"15min":1.5429999999999999},"min":{"1min":0.26300000000000001,"5min":0.25,"15min":0.249},"max":{"1min":3.5259999999999998,"5min":113.35899999999999,"15min":113.35899999999999},"last":0.312},{"interface":"front","average":{"1min":0.61399999999999999,"5min":3.302,"15min":1.52},"min":{"1min":0.35499999999999998,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":3.2160000000000002,"5min":113.32299999999999,"15min":113.32299999999999},"last":0.42099999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:39:57 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.999,"15min":0.70899999999999996},"min":{"1min":0.317,"5min":0.28999999999999998,"15min":0.28100000000000003},"max":{"1min":3.4390000000000001,"5min":11.944000000000001,"15min":11.944000000000001},"last":0.379},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.3300000000000001,"15min":0.85899999999999999},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.6500000000000004,"5min":10.356,"15min":10.356},"last":1.226}]},{"osd":7,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":1.4810000000000001,"15min":2.7400000000000002},"min":{"1min":0.37,"5min":0.312,"15min":0.312},"max":{"1min":4.5819999999999999,"5min":30.393999999999998,"15min":30.393999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.601,"15min":3.0870000000000002},"min":{"1min":0.36499999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5999999999999996,"5min":30.334,"15min":30.334},"last":0.44500000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822919,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":645416,"kb_used_data":335904,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93099736,"statfs":{"total":95995035648,"available":95334129664,"internally_reserved":0,"allocated":343965696,"data_stored":343451584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":4.4219999999999997,"15min":2.9369999999999998},"min":{"1min":0.22800000000000001,"5min":0.22700000000000001,"15min":0.19400000000000001},"max":{"1min":8.3079999999999998,"5min":165.90199999999999,"15min":165.90199999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.377,"5min":4.2290000000000001,"15min":2.9910000000000001},"min":{"1min":0.28399999999999997,"5min":0.215,"15min":0.215},"max":{"1min":5.4660000000000002,"5min":165.96700000000001,"15min":165.96700000000001},"last":0.39500000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":5.3810000000000002,"5min":3.7469999999999999,"15min":1.738},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":58.176000000000002,"5min":87.769000000000005,"15min":87.769000000000005},"last":6.9059999999999997},{"interface":"front","average":{"1min":7.1660000000000004,"5min":3.8940000000000001,"15min":1.7769999999999999},"min":{"1min":0.32200000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":54.591000000000001,"5min":87.864000000000004,"15min":87.864000000000004},"last":3.7839999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:40:20 2023","interfaces":[{"interface":"back","average":{"1min":7.2709999999999999,"5min":6.7069999999999999,"15min":4.008},"min":{"1min":0.33600000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":73.311999999999998,"5min":118.255,"15min":290.697},"last":0.26700000000000002},{"interface":"front","average":{"1min":6.1639999999999997,"5min":6.0199999999999996,"15min":3.7810000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":73.280000000000001,"5min":118.438,"15min":290.786},"last":6.8780000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":16.713999999999999,"5min":5.8200000000000003,"15min":2.4249999999999998},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":95.236999999999995,"5min":95.236999999999995,"15min":95.236999999999995},"last":6.8310000000000004},{"interface":"front","average":{"1min":16.225000000000001,"5min":5.3680000000000003,"15min":2.3100000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":93.471000000000004,"5min":93.471000000000004,"15min":93.471000000000004},"last":0.58499999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:39:41 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.94399999999999995,"15min":0.68100000000000005},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":0.91100000000000003,"5min":9.2729999999999997,"15min":9.3510000000000009},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.4990000000000001,"15min":0.92900000000000005},"min":{"1min":0.32700000000000001,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":1.4259999999999999,"5min":14.236000000000001,"15min":14.236000000000001},"last":0.72299999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:40:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1139999999999999,"5min":1.8120000000000001,"15min":1.3420000000000001},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.26000000000000001},"max":{"1min":27.146000000000001,"5min":27.146000000000001,"15min":95.010999999999996},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.1269999999999998,"5min":1.4690000000000001,"15min":1.2290000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27700000000000002},"max":{"1min":27.215,"5min":27.215,"15min":94.194000000000003},"last":1.157}]},{"osd":7,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.27,"15min":1.573},"min":{"1min":0.34399999999999997,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.94099999999999995,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":1.238},{"interface":"front","average":{"1min":0.52100000000000002,"5min":1.3400000000000001,"15min":1.8620000000000001},"min":{"1min":0.35899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.5029999999999999,"5min":14.432,"15min":14.432},"last":1.3220000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":233181184,"data_stored":233177112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":341053440,"data_stored":341049368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":551419904,"data_stored":551419904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":689700864,"data_stored":689700864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":325255168,"data_stored":325255168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":331354112,"data_stored":331350040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":520552448,"data_stored":520552448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":685899776,"data_stored":685899776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:40:46.622 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:40:46.684 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:46.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:46.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:46.789 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:46.816 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:46.863 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:46.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:46.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:46.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:47.007 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:47.061 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:47.068 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:46 smithi191 ceph-mon[161669]: pgmap v571: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 885 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 6303/42348 objects degraded (14.884%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:47.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:47.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:47.171 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:47.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:46 smithi008 ceph-mon[184418]: pgmap v571: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 885 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 6303/42348 objects degraded (14.884%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:46 smithi008 ceph-mon[191227]: pgmap v571: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 885 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 6303/42348 objects degraded (14.884%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:47.300 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:47.741 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:47.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:47.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:47.848 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:47.888 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:47.947 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:47.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:47.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:47.953 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:47 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 6303/42348 objects degraded (14.884%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:48.053 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:48.166 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:48.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:47 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 6303/42348 objects degraded (14.884%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:48.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:47 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 6303/42348 objects degraded (14.884%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:48.682 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:48.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:48.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:48.791 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:48.795 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:48.822 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:48.870 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:48.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:48.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:48.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:49.012 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:49.025 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:40:49.064 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:49.070 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:48 smithi191 ceph-mon[161669]: pgmap v572: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 830 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 6303/39696 objects degraded (15.878%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:49.070 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:48 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4256193860' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:49.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:49.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:49.173 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:40:49.278 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:48 smithi008 ceph-mon[184418]: pgmap v572: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 830 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 6303/39696 objects degraded (15.878%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:49.279 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:48 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4256193860' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:49.279 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:48 smithi008 ceph-mon[191227]: pgmap v572: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 830 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 6303/39696 objects degraded (15.878%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:49.279 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:48 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4256193860' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:40:49.289 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:49.530 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:40:49.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:49.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:40:49.636 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:49.746 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:49.846 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:50.105 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:50.105 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[184418]: failed to encode map e202 with expected crc 2023-10-09T05:40:50.105 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[184418]: failed to encode map e202 with expected crc 2023-10-09T05:40:50.105 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[184418]: failed to encode map e202 with expected crc 2023-10-09T05:40:50.105 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4256193860' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:50.105 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[184418]: osdmap e202: 8 total, 8 up, 7 in 2023-10-09T05:40:50.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:50.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[191227]: failed to encode map e202 with expected crc 2023-10-09T05:40:50.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[191227]: failed to encode map e202 with expected crc 2023-10-09T05:40:50.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[191227]: failed to encode map e202 with expected crc 2023-10-09T05:40:50.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4256193860' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:50.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:49 smithi008 ceph-mon[191227]: osdmap e202: 8 total, 8 up, 7 in 2023-10-09T05:40:50.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:49 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:40:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:49 smithi191 ceph-mon[161669]: failed to encode map e202 with expected crc 2023-10-09T05:40:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:49 smithi191 ceph-mon[161669]: failed to encode map e202 with expected crc 2023-10-09T05:40:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:49 smithi191 ceph-mon[161669]: failed to encode map e202 with expected crc 2023-10-09T05:40:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4256193860' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:40:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:49 smithi191 ceph-mon[161669]: osdmap e202: 8 total, 8 up, 7 in 2023-10-09T05:40:50.305 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:40:50.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:50.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:50.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:50.447 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:50.498 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:40:50.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:50.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:50.605 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:50.652 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:40:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:40:50] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.33.4" 2023-10-09T05:40:50.655 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:51.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:51 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:51.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:51 smithi008 ceph-mon[191227]: pgmap v574: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 732 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 6303/35001 objects degraded (18.008%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:51.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:51 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:51.195 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:51 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:51.195 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:51 smithi008 ceph-mon[184418]: pgmap v574: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 732 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 6303/35001 objects degraded (18.008%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:51.195 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:51 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:51.209 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:51.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:51.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:51.319 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:40:51.349 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:51.399 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:40:51.405 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:51 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:51.405 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:51 smithi191 ceph-mon[161669]: pgmap v574: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 732 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 6303/35001 objects degraded (18.008%); 0 B/s, 54 objects/s recovering 2023-10-09T05:40:51.405 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:51 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:51.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:51.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:40:51.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:40:51.534 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:51.585 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:40:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:51.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:40:51.626 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:40:51.626 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:40:51.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:40:51.627 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:40:51.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.627 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54034 5.6 5:6bcc9ee5:::benchmark_data_smithi008_238708_object54033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:40:51.627 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:40:51.627 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 451.104251783, 2023-10-09T05:40:51.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0234612670000001, 2023-10-09T05:40:51.627 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.627 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:40:51.627 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.628 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.628 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.628 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54034 2023-10-09T05:40:51.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.628 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:40:51.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517426+0000", 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517446+0000", 2023-10-09T05:40:51.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0636999999999999e-05 2023-10-09T05:40:51.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517453+0000", 2023-10-09T05:40:51.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5390000000000004e-06 2023-10-09T05:40:51.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.517458+0000", 2023-10-09T05:40:51.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0839999999999999e-06 2023-10-09T05:40:51.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.521341+0000", 2023-10-09T05:40:51.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038830990000000001 2023-10-09T05:40:51.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.521502+0000", 2023-10-09T05:40:51.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016102400000000001 2023-10-09T05:40:51.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:40:51.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.521674+0000", 2023-10-09T05:40:51.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017155600000000001 2023-10-09T05:40:51.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.524020+0000", 2023-10-09T05:40:51.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002346295 2023-10-09T05:40:51.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.529910+0000", 2023-10-09T05:40:51.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058903289999999997 2023-10-09T05:40:51.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541133+0000", 2023-10-09T05:40:51.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011222492000000001 2023-10-09T05:40:51.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541145+0000", 2023-10-09T05:40:51.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1871999999999999e-05 2023-10-09T05:40:51.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540792+0000", 2023-10-09T05:40:51.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99964734499999996 2023-10-09T05:40:51.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540887+0000", 2023-10-09T05:40:51.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5445999999999994e-05 2023-10-09T05:40:51.635 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.635 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.635 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.635 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54039 5.6 5:6a5b7d04:::benchmark_data_smithi008_238708_object54038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:40:51.635 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.527531+0000", 2023-10-09T05:40:51.635 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 451.094147271, 2023-10-09T05:40:51.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.017534768, 2023-10-09T05:40:51.636 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.636 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:40:51.636 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.636 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.636 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.636 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54039 2023-10-09T05:40:51.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.636 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527531+0000", 2023-10-09T05:40:51.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527531+0000", 2023-10-09T05:40:51.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527529+0000", 2023-10-09T05:40:51.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:40:51.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527558+0000", 2023-10-09T05:40:51.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9082e-05 2023-10-09T05:40:51.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527573+0000", 2023-10-09T05:40:51.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5038000000000001e-05 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527583+0000", 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3249999999999997e-06 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527639+0000", 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6297e-05 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527793+0000", 2023-10-09T05:40:51.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015416599999999999 2023-10-09T05:40:51.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:40:51.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.527952+0000", 2023-10-09T05:40:51.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015940799999999999 2023-10-09T05:40:51.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.530713+0000", 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002760878 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541193+0000", 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010480041000000001 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.541203+0000", 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1980000000000007e-06 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.544999+0000", 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.00379627 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545065+0000", 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6607999999999997e-05 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.642 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54041 5.4 5:2173e5b4:::benchmark_data_smithi008_238708_object54040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e112)", 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.535206+0000", 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 451.08647170900002, 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.006119118, 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.643 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54041 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535206+0000", 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535206+0000", 2023-10-09T05:40:51.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535205+0000", 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535227+0000", 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2062999999999999e-05 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535240+0000", 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2991e-05 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535248+0000", 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7789999999999996e-06 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535783+0000", 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000534484 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535798+0000", 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5403000000000001e-05 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540201+0000", 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0044033379999999 2023-10-09T05:40:51.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.541325+0000", 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001123906 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54046 5.4 5:2050c775:::benchmark_data_smithi008_238708_object54045:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e112)", 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:40:51.648 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 451.08641856100002, 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0145929970000001, 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54046 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:40:51.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:40:51.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535259+0000", 2023-10-09T05:40:51.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535285+0000", 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6137999999999999e-05 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535294+0000", 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9299999999999992e-06 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535302+0000", 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0250000000000002e-06 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535822+0000", 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052059000000000003 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.535829+0000", 2023-10-09T05:40:51.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5420000000000002e-06 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.541576+0000", 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.005747071 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.541934+0000", 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00035752700000000001 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.542079+0000", 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014546200000000001 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549695+0000", 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0076156389999999996 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549765+0000", 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0362000000000002e-05 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549796+0000", 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1239999999999999e-05 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549852+0000", 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6103000000000001e-05 2023-10-09T05:40:51.655 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.656 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.656 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.656 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54078 5.4 5:286f5610:::benchmark_data_smithi008_238708_object54077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:40:51.656 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:40:51.656 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 451.04034779900002, 2023-10-09T05:40:51.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96860710699999997, 2023-10-09T05:40:51.656 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.656 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.656 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54078 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:40:51.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581330+0000", 2023-10-09T05:40:51.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:51.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581351+0000", 2023-10-09T05:40:51.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1622000000000001e-05 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581359+0000", 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5129999999999999e-06 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581366+0000", 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8000000000000001e-06 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581421+0000", 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5843e-05 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.581430+0000", 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7530000000000007e-06 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543690+0000", 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96226007599999996 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543809+0000", 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000118658 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543945+0000", 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013617699999999999 2023-10-09T05:40:51.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549727+0000", 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0057819430000000003 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549865+0000", 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013792899999999999 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549891+0000", 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6105000000000002e-05 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549937+0000", 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6213e-05 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54079 5.4 5:29366199:::benchmark_data_smithi008_238708_object54078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.582855+0000", 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 451.03882291299999, 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96716001200000001, 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54079 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.664 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582855+0000", 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582855+0000", 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582854+0000", 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582876+0000", 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1637999999999998e-05 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582891+0000", 2023-10-09T05:40:51.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5489999999999999e-05 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582899+0000", 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5950000000000001e-06 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582953+0000", 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4355999999999999e-05 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.582962+0000", 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9530000000000005e-06 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545663+0000", 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96270056100000001 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545753+0000", 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0152e-05 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.545860+0000", 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000107358 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549748+0000", 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038876010000000001 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549948+0000", 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020022200000000001 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.549973+0000", 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5219e-05 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550015+0000", 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1847000000000001e-05 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54081 5.4 5:25d3ef94:::benchmark_data_smithi008_238708_object54080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 451.038313593, 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96692271200000002, 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.671 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54081 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583364+0000", 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583385+0000", 2023-10-09T05:40:51.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1100000000000001e-05 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583399+0000", 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4654000000000001e-05 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583406+0000", 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1250000000000004e-06 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583456+0000", 2023-10-09T05:40:51.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9496000000000002e-05 2023-10-09T05:40:51.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.583464+0000", 2023-10-09T05:40:51.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9300000000000003e-06 2023-10-09T05:40:51.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547622+0000", 2023-10-09T05:40:51.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96415766000000003 2023-10-09T05:40:51.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547731+0000", 2023-10-09T05:40:51.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000109104 2023-10-09T05:40:51.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:40:51.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547847+0000", 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000115983 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550186+0000", 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023388979999999998 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550198+0000", 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2286000000000001e-05 2023-10-09T05:40:51.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550244+0000", 2023-10-09T05:40:51.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6045999999999999e-05 2023-10-09T05:40:51.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.550287+0000", 2023-10-09T05:40:51.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.32e-05 2023-10-09T05:40:51.678 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.678 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:54082 5.d 5:bd5f0afe:::benchmark_data_smithi008_238708_object54081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 451.03614748299998, 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96213848199999996, 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.679 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54082 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:40:51.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585530+0000", 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585549+0000", 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8794999999999999e-05 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585559+0000", 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0594999999999999e-05 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585564+0000", 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1019999999999996e-06 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585609+0000", 2023-10-09T05:40:51.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4693000000000001e-05 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:20.585619+0000", 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9839999999999996e-06 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540181+0000", 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.954562456 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540364+0000", 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000182428 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.540531+0000", 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016716999999999999 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.543639+0000", 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003108235 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547559+0000", 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039201629999999999 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547601+0000", 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1631999999999997e-05 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:33:21.547669+0000", 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7916000000000005e-05 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98105 5.8 5:13fcc3a0:::benchmark_data_smithi008_238708_object98104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695499+0000", 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 410.92617891200001, 2023-10-09T05:40:51.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93824070100000001, 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98105 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695499+0000", 2023-10-09T05:40:51.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695499+0000", 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695497+0000", 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695527+0000", 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9377999999999999e-05 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695541+0000", 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4511e-05 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695552+0000", 2023-10-09T05:40:51.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0711e-05 2023-10-09T05:40:51.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881129+0000", 2023-10-09T05:40:51.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18557714 2023-10-09T05:40:51.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881142+0000", 2023-10-09T05:40:51.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2874000000000001e-05 2023-10-09T05:40:51.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633642+0000", 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75249991800000005 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633740+0000", 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8040000000000003e-05 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.691 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.692 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98114 5.6 5:639cccb8:::benchmark_data_smithi008_238708_object98113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:40:51.692 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695561+0000", 2023-10-09T05:40:51.692 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 410.92611731800002, 2023-10-09T05:40:51.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94775633000000004, 2023-10-09T05:40:51.692 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.692 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:40:51.692 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.692 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.692 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.693 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98114 2023-10-09T05:40:51.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.693 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695561+0000", 2023-10-09T05:40:51.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695561+0000", 2023-10-09T05:40:51.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695560+0000", 2023-10-09T05:40:51.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:51.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695587+0000", 2023-10-09T05:40:51.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6925000000000001e-05 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695596+0000", 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8750000000000006e-06 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695603+0000", 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.815e-06 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887612+0000", 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19200974600000001 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887622+0000", 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3079999999999997e-06 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.626477+0000", 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73885584199999998 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627930+0000", 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014526540000000001 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.643236+0000", 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015305859999999999 2023-10-09T05:40:51.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.643317+0000", 2023-10-09T05:40:51.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0939999999999994e-05 2023-10-09T05:40:51.698 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.698 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.698 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.698 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98120 5.18 5:1c597501:::benchmark_data_smithi008_238708_object98119:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:40:51.698 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695612+0000", 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 410.92606591800001, 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96914215000000004, 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98120 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695612+0000", 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695612+0000", 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695611+0000", 2023-10-09T05:40:51.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695637+0000", 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5809999999999999e-05 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695646+0000", 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.303e-06 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695652+0000", 2023-10-09T05:40:51.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3300000000000004e-06 2023-10-09T05:40:51.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881169+0000", 2023-10-09T05:40:51.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18551722700000001 2023-10-09T05:40:51.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881176+0000", 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1720000000000001e-06 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.658602+0000", 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77742560599999999 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659106+0000", 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000503821 2023-10-09T05:40:51.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.664640+0000", 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0055338050000000001 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.664754+0000", 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114644 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.705 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98126 5.8 5:11d9570b:::benchmark_data_smithi008_238708_object98125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:40:51.705 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695658+0000", 2023-10-09T05:40:51.705 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 410.92602009699999, 2023-10-09T05:40:51.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96035853800000004, 2023-10-09T05:40:51.705 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.705 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.705 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.705 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.705 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.705 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98126 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695658+0000", 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695658+0000", 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695657+0000", 2023-10-09T05:40:51.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:51.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695683+0000", 2023-10-09T05:40:51.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5728000000000002e-05 2023-10-09T05:40:51.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695691+0000", 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9869999999999999e-06 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695697+0000", 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9120000000000003e-06 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881200+0000", 2023-10-09T05:40:51.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.185503105 2023-10-09T05:40:51.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881210+0000", 2023-10-09T05:40:51.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4259999999999992e-06 2023-10-09T05:40:51.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633764+0000", 2023-10-09T05:40:51.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75255434799999998 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633873+0000", 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010911700000000001 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.634034+0000", 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000161149 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.651360+0000", 2023-10-09T05:40:51.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017325393000000001 2023-10-09T05:40:51.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.654621+0000", 2023-10-09T05:40:51.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032617940000000002 2023-10-09T05:40:51.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.655917+0000", 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001295678 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.655974+0000", 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6629999999999998e-05 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.656016+0000", 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2697999999999997e-05 2023-10-09T05:40:51.712 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98127 5.4 5:253a6397:::benchmark_data_smithi008_238708_object98126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695703+0000", 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 410.92597508699998, 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.927888991, 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.713 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.714 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.714 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98127 2023-10-09T05:40:51.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.714 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695703+0000", 2023-10-09T05:40:51.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695703+0000", 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695702+0000", 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695732+0000", 2023-10-09T05:40:51.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9261999999999999e-05 2023-10-09T05:40:51.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695740+0000", 2023-10-09T05:40:51.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1280000000000008e-06 2023-10-09T05:40:51.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695746+0000", 2023-10-09T05:40:51.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1310000000000003e-06 2023-10-09T05:40:51.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.866359+0000", 2023-10-09T05:40:51.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17061283099999999 2023-10-09T05:40:51.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.866369+0000", 2023-10-09T05:40:51.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.043e-05 2023-10-09T05:40:51.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.604226+0000", 2023-10-09T05:40:51.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.737856874 2023-10-09T05:40:51.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.604398+0000", 2023-10-09T05:40:51.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017228299999999999 2023-10-09T05:40:51.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.604580+0000", 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000182213 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.616540+0000", 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011959505 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.623441+0000", 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0069005589999999997 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.623503+0000", 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2378000000000005e-05 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.623592+0000", 2023-10-09T05:40:51.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8899000000000004e-05 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98129 5.18 5:1d3b051a:::benchmark_data_smithi008_238708_object98128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.695754+0000", 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 410.92592404700002, 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97840130400000003, 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.721 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98129 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695754+0000", 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695754+0000", 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695753+0000", 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695781+0000", 2023-10-09T05:40:51.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7693000000000001e-05 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695789+0000", 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9589999999999995e-06 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.695795+0000", 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4239999999999998e-06 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881231+0000", 2023-10-09T05:40:51.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18543580200000001 2023-10-09T05:40:51.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.881238+0000", 2023-10-09T05:40:51.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8290000000000002e-06 2023-10-09T05:40:51.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659365+0000", 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77812663800000004 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659507+0000", 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014198800000000001 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.659616+0000", 2023-10-09T05:40:51.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010964 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.663796+0000", 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041793669999999998 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.664218+0000", 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00042226699999999999 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.728 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.728 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.674041+0000", 2023-10-09T05:40:51.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0098232590000000009 2023-10-09T05:40:51.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.728 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.728 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.674109+0000", 2023-10-09T05:40:51.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8119000000000003e-05 2023-10-09T05:40:51.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.729 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.674155+0000", 2023-10-09T05:40:51.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5831999999999998e-05 2023-10-09T05:40:51.729 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.729 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.729 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.729 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.729 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98131 5.6 5:62d535ca:::benchmark_data_smithi008_238708_object98130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-10-09T05:40:51.729 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.711769+0000", 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 410.90990892100001, 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92605176700000003, 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98131 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711769+0000", 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711769+0000", 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711766+0000", 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999967 2023-10-09T05:40:51.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711806+0000", 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0120000000000002e-05 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711836+0000", 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0463e-05 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.711854+0000", 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7082999999999999e-05 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.890394+0000", 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17854087599999999 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.890407+0000", 2023-10-09T05:40:51.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2317000000000001e-05 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.628026+0000", 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73761953099999999 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.628147+0000", 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012068700000000001 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:40:51.735 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.628304+0000", 2023-10-09T05:40:51.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015666500000000001 2023-10-09T05:40:51.735 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.735 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.633549+0000", 2023-10-09T05:40:51.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0052457479999999997 2023-10-09T05:40:51.735 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.735 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637688+0000", 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041385900000000002 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637744+0000", 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6211000000000001e-05 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.637821+0000", 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6623000000000001e-05 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.736 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98136 5.16 5:6f5da42e:::benchmark_data_smithi008_238708_object98135:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.790198+0000", 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 410.83148015099999, 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85936307599999995, 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.737 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98136 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790198+0000", 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790198+0000", 2023-10-09T05:40:51.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790195+0000", 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790227+0000", 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2187999999999997e-05 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.740 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790252+0000", 2023-10-09T05:40:51.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4901000000000001e-05 2023-10-09T05:40:51.740 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.740 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.740 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.790264+0000", 2023-10-09T05:40:51.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2323e-05 2023-10-09T05:40:51.740 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.740 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887643+0000", 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.097379027000000007 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.887650+0000", 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4860000000000002e-06 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.645624+0000", 2023-10-09T05:40:51.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75797419499999996 2023-10-09T05:40:51.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.742 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.742 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.742 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.645742+0000", 2023-10-09T05:40:51.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000117934 2023-10-09T05:40:51.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.742 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.742 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:40:51.742 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.645860+0000", 2023-10-09T05:40:51.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000117945 2023-10-09T05:40:51.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.647879+0000", 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020193310000000001 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.649421+0000", 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015419749999999999 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.649474+0000", 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3298999999999999e-05 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.649561+0000", 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6372000000000006e-05 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.745 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119015 5.1d 5:b91a094f:::benchmark_data_smithi008_238708_object879:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:40:51.745 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.007040+0000", 2023-10-09T05:40:51.745 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 387.61463806099999, 2023-10-09T05:40:51.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.032601662, 2023-10-09T05:40:51.745 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.745 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:40:51.745 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.745 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.745 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.745 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119015 2023-10-09T05:40:51.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.746 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007040+0000", 2023-10-09T05:40:51.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007040+0000", 2023-10-09T05:40:51.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007038+0000", 2023-10-09T05:40:51.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:40:51.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007048+0000", 2023-10-09T05:40:51.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6420000000000002e-06 2023-10-09T05:40:51.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007071+0000", 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3663999999999999e-05 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007081+0000", 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8069999999999994e-06 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201929+0000", 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19484790399999999 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201935+0000", 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8259999999999998e-06 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.039550+0000", 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83761566499999995 2023-10-09T05:40:51.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.750 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.750 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.750 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.039642+0000", 2023-10-09T05:40:51.750 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1113999999999995e-05 2023-10-09T05:40:51.751 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.751 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.751 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.751 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.751 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.751 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119017 5.d 5:b096558c:::benchmark_data_smithi008_238708_object881:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 387.61458664700001, 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.010292135, 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119017 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.752 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.753 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007091+0000", 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007094+0000", 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7e-06 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007099+0000", 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5899999999999998e-06 2023-10-09T05:40:51.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007104+0000", 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7180000000000004e-06 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201969+0000", 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19486541099999999 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.755 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.201973+0000", 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1320000000000004e-06 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011724+0000", 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.80975027700000002 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011860+0000", 2023-10-09T05:40:51.756 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013602500000000001 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5", 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011985+0000", 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012507600000000001 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.016288+0000", 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0043027300000000003 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.017244+0000", 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095609500000000001 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.017296+0000", 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2408999999999999e-05 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.017383+0000", 2023-10-09T05:40:51.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7373999999999994e-05 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119025 5.1b 5:dce538de:::benchmark_data_smithi008_238708_object889:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 387.61456963400002, 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.063606593, 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.759 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119025 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.760 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007108+0000", 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007110+0000", 2023-10-09T05:40:51.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9549999999999999e-06 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007117+0000", 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5860000000000001e-06 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.007121+0000", 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.425e-06 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.226628+0000", 2023-10-09T05:40:51.763 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.21950726800000001 2023-10-09T05:40:51.763 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.763 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.226637+0000", 2023-10-09T05:40:51.763 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7229999999999993e-06 2023-10-09T05:40:51.763 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.763 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.063102+0000", 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83646540999999996 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.063229+0000", 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012684600000000001 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.063346+0000", 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011715899999999999 2023-10-09T05:40:51.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.065378+0000", 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020315820000000001 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070208+0000", 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048300499999999998 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.765 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070644+0000", 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00043635199999999999 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070683+0000", 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8967000000000001e-05 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.070715+0000", 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1566000000000002e-05 2023-10-09T05:40:51.766 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:119036 5.a 5:5126f622:::benchmark_data_smithi008_238708_object900:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:24.063601+0000", 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 387.55807657299999, 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.98084244200000004, 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:40:51.767 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119036 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063601+0000", 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063601+0000", 2023-10-09T05:40:51.768 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063599+0000", 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063609+0000", 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5370000000000003e-06 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.769 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.770 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:40:51.770 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063623+0000", 2023-10-09T05:40:51.770 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4154e-05 2023-10-09T05:40:51.770 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.770 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.770 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:40:51.770 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.063633+0000", 2023-10-09T05:40:51.770 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0023999999999999e-05 2023-10-09T05:40:51.770 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.770 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.770 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.162124+0000", 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.098490792999999993 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:24.162139+0000", 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4732000000000001e-05 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011770+0000", 2023-10-09T05:40:51.771 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.849631256 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011877+0000", 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010692999999999999 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.011982+0000", 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010477 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.772 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.016352+0000", 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004370624 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.022811+0000", 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064583419999999997 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:40:51.773 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.044346+0000", 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021535813000000001 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.044401+0000", 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4834999999999999e-05 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:25.044444+0000", 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2522999999999999e-05 2023-10-09T05:40:51.774 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.775 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.775 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.775 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:40:51.775 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:40:51.775 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:40:51.775 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:51.897 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:52.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:52.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:52.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:40:52.916 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:40:53.097 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:40:53.108 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:40:53.108 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:40:53.108 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:40:53.108 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:40:53.109 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:40:53.109 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:40:53.109 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:40:53.109 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:40:53.109 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:40:53.110 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:40:53.110 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:40:53.110 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:40:53.110 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:40:53.110 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:40:53.111 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:40:53.111 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:40:53.139 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:40:53.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:53.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:53.250 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:53.379 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:53.541 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:40:53.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:53.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:53.647 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:53.731 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:53.891 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:53.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:53.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:53.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:53 smithi008 ceph-mon[184418]: pgmap v575: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 651 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 6006/31098 objects degraded (19.313%); 0 B/s, 58 objects/s recovering 2023-10-09T05:40:53.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3147525776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:53.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 6006/31098 objects degraded (19.313%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:53.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:53 smithi008 ceph-mon[191227]: pgmap v575: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 651 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 6006/31098 objects degraded (19.313%); 0 B/s, 58 objects/s recovering 2023-10-09T05:40:53.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3147525776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:53.911 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 6006/31098 objects degraded (19.313%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:54.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:54.036 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:53 smithi191 ceph-mon[161669]: pgmap v575: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 651 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 6006/31098 objects degraded (19.313%); 0 B/s, 58 objects/s recovering 2023-10-09T05:40:54.036 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3147525776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:40:54.036 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 6006/31098 objects degraded (19.313%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:54.039 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:54.101 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:54.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:54.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:54.206 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:54.324 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:54.361 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:54.503 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:40:54.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:54.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:54.606 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:54.613 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:54.751 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:54.958 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:40:54.958 INFO:teuthology.orchestra.run.smithi008.stdout:{"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6006/28992 objects degraded (20.716%), 4 pgs degraded, 4 pgs undersized","count":8},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":748,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":202,"num_osds":8,"num_up_osds":8,"osd_up_since":1696829680,"num_in_osds":7,"osd_in_since":1696829854,"num_remapped_pgs":4},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":85},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":9664,"data_bytes":636229579,"bytes_used":4305072128,"bytes_avail":667660177408,"bytes_total":671965249536,"degraded_objects":6006,"degraded_total":28992,"degraded_ratio":0.20716059602649006,"recovering_objects_per_sec":58,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":584,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":562},"fsmap":{"epoch":6,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi008.zfejpl","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.8:8443/","prometheus":"http://172.21.15.8:9283/"}},"servicemap":{"epoch":23,"modified":"2023-10-09T05:35:39.975308+0000","services":{}},"progress_events":{"a1c2a5cb-a243-4c8c-845f-d0215fa7ce6b":{"message":"Global Recovery Event (3m)\n [==========================..] (remaining: 8s)","progress":0.95505619049072266,"add_to_ceph_s":true},"e7a35707-3c6c-469d-ad94-cd0d5825f733":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T05:40:55.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:55 smithi008 ceph-mon[184418]: pgmap v576: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 607 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 6006/28992 objects degraded (20.716%); 0 B/s, 58 objects/s recovering 2023-10-09T05:40:55.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:55 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/177711794' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:40:55.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:55 smithi008 ceph-mon[191227]: pgmap v576: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 607 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 6006/28992 objects degraded (20.716%); 0 B/s, 58 objects/s recovering 2023-10-09T05:40:55.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:55 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/177711794' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:40:56.011 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:40:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:55 smithi191 ceph-mon[161669]: pgmap v576: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 607 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 6006/28992 objects degraded (20.716%); 0 B/s, 58 objects/s recovering 2023-10-09T05:40:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:55 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/177711794' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:40:56.163 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:56.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:56.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:56.267 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:56.292 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:56.339 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:56.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:56.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:56.444 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:56.474 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:56.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:56 smithi008 ceph-mon[184418]: failed to encode map e203 with expected crc 2023-10-09T05:40:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:56 smithi008 ceph-mon[184418]: osdmap e203: 8 total, 8 up, 7 in 2023-10-09T05:40:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:56 smithi008 ceph-mon[184418]: failed to encode map e203 with expected crc 2023-10-09T05:40:56.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:56 smithi008 ceph-mon[191227]: failed to encode map e203 with expected crc 2023-10-09T05:40:56.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:56 smithi008 ceph-mon[191227]: osdmap e203: 8 total, 8 up, 7 in 2023-10-09T05:40:56.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:56 smithi008 ceph-mon[191227]: failed to encode map e203 with expected crc 2023-10-09T05:40:57.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:56 smithi191 ceph-mon[161669]: failed to encode map e203 with expected crc 2023-10-09T05:40:57.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:56 smithi191 ceph-mon[161669]: osdmap e203: 8 total, 8 up, 7 in 2023-10-09T05:40:57.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:56 smithi191 ceph-mon[161669]: failed to encode map e203 with expected crc 2023-10-09T05:40:57.212 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T05:40:57.212 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:40:57.369 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:57.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:57.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:57.482 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:40:57.514 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:57.581 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:40:57.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:57.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:40:57.680 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[184418]: pgmap v578: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 532 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 5723/25410 objects degraded (22.523%); 0 B/s, 72 objects/s recovering 2023-10-09T05:40:57.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[184418]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[184418]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[184418]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[184418]: osdmap e204: 8 total, 8 up, 7 in 2023-10-09T05:40:57.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[184418]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[184418]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:40:57.725 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:57.784 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:40:57.787 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:57 smithi191 ceph-mon[161669]: pgmap v578: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 532 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 5723/25410 objects degraded (22.523%); 0 B/s, 72 objects/s recovering 2023-10-09T05:40:57.787 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:57 smithi191 ceph-mon[161669]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.787 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:57 smithi191 ceph-mon[161669]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.787 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:57 smithi191 ceph-mon[161669]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.787 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:57 smithi191 ceph-mon[161669]: osdmap e204: 8 total, 8 up, 7 in 2023-10-09T05:40:57.787 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:57 smithi191 ceph-mon[161669]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.787 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:57 smithi191 ceph-mon[161669]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:57.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:40:57.889 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:57.932 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[191227]: pgmap v578: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 532 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 5723/25410 objects degraded (22.523%); 0 B/s, 72 objects/s recovering 2023-10-09T05:40:57.932 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[191227]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.932 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[191227]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.932 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[191227]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.932 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[191227]: osdmap e204: 8 total, 8 up, 7 in 2023-10-09T05:40:57.932 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[191227]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.933 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:57 smithi008 ceph-mon[191227]: failed to encode map e204 with expected crc 2023-10-09T05:40:57.935 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:58.410 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:40:58.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:58.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:40:58.519 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:40:58.613 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:58.687 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 5723/25410 objects degraded (22.523%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:58.688 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 5723/25410 objects degraded (22.523%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:58.861 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:58.982 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:40:58.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:58.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:40:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 5723/25410 objects degraded (22.523%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:40:59.088 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:40:59.185 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:59.281 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:59.416 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:40:59.416 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:40:59.479 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:40:59.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:40:59.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:40:59.598 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:40:59.726 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:40:59.824 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:40:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:59 smithi008 ceph-mon[184418]: pgmap v580: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 479 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 5602/22842 objects degraded (24.525%); 0 B/s, 87 objects/s recovering 2023-10-09T05:40:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2821342793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:40:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:40:59 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:40:59.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:59 smithi008 ceph-mon[191227]: pgmap v580: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 479 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 5602/22842 objects degraded (24.525%); 0 B/s, 87 objects/s recovering 2023-10-09T05:40:59.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:59.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2821342793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:40:59.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:40:59 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:59 smithi191 ceph-mon[161669]: pgmap v580: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 479 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 5602/22842 objects degraded (24.525%); 0 B/s, 87 objects/s recovering 2023-10-09T05:41:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2821342793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:40:59 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:00.826 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:41:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:41:00] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.33.4" 2023-10-09T05:41:01.178 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:01.178 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[184418]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.179 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:41:01.179 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[184418]: osdmap e205: 8 total, 8 up, 7 in 2023-10-09T05:41:01.179 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[184418]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.179 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[184418]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.179 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[184418]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.179 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:01.179 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[191227]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.179 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:41:01.179 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[191227]: osdmap e205: 8 total, 8 up, 7 in 2023-10-09T05:41:01.179 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[191227]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.180 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[191227]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.180 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:00 smithi008 ceph-mon[191227]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:00 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:01.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:00 smithi191 ceph-mon[161669]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:00 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:41:01.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:00 smithi191 ceph-mon[161669]: osdmap e205: 8 total, 8 up, 7 in 2023-10-09T05:41:01.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:00 smithi191 ceph-mon[161669]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:00 smithi191 ceph-mon[161669]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:00 smithi191 ceph-mon[161669]: failed to encode map e205 with expected crc 2023-10-09T05:41:01.647 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:01.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:01.759 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:01.890 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:01.916 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:01 smithi008 ceph-mon[191227]: pgmap v582: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 389 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 5602/18546 objects degraded (30.206%); 0 B/s, 67 objects/s recovering 2023-10-09T05:41:01.922 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:01 smithi008 ceph-mon[184418]: pgmap v582: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 389 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 5602/18546 objects degraded (30.206%); 0 B/s, 67 objects/s recovering 2023-10-09T05:41:02.056 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:41:02.251 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:02.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:02.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:01 smithi191 ceph-mon[161669]: pgmap v582: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 389 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 5602/18546 objects degraded (30.206%); 0 B/s, 67 objects/s recovering 2023-10-09T05:41:02.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:02.395 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:02.461 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:02.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:02.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:02.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:02.605 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:02.661 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:02.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:02.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:02.773 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:02.863 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:03.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:02 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 5602/18546 objects degraded (30.206%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:03.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:02 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 5602/18546 objects degraded (30.206%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:03.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:02 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 5602/18546 objects degraded (30.206%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:03.348 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:03.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:03.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:03.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:03.514 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:03.581 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:03.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:03.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:03.692 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:03.726 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:03.791 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:03.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:03.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:03.901 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:03.990 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:03 smithi191 ceph-mon[161669]: pgmap v583: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 317 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 5160/15069 objects degraded (34.242%); 0 B/s, 130 objects/s recovering 2023-10-09T05:41:04.108 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:03 smithi008 ceph-mon[184418]: pgmap v583: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 317 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 5160/15069 objects degraded (34.242%); 0 B/s, 130 objects/s recovering 2023-10-09T05:41:04.109 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:03 smithi008 ceph-mon[191227]: pgmap v583: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 317 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 5160/15069 objects degraded (34.242%); 0 B/s, 130 objects/s recovering 2023-10-09T05:41:04.318 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:04.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:04.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:04.436 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:04.577 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:04.600 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:41:04.670 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:41:04.727 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:41:04.747 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:04.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:04.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:04.789 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:41:04.823 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:41:04.859 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:04.878 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:41:04.898 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:04.935 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:41:04.979 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:04.993 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:41:05.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:41:05.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:41:05.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:41:05.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:41:05.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:41:05.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:41:05.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:41:05.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:41:05.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:04 smithi008 ceph-mon[184418]: pgmap v584: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 268 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 5160/12711 objects degraded (40.595%); 0 B/s, 70 objects/s recovering 2023-10-09T05:41:05.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1280905751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:04 smithi008 ceph-mon[191227]: pgmap v584: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 268 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 5160/12711 objects degraded (40.595%); 0 B/s, 70 objects/s recovering 2023-10-09T05:41:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1280905751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:05.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:04 smithi191 ceph-mon[161669]: pgmap v584: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 268 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 5160/12711 objects degraded (40.595%); 0 B/s, 70 objects/s recovering 2023-10-09T05:41:05.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1280905751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:05.399 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:41:05.427 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:41:05.593 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:05.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:05.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:05.697 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:05.738 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:05.810 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:05.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:05.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:05.921 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:05.953 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:06.024 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:06.035 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:05 smithi191 ceph-mon[161669]: from='client.45284 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:06.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:06.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:06.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:06.181 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:06.189 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:41:06.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:05 smithi008 ceph-mon[184418]: from='client.45284 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:06.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:05 smithi008 ceph-mon[191227]: from='client.45284 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:06.245 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:06.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:06.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:06.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:06.384 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:06.431 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:06.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:06.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:06.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:06.584 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:06.652 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:06.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:06.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:06.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:06.797 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:06.904 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:06.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:06.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:06.933 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:06 smithi008 ceph-mon[191227]: pgmap v585: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 186 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 5047/8790 objects degraded (57.418%); 0 B/s, 58 objects/s recovering 2023-10-09T05:41:07.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:07.039 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:07.081 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:07.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:07.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:07.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:07.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:06 smithi008 ceph-mon[184418]: pgmap v585: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 186 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 5047/8790 objects degraded (57.418%); 0 B/s, 58 objects/s recovering 2023-10-09T05:41:07.219 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:06 smithi191 ceph-mon[161669]: pgmap v585: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 186 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 5047/8790 objects degraded (57.418%); 0 B/s, 58 objects/s recovering 2023-10-09T05:41:07.227 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:07.302 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:07.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:07.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:07.410 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:07.484 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:07.697 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":584,"stamp":"2023-10-09T05:41:04.150154+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":280565707,"num_objects":4237,"num_object_clones":0,"num_object_copies":12711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5160,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4237,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":173619,"num_write_kb":5670182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12381,"num_bytes_recovered":531043702,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130883,"ondisk_log_size":130883,"up":267,"acting":264,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3161924,"kb_used_data":903756,"kb_used_omap":31,"kb_used_meta":2257952,"kb_avail":653054140,"statfs":{"total":671965249536,"available":668727439360,"internally_reserved":0,"allocated":925446144,"data_stored":920796306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2312143305},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-277413888,"num_objects":-4233,"num_object_clones":0,"num_object_copies":-12699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-563,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4233,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":563,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.006290"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":188,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650518+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:40:56.650518+0000","last_peered":"2023-10-09T05:40:56.650518+0000","last_clean":"2023-10-09T05:40:56.650518+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:40:56.650518+0000","last_undegraded":"2023-10-09T05:40:56.650518+0000","last_fullsized":"2023-10-09T05:40:56.650518+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":177,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.689539+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:40:56.689539+0000","last_peered":"2023-10-09T05:40:56.689539+0000","last_clean":"2023-10-09T05:40:56.689539+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:40:56.689539+0000","last_undegraded":"2023-10-09T05:40:56.689539+0000","last_fullsized":"2023-10-09T05:40:56.689539+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":152,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.647223+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:40:56.647223+0000","last_peered":"2023-10-09T05:40:56.647223+0000","last_clean":"2023-10-09T05:40:56.647223+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:40:56.647223+0000","last_undegraded":"2023-10-09T05:40:56.647223+0000","last_fullsized":"2023-10-09T05:40:56.647223+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":182,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694748+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:40:56.694748+0000","last_peered":"2023-10-09T05:40:56.694748+0000","last_clean":"2023-10-09T05:40:56.694748+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:40:56.694748+0000","last_undegraded":"2023-10-09T05:40:56.694748+0000","last_fullsized":"2023-10-09T05:40:56.694748+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":193,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.692082+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:40:56.692082+0000","last_peered":"2023-10-09T05:40:56.692082+0000","last_clean":"2023-10-09T05:40:56.692082+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:40:56.692082+0000","last_undegraded":"2023-10-09T05:40:56.692082+0000","last_fullsized":"2023-10-09T05:40:56.692082+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":171,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.652673+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:40:56.652673+0000","last_peered":"2023-10-09T05:40:56.652673+0000","last_clean":"2023-10-09T05:40:56.652673+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:40:56.652673+0000","last_undegraded":"2023-10-09T05:40:56.652673+0000","last_fullsized":"2023-10-09T05:40:56.652673+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":157,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.695098+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:40:56.695098+0000","last_peered":"2023-10-09T05:40:56.695098+0000","last_clean":"2023-10-09T05:40:56.695098+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:40:56.695098+0000","last_undegraded":"2023-10-09T05:40:56.695098+0000","last_fullsized":"2023-10-09T05:40:56.695098+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":178,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.692118+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:40:56.692118+0000","last_peered":"2023-10-09T05:40:56.692118+0000","last_clean":"2023-10-09T05:40:56.692118+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:40:56.692118+0000","last_undegraded":"2023-10-09T05:40:56.692118+0000","last_fullsized":"2023-10-09T05:40:56.692118+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":196,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.706852+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:40:56.706852+0000","last_peered":"2023-10-09T05:40:56.706852+0000","last_clean":"2023-10-09T05:40:56.706852+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:40:56.706852+0000","last_undegraded":"2023-10-09T05:40:56.706852+0000","last_fullsized":"2023-10-09T05:40:56.706852+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":192,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.688883+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:40:56.688883+0000","last_peered":"2023-10-09T05:40:56.688883+0000","last_clean":"2023-10-09T05:40:56.688883+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:40:56.688883+0000","last_undegraded":"2023-10-09T05:40:56.688883+0000","last_fullsized":"2023-10-09T05:40:56.688883+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":85,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.689468+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:40:56.689468+0000","last_peered":"2023-10-09T05:40:56.689468+0000","last_clean":"2023-10-09T05:40:56.689468+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:40:56.689468+0000","last_undegraded":"2023-10-09T05:40:56.689468+0000","last_fullsized":"2023-10-09T05:40:56.689468+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":152,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.647406+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:40:56.647406+0000","last_peered":"2023-10-09T05:40:56.647406+0000","last_clean":"2023-10-09T05:40:56.647406+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:40:56.647406+0000","last_undegraded":"2023-10-09T05:40:56.647406+0000","last_fullsized":"2023-10-09T05:40:56.647406+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":213,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650370+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:40:56.650370+0000","last_peered":"2023-10-09T05:40:56.650370+0000","last_clean":"2023-10-09T05:40:56.650370+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:40:56.650370+0000","last_undegraded":"2023-10-09T05:40:56.650370+0000","last_fullsized":"2023-10-09T05:40:56.650370+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":171,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650291+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:40:56.650291+0000","last_peered":"2023-10-09T05:40:56.650291+0000","last_clean":"2023-10-09T05:40:56.650291+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:40:56.650291+0000","last_undegraded":"2023-10-09T05:40:56.650291+0000","last_fullsized":"2023-10-09T05:40:56.650291+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":84,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650379+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:40:56.650379+0000","last_peered":"2023-10-09T05:40:56.650379+0000","last_clean":"2023-10-09T05:40:56.650379+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:40:56.650379+0000","last_undegraded":"2023-10-09T05:40:56.650379+0000","last_fullsized":"2023-10-09T05:40:56.650379+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":202,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.688980+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:40:56.688980+0000","last_peered":"2023-10-09T05:40:56.688980+0000","last_clean":"2023-10-09T05:40:56.688980+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:40:56.688980+0000","last_undegraded":"2023-10-09T05:40:56.688980+0000","last_fullsized":"2023-10-09T05:40:56.688980+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":181,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694622+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:40:56.694622+0000","last_peered":"2023-10-09T05:40:56.694622+0000","last_clean":"2023-10-09T05:40:56.694622+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:40:56.694622+0000","last_undegraded":"2023-10-09T05:40:56.694622+0000","last_fullsized":"2023-10-09T05:40:56.694622+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":195,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.706795+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:40:56.706795+0000","last_peered":"2023-10-09T05:40:56.706795+0000","last_clean":"2023-10-09T05:40:56.706795+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:40:56.706795+0000","last_undegraded":"2023-10-09T05:40:56.706795+0000","last_fullsized":"2023-10-09T05:40:56.706795+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":102,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650310+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:40:56.650310+0000","last_peered":"2023-10-09T05:40:56.650310+0000","last_clean":"2023-10-09T05:40:56.650310+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:40:56.650310+0000","last_undegraded":"2023-10-09T05:40:56.650310+0000","last_fullsized":"2023-10-09T05:40:56.650310+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":181,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694650+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:40:56.694650+0000","last_peered":"2023-10-09T05:40:56.694650+0000","last_clean":"2023-10-09T05:40:56.694650+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:40:56.694650+0000","last_undegraded":"2023-10-09T05:40:56.694650+0000","last_fullsized":"2023-10-09T05:40:56.694650+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":180,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.651824+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:40:56.651824+0000","last_peered":"2023-10-09T05:40:56.651824+0000","last_clean":"2023-10-09T05:40:56.651824+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:40:56.651824+0000","last_undegraded":"2023-10-09T05:40:56.651824+0000","last_fullsized":"2023-10-09T05:40:56.651824+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":201,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.690305+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:40:56.690305+0000","last_peered":"2023-10-09T05:40:56.690305+0000","last_clean":"2023-10-09T05:40:56.690305+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:40:56.690305+0000","last_undegraded":"2023-10-09T05:40:56.690305+0000","last_fullsized":"2023-10-09T05:40:56.690305+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"205'10727","reported_seq":10778,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.924327+0000","last_change":"2023-10-09T05:37:01.636984+0000","last_active":"2023-10-09T05:40:59.924327+0000","last_peered":"2023-10-09T05:40:59.924327+0000","last_clean":"2023-10-09T05:40:59.924327+0000","last_became_active":"2023-10-09T05:37:01.636609+0000","last_became_peered":"2023-10-09T05:37:01.636609+0000","last_unstale":"2023-10-09T05:40:59.924327+0000","last_undegraded":"2023-10-09T05:40:59.924327+0000","last_fullsized":"2023-10-09T05:40:59.924327+0000","mapping_epoch":162,"log_start":"166'700","ondisk_log_start":"166'700","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":10027,"log_dups_size":0,"ondisk_log_size":10027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21561344,"num_objects":329,"num_object_clones":0,"num_object_copies":987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10727,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":82,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650421+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:40:56.650421+0000","last_peered":"2023-10-09T05:40:56.650421+0000","last_clean":"2023-10-09T05:40:56.650421+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:40:56.650421+0000","last_undegraded":"2023-10-09T05:40:56.650421+0000","last_fullsized":"2023-10-09T05:40:56.650421+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":180,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.706286+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:40:56.706286+0000","last_peered":"2023-10-09T05:40:56.706286+0000","last_clean":"2023-10-09T05:40:56.706286+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:40:56.706286+0000","last_undegraded":"2023-10-09T05:40:56.706286+0000","last_fullsized":"2023-10-09T05:40:56.706286+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"205'10735","reported_seq":11703,"reported_epoch":205,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:41:01.394317+0000","last_change":"2023-10-09T05:40:54.883374+0000","last_active":"2023-10-09T05:41:01.394317+0000","last_peered":"2023-10-09T05:41:01.394317+0000","last_clean":"2023-10-09T05:37:34.651883+0000","last_became_active":"2023-10-09T05:37:36.832502+0000","last_became_peered":"2023-10-09T05:37:36.832502+0000","last_unstale":"2023-10-09T05:41:01.394317+0000","last_undegraded":"2023-10-09T05:37:36.702797+0000","last_fullsized":"2023-10-09T05:37:36.678583+0000","mapping_epoch":170,"log_start":"178'4300","ondisk_log_start":"178'4300","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6435,"log_dups_size":0,"ondisk_log_size":6435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:57.419460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16842752,"num_objects":257,"num_object_clones":0,"num_object_copies":771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2437,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10735,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":453,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":-2180}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":82,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.651332+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:40:56.651332+0000","last_peered":"2023-10-09T05:40:56.651332+0000","last_clean":"2023-10-09T05:40:56.651332+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:40:56.651332+0000","last_undegraded":"2023-10-09T05:40:56.651332+0000","last_fullsized":"2023-10-09T05:40:56.651332+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":186,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650222+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:40:56.650222+0000","last_peered":"2023-10-09T05:40:56.650222+0000","last_clean":"2023-10-09T05:40:56.650222+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:40:56.650222+0000","last_undegraded":"2023-10-09T05:40:56.650222+0000","last_fullsized":"2023-10-09T05:40:56.650222+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"205'10963","reported_seq":17076,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:01.795686+0000","last_change":"2023-10-09T05:38:55.317808+0000","last_active":"2023-10-09T05:41:01.795686+0000","last_peered":"2023-10-09T05:41:01.795686+0000","last_clean":"2023-10-09T05:41:01.795686+0000","last_became_active":"2023-10-09T05:38:55.302739+0000","last_became_peered":"2023-10-09T05:38:55.302739+0000","last_unstale":"2023-10-09T05:41:01.795686+0000","last_undegraded":"2023-10-09T05:41:01.795686+0000","last_fullsized":"2023-10-09T05:41:01.795686+0000","mapping_epoch":186,"log_start":"167'1800","ondisk_log_start":"167'1800","created":162,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9163,"log_dups_size":0,"ondisk_log_size":9163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:25.379818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16318464,"num_objects":249,"num_object_clones":0,"num_object_copies":747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10963,"num_write_kb":358784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":198049792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":82,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.706819+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:40:56.706819+0000","last_peered":"2023-10-09T05:40:56.706819+0000","last_clean":"2023-10-09T05:40:56.706819+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:40:56.706819+0000","last_undegraded":"2023-10-09T05:40:56.706819+0000","last_fullsized":"2023-10-09T05:40:56.706819+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":204,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.652116+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:40:56.652116+0000","last_peered":"2023-10-09T05:40:56.652116+0000","last_clean":"2023-10-09T05:40:56.652116+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:40:56.652116+0000","last_undegraded":"2023-10-09T05:40:56.652116+0000","last_fullsized":"2023-10-09T05:40:56.652116+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"205'10801","reported_seq":16672,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:03.242853+0000","last_change":"2023-10-09T05:40:04.830679+0000","last_active":"2023-10-09T05:41:03.242853+0000","last_peered":"2023-10-09T05:41:03.242853+0000","last_clean":"2023-10-09T05:41:03.242853+0000","last_became_active":"2023-10-09T05:40:04.807741+0000","last_became_peered":"2023-10-09T05:40:04.807741+0000","last_unstale":"2023-10-09T05:41:03.242853+0000","last_undegraded":"2023-10-09T05:41:03.242853+0000","last_fullsized":"2023-10-09T05:41:03.242853+0000","mapping_epoch":195,"log_start":"167'1900","ondisk_log_start":"167'1900","created":162,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8901,"log_dups_size":0,"ondisk_log_size":8901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:51.289034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11075584,"num_objects":169,"num_object_clones":0,"num_object_copies":507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10801,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2899,"num_bytes_recovered":139919360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":172,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.706531+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:40:56.706531+0000","last_peered":"2023-10-09T05:40:56.706531+0000","last_clean":"2023-10-09T05:40:56.706531+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:40:56.706531+0000","last_undegraded":"2023-10-09T05:40:56.706531+0000","last_fullsized":"2023-10-09T05:40:56.706531+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":157,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694585+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:40:56.694585+0000","last_peered":"2023-10-09T05:40:56.694585+0000","last_clean":"2023-10-09T05:40:56.694585+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:40:56.694585+0000","last_undegraded":"2023-10-09T05:40:56.694585+0000","last_fullsized":"2023-10-09T05:40:56.694585+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"205'10662","reported_seq":10713,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.936265+0000","last_change":"2023-10-09T05:37:01.439878+0000","last_active":"2023-10-09T05:40:59.936265+0000","last_peered":"2023-10-09T05:40:59.936265+0000","last_clean":"2023-10-09T05:40:59.936265+0000","last_became_active":"2023-10-09T05:37:01.439481+0000","last_became_peered":"2023-10-09T05:37:01.439481+0000","last_unstale":"2023-10-09T05:40:59.936265+0000","last_undegraded":"2023-10-09T05:40:59.936265+0000","last_fullsized":"2023-10-09T05:40:59.936265+0000","mapping_epoch":162,"log_start":"174'3100","ondisk_log_start":"174'3100","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7562,"log_dups_size":0,"ondisk_log_size":7562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21233664,"num_objects":324,"num_object_clones":0,"num_object_copies":972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10662,"num_write_kb":351552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":176,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.689803+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:40:56.689803+0000","last_peered":"2023-10-09T05:40:56.689803+0000","last_clean":"2023-10-09T05:40:56.689803+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:40:56.689803+0000","last_undegraded":"2023-10-09T05:40:56.689803+0000","last_fullsized":"2023-10-09T05:40:56.689803+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":171,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650278+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:40:56.650278+0000","last_peered":"2023-10-09T05:40:56.650278+0000","last_clean":"2023-10-09T05:40:56.650278+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:40:56.650278+0000","last_undegraded":"2023-10-09T05:40:56.650278+0000","last_fullsized":"2023-10-09T05:40:56.650278+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"205'10806","reported_seq":10859,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:01.723551+0000","last_change":"2023-10-09T05:37:01.440175+0000","last_active":"2023-10-09T05:41:01.723551+0000","last_peered":"2023-10-09T05:41:01.723551+0000","last_clean":"2023-10-09T05:41:01.723551+0000","last_became_active":"2023-10-09T05:37:01.439846+0000","last_became_peered":"2023-10-09T05:37:01.439846+0000","last_unstale":"2023-10-09T05:41:01.723551+0000","last_undegraded":"2023-10-09T05:41:01.723551+0000","last_fullsized":"2023-10-09T05:41:01.723551+0000","mapping_epoch":162,"log_start":"167'1700","ondisk_log_start":"167'1700","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9106,"log_dups_size":0,"ondisk_log_size":9106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:27.883265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16252928,"num_objects":248,"num_object_clones":0,"num_object_copies":744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10806,"num_write_kb":353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":172,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.652030+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:40:56.652030+0000","last_peered":"2023-10-09T05:40:56.652030+0000","last_clean":"2023-10-09T05:40:56.652030+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:40:56.652030+0000","last_undegraded":"2023-10-09T05:40:56.652030+0000","last_fullsized":"2023-10-09T05:40:56.652030+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":186,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650520+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:40:56.650520+0000","last_peered":"2023-10-09T05:40:56.650520+0000","last_clean":"2023-10-09T05:40:56.650520+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:40:56.650520+0000","last_undegraded":"2023-10-09T05:40:56.650520+0000","last_fullsized":"2023-10-09T05:40:56.650520+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"205'10844","reported_seq":10897,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:01.384975+0000","last_change":"2023-10-09T05:37:01.638178+0000","last_active":"2023-10-09T05:41:01.384975+0000","last_peered":"2023-10-09T05:41:01.384975+0000","last_clean":"2023-10-09T05:41:01.384975+0000","last_became_active":"2023-10-09T05:37:01.637822+0000","last_became_peered":"2023-10-09T05:37:01.637822+0000","last_unstale":"2023-10-09T05:41:01.384975+0000","last_undegraded":"2023-10-09T05:41:01.384975+0000","last_fullsized":"2023-10-09T05:41:01.384975+0000","mapping_epoch":162,"log_start":"179'4400","ondisk_log_start":"179'4400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6444,"log_dups_size":0,"ondisk_log_size":6444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:13.240069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17694720,"num_objects":270,"num_object_clones":0,"num_object_copies":810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10844,"num_write_kb":355648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":196,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650049+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:40:56.650049+0000","last_peered":"2023-10-09T05:40:56.650049+0000","last_clean":"2023-10-09T05:40:56.650049+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:40:56.650049+0000","last_undegraded":"2023-10-09T05:40:56.650049+0000","last_fullsized":"2023-10-09T05:40:56.650049+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":180,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.651386+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:40:56.651386+0000","last_peered":"2023-10-09T05:40:56.651386+0000","last_clean":"2023-10-09T05:40:56.651386+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:40:56.651386+0000","last_undegraded":"2023-10-09T05:40:56.651386+0000","last_fullsized":"2023-10-09T05:40:56.651386+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"205'10744","reported_seq":10796,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.957300+0000","last_change":"2023-10-09T05:37:01.438917+0000","last_active":"2023-10-09T05:40:59.957300+0000","last_peered":"2023-10-09T05:40:59.957300+0000","last_clean":"2023-10-09T05:40:59.957300+0000","last_became_active":"2023-10-09T05:37:01.438658+0000","last_became_peered":"2023-10-09T05:37:01.438658+0000","last_unstale":"2023-10-09T05:40:59.957300+0000","last_undegraded":"2023-10-09T05:40:59.957300+0000","last_fullsized":"2023-10-09T05:40:59.957300+0000","mapping_epoch":162,"log_start":"166'700","ondisk_log_start":"166'700","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":10044,"log_dups_size":0,"ondisk_log_size":10044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20774936,"num_objects":318,"num_object_clones":0,"num_object_copies":954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":318,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10744,"num_write_kb":353921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":203,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.692023+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:40:56.692023+0000","last_peered":"2023-10-09T05:40:56.692023+0000","last_clean":"2023-10-09T05:40:56.692023+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:40:56.692023+0000","last_undegraded":"2023-10-09T05:40:56.692023+0000","last_fullsized":"2023-10-09T05:40:56.692023+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":152,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.647638+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:40:56.647638+0000","last_peered":"2023-10-09T05:40:56.647638+0000","last_clean":"2023-10-09T05:40:56.647638+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:40:56.647638+0000","last_undegraded":"2023-10-09T05:40:56.647638+0000","last_fullsized":"2023-10-09T05:40:56.647638+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"205'10902","reported_seq":10956,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:03.264672+0000","last_change":"2023-10-09T05:37:01.440093+0000","last_active":"2023-10-09T05:41:03.264672+0000","last_peered":"2023-10-09T05:41:03.264672+0000","last_clean":"2023-10-09T05:41:03.264672+0000","last_became_active":"2023-10-09T05:37:01.439799+0000","last_became_peered":"2023-10-09T05:37:01.439799+0000","last_unstale":"2023-10-09T05:41:03.264672+0000","last_undegraded":"2023-10-09T05:41:03.264672+0000","last_fullsized":"2023-10-09T05:41:03.264672+0000","mapping_epoch":162,"log_start":"167'2000","ondisk_log_start":"167'2000","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8902,"log_dups_size":0,"ondisk_log_size":8902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:28.734471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13369344,"num_objects":204,"num_object_clones":0,"num_object_copies":612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10902,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":775,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.647856+0000","last_change":"2023-10-09T05:40:04.806908+0000","last_active":"2023-10-09T05:40:56.647856+0000","last_peered":"2023-10-09T05:40:56.647856+0000","last_clean":"2023-10-09T05:40:56.647856+0000","last_became_active":"2023-10-09T05:40:04.806232+0000","last_became_peered":"2023-10-09T05:40:04.806232+0000","last_unstale":"2023-10-09T05:40:56.647856+0000","last_undegraded":"2023-10-09T05:40:56.647856+0000","last_fullsized":"2023-10-09T05:40:56.647856+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:34.087553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":187,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.706716+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:40:56.706716+0000","last_peered":"2023-10-09T05:40:56.706716+0000","last_clean":"2023-10-09T05:40:56.706716+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:40:56.706716+0000","last_undegraded":"2023-10-09T05:40:56.706716+0000","last_fullsized":"2023-10-09T05:40:56.706716+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":138,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.647381+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:40:56.647381+0000","last_peered":"2023-10-09T05:40:56.647381+0000","last_clean":"2023-10-09T05:40:56.647381+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:40:56.647381+0000","last_undegraded":"2023-10-09T05:40:56.647381+0000","last_fullsized":"2023-10-09T05:40:56.647381+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"205'10884","reported_seq":16827,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:01.385017+0000","last_change":"2023-10-09T05:38:10.132330+0000","last_active":"2023-10-09T05:41:01.385017+0000","last_peered":"2023-10-09T05:41:01.385017+0000","last_clean":"2023-10-09T05:41:01.385017+0000","last_became_active":"2023-10-09T05:38:10.127080+0000","last_became_peered":"2023-10-09T05:38:10.127080+0000","last_unstale":"2023-10-09T05:41:01.385017+0000","last_undegraded":"2023-10-09T05:41:01.385017+0000","last_fullsized":"2023-10-09T05:41:01.385017+0000","mapping_epoch":179,"log_start":"180'4500","ondisk_log_start":"180'4500","created":162,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6384,"log_dups_size":0,"ondisk_log_size":6384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:12.659227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17301504,"num_objects":264,"num_object_clones":0,"num_object_copies":792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10884,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":192413696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":641,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.706685+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:40:56.706685+0000","last_peered":"2023-10-09T05:40:56.706685+0000","last_clean":"2023-10-09T05:40:56.706685+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:40:56.706685+0000","last_undegraded":"2023-10-09T05:40:56.706685+0000","last_fullsized":"2023-10-09T05:40:56.706685+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":172,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650463+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:40:56.650463+0000","last_peered":"2023-10-09T05:40:56.650463+0000","last_clean":"2023-10-09T05:40:56.650463+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:40:56.650463+0000","last_undegraded":"2023-10-09T05:40:56.650463+0000","last_fullsized":"2023-10-09T05:40:56.650463+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":184,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.706628+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:40:56.706628+0000","last_peered":"2023-10-09T05:40:56.706628+0000","last_clean":"2023-10-09T05:40:56.706628+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:40:56.706628+0000","last_undegraded":"2023-10-09T05:40:56.706628+0000","last_fullsized":"2023-10-09T05:40:56.706628+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"205'10692","reported_seq":11015,"reported_epoch":205,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:41:00.689505+0000","last_change":"2023-10-09T05:40:54.908423+0000","last_active":"2023-10-09T05:41:00.689505+0000","last_peered":"2023-10-09T05:41:00.689505+0000","last_clean":"2023-10-09T05:37:34.677100+0000","last_became_active":"2023-10-09T05:37:40.593033+0000","last_became_peered":"2023-10-09T05:37:40.593033+0000","last_unstale":"2023-10-09T05:41:00.689505+0000","last_undegraded":"2023-10-09T05:37:40.546647+0000","last_fullsized":"2023-10-09T05:37:40.543498+0000","mapping_epoch":170,"log_start":"167'2500","ondisk_log_start":"167'2500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8192,"log_dups_size":0,"ondisk_log_size":8192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19529728,"num_objects":298,"num_object_clones":0,"num_object_copies":894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2720,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10692,"num_write_kb":351680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":131,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":-2422}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":519,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.653472+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:40:56.653472+0000","last_peered":"2023-10-09T05:40:56.653472+0000","last_clean":"2023-10-09T05:40:56.653472+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:40:56.653472+0000","last_undegraded":"2023-10-09T05:40:56.653472+0000","last_fullsized":"2023-10-09T05:40:56.653472+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":153,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.647125+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:40:56.647125+0000","last_peered":"2023-10-09T05:40:56.647125+0000","last_clean":"2023-10-09T05:40:56.647125+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:40:56.647125+0000","last_undegraded":"2023-10-09T05:40:56.647125+0000","last_fullsized":"2023-10-09T05:40:56.647125+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":182,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.647635+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:40:56.647635+0000","last_peered":"2023-10-09T05:40:56.647635+0000","last_clean":"2023-10-09T05:40:56.647635+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:40:56.647635+0000","last_undegraded":"2023-10-09T05:40:56.647635+0000","last_fullsized":"2023-10-09T05:40:56.647635+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"205'10724","reported_seq":10775,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.860497+0000","last_change":"2023-10-09T05:37:01.438236+0000","last_active":"2023-10-09T05:40:59.860497+0000","last_peered":"2023-10-09T05:40:59.860497+0000","last_clean":"2023-10-09T05:40:59.860497+0000","last_became_active":"2023-10-09T05:37:01.437917+0000","last_became_peered":"2023-10-09T05:37:01.437917+0000","last_unstale":"2023-10-09T05:40:59.860497+0000","last_undegraded":"2023-10-09T05:40:59.860497+0000","last_fullsized":"2023-10-09T05:40:59.860497+0000","mapping_epoch":162,"log_start":"166'700","ondisk_log_start":"166'700","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":10024,"log_dups_size":0,"ondisk_log_size":10024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22937600,"num_objects":350,"num_object_clones":0,"num_object_copies":1050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10724,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":816,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.706424+0000","last_change":"2023-10-09T05:40:04.805258+0000","last_active":"2023-10-09T05:40:56.706424+0000","last_peered":"2023-10-09T05:40:56.706424+0000","last_clean":"2023-10-09T05:40:56.706424+0000","last_became_active":"2023-10-09T05:40:04.804776+0000","last_became_peered":"2023-10-09T05:40:04.804776+0000","last_unstale":"2023-10-09T05:40:56.706424+0000","last_undegraded":"2023-10-09T05:40:56.706424+0000","last_fullsized":"2023-10-09T05:40:56.706424+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":353,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694399+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:40:56.694399+0000","last_peered":"2023-10-09T05:40:56.694399+0000","last_clean":"2023-10-09T05:40:56.694399+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:40:56.694399+0000","last_undegraded":"2023-10-09T05:40:56.694399+0000","last_fullsized":"2023-10-09T05:40:56.694399+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":169,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.689199+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:40:56.689199+0000","last_peered":"2023-10-09T05:40:56.689199+0000","last_clean":"2023-10-09T05:40:56.689199+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:40:56.689199+0000","last_undegraded":"2023-10-09T05:40:56.689199+0000","last_fullsized":"2023-10-09T05:40:56.689199+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":178,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.652450+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:40:56.652450+0000","last_peered":"2023-10-09T05:40:56.652450+0000","last_clean":"2023-10-09T05:40:56.652450+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:40:56.652450+0000","last_undegraded":"2023-10-09T05:40:56.652450+0000","last_fullsized":"2023-10-09T05:40:56.652450+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"205'10794","reported_seq":10847,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:03.901616+0000","last_change":"2023-10-09T05:37:01.440172+0000","last_active":"2023-10-09T05:41:03.901616+0000","last_peered":"2023-10-09T05:41:03.901616+0000","last_clean":"2023-10-09T05:41:03.901616+0000","last_became_active":"2023-10-09T05:37:01.439932+0000","last_became_peered":"2023-10-09T05:37:01.439932+0000","last_unstale":"2023-10-09T05:41:03.901616+0000","last_undegraded":"2023-10-09T05:41:03.901616+0000","last_fullsized":"2023-10-09T05:41:03.901616+0000","mapping_epoch":162,"log_start":"180'4400","ondisk_log_start":"180'4400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6394,"log_dups_size":0,"ondisk_log_size":6394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:57.826704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10878976,"num_objects":166,"num_object_clones":0,"num_object_copies":498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10794,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":762,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.647229+0000","last_change":"2023-10-09T05:40:05.930112+0000","last_active":"2023-10-09T05:40:56.647229+0000","last_peered":"2023-10-09T05:40:56.647229+0000","last_clean":"2023-10-09T05:40:56.647229+0000","last_became_active":"2023-10-09T05:40:05.929574+0000","last_became_peered":"2023-10-09T05:40:05.929574+0000","last_unstale":"2023-10-09T05:40:56.647229+0000","last_undegraded":"2023-10-09T05:40:56.647229+0000","last_fullsized":"2023-10-09T05:40:56.647229+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:06.982326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":158,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694444+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:40:56.694444+0000","last_peered":"2023-10-09T05:40:56.694444+0000","last_clean":"2023-10-09T05:40:56.694444+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:40:56.694444+0000","last_undegraded":"2023-10-09T05:40:56.694444+0000","last_fullsized":"2023-10-09T05:40:56.694444+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":173,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694894+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:40:56.694894+0000","last_peered":"2023-10-09T05:40:56.694894+0000","last_clean":"2023-10-09T05:40:56.694894+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:40:56.694894+0000","last_undegraded":"2023-10-09T05:40:56.694894+0000","last_fullsized":"2023-10-09T05:40:56.694894+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"205'10919","reported_seq":10972,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:03.939836+0000","last_change":"2023-10-09T05:37:01.636967+0000","last_active":"2023-10-09T05:41:03.939836+0000","last_peered":"2023-10-09T05:41:03.939836+0000","last_clean":"2023-10-09T05:41:03.939836+0000","last_became_active":"2023-10-09T05:37:01.636602+0000","last_became_peered":"2023-10-09T05:37:01.636602+0000","last_unstale":"2023-10-09T05:41:03.939836+0000","last_undegraded":"2023-10-09T05:41:03.939836+0000","last_fullsized":"2023-10-09T05:41:03.939836+0000","mapping_epoch":162,"log_start":"180'4500","ondisk_log_start":"180'4500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6419,"log_dups_size":0,"ondisk_log_size":6419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:11.668143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11337728,"num_objects":173,"num_object_clones":0,"num_object_copies":519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10919,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":394,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.695420+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:40:56.695420+0000","last_peered":"2023-10-09T05:40:56.695420+0000","last_clean":"2023-10-09T05:40:56.695420+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:40:56.695420+0000","last_undegraded":"2023-10-09T05:40:56.695420+0000","last_fullsized":"2023-10-09T05:40:56.695420+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":153,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.648110+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:40:56.648110+0000","last_peered":"2023-10-09T05:40:56.648110+0000","last_clean":"2023-10-09T05:40:56.648110+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:40:56.648110+0000","last_undegraded":"2023-10-09T05:40:56.648110+0000","last_fullsized":"2023-10-09T05:40:56.648110+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":167,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.695292+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:40:56.695292+0000","last_peered":"2023-10-09T05:40:56.695292+0000","last_clean":"2023-10-09T05:40:56.695292+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:40:56.695292+0000","last_undegraded":"2023-10-09T05:40:56.695292+0000","last_fullsized":"2023-10-09T05:40:56.695292+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"205'10809","reported_seq":10859,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.908595+0000","last_change":"2023-10-09T05:37:01.440074+0000","last_active":"2023-10-09T05:40:59.908595+0000","last_peered":"2023-10-09T05:40:59.908595+0000","last_clean":"2023-10-09T05:40:59.908595+0000","last_became_active":"2023-10-09T05:37:01.439808+0000","last_became_peered":"2023-10-09T05:37:01.439808+0000","last_unstale":"2023-10-09T05:40:59.908595+0000","last_undegraded":"2023-10-09T05:40:59.908595+0000","last_fullsized":"2023-10-09T05:40:59.908595+0000","mapping_epoch":162,"log_start":"174'3300","ondisk_log_start":"174'3300","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7509,"log_dups_size":0,"ondisk_log_size":7509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20250624,"num_objects":309,"num_object_clones":0,"num_object_copies":927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10809,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":204,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.690222+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:40:56.690222+0000","last_peered":"2023-10-09T05:40:56.690222+0000","last_clean":"2023-10-09T05:40:56.690222+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:40:56.690222+0000","last_undegraded":"2023-10-09T05:40:56.690222+0000","last_fullsized":"2023-10-09T05:40:56.690222+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":593,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.647589+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:40:56.647589+0000","last_peered":"2023-10-09T05:40:56.647589+0000","last_clean":"2023-10-09T05:40:56.647589+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:40:56.647589+0000","last_undegraded":"2023-10-09T05:40:56.647589+0000","last_fullsized":"2023-10-09T05:40:56.647589+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":197,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.692138+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:40:56.692138+0000","last_peered":"2023-10-09T05:40:56.692138+0000","last_clean":"2023-10-09T05:40:56.692138+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:40:56.692138+0000","last_undegraded":"2023-10-09T05:40:56.692138+0000","last_fullsized":"2023-10-09T05:40:56.692138+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"205'10507","reported_seq":16328,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:00.669710+0000","last_change":"2023-10-09T05:40:56.681995+0000","last_active":"2023-10-09T05:41:00.669710+0000","last_peered":"2023-10-09T05:41:00.669710+0000","last_clean":"2023-10-09T05:41:00.669710+0000","last_became_active":"2023-10-09T05:40:56.654153+0000","last_became_peered":"2023-10-09T05:40:56.654153+0000","last_unstale":"2023-10-09T05:41:00.669710+0000","last_undegraded":"2023-10-09T05:41:00.669710+0000","last_fullsized":"2023-10-09T05:41:00.669710+0000","mapping_epoch":203,"log_start":"167'2300","ondisk_log_start":"167'2300","created":162,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8207,"log_dups_size":0,"ondisk_log_size":8207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18415616,"num_objects":281,"num_object_clones":0,"num_object_copies":843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10507,"num_write_kb":345216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2877,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":689,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:56.652712+0000","last_change":"2023-10-09T05:37:36.917512+0000","last_active":"2023-10-09T05:40:56.652712+0000","last_peered":"2023-10-09T05:40:56.652712+0000","last_clean":"2023-10-09T05:37:30.688967+0000","last_became_active":"2023-10-09T05:37:36.700218+0000","last_became_peered":"2023-10-09T05:37:36.700218+0000","last_unstale":"2023-10-09T05:40:56.652712+0000","last_undegraded":"2023-10-09T05:37:36.917335+0000","last_fullsized":"2023-10-09T05:37:36.682410+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":161,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694837+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:40:56.694837+0000","last_peered":"2023-10-09T05:40:56.694837+0000","last_clean":"2023-10-09T05:40:56.694837+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:40:56.694837+0000","last_undegraded":"2023-10-09T05:40:56.694837+0000","last_fullsized":"2023-10-09T05:40:56.694837+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":156,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650656+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:40:56.650656+0000","last_peered":"2023-10-09T05:40:56.650656+0000","last_clean":"2023-10-09T05:40:56.650656+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:40:56.650656+0000","last_undegraded":"2023-10-09T05:40:56.650656+0000","last_fullsized":"2023-10-09T05:40:56.650656+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":169,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694787+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:40:56.694787+0000","last_peered":"2023-10-09T05:40:56.694787+0000","last_clean":"2023-10-09T05:40:56.694787+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:40:56.694787+0000","last_undegraded":"2023-10-09T05:40:56.694787+0000","last_fullsized":"2023-10-09T05:40:56.694787+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":197,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.818134+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:40:59.818134+0000","last_peered":"2023-10-09T05:40:59.818134+0000","last_clean":"2023-10-09T05:40:59.818134+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:40:59.818134+0000","last_undegraded":"2023-10-09T05:40:59.818134+0000","last_fullsized":"2023-10-09T05:40:59.818134+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":174,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.706533+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:40:56.706533+0000","last_peered":"2023-10-09T05:40:56.706533+0000","last_clean":"2023-10-09T05:40:56.706533+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:40:56.706533+0000","last_undegraded":"2023-10-09T05:40:56.706533+0000","last_fullsized":"2023-10-09T05:40:56.706533+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":186,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650534+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:40:56.650534+0000","last_peered":"2023-10-09T05:40:56.650534+0000","last_clean":"2023-10-09T05:40:56.650534+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:40:56.650534+0000","last_undegraded":"2023-10-09T05:40:56.650534+0000","last_fullsized":"2023-10-09T05:40:56.650534+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":159,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.706620+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:40:56.706620+0000","last_peered":"2023-10-09T05:40:56.706620+0000","last_clean":"2023-10-09T05:40:56.706620+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:40:56.706620+0000","last_undegraded":"2023-10-09T05:40:56.706620+0000","last_fullsized":"2023-10-09T05:40:56.706620+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":186,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650635+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:40:56.650635+0000","last_peered":"2023-10-09T05:40:56.650635+0000","last_clean":"2023-10-09T05:40:56.650635+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:40:56.650635+0000","last_undegraded":"2023-10-09T05:40:56.650635+0000","last_fullsized":"2023-10-09T05:40:56.650635+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":184,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.689874+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:40:56.689874+0000","last_peered":"2023-10-09T05:40:56.689874+0000","last_clean":"2023-10-09T05:40:56.689874+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:40:56.689874+0000","last_undegraded":"2023-10-09T05:40:56.689874+0000","last_fullsized":"2023-10-09T05:40:56.689874+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":196,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.688814+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:40:56.688814+0000","last_peered":"2023-10-09T05:40:56.688814+0000","last_clean":"2023-10-09T05:40:56.688814+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:40:56.688814+0000","last_undegraded":"2023-10-09T05:40:56.688814+0000","last_fullsized":"2023-10-09T05:40:56.688814+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":161,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.695147+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:40:56.695147+0000","last_peered":"2023-10-09T05:40:56.695147+0000","last_clean":"2023-10-09T05:40:56.695147+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:40:56.695147+0000","last_undegraded":"2023-10-09T05:40:56.695147+0000","last_fullsized":"2023-10-09T05:40:56.695147+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":171,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.650054+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:40:56.650054+0000","last_peered":"2023-10-09T05:40:56.650054+0000","last_clean":"2023-10-09T05:40:56.650054+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:40:56.650054+0000","last_undegraded":"2023-10-09T05:40:56.650054+0000","last_fullsized":"2023-10-09T05:40:56.650054+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":275775512,"num_objects":4209,"num_object_clones":0,"num_object_copies":12627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5157,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4209,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172513,"num_write_kb":5655041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12318,"num_bytes_recovered":530448384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1458778112,"data_stored":1458765896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129713,"ondisk_log_size":129713,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846669,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626695,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":478228,"kb_used_data":172812,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93266924,"statfs":{"total":95995035648,"available":95505330176,"internally_reserved":0,"allocated":176959488,"data_stored":176436825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:25 2023","interfaces":[{"interface":"back","average":{"1min":14.522,"5min":6.984,"15min":3.121},"min":{"1min":0.24199999999999999,"5min":0.214,"15min":0.214},"max":{"1min":115.377,"5min":115.377,"15min":115.377},"last":14.23},{"interface":"front","average":{"1min":14.285,"5min":6.6109999999999998,"15min":2.9870000000000001},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.23999999999999999},"max":{"1min":116.428,"5min":116.428,"15min":116.428},"last":3.153}]},{"osd":1,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":7.274,"5min":6.056,"15min":2.7170000000000001},"min":{"1min":0.315,"5min":0.25,"15min":0.25},"max":{"1min":93.489000000000004,"5min":137.02199999999999,"15min":137.02199999999999},"last":3.2290000000000001},{"interface":"front","average":{"1min":6.6299999999999999,"5min":5.9379999999999997,"15min":2.758},"min":{"1min":0.318,"5min":0.252,"15min":0.252},"max":{"1min":93.471999999999994,"5min":137.06700000000001,"15min":137.06700000000001},"last":3.2069999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:40:25 2023","interfaces":[{"interface":"back","average":{"1min":8.1809999999999992,"5min":5.5670000000000002,"15min":3.0070000000000001},"min":{"1min":0.443,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":115.40600000000001,"5min":115.40600000000001,"15min":115.40600000000001},"last":13.73},{"interface":"front","average":{"1min":8.7759999999999998,"5min":5.6310000000000002,"15min":3.0070000000000001},"min":{"1min":0.434,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":115.297,"5min":115.297,"15min":115.297},"last":13.742000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":11.682,"5min":6.7359999999999998,"15min":3.1219999999999999},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":99.543999999999997,"5min":115.244,"15min":115.244},"last":8.7089999999999996},{"interface":"front","average":{"1min":10.734,"5min":6.556,"15min":3.1499999999999999},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":98.677999999999997,"5min":115.34999999999999,"15min":115.34999999999999},"last":0.60799999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":5.181,"5min":2.177,"15min":1.7150000000000001},"min":{"1min":0.249,"5min":0.245,"15min":0.23400000000000001},"max":{"1min":115.214,"5min":115.214,"15min":125.64},"last":0.51900000000000002},{"interface":"front","average":{"1min":5.282,"5min":1.766,"15min":1.597},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":115.22799999999999,"5min":115.22799999999999,"15min":125.625},"last":0.80900000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":7.7560000000000002,"5min":2.8050000000000002,"15min":1.819},"min":{"1min":0.31,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":115.316,"5min":115.316,"15min":127.35899999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":7.601,"5min":2.339,"15min":1.704},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29199999999999998},"max":{"1min":115.30500000000001,"5min":115.30500000000001,"15min":128.93799999999999},"last":0.83399999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":2.077,"15min":2.7909999999999999},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":5.0110000000000001,"5min":115.514,"15min":115.514},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.8240000000000001,"15min":2.0019999999999998},"min":{"1min":0.34899999999999998,"5min":0.317,"15min":0.317},"max":{"1min":3.6600000000000001,"5min":116.53100000000001,"15min":116.53100000000001},"last":0.85199999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958657,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":342468,"kb_used_data":80572,"kb_used_omap":7,"kb_used_meta":261880,"kb_avail":93402684,"statfs":{"total":95995035648,"available":95644348416,"internally_reserved":0,"allocated":82505728,"data_stored":81765314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":268165646},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":4.694,"5min":5.1500000000000004,"15min":2.6080000000000001},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":39.189,"5min":207.858,"15min":207.858},"last":0.249},{"interface":"front","average":{"1min":6.1989999999999998,"5min":6.4420000000000002,"15min":3.0720000000000001},"min":{"1min":0.23000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":42.012999999999998,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.48499999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:40:11 2023","interfaces":[{"interface":"back","average":{"1min":4.6580000000000004,"5min":6.79,"15min":3.1099999999999999},"min":{"1min":0.22,"5min":0.184,"15min":0.184},"max":{"1min":31.029,"5min":101.367,"15min":101.367},"last":0.95499999999999996},{"interface":"front","average":{"1min":5.5090000000000003,"5min":6.6219999999999999,"15min":2.9649999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":38.622999999999998,"5min":97.286000000000001,"15min":98.409999999999997},"last":0.27600000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":18.629999999999999,"5min":7.2770000000000001,"15min":2.9540000000000002},"min":{"1min":0.28899999999999998,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":162.33699999999999,"5min":162.33699999999999,"15min":162.33699999999999},"last":2.133},{"interface":"front","average":{"1min":19.029,"5min":7.9980000000000002,"15min":3.1960000000000002},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":162.376,"5min":162.376,"15min":162.376},"last":0.307}]},{"osd":4,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.1749999999999998,"5min":4.7249999999999996,"15min":2.2690000000000001},"min":{"1min":0.39700000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":42.106999999999999,"5min":141.32900000000001,"15min":141.32900000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":5.2800000000000002,"5min":3.6819999999999999,"15min":1.8400000000000001},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":40.475999999999999,"5min":141.15899999999999,"15min":141.15899999999999},"last":0.36299999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.6420000000000003,"5min":4.7009999999999996,"15min":2.2349999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":40.633000000000003,"5min":235.18700000000001,"15min":235.18700000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":5.258,"5min":5.5570000000000004,"15min":2.6240000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":42.078000000000003,"5min":235.79300000000001,"15min":235.79300000000001},"last":0.56299999999999994}]},{"osd":6,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.8860000000000001,"5min":3.8109999999999999,"15min":1.8779999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":40.558999999999997,"5min":103.60899999999999,"15min":103.60899999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":5.234,"5min":4.4950000000000001,"15min":2.141},"min":{"1min":0.41299999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":42.134999999999998,"5min":103.679,"15min":103.679},"last":0.59299999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.4950000000000001,"5min":3.649,"15min":2.5489999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":42.204000000000001,"5min":126.68300000000001,"15min":126.68300000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":5.4050000000000002,"5min":3.7450000000000001,"15min":2.6339999999999999},"min":{"1min":0.46999999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":42.228999999999999,"5min":126.738,"15min":126.738},"last":0.53900000000000003}]}]},{"osd":0,"up_from":88,"seq":377957122183,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":409520,"kb_used_data":57576,"kb_used_omap":8,"kb_used_meta":351927,"kb_avail":93335632,"statfs":{"total":95995035648,"available":95575687168,"internally_reserved":0,"allocated":58957824,"data_stored":58175974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":360374024},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4910000000000001,"5min":2.3340000000000001,"15min":1.282},"min":{"1min":0.20699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":17.760000000000002,"5min":17.760000000000002,"15min":17.760000000000002},"last":3.4039999999999999},{"interface":"front","average":{"1min":6.0330000000000004,"5min":3.3679999999999999,"15min":1.6539999999999999},"min":{"1min":0.25,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":26.163,"5min":26.163,"15min":26.163},"last":2.1829999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":5.0410000000000004,"5min":4.3979999999999997,"15min":2.2000000000000002},"min":{"1min":0.26800000000000002,"5min":0.223,"15min":0.214},"max":{"1min":21.648,"5min":79.911000000000001,"15min":79.911000000000001},"last":8.3829999999999991},{"interface":"front","average":{"1min":5.4370000000000003,"5min":4.2549999999999999,"15min":2.1040000000000001},"min":{"1min":0.26000000000000001,"5min":0.221,"15min":0.218},"max":{"1min":32.344000000000001,"5min":80.105000000000004,"15min":80.105000000000004},"last":8.5790000000000006}]},{"osd":3,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":18.498999999999999,"5min":6.4859999999999998,"15min":2.7669999999999999},"min":{"1min":0.27500000000000002,"5min":0.26000000000000001,"15min":0.22800000000000001},"max":{"1min":123.024,"5min":123.024,"15min":123.024},"last":2.3519999999999999},{"interface":"front","average":{"1min":12.829000000000001,"5min":5.4359999999999999,"15min":2.48},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":99.162000000000006,"5min":99.162000000000006,"15min":99.162000000000006},"last":2.9260000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.8420000000000001,"15min":1.2050000000000001},"min":{"1min":0.39500000000000002,"5min":0.27500000000000002,"15min":0.27200000000000002},"max":{"1min":5.157,"5min":69.082999999999998,"15min":69.082999999999998},"last":8.6470000000000002},{"interface":"front","average":{"1min":1.9119999999999999,"5min":2.5409999999999999,"15min":1.5149999999999999},"min":{"1min":0.34300000000000003,"5min":0.33000000000000002,"15min":0.30399999999999999},"max":{"1min":9.5190000000000001,"5min":69.102000000000004,"15min":69.102000000000004},"last":2.2250000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":18.456,"5min":6.2709999999999999,"15min":2.7360000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":281.733,"5min":281.733,"15min":281.733},"last":2.2719999999999998},{"interface":"front","average":{"1min":19.111999999999998,"5min":6.2210000000000001,"15min":2.7370000000000001},"min":{"1min":0.40899999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":282.95400000000001,"5min":282.95400000000001,"15min":282.95400000000001},"last":8.4269999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5529999999999999,"5min":2.0110000000000001,"15min":1.3280000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":2.2999999999999998},{"interface":"front","average":{"1min":3.5190000000000001,"5min":2.1779999999999999,"15min":1.387},"min":{"1min":0.46200000000000002,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":10.83,"5min":10.83,"15min":10.83},"last":2.9700000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.679,"15min":1.4450000000000001},"min":{"1min":0.41899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.5090000000000003,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":8.6159999999999997},{"interface":"front","average":{"1min":3.1859999999999999,"5min":2.548,"15min":2.4420000000000002},"min":{"1min":0.46800000000000003,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.6329999999999991,"5min":9.7159999999999993,"15min":11.068},"last":2.3260000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795130,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":570404,"kb_used_data":166812,"kb_used_omap":11,"kb_used_meta":403572,"kb_avail":93174748,"statfs":{"total":95995035648,"available":95410941952,"internally_reserved":0,"allocated":170815488,"data_stored":169989383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":413258325},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":4.3079999999999998,"5min":3.0630000000000002,"15min":1.53},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17199999999999999},"max":{"1min":20.771000000000001,"5min":20.771000000000001,"15min":32.122},"last":5.4669999999999996},{"interface":"front","average":{"1min":6.133,"5min":3.0640000000000001,"15min":1.5229999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":21.997,"5min":21.997,"15min":34.066000000000003},"last":9.0559999999999992}]},{"osd":1,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":4.8250000000000002,"15min":2.3109999999999999},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.20699999999999999},"max":{"1min":9.3469999999999995,"5min":98.575000000000003,"15min":98.575000000000003},"last":4.5359999999999996},{"interface":"front","average":{"1min":2.431,"5min":3.9820000000000002,"15min":2.0030000000000001},"min":{"1min":0.23899999999999999,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":9.3260000000000005,"5min":90.135000000000005,"15min":98.397000000000006},"last":4.4880000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":5.7240000000000002,"5min":6.1200000000000001,"15min":2.6499999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.24199999999999999},"max":{"1min":24.471,"5min":150.65899999999999,"15min":150.65899999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":6.6200000000000001,"5min":6.9770000000000003,"15min":3.016},"min":{"1min":0.46000000000000002,"5min":0.29899999999999999,"15min":0.26500000000000001},"max":{"1min":22.631,"5min":150.851,"15min":150.851},"last":5.4189999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2629999999999999,"5min":3.0019999999999998,"15min":1.5149999999999999},"min":{"1min":0.29799999999999999,"5min":0.28999999999999998,"15min":0.27700000000000002},"max":{"1min":9.3000000000000007,"5min":115.84999999999999,"15min":115.84999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":3.2669999999999999,"5min":3.887,"15min":1.8500000000000001},"min":{"1min":0.48399999999999999,"5min":0.312,"15min":0.29799999999999999},"max":{"1min":10.818,"5min":116.149,"15min":116.149},"last":0.41299999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":2.964,"5min":2.7690000000000001,"15min":1.524},"min":{"1min":0.44,"5min":0.312,"15min":0.312},"max":{"1min":8.7509999999999994,"5min":24.597999999999999,"15min":24.597999999999999},"last":4.5700000000000003},{"interface":"front","average":{"1min":2.3039999999999998,"5min":2.282,"15min":1.292},"min":{"1min":0.32700000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.2590000000000003,"5min":23.919,"15min":23.919},"last":0.51900000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":3.2869999999999999,"5min":2.7719999999999998,"15min":1.5049999999999999},"min":{"1min":0.41199999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.864000000000001,"5min":20.503,"15min":20.503},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.9940000000000002,"5min":2.4489999999999998,"15min":1.401},"min":{"1min":0.46800000000000003,"5min":0.39600000000000002,"15min":0.34499999999999997},"max":{"1min":8.7949999999999999,"5min":9.093,"15min":9.4049999999999994},"last":4.5949999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:40:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":3.2509999999999999,"15min":2.766},"min":{"1min":0.47199999999999998,"5min":0.40200000000000002,"15min":0.374},"max":{"1min":13.865,"5min":24.030999999999999,"15min":24.030999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":3.5070000000000001,"5min":3.1110000000000002,"15min":2.1890000000000001},"min":{"1min":0.52700000000000002,"5min":0.41899999999999998,"15min":0.41599999999999998},"max":{"1min":13.781000000000001,"5min":24.178000000000001,"15min":24.178000000000001},"last":4.6289999999999996}]}]},{"osd":3,"up_from":103,"seq":442381631604,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":533060,"kb_used_data":204476,"kb_used_omap":4,"kb_used_meta":328443,"kb_avail":93212092,"statfs":{"total":95995035648,"available":95449182208,"internally_reserved":0,"allocated":209383424,"data_stored":208642427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":336326238},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":3.8690000000000002,"5min":5.0350000000000001,"15min":2.3599999999999999},"min":{"1min":0.311,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":26.614999999999998,"5min":167.82400000000001,"15min":167.82400000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":5.6230000000000002,"5min":5.7380000000000004,"15min":2.6829999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":26.140000000000001,"5min":167.89599999999999,"15min":167.89599999999999},"last":3.278}]},{"osd":1,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":5.931,"5min":4.8170000000000002,"15min":2.0249999999999999},"min":{"1min":0.27400000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":48.052,"5min":83.921999999999997,"15min":83.921999999999997},"last":2.4550000000000001},{"interface":"front","average":{"1min":5.6310000000000002,"5min":4.7140000000000004,"15min":2.0009999999999999},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.247},"max":{"1min":13.551,"5min":83.707999999999998,"15min":83.707999999999998},"last":2.3639999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3920000000000003,"5min":7.3719999999999999,"15min":3.629},"min":{"1min":0.307,"5min":0.218,"15min":0.215},"max":{"1min":41.542000000000002,"5min":190.43299999999999,"15min":190.43299999999999},"last":0.746},{"interface":"front","average":{"1min":5.3739999999999997,"5min":8.4800000000000004,"15min":4.0250000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.26000000000000001},"max":{"1min":26.097999999999999,"5min":190.267,"15min":190.267},"last":3.6779999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0030000000000001,"5min":3.2879999999999998,"15min":1.966},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":25.946999999999999,"5min":51.591999999999999,"15min":51.591999999999999},"last":3.3540000000000001},{"interface":"front","average":{"1min":3.1299999999999999,"5min":2.2530000000000001,"15min":1.5269999999999999},"min":{"1min":0.33500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":26.774999999999999,"5min":26.774999999999999,"15min":45.735999999999997},"last":0.72099999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":4.5890000000000004,"5min":2.427,"15min":1.5880000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":13.114000000000001,"5min":13.114000000000001,"15min":47.948999999999998},"last":2.7850000000000001},{"interface":"front","average":{"1min":3.1699999999999999,"5min":2.2970000000000002,"15min":1.5960000000000001},"min":{"1min":0.499,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":26.728000000000002,"5min":26.728000000000002,"15min":47.968000000000004},"last":0.67900000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":2.387,"15min":1.504},"min":{"1min":0.50700000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":26.847999999999999,"5min":26.847999999999999,"15min":26.847999999999999},"last":0.78500000000000003},{"interface":"front","average":{"1min":4.8730000000000002,"5min":2.6619999999999999,"15min":1.4139999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":13.058,"5min":13.058,"15min":13.058},"last":1.8919999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:40:56 2023","interfaces":[{"interface":"back","average":{"1min":4.032,"5min":3.0249999999999999,"15min":2.5150000000000001},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":26.024999999999999,"5min":26.024999999999999,"15min":26.024999999999999},"last":3.3900000000000001},{"interface":"front","average":{"1min":4.2720000000000002,"5min":3.125,"15min":2.3500000000000001},"min":{"1min":0.42499999999999999,"5min":0.40899999999999997,"15min":0.35699999999999998},"max":{"1min":25.995999999999999,"5min":25.995999999999999,"15min":25.995999999999999},"last":8.6039999999999992}]}]},{"osd":4,"up_from":30,"seq":128849019151,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":409228,"kb_used_data":112004,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93335924,"statfs":{"total":95995035648,"available":95575986176,"internally_reserved":0,"allocated":114692096,"data_stored":114169446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:22 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":1.929,"15min":1.3180000000000001},"min":{"1min":0.248,"5min":0.214,"15min":0.214},"max":{"1min":6.2590000000000003,"5min":36.406999999999996,"15min":36.406999999999996},"last":100.504},{"interface":"front","average":{"1min":2.4460000000000002,"5min":2.556,"15min":1.5800000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.7629999999999999,"5min":36.454999999999998,"15min":36.454999999999998},"last":100.717}]},{"osd":1,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":1.6699999999999999,"15min":1.21},"min":{"1min":0.29999999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8140000000000001,"5min":7.9550000000000001,"15min":31.073},"last":110.899},{"interface":"front","average":{"1min":2.492,"5min":1.8520000000000001,"15min":1.3300000000000001},"min":{"1min":0.29299999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.349,"5min":12.349,"15min":31.100000000000001},"last":110.911}]},{"osd":2,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":6.4269999999999996,"15min":3.476},"min":{"1min":0.31,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":10.099,"5min":196.93700000000001,"15min":196.93700000000001},"last":100.148},{"interface":"front","average":{"1min":3.048,"5min":6.1829999999999998,"15min":3.3679999999999999},"min":{"1min":0.34399999999999997,"5min":0.28299999999999997,"15min":0.28100000000000003},"max":{"1min":11.534000000000001,"5min":197.08099999999999,"15min":197.08099999999999},"last":100.086}]},{"osd":3,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":4.3570000000000002,"5min":7.3419999999999996,"15min":3.097},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.509,"5min":211.48699999999999,"15min":211.48699999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":3.2109999999999999,"5min":7.157,"15min":2.9740000000000002},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.309},"max":{"1min":13.007,"5min":211.31100000000001,"15min":211.31100000000001},"last":0.35899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.0310000000000001,"15min":1.5649999999999999},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.249},"max":{"1min":3.2730000000000001,"5min":94.646000000000001,"15min":113.35899999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.883,"15min":1.5229999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.30599999999999999},"max":{"1min":3.3849999999999998,"5min":94.692999999999998,"15min":113.32299999999999},"last":0.39700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:40:57 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.72899999999999998,"15min":0.70499999999999996},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":1.8740000000000001,"5min":6.1779999999999999,"15min":11.944000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.93500000000000005,"15min":0.86399999999999999},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.4810000000000001,"5min":7.6500000000000004,"15min":10.356},"last":0.42799999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.94799999999999995,"15min":2.5619999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.312},"max":{"1min":3.415,"5min":11.845000000000001,"15min":30.393999999999998},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.97599999999999998,"15min":2.8780000000000001},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.5089999999999999,"5min":10.837,"15min":30.334},"last":0.38900000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822923,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":419016,"kb_used_data":109504,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93326136,"statfs":{"total":95995035648,"available":95565963264,"internally_reserved":0,"allocated":112132096,"data_stored":111616937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":4.4219999999999997,"15min":2.9369999999999998},"min":{"1min":0.22800000000000001,"5min":0.22700000000000001,"15min":0.19400000000000001},"max":{"1min":8.3079999999999998,"5min":165.90199999999999,"15min":165.90199999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.377,"5min":4.2290000000000001,"15min":2.9910000000000001},"min":{"1min":0.28399999999999997,"5min":0.215,"15min":0.215},"max":{"1min":5.4660000000000002,"5min":165.96700000000001,"15min":165.96700000000001},"last":0.36699999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":7.0419999999999998,"5min":4.6070000000000002,"15min":2.1699999999999999},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":71.209999999999994,"5min":87.769000000000005,"15min":87.769000000000005},"last":0.26700000000000002},{"interface":"front","average":{"1min":8.1159999999999997,"5min":5.2130000000000001,"15min":2.2810000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.24399999999999999},"max":{"1min":71.192999999999998,"5min":87.864000000000004,"15min":87.864000000000004},"last":0.29899999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:40:20 2023","interfaces":[{"interface":"back","average":{"1min":7.2709999999999999,"5min":6.7069999999999999,"15min":4.008},"min":{"1min":0.33600000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":73.311999999999998,"5min":118.255,"15min":290.697},"last":0.35299999999999998},{"interface":"front","average":{"1min":6.1639999999999997,"5min":6.0199999999999996,"15min":3.7810000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":73.280000000000001,"5min":118.438,"15min":290.786},"last":0.33200000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":7.5910000000000002,"5min":6.8520000000000003,"15min":2.8919999999999999},"min":{"1min":0.35199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":85.007999999999996,"5min":95.236999999999995,"15min":95.236999999999995},"last":36.395000000000003},{"interface":"front","average":{"1min":6.7199999999999998,"5min":6.2770000000000001,"15min":2.7189999999999999},"min":{"1min":0.32900000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":83.147999999999996,"5min":93.471000000000004,"15min":93.471000000000004},"last":36.406999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.81299999999999994,"15min":0.72099999999999997},"min":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":5.6459999999999999,"5min":7.2060000000000004,"15min":9.3510000000000009},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.1970000000000001,"15min":0.94399999999999995},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":4.8209999999999997,"5min":10.913,"15min":14.236000000000001},"last":0.38700000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:40:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1139999999999999,"5min":1.8120000000000001,"15min":1.3420000000000001},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.26000000000000001},"max":{"1min":27.146000000000001,"5min":27.146000000000001,"15min":95.010999999999996},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.1269999999999998,"5min":1.4690000000000001,"15min":1.2290000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27700000000000002},"max":{"1min":27.215,"5min":27.215,"15min":94.194000000000003},"last":0.42299999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:39:59 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.27,"15min":1.573},"min":{"1min":0.34399999999999997,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.94099999999999995,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.52100000000000002,"5min":1.3400000000000001,"15min":1.8620000000000001},"min":{"1min":0.35899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.5029999999999999,"5min":14.432,"15min":14.432},"last":0.46000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":49156096,"data_stored":49152024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":63705088,"data_stored":63701016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":144506880,"data_stored":144506880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":193855488,"data_stored":193855488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":106299392,"data_stored":106299392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":109776896,"data_stored":109772824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":152698880,"data_stored":152698880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":638779392,"data_stored":638779392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:41:07.829 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:07.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:07.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:07.947 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:07 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 5047/8790 objects degraded (57.418%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:07.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:07.994 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:08.050 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:08.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:08.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:08.158 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:08.183 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:08.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:07 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 5047/8790 objects degraded (57.418%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:08.240 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:08.247 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:07 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 5047/8790 objects degraded (57.418%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:08.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:08.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:08.349 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:08.454 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:08.600 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:08.666 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:08.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:08.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:08.713 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Removed 88360 objects 2023-10-09T05:41:08.713 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Clean up completed and total clean up time :151.82 2023-10-09T05:41:08.785 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:08.813 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:08.917 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:08.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:08.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:09.020 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:09.052 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:09.079 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:08 smithi008 ceph-mon[184418]: pgmap v586: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 128 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 4728/6006 objects degraded (78.721%); 0 B/s, 87 objects/s recovering 2023-10-09T05:41:09.080 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:08 smithi008 ceph-mon[191227]: pgmap v586: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 128 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 4728/6006 objects degraded (78.721%); 0 B/s, 87 objects/s recovering 2023-10-09T05:41:09.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:08 smithi191 ceph-mon[161669]: pgmap v586: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 128 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 4728/6006 objects degraded (78.721%); 0 B/s, 87 objects/s recovering 2023-10-09T05:41:09.328 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:41:09.605 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:09.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:09.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:09.708 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:09.734 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:09.778 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:09.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:09.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:09.885 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:09.913 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:10.232 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:10.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:10.335 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:10.366 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:10.526 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:10.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:10.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:10.565 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:10.599 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:41:10.599 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:41:10.599 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:41:10.599 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:41:10.599 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:41:10.631 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:10.664 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:10.704 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:41:10.778 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:41:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:41:10] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.33.4" 2023-10-09T05:41:10.935 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T05:41:10.935 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T05:41:11.327 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:11.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:11.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:11.430 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:11.455 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:11 smithi191 ceph-mon[161669]: pgmap v587: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 42 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 4728/1899 objects degraded (248.973%); 0 B/s, 84 objects/s recovering 2023-10-09T05:41:11.456 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:11.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:11 smithi008 ceph-mon[184418]: pgmap v587: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 42 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 4728/1899 objects degraded (248.973%); 0 B/s, 84 objects/s recovering 2023-10-09T05:41:11.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:11 smithi008 ceph-mon[191227]: pgmap v587: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 42 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 4728/1899 objects degraded (248.973%); 0 B/s, 84 objects/s recovering 2023-10-09T05:41:11.499 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:11.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:11.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:11.604 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:11.630 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:11.676 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:11.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:11.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:11.782 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:11.810 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:11.855 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:11.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:11.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:11.987 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:12.032 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:12.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:12.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:12.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:12.170 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:12.215 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:41:12.218 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:12.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:12.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:12.326 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:12.363 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:12.397 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:41:12.426 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:12.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:12.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:12.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1484515384' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:12.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:12 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:12.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1484515384' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:12.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:12 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:12.530 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:12.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1484515384' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:12.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:12 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:12.558 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:12.602 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:12.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:12.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:12.707 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:12.734 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:12.779 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:12.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:12.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:12.884 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:12.910 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:12.954 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:12.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:12.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:13.060 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:13.087 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:13.131 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:13.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:13.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:13.239 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:13.265 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:13.309 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:13.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:13.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:13.416 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:13.445 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:13.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[184418]: pgmap v588: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.2 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 3888/162 objects degraded (2400.000%); 0 B/s, 142 objects/s recovering 2023-10-09T05:41:13.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:41:13.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[184418]: failed to encode map e206 with expected crc 2023-10-09T05:41:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[184418]: failed to encode map e206 with expected crc 2023-10-09T05:41:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[184418]: failed to encode map e206 with expected crc 2023-10-09T05:41:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[184418]: osdmap e206: 8 total, 8 up, 7 in 2023-10-09T05:41:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 3888/162 objects degraded (2400.000%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:13.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[191227]: pgmap v588: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.2 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 3888/162 objects degraded (2400.000%); 0 B/s, 142 objects/s recovering 2023-10-09T05:41:13.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:41:13.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[191227]: failed to encode map e206 with expected crc 2023-10-09T05:41:13.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[191227]: failed to encode map e206 with expected crc 2023-10-09T05:41:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[191227]: failed to encode map e206 with expected crc 2023-10-09T05:41:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[191227]: osdmap e206: 8 total, 8 up, 7 in 2023-10-09T05:41:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 3888/162 objects degraded (2400.000%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:13 smithi191 ceph-mon[161669]: pgmap v588: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 6.2 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 3888/162 objects degraded (2400.000%); 0 B/s, 142 objects/s recovering 2023-10-09T05:41:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:13 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:41:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:13 smithi191 ceph-mon[161669]: failed to encode map e206 with expected crc 2023-10-09T05:41:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:13 smithi191 ceph-mon[161669]: failed to encode map e206 with expected crc 2023-10-09T05:41:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:13 smithi191 ceph-mon[161669]: failed to encode map e206 with expected crc 2023-10-09T05:41:13.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:13 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:13.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:13 smithi191 ceph-mon[161669]: osdmap e206: 8 total, 8 up, 7 in 2023-10-09T05:41:13.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 3888/162 objects degraded (2400.000%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:14.522 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:14.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:14.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:14.594 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:14.635 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:14.646 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:14.675 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:14.716 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:14.727 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:14.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:14.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:14.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:14.857 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:14.901 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:14.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:14.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:15.007 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:15.038 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:15.089 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:41:15.109 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:41:15.129 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:41:15.150 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:41:15.168 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:41:15.189 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:41:15.211 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:41:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:14 smithi008 ceph-mon[184418]: pgmap v590: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 3888/84 objects degraded (4628.571%); 0 B/s, 127 objects/s recovering 2023-10-09T05:41:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:14 smithi008 ceph-mon[191227]: pgmap v590: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 3888/84 objects degraded (4628.571%); 0 B/s, 127 objects/s recovering 2023-10-09T05:41:15.232 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:41:15.234 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:41:15.234 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:41:15.234 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:41:15.234 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:41:15.234 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:41:15.234 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:41:15.234 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:41:15.235 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:41:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:14 smithi191 ceph-mon[161669]: pgmap v590: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 3888/84 objects degraded (4628.571%); 0 B/s, 127 objects/s recovering 2023-10-09T05:41:16.214 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3140420321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:16.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3140420321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:16.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:16.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:16.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3140420321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:16.317 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:16.349 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:16.549 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:41:16.892 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:16 smithi008 ceph-mon[191227]: failed to encode map e207 with expected crc 2023-10-09T05:41:16.892 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:16 smithi008 ceph-mon[191227]: failed to encode map e207 with expected crc 2023-10-09T05:41:16.893 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:16 smithi008 ceph-mon[191227]: osdmap e207: 8 total, 8 up, 7 in 2023-10-09T05:41:16.893 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:16 smithi008 ceph-mon[191227]: failed to encode map e207 with expected crc 2023-10-09T05:41:16.893 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:16 smithi008 ceph-mon[191227]: pgmap v592: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 3772/84 objects degraded (4490.476%); 0 B/s, 119 objects/s recovering 2023-10-09T05:41:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:16 smithi008 ceph-mon[184418]: failed to encode map e207 with expected crc 2023-10-09T05:41:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:16 smithi008 ceph-mon[184418]: failed to encode map e207 with expected crc 2023-10-09T05:41:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:16 smithi008 ceph-mon[184418]: osdmap e207: 8 total, 8 up, 7 in 2023-10-09T05:41:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:16 smithi008 ceph-mon[184418]: failed to encode map e207 with expected crc 2023-10-09T05:41:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:16 smithi008 ceph-mon[184418]: pgmap v592: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 3772/84 objects degraded (4490.476%); 0 B/s, 119 objects/s recovering 2023-10-09T05:41:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:16 smithi191 ceph-mon[161669]: failed to encode map e207 with expected crc 2023-10-09T05:41:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:16 smithi191 ceph-mon[161669]: failed to encode map e207 with expected crc 2023-10-09T05:41:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:16 smithi191 ceph-mon[161669]: osdmap e207: 8 total, 8 up, 7 in 2023-10-09T05:41:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:16 smithi191 ceph-mon[161669]: failed to encode map e207 with expected crc 2023-10-09T05:41:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:16 smithi191 ceph-mon[161669]: pgmap v592: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 3772/84 objects degraded (4490.476%); 0 B/s, 119 objects/s recovering 2023-10-09T05:41:17.699 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:17.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:17.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:17.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:17.831 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:17.873 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:17.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:17.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:17.959 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[184418]: failed to encode map e207 with expected crc 2023-10-09T05:41:17.959 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[184418]: failed to encode map e208 with expected crc 2023-10-09T05:41:17.959 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[184418]: osdmap e208: 8 total, 8 up, 7 in 2023-10-09T05:41:17.959 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[184418]: failed to encode map e208 with expected crc 2023-10-09T05:41:17.959 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[184418]: failed to encode map e208 with expected crc 2023-10-09T05:41:17.959 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 3772/84 objects degraded (4490.476%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:17.959 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[191227]: failed to encode map e207 with expected crc 2023-10-09T05:41:17.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[191227]: failed to encode map e208 with expected crc 2023-10-09T05:41:17.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[191227]: osdmap e208: 8 total, 8 up, 7 in 2023-10-09T05:41:17.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[191227]: failed to encode map e208 with expected crc 2023-10-09T05:41:17.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[191227]: failed to encode map e208 with expected crc 2023-10-09T05:41:17.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:17 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 3772/84 objects degraded (4490.476%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:17.978 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:18.003 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:18.045 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:18.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:18.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:18.150 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:18.179 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:18.278 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:18.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:18.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:18.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:17 smithi191 ceph-mon[161669]: failed to encode map e207 with expected crc 2023-10-09T05:41:18.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:17 smithi191 ceph-mon[161669]: failed to encode map e208 with expected crc 2023-10-09T05:41:18.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:17 smithi191 ceph-mon[161669]: osdmap e208: 8 total, 8 up, 7 in 2023-10-09T05:41:18.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:17 smithi191 ceph-mon[161669]: failed to encode map e208 with expected crc 2023-10-09T05:41:18.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:17 smithi191 ceph-mon[161669]: failed to encode map e208 with expected crc 2023-10-09T05:41:18.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:17 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 3772/84 objects degraded (4490.476%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:41:18.382 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:18.418 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:18.567 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:18.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:18.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:18.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:18.695 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:18.736 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:18.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:18.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:18.837 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:41:18.840 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:41:18.841 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:18.875 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:18 smithi191 ceph-mon[161669]: failed to encode map e208 with expected crc 2023-10-09T05:41:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:18 smithi191 ceph-mon[161669]: pgmap v594: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 2381/84 objects degraded (2834.524%); 0 B/s, 251 objects/s recovering 2023-10-09T05:41:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:18 smithi191 ceph-mon[161669]: from='client.35331 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:19.083 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:19.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:19.138 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:41:19.138 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:41:19.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:18 smithi008 ceph-mon[184418]: failed to encode map e208 with expected crc 2023-10-09T05:41:19.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:18 smithi008 ceph-mon[184418]: pgmap v594: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 2381/84 objects degraded (2834.524%); 0 B/s, 251 objects/s recovering 2023-10-09T05:41:19.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:18 smithi008 ceph-mon[184418]: from='client.35331 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:19.162 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:18 smithi008 ceph-mon[191227]: failed to encode map e208 with expected crc 2023-10-09T05:41:19.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:18 smithi008 ceph-mon[191227]: pgmap v594: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 2381/84 objects degraded (2834.524%); 0 B/s, 251 objects/s recovering 2023-10-09T05:41:19.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:18 smithi008 ceph-mon[191227]: from='client.35331 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:19.186 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:19.220 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:19.643 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:19.710 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:41:19.942 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_1' removed 2023-10-09T05:41:19.951 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:19.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:19.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:20.061 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:20.088 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:20.090 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:19 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1371608972' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:41:20.090 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:41:20.090 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:19 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1371608972' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:41:20.090 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:41:20.118 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:41:20.129 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:20.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:20.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:20.235 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:20.263 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:19 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1371608972' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:41:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:41:20.468 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:20.650 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:41:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:41:20] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.33.4" 2023-10-09T05:41:20.908 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[184418]: failed to encode map e209 with expected crc 2023-10-09T05:41:20.908 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[184418]: failed to encode map e209 with expected crc 2023-10-09T05:41:20.908 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:41:20.908 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[184418]: osdmap e209: 8 total, 8 up, 7 in 2023-10-09T05:41:20.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[184418]: failed to encode map e209 with expected crc 2023-10-09T05:41:20.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[184418]: pgmap v596: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:41:20.909 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[191227]: failed to encode map e209 with expected crc 2023-10-09T05:41:20.909 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[191227]: failed to encode map e209 with expected crc 2023-10-09T05:41:20.909 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:41:20.909 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[191227]: osdmap e209: 8 total, 8 up, 7 in 2023-10-09T05:41:20.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[191227]: failed to encode map e209 with expected crc 2023-10-09T05:41:20.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:20 smithi008 ceph-mon[191227]: pgmap v596: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:41:20.993 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":594,"stamp":"2023-10-09T05:41:18.154462+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2381,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":177828,"num_write_kb":5670182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15160,"num_bytes_recovered":531043702,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130992,"ondisk_log_size":130992,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2269884,"kb_used_data":24132,"kb_used_omap":31,"kb_used_meta":2245664,"kb_avail":653946180,"statfs":{"total":671965249536,"available":669640888320,"internally_reserved":0,"allocated":24711168,"data_stored":20067629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2299560393},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1703936,"num_objects":-26,"num_object_clones":0,"num_object_copies":-78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1507,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1507,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001583"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":189,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823665+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:40:59.823665+0000","last_peered":"2023-10-09T05:40:59.823665+0000","last_clean":"2023-10-09T05:40:59.823665+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:40:59.823665+0000","last_undegraded":"2023-10-09T05:40:59.823665+0000","last_fullsized":"2023-10-09T05:40:59.823665+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":179,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.557360+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:41:13.557360+0000","last_peered":"2023-10-09T05:41:13.557360+0000","last_clean":"2023-10-09T05:41:13.557360+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:41:13.557360+0000","last_undegraded":"2023-10-09T05:41:13.557360+0000","last_fullsized":"2023-10-09T05:41:13.557360+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":153,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.842660+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:40:59.842660+0000","last_peered":"2023-10-09T05:40:59.842660+0000","last_clean":"2023-10-09T05:40:59.842660+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:40:59.842660+0000","last_undegraded":"2023-10-09T05:40:59.842660+0000","last_fullsized":"2023-10-09T05:40:59.842660+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":182,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694748+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:40:56.694748+0000","last_peered":"2023-10-09T05:40:56.694748+0000","last_clean":"2023-10-09T05:40:56.694748+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:40:56.694748+0000","last_undegraded":"2023-10-09T05:40:56.694748+0000","last_fullsized":"2023-10-09T05:40:56.694748+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":196,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.533383+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:41:13.533383+0000","last_peered":"2023-10-09T05:41:13.533383+0000","last_clean":"2023-10-09T05:41:13.533383+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:41:13.533383+0000","last_undegraded":"2023-10-09T05:41:13.533383+0000","last_fullsized":"2023-10-09T05:41:13.533383+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":172,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.833459+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:40:59.833459+0000","last_peered":"2023-10-09T05:40:59.833459+0000","last_clean":"2023-10-09T05:40:59.833459+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:40:59.833459+0000","last_undegraded":"2023-10-09T05:40:59.833459+0000","last_fullsized":"2023-10-09T05:40:59.833459+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":157,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.695098+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:40:56.695098+0000","last_peered":"2023-10-09T05:40:56.695098+0000","last_clean":"2023-10-09T05:40:56.695098+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:40:56.695098+0000","last_undegraded":"2023-10-09T05:40:56.695098+0000","last_fullsized":"2023-10-09T05:40:56.695098+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":180,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.533467+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:41:13.533467+0000","last_peered":"2023-10-09T05:41:13.533467+0000","last_clean":"2023-10-09T05:41:13.533467+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:41:13.533467+0000","last_undegraded":"2023-10-09T05:41:13.533467+0000","last_fullsized":"2023-10-09T05:41:13.533467+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":197,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.832754+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:40:59.832754+0000","last_peered":"2023-10-09T05:40:59.832754+0000","last_clean":"2023-10-09T05:40:59.832754+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:40:59.832754+0000","last_undegraded":"2023-10-09T05:40:59.832754+0000","last_fullsized":"2023-10-09T05:40:59.832754+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":194,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.533821+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:41:13.533821+0000","last_peered":"2023-10-09T05:41:13.533821+0000","last_clean":"2023-10-09T05:41:13.533821+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:41:13.533821+0000","last_undegraded":"2023-10-09T05:41:13.533821+0000","last_fullsized":"2023-10-09T05:41:13.533821+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":87,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.557327+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:41:13.557327+0000","last_peered":"2023-10-09T05:41:13.557327+0000","last_clean":"2023-10-09T05:41:13.557327+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:41:13.557327+0000","last_undegraded":"2023-10-09T05:41:13.557327+0000","last_fullsized":"2023-10-09T05:41:13.557327+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":153,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.842502+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:40:59.842502+0000","last_peered":"2023-10-09T05:40:59.842502+0000","last_clean":"2023-10-09T05:40:59.842502+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:40:59.842502+0000","last_undegraded":"2023-10-09T05:40:59.842502+0000","last_fullsized":"2023-10-09T05:40:59.842502+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":214,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823744+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:40:59.823744+0000","last_peered":"2023-10-09T05:40:59.823744+0000","last_clean":"2023-10-09T05:40:59.823744+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:40:59.823744+0000","last_undegraded":"2023-10-09T05:40:59.823744+0000","last_fullsized":"2023-10-09T05:40:59.823744+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":172,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823648+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:40:59.823648+0000","last_peered":"2023-10-09T05:40:59.823648+0000","last_clean":"2023-10-09T05:40:59.823648+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:40:59.823648+0000","last_undegraded":"2023-10-09T05:40:59.823648+0000","last_fullsized":"2023-10-09T05:40:59.823648+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":85,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823744+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:40:59.823744+0000","last_peered":"2023-10-09T05:40:59.823744+0000","last_clean":"2023-10-09T05:40:59.823744+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:40:59.823744+0000","last_undegraded":"2023-10-09T05:40:59.823744+0000","last_fullsized":"2023-10-09T05:40:59.823744+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":204,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.534139+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:41:13.534139+0000","last_peered":"2023-10-09T05:41:13.534139+0000","last_clean":"2023-10-09T05:41:13.534139+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:41:13.534139+0000","last_undegraded":"2023-10-09T05:41:13.534139+0000","last_fullsized":"2023-10-09T05:41:13.534139+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":181,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694622+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:40:56.694622+0000","last_peered":"2023-10-09T05:40:56.694622+0000","last_clean":"2023-10-09T05:40:56.694622+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:40:56.694622+0000","last_undegraded":"2023-10-09T05:40:56.694622+0000","last_fullsized":"2023-10-09T05:40:56.694622+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":196,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.832889+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:40:59.832889+0000","last_peered":"2023-10-09T05:40:59.832889+0000","last_clean":"2023-10-09T05:40:59.832889+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:40:59.832889+0000","last_undegraded":"2023-10-09T05:40:59.832889+0000","last_fullsized":"2023-10-09T05:40:59.832889+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":103,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823665+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:40:59.823665+0000","last_peered":"2023-10-09T05:40:59.823665+0000","last_clean":"2023-10-09T05:40:59.823665+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:40:59.823665+0000","last_undegraded":"2023-10-09T05:40:59.823665+0000","last_fullsized":"2023-10-09T05:40:59.823665+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":181,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694650+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:40:56.694650+0000","last_peered":"2023-10-09T05:40:56.694650+0000","last_clean":"2023-10-09T05:40:56.694650+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:40:56.694650+0000","last_undegraded":"2023-10-09T05:40:56.694650+0000","last_fullsized":"2023-10-09T05:40:56.694650+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":181,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.832467+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:40:59.832467+0000","last_peered":"2023-10-09T05:40:59.832467+0000","last_clean":"2023-10-09T05:40:59.832467+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:40:59.832467+0000","last_undegraded":"2023-10-09T05:40:59.832467+0000","last_fullsized":"2023-10-09T05:40:59.832467+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":204,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.557694+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:41:13.557694+0000","last_peered":"2023-10-09T05:41:13.557694+0000","last_clean":"2023-10-09T05:41:13.557694+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:41:13.557694+0000","last_undegraded":"2023-10-09T05:41:13.557694+0000","last_fullsized":"2023-10-09T05:41:13.557694+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"205'11056","reported_seq":11108,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:08.711652+0000","last_change":"2023-10-09T05:37:01.636984+0000","last_active":"2023-10-09T05:41:08.711652+0000","last_peered":"2023-10-09T05:41:08.711652+0000","last_clean":"2023-10-09T05:41:08.711652+0000","last_became_active":"2023-10-09T05:37:01.636609+0000","last_became_peered":"2023-10-09T05:37:01.636609+0000","last_unstale":"2023-10-09T05:41:08.711652+0000","last_undegraded":"2023-10-09T05:41:08.711652+0000","last_fullsized":"2023-10-09T05:41:08.711652+0000","mapping_epoch":162,"log_start":"166'1000","ondisk_log_start":"166'1000","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":10056,"log_dups_size":0,"ondisk_log_size":10056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11056,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":83,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.824101+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:40:59.824101+0000","last_peered":"2023-10-09T05:40:59.824101+0000","last_clean":"2023-10-09T05:40:59.824101+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:40:59.824101+0000","last_undegraded":"2023-10-09T05:40:59.824101+0000","last_fullsized":"2023-10-09T05:40:59.824101+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":181,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.832208+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:40:59.832208+0000","last_peered":"2023-10-09T05:40:59.832208+0000","last_clean":"2023-10-09T05:40:59.832208+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:40:59.832208+0000","last_undegraded":"2023-10-09T05:40:59.832208+0000","last_fullsized":"2023-10-09T05:40:59.832208+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"205'10992","reported_seq":16838,"reported_epoch":207,"state":"peering","last_fresh":"2023-10-09T05:41:15.893679+0000","last_change":"2023-10-09T05:41:15.893778+0000","last_active":"2023-10-09T05:41:15.893679+0000","last_peered":"2023-10-09T05:41:15.466522+0000","last_clean":"2023-10-09T05:37:34.651883+0000","last_became_active":"2023-10-09T05:37:36.832502+0000","last_became_peered":"2023-10-09T05:37:36.832502+0000","last_unstale":"2023-10-09T05:41:15.893679+0000","last_undegraded":"2023-10-09T05:41:15.893679+0000","last_fullsized":"2023-10-09T05:41:15.893679+0000","mapping_epoch":207,"log_start":"181'4600","ondisk_log_start":"181'4600","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6392,"log_dups_size":0,"ondisk_log_size":6392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:30:03.336304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10992,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2890,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,6,7],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":83,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823996+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:40:59.823996+0000","last_peered":"2023-10-09T05:40:59.823996+0000","last_clean":"2023-10-09T05:40:59.823996+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:40:59.823996+0000","last_undegraded":"2023-10-09T05:40:59.823996+0000","last_fullsized":"2023-10-09T05:40:59.823996+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":187,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823531+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:40:59.823531+0000","last_peered":"2023-10-09T05:40:59.823531+0000","last_clean":"2023-10-09T05:40:59.823531+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:40:59.823531+0000","last_undegraded":"2023-10-09T05:40:59.823531+0000","last_fullsized":"2023-10-09T05:40:59.823531+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"205'11212","reported_seq":17327,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.533257+0000","last_change":"2023-10-09T05:38:55.317808+0000","last_active":"2023-10-09T05:41:13.533257+0000","last_peered":"2023-10-09T05:41:13.533257+0000","last_clean":"2023-10-09T05:41:13.533257+0000","last_became_active":"2023-10-09T05:38:55.302739+0000","last_became_peered":"2023-10-09T05:38:55.302739+0000","last_unstale":"2023-10-09T05:41:13.533257+0000","last_undegraded":"2023-10-09T05:41:13.533257+0000","last_fullsized":"2023-10-09T05:41:13.533257+0000","mapping_epoch":186,"log_start":"167'2100","ondisk_log_start":"167'2100","created":162,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9112,"log_dups_size":0,"ondisk_log_size":9112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:25.379818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11212,"num_write_kb":358784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3022,"num_bytes_recovered":198049792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":83,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.832991+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:40:59.832991+0000","last_peered":"2023-10-09T05:40:59.832991+0000","last_clean":"2023-10-09T05:40:59.832991+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:40:59.832991+0000","last_undegraded":"2023-10-09T05:40:59.832991+0000","last_fullsized":"2023-10-09T05:40:59.832991+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":205,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.832809+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:40:59.832809+0000","last_peered":"2023-10-09T05:40:59.832809+0000","last_clean":"2023-10-09T05:40:59.832809+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:40:59.832809+0000","last_undegraded":"2023-10-09T05:40:59.832809+0000","last_fullsized":"2023-10-09T05:40:59.832809+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"205'10970","reported_seq":16841,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:08.671219+0000","last_change":"2023-10-09T05:40:04.830679+0000","last_active":"2023-10-09T05:41:08.671219+0000","last_peered":"2023-10-09T05:41:08.671219+0000","last_clean":"2023-10-09T05:41:08.671219+0000","last_became_active":"2023-10-09T05:40:04.807741+0000","last_became_peered":"2023-10-09T05:40:04.807741+0000","last_unstale":"2023-10-09T05:41:08.671219+0000","last_undegraded":"2023-10-09T05:41:08.671219+0000","last_fullsized":"2023-10-09T05:41:08.671219+0000","mapping_epoch":195,"log_start":"167'2100","ondisk_log_start":"167'2100","created":162,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8870,"log_dups_size":0,"ondisk_log_size":8870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:51.289034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10970,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2899,"num_bytes_recovered":139919360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":173,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.833704+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:40:59.833704+0000","last_peered":"2023-10-09T05:40:59.833704+0000","last_clean":"2023-10-09T05:40:59.833704+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:40:59.833704+0000","last_undegraded":"2023-10-09T05:40:59.833704+0000","last_fullsized":"2023-10-09T05:40:59.833704+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":158,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.843345+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:40:59.843345+0000","last_peered":"2023-10-09T05:40:59.843345+0000","last_clean":"2023-10-09T05:40:59.843345+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:40:59.843345+0000","last_undegraded":"2023-10-09T05:40:59.843345+0000","last_fullsized":"2023-10-09T05:40:59.843345+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"205'10986","reported_seq":11038,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:08.689671+0000","last_change":"2023-10-09T05:37:01.439878+0000","last_active":"2023-10-09T05:41:08.689671+0000","last_peered":"2023-10-09T05:41:08.689671+0000","last_clean":"2023-10-09T05:41:08.689671+0000","last_became_active":"2023-10-09T05:37:01.439481+0000","last_became_peered":"2023-10-09T05:37:01.439481+0000","last_unstale":"2023-10-09T05:41:08.689671+0000","last_undegraded":"2023-10-09T05:41:08.689671+0000","last_fullsized":"2023-10-09T05:41:08.689671+0000","mapping_epoch":162,"log_start":"175'3400","ondisk_log_start":"175'3400","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7586,"log_dups_size":0,"ondisk_log_size":7586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10986,"num_write_kb":351552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":178,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.534031+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:41:13.534031+0000","last_peered":"2023-10-09T05:41:13.534031+0000","last_clean":"2023-10-09T05:41:13.534031+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:41:13.534031+0000","last_undegraded":"2023-10-09T05:41:13.534031+0000","last_fullsized":"2023-10-09T05:41:13.534031+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":172,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823729+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:40:59.823729+0000","last_peered":"2023-10-09T05:40:59.823729+0000","last_clean":"2023-10-09T05:40:59.823729+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:40:59.823729+0000","last_undegraded":"2023-10-09T05:40:59.823729+0000","last_fullsized":"2023-10-09T05:40:59.823729+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"205'11054","reported_seq":11109,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.534374+0000","last_change":"2023-10-09T05:37:01.440175+0000","last_active":"2023-10-09T05:41:13.534374+0000","last_peered":"2023-10-09T05:41:13.534374+0000","last_clean":"2023-10-09T05:41:13.534374+0000","last_became_active":"2023-10-09T05:37:01.439846+0000","last_became_peered":"2023-10-09T05:37:01.439846+0000","last_unstale":"2023-10-09T05:41:13.534374+0000","last_undegraded":"2023-10-09T05:41:13.534374+0000","last_fullsized":"2023-10-09T05:41:13.534374+0000","mapping_epoch":162,"log_start":"167'1900","ondisk_log_start":"167'1900","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":9154,"log_dups_size":0,"ondisk_log_size":9154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:27.883265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11054,"num_write_kb":353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":173,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.832694+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:40:59.832694+0000","last_peered":"2023-10-09T05:40:59.832694+0000","last_clean":"2023-10-09T05:40:59.832694+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:40:59.832694+0000","last_undegraded":"2023-10-09T05:40:59.832694+0000","last_fullsized":"2023-10-09T05:40:59.832694+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":187,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823772+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:40:59.823772+0000","last_peered":"2023-10-09T05:40:59.823772+0000","last_clean":"2023-10-09T05:40:59.823772+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:40:59.823772+0000","last_undegraded":"2023-10-09T05:40:59.823772+0000","last_fullsized":"2023-10-09T05:40:59.823772+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"205'11114","reported_seq":11169,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.534288+0000","last_change":"2023-10-09T05:37:01.638178+0000","last_active":"2023-10-09T05:41:13.534288+0000","last_peered":"2023-10-09T05:41:13.534288+0000","last_clean":"2023-10-09T05:41:13.534288+0000","last_became_active":"2023-10-09T05:37:01.637822+0000","last_became_peered":"2023-10-09T05:37:01.637822+0000","last_unstale":"2023-10-09T05:41:13.534288+0000","last_undegraded":"2023-10-09T05:41:13.534288+0000","last_fullsized":"2023-10-09T05:41:13.534288+0000","mapping_epoch":162,"log_start":"181'4700","ondisk_log_start":"181'4700","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6414,"log_dups_size":0,"ondisk_log_size":6414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:13.240069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11114,"num_write_kb":355648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":197,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823407+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:40:59.823407+0000","last_peered":"2023-10-09T05:40:59.823407+0000","last_clean":"2023-10-09T05:40:59.823407+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:40:59.823407+0000","last_undegraded":"2023-10-09T05:40:59.823407+0000","last_fullsized":"2023-10-09T05:40:59.823407+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":181,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.824021+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:40:59.824021+0000","last_peered":"2023-10-09T05:40:59.824021+0000","last_clean":"2023-10-09T05:40:59.824021+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:40:59.824021+0000","last_undegraded":"2023-10-09T05:40:59.824021+0000","last_fullsized":"2023-10-09T05:40:59.824021+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"205'11062","reported_seq":11115,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:08.720421+0000","last_change":"2023-10-09T05:37:01.438917+0000","last_active":"2023-10-09T05:41:08.720421+0000","last_peered":"2023-10-09T05:41:08.720421+0000","last_clean":"2023-10-09T05:41:08.720421+0000","last_became_active":"2023-10-09T05:37:01.438658+0000","last_became_peered":"2023-10-09T05:37:01.438658+0000","last_unstale":"2023-10-09T05:41:08.720421+0000","last_undegraded":"2023-10-09T05:41:08.720421+0000","last_fullsized":"2023-10-09T05:41:08.720421+0000","mapping_epoch":162,"log_start":"166'1000","ondisk_log_start":"166'1000","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":10062,"log_dups_size":0,"ondisk_log_size":10062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11062,"num_write_kb":353921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":205,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.533300+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:41:13.533300+0000","last_peered":"2023-10-09T05:41:13.533300+0000","last_clean":"2023-10-09T05:41:13.533300+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:41:13.533300+0000","last_undegraded":"2023-10-09T05:41:13.533300+0000","last_fullsized":"2023-10-09T05:41:13.533300+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":153,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.842919+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:40:59.842919+0000","last_peered":"2023-10-09T05:40:59.842919+0000","last_clean":"2023-10-09T05:40:59.842919+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:40:59.842919+0000","last_undegraded":"2023-10-09T05:40:59.842919+0000","last_fullsized":"2023-10-09T05:40:59.842919+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"205'11106","reported_seq":11160,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:08.705282+0000","last_change":"2023-10-09T05:37:01.440093+0000","last_active":"2023-10-09T05:41:08.705282+0000","last_peered":"2023-10-09T05:41:08.705282+0000","last_clean":"2023-10-09T05:41:08.705282+0000","last_became_active":"2023-10-09T05:37:01.439799+0000","last_became_peered":"2023-10-09T05:37:01.439799+0000","last_unstale":"2023-10-09T05:41:08.705282+0000","last_undegraded":"2023-10-09T05:41:08.705282+0000","last_fullsized":"2023-10-09T05:41:08.705282+0000","mapping_epoch":162,"log_start":"167'2200","ondisk_log_start":"167'2200","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8906,"log_dups_size":0,"ondisk_log_size":8906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:28.734471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11106,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":776,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.843369+0000","last_change":"2023-10-09T05:40:04.806908+0000","last_active":"2023-10-09T05:40:59.843369+0000","last_peered":"2023-10-09T05:40:59.843369+0000","last_clean":"2023-10-09T05:40:59.843369+0000","last_became_active":"2023-10-09T05:40:04.806232+0000","last_became_peered":"2023-10-09T05:40:04.806232+0000","last_unstale":"2023-10-09T05:40:59.843369+0000","last_undegraded":"2023-10-09T05:40:59.843369+0000","last_fullsized":"2023-10-09T05:40:59.843369+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:34.087553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":188,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.832783+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:40:59.832783+0000","last_peered":"2023-10-09T05:40:59.832783+0000","last_clean":"2023-10-09T05:40:59.832783+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:40:59.832783+0000","last_undegraded":"2023-10-09T05:40:59.832783+0000","last_fullsized":"2023-10-09T05:40:59.832783+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":139,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.842431+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:40:59.842431+0000","last_peered":"2023-10-09T05:40:59.842431+0000","last_clean":"2023-10-09T05:40:59.842431+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:40:59.842431+0000","last_undegraded":"2023-10-09T05:40:59.842431+0000","last_fullsized":"2023-10-09T05:40:59.842431+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"205'11148","reported_seq":17093,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.557504+0000","last_change":"2023-10-09T05:38:10.132330+0000","last_active":"2023-10-09T05:41:13.557504+0000","last_peered":"2023-10-09T05:41:13.557504+0000","last_clean":"2023-10-09T05:41:13.557504+0000","last_became_active":"2023-10-09T05:38:10.127080+0000","last_became_peered":"2023-10-09T05:38:10.127080+0000","last_unstale":"2023-10-09T05:41:13.557504+0000","last_undegraded":"2023-10-09T05:41:13.557504+0000","last_fullsized":"2023-10-09T05:41:13.557504+0000","mapping_epoch":179,"log_start":"181'4700","ondisk_log_start":"181'4700","created":162,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6448,"log_dups_size":0,"ondisk_log_size":6448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:12.659227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11148,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":192413696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":642,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.832948+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:40:59.832948+0000","last_peered":"2023-10-09T05:40:59.832948+0000","last_clean":"2023-10-09T05:40:59.832948+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:40:59.832948+0000","last_undegraded":"2023-10-09T05:40:59.832948+0000","last_fullsized":"2023-10-09T05:40:59.832948+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":173,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823743+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:40:59.823743+0000","last_peered":"2023-10-09T05:40:59.823743+0000","last_clean":"2023-10-09T05:40:59.823743+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:40:59.823743+0000","last_undegraded":"2023-10-09T05:40:59.823743+0000","last_fullsized":"2023-10-09T05:40:59.823743+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":185,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.833755+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:40:59.833755+0000","last_peered":"2023-10-09T05:40:59.833755+0000","last_clean":"2023-10-09T05:40:59.833755+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:40:59.833755+0000","last_undegraded":"2023-10-09T05:40:59.833755+0000","last_fullsized":"2023-10-09T05:40:59.833755+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"205'10990","reported_seq":11998,"reported_epoch":206,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:41:15.784721+0000","last_change":"2023-10-09T05:40:54.908423+0000","last_active":"2023-10-09T05:41:15.784721+0000","last_peered":"2023-10-09T05:41:15.784721+0000","last_clean":"2023-10-09T05:37:34.677100+0000","last_became_active":"2023-10-09T05:37:40.593033+0000","last_became_peered":"2023-10-09T05:37:40.593033+0000","last_unstale":"2023-10-09T05:41:15.784721+0000","last_undegraded":"2023-10-09T05:37:40.546647+0000","last_fullsized":"2023-10-09T05:37:40.543498+0000","mapping_epoch":170,"log_start":"168'2800","ondisk_log_start":"168'2800","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8190,"log_dups_size":0,"ondisk_log_size":8190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2378,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10990,"num_write_kb":351680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":473,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":-2378}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":521,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.534114+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:41:13.534114+0000","last_peered":"2023-10-09T05:41:13.534114+0000","last_clean":"2023-10-09T05:41:13.534114+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:41:13.534114+0000","last_undegraded":"2023-10-09T05:41:13.534114+0000","last_fullsized":"2023-10-09T05:41:13.534114+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":154,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.842334+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:40:59.842334+0000","last_peered":"2023-10-09T05:40:59.842334+0000","last_clean":"2023-10-09T05:40:59.842334+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:40:59.842334+0000","last_undegraded":"2023-10-09T05:40:59.842334+0000","last_fullsized":"2023-10-09T05:40:59.842334+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":183,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.842958+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:40:59.842958+0000","last_peered":"2023-10-09T05:40:59.842958+0000","last_clean":"2023-10-09T05:40:59.842958+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:40:59.842958+0000","last_undegraded":"2023-10-09T05:40:59.842958+0000","last_fullsized":"2023-10-09T05:40:59.842958+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"205'11074","reported_seq":11126,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:08.673909+0000","last_change":"2023-10-09T05:37:01.438236+0000","last_active":"2023-10-09T05:41:08.673909+0000","last_peered":"2023-10-09T05:41:08.673909+0000","last_clean":"2023-10-09T05:41:08.673909+0000","last_became_active":"2023-10-09T05:37:01.437917+0000","last_became_peered":"2023-10-09T05:37:01.437917+0000","last_unstale":"2023-10-09T05:41:08.673909+0000","last_undegraded":"2023-10-09T05:41:08.673909+0000","last_fullsized":"2023-10-09T05:41:08.673909+0000","mapping_epoch":162,"log_start":"166'1000","ondisk_log_start":"166'1000","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":10074,"log_dups_size":0,"ondisk_log_size":10074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11074,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":817,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.833020+0000","last_change":"2023-10-09T05:40:04.805258+0000","last_active":"2023-10-09T05:40:59.833020+0000","last_peered":"2023-10-09T05:40:59.833020+0000","last_clean":"2023-10-09T05:40:59.833020+0000","last_became_active":"2023-10-09T05:40:04.804776+0000","last_became_peered":"2023-10-09T05:40:04.804776+0000","last_unstale":"2023-10-09T05:40:59.833020+0000","last_undegraded":"2023-10-09T05:40:59.833020+0000","last_fullsized":"2023-10-09T05:40:59.833020+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":353,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694399+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:40:56.694399+0000","last_peered":"2023-10-09T05:40:56.694399+0000","last_clean":"2023-10-09T05:40:56.694399+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:40:56.694399+0000","last_undegraded":"2023-10-09T05:40:56.694399+0000","last_fullsized":"2023-10-09T05:40:56.694399+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":171,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.533598+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:41:13.533598+0000","last_peered":"2023-10-09T05:41:13.533598+0000","last_clean":"2023-10-09T05:41:13.533598+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:41:13.533598+0000","last_undegraded":"2023-10-09T05:41:13.533598+0000","last_fullsized":"2023-10-09T05:41:13.533598+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":179,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.834382+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:40:59.834382+0000","last_peered":"2023-10-09T05:40:59.834382+0000","last_clean":"2023-10-09T05:40:59.834382+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:40:59.834382+0000","last_undegraded":"2023-10-09T05:40:59.834382+0000","last_fullsized":"2023-10-09T05:40:59.834382+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"205'10960","reported_seq":11015,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.160680+0000","last_change":"2023-10-09T05:37:01.440172+0000","last_active":"2023-10-09T05:41:13.160680+0000","last_peered":"2023-10-09T05:41:13.160680+0000","last_clean":"2023-10-09T05:41:13.160680+0000","last_became_active":"2023-10-09T05:37:01.439932+0000","last_became_peered":"2023-10-09T05:37:01.439932+0000","last_unstale":"2023-10-09T05:41:13.160680+0000","last_undegraded":"2023-10-09T05:41:13.160680+0000","last_fullsized":"2023-10-09T05:41:13.160680+0000","mapping_epoch":162,"log_start":"180'4500","ondisk_log_start":"180'4500","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6460,"log_dups_size":0,"ondisk_log_size":6460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:57.826704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10960,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":763,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.842461+0000","last_change":"2023-10-09T05:40:05.930112+0000","last_active":"2023-10-09T05:40:59.842461+0000","last_peered":"2023-10-09T05:40:59.842461+0000","last_clean":"2023-10-09T05:40:59.842461+0000","last_became_active":"2023-10-09T05:40:05.929574+0000","last_became_peered":"2023-10-09T05:40:05.929574+0000","last_unstale":"2023-10-09T05:40:59.842461+0000","last_undegraded":"2023-10-09T05:40:59.842461+0000","last_fullsized":"2023-10-09T05:40:59.842461+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:06.982326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":158,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694444+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:40:56.694444+0000","last_peered":"2023-10-09T05:40:56.694444+0000","last_clean":"2023-10-09T05:40:56.694444+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:40:56.694444+0000","last_undegraded":"2023-10-09T05:40:56.694444+0000","last_fullsized":"2023-10-09T05:40:56.694444+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":173,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694894+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:40:56.694894+0000","last_peered":"2023-10-09T05:40:56.694894+0000","last_clean":"2023-10-09T05:40:56.694894+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:40:56.694894+0000","last_undegraded":"2023-10-09T05:40:56.694894+0000","last_fullsized":"2023-10-09T05:40:56.694894+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"205'11092","reported_seq":11147,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.160546+0000","last_change":"2023-10-09T05:37:01.636967+0000","last_active":"2023-10-09T05:41:13.160546+0000","last_peered":"2023-10-09T05:41:13.160546+0000","last_clean":"2023-10-09T05:41:13.160546+0000","last_became_active":"2023-10-09T05:37:01.636602+0000","last_became_peered":"2023-10-09T05:37:01.636602+0000","last_unstale":"2023-10-09T05:41:13.160546+0000","last_undegraded":"2023-10-09T05:41:13.160546+0000","last_fullsized":"2023-10-09T05:41:13.160546+0000","mapping_epoch":162,"log_start":"181'4700","ondisk_log_start":"181'4700","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":6392,"log_dups_size":0,"ondisk_log_size":6392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:11.668143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11092,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":394,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.695420+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:40:56.695420+0000","last_peered":"2023-10-09T05:40:56.695420+0000","last_clean":"2023-10-09T05:40:56.695420+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:40:56.695420+0000","last_undegraded":"2023-10-09T05:40:56.695420+0000","last_fullsized":"2023-10-09T05:40:56.695420+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":154,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.843618+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:40:59.843618+0000","last_peered":"2023-10-09T05:40:59.843618+0000","last_clean":"2023-10-09T05:40:59.843618+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:40:59.843618+0000","last_undegraded":"2023-10-09T05:40:59.843618+0000","last_fullsized":"2023-10-09T05:40:59.843618+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":167,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.695292+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:40:56.695292+0000","last_peered":"2023-10-09T05:40:56.695292+0000","last_clean":"2023-10-09T05:40:56.695292+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:40:56.695292+0000","last_undegraded":"2023-10-09T05:40:56.695292+0000","last_fullsized":"2023-10-09T05:40:56.695292+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"205'11118","reported_seq":11170,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:08.687800+0000","last_change":"2023-10-09T05:37:01.440074+0000","last_active":"2023-10-09T05:41:08.687800+0000","last_peered":"2023-10-09T05:41:08.687800+0000","last_clean":"2023-10-09T05:41:08.687800+0000","last_became_active":"2023-10-09T05:37:01.439808+0000","last_became_peered":"2023-10-09T05:37:01.439808+0000","last_unstale":"2023-10-09T05:41:08.687800+0000","last_undegraded":"2023-10-09T05:41:08.687800+0000","last_fullsized":"2023-10-09T05:41:08.687800+0000","mapping_epoch":162,"log_start":"175'3600","ondisk_log_start":"175'3600","created":162,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":7518,"log_dups_size":0,"ondisk_log_size":7518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11118,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":206,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.557660+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:41:13.557660+0000","last_peered":"2023-10-09T05:41:13.557660+0000","last_clean":"2023-10-09T05:41:13.557660+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:41:13.557660+0000","last_undegraded":"2023-10-09T05:41:13.557660+0000","last_fullsized":"2023-10-09T05:41:13.557660+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":594,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.844513+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:40:59.844513+0000","last_peered":"2023-10-09T05:40:59.844513+0000","last_clean":"2023-10-09T05:40:59.844513+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:40:59.844513+0000","last_undegraded":"2023-10-09T05:40:59.844513+0000","last_fullsized":"2023-10-09T05:40:59.844513+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":199,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.533550+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:41:13.533550+0000","last_peered":"2023-10-09T05:41:13.533550+0000","last_clean":"2023-10-09T05:41:13.533550+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:41:13.533550+0000","last_undegraded":"2023-10-09T05:41:13.533550+0000","last_fullsized":"2023-10-09T05:41:13.533550+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"205'10788","reported_seq":16611,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:41:08.711881+0000","last_change":"2023-10-09T05:40:56.681995+0000","last_active":"2023-10-09T05:41:08.711881+0000","last_peered":"2023-10-09T05:41:08.711881+0000","last_clean":"2023-10-09T05:41:08.711881+0000","last_became_active":"2023-10-09T05:40:56.654153+0000","last_became_peered":"2023-10-09T05:40:56.654153+0000","last_unstale":"2023-10-09T05:41:08.711881+0000","last_undegraded":"2023-10-09T05:41:08.711881+0000","last_fullsized":"2023-10-09T05:41:08.711881+0000","mapping_epoch":203,"log_start":"168'2600","ondisk_log_start":"168'2600","created":162,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:37:00.418943+0000","last_clean_scrub_stamp":"2023-10-09T05:37:00.418943+0000","objects_scrubbed":0,"log_size":8188,"log_dups_size":0,"ondisk_log_size":8188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10788,"num_write_kb":345216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2877,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":690,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:40:59.833289+0000","last_change":"2023-10-09T05:37:36.917512+0000","last_active":"2023-10-09T05:40:59.833289+0000","last_peered":"2023-10-09T05:40:59.833289+0000","last_clean":"2023-10-09T05:37:30.688967+0000","last_became_active":"2023-10-09T05:37:36.700218+0000","last_became_peered":"2023-10-09T05:37:36.700218+0000","last_unstale":"2023-10-09T05:40:59.833289+0000","last_undegraded":"2023-10-09T05:37:36.917335+0000","last_fullsized":"2023-10-09T05:37:36.682410+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":161,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694837+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:40:56.694837+0000","last_peered":"2023-10-09T05:40:56.694837+0000","last_clean":"2023-10-09T05:40:56.694837+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:40:56.694837+0000","last_undegraded":"2023-10-09T05:40:56.694837+0000","last_fullsized":"2023-10-09T05:40:56.694837+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":157,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823445+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:40:59.823445+0000","last_peered":"2023-10-09T05:40:59.823445+0000","last_clean":"2023-10-09T05:40:59.823445+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:40:59.823445+0000","last_undegraded":"2023-10-09T05:40:59.823445+0000","last_fullsized":"2023-10-09T05:40:59.823445+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":169,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.694787+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:40:56.694787+0000","last_peered":"2023-10-09T05:40:56.694787+0000","last_clean":"2023-10-09T05:40:56.694787+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:40:56.694787+0000","last_undegraded":"2023-10-09T05:40:56.694787+0000","last_fullsized":"2023-10-09T05:40:56.694787+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":199,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.534294+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:41:13.534294+0000","last_peered":"2023-10-09T05:41:13.534294+0000","last_clean":"2023-10-09T05:41:13.534294+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:41:13.534294+0000","last_undegraded":"2023-10-09T05:41:13.534294+0000","last_fullsized":"2023-10-09T05:41:13.534294+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":175,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.833705+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:40:59.833705+0000","last_peered":"2023-10-09T05:40:59.833705+0000","last_clean":"2023-10-09T05:40:59.833705+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:40:59.833705+0000","last_undegraded":"2023-10-09T05:40:59.833705+0000","last_fullsized":"2023-10-09T05:40:59.833705+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":187,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823847+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:40:59.823847+0000","last_peered":"2023-10-09T05:40:59.823847+0000","last_clean":"2023-10-09T05:40:59.823847+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:40:59.823847+0000","last_undegraded":"2023-10-09T05:40:59.823847+0000","last_fullsized":"2023-10-09T05:40:59.823847+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":160,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.832500+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:40:59.832500+0000","last_peered":"2023-10-09T05:40:59.832500+0000","last_clean":"2023-10-09T05:40:59.832500+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:40:59.832500+0000","last_undegraded":"2023-10-09T05:40:59.832500+0000","last_fullsized":"2023-10-09T05:40:59.832500+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":187,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823551+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:40:59.823551+0000","last_peered":"2023-10-09T05:40:59.823551+0000","last_clean":"2023-10-09T05:40:59.823551+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:40:59.823551+0000","last_undegraded":"2023-10-09T05:40:59.823551+0000","last_fullsized":"2023-10-09T05:40:59.823551+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":186,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.557601+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:41:13.557601+0000","last_peered":"2023-10-09T05:41:13.557601+0000","last_clean":"2023-10-09T05:41:13.557601+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:41:13.557601+0000","last_undegraded":"2023-10-09T05:41:13.557601+0000","last_fullsized":"2023-10-09T05:41:13.557601+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":199,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T05:41:13.533781+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:41:13.533781+0000","last_peered":"2023-10-09T05:41:13.533781+0000","last_clean":"2023-10-09T05:41:13.533781+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:41:13.533781+0000","last_undegraded":"2023-10-09T05:41:13.533781+0000","last_fullsized":"2023-10-09T05:41:13.533781+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":161,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:40:56.695147+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:40:56.695147+0000","last_peered":"2023-10-09T05:40:56.695147+0000","last_clean":"2023-10-09T05:40:56.695147+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:40:56.695147+0000","last_undegraded":"2023-10-09T05:40:56.695147+0000","last_fullsized":"2023-10-09T05:40:56.695147+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":172,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T05:40:59.823453+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:40:59.823453+0000","last_peered":"2023-10-09T05:40:59.823453+0000","last_clean":"2023-10-09T05:40:59.823453+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:40:59.823453+0000","last_undegraded":"2023-10-09T05:40:59.823453+0000","last_fullsized":"2023-10-09T05:40:59.823453+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2378,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":176722,"num_write_kb":5655041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15097,"num_bytes_recovered":530448384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":579796992,"data_stored":579796992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129822,"ondisk_log_size":129822,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846672,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626698,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307868,"kb_used_data":6548,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93437284,"statfs":{"total":95995035648,"available":95679778816,"internally_reserved":0,"allocated":6705152,"data_stored":6182220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:25 2023","interfaces":[{"interface":"back","average":{"1min":14.522,"5min":6.984,"15min":3.121},"min":{"1min":0.24199999999999999,"5min":0.214,"15min":0.214},"max":{"1min":115.377,"5min":115.377,"15min":115.377},"last":0.39100000000000001},{"interface":"front","average":{"1min":14.285,"5min":6.6109999999999998,"15min":2.9870000000000001},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.23999999999999999},"max":{"1min":116.428,"5min":116.428,"15min":116.428},"last":0.29699999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":7.274,"5min":6.056,"15min":2.7170000000000001},"min":{"1min":0.315,"5min":0.25,"15min":0.25},"max":{"1min":93.489000000000004,"5min":137.02199999999999,"15min":137.02199999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":6.6299999999999999,"5min":5.9379999999999997,"15min":2.758},"min":{"1min":0.318,"5min":0.252,"15min":0.252},"max":{"1min":93.471999999999994,"5min":137.06700000000001,"15min":137.06700000000001},"last":0.52600000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:40:25 2023","interfaces":[{"interface":"back","average":{"1min":8.1809999999999992,"5min":5.5670000000000002,"15min":3.0070000000000001},"min":{"1min":0.443,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":115.40600000000001,"5min":115.40600000000001,"15min":115.40600000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":8.7759999999999998,"5min":5.6310000000000002,"15min":3.0070000000000001},"min":{"1min":0.434,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":115.297,"5min":115.297,"15min":115.297},"last":0.48399999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":11.682,"5min":6.7359999999999998,"15min":3.1219999999999999},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":99.543999999999997,"5min":115.244,"15min":115.244},"last":0.55800000000000005},{"interface":"front","average":{"1min":10.734,"5min":6.556,"15min":3.1499999999999999},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":98.677999999999997,"5min":115.34999999999999,"15min":115.34999999999999},"last":0.42599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.8200000000000001,"15min":1.7110000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":0.72399999999999998,"5min":115.214,"15min":125.64},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.70399999999999996,"5min":1.5820000000000001,"15min":1.6120000000000001},"min":{"1min":0.27400000000000002,"5min":0.254,"15min":0.254},"max":{"1min":2.7189999999999999,"5min":115.22799999999999,"15min":125.625},"last":0.54300000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":2.5600000000000001,"15min":1.855},"min":{"1min":0.33500000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4690000000000003,"5min":115.316,"15min":127.35899999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.0720000000000001,"5min":2.1549999999999998,"15min":1.7370000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.6079999999999997,"5min":115.30500000000001,"15min":128.93799999999999},"last":0.61099999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":2.077,"15min":2.7909999999999999},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":5.0110000000000001,"5min":115.514,"15min":115.514},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.8240000000000001,"15min":2.0019999999999998},"min":{"1min":0.34899999999999998,"5min":0.317,"15min":0.317},"max":{"1min":3.6600000000000001,"5min":116.53100000000001,"15min":116.53100000000001},"last":0.58399999999999996}]}]},{"osd":1,"up_from":93,"seq":399431958659,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":268680,"kb_used_data":2688,"kb_used_omap":7,"kb_used_meta":265976,"kb_avail":93476472,"statfs":{"total":95995035648,"available":95719907328,"internally_reserved":0,"allocated":2752512,"data_stored":2015901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":272359950},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":4.694,"5min":5.1500000000000004,"15min":2.6080000000000001},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":39.189,"5min":207.858,"15min":207.858},"last":0.35499999999999998},{"interface":"front","average":{"1min":6.1989999999999998,"5min":6.4420000000000002,"15min":3.0720000000000001},"min":{"1min":0.23000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":42.012999999999998,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.33800000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:41:11 2023","interfaces":[{"interface":"back","average":{"1min":6.2229999999999999,"5min":7.3860000000000001,"15min":3.5019999999999998},"min":{"1min":0.26000000000000001,"5min":0.22,"15min":0.184},"max":{"1min":42.469999999999999,"5min":101.367,"15min":101.367},"last":0.32000000000000001},{"interface":"front","average":{"1min":5.3369999999999997,"5min":6.9020000000000001,"15min":3.294},"min":{"1min":0.23300000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":42.151000000000003,"5min":97.286000000000001,"15min":98.409999999999997},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":18.629999999999999,"5min":7.2770000000000001,"15min":2.9540000000000002},"min":{"1min":0.28899999999999998,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":162.33699999999999,"5min":162.33699999999999,"15min":162.33699999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":19.029,"5min":7.9980000000000002,"15min":3.1960000000000002},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":162.376,"5min":162.376,"15min":162.376},"last":0.5}]},{"osd":4,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.1749999999999998,"5min":4.7249999999999996,"15min":2.2690000000000001},"min":{"1min":0.39700000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":42.106999999999999,"5min":141.32900000000001,"15min":141.32900000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":5.2800000000000002,"5min":3.6819999999999999,"15min":1.8400000000000001},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":40.475999999999999,"5min":141.15899999999999,"15min":141.15899999999999},"last":0.54300000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.6420000000000003,"5min":4.7009999999999996,"15min":2.2349999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":40.633000000000003,"5min":235.18700000000001,"15min":235.18700000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":5.258,"5min":5.5570000000000004,"15min":2.6240000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":42.078000000000003,"5min":235.79300000000001,"15min":235.79300000000001},"last":0.57599999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.8860000000000001,"5min":3.8109999999999999,"15min":1.8779999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":40.558999999999997,"5min":103.60899999999999,"15min":103.60899999999999},"last":0.628},{"interface":"front","average":{"1min":5.234,"5min":4.4950000000000001,"15min":2.141},"min":{"1min":0.41299999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":42.134999999999998,"5min":103.679,"15min":103.679},"last":0.52300000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.4950000000000001,"5min":3.649,"15min":2.5489999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":42.204000000000001,"5min":126.68300000000001,"15min":126.68300000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":5.4050000000000002,"5min":3.7450000000000001,"15min":2.6339999999999999},"min":{"1min":0.46999999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":42.228999999999999,"5min":126.738,"15min":126.738},"last":0.70199999999999996}]}]},{"osd":0,"up_from":88,"seq":377957122185,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":358764,"kb_used_data":2724,"kb_used_omap":8,"kb_used_meta":356023,"kb_avail":93386388,"statfs":{"total":95995035648,"available":95627661312,"internally_reserved":0,"allocated":2789376,"data_stored":2011598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":364568328},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4910000000000001,"5min":2.3340000000000001,"15min":1.282},"min":{"1min":0.20699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":17.760000000000002,"5min":17.760000000000002,"15min":17.760000000000002},"last":0.23599999999999999},{"interface":"front","average":{"1min":6.0330000000000004,"5min":3.3679999999999999,"15min":1.6539999999999999},"min":{"1min":0.25,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":26.163,"5min":26.163,"15min":26.163},"last":0.35299999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":5.0410000000000004,"5min":4.3979999999999997,"15min":2.2000000000000002},"min":{"1min":0.26800000000000002,"5min":0.223,"15min":0.214},"max":{"1min":21.648,"5min":79.911000000000001,"15min":79.911000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":5.4370000000000003,"5min":4.2549999999999999,"15min":2.1040000000000001},"min":{"1min":0.26000000000000001,"5min":0.221,"15min":0.218},"max":{"1min":32.344000000000001,"5min":80.105000000000004,"15min":80.105000000000004},"last":0.317}]},{"osd":3,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":18.498999999999999,"5min":6.4859999999999998,"15min":2.7669999999999999},"min":{"1min":0.27500000000000002,"5min":0.26000000000000001,"15min":0.22800000000000001},"max":{"1min":123.024,"5min":123.024,"15min":123.024},"last":0.255},{"interface":"front","average":{"1min":12.829000000000001,"5min":5.4359999999999999,"15min":2.48},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":99.162000000000006,"5min":99.162000000000006,"15min":99.162000000000006},"last":0.45400000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.8420000000000001,"15min":1.2050000000000001},"min":{"1min":0.39500000000000002,"5min":0.27500000000000002,"15min":0.27200000000000002},"max":{"1min":5.157,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.9119999999999999,"5min":2.5409999999999999,"15min":1.5149999999999999},"min":{"1min":0.34300000000000003,"5min":0.33000000000000002,"15min":0.30399999999999999},"max":{"1min":9.5190000000000001,"5min":69.102000000000004,"15min":69.102000000000004},"last":0.49099999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":18.456,"5min":6.2709999999999999,"15min":2.7360000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":281.733,"5min":281.733,"15min":281.733},"last":0.40200000000000002},{"interface":"front","average":{"1min":19.111999999999998,"5min":6.2210000000000001,"15min":2.7370000000000001},"min":{"1min":0.40899999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":282.95400000000001,"5min":282.95400000000001,"15min":282.95400000000001},"last":0.503}]},{"osd":6,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5529999999999999,"5min":2.0110000000000001,"15min":1.3280000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":0.434},{"interface":"front","average":{"1min":3.5190000000000001,"5min":2.1779999999999999,"15min":1.387},"min":{"1min":0.46200000000000002,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":10.83,"5min":10.83,"15min":10.83},"last":0.54100000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.679,"15min":1.4450000000000001},"min":{"1min":0.41899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.5090000000000003,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.46800000000000003},{"interface":"front","average":{"1min":3.1859999999999999,"5min":2.548,"15min":2.4420000000000002},"min":{"1min":0.46800000000000003,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.6329999999999991,"5min":9.7159999999999993,"15min":11.068},"last":0.52400000000000002}]}]},{"osd":2,"up_from":98,"seq":420906795133,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":405816,"kb_used_data":6320,"kb_used_omap":11,"kb_used_meta":399476,"kb_avail":93339336,"statfs":{"total":95995035648,"available":95579480064,"internally_reserved":0,"allocated":6471680,"data_stored":5641272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":409064021},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":4.3230000000000004,"5min":3.4670000000000001,"15min":1.7949999999999999},"min":{"1min":0.215,"5min":0.17799999999999999,"15min":0.17199999999999999},"max":{"1min":21.587,"5min":21.587,"15min":32.122},"last":0.215},{"interface":"front","average":{"1min":3.702,"5min":3.4889999999999999,"15min":1.7430000000000001},"min":{"1min":0.214,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":15.468,"5min":21.997,"15min":34.066000000000003},"last":0.33300000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":5.5529999999999999,"15min":2.6539999999999999},"min":{"1min":0.29199999999999998,"5min":0.23400000000000001,"15min":0.20699999999999999},"max":{"1min":24.265000000000001,"5min":98.575000000000003,"15min":98.575000000000003},"last":0.29199999999999998},{"interface":"front","average":{"1min":4.5599999999999996,"5min":4.5069999999999997,"15min":2.282},"min":{"1min":0.23100000000000001,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":18.882999999999999,"5min":90.135000000000005,"15min":98.397000000000006},"last":0.27900000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":5.7240000000000002,"5min":6.1200000000000001,"15min":2.6499999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.24199999999999999},"max":{"1min":24.471,"5min":150.65899999999999,"15min":150.65899999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":6.6200000000000001,"5min":6.9770000000000003,"15min":3.016},"min":{"1min":0.46000000000000002,"5min":0.29899999999999999,"15min":0.26500000000000001},"max":{"1min":22.631,"5min":150.851,"15min":150.851},"last":0.25800000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6629999999999998,"5min":3.2719999999999998,"15min":1.7210000000000001},"min":{"1min":0.32200000000000001,"5min":0.29099999999999998,"15min":0.27700000000000002},"max":{"1min":14.382,"5min":115.84999999999999,"15min":115.84999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.5449999999999999,"5min":3.7109999999999999,"15min":1.9770000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.29799999999999999},"max":{"1min":13.815,"5min":116.149,"15min":116.149},"last":0.376}]},{"osd":5,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":2.8959999999999999,"15min":1.6870000000000001},"min":{"1min":0.36699999999999999,"5min":0.312,"15min":0.312},"max":{"1min":13.712999999999999,"5min":24.597999999999999,"15min":24.597999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":3.222,"5min":2.302,"15min":1.466},"min":{"1min":0.35499999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":14.426,"5min":23.919,"15min":23.919},"last":0.39600000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":2.734,"15min":1.637},"min":{"1min":0.36499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":13.839,"5min":20.503,"15min":20.503},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.71,"5min":2.5619999999999998,"15min":1.536},"min":{"1min":0.40600000000000003,"5min":0.39600000000000002,"15min":0.34499999999999997},"max":{"1min":13.757,"5min":13.757,"15min":13.757},"last":0.59299999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:40:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":3.2509999999999999,"15min":2.766},"min":{"1min":0.47199999999999998,"5min":0.40200000000000002,"15min":0.374},"max":{"1min":13.865,"5min":24.030999999999999,"15min":24.030999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":3.5070000000000001,"5min":3.1110000000000002,"15min":2.1890000000000001},"min":{"1min":0.52700000000000002,"5min":0.41899999999999998,"15min":0.41599999999999998},"max":{"1min":13.781000000000001,"5min":24.178000000000001,"15min":24.178000000000001},"last":0.628}]}]},{"osd":3,"up_from":103,"seq":442381631607,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":322380,"kb_used_data":2116,"kb_used_omap":4,"kb_used_meta":320251,"kb_avail":93422772,"statfs":{"total":95995035648,"available":95664918528,"internally_reserved":0,"allocated":2166784,"data_stored":1425518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":327937630},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":3.8690000000000002,"5min":5.0350000000000001,"15min":2.3599999999999999},"min":{"1min":0.311,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":26.614999999999998,"5min":167.82400000000001,"15min":167.82400000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":5.6230000000000002,"5min":5.7380000000000004,"15min":2.6829999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":26.140000000000001,"5min":167.89599999999999,"15min":167.89599999999999},"last":0.25}]},{"osd":1,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":5.931,"5min":4.8170000000000002,"15min":2.0249999999999999},"min":{"1min":0.27400000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":48.052,"5min":83.921999999999997,"15min":83.921999999999997},"last":0.26000000000000001},{"interface":"front","average":{"1min":5.6310000000000002,"5min":4.7140000000000004,"15min":2.0009999999999999},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.247},"max":{"1min":13.551,"5min":83.707999999999998,"15min":83.707999999999998},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3920000000000003,"5min":7.3719999999999999,"15min":3.629},"min":{"1min":0.307,"5min":0.218,"15min":0.215},"max":{"1min":41.542000000000002,"5min":190.43299999999999,"15min":190.43299999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":5.3739999999999997,"5min":8.4800000000000004,"15min":4.0250000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.26000000000000001},"max":{"1min":26.097999999999999,"5min":190.267,"15min":190.267},"last":0.318}]},{"osd":4,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0030000000000001,"5min":3.2879999999999998,"15min":1.966},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":25.946999999999999,"5min":51.591999999999999,"15min":51.591999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":3.1299999999999999,"5min":2.2530000000000001,"15min":1.5269999999999999},"min":{"1min":0.33500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":26.774999999999999,"5min":26.774999999999999,"15min":45.735999999999997},"last":0.53300000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":4.5890000000000004,"5min":2.427,"15min":1.5880000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":13.114000000000001,"5min":13.114000000000001,"15min":47.948999999999998},"last":0.47799999999999998},{"interface":"front","average":{"1min":3.1699999999999999,"5min":2.2970000000000002,"15min":1.5960000000000001},"min":{"1min":0.499,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":26.728000000000002,"5min":26.728000000000002,"15min":47.968000000000004},"last":0.56999999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":2.387,"15min":1.504},"min":{"1min":0.50700000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":26.847999999999999,"5min":26.847999999999999,"15min":26.847999999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":4.8730000000000002,"5min":2.6619999999999999,"15min":1.4139999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":13.058,"5min":13.058,"15min":13.058},"last":0.44900000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:40:56 2023","interfaces":[{"interface":"back","average":{"1min":4.032,"5min":3.0249999999999999,"15min":2.5150000000000001},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":26.024999999999999,"5min":26.024999999999999,"15min":26.024999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":4.2720000000000002,"5min":3.125,"15min":2.3500000000000001},"min":{"1min":0.42499999999999999,"5min":0.40899999999999997,"15min":0.35699999999999998},"max":{"1min":25.995999999999999,"5min":25.995999999999999,"15min":25.995999999999999},"last":0.48299999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019154,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307292,"kb_used_data":1876,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437860,"statfs":{"total":95995035648,"available":95680368640,"internally_reserved":0,"allocated":1921024,"data_stored":1397836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:22 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":1.929,"15min":1.3180000000000001},"min":{"1min":0.248,"5min":0.214,"15min":0.214},"max":{"1min":6.2590000000000003,"5min":36.406999999999996,"15min":36.406999999999996},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.4460000000000002,"5min":2.556,"15min":1.5800000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.7629999999999999,"5min":36.454999999999998,"15min":36.454999999999998},"last":0.48699999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":1.6699999999999999,"15min":1.21},"min":{"1min":0.29999999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8140000000000001,"5min":7.9550000000000001,"15min":31.073},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.492,"5min":1.8520000000000001,"15min":1.3300000000000001},"min":{"1min":0.29299999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.349,"5min":12.349,"15min":31.100000000000001},"last":0.62}]},{"osd":2,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":6.4269999999999996,"15min":3.476},"min":{"1min":0.31,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":10.099,"5min":196.93700000000001,"15min":196.93700000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":3.048,"5min":6.1829999999999998,"15min":3.3679999999999999},"min":{"1min":0.34399999999999997,"5min":0.28299999999999997,"15min":0.28100000000000003},"max":{"1min":11.534000000000001,"5min":197.08099999999999,"15min":197.08099999999999},"last":0.52400000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":4.3570000000000002,"5min":7.3419999999999996,"15min":3.097},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.509,"5min":211.48699999999999,"15min":211.48699999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":3.2109999999999999,"5min":7.157,"15min":2.9740000000000002},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.309},"max":{"1min":13.007,"5min":211.31100000000001,"15min":211.31100000000001},"last":0.378}]},{"osd":5,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.0310000000000001,"15min":1.5649999999999999},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.249},"max":{"1min":3.2730000000000001,"5min":94.646000000000001,"15min":113.35899999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.883,"15min":1.5229999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.30599999999999999},"max":{"1min":3.3849999999999998,"5min":94.692999999999998,"15min":113.32299999999999},"last":0.66200000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:40:57 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.72899999999999998,"15min":0.70499999999999996},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":1.8740000000000001,"5min":6.1779999999999999,"15min":11.944000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.93500000000000005,"15min":0.86399999999999999},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.4810000000000001,"5min":7.6500000000000004,"15min":10.356},"last":0.54000000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.94799999999999995,"15min":2.5619999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.312},"max":{"1min":3.415,"5min":11.845000000000001,"15min":30.393999999999998},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.97599999999999998,"15min":2.8780000000000001},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.5089999999999999,"5min":10.837,"15min":30.334},"last":0.59699999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822926,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299084,"kb_used_data":1860,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93446068,"statfs":{"total":95995035648,"available":95688773632,"internally_reserved":0,"allocated":1904640,"data_stored":1393284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":4.4219999999999997,"15min":2.9369999999999998},"min":{"1min":0.22800000000000001,"5min":0.22700000000000001,"15min":0.19400000000000001},"max":{"1min":8.3079999999999998,"5min":165.90199999999999,"15min":165.90199999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.377,"5min":4.2290000000000001,"15min":2.9910000000000001},"min":{"1min":0.28399999999999997,"5min":0.215,"15min":0.215},"max":{"1min":5.4660000000000002,"5min":165.96700000000001,"15min":165.96700000000001},"last":0.30399999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":7.0419999999999998,"5min":4.6070000000000002,"15min":2.1699999999999999},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":71.209999999999994,"5min":87.769000000000005,"15min":87.769000000000005},"last":0.55400000000000005},{"interface":"front","average":{"1min":8.1159999999999997,"5min":5.2130000000000001,"15min":2.2810000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.24399999999999999},"max":{"1min":71.192999999999998,"5min":87.864000000000004,"15min":87.864000000000004},"last":0.34100000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:40:20 2023","interfaces":[{"interface":"back","average":{"1min":7.2709999999999999,"5min":6.7069999999999999,"15min":4.008},"min":{"1min":0.33600000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":73.311999999999998,"5min":118.255,"15min":290.697},"last":0.442},{"interface":"front","average":{"1min":6.1639999999999997,"5min":6.0199999999999996,"15min":3.7810000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":73.280000000000001,"5min":118.438,"15min":290.786},"last":1.633}]},{"osd":3,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":7.5910000000000002,"5min":6.8520000000000003,"15min":2.8919999999999999},"min":{"1min":0.35199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":85.007999999999996,"5min":95.236999999999995,"15min":95.236999999999995},"last":0.52800000000000002},{"interface":"front","average":{"1min":6.7199999999999998,"5min":6.2770000000000001,"15min":2.7189999999999999},"min":{"1min":0.32900000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":83.147999999999996,"5min":93.471000000000004,"15min":93.471000000000004},"last":1.7390000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.81299999999999994,"15min":0.72099999999999997},"min":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":5.6459999999999999,"5min":7.2060000000000004,"15min":9.3510000000000009},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.1970000000000001,"15min":0.94399999999999995},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":4.8209999999999997,"5min":10.913,"15min":14.236000000000001},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:41:04 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.2370000000000001,"15min":1.3600000000000001},"min":{"1min":0.35699999999999998,"5min":0.27600000000000002,"15min":0.26600000000000001},"max":{"1min":4.79,"5min":27.146000000000001,"15min":95.010999999999996},"last":0.374},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.1359999999999999,"15min":1.2609999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27700000000000002},"max":{"1min":4.6600000000000001,"5min":27.215,"15min":94.194000000000003},"last":0.57699999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:41:00 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":1.228,"15min":1.5589999999999999},"min":{"1min":0.39900000000000002,"5min":0.34399999999999997,"15min":0.28999999999999998},"max":{"1min":5.5570000000000004,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.212,"5min":1.157,"15min":1.8009999999999999},"min":{"1min":0.373,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.0549999999999997,"5min":10.779,"15min":14.432},"last":0.48499999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":579796992,"data_stored":579796992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:41:20.999 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:41:21.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:20 smithi191 ceph-mon[161669]: failed to encode map e209 with expected crc 2023-10-09T05:41:21.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:20 smithi191 ceph-mon[161669]: failed to encode map e209 with expected crc 2023-10-09T05:41:21.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:20 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:41:21.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:20 smithi191 ceph-mon[161669]: osdmap e209: 8 total, 8 up, 7 in 2023-10-09T05:41:21.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:20 smithi191 ceph-mon[161669]: failed to encode map e209 with expected crc 2023-10-09T05:41:21.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:20 smithi191 ceph-mon[161669]: pgmap v596: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T05:41:21.215 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:21.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:21.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:21.241 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:41:21.242 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:41:21.242 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T05:41:21.242 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T05:41:21.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:21.355 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:21.404 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:21.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:21.508 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:21.541 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:21.816 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:21.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:21.919 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:21.950 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:21.950 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:21.976 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:21.996 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:21.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:21.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:22.032 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:22.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:22.127 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:22.140 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[184418]: failed to encode map e209 with expected crc 2023-10-09T05:41:22.140 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[184418]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.140 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[184418]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.140 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[184418]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.141 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[184418]: osdmap e210: 8 total, 8 up, 7 in 2023-10-09T05:41:22.141 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[184418]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.141 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[191227]: failed to encode map e209 with expected crc 2023-10-09T05:41:22.141 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[191227]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.141 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[191227]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.141 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[191227]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.142 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[191227]: osdmap e210: 8 total, 8 up, 7 in 2023-10-09T05:41:22.142 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:21 smithi008 ceph-mon[191227]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.293 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:22.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:21 smithi191 ceph-mon[161669]: failed to encode map e209 with expected crc 2023-10-09T05:41:22.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:21 smithi191 ceph-mon[161669]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:21 smithi191 ceph-mon[161669]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.296 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:21 smithi191 ceph-mon[161669]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.296 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:21 smithi191 ceph-mon[161669]: osdmap e210: 8 total, 8 up, 7 in 2023-10-09T05:41:22.296 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:21 smithi191 ceph-mon[161669]: failed to encode map e210 with expected crc 2023-10-09T05:41:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:22.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:22.397 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:22.429 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:22.588 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:22.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:22.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:22.694 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:22.728 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:22.780 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:41:22.781 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:41:22.923 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:41:23.106 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[184418]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.106 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[184418]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.106 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[184418]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.106 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[184418]: osdmap e211: 8 total, 8 up, 7 in 2023-10-09T05:41:23.107 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[184418]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.107 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[184418]: pgmap v599: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:23.107 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1733797384' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:41:23.107 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:41:23.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[191227]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[191227]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[191227]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.108 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[191227]: osdmap e211: 8 total, 8 up, 7 in 2023-10-09T05:41:23.108 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[191227]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.108 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[191227]: pgmap v599: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:23.108 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1733797384' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:41:23.108 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:22 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:41:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:22 smithi191 ceph-mon[161669]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:22 smithi191 ceph-mon[161669]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:22 smithi191 ceph-mon[161669]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:22 smithi191 ceph-mon[161669]: osdmap e211: 8 total, 8 up, 7 in 2023-10-09T05:41:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:22 smithi191 ceph-mon[161669]: failed to encode map e211 with expected crc 2023-10-09T05:41:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:22 smithi191 ceph-mon[161669]: pgmap v599: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:22 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1733797384' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:41:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:22 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:41:23.493 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:41:23.839 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:24.011 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:24.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:24.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:24.117 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:24.151 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:24.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[191227]: failed to encode map e211 with expected crc 2023-10-09T05:41:24.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:24.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[191227]: failed to encode map e212 with expected crc 2023-10-09T05:41:24.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[191227]: failed to encode map e212 with expected crc 2023-10-09T05:41:24.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1733797384' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:41:24.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[191227]: osdmap e212: 8 total, 8 up, 7 in 2023-10-09T05:41:24.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[191227]: failed to encode map e212 with expected crc 2023-10-09T05:41:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[184418]: failed to encode map e211 with expected crc 2023-10-09T05:41:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[184418]: failed to encode map e212 with expected crc 2023-10-09T05:41:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[184418]: failed to encode map e212 with expected crc 2023-10-09T05:41:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1733797384' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:41:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[184418]: osdmap e212: 8 total, 8 up, 7 in 2023-10-09T05:41:24.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:23 smithi008 ceph-mon[184418]: failed to encode map e212 with expected crc 2023-10-09T05:41:24.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:23 smithi191 ceph-mon[161669]: failed to encode map e211 with expected crc 2023-10-09T05:41:24.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:23 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:24.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:23 smithi191 ceph-mon[161669]: failed to encode map e212 with expected crc 2023-10-09T05:41:24.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:23 smithi191 ceph-mon[161669]: failed to encode map e212 with expected crc 2023-10-09T05:41:24.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1733797384' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:41:24.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:23 smithi191 ceph-mon[161669]: osdmap e212: 8 total, 8 up, 7 in 2023-10-09T05:41:24.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:23 smithi191 ceph-mon[161669]: failed to encode map e212 with expected crc 2023-10-09T05:41:24.455 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:24.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:24.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:24.559 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:24.587 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:24.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:24 smithi008 ceph-mon[191227]: failed to encode map e212 with expected crc 2023-10-09T05:41:24.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:24 smithi008 ceph-mon[191227]: pgmap v601: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 963 KiB/s, 0 objects/s recovering 2023-10-09T05:41:24.948 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:24.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:24.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:25.052 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:25.084 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:25.122 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:24 smithi191 ceph-mon[161669]: failed to encode map e212 with expected crc 2023-10-09T05:41:25.122 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:24 smithi191 ceph-mon[161669]: pgmap v601: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 963 KiB/s, 0 objects/s recovering 2023-10-09T05:41:25.197 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:24 smithi008 ceph-mon[184418]: failed to encode map e212 with expected crc 2023-10-09T05:41:25.197 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:24 smithi008 ceph-mon[184418]: pgmap v601: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 963 KiB/s, 0 objects/s recovering 2023-10-09T05:41:25.230 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:25.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:25.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:25.275 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:41:25.275 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:41:25.334 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:25.363 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:25.457 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:25.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:25.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:25.561 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:25.588 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:25.630 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:25.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:25.733 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:25.763 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:25.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:41:25.964 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:25.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:25.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:26.027 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:41:26.049 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:41:26.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:26.070 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:41:26.089 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:41:26.096 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2659283356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:26.099 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:26.108 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:41:26.127 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:41:26.146 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:41:26.195 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:41:26.195 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:41:26.195 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:41:26.195 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:41:26.196 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:41:26.196 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:41:26.196 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:41:26.196 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:41:26.196 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:41:26.205 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:26.246 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:26.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:26.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:26.314 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2659283356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:26.314 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2659283356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:26.354 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:26.380 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:26.426 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:26.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:26.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:26.530 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:26.561 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:26.563 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:41:26.563 INFO:teuthology.orchestra.run.smithi008.stdout:{"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 1 pg undersized","count":1},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":780,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":212,"num_osds":8,"num_up_osds":8,"osd_up_since":1696829680,"num_in_osds":7,"osd_in_since":1696829854,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":72},{"state_name":"active+undersized+remapped","count":1}],"num_pgs":73,"num_pools":4,"num_objects":28,"data_bytes":4790195,"bytes_used":2351521792,"bytes_avail":669613727744,"bytes_total":671965249536,"recovering_objects_per_sec":0,"recovering_bytes_per_sec":986304,"recovering_keys_per_sec":0,"num_objects_recovered":3,"num_bytes_recovered":4194304,"num_keys_recovered":0},"fsmap":{"epoch":6,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi008.zfejpl","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.8:8443/","prometheus":"http://172.21.15.8:9283/"}},"servicemap":{"epoch":23,"modified":"2023-10-09T05:35:39.975308+0000","services":{}},"progress_events":{"a1c2a5cb-a243-4c8c-845f-d0215fa7ce6b":{"message":"Global Recovery Event (3m)\n [===========================.] (remaining: 2s)","progress":0.98630136251449585,"add_to_ceph_s":true},"e7a35707-3c6c-469d-ad94-cd0d5825f733":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T05:41:27.088 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:41:27.163 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:27.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:27.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:27.267 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:27.294 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:27.503 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:41:27.504 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:41:27.504 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:41:27.504 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:41:27.504 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.505 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54043 5.1 5:88ab6307:::benchmark_data_smithi008_238708_object54042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:41:27.505 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.530687+0000", 2023-10-09T05:41:27.505 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 486.97028280299998, 2023-10-09T05:41:27.505 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0331553470000001, 2023-10-09T05:41:27.505 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.506 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.506 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.506 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.506 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.506 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54043 2023-10-09T05:41:27.507 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.507 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.507 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.507 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.507 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530687+0000", 2023-10-09T05:41:27.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530687+0000", 2023-10-09T05:41:27.509 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:41:27.509 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.509 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.510 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530685+0000", 2023-10-09T05:41:27.510 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0238999999999999e-05 2023-10-09T05:41:27.510 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.511 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.511 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.511 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530726+0000", 2023-10-09T05:41:27.511 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5817000000000001e-05 2023-10-09T05:41:27.511 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.512 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.512 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.512 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530741+0000", 2023-10-09T05:41:27.512 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3200000000000002e-06 2023-10-09T05:41:27.512 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.512 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.513 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.513 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.530749+0000", 2023-10-09T05:41:27.513 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0053712409999999997 2023-10-09T05:41:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536120+0000", 2023-10-09T05:41:27.514 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1669999999999999e-06 2023-10-09T05:41:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536128+0000", 2023-10-09T05:41:27.515 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0145035060000001 2023-10-09T05:41:27.516 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.516 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.516 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.516 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.550632+0000", 2023-10-09T05:41:27.516 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115936 2023-10-09T05:41:27.517 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.517 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.517 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.517 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.550747+0000", 2023-10-09T05:41:27.517 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014306200000000001 2023-10-09T05:41:27.517 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.518 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.518 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:41:27.518 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.550891+0000", 2023-10-09T05:41:27.518 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059699740000000003 2023-10-09T05:41:27.518 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.519 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.519 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.519 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.556861+0000", 2023-10-09T05:41:27.520 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046747400000000001 2023-10-09T05:41:27.520 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.520 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.520 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.520 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.561535+0000", 2023-10-09T05:41:27.521 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022096709999999999 2023-10-09T05:41:27.521 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.521 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.521 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.521 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563745+0000", 2023-10-09T05:41:27.521 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7768000000000003e-05 2023-10-09T05:41:27.522 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.522 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.522 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.522 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563843+0000", 2023-10-09T05:41:27.522 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0331553470000001 2023-10-09T05:41:27.522 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.523 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.523 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.523 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.523 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.523 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54047 5.1 5:8c453526:::benchmark_data_smithi008_238708_object54046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-09T05:41:27.523 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.534316+0000", 2023-10-09T05:41:27.523 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 486.96665433800001, 2023-10-09T05:41:27.523 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.029737098, 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54047 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534316+0000", 2023-10-09T05:41:27.524 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534316+0000", 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534314+0000", 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6623999999999999e-05 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.525 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.526 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.526 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534360+0000", 2023-10-09T05:41:27.526 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0910000000000001e-05 2023-10-09T05:41:27.526 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.526 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.526 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.526 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534371+0000", 2023-10-09T05:41:27.526 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6819999999999997e-06 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.534378+0000", 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001771051 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536149+0000", 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5040000000000002e-06 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.527 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.536155+0000", 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.018304179 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554459+0000", 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014521 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.528 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554604+0000", 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019115700000000001 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.554795+0000", 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0091080610000000006 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563903+0000", 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1428e-05 2023-10-09T05:41:27.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563915+0000", 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3801000000000002e-05 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.563968+0000", 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4501999999999998e-05 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.530 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564053+0000", 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.029737098 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54057 5.1 5:851293d1:::benchmark_data_smithi008_238708_object54056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.550289+0000", 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 486.95068088800002, 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.021774086, 2023-10-09T05:41:27.531 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54057 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550289+0000", 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.532 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550289+0000", 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550287+0000", 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2347e-05 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550330+0000", 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2855000000000001e-05 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550353+0000", 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4370000000000001e-06 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550360+0000", 2023-10-09T05:41:27.534 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5225e-05 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550425+0000", 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0944999999999999e-05 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.550436+0000", 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.006471347 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.535 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.556908+0000", 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014533999999999999 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.557053+0000", 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00025040400000000002 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.557303+0000", 2023-10-09T05:41:27.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068016099999999996 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564105+0000", 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078524939999999998 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.571957+0000", 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6785e-05 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572004+0000", 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9092000000000003e-05 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572063+0000", 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.021774086 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.538 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54062 5.1 5:890b0b94:::benchmark_data_smithi008_238708_object54061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.563895+0000", 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 486.93707486, 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082578959999999, 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54062 2023-10-09T05:41:27.539 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563895+0000", 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563895+0000", 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563894+0000", 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4879000000000001e-05 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563949+0000", 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6708999999999999e-05 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563966+0000", 2023-10-09T05:41:27.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9090000000000003e-06 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.563972+0000", 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7568000000000002e-05 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.564030+0000", 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0219e-05 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.564040+0000", 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99530809600000003 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.559348+0000", 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011862999999999999 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.559467+0000", 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017885700000000001 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.559646+0000", 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0081993729999999994 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.567845+0000", 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042279680000000004 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.544 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572073+0000", 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1065999999999997e-05 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572104+0000", 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8968000000000003e-05 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572153+0000", 2023-10-09T05:41:27.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082578959999999 2023-10-09T05:41:27.546 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.546 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.546 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.546 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.546 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.546 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54068 5.1 5:8319935a:::benchmark_data_smithi008_238708_object54067:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:41:27.546 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.569507+0000", 2023-10-09T05:41:27.546 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 486.93146356199998, 2023-10-09T05:41:27.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082394779999999, 2023-10-09T05:41:27.546 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54068 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569507+0000", 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.547 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569507+0000", 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569505+0000", 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8206999999999999e-05 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569533+0000", 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3752e-05 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569547+0000", 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0120000000000002e-06 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569553+0000", 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3130999999999997e-05 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569606+0000", 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4290000000000001e-06 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.569614+0000", 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99451796599999998 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564132+0000", 2023-10-09T05:41:27.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9108e-05 2023-10-09T05:41:27.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.551 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.551 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564232+0000", 2023-10-09T05:41:27.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115476 2023-10-09T05:41:27.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.551 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.551 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:41:27.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.564347+0000", 2023-10-09T05:41:27.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078146750000000001 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572162+0000", 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0053927940000000002 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.577554+0000", 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8337999999999995e-05 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.577623+0000", 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012327700000000001 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.577746+0000", 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0082394779999999 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.553 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54070 5.1 5:819fd021:::benchmark_data_smithi008_238708_object54069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:41:27.554 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.571140+0000", 2023-10-09T05:41:27.554 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 486.929830423, 2023-10-09T05:41:27.554 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.009647422, 2023-10-09T05:41:27.554 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.554 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.554 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.554 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.554 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.554 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54070 2023-10-09T05:41:27.554 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.554 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571140+0000", 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571140+0000", 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.555 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571138+0000", 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9555999999999998e-05 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571188+0000", 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.417e-05 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571202+0000", 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7789999999999998e-06 2023-10-09T05:41:27.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571209+0000", 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6211000000000002e-05 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571255+0000", 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2060000000000001e-06 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.571262+0000", 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99464801999999997 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.565910+0000", 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000141688 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.566052+0000", 2023-10-09T05:41:27.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000160173 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.566212+0000", 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059532430000000004 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572165+0000", 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0084587169999999993 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.580624+0000", 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9176999999999999e-05 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.580683+0000", 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010398 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.580787+0000", 2023-10-09T05:41:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.009647422 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:54080 5.1 5:84267530:::benchmark_data_smithi008_238708_object54079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:33:20.584125+0000", 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 486.91684560200002, 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99768334199999997, 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.561 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 54080 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584125+0000", 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.562 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584125+0000", 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584123+0000", 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5760000000000001e-05 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584149+0000", 2023-10-09T05:41:27.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3940999999999999e-05 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584163+0000", 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6529999999999994e-06 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584171+0000", 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0436999999999998e-05 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584221+0000", 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.153e-06 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:20.584228+0000", 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98363814599999999 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.567866+0000", 2023-10-09T05:41:27.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013288699999999999 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.567999+0000", 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015074399999999999 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3", 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.568150+0000", 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040185330000000003 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.572168+0000", 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0094922050000000001 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.581661+0000", 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6870000000000003e-05 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.581717+0000", 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0416999999999998e-05 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:33:21.581808+0000", 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99768334199999997 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119018 5.e 5:7572eb82:::benchmark_data_smithi008_238708_object882:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:41:27.568 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.007070+0000", 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 423.493899709, 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0282413180000001, 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119018 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007070+0000", 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007070+0000", 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007068+0000", 2023-10-09T05:41:27.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2640000000000008e-06 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007077+0000", 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5773999999999998e-05 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007092+0000", 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9e-06 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.571 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007099+0000", 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5432000000000005e-05 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007185+0000", 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9950000000000005e-06 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007193+0000", 2023-10-09T05:41:27.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.248123126 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255316+0000", 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8770000000000004e-06 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255323+0000", 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.77988959099999999 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035212+0000", 2023-10-09T05:41:27.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9400000000000004e-05 2023-10-09T05:41:27.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035312+0000", 2023-10-09T05:41:27.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0282413180000001 2023-10-09T05:41:27.574 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.574 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.574 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119020 5.1e 5:7d0c0384:::benchmark_data_smithi008_238708_object884:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.007113+0000", 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 423.49385726999998, 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0326345889999999, 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119020 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.575 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007113+0000", 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007113+0000", 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007112+0000", 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.49e-06 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007116+0000", 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.498e-06 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007123+0000", 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.788e-06 2023-10-09T05:41:27.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007127+0000", 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.248267246 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255395+0000", 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5020000000000001e-06 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255398+0000", 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78425046499999995 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.039648+0000", 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9006999999999996e-05 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.039747+0000", 2023-10-09T05:41:27.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0326345889999999 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119026 5.e 5:7513f17c:::benchmark_data_smithi008_238708_object890:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.007132+0000", 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 423.49383848999997, 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0283243520000001, 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:27.580 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119026 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007132+0000", 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007132+0000", 2023-10-09T05:41:27.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:27.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007131+0000", 2023-10-09T05:41:27.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.322e-06 2023-10-09T05:41:27.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007134+0000", 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5490000000000003e-06 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007140+0000", 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4379999999999999e-06 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007144+0000", 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00092436700000000005 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.008068+0000", 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.917e-06 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.008075+0000", 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.247270726 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255346+0000", 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2480000000000002e-06 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255355+0000", 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78001380399999998 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035369+0000", 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7331999999999997e-05 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035456+0000", 2023-10-09T05:41:27.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0283243520000001 2023-10-09T05:41:27.586 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.586 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.586 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.586 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163684 5.21 5:84c14237:::benchmark_data_smithi008_238708_object45548:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:41:27.586 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.609434+0000", 2023-10-09T05:41:27.586 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 379.89153587800001, 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99344012400000004, 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163684 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609434+0000", 2023-10-09T05:41:27.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609434+0000", 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609433+0000", 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.511e-06 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609440+0000", 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2476999999999999e-05 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609452+0000", 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0360000000000001e-06 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609459+0000", 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9319000000000004e-05 2023-10-09T05:41:27.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609539+0000", 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0864e-05 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609549+0000", 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99053912899999996 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.600089+0000", 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103931 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.600193+0000", 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000164618 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.600357+0000", 2023-10-09T05:41:27.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023727349999999999 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602730+0000", 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1140000000000003e-05 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602771+0000", 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7978e-05 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602789+0000", 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5139000000000002e-05 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602844+0000", 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0287e-05 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602874+0000", 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99344012400000004 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163704 5.21 5:87d25abf:::benchmark_data_smithi008_238708_object45568:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.629811+0000", 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 379.87115959400001, 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97411889600000001, 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.594 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163704 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629811+0000", 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629811+0000", 2023-10-09T05:41:27.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:27.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629810+0000", 2023-10-09T05:41:27.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.638e-06 2023-10-09T05:41:27.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629813+0000", 2023-10-09T05:41:27.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8369999999999996e-06 2023-10-09T05:41:27.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629817+0000", 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4340000000000001e-06 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629821+0000", 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9418999999999994e-05 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629890+0000", 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0139999999999998e-06 2023-10-09T05:41:27.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629898+0000", 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97170687300000003 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.601605+0000", 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6179000000000002e-05 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.601701+0000", 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019179700000000001 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.601893+0000", 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001016667 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602910+0000", 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6458999999999998e-05 2023-10-09T05:41:27.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602946+0000", 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091263200000000003 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603859+0000", 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7240999999999998e-05 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603896+0000", 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3331000000000003e-05 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603929+0000", 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97411889600000001 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163753 5.21 5:845d2bea:::benchmark_data_smithi008_238708_object45617:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.676299+0000", 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 379.82467089900001, 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92971798100000003, 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163753 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.603 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676299+0000", 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676299+0000", 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676298+0000", 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4119999999999998e-06 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676304+0000", 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4579999999999999e-05 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676319+0000", 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5139999999999998e-06 2023-10-09T05:41:27.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676325+0000", 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0550999999999998e-05 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676376+0000", 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6689999999999995e-06 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676385+0000", 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92657416500000001 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602959+0000", 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2798000000000003e-05 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603031+0000", 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013177499999999999 2023-10-09T05:41:27.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603163+0000", 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027447750000000001 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605908+0000", 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7929999999999997e-05 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605956+0000", 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4289999999999998e-06 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605962+0000", 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1736000000000002e-05 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605994+0000", 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3096999999999999e-05 2023-10-09T05:41:27.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606017+0000", 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92971798100000003 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163923 5.21 5:843e313c:::benchmark_data_smithi008_238708_object45787:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.853413+0000", 2023-10-09T05:41:27.610 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 379.64755731499997, 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.75769858499999998, 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163923 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853413+0000", 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853413+0000", 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853412+0000", 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0589999999999999e-06 2023-10-09T05:41:27.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853418+0000", 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2941e-05 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853431+0000", 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7400000000000001e-06 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853436+0000", 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7589e-05 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853494+0000", 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2001000000000001e-05 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853506+0000", 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.75046967899999995 2023-10-09T05:41:27.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603976+0000", 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109487 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.604085+0000", 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019979399999999999 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.604285+0000", 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001741006 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606026+0000", 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4724e-05 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606081+0000", 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004942559 2023-10-09T05:41:27.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611023+0000", 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8794000000000001e-05 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611072+0000", 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9496000000000003e-05 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611111+0000", 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.75769858499999998 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164030 5.21 5:866422a9:::benchmark_data_smithi008_238708_object45894:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 379.55241968899998, 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66267309900000004, 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.618 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164030 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7736000000000001e-05 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948567+0000", 2023-10-09T05:41:27.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8230999999999999e-05 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948585+0000", 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3960000000000003e-06 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948592+0000", 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.498e-05 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948637+0000", 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8820000000000003e-06 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948644+0000", 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65639657299999998 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605040+0000", 2023-10-09T05:41:27.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7799000000000003e-05 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605138+0000", 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013080500000000001 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605269+0000", 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017788770000000001 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607048+0000", 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012428719999999999 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608291+0000", 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002871307 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611162+0000", 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2058999999999999e-05 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611194+0000", 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9536999999999999e-05 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611224+0000", 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66267309900000004 2023-10-09T05:41:27.625 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164042 5.21 5:85682928:::benchmark_data_smithi008_238708_object45906:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.959315+0000", 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 379.54165539100001, 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65788200699999999, 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.626 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164042 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959315+0000", 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959315+0000", 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959314+0000", 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.07e-06 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959318+0000", 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7539999999999995e-06 2023-10-09T05:41:27.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959327+0000", 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.23e-06 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959330+0000", 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1047999999999998e-05 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959361+0000", 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0820000000000004e-06 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959367+0000", 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.64674576399999995 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606113+0000", 2023-10-09T05:41:27.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0732e-05 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606184+0000", 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013076400000000001 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606315+0000", 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048103449999999997 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611125+0000", 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014353599999999999 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611268+0000", 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0058299170000000004 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617098+0000", 2023-10-09T05:41:27.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4318999999999999e-05 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617153+0000", 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4100000000000001e-05 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617197+0000", 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65788200699999999 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.633 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164098 5.21 5:85741b6f:::benchmark_data_smithi008_238708_object45962:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.014979+0000", 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 379.48599151899998, 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.60237710200000005, 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.634 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164098 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014979+0000", 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014979+0000", 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:27.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014977+0000", 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6270000000000002e-06 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014984+0000", 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2714000000000001e-05 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.636 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014996+0000", 2023-10-09T05:41:27.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2259999999999996e-06 2023-10-09T05:41:27.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.637 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.015002+0000", 2023-10-09T05:41:27.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5050999999999999e-05 2023-10-09T05:41:27.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.637 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.015057+0000", 2023-10-09T05:41:27.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0011e-05 2023-10-09T05:41:27.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.015067+0000", 2023-10-09T05:41:27.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.59200529499999999 2023-10-09T05:41:27.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607072+0000", 2023-10-09T05:41:27.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7960999999999994e-05 2023-10-09T05:41:27.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607160+0000", 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017280900000000001 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607333+0000", 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037971039999999999 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611130+0000", 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000167384 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611297+0000", 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059718460000000003 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617269+0000", 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0393999999999999e-05 2023-10-09T05:41:27.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617320+0000", 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6199999999999999e-05 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617356+0000", 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.60237710200000005 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.641 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164190 5.21 5:85c1fc08:::benchmark_data_smithi008_238708_object46054:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.100063+0000", 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 379.40090710700002, 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.517522814, 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164190 2023-10-09T05:41:27.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100063+0000", 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100063+0000", 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100062+0000", 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2819999999999996e-06 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100070+0000", 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5238000000000001e-05 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100095+0000", 2023-10-09T05:41:27.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2259999999999999e-06 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100102+0000", 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3122999999999999e-05 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100166+0000", 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1447000000000001e-05 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100177+0000", 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.50813475200000002 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608312+0000", 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102922 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608415+0000", 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019721800000000001 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608612+0000", 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026200059999999998 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611232+0000", 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3609000000000004e-05 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611326+0000", 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0061575960000000004 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617483+0000", 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1966999999999999e-05 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.648 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617535+0000", 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0797999999999997e-05 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617586+0000", 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.517522814 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164259 5.21 5:8505256a:::benchmark_data_smithi008_238708_object46123:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:41:27.649 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.167748+0000", 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 379.333222342, 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45063465699999999, 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164259 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.650 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167748+0000", 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167748+0000", 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167746+0000", 2023-10-09T05:41:27.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4189999999999996e-06 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167753+0000", 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4399999999999999e-05 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167768+0000", 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6309999999999997e-06 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.652 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167774+0000", 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7620000000000006e-05 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167852+0000", 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0051000000000001e-05 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167862+0000", 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.441777105 2023-10-09T05:41:27.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.609639+0000", 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105538 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.609744+0000", 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018787400000000001 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.654 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.609932+0000", 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041432689999999998 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.614076+0000", 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033595819999999998 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617435+0000", 2023-10-09T05:41:27.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00086552299999999999 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618301+0000", 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7574999999999999e-05 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618338+0000", 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4168000000000001e-05 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618382+0000", 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45063465699999999 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164297 5.21 5:85a0851f:::benchmark_data_smithi008_238708_object46161:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.207114+0000", 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 379.29385660899999, 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.41141557899999998, 2023-10-09T05:41:27.657 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164297 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207114+0000", 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:27.658 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207114+0000", 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207112+0000", 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5089999999999999e-06 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207118+0000", 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1423000000000001e-05 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207130+0000", 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7730000000000002e-06 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207137+0000", 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011262379999999999 2023-10-09T05:41:27.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.208263+0000", 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1843e-05 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.208275+0000", 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.40307073700000001 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611345+0000", 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6700999999999995e-05 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611442+0000", 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00022087699999999999 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611663+0000", 2023-10-09T05:41:27.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002453359 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.614116+0000", 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033269020000000001 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617443+0000", 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001025317 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.663 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618469+0000", 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4535000000000001e-05 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618493+0000", 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6001000000000003e-05 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618529+0000", 2023-10-09T05:41:27.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.41141557899999998 2023-10-09T05:41:27.665 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.665 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.665 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.665 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:27.665 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:27.665 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:41:27.741 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:27.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:27.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:27.785 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:27 smithi008 ceph-mon[184418]: pgmap v602: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 683 KiB/s, 0 objects/s recovering 2023-10-09T05:41:27.785 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1670448349' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:41:27.785 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:27 smithi008 ceph-mon[191227]: pgmap v602: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 683 KiB/s, 0 objects/s recovering 2023-10-09T05:41:27.785 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1670448349' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:41:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:27 smithi191 ceph-mon[161669]: pgmap v602: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 683 KiB/s, 0 objects/s recovering 2023-10-09T05:41:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1670448349' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:41:27.807 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T05:41:27.807 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:41:27.845 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:27.878 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:27.881 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:27.931 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:27.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:28.035 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:28.066 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:41:28.068 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:28.075 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_2' created 2023-10-09T05:41:28.420 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[184418]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T05:41:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2288693718' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T05:41:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T05:41:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T05:41:28.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[184418]: osdmap e213: 8 total, 8 up, 7 in 2023-10-09T05:41:28.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[184418]: failed to encode map e213 with expected crc 2023-10-09T05:41:28.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[184418]: failed to encode map e213 with expected crc 2023-10-09T05:41:28.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[191227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T05:41:28.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2288693718' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T05:41:28.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T05:41:28.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T05:41:28.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[191227]: osdmap e213: 8 total, 8 up, 7 in 2023-10-09T05:41:28.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[191227]: failed to encode map e213 with expected crc 2023-10-09T05:41:28.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:28 smithi008 ceph-mon[191227]: failed to encode map e213 with expected crc 2023-10-09T05:41:28.784 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:28 smithi191 ceph-mon[161669]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T05:41:28.784 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2288693718' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T05:41:28.784 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:28 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T05:41:28.784 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:28 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T05:41:28.784 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:28 smithi191 ceph-mon[161669]: osdmap e213: 8 total, 8 up, 7 in 2023-10-09T05:41:28.784 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:28 smithi191 ceph-mon[161669]: failed to encode map e213 with expected crc 2023-10-09T05:41:28.784 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:28 smithi191 ceph-mon[161669]: failed to encode map e213 with expected crc 2023-10-09T05:41:29.028 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:29.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:29.132 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:29.160 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:29.254 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T05:41:29.318 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:29.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:29.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:29.425 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:29.455 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:29.556 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:29.563 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[184418]: failed to encode map e213 with expected crc 2023-10-09T05:41:29.563 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[184418]: failed to encode map e213 with expected crc 2023-10-09T05:41:29.563 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[184418]: pgmap v604: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:29.563 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:29.563 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:29.563 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[184418]: failed to encode map e214 with expected crc 2023-10-09T05:41:29.563 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[184418]: osdmap e214: 8 total, 8 up, 7 in 2023-10-09T05:41:29.563 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[184418]: failed to encode map e214 with expected crc 2023-10-09T05:41:29.563 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[184418]: failed to encode map e214 with expected crc 2023-10-09T05:41:29.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:29.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:29.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[191227]: failed to encode map e213 with expected crc 2023-10-09T05:41:29.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[191227]: failed to encode map e213 with expected crc 2023-10-09T05:41:29.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[191227]: pgmap v604: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:29.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:29.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:29.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[191227]: failed to encode map e214 with expected crc 2023-10-09T05:41:29.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[191227]: osdmap e214: 8 total, 8 up, 7 in 2023-10-09T05:41:29.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[191227]: failed to encode map e214 with expected crc 2023-10-09T05:41:29.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:29 smithi008 ceph-mon[191227]: failed to encode map e214 with expected crc 2023-10-09T05:41:29.665 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:29.666 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:41:29.676 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:29.698 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:29.707 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:41:29.707 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:41:29.707 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T05:41:29.707 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:41:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:29 smithi191 ceph-mon[161669]: failed to encode map e213 with expected crc 2023-10-09T05:41:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:29 smithi191 ceph-mon[161669]: failed to encode map e213 with expected crc 2023-10-09T05:41:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:29 smithi191 ceph-mon[161669]: pgmap v604: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:29 smithi191 ceph-mon[161669]: failed to encode map e214 with expected crc 2023-10-09T05:41:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:29 smithi191 ceph-mon[161669]: osdmap e214: 8 total, 8 up, 7 in 2023-10-09T05:41:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:29 smithi191 ceph-mon[161669]: failed to encode map e214 with expected crc 2023-10-09T05:41:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:29 smithi191 ceph-mon[161669]: failed to encode map e214 with expected crc 2023-10-09T05:41:29.802 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:29.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:29.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:29.908 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:29.938 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:29.982 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:29.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:29.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:30.086 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:30.114 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:30.238 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:30.279 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:30.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:30.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:30.339 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:41:30.383 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:30.409 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:30.451 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:30.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:30.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:30.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:30 smithi191 ceph-mon[161669]: failed to encode map e214 with expected crc 2023-10-09T05:41:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:30 smithi191 ceph-mon[161669]: failed to encode map e214 with expected crc 2023-10-09T05:41:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:30 smithi191 ceph-mon[161669]: failed to encode map e215 with expected crc 2023-10-09T05:41:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:30 smithi191 ceph-mon[161669]: osdmap e215: 8 total, 8 up, 7 in 2023-10-09T05:41:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3851906785' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:30 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:30.558 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:30.589 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:30.688 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:30.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:41:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:41:30] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.33.4" 2023-10-09T05:41:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[184418]: failed to encode map e214 with expected crc 2023-10-09T05:41:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[184418]: failed to encode map e214 with expected crc 2023-10-09T05:41:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[184418]: failed to encode map e215 with expected crc 2023-10-09T05:41:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[184418]: osdmap e215: 8 total, 8 up, 7 in 2023-10-09T05:41:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3851906785' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:30.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[191227]: failed to encode map e214 with expected crc 2023-10-09T05:41:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[191227]: failed to encode map e214 with expected crc 2023-10-09T05:41:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[191227]: failed to encode map e215 with expected crc 2023-10-09T05:41:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[191227]: osdmap e215: 8 total, 8 up, 7 in 2023-10-09T05:41:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3851906785' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:30.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:30 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:30.985 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:30.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:30.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:31.078 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:41:31.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:31.117 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:31.210 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:31.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:31.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:31.314 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:31.344 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:31.541 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:31.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:31.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:31.645 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:31.674 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:31.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[184418]: failed to encode map e215 with expected crc 2023-10-09T05:41:31.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[184418]: pgmap v607: 89 pgs: 5 creating+peering, 9 unknown, 75 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:31.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:31.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[184418]: failed to encode map e216 with expected crc 2023-10-09T05:41:31.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:31.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[184418]: osdmap e216: 8 total, 8 up, 7 in 2023-10-09T05:41:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[191227]: failed to encode map e215 with expected crc 2023-10-09T05:41:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[191227]: pgmap v607: 89 pgs: 5 creating+peering, 9 unknown, 75 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[191227]: failed to encode map e216 with expected crc 2023-10-09T05:41:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:31 smithi008 ceph-mon[191227]: osdmap e216: 8 total, 8 up, 7 in 2023-10-09T05:41:31.720 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:31.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:31.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:31 smithi191 ceph-mon[161669]: failed to encode map e215 with expected crc 2023-10-09T05:41:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:31 smithi191 ceph-mon[161669]: pgmap v607: 89 pgs: 5 creating+peering, 9 unknown, 75 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:31 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:31 smithi191 ceph-mon[161669]: failed to encode map e216 with expected crc 2023-10-09T05:41:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:31 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:31 smithi191 ceph-mon[161669]: osdmap e216: 8 total, 8 up, 7 in 2023-10-09T05:41:31.824 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:31.853 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:32.040 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:41:32.040 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:41:32.040 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:41:32.040 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:41:32.040 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:41:32.111 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:32.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:32.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:32.174 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:32.217 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:32.245 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:32.344 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:32.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:32.448 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:32.476 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:32.513 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:41:32.515 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:41:32.705 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:41:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:32 smithi191 ceph-mon[161669]: failed to encode map e216 with expected crc 2023-10-09T05:41:32.895 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:32 smithi008 ceph-mon[184418]: failed to encode map e216 with expected crc 2023-10-09T05:41:32.895 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:32 smithi008 ceph-mon[191227]: failed to encode map e216 with expected crc 2023-10-09T05:41:33.057 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:33.377 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:33.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:33.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:33.482 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:33.515 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:33.658 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:33.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:33.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:33.759 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":609,"stamp":"2023-10-09T05:41:32.157415+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":4789622,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":233,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2296692,"kb_used_data":28796,"kb_used_omap":31,"kb_used_meta":2267808,"kb_avail":653919372,"statfs":{"total":671965249536,"available":669613436928,"internally_reserved":0,"allocated":29487104,"data_stored":24695626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2322235849},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.094027"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":197,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067967+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:41:28.067967+0000","last_peered":"2023-10-09T05:41:28.067967+0000","last_clean":"2023-10-09T05:41:28.067967+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:41:28.067967+0000","last_undegraded":"2023-10-09T05:41:28.067967+0000","last_fullsized":"2023-10-09T05:41:28.067967+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":187,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T05:41:29.070590+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:41:29.070590+0000","last_peered":"2023-10-09T05:41:29.070590+0000","last_clean":"2023-10-09T05:41:29.070590+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:41:29.070590+0000","last_undegraded":"2023-10-09T05:41:29.070590+0000","last_fullsized":"2023-10-09T05:41:29.070590+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":160,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.264017+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:41:23.264017+0000","last_peered":"2023-10-09T05:41:23.264017+0000","last_clean":"2023-10-09T05:41:23.264017+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:41:23.264017+0000","last_undegraded":"2023-10-09T05:41:23.264017+0000","last_fullsized":"2023-10-09T05:41:23.264017+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":190,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.809906+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:41:23.809906+0000","last_peered":"2023-10-09T05:41:23.809906+0000","last_clean":"2023-10-09T05:41:23.809906+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:41:23.809906+0000","last_undegraded":"2023-10-09T05:41:23.809906+0000","last_fullsized":"2023-10-09T05:41:23.809906+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":203,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.071122+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:41:28.071122+0000","last_peered":"2023-10-09T05:41:28.071122+0000","last_clean":"2023-10-09T05:41:28.071122+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:41:28.071122+0000","last_undegraded":"2023-10-09T05:41:28.071122+0000","last_fullsized":"2023-10-09T05:41:28.071122+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":180,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.068819+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:41:28.068819+0000","last_peered":"2023-10-09T05:41:28.068819+0000","last_clean":"2023-10-09T05:41:28.068819+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:41:28.068819+0000","last_undegraded":"2023-10-09T05:41:28.068819+0000","last_fullsized":"2023-10-09T05:41:28.068819+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":165,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.809836+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:41:23.809836+0000","last_peered":"2023-10-09T05:41:23.809836+0000","last_clean":"2023-10-09T05:41:23.809836+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:41:23.809836+0000","last_undegraded":"2023-10-09T05:41:23.809836+0000","last_fullsized":"2023-10-09T05:41:23.809836+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":187,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.071234+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:41:28.071234+0000","last_peered":"2023-10-09T05:41:28.071234+0000","last_clean":"2023-10-09T05:41:28.071234+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:41:28.071234+0000","last_undegraded":"2023-10-09T05:41:28.071234+0000","last_fullsized":"2023-10-09T05:41:28.071234+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":205,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.069628+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:41:28.069628+0000","last_peered":"2023-10-09T05:41:28.069628+0000","last_clean":"2023-10-09T05:41:28.069628+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:41:28.069628+0000","last_undegraded":"2023-10-09T05:41:28.069628+0000","last_fullsized":"2023-10-09T05:41:28.069628+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":201,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.071913+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:41:28.071913+0000","last_peered":"2023-10-09T05:41:28.071913+0000","last_clean":"2023-10-09T05:41:28.071913+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:41:28.071913+0000","last_undegraded":"2023-10-09T05:41:28.071913+0000","last_fullsized":"2023-10-09T05:41:28.071913+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":94,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.071347+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:41:28.071347+0000","last_peered":"2023-10-09T05:41:28.071347+0000","last_clean":"2023-10-09T05:41:28.071347+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:41:28.071347+0000","last_undegraded":"2023-10-09T05:41:28.071347+0000","last_fullsized":"2023-10-09T05:41:28.071347+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":160,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.264235+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:41:23.264235+0000","last_peered":"2023-10-09T05:41:23.264235+0000","last_clean":"2023-10-09T05:41:23.264235+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:41:23.264235+0000","last_undegraded":"2023-10-09T05:41:23.264235+0000","last_fullsized":"2023-10-09T05:41:23.264235+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":222,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067217+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:41:28.067217+0000","last_peered":"2023-10-09T05:41:28.067217+0000","last_clean":"2023-10-09T05:41:28.067217+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:41:28.067217+0000","last_undegraded":"2023-10-09T05:41:28.067217+0000","last_fullsized":"2023-10-09T05:41:28.067217+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":180,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067448+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:41:28.067448+0000","last_peered":"2023-10-09T05:41:28.067448+0000","last_clean":"2023-10-09T05:41:28.067448+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:41:28.067448+0000","last_undegraded":"2023-10-09T05:41:28.067448+0000","last_fullsized":"2023-10-09T05:41:28.067448+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":93,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067549+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:41:28.067549+0000","last_peered":"2023-10-09T05:41:28.067549+0000","last_clean":"2023-10-09T05:41:28.067549+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:41:28.067549+0000","last_undegraded":"2023-10-09T05:41:28.067549+0000","last_fullsized":"2023-10-09T05:41:28.067549+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":211,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.071202+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:41:28.071202+0000","last_peered":"2023-10-09T05:41:28.071202+0000","last_clean":"2023-10-09T05:41:28.071202+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:41:28.071202+0000","last_undegraded":"2023-10-09T05:41:28.071202+0000","last_fullsized":"2023-10-09T05:41:28.071202+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":189,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.809333+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:41:23.809333+0000","last_peered":"2023-10-09T05:41:23.809333+0000","last_clean":"2023-10-09T05:41:23.809333+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:41:23.809333+0000","last_undegraded":"2023-10-09T05:41:23.809333+0000","last_fullsized":"2023-10-09T05:41:23.809333+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":204,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.069857+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:41:28.069857+0000","last_peered":"2023-10-09T05:41:28.069857+0000","last_clean":"2023-10-09T05:41:28.069857+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:41:28.069857+0000","last_undegraded":"2023-10-09T05:41:28.069857+0000","last_fullsized":"2023-10-09T05:41:28.069857+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":111,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.066940+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:41:28.066940+0000","last_peered":"2023-10-09T05:41:28.066940+0000","last_clean":"2023-10-09T05:41:28.066940+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:41:28.066940+0000","last_undegraded":"2023-10-09T05:41:28.066940+0000","last_fullsized":"2023-10-09T05:41:28.066940+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":189,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.809816+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:41:23.809816+0000","last_peered":"2023-10-09T05:41:23.809816+0000","last_clean":"2023-10-09T05:41:23.809816+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:41:23.809816+0000","last_undegraded":"2023-10-09T05:41:23.809816+0000","last_fullsized":"2023-10-09T05:41:23.809816+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067535+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:41:28.067535+0000","last_peered":"2023-10-09T05:41:28.067535+0000","last_clean":"2023-10-09T05:41:28.067535+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:41:28.067535+0000","last_undegraded":"2023-10-09T05:41:28.067535+0000","last_fullsized":"2023-10-09T05:41:28.067535+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":211,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.072665+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:41:28.072665+0000","last_peered":"2023-10-09T05:41:28.072665+0000","last_clean":"2023-10-09T05:41:28.072665+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:41:28.072665+0000","last_undegraded":"2023-10-09T05:41:28.072665+0000","last_fullsized":"2023-10-09T05:41:28.072665+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":1,"reported_epoch":213,"state":"creating+peering","last_fresh":"2023-10-09T05:41:28.063654+0000","last_change":"2023-10-09T05:41:28.063654+0000","last_active":"2023-10-09T05:41:28.058397+0000","last_peered":"2023-10-09T05:41:28.058397+0000","last_clean":"2023-10-09T05:41:28.058397+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T05:41:28.063654+0000","last_undegraded":"2023-10-09T05:41:28.063654+0000","last_fullsized":"2023-10-09T05:41:28.063654+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:29.784837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":91,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067393+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:41:28.067393+0000","last_peered":"2023-10-09T05:41:28.067393+0000","last_clean":"2023-10-09T05:41:28.067393+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:41:28.067393+0000","last_undegraded":"2023-10-09T05:41:28.067393+0000","last_fullsized":"2023-10-09T05:41:28.067393+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.069363+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:41:28.069363+0000","last_peered":"2023-10-09T05:41:28.069363+0000","last_clean":"2023-10-09T05:41:28.069363+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:41:28.069363+0000","last_undegraded":"2023-10-09T05:41:28.069363+0000","last_fullsized":"2023-10-09T05:41:28.069363+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":10,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T05:41:29.081574+0000","last_change":"2023-10-09T05:41:29.081067+0000","last_active":"2023-10-09T05:41:29.081574+0000","last_peered":"2023-10-09T05:41:29.081574+0000","last_clean":"2023-10-09T05:41:29.081574+0000","last_became_active":"2023-10-09T05:41:29.080704+0000","last_became_peered":"2023-10-09T05:41:29.080704+0000","last_unstale":"2023-10-09T05:41:29.081574+0000","last_undegraded":"2023-10-09T05:41:29.081574+0000","last_fullsized":"2023-10-09T05:41:29.081574+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:29.483205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":91,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067454+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:41:28.067454+0000","last_peered":"2023-10-09T05:41:28.067454+0000","last_clean":"2023-10-09T05:41:28.067454+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:41:28.067454+0000","last_undegraded":"2023-10-09T05:41:28.067454+0000","last_fullsized":"2023-10-09T05:41:28.067454+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":195,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.066890+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:41:28.066890+0000","last_peered":"2023-10-09T05:41:28.066890+0000","last_clean":"2023-10-09T05:41:28.066890+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:41:28.066890+0000","last_undegraded":"2023-10-09T05:41:28.066890+0000","last_fullsized":"2023-10-09T05:41:28.066890+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":1,"reported_epoch":213,"state":"creating+peering","last_fresh":"2023-10-09T05:41:28.070466+0000","last_change":"2023-10-09T05:41:28.070466+0000","last_active":"2023-10-09T05:41:28.058397+0000","last_peered":"2023-10-09T05:41:28.058397+0000","last_clean":"2023-10-09T05:41:28.058397+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T05:41:28.070466+0000","last_undegraded":"2023-10-09T05:41:28.070466+0000","last_fullsized":"2023-10-09T05:41:28.070466+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:48.684759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":91,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.069944+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:41:28.069944+0000","last_peered":"2023-10-09T05:41:28.069944+0000","last_clean":"2023-10-09T05:41:28.069944+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:41:28.069944+0000","last_undegraded":"2023-10-09T05:41:28.069944+0000","last_fullsized":"2023-10-09T05:41:28.069944+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":213,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.068444+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:41:28.068444+0000","last_peered":"2023-10-09T05:41:28.068444+0000","last_clean":"2023-10-09T05:41:28.068444+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:41:28.068444+0000","last_undegraded":"2023-10-09T05:41:28.068444+0000","last_fullsized":"2023-10-09T05:41:28.068444+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":12,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:31.073704+0000","last_change":"2023-10-09T05:41:29.073764+0000","last_active":"2023-10-09T05:41:31.073704+0000","last_peered":"2023-10-09T05:41:31.073704+0000","last_clean":"2023-10-09T05:41:31.073704+0000","last_became_active":"2023-10-09T05:41:29.073452+0000","last_became_peered":"2023-10-09T05:41:29.073452+0000","last_unstale":"2023-10-09T05:41:31.073704+0000","last_undegraded":"2023-10-09T05:41:31.073704+0000","last_fullsized":"2023-10-09T05:41:31.073704+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:22.328530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":181,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.070164+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:41:28.070164+0000","last_peered":"2023-10-09T05:41:28.070164+0000","last_clean":"2023-10-09T05:41:28.070164+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:41:28.070164+0000","last_undegraded":"2023-10-09T05:41:28.070164+0000","last_fullsized":"2023-10-09T05:41:28.070164+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":166,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.809603+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:41:23.809603+0000","last_peered":"2023-10-09T05:41:23.809603+0000","last_clean":"2023-10-09T05:41:23.809603+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:41:23.809603+0000","last_undegraded":"2023-10-09T05:41:23.809603+0000","last_fullsized":"2023-10-09T05:41:23.809603+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":1,"reported_epoch":213,"state":"creating+peering","last_fresh":"2023-10-09T05:41:28.070504+0000","last_change":"2023-10-09T05:41:28.070504+0000","last_active":"2023-10-09T05:41:28.058397+0000","last_peered":"2023-10-09T05:41:28.058397+0000","last_clean":"2023-10-09T05:41:28.058397+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T05:41:28.070504+0000","last_undegraded":"2023-10-09T05:41:28.070504+0000","last_fullsized":"2023-10-09T05:41:28.070504+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:18.625214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":185,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.073397+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:41:28.073397+0000","last_peered":"2023-10-09T05:41:28.073397+0000","last_clean":"2023-10-09T05:41:28.073397+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:41:28.073397+0000","last_undegraded":"2023-10-09T05:41:28.073397+0000","last_fullsized":"2023-10-09T05:41:28.073397+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":180,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067119+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:41:28.067119+0000","last_peered":"2023-10-09T05:41:28.067119+0000","last_clean":"2023-10-09T05:41:28.067119+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:41:28.067119+0000","last_undegraded":"2023-10-09T05:41:28.067119+0000","last_fullsized":"2023-10-09T05:41:28.067119+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":12,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:31.074186+0000","last_change":"2023-10-09T05:41:29.078791+0000","last_active":"2023-10-09T05:41:31.074186+0000","last_peered":"2023-10-09T05:41:31.074186+0000","last_clean":"2023-10-09T05:41:31.074186+0000","last_became_active":"2023-10-09T05:41:29.078491+0000","last_became_peered":"2023-10-09T05:41:29.078491+0000","last_unstale":"2023-10-09T05:41:31.074186+0000","last_undegraded":"2023-10-09T05:41:31.074186+0000","last_fullsized":"2023-10-09T05:41:31.074186+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:37.817282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":181,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.068252+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:41:28.068252+0000","last_peered":"2023-10-09T05:41:28.068252+0000","last_clean":"2023-10-09T05:41:28.068252+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:41:28.068252+0000","last_undegraded":"2023-10-09T05:41:28.068252+0000","last_fullsized":"2023-10-09T05:41:28.068252+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":195,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067076+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:41:28.067076+0000","last_peered":"2023-10-09T05:41:28.067076+0000","last_clean":"2023-10-09T05:41:28.067076+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:41:28.067076+0000","last_undegraded":"2023-10-09T05:41:28.067076+0000","last_fullsized":"2023-10-09T05:41:28.067076+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":9,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T05:41:29.078645+0000","last_change":"2023-10-09T05:41:29.078645+0000","last_active":"2023-10-09T05:41:29.078645+0000","last_peered":"2023-10-09T05:41:29.078645+0000","last_clean":"2023-10-09T05:41:29.078645+0000","last_became_active":"2023-10-09T05:41:29.078355+0000","last_became_peered":"2023-10-09T05:41:29.078355+0000","last_unstale":"2023-10-09T05:41:29.078645+0000","last_undegraded":"2023-10-09T05:41:29.078645+0000","last_fullsized":"2023-10-09T05:41:29.078645+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":205,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.066806+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:41:28.066806+0000","last_peered":"2023-10-09T05:41:28.066806+0000","last_clean":"2023-10-09T05:41:28.066806+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:41:28.066806+0000","last_undegraded":"2023-10-09T05:41:28.066806+0000","last_fullsized":"2023-10-09T05:41:28.066806+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067540+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:41:28.067540+0000","last_peered":"2023-10-09T05:41:28.067540+0000","last_clean":"2023-10-09T05:41:28.067540+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:41:28.067540+0000","last_undegraded":"2023-10-09T05:41:28.067540+0000","last_fullsized":"2023-10-09T05:41:28.067540+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":9,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T05:41:29.082194+0000","last_change":"2023-10-09T05:41:29.082194+0000","last_active":"2023-10-09T05:41:29.082194+0000","last_peered":"2023-10-09T05:41:29.082194+0000","last_clean":"2023-10-09T05:41:29.082194+0000","last_became_active":"2023-10-09T05:41:29.081976+0000","last_became_peered":"2023-10-09T05:41:29.081976+0000","last_unstale":"2023-10-09T05:41:29.082194+0000","last_undegraded":"2023-10-09T05:41:29.082194+0000","last_fullsized":"2023-10-09T05:41:29.082194+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":212,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.072246+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:41:28.072246+0000","last_peered":"2023-10-09T05:41:28.072246+0000","last_clean":"2023-10-09T05:41:28.072246+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:41:28.072246+0000","last_undegraded":"2023-10-09T05:41:28.072246+0000","last_fullsized":"2023-10-09T05:41:28.072246+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":160,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.264399+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:41:23.264399+0000","last_peered":"2023-10-09T05:41:23.264399+0000","last_clean":"2023-10-09T05:41:23.264399+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:41:23.264399+0000","last_undegraded":"2023-10-09T05:41:23.264399+0000","last_fullsized":"2023-10-09T05:41:23.264399+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":9,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T05:41:29.079588+0000","last_change":"2023-10-09T05:41:29.079588+0000","last_active":"2023-10-09T05:41:29.079588+0000","last_peered":"2023-10-09T05:41:29.079588+0000","last_clean":"2023-10-09T05:41:29.079588+0000","last_became_active":"2023-10-09T05:41:29.079288+0000","last_became_peered":"2023-10-09T05:41:29.079288+0000","last_unstale":"2023-10-09T05:41:29.079588+0000","last_undegraded":"2023-10-09T05:41:29.079588+0000","last_fullsized":"2023-10-09T05:41:29.079588+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":783,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.264133+0000","last_change":"2023-10-09T05:40:04.806908+0000","last_active":"2023-10-09T05:41:23.264133+0000","last_peered":"2023-10-09T05:41:23.264133+0000","last_clean":"2023-10-09T05:41:23.264133+0000","last_became_active":"2023-10-09T05:40:04.806232+0000","last_became_peered":"2023-10-09T05:40:04.806232+0000","last_unstale":"2023-10-09T05:41:23.264133+0000","last_undegraded":"2023-10-09T05:41:23.264133+0000","last_fullsized":"2023-10-09T05:41:23.264133+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:34.087553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":196,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.069774+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:41:28.069774+0000","last_peered":"2023-10-09T05:41:28.069774+0000","last_clean":"2023-10-09T05:41:28.069774+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:41:28.069774+0000","last_undegraded":"2023-10-09T05:41:28.069774+0000","last_fullsized":"2023-10-09T05:41:28.069774+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":146,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.264212+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:41:23.264212+0000","last_peered":"2023-10-09T05:41:23.264212+0000","last_clean":"2023-10-09T05:41:23.264212+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:41:23.264212+0000","last_undegraded":"2023-10-09T05:41:23.264212+0000","last_fullsized":"2023-10-09T05:41:23.264212+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":9,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T05:41:29.082691+0000","last_change":"2023-10-09T05:41:29.082691+0000","last_active":"2023-10-09T05:41:29.082691+0000","last_peered":"2023-10-09T05:41:29.082691+0000","last_clean":"2023-10-09T05:41:29.082691+0000","last_became_active":"2023-10-09T05:41:29.082385+0000","last_became_peered":"2023-10-09T05:41:29.082385+0000","last_unstale":"2023-10-09T05:41:29.082691+0000","last_undegraded":"2023-10-09T05:41:29.082691+0000","last_fullsized":"2023-10-09T05:41:29.082691+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":650,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.069597+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:41:28.069597+0000","last_peered":"2023-10-09T05:41:28.069597+0000","last_clean":"2023-10-09T05:41:28.069597+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:41:28.069597+0000","last_undegraded":"2023-10-09T05:41:28.069597+0000","last_fullsized":"2023-10-09T05:41:28.069597+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":181,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067643+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:41:28.067643+0000","last_peered":"2023-10-09T05:41:28.067643+0000","last_clean":"2023-10-09T05:41:28.067643+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:41:28.067643+0000","last_undegraded":"2023-10-09T05:41:28.067643+0000","last_fullsized":"2023-10-09T05:41:28.067643+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":193,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.070289+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:41:28.070289+0000","last_peered":"2023-10-09T05:41:28.070289+0000","last_clean":"2023-10-09T05:41:28.070289+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:41:28.070289+0000","last_undegraded":"2023-10-09T05:41:28.070289+0000","last_fullsized":"2023-10-09T05:41:28.070289+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":1,"reported_epoch":213,"state":"creating+peering","last_fresh":"2023-10-09T05:41:28.070488+0000","last_change":"2023-10-09T05:41:28.070488+0000","last_active":"2023-10-09T05:41:28.058397+0000","last_peered":"2023-10-09T05:41:28.058397+0000","last_clean":"2023-10-09T05:41:28.058397+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T05:41:28.070488+0000","last_undegraded":"2023-10-09T05:41:28.070488+0000","last_fullsized":"2023-10-09T05:41:28.070488+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:53.470052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":531,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T05:41:30.068428+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:41:30.068428+0000","last_peered":"2023-10-09T05:41:30.068428+0000","last_clean":"2023-10-09T05:41:30.068428+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:41:30.068428+0000","last_undegraded":"2023-10-09T05:41:30.068428+0000","last_fullsized":"2023-10-09T05:41:30.068428+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":161,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.264017+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:41:23.264017+0000","last_peered":"2023-10-09T05:41:23.264017+0000","last_clean":"2023-10-09T05:41:23.264017+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:41:23.264017+0000","last_undegraded":"2023-10-09T05:41:23.264017+0000","last_fullsized":"2023-10-09T05:41:23.264017+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":191,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.264405+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:41:23.264405+0000","last_peered":"2023-10-09T05:41:23.264405+0000","last_clean":"2023-10-09T05:41:23.264405+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:41:23.264405+0000","last_undegraded":"2023-10-09T05:41:23.264405+0000","last_fullsized":"2023-10-09T05:41:23.264405+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":1,"reported_epoch":213,"state":"creating+peering","last_fresh":"2023-10-09T05:41:28.070610+0000","last_change":"2023-10-09T05:41:28.070610+0000","last_active":"2023-10-09T05:41:28.058397+0000","last_peered":"2023-10-09T05:41:28.058397+0000","last_clean":"2023-10-09T05:41:28.058397+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T05:41:28.070610+0000","last_undegraded":"2023-10-09T05:41:28.070610+0000","last_fullsized":"2023-10-09T05:41:28.070610+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:51.574989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":825,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.069567+0000","last_change":"2023-10-09T05:40:04.805258+0000","last_active":"2023-10-09T05:41:28.069567+0000","last_peered":"2023-10-09T05:41:28.069567+0000","last_clean":"2023-10-09T05:41:28.069567+0000","last_became_active":"2023-10-09T05:40:04.804776+0000","last_became_peered":"2023-10-09T05:40:04.804776+0000","last_unstale":"2023-10-09T05:41:28.069567+0000","last_undegraded":"2023-10-09T05:41:28.069567+0000","last_fullsized":"2023-10-09T05:41:28.069567+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":361,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.809135+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:41:23.809135+0000","last_peered":"2023-10-09T05:41:23.809135+0000","last_clean":"2023-10-09T05:41:23.809135+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:41:23.809135+0000","last_undegraded":"2023-10-09T05:41:23.809135+0000","last_fullsized":"2023-10-09T05:41:23.809135+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.071584+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:41:28.071584+0000","last_peered":"2023-10-09T05:41:28.071584+0000","last_clean":"2023-10-09T05:41:28.071584+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:41:28.071584+0000","last_undegraded":"2023-10-09T05:41:28.071584+0000","last_fullsized":"2023-10-09T05:41:28.071584+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":187,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.068991+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:41:28.068991+0000","last_peered":"2023-10-09T05:41:28.068991+0000","last_clean":"2023-10-09T05:41:28.068991+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:41:28.068991+0000","last_undegraded":"2023-10-09T05:41:28.068991+0000","last_fullsized":"2023-10-09T05:41:28.068991+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":12,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:31.074966+0000","last_change":"2023-10-09T05:41:29.076367+0000","last_active":"2023-10-09T05:41:31.074966+0000","last_peered":"2023-10-09T05:41:31.074966+0000","last_clean":"2023-10-09T05:41:31.074966+0000","last_became_active":"2023-10-09T05:41:29.076035+0000","last_became_peered":"2023-10-09T05:41:29.076035+0000","last_unstale":"2023-10-09T05:41:31.074966+0000","last_undegraded":"2023-10-09T05:41:31.074966+0000","last_fullsized":"2023-10-09T05:41:31.074966+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:51.713835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":770,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.264206+0000","last_change":"2023-10-09T05:40:05.930112+0000","last_active":"2023-10-09T05:41:23.264206+0000","last_peered":"2023-10-09T05:41:23.264206+0000","last_clean":"2023-10-09T05:41:23.264206+0000","last_became_active":"2023-10-09T05:40:05.929574+0000","last_became_peered":"2023-10-09T05:40:05.929574+0000","last_unstale":"2023-10-09T05:41:23.264206+0000","last_undegraded":"2023-10-09T05:41:23.264206+0000","last_fullsized":"2023-10-09T05:41:23.264206+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:06.982326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":166,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.809275+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:41:23.809275+0000","last_peered":"2023-10-09T05:41:23.809275+0000","last_clean":"2023-10-09T05:41:23.809275+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:41:23.809275+0000","last_undegraded":"2023-10-09T05:41:23.809275+0000","last_fullsized":"2023-10-09T05:41:23.809275+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":181,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.810079+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:41:23.810079+0000","last_peered":"2023-10-09T05:41:23.810079+0000","last_clean":"2023-10-09T05:41:23.810079+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:41:23.810079+0000","last_undegraded":"2023-10-09T05:41:23.810079+0000","last_fullsized":"2023-10-09T05:41:23.810079+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":9,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T05:41:29.073701+0000","last_change":"2023-10-09T05:41:29.073701+0000","last_active":"2023-10-09T05:41:29.073701+0000","last_peered":"2023-10-09T05:41:29.073701+0000","last_clean":"2023-10-09T05:41:29.073701+0000","last_became_active":"2023-10-09T05:41:29.073372+0000","last_became_peered":"2023-10-09T05:41:29.073372+0000","last_unstale":"2023-10-09T05:41:29.073701+0000","last_undegraded":"2023-10-09T05:41:29.073701+0000","last_fullsized":"2023-10-09T05:41:29.073701+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":402,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.810266+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:41:23.810266+0000","last_peered":"2023-10-09T05:41:23.810266+0000","last_clean":"2023-10-09T05:41:23.810266+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:41:23.810266+0000","last_undegraded":"2023-10-09T05:41:23.810266+0000","last_fullsized":"2023-10-09T05:41:23.810266+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":161,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.264414+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:41:23.264414+0000","last_peered":"2023-10-09T05:41:23.264414+0000","last_clean":"2023-10-09T05:41:23.264414+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:41:23.264414+0000","last_undegraded":"2023-10-09T05:41:23.264414+0000","last_fullsized":"2023-10-09T05:41:23.264414+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":175,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.809751+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:41:23.809751+0000","last_peered":"2023-10-09T05:41:23.809751+0000","last_clean":"2023-10-09T05:41:23.809751+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:41:23.809751+0000","last_undegraded":"2023-10-09T05:41:23.809751+0000","last_fullsized":"2023-10-09T05:41:23.809751+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":10,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T05:41:29.081618+0000","last_change":"2023-10-09T05:41:29.080413+0000","last_active":"2023-10-09T05:41:29.081618+0000","last_peered":"2023-10-09T05:41:29.081618+0000","last_clean":"2023-10-09T05:41:29.081618+0000","last_became_active":"2023-10-09T05:41:29.080184+0000","last_became_peered":"2023-10-09T05:41:29.080184+0000","last_unstale":"2023-10-09T05:41:29.081618+0000","last_undegraded":"2023-10-09T05:41:29.081618+0000","last_fullsized":"2023-10-09T05:41:29.081618+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:47.929696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":213,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.072580+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:41:28.072580+0000","last_peered":"2023-10-09T05:41:28.072580+0000","last_clean":"2023-10-09T05:41:28.072580+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:41:28.072580+0000","last_undegraded":"2023-10-09T05:41:28.072580+0000","last_fullsized":"2023-10-09T05:41:28.072580+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":602,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.264287+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:41:23.264287+0000","last_peered":"2023-10-09T05:41:23.264287+0000","last_clean":"2023-10-09T05:41:23.264287+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:41:23.264287+0000","last_undegraded":"2023-10-09T05:41:23.264287+0000","last_fullsized":"2023-10-09T05:41:23.264287+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":207,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T05:41:29.070574+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:41:29.070574+0000","last_peered":"2023-10-09T05:41:29.070574+0000","last_clean":"2023-10-09T05:41:29.070574+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:41:29.070574+0000","last_undegraded":"2023-10-09T05:41:29.070574+0000","last_fullsized":"2023-10-09T05:41:29.070574+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":10,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T05:41:29.081594+0000","last_change":"2023-10-09T05:41:29.081230+0000","last_active":"2023-10-09T05:41:29.081594+0000","last_peered":"2023-10-09T05:41:29.081594+0000","last_clean":"2023-10-09T05:41:29.081594+0000","last_became_active":"2023-10-09T05:41:29.080911+0000","last_became_peered":"2023-10-09T05:41:29.080911+0000","last_unstale":"2023-10-09T05:41:29.081594+0000","last_undegraded":"2023-10-09T05:41:29.081594+0000","last_fullsized":"2023-10-09T05:41:29.081594+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:22.580967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":715,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.068807+0000","last_change":"2023-10-09T05:41:21.917602+0000","last_active":"2023-10-09T05:41:28.068807+0000","last_peered":"2023-10-09T05:41:28.068807+0000","last_clean":"2023-10-09T05:41:28.068807+0000","last_became_active":"2023-10-09T05:41:21.917116+0000","last_became_peered":"2023-10-09T05:41:21.917116+0000","last_unstale":"2023-10-09T05:41:28.068807+0000","last_undegraded":"2023-10-09T05:41:28.068807+0000","last_fullsized":"2023-10-09T05:41:28.068807+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":169,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.810013+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:41:23.810013+0000","last_peered":"2023-10-09T05:41:23.810013+0000","last_clean":"2023-10-09T05:41:23.810013+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:41:23.810013+0000","last_undegraded":"2023-10-09T05:41:23.810013+0000","last_fullsized":"2023-10-09T05:41:23.810013+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":165,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.066802+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:41:28.066802+0000","last_peered":"2023-10-09T05:41:28.066802+0000","last_clean":"2023-10-09T05:41:28.066802+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:41:28.066802+0000","last_undegraded":"2023-10-09T05:41:28.066802+0000","last_fullsized":"2023-10-09T05:41:28.066802+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":177,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.809650+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:41:23.809650+0000","last_peered":"2023-10-09T05:41:23.809650+0000","last_clean":"2023-10-09T05:41:23.809650+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:41:23.809650+0000","last_undegraded":"2023-10-09T05:41:23.809650+0000","last_fullsized":"2023-10-09T05:41:23.809650+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":208,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T05:41:30.069107+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:41:30.069107+0000","last_peered":"2023-10-09T05:41:30.069107+0000","last_clean":"2023-10-09T05:41:30.069107+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:41:30.069107+0000","last_undegraded":"2023-10-09T05:41:30.069107+0000","last_fullsized":"2023-10-09T05:41:30.069107+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":183,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.070081+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:41:28.070081+0000","last_peered":"2023-10-09T05:41:28.070081+0000","last_clean":"2023-10-09T05:41:28.070081+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:41:28.070081+0000","last_undegraded":"2023-10-09T05:41:28.070081+0000","last_fullsized":"2023-10-09T05:41:28.070081+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":195,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.067722+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:41:28.067722+0000","last_peered":"2023-10-09T05:41:28.067722+0000","last_clean":"2023-10-09T05:41:28.067722+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:41:28.067722+0000","last_undegraded":"2023-10-09T05:41:28.067722+0000","last_fullsized":"2023-10-09T05:41:28.067722+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":168,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.069687+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:41:28.069687+0000","last_peered":"2023-10-09T05:41:28.069687+0000","last_clean":"2023-10-09T05:41:28.069687+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:41:28.069687+0000","last_undegraded":"2023-10-09T05:41:28.069687+0000","last_fullsized":"2023-10-09T05:41:28.069687+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":195,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.066862+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:41:28.066862+0000","last_peered":"2023-10-09T05:41:28.066862+0000","last_clean":"2023-10-09T05:41:28.066862+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:41:28.066862+0000","last_undegraded":"2023-10-09T05:41:28.066862+0000","last_fullsized":"2023-10-09T05:41:28.066862+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":194,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T05:41:29.069902+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:41:29.069902+0000","last_peered":"2023-10-09T05:41:29.069902+0000","last_clean":"2023-10-09T05:41:29.069902+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:41:29.069902+0000","last_undegraded":"2023-10-09T05:41:29.069902+0000","last_fullsized":"2023-10-09T05:41:29.069902+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":206,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.071831+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:41:28.071831+0000","last_peered":"2023-10-09T05:41:28.071831+0000","last_clean":"2023-10-09T05:41:28.071831+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:41:28.071831+0000","last_undegraded":"2023-10-09T05:41:28.071831+0000","last_fullsized":"2023-10-09T05:41:28.071831+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":169,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T05:41:23.809848+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:41:23.809848+0000","last_peered":"2023-10-09T05:41:23.809848+0000","last_clean":"2023-10-09T05:41:23.809848+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:41:23.809848+0000","last_undegraded":"2023-10-09T05:41:23.809848+0000","last_fullsized":"2023-10-09T05:41:23.809848+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":180,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T05:41:28.066666+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:41:28.066666+0000","last_peered":"2023-10-09T05:41:28.066666+0000","last_clean":"2023-10-09T05:41:28.066666+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:41:28.066666+0000","last_undegraded":"2023-10-09T05:41:28.066666+0000","last_fullsized":"2023-10-09T05:41:28.066666+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846675,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626701,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312032,"kb_used_data":6616,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93433120,"statfs":{"total":95995035648,"available":95675514880,"internally_reserved":0,"allocated":6774784,"data_stored":6243952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:28 2023","interfaces":[{"interface":"back","average":{"1min":2.032,"5min":5.6829999999999998,"15min":3.2210000000000001},"min":{"1min":0.23100000000000001,"5min":0.214,"15min":0.214},"max":{"1min":14.23,"5min":115.377,"15min":115.377},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.9350000000000001,"5min":5.3710000000000004,"15min":3.0800000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23999999999999999},"max":{"1min":11.727,"5min":116.428,"15min":116.428},"last":0.49399999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":7.274,"5min":6.056,"15min":2.7170000000000001},"min":{"1min":0.315,"5min":0.25,"15min":0.25},"max":{"1min":93.489000000000004,"5min":137.02199999999999,"15min":137.02199999999999},"last":0.433},{"interface":"front","average":{"1min":6.6299999999999999,"5min":5.9379999999999997,"15min":2.758},"min":{"1min":0.318,"5min":0.252,"15min":0.252},"max":{"1min":93.471999999999994,"5min":137.06700000000001,"15min":137.06700000000001},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:41:28 2023","interfaces":[{"interface":"back","average":{"1min":7.8099999999999996,"5min":6.226,"15min":3.456},"min":{"1min":0.32200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":87.703000000000003,"5min":115.40600000000001,"15min":115.40600000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":8.4559999999999995,"5min":6.5220000000000002,"15min":3.4980000000000002},"min":{"1min":0.35499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":87.689999999999998,"5min":115.297,"15min":115.297},"last":0.46500000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":11.682,"5min":6.7359999999999998,"15min":3.1219999999999999},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":99.543999999999997,"5min":115.244,"15min":115.244},"last":0.40600000000000003},{"interface":"front","average":{"1min":10.734,"5min":6.556,"15min":3.1499999999999999},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":98.677999999999997,"5min":115.34999999999999,"15min":115.34999999999999},"last":0.56399999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.8200000000000001,"15min":1.7110000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":0.72399999999999998,"5min":115.214,"15min":125.64},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.70399999999999996,"5min":1.5820000000000001,"15min":1.6120000000000001},"min":{"1min":0.27400000000000002,"5min":0.254,"15min":0.254},"max":{"1min":2.7189999999999999,"5min":115.22799999999999,"15min":125.625},"last":0.54600000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":2.5600000000000001,"15min":1.855},"min":{"1min":0.33500000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4690000000000003,"5min":115.316,"15min":127.35899999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":2.1549999999999998,"15min":1.7370000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.6079999999999997,"5min":115.30500000000001,"15min":128.93799999999999},"last":0.51900000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":2.077,"15min":2.7909999999999999},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":5.0110000000000001,"5min":115.514,"15min":115.514},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.8240000000000001,"15min":2.0019999999999998},"min":{"1min":0.34899999999999998,"5min":0.317,"15min":0.317},"max":{"1min":3.6600000000000001,"5min":116.53100000000001,"15min":116.53100000000001},"last":0.57299999999999995}]}]},{"osd":1,"up_from":93,"seq":399431958662,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":268768,"kb_used_data":2776,"kb_used_omap":7,"kb_used_meta":265976,"kb_avail":93476384,"statfs":{"total":95995035648,"available":95719817216,"internally_reserved":0,"allocated":2842624,"data_stored":2072582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":272359950},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":4.694,"5min":5.1500000000000004,"15min":2.6080000000000001},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":39.189,"5min":207.858,"15min":207.858},"last":0.33000000000000002},{"interface":"front","average":{"1min":6.1989999999999998,"5min":6.4420000000000002,"15min":3.0720000000000001},"min":{"1min":0.23000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":42.012999999999998,"5min":206.97200000000001,"15min":206.97200000000001},"last":0.35399999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:41:11 2023","interfaces":[{"interface":"back","average":{"1min":6.2229999999999999,"5min":7.3860000000000001,"15min":3.5019999999999998},"min":{"1min":0.26000000000000001,"5min":0.22,"15min":0.184},"max":{"1min":42.469999999999999,"5min":101.367,"15min":101.367},"last":0.34000000000000002},{"interface":"front","average":{"1min":5.3369999999999997,"5min":6.9020000000000001,"15min":3.294},"min":{"1min":0.23300000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":42.151000000000003,"5min":97.286000000000001,"15min":98.409999999999997},"last":0.28999999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":18.629999999999999,"5min":7.2770000000000001,"15min":2.9540000000000002},"min":{"1min":0.28899999999999998,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":162.33699999999999,"5min":162.33699999999999,"15min":162.33699999999999},"last":0.31},{"interface":"front","average":{"1min":19.029,"5min":7.9980000000000002,"15min":3.1960000000000002},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":162.376,"5min":162.376,"15min":162.376},"last":0.372}]},{"osd":4,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.1749999999999998,"5min":4.7249999999999996,"15min":2.2690000000000001},"min":{"1min":0.39700000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":42.106999999999999,"5min":141.32900000000001,"15min":141.32900000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":5.2800000000000002,"5min":3.6819999999999999,"15min":1.8400000000000001},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":40.475999999999999,"5min":141.15899999999999,"15min":141.15899999999999},"last":0.47399999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.6420000000000003,"5min":4.7009999999999996,"15min":2.2349999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":40.633000000000003,"5min":235.18700000000001,"15min":235.18700000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":5.258,"5min":5.5570000000000004,"15min":2.6240000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":42.078000000000003,"5min":235.79300000000001,"15min":235.79300000000001},"last":0.501}]},{"osd":6,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.8860000000000001,"5min":3.8109999999999999,"15min":1.8779999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":40.558999999999997,"5min":103.60899999999999,"15min":103.60899999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":5.234,"5min":4.4950000000000001,"15min":2.141},"min":{"1min":0.41299999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":42.134999999999998,"5min":103.679,"15min":103.679},"last":0.40500000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.4950000000000001,"5min":3.649,"15min":2.5489999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":42.204000000000001,"5min":126.68300000000001,"15min":126.68300000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":5.4050000000000002,"5min":3.7450000000000001,"15min":2.6339999999999999},"min":{"1min":0.46999999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":42.228999999999999,"5min":126.738,"15min":126.738},"last":0.51500000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122188,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":358844,"kb_used_data":2804,"kb_used_omap":8,"kb_used_meta":356023,"kb_avail":93386308,"statfs":{"total":95995035648,"available":95627579392,"internally_reserved":0,"allocated":2871296,"data_stored":2067316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":364568328},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4910000000000001,"5min":2.3340000000000001,"15min":1.282},"min":{"1min":0.20699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":17.760000000000002,"5min":17.760000000000002,"15min":17.760000000000002},"last":0.221},{"interface":"front","average":{"1min":6.0330000000000004,"5min":3.3679999999999999,"15min":1.6539999999999999},"min":{"1min":0.25,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":26.163,"5min":26.163,"15min":26.163},"last":0.35099999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":7.7830000000000004,"5min":4.2309999999999999,"15min":2.6949999999999998},"min":{"1min":0.23999999999999999,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":92.230000000000004,"5min":92.230000000000004,"15min":92.230000000000004},"last":0.248},{"interface":"front","average":{"1min":8.4440000000000008,"5min":4.3079999999999998,"15min":2.6419999999999999},"min":{"1min":0.253,"5min":0.221,"15min":0.218},"max":{"1min":92.247,"5min":92.247,"15min":92.247},"last":0.29699999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":18.498999999999999,"5min":6.4859999999999998,"15min":2.7669999999999999},"min":{"1min":0.27500000000000002,"5min":0.26000000000000001,"15min":0.22800000000000001},"max":{"1min":123.024,"5min":123.024,"15min":123.024},"last":0.316},{"interface":"front","average":{"1min":12.829000000000001,"5min":5.4359999999999999,"15min":2.48},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":99.162000000000006,"5min":99.162000000000006,"15min":99.162000000000006},"last":0.28699999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.9970000000000001,"15min":1.288},"min":{"1min":0.32200000000000001,"5min":0.27500000000000002,"15min":0.27200000000000002},"max":{"1min":8.6470000000000002,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.206,"5min":2.778,"15min":1.6799999999999999},"min":{"1min":0.41399999999999998,"5min":0.33000000000000002,"15min":0.30399999999999999},"max":{"1min":7.9240000000000004,"5min":69.102000000000004,"15min":69.102000000000004},"last":0.53500000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":3.2450000000000001,"5min":6.4889999999999999,"15min":2.907},"min":{"1min":0.36299999999999999,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":8.0289999999999999,"5min":281.733,"15min":281.733},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.6080000000000001,"5min":6.3609999999999998,"15min":2.8650000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.4269999999999996,"5min":282.95400000000001,"15min":282.95400000000001},"last":0.51700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":3.266,"5min":2.2610000000000001,"15min":1.4990000000000001},"min":{"1min":0.39800000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9969999999999999,"5min":11.664,"15min":11.664},"last":0.50600000000000001},{"interface":"front","average":{"1min":2.355,"5min":2.2200000000000002,"15min":1.4990000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.8490000000000002,"5min":10.83,"15min":10.83},"last":0.54300000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.679,"15min":1.4450000000000001},"min":{"1min":0.41899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.5090000000000003,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.49199999999999999},{"interface":"front","average":{"1min":3.1859999999999999,"5min":2.548,"15min":2.4420000000000002},"min":{"1min":0.46800000000000003,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.6329999999999991,"5min":9.7159999999999993,"15min":11.068},"last":0.55400000000000005}]}]},{"osd":2,"up_from":98,"seq":420906795136,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":405924,"kb_used_data":6428,"kb_used_omap":11,"kb_used_meta":399476,"kb_avail":93339228,"statfs":{"total":95995035648,"available":95579369472,"internally_reserved":0,"allocated":6582272,"data_stored":5713861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":409064021},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":4.3230000000000004,"5min":3.4670000000000001,"15min":1.7949999999999999},"min":{"1min":0.215,"5min":0.17799999999999999,"15min":0.17199999999999999},"max":{"1min":21.587,"5min":21.587,"15min":32.122},"last":0.23899999999999999},{"interface":"front","average":{"1min":3.702,"5min":3.4889999999999999,"15min":1.7430000000000001},"min":{"1min":0.214,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":15.468,"5min":21.997,"15min":34.066000000000003},"last":0.27000000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":5.5529999999999999,"15min":2.6539999999999999},"min":{"1min":0.29199999999999998,"5min":0.23400000000000001,"15min":0.20699999999999999},"max":{"1min":24.265000000000001,"5min":98.575000000000003,"15min":98.575000000000003},"last":0.30499999999999999},{"interface":"front","average":{"1min":4.5599999999999996,"5min":4.5069999999999997,"15min":2.282},"min":{"1min":0.23100000000000001,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":18.882999999999999,"5min":90.135000000000005,"15min":98.397000000000006},"last":0.217}]},{"osd":3,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":5.7240000000000002,"5min":6.1200000000000001,"15min":2.6499999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.24199999999999999},"max":{"1min":24.471,"5min":150.65899999999999,"15min":150.65899999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":6.6200000000000001,"5min":6.9770000000000003,"15min":3.016},"min":{"1min":0.46000000000000002,"5min":0.29899999999999999,"15min":0.26500000000000001},"max":{"1min":22.631,"5min":150.851,"15min":150.851},"last":0.32400000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6629999999999998,"5min":3.2719999999999998,"15min":1.7210000000000001},"min":{"1min":0.32200000000000001,"5min":0.29099999999999998,"15min":0.27700000000000002},"max":{"1min":14.382,"5min":115.84999999999999,"15min":115.84999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.5449999999999999,"5min":3.7109999999999999,"15min":1.9770000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.29799999999999999},"max":{"1min":13.815,"5min":116.149,"15min":116.149},"last":0.48799999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":2.8959999999999999,"15min":1.6870000000000001},"min":{"1min":0.36699999999999999,"5min":0.312,"15min":0.312},"max":{"1min":13.712999999999999,"5min":24.597999999999999,"15min":24.597999999999999},"last":0.434},{"interface":"front","average":{"1min":3.222,"5min":2.302,"15min":1.466},"min":{"1min":0.35499999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":14.426,"5min":23.919,"15min":23.919},"last":0.52800000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":2.734,"15min":1.637},"min":{"1min":0.36499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":13.839,"5min":20.503,"15min":20.503},"last":0.54400000000000004},{"interface":"front","average":{"1min":2.71,"5min":2.5619999999999998,"15min":1.536},"min":{"1min":0.40600000000000003,"5min":0.39600000000000002,"15min":0.34499999999999997},"max":{"1min":13.757,"5min":13.757,"15min":13.757},"last":0.55300000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:40:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":3.2509999999999999,"15min":2.766},"min":{"1min":0.47199999999999998,"5min":0.40200000000000002,"15min":0.374},"max":{"1min":13.865,"5min":24.030999999999999,"15min":24.030999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":3.5070000000000001,"5min":3.1110000000000002,"15min":2.1890000000000001},"min":{"1min":0.52700000000000002,"5min":0.41899999999999998,"15min":0.41599999999999998},"max":{"1min":13.781000000000001,"5min":24.178000000000001,"15min":24.178000000000001},"last":0.503}]}]},{"osd":3,"up_from":103,"seq":442381631610,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":344612,"kb_used_data":6300,"kb_used_omap":4,"kb_used_meta":338299,"kb_avail":93400540,"statfs":{"total":95995035648,"available":95642152960,"internally_reserved":0,"allocated":6451200,"data_stored":5683331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":346418782},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":3.8690000000000002,"5min":5.0350000000000001,"15min":2.3599999999999999},"min":{"1min":0.311,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":26.614999999999998,"5min":167.82400000000001,"15min":167.82400000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":5.6230000000000002,"5min":5.7380000000000004,"15min":2.6829999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":26.140000000000001,"5min":167.89599999999999,"15min":167.89599999999999},"last":0.29299999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":5.931,"5min":4.8170000000000002,"15min":2.0249999999999999},"min":{"1min":0.27400000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":48.052,"5min":83.921999999999997,"15min":83.921999999999997},"last":0.38200000000000001},{"interface":"front","average":{"1min":5.6310000000000002,"5min":4.7140000000000004,"15min":2.0009999999999999},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.247},"max":{"1min":13.551,"5min":83.707999999999998,"15min":83.707999999999998},"last":0.35999999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3920000000000003,"5min":7.3719999999999999,"15min":3.629},"min":{"1min":0.307,"5min":0.218,"15min":0.215},"max":{"1min":41.542000000000002,"5min":190.43299999999999,"15min":190.43299999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":5.3739999999999997,"5min":8.4800000000000004,"15min":4.0250000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.26000000000000001},"max":{"1min":26.097999999999999,"5min":190.267,"15min":190.267},"last":0.55100000000000005}]},{"osd":4,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0030000000000001,"5min":3.2879999999999998,"15min":1.966},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":25.946999999999999,"5min":51.591999999999999,"15min":51.591999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":3.1299999999999999,"5min":2.2530000000000001,"15min":1.5269999999999999},"min":{"1min":0.33500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":26.774999999999999,"5min":26.774999999999999,"15min":45.735999999999997},"last":0.63400000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":4.5890000000000004,"5min":2.427,"15min":1.5880000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":13.114000000000001,"5min":13.114000000000001,"15min":47.948999999999998},"last":0.58199999999999996},{"interface":"front","average":{"1min":3.1699999999999999,"5min":2.2970000000000002,"15min":1.5960000000000001},"min":{"1min":0.499,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":26.728000000000002,"5min":26.728000000000002,"15min":47.968000000000004},"last":0.53500000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":2.387,"15min":1.504},"min":{"1min":0.50700000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":26.847999999999999,"5min":26.847999999999999,"15min":26.847999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":4.8730000000000002,"5min":2.6619999999999999,"15min":1.4139999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":13.058,"5min":13.058,"15min":13.058},"last":0.621}]},{"osd":7,"last update":"Mon Oct 9 05:40:56 2023","interfaces":[{"interface":"back","average":{"1min":4.032,"5min":3.0249999999999999,"15min":2.5150000000000001},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":26.024999999999999,"5min":26.024999999999999,"15min":26.024999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":4.2720000000000002,"5min":3.125,"15min":2.3500000000000001},"min":{"1min":0.42499999999999999,"5min":0.40899999999999997,"15min":0.35699999999999998},"max":{"1min":25.995999999999999,"5min":25.995999999999999,"15min":25.995999999999999},"last":0.51500000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019157,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307360,"kb_used_data":1944,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437792,"statfs":{"total":95995035648,"available":95680299008,"internally_reserved":0,"allocated":1990656,"data_stored":1459568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:24 2023","interfaces":[{"interface":"back","average":{"1min":6.5049999999999999,"5min":2.403,"15min":1.7170000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.214},"max":{"1min":100.504,"5min":100.504,"15min":100.504},"last":0.40500000000000003},{"interface":"front","average":{"1min":6.827,"5min":2.9089999999999998,"15min":1.998},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":100.717,"5min":100.717,"15min":100.717},"last":0.52700000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":1.6699999999999999,"15min":1.21},"min":{"1min":0.29999999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8140000000000001,"5min":7.9550000000000001,"15min":31.073},"last":0.64400000000000002},{"interface":"front","average":{"1min":2.492,"5min":1.8520000000000001,"15min":1.3300000000000001},"min":{"1min":0.29299999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.349,"5min":12.349,"15min":31.100000000000001},"last":0.45300000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":8.7859999999999996,"5min":5.681,"15min":4.0270000000000001},"min":{"1min":0.315,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":100.148,"5min":196.93700000000001,"15min":196.93700000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":9.0389999999999997,"5min":5.492,"15min":3.931},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":100.086,"5min":197.08099999999999,"15min":197.08099999999999},"last":0.50600000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":4.3570000000000002,"5min":7.3419999999999996,"15min":3.097},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.509,"5min":211.48699999999999,"15min":211.48699999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":3.2109999999999999,"5min":7.157,"15min":2.9740000000000002},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.309},"max":{"1min":13.007,"5min":211.31100000000001,"15min":211.31100000000001},"last":0.57099999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.0310000000000001,"15min":1.5649999999999999},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.249},"max":{"1min":3.2730000000000001,"5min":94.646000000000001,"15min":113.35899999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.883,"15min":1.5229999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.30599999999999999},"max":{"1min":3.3849999999999998,"5min":94.692999999999998,"15min":113.32299999999999},"last":0.54100000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:40:57 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.72899999999999998,"15min":0.70499999999999996},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":1.8740000000000001,"5min":6.1779999999999999,"15min":11.944000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.93500000000000005,"15min":0.86399999999999999},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.4810000000000001,"5min":7.6500000000000004,"15min":10.356},"last":0.58899999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.94799999999999995,"15min":2.5619999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.312},"max":{"1min":3.415,"5min":11.845000000000001,"15min":30.393999999999998},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.97599999999999998,"15min":2.8780000000000001},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.5089999999999999,"5min":10.837,"15min":30.334},"last":0.66800000000000004}]}]},{"osd":5,"up_from":36,"seq":154618822929,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299152,"kb_used_data":1928,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93446000,"statfs":{"total":95995035648,"available":95688704000,"internally_reserved":0,"allocated":1974272,"data_stored":1455016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.879,"15min":3.008},"min":{"1min":0.217,"5min":0.217,"15min":0.19400000000000001},"max":{"1min":9.032,"5min":10.712999999999999,"15min":165.90199999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.849,"5min":1.8300000000000001,"15min":3.077},"min":{"1min":0.30399999999999999,"5min":0.28399999999999997,"15min":0.215},"max":{"1min":7.4050000000000002,"5min":12.263999999999999,"15min":165.96700000000001},"last":0.48399999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":7.0419999999999998,"5min":4.6070000000000002,"15min":2.1699999999999999},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":71.209999999999994,"5min":87.769000000000005,"15min":87.769000000000005},"last":0.51500000000000001},{"interface":"front","average":{"1min":8.1159999999999997,"5min":5.2130000000000001,"15min":2.2810000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.24399999999999999},"max":{"1min":71.192999999999998,"5min":87.864000000000004,"15min":87.864000000000004},"last":0.42399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":3.5299999999999998,"15min":4.0659999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.1910000000000007,"5min":73.311999999999998,"15min":290.697},"last":0.54100000000000004},{"interface":"front","average":{"1min":3.1920000000000002,"5min":3.3929999999999998,"15min":3.956},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":9.2140000000000004,"5min":73.280000000000001,"15min":290.786},"last":0.53300000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":7.5910000000000002,"5min":6.8520000000000003,"15min":2.8919999999999999},"min":{"1min":0.35199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":85.007999999999996,"5min":95.236999999999995,"15min":95.236999999999995},"last":0.45800000000000002},{"interface":"front","average":{"1min":6.7199999999999998,"5min":6.2770000000000001,"15min":2.7189999999999999},"min":{"1min":0.32900000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":83.147999999999996,"5min":93.471000000000004,"15min":93.471000000000004},"last":0.57999999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.81299999999999994,"15min":0.72099999999999997},"min":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":5.6459999999999999,"5min":7.2060000000000004,"15min":9.3510000000000009},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.1970000000000001,"15min":0.94399999999999995},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":4.8209999999999997,"5min":10.913,"15min":14.236000000000001},"last":0.59399999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:41:04 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.2370000000000001,"15min":1.3600000000000001},"min":{"1min":0.35699999999999998,"5min":0.27600000000000002,"15min":0.26600000000000001},"max":{"1min":4.79,"5min":27.146000000000001,"15min":95.010999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.1359999999999999,"15min":1.2609999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27700000000000002},"max":{"1min":4.6600000000000001,"5min":27.215,"15min":94.194000000000003},"last":0.497}]},{"osd":7,"last update":"Mon Oct 9 05:41:00 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":1.228,"15min":1.5589999999999999},"min":{"1min":0.39900000000000002,"5min":0.34399999999999997,"15min":0.28999999999999998},"max":{"1min":5.5570000000000004,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.212,"5min":1.157,"15min":1.8009999999999999},"min":{"1min":0.373,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.0549999999999997,"5min":10.779,"15min":14.432},"last":0.627}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:41:33.767 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:33.793 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:33.949 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:33.951 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:33 smithi191 ceph-mon[161669]: pgmap v609: 89 pgs: 5 creating+peering, 84 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:33.951 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:33 smithi191 ceph-mon[161669]: from='client.45344 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:33.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:33.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:33 smithi008 ceph-mon[184418]: pgmap v609: 89 pgs: 5 creating+peering, 84 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:33 smithi008 ceph-mon[184418]: from='client.45344 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:33 smithi008 ceph-mon[191227]: pgmap v609: 89 pgs: 5 creating+peering, 84 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:33 smithi008 ceph-mon[191227]: from='client.45344 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:34.007 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:41:34.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:34.081 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:34.150 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:34.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:34.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:34.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:34.282 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:34.376 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:34.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:34.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:34.446 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:41:34.446 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:41:34.446 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:41:34.446 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:41:34.447 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.447 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119031 5.f 5:f62bfa5d:::benchmark_data_smithi008_238708_object895:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:41:34.447 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.042073+0000", 2023-10-09T05:41:34.447 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.40159301699998, 2023-10-09T05:41:34.447 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99245742100000001, 2023-10-09T05:41:34.447 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.447 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.447 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.447 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:34.447 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:34.448 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119031 2023-10-09T05:41:34.448 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.448 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.448 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.448 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.448 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042073+0000", 2023-10-09T05:41:34.448 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.448 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.448 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.448 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.448 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042073+0000", 2023-10-09T05:41:34.449 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999962 2023-10-09T05:41:34.449 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.449 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.449 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.449 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042070+0000", 2023-10-09T05:41:34.449 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8694000000000001e-05 2023-10-09T05:41:34.449 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.449 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.449 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.449 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042088+0000", 2023-10-09T05:41:34.449 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3565e-05 2023-10-09T05:41:34.450 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.450 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.450 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.450 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042132+0000", 2023-10-09T05:41:34.450 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.365e-05 2023-10-09T05:41:34.450 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.450 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.450 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.450 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042145+0000", 2023-10-09T05:41:34.450 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14325328600000001 2023-10-09T05:41:34.450 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185399+0000", 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8292e-05 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185417+0000", 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83638978799999997 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.451 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.452 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.021807+0000", 2023-10-09T05:41:34.452 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012924400000000001 2023-10-09T05:41:34.452 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.452 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.452 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.452 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.021936+0000", 2023-10-09T05:41:34.452 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014705600000000001 2023-10-09T05:41:34.452 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.453 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.453 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:34.453 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.022083+0000", 2023-10-09T05:41:34.454 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041951740000000003 2023-10-09T05:41:34.454 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.454 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.454 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.454 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.026278+0000", 2023-10-09T05:41:34.454 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0043576520000000001 2023-10-09T05:41:34.454 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.454 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.454 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.454 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.030636+0000", 2023-10-09T05:41:34.455 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038215580000000001 2023-10-09T05:41:34.455 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.455 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.455 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.457 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034457+0000", 2023-10-09T05:41:34.457 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4993999999999997e-05 2023-10-09T05:41:34.457 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.457 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.457 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.458 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034492+0000", 2023-10-09T05:41:34.458 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8413e-05 2023-10-09T05:41:34.458 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.458 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.458 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.458 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034531+0000", 2023-10-09T05:41:34.458 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99245742100000001 2023-10-09T05:41:34.459 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.459 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.459 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.459 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.459 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119032 5.1f 5:fba7d45d:::benchmark_data_smithi008_238708_object896:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:41:34.459 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:41:34.460 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.40150233700001, 2023-10-09T05:41:34.460 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0020802900000001, 2023-10-09T05:41:34.460 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.460 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.460 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.460 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:34.460 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:34.460 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119032 2023-10-09T05:41:34.461 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.461 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.461 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.461 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.461 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:41:34.461 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.462 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.462 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.462 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.462 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:41:34.462 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:34.462 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.462 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.463 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.463 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042164+0000", 2023-10-09T05:41:34.463 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8399999999999997e-06 2023-10-09T05:41:34.463 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.463 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.463 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.463 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042167+0000", 2023-10-09T05:41:34.464 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7719999999999997e-06 2023-10-09T05:41:34.464 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.464 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.464 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042174+0000", 2023-10-09T05:41:34.465 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5770000000000004e-06 2023-10-09T05:41:34.465 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.465 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.465 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.465 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042179+0000", 2023-10-09T05:41:34.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14336328700000001 2023-10-09T05:41:34.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.466 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185542+0000", 2023-10-09T05:41:34.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4260000000000002e-06 2023-10-09T05:41:34.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.467 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:34.467 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185547+0000", 2023-10-09T05:41:34.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85482789199999998 2023-10-09T05:41:34.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.467 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.467 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.040375+0000", 2023-10-09T05:41:34.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012875500000000001 2023-10-09T05:41:34.468 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.468 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.468 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.468 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.040504+0000", 2023-10-09T05:41:34.468 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017535599999999999 2023-10-09T05:41:34.468 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.468 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.469 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:34.469 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.040679+0000", 2023-10-09T05:41:34.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032237390000000002 2023-10-09T05:41:34.469 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.469 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.469 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.469 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.043903+0000", 2023-10-09T05:41:34.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021366899999999999 2023-10-09T05:41:34.469 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.469 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.470 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.470 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044117+0000", 2023-10-09T05:41:34.470 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7777000000000001e-05 2023-10-09T05:41:34.470 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.470 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.470 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.470 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044155+0000", 2023-10-09T05:41:34.470 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9549999999999997e-05 2023-10-09T05:41:34.470 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.470 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044214+0000", 2023-10-09T05:41:34.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0266000000000001e-05 2023-10-09T05:41:34.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044244+0000", 2023-10-09T05:41:34.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0020802900000001 2023-10-09T05:41:34.471 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.471 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.471 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.472 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.472 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.472 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119033 5.1f 5:fb715b35:::benchmark_data_smithi008_238708_object897:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:41:34.472 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.042184+0000", 2023-10-09T05:41:34.472 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.40148284700001, 2023-10-09T05:41:34.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0031231839999999, 2023-10-09T05:41:34.472 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.472 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.472 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.473 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:34.473 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:34.473 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119033 2023-10-09T05:41:34.473 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.473 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.474 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042184+0000", 2023-10-09T05:41:34.474 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.474 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.474 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.474 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042184+0000", 2023-10-09T05:41:34.474 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:34.474 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.474 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.474 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042183+0000", 2023-10-09T05:41:34.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5550000000000001e-06 2023-10-09T05:41:34.475 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042186+0000", 2023-10-09T05:41:34.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4040000000000003e-06 2023-10-09T05:41:34.475 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.476 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.476 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042191+0000", 2023-10-09T05:41:34.476 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5389999999999999e-06 2023-10-09T05:41:34.476 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.476 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.476 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.042195+0000", 2023-10-09T05:41:34.477 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.143397471 2023-10-09T05:41:34.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.477 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.477 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185592+0000", 2023-10-09T05:41:34.477 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4479999999999996e-06 2023-10-09T05:41:34.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.477 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.478 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:34.478 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185597+0000", 2023-10-09T05:41:34.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85661535899999997 2023-10-09T05:41:34.478 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.478 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.478 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.478 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.042212+0000", 2023-10-09T05:41:34.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5107999999999998e-05 2023-10-09T05:41:34.478 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.479 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.479 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.479 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.042307+0000", 2023-10-09T05:41:34.479 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3749000000000001e-05 2023-10-09T05:41:34.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.479 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.479 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:34.479 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.042391+0000", 2023-10-09T05:41:34.479 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001859688 2023-10-09T05:41:34.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.480 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.480 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.480 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044251+0000", 2023-10-09T05:41:34.480 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00037112400000000002 2023-10-09T05:41:34.480 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.480 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.480 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.480 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.044622+0000", 2023-10-09T05:41:34.481 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060215500000000005 2023-10-09T05:41:34.481 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.481 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.481 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.481 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.045224+0000", 2023-10-09T05:41:34.481 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6007999999999997e-05 2023-10-09T05:41:34.481 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.481 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.481 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.482 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.045270+0000", 2023-10-09T05:41:34.482 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6947999999999999e-05 2023-10-09T05:41:34.482 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.482 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.482 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.482 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.045307+0000", 2023-10-09T05:41:34.482 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0031231839999999 2023-10-09T05:41:34.482 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.482 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.482 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.483 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.483 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.483 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119034 5.f 5:f790e635:::benchmark_data_smithi008_238708_object898:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:41:34.483 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.066949+0000", 2023-10-09T05:41:34.483 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.37671748000002, 2023-10-09T05:41:34.483 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.967741929, 2023-10-09T05:41:34.483 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.483 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.484 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.484 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:34.484 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:34.484 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119034 2023-10-09T05:41:34.484 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.484 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.484 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.484 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.484 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066949+0000", 2023-10-09T05:41:34.484 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.484 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.485 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.485 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.485 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066949+0000", 2023-10-09T05:41:34.485 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999962 2023-10-09T05:41:34.485 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.485 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.485 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.485 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066945+0000", 2023-10-09T05:41:34.485 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3286e-05 2023-10-09T05:41:34.486 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.486 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.486 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.486 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066959+0000", 2023-10-09T05:41:34.486 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5304999999999999e-05 2023-10-09T05:41:34.486 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.486 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.486 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.486 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066984+0000", 2023-10-09T05:41:34.487 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2354999999999999e-05 2023-10-09T05:41:34.487 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.487 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.487 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.487 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.066996+0000", 2023-10-09T05:41:34.487 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.118639008 2023-10-09T05:41:34.487 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.488 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.488 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.488 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185635+0000", 2023-10-09T05:41:34.488 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0820000000000004e-06 2023-10-09T05:41:34.488 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.488 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.488 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:34.488 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.185641+0000", 2023-10-09T05:41:34.488 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83823218700000002 2023-10-09T05:41:34.488 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.489 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.489 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.489 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.023874+0000", 2023-10-09T05:41:34.489 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011226 2023-10-09T05:41:34.489 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.489 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.489 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.489 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.023986+0000", 2023-10-09T05:41:34.489 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013421799999999999 2023-10-09T05:41:34.489 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.490 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.490 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:34.490 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.024120+0000", 2023-10-09T05:41:34.490 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003618137 2023-10-09T05:41:34.490 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.490 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.490 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.490 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.027738+0000", 2023-10-09T05:41:34.490 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029752229999999999 2023-10-09T05:41:34.490 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.491 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.491 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.491 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.030713+0000", 2023-10-09T05:41:34.491 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039165859999999997 2023-10-09T05:41:34.491 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.491 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.491 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.491 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034630+0000", 2023-10-09T05:41:34.491 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7750999999999999e-05 2023-10-09T05:41:34.492 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.492 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.492 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.492 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034658+0000", 2023-10-09T05:41:34.492 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3167000000000003e-05 2023-10-09T05:41:34.492 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.492 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.493 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.493 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.034691+0000", 2023-10-09T05:41:34.493 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.967741929 2023-10-09T05:41:34.493 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.493 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.493 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.493 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.494 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.494 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4048 6.a 6:51c21166:::benchmark_data_smithi008_295510_object4047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:41:34.494 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.480144+0000", 2023-10-09T05:41:34.494 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 265.96352247599998, 2023-10-09T05:41:34.494 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48056587499999998, 2023-10-09T05:41:34.494 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.494 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.495 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.495 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.495 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.495 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4048 2023-10-09T05:41:34.495 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.495 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.495 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.495 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.495 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480144+0000", 2023-10-09T05:41:34.495 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.495 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480144+0000", 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480143+0000", 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7996e-05 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480181+0000", 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0966e-05 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480192+0000", 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6390000000000003e-06 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480199+0000", 2023-10-09T05:41:34.497 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017538499999999999 2023-10-09T05:41:34.498 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.498 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480374+0000", 2023-10-09T05:41:34.498 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014747999999999999 2023-10-09T05:41:34.498 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.498 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480521+0000", 2023-10-09T05:41:34.498 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020043799999999999 2023-10-09T05:41:34.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.499 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:41:34.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480722+0000", 2023-10-09T05:41:34.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00076789800000000004 2023-10-09T05:41:34.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.499 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.481490+0000", 2023-10-09T05:41:34.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00070511100000000002 2023-10-09T05:41:34.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.482195+0000", 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.478412911 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.960608+0000", 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5300000000000002e-05 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.500 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.960663+0000", 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6805999999999999e-05 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.960710+0000", 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48056587499999998 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.501 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.502 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4066 6.a 6:51b3cfb1:::benchmark_data_smithi008_295510_object4065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:41:34.502 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.483733+0000", 2023-10-09T05:41:34.502 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 265.959933095, 2023-10-09T05:41:34.502 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47826635200000001, 2023-10-09T05:41:34.503 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.503 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.503 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.503 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.503 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.503 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4066 2023-10-09T05:41:34.503 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.503 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.503 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.503 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.504 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483733+0000", 2023-10-09T05:41:34.504 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.504 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.504 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.504 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.504 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483733+0000", 2023-10-09T05:41:34.504 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:34.504 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.504 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.505 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.505 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483732+0000", 2023-10-09T05:41:34.505 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3579999999999995e-05 2023-10-09T05:41:34.505 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.505 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.505 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.505 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483826+0000", 2023-10-09T05:41:34.505 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.503e-05 2023-10-09T05:41:34.505 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.506 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.506 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.506 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483841+0000", 2023-10-09T05:41:34.506 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4710000000000004e-06 2023-10-09T05:41:34.506 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.506 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.506 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.506 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483847+0000", 2023-10-09T05:41:34.507 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8745999999999997e-05 2023-10-09T05:41:34.507 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.507 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.507 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.507 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483896+0000", 2023-10-09T05:41:34.507 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013626099999999999 2023-10-09T05:41:34.507 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.507 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.507 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.507 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484032+0000", 2023-10-09T05:41:34.507 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016087400000000001 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484193+0000", 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060899000000000001 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484802+0000", 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001503536 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486306+0000", 2023-10-09T05:41:34.509 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.475630736 2023-10-09T05:41:34.509 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.509 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.961936+0000", 2023-10-09T05:41:34.510 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1003000000000001e-05 2023-10-09T05:41:34.510 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.510 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.510 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.510 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.961967+0000", 2023-10-09T05:41:34.510 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2406000000000001e-05 2023-10-09T05:41:34.510 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.510 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.510 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.510 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962000+0000", 2023-10-09T05:41:34.510 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47826635200000001 2023-10-09T05:41:34.511 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.511 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.511 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.511 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.511 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.511 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4069 6.d 6:be699dd1:::benchmark_data_smithi008_295510_object4068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:41:34.511 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.484947+0000", 2023-10-09T05:41:34.511 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 265.95871964299999, 2023-10-09T05:41:34.511 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47790969900000002, 2023-10-09T05:41:34.511 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.512 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.512 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.512 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.512 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.512 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4069 2023-10-09T05:41:34.512 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.512 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.512 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.512 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.512 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484947+0000", 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484947+0000", 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484945+0000", 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019272500000000001 2023-10-09T05:41:34.513 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485138+0000", 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1980999999999999e-05 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485150+0000", 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2870000000000004e-06 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.514 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485155+0000", 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2120999999999998e-05 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485207+0000", 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117552 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485325+0000", 2023-10-09T05:41:34.515 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000151662 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485477+0000", 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011680009999999999 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486645+0000", 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6749999999999999e-05 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.516 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.517 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.517 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486681+0000", 2023-10-09T05:41:34.517 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47611776500000003 2023-10-09T05:41:34.517 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.517 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.517 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.517 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962799+0000", 2023-10-09T05:41:34.517 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4666999999999999e-05 2023-10-09T05:41:34.517 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.517 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.517 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.518 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962834+0000", 2023-10-09T05:41:34.518 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2668999999999999e-05 2023-10-09T05:41:34.518 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.518 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.518 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.518 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962857+0000", 2023-10-09T05:41:34.518 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47790969900000002 2023-10-09T05:41:34.518 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.518 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.518 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.519 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.519 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.519 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4072 6.d 6:b4da5f17:::benchmark_data_smithi008_295510_object4071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:41:34.519 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.485343+0000", 2023-10-09T05:41:34.519 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 265.95832319499999, 2023-10-09T05:41:34.519 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47815177399999997, 2023-10-09T05:41:34.519 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.519 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.519 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.519 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.519 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.520 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4072 2023-10-09T05:41:34.520 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.520 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.520 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.520 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.520 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485343+0000", 2023-10-09T05:41:34.520 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.520 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.520 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.520 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485343+0000", 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485342+0000", 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9259000000000006e-05 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485442+0000", 2023-10-09T05:41:34.521 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0434999999999999e-05 2023-10-09T05:41:34.522 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.522 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.522 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.522 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485452+0000", 2023-10-09T05:41:34.522 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2230000000000004e-06 2023-10-09T05:41:34.522 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.522 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.522 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.522 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485458+0000", 2023-10-09T05:41:34.522 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000491495 2023-10-09T05:41:34.523 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.523 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.523 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.523 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485950+0000", 2023-10-09T05:41:34.523 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013497300000000001 2023-10-09T05:41:34.523 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.523 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.523 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.523 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486085+0000", 2023-10-09T05:41:34.524 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018227900000000001 2023-10-09T05:41:34.524 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.524 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486267+0000", 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00061199699999999998 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486879+0000", 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00083038800000000005 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.525 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.487709+0000", 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47569748899999997 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963407+0000", 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8826000000000003e-05 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963456+0000", 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9415000000000001e-05 2023-10-09T05:41:34.526 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963495+0000", 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47815177399999997 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4084 6.d 6:b4676de6:::benchmark_data_smithi008_295510_object4083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.489711+0000", 2023-10-09T05:41:34.527 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 265.953955882, 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47387225700000002, 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4084 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489711+0000", 2023-10-09T05:41:34.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489711+0000", 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489710+0000", 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7155000000000003e-05 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.530 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.530 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489777+0000", 2023-10-09T05:41:34.530 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4321000000000001e-05 2023-10-09T05:41:34.530 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.530 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.530 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.530 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489791+0000", 2023-10-09T05:41:34.530 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6220000000000003e-06 2023-10-09T05:41:34.530 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.530 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.530 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489798+0000", 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.515e-05 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489843+0000", 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6768000000000006e-05 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489940+0000", 2023-10-09T05:41:34.531 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000134702 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.490074+0000", 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000556044 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.490630+0000", 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00032459399999999999 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.532 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.490955+0000", 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47257032300000001 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963525+0000", 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8180000000000001e-05 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963554+0000", 2023-10-09T05:41:34.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9314000000000001e-05 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963583+0000", 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47387225700000002 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4097 6.a 6:56ff3694:::benchmark_data_smithi008_295510_object4096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:41:34.534 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.498925+0000", 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 265.94474162099999, 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46416822400000002, 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4097 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.535 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498925+0000", 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498925+0000", 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498923+0000", 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8837e-05 2023-10-09T05:41:34.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498972+0000", 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7210999999999998e-05 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498989+0000", 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1210000000000003e-06 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498995+0000", 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0176000000000001e-05 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499056+0000", 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016777900000000001 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499223+0000", 2023-10-09T05:41:34.538 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000203718 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499427+0000", 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00054037499999999995 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499967+0000", 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000180817 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.539 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.500148+0000", 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46288928400000001 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963038+0000", 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3631999999999999e-05 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963071+0000", 2023-10-09T05:41:34.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1931999999999999e-05 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963093+0000", 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46416822400000002 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:13694 6.7 6:e0b43690:::benchmark_data_smithi008_295510_object13693:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:41:34.541 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:13.266020+0000", 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 261.17764695699998, 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.317964937, 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 13694 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266020+0000", 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266020+0000", 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266018+0000", 2023-10-09T05:41:34.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015691500000000001 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266175+0000", 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9809999999999998e-06 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266185+0000", 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4829999999999999e-06 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.544 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266191+0000", 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6820000000000002e-05 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266238+0000", 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011752900000000001 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266355+0000", 2023-10-09T05:41:34.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015299500000000001 2023-10-09T05:41:34.546 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.546 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.546 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:41:34.546 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266508+0000", 2023-10-09T05:41:34.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014897070000000001 2023-10-09T05:41:34.546 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.546 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.546 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.546 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.267998+0000", 2023-10-09T05:41:34.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00064413000000000003 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.268642+0000", 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31523500300000001 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583877+0000", 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0569000000000002e-05 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.547 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583937+0000", 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6968000000000002e-05 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.583984+0000", 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.317964937 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.548 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:13699 6.7 6:e2dc0f5f:::benchmark_data_smithi008_295510_object13698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:13.266204+0000", 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 261.17746237099999, 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31789818199999997, 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 13699 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.549 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266204+0000", 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266204+0000", 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266203+0000", 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9275999999999999e-05 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266233+0000", 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3199999999999996e-06 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266239+0000", 2023-10-09T05:41:34.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3409999999999999e-06 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.266242+0000", 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00098051400000000004 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.267223+0000", 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108423 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.267331+0000", 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000150328 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.267482+0000", 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00078777399999999998 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.268269+0000", 2023-10-09T05:41:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00047432399999999998 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.268744+0000", 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.315280326 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584024+0000", 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3627000000000001e-05 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.554 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584068+0000", 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.464e-05 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584102+0000", 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31789818199999997 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.555 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.556 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.556 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:13705 6.d 6:b16d4aa8:::benchmark_data_smithi008_295510_object13704:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:41:34.556 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:13.269168+0000", 2023-10-09T05:41:34.556 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 261.174498718, 2023-10-09T05:41:34.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31545825999999999, 2023-10-09T05:41:34.556 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.556 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.556 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.556 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.556 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 13705 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269168+0000", 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269168+0000", 2023-10-09T05:41:34.557 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269167+0000", 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7036e-05 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269234+0000", 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7920000000000004e-06 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269241+0000", 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8670000000000004e-06 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269245+0000", 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7756000000000002e-05 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269292+0000", 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000116999 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269409+0000", 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122028 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.269531+0000", 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00063579900000000004 2023-10-09T05:41:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.270167+0000", 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002216871 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.272384+0000", 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31213664800000002 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584521+0000", 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2773000000000004e-05 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584583+0000", 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2614000000000002e-05 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:13.584626+0000", 2023-10-09T05:41:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31545825999999999 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:142206 6.d 6:b4d13e7d:::benchmark_data_smithi008_295510_object53843:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.366419+0000", 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 87.077247139999997, 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45887450299999999, 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 142206 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366419+0000", 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366419+0000", 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366418+0000", 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9809999999999997e-06 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366425+0000", 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1507e-05 2023-10-09T05:41:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366436+0000", 2023-10-09T05:41:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6210000000000001e-06 2023-10-09T05:41:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366442+0000", 2023-10-09T05:41:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8337999999999997e-05 2023-10-09T05:41:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366490+0000", 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000129345 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366620+0000", 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012503699999999999 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:41:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366745+0000", 2023-10-09T05:41:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041122060000000002 2023-10-09T05:41:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.370857+0000", 2023-10-09T05:41:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45407035200000001 2023-10-09T05:41:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824927+0000", 2023-10-09T05:41:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000271425 2023-10-09T05:41:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.825199+0000", 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0005000000000001e-05 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.825239+0000", 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5108999999999997e-05 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.825294+0000", 2023-10-09T05:41:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45887450299999999 2023-10-09T05:41:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:142208 6.a 6:5453517f:::benchmark_data_smithi008_295510_object53845:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:41:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.366458+0000", 2023-10-09T05:41:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 87.077208592000005, 2023-10-09T05:41:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46771891900000001, 2023-10-09T05:41:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 142208 2023-10-09T05:41:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366458+0000", 2023-10-09T05:41:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366458+0000", 2023-10-09T05:41:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366457+0000", 2023-10-09T05:41:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5499999999999999e-06 2023-10-09T05:41:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366461+0000", 2023-10-09T05:41:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4989999999999999e-06 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366467+0000", 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1790000000000001e-06 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366472+0000", 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4431000000000002e-05 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366516+0000", 2023-10-09T05:41:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8264999999999996e-05 2023-10-09T05:41:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366604+0000", 2023-10-09T05:41:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5539999999999998e-05 2023-10-09T05:41:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:41:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366690+0000", 2023-10-09T05:41:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46729868499999999 2023-10-09T05:41:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.833989+0000", 2023-10-09T05:41:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0900999999999999e-05 2023-10-09T05:41:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834039+0000", 2023-10-09T05:41:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9656000000000001e-05 2023-10-09T05:41:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834079+0000", 2023-10-09T05:41:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5481e-05 2023-10-09T05:41:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834135+0000", 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2240000000000002e-05 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834177+0000", 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46771891900000001 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:142210 6.a 6:59802c8c:::benchmark_data_smithi008_295510_object53847:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.366485+0000", 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 87.077181308999997, 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46783791200000002, 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 142210 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366485+0000", 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366485+0000", 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366485+0000", 2023-10-09T05:41:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2109999999999998e-06 2023-10-09T05:41:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366488+0000", 2023-10-09T05:41:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7570000000000004e-06 2023-10-09T05:41:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366493+0000", 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0410000000000002e-06 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366496+0000", 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047554600000000004 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371251+0000", 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012267699999999999 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371374+0000", 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011798600000000001 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371492+0000", 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46255569000000002 2023-10-09T05:41:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834047+0000", 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000167675 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834215+0000", 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8153e-05 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834243+0000", 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0781000000000001e-05 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834284+0000", 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8974000000000003e-05 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834323+0000", 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46783791200000002 2023-10-09T05:41:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:142213 6.a 6:5635753f:::benchmark_data_smithi008_295510_object53850:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.371323+0000", 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 87.072343281000002, 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46515832600000001, 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 142213 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371323+0000", 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371323+0000", 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371322+0000", 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9589999999999995e-06 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371330+0000", 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3706000000000001e-05 2023-10-09T05:41:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371343+0000", 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6529999999999994e-06 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371351+0000", 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45795592800000001 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.829307+0000", 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000141691 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.829449+0000", 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000160099 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.829609+0000", 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047785500000000003 2023-10-09T05:41:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834387+0000", 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00086802500000000002 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.835255+0000", 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011420340000000001 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.836397+0000", 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7500000000000003e-05 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.836445+0000", 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6659000000000003e-05 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.836482+0000", 2023-10-09T05:41:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46515832600000001 2023-10-09T05:41:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.45029.0:142208 6.a e198/162 ondisk, result = 0)", 2023-10-09T05:41:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.371518+0000", 2023-10-09T05:41:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 87.072148779000003, 2023-10-09T05:41:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46253101000000002, 2023-10-09T05:41:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371518+0000", 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371518+0000", 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371517+0000", 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7789999999999998e-06 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371523+0000", 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0664e-05 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371534+0000", 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1079999999999998e-06 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371540+0000", 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46248180700000002 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834022+0000", 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3530000000000001e-06 2023-10-09T05:41:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834031+0000", 2023-10-09T05:41:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7370000000000001e-05 2023-10-09T05:41:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834049+0000", 2023-10-09T05:41:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46253101000000002 2023-10-09T05:41:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.45029.0:142206 6.d e198/162 ondisk, result = 0)", 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.373159+0000", 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 87.070507315, 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45177083299999998, 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.373159+0000", 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.373159+0000", 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.373158+0000", 2023-10-09T05:41:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1810000000000001e-06 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.373166+0000", 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45168935799999999 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824855+0000", 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1770000000000001e-05 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824867+0000", 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1879999999999999e-05 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824909+0000", 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7610000000000002e-06 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824919+0000", 2023-10-09T05:41:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1137e-05 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824930+0000", 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45177083299999998 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.45029.0:142210 6.a e198/162 ondisk, result = 0)", 2023-10-09T05:41:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.375554+0000", 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 87.068112933999998, 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45866283400000002, 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.375554+0000", 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.375554+0000", 2023-10-09T05:41:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:41:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.375553+0000", 2023-10-09T05:41:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4059999999999999e-06 2023-10-09T05:41:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.375560+0000", 2023-10-09T05:41:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.44929598900000001 2023-10-09T05:41:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824856+0000", 2023-10-09T05:41:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0549e-05 2023-10-09T05:41:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824867+0000", 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.009337382 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834204+0000", 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6239999999999999e-06 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834210+0000", 2023-10-09T05:41:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7750000000000004e-06 2023-10-09T05:41:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834216+0000", 2023-10-09T05:41:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45866283400000002 2023-10-09T05:41:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:34.608 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:41:34.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:34.636 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:34.679 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:34.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:34.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:34.743 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:34.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:34.796 INFO:teuthology.orchestra.run.smithi008.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T05:41:34.810 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:34.836 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2670724098' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:34.837 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:34 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:34.837 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2670724098' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:34.837 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:34 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:35.020 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:35.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2670724098' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:35.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:34 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:35.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:35.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:35.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:41:35.124 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:35.152 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:35.197 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:35.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:35.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:35.305 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:35.336 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:35.439 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:35.836 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:41:35.836 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:41:35.837 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:41:35.837 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:41:35.837 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:41:35.837 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:41:35.837 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:41:35.837 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:41:35.837 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:41:35.838 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:41:35.838 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:41:35.838 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:41:35.838 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:41:35.838 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:41:35.838 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:41:35.838 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:41:35.973 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:35 smithi191 ceph-mon[161669]: pgmap v610: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:35.973 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:35 smithi191 ceph-mon[161669]: failed to encode map e217 with expected crc 2023-10-09T05:41:35.973 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:35 smithi191 ceph-mon[161669]: failed to encode map e217 with expected crc 2023-10-09T05:41:35.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:35 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:41:35.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:35 smithi191 ceph-mon[161669]: osdmap e217: 8 total, 8 up, 7 in 2023-10-09T05:41:35.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1901726310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:36.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[184418]: pgmap v610: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:36.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[184418]: failed to encode map e217 with expected crc 2023-10-09T05:41:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[184418]: failed to encode map e217 with expected crc 2023-10-09T05:41:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:41:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[184418]: osdmap e217: 8 total, 8 up, 7 in 2023-10-09T05:41:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1901726310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[191227]: pgmap v610: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[191227]: failed to encode map e217 with expected crc 2023-10-09T05:41:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[191227]: failed to encode map e217 with expected crc 2023-10-09T05:41:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:41:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[191227]: osdmap e217: 8 total, 8 up, 7 in 2023-10-09T05:41:36.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1901726310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:36.593 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:36.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:36.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:36.697 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:36.725 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:36.767 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:41:36.771 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:36.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:36.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:36.817 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:41:36.818 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:41:36.874 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:36.901 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:37.036 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:37.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:37.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:37.140 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:37.170 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:37.211 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:37.268 DEBUG:teuthology.orchestra.run.smithi008:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T05:41:37.270 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:41:37.271 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:41:37.318 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:hints = 1 2023-10-09T05:41:37.319 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:41:37.319 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object prefix: benchmark_data_smithi008_354649 2023-10-09T05:41:37.319 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:41:37.319 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:41:37.415 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:37.418 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:41:37.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:37.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:37.522 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:37.558 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:37.621 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:37.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:37.729 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:37.795 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:37.935 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:38.070 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:38.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:38.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:38.079 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:37 smithi008 ceph-mon[184418]: pgmap v612: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:38.079 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:37 smithi008 ceph-mon[191227]: pgmap v612: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:38.136 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:37 smithi191 ceph-mon[161669]: pgmap v612: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:38.179 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:38.216 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:38.319 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 1 16 2747 2731 170.424 170.688 0.0030858 0.00584612 2023-10-09T05:41:38.532 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:38.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:38.594 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:41:38.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:38.670 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:38.721 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:38.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:38.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:38.824 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:38.828 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:41:38.853 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:38.901 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:38.902 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:38 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1347000770' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:38.903 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:38 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:38.903 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:38 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1347000770' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:38.903 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:38 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:38.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:38.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:39.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:39.039 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:39.237 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:39.239 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:38 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1347000770' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:39.239 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:38 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:41:39.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:39.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:39.319 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 2 16 5153 5137 160.4 150.375 0.00840685 0.00621769 2023-10-09T05:41:39.341 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:39.398 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:39.723 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:39.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:39.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:39.746 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:41:39.746 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:41:39.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:39.871 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:39.937 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:39.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:39.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[184418]: pgmap v613: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[184418]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:41:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[184418]: osdmap e218: 8 total, 8 up, 7 in 2023-10-09T05:41:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[184418]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[184418]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[184418]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[184418]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[184418]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[184418]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[191227]: pgmap v613: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[191227]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:41:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[191227]: osdmap e218: 8 total, 8 up, 7 in 2023-10-09T05:41:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[191227]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[191227]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[191227]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[191227]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[191227]: failed to encode map e218 with expected crc 2023-10-09T05:41:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:39 smithi008 ceph-mon[191227]: failed to encode map e218 with expected crc 2023-10-09T05:41:40.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:39 smithi191 ceph-mon[161669]: pgmap v613: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:39 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:39 smithi191 ceph-mon[161669]: failed to encode map e218 with expected crc 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:39 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:39 smithi191 ceph-mon[161669]: osdmap e218: 8 total, 8 up, 7 in 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:39 smithi191 ceph-mon[161669]: failed to encode map e218 with expected crc 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:39 smithi191 ceph-mon[161669]: failed to encode map e218 with expected crc 2023-10-09T05:41:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:39 smithi191 ceph-mon[161669]: failed to encode map e218 with expected crc 2023-10-09T05:41:40.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:39 smithi191 ceph-mon[161669]: failed to encode map e218 with expected crc 2023-10-09T05:41:40.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:39 smithi191 ceph-mon[161669]: failed to encode map e218 with expected crc 2023-10-09T05:41:40.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:39 smithi191 ceph-mon[161669]: failed to encode map e218 with expected crc 2023-10-09T05:41:40.076 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:40.134 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:40.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:40.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:40.239 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:40.266 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:40.319 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 3 16 7500 7484 155.827 146.688 0.00150275 0.00610991 2023-10-09T05:41:40.716 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:41:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:41:40] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.33.4" 2023-10-09T05:41:40.724 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:41:40.946 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:40.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:41.053 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:41.087 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:41.152 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:41.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:41.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:41.208 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:41.250 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:41.257 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:41.291 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:41.323 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 4 16 9598 9582 149.504 131.125 0.00344384 0.00667911 2023-10-09T05:41:41.349 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:41.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:41.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:41.452 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:41.495 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:41.552 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:41.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:41.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:41.657 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:41.693 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:41.809 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:41:41.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:41.811 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:41.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:41.821 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:41:41.912 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:41.987 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:42.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:41 smithi191 ceph-mon[161669]: pgmap v615: 89 pgs: 89 active+clean; 115 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 221 op/s 2023-10-09T05:41:42.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:41 smithi008 ceph-mon[184418]: pgmap v615: 89 pgs: 89 active+clean; 115 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 221 op/s 2023-10-09T05:41:42.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:41 smithi008 ceph-mon[191227]: pgmap v615: 89 pgs: 89 active+clean; 115 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 221 op/s 2023-10-09T05:41:42.323 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 5 16 11612 11596 144.78 125.875 0.0014685 0.00676605 2023-10-09T05:41:42.953 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:42 smithi191 ceph-mon[161669]: from='client.55264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:43.003 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:41:43.183 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:41:43.184 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:41:43.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:42 smithi008 ceph-mon[184418]: from='client.55264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:43.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:42 smithi008 ceph-mon[191227]: from='client.55264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:43.268 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:43.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:43.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:43.329 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 6 16 13829 13813 143.615 138.562 0.00590004 0.00695482 2023-10-09T05:41:43.377 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:43.419 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:43.477 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:43.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:43.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:43.582 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:43.639 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:43.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:43 smithi008 ceph-mon[184418]: pgmap v616: 89 pgs: 89 active+clean; 454 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T05:41:43.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]: dispatch 2023-10-09T05:41:43.975 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T05:41:43.976 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:43 smithi008 ceph-mon[191227]: pgmap v616: 89 pgs: 89 active+clean; 454 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T05:41:43.976 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]: dispatch 2023-10-09T05:41:43.976 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T05:41:44.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:41:44.170 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:43 smithi191 ceph-mon[161669]: pgmap v616: 89 pgs: 89 active+clean; 454 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T05:41:44.170 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]: dispatch 2023-10-09T05:41:44.171 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T05:41:44.329 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 7 16 15222 15206 135.548 87.0625 0.00593904 0.00737123 2023-10-09T05:41:44.386 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:44.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:44.490 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:44.531 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:44.583 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:44.595 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:44.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:44.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:44.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:44.733 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":615,"stamp":"2023-10-09T05:41:40.160032+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":120723379,"num_objects":1797,"num_object_clones":0,"num_object_copies":5391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1797,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":2875,"num_write_kb":128357,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":4789622,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2939,"ondisk_log_size":2939,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2682820,"kb_used_data":398540,"kb_used_omap":31,"kb_used_meta":2284192,"kb_avail":653533244,"statfs":{"total":671965249536,"available":669218041856,"internally_reserved":0,"allocated":408104960,"data_stored":403290495,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2339013065},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":115933184,"num_objects":1769,"num_object_clones":0,"num_object_copies":5307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1769,"num_write_kb":113216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002617"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":200,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954346+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:41:34.954346+0000","last_peered":"2023-10-09T05:41:34.954346+0000","last_clean":"2023-10-09T05:41:34.954346+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:41:34.954346+0000","last_undegraded":"2023-10-09T05:41:34.954346+0000","last_fullsized":"2023-10-09T05:41:34.954346+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":189,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154984+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:41:32.154984+0000","last_peered":"2023-10-09T05:41:32.154984+0000","last_clean":"2023-10-09T05:41:32.154984+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:41:32.154984+0000","last_undegraded":"2023-10-09T05:41:32.154984+0000","last_fullsized":"2023-10-09T05:41:32.154984+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":164,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.861429+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:41:34.861429+0000","last_peered":"2023-10-09T05:41:34.861429+0000","last_clean":"2023-10-09T05:41:34.861429+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:41:34.861429+0000","last_undegraded":"2023-10-09T05:41:34.861429+0000","last_fullsized":"2023-10-09T05:41:34.861429+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":193,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.155247+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:41:32.155247+0000","last_peered":"2023-10-09T05:41:32.155247+0000","last_clean":"2023-10-09T05:41:32.155247+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:41:32.155247+0000","last_undegraded":"2023-10-09T05:41:32.155247+0000","last_fullsized":"2023-10-09T05:41:32.155247+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":205,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.153871+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:41:32.153871+0000","last_peered":"2023-10-09T05:41:32.153871+0000","last_clean":"2023-10-09T05:41:32.153871+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:41:32.153871+0000","last_undegraded":"2023-10-09T05:41:32.153871+0000","last_fullsized":"2023-10-09T05:41:32.153871+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":182,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154276+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:41:32.154276+0000","last_peered":"2023-10-09T05:41:32.154276+0000","last_clean":"2023-10-09T05:41:32.154276+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:41:32.154276+0000","last_undegraded":"2023-10-09T05:41:32.154276+0000","last_fullsized":"2023-10-09T05:41:32.154276+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":168,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154592+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:41:32.154592+0000","last_peered":"2023-10-09T05:41:32.154592+0000","last_clean":"2023-10-09T05:41:32.154592+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:41:32.154592+0000","last_undegraded":"2023-10-09T05:41:32.154592+0000","last_fullsized":"2023-10-09T05:41:32.154592+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":189,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.153970+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:41:32.153970+0000","last_peered":"2023-10-09T05:41:32.153970+0000","last_clean":"2023-10-09T05:41:32.153970+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:41:32.153970+0000","last_undegraded":"2023-10-09T05:41:32.153970+0000","last_fullsized":"2023-10-09T05:41:32.153970+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":209,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.795205+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:41:34.795205+0000","last_peered":"2023-10-09T05:41:34.795205+0000","last_clean":"2023-10-09T05:41:34.795205+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:41:34.795205+0000","last_undegraded":"2023-10-09T05:41:34.795205+0000","last_fullsized":"2023-10-09T05:41:34.795205+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":203,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.155653+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:41:32.155653+0000","last_peered":"2023-10-09T05:41:32.155653+0000","last_clean":"2023-10-09T05:41:32.155653+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:41:32.155653+0000","last_undegraded":"2023-10-09T05:41:32.155653+0000","last_fullsized":"2023-10-09T05:41:32.155653+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":96,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154882+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:41:32.154882+0000","last_peered":"2023-10-09T05:41:32.154882+0000","last_clean":"2023-10-09T05:41:32.154882+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:41:32.154882+0000","last_undegraded":"2023-10-09T05:41:32.154882+0000","last_fullsized":"2023-10-09T05:41:32.154882+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":164,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.860911+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:41:34.860911+0000","last_peered":"2023-10-09T05:41:34.860911+0000","last_clean":"2023-10-09T05:41:34.860911+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:41:34.860911+0000","last_undegraded":"2023-10-09T05:41:34.860911+0000","last_fullsized":"2023-10-09T05:41:34.860911+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":225,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.955001+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:41:34.955001+0000","last_peered":"2023-10-09T05:41:34.955001+0000","last_clean":"2023-10-09T05:41:34.955001+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:41:34.955001+0000","last_undegraded":"2023-10-09T05:41:34.955001+0000","last_fullsized":"2023-10-09T05:41:34.955001+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":183,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954649+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:41:34.954649+0000","last_peered":"2023-10-09T05:41:34.954649+0000","last_clean":"2023-10-09T05:41:34.954649+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:41:34.954649+0000","last_undegraded":"2023-10-09T05:41:34.954649+0000","last_fullsized":"2023-10-09T05:41:34.954649+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":96,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954760+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:41:34.954760+0000","last_peered":"2023-10-09T05:41:34.954760+0000","last_clean":"2023-10-09T05:41:34.954760+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:41:34.954760+0000","last_undegraded":"2023-10-09T05:41:34.954760+0000","last_fullsized":"2023-10-09T05:41:34.954760+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":213,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.155275+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:41:32.155275+0000","last_peered":"2023-10-09T05:41:32.155275+0000","last_clean":"2023-10-09T05:41:32.155275+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:41:32.155275+0000","last_undegraded":"2023-10-09T05:41:32.155275+0000","last_fullsized":"2023-10-09T05:41:32.155275+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":193,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954119+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:41:34.954119+0000","last_peered":"2023-10-09T05:41:34.954119+0000","last_clean":"2023-10-09T05:41:34.954119+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:41:34.954119+0000","last_undegraded":"2023-10-09T05:41:34.954119+0000","last_fullsized":"2023-10-09T05:41:34.954119+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":208,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.794768+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:41:34.794768+0000","last_peered":"2023-10-09T05:41:34.794768+0000","last_clean":"2023-10-09T05:41:34.794768+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:41:34.794768+0000","last_undegraded":"2023-10-09T05:41:34.794768+0000","last_fullsized":"2023-10-09T05:41:34.794768+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":114,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954209+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:41:34.954209+0000","last_peered":"2023-10-09T05:41:34.954209+0000","last_clean":"2023-10-09T05:41:34.954209+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:41:34.954209+0000","last_undegraded":"2023-10-09T05:41:34.954209+0000","last_fullsized":"2023-10-09T05:41:34.954209+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":192,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.155447+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:41:32.155447+0000","last_peered":"2023-10-09T05:41:32.155447+0000","last_clean":"2023-10-09T05:41:32.155447+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:41:32.155447+0000","last_undegraded":"2023-10-09T05:41:32.155447+0000","last_fullsized":"2023-10-09T05:41:32.155447+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":191,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.153117+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:41:32.153117+0000","last_peered":"2023-10-09T05:41:32.153117+0000","last_clean":"2023-10-09T05:41:32.153117+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:41:32.153117+0000","last_undegraded":"2023-10-09T05:41:32.153117+0000","last_fullsized":"2023-10-09T05:41:32.153117+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":213,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.156121+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:41:32.156121+0000","last_peered":"2023-10-09T05:41:32.156121+0000","last_clean":"2023-10-09T05:41:32.156121+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:41:32.156121+0000","last_undegraded":"2023-10-09T05:41:32.156121+0000","last_fullsized":"2023-10-09T05:41:32.156121+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"218'254","reported_seq":267,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T05:41:38.951076+0000","last_change":"2023-10-09T05:41:29.078278+0000","last_active":"2023-10-09T05:41:38.951076+0000","last_peered":"2023-10-09T05:41:38.951076+0000","last_clean":"2023-10-09T05:41:38.951076+0000","last_became_active":"2023-10-09T05:41:29.077562+0000","last_became_peered":"2023-10-09T05:41:29.077562+0000","last_unstale":"2023-10-09T05:41:38.951076+0000","last_undegraded":"2023-10-09T05:41:38.951076+0000","last_fullsized":"2023-10-09T05:41:38.951076+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":254,"log_dups_size":0,"ondisk_log_size":254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:29.784837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16646144,"num_objects":254,"num_object_clones":0,"num_object_copies":762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":254,"num_write_kb":16256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":94,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954643+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:41:34.954643+0000","last_peered":"2023-10-09T05:41:34.954643+0000","last_clean":"2023-10-09T05:41:34.954643+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:41:34.954643+0000","last_undegraded":"2023-10-09T05:41:34.954643+0000","last_fullsized":"2023-10-09T05:41:34.954643+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":193,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.794400+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:41:34.794400+0000","last_peered":"2023-10-09T05:41:34.794400+0000","last_clean":"2023-10-09T05:41:34.794400+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:41:34.794400+0000","last_undegraded":"2023-10-09T05:41:34.794400+0000","last_fullsized":"2023-10-09T05:41:34.794400+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":12,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154949+0000","last_change":"2023-10-09T05:41:29.081067+0000","last_active":"2023-10-09T05:41:32.154949+0000","last_peered":"2023-10-09T05:41:32.154949+0000","last_clean":"2023-10-09T05:41:32.154949+0000","last_became_active":"2023-10-09T05:41:29.080704+0000","last_became_peered":"2023-10-09T05:41:29.080704+0000","last_unstale":"2023-10-09T05:41:32.154949+0000","last_undegraded":"2023-10-09T05:41:32.154949+0000","last_fullsized":"2023-10-09T05:41:32.154949+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:29.483205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":94,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954569+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:41:34.954569+0000","last_peered":"2023-10-09T05:41:34.954569+0000","last_clean":"2023-10-09T05:41:34.954569+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:41:34.954569+0000","last_undegraded":"2023-10-09T05:41:34.954569+0000","last_fullsized":"2023-10-09T05:41:34.954569+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":198,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954183+0000","last_change":"2023-10-09T05:31:32.869504+0000","last_active":"2023-10-09T05:41:34.954183+0000","last_peered":"2023-10-09T05:41:34.954183+0000","last_clean":"2023-10-09T05:41:34.954183+0000","last_became_active":"2023-10-09T05:31:32.869181+0000","last_became_peered":"2023-10-09T05:31:32.869181+0000","last_unstale":"2023-10-09T05:41:34.954183+0000","last_undegraded":"2023-10-09T05:41:34.954183+0000","last_fullsized":"2023-10-09T05:41:34.954183+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"217'149","reported_seq":161,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:38.270116+0000","last_change":"2023-10-09T05:41:29.077986+0000","last_active":"2023-10-09T05:41:38.270116+0000","last_peered":"2023-10-09T05:41:38.270116+0000","last_clean":"2023-10-09T05:41:38.270116+0000","last_became_active":"2023-10-09T05:41:29.077631+0000","last_became_peered":"2023-10-09T05:41:29.077631+0000","last_unstale":"2023-10-09T05:41:38.270116+0000","last_undegraded":"2023-10-09T05:41:38.270116+0000","last_fullsized":"2023-10-09T05:41:38.270116+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:48.684759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9764864,"num_objects":149,"num_object_clones":0,"num_object_copies":447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":149,"num_write_kb":9536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":95,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.794796+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:41:34.794796+0000","last_peered":"2023-10-09T05:41:34.794796+0000","last_clean":"2023-10-09T05:41:34.794796+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:41:34.794796+0000","last_undegraded":"2023-10-09T05:41:34.794796+0000","last_fullsized":"2023-10-09T05:41:34.794796+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":215,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.153814+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:41:32.153814+0000","last_peered":"2023-10-09T05:41:32.153814+0000","last_clean":"2023-10-09T05:41:32.153814+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:41:32.153814+0000","last_undegraded":"2023-10-09T05:41:32.153814+0000","last_fullsized":"2023-10-09T05:41:32.153814+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":13,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:31.836168+0000","last_change":"2023-10-09T05:41:29.073764+0000","last_active":"2023-10-09T05:41:31.836168+0000","last_peered":"2023-10-09T05:41:31.836168+0000","last_clean":"2023-10-09T05:41:31.836168+0000","last_became_active":"2023-10-09T05:41:29.073452+0000","last_became_peered":"2023-10-09T05:41:29.073452+0000","last_unstale":"2023-10-09T05:41:31.836168+0000","last_undegraded":"2023-10-09T05:41:31.836168+0000","last_fullsized":"2023-10-09T05:41:31.836168+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:22.328530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":185,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.794793+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:41:34.794793+0000","last_peered":"2023-10-09T05:41:34.794793+0000","last_clean":"2023-10-09T05:41:34.794793+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:41:34.794793+0000","last_undegraded":"2023-10-09T05:41:34.794793+0000","last_fullsized":"2023-10-09T05:41:34.794793+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":169,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.153691+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:41:32.153691+0000","last_peered":"2023-10-09T05:41:32.153691+0000","last_clean":"2023-10-09T05:41:32.153691+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:41:32.153691+0000","last_undegraded":"2023-10-09T05:41:32.153691+0000","last_fullsized":"2023-10-09T05:41:32.153691+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"217'164","reported_seq":176,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:38.277804+0000","last_change":"2023-10-09T05:41:29.082598+0000","last_active":"2023-10-09T05:41:38.277804+0000","last_peered":"2023-10-09T05:41:38.277804+0000","last_clean":"2023-10-09T05:41:38.277804+0000","last_became_active":"2023-10-09T05:41:29.082285+0000","last_became_peered":"2023-10-09T05:41:29.082285+0000","last_unstale":"2023-10-09T05:41:38.277804+0000","last_undegraded":"2023-10-09T05:41:38.277804+0000","last_fullsized":"2023-10-09T05:41:38.277804+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:18.625214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10747904,"num_objects":164,"num_object_clones":0,"num_object_copies":492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":164,"num_write_kb":10496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":187,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154562+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:41:32.154562+0000","last_peered":"2023-10-09T05:41:32.154562+0000","last_clean":"2023-10-09T05:41:32.154562+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:41:32.154562+0000","last_undegraded":"2023-10-09T05:41:32.154562+0000","last_fullsized":"2023-10-09T05:41:32.154562+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":183,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954453+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:41:34.954453+0000","last_peered":"2023-10-09T05:41:34.954453+0000","last_clean":"2023-10-09T05:41:34.954453+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:41:34.954453+0000","last_undegraded":"2023-10-09T05:41:34.954453+0000","last_fullsized":"2023-10-09T05:41:34.954453+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":13,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:31.836253+0000","last_change":"2023-10-09T05:41:29.078791+0000","last_active":"2023-10-09T05:41:31.836253+0000","last_peered":"2023-10-09T05:41:31.836253+0000","last_clean":"2023-10-09T05:41:31.836253+0000","last_became_active":"2023-10-09T05:41:29.078491+0000","last_became_peered":"2023-10-09T05:41:29.078491+0000","last_unstale":"2023-10-09T05:41:31.836253+0000","last_undegraded":"2023-10-09T05:41:31.836253+0000","last_fullsized":"2023-10-09T05:41:31.836253+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:37.817282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":183,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.153401+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:41:32.153401+0000","last_peered":"2023-10-09T05:41:32.153401+0000","last_clean":"2023-10-09T05:41:32.153401+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:41:32.153401+0000","last_undegraded":"2023-10-09T05:41:32.153401+0000","last_fullsized":"2023-10-09T05:41:32.153401+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":198,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954505+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:41:34.954505+0000","last_peered":"2023-10-09T05:41:34.954505+0000","last_clean":"2023-10-09T05:41:34.954505+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:41:34.954505+0000","last_undegraded":"2023-10-09T05:41:34.954505+0000","last_fullsized":"2023-10-09T05:41:34.954505+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":11,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154397+0000","last_change":"2023-10-09T05:41:29.078645+0000","last_active":"2023-10-09T05:41:32.154397+0000","last_peered":"2023-10-09T05:41:32.154397+0000","last_clean":"2023-10-09T05:41:32.154397+0000","last_became_active":"2023-10-09T05:41:29.078355+0000","last_became_peered":"2023-10-09T05:41:29.078355+0000","last_unstale":"2023-10-09T05:41:32.154397+0000","last_undegraded":"2023-10-09T05:41:32.154397+0000","last_fullsized":"2023-10-09T05:41:32.154397+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":208,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954021+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:41:34.954021+0000","last_peered":"2023-10-09T05:41:34.954021+0000","last_clean":"2023-10-09T05:41:34.954021+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:41:34.954021+0000","last_undegraded":"2023-10-09T05:41:34.954021+0000","last_fullsized":"2023-10-09T05:41:34.954021+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":192,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954659+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:41:34.954659+0000","last_peered":"2023-10-09T05:41:34.954659+0000","last_clean":"2023-10-09T05:41:34.954659+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:41:34.954659+0000","last_undegraded":"2023-10-09T05:41:34.954659+0000","last_fullsized":"2023-10-09T05:41:34.954659+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":11,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154692+0000","last_change":"2023-10-09T05:41:29.082194+0000","last_active":"2023-10-09T05:41:32.154692+0000","last_peered":"2023-10-09T05:41:32.154692+0000","last_clean":"2023-10-09T05:41:32.154692+0000","last_became_active":"2023-10-09T05:41:29.081976+0000","last_became_peered":"2023-10-09T05:41:29.081976+0000","last_unstale":"2023-10-09T05:41:32.154692+0000","last_undegraded":"2023-10-09T05:41:32.154692+0000","last_fullsized":"2023-10-09T05:41:32.154692+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":214,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.153766+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:41:32.153766+0000","last_peered":"2023-10-09T05:41:32.153766+0000","last_clean":"2023-10-09T05:41:32.153766+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:41:32.153766+0000","last_undegraded":"2023-10-09T05:41:32.153766+0000","last_fullsized":"2023-10-09T05:41:32.153766+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":164,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.862269+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:41:34.862269+0000","last_peered":"2023-10-09T05:41:34.862269+0000","last_clean":"2023-10-09T05:41:34.862269+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:41:34.862269+0000","last_undegraded":"2023-10-09T05:41:34.862269+0000","last_fullsized":"2023-10-09T05:41:34.862269+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"218'453","reported_seq":465,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T05:41:39.992547+0000","last_change":"2023-10-09T05:41:29.079588+0000","last_active":"2023-10-09T05:41:39.992547+0000","last_peered":"2023-10-09T05:41:39.992547+0000","last_clean":"2023-10-09T05:41:39.992547+0000","last_became_active":"2023-10-09T05:41:29.079288+0000","last_became_peered":"2023-10-09T05:41:29.079288+0000","last_unstale":"2023-10-09T05:41:39.992547+0000","last_undegraded":"2023-10-09T05:41:39.992547+0000","last_fullsized":"2023-10-09T05:41:39.992547+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":453,"log_dups_size":0,"ondisk_log_size":453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29687808,"num_objects":453,"num_object_clones":0,"num_object_copies":1359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":453,"num_write_kb":28992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":787,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.861695+0000","last_change":"2023-10-09T05:40:04.806908+0000","last_active":"2023-10-09T05:41:34.861695+0000","last_peered":"2023-10-09T05:41:34.861695+0000","last_clean":"2023-10-09T05:41:34.861695+0000","last_became_active":"2023-10-09T05:40:04.806232+0000","last_became_peered":"2023-10-09T05:40:04.806232+0000","last_unstale":"2023-10-09T05:41:34.861695+0000","last_undegraded":"2023-10-09T05:41:34.861695+0000","last_fullsized":"2023-10-09T05:41:34.861695+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:34.087553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":200,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.794691+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:41:34.794691+0000","last_peered":"2023-10-09T05:41:34.794691+0000","last_clean":"2023-10-09T05:41:34.794691+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:41:34.794691+0000","last_undegraded":"2023-10-09T05:41:34.794691+0000","last_fullsized":"2023-10-09T05:41:34.794691+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":150,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.860904+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:41:34.860904+0000","last_peered":"2023-10-09T05:41:34.860904+0000","last_clean":"2023-10-09T05:41:34.860904+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:41:34.860904+0000","last_undegraded":"2023-10-09T05:41:34.860904+0000","last_fullsized":"2023-10-09T05:41:34.860904+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":11,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154042+0000","last_change":"2023-10-09T05:41:29.082691+0000","last_active":"2023-10-09T05:41:32.154042+0000","last_peered":"2023-10-09T05:41:32.154042+0000","last_clean":"2023-10-09T05:41:32.154042+0000","last_became_active":"2023-10-09T05:41:29.082385+0000","last_became_peered":"2023-10-09T05:41:29.082385+0000","last_unstale":"2023-10-09T05:41:32.154042+0000","last_undegraded":"2023-10-09T05:41:32.154042+0000","last_fullsized":"2023-10-09T05:41:32.154042+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":654,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.794624+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:41:34.794624+0000","last_peered":"2023-10-09T05:41:34.794624+0000","last_clean":"2023-10-09T05:41:34.794624+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:41:34.794624+0000","last_undegraded":"2023-10-09T05:41:34.794624+0000","last_fullsized":"2023-10-09T05:41:34.794624+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":184,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954836+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:41:34.954836+0000","last_peered":"2023-10-09T05:41:34.954836+0000","last_clean":"2023-10-09T05:41:34.954836+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:41:34.954836+0000","last_undegraded":"2023-10-09T05:41:34.954836+0000","last_fullsized":"2023-10-09T05:41:34.954836+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":197,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.794923+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:41:34.794923+0000","last_peered":"2023-10-09T05:41:34.794923+0000","last_clean":"2023-10-09T05:41:34.794923+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:41:34.794923+0000","last_undegraded":"2023-10-09T05:41:34.794923+0000","last_fullsized":"2023-10-09T05:41:34.794923+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"217'148","reported_seq":160,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:38.282336+0000","last_change":"2023-10-09T05:41:29.077953+0000","last_active":"2023-10-09T05:41:38.282336+0000","last_peered":"2023-10-09T05:41:38.282336+0000","last_clean":"2023-10-09T05:41:38.282336+0000","last_became_active":"2023-10-09T05:41:29.077652+0000","last_became_peered":"2023-10-09T05:41:29.077652+0000","last_unstale":"2023-10-09T05:41:38.282336+0000","last_undegraded":"2023-10-09T05:41:38.282336+0000","last_fullsized":"2023-10-09T05:41:38.282336+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:53.470052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9699328,"num_objects":148,"num_object_clones":0,"num_object_copies":444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":148,"num_write_kb":9472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":532,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:31.073894+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:41:31.073894+0000","last_peered":"2023-10-09T05:41:31.073894+0000","last_clean":"2023-10-09T05:41:31.073894+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:41:31.073894+0000","last_undegraded":"2023-10-09T05:41:31.073894+0000","last_fullsized":"2023-10-09T05:41:31.073894+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":165,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.860875+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:41:34.860875+0000","last_peered":"2023-10-09T05:41:34.860875+0000","last_clean":"2023-10-09T05:41:34.860875+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:41:34.860875+0000","last_undegraded":"2023-10-09T05:41:34.860875+0000","last_fullsized":"2023-10-09T05:41:34.860875+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":195,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.861024+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:41:34.861024+0000","last_peered":"2023-10-09T05:41:34.861024+0000","last_clean":"2023-10-09T05:41:34.861024+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:41:34.861024+0000","last_undegraded":"2023-10-09T05:41:34.861024+0000","last_fullsized":"2023-10-09T05:41:34.861024+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"217'167","reported_seq":179,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:38.280695+0000","last_change":"2023-10-09T05:41:29.082532+0000","last_active":"2023-10-09T05:41:38.280695+0000","last_peered":"2023-10-09T05:41:38.280695+0000","last_clean":"2023-10-09T05:41:38.280695+0000","last_became_active":"2023-10-09T05:41:29.082272+0000","last_became_peered":"2023-10-09T05:41:29.082272+0000","last_unstale":"2023-10-09T05:41:38.280695+0000","last_undegraded":"2023-10-09T05:41:38.280695+0000","last_fullsized":"2023-10-09T05:41:38.280695+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:51.574989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10944512,"num_objects":167,"num_object_clones":0,"num_object_copies":501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":167,"num_write_kb":10688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":829,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.794635+0000","last_change":"2023-10-09T05:40:04.805258+0000","last_active":"2023-10-09T05:41:34.794635+0000","last_peered":"2023-10-09T05:41:34.794635+0000","last_clean":"2023-10-09T05:41:34.794635+0000","last_became_active":"2023-10-09T05:40:04.804776+0000","last_became_peered":"2023-10-09T05:40:04.804776+0000","last_unstale":"2023-10-09T05:41:34.794635+0000","last_undegraded":"2023-10-09T05:41:34.794635+0000","last_fullsized":"2023-10-09T05:41:34.794635+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":365,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954077+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:41:34.954077+0000","last_peered":"2023-10-09T05:41:34.954077+0000","last_clean":"2023-10-09T05:41:34.954077+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:41:34.954077+0000","last_undegraded":"2023-10-09T05:41:34.954077+0000","last_fullsized":"2023-10-09T05:41:34.954077+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":180,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.155185+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:41:32.155185+0000","last_peered":"2023-10-09T05:41:32.155185+0000","last_clean":"2023-10-09T05:41:32.155185+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:41:32.155185+0000","last_undegraded":"2023-10-09T05:41:32.155185+0000","last_fullsized":"2023-10-09T05:41:32.155185+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":189,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154680+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:41:32.154680+0000","last_peered":"2023-10-09T05:41:32.154680+0000","last_clean":"2023-10-09T05:41:32.154680+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:41:32.154680+0000","last_undegraded":"2023-10-09T05:41:32.154680+0000","last_fullsized":"2023-10-09T05:41:32.154680+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":13,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:31.836220+0000","last_change":"2023-10-09T05:41:29.076367+0000","last_active":"2023-10-09T05:41:31.836220+0000","last_peered":"2023-10-09T05:41:31.836220+0000","last_clean":"2023-10-09T05:41:31.836220+0000","last_became_active":"2023-10-09T05:41:29.076035+0000","last_became_peered":"2023-10-09T05:41:29.076035+0000","last_unstale":"2023-10-09T05:41:31.836220+0000","last_undegraded":"2023-10-09T05:41:31.836220+0000","last_fullsized":"2023-10-09T05:41:31.836220+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:51.713835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":775,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T05:41:38.922033+0000","last_change":"2023-10-09T05:40:05.930112+0000","last_active":"2023-10-09T05:41:38.922033+0000","last_peered":"2023-10-09T05:41:38.922033+0000","last_clean":"2023-10-09T05:41:38.922033+0000","last_became_active":"2023-10-09T05:40:05.929574+0000","last_became_peered":"2023-10-09T05:40:05.929574+0000","last_unstale":"2023-10-09T05:41:38.922033+0000","last_undegraded":"2023-10-09T05:41:38.922033+0000","last_fullsized":"2023-10-09T05:41:38.922033+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:06.982326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":169,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154946+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:41:32.154946+0000","last_peered":"2023-10-09T05:41:32.154946+0000","last_clean":"2023-10-09T05:41:32.154946+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:41:32.154946+0000","last_undegraded":"2023-10-09T05:41:32.154946+0000","last_fullsized":"2023-10-09T05:41:32.154946+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":184,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154927+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:41:32.154927+0000","last_peered":"2023-10-09T05:41:32.154927+0000","last_clean":"2023-10-09T05:41:32.154927+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:41:32.154927+0000","last_undegraded":"2023-10-09T05:41:32.154927+0000","last_fullsized":"2023-10-09T05:41:32.154927+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"218'434","reported_seq":447,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T05:41:39.968368+0000","last_change":"2023-10-09T05:41:29.073701+0000","last_active":"2023-10-09T05:41:39.968368+0000","last_peered":"2023-10-09T05:41:39.968368+0000","last_clean":"2023-10-09T05:41:39.968368+0000","last_became_active":"2023-10-09T05:41:29.073372+0000","last_became_peered":"2023-10-09T05:41:29.073372+0000","last_unstale":"2023-10-09T05:41:39.968368+0000","last_undegraded":"2023-10-09T05:41:39.968368+0000","last_fullsized":"2023-10-09T05:41:39.968368+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":434,"log_dups_size":0,"ondisk_log_size":434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28442624,"num_objects":434,"num_object_clones":0,"num_object_copies":1302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":434,"num_write_kb":27776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":405,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154616+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:41:32.154616+0000","last_peered":"2023-10-09T05:41:32.154616+0000","last_clean":"2023-10-09T05:41:32.154616+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:41:32.154616+0000","last_undegraded":"2023-10-09T05:41:32.154616+0000","last_fullsized":"2023-10-09T05:41:32.154616+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":165,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.860903+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:41:34.860903+0000","last_peered":"2023-10-09T05:41:34.860903+0000","last_clean":"2023-10-09T05:41:34.860903+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:41:34.860903+0000","last_undegraded":"2023-10-09T05:41:34.860903+0000","last_fullsized":"2023-10-09T05:41:34.860903+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":178,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154033+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:41:32.154033+0000","last_peered":"2023-10-09T05:41:32.154033+0000","last_clean":"2023-10-09T05:41:32.154033+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:41:32.154033+0000","last_undegraded":"2023-10-09T05:41:32.154033+0000","last_fullsized":"2023-10-09T05:41:32.154033+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":12,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154276+0000","last_change":"2023-10-09T05:41:29.080413+0000","last_active":"2023-10-09T05:41:32.154276+0000","last_peered":"2023-10-09T05:41:32.154276+0000","last_clean":"2023-10-09T05:41:32.154276+0000","last_became_active":"2023-10-09T05:41:29.080184+0000","last_became_peered":"2023-10-09T05:41:29.080184+0000","last_unstale":"2023-10-09T05:41:32.154276+0000","last_undegraded":"2023-10-09T05:41:32.154276+0000","last_fullsized":"2023-10-09T05:41:32.154276+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:47.929696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":215,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.156067+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:41:32.156067+0000","last_peered":"2023-10-09T05:41:32.156067+0000","last_clean":"2023-10-09T05:41:32.156067+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:41:32.156067+0000","last_undegraded":"2023-10-09T05:41:32.156067+0000","last_fullsized":"2023-10-09T05:41:32.156067+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":606,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.861899+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:41:34.861899+0000","last_peered":"2023-10-09T05:41:34.861899+0000","last_clean":"2023-10-09T05:41:34.861899+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:41:34.861899+0000","last_undegraded":"2023-10-09T05:41:34.861899+0000","last_fullsized":"2023-10-09T05:41:34.861899+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":209,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154076+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:41:32.154076+0000","last_peered":"2023-10-09T05:41:32.154076+0000","last_clean":"2023-10-09T05:41:32.154076+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:41:32.154076+0000","last_undegraded":"2023-10-09T05:41:32.154076+0000","last_fullsized":"2023-10-09T05:41:32.154076+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":12,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.155063+0000","last_change":"2023-10-09T05:41:29.081230+0000","last_active":"2023-10-09T05:41:32.155063+0000","last_peered":"2023-10-09T05:41:32.155063+0000","last_clean":"2023-10-09T05:41:32.155063+0000","last_became_active":"2023-10-09T05:41:29.080911+0000","last_became_peered":"2023-10-09T05:41:29.080911+0000","last_unstale":"2023-10-09T05:41:32.155063+0000","last_undegraded":"2023-10-09T05:41:32.155063+0000","last_fullsized":"2023-10-09T05:41:32.155063+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:22.580967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":717,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154168+0000","last_change":"2023-10-09T05:41:21.917602+0000","last_active":"2023-10-09T05:41:32.154168+0000","last_peered":"2023-10-09T05:41:32.154168+0000","last_clean":"2023-10-09T05:41:32.154168+0000","last_became_active":"2023-10-09T05:41:21.917116+0000","last_became_peered":"2023-10-09T05:41:21.917116+0000","last_unstale":"2023-10-09T05:41:32.154168+0000","last_undegraded":"2023-10-09T05:41:32.154168+0000","last_fullsized":"2023-10-09T05:41:32.154168+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":172,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.155046+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:41:32.155046+0000","last_peered":"2023-10-09T05:41:32.155046+0000","last_clean":"2023-10-09T05:41:32.155046+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:41:32.155046+0000","last_undegraded":"2023-10-09T05:41:32.155046+0000","last_fullsized":"2023-10-09T05:41:32.155046+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":168,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954107+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:41:34.954107+0000","last_peered":"2023-10-09T05:41:34.954107+0000","last_clean":"2023-10-09T05:41:34.954107+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:41:34.954107+0000","last_undegraded":"2023-10-09T05:41:34.954107+0000","last_fullsized":"2023-10-09T05:41:34.954107+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":180,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.153927+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:41:32.153927+0000","last_peered":"2023-10-09T05:41:32.153927+0000","last_clean":"2023-10-09T05:41:32.153927+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:41:32.153927+0000","last_undegraded":"2023-10-09T05:41:32.153927+0000","last_fullsized":"2023-10-09T05:41:32.153927+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":209,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:31.074471+0000","last_change":"2023-10-09T05:31:32.870694+0000","last_active":"2023-10-09T05:41:31.074471+0000","last_peered":"2023-10-09T05:41:31.074471+0000","last_clean":"2023-10-09T05:41:31.074471+0000","last_became_active":"2023-10-09T05:31:32.870386+0000","last_became_peered":"2023-10-09T05:31:32.870386+0000","last_unstale":"2023-10-09T05:41:31.074471+0000","last_undegraded":"2023-10-09T05:41:31.074471+0000","last_fullsized":"2023-10-09T05:41:31.074471+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:10.334525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":187,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.794607+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:41:34.794607+0000","last_peered":"2023-10-09T05:41:34.794607+0000","last_clean":"2023-10-09T05:41:34.794607+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:41:34.794607+0000","last_undegraded":"2023-10-09T05:41:34.794607+0000","last_fullsized":"2023-10-09T05:41:34.794607+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":198,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.954896+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:41:34.954896+0000","last_peered":"2023-10-09T05:41:34.954896+0000","last_clean":"2023-10-09T05:41:34.954896+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:41:34.954896+0000","last_undegraded":"2023-10-09T05:41:34.954896+0000","last_fullsized":"2023-10-09T05:41:34.954896+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":172,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.794682+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:41:34.794682+0000","last_peered":"2023-10-09T05:41:34.794682+0000","last_clean":"2023-10-09T05:41:34.794682+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:41:34.794682+0000","last_undegraded":"2023-10-09T05:41:34.794682+0000","last_fullsized":"2023-10-09T05:41:34.794682+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":198,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.953999+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:41:34.953999+0000","last_peered":"2023-10-09T05:41:34.953999+0000","last_clean":"2023-10-09T05:41:34.953999+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:41:34.953999+0000","last_undegraded":"2023-10-09T05:41:34.953999+0000","last_fullsized":"2023-10-09T05:41:34.953999+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":196,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.153984+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:41:32.153984+0000","last_peered":"2023-10-09T05:41:32.153984+0000","last_clean":"2023-10-09T05:41:32.153984+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:41:32.153984+0000","last_undegraded":"2023-10-09T05:41:32.153984+0000","last_fullsized":"2023-10-09T05:41:32.153984+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":208,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.155603+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:41:32.155603+0000","last_peered":"2023-10-09T05:41:32.155603+0000","last_clean":"2023-10-09T05:41:32.155603+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:41:32.155603+0000","last_undegraded":"2023-10-09T05:41:32.155603+0000","last_fullsized":"2023-10-09T05:41:32.155603+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":172,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T05:41:32.154711+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:41:32.154711+0000","last_peered":"2023-10-09T05:41:32.154711+0000","last_clean":"2023-10-09T05:41:32.154711+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:41:32.154711+0000","last_undegraded":"2023-10-09T05:41:32.154711+0000","last_fullsized":"2023-10-09T05:41:32.154711+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":183,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T05:41:34.953974+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:41:34.953974+0000","last_peered":"2023-10-09T05:41:34.953974+0000","last_clean":"2023-10-09T05:41:34.953974+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:41:34.953974+0000","last_undegraded":"2023-10-09T05:41:34.953974+0000","last_fullsized":"2023-10-09T05:41:34.953974+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":115933184,"num_objects":1769,"num_object_clones":0,"num_object_copies":5307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1769,"num_write_kb":113216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":516620288,"data_stored":516620288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1769,"ondisk_log_size":1769,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846677,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626702,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312056,"kb_used_data":6640,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93433096,"statfs":{"total":95995035648,"available":95675490304,"internally_reserved":0,"allocated":6799360,"data_stored":6267434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:28 2023","interfaces":[{"interface":"back","average":{"1min":2.032,"5min":5.6829999999999998,"15min":3.2210000000000001},"min":{"1min":0.23100000000000001,"5min":0.214,"15min":0.214},"max":{"1min":14.23,"5min":115.377,"15min":115.377},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.9350000000000001,"5min":5.3710000000000004,"15min":3.0800000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23999999999999999},"max":{"1min":11.727,"5min":116.428,"15min":116.428},"last":0.49399999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":7.274,"5min":6.056,"15min":2.7170000000000001},"min":{"1min":0.315,"5min":0.25,"15min":0.25},"max":{"1min":93.489000000000004,"5min":137.02199999999999,"15min":137.02199999999999},"last":0.433},{"interface":"front","average":{"1min":6.6299999999999999,"5min":5.9379999999999997,"15min":2.758},"min":{"1min":0.318,"5min":0.252,"15min":0.252},"max":{"1min":93.471999999999994,"5min":137.06700000000001,"15min":137.06700000000001},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:41:28 2023","interfaces":[{"interface":"back","average":{"1min":7.8099999999999996,"5min":6.226,"15min":3.456},"min":{"1min":0.32200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":87.703000000000003,"5min":115.40600000000001,"15min":115.40600000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":8.4559999999999995,"5min":6.5220000000000002,"15min":3.4980000000000002},"min":{"1min":0.35499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":87.689999999999998,"5min":115.297,"15min":115.297},"last":0.46500000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":11.682,"5min":6.7359999999999998,"15min":3.1219999999999999},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":99.543999999999997,"5min":115.244,"15min":115.244},"last":0.40600000000000003},{"interface":"front","average":{"1min":10.734,"5min":6.556,"15min":3.1499999999999999},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":98.677999999999997,"5min":115.34999999999999,"15min":115.34999999999999},"last":0.56399999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.8200000000000001,"15min":1.7110000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":0.72399999999999998,"5min":115.214,"15min":125.64},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.70399999999999996,"5min":1.5820000000000001,"15min":1.6120000000000001},"min":{"1min":0.27400000000000002,"5min":0.254,"15min":0.254},"max":{"1min":2.7189999999999999,"5min":115.22799999999999,"15min":125.625},"last":0.54600000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":2.5600000000000001,"15min":1.855},"min":{"1min":0.33500000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4690000000000003,"5min":115.316,"15min":127.35899999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":2.1549999999999998,"15min":1.7370000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.6079999999999997,"5min":115.30500000000001,"15min":128.93799999999999},"last":0.51900000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":2.077,"15min":2.7909999999999999},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":5.0110000000000001,"5min":115.514,"15min":115.514},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.8240000000000001,"15min":2.0019999999999998},"min":{"1min":0.34899999999999998,"5min":0.317,"15min":0.317},"max":{"1min":3.6600000000000001,"5min":116.53100000000001,"15min":116.53100000000001},"last":0.57299999999999995}]}]},{"osd":1,"up_from":93,"seq":399431958664,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":383880,"kb_used_data":113792,"kb_used_omap":7,"kb_used_meta":270072,"kb_avail":93361272,"statfs":{"total":95995035648,"available":95601942528,"internally_reserved":0,"allocated":116523008,"data_stored":115744369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":276554254},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":4.694,"5min":5.1500000000000004,"15min":2.6080000000000001},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":39.189,"5min":207.858,"15min":207.858},"last":1.3280000000000001},{"interface":"front","average":{"1min":6.1989999999999998,"5min":6.4420000000000002,"15min":3.0720000000000001},"min":{"1min":0.23000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":42.012999999999998,"5min":206.97200000000001,"15min":206.97200000000001},"last":1.526}]},{"osd":2,"last update":"Mon Oct 9 05:41:11 2023","interfaces":[{"interface":"back","average":{"1min":6.2229999999999999,"5min":7.3860000000000001,"15min":3.5019999999999998},"min":{"1min":0.26000000000000001,"5min":0.22,"15min":0.184},"max":{"1min":42.469999999999999,"5min":101.367,"15min":101.367},"last":0.76500000000000001},{"interface":"front","average":{"1min":5.3369999999999997,"5min":6.9020000000000001,"15min":3.294},"min":{"1min":0.23300000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":42.151000000000003,"5min":97.286000000000001,"15min":98.409999999999997},"last":1.635}]},{"osd":3,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":18.629999999999999,"5min":7.2770000000000001,"15min":2.9540000000000002},"min":{"1min":0.28899999999999998,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":162.33699999999999,"5min":162.33699999999999,"15min":162.33699999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":19.029,"5min":7.9980000000000002,"15min":3.1960000000000002},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":162.376,"5min":162.376,"15min":162.376},"last":1.5029999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.1749999999999998,"5min":4.7249999999999996,"15min":2.2690000000000001},"min":{"1min":0.39700000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":42.106999999999999,"5min":141.32900000000001,"15min":141.32900000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":5.2800000000000002,"5min":3.6819999999999999,"15min":1.8400000000000001},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":40.475999999999999,"5min":141.15899999999999,"15min":141.15899999999999},"last":1.601}]},{"osd":5,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.6420000000000003,"5min":4.7009999999999996,"15min":2.2349999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":40.633000000000003,"5min":235.18700000000001,"15min":235.18700000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":5.258,"5min":5.5570000000000004,"15min":2.6240000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":42.078000000000003,"5min":235.79300000000001,"15min":235.79300000000001},"last":0.63300000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.8860000000000001,"5min":3.8109999999999999,"15min":1.8779999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":40.558999999999997,"5min":103.60899999999999,"15min":103.60899999999999},"last":1.0580000000000001},{"interface":"front","average":{"1min":5.234,"5min":4.4950000000000001,"15min":2.141},"min":{"1min":0.41299999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":42.134999999999998,"5min":103.679,"15min":103.679},"last":0.60099999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.4950000000000001,"5min":3.649,"15min":2.5489999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":42.204000000000001,"5min":126.68300000000001,"15min":126.68300000000001},"last":1.669},{"interface":"front","average":{"1min":5.4050000000000002,"5min":3.7450000000000001,"15min":2.6339999999999999},"min":{"1min":0.46999999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":42.228999999999999,"5min":126.738,"15min":126.738},"last":1.875}]}]},{"osd":0,"up_from":88,"seq":377957122190,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":394920,"kb_used_data":38880,"kb_used_omap":8,"kb_used_meta":356023,"kb_avail":93350232,"statfs":{"total":95995035648,"available":95590637568,"internally_reserved":0,"allocated":39813120,"data_stored":39003619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":364568328},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4910000000000001,"5min":2.3340000000000001,"15min":1.282},"min":{"1min":0.20699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":17.760000000000002,"5min":17.760000000000002,"15min":17.760000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":6.0330000000000004,"5min":3.3679999999999999,"15min":1.6539999999999999},"min":{"1min":0.25,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":26.163,"5min":26.163,"15min":26.163},"last":0.501}]},{"osd":2,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":7.7830000000000004,"5min":4.2309999999999999,"15min":2.6949999999999998},"min":{"1min":0.23999999999999999,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":92.230000000000004,"5min":92.230000000000004,"15min":92.230000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":8.4440000000000008,"5min":4.3079999999999998,"15min":2.6419999999999999},"min":{"1min":0.253,"5min":0.221,"15min":0.218},"max":{"1min":92.247,"5min":92.247,"15min":92.247},"last":0.51700000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":18.498999999999999,"5min":6.4859999999999998,"15min":2.7669999999999999},"min":{"1min":0.27500000000000002,"5min":0.26000000000000001,"15min":0.22800000000000001},"max":{"1min":123.024,"5min":123.024,"15min":123.024},"last":0.56999999999999995},{"interface":"front","average":{"1min":12.829000000000001,"5min":5.4359999999999999,"15min":2.48},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":99.162000000000006,"5min":99.162000000000006,"15min":99.162000000000006},"last":0.55400000000000005}]},{"osd":4,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.9970000000000001,"15min":1.288},"min":{"1min":0.32200000000000001,"5min":0.27500000000000002,"15min":0.27200000000000002},"max":{"1min":8.6470000000000002,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.878},{"interface":"front","average":{"1min":3.206,"5min":2.778,"15min":1.6799999999999999},"min":{"1min":0.41399999999999998,"5min":0.33000000000000002,"15min":0.30399999999999999},"max":{"1min":7.9240000000000004,"5min":69.102000000000004,"15min":69.102000000000004},"last":0.90900000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":3.2450000000000001,"5min":6.4889999999999999,"15min":2.907},"min":{"1min":0.36299999999999999,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":8.0289999999999999,"5min":281.733,"15min":281.733},"last":0.755},{"interface":"front","average":{"1min":2.6080000000000001,"5min":6.3609999999999998,"15min":2.8650000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.4269999999999996,"5min":282.95400000000001,"15min":282.95400000000001},"last":0.89000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":3.266,"5min":2.2610000000000001,"15min":1.4990000000000001},"min":{"1min":0.39800000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9969999999999999,"5min":11.664,"15min":11.664},"last":0.78700000000000003},{"interface":"front","average":{"1min":2.355,"5min":2.2200000000000002,"15min":1.4990000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.8490000000000002,"5min":10.83,"15min":10.83},"last":0.93300000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.679,"15min":1.4450000000000001},"min":{"1min":0.41899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.5090000000000003,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.94299999999999995},{"interface":"front","average":{"1min":3.1859999999999999,"5min":2.548,"15min":2.4420000000000002},"min":{"1min":0.46800000000000003,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.6329999999999991,"5min":9.7159999999999993,"15min":11.068},"last":0.95199999999999996}]}]},{"osd":2,"up_from":98,"seq":420906795137,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":405936,"kb_used_data":6440,"kb_used_omap":11,"kb_used_meta":399476,"kb_avail":93339216,"statfs":{"total":95995035648,"available":95579357184,"internally_reserved":0,"allocated":6594560,"data_stored":5722368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":409064021},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":4.3230000000000004,"5min":3.4670000000000001,"15min":1.7949999999999999},"min":{"1min":0.215,"5min":0.17799999999999999,"15min":0.17199999999999999},"max":{"1min":21.587,"5min":21.587,"15min":32.122},"last":0.378},{"interface":"front","average":{"1min":3.702,"5min":3.4889999999999999,"15min":1.7430000000000001},"min":{"1min":0.214,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":15.468,"5min":21.997,"15min":34.066000000000003},"last":0.46100000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":5.5529999999999999,"15min":2.6539999999999999},"min":{"1min":0.29199999999999998,"5min":0.23400000000000001,"15min":0.20699999999999999},"max":{"1min":24.265000000000001,"5min":98.575000000000003,"15min":98.575000000000003},"last":0.41999999999999998},{"interface":"front","average":{"1min":4.5599999999999996,"5min":4.5069999999999997,"15min":2.282},"min":{"1min":0.23100000000000001,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":18.882999999999999,"5min":90.135000000000005,"15min":98.397000000000006},"last":0.40699999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":5.7240000000000002,"5min":6.1200000000000001,"15min":2.6499999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.24199999999999999},"max":{"1min":24.471,"5min":150.65899999999999,"15min":150.65899999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":6.6200000000000001,"5min":6.9770000000000003,"15min":3.016},"min":{"1min":0.46000000000000002,"5min":0.29899999999999999,"15min":0.26500000000000001},"max":{"1min":22.631,"5min":150.851,"15min":150.851},"last":0.44900000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6629999999999998,"5min":3.2719999999999998,"15min":1.7210000000000001},"min":{"1min":0.32200000000000001,"5min":0.29099999999999998,"15min":0.27700000000000002},"max":{"1min":14.382,"5min":115.84999999999999,"15min":115.84999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.5449999999999999,"5min":3.7109999999999999,"15min":1.9770000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.29799999999999999},"max":{"1min":13.815,"5min":116.149,"15min":116.149},"last":0.49299999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":2.8959999999999999,"15min":1.6870000000000001},"min":{"1min":0.36699999999999999,"5min":0.312,"15min":0.312},"max":{"1min":13.712999999999999,"5min":24.597999999999999,"15min":24.597999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":3.222,"5min":2.302,"15min":1.466},"min":{"1min":0.35499999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":14.426,"5min":23.919,"15min":23.919},"last":0.52300000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":2.734,"15min":1.637},"min":{"1min":0.36499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":13.839,"5min":20.503,"15min":20.503},"last":0.624},{"interface":"front","average":{"1min":2.71,"5min":2.5619999999999998,"15min":1.536},"min":{"1min":0.40600000000000003,"5min":0.39600000000000002,"15min":0.34499999999999997},"max":{"1min":13.757,"5min":13.757,"15min":13.757},"last":0.53800000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:40:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":3.2509999999999999,"15min":2.766},"min":{"1min":0.47199999999999998,"5min":0.40200000000000002,"15min":0.374},"max":{"1min":13.865,"5min":24.030999999999999,"15min":24.030999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":3.5070000000000001,"5min":3.1110000000000002,"15min":2.1890000000000001},"min":{"1min":0.52700000000000002,"5min":0.41899999999999998,"15min":0.41599999999999998},"max":{"1min":13.781000000000001,"5min":24.178000000000001,"15min":24.178000000000001},"last":0.58099999999999996}]}]},{"osd":3,"up_from":103,"seq":442381631611,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":344636,"kb_used_data":6324,"kb_used_omap":4,"kb_used_meta":338299,"kb_avail":93400516,"statfs":{"total":95995035648,"available":95642128384,"internally_reserved":0,"allocated":6475776,"data_stored":5706813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":346418782},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":3.8690000000000002,"5min":5.0350000000000001,"15min":2.3599999999999999},"min":{"1min":0.311,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":26.614999999999998,"5min":167.82400000000001,"15min":167.82400000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":5.6230000000000002,"5min":5.7380000000000004,"15min":2.6829999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":26.140000000000001,"5min":167.89599999999999,"15min":167.89599999999999},"last":0.24099999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":5.931,"5min":4.8170000000000002,"15min":2.0249999999999999},"min":{"1min":0.27400000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":48.052,"5min":83.921999999999997,"15min":83.921999999999997},"last":0.36499999999999999},{"interface":"front","average":{"1min":5.6310000000000002,"5min":4.7140000000000004,"15min":2.0009999999999999},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.247},"max":{"1min":13.551,"5min":83.707999999999998,"15min":83.707999999999998},"last":0.29699999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3920000000000003,"5min":7.3719999999999999,"15min":3.629},"min":{"1min":0.307,"5min":0.218,"15min":0.215},"max":{"1min":41.542000000000002,"5min":190.43299999999999,"15min":190.43299999999999},"last":0.28100000000000003},{"interface":"front","average":{"1min":5.3739999999999997,"5min":8.4800000000000004,"15min":4.0250000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.26000000000000001},"max":{"1min":26.097999999999999,"5min":190.267,"15min":190.267},"last":0.312}]},{"osd":4,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0030000000000001,"5min":3.2879999999999998,"15min":1.966},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":25.946999999999999,"5min":51.591999999999999,"15min":51.591999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":3.1299999999999999,"5min":2.2530000000000001,"15min":1.5269999999999999},"min":{"1min":0.33500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":26.774999999999999,"5min":26.774999999999999,"15min":45.735999999999997},"last":0.40699999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":4.5890000000000004,"5min":2.427,"15min":1.5880000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":13.114000000000001,"5min":13.114000000000001,"15min":47.948999999999998},"last":0.503},{"interface":"front","average":{"1min":3.1699999999999999,"5min":2.2970000000000002,"15min":1.5960000000000001},"min":{"1min":0.499,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":26.728000000000002,"5min":26.728000000000002,"15min":47.968000000000004},"last":0.45600000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:40:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":2.387,"15min":1.504},"min":{"1min":0.50700000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":26.847999999999999,"5min":26.847999999999999,"15min":26.847999999999999},"last":0.434},{"interface":"front","average":{"1min":4.8730000000000002,"5min":2.6619999999999999,"15min":1.4139999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":13.058,"5min":13.058,"15min":13.058},"last":0.52600000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:40:56 2023","interfaces":[{"interface":"back","average":{"1min":4.032,"5min":3.0249999999999999,"15min":2.5150000000000001},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":26.024999999999999,"5min":26.024999999999999,"15min":26.024999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":4.2720000000000002,"5min":3.125,"15min":2.3500000000000001},"min":{"1min":0.42499999999999999,"5min":0.40899999999999997,"15min":0.35699999999999998},"max":{"1min":25.995999999999999,"5min":25.995999999999999,"15min":25.995999999999999},"last":0.44800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019159,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":432448,"kb_used_data":122936,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93312704,"statfs":{"total":95995035648,"available":95552208896,"internally_reserved":0,"allocated":125886464,"data_stored":125353926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:24 2023","interfaces":[{"interface":"back","average":{"1min":6.5049999999999999,"5min":2.403,"15min":1.7170000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.214},"max":{"1min":100.504,"5min":100.504,"15min":100.504},"last":0.26800000000000002},{"interface":"front","average":{"1min":6.827,"5min":2.9089999999999998,"15min":1.998},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":100.717,"5min":100.717,"15min":100.717},"last":0.60799999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":1.6699999999999999,"15min":1.21},"min":{"1min":0.29999999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8140000000000001,"5min":7.9550000000000001,"15min":31.073},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.492,"5min":1.8520000000000001,"15min":1.3300000000000001},"min":{"1min":0.29299999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.349,"5min":12.349,"15min":31.100000000000001},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":8.7859999999999996,"5min":5.681,"15min":4.0270000000000001},"min":{"1min":0.315,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":100.148,"5min":196.93700000000001,"15min":196.93700000000001},"last":1.1879999999999999},{"interface":"front","average":{"1min":9.0389999999999997,"5min":5.492,"15min":3.931},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":100.086,"5min":197.08099999999999,"15min":197.08099999999999},"last":0.495}]},{"osd":3,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":4.3570000000000002,"5min":7.3419999999999996,"15min":3.097},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.509,"5min":211.48699999999999,"15min":211.48699999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":3.2109999999999999,"5min":7.157,"15min":2.9740000000000002},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.309},"max":{"1min":13.007,"5min":211.31100000000001,"15min":211.31100000000001},"last":1.637}]},{"osd":5,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.0310000000000001,"15min":1.5649999999999999},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.249},"max":{"1min":3.2730000000000001,"5min":94.646000000000001,"15min":113.35899999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.883,"15min":1.5229999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.30599999999999999},"max":{"1min":3.3849999999999998,"5min":94.692999999999998,"15min":113.32299999999999},"last":0.38400000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:40:57 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.72899999999999998,"15min":0.70499999999999996},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":1.8740000000000001,"5min":6.1779999999999999,"15min":11.944000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.93500000000000005,"15min":0.86399999999999999},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.4810000000000001,"5min":7.6500000000000004,"15min":10.356},"last":0.33400000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.94799999999999995,"15min":2.5619999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.312},"max":{"1min":3.415,"5min":11.845000000000001,"15min":30.393999999999998},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.97599999999999998,"15min":2.8780000000000001},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.5089999999999999,"5min":10.837,"15min":30.334},"last":0.51500000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822931,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":408944,"kb_used_data":103528,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93336208,"statfs":{"total":95995035648,"available":95576276992,"internally_reserved":0,"allocated":106012672,"data_stored":105491966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.879,"15min":3.008},"min":{"1min":0.217,"5min":0.217,"15min":0.19400000000000001},"max":{"1min":9.032,"5min":10.712999999999999,"15min":165.90199999999999},"last":0.499},{"interface":"front","average":{"1min":1.849,"5min":1.8300000000000001,"15min":3.077},"min":{"1min":0.30399999999999999,"5min":0.28399999999999997,"15min":0.215},"max":{"1min":7.4050000000000002,"5min":12.263999999999999,"15min":165.96700000000001},"last":0.38400000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":7.0419999999999998,"5min":4.6070000000000002,"15min":2.1699999999999999},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":71.209999999999994,"5min":87.769000000000005,"15min":87.769000000000005},"last":0.55200000000000005},{"interface":"front","average":{"1min":8.1159999999999997,"5min":5.2130000000000001,"15min":2.2810000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.24399999999999999},"max":{"1min":71.192999999999998,"5min":87.864000000000004,"15min":87.864000000000004},"last":0.41999999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":3.5299999999999998,"15min":4.0659999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.1910000000000007,"5min":73.311999999999998,"15min":290.697},"last":0.441},{"interface":"front","average":{"1min":3.1920000000000002,"5min":3.3929999999999998,"15min":3.956},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":9.2140000000000004,"5min":73.280000000000001,"15min":290.786},"last":0.56399999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":7.5910000000000002,"5min":6.8520000000000003,"15min":2.8919999999999999},"min":{"1min":0.35199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":85.007999999999996,"5min":95.236999999999995,"15min":95.236999999999995},"last":0.52000000000000002},{"interface":"front","average":{"1min":6.7199999999999998,"5min":6.2770000000000001,"15min":2.7189999999999999},"min":{"1min":0.32900000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":83.147999999999996,"5min":93.471000000000004,"15min":93.471000000000004},"last":0.41199999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:40:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.81299999999999994,"15min":0.72099999999999997},"min":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":5.6459999999999999,"5min":7.2060000000000004,"15min":9.3510000000000009},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.1970000000000001,"15min":0.94399999999999995},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.23400000000000001},"max":{"1min":4.8209999999999997,"5min":10.913,"15min":14.236000000000001},"last":0.37}]},{"osd":6,"last update":"Mon Oct 9 05:41:04 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.2370000000000001,"15min":1.3600000000000001},"min":{"1min":0.35699999999999998,"5min":0.27600000000000002,"15min":0.26600000000000001},"max":{"1min":4.79,"5min":27.146000000000001,"15min":95.010999999999996},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.1359999999999999,"15min":1.2609999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27700000000000002},"max":{"1min":4.6600000000000001,"5min":27.215,"15min":94.194000000000003},"last":0.53700000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:41:00 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":1.228,"15min":1.5589999999999999},"min":{"1min":0.39900000000000002,"5min":0.34399999999999997,"15min":0.28999999999999998},"max":{"1min":5.5570000000000004,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.436},{"interface":"front","average":{"1min":1.212,"5min":1.157,"15min":1.8009999999999999},"min":{"1min":0.373,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.0549999999999997,"5min":10.779,"15min":14.432},"last":0.47999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":85458944,"data_stored":85458944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":116785152,"data_stored":116785152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":171311104,"data_stored":171311104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":143065088,"data_stored":143065088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:41:44.744 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T05:41:44.744 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:41:44.744 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2023-10-09T05:41:44.745 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:41:44.745 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:44.842 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:44.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:44.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:44.870 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:44.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:44.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]': finished 2023-10-09T05:41:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T05:41:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: osdmap e219: 8 total, 8 up, 7 in 2023-10-09T05:41:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:44.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:44.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[184418]: osdmap e220: 8 total, 8 up, 7 in 2023-10-09T05:41:44.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]': finished 2023-10-09T05:41:44.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: osdmap e219: 8 total, 8 up, 7 in 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: failed to encode map e219 with expected crc 2023-10-09T05:41:44.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:44.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:44.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:44 smithi008 ceph-mon[191227]: osdmap e220: 8 total, 8 up, 7 in 2023-10-09T05:41:44.979 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:45.089 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:45.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: failed to encode map e219 with expected crc 2023-10-09T05:41:45.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]': finished 2023-10-09T05:41:45.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T05:41:45.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: osdmap e219: 8 total, 8 up, 7 in 2023-10-09T05:41:45.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: failed to encode map e219 with expected crc 2023-10-09T05:41:45.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: failed to encode map e219 with expected crc 2023-10-09T05:41:45.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: failed to encode map e219 with expected crc 2023-10-09T05:41:45.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: failed to encode map e219 with expected crc 2023-10-09T05:41:45.091 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: failed to encode map e219 with expected crc 2023-10-09T05:41:45.091 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: failed to encode map e219 with expected crc 2023-10-09T05:41:45.091 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: failed to encode map e219 with expected crc 2023-10-09T05:41:45.091 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: failed to encode map e219 with expected crc 2023-10-09T05:41:45.091 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: failed to encode map e219 with expected crc 2023-10-09T05:41:45.091 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: failed to encode map e219 with expected crc 2023-10-09T05:41:45.091 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:45.091 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:45.091 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:44 smithi191 ceph-mon[161669]: osdmap e220: 8 total, 8 up, 7 in 2023-10-09T05:41:45.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:45.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:45.193 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:45.242 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:45.287 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:45.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:45.329 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 8 16 16905 16889 131.757 105.188 0.105579 0.00755228 2023-10-09T05:41:45.391 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:45.420 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:41:45.448 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:41:45.449 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:45.472 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:41:45.493 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:41:45.497 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:45.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:45.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:45.549 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:41:45.583 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:41:45.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:45.618 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:41:45.633 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:45.660 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:41:45.672 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:41:45.672 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:41:45.672 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:41:45.673 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:41:45.673 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:41:45.673 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:41:45.673 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:41:45.673 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:41:45.889 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:45.894 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:45 smithi191 ceph-mon[161669]: pgmap v618: 89 pgs: 89 active+clean; 671 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:41:45.894 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:45 smithi191 ceph-mon[161669]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.894 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:45 smithi191 ceph-mon[161669]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.894 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:45 smithi191 ceph-mon[161669]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.894 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:45 smithi191 ceph-mon[161669]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.894 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:45 smithi191 ceph-mon[161669]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.894 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:45 smithi191 ceph-mon[161669]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.894 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/153286736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:45.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:45.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:45.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[184418]: pgmap v618: 89 pgs: 89 active+clean; 671 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:41:45.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[184418]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[184418]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[184418]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[184418]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[184418]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[184418]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/153286736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:45.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[191227]: pgmap v618: 89 pgs: 89 active+clean; 671 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:41:45.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[191227]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[191227]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[191227]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[191227]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[191227]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[191227]: failed to encode map e220 with expected crc 2023-10-09T05:41:45.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/153286736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:45.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:46.036 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:46.152 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:46.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:46.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:46.256 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:46.326 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:46.332 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 9 16 18725 18709 129.712 113.75 0.0115811 0.00770169 2023-10-09T05:41:46.401 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:41:46.401 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:41:46.401 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T05:41:46.401 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:41:46.423 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:46.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:46.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:46.530 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:46.592 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:46.679 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:46.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:46.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:46.783 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:46.809 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:46.853 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:46.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:46.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:46.956 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:46.984 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:47.094 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:47.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:47.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:47.126 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:46 smithi008 ceph-mon[191227]: pgmap v620: 89 pgs: 89 active+clean; 850 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T05:41:47.127 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:46 smithi008 ceph-mon[184418]: pgmap v620: 89 pgs: 89 active+clean; 850 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T05:41:47.199 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:47.227 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:47.278 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:46 smithi191 ceph-mon[161669]: pgmap v620: 89 pgs: 89 active+clean; 850 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T05:41:47.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:41:47.333 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 10 16 19820 19804 123.592 68.4375 0.00255048 0.00807266 2023-10-09T05:41:47.731 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:47.827 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:47.857 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:47.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:47.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:47.970 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:48.043 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:48.332 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 11 16 21435 21419 121.534 100.938 0.00756101 0.00822018 2023-10-09T05:41:48.393 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:48.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:48.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:48.500 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:48.578 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:48.741 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:48.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:48.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:48.850 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:48.952 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:49.075 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:49.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:49.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:49.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:49.223 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:49.293 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:49.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:49.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:49.333 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 12 16 23066 23050 119.902 101.938 0.00762442 0.00833269 2023-10-09T05:41:49.399 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:49.442 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:49.512 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:49.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:49.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:49.527 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:41:49.527 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:41:49.527 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:41:49.527 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:41:49.527 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:41:49.616 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:49.667 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:49.783 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:49.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:49.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:49.796 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:41:49.826 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:41:49.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:49.914 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:49.960 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:49.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:49.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:49 smithi008 ceph-mon[184418]: pgmap v621: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T05:41:49.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2285896097' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:49.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:49 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:49 smithi008 ceph-mon[191227]: pgmap v621: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T05:41:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2285896097' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:49 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:50.064 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:50.121 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:50.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:49 smithi191 ceph-mon[161669]: pgmap v621: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T05:41:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2285896097' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:49 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:41:50.333 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 13 16 24518 24502 117.662 90.75 0.0154226 0.0084938 2023-10-09T05:41:50.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:41:50.644 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:50.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:50.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:50.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:41:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:41:50] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.33.4" 2023-10-09T05:41:50.752 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:50.780 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:50.804 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:41:51.175 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:50 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:41:51.175 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:50 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:41:51.176 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:50 smithi191 ceph-mon[161669]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.176 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:50 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:51.176 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:50 smithi191 ceph-mon[161669]: osdmap e221: 8 total, 8 up, 7 in 2023-10-09T05:41:51.176 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:50 smithi191 ceph-mon[161669]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.176 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:50 smithi191 ceph-mon[161669]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.177 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:50 smithi191 ceph-mon[161669]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.177 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:50 smithi191 ceph-mon[161669]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.202 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:51.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:41:51.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:41:51.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[184418]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:51.208 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[184418]: osdmap e221: 8 total, 8 up, 7 in 2023-10-09T05:41:51.208 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[184418]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.208 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[184418]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.208 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[184418]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.208 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[184418]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:51.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:51.209 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:41:51.209 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:41:51.209 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[191227]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.210 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:41:51.210 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[191227]: osdmap e221: 8 total, 8 up, 7 in 2023-10-09T05:41:51.210 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[191227]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.210 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[191227]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.210 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[191227]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.210 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:50 smithi008 ceph-mon[191227]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.310 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:51.333 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 14 16 26177 26161 116.664 103.688 0.013015 0.00856415 2023-10-09T05:41:51.429 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:51.524 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:51.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:51.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:51.634 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:41:51.664 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:51.687 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:51.715 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:41:51.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:51.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:41:51.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:51.834 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:51 smithi008 ceph-mon[184418]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.834 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:51 smithi008 ceph-mon[184418]: pgmap v623: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T05:41:51.834 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:51 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:51.835 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:51 smithi008 ceph-mon[191227]: failed to encode map e221 with expected crc 2023-10-09T05:41:51.837 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:51 smithi008 ceph-mon[191227]: pgmap v623: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T05:41:51.837 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:51 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:51.852 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:52.085 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:51 smithi191 ceph-mon[161669]: failed to encode map e221 with expected crc 2023-10-09T05:41:52.086 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:51 smithi191 ceph-mon[161669]: pgmap v623: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T05:41:52.086 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:51 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:52.115 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:52.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:52.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:52.204 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:41:52.214 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:41:52.224 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:52.263 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:52.339 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 15 16 27422 27406 114.029 77.8125 0.00764546 0.00875945 2023-10-09T05:41:52.384 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:52.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:52.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:52.489 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:52.533 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:52.590 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:52.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:52.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:52.611 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:41:52.611 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:41:52.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:41:52.611 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:41:52.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.611 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138523 5.4 5.84db7384 (undecoded) ondisk+retry+write+known_if_redirected e127)", 2023-10-09T05:41:52.612 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.658381+0000", 2023-10-09T05:41:52.612 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 431.949302816, 2023-10-09T05:41:52.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99745561100000002, 2023-10-09T05:41:52.612 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.612 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:52.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.612 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138523 2023-10-09T05:41:52.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.613 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658381+0000", 2023-10-09T05:41:52.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658381+0000", 2023-10-09T05:41:52.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:52.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658380+0000", 2023-10-09T05:41:52.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7150000000000001e-06 2023-10-09T05:41:52.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658386+0000", 2023-10-09T05:41:52.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2750000000000001e-06 2023-10-09T05:41:52.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658396+0000", 2023-10-09T05:41:52.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7730000000000002e-06 2023-10-09T05:41:52.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658402+0000", 2023-10-09T05:41:52.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00063723800000000004 2023-10-09T05:41:52.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659040+0000", 2023-10-09T05:41:52.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2666999999999999e-05 2023-10-09T05:41:52.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659052+0000", 2023-10-09T05:41:52.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99677223800000003 2023-10-09T05:41:52.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655825+0000", 2023-10-09T05:41:52.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2245999999999999e-05 2023-10-09T05:41:52.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655837+0000", 2023-10-09T05:41:52.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99745561100000002 2023-10-09T05:41:52.619 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.619 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.619 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.620 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138528 5.4 5.bf234c24 (undecoded) ondisk+retry+write+known_if_redirected e127)", 2023-10-09T05:41:52.620 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.658412+0000", 2023-10-09T05:41:52.620 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 431.94927216000002, 2023-10-09T05:41:52.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.997458705, 2023-10-09T05:41:52.620 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.620 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:52.620 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.620 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.621 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.621 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138528 2023-10-09T05:41:52.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.621 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658412+0000", 2023-10-09T05:41:52.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658412+0000", 2023-10-09T05:41:52.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:52.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658411+0000", 2023-10-09T05:41:52.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6459999999999999e-06 2023-10-09T05:41:52.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658415+0000", 2023-10-09T05:41:52.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.49e-06 2023-10-09T05:41:52.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658420+0000", 2023-10-09T05:41:52.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6770000000000003e-06 2023-10-09T05:41:52.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658425+0000", 2023-10-09T05:41:52.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000652908 2023-10-09T05:41:52.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659078+0000", 2023-10-09T05:41:52.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7360000000000001e-06 2023-10-09T05:41:52.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659086+0000", 2023-10-09T05:41:52.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99677464199999999 2023-10-09T05:41:52.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655860+0000", 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0182e-05 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655871+0000", 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.997458705 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138537 5.14 5.de8f0eb4 (undecoded) ondisk+retry+write+known_if_redirected e127)", 2023-10-09T05:41:52.627 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.658438+0000", 2023-10-09T05:41:52.628 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 431.949245738, 2023-10-09T05:41:52.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99594088300000005, 2023-10-09T05:41:52.628 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.628 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:52.628 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.628 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.628 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.628 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138537 2023-10-09T05:41:52.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.628 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658438+0000", 2023-10-09T05:41:52.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658438+0000", 2023-10-09T05:41:52.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:52.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.630 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.630 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658438+0000", 2023-10-09T05:41:52.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0450000000000001e-06 2023-10-09T05:41:52.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.630 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.630 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658441+0000", 2023-10-09T05:41:52.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2435e-05 2023-10-09T05:41:52.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658453+0000", 2023-10-09T05:41:52.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7509999999999999e-06 2023-10-09T05:41:52.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658458+0000", 2023-10-09T05:41:52.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00090573200000000002 2023-10-09T05:41:52.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659364+0000", 2023-10-09T05:41:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1409999999999999e-06 2023-10-09T05:41:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659371+0000", 2023-10-09T05:41:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99499399300000002 2023-10-09T05:41:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654365+0000", 2023-10-09T05:41:52.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4296e-05 2023-10-09T05:41:52.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654379+0000", 2023-10-09T05:41:52.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99594088300000005 2023-10-09T05:41:52.633 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.634 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.634 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.634 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138539 5.14 5.df7ef574 (undecoded) ondisk+retry+write+known_if_redirected e127)", 2023-10-09T05:41:52.634 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.658463+0000", 2023-10-09T05:41:52.635 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 431.949220755, 2023-10-09T05:41:52.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99595421100000003, 2023-10-09T05:41:52.635 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.635 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:52.635 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.636 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.636 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.636 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138539 2023-10-09T05:41:52.636 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.636 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.636 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.636 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.637 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658463+0000", 2023-10-09T05:41:52.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.637 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658463+0000", 2023-10-09T05:41:52.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:52.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658463+0000", 2023-10-09T05:41:52.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3050000000000001e-06 2023-10-09T05:41:52.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658466+0000", 2023-10-09T05:41:52.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9939999999999997e-06 2023-10-09T05:41:52.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658472+0000", 2023-10-09T05:41:52.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3320000000000002e-06 2023-10-09T05:41:52.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658476+0000", 2023-10-09T05:41:52.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001204175 2023-10-09T05:41:52.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659681+0000", 2023-10-09T05:41:52.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6250000000000002e-06 2023-10-09T05:41:52.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.659690+0000", 2023-10-09T05:41:52.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99471561100000006 2023-10-09T05:41:52.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654406+0000", 2023-10-09T05:41:52.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1659e-05 2023-10-09T05:41:52.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654418+0000", 2023-10-09T05:41:52.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99595421100000003 2023-10-09T05:41:52.641 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.641 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.642 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.642 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.642 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138541 5.9 5.c283ca09 (undecoded) ondisk+write+known_if_redirected e127)", 2023-10-09T05:41:52.642 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.658481+0000", 2023-10-09T05:41:52.642 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 431.94920352899999, 2023-10-09T05:41:52.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0067079050000001, 2023-10-09T05:41:52.642 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.643 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:52.643 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.643 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.643 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.643 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138541 2023-10-09T05:41:52.643 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.643 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.643 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658481+0000", 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658481+0000", 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658480+0000", 2023-10-09T05:41:52.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3970000000000001e-06 2023-10-09T05:41:52.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658483+0000", 2023-10-09T05:41:52.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2420000000000002e-06 2023-10-09T05:41:52.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658487+0000", 2023-10-09T05:41:52.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4499999999999998e-06 2023-10-09T05:41:52.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658489+0000", 2023-10-09T05:41:52.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4831000000000001e-05 2023-10-09T05:41:52.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658584+0000", 2023-10-09T05:41:52.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7370000000000001e-06 2023-10-09T05:41:52.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.658594+0000", 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.006582222 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.665176+0000", 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2413e-05 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.665188+0000", 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0067079050000001 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138551 5.19 5.f6e6d179 (undecoded) ondisk+write+known_if_redirected e127)", 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.681781+0000", 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 431.925902705, 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97309186599999997, 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.648 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138551 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681781+0000", 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681781+0000", 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681780+0000", 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0450000000000001e-06 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.650 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681788+0000", 2023-10-09T05:41:52.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0400999999999999e-05 2023-10-09T05:41:52.651 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.651 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.651 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681798+0000", 2023-10-09T05:41:52.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.439e-06 2023-10-09T05:41:52.651 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.651 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.651 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.681807+0000", 2023-10-09T05:41:52.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002494975 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684302+0000", 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0268999999999999e-05 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684312+0000", 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97055105699999999 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654863+0000", 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0227999999999999e-05 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654873+0000", 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97309186599999997 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.654 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138552 5.4 5.ef384104 (undecoded) ondisk+write+known_if_redirected e127)", 2023-10-09T05:41:52.654 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.684234+0000", 2023-10-09T05:41:52.654 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 431.92344958699999, 2023-10-09T05:41:52.654 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97166571400000001, 2023-10-09T05:41:52.654 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.654 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:52.654 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.654 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.654 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.655 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138552 2023-10-09T05:41:52.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.655 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.655 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.655 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684234+0000", 2023-10-09T05:41:52.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.655 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.655 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684234+0000", 2023-10-09T05:41:52.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:41:52.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.656 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684232+0000", 2023-10-09T05:41:52.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4124999999999999e-05 2023-10-09T05:41:52.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.658 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.658 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.658 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684247+0000", 2023-10-09T05:41:52.658 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3252999999999999e-05 2023-10-09T05:41:52.658 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.658 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684270+0000", 2023-10-09T05:41:52.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4730000000000006e-06 2023-10-09T05:41:52.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684279+0000", 2023-10-09T05:41:52.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028417399999999999 2023-10-09T05:41:52.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684563+0000", 2023-10-09T05:41:52.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.842e-05 2023-10-09T05:41:52.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684582+0000", 2023-10-09T05:41:52.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97130895799999994 2023-10-09T05:41:52.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655891+0000", 2023-10-09T05:41:52.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3510000000000008e-06 2023-10-09T05:41:52.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.655900+0000", 2023-10-09T05:41:52.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97166571400000001 2023-10-09T05:41:52.661 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.661 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.661 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.662 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.662 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138555 5.14 5.8803b654 (undecoded) ondisk+write+known_if_redirected e127)", 2023-10-09T05:41:52.662 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.684293+0000", 2023-10-09T05:41:52.662 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 431.92339095800003, 2023-10-09T05:41:52.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97015670499999995, 2023-10-09T05:41:52.662 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.662 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:52.662 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.663 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.663 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.663 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138555 2023-10-09T05:41:52.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.663 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.663 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.663 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684293+0000", 2023-10-09T05:41:52.663 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684293+0000", 2023-10-09T05:41:52.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:52.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684292+0000", 2023-10-09T05:41:52.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9009999999999996e-06 2023-10-09T05:41:52.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.665 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.665 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684296+0000", 2023-10-09T05:41:52.665 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6510000000000004e-06 2023-10-09T05:41:52.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.665 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.665 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.665 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684303+0000", 2023-10-09T05:41:52.665 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4569999999999998e-06 2023-10-09T05:41:52.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.665 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684307+0000", 2023-10-09T05:41:52.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00047098700000000002 2023-10-09T05:41:52.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684778+0000", 2023-10-09T05:41:52.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3480000000000004e-06 2023-10-09T05:41:52.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.667 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.667 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.684787+0000", 2023-10-09T05:41:52.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96965241499999999 2023-10-09T05:41:52.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.667 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.667 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.667 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654439+0000", 2023-10-09T05:41:52.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0606e-05 2023-10-09T05:41:52.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.667 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.654450+0000", 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97015670499999995 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:138562 5.9 5.6835f569 (undecoded) ondisk+write+known_if_redirected e127)", 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:40.699110+0000", 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 431.908573698, 2023-10-09T05:41:52.668 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96611003200000001, 2023-10-09T05:41:52.669 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.669 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:52.669 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.669 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.669 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.669 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138562 2023-10-09T05:41:52.669 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.669 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.669 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.669 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.669 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699110+0000", 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699110+0000", 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699109+0000", 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1216999999999999e-05 2023-10-09T05:41:52.670 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.671 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.671 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699120+0000", 2023-10-09T05:41:52.671 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7193e-05 2023-10-09T05:41:52.671 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.671 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.671 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699137+0000", 2023-10-09T05:41:52.671 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5379999999999999e-06 2023-10-09T05:41:52.671 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699144+0000", 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4489e-05 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699198+0000", 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9580000000000002e-06 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.672 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:40.699208+0000", 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96600283600000003 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.665211+0000", 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2779999999999999e-06 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:41.665220+0000", 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96611003200000001 2023-10-09T05:41:52.673 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:218946 5.19 5:9ed084b7:::benchmark_data_smithi008_238708_object100810:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e145)", 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.643711+0000", 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 341.96397285299997, 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.028762001, 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.674 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.675 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.675 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 218946 2023-10-09T05:41:52.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.675 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.675 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643711+0000", 2023-10-09T05:41:52.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643711+0000", 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643709+0000", 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5990000000000001e-05 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643725+0000", 2023-10-09T05:41:52.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7798000000000001e-05 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643743+0000", 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3549999999999999e-06 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643751+0000", 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4769000000000002e-05 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643825+0000", 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.361e-05 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.643839+0000", 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.011016055 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.654855+0000", 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7491999999999994e-05 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.654952+0000", 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017458551999999999 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672411+0000", 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2207999999999998e-05 2023-10-09T05:41:52.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672473+0000", 2023-10-09T05:41:52.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.028762001 2023-10-09T05:41:52.680 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.680 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.680 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.680 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:218960 5.19 5:9c42dd17:::benchmark_data_smithi008_238708_object100824:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:41:52.680 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.653577+0000", 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 341.954106733, 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.034507125, 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 218960 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653577+0000", 2023-10-09T05:41:52.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653577+0000", 2023-10-09T05:41:52.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999919 2023-10-09T05:41:52.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653569+0000", 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2618e-05 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653582+0000", 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9889999999999998e-06 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653591+0000", 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7910000000000004e-06 2023-10-09T05:41:52.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.684 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.684 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653596+0000", 2023-10-09T05:41:52.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001950198 2023-10-09T05:41:52.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.684 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.684 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.655546+0000", 2023-10-09T05:41:52.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0429e-05 2023-10-09T05:41:52.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.685 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.685 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.685 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.655556+0000", 2023-10-09T05:41:52.685 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99943933200000001 2023-10-09T05:41:52.685 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.685 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.685 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.686 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.654996+0000", 2023-10-09T05:41:52.686 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6644999999999994e-05 2023-10-09T05:41:52.686 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.686 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.686 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:52.686 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.655092+0000", 2023-10-09T05:41:52.686 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00037147599999999998 2023-10-09T05:41:52.687 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.687 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.687 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:52.687 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.655464+0000", 2023-10-09T05:41:52.687 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0089923880000000005 2023-10-09T05:41:52.687 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.687 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:52.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.664456+0000", 2023-10-09T05:41:52.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023390363000000001 2023-10-09T05:41:52.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.687846+0000", 2023-10-09T05:41:52.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5685000000000005e-05 2023-10-09T05:41:52.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.687922+0000", 2023-10-09T05:41:52.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2720999999999998e-05 2023-10-09T05:41:52.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.690 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:52.690 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.687985+0000", 2023-10-09T05:41:52.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.959e-05 2023-10-09T05:41:52.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.690 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.690 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.690 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688084+0000", 2023-10-09T05:41:52.691 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.034507125 2023-10-09T05:41:52.691 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.691 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.691 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.691 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.691 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.691 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:218961 5.19 5:9c1d5529:::benchmark_data_smithi008_238708_object100825:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:41:52.692 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.653606+0000", 2023-10-09T05:41:52.692 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 341.95407811699999, 2023-10-09T05:41:52.692 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0346523700000001, 2023-10-09T05:41:52.692 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.692 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:52.692 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.692 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.693 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.693 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 218961 2023-10-09T05:41:52.693 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.693 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.693 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.693 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653606+0000", 2023-10-09T05:41:52.694 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.694 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.694 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.694 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653606+0000", 2023-10-09T05:41:52.694 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:52.694 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.695 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.695 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653606+0000", 2023-10-09T05:41:52.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.34e-06 2023-10-09T05:41:52.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.695 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653608+0000", 2023-10-09T05:41:52.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8399999999999995e-06 2023-10-09T05:41:52.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.696 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.696 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653616+0000", 2023-10-09T05:41:52.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9369999999999997e-06 2023-10-09T05:41:52.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.697 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.697 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.653620+0000", 2023-10-09T05:41:52.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001972414 2023-10-09T05:41:52.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.655592+0000", 2023-10-09T05:41:52.698 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.131e-06 2023-10-09T05:41:52.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.699 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.655599+0000", 2023-10-09T05:41:52.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.001532678 2023-10-09T05:41:52.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.699 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.699 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.699 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.657132+0000", 2023-10-09T05:41:52.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6515999999999997e-05 2023-10-09T05:41:52.700 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:52.700 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.657208+0000", 2023-10-09T05:41:52.700 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021616300000000001 2023-10-09T05:41:52.700 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:52.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.657425+0000", 2023-10-09T05:41:52.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010284753000000001 2023-10-09T05:41:52.701 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.701 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.701 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:52.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.667709+0000", 2023-10-09T05:41:52.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020432143999999999 2023-10-09T05:41:52.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.702 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688141+0000", 2023-10-09T05:41:52.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4103000000000002e-05 2023-10-09T05:41:52.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.703 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.703 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688176+0000", 2023-10-09T05:41:52.703 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9569000000000001e-05 2023-10-09T05:41:52.703 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.703 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.703 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:52.703 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688205+0000", 2023-10-09T05:41:52.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3216e-05 2023-10-09T05:41:52.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688258+0000", 2023-10-09T05:41:52.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0346523700000001 2023-10-09T05:41:52.704 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.705 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.705 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.705 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219000 5.19 5:9a536baf:::benchmark_data_smithi008_238708_object100864:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:41:52.705 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.708353+0000", 2023-10-09T05:41:52.705 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 341.89933129999997, 2023-10-09T05:41:52.705 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98004795200000006, 2023-10-09T05:41:52.706 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.706 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:52.706 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.706 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.706 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.706 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219000 2023-10-09T05:41:52.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.707 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708353+0000", 2023-10-09T05:41:52.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.707 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.708 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.708 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708353+0000", 2023-10-09T05:41:52.708 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:41:52.708 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.708 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.708 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.708 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708351+0000", 2023-10-09T05:41:52.708 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3520000000000007e-06 2023-10-09T05:41:52.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.709 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.709 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.709 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708359+0000", 2023-10-09T05:41:52.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2427e-05 2023-10-09T05:41:52.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.709 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.710 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.710 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708372+0000", 2023-10-09T05:41:52.710 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3820000000000001e-06 2023-10-09T05:41:52.710 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.710 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.710 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.710 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.708378+0000", 2023-10-09T05:41:52.711 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002761337 2023-10-09T05:41:52.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.711 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.711139+0000", 2023-10-09T05:41:52.711 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0630000000000006e-06 2023-10-09T05:41:52.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.712 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.712 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.711148+0000", 2023-10-09T05:41:52.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94768007300000001 2023-10-09T05:41:52.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.712 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.712 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.658828+0000", 2023-10-09T05:41:52.713 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0051999999999998e-05 2023-10-09T05:41:52.713 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:52.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.658918+0000", 2023-10-09T05:41:52.713 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000231487 2023-10-09T05:41:52.713 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.714 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.714 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:52.714 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.659149+0000", 2023-10-09T05:41:52.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0085686330000000008 2023-10-09T05:41:52.714 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.714 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.714 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:52.715 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.667718+0000", 2023-10-09T05:41:52.715 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020580158000000001 2023-10-09T05:41:52.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.715 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.715 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.715 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688298+0000", 2023-10-09T05:41:52.715 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1848000000000002e-05 2023-10-09T05:41:52.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.716 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.716 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.716 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688320+0000", 2023-10-09T05:41:52.716 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7568000000000001e-05 2023-10-09T05:41:52.716 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.716 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.716 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:52.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688347+0000", 2023-10-09T05:41:52.717 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3371000000000001e-05 2023-10-09T05:41:52.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.717 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688401+0000", 2023-10-09T05:41:52.717 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98004795200000006 2023-10-09T05:41:52.718 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.718 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.718 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.718 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.718 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.718 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219006 5.19 5:9d7b33f6:::benchmark_data_smithi008_238708_object100870:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:41:52.718 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.718358+0000", 2023-10-09T05:41:52.718 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 341.88932586200002, 2023-10-09T05:41:52.719 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97025771199999999, 2023-10-09T05:41:52.719 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.719 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:52.719 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.719 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.719 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.720 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219006 2023-10-09T05:41:52.720 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.720 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.720 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.720 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.720 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718358+0000", 2023-10-09T05:41:52.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.721 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.721 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.721 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718358+0000", 2023-10-09T05:41:52.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:52.721 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.721 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718357+0000", 2023-10-09T05:41:52.722 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.238e-06 2023-10-09T05:41:52.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718364+0000", 2023-10-09T05:41:52.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5883999999999999e-05 2023-10-09T05:41:52.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718380+0000", 2023-10-09T05:41:52.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3749999999999997e-06 2023-10-09T05:41:52.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.724 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.724 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718387+0000", 2023-10-09T05:41:52.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5766999999999996e-05 2023-10-09T05:41:52.724 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.724 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.724 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718453+0000", 2023-10-09T05:41:52.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1304999999999999e-05 2023-10-09T05:41:52.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.718464+0000", 2023-10-09T05:41:52.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94278578400000002 2023-10-09T05:41:52.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.661250+0000", 2023-10-09T05:41:52.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119416 2023-10-09T05:41:52.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:52.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.661369+0000", 2023-10-09T05:41:52.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00037868800000000002 2023-10-09T05:41:52.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:52.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.661748+0000", 2023-10-09T05:41:52.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059772230000000003 2023-10-09T05:41:52.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:52.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.667725+0000", 2023-10-09T05:41:52.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020709274999999999 2023-10-09T05:41:52.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688435+0000", 2023-10-09T05:41:52.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7390999999999999e-05 2023-10-09T05:41:52.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.729 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.729 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.729 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688462+0000", 2023-10-09T05:41:52.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0025999999999998e-05 2023-10-09T05:41:52.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:52.730 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688512+0000", 2023-10-09T05:41:52.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103828 2023-10-09T05:41:52.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688616+0000", 2023-10-09T05:41:52.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97025771199999999 2023-10-09T05:41:52.731 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.731 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.731 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.731 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219040 5.19 5:9927ccc2:::benchmark_data_smithi008_238708_object100904:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:41:52.731 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.768643+0000", 2023-10-09T05:41:52.731 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 341.83904149300002, 2023-10-09T05:41:52.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92022723900000003, 2023-10-09T05:41:52.732 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.732 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:52.732 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.732 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.732 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.732 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219040 2023-10-09T05:41:52.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.732 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768643+0000", 2023-10-09T05:41:52.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768643+0000", 2023-10-09T05:41:52.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:41:52.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768641+0000", 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9090000000000006e-06 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768650+0000", 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5481999999999998e-05 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768665+0000", 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2269999999999998e-06 2023-10-09T05:41:52.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768671+0000", 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6886999999999999e-05 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768728+0000", 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0456e-05 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.768739+0000", 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89575502600000001 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.664494+0000", 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9017000000000004e-05 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.664593+0000", 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00022533399999999999 2023-10-09T05:41:52.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.664818+0000", 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075507980000000001 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672369+0000", 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016304721000000001 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688673+0000", 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0516000000000001e-05 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688734+0000", 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4342000000000003e-05 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688778+0000", 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1475000000000001e-05 2023-10-09T05:41:52.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688870+0000", 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92022723900000003 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219067 5.19 5:9a00ab2f:::benchmark_data_smithi008_238708_object100931:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.806455+0000", 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 341.80122952300002, 2023-10-09T05:41:52.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88268894200000003, 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219067 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806455+0000", 2023-10-09T05:41:52.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806455+0000", 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806453+0000", 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7700000000000004e-06 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806460+0000", 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2864000000000001e-05 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806473+0000", 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0069999999999997e-06 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806478+0000", 2023-10-09T05:41:52.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1210999999999994e-05 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806539+0000", 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2235000000000001e-05 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806551+0000", 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85952352399999998 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.666075+0000", 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4686999999999998e-05 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.666149+0000", 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021614200000000001 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.666365+0000", 2023-10-09T05:41:52.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087013460000000004 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675067+0000", 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013862637000000001 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688929+0000", 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6715999999999999e-05 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.688966+0000", 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9138000000000002e-05 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689025+0000", 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118199 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689143+0000", 2023-10-09T05:41:52.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88268894200000003 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219068 5.19 5:9c85096d:::benchmark_data_smithi008_238708_object100932:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.806490+0000", 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 341.80119452600002, 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88291725200000004, 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.747 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219068 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806490+0000", 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806490+0000", 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806489+0000", 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2910000000000002e-06 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806491+0000", 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6740000000000002e-06 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806497+0000", 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5790000000000002e-06 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806500+0000", 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00047362400000000002 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806973+0000", 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0248000000000001e-05 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.806984+0000", 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86079349100000002 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.667777+0000", 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012534800000000001 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.667902+0000", 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000286147 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.668189+0000", 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068859680000000001 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:52.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675075+0000", 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014158188 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689233+0000", 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6288e-05 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689289+0000", 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5240000000000001e-05 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:52.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689334+0000", 2023-10-09T05:41:52.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2529999999999998e-05 2023-10-09T05:41:52.754 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.754 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689407+0000", 2023-10-09T05:41:52.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88291725200000004 2023-10-09T05:41:52.754 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.754 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.754 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219080 5.19 5:9b22456a:::benchmark_data_smithi008_238708_object100944:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.825998+0000", 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 341.78168613100001, 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86367064199999999, 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.755 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219080 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.825998+0000", 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.825998+0000", 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.825996+0000", 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6950000000000006e-06 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.826005+0000", 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6087999999999999e-05 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.826021+0000", 2023-10-09T05:41:52.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0870000000000001e-06 2023-10-09T05:41:52.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.826028+0000", 2023-10-09T05:41:52.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7341999999999993e-05 2023-10-09T05:41:52.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.826096+0000", 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0266e-05 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.826106+0000", 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.84409459399999998 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.670201+0000", 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4676e-05 2023-10-09T05:41:52.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.670285+0000", 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000226407 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.670512+0000", 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045699679999999998 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675082+0000", 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01438443 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689466+0000", 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4876000000000003e-05 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689501+0000", 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5906999999999999e-05 2023-10-09T05:41:52.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689557+0000", 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011178799999999999 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689669+0000", 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86367064199999999 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.762 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.763 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219108 5.19 5:9a26fec5:::benchmark_data_smithi008_238708_object100972:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:41:52.763 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.859141+0000", 2023-10-09T05:41:52.763 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 341.74854306399999, 2023-10-09T05:41:52.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83075073300000002, 2023-10-09T05:41:52.763 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.763 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:52.763 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.764 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.764 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219108 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859141+0000", 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859141+0000", 2023-10-09T05:41:52.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859140+0000", 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8099999999999997e-06 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859144+0000", 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0951000000000001e-05 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859155+0000", 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.048e-06 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859159+0000", 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8704e-05 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859208+0000", 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6360000000000006e-06 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.859218+0000", 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81329287100000003 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672511+0000", 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000123986 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672635+0000", 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000329876 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.672964+0000", 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010650517999999999 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.683615+0000", 2023-10-09T05:41:52.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060958770000000004 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689711+0000", 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1442999999999997e-05 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689742+0000", 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6600000000000001e-05 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689789+0000", 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102797 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689892+0000", 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.83075073300000002 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:219141 5.19 5:9c1c6174:::benchmark_data_smithi008_238708_object101005:head [delete] snapc 0=[] ondisk+write+known_if_redirected e145)", 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:36:10.913601+0000", 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 341.69408335499998, 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.77651063099999995, 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 219141 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.772 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913601+0000", 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913601+0000", 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:41:52.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913599+0000", 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7490000000000001e-06 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913606+0000", 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2513e-05 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913618+0000", 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1380000000000001e-06 2023-10-09T05:41:52.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913626+0000", 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8097999999999998e-05 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913694+0000", 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3261000000000001e-05 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:41:52.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:10.913707+0000", 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.76142268800000001 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675130+0000", 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000116388 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675246+0000", 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000282212 2023-10-09T05:41:52.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.675528+0000", 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080939329999999993 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.683622+0000", 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063094969999999998 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.777 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689932+0000", 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1736000000000002e-05 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.689963+0000", 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4965000000000001e-05 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.690008+0000", 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102929 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:41:52.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:41:52.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:41:52.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:36:11.690111+0000", 2023-10-09T05:41:52.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.77651063099999995 2023-10-09T05:41:52.779 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.779 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.779 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.779 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:41:52.779 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:41:52.779 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:41:52.779 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:52.847 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:53.339 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 16 16 28744 28728 112.068 82.625 0.0111098 0.00891634 2023-10-09T05:41:53.359 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:41:53.477 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:41:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:53 smithi008 ceph-mon[184418]: pgmap v624: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T05:41:53.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:53 smithi008 ceph-mon[184418]: from='client.45380 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:53.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:53 smithi008 ceph-mon[191227]: pgmap v624: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T05:41:53.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:53 smithi008 ceph-mon[191227]: from='client.45380 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:53 smithi191 ceph-mon[161669]: pgmap v624: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T05:41:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:53 smithi191 ceph-mon[161669]: from='client.45380 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:54.339 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 17 16 29622 29606 108.708 54.875 0.00786112 0.00919144 2023-10-09T05:41:54.449 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-10-09T05:41:52.163545+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1601116083,"num_objects":24386,"num_object_clones":0,"num_object_copies":73158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24386,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":25464,"num_write_kb":1574053,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":4789622,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":25528,"ondisk_log_size":25528,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":266,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6859232,"kb_used_data":4551144,"kb_used_omap":31,"kb_used_meta":2308000,"kb_avail":649356832,"statfs":{"total":671965249536,"available":664941395968,"internally_reserved":0,"allocated":4660371456,"data_stored":4655509300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2363392457},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,5,5,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":897974272,"num_objects":13702,"num_object_clones":0,"num_object_copies":41106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13702,"num_write_kb":876928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002263"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":203,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858127+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:41:44.858127+0000","last_peered":"2023-10-09T05:41:44.858127+0000","last_clean":"2023-10-09T05:41:44.858127+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:41:44.858127+0000","last_undegraded":"2023-10-09T05:41:44.858127+0000","last_fullsized":"2023-10-09T05:41:44.858127+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":193,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.875575+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:41:44.875575+0000","last_peered":"2023-10-09T05:41:44.875575+0000","last_clean":"2023-10-09T05:41:44.875575+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:41:44.875575+0000","last_undegraded":"2023-10-09T05:41:44.875575+0000","last_fullsized":"2023-10-09T05:41:44.875575+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":166,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.854445+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:41:43.854445+0000","last_peered":"2023-10-09T05:41:43.854445+0000","last_clean":"2023-10-09T05:41:43.854445+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:41:43.854445+0000","last_undegraded":"2023-10-09T05:41:43.854445+0000","last_fullsized":"2023-10-09T05:41:43.854445+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":196,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.864570+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:41:43.864570+0000","last_peered":"2023-10-09T05:41:43.864570+0000","last_clean":"2023-10-09T05:41:43.864570+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:41:43.864570+0000","last_undegraded":"2023-10-09T05:41:43.864570+0000","last_fullsized":"2023-10-09T05:41:43.864570+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":209,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.875918+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:41:44.875918+0000","last_peered":"2023-10-09T05:41:44.875918+0000","last_clean":"2023-10-09T05:41:44.875918+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:41:44.875918+0000","last_undegraded":"2023-10-09T05:41:44.875918+0000","last_fullsized":"2023-10-09T05:41:44.875918+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":186,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.871447+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:41:44.871447+0000","last_peered":"2023-10-09T05:41:44.871447+0000","last_clean":"2023-10-09T05:41:44.871447+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:41:44.871447+0000","last_undegraded":"2023-10-09T05:41:44.871447+0000","last_fullsized":"2023-10-09T05:41:44.871447+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":171,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.869092+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:41:43.869092+0000","last_peered":"2023-10-09T05:41:43.869092+0000","last_clean":"2023-10-09T05:41:43.869092+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:41:43.869092+0000","last_undegraded":"2023-10-09T05:41:43.869092+0000","last_fullsized":"2023-10-09T05:41:43.869092+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":193,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.875926+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:41:44.875926+0000","last_peered":"2023-10-09T05:41:44.875926+0000","last_clean":"2023-10-09T05:41:44.875926+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:41:44.875926+0000","last_undegraded":"2023-10-09T05:41:44.875926+0000","last_fullsized":"2023-10-09T05:41:44.875926+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":212,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.874787+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:41:44.874787+0000","last_peered":"2023-10-09T05:41:44.874787+0000","last_clean":"2023-10-09T05:41:44.874787+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:41:44.874787+0000","last_undegraded":"2023-10-09T05:41:44.874787+0000","last_fullsized":"2023-10-09T05:41:44.874787+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":207,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.875157+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:41:44.875157+0000","last_peered":"2023-10-09T05:41:44.875157+0000","last_clean":"2023-10-09T05:41:44.875157+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:41:44.875157+0000","last_undegraded":"2023-10-09T05:41:44.875157+0000","last_fullsized":"2023-10-09T05:41:44.875157+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":100,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.875489+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:41:44.875489+0000","last_peered":"2023-10-09T05:41:44.875489+0000","last_clean":"2023-10-09T05:41:44.875489+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:41:44.875489+0000","last_undegraded":"2023-10-09T05:41:44.875489+0000","last_fullsized":"2023-10-09T05:41:44.875489+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":166,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.854041+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:41:43.854041+0000","last_peered":"2023-10-09T05:41:43.854041+0000","last_clean":"2023-10-09T05:41:43.854041+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:41:43.854041+0000","last_undegraded":"2023-10-09T05:41:43.854041+0000","last_fullsized":"2023-10-09T05:41:43.854041+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":228,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858606+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:41:44.858606+0000","last_peered":"2023-10-09T05:41:44.858606+0000","last_clean":"2023-10-09T05:41:44.858606+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:41:44.858606+0000","last_undegraded":"2023-10-09T05:41:44.858606+0000","last_fullsized":"2023-10-09T05:41:44.858606+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":186,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858236+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:41:44.858236+0000","last_peered":"2023-10-09T05:41:44.858236+0000","last_clean":"2023-10-09T05:41:44.858236+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:41:44.858236+0000","last_undegraded":"2023-10-09T05:41:44.858236+0000","last_fullsized":"2023-10-09T05:41:44.858236+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":99,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858306+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:41:44.858306+0000","last_peered":"2023-10-09T05:41:44.858306+0000","last_clean":"2023-10-09T05:41:44.858306+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:41:44.858306+0000","last_undegraded":"2023-10-09T05:41:44.858306+0000","last_fullsized":"2023-10-09T05:41:44.858306+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":217,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.876068+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:41:44.876068+0000","last_peered":"2023-10-09T05:41:44.876068+0000","last_clean":"2023-10-09T05:41:44.876068+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:41:44.876068+0000","last_undegraded":"2023-10-09T05:41:44.876068+0000","last_fullsized":"2023-10-09T05:41:44.876068+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":196,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.864636+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:41:43.864636+0000","last_peered":"2023-10-09T05:41:43.864636+0000","last_clean":"2023-10-09T05:41:43.864636+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:41:43.864636+0000","last_undegraded":"2023-10-09T05:41:43.864636+0000","last_fullsized":"2023-10-09T05:41:43.864636+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":211,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.874819+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:41:44.874819+0000","last_peered":"2023-10-09T05:41:44.874819+0000","last_clean":"2023-10-09T05:41:44.874819+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:41:44.874819+0000","last_undegraded":"2023-10-09T05:41:44.874819+0000","last_fullsized":"2023-10-09T05:41:44.874819+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":117,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858207+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:41:44.858207+0000","last_peered":"2023-10-09T05:41:44.858207+0000","last_clean":"2023-10-09T05:41:44.858207+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:41:44.858207+0000","last_undegraded":"2023-10-09T05:41:44.858207+0000","last_fullsized":"2023-10-09T05:41:44.858207+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":195,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.869774+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:41:43.869774+0000","last_peered":"2023-10-09T05:41:43.869774+0000","last_clean":"2023-10-09T05:41:43.869774+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:41:43.869774+0000","last_undegraded":"2023-10-09T05:41:43.869774+0000","last_fullsized":"2023-10-09T05:41:43.869774+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":195,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.869720+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:41:44.869720+0000","last_peered":"2023-10-09T05:41:44.869720+0000","last_clean":"2023-10-09T05:41:44.869720+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:41:44.869720+0000","last_undegraded":"2023-10-09T05:41:44.869720+0000","last_fullsized":"2023-10-09T05:41:44.869720+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":217,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.875606+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:41:44.875606+0000","last_peered":"2023-10-09T05:41:44.875606+0000","last_clean":"2023-10-09T05:41:44.875606+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:41:44.875606+0000","last_undegraded":"2023-10-09T05:41:44.875606+0000","last_fullsized":"2023-10-09T05:41:44.875606+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"220'1353","reported_seq":1368,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:48.959407+0000","last_change":"2023-10-09T05:41:29.078278+0000","last_active":"2023-10-09T05:41:48.959407+0000","last_peered":"2023-10-09T05:41:48.959407+0000","last_clean":"2023-10-09T05:41:48.959407+0000","last_became_active":"2023-10-09T05:41:29.077562+0000","last_became_peered":"2023-10-09T05:41:29.077562+0000","last_unstale":"2023-10-09T05:41:48.959407+0000","last_undegraded":"2023-10-09T05:41:48.959407+0000","last_fullsized":"2023-10-09T05:41:48.959407+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1353,"log_dups_size":0,"ondisk_log_size":1353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:29.784837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1353,"num_write_kb":86592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":97,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858871+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:41:44.858871+0000","last_peered":"2023-10-09T05:41:44.858871+0000","last_clean":"2023-10-09T05:41:44.858871+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:41:44.858871+0000","last_undegraded":"2023-10-09T05:41:44.858871+0000","last_fullsized":"2023-10-09T05:41:44.858871+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":196,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.874394+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:41:44.874394+0000","last_peered":"2023-10-09T05:41:44.874394+0000","last_clean":"2023-10-09T05:41:44.874394+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:41:44.874394+0000","last_undegraded":"2023-10-09T05:41:44.874394+0000","last_fullsized":"2023-10-09T05:41:44.874394+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"221'1633","reported_seq":1649,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:41:51.411080+0000","last_change":"2023-10-09T05:41:29.081067+0000","last_active":"2023-10-09T05:41:51.411080+0000","last_peered":"2023-10-09T05:41:51.411080+0000","last_clean":"2023-10-09T05:41:51.411080+0000","last_became_active":"2023-10-09T05:41:29.080704+0000","last_became_peered":"2023-10-09T05:41:29.080704+0000","last_unstale":"2023-10-09T05:41:51.411080+0000","last_undegraded":"2023-10-09T05:41:51.411080+0000","last_fullsized":"2023-10-09T05:41:51.411080+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1633,"log_dups_size":0,"ondisk_log_size":1633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:29.483205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107020288,"num_objects":1633,"num_object_clones":0,"num_object_copies":4899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1633,"num_write_kb":104512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":97,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858711+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:41:44.858711+0000","last_peered":"2023-10-09T05:41:44.858711+0000","last_clean":"2023-10-09T05:41:44.858711+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:41:44.858711+0000","last_undegraded":"2023-10-09T05:41:44.858711+0000","last_fullsized":"2023-10-09T05:41:44.858711+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":210,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.881539+0000","last_change":"2023-10-09T05:41:44.881327+0000","last_active":"2023-10-09T05:41:44.881539+0000","last_peered":"2023-10-09T05:41:44.881539+0000","last_clean":"2023-10-09T05:41:44.881539+0000","last_became_active":"2023-10-09T05:41:44.881044+0000","last_became_peered":"2023-10-09T05:41:44.881044+0000","last_unstale":"2023-10-09T05:41:44.881539+0000","last_undegraded":"2023-10-09T05:41:44.881539+0000","last_fullsized":"2023-10-09T05:41:44.881539+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"220'1297","reported_seq":1312,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:48.259207+0000","last_change":"2023-10-09T05:41:29.077986+0000","last_active":"2023-10-09T05:41:48.259207+0000","last_peered":"2023-10-09T05:41:48.259207+0000","last_clean":"2023-10-09T05:41:48.259207+0000","last_became_active":"2023-10-09T05:41:29.077631+0000","last_became_peered":"2023-10-09T05:41:29.077631+0000","last_unstale":"2023-10-09T05:41:48.259207+0000","last_undegraded":"2023-10-09T05:41:48.259207+0000","last_fullsized":"2023-10-09T05:41:48.259207+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1297,"log_dups_size":0,"ondisk_log_size":1297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:48.684759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85000192,"num_objects":1297,"num_object_clones":0,"num_object_copies":3891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1297,"num_write_kb":83008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":98,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.874906+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:41:44.874906+0000","last_peered":"2023-10-09T05:41:44.874906+0000","last_clean":"2023-10-09T05:41:44.874906+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:41:44.874906+0000","last_undegraded":"2023-10-09T05:41:44.874906+0000","last_fullsized":"2023-10-09T05:41:44.874906+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":219,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.869928+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:41:44.869928+0000","last_peered":"2023-10-09T05:41:44.869928+0000","last_clean":"2023-10-09T05:41:44.869928+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:41:44.869928+0000","last_undegraded":"2023-10-09T05:41:44.869928+0000","last_fullsized":"2023-10-09T05:41:44.869928+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"221'1642","reported_seq":1659,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:41:51.799415+0000","last_change":"2023-10-09T05:41:29.073764+0000","last_active":"2023-10-09T05:41:51.799415+0000","last_peered":"2023-10-09T05:41:51.799415+0000","last_clean":"2023-10-09T05:41:51.799415+0000","last_became_active":"2023-10-09T05:41:29.073452+0000","last_became_peered":"2023-10-09T05:41:29.073452+0000","last_unstale":"2023-10-09T05:41:51.799415+0000","last_undegraded":"2023-10-09T05:41:51.799415+0000","last_fullsized":"2023-10-09T05:41:51.799415+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1642,"log_dups_size":0,"ondisk_log_size":1642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:22.328530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107610112,"num_objects":1642,"num_object_clones":0,"num_object_copies":4926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1642,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":188,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.874727+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:41:44.874727+0000","last_peered":"2023-10-09T05:41:44.874727+0000","last_clean":"2023-10-09T05:41:44.874727+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:41:44.874727+0000","last_undegraded":"2023-10-09T05:41:44.874727+0000","last_fullsized":"2023-10-09T05:41:44.874727+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":172,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.864391+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:41:43.864391+0000","last_peered":"2023-10-09T05:41:43.864391+0000","last_clean":"2023-10-09T05:41:43.864391+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:41:43.864391+0000","last_undegraded":"2023-10-09T05:41:43.864391+0000","last_fullsized":"2023-10-09T05:41:43.864391+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"220'1353","reported_seq":1368,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:48.258106+0000","last_change":"2023-10-09T05:41:29.082598+0000","last_active":"2023-10-09T05:41:48.258106+0000","last_peered":"2023-10-09T05:41:48.258106+0000","last_clean":"2023-10-09T05:41:48.258106+0000","last_became_active":"2023-10-09T05:41:29.082285+0000","last_became_peered":"2023-10-09T05:41:29.082285+0000","last_unstale":"2023-10-09T05:41:48.258106+0000","last_undegraded":"2023-10-09T05:41:48.258106+0000","last_fullsized":"2023-10-09T05:41:48.258106+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1353,"log_dups_size":0,"ondisk_log_size":1353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:18.625214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1353,"num_write_kb":86592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":191,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.875259+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:41:44.875259+0000","last_peered":"2023-10-09T05:41:44.875259+0000","last_clean":"2023-10-09T05:41:44.875259+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:41:44.875259+0000","last_undegraded":"2023-10-09T05:41:44.875259+0000","last_fullsized":"2023-10-09T05:41:44.875259+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":186,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858588+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:41:44.858588+0000","last_peered":"2023-10-09T05:41:44.858588+0000","last_clean":"2023-10-09T05:41:44.858588+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:41:44.858588+0000","last_undegraded":"2023-10-09T05:41:44.858588+0000","last_fullsized":"2023-10-09T05:41:44.858588+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"221'1707","reported_seq":1724,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:41:51.830902+0000","last_change":"2023-10-09T05:41:29.078791+0000","last_active":"2023-10-09T05:41:51.830902+0000","last_peered":"2023-10-09T05:41:51.830902+0000","last_clean":"2023-10-09T05:41:51.830902+0000","last_became_active":"2023-10-09T05:41:29.078491+0000","last_became_peered":"2023-10-09T05:41:29.078491+0000","last_unstale":"2023-10-09T05:41:51.830902+0000","last_undegraded":"2023-10-09T05:41:51.830902+0000","last_fullsized":"2023-10-09T05:41:51.830902+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1707,"log_dups_size":0,"ondisk_log_size":1707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:37.817282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111869952,"num_objects":1707,"num_object_clones":0,"num_object_copies":5121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1707,"num_write_kb":109248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":187,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.869780+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:41:44.869780+0000","last_peered":"2023-10-09T05:41:44.869780+0000","last_clean":"2023-10-09T05:41:44.869780+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:41:44.869780+0000","last_undegraded":"2023-10-09T05:41:44.869780+0000","last_fullsized":"2023-10-09T05:41:44.869780+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":201,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858562+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:41:44.858562+0000","last_peered":"2023-10-09T05:41:44.858562+0000","last_clean":"2023-10-09T05:41:44.858562+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:41:44.858562+0000","last_undegraded":"2023-10-09T05:41:44.858562+0000","last_fullsized":"2023-10-09T05:41:44.858562+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"221'1559","reported_seq":1574,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:41:50.717050+0000","last_change":"2023-10-09T05:41:29.078645+0000","last_active":"2023-10-09T05:41:50.717050+0000","last_peered":"2023-10-09T05:41:50.717050+0000","last_clean":"2023-10-09T05:41:50.717050+0000","last_became_active":"2023-10-09T05:41:29.078355+0000","last_became_peered":"2023-10-09T05:41:29.078355+0000","last_unstale":"2023-10-09T05:41:50.717050+0000","last_undegraded":"2023-10-09T05:41:50.717050+0000","last_fullsized":"2023-10-09T05:41:50.717050+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1559,"log_dups_size":0,"ondisk_log_size":1559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102170624,"num_objects":1559,"num_object_clones":0,"num_object_copies":4677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1559,"num_write_kb":99776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":211,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858054+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:41:44.858054+0000","last_peered":"2023-10-09T05:41:44.858054+0000","last_clean":"2023-10-09T05:41:44.858054+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:41:44.858054+0000","last_undegraded":"2023-10-09T05:41:44.858054+0000","last_fullsized":"2023-10-09T05:41:44.858054+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":195,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858793+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:41:44.858793+0000","last_peered":"2023-10-09T05:41:44.858793+0000","last_clean":"2023-10-09T05:41:44.858793+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:41:44.858793+0000","last_undegraded":"2023-10-09T05:41:44.858793+0000","last_fullsized":"2023-10-09T05:41:44.858793+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"221'1521","reported_seq":1536,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:41:50.812800+0000","last_change":"2023-10-09T05:41:29.082194+0000","last_active":"2023-10-09T05:41:50.812800+0000","last_peered":"2023-10-09T05:41:50.812800+0000","last_clean":"2023-10-09T05:41:50.812800+0000","last_became_active":"2023-10-09T05:41:29.081976+0000","last_became_peered":"2023-10-09T05:41:29.081976+0000","last_unstale":"2023-10-09T05:41:50.812800+0000","last_undegraded":"2023-10-09T05:41:50.812800+0000","last_fullsized":"2023-10-09T05:41:50.812800+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1521,"log_dups_size":0,"ondisk_log_size":1521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99680256,"num_objects":1521,"num_object_clones":0,"num_object_copies":4563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1521,"num_write_kb":97344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":218,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.875841+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:41:44.875841+0000","last_peered":"2023-10-09T05:41:44.875841+0000","last_clean":"2023-10-09T05:41:44.875841+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:41:44.875841+0000","last_undegraded":"2023-10-09T05:41:44.875841+0000","last_fullsized":"2023-10-09T05:41:44.875841+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":167,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.960561+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:41:43.960561+0000","last_peered":"2023-10-09T05:41:43.960561+0000","last_clean":"2023-10-09T05:41:43.960561+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:41:43.960561+0000","last_undegraded":"2023-10-09T05:41:43.960561+0000","last_fullsized":"2023-10-09T05:41:43.960561+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"220'1483","reported_seq":1498,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:49.935508+0000","last_change":"2023-10-09T05:41:29.079588+0000","last_active":"2023-10-09T05:41:49.935508+0000","last_peered":"2023-10-09T05:41:49.935508+0000","last_clean":"2023-10-09T05:41:49.935508+0000","last_became_active":"2023-10-09T05:41:29.079288+0000","last_became_peered":"2023-10-09T05:41:29.079288+0000","last_unstale":"2023-10-09T05:41:49.935508+0000","last_undegraded":"2023-10-09T05:41:49.935508+0000","last_fullsized":"2023-10-09T05:41:49.935508+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1483,"log_dups_size":0,"ondisk_log_size":1483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97189888,"num_objects":1483,"num_object_clones":0,"num_object_copies":4449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1483,"num_write_kb":94912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":789,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.854575+0000","last_change":"2023-10-09T05:40:04.806908+0000","last_active":"2023-10-09T05:41:43.854575+0000","last_peered":"2023-10-09T05:41:43.854575+0000","last_clean":"2023-10-09T05:41:43.854575+0000","last_became_active":"2023-10-09T05:40:04.806232+0000","last_became_peered":"2023-10-09T05:40:04.806232+0000","last_unstale":"2023-10-09T05:41:43.854575+0000","last_undegraded":"2023-10-09T05:41:43.854575+0000","last_fullsized":"2023-10-09T05:41:43.854575+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:34.087553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":203,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.874733+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:41:44.874733+0000","last_peered":"2023-10-09T05:41:44.874733+0000","last_clean":"2023-10-09T05:41:44.874733+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:41:44.874733+0000","last_undegraded":"2023-10-09T05:41:44.874733+0000","last_fullsized":"2023-10-09T05:41:44.874733+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":152,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.854117+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:41:43.854117+0000","last_peered":"2023-10-09T05:41:43.854117+0000","last_clean":"2023-10-09T05:41:43.854117+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:41:43.854117+0000","last_undegraded":"2023-10-09T05:41:43.854117+0000","last_fullsized":"2023-10-09T05:41:43.854117+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"221'1579","reported_seq":1594,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:41:50.816538+0000","last_change":"2023-10-09T05:41:29.082691+0000","last_active":"2023-10-09T05:41:50.816538+0000","last_peered":"2023-10-09T05:41:50.816538+0000","last_clean":"2023-10-09T05:41:50.816538+0000","last_became_active":"2023-10-09T05:41:29.082385+0000","last_became_peered":"2023-10-09T05:41:29.082385+0000","last_unstale":"2023-10-09T05:41:50.816538+0000","last_undegraded":"2023-10-09T05:41:50.816538+0000","last_fullsized":"2023-10-09T05:41:50.816538+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1579,"log_dups_size":0,"ondisk_log_size":1579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103481344,"num_objects":1579,"num_object_clones":0,"num_object_copies":4737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1579,"num_write_kb":101056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":657,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.874629+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:41:44.874629+0000","last_peered":"2023-10-09T05:41:44.874629+0000","last_clean":"2023-10-09T05:41:44.874629+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:41:44.874629+0000","last_undegraded":"2023-10-09T05:41:44.874629+0000","last_fullsized":"2023-10-09T05:41:44.874629+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":187,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858311+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:41:44.858311+0000","last_peered":"2023-10-09T05:41:44.858311+0000","last_clean":"2023-10-09T05:41:44.858311+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:41:44.858311+0000","last_undegraded":"2023-10-09T05:41:44.858311+0000","last_fullsized":"2023-10-09T05:41:44.858311+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":200,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.874884+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:41:44.874884+0000","last_peered":"2023-10-09T05:41:44.874884+0000","last_clean":"2023-10-09T05:41:44.874884+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:41:44.874884+0000","last_undegraded":"2023-10-09T05:41:44.874884+0000","last_fullsized":"2023-10-09T05:41:44.874884+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"220'1323","reported_seq":1338,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:48.276685+0000","last_change":"2023-10-09T05:41:29.077953+0000","last_active":"2023-10-09T05:41:48.276685+0000","last_peered":"2023-10-09T05:41:48.276685+0000","last_clean":"2023-10-09T05:41:48.276685+0000","last_became_active":"2023-10-09T05:41:29.077652+0000","last_became_peered":"2023-10-09T05:41:29.077652+0000","last_unstale":"2023-10-09T05:41:48.276685+0000","last_undegraded":"2023-10-09T05:41:48.276685+0000","last_fullsized":"2023-10-09T05:41:48.276685+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1323,"log_dups_size":0,"ondisk_log_size":1323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:53.470052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86704128,"num_objects":1323,"num_object_clones":0,"num_object_copies":3969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1323,"num_write_kb":84672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":536,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.857337+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:41:44.857337+0000","last_peered":"2023-10-09T05:41:44.857337+0000","last_clean":"2023-10-09T05:41:44.857337+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:41:44.857337+0000","last_undegraded":"2023-10-09T05:41:44.857337+0000","last_fullsized":"2023-10-09T05:41:44.857337+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":167,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.855811+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:41:43.855811+0000","last_peered":"2023-10-09T05:41:43.855811+0000","last_clean":"2023-10-09T05:41:43.855811+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:41:43.855811+0000","last_undegraded":"2023-10-09T05:41:43.855811+0000","last_fullsized":"2023-10-09T05:41:43.855811+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":197,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.854578+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:41:43.854578+0000","last_peered":"2023-10-09T05:41:43.854578+0000","last_clean":"2023-10-09T05:41:43.854578+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:41:43.854578+0000","last_undegraded":"2023-10-09T05:41:43.854578+0000","last_fullsized":"2023-10-09T05:41:43.854578+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"220'1363","reported_seq":1378,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:48.281374+0000","last_change":"2023-10-09T05:41:29.082532+0000","last_active":"2023-10-09T05:41:48.281374+0000","last_peered":"2023-10-09T05:41:48.281374+0000","last_clean":"2023-10-09T05:41:48.281374+0000","last_became_active":"2023-10-09T05:41:29.082272+0000","last_became_peered":"2023-10-09T05:41:29.082272+0000","last_unstale":"2023-10-09T05:41:48.281374+0000","last_undegraded":"2023-10-09T05:41:48.281374+0000","last_fullsized":"2023-10-09T05:41:48.281374+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1363,"log_dups_size":0,"ondisk_log_size":1363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:51.574989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89325568,"num_objects":1363,"num_object_clones":0,"num_object_copies":4089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1363,"num_write_kb":87232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":832,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.874536+0000","last_change":"2023-10-09T05:40:04.805258+0000","last_active":"2023-10-09T05:41:44.874536+0000","last_peered":"2023-10-09T05:41:44.874536+0000","last_clean":"2023-10-09T05:41:44.874536+0000","last_became_active":"2023-10-09T05:40:04.804776+0000","last_became_peered":"2023-10-09T05:40:04.804776+0000","last_unstale":"2023-10-09T05:41:44.874536+0000","last_undegraded":"2023-10-09T05:41:44.874536+0000","last_fullsized":"2023-10-09T05:41:44.874536+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":368,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.864246+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:41:43.864246+0000","last_peered":"2023-10-09T05:41:43.864246+0000","last_clean":"2023-10-09T05:41:43.864246+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:41:43.864246+0000","last_undegraded":"2023-10-09T05:41:43.864246+0000","last_fullsized":"2023-10-09T05:41:43.864246+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":184,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.876771+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:41:44.876771+0000","last_peered":"2023-10-09T05:41:44.876771+0000","last_clean":"2023-10-09T05:41:44.876771+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:41:44.876771+0000","last_undegraded":"2023-10-09T05:41:44.876771+0000","last_fullsized":"2023-10-09T05:41:44.876771+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":193,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.870010+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:41:44.870010+0000","last_peered":"2023-10-09T05:41:44.870010+0000","last_clean":"2023-10-09T05:41:44.870010+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:41:44.870010+0000","last_undegraded":"2023-10-09T05:41:44.870010+0000","last_fullsized":"2023-10-09T05:41:44.870010+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"221'1747","reported_seq":1764,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:41:51.817608+0000","last_change":"2023-10-09T05:41:29.076367+0000","last_active":"2023-10-09T05:41:51.817608+0000","last_peered":"2023-10-09T05:41:51.817608+0000","last_clean":"2023-10-09T05:41:51.817608+0000","last_became_active":"2023-10-09T05:41:29.076035+0000","last_became_peered":"2023-10-09T05:41:29.076035+0000","last_unstale":"2023-10-09T05:41:51.817608+0000","last_undegraded":"2023-10-09T05:41:51.817608+0000","last_fullsized":"2023-10-09T05:41:51.817608+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1747,"log_dups_size":0,"ondisk_log_size":1747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:51.713835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114491392,"num_objects":1747,"num_object_clones":0,"num_object_copies":5241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1747,"num_write_kb":111808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":777,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.855956+0000","last_change":"2023-10-09T05:40:05.930112+0000","last_active":"2023-10-09T05:41:43.855956+0000","last_peered":"2023-10-09T05:41:43.855956+0000","last_clean":"2023-10-09T05:41:43.855956+0000","last_became_active":"2023-10-09T05:40:05.929574+0000","last_became_peered":"2023-10-09T05:40:05.929574+0000","last_unstale":"2023-10-09T05:41:43.855956+0000","last_undegraded":"2023-10-09T05:41:43.855956+0000","last_fullsized":"2023-10-09T05:41:43.855956+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:06.982326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":172,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.864424+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:41:43.864424+0000","last_peered":"2023-10-09T05:41:43.864424+0000","last_clean":"2023-10-09T05:41:43.864424+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:41:43.864424+0000","last_undegraded":"2023-10-09T05:41:43.864424+0000","last_fullsized":"2023-10-09T05:41:43.864424+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":187,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.864280+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:41:43.864280+0000","last_peered":"2023-10-09T05:41:43.864280+0000","last_clean":"2023-10-09T05:41:43.864280+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:41:43.864280+0000","last_undegraded":"2023-10-09T05:41:43.864280+0000","last_fullsized":"2023-10-09T05:41:43.864280+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"220'1476","reported_seq":1492,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:49.943921+0000","last_change":"2023-10-09T05:41:29.073701+0000","last_active":"2023-10-09T05:41:49.943921+0000","last_peered":"2023-10-09T05:41:49.943921+0000","last_clean":"2023-10-09T05:41:49.943921+0000","last_became_active":"2023-10-09T05:41:29.073372+0000","last_became_peered":"2023-10-09T05:41:29.073372+0000","last_unstale":"2023-10-09T05:41:49.943921+0000","last_undegraded":"2023-10-09T05:41:49.943921+0000","last_fullsized":"2023-10-09T05:41:49.943921+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1476,"log_dups_size":0,"ondisk_log_size":1476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96731136,"num_objects":1476,"num_object_clones":0,"num_object_copies":4428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1476,"num_write_kb":94464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":409,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.862645+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:41:44.862645+0000","last_peered":"2023-10-09T05:41:44.862645+0000","last_clean":"2023-10-09T05:41:44.862645+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:41:44.862645+0000","last_undegraded":"2023-10-09T05:41:44.862645+0000","last_fullsized":"2023-10-09T05:41:44.862645+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":167,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.854890+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:41:43.854890+0000","last_peered":"2023-10-09T05:41:43.854890+0000","last_clean":"2023-10-09T05:41:43.854890+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:41:43.854890+0000","last_undegraded":"2023-10-09T05:41:43.854890+0000","last_fullsized":"2023-10-09T05:41:43.854890+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":181,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.864665+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:41:43.864665+0000","last_peered":"2023-10-09T05:41:43.864665+0000","last_clean":"2023-10-09T05:41:43.864665+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:41:43.864665+0000","last_undegraded":"2023-10-09T05:41:43.864665+0000","last_fullsized":"2023-10-09T05:41:43.864665+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"221'1622","reported_seq":1638,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:41:51.393048+0000","last_change":"2023-10-09T05:41:29.080413+0000","last_active":"2023-10-09T05:41:51.393048+0000","last_peered":"2023-10-09T05:41:51.393048+0000","last_clean":"2023-10-09T05:41:51.393048+0000","last_became_active":"2023-10-09T05:41:29.080184+0000","last_became_peered":"2023-10-09T05:41:29.080184+0000","last_unstale":"2023-10-09T05:41:51.393048+0000","last_undegraded":"2023-10-09T05:41:51.393048+0000","last_fullsized":"2023-10-09T05:41:51.393048+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1622,"log_dups_size":0,"ondisk_log_size":1622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:47.929696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106299392,"num_objects":1622,"num_object_clones":0,"num_object_copies":4866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1622,"num_write_kb":103808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":219,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.875520+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:41:44.875520+0000","last_peered":"2023-10-09T05:41:44.875520+0000","last_clean":"2023-10-09T05:41:44.875520+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:41:44.875520+0000","last_undegraded":"2023-10-09T05:41:44.875520+0000","last_fullsized":"2023-10-09T05:41:44.875520+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":608,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.854090+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:41:43.854090+0000","last_peered":"2023-10-09T05:41:43.854090+0000","last_clean":"2023-10-09T05:41:43.854090+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:41:43.854090+0000","last_undegraded":"2023-10-09T05:41:43.854090+0000","last_fullsized":"2023-10-09T05:41:43.854090+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":213,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.876002+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:41:44.876002+0000","last_peered":"2023-10-09T05:41:44.876002+0000","last_clean":"2023-10-09T05:41:44.876002+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:41:44.876002+0000","last_undegraded":"2023-10-09T05:41:44.876002+0000","last_fullsized":"2023-10-09T05:41:44.876002+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"221'1700","reported_seq":1716,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:41:51.402332+0000","last_change":"2023-10-09T05:41:29.081230+0000","last_active":"2023-10-09T05:41:51.402332+0000","last_peered":"2023-10-09T05:41:51.402332+0000","last_clean":"2023-10-09T05:41:51.402332+0000","last_became_active":"2023-10-09T05:41:29.080911+0000","last_became_peered":"2023-10-09T05:41:29.080911+0000","last_unstale":"2023-10-09T05:41:51.402332+0000","last_undegraded":"2023-10-09T05:41:51.402332+0000","last_fullsized":"2023-10-09T05:41:51.402332+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":1700,"log_dups_size":0,"ondisk_log_size":1700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:22.580967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111411200,"num_objects":1700,"num_object_clones":0,"num_object_copies":5100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1700,"num_write_kb":108800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":721,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.871417+0000","last_change":"2023-10-09T05:41:21.917602+0000","last_active":"2023-10-09T05:41:44.871417+0000","last_peered":"2023-10-09T05:41:44.871417+0000","last_clean":"2023-10-09T05:41:44.871417+0000","last_became_active":"2023-10-09T05:41:21.917116+0000","last_became_peered":"2023-10-09T05:41:21.917116+0000","last_unstale":"2023-10-09T05:41:44.871417+0000","last_undegraded":"2023-10-09T05:41:44.871417+0000","last_fullsized":"2023-10-09T05:41:44.871417+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":175,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.864390+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:41:43.864390+0000","last_peered":"2023-10-09T05:41:43.864390+0000","last_clean":"2023-10-09T05:41:43.864390+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:41:43.864390+0000","last_undegraded":"2023-10-09T05:41:43.864390+0000","last_fullsized":"2023-10-09T05:41:43.864390+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858095+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:41:44.858095+0000","last_peered":"2023-10-09T05:41:44.858095+0000","last_clean":"2023-10-09T05:41:44.858095+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:41:44.858095+0000","last_undegraded":"2023-10-09T05:41:44.858095+0000","last_fullsized":"2023-10-09T05:41:44.858095+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":183,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.864579+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:41:43.864579+0000","last_peered":"2023-10-09T05:41:43.864579+0000","last_clean":"2023-10-09T05:41:43.864579+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:41:43.864579+0000","last_undegraded":"2023-10-09T05:41:43.864579+0000","last_fullsized":"2023-10-09T05:41:43.864579+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":222,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.888145+0000","last_change":"2023-10-09T05:41:44.887519+0000","last_active":"2023-10-09T05:41:44.888145+0000","last_peered":"2023-10-09T05:41:44.888145+0000","last_clean":"2023-10-09T05:41:44.888145+0000","last_became_active":"2023-10-09T05:41:44.887047+0000","last_became_peered":"2023-10-09T05:41:44.887047+0000","last_unstale":"2023-10-09T05:41:44.888145+0000","last_undegraded":"2023-10-09T05:41:44.888145+0000","last_fullsized":"2023-10-09T05:41:44.888145+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:49.475913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":190,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.874566+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:41:44.874566+0000","last_peered":"2023-10-09T05:41:44.874566+0000","last_clean":"2023-10-09T05:41:44.874566+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:41:44.874566+0000","last_undegraded":"2023-10-09T05:41:44.874566+0000","last_fullsized":"2023-10-09T05:41:44.874566+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":201,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858354+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:41:44.858354+0000","last_peered":"2023-10-09T05:41:44.858354+0000","last_clean":"2023-10-09T05:41:44.858354+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:41:44.858354+0000","last_undegraded":"2023-10-09T05:41:44.858354+0000","last_fullsized":"2023-10-09T05:41:44.858354+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":175,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.874955+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:41:44.874955+0000","last_peered":"2023-10-09T05:41:44.874955+0000","last_clean":"2023-10-09T05:41:44.874955+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:41:44.874955+0000","last_undegraded":"2023-10-09T05:41:44.874955+0000","last_fullsized":"2023-10-09T05:41:44.874955+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":201,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858031+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:41:44.858031+0000","last_peered":"2023-10-09T05:41:44.858031+0000","last_clean":"2023-10-09T05:41:44.858031+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:41:44.858031+0000","last_undegraded":"2023-10-09T05:41:44.858031+0000","last_fullsized":"2023-10-09T05:41:44.858031+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":200,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.875602+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:41:44.875602+0000","last_peered":"2023-10-09T05:41:44.875602+0000","last_clean":"2023-10-09T05:41:44.875602+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:41:44.875602+0000","last_undegraded":"2023-10-09T05:41:44.875602+0000","last_fullsized":"2023-10-09T05:41:44.875602+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":212,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.875112+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:41:44.875112+0000","last_peered":"2023-10-09T05:41:44.875112+0000","last_clean":"2023-10-09T05:41:44.875112+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:41:44.875112+0000","last_undegraded":"2023-10-09T05:41:44.875112+0000","last_fullsized":"2023-10-09T05:41:44.875112+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":175,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T05:41:43.869194+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:41:43.869194+0000","last_peered":"2023-10-09T05:41:43.869194+0000","last_clean":"2023-10-09T05:41:43.869194+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:41:43.869194+0000","last_undegraded":"2023-10-09T05:41:43.869194+0000","last_fullsized":"2023-10-09T05:41:43.869194+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":186,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T05:41:44.858014+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:41:44.858014+0000","last_peered":"2023-10-09T05:41:44.858014+0000","last_clean":"2023-10-09T05:41:44.858014+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:41:44.858014+0000","last_undegraded":"2023-10-09T05:41:44.858014+0000","last_fullsized":"2023-10-09T05:41:44.858014+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1596325888,"num_objects":24358,"num_object_clones":0,"num_object_copies":73074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":24358,"num_write_kb":1558912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4777377792,"data_stored":4777377792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":24358,"ondisk_log_size":24358,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846679,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626705,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":778136,"kb_used_data":480912,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92967016,"statfs":{"total":95995035648,"available":95198224384,"internally_reserved":0,"allocated":492453888,"data_stored":491920712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:28 2023","interfaces":[{"interface":"back","average":{"1min":2.032,"5min":5.6829999999999998,"15min":3.2210000000000001},"min":{"1min":0.23100000000000001,"5min":0.214,"15min":0.214},"max":{"1min":14.23,"5min":115.377,"15min":115.377},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.9350000000000001,"5min":5.3710000000000004,"15min":3.0800000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23999999999999999},"max":{"1min":11.727,"5min":116.428,"15min":116.428},"last":1.4410000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":7.274,"5min":6.056,"15min":2.7170000000000001},"min":{"1min":0.315,"5min":0.25,"15min":0.25},"max":{"1min":93.489000000000004,"5min":137.02199999999999,"15min":137.02199999999999},"last":1.984},{"interface":"front","average":{"1min":6.6299999999999999,"5min":5.9379999999999997,"15min":2.758},"min":{"1min":0.318,"5min":0.252,"15min":0.252},"max":{"1min":93.471999999999994,"5min":137.06700000000001,"15min":137.06700000000001},"last":2.0710000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:41:28 2023","interfaces":[{"interface":"back","average":{"1min":7.8099999999999996,"5min":6.226,"15min":3.456},"min":{"1min":0.32200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":87.703000000000003,"5min":115.40600000000001,"15min":115.40600000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":8.4559999999999995,"5min":6.5220000000000002,"15min":3.4980000000000002},"min":{"1min":0.35499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":87.689999999999998,"5min":115.297,"15min":115.297},"last":0.42899999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":11.682,"5min":6.7359999999999998,"15min":3.1219999999999999},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":99.543999999999997,"5min":115.244,"15min":115.244},"last":0.46899999999999997},{"interface":"front","average":{"1min":10.734,"5min":6.556,"15min":3.1499999999999999},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":98.677999999999997,"5min":115.34999999999999,"15min":115.34999999999999},"last":0.45200000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.8200000000000001,"15min":1.7110000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":0.72399999999999998,"5min":115.214,"15min":125.64},"last":0.376},{"interface":"front","average":{"1min":0.70399999999999996,"5min":1.5820000000000001,"15min":1.6120000000000001},"min":{"1min":0.27400000000000002,"5min":0.254,"15min":0.254},"max":{"1min":2.7189999999999999,"5min":115.22799999999999,"15min":125.625},"last":0.317}]},{"osd":5,"last update":"Mon Oct 9 05:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":2.5600000000000001,"15min":1.855},"min":{"1min":0.33500000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4690000000000003,"5min":115.316,"15min":127.35899999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.0720000000000001,"5min":2.1549999999999998,"15min":1.7370000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.6079999999999997,"5min":115.30500000000001,"15min":128.93799999999999},"last":1.228}]},{"osd":7,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":2.077,"15min":2.7909999999999999},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":5.0110000000000001,"5min":115.514,"15min":115.514},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.8240000000000001,"15min":2.0019999999999998},"min":{"1min":0.34899999999999998,"5min":0.317,"15min":0.317},"max":{"1min":3.6600000000000001,"5min":116.53100000000001,"15min":116.53100000000001},"last":0.41199999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958666,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":887276,"kb_used_data":625380,"kb_used_omap":7,"kb_used_meta":261880,"kb_avail":92857876,"statfs":{"total":95995035648,"available":95086465024,"internally_reserved":0,"allocated":640389120,"data_stored":639598366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":268165646},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":4.694,"5min":5.1500000000000004,"15min":2.6080000000000001},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":39.189,"5min":207.858,"15min":207.858},"last":2.4689999999999999},{"interface":"front","average":{"1min":6.1989999999999998,"5min":6.4420000000000002,"15min":3.0720000000000001},"min":{"1min":0.23000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":42.012999999999998,"5min":206.97200000000001,"15min":206.97200000000001},"last":1.276}]},{"osd":2,"last update":"Mon Oct 9 05:41:11 2023","interfaces":[{"interface":"back","average":{"1min":6.2229999999999999,"5min":7.3860000000000001,"15min":3.5019999999999998},"min":{"1min":0.26000000000000001,"5min":0.22,"15min":0.184},"max":{"1min":42.469999999999999,"5min":101.367,"15min":101.367},"last":3.206},{"interface":"front","average":{"1min":5.3369999999999997,"5min":6.9020000000000001,"15min":3.294},"min":{"1min":0.23300000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":42.151000000000003,"5min":97.286000000000001,"15min":98.409999999999997},"last":6.3079999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":18.629999999999999,"5min":7.2770000000000001,"15min":2.9540000000000002},"min":{"1min":0.28899999999999998,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":162.33699999999999,"5min":162.33699999999999,"15min":162.33699999999999},"last":3.234},{"interface":"front","average":{"1min":19.029,"5min":7.9980000000000002,"15min":3.1960000000000002},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":162.376,"5min":162.376,"15min":162.376},"last":2.504}]},{"osd":4,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.1749999999999998,"5min":4.7249999999999996,"15min":2.2690000000000001},"min":{"1min":0.39700000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":42.106999999999999,"5min":141.32900000000001,"15min":141.32900000000001},"last":1.244},{"interface":"front","average":{"1min":5.2800000000000002,"5min":3.6819999999999999,"15min":1.8400000000000001},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":40.475999999999999,"5min":141.15899999999999,"15min":141.15899999999999},"last":2.351}]},{"osd":5,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.6420000000000003,"5min":4.7009999999999996,"15min":2.2349999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":40.633000000000003,"5min":235.18700000000001,"15min":235.18700000000001},"last":3.1749999999999998},{"interface":"front","average":{"1min":5.258,"5min":5.5570000000000004,"15min":2.6240000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":42.078000000000003,"5min":235.79300000000001,"15min":235.79300000000001},"last":1.216}]},{"osd":6,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.8860000000000001,"5min":3.8109999999999999,"15min":1.8779999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":40.558999999999997,"5min":103.60899999999999,"15min":103.60899999999999},"last":3.1230000000000002},{"interface":"front","average":{"1min":5.234,"5min":4.4950000000000001,"15min":2.141},"min":{"1min":0.41299999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":42.134999999999998,"5min":103.679,"15min":103.679},"last":1.298}]},{"osd":7,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":5.4950000000000001,"5min":3.649,"15min":2.5489999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":42.204000000000001,"5min":126.68300000000001,"15min":126.68300000000001},"last":2.4329999999999998},{"interface":"front","average":{"1min":5.4050000000000002,"5min":3.7450000000000001,"15min":2.6339999999999999},"min":{"1min":0.46999999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":42.228999999999999,"5min":126.738,"15min":126.738},"last":2.399}]}]},{"osd":0,"up_from":88,"seq":377957122192,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1016328,"kb_used_data":664384,"kb_used_omap":8,"kb_used_meta":351927,"kb_avail":92728824,"statfs":{"total":95995035648,"available":94954315776,"internally_reserved":0,"allocated":680329216,"data_stored":679511365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":360374024},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4910000000000001,"5min":2.3340000000000001,"15min":1.282},"min":{"1min":0.20699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":17.760000000000002,"5min":17.760000000000002,"15min":17.760000000000002},"last":0.20200000000000001},{"interface":"front","average":{"1min":6.0330000000000004,"5min":3.3679999999999999,"15min":1.6539999999999999},"min":{"1min":0.25,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":26.163,"5min":26.163,"15min":26.163},"last":0.25900000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":7.7830000000000004,"5min":4.2309999999999999,"15min":2.6949999999999998},"min":{"1min":0.23999999999999999,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":92.230000000000004,"5min":92.230000000000004,"15min":92.230000000000004},"last":15.894},{"interface":"front","average":{"1min":8.4440000000000008,"5min":4.3079999999999998,"15min":2.6419999999999999},"min":{"1min":0.253,"5min":0.221,"15min":0.218},"max":{"1min":92.247,"5min":92.247,"15min":92.247},"last":17.824000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":18.498999999999999,"5min":6.4859999999999998,"15min":2.7669999999999999},"min":{"1min":0.27500000000000002,"5min":0.26000000000000001,"15min":0.22800000000000001},"max":{"1min":123.024,"5min":123.024,"15min":123.024},"last":0.29699999999999999},{"interface":"front","average":{"1min":12.829000000000001,"5min":5.4359999999999999,"15min":2.48},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":99.162000000000006,"5min":99.162000000000006,"15min":99.162000000000006},"last":0.28499999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.9970000000000001,"15min":1.288},"min":{"1min":0.32200000000000001,"5min":0.27500000000000002,"15min":0.27200000000000002},"max":{"1min":8.6470000000000002,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":3.206,"5min":2.778,"15min":1.6799999999999999},"min":{"1min":0.41399999999999998,"5min":0.33000000000000002,"15min":0.30399999999999999},"max":{"1min":7.9240000000000004,"5min":69.102000000000004,"15min":69.102000000000004},"last":0.59599999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":3.2450000000000001,"5min":6.4889999999999999,"15min":2.907},"min":{"1min":0.36299999999999999,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":8.0289999999999999,"5min":281.733,"15min":281.733},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.6080000000000001,"5min":6.3609999999999998,"15min":2.8650000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.4269999999999996,"5min":282.95400000000001,"15min":282.95400000000001},"last":0.55100000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":3.266,"5min":2.2610000000000001,"15min":1.4990000000000001},"min":{"1min":0.39800000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9969999999999999,"5min":11.664,"15min":11.664},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.355,"5min":2.2200000000000002,"15min":1.4990000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.8490000000000002,"5min":10.83,"15min":10.83},"last":0.44900000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.679,"15min":1.4450000000000001},"min":{"1min":0.41899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.5090000000000003,"5min":9.2400000000000002,"15min":9.2400000000000002},"last":0.624},{"interface":"front","average":{"1min":3.1859999999999999,"5min":2.548,"15min":2.4420000000000002},"min":{"1min":0.46800000000000003,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.6329999999999991,"5min":9.7159999999999993,"15min":11.068},"last":0.46800000000000003}]}]},{"osd":2,"up_from":98,"seq":420906795140,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1166108,"kb_used_data":748564,"kb_used_omap":11,"kb_used_meta":417524,"kb_avail":92579044,"statfs":{"total":95995035648,"available":94800941056,"internally_reserved":0,"allocated":766529536,"data_stored":765644989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":427545173},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":4.3230000000000004,"5min":3.4670000000000001,"15min":1.7949999999999999},"min":{"1min":0.215,"5min":0.17799999999999999,"15min":0.17199999999999999},"max":{"1min":21.587,"5min":21.587,"15min":32.122},"last":1.1419999999999999},{"interface":"front","average":{"1min":3.702,"5min":3.4889999999999999,"15min":1.7430000000000001},"min":{"1min":0.214,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":15.468,"5min":21.997,"15min":34.066000000000003},"last":0.42899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":5.5529999999999999,"15min":2.6539999999999999},"min":{"1min":0.29199999999999998,"5min":0.23400000000000001,"15min":0.20699999999999999},"max":{"1min":24.265000000000001,"5min":98.575000000000003,"15min":98.575000000000003},"last":1.3100000000000001},{"interface":"front","average":{"1min":4.5599999999999996,"5min":4.5069999999999997,"15min":2.282},"min":{"1min":0.23100000000000001,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":18.882999999999999,"5min":90.135000000000005,"15min":98.397000000000006},"last":1.296}]},{"osd":3,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":5.7240000000000002,"5min":6.1200000000000001,"15min":2.6499999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.24199999999999999},"max":{"1min":24.471,"5min":150.65899999999999,"15min":150.65899999999999},"last":1.327},{"interface":"front","average":{"1min":6.6200000000000001,"5min":6.9770000000000003,"15min":3.016},"min":{"1min":0.46000000000000002,"5min":0.29899999999999999,"15min":0.26500000000000001},"max":{"1min":22.631,"5min":150.851,"15min":150.851},"last":1.0820000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6629999999999998,"5min":3.2719999999999998,"15min":1.7210000000000001},"min":{"1min":0.32200000000000001,"5min":0.29099999999999998,"15min":0.27700000000000002},"max":{"1min":14.382,"5min":115.84999999999999,"15min":115.84999999999999},"last":1.355},{"interface":"front","average":{"1min":2.5449999999999999,"5min":3.7109999999999999,"15min":1.9770000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.29799999999999999},"max":{"1min":13.815,"5min":116.149,"15min":116.149},"last":0.60999999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":2.8959999999999999,"15min":1.6870000000000001},"min":{"1min":0.36699999999999999,"5min":0.312,"15min":0.312},"max":{"1min":13.712999999999999,"5min":24.597999999999999,"15min":24.597999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":3.222,"5min":2.302,"15min":1.466},"min":{"1min":0.35499999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":14.426,"5min":23.919,"15min":23.919},"last":1.4399999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:41:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":2.734,"15min":1.637},"min":{"1min":0.36499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":13.839,"5min":20.503,"15min":20.503},"last":0.58199999999999996},{"interface":"front","average":{"1min":2.71,"5min":2.5619999999999998,"15min":1.536},"min":{"1min":0.40600000000000003,"5min":0.39600000000000002,"15min":0.34499999999999997},"max":{"1min":13.757,"5min":13.757,"15min":13.757},"last":0.72199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:40:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":3.2509999999999999,"15min":2.766},"min":{"1min":0.47199999999999998,"5min":0.40200000000000002,"15min":0.374},"max":{"1min":13.865,"5min":24.030999999999999,"15min":24.030999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":3.5070000000000001,"5min":3.1110000000000002,"15min":2.1890000000000001},"min":{"1min":0.52700000000000002,"5min":0.41899999999999998,"15min":0.41599999999999998},"max":{"1min":13.781000000000001,"5min":24.178000000000001,"15min":24.178000000000001},"last":0.751}]}]},{"osd":3,"up_from":103,"seq":442381631614,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1378152,"kb_used_data":1013600,"kb_used_omap":4,"kb_used_meta":364539,"kb_avail":92367000,"statfs":{"total":95995035648,"available":94583808000,"internally_reserved":0,"allocated":1037926400,"data_stored":1037145008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":373288542},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":4.7149999999999999,"15min":2.3900000000000001},"min":{"1min":0.25800000000000001,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":4.234,"5min":167.82400000000001,"15min":167.82400000000001},"last":4.0839999999999996},{"interface":"front","average":{"1min":1.323,"5min":5.5289999999999999,"15min":2.746},"min":{"1min":0.24099999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.1959999999999997,"5min":167.89599999999999,"15min":167.89599999999999},"last":1.1080000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":4.0620000000000003,"15min":2.109},"min":{"1min":0.26000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.09,"5min":83.921999999999997,"15min":83.921999999999997},"last":3.407},{"interface":"front","average":{"1min":1.3480000000000001,"5min":3.8279999999999998,"15min":2.0659999999999998},"min":{"1min":0.27400000000000002,"5min":0.25,"15min":0.247},"max":{"1min":9.4420000000000002,"5min":83.707999999999998,"15min":83.707999999999998},"last":1.413}]},{"osd":2,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":6.2569999999999997,"15min":3.7050000000000001},"min":{"1min":0.28100000000000003,"5min":0.25,"15min":0.215},"max":{"1min":11.619,"5min":190.43299999999999,"15min":190.43299999999999},"last":3.4289999999999998},{"interface":"front","average":{"1min":1.9970000000000001,"5min":7.3390000000000004,"15min":4.1289999999999996},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.26000000000000001},"max":{"1min":11.738,"5min":190.267,"15min":190.267},"last":0.96399999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":3.1280000000000001,"15min":2.012},"min":{"1min":0.35299999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.1859999999999999,"5min":51.591999999999999,"15min":51.591999999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":2.1059999999999999,"15min":1.5489999999999999},"min":{"1min":0.40699999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.2089999999999996,"5min":26.774999999999999,"15min":45.735999999999997},"last":3.355}]},{"osd":5,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":2.331,"15min":1.623},"min":{"1min":0.374,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":9.5180000000000007,"5min":13.114000000000001,"15min":47.948999999999998},"last":1.446},{"interface":"front","average":{"1min":1.5149999999999999,"5min":2.2200000000000002,"15min":1.6519999999999999},"min":{"1min":0.439,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.125,"5min":26.728000000000002,"15min":47.968000000000004},"last":3.3180000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":2.1179999999999999,"15min":1.524},"min":{"1min":0.40500000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.1699999999999999,"5min":26.847999999999999,"15min":26.847999999999999},"last":3.383},{"interface":"front","average":{"1min":1.3200000000000001,"5min":2.415,"15min":1.46},"min":{"1min":0.39600000000000002,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":9.4809999999999999,"5min":13.058,"15min":13.058},"last":1.472}]},{"osd":7,"last update":"Mon Oct 9 05:40:56 2023","interfaces":[{"interface":"back","average":{"1min":4.032,"5min":3.0249999999999999,"15min":2.5150000000000001},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":26.024999999999999,"5min":26.024999999999999,"15min":26.024999999999999},"last":1.016},{"interface":"front","average":{"1min":4.2720000000000002,"5min":3.125,"15min":2.3500000000000001},"min":{"1min":0.42499999999999999,"5min":0.40899999999999997,"15min":0.35699999999999998},"max":{"1min":25.995999999999999,"5min":25.995999999999999,"15min":25.995999999999999},"last":1.0700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019161,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":856976,"kb_used_data":551560,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92888176,"statfs":{"total":95995035648,"available":95117492224,"internally_reserved":0,"allocated":564797440,"data_stored":564264306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:24 2023","interfaces":[{"interface":"back","average":{"1min":6.5049999999999999,"5min":2.403,"15min":1.7170000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.214},"max":{"1min":100.504,"5min":100.504,"15min":100.504},"last":0.46000000000000002},{"interface":"front","average":{"1min":6.827,"5min":2.9089999999999998,"15min":1.998},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":100.717,"5min":100.717,"15min":100.717},"last":0.45300000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":1.6699999999999999,"15min":1.21},"min":{"1min":0.29999999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8140000000000001,"5min":7.9550000000000001,"15min":31.073},"last":187.369},{"interface":"front","average":{"1min":2.492,"5min":1.8520000000000001,"15min":1.3300000000000001},"min":{"1min":0.29299999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.349,"5min":12.349,"15min":31.100000000000001},"last":187.35300000000001}]},{"osd":2,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":8.7859999999999996,"5min":5.681,"15min":4.0270000000000001},"min":{"1min":0.315,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":100.148,"5min":196.93700000000001,"15min":196.93700000000001},"last":0.5},{"interface":"front","average":{"1min":9.0389999999999997,"5min":5.492,"15min":3.931},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":100.086,"5min":197.08099999999999,"15min":197.08099999999999},"last":0.49299999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:41:43 2023","interfaces":[{"interface":"back","average":{"1min":5.4669999999999996,"5min":8.048,"15min":3.4239999999999999},"min":{"1min":0.35899999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":70.962999999999994,"5min":211.48699999999999,"15min":211.48699999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":5.883,"5min":7.9420000000000002,"15min":3.3250000000000002},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.309},"max":{"1min":70.838999999999999,"5min":211.31100000000001,"15min":211.31100000000001},"last":0.51100000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":1.758,"15min":1.5629999999999999},"min":{"1min":0.255,"5min":0.25,"15min":0.249},"max":{"1min":0.67300000000000004,"5min":94.646000000000001,"15min":113.35899999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.46100000000000002,"5min":1.6910000000000001,"15min":1.516},"min":{"1min":0.32100000000000001,"5min":0.317,"15min":0.30599999999999999},"max":{"1min":0.73999999999999999,"5min":94.692999999999998,"15min":113.32299999999999},"last":0.434}]},{"osd":6,"last update":"Mon Oct 9 05:40:57 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.72899999999999998,"15min":0.70499999999999996},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":1.8740000000000001,"5min":6.1779999999999999,"15min":11.944000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.93500000000000005,"15min":0.86399999999999999},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.4810000000000001,"5min":7.6500000000000004,"15min":10.356},"last":0.34300000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.94799999999999995,"15min":2.5619999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.312},"max":{"1min":3.415,"5min":11.845000000000001,"15min":30.393999999999998},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.97599999999999998,"15min":2.8780000000000001},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.5089999999999999,"5min":10.837,"15min":30.334},"last":0.39200000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822933,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":776256,"kb_used_data":466744,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92968896,"statfs":{"total":95995035648,"available":95200149504,"internally_reserved":0,"allocated":477945856,"data_stored":477424554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:18 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.879,"15min":3.008},"min":{"1min":0.217,"5min":0.217,"15min":0.19400000000000001},"max":{"1min":9.032,"5min":10.712999999999999,"15min":165.90199999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.849,"5min":1.8300000000000001,"15min":3.077},"min":{"1min":0.30399999999999999,"5min":0.28399999999999997,"15min":0.215},"max":{"1min":7.4050000000000002,"5min":12.263999999999999,"15min":165.96700000000001},"last":0.34300000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:41:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":4.3390000000000004,"15min":2.214},"min":{"1min":0.26700000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.0229999999999997,"5min":87.769000000000005,"15min":87.769000000000005},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.4239999999999999,"5min":4.891,"15min":2.339},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.24399999999999999},"max":{"1min":12.228,"5min":87.864000000000004,"15min":87.864000000000004},"last":1.9810000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":3.5299999999999998,"15min":4.0659999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.1910000000000007,"5min":73.311999999999998,"15min":290.697},"last":0.64100000000000001},{"interface":"front","average":{"1min":3.1920000000000002,"5min":3.3929999999999998,"15min":3.956},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":9.2140000000000004,"5min":73.280000000000001,"15min":290.786},"last":0.40799999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:41:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0489999999999999,"5min":6.7690000000000001,"15min":3.0550000000000002},"min":{"1min":0.40799999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":36.395000000000003,"5min":95.236999999999995,"15min":95.236999999999995},"last":2.266},{"interface":"front","average":{"1min":2.758,"5min":6.3600000000000003,"15min":2.8639999999999999},"min":{"1min":0.41199999999999998,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":36.406999999999996,"5min":93.471000000000004,"15min":93.471000000000004},"last":2.4209999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:41:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.60999999999999999,"15min":0.72099999999999997},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":0.84999999999999998,"5min":5.6459999999999999,"15min":9.3510000000000009},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.58999999999999997,"15min":0.94499999999999995},"min":{"1min":0.254,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":0.78100000000000003,"5min":4.8209999999999997,"15min":14.236000000000001},"last":0.50700000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:41:04 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.2370000000000001,"15min":1.3600000000000001},"min":{"1min":0.35699999999999998,"5min":0.27600000000000002,"15min":0.26600000000000001},"max":{"1min":4.79,"5min":27.146000000000001,"15min":95.010999999999996},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.1359999999999999,"15min":1.2609999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27700000000000002},"max":{"1min":4.6600000000000001,"5min":27.215,"15min":94.194000000000003},"last":0.65600000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:41:00 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":1.228,"15min":1.5589999999999999},"min":{"1min":0.39900000000000002,"5min":0.34399999999999997,"15min":0.28999999999999998},"max":{"1min":5.5570000000000004,"5min":9.9849999999999994,"15min":9.9849999999999994},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.212,"5min":1.157,"15min":1.8009999999999999},"min":{"1min":0.373,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.0549999999999997,"5min":10.779,"15min":14.432},"last":0.47199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":706609152,"data_stored":706609152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":639893504,"data_stored":639893504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":769851392,"data_stored":769851392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1074462720,"data_stored":1074462720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":585236480,"data_stored":585236480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":492044288,"data_stored":492044288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":509280256,"data_stored":509280256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:41:54.458 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_2 pgp_num 16 2023-10-09T05:41:54.509 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:54.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:54.620 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:41:54.656 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:54.759 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:41:54.761 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:54.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:54.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:41:54.865 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:41:54.947 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:55.117 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:41:55.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:55.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:41:55.196 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:55.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:55.256 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:55.312 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:55.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:55.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:55.339 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 18 16 30824 30808 106.844 75.125 0.0115772 0.00935194 2023-10-09T05:41:55.371 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:41:55.416 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:55.421 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:41:55.449 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:55.460 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:41:55.503 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:41:55.504 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:55.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:55.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:55.542 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:41:55.580 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:41:55.613 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:55.630 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:41:55.647 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:55.675 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:41:55.708 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:41:55.708 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:41:55.708 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:41:55.708 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:41:55.708 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:41:55.709 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:41:55.709 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:41:55.709 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:41:55.710 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:55.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:55.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:55.814 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:55.850 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:55.854 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:55 smithi008 ceph-mon[184418]: pgmap v625: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-10-09T05:41:55.854 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:55 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1033685594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:55.855 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:55 smithi008 ceph-mon[191227]: pgmap v625: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-10-09T05:41:55.855 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:55 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1033685594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:56.077 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:56.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:56.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:56.183 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:56.213 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:55 smithi191 ceph-mon[161669]: pgmap v625: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-10-09T05:41:56.214 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:55 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1033685594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:41:56.216 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:56.273 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:56.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:56.339 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 19 16 32342 32326 106.214 94.875 0.0126006 0.00941009 2023-10-09T05:41:56.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:56.415 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:56.483 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:56.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:56.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:56.590 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:56.625 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:56.700 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:56.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:56.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:56.809 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:41:56.892 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:57.339 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:41:57.340271+0000 min lat: 0.00134385 max lat: 0.284215 avg lat: 0.00953156 2023-10-09T05:41:57.340 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:41:57.340 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 20 16 33435 33419 104.32 68.3125 0.109856 0.00953156 2023-10-09T05:41:57.430 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:41:57.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:57.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:41:57.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:57 smithi008 ceph-mon[184418]: pgmap v626: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T05:41:57.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:57 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:57.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:57 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:57 smithi008 ceph-mon[191227]: pgmap v626: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T05:41:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:57 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:57 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:57.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:41:57.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:57 smithi191 ceph-mon[161669]: pgmap v626: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T05:41:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:57 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:57 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:41:57.568 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:57.625 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:41:57.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:57.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:41:57.732 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:41:57.826 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:58.339 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 21 16 34695 34679 103.103 78.75 0.103423 0.00967263 2023-10-09T05:41:59.278 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:59.340 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 22 16 35970 35954 102.036 79.6875 0.0197375 0.00979308 2023-10-09T05:41:59.362 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:41:59.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:59.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:41:59.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:41:59.512 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:59.564 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:41:59.564 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:41:59.567 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:41:59.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:59.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:41:59.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:41:59.704 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:41:59.757 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:41:59.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:41:59.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:41:59.865 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:00.132 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:59 smithi191 ceph-mon[161669]: pgmap v627: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T05:42:00.132 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:00.132 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:41:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:00.165 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:59 smithi008 ceph-mon[184418]: pgmap v627: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T05:42:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:00.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:41:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:00.217 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:00.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:59 smithi008 ceph-mon[191227]: pgmap v627: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T05:42:00.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:00.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:41:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:00.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:00.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:00.323 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:00.340 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 23 16 37349 37333 101.347 86.1875 0.0121156 0.00986035 2023-10-09T05:42:00.353 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:00.412 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:00.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:00.518 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:00.549 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:00.598 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:00.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:00.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:00.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:00.731 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:00.783 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:00.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:00.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:00.853 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:42:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:42:00] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.33.4" 2023-10-09T05:42:00.854 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1189739765' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:42:00.854 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:00 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:42:00.888 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:00.895 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:42:00.966 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1189739765' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:42:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:00 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:42:01.117 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:42:01.173 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1189739765' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:42:01.173 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:00 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:42:01.344 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 24 16 38400 38384 99.8469 65.6875 0.016719 0.0100089 2023-10-09T05:42:02.133 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[184418]: pgmap v628: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T05:42:02.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:02.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[184418]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:42:02.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[184418]: osdmap e222: 8 total, 8 up, 7 in 2023-10-09T05:42:02.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[184418]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[184418]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.135 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[184418]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.135 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[191227]: pgmap v628: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T05:42:02.135 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:02.135 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[191227]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.135 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:42:02.136 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[191227]: osdmap e222: 8 total, 8 up, 7 in 2023-10-09T05:42:02.136 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[191227]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.136 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[191227]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.136 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:01 smithi008 ceph-mon[191227]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.167 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:42:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:01 smithi191 ceph-mon[161669]: pgmap v628: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T05:42:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:01 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:01 smithi191 ceph-mon[161669]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:01 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:42:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:01 smithi191 ceph-mon[161669]: osdmap e222: 8 total, 8 up, 7 in 2023-10-09T05:42:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:01 smithi191 ceph-mon[161669]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:01 smithi191 ceph-mon[161669]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:01 smithi191 ceph-mon[161669]: failed to encode map e222 with expected crc 2023-10-09T05:42:02.347 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 25 16 39239 39223 97.9432 52.4375 0.0118534 0.0102031 2023-10-09T05:42:02.370 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:02.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:02.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:02.508 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:02.553 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:02.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:02.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:02.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:02.689 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:02.742 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:02.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:02.850 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:02.922 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:03.145 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:03.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:03.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:03.259 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:03.292 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:03.345 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:03.347 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 26 15 40353 40338 96.8573 69.6875 0.010219 0.0103191 2023-10-09T05:42:03.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:03.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:03.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:03.479 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:03.533 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:03.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:03.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:03.638 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:03.703 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:03.716 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:03.734 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:03.864 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:03.867 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:03 smithi008 ceph-mon[184418]: pgmap v630: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T05:42:03.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:03.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:03.974 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:04.057 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:04.166 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:04.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:04.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:03 smithi008 ceph-mon[191227]: pgmap v630: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T05:42:04.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:04.289 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:03 smithi191 ceph-mon[161669]: pgmap v630: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T05:42:04.319 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:04.347 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 27 16 41419 41403 95.7359 66.5625 0.0149978 0.0104395 2023-10-09T05:42:04.369 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:04.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:04.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:04.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:04.505 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:04.560 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:04.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:04.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:04.667 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:04.728 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:04.873 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/211948325' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:42:04.873 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:04 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:42:04.873 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/211948325' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:42:04.874 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:04 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:42:04.912 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 7 pgp_num to 16 2023-10-09T05:42:05.081 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:42:05.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/211948325' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:42:05.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:04 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T05:42:05.349 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 28 16 42676 42660 95.1163 78.5625 0.0253409 0.0105061 2023-10-09T05:42:06.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[184418]: pgmap v631: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T05:42:06.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[184418]: failed to encode map e223 with expected crc 2023-10-09T05:42:06.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[184418]: failed to encode map e223 with expected crc 2023-10-09T05:42:06.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:42:06.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[184418]: osdmap e223: 8 total, 8 up, 7 in 2023-10-09T05:42:06.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[184418]: failed to encode map e223 with expected crc 2023-10-09T05:42:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[191227]: pgmap v631: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T05:42:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[191227]: failed to encode map e223 with expected crc 2023-10-09T05:42:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[191227]: failed to encode map e223 with expected crc 2023-10-09T05:42:06.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:42:06.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[191227]: osdmap e223: 8 total, 8 up, 7 in 2023-10-09T05:42:06.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:05 smithi008 ceph-mon[191227]: failed to encode map e223 with expected crc 2023-10-09T05:42:06.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:05 smithi191 ceph-mon[161669]: pgmap v631: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T05:42:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:05 smithi191 ceph-mon[161669]: failed to encode map e223 with expected crc 2023-10-09T05:42:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:05 smithi191 ceph-mon[161669]: failed to encode map e223 with expected crc 2023-10-09T05:42:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:05 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T05:42:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:05 smithi191 ceph-mon[161669]: osdmap e223: 8 total, 8 up, 7 in 2023-10-09T05:42:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:05 smithi191 ceph-mon[161669]: failed to encode map e223 with expected crc 2023-10-09T05:42:06.349 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 29 16 44039 44023 94.774 85.1875 0.0102144 0.0105455 2023-10-09T05:42:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:06 smithi008 ceph-mon[191227]: pgmap v633: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T05:42:07.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:06 smithi008 ceph-mon[184418]: pgmap v633: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T05:42:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:06 smithi191 ceph-mon[161669]: pgmap v633: 89 pgs: 89 active+clean; 2.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T05:42:07.350 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 30 16 45001 44985 93.62 60.125 0.00213588 0.0106668 2023-10-09T05:42:07.794 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:07.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:07.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:07.902 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:07.972 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:08.224 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:08.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:08.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:08.339 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:08.350 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 31 16 46138 46122 92.8922 71.0625 0.0167162 0.0107597 2023-10-09T05:42:08.417 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:08.631 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:08.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:08.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:08.727 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:42:08.727 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:42:08.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:08.789 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:08.839 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:08.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:08.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:08.858 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:42:08.858 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:42:08.896 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:42:08.912 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:08.946 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:42:08.950 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:08.970 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3302854215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:08.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3302854215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:08.989 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:42:09.026 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:42:09.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3302854215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:09.057 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:09.074 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:42:09.115 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:42:09.133 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:42:09.133 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:42:09.133 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:42:09.133 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:42:09.133 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:42:09.133 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:42:09.133 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:42:09.133 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:42:09.350 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 32 16 47128 47112 91.9223 61.875 0.0132427 0.0108728 2023-10-09T05:42:10.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:09 smithi008 ceph-mon[184418]: pgmap v634: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:42:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:09 smithi008 ceph-mon[191227]: pgmap v634: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:42:10.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:09 smithi191 ceph-mon[161669]: pgmap v634: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:42:10.350 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 33 16 48164 48148 91.0995 64.75 0.0150169 0.01097 2023-10-09T05:42:10.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:42:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:42:10] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.33.4" 2023-10-09T05:42:11.236 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:11.243 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:42:11.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:11.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:11.350 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 34 16 49235 49219 90.3891 66.9375 0.00639169 0.0110564 2023-10-09T05:42:11.352 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:11.455 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:11.957 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:11 smithi008 ceph-mon[191227]: pgmap v635: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T05:42:11.957 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:11 smithi008 ceph-mon[184418]: pgmap v635: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T05:42:11.973 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:11.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:11.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:12.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:12.113 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:12.175 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:12.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:12.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:12.182 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:11 smithi191 ceph-mon[161669]: pgmap v635: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T05:42:12.283 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:12.313 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:12.350 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 35 15 50340 50325 89.7819 69.125 0.0140451 0.0111312 2023-10-09T05:42:12.365 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:12.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:12.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:12.472 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:12.501 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:12.988 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:12.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:13.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:13.129 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:13.179 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:42:13.179 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:42:13.179 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T05:42:13.180 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:42:13.180 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:42:13.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T05:42:13.183 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:13.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:13.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:13.275 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:42:13.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:13.318 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:13.360 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 36 16 51333 51317 89.0083 62 0.0194449 0.0112282 2023-10-09T05:42:13.380 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:13.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:13.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:13.419 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:13.488 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:42:13.599 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:42:13.630 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:13.827 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:42:13.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:13.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:42:13.866 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:13 smithi008 ceph-mon[184418]: pgmap v636: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T05:42:13.867 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:13.867 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:13.867 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:13 smithi008 ceph-mon[191227]: pgmap v636: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T05:42:13.867 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:13.867 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:13.941 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:13.974 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:13 smithi191 ceph-mon[161669]: pgmap v636: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T05:42:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:14.233 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:14.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:14.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:14.351 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 37 16 52212 52196 88.0881 54.9375 0.0150854 0.0113468 2023-10-09T05:42:14.369 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:14.429 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:14.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:14.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:14.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:14.564 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:14.615 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:14.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:14.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:14.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:14.743 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:14.785 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:14.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:14.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:14.888 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:42:14.950 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:42:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:14 smithi191 ceph-mon[161669]: pgmap v637: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T05:42:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:14 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/5108281' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:15.088 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:42:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:42:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:15.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[184418]: pgmap v637: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T05:42:15.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/5108281' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:42:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[191227]: pgmap v637: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T05:42:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:14 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/5108281' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:15.246 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:42:15.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:15.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:42:15.354 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 38 16 53100 53084 87.2294 55.5 0.00775898 0.0114593 2023-10-09T05:42:15.356 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:15.430 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:15.541 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:15.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:15.644 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:42:15.674 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:15.717 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:42:15.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:15.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:42:15.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:15.853 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:15.892 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:15 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:15.893 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/5108281' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:42:15.893 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:15 smithi008 ceph-mon[184418]: osdmap e224: 8 total, 8 up, 7 in 2023-10-09T05:42:15.893 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:15 smithi008 ceph-mon[184418]: failed to encode map e224 with expected crc 2023-10-09T05:42:15.893 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:15 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:15.893 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/5108281' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:42:15.893 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:15 smithi008 ceph-mon[191227]: osdmap e224: 8 total, 8 up, 7 in 2023-10-09T05:42:15.894 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:15 smithi008 ceph-mon[191227]: failed to encode map e224 with expected crc 2023-10-09T05:42:15.915 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:15.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:15.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:16.019 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:16.079 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:16.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:15 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/5108281' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:42:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:15 smithi191 ceph-mon[161669]: osdmap e224: 8 total, 8 up, 7 in 2023-10-09T05:42:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:15 smithi191 ceph-mon[161669]: failed to encode map e224 with expected crc 2023-10-09T05:42:16.325 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:16.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:16.340 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:16.353 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 39 16 53986 53970 86.4132 55.375 0.0130263 0.0115666 2023-10-09T05:42:16.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:16.464 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:16.518 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:16.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:16.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:16.584 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:42:16.585 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:42:16.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:42:16.585 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:42:16.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.585 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98118 5.5 5:a3d60b4a:::benchmark_data_smithi008_238708_object98117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T05:42:16.585 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.694337+0000", 2023-10-09T05:42:16.585 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 495.88764806199998, 2023-10-09T05:42:16.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.914715308, 2023-10-09T05:42:16.585 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.585 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:42:16.585 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98118 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694337+0000", 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694337+0000", 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694334+0000", 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694371+0000", 2023-10-09T05:42:16.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7106e-05 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694394+0000", 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3269e-05 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.694410+0000", 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5454000000000001e-05 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.879229+0000", 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.184819179 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.879238+0000", 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6590000000000002e-06 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.589 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.608962+0000", 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72972381399999997 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.609052+0000", 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9624999999999997e-05 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.590 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.34770.0:98137 5.3 5:c7026034:::benchmark_data_smithi008_238708_object98136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e118)", 2023-10-09T05:42:16.591 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:34:00.815708+0000", 2023-10-09T05:42:16.591 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 495.76627706300002, 2023-10-09T05:42:16.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.811726473, 2023-10-09T05:42:16.591 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.591 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.591 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.591 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.34770", 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 98137 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815708+0000", 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815708+0000", 2023-10-09T05:42:16.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815705+0000", 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815742+0000", 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6514999999999999e-05 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815763+0000", 2023-10-09T05:42:16.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1107999999999998e-05 2023-10-09T05:42:16.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.815778+0000", 2023-10-09T05:42:16.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4718999999999999e-05 2023-10-09T05:42:16.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.848379+0000", 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.032600899000000003 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:00.848390+0000", 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1735e-05 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.619939+0000", 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77154830200000002 2023-10-09T05:42:16.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.596 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.596 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.620114+0000", 2023-10-09T05:42:16.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017572099999999999 2023-10-09T05:42:16.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.596 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:42:16.596 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.620276+0000", 2023-10-09T05:42:16.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016124600000000001 2023-10-09T05:42:16.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.622357+0000", 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020815999999999999 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.625946+0000", 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035886339999999998 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.597 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627349+0000", 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014031130000000001 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627400+0000", 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1385999999999999e-05 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:34:01.627434+0000", 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3640999999999999e-05 2023-10-09T05:42:16.598 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4050 6.0 6:0be16091:::benchmark_data_smithi008_295510_object4049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.480314+0000", 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 308.10167064500001, 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48189201799999998, 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.599 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4050 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480314+0000", 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480314+0000", 2023-10-09T05:42:16.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.601 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480313+0000", 2023-10-09T05:42:16.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:42:16.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.601 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480347+0000", 2023-10-09T05:42:16.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3819000000000003e-05 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480359+0000", 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2092999999999999e-05 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480374+0000", 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5096000000000001e-05 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480421+0000", 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7611999999999997e-05 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480552+0000", 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013091099999999999 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.480687+0000", 2023-10-09T05:42:16.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013453799999999999 2023-10-09T05:42:16.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959945+0000", 2023-10-09T05:42:16.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47925849799999998 2023-10-09T05:42:16.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959955+0000", 2023-10-09T05:42:16.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8859999999999999e-06 2023-10-09T05:42:16.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962107+0000", 2023-10-09T05:42:16.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002151975 2023-10-09T05:42:16.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962152+0000", 2023-10-09T05:42:16.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5049999999999997e-05 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.962206+0000", 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3773999999999997e-05 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4060 6.0 6:04bff711:::benchmark_data_smithi008_295510_object4059:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:42:16.606 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:08.482112+0000", 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 308.099873118, 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48355848200000001, 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4060 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482112+0000", 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482112+0000", 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482111+0000", 2023-10-09T05:42:16.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.482141+0000", 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0737000000000001e-05 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959856+0000", 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47771430100000001 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959871+0000", 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5611999999999999e-05 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.959984+0000", 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000112897 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.960209+0000", 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00022461200000000001 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.960939+0000", 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000730622 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.963896+0000", 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029571720000000001 2023-10-09T05:42:16.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965560+0000", 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016638639999999999 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965601+0000", 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0173999999999999e-05 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965632+0000", 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1591000000000003e-05 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:08.965670+0000", 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7867000000000002e-05 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.613 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140594 6.9 6:972114e9:::benchmark_data_smithi008_295510_object52231:head [delete] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.801636+0000", 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.780348511, 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.041270468, 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140594 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.801636+0000", 2023-10-09T05:42:16.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.801636+0000", 2023-10-09T05:42:16.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.801635+0000", 2023-10-09T05:42:16.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:42:16.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.801642+0000", 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7689999999999996e-06 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.825379+0000", 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.023737011999999998 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.825394+0000", 2023-10-09T05:42:16.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4861999999999999e-05 2023-10-09T05:42:16.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.835389+0000", 2023-10-09T05:42:16.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0099948799999999994 2023-10-09T05:42:16.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.835401+0000", 2023-10-09T05:42:16.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1549000000000001e-05 2023-10-09T05:42:16.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.826729+0000", 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99132818599999994 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.826827+0000", 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7775999999999994e-05 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.826945+0000", 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000118439 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.830778+0000", 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038329269999999999 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842771+0000", 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011992505000000001 2023-10-09T05:42:16.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842811+0000", 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0772000000000002e-05 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842865+0000", 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3745000000000001e-05 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842907+0000", 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1482e-05 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140616 6.9 6:92cdb345:::benchmark_data_smithi008_295510_object52253:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.868684+0000", 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.71330017599999, 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97444081699999996, 2023-10-09T05:42:16.626 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140616 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868684+0000", 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868684+0000", 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868683+0000", 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868690+0000", 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.849e-06 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868702+0000", 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2642e-05 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868711+0000", 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1200000000000002e-06 2023-10-09T05:42:16.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868775+0000", 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4764999999999994e-05 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868787+0000", 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1617000000000001e-05 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.830809+0000", 2023-10-09T05:42:16.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96202192200000003 2023-10-09T05:42:16.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.830938+0000", 2023-10-09T05:42:16.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012940799999999999 2023-10-09T05:42:16.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.831034+0000", 2023-10-09T05:42:16.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5957000000000003e-05 2023-10-09T05:42:16.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842752+0000", 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011717455 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842996+0000", 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00024453700000000002 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843074+0000", 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8015000000000004e-05 2023-10-09T05:42:16.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843102+0000", 2023-10-09T05:42:16.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7381000000000001e-05 2023-10-09T05:42:16.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843125+0000", 2023-10-09T05:42:16.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3686e-05 2023-10-09T05:42:16.635 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.635 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140627 6.9 6:9a23699b:::benchmark_data_smithi008_295510_object52264:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.891296+0000", 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.69068816800001, 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.95198831100000003, 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.636 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140627 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891296+0000", 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891296+0000", 2023-10-09T05:42:16.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891294+0000", 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891305+0000", 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1089e-05 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891322+0000", 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6232e-05 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891333+0000", 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1722999999999999e-05 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891404+0000", 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0164000000000001e-05 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891415+0000", 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1926000000000001e-05 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.834610+0000", 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94319403899999998 2023-10-09T05:42:16.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.834746+0000", 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013667 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.834874+0000", 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012732499999999999 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843048+0000", 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081748810000000002 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843142+0000", 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3560999999999994e-05 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843196+0000", 2023-10-09T05:42:16.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4468000000000001e-05 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843248+0000", 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1266e-05 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843285+0000", 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7061000000000003e-05 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.643 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140639 6.9 6:91830589:::benchmark_data_smithi008_295510_object52276:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.911940+0000", 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.67004427699999, 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94121215400000002, 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.644 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140639 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911940+0000", 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911940+0000", 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911940+0000", 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911943+0000", 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7889999999999999e-06 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911949+0000", 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6769999999999998e-06 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911955+0000", 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.015e-06 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.912210+0000", 2023-10-09T05:42:16.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000254338 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.912220+0000", 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0635999999999999e-05 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.839273+0000", 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92705263999999998 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.839367+0000", 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4503e-05 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.839438+0000", 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0965999999999998e-05 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843330+0000", 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038915070000000002 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853009+0000", 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0096787590000000003 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853053+0000", 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4736e-05 2023-10-09T05:42:16.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853105+0000", 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1403999999999998e-05 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853152+0000", 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7645999999999997e-05 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.651 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140645 6.9 6:9412362f:::benchmark_data_smithi008_295510_object52282:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.933367+0000", 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.64861723000001, 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93559094200000004, 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140645 2023-10-09T05:42:16.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933367+0000", 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933367+0000", 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933365+0000", 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933375+0000", 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6840000000000008e-06 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933396+0000", 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1433999999999998e-05 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933408+0000", 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1389999999999999e-05 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933515+0000", 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000107681 2023-10-09T05:42:16.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933528+0000", 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2191000000000001e-05 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843353+0000", 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90982495399999996 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843428+0000", 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5394000000000004e-05 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843498+0000", 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9961000000000003e-05 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853172+0000", 2023-10-09T05:42:16.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0096736459999999993 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868740+0000", 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015568584999999999 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868872+0000", 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013186599999999999 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868919+0000", 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7134e-05 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868958+0000", 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9171000000000002e-05 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140663 6.9 6:9f610fbc:::benchmark_data_smithi008_295510_object52300:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.969184+0000", 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.612800502, 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89992256199999998, 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140663 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.660 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969184+0000", 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969184+0000", 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969182+0000", 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969193+0000", 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0937e-05 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969215+0000", 2023-10-09T05:42:16.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2320999999999999e-05 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969226+0000", 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0558e-05 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969298+0000", 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2253000000000001e-05 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969312+0000", 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3594999999999999e-05 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.848260+0000", 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.87894824199999999 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.848390+0000", 2023-10-09T05:42:16.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013013200000000001 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.848515+0000", 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012543100000000001 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.857837+0000", 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0093214249999999995 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868779+0000", 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010942062000000001 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.869018+0000", 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00023956199999999999 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.869067+0000", 2023-10-09T05:42:16.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8427000000000002e-05 2023-10-09T05:42:16.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.869107+0000", 2023-10-09T05:42:16.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9813e-05 2023-10-09T05:42:16.667 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.667 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.667 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.668 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140674 6.9 6:9fbec6cb:::benchmark_data_smithi008_295510_object52311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.668 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.981500+0000", 2023-10-09T05:42:16.668 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.60048436400001, 2023-10-09T05:42:16.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.91512313499999998, 2023-10-09T05:42:16.668 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.668 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.668 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.668 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140674 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981500+0000", 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981500+0000", 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981498+0000", 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981508+0000", 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7890000000000006e-06 2023-10-09T05:42:16.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981528+0000", 2023-10-09T05:42:16.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0392999999999999e-05 2023-10-09T05:42:16.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981538+0000", 2023-10-09T05:42:16.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0149999999999999e-05 2023-10-09T05:42:16.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981604+0000", 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5709000000000004e-05 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981615+0000", 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1090999999999999e-05 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853195+0000", 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.87157961399999995 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853323+0000", 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012834899999999999 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853450+0000", 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012675300000000001 2023-10-09T05:42:16.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868789+0000", 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015338991999999999 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868838+0000", 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9543999999999999e-05 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896521+0000", 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.027682469000000001 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896573+0000", 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1724000000000001e-05 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896623+0000", 2023-10-09T05:42:16.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0781000000000001e-05 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140703 6.9 6:9ba5b182:::benchmark_data_smithi008_295510_object52340:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:04.038296+0000", 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.54368873600001, 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85847289500000001, 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.676 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140703 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.038296+0000", 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.038296+0000", 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.038294+0000", 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.038302+0000", 2023-10-09T05:42:16.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6830000000000008e-06 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.038320+0000", 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8669999999999999e-05 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.038332+0000", 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1051e-05 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.038422+0000", 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0773999999999995e-05 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.038435+0000", 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2658e-05 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.857883+0000", 2023-10-09T05:42:16.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.81944798900000004 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.858031+0000", 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014843699999999999 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.858189+0000", 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015717899999999999 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868796+0000", 2023-10-09T05:42:16.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010607798 2023-10-09T05:42:16.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.887527+0000", 2023-10-09T05:42:16.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018730824 2023-10-09T05:42:16.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896679+0000", 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0091522400000000007 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896728+0000", 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8436000000000002e-05 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896769+0000", 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0929000000000001e-05 2023-10-09T05:42:16.683 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140710 6.9 6:98cfa2f1:::benchmark_data_smithi008_295510_object52347:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:04.053605+0000", 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.52837959999999, 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85610101500000002, 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.684 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140710 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.053605+0000", 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.053605+0000", 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.053603+0000", 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.053613+0000", 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5570000000000001e-06 2023-10-09T05:42:16.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.053630+0000", 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7462999999999999e-05 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.053640+0000", 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8579999999999995e-06 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.053704+0000", 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3660999999999999e-05 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.053713+0000", 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8510000000000005e-06 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.863996+0000", 2023-10-09T05:42:16.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.81028331600000003 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.864172+0000", 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017662099999999999 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.864321+0000", 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014826100000000001 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.873848+0000", 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0095276049999999998 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.887554+0000", 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013705701000000001 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.909613+0000", 2023-10-09T05:42:16.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022058787 2023-10-09T05:42:16.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.909661+0000", 2023-10-09T05:42:16.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8624000000000001e-05 2023-10-09T05:42:16.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.909706+0000", 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4576999999999997e-05 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140730 6.9 6:936bd551:::benchmark_data_smithi008_295510_object52367:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:04.093283+0000", 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.48870124000001, 2023-10-09T05:42:16.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.81652804499999998, 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140730 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.093283+0000", 2023-10-09T05:42:16.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.093283+0000", 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.093282+0000", 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.093289+0000", 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.888e-06 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.093305+0000", 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5645999999999999e-05 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.093313+0000", 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0849999999999997e-06 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.093405+0000", 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2666999999999999e-05 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.093416+0000", 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1079e-05 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.869153+0000", 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77573635100000005 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.869284+0000", 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013161 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.869389+0000", 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000104498 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.883577+0000", 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.01418837 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896822+0000", 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013244588 2023-10-09T05:42:16.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.909739+0000", 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012917069999999999 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.909780+0000", 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1454e-05 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.909811+0000", 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1007999999999999e-05 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140741 6.9 6:919e5004:::benchmark_data_smithi008_295510_object52378:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:04.113106+0000", 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.468878512, 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.79680374499999995, 2023-10-09T05:42:16.700 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140741 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.113106+0000", 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.113106+0000", 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.113104+0000", 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.113115+0000", 2023-10-09T05:42:16.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0967e-05 2023-10-09T05:42:16.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.113138+0000", 2023-10-09T05:42:16.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2861e-05 2023-10-09T05:42:16.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.113148+0000", 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0118999999999999e-05 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.113235+0000", 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6327000000000002e-05 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.113249+0000", 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4593000000000001e-05 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.873885+0000", 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.76063538399999997 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.874025+0000", 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014007599999999999 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.874150+0000", 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000125773 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.887484+0000", 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.01333315 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.906244+0000", 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018760348999999999 2023-10-09T05:42:16.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.909842+0000", 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035976860000000001 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.909879+0000", 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7292999999999997e-05 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.909910+0000", 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0998999999999999e-05 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140761 6.9 6:92b4b758:::benchmark_data_smithi008_295510_object52398:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:04.147066+0000", 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.434918522, 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.76295238700000001, 2023-10-09T05:42:16.708 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140761 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.147066+0000", 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.147066+0000", 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.147065+0000", 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.147069+0000", 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4599999999999999e-06 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.147078+0000", 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7970000000000006e-06 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.147082+0000", 2023-10-09T05:42:16.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3800000000000004e-06 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.147131+0000", 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8949000000000002e-05 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.147141+0000", 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0363999999999999e-05 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.878566+0000", 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73142475699999998 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.878678+0000", 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011194200000000001 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.878769+0000", 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1263999999999999e-05 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896458+0000", 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017689177 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.906272+0000", 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.009813647 2023-10-09T05:42:16.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.909947+0000", 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036753380000000002 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.909988+0000", 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0225000000000002e-05 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.910018+0000", 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0817e-05 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140766 6.9 6:9daea187:::benchmark_data_smithi008_295510_object52403:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:04.157780+0000", 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.42420501399999, 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75233649499999999, 2023-10-09T05:42:16.716 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140766 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.157780+0000", 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.157780+0000", 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.157779+0000", 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.157785+0000", 2023-10-09T05:42:16.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2700000000000001e-06 2023-10-09T05:42:16.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.157797+0000", 2023-10-09T05:42:16.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2376e-05 2023-10-09T05:42:16.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.157815+0000", 2023-10-09T05:42:16.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7924999999999999e-05 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.157915+0000", 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9337000000000001e-05 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.157926+0000", 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1763000000000001e-05 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.883604+0000", 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72567739600000003 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.883726+0000", 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012208600000000001 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.883846+0000", 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000119831 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896467+0000", 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.01262112 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.906298+0000", 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0098315030000000001 2023-10-09T05:42:16.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.910048+0000", 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037493159999999999 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.910085+0000", 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7132000000000003e-05 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.910116+0000", 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1430000000000002e-05 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140793 6.9 6:9cef5f6c:::benchmark_data_smithi008_295510_object52430:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:04.217831+0000", 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.36415315100001, 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.71209486600000005, 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.724 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140793 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.217831+0000", 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.217831+0000", 2023-10-09T05:42:16.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.217830+0000", 2023-10-09T05:42:16.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:42:16.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.217836+0000", 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3040000000000001e-06 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.217850+0000", 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3122000000000001e-05 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.217859+0000", 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1419999999999999e-06 2023-10-09T05:42:16.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.729 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.217925+0000", 2023-10-09T05:42:16.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6088999999999997e-05 2023-10-09T05:42:16.729 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.729 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.217939+0000", 2023-10-09T05:42:16.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3743e-05 2023-10-09T05:42:16.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.887571+0000", 2023-10-09T05:42:16.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.66963220300000004 2023-10-09T05:42:16.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.887659+0000", 2023-10-09T05:42:16.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8642999999999998e-05 2023-10-09T05:42:16.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.887735+0000", 2023-10-09T05:42:16.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5147000000000004e-05 2023-10-09T05:42:16.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896637+0000", 2023-10-09T05:42:16.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0089022749999999994 2023-10-09T05:42:16.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.929704+0000", 2023-10-09T05:42:16.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.033067325000000002 2023-10-09T05:42:16.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.929830+0000", 2023-10-09T05:42:16.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012635500000000001 2023-10-09T05:42:16.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.929885+0000", 2023-10-09T05:42:16.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4710000000000003e-05 2023-10-09T05:42:16.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.929926+0000", 2023-10-09T05:42:16.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1164000000000001e-05 2023-10-09T05:42:16.734 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.734 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.734 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.734 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140816 6.9 6:956a9e45:::benchmark_data_smithi008_295510_object52453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.734 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:04.295038+0000", 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.28694653299999, 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.63501496999999996, 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140816 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.295038+0000", 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.295038+0000", 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.295037+0000", 2023-10-09T05:42:16.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.295044+0000", 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5920000000000003e-06 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.295057+0000", 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2828e-05 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.295069+0000", 2023-10-09T05:42:16.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1883e-05 2023-10-09T05:42:16.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.295129+0000", 2023-10-09T05:42:16.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9734000000000001e-05 2023-10-09T05:42:16.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.295145+0000", 2023-10-09T05:42:16.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6629e-05 2023-10-09T05:42:16.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.891996+0000", 2023-10-09T05:42:16.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.59685096699999995 2023-10-09T05:42:16.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.892096+0000", 2023-10-09T05:42:16.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9383999999999996e-05 2023-10-09T05:42:16.740 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.740 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.740 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.892217+0000", 2023-10-09T05:42:16.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000120723 2023-10-09T05:42:16.740 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.740 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.740 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896779+0000", 2023-10-09T05:42:16.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045627660000000002 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.929738+0000", 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.032959169000000003 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.929965+0000", 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000226574 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.930014+0000", 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8819999999999997e-05 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.930053+0000", 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9221000000000003e-05 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140825 6.9 6:9a3f4eab:::benchmark_data_smithi008_295510_object52462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:04.337316+0000", 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 132.244668773, 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.59287589600000001, 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140825 2023-10-09T05:42:16.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.337316+0000", 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.337316+0000", 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.337315+0000", 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.337320+0000", 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3890000000000004e-06 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.337333+0000", 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2785e-05 2023-10-09T05:42:16.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.337341+0000", 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5339999999999999e-06 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.337400+0000", 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9018000000000003e-05 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.337410+0000", 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2969999999999992e-06 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896854+0000", 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.55944470800000001 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.897009+0000", 2023-10-09T05:42:16.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015443 2023-10-09T05:42:16.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:42:16.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.897142+0000", 2023-10-09T05:42:16.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013297999999999999 2023-10-09T05:42:16.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:16.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.906676+0000", 2023-10-09T05:42:16.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.009534687 2023-10-09T05:42:16.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.929770+0000", 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.023093196999999999 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.930087+0000", 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00031743 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:42:16.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.930145+0000", 2023-10-09T05:42:16.750 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8239999999999998e-05 2023-10-09T05:42:16.750 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:16.750 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:16.751 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:42:16.751 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.930192+0000", 2023-10-09T05:42:16.751 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6455999999999997e-05 2023-10-09T05:42:16.751 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.751 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.751 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.752 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:16.752 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:16.752 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:42:16.753 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:16.782 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:17.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:16 smithi191 ceph-mon[161669]: failed to encode map e224 with expected crc 2023-10-09T05:42:17.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:16 smithi191 ceph-mon[161669]: failed to encode map e224 with expected crc 2023-10-09T05:42:17.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:16 smithi191 ceph-mon[161669]: pgmap v639: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:42:17.054 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:17.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:17.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:17.159 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:17.195 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:16 smithi008 ceph-mon[184418]: failed to encode map e224 with expected crc 2023-10-09T05:42:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:16 smithi008 ceph-mon[184418]: failed to encode map e224 with expected crc 2023-10-09T05:42:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:16 smithi008 ceph-mon[184418]: pgmap v639: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:42:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:16 smithi008 ceph-mon[191227]: failed to encode map e224 with expected crc 2023-10-09T05:42:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:16 smithi008 ceph-mon[191227]: failed to encode map e224 with expected crc 2023-10-09T05:42:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:16 smithi008 ceph-mon[191227]: pgmap v639: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:42:17.309 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:17.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:17.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:17.352 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:42:17.353338+0000 min lat: 0.00134385 max lat: 0.28677 avg lat: 0.0116403 2023-10-09T05:42:17.353 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:42:17.353 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 40 16 55017 55001 85.8641 64.4375 0.0113946 0.0116403 2023-10-09T05:42:17.416 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:17.500 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:17.511 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:42:17.511 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:42:17.511 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:42:17.937 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:17.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:17.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:18.041 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:42:18.069 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:18.331 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:42:18.353 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 41 16 55833 55817 85.0142 51 0.0205124 0.0117564 2023-10-09T05:42:18.406 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:42:18.593 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:42:18.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:18.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:42:18.716 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:42:18.862 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:19.304 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:42:19.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:19.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:42:19.353 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 42 16 56792 56776 84.4175 59.9375 0.0184712 0.0118412 2023-10-09T05:42:19.418 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:19.466 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:19.504 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:19.601 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:19.622 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:19.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:19.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:19.661 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:42:19.705 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:42:19.731 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:42:19.746 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:42:19.798 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:19.821 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:42:19.856 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:42:19.896 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:42:19.904 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:19 smithi008 ceph-mon[184418]: pgmap v640: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T05:42:19.904 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:19 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3721998317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:19.905 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:19 smithi008 ceph-mon[191227]: pgmap v640: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T05:42:19.905 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:19 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3721998317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:19.935 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:42:19.976 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:42:19.984 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:42:19.985 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:42:19.985 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:42:19.985 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:42:19.985 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:42:19.985 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:42:19.985 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:42:19.985 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:42:20.111 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:42:20.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:20.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:42:20.222 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:42:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:19 smithi191 ceph-mon[161669]: pgmap v640: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T05:42:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:19 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3721998317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:20.325 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:20.353 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 43 16 57728 57712 83.8151 58.5 0.0112596 0.0119249 2023-10-09T05:42:20.972 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:42:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:42:20] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.33.4" 2023-10-09T05:42:21.259 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:42:21.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:21.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:42:21.354 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 44 16 58906 58890 83.5808 73.625 0.00914223 0.011959 2023-10-09T05:42:21.366 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:21.402 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:21.450 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:21.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:21.558 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:21.613 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:21.658 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:21.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:21.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:21.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:21.792 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:21.849 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:21.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:21.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:21.955 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:21.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:21 smithi008 ceph-mon[184418]: pgmap v641: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-09T05:42:21.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:21 smithi008 ceph-mon[191227]: pgmap v641: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-09T05:42:21.987 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:22.031 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:22.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:22.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:21 smithi191 ceph-mon[161669]: pgmap v641: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-09T05:42:22.136 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:22.204 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:22.354 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 45 16 59566 59550 82.6407 41.25 0.0147136 0.0120955 2023-10-09T05:42:22.479 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:22.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:22.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:22.589 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:22.616 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:23.201 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:23.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:23.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:23.274 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:42:23.274 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:42:23.275 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T05:42:23.275 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:42:23.317 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:23.355 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 46 16 60526 60510 82.1472 60 0.00985089 0.0121678 2023-10-09T05:42:23.421 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:23.858 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:42:23.858 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:42:23.862 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:23.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:23.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:23.896 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:23 smithi008 ceph-mon[184418]: pgmap v642: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T05:42:23.897 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:23 smithi008 ceph-mon[191227]: pgmap v642: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T05:42:23.977 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:24.010 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:24.061 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:24.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:24.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:24.167 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:24.260 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:23 smithi191 ceph-mon[161669]: pgmap v642: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T05:42:24.341 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:24.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:24.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:24.356 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 47 16 61361 61345 81.5101 52.1875 0.00338866 0.0122537 2023-10-09T05:42:24.445 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:24.530 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:24.614 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:24.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:24.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:24.718 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:24.782 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:24.897 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:42:25.142 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:42:25.201 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:24 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3550391878' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:25.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:24 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:25.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:24 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3550391878' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:25.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:24 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:25.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:24 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3550391878' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:25.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:24 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:42:25.355 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 48 16 62308 62292 81.0453 59.1875 0.00262641 0.0123251 2023-10-09T05:42:25.589 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:25.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:25.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:25.692 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:25.722 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:25.774 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:25.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:25.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:25.815 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:42:25.882 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:26.038 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:26.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:25 smithi191 ceph-mon[161669]: pgmap v643: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T05:42:26.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:25 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:25 smithi191 ceph-mon[161669]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:25 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:42:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:25 smithi191 ceph-mon[161669]: osdmap e225: 8 total, 8 up, 7 in 2023-10-09T05:42:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:25 smithi191 ceph-mon[161669]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:25 smithi191 ceph-mon[161669]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:25 smithi191 ceph-mon[161669]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.176 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:26.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[184418]: pgmap v643: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T05:42:26.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:26.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[184418]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:42:26.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[184418]: osdmap e225: 8 total, 8 up, 7 in 2023-10-09T05:42:26.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[184418]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[184418]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[184418]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:26.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:26.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[191227]: pgmap v643: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T05:42:26.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:26.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[191227]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:42:26.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[191227]: osdmap e225: 8 total, 8 up, 7 in 2023-10-09T05:42:26.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[191227]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[191227]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:25 smithi008 ceph-mon[191227]: failed to encode map e225 with expected crc 2023-10-09T05:42:26.219 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:26.290 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:26.325 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:26.356 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 49 16 63003 62987 80.278 43.4375 0.0203943 0.0124484 2023-10-09T05:42:26.390 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:26.743 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:26.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:26.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:26.850 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:26.934 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:27.357 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 50 16 63991 63975 79.9075 61.75 0.0217039 0.0125089 2023-10-09T05:42:28.060 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:42:28.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:27 smithi008 ceph-mon[184418]: pgmap v645: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T05:42:28.130 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:27 smithi008 ceph-mon[191227]: pgmap v645: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T05:42:28.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:27 smithi191 ceph-mon[161669]: pgmap v645: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T05:42:28.357 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 51 16 64828 64812 79.366 52.3125 0.00718879 0.0125947 2023-10-09T05:42:28.784 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:28.786 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:28.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:28.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:28.904 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:28.934 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:28.991 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:28.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:28.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:29.096 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:29.188 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:29.359 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 52 16 65778 65762 78.9784 59.375 0.0185032 0.0126564 2023-10-09T05:42:29.643 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:29.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:29.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:29.755 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:29.852 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:29.884 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:29.987 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:42:30.019 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:42:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:29 smithi008 ceph-mon[184418]: pgmap v646: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s 2023-10-09T05:42:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:29 smithi008 ceph-mon[191227]: pgmap v646: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s 2023-10-09T05:42:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:30.291 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:30.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:30.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:30.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:29 smithi191 ceph-mon[161669]: pgmap v646: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s 2023-10-09T05:42:30.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:30.359 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 53 16 66687 66671 78.5603 56.8125 0.0158003 0.0127231 2023-10-09T05:42:30.398 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:30.505 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:30.614 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:42:30.614 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:42:30.614 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:42:30.649 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:42:30.678 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:42:30.717 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:42:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:42:30] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.33.4" 2023-10-09T05:42:30.811 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:42:30.812 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:42:30.812 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:42:30.812 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:42:30.812 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:42:30.812 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:42:30.812 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:42:30.812 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:42:30.812 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:42:30.813 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:42:30.813 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:42:31.062 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:31.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:31.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:31.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3847132048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:31.077 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3847132048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:31.172 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:31.261 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3847132048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:31.359 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 54 16 67579 67563 78.1381 55.75 0.0150191 0.0127931 2023-10-09T05:42:32.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:31 smithi008 ceph-mon[184418]: from='client.35451 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:42:32.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:31 smithi008 ceph-mon[184418]: pgmap v647: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 56 MiB/s wr, 902 op/s 2023-10-09T05:42:32.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:31 smithi008 ceph-mon[191227]: from='client.35451 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:42:32.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:31 smithi008 ceph-mon[191227]: pgmap v647: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 56 MiB/s wr, 902 op/s 2023-10-09T05:42:32.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:31 smithi191 ceph-mon[161669]: from='client.35451 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:42:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:31 smithi191 ceph-mon[161669]: pgmap v647: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 56 MiB/s wr, 902 op/s 2023-10-09T05:42:32.359 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 55 16 68387 68371 77.6358 50.5 0.0272754 0.0128744 2023-10-09T05:42:32.570 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:32.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:32.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:32.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:32.709 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:32.760 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:32.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:32.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:32.769 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":646,"stamp":"2023-10-09T05:42:28.183156+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4081129395,"num_objects":62228,"num_object_clones":0,"num_object_copies":186684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62228,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":63306,"num_write_kb":3995941,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":4789622,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":63370,"ondisk_log_size":63370,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14340124,"kb_used_data":11845028,"kb_used_omap":31,"kb_used_meta":2495008,"kb_avail":641875940,"statfs":{"total":671965249536,"available":657280962560,"internally_reserved":0,"allocated":12129308672,"data_stored":12124412223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2554888649},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":602210304,"num_objects":9189,"num_object_clones":0,"num_object_copies":27567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9189,"num_write_kb":588096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006601"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":207,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.075725+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:42:15.075725+0000","last_peered":"2023-10-09T05:42:15.075725+0000","last_clean":"2023-10-09T05:42:15.075725+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:42:15.075725+0000","last_undegraded":"2023-10-09T05:42:15.075725+0000","last_fullsized":"2023-10-09T05:42:15.075725+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":197,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.110734+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:42:15.110734+0000","last_peered":"2023-10-09T05:42:15.110734+0000","last_clean":"2023-10-09T05:42:15.110734+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:42:15.110734+0000","last_undegraded":"2023-10-09T05:42:15.110734+0000","last_fullsized":"2023-10-09T05:42:15.110734+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":170,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.895814+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:42:04.895814+0000","last_peered":"2023-10-09T05:42:04.895814+0000","last_clean":"2023-10-09T05:42:04.895814+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:42:04.895814+0000","last_undegraded":"2023-10-09T05:42:04.895814+0000","last_fullsized":"2023-10-09T05:42:04.895814+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":201,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.894107+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:42:04.894107+0000","last_peered":"2023-10-09T05:42:04.894107+0000","last_clean":"2023-10-09T05:42:04.894107+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:42:04.894107+0000","last_undegraded":"2023-10-09T05:42:04.894107+0000","last_fullsized":"2023-10-09T05:42:04.894107+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":213,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.104964+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:42:15.104964+0000","last_peered":"2023-10-09T05:42:15.104964+0000","last_clean":"2023-10-09T05:42:15.104964+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:42:15.104964+0000","last_undegraded":"2023-10-09T05:42:15.104964+0000","last_fullsized":"2023-10-09T05:42:15.104964+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":190,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.091935+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:42:15.091935+0000","last_peered":"2023-10-09T05:42:15.091935+0000","last_clean":"2023-10-09T05:42:15.091935+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:42:15.091935+0000","last_undegraded":"2023-10-09T05:42:15.091935+0000","last_fullsized":"2023-10-09T05:42:15.091935+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":176,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.894569+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:42:04.894569+0000","last_peered":"2023-10-09T05:42:04.894569+0000","last_clean":"2023-10-09T05:42:04.894569+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:42:04.894569+0000","last_undegraded":"2023-10-09T05:42:04.894569+0000","last_fullsized":"2023-10-09T05:42:04.894569+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":197,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.104998+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:42:15.104998+0000","last_peered":"2023-10-09T05:42:15.104998+0000","last_clean":"2023-10-09T05:42:15.104998+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:42:15.104998+0000","last_undegraded":"2023-10-09T05:42:15.104998+0000","last_fullsized":"2023-10-09T05:42:15.104998+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":216,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.084659+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:42:15.084659+0000","last_peered":"2023-10-09T05:42:15.084659+0000","last_clean":"2023-10-09T05:42:15.084659+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:42:15.084659+0000","last_undegraded":"2023-10-09T05:42:15.084659+0000","last_fullsized":"2023-10-09T05:42:15.084659+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":211,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.104362+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:42:15.104362+0000","last_peered":"2023-10-09T05:42:15.104362+0000","last_clean":"2023-10-09T05:42:15.104362+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:42:15.104362+0000","last_undegraded":"2023-10-09T05:42:15.104362+0000","last_fullsized":"2023-10-09T05:42:15.104362+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":105,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:24.892360+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:42:24.892360+0000","last_peered":"2023-10-09T05:42:24.892360+0000","last_clean":"2023-10-09T05:42:24.892360+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:42:24.892360+0000","last_undegraded":"2023-10-09T05:42:24.892360+0000","last_fullsized":"2023-10-09T05:42:24.892360+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":171,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:09.014724+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:42:09.014724+0000","last_peered":"2023-10-09T05:42:09.014724+0000","last_clean":"2023-10-09T05:42:09.014724+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:42:09.014724+0000","last_undegraded":"2023-10-09T05:42:09.014724+0000","last_fullsized":"2023-10-09T05:42:09.014724+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":232,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.075890+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:42:15.075890+0000","last_peered":"2023-10-09T05:42:15.075890+0000","last_clean":"2023-10-09T05:42:15.075890+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:42:15.075890+0000","last_undegraded":"2023-10-09T05:42:15.075890+0000","last_fullsized":"2023-10-09T05:42:15.075890+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":190,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.075804+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:42:15.075804+0000","last_peered":"2023-10-09T05:42:15.075804+0000","last_clean":"2023-10-09T05:42:15.075804+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:42:15.075804+0000","last_undegraded":"2023-10-09T05:42:15.075804+0000","last_fullsized":"2023-10-09T05:42:15.075804+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":103,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.075891+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:42:15.075891+0000","last_peered":"2023-10-09T05:42:15.075891+0000","last_clean":"2023-10-09T05:42:15.075891+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:42:15.075891+0000","last_undegraded":"2023-10-09T05:42:15.075891+0000","last_fullsized":"2023-10-09T05:42:15.075891+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":221,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.110933+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:42:15.110933+0000","last_peered":"2023-10-09T05:42:15.110933+0000","last_clean":"2023-10-09T05:42:15.110933+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:42:15.110933+0000","last_undegraded":"2023-10-09T05:42:15.110933+0000","last_fullsized":"2023-10-09T05:42:15.110933+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":200,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.893253+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:42:04.893253+0000","last_peered":"2023-10-09T05:42:04.893253+0000","last_clean":"2023-10-09T05:42:04.893253+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:42:04.893253+0000","last_undegraded":"2023-10-09T05:42:04.893253+0000","last_fullsized":"2023-10-09T05:42:04.893253+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":215,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.084512+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:42:15.084512+0000","last_peered":"2023-10-09T05:42:15.084512+0000","last_clean":"2023-10-09T05:42:15.084512+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:42:15.084512+0000","last_undegraded":"2023-10-09T05:42:15.084512+0000","last_fullsized":"2023-10-09T05:42:15.084512+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":121,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.076353+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:42:15.076353+0000","last_peered":"2023-10-09T05:42:15.076353+0000","last_clean":"2023-10-09T05:42:15.076353+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:42:15.076353+0000","last_undegraded":"2023-10-09T05:42:15.076353+0000","last_fullsized":"2023-10-09T05:42:15.076353+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":200,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.894267+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:42:04.894267+0000","last_peered":"2023-10-09T05:42:04.894267+0000","last_clean":"2023-10-09T05:42:04.894267+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:42:04.894267+0000","last_undegraded":"2023-10-09T05:42:04.894267+0000","last_fullsized":"2023-10-09T05:42:04.894267+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":199,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.091563+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:42:15.091563+0000","last_peered":"2023-10-09T05:42:15.091563+0000","last_clean":"2023-10-09T05:42:15.091563+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:42:15.091563+0000","last_undegraded":"2023-10-09T05:42:15.091563+0000","last_fullsized":"2023-10-09T05:42:15.091563+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":221,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.111668+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:42:15.111668+0000","last_peered":"2023-10-09T05:42:15.111668+0000","last_clean":"2023-10-09T05:42:15.111668+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:42:15.111668+0000","last_undegraded":"2023-10-09T05:42:15.111668+0000","last_fullsized":"2023-10-09T05:42:15.111668+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"224'3724","reported_seq":3743,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:24.005497+0000","last_change":"2023-10-09T05:41:29.078278+0000","last_active":"2023-10-09T05:42:24.005497+0000","last_peered":"2023-10-09T05:42:24.005497+0000","last_clean":"2023-10-09T05:42:24.005497+0000","last_became_active":"2023-10-09T05:41:29.077562+0000","last_became_peered":"2023-10-09T05:41:29.077562+0000","last_unstale":"2023-10-09T05:42:24.005497+0000","last_undegraded":"2023-10-09T05:42:24.005497+0000","last_fullsized":"2023-10-09T05:42:24.005497+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3724,"log_dups_size":0,"ondisk_log_size":3724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:29.784837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244056064,"num_objects":3724,"num_object_clones":0,"num_object_copies":11172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3724,"num_write_kb":238336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":101,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.076173+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:42:15.076173+0000","last_peered":"2023-10-09T05:42:15.076173+0000","last_clean":"2023-10-09T05:42:15.076173+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:42:15.076173+0000","last_undegraded":"2023-10-09T05:42:15.076173+0000","last_fullsized":"2023-10-09T05:42:15.076173+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":200,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.084143+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:42:15.084143+0000","last_peered":"2023-10-09T05:42:15.084143+0000","last_clean":"2023-10-09T05:42:15.084143+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:42:15.084143+0000","last_undegraded":"2023-10-09T05:42:15.084143+0000","last_fullsized":"2023-10-09T05:42:15.084143+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"225'3870","reported_seq":3890,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:26.528795+0000","last_change":"2023-10-09T05:41:29.081067+0000","last_active":"2023-10-09T05:42:26.528795+0000","last_peered":"2023-10-09T05:42:26.528795+0000","last_clean":"2023-10-09T05:42:26.528795+0000","last_became_active":"2023-10-09T05:41:29.080704+0000","last_became_peered":"2023-10-09T05:41:29.080704+0000","last_unstale":"2023-10-09T05:42:26.528795+0000","last_undegraded":"2023-10-09T05:42:26.528795+0000","last_fullsized":"2023-10-09T05:42:26.528795+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3870,"log_dups_size":0,"ondisk_log_size":3870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:29.483205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253624320,"num_objects":3870,"num_object_clones":0,"num_object_copies":11610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3870,"num_write_kb":247680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":101,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.076137+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:42:15.076137+0000","last_peered":"2023-10-09T05:42:15.076137+0000","last_clean":"2023-10-09T05:42:15.076137+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:42:15.076137+0000","last_undegraded":"2023-10-09T05:42:15.076137+0000","last_fullsized":"2023-10-09T05:42:15.076137+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":214,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.076285+0000","last_change":"2023-10-09T05:41:44.881327+0000","last_active":"2023-10-09T05:42:15.076285+0000","last_peered":"2023-10-09T05:42:15.076285+0000","last_clean":"2023-10-09T05:42:15.076285+0000","last_became_active":"2023-10-09T05:41:44.881044+0000","last_became_peered":"2023-10-09T05:41:44.881044+0000","last_unstale":"2023-10-09T05:42:15.076285+0000","last_undegraded":"2023-10-09T05:42:15.076285+0000","last_fullsized":"2023-10-09T05:42:15.076285+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"224'3715","reported_seq":3734,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:23.247776+0000","last_change":"2023-10-09T05:41:29.077986+0000","last_active":"2023-10-09T05:42:23.247776+0000","last_peered":"2023-10-09T05:42:23.247776+0000","last_clean":"2023-10-09T05:42:23.247776+0000","last_became_active":"2023-10-09T05:41:29.077631+0000","last_became_peered":"2023-10-09T05:41:29.077631+0000","last_unstale":"2023-10-09T05:42:23.247776+0000","last_undegraded":"2023-10-09T05:42:23.247776+0000","last_fullsized":"2023-10-09T05:42:23.247776+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3715,"log_dups_size":0,"ondisk_log_size":3715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:48.684759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243466240,"num_objects":3715,"num_object_clones":0,"num_object_copies":11145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3715,"num_write_kb":237760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":102,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.084596+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:42:15.084596+0000","last_peered":"2023-10-09T05:42:15.084596+0000","last_clean":"2023-10-09T05:42:15.084596+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:42:15.084596+0000","last_undegraded":"2023-10-09T05:42:15.084596+0000","last_fullsized":"2023-10-09T05:42:15.084596+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":223,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.092022+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:42:15.092022+0000","last_peered":"2023-10-09T05:42:15.092022+0000","last_clean":"2023-10-09T05:42:15.092022+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:42:15.092022+0000","last_undegraded":"2023-10-09T05:42:15.092022+0000","last_fullsized":"2023-10-09T05:42:15.092022+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"225'3983","reported_seq":4004,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:26.842398+0000","last_change":"2023-10-09T05:41:29.073764+0000","last_active":"2023-10-09T05:42:26.842398+0000","last_peered":"2023-10-09T05:42:26.842398+0000","last_clean":"2023-10-09T05:42:26.842398+0000","last_became_active":"2023-10-09T05:41:29.073452+0000","last_became_peered":"2023-10-09T05:41:29.073452+0000","last_unstale":"2023-10-09T05:42:26.842398+0000","last_undegraded":"2023-10-09T05:42:26.842398+0000","last_fullsized":"2023-10-09T05:42:26.842398+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3983,"log_dups_size":0,"ondisk_log_size":3983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:22.328530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261029888,"num_objects":3983,"num_object_clones":0,"num_object_copies":11949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3983,"num_write_kb":254912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":192,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.084356+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:42:15.084356+0000","last_peered":"2023-10-09T05:42:15.084356+0000","last_clean":"2023-10-09T05:42:15.084356+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:42:15.084356+0000","last_undegraded":"2023-10-09T05:42:15.084356+0000","last_fullsized":"2023-10-09T05:42:15.084356+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":176,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.893343+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:42:04.893343+0000","last_peered":"2023-10-09T05:42:04.893343+0000","last_clean":"2023-10-09T05:42:04.893343+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:42:04.893343+0000","last_undegraded":"2023-10-09T05:42:04.893343+0000","last_fullsized":"2023-10-09T05:42:04.893343+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"224'3875","reported_seq":3894,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:23.271984+0000","last_change":"2023-10-09T05:41:29.082598+0000","last_active":"2023-10-09T05:42:23.271984+0000","last_peered":"2023-10-09T05:42:23.271984+0000","last_clean":"2023-10-09T05:42:23.271984+0000","last_became_active":"2023-10-09T05:41:29.082285+0000","last_became_peered":"2023-10-09T05:41:29.082285+0000","last_unstale":"2023-10-09T05:42:23.271984+0000","last_undegraded":"2023-10-09T05:42:23.271984+0000","last_fullsized":"2023-10-09T05:42:23.271984+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3875,"log_dups_size":0,"ondisk_log_size":3875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:18.625214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253952000,"num_objects":3875,"num_object_clones":0,"num_object_copies":11625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3875,"num_write_kb":248000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":195,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.110574+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:42:15.110574+0000","last_peered":"2023-10-09T05:42:15.110574+0000","last_clean":"2023-10-09T05:42:15.110574+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:42:15.110574+0000","last_undegraded":"2023-10-09T05:42:15.110574+0000","last_fullsized":"2023-10-09T05:42:15.110574+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":190,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.075785+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:42:15.075785+0000","last_peered":"2023-10-09T05:42:15.075785+0000","last_clean":"2023-10-09T05:42:15.075785+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:42:15.075785+0000","last_undegraded":"2023-10-09T05:42:15.075785+0000","last_fullsized":"2023-10-09T05:42:15.075785+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"225'4079","reported_seq":4100,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:26.792113+0000","last_change":"2023-10-09T05:41:29.078791+0000","last_active":"2023-10-09T05:42:26.792113+0000","last_peered":"2023-10-09T05:42:26.792113+0000","last_clean":"2023-10-09T05:42:26.792113+0000","last_became_active":"2023-10-09T05:41:29.078491+0000","last_became_peered":"2023-10-09T05:41:29.078491+0000","last_unstale":"2023-10-09T05:42:26.792113+0000","last_undegraded":"2023-10-09T05:42:26.792113+0000","last_fullsized":"2023-10-09T05:42:26.792113+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":4079,"log_dups_size":0,"ondisk_log_size":4079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:37.817282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267321344,"num_objects":4079,"num_object_clones":0,"num_object_copies":12237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4079,"num_write_kb":261056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":191,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.091632+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:42:15.091632+0000","last_peered":"2023-10-09T05:42:15.091632+0000","last_clean":"2023-10-09T05:42:15.091632+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:42:15.091632+0000","last_undegraded":"2023-10-09T05:42:15.091632+0000","last_fullsized":"2023-10-09T05:42:15.091632+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":205,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.075930+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:42:15.075930+0000","last_peered":"2023-10-09T05:42:15.075930+0000","last_clean":"2023-10-09T05:42:15.075930+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:42:15.075930+0000","last_undegraded":"2023-10-09T05:42:15.075930+0000","last_fullsized":"2023-10-09T05:42:15.075930+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"225'3811","reported_seq":3830,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:25.702341+0000","last_change":"2023-10-09T05:41:29.078645+0000","last_active":"2023-10-09T05:42:25.702341+0000","last_peered":"2023-10-09T05:42:25.702341+0000","last_clean":"2023-10-09T05:42:25.702341+0000","last_became_active":"2023-10-09T05:41:29.078355+0000","last_became_peered":"2023-10-09T05:41:29.078355+0000","last_unstale":"2023-10-09T05:42:25.702341+0000","last_undegraded":"2023-10-09T05:42:25.702341+0000","last_fullsized":"2023-10-09T05:42:25.702341+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3811,"log_dups_size":0,"ondisk_log_size":3811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249757696,"num_objects":3811,"num_object_clones":0,"num_object_copies":11433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3811,"num_write_kb":243904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":215,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.076097+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:42:15.076097+0000","last_peered":"2023-10-09T05:42:15.076097+0000","last_clean":"2023-10-09T05:42:15.076097+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:42:15.076097+0000","last_undegraded":"2023-10-09T05:42:15.076097+0000","last_fullsized":"2023-10-09T05:42:15.076097+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":199,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.076214+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:42:15.076214+0000","last_peered":"2023-10-09T05:42:15.076214+0000","last_clean":"2023-10-09T05:42:15.076214+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:42:15.076214+0000","last_undegraded":"2023-10-09T05:42:15.076214+0000","last_fullsized":"2023-10-09T05:42:15.076214+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"225'3852","reported_seq":3870,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:25.824802+0000","last_change":"2023-10-09T05:41:29.082194+0000","last_active":"2023-10-09T05:42:25.824802+0000","last_peered":"2023-10-09T05:42:25.824802+0000","last_clean":"2023-10-09T05:42:25.824802+0000","last_became_active":"2023-10-09T05:41:29.081976+0000","last_became_peered":"2023-10-09T05:41:29.081976+0000","last_unstale":"2023-10-09T05:42:25.824802+0000","last_undegraded":"2023-10-09T05:42:25.824802+0000","last_fullsized":"2023-10-09T05:42:25.824802+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3852,"log_dups_size":0,"ondisk_log_size":3852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252444672,"num_objects":3852,"num_object_clones":0,"num_object_copies":11556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3852,"num_write_kb":246528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":223,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:21.554499+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:42:21.554499+0000","last_peered":"2023-10-09T05:42:21.554499+0000","last_clean":"2023-10-09T05:42:21.554499+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:42:21.554499+0000","last_undegraded":"2023-10-09T05:42:21.554499+0000","last_fullsized":"2023-10-09T05:42:21.554499+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":171,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.895342+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:42:04.895342+0000","last_peered":"2023-10-09T05:42:04.895342+0000","last_clean":"2023-10-09T05:42:04.895342+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:42:04.895342+0000","last_undegraded":"2023-10-09T05:42:04.895342+0000","last_fullsized":"2023-10-09T05:42:04.895342+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"225'3856","reported_seq":3875,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:24.991427+0000","last_change":"2023-10-09T05:41:29.079588+0000","last_active":"2023-10-09T05:42:24.991427+0000","last_peered":"2023-10-09T05:42:24.991427+0000","last_clean":"2023-10-09T05:42:24.991427+0000","last_became_active":"2023-10-09T05:41:29.079288+0000","last_became_peered":"2023-10-09T05:41:29.079288+0000","last_unstale":"2023-10-09T05:42:24.991427+0000","last_undegraded":"2023-10-09T05:42:24.991427+0000","last_fullsized":"2023-10-09T05:42:24.991427+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3856,"log_dups_size":0,"ondisk_log_size":3856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252706816,"num_objects":3856,"num_object_clones":0,"num_object_copies":11568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3856,"num_write_kb":246784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":793,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.895284+0000","last_change":"2023-10-09T05:40:04.806908+0000","last_active":"2023-10-09T05:42:04.895284+0000","last_peered":"2023-10-09T05:42:04.895284+0000","last_clean":"2023-10-09T05:42:04.895284+0000","last_became_active":"2023-10-09T05:40:04.806232+0000","last_became_peered":"2023-10-09T05:40:04.806232+0000","last_unstale":"2023-10-09T05:42:04.895284+0000","last_undegraded":"2023-10-09T05:42:04.895284+0000","last_fullsized":"2023-10-09T05:42:04.895284+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:34.087553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":207,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.084429+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:42:15.084429+0000","last_peered":"2023-10-09T05:42:15.084429+0000","last_clean":"2023-10-09T05:42:15.084429+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:42:15.084429+0000","last_undegraded":"2023-10-09T05:42:15.084429+0000","last_fullsized":"2023-10-09T05:42:15.084429+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":156,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.895977+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:42:04.895977+0000","last_peered":"2023-10-09T05:42:04.895977+0000","last_clean":"2023-10-09T05:42:04.895977+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:42:04.895977+0000","last_undegraded":"2023-10-09T05:42:04.895977+0000","last_fullsized":"2023-10-09T05:42:04.895977+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"225'3861","reported_seq":3880,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:25.751038+0000","last_change":"2023-10-09T05:41:29.082691+0000","last_active":"2023-10-09T05:42:25.751038+0000","last_peered":"2023-10-09T05:42:25.751038+0000","last_clean":"2023-10-09T05:42:25.751038+0000","last_became_active":"2023-10-09T05:41:29.082385+0000","last_became_peered":"2023-10-09T05:41:29.082385+0000","last_unstale":"2023-10-09T05:42:25.751038+0000","last_undegraded":"2023-10-09T05:42:25.751038+0000","last_fullsized":"2023-10-09T05:42:25.751038+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3861,"log_dups_size":0,"ondisk_log_size":3861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253034496,"num_objects":3861,"num_object_clones":0,"num_object_copies":11583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3861,"num_write_kb":247104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":661,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.084454+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:42:15.084454+0000","last_peered":"2023-10-09T05:42:15.084454+0000","last_clean":"2023-10-09T05:42:15.084454+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:42:15.084454+0000","last_undegraded":"2023-10-09T05:42:15.084454+0000","last_fullsized":"2023-10-09T05:42:15.084454+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":191,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.075992+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:42:15.075992+0000","last_peered":"2023-10-09T05:42:15.075992+0000","last_clean":"2023-10-09T05:42:15.075992+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:42:15.075992+0000","last_undegraded":"2023-10-09T05:42:15.075992+0000","last_fullsized":"2023-10-09T05:42:15.075992+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":204,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.084555+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:42:15.084555+0000","last_peered":"2023-10-09T05:42:15.084555+0000","last_clean":"2023-10-09T05:42:15.084555+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:42:15.084555+0000","last_undegraded":"2023-10-09T05:42:15.084555+0000","last_fullsized":"2023-10-09T05:42:15.084555+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"224'3767","reported_seq":3786,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:23.314934+0000","last_change":"2023-10-09T05:41:29.077953+0000","last_active":"2023-10-09T05:42:23.314934+0000","last_peered":"2023-10-09T05:42:23.314934+0000","last_clean":"2023-10-09T05:42:23.314934+0000","last_became_active":"2023-10-09T05:41:29.077652+0000","last_became_peered":"2023-10-09T05:41:29.077652+0000","last_unstale":"2023-10-09T05:42:23.314934+0000","last_undegraded":"2023-10-09T05:42:23.314934+0000","last_fullsized":"2023-10-09T05:42:23.314934+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3767,"log_dups_size":0,"ondisk_log_size":3767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:53.470052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246874112,"num_objects":3767,"num_object_clones":0,"num_object_copies":11301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3767,"num_write_kb":241088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":540,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.087984+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:42:15.087984+0000","last_peered":"2023-10-09T05:42:15.087984+0000","last_clean":"2023-10-09T05:42:15.087984+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:42:15.087984+0000","last_undegraded":"2023-10-09T05:42:15.087984+0000","last_fullsized":"2023-10-09T05:42:15.087984+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":171,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.895747+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:42:04.895747+0000","last_peered":"2023-10-09T05:42:04.895747+0000","last_clean":"2023-10-09T05:42:04.895747+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:42:04.895747+0000","last_undegraded":"2023-10-09T05:42:04.895747+0000","last_fullsized":"2023-10-09T05:42:04.895747+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":201,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.895423+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:42:04.895423+0000","last_peered":"2023-10-09T05:42:04.895423+0000","last_clean":"2023-10-09T05:42:04.895423+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:42:04.895423+0000","last_undegraded":"2023-10-09T05:42:04.895423+0000","last_fullsized":"2023-10-09T05:42:04.895423+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"224'3839","reported_seq":3857,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:23.309998+0000","last_change":"2023-10-09T05:41:29.082532+0000","last_active":"2023-10-09T05:42:23.309998+0000","last_peered":"2023-10-09T05:42:23.309998+0000","last_clean":"2023-10-09T05:42:23.309998+0000","last_became_active":"2023-10-09T05:41:29.082272+0000","last_became_peered":"2023-10-09T05:41:29.082272+0000","last_unstale":"2023-10-09T05:42:23.309998+0000","last_undegraded":"2023-10-09T05:42:23.309998+0000","last_fullsized":"2023-10-09T05:42:23.309998+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3839,"log_dups_size":0,"ondisk_log_size":3839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:51.574989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251592704,"num_objects":3839,"num_object_clones":0,"num_object_copies":11517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3839,"num_write_kb":245696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":836,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.084724+0000","last_change":"2023-10-09T05:40:04.805258+0000","last_active":"2023-10-09T05:42:15.084724+0000","last_peered":"2023-10-09T05:42:15.084724+0000","last_clean":"2023-10-09T05:42:15.084724+0000","last_became_active":"2023-10-09T05:40:04.804776+0000","last_became_peered":"2023-10-09T05:40:04.804776+0000","last_unstale":"2023-10-09T05:42:15.084724+0000","last_undegraded":"2023-10-09T05:42:15.084724+0000","last_fullsized":"2023-10-09T05:42:15.084724+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":372,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.893092+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:42:04.893092+0000","last_peered":"2023-10-09T05:42:04.893092+0000","last_clean":"2023-10-09T05:42:04.893092+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:42:04.893092+0000","last_undegraded":"2023-10-09T05:42:04.893092+0000","last_fullsized":"2023-10-09T05:42:04.893092+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":189,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.110534+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:42:15.110534+0000","last_peered":"2023-10-09T05:42:15.110534+0000","last_clean":"2023-10-09T05:42:15.110534+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:42:15.110534+0000","last_undegraded":"2023-10-09T05:42:15.110534+0000","last_fullsized":"2023-10-09T05:42:15.110534+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":197,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.092116+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:42:15.092116+0000","last_peered":"2023-10-09T05:42:15.092116+0000","last_clean":"2023-10-09T05:42:15.092116+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:42:15.092116+0000","last_undegraded":"2023-10-09T05:42:15.092116+0000","last_fullsized":"2023-10-09T05:42:15.092116+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"225'4027","reported_seq":4048,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:26.841961+0000","last_change":"2023-10-09T05:41:29.076367+0000","last_active":"2023-10-09T05:42:26.841961+0000","last_peered":"2023-10-09T05:42:26.841961+0000","last_clean":"2023-10-09T05:42:26.841961+0000","last_became_active":"2023-10-09T05:41:29.076035+0000","last_became_peered":"2023-10-09T05:41:29.076035+0000","last_unstale":"2023-10-09T05:42:26.841961+0000","last_undegraded":"2023-10-09T05:42:26.841961+0000","last_fullsized":"2023-10-09T05:42:26.841961+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":4027,"log_dups_size":0,"ondisk_log_size":4027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:51.713835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263913472,"num_objects":4027,"num_object_clones":0,"num_object_copies":12081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4027,"num_write_kb":257728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":781,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.895880+0000","last_change":"2023-10-09T05:40:05.930112+0000","last_active":"2023-10-09T05:42:04.895880+0000","last_peered":"2023-10-09T05:42:04.895880+0000","last_clean":"2023-10-09T05:42:04.895880+0000","last_became_active":"2023-10-09T05:40:05.929574+0000","last_became_peered":"2023-10-09T05:40:05.929574+0000","last_unstale":"2023-10-09T05:42:04.895880+0000","last_undegraded":"2023-10-09T05:42:04.895880+0000","last_fullsized":"2023-10-09T05:42:04.895880+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:06.982326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":176,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.893436+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:42:04.893436+0000","last_peered":"2023-10-09T05:42:04.893436+0000","last_clean":"2023-10-09T05:42:04.893436+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:42:04.893436+0000","last_undegraded":"2023-10-09T05:42:04.893436+0000","last_fullsized":"2023-10-09T05:42:04.893436+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":191,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.893962+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:42:04.893962+0000","last_peered":"2023-10-09T05:42:04.893962+0000","last_clean":"2023-10-09T05:42:04.893962+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:42:04.893962+0000","last_undegraded":"2023-10-09T05:42:04.893962+0000","last_fullsized":"2023-10-09T05:42:04.893962+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"225'3862","reported_seq":3882,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:25.021905+0000","last_change":"2023-10-09T05:41:29.073701+0000","last_active":"2023-10-09T05:42:25.021905+0000","last_peered":"2023-10-09T05:42:25.021905+0000","last_clean":"2023-10-09T05:42:25.021905+0000","last_became_active":"2023-10-09T05:41:29.073372+0000","last_became_peered":"2023-10-09T05:41:29.073372+0000","last_unstale":"2023-10-09T05:42:25.021905+0000","last_undegraded":"2023-10-09T05:42:25.021905+0000","last_fullsized":"2023-10-09T05:42:25.021905+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3862,"log_dups_size":0,"ondisk_log_size":3862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253100032,"num_objects":3862,"num_object_clones":0,"num_object_copies":11586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3862,"num_write_kb":247168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":413,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.894349+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:42:04.894349+0000","last_peered":"2023-10-09T05:42:04.894349+0000","last_clean":"2023-10-09T05:42:04.894349+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:42:04.894349+0000","last_undegraded":"2023-10-09T05:42:04.894349+0000","last_fullsized":"2023-10-09T05:42:04.894349+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":171,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.895479+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:42:04.895479+0000","last_peered":"2023-10-09T05:42:04.895479+0000","last_clean":"2023-10-09T05:42:04.895479+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:42:04.895479+0000","last_undegraded":"2023-10-09T05:42:04.895479+0000","last_fullsized":"2023-10-09T05:42:04.895479+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":185,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.893565+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:42:04.893565+0000","last_peered":"2023-10-09T05:42:04.893565+0000","last_clean":"2023-10-09T05:42:04.893565+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:42:04.893565+0000","last_undegraded":"2023-10-09T05:42:04.893565+0000","last_fullsized":"2023-10-09T05:42:04.893565+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"225'3989","reported_seq":4009,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:26.501669+0000","last_change":"2023-10-09T05:41:29.080413+0000","last_active":"2023-10-09T05:42:26.501669+0000","last_peered":"2023-10-09T05:42:26.501669+0000","last_clean":"2023-10-09T05:42:26.501669+0000","last_became_active":"2023-10-09T05:41:29.080184+0000","last_became_peered":"2023-10-09T05:41:29.080184+0000","last_unstale":"2023-10-09T05:42:26.501669+0000","last_undegraded":"2023-10-09T05:42:26.501669+0000","last_fullsized":"2023-10-09T05:42:26.501669+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":3989,"log_dups_size":0,"ondisk_log_size":3989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:47.929696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261423104,"num_objects":3989,"num_object_clones":0,"num_object_copies":11967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3989,"num_write_kb":255296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":224,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:21.556862+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:42:21.556862+0000","last_peered":"2023-10-09T05:42:21.556862+0000","last_clean":"2023-10-09T05:42:21.556862+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:42:21.556862+0000","last_undegraded":"2023-10-09T05:42:21.556862+0000","last_fullsized":"2023-10-09T05:42:21.556862+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":612,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.895214+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:42:04.895214+0000","last_peered":"2023-10-09T05:42:04.895214+0000","last_clean":"2023-10-09T05:42:04.895214+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:42:04.895214+0000","last_undegraded":"2023-10-09T05:42:04.895214+0000","last_fullsized":"2023-10-09T05:42:04.895214+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":217,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.105030+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:42:15.105030+0000","last_peered":"2023-10-09T05:42:15.105030+0000","last_clean":"2023-10-09T05:42:15.105030+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:42:15.105030+0000","last_undegraded":"2023-10-09T05:42:15.105030+0000","last_fullsized":"2023-10-09T05:42:15.105030+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"225'4090","reported_seq":4110,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:42:26.518094+0000","last_change":"2023-10-09T05:41:29.081230+0000","last_active":"2023-10-09T05:42:26.518094+0000","last_peered":"2023-10-09T05:42:26.518094+0000","last_clean":"2023-10-09T05:42:26.518094+0000","last_became_active":"2023-10-09T05:41:29.080911+0000","last_became_peered":"2023-10-09T05:41:29.080911+0000","last_unstale":"2023-10-09T05:42:26.518094+0000","last_undegraded":"2023-10-09T05:42:26.518094+0000","last_fullsized":"2023-10-09T05:42:26.518094+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":4090,"log_dups_size":0,"ondisk_log_size":4090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:22.580967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268042240,"num_objects":4090,"num_object_clones":0,"num_object_copies":12270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4090,"num_write_kb":261760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":725,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.091860+0000","last_change":"2023-10-09T05:41:21.917602+0000","last_active":"2023-10-09T05:42:15.091860+0000","last_peered":"2023-10-09T05:42:15.091860+0000","last_clean":"2023-10-09T05:42:15.091860+0000","last_became_active":"2023-10-09T05:41:21.917116+0000","last_became_peered":"2023-10-09T05:41:21.917116+0000","last_unstale":"2023-10-09T05:42:15.091860+0000","last_undegraded":"2023-10-09T05:42:15.091860+0000","last_fullsized":"2023-10-09T05:42:15.091860+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":179,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.894286+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:42:04.894286+0000","last_peered":"2023-10-09T05:42:04.894286+0000","last_clean":"2023-10-09T05:42:04.894286+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:42:04.894286+0000","last_undegraded":"2023-10-09T05:42:04.894286+0000","last_fullsized":"2023-10-09T05:42:04.894286+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":175,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.076166+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:42:15.076166+0000","last_peered":"2023-10-09T05:42:15.076166+0000","last_clean":"2023-10-09T05:42:15.076166+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:42:15.076166+0000","last_undegraded":"2023-10-09T05:42:15.076166+0000","last_fullsized":"2023-10-09T05:42:15.076166+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":187,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.893495+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:42:04.893495+0000","last_peered":"2023-10-09T05:42:04.893495+0000","last_clean":"2023-10-09T05:42:04.893495+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:42:04.893495+0000","last_undegraded":"2023-10-09T05:42:04.893495+0000","last_fullsized":"2023-10-09T05:42:04.893495+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":226,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.089377+0000","last_change":"2023-10-09T05:41:44.887519+0000","last_active":"2023-10-09T05:42:15.089377+0000","last_peered":"2023-10-09T05:42:15.089377+0000","last_clean":"2023-10-09T05:42:15.089377+0000","last_became_active":"2023-10-09T05:41:44.887047+0000","last_became_peered":"2023-10-09T05:41:44.887047+0000","last_unstale":"2023-10-09T05:42:15.089377+0000","last_undegraded":"2023-10-09T05:42:15.089377+0000","last_fullsized":"2023-10-09T05:42:15.089377+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:49.475913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":194,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.084274+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:42:15.084274+0000","last_peered":"2023-10-09T05:42:15.084274+0000","last_clean":"2023-10-09T05:42:15.084274+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:42:15.084274+0000","last_undegraded":"2023-10-09T05:42:15.084274+0000","last_fullsized":"2023-10-09T05:42:15.084274+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":205,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.076067+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:42:15.076067+0000","last_peered":"2023-10-09T05:42:15.076067+0000","last_clean":"2023-10-09T05:42:15.076067+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:42:15.076067+0000","last_undegraded":"2023-10-09T05:42:15.076067+0000","last_fullsized":"2023-10-09T05:42:15.076067+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":179,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.084346+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:42:15.084346+0000","last_peered":"2023-10-09T05:42:15.084346+0000","last_clean":"2023-10-09T05:42:15.084346+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:42:15.084346+0000","last_undegraded":"2023-10-09T05:42:15.084346+0000","last_fullsized":"2023-10-09T05:42:15.084346+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":205,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.075551+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:42:15.075551+0000","last_peered":"2023-10-09T05:42:15.075551+0000","last_clean":"2023-10-09T05:42:15.075551+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:42:15.075551+0000","last_undegraded":"2023-10-09T05:42:15.075551+0000","last_fullsized":"2023-10-09T05:42:15.075551+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":204,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.111684+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:42:15.111684+0000","last_peered":"2023-10-09T05:42:15.111684+0000","last_clean":"2023-10-09T05:42:15.111684+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:42:15.111684+0000","last_undegraded":"2023-10-09T05:42:15.111684+0000","last_fullsized":"2023-10-09T05:42:15.111684+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":216,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.104284+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:42:15.104284+0000","last_peered":"2023-10-09T05:42:15.104284+0000","last_clean":"2023-10-09T05:42:15.104284+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:42:15.104284+0000","last_undegraded":"2023-10-09T05:42:15.104284+0000","last_fullsized":"2023-10-09T05:42:15.104284+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":179,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:42:04.894648+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:42:04.894648+0000","last_peered":"2023-10-09T05:42:04.894648+0000","last_clean":"2023-10-09T05:42:04.894648+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:42:04.894648+0000","last_undegraded":"2023-10-09T05:42:04.894648+0000","last_fullsized":"2023-10-09T05:42:04.894648+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":190,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T05:42:15.075535+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:42:15.075535+0000","last_peered":"2023-10-09T05:42:15.075535+0000","last_clean":"2023-10-09T05:42:15.075535+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:42:15.075535+0000","last_undegraded":"2023-10-09T05:42:15.075535+0000","last_fullsized":"2023-10-09T05:42:15.075535+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4076339200,"num_objects":62200,"num_object_clones":0,"num_object_copies":186600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62200,"num_write_kb":3980800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12217810944,"data_stored":12217810944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":62200,"ondisk_log_size":62200,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846686,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626712,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1541880,"kb_used_data":1232368,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92203272,"statfs":{"total":95995035648,"available":94416150528,"internally_reserved":0,"allocated":1261944832,"data_stored":1261410464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:28 2023","interfaces":[{"interface":"back","average":{"1min":2.032,"5min":5.6829999999999998,"15min":3.2210000000000001},"min":{"1min":0.23100000000000001,"5min":0.214,"15min":0.214},"max":{"1min":14.23,"5min":115.377,"15min":115.377},"last":3.3330000000000002},{"interface":"front","average":{"1min":1.9350000000000001,"5min":5.3710000000000004,"15min":3.0800000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23999999999999999},"max":{"1min":11.727,"5min":116.428,"15min":116.428},"last":0.34000000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:41:51 2023","interfaces":[{"interface":"back","average":{"1min":11.552,"5min":7.8680000000000003,"15min":3.4500000000000002},"min":{"1min":0.28799999999999998,"5min":0.25,"15min":0.25},"max":{"1min":228.24299999999999,"5min":228.24299999999999,"15min":228.24299999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":11.526999999999999,"5min":7.6799999999999997,"15min":3.488},"min":{"1min":0.29699999999999999,"5min":0.252,"15min":0.252},"max":{"1min":228.22499999999999,"5min":228.22499999999999,"15min":228.22499999999999},"last":0.40200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:41:28 2023","interfaces":[{"interface":"back","average":{"1min":7.8099999999999996,"5min":6.226,"15min":3.456},"min":{"1min":0.32200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":87.703000000000003,"5min":115.40600000000001,"15min":115.40600000000001},"last":3.3149999999999999},{"interface":"front","average":{"1min":8.4559999999999995,"5min":6.5220000000000002,"15min":3.4980000000000002},"min":{"1min":0.35499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":87.689999999999998,"5min":115.297,"15min":115.297},"last":1.9390000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":6.6369999999999996,"15min":3.161},"min":{"1min":0.32000000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.7089999999999996,"5min":115.244,"15min":115.244},"last":2.6709999999999998},{"interface":"front","average":{"1min":0.80200000000000005,"5min":6.2409999999999997,"15min":3.129},"min":{"1min":0.372,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.0089999999999999,"5min":115.34999999999999,"15min":115.34999999999999},"last":3.278}]},{"osd":4,"last update":"Mon Oct 9 05:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":1.361,"15min":1.7030000000000001},"min":{"1min":0.26100000000000001,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":0.501,"5min":115.214,"15min":125.64},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.44700000000000001,"5min":1.4810000000000001,"15min":1.609},"min":{"1min":0.26000000000000001,"5min":0.254,"15min":0.254},"max":{"1min":0.88500000000000001,"5min":115.22799999999999,"15min":125.625},"last":0.39200000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:42:04 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":2.0979999999999999,"15min":1.851},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.60899999999999999,"5min":115.316,"15min":127.35899999999999},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.50600000000000001,"5min":2.0510000000000002,"15min":1.7370000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.228,"5min":115.30500000000001,"15min":128.93799999999999},"last":0.372}]},{"osd":7,"last update":"Mon Oct 9 05:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.6850000000000001,"15min":2.6360000000000001},"min":{"1min":0.33900000000000002,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":7.319,"5min":115.514,"15min":115.514},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.77000000000000002,"5min":1.716,"15min":1.9139999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.641,"5min":116.53100000000001,"15min":116.53100000000001},"last":0.55600000000000005}]}]},{"osd":1,"up_from":93,"seq":399431958673,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2045972,"kb_used_data":1689740,"kb_used_omap":7,"kb_used_meta":356216,"kb_avail":91699180,"statfs":{"total":95995035648,"available":93899960320,"internally_reserved":0,"allocated":1730293760,"data_stored":1729494400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":364765710},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:53 2023","interfaces":[{"interface":"back","average":{"1min":2.9420000000000002,"5min":5.2800000000000002,"15min":2.7770000000000001},"min":{"1min":0.22500000000000001,"5min":0.20300000000000001,"15min":0.185},"max":{"1min":42.034999999999997,"5min":207.858,"15min":207.858},"last":48.176000000000002},{"interface":"front","average":{"1min":3.375,"5min":6.4429999999999996,"15min":3.27},"min":{"1min":0.30399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":42.049999999999997,"5min":206.97200000000001,"15min":206.97200000000001},"last":48.119999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":5.4800000000000004,"15min":3.532},"min":{"1min":0.247,"5min":0.22,"15min":0.184},"max":{"1min":4.3819999999999997,"5min":101.367,"15min":101.367},"last":0.374},{"interface":"front","average":{"1min":0.90400000000000003,"5min":5.1459999999999999,"15min":3.3260000000000001},"min":{"1min":0.26600000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.3079999999999998,"5min":97.286000000000001,"15min":98.409999999999997},"last":0.34699999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:41:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":7.141,"15min":3.0019999999999998},"min":{"1min":0.28399999999999997,"5min":0.24199999999999999,"15min":0.223},"max":{"1min":7.8319999999999999,"5min":162.33699999999999,"15min":162.33699999999999},"last":45.512},{"interface":"front","average":{"1min":1.167,"5min":7.9210000000000003,"15min":3.2469999999999999},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":7.734,"5min":162.376,"15min":162.376},"last":45.137}]},{"osd":4,"last update":"Mon Oct 9 05:41:53 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":4.3860000000000001,"15min":2.2799999999999998},"min":{"1min":0.36799999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.9610000000000003,"5min":141.32900000000001,"15min":141.32900000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.81200000000000006,"5min":3.452,"15min":1.8500000000000001},"min":{"1min":0.35399999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.492,"5min":141.15899999999999,"15min":141.15899999999999},"last":0.77700000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:41:53 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":4.3979999999999997,"15min":2.2570000000000001},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.307},"max":{"1min":4.8730000000000002,"5min":235.18700000000001,"15min":235.18700000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.89400000000000002,"5min":5.2750000000000004,"15min":2.6389999999999998},"min":{"1min":0.38700000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.9050000000000002,"5min":235.79300000000001,"15min":235.79300000000001},"last":0.81299999999999994}]},{"osd":6,"last update":"Mon Oct 9 05:41:53 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":3.5310000000000001,"15min":1.8979999999999999},"min":{"1min":0.38200000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.9409999999999998,"5min":103.60899999999999,"15min":103.60899999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.95099999999999996,"5min":4.2039999999999997,"15min":2.1579999999999999},"min":{"1min":0.378,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.9340000000000002,"5min":103.679,"15min":103.679},"last":0.78800000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:41:53 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":3.4969999999999999,"15min":2.4740000000000002},"min":{"1min":0.41399999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.4329999999999998,"5min":126.68300000000001,"15min":126.68300000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.147,"5min":3.5449999999999999,"15min":2.5720000000000001},"min":{"1min":0.44,"5min":0.38700000000000001,"15min":0.34399999999999997},"max":{"1min":8.0039999999999996,"5min":126.738,"15min":126.738},"last":0.80000000000000004}]}]},{"osd":0,"up_from":88,"seq":377957122199,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2307440,"kb_used_data":1915304,"kb_used_omap":8,"kb_used_meta":392119,"kb_avail":91437712,"statfs":{"total":95995035648,"available":93632217088,"internally_reserved":0,"allocated":1961271296,"data_stored":1960444839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":401530632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":2.0339999999999998,"15min":1.3140000000000001},"min":{"1min":0.20200000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":4.9870000000000001,"5min":17.760000000000002,"15min":17.760000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.97199999999999998,"5min":3.008,"15min":1.6899999999999999},"min":{"1min":0.25900000000000001,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":5.6600000000000001,"5min":26.163,"15min":26.163},"last":0.70199999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:42:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":4.3630000000000004,"15min":2.8380000000000001},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":15.894,"5min":92.230000000000004,"15min":92.230000000000004},"last":1.907},{"interface":"front","average":{"1min":1.6850000000000001,"5min":4.282,"15min":2.7290000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.218},"max":{"1min":17.824000000000002,"5min":92.247,"15min":92.247},"last":0.58499999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:41:52 2023","interfaces":[{"interface":"back","average":{"1min":5.4939999999999998,"5min":7.101,"15min":3.1070000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":79.159000000000006,"5min":123.024,"15min":123.024},"last":1.26},{"interface":"front","average":{"1min":5.423,"5min":5.9640000000000004,"15min":2.8140000000000001},"min":{"1min":0.28499999999999998,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":79.353999999999999,"5min":99.162000000000006,"15min":99.162000000000006},"last":1.9339999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:42:21 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.948,"15min":1.3069999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.2590000000000003,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.6639999999999999,"5min":2.8490000000000002,"15min":1.7430000000000001},"min":{"1min":0.46700000000000003,"5min":0.34300000000000003,"15min":0.30399999999999999},"max":{"1min":7.1920000000000002,"5min":69.102000000000004,"15min":69.102000000000004},"last":0.73799999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:42:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":6.484,"15min":2.9580000000000002},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":7.1340000000000003,"5min":281.733,"15min":281.733},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.7370000000000001,"5min":6.3979999999999997,"15min":2.9359999999999999},"min":{"1min":0.437,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.0999999999999996,"5min":282.95400000000001,"15min":282.95400000000001},"last":1.8120000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:42:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":2.286,"15min":1.5509999999999999},"min":{"1min":0.35999999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.1680000000000001,"5min":11.664,"15min":11.664},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.7210000000000001,"5min":2.3340000000000001,"15min":1.5680000000000001},"min":{"1min":0.44900000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.0700000000000003,"5min":10.83,"15min":10.83},"last":1.78}]},{"osd":7,"last update":"Mon Oct 9 05:41:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.4710000000000001,"15min":1.425},"min":{"1min":0.39200000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.6159999999999997,"5min":8.9740000000000002,"15min":9.2400000000000002},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.123,"5min":2.2349999999999999,"15min":2.3580000000000001},"min":{"1min":0.42599999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":5.6070000000000002,"5min":9.6329999999999991,"15min":11.068},"last":0.76400000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795147,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2216512,"kb_used_data":1780920,"kb_used_omap":11,"kb_used_meta":435572,"kb_avail":91528640,"statfs":{"total":95995035648,"available":93725327360,"internally_reserved":0,"allocated":1823662080,"data_stored":1822772634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":446026325},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:42:17 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":3.089,"15min":1.8380000000000001},"min":{"1min":0.22500000000000001,"5min":0.17799999999999999,"15min":0.17199999999999999},"max":{"1min":6.2389999999999999,"5min":21.587,"15min":32.122},"last":0.55000000000000004},{"interface":"front","average":{"1min":2.1850000000000001,"5min":3.5990000000000002,"15min":1.8620000000000001},"min":{"1min":0.27000000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.525,"5min":21.997,"15min":34.066000000000003},"last":0.39900000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":5.5309999999999997,"15min":2.7589999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20699999999999999},"max":{"1min":9.0909999999999993,"5min":98.575000000000003,"15min":98.575000000000003},"last":6.4029999999999996},{"interface":"front","average":{"1min":1.454,"5min":4.4219999999999997,"15min":2.3530000000000002},"min":{"1min":0.217,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":8.8510000000000009,"5min":90.135000000000005,"15min":98.397000000000006},"last":4.2549999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:41:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":5.9580000000000002,"15min":2.7280000000000002},"min":{"1min":0.29999999999999999,"5min":0.26700000000000002,"15min":0.24199999999999999},"max":{"1min":13.173,"5min":150.65899999999999,"15min":150.65899999999999},"last":4.3310000000000004},{"interface":"front","average":{"1min":0.97499999999999998,"5min":6.6779999999999999,"15min":3.0539999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.4189999999999996,"5min":150.851,"15min":150.851},"last":0.83599999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":3.1200000000000001,"15min":1.7769999999999999},"min":{"1min":0.311,"5min":0.29099999999999998,"15min":0.27700000000000002},"max":{"1min":8.7769999999999992,"5min":115.84999999999999,"15min":115.84999999999999},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.84699999999999998,"5min":3.2450000000000001,"15min":1.99},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.359,"5min":116.149,"15min":116.149},"last":0.85899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":2.7469999999999999,"15min":1.738},"min":{"1min":0.42099999999999999,"5min":0.312,"15min":0.312},"max":{"1min":8.6639999999999997,"5min":24.597999999999999,"15min":24.597999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.4119999999999999,"5min":2.2250000000000001,"15min":1.52},"min":{"1min":0.372,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.8190000000000008,"5min":23.919,"15min":23.919},"last":0.73099999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":2.29,"15min":1.669},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.3179999999999996,"5min":13.839,"15min":20.503},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.536,"5min":2.4630000000000001,"15min":1.593},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.34499999999999997},"max":{"1min":8.6180000000000003,"5min":13.757,"15min":13.757},"last":0.77800000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":2.698,"15min":2.641},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.374},"max":{"1min":1.3009999999999999,"5min":24.030999999999999,"15min":24.030999999999999},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.77200000000000002,"5min":2.6219999999999999,"15min":2.125},"min":{"1min":0.42399999999999999,"5min":0.41899999999999998,"15min":0.41599999999999998},"max":{"1min":2.3580000000000001,"5min":24.178000000000001,"15min":24.178000000000001},"last":0.63100000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631621,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2906768,"kb_used_data":2524168,"kb_used_omap":4,"kb_used_meta":382587,"kb_avail":90838384,"statfs":{"total":95995035648,"available":93018505216,"internally_reserved":0,"allocated":2584748032,"data_stored":2583958034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":391769694},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":4.7149999999999999,"15min":2.3900000000000001},"min":{"1min":0.25800000000000001,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":4.234,"5min":167.82400000000001,"15min":167.82400000000001},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.323,"5min":5.5289999999999999,"15min":2.746},"min":{"1min":0.24099999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.1959999999999997,"5min":167.89599999999999,"15min":167.89599999999999},"last":3.4569999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":4.0620000000000003,"15min":2.109},"min":{"1min":0.26000000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.09,"5min":83.921999999999997,"15min":83.921999999999997},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.3480000000000001,"5min":3.8279999999999998,"15min":2.0659999999999998},"min":{"1min":0.27400000000000002,"5min":0.25,"15min":0.247},"max":{"1min":9.4420000000000002,"5min":83.707999999999998,"15min":83.707999999999998},"last":11.452}]},{"osd":2,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":6.2569999999999997,"15min":3.7050000000000001},"min":{"1min":0.28100000000000003,"5min":0.25,"15min":0.215},"max":{"1min":11.619,"5min":190.43299999999999,"15min":190.43299999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.9970000000000001,"5min":7.3390000000000004,"15min":4.1289999999999996},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.26000000000000001},"max":{"1min":11.738,"5min":190.267,"15min":190.267},"last":3.319}]},{"osd":4,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":3.1280000000000001,"15min":2.012},"min":{"1min":0.35299999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.1859999999999999,"5min":51.591999999999999,"15min":51.591999999999999},"last":3.3719999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":2.1059999999999999,"15min":1.5489999999999999},"min":{"1min":0.40699999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.2089999999999996,"5min":26.774999999999999,"15min":45.735999999999997},"last":0.55700000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":2.331,"15min":1.623},"min":{"1min":0.374,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":9.5180000000000007,"5min":13.114000000000001,"15min":47.948999999999998},"last":11.372999999999999},{"interface":"front","average":{"1min":1.5149999999999999,"5min":2.2200000000000002,"15min":1.6519999999999999},"min":{"1min":0.439,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.125,"5min":26.728000000000002,"15min":47.968000000000004},"last":0.61699999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:41:49 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":2.1179999999999999,"15min":1.524},"min":{"1min":0.40500000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.1699999999999999,"5min":26.847999999999999,"15min":26.847999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.3200000000000001,"5min":2.415,"15min":1.46},"min":{"1min":0.39600000000000002,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":9.4809999999999999,"5min":13.058,"15min":13.058},"last":11.423}]},{"osd":7,"last update":"Mon Oct 9 05:41:58 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":2.702,"15min":2.4399999999999999},"min":{"1min":0.435,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":3.9950000000000001,"5min":26.024999999999999,"15min":26.024999999999999},"last":3.4380000000000002},{"interface":"front","average":{"1min":1.4510000000000001,"5min":2.9340000000000002,"15min":2.3159999999999998},"min":{"1min":0.44800000000000001,"5min":0.40899999999999997,"15min":0.35699999999999998},"max":{"1min":8.6039999999999992,"5min":25.995999999999999,"15min":25.995999999999999},"last":3.4119999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019168,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1784176,"kb_used_data":1474664,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91960976,"statfs":{"total":95995035648,"available":94168039424,"internally_reserved":0,"allocated":1510055936,"data_stored":1509521610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:41:24 2023","interfaces":[{"interface":"back","average":{"1min":6.5049999999999999,"5min":2.403,"15min":1.7170000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.214},"max":{"1min":100.504,"5min":100.504,"15min":100.504},"last":0.28799999999999998},{"interface":"front","average":{"1min":6.827,"5min":2.9089999999999998,"15min":1.998},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":100.717,"5min":100.717,"15min":100.717},"last":0.27100000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:41:56 2023","interfaces":[{"interface":"back","average":{"1min":16.344000000000001,"5min":4.601,"15min":2.2639999999999998},"min":{"1min":0.32600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":187.369,"5min":187.369,"15min":187.369},"last":0.38800000000000001},{"interface":"front","average":{"1min":16.341999999999999,"5min":4.8220000000000001,"15min":2.3820000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":187.35300000000001,"5min":187.35300000000001,"15min":187.35300000000001},"last":0.36299999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":5.6040000000000001,"15min":4.0460000000000003},"min":{"1min":0.313,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":1.694,"5min":196.93700000000001,"15min":196.93700000000001},"last":0.91900000000000004},{"interface":"front","average":{"1min":1.2070000000000001,"5min":5.508,"15min":3.9729999999999999},"min":{"1min":0.32400000000000001,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":6.6429999999999998,"5min":197.08099999999999,"15min":197.08099999999999},"last":0.32400000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:41:43 2023","interfaces":[{"interface":"back","average":{"1min":5.4669999999999996,"5min":8.048,"15min":3.4239999999999999},"min":{"1min":0.35899999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":70.962999999999994,"5min":211.48699999999999,"15min":211.48699999999999},"last":0.89900000000000002},{"interface":"front","average":{"1min":5.883,"5min":7.9420000000000002,"15min":3.3250000000000002},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.309},"max":{"1min":70.838999999999999,"5min":211.31100000000001,"15min":211.31100000000001},"last":0.34899999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":1.758,"15min":1.5629999999999999},"min":{"1min":0.255,"5min":0.25,"15min":0.249},"max":{"1min":0.67300000000000004,"5min":94.646000000000001,"15min":113.35899999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":1.6910000000000001,"15min":1.516},"min":{"1min":0.32100000000000001,"5min":0.317,"15min":0.30599999999999999},"max":{"1min":0.73999999999999999,"5min":94.692999999999998,"15min":113.32299999999999},"last":0.39700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:42:01 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.58599999999999997,"15min":0.70099999999999996},"min":{"1min":0.28599999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":0.79600000000000004,"5min":5.7599999999999998,"15min":11.944000000000001},"last":0.374},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.82399999999999995,"15min":0.85899999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.70199999999999996,"5min":7.6500000000000004,"15min":10.356},"last":0.41399999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:41:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.67100000000000004,"15min":2.3809999999999998},"min":{"1min":0.33300000000000002,"5min":0.31900000000000001,"15min":0.312},"max":{"1min":2.3660000000000001,"5min":4.5819999999999999,"15min":30.393999999999998},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.71199999999999997,"15min":2.6589999999999998},"min":{"1min":0.378,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.3540000000000001,"5min":4.5999999999999996,"15min":30.334},"last":0.67000000000000004}]}]},{"osd":5,"up_from":36,"seq":154618822940,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1537376,"kb_used_data":1227864,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92207776,"statfs":{"total":95995035648,"available":94420762624,"internally_reserved":0,"allocated":1257332736,"data_stored":1256810242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:42:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.613,"15min":3.032},"min":{"1min":0.26500000000000001,"5min":0.217,"15min":0.19400000000000001},"max":{"1min":3.198,"5min":10.712999999999999,"15min":165.90199999999999},"last":3.02},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.6559999999999999,"15min":3.0800000000000001},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.215},"max":{"1min":1.742,"5min":12.263999999999999,"15min":165.96700000000001},"last":0.317}]},{"osd":1,"last update":"Mon Oct 9 05:41:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":4.3390000000000004,"15min":2.214},"min":{"1min":0.26700000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.0229999999999997,"5min":87.769000000000005,"15min":87.769000000000005},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.4239999999999999,"5min":4.891,"15min":2.339},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.24399999999999999},"max":{"1min":12.228,"5min":87.864000000000004,"15min":87.864000000000004},"last":1.6419999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:42:24 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":3.5089999999999999,"15min":4.093},"min":{"1min":0.36899999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.7850000000000001,"5min":73.311999999999998,"15min":290.697},"last":2.5670000000000002},{"interface":"front","average":{"1min":1.21,"5min":3.3860000000000001,"15min":3.9990000000000001},"min":{"1min":0.40799999999999997,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":4.1299999999999999,"5min":73.280000000000001,"15min":290.786},"last":0.45800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:41:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0489999999999999,"5min":6.7690000000000001,"15min":3.0550000000000002},"min":{"1min":0.40799999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":36.395000000000003,"5min":95.236999999999995,"15min":95.236999999999995},"last":1.5449999999999999},{"interface":"front","average":{"1min":2.758,"5min":6.3600000000000003,"15min":2.8639999999999999},"min":{"1min":0.41199999999999998,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":36.406999999999996,"5min":93.471000000000004,"15min":93.471000000000004},"last":2.6589999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:41:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.60999999999999999,"15min":0.72099999999999997},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":0.84999999999999998,"5min":5.6459999999999999,"15min":9.3510000000000009},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.58999999999999997,"15min":0.94499999999999995},"min":{"1min":0.254,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":0.78100000000000003,"5min":4.8209999999999997,"15min":14.236000000000001},"last":0.37}]},{"osd":6,"last update":"Mon Oct 9 05:42:05 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.96299999999999997,"15min":1.365},"min":{"1min":0.34999999999999998,"5min":0.27600000000000002,"15min":0.26600000000000001},"max":{"1min":1.3080000000000001,"5min":27.146000000000001,"15min":95.010999999999996},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.64500000000000002,"5min":1.002,"15min":1.27},"min":{"1min":0.36699999999999999,"5min":0.28499999999999998,"15min":0.27700000000000002},"max":{"1min":2.2389999999999999,"5min":27.215,"15min":94.194000000000003},"last":0.434}]},{"osd":7,"last update":"Mon Oct 9 05:42:05 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.84099999999999997,"15min":1.486},"min":{"1min":0.39900000000000002,"5min":0.34399999999999997,"15min":0.28999999999999998},"max":{"1min":2.4769999999999999,"5min":5.5570000000000004,"15min":9.9849999999999994},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.76900000000000002,"15min":1.7},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.2270000000000001,"5min":5.0549999999999997,"15min":14.432},"last":0.44700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1987641344,"data_stored":1987641344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1744306176,"data_stored":1744306176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1836318720,"data_stored":1836318720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2595880960,"data_stored":2595880960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1525153792,"data_stored":1525153792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1267793920,"data_stored":1267793920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1260716032,"data_stored":1260716032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:42:32.776 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:42:32.776 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 21 2023-10-09T05:42:32.867 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:32.897 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:32.951 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:32.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:32.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:33.058 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:33.085 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:33.135 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:33.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:33.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:33.244 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:33.272 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:33.333 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:33.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:33.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:33.361 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 56 16 69202 69186 77.157 50.9375 0.0192439 0.012955 2023-10-09T05:42:33.439 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:33.538 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:33.763 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:33.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:33.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:33.872 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:33.976 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:34.073 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:42:34.073 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:42:34.073 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:42:34.073 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:42:34.073 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:42:34.167 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:33 smithi008 ceph-mon[191227]: pgmap v648: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s 2023-10-09T05:42:34.168 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:33 smithi008 ceph-mon[184418]: pgmap v648: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s 2023-10-09T05:42:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:33 smithi191 ceph-mon[161669]: pgmap v648: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s 2023-10-09T05:42:34.361 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 57 16 70076 70060 76.7619 54.625 0.0190717 0.0130231 2023-10-09T05:42:34.433 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:42:34.489 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:34.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:34.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:34.597 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:34.625 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:34.724 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:34.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:34.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:34.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:34.844 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3111803006' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:34.845 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:34 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:34.860 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:34.891 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:42:34.920 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:34.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:34.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:35.025 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:42:35.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3111803006' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:35.098 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:34 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:35.119 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3111803006' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:34 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:35.361 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 58 15 70826 70811 76.2479 46.9375 0.0145316 0.0131103 2023-10-09T05:42:35.889 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:42:36.002 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:42:36.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:36.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:42:36.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:36.133 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[184418]: pgmap v649: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 875 op/s 2023-10-09T05:42:36.133 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:42:36.133 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:42:36.133 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[184418]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.133 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:42:36.133 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[184418]: osdmap e226: 8 total, 8 up, 7 in 2023-10-09T05:42:36.133 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[184418]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[184418]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[184418]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[184418]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:35 smithi191 ceph-mon[161669]: pgmap v649: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 875 op/s 2023-10-09T05:42:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:35 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:42:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:35 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:42:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:35 smithi191 ceph-mon[161669]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:35 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:42:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:35 smithi191 ceph-mon[161669]: osdmap e226: 8 total, 8 up, 7 in 2023-10-09T05:42:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:35 smithi191 ceph-mon[161669]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:35 smithi191 ceph-mon[161669]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.135 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:35 smithi191 ceph-mon[161669]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.135 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:35 smithi191 ceph-mon[161669]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.135 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:36.138 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[191227]: pgmap v649: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 875 op/s 2023-10-09T05:42:36.139 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:42:36.139 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:42:36.139 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[191227]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.139 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:42:36.139 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[191227]: osdmap e226: 8 total, 8 up, 7 in 2023-10-09T05:42:36.139 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[191227]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.139 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[191227]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.139 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[191227]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.139 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:35 smithi008 ceph-mon[191227]: failed to encode map e226 with expected crc 2023-10-09T05:42:36.149 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:36.185 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:36.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:36.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:36.291 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:36.359 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:36.368 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 59 16 71604 71588 75.7697 48.5625 0.0114739 0.0131914 2023-10-09T05:42:36.402 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:36.523 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:36.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:36.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:36.635 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:36.666 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:36.721 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:36.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:36.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:36.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:36.858 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:36.917 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:36.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:36.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:37.027 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:37.091 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:37.367 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:37.368 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:42:37.369171+0000 min lat: 0.00134385 max lat: 0.28677 avg lat: 0.0132515 2023-10-09T05:42:37.368 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:42:37.368 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 60 16 72489 72473 75.4289 55.3125 0.00979817 0.0132515 2023-10-09T05:42:37.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:37.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:37.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:37.507 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:37.548 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:37.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:37.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:37.656 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:37.684 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:37.726 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:37.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:37.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:37.834 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:37.918 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:38.167 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:42:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:37 smithi008 ceph-mon[191227]: pgmap v651: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s 2023-10-09T05:42:38.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:37 smithi008 ceph-mon[184418]: pgmap v651: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s 2023-10-09T05:42:38.395 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:37 smithi191 ceph-mon[161669]: pgmap v651: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s 2023-10-09T05:42:38.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:38.396 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:38.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:38.397 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 61 16 73132 73116 74.8471 40.1875 0.00581973 0.013355 2023-10-09T05:42:38.498 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:38.526 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:38.563 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:38.574 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:38.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:38.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:38.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:38.711 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:38.768 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:38.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:38.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:38.875 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:38.907 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:38.961 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:38.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:38.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:39.068 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:42:39.156 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:39.298 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:42:39.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:39.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:42:39.375 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 62 16 73962 73946 74.473 51.875 0.0213637 0.0134213 2023-10-09T05:42:39.421 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:39.540 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:39.718 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:39.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:39.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:39.729 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:42:39.820 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:42:39.830 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:39.887 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:42:39.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:39 smithi008 ceph-mon[184418]: pgmap v652: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T05:42:39.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2892119357' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T05:42:39.928 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/793136923' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:39.928 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:39 smithi008 ceph-mon[191227]: pgmap v652: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T05:42:39.929 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2892119357' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T05:42:39.929 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/793136923' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:39.934 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:42:39.935 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:39.983 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:42:40.034 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:42:40.066 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:42:40.109 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:42:40.145 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:42:40.160 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:42:40.160 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:42:40.161 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:42:40.161 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:42:40.161 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:42:40.161 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:42:40.161 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:42:40.161 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:42:40.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:39 smithi191 ceph-mon[161669]: pgmap v652: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T05:42:40.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2892119357' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T05:42:40.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/793136923' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:40.375 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 63 16 74670 74654 73.9936 44.25 0.0031986 0.0135002 2023-10-09T05:42:40.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:42:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:42:40] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.33.4" 2023-10-09T05:42:41.375 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 64 16 75403 75387 73.5535 45.8125 0.0219664 0.0135908 2023-10-09T05:42:41.721 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:42:41.722 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:41.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:41.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:41.735 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:41.827 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:41.910 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:41.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:41 smithi008 ceph-mon[184418]: pgmap v653: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T05:42:41.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:41 smithi008 ceph-mon[191227]: pgmap v653: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T05:42:42.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:41 smithi191 ceph-mon[161669]: pgmap v653: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T05:42:42.349 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:42.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:42.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:42.376 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 65 16 76337 76321 73.3201 58.375 0.0121506 0.0136334 2023-10-09T05:42:42.454 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:42.482 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:42.537 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:42.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:42.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:42.642 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:42.709 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:42.955 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:42.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:42.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:43.031 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:42:43.031 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:42:43.073 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:43.104 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:43.155 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:43.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:43.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:43.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:43.295 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:43.348 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:43.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:43.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:43.377 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 66 16 77207 77191 73.0314 54.375 0.0173878 0.0136865 2023-10-09T05:42:43.456 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:43.482 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:43.523 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:43.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:43.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:43.631 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:43.682 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:43.733 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:43.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:43.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:43.840 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:43.935 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:43 smithi008 ceph-mon[184418]: pgmap v654: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T05:42:43.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:43 smithi008 ceph-mon[191227]: pgmap v654: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T05:42:43.942 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:44.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:43 smithi191 ceph-mon[161669]: pgmap v654: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T05:42:44.233 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:44.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:44.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:44.344 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:44.378 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:44.384 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 67 16 77970 77954 72.6473 47.6875 0.0287846 0.0137599 2023-10-09T05:42:44.431 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:44.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:44.514 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:42:44.537 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:42:44.597 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:44.922 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:44.922 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:44.922 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:44.923 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:44.927 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:42:44.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:44.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:42:45.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:45.084 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:45.099 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:42:45.143 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:45.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:45.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:45.249 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:45.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:45.361 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:45.387 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 68 16 78676 78660 72.2261 44.125 0.026975 0.0138399 2023-10-09T05:42:45.479 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:45.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:45.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:45.590 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:45.603 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:45.671 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:45.785 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:45.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:45.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:45.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:45.929 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:45.978 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:45.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:45.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:46.050 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:45 smithi008 ceph-mon[184418]: pgmap v655: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T05:42:46.050 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:45 smithi008 ceph-mon[191227]: pgmap v655: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T05:42:46.085 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:46.166 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:45 smithi191 ceph-mon[161669]: pgmap v655: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T05:42:46.323 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:46.385 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 69 16 79401 79385 71.8363 45.3125 0.0270482 0.0139141 2023-10-09T05:42:46.515 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:46.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:46.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:46.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:46.657 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:46.713 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:46.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:46.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:46.723 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:42:46.723 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:42:46.723 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T05:42:46.723 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:42:46.818 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:46.866 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:46.935 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:42:47.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4284060123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:42:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:46 smithi008 ceph-mon[184418]: pgmap v656: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T05:42:47.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4284060123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:42:47.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:46 smithi008 ceph-mon[191227]: pgmap v656: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T05:42:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4284060123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:42:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:46 smithi191 ceph-mon[161669]: pgmap v656: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T05:42:47.380 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:47.385 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 70 16 80148 80132 71.4773 46.6875 0.0177224 0.0139855 2023-10-09T05:42:47.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:47.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:47.490 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:47.519 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:47.567 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:47.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:47.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:47.675 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:47.708 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:47.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:47 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:47.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:47 smithi008 ceph-mon[184418]: failed to encode map e227 with expected crc 2023-10-09T05:42:47.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:47 smithi008 ceph-mon[184418]: failed to encode map e227 with expected crc 2023-10-09T05:42:47.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4284060123' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:42:47.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:47 smithi008 ceph-mon[184418]: osdmap e227: 8 total, 8 up, 7 in 2023-10-09T05:42:47.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:47 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:47.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:47 smithi008 ceph-mon[191227]: failed to encode map e227 with expected crc 2023-10-09T05:42:47.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:47 smithi008 ceph-mon[191227]: failed to encode map e227 with expected crc 2023-10-09T05:42:47.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4284060123' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:42:47.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:47 smithi008 ceph-mon[191227]: osdmap e227: 8 total, 8 up, 7 in 2023-10-09T05:42:48.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:47 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:42:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:47 smithi191 ceph-mon[161669]: failed to encode map e227 with expected crc 2023-10-09T05:42:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:47 smithi191 ceph-mon[161669]: failed to encode map e227 with expected crc 2023-10-09T05:42:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4284060123' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:42:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:47 smithi191 ceph-mon[161669]: osdmap e227: 8 total, 8 up, 7 in 2023-10-09T05:42:48.385 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 71 16 80800 80784 71.0448 40.75 0.00813962 0.0140694 2023-10-09T05:42:48.705 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:48.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:48.814 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:48.904 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:48.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:48 smithi008 ceph-mon[184418]: pgmap v658: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s 2023-10-09T05:42:48.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:48 smithi008 ceph-mon[191227]: pgmap v658: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s 2023-10-09T05:42:49.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:48 smithi191 ceph-mon[161669]: pgmap v658: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s 2023-10-09T05:42:49.336 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:49.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:49.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:49.385 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 72 16 81695 81679 70.8351 55.9375 0.0102958 0.0141121 2023-10-09T05:42:49.449 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:49.534 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:49.758 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:49.763 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:49.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:49.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:49.868 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:49.894 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:49.946 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:49.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:49.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:49.963 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:42:50.006 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:42:50.042 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:42:50.052 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:50.101 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:42:50.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1878260501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:50.134 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:42:50.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1878260501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:50.142 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:50.177 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:42:50.238 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:42:50.256 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:42:50.256 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:42:50.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.256 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:82289 7.8 7:19eebdc3:::benchmark_data_smithi008_354649_object82288:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-10-09T05:42:50.256 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:42:50.233686+0000", 2023-10-09T05:42:50.256 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.016558844999999999, 2023-10-09T05:42:50.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016602789999999999, 2023-10-09T05:42:50.257 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:50.257 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:42:50.257 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:50.257 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:42:50.257 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:42:50.257 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 82289 2023-10-09T05:42:50.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.257 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:50.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:50.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233686+0000", 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233686+0000", 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233684+0000", 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:42:50.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233706+0000", 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1645e-05 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233718+0000", 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2354999999999999e-05 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233725+0000", 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4899999999999997e-06 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.237364+0000", 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036385639999999999 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.237527+0000", 2023-10-09T05:42:50.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016300200000000001 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.237673+0000", 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000146236 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.244348+0000", 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066755809999999999 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245240+0000", 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00089143700000000002 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:82291 7.5 7:a438a61d:::benchmark_data_smithi008_354649_object82290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:42:50.233740+0000", 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.016504835999999998, 2023-10-09T05:42:50.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016654164999999999, 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 82291 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233740+0000", 2023-10-09T05:42:50.263 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233740+0000", 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233739+0000", 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233766+0000", 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6183999999999999e-05 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233776+0000", 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0791999999999999e-05 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:50.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233782+0000", 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2009999999999998e-06 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233951+0000", 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016919999999999999 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.234089+0000", 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013774800000000001 2023-10-09T05:42:50.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.234239+0000", 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015081799999999999 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.244552+0000", 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010312902000000001 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:50.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245200+0000", 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00064802100000000004 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:82295 7.e 7:73521bae:::benchmark_data_smithi008_354649_object82294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:42:50.233797+0000", 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.016447847000000002, 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016688288999999999, 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:50.268 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 82295 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233797+0000", 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:50.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233797+0000", 2023-10-09T05:42:50.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:50.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233796+0000", 2023-10-09T05:42:50.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:42:50.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233819+0000", 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2263e-05 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233830+0000", 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1257000000000001e-05 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233835+0000", 2023-10-09T05:42:50.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5130000000000004e-06 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233889+0000", 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3607999999999999e-05 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.234066+0000", 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017740599999999999 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.234239+0000", 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000172831 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245174+0000", 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010935106999999999 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245184+0000", 2023-10-09T05:42:50.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6250000000000002e-06 2023-10-09T05:42:50.274 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.274 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:50.274 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.274 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.274 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:82297 7.8 7:1b15fce8:::benchmark_data_smithi008_354649_object82296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-10-09T05:42:50.274 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:42:50.233849+0000", 2023-10-09T05:42:50.274 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.016395789000000001, 2023-10-09T05:42:50.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016709781, 2023-10-09T05:42:50.274 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:50.274 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 82297 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233849+0000", 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233849+0000", 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233848+0000", 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:50.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233872+0000", 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3663999999999999e-05 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233879+0000", 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0650000000000001e-06 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.233885+0000", 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4110000000000004e-06 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.240554+0000", 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066694900000000001 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.240684+0000", 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012994600000000001 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:42:50.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.240819+0000", 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013480499999999999 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245247+0000", 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0044276999999999997 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.279 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:82302 7.e 7:7fc21c55:::benchmark_data_smithi008_354649_object82301:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:42:50.237630+0000", 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.012614246000000001, 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012999032000000001, 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 82302 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.237630+0000", 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.237630+0000", 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.237629+0000", 2023-10-09T05:42:50.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.237665+0000", 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6566000000000002e-05 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.237680+0000", 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4673e-05 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.237689+0000", 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2900000000000008e-06 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245214+0000", 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0075242549999999997 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245361+0000", 2023-10-09T05:42:50.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000147052 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245505+0000", 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014419199999999999 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.248644+0000", 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031386600000000001 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.284 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:82294 7.a e227/213 7:5157af24:::benchmark_data_smithi008_354649_object82293:head v 227'5057, mlcod=227'5054)", 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:42:50.245034+0000", 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0052105559999999999, 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0056805629999999996, 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:50.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245034+0000", 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245034+0000", 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245034+0000", 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:42:50.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245060+0000", 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5964000000000001e-05 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245078+0000", 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8124000000000001e-05 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.245085+0000", 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7800000000000001e-06 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.246335+0000", 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001249514 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.246359+0000", 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4287e-05 2023-10-09T05:42:50.288 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:82300 7.3 e227/213 7:c927dde1:::benchmark_data_smithi008_354649_object82299:head v 227'5158, mlcod=227'5156)", 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:42:50.247982+0000", 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0022627559999999999, 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002794557, 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:42:50.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:42:50.290 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.247982+0000", 2023-10-09T05:42:50.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:42:50.290 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.247982+0000", 2023-10-09T05:42:50.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:42:50.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.247981+0000", 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.248008+0000", 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7152e-05 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.248022+0000", 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4304999999999999e-05 2023-10-09T05:42:50.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.248030+0000", 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7589999999999997e-06 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.248517+0000", 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000487226 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:42:50.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:42:50.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:42:50.248537+0000", 2023-10-09T05:42:50.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9772e-05 2023-10-09T05:42:50.293 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.293 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:42:50.293 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.293 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:42:50.293 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:42:50.293 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 7 2023-10-09T05:42:50.293 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:42:50.294 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:42:50.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:50.295 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:50.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1878260501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:42:50.306 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:42:50.306 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:42:50.306 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:42:50.306 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:42:50.306 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:42:50.307 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:42:50.307 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:42:50.307 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:42:50.391 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 73 16 82413 82397 70.4764 44.875 0.0165961 0.0141835 2023-10-09T05:42:50.396 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:50.426 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:50.474 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:42:50.479 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:50.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:50.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:50.585 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:50.618 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:50.668 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:50.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:50.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:50.778 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:50.873 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:42:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:42:50] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.33.4" 2023-10-09T05:42:50.877 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:50.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:50 smithi191 ceph-mon[161669]: pgmap v659: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 774 op/s 2023-10-09T05:42:51.126 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:50 smithi008 ceph-mon[184418]: pgmap v659: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 774 op/s 2023-10-09T05:42:51.127 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:50 smithi008 ceph-mon[191227]: pgmap v659: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 774 op/s 2023-10-09T05:42:51.389 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 74 16 83082 83066 70.0893 41.8125 0.00287211 0.0142457 2023-10-09T05:42:52.260 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:52.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:52.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:52.366 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:42:52.373 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:42:52.390 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 75 16 83815 83799 69.7656 45.8125 0.00856096 0.0143279 2023-10-09T05:42:52.464 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:52.545 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:42:52.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:52.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:42:52.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:52.683 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:52.728 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:52.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:52.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:52.833 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:52.861 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:52.918 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:52.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:52.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:53.026 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:53.100 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:53.229 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:42:53.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:53.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:42:53.283 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:53.332 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:53.340 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:53.370 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:53.389 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 76 16 84470 84454 69.3865 40.9375 0.030057 0.0144062 2023-10-09T05:42:53.420 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:53.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:53.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:53.525 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:53.626 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:53.983 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:53.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:53.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:54.095 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:54.124 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:53 smithi191 ceph-mon[161669]: pgmap v660: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T05:42:54.127 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:54.181 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:54.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:54.189 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:53 smithi008 ceph-mon[184418]: pgmap v660: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T05:42:54.190 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:53 smithi008 ceph-mon[191227]: pgmap v660: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T05:42:54.207 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:42:54.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:54.314 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:54.338 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:42:54.354 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:42:54.364 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:54.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:54.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:54.394 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 77 16 85214 85198 69.0853 46.5 0.0147992 0.0144698 2023-10-09T05:42:54.472 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:42:54.505 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:54.554 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:42:54.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:54.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:42:54.660 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:54.737 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:55.394 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 78 16 86019 86003 68.8448 50.3125 0.0225946 0.0145195 2023-10-09T05:42:55.869 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:55 smithi008 ceph-mon[184418]: pgmap v661: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s 2023-10-09T05:42:55.869 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:55 smithi008 ceph-mon[184418]: from='client.45431 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:42:56.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:55 smithi008 ceph-mon[191227]: pgmap v661: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s 2023-10-09T05:42:56.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:55 smithi008 ceph-mon[191227]: from='client.45431 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:42:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:55 smithi191 ceph-mon[161669]: pgmap v661: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s 2023-10-09T05:42:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:55 smithi191 ceph-mon[161669]: from='client.45431 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:42:56.395 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 79 16 86710 86694 68.5194 43.1875 0.128537 0.0145753 2023-10-09T05:42:56.874 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:56.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:56.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:56.904 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":661,"stamp":"2023-10-09T05:42:54.211937+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5471410099,"num_objects":83442,"num_object_clones":0,"num_object_copies":250326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83442,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":84520,"num_write_kb":5353637,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":4789622,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84584,"ondisk_log_size":84584,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18549696,"kb_used_data":15974792,"kb_used_omap":31,"kb_used_meta":2574816,"kb_avail":637666368,"statfs":{"total":671965249536,"available":652970360832,"internally_reserved":0,"allocated":16358187008,"data_stored":16353266660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":2636612041},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,0,4,8,0,0,2,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":490995712,"num_objects":7492,"num_object_clones":0,"num_object_copies":22476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7492,"num_write_kb":479488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.015689"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":209,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874593+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:42:34.874593+0000","last_peered":"2023-10-09T05:42:34.874593+0000","last_clean":"2023-10-09T05:42:34.874593+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:42:34.874593+0000","last_undegraded":"2023-10-09T05:42:34.874593+0000","last_fullsized":"2023-10-09T05:42:34.874593+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":199,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.868969+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:42:34.868969+0000","last_peered":"2023-10-09T05:42:34.868969+0000","last_clean":"2023-10-09T05:42:34.868969+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:42:34.868969+0000","last_undegraded":"2023-10-09T05:42:34.868969+0000","last_fullsized":"2023-10-09T05:42:34.868969+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":174,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874008+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:42:34.874008+0000","last_peered":"2023-10-09T05:42:34.874008+0000","last_clean":"2023-10-09T05:42:34.874008+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:42:34.874008+0000","last_undegraded":"2023-10-09T05:42:34.874008+0000","last_fullsized":"2023-10-09T05:42:34.874008+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":204,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.873062+0000","last_change":"2023-10-09T05:37:35.695995+0000","last_active":"2023-10-09T05:42:34.873062+0000","last_peered":"2023-10-09T05:42:34.873062+0000","last_clean":"2023-10-09T05:42:34.873062+0000","last_became_active":"2023-10-09T05:37:35.695682+0000","last_became_peered":"2023-10-09T05:37:35.695682+0000","last_unstale":"2023-10-09T05:42:34.873062+0000","last_undegraded":"2023-10-09T05:42:34.873062+0000","last_fullsized":"2023-10-09T05:42:34.873062+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:25.779103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":215,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.869612+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:42:34.869612+0000","last_peered":"2023-10-09T05:42:34.869612+0000","last_clean":"2023-10-09T05:42:34.869612+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:42:34.869612+0000","last_undegraded":"2023-10-09T05:42:34.869612+0000","last_fullsized":"2023-10-09T05:42:34.869612+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":192,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.875082+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:42:34.875082+0000","last_peered":"2023-10-09T05:42:34.875082+0000","last_clean":"2023-10-09T05:42:34.875082+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:42:34.875082+0000","last_undegraded":"2023-10-09T05:42:34.875082+0000","last_fullsized":"2023-10-09T05:42:34.875082+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":179,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874578+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:42:34.874578+0000","last_peered":"2023-10-09T05:42:34.874578+0000","last_clean":"2023-10-09T05:42:34.874578+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:42:34.874578+0000","last_undegraded":"2023-10-09T05:42:34.874578+0000","last_fullsized":"2023-10-09T05:42:34.874578+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":199,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.869687+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:42:34.869687+0000","last_peered":"2023-10-09T05:42:34.869687+0000","last_clean":"2023-10-09T05:42:34.869687+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:42:34.869687+0000","last_undegraded":"2023-10-09T05:42:34.869687+0000","last_fullsized":"2023-10-09T05:42:34.869687+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":218,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.862280+0000","last_change":"2023-10-09T05:37:35.695194+0000","last_active":"2023-10-09T05:42:34.862280+0000","last_peered":"2023-10-09T05:42:34.862280+0000","last_clean":"2023-10-09T05:42:34.862280+0000","last_became_active":"2023-10-09T05:37:35.694808+0000","last_became_peered":"2023-10-09T05:37:35.694808+0000","last_unstale":"2023-10-09T05:42:34.862280+0000","last_undegraded":"2023-10-09T05:42:34.862280+0000","last_fullsized":"2023-10-09T05:42:34.862280+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":213,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.867958+0000","last_change":"2023-10-09T05:37:35.689884+0000","last_active":"2023-10-09T05:42:34.867958+0000","last_peered":"2023-10-09T05:42:34.867958+0000","last_clean":"2023-10-09T05:42:34.867958+0000","last_became_active":"2023-10-09T05:37:35.689497+0000","last_became_peered":"2023-10-09T05:37:35.689497+0000","last_unstale":"2023-10-09T05:42:34.867958+0000","last_undegraded":"2023-10-09T05:42:34.867958+0000","last_fullsized":"2023-10-09T05:42:34.867958+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:26.622697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":107,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.868884+0000","last_change":"2023-10-09T05:37:35.709449+0000","last_active":"2023-10-09T05:42:34.868884+0000","last_peered":"2023-10-09T05:42:34.868884+0000","last_clean":"2023-10-09T05:42:34.868884+0000","last_became_active":"2023-10-09T05:37:35.700107+0000","last_became_peered":"2023-10-09T05:37:35.700107+0000","last_unstale":"2023-10-09T05:42:34.868884+0000","last_undegraded":"2023-10-09T05:42:34.868884+0000","last_fullsized":"2023-10-09T05:42:34.868884+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:51.388880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":174,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.873410+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:42:34.873410+0000","last_peered":"2023-10-09T05:42:34.873410+0000","last_clean":"2023-10-09T05:42:34.873410+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:42:34.873410+0000","last_undegraded":"2023-10-09T05:42:34.873410+0000","last_fullsized":"2023-10-09T05:42:34.873410+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":234,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.875210+0000","last_change":"2023-10-09T05:37:35.912663+0000","last_active":"2023-10-09T05:42:34.875210+0000","last_peered":"2023-10-09T05:42:34.875210+0000","last_clean":"2023-10-09T05:42:34.875210+0000","last_became_active":"2023-10-09T05:37:35.690596+0000","last_became_peered":"2023-10-09T05:37:35.690596+0000","last_unstale":"2023-10-09T05:42:34.875210+0000","last_undegraded":"2023-10-09T05:42:34.875210+0000","last_fullsized":"2023-10-09T05:42:34.875210+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":192,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.875074+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:42:34.875074+0000","last_peered":"2023-10-09T05:42:34.875074+0000","last_clean":"2023-10-09T05:42:34.875074+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:42:34.875074+0000","last_undegraded":"2023-10-09T05:42:34.875074+0000","last_fullsized":"2023-10-09T05:42:34.875074+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":105,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874723+0000","last_change":"2023-10-09T05:37:35.999390+0000","last_active":"2023-10-09T05:42:34.874723+0000","last_peered":"2023-10-09T05:42:34.874723+0000","last_clean":"2023-10-09T05:42:34.874723+0000","last_became_active":"2023-10-09T05:37:35.690796+0000","last_became_peered":"2023-10-09T05:37:35.690796+0000","last_unstale":"2023-10-09T05:42:34.874723+0000","last_undegraded":"2023-10-09T05:42:34.874723+0000","last_fullsized":"2023-10-09T05:42:34.874723+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":223,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.868538+0000","last_change":"2023-10-09T05:37:35.695711+0000","last_active":"2023-10-09T05:42:34.868538+0000","last_peered":"2023-10-09T05:42:34.868538+0000","last_clean":"2023-10-09T05:42:34.868538+0000","last_became_active":"2023-10-09T05:37:35.695394+0000","last_became_peered":"2023-10-09T05:37:35.695394+0000","last_unstale":"2023-10-09T05:42:34.868538+0000","last_undegraded":"2023-10-09T05:42:34.868538+0000","last_fullsized":"2023-10-09T05:42:34.868538+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:18.818723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":203,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.871825+0000","last_change":"2023-10-09T05:37:35.694698+0000","last_active":"2023-10-09T05:42:34.871825+0000","last_peered":"2023-10-09T05:42:34.871825+0000","last_clean":"2023-10-09T05:42:34.871825+0000","last_became_active":"2023-10-09T05:37:35.694419+0000","last_became_peered":"2023-10-09T05:37:35.694419+0000","last_unstale":"2023-10-09T05:42:34.871825+0000","last_undegraded":"2023-10-09T05:42:34.871825+0000","last_fullsized":"2023-10-09T05:42:34.871825+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:47.514392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":217,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.862163+0000","last_change":"2023-10-09T05:37:35.692253+0000","last_active":"2023-10-09T05:42:34.862163+0000","last_peered":"2023-10-09T05:42:34.862163+0000","last_clean":"2023-10-09T05:42:34.862163+0000","last_became_active":"2023-10-09T05:37:35.691861+0000","last_became_peered":"2023-10-09T05:37:35.691861+0000","last_unstale":"2023-10-09T05:42:34.862163+0000","last_undegraded":"2023-10-09T05:42:34.862163+0000","last_fullsized":"2023-10-09T05:42:34.862163+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":123,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.875286+0000","last_change":"2023-10-09T05:37:35.691785+0000","last_active":"2023-10-09T05:42:34.875286+0000","last_peered":"2023-10-09T05:42:34.875286+0000","last_clean":"2023-10-09T05:42:34.875286+0000","last_became_active":"2023-10-09T05:37:35.691416+0000","last_became_peered":"2023-10-09T05:37:35.691416+0000","last_unstale":"2023-10-09T05:42:34.875286+0000","last_undegraded":"2023-10-09T05:42:34.875286+0000","last_fullsized":"2023-10-09T05:42:34.875286+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":203,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.873221+0000","last_change":"2023-10-09T05:37:35.694887+0000","last_active":"2023-10-09T05:42:34.873221+0000","last_peered":"2023-10-09T05:42:34.873221+0000","last_clean":"2023-10-09T05:42:34.873221+0000","last_became_active":"2023-10-09T05:37:35.694532+0000","last_became_peered":"2023-10-09T05:37:35.694532+0000","last_unstale":"2023-10-09T05:42:34.873221+0000","last_undegraded":"2023-10-09T05:42:34.873221+0000","last_fullsized":"2023-10-09T05:42:34.873221+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.579095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":201,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874884+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:42:34.874884+0000","last_peered":"2023-10-09T05:42:34.874884+0000","last_clean":"2023-10-09T05:42:34.874884+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:42:34.874884+0000","last_undegraded":"2023-10-09T05:42:34.874884+0000","last_fullsized":"2023-10-09T05:42:34.874884+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":223,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.869381+0000","last_change":"2023-10-09T05:37:35.695684+0000","last_active":"2023-10-09T05:42:34.869381+0000","last_peered":"2023-10-09T05:42:34.869381+0000","last_clean":"2023-10-09T05:42:34.869381+0000","last_became_active":"2023-10-09T05:37:35.695395+0000","last_became_peered":"2023-10-09T05:37:35.695395+0000","last_unstale":"2023-10-09T05:42:34.869381+0000","last_undegraded":"2023-10-09T05:42:34.869381+0000","last_fullsized":"2023-10-09T05:42:34.869381+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.098596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"227'5187","reported_seq":5209,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:54.009165+0000","last_change":"2023-10-09T05:41:29.078278+0000","last_active":"2023-10-09T05:42:54.009165+0000","last_peered":"2023-10-09T05:42:54.009165+0000","last_clean":"2023-10-09T05:42:54.009165+0000","last_became_active":"2023-10-09T05:41:29.077562+0000","last_became_peered":"2023-10-09T05:41:29.077562+0000","last_unstale":"2023-10-09T05:42:54.009165+0000","last_undegraded":"2023-10-09T05:42:54.009165+0000","last_fullsized":"2023-10-09T05:42:54.009165+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5187,"log_dups_size":0,"ondisk_log_size":5187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:29.784837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339935232,"num_objects":5187,"num_object_clones":0,"num_object_copies":15561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5187,"num_write_kb":331968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":103,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.875616+0000","last_change":"2023-10-09T05:37:36.001540+0000","last_active":"2023-10-09T05:42:34.875616+0000","last_peered":"2023-10-09T05:42:34.875616+0000","last_clean":"2023-10-09T05:42:34.875616+0000","last_became_active":"2023-10-09T05:37:35.695981+0000","last_became_peered":"2023-10-09T05:37:35.695981+0000","last_unstale":"2023-10-09T05:42:34.875616+0000","last_undegraded":"2023-10-09T05:42:34.875616+0000","last_fullsized":"2023-10-09T05:42:34.875616+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":202,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.861825+0000","last_change":"2023-10-09T05:37:35.692404+0000","last_active":"2023-10-09T05:42:34.861825+0000","last_peered":"2023-10-09T05:42:34.861825+0000","last_clean":"2023-10-09T05:42:34.861825+0000","last_became_active":"2023-10-09T05:37:35.692044+0000","last_became_peered":"2023-10-09T05:37:35.692044+0000","last_unstale":"2023-10-09T05:42:34.861825+0000","last_undegraded":"2023-10-09T05:42:34.861825+0000","last_fullsized":"2023-10-09T05:42:34.861825+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"227'5118","reported_seq":5140,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:51.597116+0000","last_change":"2023-10-09T05:41:29.081067+0000","last_active":"2023-10-09T05:42:51.597116+0000","last_peered":"2023-10-09T05:42:51.597116+0000","last_clean":"2023-10-09T05:42:51.597116+0000","last_became_active":"2023-10-09T05:41:29.080704+0000","last_became_peered":"2023-10-09T05:41:29.080704+0000","last_unstale":"2023-10-09T05:42:51.597116+0000","last_undegraded":"2023-10-09T05:42:51.597116+0000","last_fullsized":"2023-10-09T05:42:51.597116+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5118,"log_dups_size":0,"ondisk_log_size":5118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:29.483205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335413248,"num_objects":5118,"num_object_clones":0,"num_object_copies":15354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5118,"num_write_kb":327552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":103,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.875330+0000","last_change":"2023-10-09T05:37:36.047485+0000","last_active":"2023-10-09T05:42:34.875330+0000","last_peered":"2023-10-09T05:42:34.875330+0000","last_clean":"2023-10-09T05:42:34.875330+0000","last_became_active":"2023-10-09T05:37:35.696098+0000","last_became_peered":"2023-10-09T05:37:35.696098+0000","last_unstale":"2023-10-09T05:42:34.875330+0000","last_undegraded":"2023-10-09T05:42:34.875330+0000","last_fullsized":"2023-10-09T05:42:34.875330+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":216,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874550+0000","last_change":"2023-10-09T05:41:44.881327+0000","last_active":"2023-10-09T05:42:34.874550+0000","last_peered":"2023-10-09T05:42:34.874550+0000","last_clean":"2023-10-09T05:42:34.874550+0000","last_became_active":"2023-10-09T05:41:44.881044+0000","last_became_peered":"2023-10-09T05:41:44.881044+0000","last_unstale":"2023-10-09T05:42:34.874550+0000","last_undegraded":"2023-10-09T05:42:34.874550+0000","last_fullsized":"2023-10-09T05:42:34.874550+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"227'5217","reported_seq":5238,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:53.360770+0000","last_change":"2023-10-09T05:41:29.077986+0000","last_active":"2023-10-09T05:42:53.360770+0000","last_peered":"2023-10-09T05:42:53.360770+0000","last_clean":"2023-10-09T05:42:53.360770+0000","last_became_active":"2023-10-09T05:41:29.077631+0000","last_became_peered":"2023-10-09T05:41:29.077631+0000","last_unstale":"2023-10-09T05:42:53.360770+0000","last_undegraded":"2023-10-09T05:42:53.360770+0000","last_fullsized":"2023-10-09T05:42:53.360770+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5217,"log_dups_size":0,"ondisk_log_size":5217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:48.684759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341901312,"num_objects":5217,"num_object_clones":0,"num_object_copies":15651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5217,"num_write_kb":333888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":104,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.862189+0000","last_change":"2023-10-09T05:37:35.740157+0000","last_active":"2023-10-09T05:42:34.862189+0000","last_peered":"2023-10-09T05:42:34.862189+0000","last_clean":"2023-10-09T05:42:34.862189+0000","last_became_active":"2023-10-09T05:37:35.691857+0000","last_became_peered":"2023-10-09T05:37:35.691857+0000","last_unstale":"2023-10-09T05:42:34.862189+0000","last_undegraded":"2023-10-09T05:42:34.862189+0000","last_fullsized":"2023-10-09T05:42:34.862189+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":225,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874985+0000","last_change":"2023-10-09T05:37:35.693161+0000","last_active":"2023-10-09T05:42:34.874985+0000","last_peered":"2023-10-09T05:42:34.874985+0000","last_clean":"2023-10-09T05:42:34.874985+0000","last_became_active":"2023-10-09T05:37:35.692817+0000","last_became_peered":"2023-10-09T05:37:35.692817+0000","last_unstale":"2023-10-09T05:42:34.874985+0000","last_undegraded":"2023-10-09T05:42:34.874985+0000","last_fullsized":"2023-10-09T05:42:34.874985+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"227'5262","reported_seq":5285,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:51.842978+0000","last_change":"2023-10-09T05:41:29.073764+0000","last_active":"2023-10-09T05:42:51.842978+0000","last_peered":"2023-10-09T05:42:51.842978+0000","last_clean":"2023-10-09T05:42:51.842978+0000","last_became_active":"2023-10-09T05:41:29.073452+0000","last_became_peered":"2023-10-09T05:41:29.073452+0000","last_unstale":"2023-10-09T05:42:51.842978+0000","last_undegraded":"2023-10-09T05:42:51.842978+0000","last_fullsized":"2023-10-09T05:42:51.842978+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5262,"log_dups_size":0,"ondisk_log_size":5262,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:22.328530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344850432,"num_objects":5262,"num_object_clones":0,"num_object_copies":15786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5262,"num_write_kb":336768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":194,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.862083+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:42:34.862083+0000","last_peered":"2023-10-09T05:42:34.862083+0000","last_clean":"2023-10-09T05:42:34.862083+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:42:34.862083+0000","last_undegraded":"2023-10-09T05:42:34.862083+0000","last_fullsized":"2023-10-09T05:42:34.862083+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":179,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.870866+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:42:34.870866+0000","last_peered":"2023-10-09T05:42:34.870866+0000","last_clean":"2023-10-09T05:42:34.870866+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:42:34.870866+0000","last_undegraded":"2023-10-09T05:42:34.870866+0000","last_fullsized":"2023-10-09T05:42:34.870866+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"227'5342","reported_seq":5364,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:53.272133+0000","last_change":"2023-10-09T05:41:29.082598+0000","last_active":"2023-10-09T05:42:53.272133+0000","last_peered":"2023-10-09T05:42:53.272133+0000","last_clean":"2023-10-09T05:42:53.272133+0000","last_became_active":"2023-10-09T05:41:29.082285+0000","last_became_peered":"2023-10-09T05:41:29.082285+0000","last_unstale":"2023-10-09T05:42:53.272133+0000","last_undegraded":"2023-10-09T05:42:53.272133+0000","last_fullsized":"2023-10-09T05:42:53.272133+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5342,"log_dups_size":0,"ondisk_log_size":5342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:18.625214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350093312,"num_objects":5342,"num_object_clones":0,"num_object_copies":16026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5342,"num_write_kb":341888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":198,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:46.673548+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:42:46.673548+0000","last_peered":"2023-10-09T05:42:46.673548+0000","last_clean":"2023-10-09T05:42:46.673548+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:42:46.673548+0000","last_undegraded":"2023-10-09T05:42:46.673548+0000","last_fullsized":"2023-10-09T05:42:46.673548+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":192,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.875122+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:42:34.875122+0000","last_peered":"2023-10-09T05:42:34.875122+0000","last_clean":"2023-10-09T05:42:34.875122+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:42:34.875122+0000","last_undegraded":"2023-10-09T05:42:34.875122+0000","last_fullsized":"2023-10-09T05:42:34.875122+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"227'5340","reported_seq":5363,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:51.834950+0000","last_change":"2023-10-09T05:41:29.078791+0000","last_active":"2023-10-09T05:42:51.834950+0000","last_peered":"2023-10-09T05:42:51.834950+0000","last_clean":"2023-10-09T05:42:51.834950+0000","last_became_active":"2023-10-09T05:41:29.078491+0000","last_became_peered":"2023-10-09T05:41:29.078491+0000","last_unstale":"2023-10-09T05:42:51.834950+0000","last_undegraded":"2023-10-09T05:42:51.834950+0000","last_fullsized":"2023-10-09T05:42:51.834950+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5340,"log_dups_size":0,"ondisk_log_size":5340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:37.817282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349962240,"num_objects":5340,"num_object_clones":0,"num_object_copies":16020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5340,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":193,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874670+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:42:34.874670+0000","last_peered":"2023-10-09T05:42:34.874670+0000","last_clean":"2023-10-09T05:42:34.874670+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:42:34.874670+0000","last_undegraded":"2023-10-09T05:42:34.874670+0000","last_fullsized":"2023-10-09T05:42:34.874670+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":207,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.875093+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:42:34.875093+0000","last_peered":"2023-10-09T05:42:34.875093+0000","last_clean":"2023-10-09T05:42:34.875093+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:42:34.875093+0000","last_undegraded":"2023-10-09T05:42:34.875093+0000","last_fullsized":"2023-10-09T05:42:34.875093+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"227'5126","reported_seq":5147,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:50.817263+0000","last_change":"2023-10-09T05:41:29.078645+0000","last_active":"2023-10-09T05:42:50.817263+0000","last_peered":"2023-10-09T05:42:50.817263+0000","last_clean":"2023-10-09T05:42:50.817263+0000","last_became_active":"2023-10-09T05:41:29.078355+0000","last_became_peered":"2023-10-09T05:41:29.078355+0000","last_unstale":"2023-10-09T05:42:50.817263+0000","last_undegraded":"2023-10-09T05:42:50.817263+0000","last_fullsized":"2023-10-09T05:42:50.817263+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5126,"log_dups_size":0,"ondisk_log_size":5126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335937536,"num_objects":5126,"num_object_clones":0,"num_object_copies":15378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5126,"num_write_kb":328064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":217,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874434+0000","last_change":"2023-10-09T05:37:35.696148+0000","last_active":"2023-10-09T05:42:34.874434+0000","last_peered":"2023-10-09T05:42:34.874434+0000","last_clean":"2023-10-09T05:42:34.874434+0000","last_became_active":"2023-10-09T05:37:35.695750+0000","last_became_peered":"2023-10-09T05:37:35.695750+0000","last_unstale":"2023-10-09T05:42:34.874434+0000","last_undegraded":"2023-10-09T05:42:34.874434+0000","last_fullsized":"2023-10-09T05:42:34.874434+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":201,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.875465+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:42:34.875465+0000","last_peered":"2023-10-09T05:42:34.875465+0000","last_clean":"2023-10-09T05:42:34.875465+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:42:34.875465+0000","last_undegraded":"2023-10-09T05:42:34.875465+0000","last_fullsized":"2023-10-09T05:42:34.875465+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"227'5119","reported_seq":5140,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:50.824525+0000","last_change":"2023-10-09T05:41:29.082194+0000","last_active":"2023-10-09T05:42:50.824525+0000","last_peered":"2023-10-09T05:42:50.824525+0000","last_clean":"2023-10-09T05:42:50.824525+0000","last_became_active":"2023-10-09T05:41:29.081976+0000","last_became_peered":"2023-10-09T05:41:29.081976+0000","last_unstale":"2023-10-09T05:42:50.824525+0000","last_undegraded":"2023-10-09T05:42:50.824525+0000","last_fullsized":"2023-10-09T05:42:50.824525+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5119,"log_dups_size":0,"ondisk_log_size":5119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335478784,"num_objects":5119,"num_object_clones":0,"num_object_copies":15357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5119,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":225,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.869537+0000","last_change":"2023-10-09T05:37:35.699716+0000","last_active":"2023-10-09T05:42:34.869537+0000","last_peered":"2023-10-09T05:42:34.869537+0000","last_clean":"2023-10-09T05:42:34.869537+0000","last_became_active":"2023-10-09T05:37:35.699413+0000","last_became_peered":"2023-10-09T05:37:35.699413+0000","last_unstale":"2023-10-09T05:42:34.869537+0000","last_undegraded":"2023-10-09T05:42:34.869537+0000","last_fullsized":"2023-10-09T05:42:34.869537+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:47.360897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":174,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.873148+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:42:34.873148+0000","last_peered":"2023-10-09T05:42:34.873148+0000","last_clean":"2023-10-09T05:42:34.873148+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:42:34.873148+0000","last_undegraded":"2023-10-09T05:42:34.873148+0000","last_fullsized":"2023-10-09T05:42:34.873148+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"227'5077","reported_seq":5098,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:49.961644+0000","last_change":"2023-10-09T05:41:29.079588+0000","last_active":"2023-10-09T05:42:49.961644+0000","last_peered":"2023-10-09T05:42:49.961644+0000","last_clean":"2023-10-09T05:42:49.961644+0000","last_became_active":"2023-10-09T05:41:29.079288+0000","last_became_peered":"2023-10-09T05:41:29.079288+0000","last_unstale":"2023-10-09T05:42:49.961644+0000","last_undegraded":"2023-10-09T05:42:49.961644+0000","last_fullsized":"2023-10-09T05:42:49.961644+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5077,"log_dups_size":0,"ondisk_log_size":5077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332726272,"num_objects":5077,"num_object_clones":0,"num_object_copies":15231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5077,"num_write_kb":324928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":796,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874955+0000","last_change":"2023-10-09T05:40:04.806908+0000","last_active":"2023-10-09T05:42:34.874955+0000","last_peered":"2023-10-09T05:42:34.874955+0000","last_clean":"2023-10-09T05:42:34.874955+0000","last_became_active":"2023-10-09T05:40:04.806232+0000","last_became_peered":"2023-10-09T05:40:04.806232+0000","last_unstale":"2023-10-09T05:42:34.874955+0000","last_undegraded":"2023-10-09T05:42:34.874955+0000","last_fullsized":"2023-10-09T05:42:34.874955+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:34.087553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":209,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.862068+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:42:34.862068+0000","last_peered":"2023-10-09T05:42:34.862068+0000","last_clean":"2023-10-09T05:42:34.862068+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:42:34.862068+0000","last_undegraded":"2023-10-09T05:42:34.862068+0000","last_fullsized":"2023-10-09T05:42:34.862068+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":159,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.873481+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:42:34.873481+0000","last_peered":"2023-10-09T05:42:34.873481+0000","last_clean":"2023-10-09T05:42:34.873481+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:42:34.873481+0000","last_undegraded":"2023-10-09T05:42:34.873481+0000","last_fullsized":"2023-10-09T05:42:34.873481+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"227'5117","reported_seq":5138,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:50.799594+0000","last_change":"2023-10-09T05:41:29.082691+0000","last_active":"2023-10-09T05:42:50.799594+0000","last_peered":"2023-10-09T05:42:50.799594+0000","last_clean":"2023-10-09T05:42:50.799594+0000","last_became_active":"2023-10-09T05:41:29.082385+0000","last_became_peered":"2023-10-09T05:41:29.082385+0000","last_unstale":"2023-10-09T05:42:50.799594+0000","last_undegraded":"2023-10-09T05:42:50.799594+0000","last_fullsized":"2023-10-09T05:42:50.799594+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5117,"log_dups_size":0,"ondisk_log_size":5117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335347712,"num_objects":5117,"num_object_clones":0,"num_object_copies":15351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5117,"num_write_kb":327488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":663,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.863007+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:42:34.863007+0000","last_peered":"2023-10-09T05:42:34.863007+0000","last_clean":"2023-10-09T05:42:34.863007+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:42:34.863007+0000","last_undegraded":"2023-10-09T05:42:34.863007+0000","last_fullsized":"2023-10-09T05:42:34.863007+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":193,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874811+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:42:34.874811+0000","last_peered":"2023-10-09T05:42:34.874811+0000","last_clean":"2023-10-09T05:42:34.874811+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:42:34.874811+0000","last_undegraded":"2023-10-09T05:42:34.874811+0000","last_fullsized":"2023-10-09T05:42:34.874811+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":206,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.862274+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:42:34.862274+0000","last_peered":"2023-10-09T05:42:34.862274+0000","last_clean":"2023-10-09T05:42:34.862274+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:42:34.862274+0000","last_undegraded":"2023-10-09T05:42:34.862274+0000","last_fullsized":"2023-10-09T05:42:34.862274+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"227'5232","reported_seq":5254,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:53.334607+0000","last_change":"2023-10-09T05:41:29.077953+0000","last_active":"2023-10-09T05:42:53.334607+0000","last_peered":"2023-10-09T05:42:53.334607+0000","last_clean":"2023-10-09T05:42:53.334607+0000","last_became_active":"2023-10-09T05:41:29.077652+0000","last_became_peered":"2023-10-09T05:41:29.077652+0000","last_unstale":"2023-10-09T05:42:53.334607+0000","last_undegraded":"2023-10-09T05:42:53.334607+0000","last_fullsized":"2023-10-09T05:42:53.334607+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5232,"log_dups_size":0,"ondisk_log_size":5232,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:53.470052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342884352,"num_objects":5232,"num_object_clones":0,"num_object_copies":15696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5232,"num_write_kb":334848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":542,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.863714+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:42:34.863714+0000","last_peered":"2023-10-09T05:42:34.863714+0000","last_clean":"2023-10-09T05:42:34.863714+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:42:34.863714+0000","last_undegraded":"2023-10-09T05:42:34.863714+0000","last_fullsized":"2023-10-09T05:42:34.863714+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":175,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.876744+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:42:34.876744+0000","last_peered":"2023-10-09T05:42:34.876744+0000","last_clean":"2023-10-09T05:42:34.876744+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:42:34.876744+0000","last_undegraded":"2023-10-09T05:42:34.876744+0000","last_fullsized":"2023-10-09T05:42:34.876744+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":204,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.873227+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:42:34.873227+0000","last_peered":"2023-10-09T05:42:34.873227+0000","last_clean":"2023-10-09T05:42:34.873227+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:42:34.873227+0000","last_undegraded":"2023-10-09T05:42:34.873227+0000","last_fullsized":"2023-10-09T05:42:34.873227+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"227'5345","reported_seq":5366,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:53.376521+0000","last_change":"2023-10-09T05:41:29.082532+0000","last_active":"2023-10-09T05:42:53.376521+0000","last_peered":"2023-10-09T05:42:53.376521+0000","last_clean":"2023-10-09T05:42:53.376521+0000","last_became_active":"2023-10-09T05:41:29.082272+0000","last_became_peered":"2023-10-09T05:41:29.082272+0000","last_unstale":"2023-10-09T05:42:53.376521+0000","last_undegraded":"2023-10-09T05:42:53.376521+0000","last_fullsized":"2023-10-09T05:42:53.376521+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5345,"log_dups_size":0,"ondisk_log_size":5345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:51.574989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350289920,"num_objects":5345,"num_object_clones":0,"num_object_copies":16035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5345,"num_write_kb":342080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":838,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.862033+0000","last_change":"2023-10-09T05:40:04.805258+0000","last_active":"2023-10-09T05:42:34.862033+0000","last_peered":"2023-10-09T05:42:34.862033+0000","last_clean":"2023-10-09T05:42:34.862033+0000","last_became_active":"2023-10-09T05:40:04.804776+0000","last_became_peered":"2023-10-09T05:40:04.804776+0000","last_unstale":"2023-10-09T05:42:34.862033+0000","last_undegraded":"2023-10-09T05:42:34.862033+0000","last_fullsized":"2023-10-09T05:42:34.862033+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":375,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.871241+0000","last_change":"2023-10-09T05:37:38.649806+0000","last_active":"2023-10-09T05:42:34.871241+0000","last_peered":"2023-10-09T05:42:34.871241+0000","last_clean":"2023-10-09T05:42:34.871241+0000","last_became_active":"2023-10-09T05:37:38.649407+0000","last_became_peered":"2023-10-09T05:37:38.649407+0000","last_unstale":"2023-10-09T05:42:34.871241+0000","last_undegraded":"2023-10-09T05:42:34.871241+0000","last_fullsized":"2023-10-09T05:42:34.871241+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.736601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":191,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.872133+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:42:34.872133+0000","last_peered":"2023-10-09T05:42:34.872133+0000","last_clean":"2023-10-09T05:42:34.872133+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:42:34.872133+0000","last_undegraded":"2023-10-09T05:42:34.872133+0000","last_fullsized":"2023-10-09T05:42:34.872133+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":199,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.875780+0000","last_change":"2023-10-09T05:37:35.693322+0000","last_active":"2023-10-09T05:42:34.875780+0000","last_peered":"2023-10-09T05:42:34.875780+0000","last_clean":"2023-10-09T05:42:34.875780+0000","last_became_active":"2023-10-09T05:37:35.692925+0000","last_became_peered":"2023-10-09T05:37:35.692925+0000","last_unstale":"2023-10-09T05:42:34.875780+0000","last_undegraded":"2023-10-09T05:42:34.875780+0000","last_fullsized":"2023-10-09T05:42:34.875780+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"227'5274","reported_seq":5297,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:51.853744+0000","last_change":"2023-10-09T05:41:29.076367+0000","last_active":"2023-10-09T05:42:51.853744+0000","last_peered":"2023-10-09T05:42:51.853744+0000","last_clean":"2023-10-09T05:42:51.853744+0000","last_became_active":"2023-10-09T05:41:29.076035+0000","last_became_peered":"2023-10-09T05:41:29.076035+0000","last_unstale":"2023-10-09T05:42:51.853744+0000","last_undegraded":"2023-10-09T05:42:51.853744+0000","last_fullsized":"2023-10-09T05:42:51.853744+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5274,"log_dups_size":0,"ondisk_log_size":5274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:51.713835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345636864,"num_objects":5274,"num_object_clones":0,"num_object_copies":15822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5274,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":784,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.876834+0000","last_change":"2023-10-09T05:40:05.930112+0000","last_active":"2023-10-09T05:42:34.876834+0000","last_peered":"2023-10-09T05:42:34.876834+0000","last_clean":"2023-10-09T05:42:34.876834+0000","last_became_active":"2023-10-09T05:40:05.929574+0000","last_became_peered":"2023-10-09T05:40:05.929574+0000","last_unstale":"2023-10-09T05:42:34.876834+0000","last_undegraded":"2023-10-09T05:42:34.876834+0000","last_fullsized":"2023-10-09T05:42:34.876834+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:06.982326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":180,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.871827+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:42:34.871827+0000","last_peered":"2023-10-09T05:42:34.871827+0000","last_clean":"2023-10-09T05:42:34.871827+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:42:34.871827+0000","last_undegraded":"2023-10-09T05:42:34.871827+0000","last_fullsized":"2023-10-09T05:42:34.871827+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":194,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.870875+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:42:34.870875+0000","last_peered":"2023-10-09T05:42:34.870875+0000","last_clean":"2023-10-09T05:42:34.870875+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:42:34.870875+0000","last_undegraded":"2023-10-09T05:42:34.870875+0000","last_fullsized":"2023-10-09T05:42:34.870875+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"227'5084","reported_seq":5104,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:50.030045+0000","last_change":"2023-10-09T05:41:29.073701+0000","last_active":"2023-10-09T05:42:50.030045+0000","last_peered":"2023-10-09T05:42:50.030045+0000","last_clean":"2023-10-09T05:42:50.030045+0000","last_became_active":"2023-10-09T05:41:29.073372+0000","last_became_peered":"2023-10-09T05:41:29.073372+0000","last_unstale":"2023-10-09T05:42:50.030045+0000","last_undegraded":"2023-10-09T05:42:50.030045+0000","last_fullsized":"2023-10-09T05:42:50.030045+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5084,"log_dups_size":0,"ondisk_log_size":5084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333185024,"num_objects":5084,"num_object_clones":0,"num_object_copies":15252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5084,"num_write_kb":325376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":416,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.871353+0000","last_change":"2023-10-09T05:37:38.651376+0000","last_active":"2023-10-09T05:42:34.871353+0000","last_peered":"2023-10-09T05:42:34.871353+0000","last_clean":"2023-10-09T05:42:34.871353+0000","last_became_active":"2023-10-09T05:37:38.650743+0000","last_became_peered":"2023-10-09T05:37:38.650743+0000","last_unstale":"2023-10-09T05:42:34.871353+0000","last_undegraded":"2023-10-09T05:42:34.871353+0000","last_fullsized":"2023-10-09T05:42:34.871353+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:32.986250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":174,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.875137+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:42:34.875137+0000","last_peered":"2023-10-09T05:42:34.875137+0000","last_clean":"2023-10-09T05:42:34.875137+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:42:34.875137+0000","last_undegraded":"2023-10-09T05:42:34.875137+0000","last_fullsized":"2023-10-09T05:42:34.875137+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":189,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.871085+0000","last_change":"2023-10-09T05:37:35.692906+0000","last_active":"2023-10-09T05:42:34.871085+0000","last_peered":"2023-10-09T05:42:34.871085+0000","last_clean":"2023-10-09T05:42:34.871085+0000","last_became_active":"2023-10-09T05:37:35.692357+0000","last_became_peered":"2023-10-09T05:37:35.692357+0000","last_unstale":"2023-10-09T05:42:34.871085+0000","last_undegraded":"2023-10-09T05:42:34.871085+0000","last_fullsized":"2023-10-09T05:42:34.871085+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:40.314333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"227'5226","reported_seq":5248,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:51.568873+0000","last_change":"2023-10-09T05:41:29.080413+0000","last_active":"2023-10-09T05:42:51.568873+0000","last_peered":"2023-10-09T05:42:51.568873+0000","last_clean":"2023-10-09T05:42:51.568873+0000","last_became_active":"2023-10-09T05:41:29.080184+0000","last_became_peered":"2023-10-09T05:41:29.080184+0000","last_unstale":"2023-10-09T05:42:51.568873+0000","last_undegraded":"2023-10-09T05:42:51.568873+0000","last_fullsized":"2023-10-09T05:42:51.568873+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5226,"log_dups_size":0,"ondisk_log_size":5226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:47.929696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342491136,"num_objects":5226,"num_object_clones":0,"num_object_copies":15678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5226,"num_write_kb":334464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":226,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.869297+0000","last_change":"2023-10-09T05:37:35.711850+0000","last_active":"2023-10-09T05:42:34.869297+0000","last_peered":"2023-10-09T05:42:34.869297+0000","last_clean":"2023-10-09T05:42:34.869297+0000","last_became_active":"2023-10-09T05:37:35.700668+0000","last_became_peered":"2023-10-09T05:37:35.700668+0000","last_unstale":"2023-10-09T05:42:34.869297+0000","last_undegraded":"2023-10-09T05:42:34.869297+0000","last_fullsized":"2023-10-09T05:42:34.869297+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:31.019620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":615,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.872852+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:42:34.872852+0000","last_peered":"2023-10-09T05:42:34.872852+0000","last_clean":"2023-10-09T05:42:34.872852+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:42:34.872852+0000","last_undegraded":"2023-10-09T05:42:34.872852+0000","last_fullsized":"2023-10-09T05:42:34.872852+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":219,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.869755+0000","last_change":"2023-10-09T05:37:35.699849+0000","last_active":"2023-10-09T05:42:34.869755+0000","last_peered":"2023-10-09T05:42:34.869755+0000","last_clean":"2023-10-09T05:42:34.869755+0000","last_became_active":"2023-10-09T05:37:35.699524+0000","last_became_peered":"2023-10-09T05:37:35.699524+0000","last_unstale":"2023-10-09T05:42:34.869755+0000","last_undegraded":"2023-10-09T05:42:34.869755+0000","last_fullsized":"2023-10-09T05:42:34.869755+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:55.967441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"227'5348","reported_seq":5370,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-10-09T05:42:51.578426+0000","last_change":"2023-10-09T05:41:29.081230+0000","last_active":"2023-10-09T05:42:51.578426+0000","last_peered":"2023-10-09T05:42:51.578426+0000","last_clean":"2023-10-09T05:42:51.578426+0000","last_became_active":"2023-10-09T05:41:29.080911+0000","last_became_peered":"2023-10-09T05:41:29.080911+0000","last_unstale":"2023-10-09T05:42:51.578426+0000","last_undegraded":"2023-10-09T05:42:51.578426+0000","last_fullsized":"2023-10-09T05:42:51.578426+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5348,"log_dups_size":0,"ondisk_log_size":5348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:22.580967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350486528,"num_objects":5348,"num_object_clones":0,"num_object_copies":16044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5348,"num_write_kb":342272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":727,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874892+0000","last_change":"2023-10-09T05:41:21.917602+0000","last_active":"2023-10-09T05:42:34.874892+0000","last_peered":"2023-10-09T05:42:34.874892+0000","last_clean":"2023-10-09T05:42:34.874892+0000","last_became_active":"2023-10-09T05:41:21.917116+0000","last_became_peered":"2023-10-09T05:41:21.917116+0000","last_unstale":"2023-10-09T05:42:34.874892+0000","last_undegraded":"2023-10-09T05:42:34.874892+0000","last_fullsized":"2023-10-09T05:42:34.874892+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":182,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.870966+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:42:34.870966+0000","last_peered":"2023-10-09T05:42:34.870966+0000","last_clean":"2023-10-09T05:42:34.870966+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:42:34.870966+0000","last_undegraded":"2023-10-09T05:42:34.870966+0000","last_fullsized":"2023-10-09T05:42:34.870966+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":177,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874456+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:42:34.874456+0000","last_peered":"2023-10-09T05:42:34.874456+0000","last_clean":"2023-10-09T05:42:34.874456+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:42:34.874456+0000","last_undegraded":"2023-10-09T05:42:34.874456+0000","last_fullsized":"2023-10-09T05:42:34.874456+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":191,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.871015+0000","last_change":"2023-10-09T05:37:35.692709+0000","last_active":"2023-10-09T05:42:34.871015+0000","last_peered":"2023-10-09T05:42:34.871015+0000","last_clean":"2023-10-09T05:42:34.871015+0000","last_became_active":"2023-10-09T05:37:35.691853+0000","last_became_peered":"2023-10-09T05:37:35.691853+0000","last_unstale":"2023-10-09T05:42:34.871015+0000","last_undegraded":"2023-10-09T05:42:34.871015+0000","last_fullsized":"2023-10-09T05:42:34.871015+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:39.971837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":228,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.864024+0000","last_change":"2023-10-09T05:41:44.887519+0000","last_active":"2023-10-09T05:42:34.864024+0000","last_peered":"2023-10-09T05:42:34.864024+0000","last_clean":"2023-10-09T05:42:34.864024+0000","last_became_active":"2023-10-09T05:41:44.887047+0000","last_became_peered":"2023-10-09T05:41:44.887047+0000","last_unstale":"2023-10-09T05:42:34.864024+0000","last_undegraded":"2023-10-09T05:42:34.864024+0000","last_fullsized":"2023-10-09T05:42:34.864024+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:49.475913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":196,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.862003+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:42:34.862003+0000","last_peered":"2023-10-09T05:42:34.862003+0000","last_clean":"2023-10-09T05:42:34.862003+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:42:34.862003+0000","last_undegraded":"2023-10-09T05:42:34.862003+0000","last_fullsized":"2023-10-09T05:42:34.862003+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":207,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874878+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:42:34.874878+0000","last_peered":"2023-10-09T05:42:34.874878+0000","last_clean":"2023-10-09T05:42:34.874878+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:42:34.874878+0000","last_undegraded":"2023-10-09T05:42:34.874878+0000","last_fullsized":"2023-10-09T05:42:34.874878+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":181,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.862074+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:42:34.862074+0000","last_peered":"2023-10-09T05:42:34.862074+0000","last_clean":"2023-10-09T05:42:34.862074+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:42:34.862074+0000","last_undegraded":"2023-10-09T05:42:34.862074+0000","last_fullsized":"2023-10-09T05:42:34.862074+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":207,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874491+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:42:34.874491+0000","last_peered":"2023-10-09T05:42:34.874491+0000","last_clean":"2023-10-09T05:42:34.874491+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:42:34.874491+0000","last_undegraded":"2023-10-09T05:42:34.874491+0000","last_fullsized":"2023-10-09T05:42:34.874491+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":206,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.869556+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:42:34.869556+0000","last_peered":"2023-10-09T05:42:34.869556+0000","last_clean":"2023-10-09T05:42:34.869556+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:42:34.869556+0000","last_undegraded":"2023-10-09T05:42:34.869556+0000","last_fullsized":"2023-10-09T05:42:34.869556+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":218,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.867918+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:42:34.867918+0000","last_peered":"2023-10-09T05:42:34.867918+0000","last_clean":"2023-10-09T05:42:34.867918+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:42:34.867918+0000","last_undegraded":"2023-10-09T05:42:34.867918+0000","last_fullsized":"2023-10-09T05:42:34.867918+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":183,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874653+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:42:34.874653+0000","last_peered":"2023-10-09T05:42:34.874653+0000","last_clean":"2023-10-09T05:42:34.874653+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:42:34.874653+0000","last_undegraded":"2023-10-09T05:42:34.874653+0000","last_fullsized":"2023-10-09T05:42:34.874653+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":192,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:42:34.874391+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:42:34.874391+0000","last_peered":"2023-10-09T05:42:34.874391+0000","last_clean":"2023-10-09T05:42:34.874391+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:42:34.874391+0000","last_undegraded":"2023-10-09T05:42:34.874391+0000","last_fullsized":"2023-10-09T05:42:34.874391+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5466619904,"num_objects":83414,"num_object_clones":0,"num_object_copies":250242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":83414,"num_write_kb":5338496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16398876672,"data_stored":16398876672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83414,"ondisk_log_size":83414,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846691,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626717,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1934472,"kb_used_data":1633152,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91810680,"statfs":{"total":95995035648,"available":94014136320,"internally_reserved":0,"allocated":1672347648,"data_stored":1671812684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:42:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":5.1349999999999998,"15min":3.3180000000000001},"min":{"1min":0.22800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":20.940000000000001,"5min":115.377,"15min":115.377},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.9810000000000001,"5min":4.843,"15min":3.1760000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":20.716999999999999,"5min":116.428,"15min":116.428},"last":0.437}]},{"osd":1,"last update":"Mon Oct 9 05:41:51 2023","interfaces":[{"interface":"back","average":{"1min":11.552,"5min":7.8680000000000003,"15min":3.4500000000000002},"min":{"1min":0.28799999999999998,"5min":0.25,"15min":0.25},"max":{"1min":228.24299999999999,"5min":228.24299999999999,"15min":228.24299999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":11.526999999999999,"5min":7.6799999999999997,"15min":3.488},"min":{"1min":0.29699999999999999,"5min":0.252,"15min":0.252},"max":{"1min":228.22499999999999,"5min":228.22499999999999,"15min":228.22499999999999},"last":0.623}]},{"osd":2,"last update":"Mon Oct 9 05:42:30 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":5.5979999999999999,"15min":3.5129999999999999},"min":{"1min":0.32200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":17.324999999999999,"5min":115.40600000000001,"15min":115.40600000000001},"last":0.82599999999999996},{"interface":"front","average":{"1min":1.9199999999999999,"5min":5.8719999999999999,"15min":3.5529999999999999},"min":{"1min":0.315,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":17.295000000000002,"5min":115.297,"15min":115.297},"last":0.81000000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":6.6369999999999996,"15min":3.161},"min":{"1min":0.32000000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.7089999999999996,"5min":115.244,"15min":115.244},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.80200000000000005,"5min":6.2409999999999997,"15min":3.129},"min":{"1min":0.372,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.0089999999999999,"5min":115.34999999999999,"15min":115.34999999999999},"last":0.73899999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":1.361,"15min":1.7030000000000001},"min":{"1min":0.26100000000000001,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":0.501,"5min":115.214,"15min":125.64},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.44700000000000001,"5min":1.4810000000000001,"15min":1.609},"min":{"1min":0.26000000000000001,"5min":0.254,"15min":0.254},"max":{"1min":0.88500000000000001,"5min":115.22799999999999,"15min":125.625},"last":0.56299999999999994}]},{"osd":5,"last update":"Mon Oct 9 05:42:04 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":2.0979999999999999,"15min":1.851},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.60899999999999999,"5min":115.316,"15min":127.35899999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.50600000000000001,"5min":2.0510000000000002,"15min":1.7370000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.228,"5min":115.30500000000001,"15min":128.93799999999999},"last":0.64000000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.6850000000000001,"15min":2.6360000000000001},"min":{"1min":0.33900000000000002,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":7.319,"5min":115.514,"15min":115.514},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.77000000000000002,"5min":1.716,"15min":1.9139999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.641,"5min":116.53100000000001,"15min":116.53100000000001},"last":0.59599999999999997}]}]},{"osd":1,"up_from":93,"seq":399431958679,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2738796,"kb_used_data":2364516,"kb_used_omap":7,"kb_used_meta":374264,"kb_avail":91006356,"statfs":{"total":95995035648,"available":93190508544,"internally_reserved":0,"allocated":2421264384,"data_stored":2420464130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":383246862},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":12.887,"5min":5.1360000000000001,"15min":3.6099999999999999},"min":{"1min":0.28899999999999998,"5min":0.20300000000000001,"15min":0.185},"max":{"1min":82.459999999999994,"5min":82.459999999999994,"15min":207.858},"last":3.8940000000000001},{"interface":"front","average":{"1min":12.430999999999999,"5min":6.1980000000000004,"15min":4.0720000000000001},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":80.004000000000005,"5min":80.004000000000005,"15min":206.97200000000001},"last":3.7360000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":5.4800000000000004,"15min":3.532},"min":{"1min":0.247,"5min":0.22,"15min":0.184},"max":{"1min":4.3819999999999997,"5min":101.367,"15min":101.367},"last":2.919},{"interface":"front","average":{"1min":0.90400000000000003,"5min":5.1459999999999999,"15min":3.3260000000000001},"min":{"1min":0.26600000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.3079999999999998,"5min":97.286000000000001,"15min":98.409999999999997},"last":3.9239999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":18.77,"5min":9.8010000000000002,"15min":4.2220000000000004},"min":{"1min":0.33600000000000002,"5min":0.24199999999999999,"15min":0.223},"max":{"1min":104.26600000000001,"5min":162.33699999999999,"15min":162.33699999999999},"last":2.96},{"interface":"front","average":{"1min":18.106000000000002,"5min":10.465,"15min":4.4269999999999996},"min":{"1min":0.26900000000000002,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":102.947,"5min":162.376,"15min":162.376},"last":3.847}]},{"osd":4,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":3.1120000000000001,"15min":2.2999999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.0550000000000002,"5min":63.116,"15min":141.32900000000001},"last":3.0550000000000002},{"interface":"front","average":{"1min":1.077,"5min":2.2509999999999999,"15min":1.8779999999999999},"min":{"1min":0.35099999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.956,"5min":40.475999999999999,"15min":141.15899999999999},"last":3.956}]},{"osd":5,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":2.3420000000000001,"15min":2.2850000000000001},"min":{"1min":0.38200000000000001,"5min":0.307,"15min":0.307},"max":{"1min":3.0009999999999999,"5min":40.633000000000003,"15min":235.18700000000001},"last":3.0009999999999999},{"interface":"front","average":{"1min":0.92400000000000004,"5min":3.1640000000000001,"15min":2.6549999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.0139999999999998,"5min":63.162999999999997,"15min":235.79300000000001},"last":3.0139999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":3.5129999999999999,"15min":1.931},"min":{"1min":0.435,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.0329999999999999,"5min":103.60899999999999,"15min":103.60899999999999},"last":3.0329999999999999},{"interface":"front","average":{"1min":1.1519999999999999,"5min":4.1820000000000004,"15min":2.1890000000000001},"min":{"1min":0.39300000000000002,"5min":0.378,"15min":0.33400000000000002},"max":{"1min":2.9740000000000002,"5min":103.679,"15min":103.679},"last":2.9740000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.3959999999999999,"15min":2.4129999999999998},"min":{"1min":0.45100000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.0250000000000004,"5min":42.204000000000001,"15min":126.68300000000001},"last":4.0250000000000004},{"interface":"front","average":{"1min":1.101,"5min":2.444,"15min":2.5070000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.34399999999999997},"max":{"1min":3.9940000000000002,"5min":42.228999999999999,"15min":126.738},"last":3.9940000000000002}]}]},{"osd":0,"up_from":88,"seq":377957122205,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3108112,"kb_used_data":2693832,"kb_used_omap":8,"kb_used_meta":414263,"kb_avail":90637040,"statfs":{"total":95995035648,"available":92812328960,"internally_reserved":0,"allocated":2758483968,"data_stored":2757649203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":424206088},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":2.0339999999999998,"15min":1.3140000000000001},"min":{"1min":0.20200000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":4.9870000000000001,"5min":17.760000000000002,"15min":17.760000000000002},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.97199999999999998,"5min":3.008,"15min":1.6899999999999999},"min":{"1min":0.25900000000000001,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":5.6600000000000001,"5min":26.163,"15min":26.163},"last":4.1340000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:42:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":4.3630000000000004,"15min":2.8380000000000001},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":15.894,"5min":92.230000000000004,"15min":92.230000000000004},"last":8.0649999999999995},{"interface":"front","average":{"1min":1.6850000000000001,"5min":4.282,"15min":2.7290000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.218},"max":{"1min":17.824000000000002,"5min":92.247,"15min":92.247},"last":0.48899999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:41:52 2023","interfaces":[{"interface":"back","average":{"1min":5.4939999999999998,"5min":7.101,"15min":3.1070000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":79.159000000000006,"5min":123.024,"15min":123.024},"last":8.0890000000000004},{"interface":"front","average":{"1min":5.423,"5min":5.9640000000000004,"15min":2.8140000000000001},"min":{"1min":0.28499999999999998,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":79.353999999999999,"5min":99.162000000000006,"15min":99.162000000000006},"last":8.0299999999999994}]},{"osd":4,"last update":"Mon Oct 9 05:42:21 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.948,"15min":1.3069999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.2590000000000003,"5min":69.082999999999998,"15min":69.082999999999998},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.6639999999999999,"5min":2.8490000000000002,"15min":1.7430000000000001},"min":{"1min":0.46700000000000003,"5min":0.34300000000000003,"15min":0.30399999999999999},"max":{"1min":7.1920000000000002,"5min":69.102000000000004,"15min":69.102000000000004},"last":3.27}]},{"osd":5,"last update":"Mon Oct 9 05:42:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":6.484,"15min":2.9580000000000002},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":7.1340000000000003,"5min":281.733,"15min":281.733},"last":3.3119999999999998},{"interface":"front","average":{"1min":1.7370000000000001,"5min":6.3979999999999997,"15min":2.9359999999999999},"min":{"1min":0.437,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.0999999999999996,"5min":282.95400000000001,"15min":282.95400000000001},"last":7.9870000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:42:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":2.286,"15min":1.5509999999999999},"min":{"1min":0.35999999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.1680000000000001,"5min":11.664,"15min":11.664},"last":3.3420000000000001},{"interface":"front","average":{"1min":1.7210000000000001,"5min":2.3340000000000001,"15min":1.5680000000000001},"min":{"1min":0.44900000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.0700000000000003,"5min":10.83,"15min":10.83},"last":7.9320000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:41:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.4710000000000001,"15min":1.425},"min":{"1min":0.39200000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.6159999999999997,"5min":8.9740000000000002,"15min":9.2400000000000002},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.123,"5min":2.2349999999999999,"15min":2.3580000000000001},"min":{"1min":0.42599999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":5.6070000000000002,"5min":9.6329999999999991,"15min":11.068},"last":3.367}]}]},{"osd":2,"up_from":98,"seq":420906795152,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2823384,"kb_used_data":2349840,"kb_used_omap":11,"kb_used_meta":473524,"kb_avail":90921768,"statfs":{"total":95995035648,"available":93103890432,"internally_reserved":0,"allocated":2406236160,"data_stored":2405338704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":484889173},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,7,0,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:42:17 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":3.089,"15min":1.8380000000000001},"min":{"1min":0.22500000000000001,"5min":0.17799999999999999,"15min":0.17199999999999999},"max":{"1min":6.2389999999999999,"5min":21.587,"15min":32.122},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.1850000000000001,"5min":3.5990000000000002,"15min":1.8620000000000001},"min":{"1min":0.27000000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.525,"5min":21.997,"15min":34.066000000000003},"last":0.23899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":5.5309999999999997,"15min":2.7589999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20699999999999999},"max":{"1min":9.0909999999999993,"5min":98.575000000000003,"15min":98.575000000000003},"last":1.2290000000000001},{"interface":"front","average":{"1min":1.454,"5min":4.4219999999999997,"15min":2.3530000000000002},"min":{"1min":0.217,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":8.8510000000000009,"5min":90.135000000000005,"15min":98.397000000000006},"last":3.2850000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:41:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":5.9580000000000002,"15min":2.7280000000000002},"min":{"1min":0.29999999999999999,"5min":0.26700000000000002,"15min":0.24199999999999999},"max":{"1min":13.173,"5min":150.65899999999999,"15min":150.65899999999999},"last":3.1920000000000002},{"interface":"front","average":{"1min":0.97499999999999998,"5min":6.6779999999999999,"15min":3.0539999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.4189999999999996,"5min":150.851,"15min":150.851},"last":2.3530000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":3.1200000000000001,"15min":1.7769999999999999},"min":{"1min":0.311,"5min":0.29099999999999998,"15min":0.27700000000000002},"max":{"1min":8.7769999999999992,"5min":115.84999999999999,"15min":115.84999999999999},"last":3.1669999999999998},{"interface":"front","average":{"1min":0.84699999999999998,"5min":3.2450000000000001,"15min":1.99},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.359,"5min":116.149,"15min":116.149},"last":0.48399999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":2.7469999999999999,"15min":1.738},"min":{"1min":0.42099999999999999,"5min":0.312,"15min":0.312},"max":{"1min":8.6639999999999997,"5min":24.597999999999999,"15min":24.597999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.4119999999999999,"5min":2.2250000000000001,"15min":1.52},"min":{"1min":0.372,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.8190000000000008,"5min":23.919,"15min":23.919},"last":3.129}]},{"osd":6,"last update":"Mon Oct 9 05:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":2.29,"15min":1.669},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.3179999999999996,"5min":13.839,"15min":20.503},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.536,"5min":2.4630000000000001,"15min":1.593},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.34499999999999997},"max":{"1min":8.6180000000000003,"5min":13.757,"15min":13.757},"last":0.52600000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":2.698,"15min":2.641},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.374},"max":{"1min":1.3009999999999999,"5min":24.030999999999999,"15min":24.030999999999999},"last":1.7030000000000001},{"interface":"front","average":{"1min":0.77200000000000002,"5min":2.6219999999999999,"15min":2.125},"min":{"1min":0.42399999999999999,"5min":0.41899999999999998,"15min":0.41599999999999998},"max":{"1min":2.3580000000000001,"5min":24.178000000000001,"15min":24.178000000000001},"last":0.499}]}]},{"osd":3,"up_from":103,"seq":442381631626,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3732708,"kb_used_data":3332060,"kb_used_omap":4,"kb_used_meta":400635,"kb_avail":90012444,"statfs":{"total":95995035648,"available":92172742656,"internally_reserved":0,"allocated":3412029440,"data_stored":3411235139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":410250846},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":4.8460000000000001,"15min":2.5},"min":{"1min":0.184,"5min":0.184,"15min":0.183},"max":{"1min":6.8230000000000004,"5min":167.82400000000001,"15min":167.82400000000001},"last":1.3740000000000001},{"interface":"front","average":{"1min":3.1720000000000002,"5min":5.9649999999999999,"15min":2.9319999999999999},"min":{"1min":0.435,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.3789999999999996,"5min":167.89599999999999,"15min":167.89599999999999},"last":0.77900000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":2.5750000000000002,"5min":3.2770000000000001,"15min":2.254},"min":{"1min":0.32200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.0039999999999996,"5min":48.052,"15min":83.921999999999997},"last":1.222},{"interface":"front","average":{"1min":2.996,"5min":3.286,"15min":2.2410000000000001},"min":{"1min":0.25900000000000001,"5min":0.25,"15min":0.247},"max":{"1min":11.452,"5min":13.551,"15min":83.707999999999998},"last":0.29199999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":2.625,"5min":6.4249999999999998,"15min":3.851},"min":{"1min":0.29499999999999998,"5min":0.25,"15min":0.215},"max":{"1min":12.243,"5min":190.43299999999999,"15min":190.43299999999999},"last":1.2},{"interface":"front","average":{"1min":2.371,"5min":7.5789999999999997,"15min":4.2569999999999997},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":13.788,"5min":190.267,"15min":190.267},"last":0.84999999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":3.1469999999999998,"15min":2.0600000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.7649999999999997,"5min":51.591999999999999,"15min":51.591999999999999},"last":0.752},{"interface":"front","average":{"1min":1.6870000000000001,"5min":2.125,"15min":1.6180000000000001},"min":{"1min":0.315,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.5839999999999996,"5min":26.774999999999999,"15min":45.735999999999997},"last":1.119}]},{"osd":5,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.5699999999999998,"15min":1.7330000000000001},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":11.372999999999999,"5min":13.114000000000001,"15min":47.948999999999998},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.2949999999999999,"15min":1.72},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.548,"5min":26.728000000000002,"15min":47.968000000000004},"last":1.1479999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7310000000000001,"5min":2.1930000000000001,"15min":1.595},"min":{"1min":0.33900000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.6100000000000003,"5min":26.847999999999999,"15min":26.847999999999999},"last":1.1739999999999999},{"interface":"front","average":{"1min":2.3170000000000002,"5min":2.6560000000000001,"15min":1.573},"min":{"1min":0.432,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":11.423,"5min":13.058,"15min":13.058},"last":0.56799999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:41:58 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":2.702,"15min":2.4399999999999999},"min":{"1min":0.435,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":3.9950000000000001,"5min":26.024999999999999,"15min":26.024999999999999},"last":0.82299999999999995},{"interface":"front","average":{"1min":1.4510000000000001,"5min":2.9340000000000002,"15min":2.3159999999999998},"min":{"1min":0.44800000000000001,"5min":0.40899999999999997,"15min":0.35699999999999998},"max":{"1min":8.6039999999999992,"5min":25.995999999999999,"15min":25.995999999999999},"last":0.80200000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019173,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2280136,"kb_used_data":1970624,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91465016,"statfs":{"total":95995035648,"available":93660176384,"internally_reserved":0,"allocated":2017918976,"data_stored":2017383756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:42:28 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":2.4119999999999999,"15min":1.746},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.214},"max":{"1min":4.0469999999999997,"5min":100.504,"15min":100.504},"last":2.8010000000000002},{"interface":"front","average":{"1min":0.93200000000000005,"5min":2.8780000000000001,"15min":2.0230000000000001},"min":{"1min":0.27100000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.5529999999999999,"5min":100.717,"15min":100.717},"last":0.35899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:41:56 2023","interfaces":[{"interface":"back","average":{"1min":16.344000000000001,"5min":4.601,"15min":2.2639999999999998},"min":{"1min":0.32600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":187.369,"5min":187.369,"15min":187.369},"last":0.378},{"interface":"front","average":{"1min":16.341999999999999,"5min":4.8220000000000001,"15min":2.3820000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":187.35300000000001,"5min":187.35300000000001,"15min":187.35300000000001},"last":0.41599999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":5.6040000000000001,"15min":4.0460000000000003},"min":{"1min":0.313,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":1.694,"5min":196.93700000000001,"15min":196.93700000000001},"last":0.81299999999999994},{"interface":"front","average":{"1min":1.2070000000000001,"5min":5.508,"15min":3.9729999999999999},"min":{"1min":0.32400000000000001,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":6.6429999999999998,"5min":197.08099999999999,"15min":197.08099999999999},"last":0.48199999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:42:47 2023","interfaces":[{"interface":"back","average":{"1min":6.1219999999999999,"5min":9.1140000000000008,"15min":3.7949999999999999},"min":{"1min":0.39000000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":72.391999999999996,"5min":211.48699999999999,"15min":211.48699999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":5.6529999999999996,"5min":8.8309999999999995,"15min":3.661},"min":{"1min":0.34899999999999998,"5min":0.34799999999999998,"15min":0.309},"max":{"1min":71.721999999999994,"5min":211.31100000000001,"15min":211.31100000000001},"last":0.504}]},{"osd":5,"last update":"Mon Oct 9 05:42:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.58499999999999996,"15min":1.5629999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.9099999999999999,"5min":3.5259999999999998,"15min":113.35899999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55700000000000005,"15min":1.514},"min":{"1min":0.33800000000000002,"5min":0.317,"15min":0.30599999999999999},"max":{"1min":1.9239999999999999,"5min":3.3849999999999998,"15min":113.32299999999999},"last":0.44}]},{"osd":6,"last update":"Mon Oct 9 05:42:01 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.58599999999999997,"15min":0.70099999999999996},"min":{"1min":0.28599999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":0.79600000000000004,"5min":5.7599999999999998,"15min":11.944000000000001},"last":0.37},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.82399999999999995,"15min":0.85899999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.70199999999999996,"5min":7.6500000000000004,"15min":10.356},"last":0.45200000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:41:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.67100000000000004,"15min":2.3809999999999998},"min":{"1min":0.33300000000000002,"5min":0.31900000000000001,"15min":0.312},"max":{"1min":2.3660000000000001,"5min":4.5819999999999999,"15min":30.393999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.71199999999999997,"15min":2.6589999999999998},"min":{"1min":0.378,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.3540000000000001,"5min":4.5999999999999996,"15min":30.334},"last":0.40500000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822945,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1932088,"kb_used_data":1630768,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91813064,"statfs":{"total":95995035648,"available":94016577536,"internally_reserved":0,"allocated":1669906432,"data_stored":1669383044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:42:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.613,"15min":3.032},"min":{"1min":0.26500000000000001,"5min":0.217,"15min":0.19400000000000001},"max":{"1min":3.198,"5min":10.712999999999999,"15min":165.90199999999999},"last":0.749},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.6559999999999999,"15min":3.0800000000000001},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.215},"max":{"1min":1.742,"5min":12.263999999999999,"15min":165.96700000000001},"last":8.2539999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:41:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":4.3390000000000004,"15min":2.214},"min":{"1min":0.26700000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.0229999999999997,"5min":87.769000000000005,"15min":87.769000000000005},"last":0.624},{"interface":"front","average":{"1min":1.4239999999999999,"5min":4.891,"15min":2.339},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.24399999999999999},"max":{"1min":12.228,"5min":87.864000000000004,"15min":87.864000000000004},"last":0.69899999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:42:24 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":3.5089999999999999,"15min":4.093},"min":{"1min":0.36899999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.7850000000000001,"5min":73.311999999999998,"15min":290.697},"last":2.2949999999999999},{"interface":"front","average":{"1min":1.21,"5min":3.3860000000000001,"15min":3.9990000000000001},"min":{"1min":0.40799999999999997,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":4.1299999999999999,"5min":73.280000000000001,"15min":290.786},"last":0.78400000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:41:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0489999999999999,"5min":6.7690000000000001,"15min":3.0550000000000002},"min":{"1min":0.40799999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":36.395000000000003,"5min":95.236999999999995,"15min":95.236999999999995},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.758,"5min":6.3600000000000003,"15min":2.8639999999999999},"min":{"1min":0.41199999999999998,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":36.406999999999996,"5min":93.471000000000004,"15min":93.471000000000004},"last":0.64500000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:41:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.60999999999999999,"15min":0.72099999999999997},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":0.84999999999999998,"5min":5.6459999999999999,"15min":9.3510000000000009},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.58999999999999997,"15min":0.94499999999999995},"min":{"1min":0.254,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":0.78100000000000003,"5min":4.8209999999999997,"15min":14.236000000000001},"last":0.46000000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:42:05 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.96299999999999997,"15min":1.365},"min":{"1min":0.34999999999999998,"5min":0.27600000000000002,"15min":0.26600000000000001},"max":{"1min":1.3080000000000001,"5min":27.146000000000001,"15min":95.010999999999996},"last":0.495},{"interface":"front","average":{"1min":0.64500000000000002,"5min":1.002,"15min":1.27},"min":{"1min":0.36699999999999999,"5min":0.28499999999999998,"15min":0.27700000000000002},"max":{"1min":2.2389999999999999,"5min":27.215,"15min":94.194000000000003},"last":0.67000000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:42:05 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.84099999999999997,"15min":1.486},"min":{"1min":0.39900000000000002,"5min":0.34399999999999997,"15min":0.28999999999999998},"max":{"1min":2.4769999999999999,"5min":5.5570000000000004,"15min":9.9849999999999994},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.76900000000000002,"15min":1.7},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.2270000000000001,"5min":5.0549999999999997,"15min":14.432},"last":0.59499999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2775121920,"data_stored":2775121920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2429681664,"data_stored":2429681664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2407923712,"data_stored":2407923712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3414687744,"data_stored":3414687744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2015952896,"data_stored":2015952896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1680801792,"data_stored":1680801792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1674706944,"data_stored":1674706944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:42:56.913 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 21 2023-10-09T05:42:56.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:42:57.022 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:57.071 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:42:57.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:57.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:42:57.179 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:42:57.209 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:57.260 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:42:57.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:57.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:42:57.366 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:57.394 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:57.398 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:42:57.399471+0000 min lat: 0.00134385 max lat: 0.28677 avg lat: 0.0146425 2023-10-09T05:42:57.398 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:42:57.399 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 80 16 87481 87465 68.2629 48.1875 0.0131888 0.0146425 2023-10-09T05:42:57.447 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:57.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:57.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:57.469 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:42:57.553 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:42:57.580 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:57.625 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:42:57.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:57.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:42:57.633 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:42:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:42:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:42:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:42:57.634 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:42:57.733 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:42:57.868 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:42:57.920 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:57 smithi008 ceph-mon[184418]: pgmap v662: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T05:42:57.925 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:57 smithi008 ceph-mon[191227]: pgmap v662: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T05:42:58.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:57 smithi191 ceph-mon[161669]: pgmap v662: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T05:42:58.403 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 81 16 88164 88148 67.9435 42.6875 0.0251988 0.0147126 2023-10-09T05:42:58.970 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:42:59.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2295223840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:58 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:58 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:59.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:58 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:59.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2295223840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:59.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:58 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:59.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:58 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:59.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:58 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:59.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2295223840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:59.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:58 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:42:59.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:58 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:42:59.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:58 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:59.403 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 82 16 88852 88836 67.6395 43 0.010365 0.0147799 2023-10-09T05:42:59.570 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:42:59.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:42:59.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:42:59.689 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:42:59.774 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:00.211 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[184418]: pgmap v663: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s 2023-10-09T05:43:00.212 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:00.212 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[184418]: failed to encode map e228 with expected crc 2023-10-09T05:43:00.212 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[184418]: failed to encode map e228 with expected crc 2023-10-09T05:43:00.212 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[184418]: failed to encode map e228 with expected crc 2023-10-09T05:43:00.212 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:43:00.212 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[184418]: osdmap e228: 8 total, 8 up, 7 in 2023-10-09T05:43:00.212 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[191227]: pgmap v663: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s 2023-10-09T05:43:00.212 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:00.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[191227]: failed to encode map e228 with expected crc 2023-10-09T05:43:00.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[191227]: failed to encode map e228 with expected crc 2023-10-09T05:43:00.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[191227]: failed to encode map e228 with expected crc 2023-10-09T05:43:00.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:43:00.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:42:59 smithi008 ceph-mon[191227]: osdmap e228: 8 total, 8 up, 7 in 2023-10-09T05:43:00.214 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:43:00.223 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:00.238 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:00.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:59 smithi191 ceph-mon[161669]: pgmap v663: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s 2023-10-09T05:43:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:59 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:59 smithi191 ceph-mon[161669]: failed to encode map e228 with expected crc 2023-10-09T05:43:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:59 smithi191 ceph-mon[161669]: failed to encode map e228 with expected crc 2023-10-09T05:43:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:59 smithi191 ceph-mon[161669]: failed to encode map e228 with expected crc 2023-10-09T05:43:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:59 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:43:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:42:59 smithi191 ceph-mon[161669]: osdmap e228: 8 total, 8 up, 7 in 2023-10-09T05:43:00.348 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:00.379 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:00.403 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 83 16 89641 89625 67.4189 49.3125 0.00766128 0.0148271 2023-10-09T05:43:00.436 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:00.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:00.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:00.544 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:00.657 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:00.798 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:43:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:43:00] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.33.4" 2023-10-09T05:43:00.884 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:00.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:00.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:00.911 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:43:00.971 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:43:00.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:01.021 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:43:01.024 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:01.072 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:01.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:01.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:01.078 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:43:01.087 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:00 smithi008 ceph-mon[184418]: pgmap v665: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s 2023-10-09T05:43:01.087 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1469857267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:01.087 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:00 smithi008 ceph-mon[191227]: pgmap v665: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s 2023-10-09T05:43:01.087 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1469857267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:01.111 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:43:01.152 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:43:01.178 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:01.184 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:43:01.204 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:00 smithi191 ceph-mon[161669]: pgmap v665: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s 2023-10-09T05:43:01.205 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1469857267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:01.207 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:01.217 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:43:01.236 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:43:01.236 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:43:01.236 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:43:01.236 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:43:01.236 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:43:01.236 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:43:01.236 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:43:01.236 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:43:01.265 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:01.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:01.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:01.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:01.404 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 84 16 90334 90318 67.1321 43.3125 0.0170465 0.0148904 2023-10-09T05:43:01.404 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:01.464 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:01.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:01.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:01.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:01.606 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:01.666 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:01.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:01.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:01.773 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:01.861 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:02.270 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:02.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:02.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:02.279 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:43:02.375 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:02.404 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 85 16 91127 91111 66.9255 49.5625 0.0339343 0.0149368 2023-10-09T05:43:02.460 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:02.896 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:02.941 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:02.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:02.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:03.052 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:03.131 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:03.277 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:03.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:03.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:03.383 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:03.404 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 86 16 91798 91782 66.6352 41.9375 0.0159667 0.0150013 2023-10-09T05:43:03.480 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:03.535 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:03.588 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:03.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:03.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:03.702 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:03.768 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:03.974 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:43:04.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:03 smithi008 ceph-mon[191227]: pgmap v666: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 725 op/s 2023-10-09T05:43:04.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:03 smithi008 ceph-mon[184418]: pgmap v666: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 725 op/s 2023-10-09T05:43:04.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:03 smithi191 ceph-mon[161669]: pgmap v666: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 725 op/s 2023-10-09T05:43:04.404 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 87 16 92473 92457 66.3544 42.1875 0.010028 0.0150653 2023-10-09T05:43:04.842 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:04.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:04.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:04.889 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:43:04.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:04.978 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:05.032 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:05.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:05.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:05.137 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:05.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/954415352' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:43:05.148 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/954415352' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:43:05.225 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:05.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/954415352' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:43:05.409 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 88 16 93121 93105 66.0608 40.5 0.0287778 0.0151318 2023-10-09T05:43:05.539 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:05.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:05.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:05.643 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:05.738 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:05.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:05 smithi008 ceph-mon[184418]: pgmap v667: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 730 op/s 2023-10-09T05:43:05.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:05 smithi008 ceph-mon[191227]: pgmap v667: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 730 op/s 2023-10-09T05:43:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:05 smithi191 ceph-mon[161669]: pgmap v667: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 730 op/s 2023-10-09T05:43:06.404 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 89 16 93815 93799 65.8061 43.375 0.0206192 0.0151896 2023-10-09T05:43:06.417 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:43:06.429 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:06.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:06.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:06.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:06.565 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:06.614 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:06.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:06.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:06.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:06.749 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:06.800 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:06.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:06.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:06.906 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:06.937 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:06.988 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:06.988 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:06.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:06.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:07.095 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:07.141 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:07.141 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:07.206 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:07.404 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total time run: 90.0107 2023-10-09T05:43:07.404 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total writes made: 94493 2023-10-09T05:43:07.405 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Write size: 65536 2023-10-09T05:43:07.405 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object size: 65536 2023-10-09T05:43:07.405 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Bandwidth (MB/sec): 65.6123 2023-10-09T05:43:07.405 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Bandwidth: 27.3327 2023-10-09T05:43:07.405 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max bandwidth (MB/sec): 170.688 2023-10-09T05:43:07.405 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min bandwidth (MB/sec): 40.1875 2023-10-09T05:43:07.406 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average IOPS: 1049 2023-10-09T05:43:07.406 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev IOPS: 437.323 2023-10-09T05:43:07.406 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max IOPS: 2731 2023-10-09T05:43:07.406 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min IOPS: 643 2023-10-09T05:43:07.406 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average Latency(s): 0.0152369 2023-10-09T05:43:07.406 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Latency(s): 0.0172134 2023-10-09T05:43:07.406 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max latency(s): 0.28677 2023-10-09T05:43:07.406 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min latency(s): 0.00134385 2023-10-09T05:43:07.410 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:43:08.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:07 smithi008 ceph-mon[184418]: pgmap v668: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 727 op/s 2023-10-09T05:43:08.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:07 smithi008 ceph-mon[191227]: pgmap v668: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 727 op/s 2023-10-09T05:43:08.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:07 smithi191 ceph-mon[161669]: pgmap v668: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 727 op/s 2023-10-09T05:43:08.377 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:08.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:08.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:08.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:08.509 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:08.511 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:43:08.557 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:08.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:08.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:08.662 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:08.690 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:08.747 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:08.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:08.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:08.852 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:08.923 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:43:08.962 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3893787840' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:08 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3893787840' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:08 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3893787840' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:08 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:10.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[184418]: pgmap v669: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s 2023-10-09T05:43:10.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[184418]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[184418]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[184418]: osdmap e229: 8 total, 8 up, 7 in 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[184418]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[184418]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[191227]: pgmap v669: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:10.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[191227]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[191227]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:43:10.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[191227]: osdmap e229: 8 total, 8 up, 7 in 2023-10-09T05:43:10.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[191227]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.164 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:09 smithi008 ceph-mon[191227]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.226 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:10.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:10.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:09 smithi191 ceph-mon[161669]: pgmap v669: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s 2023-10-09T05:43:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:09 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:09 smithi191 ceph-mon[161669]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:09 smithi191 ceph-mon[161669]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:09 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:43:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:09 smithi191 ceph-mon[161669]: osdmap e229: 8 total, 8 up, 7 in 2023-10-09T05:43:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:09 smithi191 ceph-mon[161669]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:09 smithi191 ceph-mon[161669]: failed to encode map e229 with expected crc 2023-10-09T05:43:10.344 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:10.398 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:10.420 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:10.609 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:10.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:10.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:10.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:10.721 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:43:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:43:10] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.33.4" 2023-10-09T05:43:10.750 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:10.811 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:10.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:10.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:10.917 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:10.980 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:43:11.016 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:43:11.018 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:11.079 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:43:11.115 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:43:11.175 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:43:11.215 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:43:11.252 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:43:11.253 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:11.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:11.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:11.292 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:43:11.328 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:43:11.328 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:43:11.329 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:43:11.329 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:43:11.329 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:43:11.329 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:43:11.329 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:43:11.329 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:43:11.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:11.392 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:11.419 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:43:11.419 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:43:11.419 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T05:43:11.419 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T05:43:11.440 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:11.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:11.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:11.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:11.580 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:11.633 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:11.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:11.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:11.743 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:11.840 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:12.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:11 smithi008 ceph-mon[191227]: pgmap v671: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 707 op/s 2023-10-09T05:43:12.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1448431848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:12.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:11 smithi008 ceph-mon[184418]: pgmap v671: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 707 op/s 2023-10-09T05:43:12.224 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1448431848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:12.248 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:43:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:11 smithi191 ceph-mon[161669]: pgmap v671: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 41 MiB/s wr, 707 op/s 2023-10-09T05:43:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1448431848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:13.061 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:13.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:13.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:13.177 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:13.260 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:13.286 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:13.417 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:13.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:13.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:13.527 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:13.622 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:13.635 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:13.739 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:13.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:13.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:13.790 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:43:13.790 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:43:13.790 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.790 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:99028 7.1 e229/213 7:867256d5:::benchmark_data_smithi008_354649_object4532:head v 229'6103, mlcod=229'6101)", 2023-10-09T05:43:13.790 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:13.776519+0000", 2023-10-09T05:43:13.790 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0051254689999999997, 2023-10-09T05:43:13.790 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051571730000000001, 2023-10-09T05:43:13.790 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:43:13.790 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:43:13.791 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:43:13.791 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.791 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:43:13.791 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.776519+0000", 2023-10-09T05:43:13.791 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:43:13.791 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.791 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.791 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:43:13.791 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.776519+0000", 2023-10-09T05:43:13.791 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:43:13.791 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.776518+0000", 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.776529+0000", 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0771000000000001e-05 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.792 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.776545+0000", 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5973000000000001e-05 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.776552+0000", 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.559e-06 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.776612+0000", 2023-10-09T05:43:13.793 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9660000000000001e-05 2023-10-09T05:43:13.794 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.794 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.794 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:43:13.794 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.776631+0000", 2023-10-09T05:43:13.794 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9298999999999999e-05 2023-10-09T05:43:13.794 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:43:13.794 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:43:13.794 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:43:13.794 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.794 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.795 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:99022 7.3 7:c6f81b5b:::benchmark_data_smithi008_354649_object4526:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T05:43:13.795 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:13.779778+0000", 2023-10-09T05:43:13.795 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.001866014, 2023-10-09T05:43:13.795 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019763770000000001, 2023-10-09T05:43:13.795 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:43:13.795 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:43:13.795 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:43:13.795 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:43:13.795 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:43:13.795 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 99022 2023-10-09T05:43:13.795 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.779778+0000", 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.779778+0000", 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.796 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.779777+0000", 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.779784+0000", 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4669999999999998e-06 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:43:13.797 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.779797+0000", 2023-10-09T05:43:13.798 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2741e-05 2023-10-09T05:43:13.798 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.798 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:43:13.798 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.779805+0000", 2023-10-09T05:43:13.798 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1980000000000001e-06 2023-10-09T05:43:13.798 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.798 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:43:13.798 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.779865+0000", 2023-10-09T05:43:13.798 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9910000000000001e-05 2023-10-09T05:43:13.799 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.799 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:43:13.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.779998+0000", 2023-10-09T05:43:13.799 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013286799999999999 2023-10-09T05:43:13.799 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.799 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:43:13.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.780113+0000", 2023-10-09T05:43:13.799 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011537200000000001 2023-10-09T05:43:13.799 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:43:13.800 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:43:13.800 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:43:13.800 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.800 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.800 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:99033 7.1 e229/213)", 2023-10-09T05:43:13.800 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:13.780583+0000", 2023-10-09T05:43:13.800 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0010609650000000001, 2023-10-09T05:43:13.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0012560290000000001, 2023-10-09T05:43:13.800 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:43:13.800 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "queued for pg", 2023-10-09T05:43:13.800 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:43:13.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.801 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:43:13.801 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.780583+0000", 2023-10-09T05:43:13.801 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:43:13.801 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.801 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:43:13.801 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.780583+0000", 2023-10-09T05:43:13.801 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:43:13.801 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.780583+0000", 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.780590+0000", 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.114e-06 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:43:13.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.780600+0000", 2023-10-09T05:43:13.803 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0400999999999999e-05 2023-10-09T05:43:13.803 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:43:13.803 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:43:13.803 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:43:13.803 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:13.780607+0000", 2023-10-09T05:43:13.803 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4969999999999998e-06 2023-10-09T05:43:13.803 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:43:13.803 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:43:13.803 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:43:13.803 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:43:13.803 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:43:13.804 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 3 2023-10-09T05:43:13.804 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:43:13.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:13.883 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:13.913 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:13 smithi008 ceph-mon[191227]: pgmap v672: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 31 MiB/s wr, 707 op/s 2023-10-09T05:43:13.913 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:13.932 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:13.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:14.038 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:14.138 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:14.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:13 smithi008 ceph-mon[184418]: pgmap v672: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 31 MiB/s wr, 707 op/s 2023-10-09T05:43:14.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:14.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:13 smithi191 ceph-mon[161669]: pgmap v672: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 31 MiB/s wr, 707 op/s 2023-10-09T05:43:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:14.363 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:14.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:14.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:14.475 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:14.591 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:15.043 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:43:15.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:15.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:15.695 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:15.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:15.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:15.792 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:43:15.798 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:15.917 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:15.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:15 smithi008 ceph-mon[184418]: pgmap v673: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 702 op/s 2023-10-09T05:43:15.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:15 smithi008 ceph-mon[191227]: pgmap v673: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 702 op/s 2023-10-09T05:43:16.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:15 smithi191 ceph-mon[161669]: pgmap v673: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 24 MiB/s wr, 702 op/s 2023-10-09T05:43:16.533 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:16.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:16.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:16.644 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:16.680 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:16.787 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:16.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:16.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:16.898 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:16.911 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:17.009 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:17.243 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:16 smithi008 ceph-mon[184418]: pgmap v674: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 17 MiB/s wr, 694 op/s 2023-10-09T05:43:17.246 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:16 smithi008 ceph-mon[191227]: pgmap v674: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 17 MiB/s wr, 694 op/s 2023-10-09T05:43:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:16 smithi191 ceph-mon[161669]: pgmap v674: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 17 MiB/s wr, 694 op/s 2023-10-09T05:43:17.466 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:17.555 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:17.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:17.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:17.668 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:17.768 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:18.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:17 smithi008 conmon[184394]: 2023-10-09T05:43:17.919+0000 7f8a66521700 -1 mon.a@0(leader).osd e229 definitely_dead 0 2023-10-09T05:43:18.283 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:18.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:18.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:18.393 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:18.433 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:18.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:18.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2994248049' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:43:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:43:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3939550792' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:18.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:18.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:18.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2994248049' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:43:18.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:43:18.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3939550792' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:18.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:18.898 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:43:18.906 INFO:teuthology.orchestra.run.smithi008.stderr:marked in osd.7. 2023-10-09T05:43:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:18 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:18 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2994248049' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:43:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:43:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3939550792' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:19.288 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:19.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:19.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:19.392 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:19.426 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:19.473 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:19.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:19.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:19.579 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:19.608 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:19 smithi191 ceph-mon[161669]: pgmap v675: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 8.7 MiB/s wr, 690 op/s 2023-10-09T05:43:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:19 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:43:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:19 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:43:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:19 smithi191 ceph-mon[161669]: failed to encode map e230 with expected crc 2023-10-09T05:43:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:19 smithi191 ceph-mon[161669]: failed to encode map e230 with expected crc 2023-10-09T05:43:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:43:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:43:19.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:19 smithi191 ceph-mon[161669]: osdmap e230: 8 total, 8 up, 8 in 2023-10-09T05:43:19.889 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[184418]: pgmap v675: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 8.7 MiB/s wr, 690 op/s 2023-10-09T05:43:19.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:43:19.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:43:19.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[184418]: failed to encode map e230 with expected crc 2023-10-09T05:43:19.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[184418]: failed to encode map e230 with expected crc 2023-10-09T05:43:19.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:43:19.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:43:19.890 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[184418]: osdmap e230: 8 total, 8 up, 8 in 2023-10-09T05:43:19.891 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[191227]: pgmap v675: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 8.7 MiB/s wr, 690 op/s 2023-10-09T05:43:19.891 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:43:19.891 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:43:19.892 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[191227]: failed to encode map e230 with expected crc 2023-10-09T05:43:19.892 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[191227]: failed to encode map e230 with expected crc 2023-10-09T05:43:19.892 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:43:19.892 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:43:19.892 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:19 smithi008 ceph-mon[191227]: osdmap e230: 8 total, 8 up, 8 in 2023-10-09T05:43:20.607 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T05:43:20.874 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[191227]: failed to encode map e231 with expected crc 2023-10-09T05:43:20.874 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[191227]: failed to encode map e231 with expected crc 2023-10-09T05:43:20.874 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[191227]: failed to encode map e231 with expected crc 2023-10-09T05:43:20.874 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[191227]: failed to encode map e231 with expected crc 2023-10-09T05:43:20.874 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[191227]: failed to encode map e231 with expected crc 2023-10-09T05:43:20.874 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[191227]: osdmap e231: 8 total, 8 up, 8 in 2023-10-09T05:43:20.874 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[191227]: failed to encode map e231 with expected crc 2023-10-09T05:43:20.874 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[191227]: failed to encode map e231 with expected crc 2023-10-09T05:43:20.875 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:43:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:43:20] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.33.4" 2023-10-09T05:43:21.110 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:21.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:21.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:21.129 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[184418]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.129 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[184418]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.129 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[184418]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.129 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[184418]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.129 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[184418]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[184418]: osdmap e231: 8 total, 8 up, 8 in 2023-10-09T05:43:21.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[184418]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:20 smithi008 ceph-mon[184418]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:21.258 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:20 smithi191 ceph-mon[161669]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.258 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:20 smithi191 ceph-mon[161669]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.258 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:20 smithi191 ceph-mon[161669]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.258 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:20 smithi191 ceph-mon[161669]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.259 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:20 smithi191 ceph-mon[161669]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.259 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:20 smithi191 ceph-mon[161669]: osdmap e231: 8 total, 8 up, 8 in 2023-10-09T05:43:21.259 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:20 smithi191 ceph-mon[161669]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.259 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:20 smithi191 ceph-mon[161669]: failed to encode map e231 with expected crc 2023-10-09T05:43:21.260 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:21.341 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:21.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:21.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:21.452 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:21.531 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:21.604 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:21.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:21.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:21.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:21.766 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:21.845 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:21.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:21.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:21.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:21.981 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:22.049 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:22.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:22.110 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:43:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:43:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:43:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:43:22.111 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:43:22.155 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:22.215 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:43:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: pgmap v678: 89 pgs: 1 remapped+peering, 7 peering, 1 active+recovering, 80 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 564 B/s, 0 objects/s recovering 2023-10-09T05:43:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: osdmap e232: 8 total, 8 up, 8 in 2023-10-09T05:43:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[184418]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: pgmap v678: 89 pgs: 1 remapped+peering, 7 peering, 1 active+recovering, 80 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 564 B/s, 0 objects/s recovering 2023-10-09T05:43:22.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: osdmap e232: 8 total, 8 up, 8 in 2023-10-09T05:43:22.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:21 smithi008 ceph-mon[191227]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.247 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:22.270 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:43:22.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: pgmap v678: 89 pgs: 1 remapped+peering, 7 peering, 1 active+recovering, 80 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 564 B/s, 0 objects/s recovering 2023-10-09T05:43:22.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: osdmap e232: 8 total, 8 up, 8 in 2023-10-09T05:43:22.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:21 smithi191 ceph-mon[161669]: failed to encode map e232 with expected crc 2023-10-09T05:43:22.331 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:43:22.379 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:43:22.428 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:22.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:22.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:22.434 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:43:22.477 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:43:22.507 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:43:22.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:22.549 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:43:22.556 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:43:22.556 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:43:22.556 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:43:22.556 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:43:22.556 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:43:22.557 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:43:22.557 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:43:22.557 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:43:22.585 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:22.673 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:22.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:22.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:22.786 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:22.820 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:22.889 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:22.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:22.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:22.998 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:23.069 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:23.132 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:43:23.205 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[184418]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.205 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[184418]: osdmap e233: 8 total, 8 up, 8 in 2023-10-09T05:43:23.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[184418]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[184418]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[184418]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[184418]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[184418]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3044142439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:23.206 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[191227]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[191227]: osdmap e233: 8 total, 8 up, 8 in 2023-10-09T05:43:23.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[191227]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[191227]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[191227]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[191227]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[191227]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:22 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3044142439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:22 smithi191 ceph-mon[161669]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:22 smithi191 ceph-mon[161669]: osdmap e233: 8 total, 8 up, 8 in 2023-10-09T05:43:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:22 smithi191 ceph-mon[161669]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:22 smithi191 ceph-mon[161669]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:22 smithi191 ceph-mon[161669]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:22 smithi191 ceph-mon[161669]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:22 smithi191 ceph-mon[161669]: failed to encode map e233 with expected crc 2023-10-09T05:43:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:22 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3044142439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: pgmap v681: 89 pgs: 3 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 32/260667 objects degraded (0.012%); 5449/260667 objects misplaced (2.090%); 65 KiB/s, 0 objects/s recovering 2023-10-09T05:43:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: Health check failed: Degraded data redundancy: 32/260667 objects degraded (0.012%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:43:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: osdmap e234: 8 total, 8 up, 8 in 2023-10-09T05:43:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[184418]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: pgmap v681: 89 pgs: 3 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 32/260667 objects degraded (0.012%); 5449/260667 objects misplaced (2.090%); 65 KiB/s, 0 objects/s recovering 2023-10-09T05:43:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: Health check failed: Degraded data redundancy: 32/260667 objects degraded (0.012%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:43:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: osdmap e234: 8 total, 8 up, 8 in 2023-10-09T05:43:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:23 smithi008 ceph-mon[191227]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: pgmap v681: 89 pgs: 3 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 32/260667 objects degraded (0.012%); 5449/260667 objects misplaced (2.090%); 65 KiB/s, 0 objects/s recovering 2023-10-09T05:43:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: Health check failed: Degraded data redundancy: 32/260667 objects degraded (0.012%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:43:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: osdmap e234: 8 total, 8 up, 8 in 2023-10-09T05:43:24.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:23 smithi191 ceph-mon[161669]: failed to encode map e234 with expected crc 2023-10-09T05:43:24.372 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:24.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:24.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:24.486 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:24.494 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:43:24.514 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:24.567 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:24.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:24.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:24.681 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:24.804 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:24.807 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:24.950 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:24 smithi008 ceph-mon[184418]: pgmap v683: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 73 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 17979/258576 objects degraded (6.953%); 5449/258576 objects misplaced (2.107%); 60 KiB/s, 5 objects/s recovering 2023-10-09T05:43:24.957 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:24 smithi008 ceph-mon[191227]: pgmap v683: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 73 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 17979/258576 objects degraded (6.953%); 5449/258576 objects misplaced (2.107%); 60 KiB/s, 5 objects/s recovering 2023-10-09T05:43:24.980 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:25.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:25.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:25.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:25.140 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:25.205 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:25.214 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:24 smithi191 ceph-mon[161669]: pgmap v683: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 73 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 17979/258576 objects degraded (6.953%); 5449/258576 objects misplaced (2.107%); 60 KiB/s, 5 objects/s recovering 2023-10-09T05:43:25.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:25.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:25.317 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:25.354 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:25.422 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:25.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:25.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:25.531 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:25.564 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:25.608 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:43:25.608 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:43:25.608 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T05:43:25.609 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:43:25.609 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:43:25.609 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T05:43:25.633 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:25.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:25.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:25.739 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:25.770 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:25.823 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:25.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:25.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:25.928 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:26.011 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:26.174 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:26.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:26.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:26.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:26.306 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:26.348 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:26.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:26.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:26.455 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:26.497 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:26.543 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:43:26.556 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:26.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:26.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:26.661 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:26.695 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:26.753 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:26.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:26.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:26.859 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:26.953 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:27.124 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:27.135 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:27.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:27.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:27.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:27.286 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:27.354 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:27.360 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:27.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:27.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:27.469 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:27.504 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:27.544 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:27 smithi008 ceph-mon[184418]: pgmap v684: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 1.5 KiB/s wr, 391 op/s; 23849/256215 objects degraded (9.308%); 5449/256215 objects misplaced (2.127%); 725 KiB/s, 1 keys/s, 7 objects/s recovering 2023-10-09T05:43:27.544 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:27 smithi008 ceph-mon[191227]: pgmap v684: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 1.5 KiB/s wr, 391 op/s; 23849/256215 objects degraded (9.308%); 5449/256215 objects misplaced (2.127%); 725 KiB/s, 1 keys/s, 7 objects/s recovering 2023-10-09T05:43:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:27 smithi191 ceph-mon[161669]: pgmap v684: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 1.5 KiB/s wr, 391 op/s; 23849/256215 objects degraded (9.308%); 5449/256215 objects misplaced (2.127%); 725 KiB/s, 1 keys/s, 7 objects/s recovering 2023-10-09T05:43:27.567 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:27.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:27.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:27.672 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:27.716 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:27.786 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:27.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:27.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:27.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:27.923 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:27.991 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:27.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:27.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:28.099 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:28.191 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:28.867 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:28.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:28.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:28.975 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:29.024 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23887/253380 objects degraded (9.427%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:29.025 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:28 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:29.025 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:28 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:29.025 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23887/253380 objects degraded (9.427%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:29.025 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:28 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:29.025 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:28 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:29.034 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:29.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23887/253380 objects degraded (9.427%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:29.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:28 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:29.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:28 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:29.840 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:29.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:29.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:29.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:29 smithi008 ceph-mon[191227]: pgmap v685: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.2 KiB/s wr, 402 op/s; 23887/253380 objects degraded (9.427%); 5274/253380 objects misplaced (2.081%); 1.7 MiB/s, 1 keys/s, 24 objects/s recovering 2023-10-09T05:43:29.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:29.992 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:30.042 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:30.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:30.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:30.147 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:30.184 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:29 smithi191 ceph-mon[161669]: pgmap v685: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.2 KiB/s wr, 402 op/s; 23887/253380 objects degraded (9.427%); 5274/253380 objects misplaced (2.081%); 1.7 MiB/s, 1 keys/s, 24 objects/s recovering 2023-10-09T05:43:30.187 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:29 smithi008 ceph-mon[184418]: pgmap v685: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.2 KiB/s wr, 402 op/s; 23887/253380 objects degraded (9.427%); 5274/253380 objects misplaced (2.081%); 1.7 MiB/s, 1 keys/s, 24 objects/s recovering 2023-10-09T05:43:30.268 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:30.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:30.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:30.378 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:30.480 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:30.686 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:30.779 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:43:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:43:30] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.33.4" 2023-10-09T05:43:30.829 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:30.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:30.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:30.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:30.971 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:31.039 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:31.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:31.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:31.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:31.207 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:31.282 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:31.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:31.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:31.393 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:31.427 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:31.478 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:31.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:31.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:31.585 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:31.617 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:31.680 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:31.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:31.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:31.788 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:31.870 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:43:31.870 INFO:teuthology.orchestra.run.smithi008.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:43:31.870 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:43:31.880 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:31.976 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:43:32.203 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:31 smithi008 ceph-mon[184418]: pgmap v686: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.1 KiB/s wr, 492 op/s; 23887/249081 objects degraded (9.590%); 5274/249081 objects misplaced (2.117%); 1.5 MiB/s, 1 keys/s, 21 objects/s recovering 2023-10-09T05:43:32.204 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:31 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3728645432' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:43:32.204 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:31 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:43:32.204 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:32.204 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:32.204 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:31 smithi008 ceph-mon[191227]: pgmap v686: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.1 KiB/s wr, 492 op/s; 23887/249081 objects degraded (9.590%); 5274/249081 objects misplaced (2.117%); 1.5 MiB/s, 1 keys/s, 21 objects/s recovering 2023-10-09T05:43:32.205 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:31 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3728645432' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:43:32.205 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:31 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:43:32.205 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:32.205 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:31 smithi191 ceph-mon[161669]: pgmap v686: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.1 KiB/s wr, 492 op/s; 23887/249081 objects degraded (9.590%); 5274/249081 objects misplaced (2.117%); 1.5 MiB/s, 1 keys/s, 21 objects/s recovering 2023-10-09T05:43:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:31 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3728645432' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:43:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:31 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:43:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:32 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:32 smithi191 ceph-mon[161669]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:32 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:43:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:32 smithi191 ceph-mon[161669]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:32 smithi191 ceph-mon[161669]: osdmap e235: 8 total, 8 up, 8 in 2023-10-09T05:43:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:32 smithi191 ceph-mon[161669]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:32 smithi191 ceph-mon[161669]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:32 smithi191 ceph-mon[161669]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:32 smithi191 ceph-mon[161669]: pgmap v688: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 990 B/s wr, 576 op/s; 23934/244638 objects degraded (9.783%); 5099/244638 objects misplaced (2.084%); 2.2 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T05:43:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[191227]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:43:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[191227]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[191227]: osdmap e235: 8 total, 8 up, 8 in 2023-10-09T05:43:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[191227]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[191227]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[191227]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[191227]: pgmap v688: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 990 B/s wr, 576 op/s; 23934/244638 objects degraded (9.783%); 5099/244638 objects misplaced (2.084%); 2.2 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T05:43:33.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:33.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[184418]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:43:33.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[184418]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[184418]: osdmap e235: 8 total, 8 up, 8 in 2023-10-09T05:43:33.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[184418]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[184418]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[184418]: failed to encode map e235 with expected crc 2023-10-09T05:43:33.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:32 smithi008 ceph-mon[184418]: pgmap v688: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 990 B/s wr, 576 op/s; 23934/244638 objects degraded (9.783%); 5099/244638 objects misplaced (2.084%); 2.2 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T05:43:34.022 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:34.023 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:34.215 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:34.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:34.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:34.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:34.371 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:34.424 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:34.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:34.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:34.533 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:34.564 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:34.639 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:34.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:34.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:34.744 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:34.800 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:35.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:35.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:35 smithi008 ceph-mon[184418]: pgmap v689: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 920 B/s wr, 576 op/s; 23934/241323 objects degraded (9.918%); 5099/241323 objects misplaced (2.113%); 2.1 MiB/s, 0 keys/s, 28 objects/s recovering 2023-10-09T05:43:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:35.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:35.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:35 smithi008 ceph-mon[191227]: pgmap v689: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 920 B/s wr, 576 op/s; 23934/241323 objects degraded (9.918%); 5099/241323 objects misplaced (2.113%); 2.1 MiB/s, 0 keys/s, 28 objects/s recovering 2023-10-09T05:43:35.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:35.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:35.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:35.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:35.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:35 smithi191 ceph-mon[161669]: pgmap v689: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 920 B/s wr, 576 op/s; 23934/241323 objects degraded (9.918%); 5099/241323 objects misplaced (2.113%); 2.1 MiB/s, 0 keys/s, 28 objects/s recovering 2023-10-09T05:43:35.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:35.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:35.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:35.977 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:43:36.017 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:36.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:36.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:36.129 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:36.193 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:36 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23934/241323 objects degraded (9.918%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:36.215 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:36 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23934/241323 objects degraded (9.918%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:36.432 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:36.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:36.449 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:36 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23934/241323 objects degraded (9.918%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:36.501 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:36.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:36.590 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:36.720 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:36.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:36.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:36.768 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:43:36.823 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:43:36.828 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:36.857 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:43:36.892 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:36.907 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:43:36.950 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:43:36.996 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:43:37.041 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:43:37.083 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:43:37.087 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:43:37.087 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:43:37.087 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:43:37.087 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:43:37.087 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:43:37.087 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:43:37.087 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:43:37.087 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:43:37.123 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:37.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:37.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:37.229 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:37.309 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:37.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:37 smithi008 ceph-mon[184418]: pgmap v690: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 23934/238725 objects degraded (10.026%); 5099/238725 objects misplaced (2.136%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:43:37.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:37 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4267438067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:37.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:37 smithi008 ceph-mon[191227]: pgmap v690: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 23934/238725 objects degraded (10.026%); 5099/238725 objects misplaced (2.136%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:43:37.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:37 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4267438067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:37.472 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:37.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:37.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:37.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:37 smithi191 ceph-mon[161669]: pgmap v690: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 23934/238725 objects degraded (10.026%); 5099/238725 objects misplaced (2.136%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:43:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:37 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4267438067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:37.582 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:37.666 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:37.804 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:37.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:37.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:37.919 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:37.946 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:37.991 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:37.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:37.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:38.036 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:43:38.094 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:38.154 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:38.457 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:38.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:38.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:38.562 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:38.678 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:38.803 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:38.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:38.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:38.894 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:38.919 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:38.976 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:39.110 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:39.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:39.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:39.181 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:43:39.181 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:43:39.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:39.182 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45365.0:112598 7.9 e235/213 7:9e1b5642:::benchmark_data_smithi008_354649_object18102:head v 235'6964, mlcod=235'6962)", 2023-10-09T05:43:39.182 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:39.178667+0000", 2023-10-09T05:43:39.182 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.002571079, 2023-10-09T05:43:39.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026037909999999998, 2023-10-09T05:43:39.182 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:39.182 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:43:39.182 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:39.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:39.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:39.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.178667+0000", 2023-10-09T05:43:39.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.178667+0000", 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.178666+0000", 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9233e-05 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:39.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.178685+0000", 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3442999999999999e-05 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.178708+0000", 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9730000000000003e-06 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:39.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.178717+0000", 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2663999999999993e-05 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.178800+0000", 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5833000000000002e-05 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.178826+0000", 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015849899999999999 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:39.185 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:39.186 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:39.186 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:39.186 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:43:39.186 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 1 2023-10-09T05:43:39.186 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:43:39.222 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:39.272 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:39.355 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:39.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:39.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:39.459 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:39.489 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:39.532 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:39.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:39.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:39.636 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:39.739 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:39.871 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:39.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:39.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:39.916 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:43:39.982 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:40.054 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:39 smithi008 ceph-mon[184418]: pgmap v691: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 23996/235905 objects degraded (10.172%); 4914/235905 objects misplaced (2.083%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:43:40.054 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:39 smithi008 ceph-mon[191227]: pgmap v691: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 23996/235905 objects degraded (10.172%); 4914/235905 objects misplaced (2.083%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:43:40.064 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:40.144 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:40.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:40.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:40.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:40.292 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:39 smithi191 ceph-mon[161669]: pgmap v691: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 23996/235905 objects degraded (10.172%); 4914/235905 objects misplaced (2.083%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:43:40.292 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:40.382 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:40.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:40.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:40.495 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:40.507 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:40.535 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:43:40.571 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:40.766 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:43:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:43:40] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.33.4" 2023-10-09T05:43:40.818 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:40.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:40.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:40.935 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:40.974 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:43:40.974 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:43:40.974 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2023-10-09T05:43:40.975 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:43:40.986 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:41.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:40 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23996/231816 objects degraded (10.351%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:41.074 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:40 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23996/231816 objects degraded (10.351%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:41.074 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:40 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23996/231816 objects degraded (10.351%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:41.109 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:41.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:41.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:41.214 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:41.394 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:41.567 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:41.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:41.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:41.617 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:41.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:41.719 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:41.820 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:41.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:41.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:41.924 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:41.929 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:41.971 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:43:41.972 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:42.109 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:42.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:42.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:42.190 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:43:42.190 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:43:42.190 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:43:42.191 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:43:42.191 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:43:42.212 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:42.216 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:41 smithi191 ceph-mon[161669]: pgmap v692: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 23996/231816 objects degraded (10.351%); 4914/231816 objects misplaced (2.120%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:43:42.216 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:41 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1308807320' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:42.216 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:41 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:42.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:41 smithi008 ceph-mon[184418]: pgmap v692: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 23996/231816 objects degraded (10.351%); 4914/231816 objects misplaced (2.120%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:43:42.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:41 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1308807320' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:42.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:41 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:42.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:41 smithi008 ceph-mon[191227]: pgmap v692: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 23996/231816 objects degraded (10.351%); 4914/231816 objects misplaced (2.120%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:43:42.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:41 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1308807320' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:42.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:41 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:43:42.290 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:42.865 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:43:42.939 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:42.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:42.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:43.044 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:43.090 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:43.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:43.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:43:43.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[184418]: osdmap e236: 8 total, 8 up, 8 in 2023-10-09T05:43:43.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[184418]: failed to encode map e236 with expected crc 2023-10-09T05:43:43.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[184418]: failed to encode map e236 with expected crc 2023-10-09T05:43:43.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[184418]: failed to encode map e236 with expected crc 2023-10-09T05:43:43.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[184418]: pgmap v694: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 24036/227484 objects degraded (10.566%); 4748/227484 objects misplaced (2.087%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:43:43.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:43.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:43:43.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[191227]: osdmap e236: 8 total, 8 up, 8 in 2023-10-09T05:43:43.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[191227]: failed to encode map e236 with expected crc 2023-10-09T05:43:43.222 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[191227]: failed to encode map e236 with expected crc 2023-10-09T05:43:43.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[191227]: failed to encode map e236 with expected crc 2023-10-09T05:43:43.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:42 smithi008 ceph-mon[191227]: pgmap v694: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 24036/227484 objects degraded (10.566%); 4748/227484 objects misplaced (2.087%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:43:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:42 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:42 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:43:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:42 smithi191 ceph-mon[161669]: osdmap e236: 8 total, 8 up, 8 in 2023-10-09T05:43:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:42 smithi191 ceph-mon[161669]: failed to encode map e236 with expected crc 2023-10-09T05:43:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:42 smithi191 ceph-mon[161669]: failed to encode map e236 with expected crc 2023-10-09T05:43:43.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:42 smithi191 ceph-mon[161669]: failed to encode map e236 with expected crc 2023-10-09T05:43:43.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:42 smithi191 ceph-mon[161669]: pgmap v694: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 24036/227484 objects degraded (10.566%); 4748/227484 objects misplaced (2.087%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:43:43.380 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:43.800 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:43.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:43.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:43.912 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:43.950 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:44.377 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:44.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:44.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:44.491 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:44.549 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:44.799 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:44.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:44.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:44.911 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:44.974 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:44.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:44.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:44.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:44.984 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:45.007 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:43:45.007 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:43:45.007 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:43:45.007 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119018 5.e 5:7572eb82:::benchmark_data_smithi008_238708_object882:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.007070+0000", 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 560.99705477999998, 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0282413180000001, 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119018 2023-10-09T05:43:45.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007070+0000", 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007070+0000", 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007068+0000", 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2640000000000008e-06 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007077+0000", 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5773999999999998e-05 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007092+0000", 2023-10-09T05:43:45.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9e-06 2023-10-09T05:43:45.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007099+0000", 2023-10-09T05:43:45.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5432000000000005e-05 2023-10-09T05:43:45.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007185+0000", 2023-10-09T05:43:45.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9950000000000005e-06 2023-10-09T05:43:45.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T05:43:45.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007193+0000", 2023-10-09T05:43:45.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.248123126 2023-10-09T05:43:45.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255316+0000", 2023-10-09T05:43:45.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8770000000000004e-06 2023-10-09T05:43:45.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255323+0000", 2023-10-09T05:43:45.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.77988959099999999 2023-10-09T05:43:45.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035212+0000", 2023-10-09T05:43:45.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9400000000000004e-05 2023-10-09T05:43:45.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035312+0000", 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0282413180000001 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119020 5.1e 5:7d0c0384:::benchmark_data_smithi008_238708_object884:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.007113+0000", 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 560.99701234099996, 2023-10-09T05:43:45.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0326345889999999, 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119020 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007113+0000", 2023-10-09T05:43:45.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007113+0000", 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007112+0000", 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.49e-06 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007116+0000", 2023-10-09T05:43:45.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.498e-06 2023-10-09T05:43:45.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007123+0000", 2023-10-09T05:43:45.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.788e-06 2023-10-09T05:43:45.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007127+0000", 2023-10-09T05:43:45.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.248267246 2023-10-09T05:43:45.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255395+0000", 2023-10-09T05:43:45.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5020000000000001e-06 2023-10-09T05:43:45.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255398+0000", 2023-10-09T05:43:45.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78425046499999995 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.039648+0000", 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9006999999999996e-05 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.039747+0000", 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0326345889999999 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.019 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119026 5.e 5:7513f17c:::benchmark_data_smithi008_238708_object890:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e123)", 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.007132+0000", 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 560.99699356099995, 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0283243520000001, 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.020 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119026 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007132+0000", 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007132+0000", 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:43:45.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007131+0000", 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.322e-06 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007134+0000", 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5490000000000003e-06 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.023 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007140+0000", 2023-10-09T05:43:45.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4379999999999999e-06 2023-10-09T05:43:45.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.023 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.023 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.023 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.007144+0000", 2023-10-09T05:43:45.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00092436700000000005 2023-10-09T05:43:45.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.023 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.023 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.023 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.008068+0000", 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.917e-06 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.008075+0000", 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.247270726 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255346+0000", 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2480000000000002e-06 2023-10-09T05:43:45.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.255355+0000", 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78001380399999998 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035369+0000", 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7331999999999997e-05 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.035456+0000", 2023-10-09T05:43:45.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0283243520000001 2023-10-09T05:43:45.026 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.026 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.026 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.026 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163684 5.21 5:84c14237:::benchmark_data_smithi008_238708_object45548:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:43:45.026 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.609434+0000", 2023-10-09T05:43:45.026 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 517.39469094900005, 2023-10-09T05:43:45.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99344012400000004, 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163684 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609434+0000", 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609434+0000", 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609433+0000", 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.511e-06 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609440+0000", 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2476999999999999e-05 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609452+0000", 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0360000000000001e-06 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609459+0000", 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9319000000000004e-05 2023-10-09T05:43:45.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609539+0000", 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0864e-05 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.609549+0000", 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99053912899999996 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.600089+0000", 2023-10-09T05:43:45.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103931 2023-10-09T05:43:45.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.600193+0000", 2023-10-09T05:43:45.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000164618 2023-10-09T05:43:45.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:43:45.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.600357+0000", 2023-10-09T05:43:45.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023727349999999999 2023-10-09T05:43:45.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602730+0000", 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1140000000000003e-05 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602771+0000", 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7978e-05 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602789+0000", 2023-10-09T05:43:45.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5139000000000002e-05 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602844+0000", 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0287e-05 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602874+0000", 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99344012400000004 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.034 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163704 5.21 5:87d25abf:::benchmark_data_smithi008_238708_object45568:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.629811+0000", 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 517.37431466500004, 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97411889600000001, 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.035 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163704 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629811+0000", 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629811+0000", 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:43:45.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629810+0000", 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.638e-06 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629813+0000", 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8369999999999996e-06 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629817+0000", 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4340000000000001e-06 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629821+0000", 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9418999999999994e-05 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629890+0000", 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0139999999999998e-06 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.629898+0000", 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97170687300000003 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.601605+0000", 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6179000000000002e-05 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.601701+0000", 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019179700000000001 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.601893+0000", 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001016667 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602910+0000", 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6458999999999998e-05 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602946+0000", 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091263200000000003 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603859+0000", 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7240999999999998e-05 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603896+0000", 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3331000000000003e-05 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603929+0000", 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97411889600000001 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.042 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163753 5.21 5:845d2bea:::benchmark_data_smithi008_238708_object45617:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.676299+0000", 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 517.32782597000005, 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92971798100000003, 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163753 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676299+0000", 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676299+0000", 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676298+0000", 2023-10-09T05:43:45.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4119999999999998e-06 2023-10-09T05:43:45.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676304+0000", 2023-10-09T05:43:45.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4579999999999999e-05 2023-10-09T05:43:45.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676319+0000", 2023-10-09T05:43:45.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5139999999999998e-06 2023-10-09T05:43:45.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676325+0000", 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0550999999999998e-05 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676376+0000", 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6689999999999995e-06 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.676385+0000", 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92657416500000001 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.602959+0000", 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2798000000000003e-05 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603031+0000", 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013177499999999999 2023-10-09T05:43:45.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603163+0000", 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027447750000000001 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605908+0000", 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7929999999999997e-05 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605956+0000", 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4289999999999998e-06 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605962+0000", 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1736000000000002e-05 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605994+0000", 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3096999999999999e-05 2023-10-09T05:43:45.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606017+0000", 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92971798100000003 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:163923 5.21 5:843e313c:::benchmark_data_smithi008_238708_object45787:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.853413+0000", 2023-10-09T05:43:45.050 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 517.15071238600001, 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.75769858499999998, 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 163923 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853413+0000", 2023-10-09T05:43:45.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853413+0000", 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853412+0000", 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0589999999999999e-06 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853418+0000", 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2941e-05 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853431+0000", 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7400000000000001e-06 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853436+0000", 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7589e-05 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853494+0000", 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2001000000000001e-05 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.853506+0000", 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.75046967899999995 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.603976+0000", 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109487 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.604085+0000", 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019979399999999999 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:43:45.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.604285+0000", 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001741006 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606026+0000", 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4724e-05 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606081+0000", 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004942559 2023-10-09T05:43:45.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611023+0000", 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8794000000000001e-05 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611072+0000", 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9496000000000003e-05 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611111+0000", 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.75769858499999998 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164030 5.21 5:866422a9:::benchmark_data_smithi008_238708_object45894:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 517.05557476000001, 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66267309900000004, 2023-10-09T05:43:45.058 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164030 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948550+0000", 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7736000000000001e-05 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.060 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948567+0000", 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8230999999999999e-05 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948585+0000", 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3960000000000003e-06 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948592+0000", 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.498e-05 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948637+0000", 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8820000000000003e-06 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.948644+0000", 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65639657299999998 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605040+0000", 2023-10-09T05:43:45.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7799000000000003e-05 2023-10-09T05:43:45.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605138+0000", 2023-10-09T05:43:45.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013080500000000001 2023-10-09T05:43:45.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:43:45.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.605269+0000", 2023-10-09T05:43:45.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017788770000000001 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607048+0000", 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012428719999999999 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608291+0000", 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002871307 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611162+0000", 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2058999999999999e-05 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611194+0000", 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9536999999999999e-05 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611224+0000", 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66267309900000004 2023-10-09T05:43:45.065 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.066 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.066 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.066 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164042 5.21 5:85682928:::benchmark_data_smithi008_238708_object45906:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:43:45.066 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:07.959315+0000", 2023-10-09T05:43:45.066 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 517.04481046199999, 2023-10-09T05:43:45.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65788200699999999, 2023-10-09T05:43:45.066 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.066 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.066 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164042 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959315+0000", 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959315+0000", 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959314+0000", 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.07e-06 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959318+0000", 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7539999999999995e-06 2023-10-09T05:43:45.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959327+0000", 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.23e-06 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959330+0000", 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1047999999999998e-05 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959361+0000", 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0820000000000004e-06 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:07.959367+0000", 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.64674576399999995 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606113+0000", 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0732e-05 2023-10-09T05:43:45.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606184+0000", 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013076400000000001 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.606315+0000", 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048103449999999997 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611125+0000", 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014353599999999999 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611268+0000", 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0058299170000000004 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617098+0000", 2023-10-09T05:43:45.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4318999999999999e-05 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617153+0000", 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4100000000000001e-05 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617197+0000", 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65788200699999999 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.073 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.074 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164098 5.21 5:85741b6f:::benchmark_data_smithi008_238708_object45962:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:43:45.074 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.014979+0000", 2023-10-09T05:43:45.074 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 516.98914659000002, 2023-10-09T05:43:45.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.60237710200000005, 2023-10-09T05:43:45.074 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.074 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.075 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.075 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.075 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.075 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164098 2023-10-09T05:43:45.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.075 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014979+0000", 2023-10-09T05:43:45.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014979+0000", 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014977+0000", 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6270000000000002e-06 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014984+0000", 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2714000000000001e-05 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.014996+0000", 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2259999999999996e-06 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.015002+0000", 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5050999999999999e-05 2023-10-09T05:43:45.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.015057+0000", 2023-10-09T05:43:45.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0011e-05 2023-10-09T05:43:45.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.015067+0000", 2023-10-09T05:43:45.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.59200529499999999 2023-10-09T05:43:45.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607072+0000", 2023-10-09T05:43:45.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7960999999999994e-05 2023-10-09T05:43:45.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607160+0000", 2023-10-09T05:43:45.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017280900000000001 2023-10-09T05:43:45.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:43:45.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.607333+0000", 2023-10-09T05:43:45.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037971039999999999 2023-10-09T05:43:45.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611130+0000", 2023-10-09T05:43:45.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000167384 2023-10-09T05:43:45.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611297+0000", 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059718460000000003 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617269+0000", 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0393999999999999e-05 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617320+0000", 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6199999999999999e-05 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617356+0000", 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.60237710200000005 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164190 5.21 5:85c1fc08:::benchmark_data_smithi008_238708_object46054:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.100063+0000", 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 516.904062178, 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.517522814, 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164190 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.083 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100063+0000", 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100063+0000", 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100062+0000", 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2819999999999996e-06 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100070+0000", 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5238000000000001e-05 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100095+0000", 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2259999999999999e-06 2023-10-09T05:43:45.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100102+0000", 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3122999999999999e-05 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100166+0000", 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1447000000000001e-05 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.100177+0000", 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.50813475200000002 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608312+0000", 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102922 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608415+0000", 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019721800000000001 2023-10-09T05:43:45.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.608612+0000", 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026200059999999998 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611232+0000", 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3609000000000004e-05 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611326+0000", 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0061575960000000004 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617483+0000", 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1966999999999999e-05 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617535+0000", 2023-10-09T05:43:45.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0797999999999997e-05 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617586+0000", 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.517522814 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164259 5.21 5:8505256a:::benchmark_data_smithi008_238708_object46123:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:43:45.090 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.167748+0000", 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 516.83637741300004, 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45063465699999999, 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164259 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167748+0000", 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167748+0000", 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167746+0000", 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4189999999999996e-06 2023-10-09T05:43:45.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167753+0000", 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4399999999999999e-05 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167768+0000", 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6309999999999997e-06 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167774+0000", 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7620000000000006e-05 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167852+0000", 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0051000000000001e-05 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.167862+0000", 2023-10-09T05:43:45.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.441777105 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.609639+0000", 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105538 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.609744+0000", 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018787400000000001 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.609932+0000", 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041432689999999998 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.614076+0000", 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033595819999999998 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617435+0000", 2023-10-09T05:43:45.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00086552299999999999 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618301+0000", 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7574999999999999e-05 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618338+0000", 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4168000000000001e-05 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618382+0000", 2023-10-09T05:43:45.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45063465699999999 2023-10-09T05:43:45.098 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.098 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.098 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.098 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:164297 5.21 5:85a0851f:::benchmark_data_smithi008_238708_object46161:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T05:43:45.098 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:35:08.207114+0000", 2023-10-09T05:43:45.098 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 516.79701167999997, 2023-10-09T05:43:45.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.41141557899999998, 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 164297 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207114+0000", 2023-10-09T05:43:45.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207114+0000", 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207112+0000", 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5089999999999999e-06 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207118+0000", 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1423000000000001e-05 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207130+0000", 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7730000000000002e-06 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.207137+0000", 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011262379999999999 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.208263+0000", 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1843e-05 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.208275+0000", 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.40307073700000001 2023-10-09T05:43:45.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611345+0000", 2023-10-09T05:43:45.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6700999999999995e-05 2023-10-09T05:43:45.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611442+0000", 2023-10-09T05:43:45.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00022087699999999999 2023-10-09T05:43:45.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.611663+0000", 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002453359 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.614116+0000", 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033269020000000001 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.617443+0000", 2023-10-09T05:43:45.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001025317 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618469+0000", 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4535000000000001e-05 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618493+0000", 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6001000000000003e-05 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:35:08.618529+0000", 2023-10-09T05:43:45.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.41141557899999998 2023-10-09T05:43:45.106 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.106 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.106 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.106 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:112010 7.1 7:8bd9d61d:::benchmark_data_smithi008_354649_object17514:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:38.184797+0000", 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 6.8193282159999997, 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.072751233999999998, 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 112010 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.184797+0000", 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.184797+0000", 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.184796+0000", 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1440000000000003e-06 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.184804+0000", 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0618999999999999e-05 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.184814+0000", 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0780000000000001e-06 2023-10-09T05:43:45.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.184820+0000", 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6521999999999999e-05 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.184867+0000", 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0043999999999994e-05 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.184957+0000", 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5792999999999997e-05 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.185043+0000", 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003213371 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.188256+0000", 2023-10-09T05:43:45.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005935699 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.194192+0000", 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.063262642999999993 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.257454+0000", 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2006999999999999e-05 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.257496+0000", 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1768000000000003e-05 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:38.257548+0000", 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.072751233999999998 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.114 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:112774 7.1 7:8d31988f:::benchmark_data_smithi008_354649_object18278:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T05:43:45.114 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:39.457287+0000", 2023-10-09T05:43:45.114 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 5.5468386990000003, 2023-10-09T05:43:45.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.10824073200000001, 2023-10-09T05:43:45.114 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.114 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.114 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.114 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:43:45.114 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:43:45.114 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 112774 2023-10-09T05:43:45.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.457287+0000", 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.457287+0000", 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.457285+0000", 2023-10-09T05:43:45.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1438e-05 2023-10-09T05:43:45.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.457296+0000", 2023-10-09T05:43:45.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9649000000000001e-05 2023-10-09T05:43:45.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.457316+0000", 2023-10-09T05:43:45.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0105999999999999e-05 2023-10-09T05:43:45.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.457326+0000", 2023-10-09T05:43:45.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4487000000000003e-05 2023-10-09T05:43:45.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.457370+0000", 2023-10-09T05:43:45.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4957999999999994e-05 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.457465+0000", 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1970000000000002e-05 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.457557+0000", 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032024219999999999 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.460760+0000", 2023-10-09T05:43:45.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099811589999999999 2023-10-09T05:43:45.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.470741+0000", 2023-10-09T05:43:45.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.094668829999999995 2023-10-09T05:43:45.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.565410+0000", 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4084000000000005e-05 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.565474+0000", 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3635999999999999e-05 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.565527+0000", 2023-10-09T05:43:45.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.10824073200000001 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:112779 7.1 7:899947e4:::benchmark_data_smithi008_354649_object18283:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:39.463220+0000", 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 5.5409048160000003, 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.102427924, 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.121 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.122 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:43:45.122 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:43:45.122 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 112779 2023-10-09T05:43:45.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.122 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.463220+0000", 2023-10-09T05:43:45.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.463220+0000", 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.463218+0000", 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2959000000000001e-05 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.463231+0000", 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5201999999999999e-05 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.463247+0000", 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3939999999999998e-06 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.463254+0000", 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9014999999999997e-05 2023-10-09T05:43:45.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.463303+0000", 2023-10-09T05:43:45.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000136217 2023-10-09T05:43:45.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.463439+0000", 2023-10-09T05:43:45.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015523300000000001 2023-10-09T05:43:45.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.463594+0000", 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071606769999999998 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.470755+0000", 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0066700350000000004 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.477425+0000", 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.088163191000000002 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.565588+0000", 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4547e-05 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.565623+0000", 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5392999999999999e-05 2023-10-09T05:43:45.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.565648+0000", 2023-10-09T05:43:45.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.102427924 2023-10-09T05:43:45.128 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.128 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.128 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.128 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:112780 7.1 7:848ea739:::benchmark_data_smithi008_354649_object18284:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T05:43:45.128 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:39.466348+0000", 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 5.53777746, 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.099427565999999995, 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 112780 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.466348+0000", 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.466348+0000", 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.466345+0000", 2023-10-09T05:43:45.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.02e-06 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.466354+0000", 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5313999999999999e-05 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.466370+0000", 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9410000000000001e-06 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.466377+0000", 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000841671 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.467218+0000", 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4957000000000007e-05 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.467303+0000", 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3467999999999994e-05 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.467387+0000", 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003491198 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.470878+0000", 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068238839999999997 2023-10-09T05:43:45.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.477702+0000", 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.087990824999999995 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.565693+0000", 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6990999999999998e-05 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.565740+0000", 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5646999999999999e-05 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.565775+0000", 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.099427565999999995 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45365.0:112799 7.4 e235/213 7:2a414afb:::benchmark_data_smithi008_354649_object18303:head v 235'7019, mlcod=235'7017)", 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:39.508984+0000", 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 5.4951412309999998, 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.050416806000000002, 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.508984+0000", 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.508984+0000", 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.508982+0000", 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1070000000000001e-05 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.508994+0000", 2023-10-09T05:43:45.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3937999999999999e-05 2023-10-09T05:43:45.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.509007+0000", 2023-10-09T05:43:45.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8000000000000001e-06 2023-10-09T05:43:45.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.509014+0000", 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.044987906000000001 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.554002+0000", 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7402e-05 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.554030+0000", 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005322144 2023-10-09T05:43:45.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.559352+0000", 2023-10-09T05:43:45.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9075e-05 2023-10-09T05:43:45.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:39.559401+0000", 2023-10-09T05:43:45.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.050416806000000002 2023-10-09T05:43:45.140 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.140 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.141 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.141 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:113848 7.1 7:8292c1dc:::benchmark_data_smithi008_354649_object19352:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T05:43:45.141 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:41.279358+0000", 2023-10-09T05:43:45.141 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 3.724767114, 2023-10-09T05:43:45.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.063417942000000005, 2023-10-09T05:43:45.141 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.141 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.141 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.142 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:43:45.142 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:43:45.142 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 113848 2023-10-09T05:43:45.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.142 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.279358+0000", 2023-10-09T05:43:45.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.279358+0000", 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.279356+0000", 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8099999999999992e-06 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.279366+0000", 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5532e-05 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.279382+0000", 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0240000000000003e-06 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.279388+0000", 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.596e-05 2023-10-09T05:43:45.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.279434+0000", 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9314999999999995e-05 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.279513+0000", 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9066000000000004e-05 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.279602+0000", 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030477659999999999 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.282650+0000", 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010750765000000001 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.293401+0000", 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.049272376999999999 2023-10-09T05:43:45.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.342673+0000", 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9728999999999997e-05 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.342733+0000", 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3346999999999997e-05 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:41.342776+0000", 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.063417942000000005 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:114634 7.1 7:8ae9d527:::benchmark_data_smithi008_354649_object20138:head [delete] snapc 0=[] ondisk+write+known_if_redirected e236)", 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:42.766293+0000", 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 2.2378321269999999, 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.086196985000000004, 2023-10-09T05:43:45.148 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:43:45.149 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:43:45.149 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:43:45.149 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:43:45.149 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:43:45.149 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 114634 2023-10-09T05:43:45.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.149 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:43:45.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:43:45.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.766293+0000", 2023-10-09T05:43:45.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.766293+0000", 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.766291+0000", 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0648e-05 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.766302+0000", 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1438999999999999e-05 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.766323+0000", 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9800000000000001e-06 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:43:45.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.766330+0000", 2023-10-09T05:43:45.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9509000000000003e-05 2023-10-09T05:43:45.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:43:45.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.766380+0000", 2023-10-09T05:43:45.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013086700000000001 2023-10-09T05:43:45.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:43:45.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.766511+0000", 2023-10-09T05:43:45.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106871 2023-10-09T05:43:45.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:43:45.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.766617+0000", 2023-10-09T05:43:45.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035677740000000001 2023-10-09T05:43:45.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:43:45.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.770185+0000", 2023-10-09T05:43:45.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000182256 2023-10-09T05:43:45.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.770368+0000", 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.082020921999999996 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.852388+0000", 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7561e-05 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:43:45.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.852446+0000", 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4097000000000001e-05 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:42.852490+0000", 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.086196985000000004 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:43:45.156 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:43:45.157 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:43:45.157 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:45.477 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:45.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:45.564 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:43:45.593 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:45.629 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:45.630 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:43:45.659 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:43:45.686 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:43:45.707 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:43:45.710 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:45.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:45.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:45.747 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:43:45.792 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:43:45.818 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:45.835 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:43:45.882 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:43:45.913 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:45.914 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:45 smithi191 ceph-mon[161669]: pgmap v695: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 24036/224397 objects degraded (10.711%); 4748/224397 objects misplaced (2.116%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:43:45.915 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/896882109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:45.964 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:45.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:45.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:46.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:43:46.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:43:46.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:43:46.014 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:43:46.015 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:43:46.015 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:43:46.015 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:43:46.015 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:43:46.068 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:46.127 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:46.150 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:46.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:45 smithi008 ceph-mon[184418]: pgmap v695: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 24036/224397 objects degraded (10.711%); 4748/224397 objects misplaced (2.116%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:43:46.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/896882109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:46.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:45 smithi008 ceph-mon[191227]: pgmap v695: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 24036/224397 objects degraded (10.711%); 4748/224397 objects misplaced (2.116%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:43:46.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/896882109' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:46.539 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:43:46.825 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:46.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:46.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:46.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:46.964 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:46 smithi191 ceph-mon[161669]: pgmap v696: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 24036/221850 objects degraded (10.834%); 4748/221850 objects misplaced (2.140%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:43:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:46 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24036/221850 objects degraded (10.834%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:47.276 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:47.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:47.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:47.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:47.433 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:47.450 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:46 smithi008 ceph-mon[191227]: pgmap v696: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 24036/221850 objects degraded (10.834%); 4748/221850 objects misplaced (2.140%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:43:47.451 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:46 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24036/221850 objects degraded (10.834%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:47.451 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:46 smithi008 ceph-mon[184418]: pgmap v696: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 24036/221850 objects degraded (10.834%); 4748/221850 objects misplaced (2.140%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T05:43:47.451 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:46 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24036/221850 objects degraded (10.834%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:47.547 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:47.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:47.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:47.653 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:47.693 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:47.763 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:47.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:47.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:47.818 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:47.818 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:47.868 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:47.927 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:48.040 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:48.100 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:48.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:48.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:48.204 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:48.241 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:48.302 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:48.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:48.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:48.406 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:48.414 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:43:48.431 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:43:48.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:43:48.710 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:49.188 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:49.470 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:43:49.670 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:49.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:49.791 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:49.830 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:49.874 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:49.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:49.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:49.977 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:50.001 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:49 smithi191 ceph-mon[161669]: pgmap v697: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 24082/219222 objects degraded (10.985%); 4566/219222 objects misplaced (2.083%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:43:50.001 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:49 smithi191 ceph-mon[161669]: from='client.35517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:43:50.004 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:49 smithi008 ceph-mon[184418]: pgmap v697: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 24082/219222 objects degraded (10.985%); 4566/219222 objects misplaced (2.083%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:43:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:49 smithi008 ceph-mon[184418]: from='client.35517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:43:50.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:49 smithi008 ceph-mon[191227]: pgmap v697: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 24082/219222 objects degraded (10.985%); 4566/219222 objects misplaced (2.083%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:43:50.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:49 smithi008 ceph-mon[191227]: from='client.35517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:43:50.312 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:50.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:50.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:50.418 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:50.468 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:50.593 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:50.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:50.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:50.697 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:50.725 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:50.770 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:50.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:50.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:50.794 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:50.794 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:50.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:50.905 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:50.967 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:43:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:43:50] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T05:43:51.076 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:51.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:51.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:51.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:43:51.216 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:51.435 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:43:51.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:43:51.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:43:51.536 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:51.922 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":697,"stamp":"2023-10-09T05:43:48.243481+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4791867339,"num_objects":73074,"num_object_clones":0,"num_object_copies":219222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24082,"num_objects_misplaced":4566,"num_objects_unfound":0,"num_objects_dirty":73074,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":117048,"num_write_kb":6062694,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":756,"num_bytes_recovered":51849204,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117012,"ondisk_log_size":117012,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":275,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17309568,"kb_used_data":14366144,"kb_used_omap":45,"kb_used_meta":2943250,"kb_avail":732651648,"statfs":{"total":767960285184,"available":750235287552,"internally_reserved":0,"allocated":14710931456,"data_stored":14705347093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":3013888291},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,8,5,2,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-364445696,"num_objects":-5561,"num_object_clones":0,"num_object_copies":-16683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":86,"num_objects_misplaced":-348,"num_objects_unfound":0,"num_objects_dirty":-5561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5561,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":262,"num_bytes_recovered":17170432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004591"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":218,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.975710+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:43:31.975710+0000","last_peered":"2023-10-09T05:43:31.975710+0000","last_clean":"2023-10-09T05:43:31.975710+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:43:31.975710+0000","last_undegraded":"2023-10-09T05:43:31.975710+0000","last_fullsized":"2023-10-09T05:43:31.975710+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":208,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.971597+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:43:31.971597+0000","last_peered":"2023-10-09T05:43:31.971597+0000","last_clean":"2023-10-09T05:43:31.971597+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:43:31.971597+0000","last_undegraded":"2023-10-09T05:43:31.971597+0000","last_fullsized":"2023-10-09T05:43:31.971597+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":183,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.984227+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:43:31.984227+0000","last_peered":"2023-10-09T05:43:31.984227+0000","last_clean":"2023-10-09T05:43:31.984227+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:43:31.984227+0000","last_undegraded":"2023-10-09T05:43:31.984227+0000","last_fullsized":"2023-10-09T05:43:31.984227+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":222,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.960782+0000","last_change":"2023-10-09T05:43:19.889677+0000","last_active":"2023-10-09T05:43:31.960782+0000","last_peered":"2023-10-09T05:43:31.960782+0000","last_clean":"2023-10-09T05:43:31.960782+0000","last_became_active":"2023-10-09T05:43:19.889320+0000","last_became_peered":"2023-10-09T05:43:19.889320+0000","last_unstale":"2023-10-09T05:43:31.960782+0000","last_undegraded":"2023-10-09T05:43:31.960782+0000","last_fullsized":"2023-10-09T05:43:31.960782+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:57.376656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":225,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:36.746395+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:43:36.746395+0000","last_peered":"2023-10-09T05:43:36.746395+0000","last_clean":"2023-10-09T05:43:36.746395+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:43:36.746395+0000","last_undegraded":"2023-10-09T05:43:36.746395+0000","last_fullsized":"2023-10-09T05:43:36.746395+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":201,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.978301+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:43:31.978301+0000","last_peered":"2023-10-09T05:43:31.978301+0000","last_clean":"2023-10-09T05:43:31.978301+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:43:31.978301+0000","last_undegraded":"2023-10-09T05:43:31.978301+0000","last_fullsized":"2023-10-09T05:43:31.978301+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":188,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.978083+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:43:31.978083+0000","last_peered":"2023-10-09T05:43:31.978083+0000","last_clean":"2023-10-09T05:43:31.978083+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:43:31.978083+0000","last_undegraded":"2023-10-09T05:43:31.978083+0000","last_fullsized":"2023-10-09T05:43:31.978083+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":209,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.963333+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:43:31.963333+0000","last_peered":"2023-10-09T05:43:31.963333+0000","last_clean":"2023-10-09T05:43:31.963333+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:43:31.963333+0000","last_undegraded":"2023-10-09T05:43:31.963333+0000","last_fullsized":"2023-10-09T05:43:31.963333+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":236,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.966598+0000","last_change":"2023-10-09T05:43:19.886014+0000","last_active":"2023-10-09T05:43:31.966598+0000","last_peered":"2023-10-09T05:43:31.966598+0000","last_clean":"2023-10-09T05:43:31.966598+0000","last_became_active":"2023-10-09T05:43:19.885659+0000","last_became_peered":"2023-10-09T05:43:19.885659+0000","last_unstale":"2023-10-09T05:43:31.966598+0000","last_undegraded":"2023-10-09T05:43:31.966598+0000","last_fullsized":"2023-10-09T05:43:31.966598+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":232,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.960752+0000","last_change":"2023-10-09T05:43:19.883115+0000","last_active":"2023-10-09T05:43:31.960752+0000","last_peered":"2023-10-09T05:43:31.960752+0000","last_clean":"2023-10-09T05:43:31.960752+0000","last_became_active":"2023-10-09T05:43:19.882674+0000","last_became_peered":"2023-10-09T05:43:19.882674+0000","last_unstale":"2023-10-09T05:43:31.960752+0000","last_undegraded":"2023-10-09T05:43:31.960752+0000","last_fullsized":"2023-10-09T05:43:31.960752+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:30.479481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":21,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.934765+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:43:31.934765+0000","last_peered":"2023-10-09T05:43:31.934765+0000","last_clean":"2023-10-09T05:43:31.934765+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:43:31.934765+0000","last_undegraded":"2023-10-09T05:43:31.934765+0000","last_fullsized":"2023-10-09T05:43:31.934765+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":183,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.985178+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:43:31.985178+0000","last_peered":"2023-10-09T05:43:31.985178+0000","last_clean":"2023-10-09T05:43:31.985178+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:43:31.985178+0000","last_undegraded":"2023-10-09T05:43:31.985178+0000","last_fullsized":"2023-10-09T05:43:31.985178+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":262,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.975431+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:43:31.975431+0000","last_peered":"2023-10-09T05:43:31.975431+0000","last_clean":"2023-10-09T05:43:31.975431+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:43:31.975431+0000","last_undegraded":"2023-10-09T05:43:31.975431+0000","last_fullsized":"2023-10-09T05:43:31.975431+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":201,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.975148+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:43:31.975148+0000","last_peered":"2023-10-09T05:43:31.975148+0000","last_clean":"2023-10-09T05:43:31.975148+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:43:31.975148+0000","last_undegraded":"2023-10-09T05:43:31.975148+0000","last_fullsized":"2023-10-09T05:43:31.975148+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":21,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.934724+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:43:31.934724+0000","last_peered":"2023-10-09T05:43:31.934724+0000","last_clean":"2023-10-09T05:43:31.934724+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:43:31.934724+0000","last_undegraded":"2023-10-09T05:43:31.934724+0000","last_fullsized":"2023-10-09T05:43:31.934724+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":242,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.960818+0000","last_change":"2023-10-09T05:43:19.882836+0000","last_active":"2023-10-09T05:43:31.960818+0000","last_peered":"2023-10-09T05:43:31.960818+0000","last_clean":"2023-10-09T05:43:31.960818+0000","last_became_active":"2023-10-09T05:43:19.882415+0000","last_became_peered":"2023-10-09T05:43:19.882415+0000","last_unstale":"2023-10-09T05:43:31.960818+0000","last_undegraded":"2023-10-09T05:43:31.960818+0000","last_fullsized":"2023-10-09T05:43:31.960818+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:20.794814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":221,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.957498+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:43:31.957498+0000","last_peered":"2023-10-09T05:43:31.957498+0000","last_clean":"2023-10-09T05:43:31.957498+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:43:31.957498+0000","last_undegraded":"2023-10-09T05:43:31.957498+0000","last_fullsized":"2023-10-09T05:43:31.957498+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":235,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.966553+0000","last_change":"2023-10-09T05:43:19.886058+0000","last_active":"2023-10-09T05:43:31.966553+0000","last_peered":"2023-10-09T05:43:31.966553+0000","last_clean":"2023-10-09T05:43:31.966553+0000","last_became_active":"2023-10-09T05:43:19.885713+0000","last_became_peered":"2023-10-09T05:43:19.885713+0000","last_unstale":"2023-10-09T05:43:31.966553+0000","last_undegraded":"2023-10-09T05:43:31.966553+0000","last_fullsized":"2023-10-09T05:43:31.966553+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.934465+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:43:31.934465+0000","last_peered":"2023-10-09T05:43:31.934465+0000","last_clean":"2023-10-09T05:43:31.934465+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:43:31.934465+0000","last_undegraded":"2023-10-09T05:43:31.934465+0000","last_fullsized":"2023-10-09T05:43:31.934465+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":221,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.960845+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:43:31.960845+0000","last_peered":"2023-10-09T05:43:31.960845+0000","last_clean":"2023-10-09T05:43:31.960845+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:43:31.960845+0000","last_undegraded":"2023-10-09T05:43:31.960845+0000","last_fullsized":"2023-10-09T05:43:31.960845+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":210,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.977287+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:43:31.977287+0000","last_peered":"2023-10-09T05:43:31.977287+0000","last_clean":"2023-10-09T05:43:31.977287+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:43:31.977287+0000","last_undegraded":"2023-10-09T05:43:31.977287+0000","last_fullsized":"2023-10-09T05:43:31.977287+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":242,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:36.746566+0000","last_change":"2023-10-09T05:43:19.883642+0000","last_active":"2023-10-09T05:43:36.746566+0000","last_peered":"2023-10-09T05:43:36.746566+0000","last_clean":"2023-10-09T05:43:36.746566+0000","last_became_active":"2023-10-09T05:43:19.883419+0000","last_became_peered":"2023-10-09T05:43:19.883419+0000","last_unstale":"2023-10-09T05:43:36.746566+0000","last_undegraded":"2023-10-09T05:43:36.746566+0000","last_fullsized":"2023-10-09T05:43:36.746566+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:13.488569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"236'7013","reported_seq":7043,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-10-09T05:43:44.108595+0000","last_change":"2023-10-09T05:41:29.078278+0000","last_active":"2023-10-09T05:43:44.108595+0000","last_peered":"2023-10-09T05:43:44.108595+0000","last_clean":"2023-10-09T05:43:44.108595+0000","last_became_active":"2023-10-09T05:41:29.077562+0000","last_became_peered":"2023-10-09T05:41:29.077562+0000","last_unstale":"2023-10-09T05:43:44.108595+0000","last_undegraded":"2023-10-09T05:43:44.108595+0000","last_fullsized":"2023-10-09T05:43:44.108595+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:29.784837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293666816,"num_objects":4481,"num_object_clones":0,"num_object_copies":13443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7013,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":19,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.934876+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:43:31.934876+0000","last_peered":"2023-10-09T05:43:31.934876+0000","last_clean":"2023-10-09T05:43:31.934876+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:43:31.934876+0000","last_undegraded":"2023-10-09T05:43:31.934876+0000","last_fullsized":"2023-10-09T05:43:31.934876+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":220,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.966137+0000","last_change":"2023-10-09T05:43:19.885929+0000","last_active":"2023-10-09T05:43:31.966137+0000","last_peered":"2023-10-09T05:43:31.966137+0000","last_clean":"2023-10-09T05:43:31.966137+0000","last_became_active":"2023-10-09T05:43:19.885542+0000","last_became_peered":"2023-10-09T05:43:19.885542+0000","last_unstale":"2023-10-09T05:43:31.966137+0000","last_undegraded":"2023-10-09T05:43:31.966137+0000","last_fullsized":"2023-10-09T05:43:31.966137+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"236'7231","reported_seq":7263,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-10-09T05:43:46.620356+0000","last_change":"2023-10-09T05:41:29.081067+0000","last_active":"2023-10-09T05:43:46.620356+0000","last_peered":"2023-10-09T05:43:46.620356+0000","last_clean":"2023-10-09T05:43:46.620356+0000","last_became_active":"2023-10-09T05:41:29.080704+0000","last_became_peered":"2023-10-09T05:41:29.080704+0000","last_unstale":"2023-10-09T05:43:46.620356+0000","last_undegraded":"2023-10-09T05:43:46.620356+0000","last_fullsized":"2023-10-09T05:43:46.620356+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7231,"log_dups_size":0,"ondisk_log_size":7231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:29.483205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288030720,"num_objects":4395,"num_object_clones":0,"num_object_copies":13185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7231,"num_write_kb":372032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":19,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.934969+0000","last_change":"2023-10-09T05:43:20.538857+0000","last_active":"2023-10-09T05:43:31.934969+0000","last_peered":"2023-10-09T05:43:31.934969+0000","last_clean":"2023-10-09T05:43:31.934969+0000","last_became_active":"2023-10-09T05:43:19.889408+0000","last_became_peered":"2023-10-09T05:43:19.889408+0000","last_unstale":"2023-10-09T05:43:31.934969+0000","last_undegraded":"2023-10-09T05:43:31.934969+0000","last_fullsized":"2023-10-09T05:43:31.934969+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":225,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.975111+0000","last_change":"2023-10-09T05:41:44.881327+0000","last_active":"2023-10-09T05:43:31.975111+0000","last_peered":"2023-10-09T05:43:31.975111+0000","last_clean":"2023-10-09T05:43:31.975111+0000","last_became_active":"2023-10-09T05:41:44.881044+0000","last_became_peered":"2023-10-09T05:41:44.881044+0000","last_unstale":"2023-10-09T05:43:31.975111+0000","last_undegraded":"2023-10-09T05:43:31.975111+0000","last_fullsized":"2023-10-09T05:43:31.975111+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"236'7103","reported_seq":7135,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-10-09T05:43:43.475217+0000","last_change":"2023-10-09T05:41:29.077986+0000","last_active":"2023-10-09T05:43:43.475217+0000","last_peered":"2023-10-09T05:43:43.475217+0000","last_clean":"2023-10-09T05:43:43.475217+0000","last_became_active":"2023-10-09T05:41:29.077631+0000","last_became_peered":"2023-10-09T05:41:29.077631+0000","last_unstale":"2023-10-09T05:43:43.475217+0000","last_undegraded":"2023-10-09T05:43:43.475217+0000","last_fullsized":"2023-10-09T05:43:43.475217+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7103,"log_dups_size":0,"ondisk_log_size":7103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:48.684759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302841856,"num_objects":4621,"num_object_clones":0,"num_object_copies":13863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7103,"num_write_kb":375168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":19,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.934758+0000","last_change":"2023-10-09T05:43:20.148955+0000","last_active":"2023-10-09T05:43:31.934758+0000","last_peered":"2023-10-09T05:43:31.934758+0000","last_clean":"2023-10-09T05:43:31.934758+0000","last_became_active":"2023-10-09T05:43:19.884767+0000","last_became_peered":"2023-10-09T05:43:19.884767+0000","last_unstale":"2023-10-09T05:43:31.934758+0000","last_undegraded":"2023-10-09T05:43:31.934758+0000","last_fullsized":"2023-10-09T05:43:31.934758+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":243,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.977471+0000","last_change":"2023-10-09T05:43:19.888083+0000","last_active":"2023-10-09T05:43:31.977471+0000","last_peered":"2023-10-09T05:43:31.977471+0000","last_clean":"2023-10-09T05:43:31.977471+0000","last_became_active":"2023-10-09T05:43:19.887781+0000","last_became_peered":"2023-10-09T05:43:19.887781+0000","last_unstale":"2023-10-09T05:43:31.977471+0000","last_undegraded":"2023-10-09T05:43:31.977471+0000","last_fullsized":"2023-10-09T05:43:31.977471+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"236'7365","reported_seq":8714,"reported_epoch":236,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:43:47.065773+0000","last_change":"2023-10-09T05:43:21.749532+0000","last_active":"2023-10-09T05:43:47.065773+0000","last_peered":"2023-10-09T05:43:47.065773+0000","last_clean":"2023-10-09T05:43:18.869044+0000","last_became_active":"2023-10-09T05:43:21.192067+0000","last_became_peered":"2023-10-09T05:43:21.192067+0000","last_unstale":"2023-10-09T05:43:47.065773+0000","last_undegraded":"2023-10-09T05:43:20.912508+0000","last_fullsized":"2023-10-09T05:43:20.876225+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7365,"log_dups_size":0,"ondisk_log_size":7365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:25.034988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299696128,"num_objects":4573,"num_object_clones":0,"num_object_copies":13719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":265,"num_objects_misplaced":4566,"num_objects_unfound":0,"num_objects_dirty":4573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7365,"num_write_kb":382016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":654,"num_bytes_recovered":42860544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":4566},{"shards":"2,4","objects":-258}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":203,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.966307+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:43:31.966307+0000","last_peered":"2023-10-09T05:43:31.966307+0000","last_clean":"2023-10-09T05:43:31.966307+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:43:31.966307+0000","last_undegraded":"2023-10-09T05:43:31.966307+0000","last_fullsized":"2023-10-09T05:43:31.966307+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":189,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.956995+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:43:31.956995+0000","last_peered":"2023-10-09T05:43:31.956995+0000","last_clean":"2023-10-09T05:43:31.956995+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:43:31.956995+0000","last_undegraded":"2023-10-09T05:43:31.956995+0000","last_fullsized":"2023-10-09T05:43:31.956995+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"236'7285","reported_seq":7324,"reported_epoch":236,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:43:43.494919+0000","last_change":"2023-10-09T05:43:21.226422+0000","last_active":"2023-10-09T05:43:43.494919+0000","last_peered":"2023-10-09T05:43:43.494919+0000","last_clean":"2023-10-09T05:43:18.871019+0000","last_became_active":"2023-10-09T05:43:21.208517+0000","last_became_peered":"2023-10-09T05:43:21.208517+0000","last_unstale":"2023-10-09T05:43:43.494919+0000","last_undegraded":"2023-10-09T05:43:20.928538+0000","last_fullsized":"2023-10-09T05:43:20.877574+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7285,"log_dups_size":0,"ondisk_log_size":7285,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:20.409497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307036160,"num_objects":4685,"num_object_clones":0,"num_object_copies":14055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5985,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7285,"num_write_kb":383040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3","objects":-835},{"shards":"0,3,4","objects":4685}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":207,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.964810+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:43:31.964810+0000","last_peered":"2023-10-09T05:43:31.964810+0000","last_clean":"2023-10-09T05:43:31.964810+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:43:31.964810+0000","last_undegraded":"2023-10-09T05:43:31.964810+0000","last_fullsized":"2023-10-09T05:43:31.964810+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":201,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.975354+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:43:31.975354+0000","last_peered":"2023-10-09T05:43:31.975354+0000","last_clean":"2023-10-09T05:43:31.975354+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:43:31.975354+0000","last_undegraded":"2023-10-09T05:43:31.975354+0000","last_fullsized":"2023-10-09T05:43:31.975354+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"236'7503","reported_seq":7535,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-10-09T05:43:47.079466+0000","last_change":"2023-10-09T05:41:29.078791+0000","last_active":"2023-10-09T05:43:47.079466+0000","last_peered":"2023-10-09T05:43:47.079466+0000","last_clean":"2023-10-09T05:43:47.079466+0000","last_became_active":"2023-10-09T05:41:29.078491+0000","last_became_peered":"2023-10-09T05:41:29.078491+0000","last_unstale":"2023-10-09T05:43:47.079466+0000","last_undegraded":"2023-10-09T05:43:47.079466+0000","last_fullsized":"2023-10-09T05:43:47.079466+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7503,"log_dups_size":0,"ondisk_log_size":7503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:37.817282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300220416,"num_objects":4581,"num_object_clones":0,"num_object_copies":13743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7503,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":202,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.977371+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:43:31.977371+0000","last_peered":"2023-10-09T05:43:31.977371+0000","last_clean":"2023-10-09T05:43:31.977371+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:43:31.977371+0000","last_undegraded":"2023-10-09T05:43:31.977371+0000","last_fullsized":"2023-10-09T05:43:31.977371+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":216,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.975287+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:43:31.975287+0000","last_peered":"2023-10-09T05:43:31.975287+0000","last_clean":"2023-10-09T05:43:31.975287+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:43:31.975287+0000","last_undegraded":"2023-10-09T05:43:31.975287+0000","last_fullsized":"2023-10-09T05:43:31.975287+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"236'7218","reported_seq":7249,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-10-09T05:43:45.792912+0000","last_change":"2023-10-09T05:41:29.078645+0000","last_active":"2023-10-09T05:43:45.792912+0000","last_peered":"2023-10-09T05:43:45.792912+0000","last_clean":"2023-10-09T05:43:45.792912+0000","last_became_active":"2023-10-09T05:41:29.078355+0000","last_became_peered":"2023-10-09T05:41:29.078355+0000","last_unstale":"2023-10-09T05:43:45.792912+0000","last_undegraded":"2023-10-09T05:43:45.792912+0000","last_fullsized":"2023-10-09T05:43:45.792912+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7218,"log_dups_size":0,"ondisk_log_size":7218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296026136,"num_objects":4518,"num_object_clones":0,"num_object_copies":13554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4518,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7218,"num_write_kb":375489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":235,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.975012+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:43:31.975012+0000","last_peered":"2023-10-09T05:43:31.975012+0000","last_clean":"2023-10-09T05:43:31.975012+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:43:31.975012+0000","last_undegraded":"2023-10-09T05:43:31.975012+0000","last_fullsized":"2023-10-09T05:43:31.975012+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":210,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.975636+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:43:31.975636+0000","last_peered":"2023-10-09T05:43:31.975636+0000","last_clean":"2023-10-09T05:43:31.975636+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:43:31.975636+0000","last_undegraded":"2023-10-09T05:43:31.975636+0000","last_fullsized":"2023-10-09T05:43:31.975636+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"236'7204","reported_seq":7245,"reported_epoch":236,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:43:45.835803+0000","last_change":"2023-10-09T05:43:21.223356+0000","last_active":"2023-10-09T05:43:45.835803+0000","last_peered":"2023-10-09T05:43:45.835803+0000","last_clean":"2023-10-09T05:43:18.861565+0000","last_became_active":"2023-10-09T05:43:21.207790+0000","last_became_peered":"2023-10-09T05:43:21.207790+0000","last_unstale":"2023-10-09T05:43:45.835803+0000","last_undegraded":"2023-10-09T05:43:21.070384+0000","last_fullsized":"2023-10-09T05:43:20.885426+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7204,"log_dups_size":0,"ondisk_log_size":7204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297402368,"num_objects":4538,"num_object_clones":0,"num_object_copies":13614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5870,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7204,"num_write_kb":375744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3],"avail_no_missing":["6","2","3"],"object_location_counts":[{"shards":"2,3,6","objects":4538},{"shards":"3,6","objects":-907}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":243,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.963154+0000","last_change":"2023-10-09T05:43:19.883213+0000","last_active":"2023-10-09T05:43:31.963154+0000","last_peered":"2023-10-09T05:43:31.963154+0000","last_clean":"2023-10-09T05:43:31.963154+0000","last_became_active":"2023-10-09T05:43:19.882914+0000","last_became_peered":"2023-10-09T05:43:19.882914+0000","last_unstale":"2023-10-09T05:43:31.963154+0000","last_undegraded":"2023-10-09T05:43:31.963154+0000","last_fullsized":"2023-10-09T05:43:31.963154+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.768555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":183,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.984157+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:43:31.984157+0000","last_peered":"2023-10-09T05:43:31.984157+0000","last_clean":"2023-10-09T05:43:31.984157+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:43:31.984157+0000","last_undegraded":"2023-10-09T05:43:31.984157+0000","last_fullsized":"2023-10-09T05:43:31.984157+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"236'7108","reported_seq":7138,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-10-09T05:43:44.957720+0000","last_change":"2023-10-09T05:41:29.079588+0000","last_active":"2023-10-09T05:43:44.957720+0000","last_peered":"2023-10-09T05:43:44.957720+0000","last_clean":"2023-10-09T05:43:44.957720+0000","last_became_active":"2023-10-09T05:41:29.079288+0000","last_became_peered":"2023-10-09T05:41:29.079288+0000","last_unstale":"2023-10-09T05:43:44.957720+0000","last_undegraded":"2023-10-09T05:43:44.957720+0000","last_fullsized":"2023-10-09T05:43:44.957720+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7108,"log_dups_size":0,"ondisk_log_size":7108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295436288,"num_objects":4508,"num_object_clones":0,"num_object_copies":13524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7108,"num_write_kb":371712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":849,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.984781+0000","last_change":"2023-10-09T05:43:22.965549+0000","last_active":"2023-10-09T05:43:31.984781+0000","last_peered":"2023-10-09T05:43:31.984781+0000","last_clean":"2023-10-09T05:43:31.984781+0000","last_became_active":"2023-10-09T05:43:22.964989+0000","last_became_peered":"2023-10-09T05:43:22.964989+0000","last_unstale":"2023-10-09T05:43:31.984781+0000","last_undegraded":"2023-10-09T05:43:31.984781+0000","last_fullsized":"2023-10-09T05:43:31.984781+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:51.536899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":218,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.966462+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:43:31.966462+0000","last_peered":"2023-10-09T05:43:31.966462+0000","last_clean":"2023-10-09T05:43:31.966462+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:43:31.966462+0000","last_undegraded":"2023-10-09T05:43:31.966462+0000","last_fullsized":"2023-10-09T05:43:31.966462+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":169,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.985252+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:43:31.985252+0000","last_peered":"2023-10-09T05:43:31.985252+0000","last_clean":"2023-10-09T05:43:31.985252+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:43:31.985252+0000","last_undegraded":"2023-10-09T05:43:31.985252+0000","last_fullsized":"2023-10-09T05:43:31.985252+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"236'7199","reported_seq":7229,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-10-09T05:43:45.783139+0000","last_change":"2023-10-09T05:41:29.082691+0000","last_active":"2023-10-09T05:43:45.783139+0000","last_peered":"2023-10-09T05:43:45.783139+0000","last_clean":"2023-10-09T05:43:45.783139+0000","last_became_active":"2023-10-09T05:41:29.082385+0000","last_became_peered":"2023-10-09T05:41:29.082385+0000","last_unstale":"2023-10-09T05:43:45.783139+0000","last_undegraded":"2023-10-09T05:43:45.783139+0000","last_fullsized":"2023-10-09T05:43:45.783139+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7199,"log_dups_size":0,"ondisk_log_size":7199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293273600,"num_objects":4475,"num_object_clones":0,"num_object_copies":13425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7199,"num_write_kb":373568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":672,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.966372+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:43:31.966372+0000","last_peered":"2023-10-09T05:43:31.966372+0000","last_clean":"2023-10-09T05:43:31.966372+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:43:31.966372+0000","last_undegraded":"2023-10-09T05:43:31.966372+0000","last_fullsized":"2023-10-09T05:43:31.966372+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":202,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.975241+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:43:31.975241+0000","last_peered":"2023-10-09T05:43:31.975241+0000","last_clean":"2023-10-09T05:43:31.975241+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:43:31.975241+0000","last_undegraded":"2023-10-09T05:43:31.975241+0000","last_fullsized":"2023-10-09T05:43:31.975241+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":215,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.966446+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:43:31.966446+0000","last_peered":"2023-10-09T05:43:31.966446+0000","last_clean":"2023-10-09T05:43:31.966446+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:43:31.966446+0000","last_undegraded":"2023-10-09T05:43:31.966446+0000","last_fullsized":"2023-10-09T05:43:31.966446+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"236'7149","reported_seq":7181,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-10-09T05:43:43.294061+0000","last_change":"2023-10-09T05:41:29.077953+0000","last_active":"2023-10-09T05:43:43.294061+0000","last_peered":"2023-10-09T05:43:43.294061+0000","last_clean":"2023-10-09T05:43:43.294061+0000","last_became_active":"2023-10-09T05:41:29.077652+0000","last_became_peered":"2023-10-09T05:41:29.077652+0000","last_unstale":"2023-10-09T05:43:43.294061+0000","last_undegraded":"2023-10-09T05:43:43.294061+0000","last_fullsized":"2023-10-09T05:43:43.294061+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7149,"log_dups_size":0,"ondisk_log_size":7149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:53.470052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302710784,"num_objects":4619,"num_object_clones":0,"num_object_copies":13857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7149,"num_write_kb":376576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":552,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:32.045011+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:43:32.045011+0000","last_peered":"2023-10-09T05:43:32.045011+0000","last_clean":"2023-10-09T05:43:32.045011+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:43:32.045011+0000","last_undegraded":"2023-10-09T05:43:32.045011+0000","last_fullsized":"2023-10-09T05:43:32.045011+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":184,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.983945+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:43:31.983945+0000","last_peered":"2023-10-09T05:43:31.983945+0000","last_clean":"2023-10-09T05:43:31.983945+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:43:31.983945+0000","last_undegraded":"2023-10-09T05:43:31.983945+0000","last_fullsized":"2023-10-09T05:43:31.983945+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":214,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.984235+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:43:31.984235+0000","last_peered":"2023-10-09T05:43:31.984235+0000","last_clean":"2023-10-09T05:43:31.984235+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:43:31.984235+0000","last_undegraded":"2023-10-09T05:43:31.984235+0000","last_fullsized":"2023-10-09T05:43:31.984235+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"236'7275","reported_seq":7318,"reported_epoch":236,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:43:43.489854+0000","last_change":"2023-10-09T05:43:21.239960+0000","last_active":"2023-10-09T05:43:43.489854+0000","last_peered":"2023-10-09T05:43:43.489854+0000","last_clean":"2023-10-09T05:43:18.711607+0000","last_became_active":"2023-10-09T05:43:21.221385+0000","last_became_peered":"2023-10-09T05:43:21.221385+0000","last_unstale":"2023-10-09T05:43:43.489854+0000","last_undegraded":"2023-10-09T05:43:21.059885+0000","last_fullsized":"2023-10-09T05:43:20.882682+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7275,"log_dups_size":0,"ondisk_log_size":7275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:07.010261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306380800,"num_objects":4675,"num_object_clones":0,"num_object_copies":14025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5973,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7275,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3","objects":-798},{"shards":"0,3,5","objects":4675}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":882,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.966310+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:43:31.966310+0000","last_peered":"2023-10-09T05:43:31.966310+0000","last_clean":"2023-10-09T05:43:31.966310+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:43:31.966310+0000","last_undegraded":"2023-10-09T05:43:31.966310+0000","last_fullsized":"2023-10-09T05:43:31.966310+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":390,"reported_epoch":235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:43:38.492647+0000","last_change":"2023-10-09T05:43:20.894197+0000","last_active":"2023-10-09T05:43:38.492647+0000","last_peered":"2023-10-09T05:43:38.492647+0000","last_clean":"2023-10-09T05:43:08.904107+0000","last_became_active":"2023-10-09T05:43:20.891297+0000","last_became_peered":"2023-10-09T05:43:20.891297+0000","last_unstale":"2023-10-09T05:43:38.492647+0000","last_undegraded":"2023-10-09T05:43:20.877647+0000","last_fullsized":"2023-10-09T05:43:20.877062+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:23.482963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":200,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.964089+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:43:31.964089+0000","last_peered":"2023-10-09T05:43:31.964089+0000","last_clean":"2023-10-09T05:43:31.964089+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:43:31.964089+0000","last_undegraded":"2023-10-09T05:43:31.964089+0000","last_fullsized":"2023-10-09T05:43:31.964089+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":217,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.977710+0000","last_change":"2023-10-09T05:43:19.888977+0000","last_active":"2023-10-09T05:43:31.977710+0000","last_peered":"2023-10-09T05:43:31.977710+0000","last_clean":"2023-10-09T05:43:31.977710+0000","last_became_active":"2023-10-09T05:43:19.888644+0000","last_became_peered":"2023-10-09T05:43:19.888644+0000","last_unstale":"2023-10-09T05:43:31.977710+0000","last_undegraded":"2023-10-09T05:43:31.977710+0000","last_fullsized":"2023-10-09T05:43:31.977710+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"236'7315","reported_seq":7352,"reported_epoch":236,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:43:43.477923+0000","last_change":"2023-10-09T05:43:21.247042+0000","last_active":"2023-10-09T05:43:43.477923+0000","last_peered":"2023-10-09T05:43:43.477923+0000","last_clean":"2023-10-09T05:43:18.738931+0000","last_became_active":"2023-10-09T05:43:21.222528+0000","last_became_peered":"2023-10-09T05:43:21.222528+0000","last_unstale":"2023-10-09T05:43:43.477923+0000","last_undegraded":"2023-10-09T05:43:20.928866+0000","last_fullsized":"2023-10-09T05:43:20.877334+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7315,"log_dups_size":0,"ondisk_log_size":7315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:54.770861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303497216,"num_objects":4631,"num_object_clones":0,"num_object_copies":13893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5973,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7315,"num_write_kb":382272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[0,2],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":4631},{"shards":"0,2","objects":-817}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":680,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.934392+0000","last_change":"2023-10-09T05:43:22.954273+0000","last_active":"2023-10-09T05:43:31.934392+0000","last_peered":"2023-10-09T05:43:31.934392+0000","last_clean":"2023-10-09T05:43:31.934392+0000","last_became_active":"2023-10-09T05:43:22.953880+0000","last_became_peered":"2023-10-09T05:43:22.953880+0000","last_unstale":"2023-10-09T05:43:31.934392+0000","last_undegraded":"2023-10-09T05:43:31.934392+0000","last_fullsized":"2023-10-09T05:43:31.934392+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":189,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.957397+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:43:31.957397+0000","last_peered":"2023-10-09T05:43:31.957397+0000","last_clean":"2023-10-09T05:43:31.957397+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:43:31.957397+0000","last_undegraded":"2023-10-09T05:43:31.957397+0000","last_fullsized":"2023-10-09T05:43:31.957397+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":204,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.960564+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:43:31.960564+0000","last_peered":"2023-10-09T05:43:31.960564+0000","last_clean":"2023-10-09T05:43:31.960564+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:43:31.960564+0000","last_undegraded":"2023-10-09T05:43:31.960564+0000","last_fullsized":"2023-10-09T05:43:31.960564+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"236'7174","reported_seq":7205,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-10-09T05:43:45.022038+0000","last_change":"2023-10-09T05:41:29.073701+0000","last_active":"2023-10-09T05:43:45.022038+0000","last_peered":"2023-10-09T05:43:45.022038+0000","last_clean":"2023-10-09T05:43:45.022038+0000","last_became_active":"2023-10-09T05:41:29.073372+0000","last_became_peered":"2023-10-09T05:41:29.073372+0000","last_unstale":"2023-10-09T05:43:45.022038+0000","last_undegraded":"2023-10-09T05:43:45.022038+0000","last_fullsized":"2023-10-09T05:43:45.022038+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7174,"log_dups_size":0,"ondisk_log_size":7174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297795584,"num_objects":4544,"num_object_clones":0,"num_object_copies":13632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7174,"num_write_kb":374976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":430,"reported_epoch":235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:43:31.974412+0000","last_change":"2023-10-09T05:43:20.934237+0000","last_active":"2023-10-09T05:43:31.974412+0000","last_peered":"2023-10-09T05:43:31.974412+0000","last_clean":"2023-10-09T05:43:08.905133+0000","last_became_active":"2023-10-09T05:43:20.932614+0000","last_became_peered":"2023-10-09T05:43:20.932614+0000","last_unstale":"2023-10-09T05:43:31.974412+0000","last_undegraded":"2023-10-09T05:43:20.878705+0000","last_fullsized":"2023-10-09T05:43:20.878227+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:53.541315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":184,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.984946+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:43:31.984946+0000","last_peered":"2023-10-09T05:43:31.984946+0000","last_clean":"2023-10-09T05:43:31.984946+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:43:31.984946+0000","last_undegraded":"2023-10-09T05:43:31.984946+0000","last_fullsized":"2023-10-09T05:43:31.984946+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":207,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.957632+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:43:31.957632+0000","last_peered":"2023-10-09T05:43:31.957632+0000","last_clean":"2023-10-09T05:43:31.957632+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:43:31.957632+0000","last_undegraded":"2023-10-09T05:43:31.957632+0000","last_fullsized":"2023-10-09T05:43:31.957632+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"236'7306","reported_seq":7338,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-10-09T05:43:46.726355+0000","last_change":"2023-10-09T05:41:29.080413+0000","last_active":"2023-10-09T05:43:46.726355+0000","last_peered":"2023-10-09T05:43:46.726355+0000","last_clean":"2023-10-09T05:43:46.726355+0000","last_became_active":"2023-10-09T05:41:29.080184+0000","last_became_peered":"2023-10-09T05:41:29.080184+0000","last_unstale":"2023-10-09T05:43:46.726355+0000","last_undegraded":"2023-10-09T05:43:46.726355+0000","last_fullsized":"2023-10-09T05:43:46.726355+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7306,"log_dups_size":0,"ondisk_log_size":7306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:47.929696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299368448,"num_objects":4568,"num_object_clones":0,"num_object_copies":13704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7306,"num_write_kb":379968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":248,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.966410+0000","last_change":"2023-10-09T05:43:20.063203+0000","last_active":"2023-10-09T05:43:31.966410+0000","last_peered":"2023-10-09T05:43:31.966410+0000","last_clean":"2023-10-09T05:43:31.966410+0000","last_became_active":"2023-10-09T05:43:19.885257+0000","last_became_peered":"2023-10-09T05:43:19.885257+0000","last_unstale":"2023-10-09T05:43:31.966410+0000","last_undegraded":"2023-10-09T05:43:31.966410+0000","last_fullsized":"2023-10-09T05:43:31.966410+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:36.424359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":625,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.983887+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:43:31.983887+0000","last_peered":"2023-10-09T05:43:31.983887+0000","last_clean":"2023-10-09T05:43:31.983887+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:43:31.983887+0000","last_undegraded":"2023-10-09T05:43:31.983887+0000","last_fullsized":"2023-10-09T05:43:31.983887+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":238,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.963408+0000","last_change":"2023-10-09T05:43:19.886969+0000","last_active":"2023-10-09T05:43:31.963408+0000","last_peered":"2023-10-09T05:43:31.963408+0000","last_clean":"2023-10-09T05:43:31.963408+0000","last_became_active":"2023-10-09T05:43:19.885111+0000","last_became_peered":"2023-10-09T05:43:19.885111+0000","last_unstale":"2023-10-09T05:43:31.963408+0000","last_undegraded":"2023-10-09T05:43:31.963408+0000","last_fullsized":"2023-10-09T05:43:31.963408+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:26.890192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"236'7494","reported_seq":7526,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-10-09T05:43:46.718126+0000","last_change":"2023-10-09T05:41:29.081230+0000","last_active":"2023-10-09T05:43:46.718126+0000","last_peered":"2023-10-09T05:43:46.718126+0000","last_clean":"2023-10-09T05:43:46.718126+0000","last_became_active":"2023-10-09T05:41:29.080911+0000","last_became_peered":"2023-10-09T05:41:29.080911+0000","last_unstale":"2023-10-09T05:43:46.718126+0000","last_undegraded":"2023-10-09T05:43:46.718126+0000","last_fullsized":"2023-10-09T05:43:46.718126+0000","mapping_epoch":213,"log_start":"217'100","ondisk_log_start":"217'100","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7394,"log_dups_size":0,"ondisk_log_size":7394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:22.580967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303693824,"num_objects":4634,"num_object_clones":0,"num_object_copies":13902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7494,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":771,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.978233+0000","last_change":"2023-10-09T05:43:22.945599+0000","last_active":"2023-10-09T05:43:31.978233+0000","last_peered":"2023-10-09T05:43:31.978233+0000","last_clean":"2023-10-09T05:43:31.978233+0000","last_became_active":"2023-10-09T05:43:22.945167+0000","last_became_peered":"2023-10-09T05:43:22.945167+0000","last_unstale":"2023-10-09T05:43:31.978233+0000","last_undegraded":"2023-10-09T05:43:31.978233+0000","last_fullsized":"2023-10-09T05:43:31.978233+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":192,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.960657+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:43:31.960657+0000","last_peered":"2023-10-09T05:43:31.960657+0000","last_clean":"2023-10-09T05:43:31.960657+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:43:31.960657+0000","last_undegraded":"2023-10-09T05:43:31.960657+0000","last_fullsized":"2023-10-09T05:43:31.960657+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":186,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.974998+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:43:31.974998+0000","last_peered":"2023-10-09T05:43:31.974998+0000","last_clean":"2023-10-09T05:43:31.974998+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:43:31.974998+0000","last_undegraded":"2023-10-09T05:43:31.974998+0000","last_fullsized":"2023-10-09T05:43:31.974998+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":209,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.957160+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:43:31.957160+0000","last_peered":"2023-10-09T05:43:31.957160+0000","last_clean":"2023-10-09T05:43:31.957160+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:43:31.957160+0000","last_undegraded":"2023-10-09T05:43:31.957160+0000","last_fullsized":"2023-10-09T05:43:31.957160+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":238,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:32.045032+0000","last_change":"2023-10-09T05:41:44.887519+0000","last_active":"2023-10-09T05:43:32.045032+0000","last_peered":"2023-10-09T05:43:32.045032+0000","last_clean":"2023-10-09T05:43:32.045032+0000","last_became_active":"2023-10-09T05:41:44.887047+0000","last_became_peered":"2023-10-09T05:41:44.887047+0000","last_unstale":"2023-10-09T05:43:32.045032+0000","last_undegraded":"2023-10-09T05:43:32.045032+0000","last_fullsized":"2023-10-09T05:43:32.045032+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:49.475913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":205,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.966292+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:43:31.966292+0000","last_peered":"2023-10-09T05:43:31.966292+0000","last_clean":"2023-10-09T05:43:31.966292+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:43:31.966292+0000","last_undegraded":"2023-10-09T05:43:31.966292+0000","last_fullsized":"2023-10-09T05:43:31.966292+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":216,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.975236+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:43:31.975236+0000","last_peered":"2023-10-09T05:43:31.975236+0000","last_clean":"2023-10-09T05:43:31.975236+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:43:31.975236+0000","last_undegraded":"2023-10-09T05:43:31.975236+0000","last_fullsized":"2023-10-09T05:43:31.975236+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":190,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.966457+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:43:31.966457+0000","last_peered":"2023-10-09T05:43:31.966457+0000","last_clean":"2023-10-09T05:43:31.966457+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:43:31.966457+0000","last_undegraded":"2023-10-09T05:43:31.966457+0000","last_fullsized":"2023-10-09T05:43:31.966457+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":216,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.975001+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:43:31.975001+0000","last_peered":"2023-10-09T05:43:31.975001+0000","last_clean":"2023-10-09T05:43:31.975001+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:43:31.975001+0000","last_undegraded":"2023-10-09T05:43:31.975001+0000","last_fullsized":"2023-10-09T05:43:31.975001+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":216,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.961176+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:43:31.961176+0000","last_peered":"2023-10-09T05:43:31.961176+0000","last_clean":"2023-10-09T05:43:31.961176+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:43:31.961176+0000","last_undegraded":"2023-10-09T05:43:31.961176+0000","last_fullsized":"2023-10-09T05:43:31.961176+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":228,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:36.746435+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:43:36.746435+0000","last_peered":"2023-10-09T05:43:36.746435+0000","last_clean":"2023-10-09T05:43:36.746435+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:43:36.746435+0000","last_undegraded":"2023-10-09T05:43:36.746435+0000","last_fullsized":"2023-10-09T05:43:36.746435+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":192,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.978173+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:43:31.978173+0000","last_peered":"2023-10-09T05:43:31.978173+0000","last_clean":"2023-10-09T05:43:31.978173+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:43:31.978173+0000","last_undegraded":"2023-10-09T05:43:31.978173+0000","last_fullsized":"2023-10-09T05:43:31.978173+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":201,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T05:43:31.974871+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:43:31.974871+0000","last_peered":"2023-10-09T05:43:31.974871+0000","last_clean":"2023-10-09T05:43:31.974871+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:43:31.974871+0000","last_undegraded":"2023-10-09T05:43:31.974871+0000","last_fullsized":"2023-10-09T05:43:31.974871+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4787077144,"num_objects":73046,"num_object_clones":0,"num_object_copies":219138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24066,"num_objects_misplaced":4566,"num_objects_unfound":0,"num_objects_dirty":73046,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":115942,"num_write_kb":6047553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":657,"num_bytes_recovered":42860544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14630465536,"data_stored":14630453320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115842,"ondisk_log_size":115842,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846702,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328988,"kb_used_data":6484,"kb_used_omap":14,"kb_used_meta":322481,"kb_avail":93416164,"statfs":{"total":95995035648,"available":95658151936,"internally_reserved":0,"allocated":6639616,"data_stored":6090946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":330221402},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:42:52 2023","interfaces":[{"interface":"back","average":{"1min":10.589,"5min":3.4049999999999998,"15min":6.0679999999999996},"min":{"1min":0.252,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":113.018,"5min":113.018,"15min":220.77500000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":10.676,"5min":3.46,"15min":6.7539999999999996},"min":{"1min":0.30499999999999999,"5min":0.29499999999999998,"15min":0.22700000000000001},"max":{"1min":113.005,"5min":113.005,"15min":220.78700000000001},"last":0.442}]},{"osd":1,"last update":"Mon Oct 9 05:42:52 2023","interfaces":[{"interface":"back","average":{"1min":5.2240000000000002,"5min":3.427,"15min":2.8119999999999998},"min":{"1min":0.28999999999999998,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":58.615000000000002,"5min":58.615000000000002,"15min":190.68100000000001},"last":6.1799999999999997},{"interface":"front","average":{"1min":4.9640000000000004,"5min":3.2949999999999999,"15min":2.5619999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":59.134999999999998,"5min":59.134999999999998,"15min":191.09200000000001},"last":1.0669999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:42:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":2.2360000000000002,"15min":4.9269999999999996},"min":{"1min":0.313,"5min":0.308,"15min":0.23899999999999999},"max":{"1min":4.3959999999999999,"5min":29.391999999999999,"15min":78.716999999999999},"last":4.5780000000000003},{"interface":"front","average":{"1min":1.29,"5min":2.3690000000000002,"15min":4.8449999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27000000000000002},"max":{"1min":5.4139999999999997,"5min":29.271999999999998,"15min":78.768000000000001},"last":0.59299999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:42:52 2023","interfaces":[{"interface":"back","average":{"1min":9.9009999999999998,"5min":6.633,"15min":2.9089999999999998},"min":{"1min":0.36299999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":108.90300000000001,"5min":185.17099999999999,"15min":185.17099999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":10.630000000000001,"5min":6.633,"15min":3.6019999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":109.08499999999999,"5min":183.35499999999999,"15min":183.35499999999999},"last":1.5629999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:42:52 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.56299999999999994,"15min":1.371},"min":{"1min":0.29099999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.60799999999999998,"5min":2.9289999999999998,"15min":12.789},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.64000000000000001,"15min":1.585},"min":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":1.234,"5min":4.3419999999999996,"15min":9.0470000000000006},"last":3.4969999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:42:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.69199999999999995,"15min":1.52},"min":{"1min":0.311,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.2110000000000001,"5min":4.4429999999999996,"15min":10.813000000000001},"last":4.0220000000000002},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.73999999999999999,"15min":1.921},"min":{"1min":0.36099999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.7029999999999998,"5min":4.6769999999999996,"15min":10.321},"last":3.9500000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:42:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.70599999999999996,"15min":1.5900000000000001},"min":{"1min":0.35199999999999998,"5min":0.318,"15min":0.318},"max":{"1min":1.258,"5min":4.468,"15min":9.2530000000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.74099999999999999,"15min":1.4810000000000001},"min":{"1min":0.32500000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.6600000000000001,"5min":4.7060000000000004,"15min":8.2680000000000007},"last":4.2279999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626728,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1759960,"kb_used_data":1450448,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91985192,"statfs":{"total":95995035648,"available":94192836608,"internally_reserved":0,"allocated":1485258752,"data_stored":1484714354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:43:30 2023","interfaces":[{"interface":"back","average":{"1min":15.058,"5min":7.9409999999999998,"15min":4.2859999999999996},"min":{"1min":0.249,"5min":0.214,"15min":0.214},"max":{"1min":162.08799999999999,"5min":162.08799999999999,"15min":162.08799999999999},"last":1.278},{"interface":"front","average":{"1min":12.91,"5min":7.2599999999999998,"15min":4},"min":{"1min":0.255,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":161.90899999999999,"5min":161.90899999999999,"15min":161.90899999999999},"last":0.45200000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":4.3289999999999997,"5min":7.8470000000000004,"15min":3.702},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":71.856999999999999,"5min":228.24299999999999,"15min":228.24299999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":4.4459999999999997,"5min":7.6920000000000002,"15min":3.746},"min":{"1min":0.312,"5min":0.252,"15min":0.252},"max":{"1min":71.873999999999995,"5min":228.22499999999999,"15min":228.22499999999999},"last":0.35299999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:43:30 2023","interfaces":[{"interface":"back","average":{"1min":6.782,"5min":6.5839999999999996,"15min":3.8929999999999998},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":77.233000000000004,"5min":115.40600000000001,"15min":115.40600000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":6.5060000000000002,"5min":6.7789999999999999,"15min":3.9140000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":77.254999999999995,"5min":115.297,"15min":115.297},"last":0.56899999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":2.5430000000000001,"5min":6.2530000000000001,"15min":3.2589999999999999},"min":{"1min":0.31,"5min":0.307,"15min":0.26800000000000002},"max":{"1min":17.510000000000002,"5min":115.244,"15min":115.244},"last":2.27},{"interface":"front","average":{"1min":3.282,"5min":6.0419999999999998,"15min":3.2730000000000001},"min":{"1min":0.35899999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":19.324999999999999,"5min":115.34999999999999,"15min":115.34999999999999},"last":2.3540000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.3540000000000001,"15min":1.7030000000000001},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":2.4409999999999998,"5min":115.214,"15min":125.64},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.40500000000000003,"5min":1.45,"15min":1.6040000000000001},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":1.089,"5min":115.22799999999999,"15min":125.625},"last":0.51900000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":2.1000000000000001,"15min":1.859},"min":{"1min":0.28000000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.9359999999999999,"5min":115.316,"15min":127.35899999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.44700000000000001,"5min":2.0379999999999998,"15min":1.7310000000000001},"min":{"1min":0.30199999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.125,"5min":115.30500000000001,"15min":128.93799999999999},"last":0.71199999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.6399999999999999,"15min":2.4660000000000002},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":3.044,"5min":115.514,"15min":115.514},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.673,"15min":1.806},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.163,"5min":116.53100000000001,"15min":116.53100000000001},"last":0.68899999999999995}]}]},{"osd":1,"up_from":93,"seq":399431958689,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2506828,"kb_used_data":2122692,"kb_used_omap":7,"kb_used_meta":384120,"kb_avail":91238324,"statfs":{"total":95995035648,"available":93428043776,"internally_reserved":0,"allocated":2173636608,"data_stored":2172812675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":393339406},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":12.887,"5min":5.1360000000000001,"15min":3.6099999999999999},"min":{"1min":0.28899999999999998,"5min":0.20300000000000001,"15min":0.185},"max":{"1min":82.459999999999994,"5min":82.459999999999994,"15min":207.858},"last":11.35},{"interface":"front","average":{"1min":12.430999999999999,"5min":6.1980000000000004,"15min":4.0720000000000001},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":80.004000000000005,"5min":80.004000000000005,"15min":206.97200000000001},"last":11.526}]},{"osd":2,"last update":"Mon Oct 9 05:43:15 2023","interfaces":[{"interface":"back","average":{"1min":7.6900000000000004,"5min":4.9370000000000003,"15min":4.0220000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.184},"max":{"1min":99.381,"5min":99.381,"15min":101.367},"last":1.0389999999999999},{"interface":"front","average":{"1min":6.6139999999999999,"5min":4.5659999999999998,"15min":3.7400000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":99.861000000000004,"5min":99.861000000000004,"15min":99.861000000000004},"last":0.75600000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":18.77,"5min":9.8010000000000002,"15min":4.2220000000000004},"min":{"1min":0.33600000000000002,"5min":0.24199999999999999,"15min":0.223},"max":{"1min":104.26600000000001,"5min":162.33699999999999,"15min":162.33699999999999},"last":1.6439999999999999},{"interface":"front","average":{"1min":18.106000000000002,"5min":10.465,"15min":4.4269999999999996},"min":{"1min":0.26900000000000002,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":102.947,"5min":162.376,"15min":162.376},"last":0.53700000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":3.1120000000000001,"15min":2.2999999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.0550000000000002,"5min":63.116,"15min":141.32900000000001},"last":11.493},{"interface":"front","average":{"1min":1.077,"5min":2.2509999999999999,"15min":1.8779999999999999},"min":{"1min":0.35099999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.956,"5min":40.475999999999999,"15min":141.15899999999999},"last":11.446}]},{"osd":5,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":2.3420000000000001,"15min":2.2850000000000001},"min":{"1min":0.38200000000000001,"5min":0.307,"15min":0.307},"max":{"1min":3.0009999999999999,"5min":40.633000000000003,"15min":235.18700000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.92400000000000004,"5min":3.1640000000000001,"15min":2.6549999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.0139999999999998,"5min":63.162999999999997,"15min":235.79300000000001},"last":11.554}]},{"osd":6,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":3.5129999999999999,"15min":1.931},"min":{"1min":0.435,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.0329999999999999,"5min":103.60899999999999,"15min":103.60899999999999},"last":5.8840000000000003},{"interface":"front","average":{"1min":1.1519999999999999,"5min":4.1820000000000004,"15min":2.1890000000000001},"min":{"1min":0.39300000000000002,"5min":0.378,"15min":0.33400000000000002},"max":{"1min":2.9740000000000002,"5min":103.679,"15min":103.679},"last":1.1930000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:42:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.3959999999999999,"15min":2.4129999999999998},"min":{"1min":0.45100000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.0250000000000004,"5min":42.204000000000001,"15min":126.68300000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.101,"5min":2.444,"15min":2.5070000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.34399999999999997},"max":{"1min":3.9940000000000002,"5min":42.228999999999999,"15min":126.738},"last":11.395}]}]},{"osd":0,"up_from":88,"seq":377957122215,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2815856,"kb_used_data":2387624,"kb_used_omap":8,"kb_used_meta":428215,"kb_avail":90929296,"statfs":{"total":95995035648,"available":93111599104,"internally_reserved":0,"allocated":2444926976,"data_stored":2444068464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":438492936},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:42:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3079999999999998,"5min":2.3919999999999999,"15min":1.4419999999999999},"min":{"1min":0.20399999999999999,"5min":0.20200000000000001,"15min":0.17199999999999999},"max":{"1min":11.289,"5min":17.760000000000002,"15min":17.760000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":3.0649999999999999,"5min":3.476,"15min":1.865},"min":{"1min":0.38100000000000001,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":13.754,"5min":26.163,"15min":26.163},"last":1.96}]},{"osd":2,"last update":"Mon Oct 9 05:43:23 2023","interfaces":[{"interface":"back","average":{"1min":4.7670000000000003,"5min":4.702,"15min":3.1320000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.214},"max":{"1min":13.302,"5min":92.230000000000004,"15min":92.230000000000004},"last":0.20499999999999999},{"interface":"front","average":{"1min":4.1230000000000002,"5min":4.7539999999999996,"15min":2.9790000000000001},"min":{"1min":0.26800000000000002,"5min":0.222,"15min":0.218},"max":{"1min":16.402999999999999,"5min":92.247,"15min":92.247},"last":0.48499999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:42:54 2023","interfaces":[{"interface":"back","average":{"1min":3.5899999999999999,"5min":7.6390000000000002,"15min":3.3199999999999998},"min":{"1min":0.35199999999999998,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":14.808999999999999,"5min":123.024,"15min":123.024},"last":2.274},{"interface":"front","average":{"1min":5.1360000000000001,"5min":6.8369999999999997,"15min":3.1299999999999999},"min":{"1min":0.40200000000000002,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":11.715999999999999,"5min":99.162000000000006,"15min":99.162000000000006},"last":2.2530000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:43:23 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.47,"15min":1.395},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.35,"5min":11.35,"15min":69.082999999999998},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.974,"5min":2.2829999999999999,"15min":1.827},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.30399999999999999},"max":{"1min":6.508,"5min":9.5190000000000001,"15min":69.102000000000004},"last":2.556}]},{"osd":5,"last update":"Mon Oct 9 05:43:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":6.6639999999999997,"15min":3.0569999999999999},"min":{"1min":0.35699999999999998,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":6.7750000000000004,"5min":281.733,"15min":281.733},"last":6.8620000000000001},{"interface":"front","average":{"1min":3.395,"5min":6.7850000000000001,"15min":3.117},"min":{"1min":0.31,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.552,"5min":282.95400000000001,"15min":282.95400000000001},"last":0.38200000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:43:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":2.4420000000000002,"15min":1.6379999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":6.625,"5min":11.664,"15min":11.664},"last":6.9109999999999996},{"interface":"front","average":{"1min":3.4180000000000001,"5min":2.633,"15min":1.7509999999999999},"min":{"1min":0.38800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":12.587999999999999,"5min":12.587999999999999,"15min":12.587999999999999},"last":0.34899999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:42:54 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":1.6759999999999999,"15min":1.448},"min":{"1min":0.35999999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":11.631,"5min":11.631,"15min":11.631},"last":7.1159999999999997},{"interface":"front","average":{"1min":2.2869999999999999,"5min":2.5089999999999999,"15min":2.351},"min":{"1min":0.439,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":7.2190000000000003,"5min":9.6329999999999991,"15min":11.068},"last":6.9580000000000002}]}]},{"osd":2,"up_from":98,"seq":420906795163,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2576940,"kb_used_data":2103396,"kb_used_omap":11,"kb_used_meta":473524,"kb_avail":91168212,"statfs":{"total":95995035648,"available":93356249088,"internally_reserved":0,"allocated":2153877504,"data_stored":2152963805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":484889173},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:43:18 2023","interfaces":[{"interface":"back","average":{"1min":4.7080000000000002,"5min":3.5619999999999998,"15min":2.1280000000000001},"min":{"1min":0.23699999999999999,"5min":0.17799999999999999,"15min":0.17199999999999999},"max":{"1min":53.709000000000003,"5min":53.709000000000003,"15min":53.709000000000003},"last":0.32800000000000001},{"interface":"front","average":{"1min":6.7439999999999998,"5min":4.4059999999999997,"15min":2.286},"min":{"1min":0.23899999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":53.664999999999999,"5min":53.664999999999999,"15min":53.664999999999999},"last":1.042}]},{"osd":1,"last update":"Mon Oct 9 05:43:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7810000000000001,"5min":5.5430000000000001,"15min":2.919},"min":{"1min":0.29099999999999998,"5min":0.23100000000000001,"15min":0.20699999999999999},"max":{"1min":10.066000000000001,"5min":98.575000000000003,"15min":98.575000000000003},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.9019999999999999,"5min":4.5199999999999996,"15min":2.4550000000000001},"min":{"1min":0.32100000000000001,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":5.9429999999999996,"5min":90.135000000000005,"15min":98.397000000000006},"last":0.41599999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:43:03 2023","interfaces":[{"interface":"back","average":{"1min":16.231999999999999,"5min":8.5359999999999996,"15min":3.7850000000000001},"min":{"1min":0.28299999999999997,"5min":0.26700000000000002,"15min":0.24199999999999999},"max":{"1min":127.58,"5min":150.65899999999999,"15min":150.65899999999999},"last":0.749},{"interface":"front","average":{"1min":10.042999999999999,"5min":7.7460000000000004,"15min":3.6970000000000001},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":127.083,"5min":150.851,"15min":150.851},"last":0.35199999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:43:18 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":2.1600000000000001,"15min":1.8500000000000001},"min":{"1min":0.42099999999999999,"5min":0.29099999999999998,"15min":0.27700000000000002},"max":{"1min":5.8739999999999997,"5min":14.382,"15min":115.84999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.92800000000000005,"5min":2.1120000000000001,"15min":2.0099999999999998},"min":{"1min":0.42699999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.9910000000000001,"5min":13.815,"15min":116.149},"last":0.46200000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:43:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2370000000000001,"5min":2.5209999999999999,"15min":1.8440000000000001},"min":{"1min":0.46800000000000003,"5min":0.36699999999999999,"15min":0.312},"max":{"1min":10.023,"5min":13.712999999999999,"15min":24.597999999999999},"last":0.505},{"interface":"front","average":{"1min":1.714,"5min":2.1030000000000002,"15min":1.593},"min":{"1min":0.44600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.915,"5min":14.426,"15min":23.919},"last":0.71399999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:43:18 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":2.1859999999999999,"15min":1.6890000000000001},"min":{"1min":0.39600000000000002,"5min":0.35599999999999998,"15min":0.35299999999999998},"max":{"1min":2.9380000000000002,"5min":13.839,"15min":20.503},"last":0.376},{"interface":"front","average":{"1min":2.2879999999999998,"5min":2.476,"15min":1.6990000000000001},"min":{"1min":0.52600000000000002,"5min":0.36799999999999999,"15min":0.34499999999999997},"max":{"1min":10.097,"5min":13.757,"15min":13.757},"last":9.8179999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:43:03 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":2.2869999999999999,"15min":2.5579999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":6.3890000000000002,"5min":13.865,"15min":24.030999999999999},"last":4.8819999999999997},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.589,"15min":2.1600000000000001},"min":{"1min":0.46500000000000002,"5min":0.42399999999999999,"15min":0.41599999999999998},"max":{"1min":10.131,"5min":13.781000000000001,"15min":24.178000000000001},"last":0.84199999999999997}]}]},{"osd":3,"up_from":103,"seq":442381631637,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3369028,"kb_used_data":2954364,"kb_used_omap":4,"kb_used_meta":414587,"kb_avail":90376124,"statfs":{"total":95995035648,"available":92545150976,"internally_reserved":0,"allocated":3025268736,"data_stored":3024451041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":424537694},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":4.8460000000000001,"15min":2.5},"min":{"1min":0.184,"5min":0.184,"15min":0.183},"max":{"1min":6.8230000000000004,"5min":167.82400000000001,"15min":167.82400000000001},"last":106.95999999999999},{"interface":"front","average":{"1min":3.1720000000000002,"5min":5.9649999999999999,"15min":2.9319999999999999},"min":{"1min":0.435,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.3789999999999996,"5min":167.89599999999999,"15min":167.89599999999999},"last":107.348}]},{"osd":1,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":2.5750000000000002,"5min":3.2770000000000001,"15min":2.254},"min":{"1min":0.32200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.0039999999999996,"5min":48.052,"15min":83.921999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.996,"5min":3.286,"15min":2.2410000000000001},"min":{"1min":0.25900000000000001,"5min":0.25,"15min":0.247},"max":{"1min":11.452,"5min":13.551,"15min":83.707999999999998},"last":0.253}]},{"osd":2,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":2.625,"5min":6.4249999999999998,"15min":3.851},"min":{"1min":0.29499999999999998,"5min":0.25,"15min":0.215},"max":{"1min":12.243,"5min":190.43299999999999,"15min":190.43299999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":2.371,"5min":7.5789999999999997,"15min":4.2569999999999997},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":13.788,"5min":190.267,"15min":190.267},"last":0.27200000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":3.1469999999999998,"15min":2.0600000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.7649999999999997,"5min":51.591999999999999,"15min":51.591999999999999},"last":0.38},{"interface":"front","average":{"1min":1.6870000000000001,"5min":2.125,"15min":1.6180000000000001},"min":{"1min":0.315,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.5839999999999996,"5min":26.774999999999999,"15min":45.735999999999997},"last":0.35299999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.5699999999999998,"15min":1.7330000000000001},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":11.372999999999999,"5min":13.114000000000001,"15min":47.948999999999998},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.2949999999999999,"15min":1.72},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.548,"5min":26.728000000000002,"15min":47.968000000000004},"last":0.45300000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7310000000000001,"5min":2.1930000000000001,"15min":1.595},"min":{"1min":0.33900000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.6100000000000003,"5min":26.847999999999999,"15min":26.847999999999999},"last":0.504},{"interface":"front","average":{"1min":2.3170000000000002,"5min":2.6560000000000001,"15min":1.573},"min":{"1min":0.432,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":11.423,"5min":13.058,"15min":13.058},"last":0.49099999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:42:58 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":2.77,"15min":2.3849999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.806,"5min":26.024999999999999,"15min":26.024999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.411,"5min":2.899,"15min":2.2789999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":4.8419999999999996,"5min":25.995999999999999,"15min":25.995999999999999},"last":0.46200000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019184,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2127192,"kb_used_data":1821776,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91617960,"statfs":{"total":95995035648,"available":93816791040,"internally_reserved":0,"allocated":1865498624,"data_stored":1864953980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:43:30 2023","interfaces":[{"interface":"back","average":{"1min":5.8680000000000003,"5min":3.4089999999999998,"15min":2.1030000000000002},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.214},"max":{"1min":73.384,"5min":100.504,"15min":100.504},"last":2.9039999999999999},{"interface":"front","average":{"1min":5.6050000000000004,"5min":3.786,"15min":2.3599999999999999},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.23200000000000001},"max":{"1min":71.884,"5min":100.717,"15min":100.717},"last":0.54000000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:42:59 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":4.5060000000000002,"15min":2.2749999999999999},"min":{"1min":0.26200000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.1760000000000002,"5min":187.369,"15min":187.369},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":4.7690000000000001,"15min":2.4060000000000001},"min":{"1min":0.28499999999999998,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":3.9929999999999999,"5min":187.35300000000001,"15min":187.35300000000001},"last":0.56399999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:43:22 2023","interfaces":[{"interface":"back","average":{"1min":5.7320000000000002,"5min":4.4169999999999998,"15min":4.3929999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":71.748000000000005,"5min":100.148,"15min":196.93700000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":5.4530000000000003,"5min":4.2729999999999997,"15min":4.2969999999999997},"min":{"1min":0.33800000000000002,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":72.096000000000004,"5min":100.086,"15min":197.08099999999999},"last":0.42999999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:42:47 2023","interfaces":[{"interface":"back","average":{"1min":6.1219999999999999,"5min":9.1140000000000008,"15min":3.7949999999999999},"min":{"1min":0.39000000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":72.391999999999996,"5min":211.48699999999999,"15min":211.48699999999999},"last":3.246},{"interface":"front","average":{"1min":5.6529999999999996,"5min":8.8309999999999995,"15min":3.661},"min":{"1min":0.34899999999999998,"5min":0.34799999999999998,"15min":0.309},"max":{"1min":71.721999999999994,"5min":211.31100000000001,"15min":211.31100000000001},"last":0.51900000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:42:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.58499999999999996,"15min":1.5629999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.9099999999999999,"5min":3.5259999999999998,"15min":113.35899999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55700000000000005,"15min":1.514},"min":{"1min":0.33800000000000002,"5min":0.317,"15min":0.30599999999999999},"max":{"1min":1.9239999999999999,"5min":3.3849999999999998,"15min":113.32299999999999},"last":0.45500000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:43:03 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.50600000000000001,"15min":0.69699999999999995},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":0.71799999999999997,"5min":3.4390000000000001,"15min":11.944000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.745,"15min":0.85699999999999998},"min":{"1min":0.32400000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.1000000000000001,"5min":7.6500000000000004,"15min":10.356},"last":0.44700000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:42:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.63500000000000001,"15min":2.1880000000000002},"min":{"1min":0.371,"5min":0.31900000000000001,"15min":0.312},"max":{"1min":0.79800000000000004,"5min":4.5819999999999999,"15min":30.393999999999998},"last":0.999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.71499999999999997,"15min":2.4449999999999998},"min":{"1min":0.34899999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.8779999999999999,"5min":4.5999999999999996,"15min":30.334},"last":0.39200000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822956,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1824776,"kb_used_data":1519360,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91920376,"statfs":{"total":95995035648,"available":94126465024,"internally_reserved":0,"allocated":1555824640,"data_stored":1555291828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:43:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":1.6930000000000001,"15min":3.1309999999999998},"min":{"1min":0.253,"5min":0.217,"15min":0.19400000000000001},"max":{"1min":9.2159999999999993,"5min":10.712999999999999,"15min":165.90199999999999},"last":3.871},{"interface":"front","average":{"1min":2.222,"5min":1.639,"15min":3.1909999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.215},"max":{"1min":8.2539999999999996,"5min":8.6099999999999994,"15min":165.96700000000001},"last":6.6180000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":4.4560000000000004,"15min":2.258},"min":{"1min":0.28899999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.6239999999999997,"5min":87.769000000000005,"15min":87.769000000000005},"last":6.3970000000000002},{"interface":"front","average":{"1min":1.1819999999999999,"5min":4.9640000000000004,"15min":2.3809999999999998},"min":{"1min":0.318,"5min":0.29899999999999999,"15min":0.24399999999999999},"max":{"1min":3.1560000000000001,"5min":87.864000000000004,"15min":87.864000000000004},"last":4.8090000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:43:29 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":3.847,"15min":4.2949999999999999},"min":{"1min":0.32800000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":31.114000000000001,"5min":73.311999999999998,"15min":290.697},"last":3.9129999999999998},{"interface":"front","average":{"1min":3.3380000000000001,"5min":3.6349999999999998,"15min":4.1829999999999998},"min":{"1min":0.378,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":33.875999999999998,"5min":73.280000000000001,"15min":290.786},"last":1.5389999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":6.9630000000000001,"15min":3.125},"min":{"1min":0.30399999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":8.0879999999999992,"5min":95.236999999999995,"15min":95.236999999999995},"last":2.6739999999999999},{"interface":"front","average":{"1min":1.738,"5min":6.4569999999999999,"15min":2.9409999999999998},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":7.9829999999999997,"5min":93.471000000000004,"15min":93.471000000000004},"last":2.7210000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:42:51 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.60199999999999998,"15min":0.71699999999999997},"min":{"1min":0.28499999999999998,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":0.69799999999999995,"5min":5.6459999999999999,"15min":9.3510000000000009},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.57499999999999996,"15min":0.94399999999999995},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":1.9079999999999999,"5min":4.8209999999999997,"15min":14.236000000000001},"last":1.117}]},{"osd":6,"last update":"Mon Oct 9 05:43:08 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.93200000000000005,"15min":1.367},"min":{"1min":0.316,"5min":0.29499999999999998,"15min":0.26600000000000001},"max":{"1min":1.958,"5min":27.146000000000001,"15min":95.010999999999996},"last":2.2269999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.98599999999999999,"15min":1.27},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.0510000000000002,"5min":27.215,"15min":94.194000000000003},"last":2.2509999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:43:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.78500000000000003,"15min":1.4019999999999999},"min":{"1min":0.35799999999999998,"5min":0.34399999999999997,"15min":0.28999999999999998},"max":{"1min":0.60899999999999999,"5min":5.5570000000000004,"15min":9.9849999999999994},"last":1.1519999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.69299999999999995,"15min":1.599},"min":{"1min":0.375,"5min":0.308,"15min":0.308},"max":{"1min":1.9890000000000001,"5min":5.0549999999999997,"15min":14.432},"last":2.0489999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2430013440,"data_stored":2430009368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2163150848,"data_stored":2163146776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2141257728,"data_stored":2141257728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3020750848,"data_stored":3020750848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1862139904,"data_stored":1862139904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1543634944,"data_stored":1543634944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1469452288,"data_stored":1469448216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":65536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:43:51.929 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:43:51.929 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_2 pg_num 26 2023-10-09T05:43:51.951 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:51.958 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:52.183 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:51 smithi191 ceph-mon[161669]: pgmap v698: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 24082/214863 objects degraded (11.208%); 4566/214863 objects misplaced (2.125%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:43:52.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:51 smithi008 ceph-mon[191227]: pgmap v698: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 24082/214863 objects degraded (11.208%); 4566/214863 objects misplaced (2.125%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:43:52.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:51 smithi008 ceph-mon[184418]: pgmap v698: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 24082/214863 objects degraded (11.208%); 4566/214863 objects misplaced (2.125%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:43:52.221 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:52.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:52.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:52.331 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:52.426 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:52.639 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:52.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:52.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:52.772 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:52.798 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:52.942 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:52 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24082/214863 objects degraded (11.208%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:53.044 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:52 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24082/214863 objects degraded (11.208%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:53.057 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:53.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:53.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:53.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:43:53.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:52 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24082/214863 objects degraded (11.208%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:53.218 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:53.318 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:43:53.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:53.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:43:53.423 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:53.477 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:53.579 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:53.579 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:53.590 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:53.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:53.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:43:53.732 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:53.818 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:43:53.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:53.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:43:53.929 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:53.983 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:43:54.049 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:53 smithi191 ceph-mon[161669]: pgmap v699: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 24090/211095 objects degraded (11.412%); 4405/211095 objects misplaced (2.087%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:43:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2377196481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:53 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:54.389 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:43:54.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:53 smithi008 ceph-mon[191227]: pgmap v699: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 24090/211095 objects degraded (11.412%); 4405/211095 objects misplaced (2.087%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:43:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2377196481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:53 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:54.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:53 smithi008 ceph-mon[184418]: pgmap v699: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 24090/211095 objects degraded (11.412%); 4405/211095 objects misplaced (2.087%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:43:54.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2377196481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:54.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:53 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:43:54.931 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:54.951 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:54.969 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:54.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:54.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:55.083 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:55.126 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:55.204 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:55.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:55.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:54 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:54 smithi191 ceph-mon[161669]: failed to encode map e237 with expected crc 2023-10-09T05:43:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:54 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:43:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:54 smithi191 ceph-mon[161669]: osdmap e237: 8 total, 8 up, 8 in 2023-10-09T05:43:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:54 smithi191 ceph-mon[161669]: failed to encode map e237 with expected crc 2023-10-09T05:43:55.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:54 smithi191 ceph-mon[161669]: pgmap v701: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 24090/208080 objects degraded (11.577%); 4405/208080 objects misplaced (2.117%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:43:55.307 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:55.401 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:55.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[184418]: failed to encode map e237 with expected crc 2023-10-09T05:43:55.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:43:55.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[184418]: osdmap e237: 8 total, 8 up, 8 in 2023-10-09T05:43:55.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[184418]: failed to encode map e237 with expected crc 2023-10-09T05:43:55.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[184418]: pgmap v701: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 24090/208080 objects degraded (11.577%); 4405/208080 objects misplaced (2.117%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:43:55.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:43:55.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[191227]: failed to encode map e237 with expected crc 2023-10-09T05:43:55.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:43:55.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[191227]: osdmap e237: 8 total, 8 up, 8 in 2023-10-09T05:43:55.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[191227]: failed to encode map e237 with expected crc 2023-10-09T05:43:55.404 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:54 smithi008 ceph-mon[191227]: pgmap v701: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 24090/208080 objects degraded (11.577%); 4405/208080 objects misplaced (2.117%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:43:55.407 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:56.542 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:56.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:56.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:56.649 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:43:56.697 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:43:56.697 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:43:56.746 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:57.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:43:57.505 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:43:57.506 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:43:57.516 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:43:57.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:57.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:43:57.523 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:57 smithi008 ceph-mon[184418]: pgmap v702: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24090/205557 objects degraded (11.719%); 4405/205557 objects misplaced (2.143%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:43:57.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:57 smithi008 ceph-mon[191227]: pgmap v702: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24090/205557 objects degraded (11.719%); 4405/205557 objects misplaced (2.143%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:43:57.540 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:43:57.597 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:43:57.622 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:57.670 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:57.720 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:57.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:57.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:57.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:57 smithi191 ceph-mon[161669]: pgmap v702: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24090/205557 objects degraded (11.719%); 4405/205557 objects misplaced (2.143%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:43:57.823 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:43:57.825 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:43:57.825 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:43:57.825 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:43:57.825 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:43:57.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:43:57.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:43:57.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:43:57.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:43:57.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:43:57.826 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:43:57.826 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:43:57.826 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:43:57.831 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:43:57.866 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:57.926 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:43:57.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:57.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:43:58.043 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:58.129 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:58.433 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3798871964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:58.433 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24090/205557 objects degraded (11.719%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:58.441 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3798871964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:58.441 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24090/205557 objects degraded (11.719%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:58.443 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3798871964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:43:58.443 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24090/205557 objects degraded (11.719%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:43:58.488 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:58.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:58.599 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:43:58.634 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:43:58.672 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:58.880 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:43:58.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:58.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:43:58.992 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:43:59.085 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:59.510 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:43:59.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:59.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:43:59.592 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:43:59.593 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:43:59.593 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:43:59.593 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:43:59.593 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:43:59.619 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:43:59.714 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:43:59.884 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:43:59.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:43:59.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:43:59.903 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:59 smithi191 ceph-mon[161669]: pgmap v703: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 24122/202791 objects degraded (11.895%); 4235/202791 objects misplaced (2.088%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T05:43:59.903 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:43:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:59 smithi008 ceph-mon[184418]: pgmap v703: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 24122/202791 objects degraded (11.895%); 4235/202791 objects misplaced (2.088%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T05:43:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:43:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:59.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:59 smithi008 ceph-mon[191227]: pgmap v703: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 24122/202791 objects degraded (11.895%); 4235/202791 objects misplaced (2.088%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T05:43:59.975 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:43:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:59.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:00.031 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:00.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:44:00.286 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:00.291 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:00.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:00.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:00.398 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:00.459 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:00.553 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:00.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:00.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:00.665 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:00.738 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:00.789 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:44:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:44:00] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T05:44:00.812 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:44:00.919 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3982133545' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:44:00.953 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 7 pg_num to 26 2023-10-09T05:44:01.021 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:01.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:01.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:01.134 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:01.200 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3982133545' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:44:01.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3982133545' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:44:01.205 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:01.413 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:44:01.955 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:01.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:01.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:02.065 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:02.150 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:02.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[184418]: pgmap v704: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24122/198579 objects degraded (12.147%); 4235/198579 objects misplaced (2.133%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T05:44:02.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[184418]: failed to encode map e238 with expected crc 2023-10-09T05:44:02.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[184418]: failed to encode map e238 with expected crc 2023-10-09T05:44:02.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3982133545' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:44:02.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[184418]: osdmap e238: 8 total, 8 up, 8 in 2023-10-09T05:44:02.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[184418]: failed to encode map e238 with expected crc 2023-10-09T05:44:02.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[191227]: pgmap v704: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24122/198579 objects degraded (12.147%); 4235/198579 objects misplaced (2.133%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T05:44:02.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[191227]: failed to encode map e238 with expected crc 2023-10-09T05:44:02.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[191227]: failed to encode map e238 with expected crc 2023-10-09T05:44:02.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3982133545' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:44:02.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[191227]: osdmap e238: 8 total, 8 up, 8 in 2023-10-09T05:44:02.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:01 smithi008 ceph-mon[191227]: failed to encode map e238 with expected crc 2023-10-09T05:44:02.270 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:01 smithi191 ceph-mon[161669]: pgmap v704: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24122/198579 objects degraded (12.147%); 4235/198579 objects misplaced (2.133%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T05:44:02.270 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:01 smithi191 ceph-mon[161669]: failed to encode map e238 with expected crc 2023-10-09T05:44:02.270 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:01 smithi191 ceph-mon[161669]: failed to encode map e238 with expected crc 2023-10-09T05:44:02.270 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:01 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3982133545' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:44:02.270 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:01 smithi191 ceph-mon[161669]: osdmap e238: 8 total, 8 up, 8 in 2023-10-09T05:44:02.270 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:01 smithi191 ceph-mon[161669]: failed to encode map e238 with expected crc 2023-10-09T05:44:02.314 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:02.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:02.421 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:02.531 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:44:02.531 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:44:02.531 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:44:02.531 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:119037 5.2 5:408f5192:::benchmark_data_smithi008_238708_object901:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:24.151111+0000", 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 578.37690408399999, 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88165001499999995, 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119037 2023-10-09T05:44:02.532 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151111+0000", 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151111+0000", 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151109+0000", 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1720000000000007e-06 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151117+0000", 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2285e-05 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151130+0000", 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6660000000000003e-06 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.151139+0000", 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016795371999999999 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.167935+0000", 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4433999999999999e-05 2023-10-09T05:44:02.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:24.167949+0000", 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85092211299999998 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.018871+0000", 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3704999999999998e-05 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.018965+0000", 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5246000000000003e-05 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.019050+0000", 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052231589999999998 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.024273+0000", 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048098300000000002 2023-10-09T05:44:02.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.029083+0000", 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035822990000000002 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.032666+0000", 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5321000000000001e-05 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.538 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.032721+0000", 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9962000000000001e-05 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:25.032761+0000", 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88165001499999995 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.539 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34770.0:121199 5.12 5:4c3a7433:::benchmark_data_smithi008_238708_object3063:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:34:26.591400+0000", 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 575.93661469699998, 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25645415500000002, 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34770", 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2362293909", 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 121199 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.540 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591400+0000", 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591400+0000", 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591399+0000", 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7750000000000004e-06 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591406+0000", 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0115000000000001e-05 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591416+0000", 2023-10-09T05:44:02.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9239999999999998e-06 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591421+0000", 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9996999999999997e-05 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591461+0000", 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7571999999999994e-05 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591548+0000", 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6479999999999999e-05 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.591635+0000", 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012101460000000001 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.592845+0000", 2023-10-09T05:44:02.544 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031766419999999999 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.596022+0000", 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25174085699999998 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847763+0000", 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1802999999999999e-05 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847814+0000", 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0043999999999998e-05 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:34:26.847854+0000", 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25645415500000002 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.546 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.547 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.547 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:57816 6.5 6:a6f014e8:::benchmark_data_smithi008_295510_object57815:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-10-09T05:44:02.547 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:53.723556+0000", 2023-10-09T05:44:02.547 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 368.804458651, 2023-10-09T05:44:02.547 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.293916552, 2023-10-09T05:44:02.547 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.547 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.547 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.547 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:44:02.547 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:02.547 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 57816 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723556+0000", 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723556+0000", 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:02.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723555+0000", 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4774000000000003e-05 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723589+0000", 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8880000000000003e-06 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723598+0000", 2023-10-09T05:44:02.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6620000000000002e-06 2023-10-09T05:44:02.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723604+0000", 2023-10-09T05:44:02.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0074562639999999998 2023-10-09T05:44:02.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731060+0000", 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000178142 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731238+0000", 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018982800000000001 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731428+0000", 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0034443920000000001 2023-10-09T05:44:02.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.734872+0000", 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028075729999999998 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.737680+0000", 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27808630400000001 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.552 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.015766+0000", 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9630999999999998e-05 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.015836+0000", 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001636813 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.017473+0000", 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.293916552 2023-10-09T05:44:02.553 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.554 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.554 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.554 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.554 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.554 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:57823 6.5 6:a9e36634:::benchmark_data_smithi008_295510_object57822:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-10-09T05:44:02.554 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:53.731562+0000", 2023-10-09T05:44:02.554 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 368.79645269700001, 2023-10-09T05:44:02.554 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28707806800000002, 2023-10-09T05:44:02.554 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.554 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.554 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 57823 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731562+0000", 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.555 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731562+0000", 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731561+0000", 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.126e-05 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731612+0000", 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5167999999999999e-05 2023-10-09T05:44:02.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731627+0000", 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1280000000000002e-06 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731634+0000", 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001613838 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.733248+0000", 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013990199999999999 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.733388+0000", 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011707800000000001 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.733505+0000", 2023-10-09T05:44:02.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014731880000000001 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.734978+0000", 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027389850000000002 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.737717+0000", 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27980261000000001 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.017520+0000", 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4610999999999997e-05 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.017565+0000", 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010756380000000001 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.018640+0000", 2023-10-09T05:44:02.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28707806800000002 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:72554 6.3 6:caefc39f:::benchmark_data_smithi008_295510_object72553:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:38:13.303447+0000", 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 349.224568035, 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30038207500000003, 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.561 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.562 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:44:02.562 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:02.562 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72554 2023-10-09T05:44:02.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.562 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.562 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303447+0000", 2023-10-09T05:44:02.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.563 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.563 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303447+0000", 2023-10-09T05:44:02.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:02.563 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.563 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.563 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303445+0000", 2023-10-09T05:44:02.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8564000000000002e-05 2023-10-09T05:44:02.563 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.563 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303524+0000", 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.117e-05 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303535+0000", 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.516e-06 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303542+0000", 2023-10-09T05:44:02.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050977210000000004 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.308639+0000", 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016793 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.308807+0000", 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013890300000000001 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:44:02.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.308946+0000", 2023-10-09T05:44:02.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021140360000000001 2023-10-09T05:44:02.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.311060+0000", 2023-10-09T05:44:02.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013415409999999999 2023-10-09T05:44:02.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.312402+0000", 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.289153361 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.601555+0000", 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8233999999999997e-05 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.601633+0000", 2023-10-09T05:44:02.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021954909999999999 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.603829+0000", 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30038207500000003 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:74957 6.3 6:ca97a06f:::benchmark_data_smithi008_295510_object74956:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e181)", 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:38:17.013260+0000", 2023-10-09T05:44:02.568 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 345.51475480300002, 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.256926671, 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 74957 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013260+0000", 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013260+0000", 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013259+0000", 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.71e-05 2023-10-09T05:44:02.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013306+0000", 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1833e-05 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013318+0000", 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8119999999999996e-06 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.571 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013323+0000", 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039195150000000002 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.017243+0000", 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017195099999999999 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.017415+0000", 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000214391 2023-10-09T05:44:02.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.017629+0000", 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048030879999999996 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.022432+0000", 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029784350000000002 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.025411+0000", 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24323214100000001 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.268643+0000", 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8260000000000002e-05 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.268691+0000", 2023-10-09T05:44:02.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014955210000000001 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.270187+0000", 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.256926671 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:74962 6.3 6:c979a0e9:::benchmark_data_smithi008_295510_object74961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e181)", 2023-10-09T05:44:02.575 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:38:17.015632+0000", 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 345.51238287199999, 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25564477200000002, 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 74962 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015632+0000", 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015632+0000", 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015631+0000", 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4298999999999997e-05 2023-10-09T05:44:02.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.578 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015675+0000", 2023-10-09T05:44:02.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3551e-05 2023-10-09T05:44:02.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.578 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015689+0000", 2023-10-09T05:44:02.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0249999999999999e-06 2023-10-09T05:44:02.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015695+0000", 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040153250000000001 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.019710+0000", 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118621 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.019828+0000", 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000193387 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.020022+0000", 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024983150000000001 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.022520+0000", 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0064664869999999999 2023-10-09T05:44:02.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.028987+0000", 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24126114400000001 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.270248+0000", 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9090000000000004e-05 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.270317+0000", 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00095988599999999996 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.271277+0000", 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25564477200000002 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.582 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:91608 6.5 6:a934938f:::benchmark_data_smithi008_295510_object3245:head [delete] snapc 0=[] ondisk+write+known_if_redirected e184)", 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:38:41.229295+0000", 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 321.29871943400002, 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24228119000000001, 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91608 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229295+0000", 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229295+0000", 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229295+0000", 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3749999999999999e-06 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229298+0000", 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3010000000000003e-06 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229305+0000", 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4959999999999998e-06 2023-10-09T05:44:02.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229309+0000", 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122569 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229432+0000", 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017970599999999999 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229611+0000", 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016554100000000001 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229777+0000", 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063023059999999997 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.236079+0000", 2023-10-09T05:44:02.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00062206499999999999 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.236701+0000", 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23290044500000001 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.469602+0000", 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0106999999999999e-05 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.469662+0000", 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019147490000000001 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.471577+0000", 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24228119000000001 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:91632 6.5 6:a980578b:::benchmark_data_smithi008_295510_object3269:head [delete] snapc 0=[] ondisk+write+known_if_redirected e184)", 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:38:41.249264+0000", 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 321.27875082700001, 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23107453999999999, 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91632 2023-10-09T05:44:02.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249264+0000", 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249264+0000", 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249263+0000", 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1389999999999998e-06 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249270+0000", 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3324e-05 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249283+0000", 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4700000000000001e-06 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249289+0000", 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5268000000000001e-05 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249334+0000", 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012688599999999999 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249461+0000", 2023-10-09T05:44:02.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013272699999999999 2023-10-09T05:44:02.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:02.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249594+0000", 2023-10-09T05:44:02.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024026540000000002 2023-10-09T05:44:02.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.251996+0000", 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.187454126 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.439450+0000", 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.039058080000000002 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.478508+0000", 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7931999999999999e-05 2023-10-09T05:44:02.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.478586+0000", 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017522530000000001 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.480339+0000", 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23107453999999999 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.596 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:19336 7.d 7:bca1948b:::benchmark_data_smithi008_354649_object19335:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 135.88075642699999, 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27934059700000002, 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:02.597 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 19336 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:44:02.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647257+0000", 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4706000000000003e-05 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647292+0000", 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0139999999999999e-05 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647302+0000", 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4790000000000004e-06 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647308+0000", 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0175000000000001e-05 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647348+0000", 2023-10-09T05:44:02.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3017999999999997e-05 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647431+0000", 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7472000000000005e-05 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647528+0000", 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00087520700000000002 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.648404+0000", 2023-10-09T05:44:02.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091703300000000004 2023-10-09T05:44:02.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.649321+0000", 2023-10-09T05:44:02.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27717746799999998 2023-10-09T05:44:02.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.603 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926498+0000", 2023-10-09T05:44:02.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2379000000000001e-05 2023-10-09T05:44:02.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.603 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.603 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926550+0000", 2023-10-09T05:44:02.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8653000000000003e-05 2023-10-09T05:44:02.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.603 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.603 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926599+0000", 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27934059700000002 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:52521 7.c 7:3208f079:::benchmark_data_smithi008_354649_object52520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 107.927086649, 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.253878241, 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.604 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.605 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.605 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:02.605 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:02.605 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52521 2023-10-09T05:44:02.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.605 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:44:02.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600927+0000", 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114644 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601041+0000", 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3705e-05 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601055+0000", 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3670000000000002e-06 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601062+0000", 2023-10-09T05:44:02.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6796000000000003e-05 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601118+0000", 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000151423 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601270+0000", 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000187224 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601457+0000", 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018848820000000001 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.603342+0000", 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017959000000000001 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.603521+0000", 2023-10-09T05:44:02.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25118068700000001 2023-10-09T05:44:02.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854702+0000", 2023-10-09T05:44:02.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0417000000000001e-05 2023-10-09T05:44:02.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854763+0000", 2023-10-09T05:44:02.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3959999999999999e-05 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854806+0000", 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.253878241 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72589 7.d 7:bac7991c:::benchmark_data_smithi008_354649_object72588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:44:02.611 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 85.069585723000003, 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.258363538, 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72589 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8671000000000004e-05 2023-10-09T05:44:02.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458527+0000", 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2957e-05 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458540+0000", 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6649999999999997e-06 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458547+0000", 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2287999999999997e-05 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458609+0000", 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012157100000000001 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458731+0000", 2023-10-09T05:44:02.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000152602 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458883+0000", 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026998410000000001 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.461583+0000", 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0065822210000000001 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.468166+0000", 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24850773600000001 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716673+0000", 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7136000000000002e-05 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716740+0000", 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2272999999999999e-05 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716793+0000", 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.258363538 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.618 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72593 7.d 7:ba3e3325:::benchmark_data_smithi008_354649_object72592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 85.062111059000003, 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25827438800000002, 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72593 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465902+0000", 2023-10-09T05:44:02.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000110436 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466013+0000", 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0246e-05 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466023+0000", 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7699999999999998e-06 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466029+0000", 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0343999999999999e-05 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466069+0000", 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4201000000000002e-05 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466153+0000", 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3833999999999996e-05 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466247+0000", 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020428099999999999 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.468290+0000", 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011037670000000001 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.469394+0000", 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25468252499999999 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724076+0000", 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0541000000000003e-05 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724127+0000", 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1499000000000003e-05 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724178+0000", 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25827438800000002 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.625 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72601 7.d 7:b4df4171:::benchmark_data_smithi008_354649_object72600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 85.056171105000004, 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25693990900000002, 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72601 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471843+0000", 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1391000000000003e-05 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471894+0000", 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3217000000000001e-05 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471907+0000", 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4e-06 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471913+0000", 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6798999999999998e-05 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471950+0000", 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6098000000000001e-05 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.472036+0000", 2023-10-09T05:44:02.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010641099999999999 2023-10-09T05:44:02.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.630 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:44:02.630 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.472142+0000", 2023-10-09T05:44:02.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001463415 2023-10-09T05:44:02.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.630 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.630 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.473605+0000", 2023-10-09T05:44:02.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00083725200000000003 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.474443+0000", 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25426129400000003 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728704+0000", 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5178e-05 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728749+0000", 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4474999999999998e-05 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728784+0000", 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25693990900000002 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.633 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102136 7.c 7:3c9f4178:::benchmark_data_smithi008_354649_object7640:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T05:44:02.633 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:44:02.633 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 43.652470403000002, 2023-10-09T05:44:02.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3695118229999999, 2023-10-09T05:44:02.633 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.633 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:44:02.633 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.633 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:02.633 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:02.633 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102136 2023-10-09T05:44:02.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875543+0000", 2023-10-09T05:44:02.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0234999999999999e-05 2023-10-09T05:44:02.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875553+0000", 2023-10-09T05:44:02.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6123000000000001e-05 2023-10-09T05:44:02.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875569+0000", 2023-10-09T05:44:02.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7334e-05 2023-10-09T05:44:02.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875586+0000", 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3238999999999997e-05 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875630+0000", 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120889 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875751+0000", 2023-10-09T05:44:02.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000121965 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875873+0000", 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068513569999999998 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.882724+0000", 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.155e-05 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.882735+0000", 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99459824299999999 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877334+0000", 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.508e-05 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:02.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877349+0000", 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.345701239 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.223050+0000", 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016460699999999999 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.223215+0000", 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021743302999999999 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.244958+0000", 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8379000000000001e-05 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245056+0000", 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3695118229999999 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.643 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102150 7.c 7:30e13689:::benchmark_data_smithi008_354649_object7654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:44:02.643 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:44:02.643 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 43.623414584999999, 2023-10-09T05:44:02.643 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3456080319999999, 2023-10-09T05:44:02.643 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.643 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102150 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904598+0000", 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9720000000000003e-06 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904605+0000", 2023-10-09T05:44:02.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3104e-05 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904619+0000", 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7699999999999998e-06 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904624+0000", 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5915000000000003e-05 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904680+0000", 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1524e-05 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904692+0000", 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97280887699999996 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877501+0000", 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9029999999999993e-06 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877510+0000", 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3503837970000001 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.227893+0000", 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017629700000000001 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.228070+0000", 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019175 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.228261+0000", 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086207179999999994 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.236882+0000", 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0081705159999999992 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245053+0000", 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4658000000000003e-05 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245117+0000", 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2007000000000004e-05 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245189+0000", 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005019061 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250208+0000", 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3456080319999999 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.651 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102158 7.c 7:380582de:::benchmark_data_smithi008_354649_object7662:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 43.618678682999999, 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3657197050000001, 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102158 2023-10-09T05:44:02.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909335+0000", 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1000000000000003e-06 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909340+0000", 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9660000000000002e-06 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.654 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909349+0000", 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3199999999999999e-06 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909354+0000", 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2012999999999999e-05 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909396+0000", 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4160000000000001e-06 2023-10-09T05:44:02.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909403+0000", 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96813495800000005 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877538+0000", 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5560000000000008e-06 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877546+0000", 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3727139369999999 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250260+0000", 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017051999999999999 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250431+0000", 2023-10-09T05:44:02.657 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019215399999999999 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250623+0000", 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010646828000000001 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261270+0000", 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6761999999999995e-05 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.658 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261337+0000", 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087281960000000006 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.270065+0000", 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9771000000000006e-05 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.270144+0000", 2023-10-09T05:44:02.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0049114320000000003 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275056+0000", 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3657197050000001 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102172 7.c 7:3ca77be9:::benchmark_data_smithi008_354649_object7676:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:44:02.660 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 43.591941613000003, 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3439011609999998, 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102172 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999971 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936071+0000", 2023-10-09T05:44:02.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4219999999999999e-05 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936085+0000", 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3510000000000003e-05 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936118+0000", 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3056e-05 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936131+0000", 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013862499999999999 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936270+0000", 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3222999999999997e-05 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936303+0000", 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.941830422 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.878134+0000", 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3155e-05 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.878147+0000", 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.375972827 2023-10-09T05:44:02.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254120+0000", 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00031237700000000002 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254432+0000", 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000179364 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254611+0000", 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0066698180000000001 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261281+0000", 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00092641700000000002 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.262208+0000", 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012930142 2023-10-09T05:44:02.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275138+0000", 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1118999999999995e-05 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275209+0000", 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047656299999999999 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.668 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.279974+0000", 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3439011609999998 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:124846 7.c 7:339e95c6:::benchmark_data_smithi008_354649_object30350:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.0240658929999999, 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19787401099999999, 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.669 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124846 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.670 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8360000000000001e-06 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.671 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503951+0000", 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.534e-06 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503954+0000", 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7929999999999999e-06 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503958+0000", 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070412340000000004 2023-10-09T05:44:02.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.510999+0000", 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012243900000000001 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.511122+0000", 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012357500000000001 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:44:02.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.511245+0000", 2023-10-09T05:44:02.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070869629999999999 2023-10-09T05:44:02.674 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.674 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.674 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.674 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.518332+0000", 2023-10-09T05:44:02.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010506108 2023-10-09T05:44:02.674 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.674 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.674 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.674 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.528838+0000", 2023-10-09T05:44:02.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070145440000000002 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.535853+0000", 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2738000000000002e-05 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.535905+0000", 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.165917541 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.701823+0000", 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19787401099999999 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:124870 7.c 7:34cc5040:::benchmark_data_smithi008_354649_object30374:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.99163194300000002, 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.194346888, 2023-10-09T05:44:02.676 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124870 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:02.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536382+0000", 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1800000000000001e-06 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536388+0000", 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8409999999999995e-06 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536398+0000", 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9701e-05 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536417+0000", 2023-10-09T05:44:02.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16904754699999999 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705465+0000", 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016865399999999999 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705634+0000", 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017818700000000001 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705812+0000", 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068761990000000004 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.712688+0000", 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078597960000000005 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.720548+0000", 2023-10-09T05:44:02.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059971089999999996 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.726545+0000", 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4085000000000006e-05 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.726619+0000", 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004110832 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:02.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:02.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:44:02.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.730730+0000", 2023-10-09T05:44:02.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.194346888 2023-10-09T05:44:02.683 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.683 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.683 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.683 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:02.683 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:02.683 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:44:02.684 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:02.718 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:02.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:02.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:02.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:02.887 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:03.037 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:03.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:03.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:03.163 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:03.213 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:02 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:44:03.214 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:02 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24122/198579 objects degraded (12.147%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:03.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:02 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:44:03.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:02 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24122/198579 objects degraded (12.147%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:03.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:02 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:44:03.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:02 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24122/198579 objects degraded (12.147%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:03.217 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:03.319 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:44:03.323 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:03.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:03.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:03.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:03.457 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:03.511 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:03.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:03.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:03.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:03.648 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:03.697 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:03.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:03.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:03.723 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:44:03.802 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:03.830 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:03.919 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:03.980 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:04.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:04.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:04.059 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[184418]: pgmap v706: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 24146/194727 objects degraded (12.400%); 4070/194727 objects misplaced (2.090%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T05:44:04.059 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[184418]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.059 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:44:04.059 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[184418]: osdmap e239: 8 total, 8 up, 8 in 2023-10-09T05:44:04.059 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[184418]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.059 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[184418]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.059 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[184418]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.059 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[184418]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.060 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[191227]: pgmap v706: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 24146/194727 objects degraded (12.400%); 4070/194727 objects misplaced (2.090%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T05:44:04.060 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[191227]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.060 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:44:04.060 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[191227]: osdmap e239: 8 total, 8 up, 8 in 2023-10-09T05:44:04.060 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[191227]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.060 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[191227]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.060 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[191227]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.060 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:03 smithi008 ceph-mon[191227]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.103 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:04.149 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:03 smithi191 ceph-mon[161669]: pgmap v706: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 24146/194727 objects degraded (12.400%); 4070/194727 objects misplaced (2.090%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T05:44:04.149 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:03 smithi191 ceph-mon[161669]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:44:04.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:03 smithi191 ceph-mon[161669]: osdmap e239: 8 total, 8 up, 8 in 2023-10-09T05:44:04.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:03 smithi191 ceph-mon[161669]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:03 smithi191 ceph-mon[161669]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:03 smithi191 ceph-mon[161669]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:03 smithi191 ceph-mon[161669]: failed to encode map e239 with expected crc 2023-10-09T05:44:04.154 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:04.320 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:04.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:04.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:04.426 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:04.455 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:04.502 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:04.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:04.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:04.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:04.634 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:04.679 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:04.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:04.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:04.786 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:04.818 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:04.866 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:04.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:04.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:04.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:04.980 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:44:05.002 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:05.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: osdmap e240: 8 total, 8 up, 8 in 2023-10-09T05:44:05.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.011 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.011 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.011 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.011 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.011 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.011 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.011 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:05.011 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/837101042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:05.011 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:05.012 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.012 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.012 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.012 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: osdmap e240: 8 total, 8 up, 8 in 2023-10-09T05:44:05.012 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.012 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.012 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.012 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.012 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.012 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:05.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/837101042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:05.013 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:04 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:05.196 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:05.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:05.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:05.236 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.236 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.236 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.236 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: osdmap e240: 8 total, 8 up, 8 in 2023-10-09T05:44:05.237 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.237 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.237 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.237 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.237 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.237 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.237 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.237 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.237 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.237 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.237 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: failed to encode map e240 with expected crc 2023-10-09T05:44:05.238 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:05.238 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/837101042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:05.238 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:04 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:05.302 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:05.336 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:05.703 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:44:05.704 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:44:05.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.704 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125692 7.5 7.d812e545 (undecoded) ondisk+retry+write+known_if_redirected e240)", 2023-10-09T05:44:05.704 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:03.983873+0000", 2023-10-09T05:44:05.704 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.718778855, 2023-10-09T05:44:05.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7187986400000002, 2023-10-09T05:44:05.704 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.704 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:44:05.704 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125692 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.983873+0000", 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.705 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.983873+0000", 2023-10-09T05:44:05.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:44:05.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.706 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.983871+0000", 2023-10-09T05:44:05.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9489999999999999e-06 2023-10-09T05:44:05.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.706 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.983878+0000", 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3216e-05 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.983891+0000", 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7199999999999995e-06 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.983900+0000", 2023-10-09T05:44:05.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6522999999999998e-05 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.983986+0000", 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2206e-05 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.983999+0000", 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080750000000000006 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.708 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.709 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:05.709 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.992074+0000", 2023-10-09T05:44:05.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0256e-05 2023-10-09T05:44:05.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.709 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.709 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:05.709 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.992084+0000", 2023-10-09T05:44:05.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0082105919999999992 2023-10-09T05:44:05.709 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.709 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.709 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.710 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.710 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.710 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125880 7.15 7.3c3389b5 (undecoded) ondisk+write+known_if_redirected e241)", 2023-10-09T05:44:05.710 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:05.177601+0000", 2023-10-09T05:44:05.710 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.52505114100000005, 2023-10-09T05:44:05.710 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.52512877000000002, 2023-10-09T05:44:05.710 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.710 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:44:05.710 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.710 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.710 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125880 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.177601+0000", 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.177601+0000", 2023-10-09T05:44:05.711 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:44:05.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.714 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.177599+0000", 2023-10-09T05:44:05.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2682e-05 2023-10-09T05:44:05.714 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.714 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.714 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.714 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.177611+0000", 2023-10-09T05:44:05.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1797000000000002e-05 2023-10-09T05:44:05.714 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.714 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.714 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.714 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.177633+0000", 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0013000000000001e-05 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.177643+0000", 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2264000000000001e-05 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.715 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125751 7.5 7.943ba925 (undecoded) ondisk+write+known_if_redirected e240)", 2023-10-09T05:44:05.716 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:04.195095+0000", 2023-10-09T05:44:05.716 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.507556986, 2023-10-09T05:44:05.716 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5076682190000001, 2023-10-09T05:44:05.716 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.716 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:44:05.716 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.716 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.716 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.716 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125751 2023-10-09T05:44:05.716 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.716 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.195095+0000", 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.195095+0000", 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.717 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.718 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.195093+0000", 2023-10-09T05:44:05.718 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9969999999999998e-06 2023-10-09T05:44:05.718 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.718 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.718 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.718 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.195101+0000", 2023-10-09T05:44:05.718 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3715e-05 2023-10-09T05:44:05.718 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.718 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.718 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.718 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.195115+0000", 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9129999999999997e-06 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.195122+0000", 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9567999999999994e-05 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.195201+0000", 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8299999999999996e-06 2023-10-09T05:44:05.719 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.195209+0000", 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114182 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125759 7.5 7.d9d957c5 (undecoded) ondisk+write+known_if_redirected e240)", 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:04.203272+0000", 2023-10-09T05:44:05.720 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.4993796239999999, 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4995299659999999, 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125759 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.203272+0000", 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.203272+0000", 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.203270+0000", 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1231e-05 2023-10-09T05:44:05.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.203282+0000", 2023-10-09T05:44:05.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0021999999999999e-05 2023-10-09T05:44:05.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.203302+0000", 2023-10-09T05:44:05.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0330000000000008e-06 2023-10-09T05:44:05.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.724 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.724 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.203310+0000", 2023-10-09T05:44:05.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.585e-05 2023-10-09T05:44:05.724 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.724 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:05.724 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.203395+0000", 2023-10-09T05:44:05.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.313e-05 2023-10-09T05:44:05.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:05.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.203409+0000", 2023-10-09T05:44:05.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013611299999999999 2023-10-09T05:44:05.725 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.725 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.725 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.726 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125715 7.5 7.83709b05 (undecoded) ondisk+write+known_if_redirected e240)", 2023-10-09T05:44:05.726 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:04.069242+0000", 2023-10-09T05:44:05.726 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.6334098880000001, 2023-10-09T05:44:05.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.633600151, 2023-10-09T05:44:05.726 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.726 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:44:05.726 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.726 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.726 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.726 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125715 2023-10-09T05:44:05.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.069242+0000", 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.069242+0000", 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.069240+0000", 2023-10-09T05:44:05.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0829999999999998e-06 2023-10-09T05:44:05.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.069247+0000", 2023-10-09T05:44:05.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.944e-06 2023-10-09T05:44:05.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.069257+0000", 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7990000000000001e-06 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.069262+0000", 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2568000000000002e-05 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:05.729 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.069315+0000", 2023-10-09T05:44:05.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1012000000000001e-05 2023-10-09T05:44:05.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:05.730 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.069326+0000", 2023-10-09T05:44:05.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3609000000000005e-05 2023-10-09T05:44:05.730 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.730 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.730 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125898 7.5 7.f2bacc25 (undecoded) ondisk+write+known_if_redirected e241)", 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:05.213371+0000", 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.489280679, 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.489508891, 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125898 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.731 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.213371+0000", 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.213371+0000", 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.213369+0000", 2023-10-09T05:44:05.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0757999999999999e-05 2023-10-09T05:44:05.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.213380+0000", 2023-10-09T05:44:05.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8419000000000001e-05 2023-10-09T05:44:05.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.213398+0000", 2023-10-09T05:44:05.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2800000000000003e-06 2023-10-09T05:44:05.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.734 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.213407+0000", 2023-10-09T05:44:05.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5224e-05 2023-10-09T05:44:05.734 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.734 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.734 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.734 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125899 7.5 7.6ac12645 (undecoded) ondisk+write+known_if_redirected e241)", 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:05.215103+0000", 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.487548649, 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48780824099999998, 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125899 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.215103+0000", 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.215103+0000", 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.215102+0000", 2023-10-09T05:44:05.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.174e-06 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.215110+0000", 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3699e-05 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.215124+0000", 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.601e-06 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.215130+0000", 2023-10-09T05:44:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6769000000000001e-05 2023-10-09T05:44:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125901 7.15 7.d6e5d2f5 (undecoded) ondisk+write+known_if_redirected e241)", 2023-10-09T05:44:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:05.216153+0000", 2023-10-09T05:44:05.738 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.48649927300000001, 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48679014500000001, 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125901 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.216153+0000", 2023-10-09T05:44:05.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.216153+0000", 2023-10-09T05:44:05.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:05.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.216152+0000", 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7190000000000004e-06 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.216158+0000", 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7249e-05 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.216175+0000", 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.144e-06 2023-10-09T05:44:05.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.216182+0000", 2023-10-09T05:44:05.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8832000000000001e-05 2023-10-09T05:44:05.742 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.742 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.742 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.742 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "MOSDPGPush(7.15 241/239 [PushOp(7:aacd2a3f:::benchmark_data_smithi008_354649_object31290:head, version: 221'2036, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:aacd2a3f:::benchmark_data_smithi008_354649_object31290:head@221'2036, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:44:05.742 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:05.042661+0000", 2023-10-09T05:44:05.743 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.65999127800000001, 2023-10-09T05:44:05.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66034003799999996, 2023-10-09T05:44:05.743 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.743 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:44:05.743 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.042661+0000", 2023-10-09T05:44:05.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.042661+0000", 2023-10-09T05:44:05.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:44:05.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.042660+0000", 2023-10-09T05:44:05.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3788000000000001e-05 2023-10-09T05:44:05.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.042714+0000", 2023-10-09T05:44:05.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7507e-05 2023-10-09T05:44:05.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.042731+0000", 2023-10-09T05:44:05.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2190000000000004e-06 2023-10-09T05:44:05.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.042739+0000", 2023-10-09T05:44:05.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8452e-05 2023-10-09T05:44:05.746 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.746 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.746 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.746 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125850 7.15 7.21be0b75 (undecoded) ondisk+write+known_if_redirected e241)", 2023-10-09T05:44:05.746 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:05.132223+0000", 2023-10-09T05:44:05.746 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.57042857499999999, 2023-10-09T05:44:05.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.57080565800000005, 2023-10-09T05:44:05.746 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125850 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.132223+0000", 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.132223+0000", 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.132222+0000", 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7999999999999993e-06 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.132231+0000", 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4588e-05 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.132246+0000", 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5530000000000004e-06 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.132253+0000", 2023-10-09T05:44:05.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9989e-05 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125786 7.15 7:aacd2a3f:::benchmark_data_smithi008_354649_object31290:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:04.276247+0000", 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.426405173, 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4268163469999999, 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:44:05.750 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125786 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.276247+0000", 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.276247+0000", 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.276245+0000", 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1040000000000007e-06 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.276253+0000", 2023-10-09T05:44:05.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4799e-05 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.276268+0000", 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5860000000000004e-06 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.276274+0000", 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2005000000000001e-05 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.276326+0000", 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1224e-05 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.276337+0000", 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.76185878799999995 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:05.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.038196+0000", 2023-10-09T05:44:05.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040833129999999999 2023-10-09T05:44:05.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.755 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:44:05.755 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.042279+0000", 2023-10-09T05:44:05.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.76603232399999999 2023-10-09T05:44:05.755 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.755 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.755 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.756 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125854 7.5 7.b0f84e5 (undecoded) ondisk+write+known_if_redirected e241)", 2023-10-09T05:44:05.756 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:05.138490+0000", 2023-10-09T05:44:05.756 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.56416180000000005, 2023-10-09T05:44:05.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.56461736900000004, 2023-10-09T05:44:05.756 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.756 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:44:05.756 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.756 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.756 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.756 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125854 2023-10-09T05:44:05.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.138490+0000", 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.138490+0000", 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.138489+0000", 2023-10-09T05:44:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8509999999999999e-06 2023-10-09T05:44:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.138496+0000", 2023-10-09T05:44:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1566000000000001e-05 2023-10-09T05:44:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.138508+0000", 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9370000000000001e-06 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.138514+0000", 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3713000000000001e-05 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.760 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125969 7.15 7.5677f395 (undecoded) ondisk+write+known_if_redirected e241)", 2023-10-09T05:44:05.760 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:05.430981+0000", 2023-10-09T05:44:05.760 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.27167098299999998, 2023-10-09T05:44:05.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27216300399999999, 2023-10-09T05:44:05.760 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.760 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:44:05.760 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.760 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.760 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.760 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125969 2023-10-09T05:44:05.760 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.761 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.430981+0000", 2023-10-09T05:44:05.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.430981+0000", 2023-10-09T05:44:05.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:05.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.430979+0000", 2023-10-09T05:44:05.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3659999999999992e-06 2023-10-09T05:44:05.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.430988+0000", 2023-10-09T05:44:05.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5526999999999999e-05 2023-10-09T05:44:05.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.431003+0000", 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9160000000000001e-06 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.431011+0000", 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9879e-05 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125871 7.5 7.48447aa5 (undecoded) ondisk+write+known_if_redirected e241)", 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:05.165470+0000", 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.53718160299999995, 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.53770531399999999, 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125871 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.764 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.165470+0000", 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.165470+0000", 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.165469+0000", 2023-10-09T05:44:05.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.993e-06 2023-10-09T05:44:05.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.165475+0000", 2023-10-09T05:44:05.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2199999999999998e-06 2023-10-09T05:44:05.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.165484+0000", 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7979999999999996e-06 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.165489+0000", 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.861e-05 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.767 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125931 7.15 7.e4b18d35 (undecoded) ondisk+write+known_if_redirected e241)", 2023-10-09T05:44:05.768 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:05.304685+0000", 2023-10-09T05:44:05.768 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.39796721600000001, 2023-10-09T05:44:05.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.39852427299999998, 2023-10-09T05:44:05.768 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.768 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:44:05.768 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.768 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.768 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.768 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125931 2023-10-09T05:44:05.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.769 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.304685+0000", 2023-10-09T05:44:05.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.304685+0000", 2023-10-09T05:44:05.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:05.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.304683+0000", 2023-10-09T05:44:05.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8849999999999999e-06 2023-10-09T05:44:05.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.304691+0000", 2023-10-09T05:44:05.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8808999999999999e-05 2023-10-09T05:44:05.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.304710+0000", 2023-10-09T05:44:05.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6189999999999996e-06 2023-10-09T05:44:05.771 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:05.304716+0000", 2023-10-09T05:44:05.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1473999999999997e-05 2023-10-09T05:44:05.771 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.771 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125743 7.5 7.3514ecc5 (undecoded) ondisk+write+known_if_redirected e240)", 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:04.180675+0000", 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.5219769749999998, 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5225651330000001, 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:44:05.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:05.773 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125743 2023-10-09T05:44:05.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.773 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:44:05.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:44:05.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.180675+0000", 2023-10-09T05:44:05.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:44:05.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:44:05.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.180675+0000", 2023-10-09T05:44:05.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:05.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:44:05.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.180673+0000", 2023-10-09T05:44:05.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7340000000000008e-06 2023-10-09T05:44:05.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:44:05.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.180681+0000", 2023-10-09T05:44:05.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4353e-05 2023-10-09T05:44:05.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:44:05.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.180695+0000", 2023-10-09T05:44:05.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3380000000000004e-06 2023-10-09T05:44:05.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:44:05.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.180701+0000", 2023-10-09T05:44:05.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0124000000000002e-05 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.180761+0000", 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3416e-05 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.180774+0000", 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8978e-05 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.776 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:44:05.777 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.777 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:44:05.777 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:44:05.777 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 16 2023-10-09T05:44:05.777 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:44:06.102 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:06.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:06.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:06.188 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[184418]: pgmap v709: 99 pgs: 2 remapped+peering, 80 active+clean, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 peering; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 12198/186084 objects degraded (6.555%); 4070/186084 objects misplaced (2.187%); 1.6 MiB/s, 23 objects/s recovering 2023-10-09T05:44:06.189 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:44:06.189 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[184418]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.189 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[184418]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.189 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:44:06.189 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[184418]: osdmap e241: 8 total, 8 up, 8 in 2023-10-09T05:44:06.189 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[184418]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.189 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[184418]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.189 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[191227]: pgmap v709: 99 pgs: 2 remapped+peering, 80 active+clean, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 peering; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 12198/186084 objects degraded (6.555%); 4070/186084 objects misplaced (2.187%); 1.6 MiB/s, 23 objects/s recovering 2023-10-09T05:44:06.189 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:44:06.190 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[191227]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.190 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[191227]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.190 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:44:06.190 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[191227]: osdmap e241: 8 total, 8 up, 8 in 2023-10-09T05:44:06.190 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[191227]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.190 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:05 smithi008 ceph-mon[191227]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.206 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:06.234 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:06.280 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:06.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:06.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:06.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:05 smithi191 ceph-mon[161669]: pgmap v709: 99 pgs: 2 remapped+peering, 80 active+clean, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 peering; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 12198/186084 objects degraded (6.555%); 4070/186084 objects misplaced (2.187%); 1.6 MiB/s, 23 objects/s recovering 2023-10-09T05:44:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:05 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:44:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:05 smithi191 ceph-mon[161669]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:05 smithi191 ceph-mon[161669]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:05 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:44:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:05 smithi191 ceph-mon[161669]: osdmap e241: 8 total, 8 up, 8 in 2023-10-09T05:44:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:05 smithi191 ceph-mon[161669]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:05 smithi191 ceph-mon[161669]: failed to encode map e241 with expected crc 2023-10-09T05:44:06.383 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:06.416 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:06.467 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:06.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:06.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:06.571 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:06.603 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:06.608 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:44:06.631 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:44:06.650 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:44:06.651 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:06.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:06.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:06.670 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:44:06.688 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:44:06.706 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:44:06.724 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:44:06.743 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:44:06.744 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:44:06.744 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:44:06.744 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:44:06.744 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:44:06.744 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:44:06.745 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:44:06.745 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:44:06.745 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:44:06.754 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:06.782 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:06.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:44:06.982 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:06.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:07.086 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:07.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1641516297' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:07.120 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:07.173 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:07.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:07.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:07.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1641516297' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1641516297' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:07.274 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:44:07.279 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:07.309 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:07.470 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:07.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:07.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:07.574 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:07.633 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:07.890 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:07.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:07.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:07.994 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:08.053 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:08.153 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:08.156 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:07 smithi191 ceph-mon[161669]: pgmap v711: 99 pgs: 1 remapped+peering, 83 active+clean, 1 creating+peering, 6 unknown, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 7794/190386 objects degraded (4.094%); 7917/190386 objects misplaced (4.158%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:44:08.156 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:07 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:08.156 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:07 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 7794/190386 objects degraded (4.094%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:08.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:08.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:08.206 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:44:08.235 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:07 smithi008 ceph-mon[184418]: pgmap v711: 99 pgs: 1 remapped+peering, 83 active+clean, 1 creating+peering, 6 unknown, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 7794/190386 objects degraded (4.094%); 7917/190386 objects misplaced (4.158%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:44:08.236 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:07 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:08.236 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:07 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 7794/190386 objects degraded (4.094%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:08.236 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:07 smithi008 ceph-mon[191227]: pgmap v711: 99 pgs: 1 remapped+peering, 83 active+clean, 1 creating+peering, 6 unknown, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 7794/190386 objects degraded (4.094%); 7917/190386 objects misplaced (4.158%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T05:44:08.236 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:07 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:08.236 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:07 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 7794/190386 objects degraded (4.094%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:08.258 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:08.296 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:08.360 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:08.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:08.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:08.467 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:08.494 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:08.549 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:08.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:08.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:08.657 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:08.718 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:44:08.718 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:44:08.718 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T05:44:08.719 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:44:08.721 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:08.788 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:08.805 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:08.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:08.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:08.913 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:08.945 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:09.064 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:09.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:09.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:09.169 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:09.205 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:09.275 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:09.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:09.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:09.380 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:09.383 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:44:09.383 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:44:09.418 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:09.475 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:09.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:09.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:09.538 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:44:09.579 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:09.610 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:09.663 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:09.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:09.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:09.767 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:09.813 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:09.877 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:09.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:09.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:09.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:10.013 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:10.227 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:10.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:09 smithi191 ceph-mon[161669]: pgmap v712: 99 pgs: 2 active+recovery_wait+degraded, 1 remapped+peering, 85 active+clean, 1 creating+peering, 3 unknown, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 298 op/s; 12306/189348 objects degraded (6.499%); 3847/189348 objects misplaced (2.032%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:44:10.232 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:09 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:10.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:10.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:10.274 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:10.333 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:10.341 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:44:10.371 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:09 smithi008 ceph-mon[184418]: pgmap v712: 99 pgs: 2 active+recovery_wait+degraded, 1 remapped+peering, 85 active+clean, 1 creating+peering, 3 unknown, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 298 op/s; 12306/189348 objects degraded (6.499%); 3847/189348 objects misplaced (2.032%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:44:10.371 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:09 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:10.371 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:09 smithi008 ceph-mon[191227]: pgmap v712: 99 pgs: 2 active+recovery_wait+degraded, 1 remapped+peering, 85 active+clean, 1 creating+peering, 3 unknown, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 298 op/s; 12306/189348 objects degraded (6.499%); 3847/189348 objects misplaced (2.032%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:44:10.371 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:09 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:10.374 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:10.423 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:10.556 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:10.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:10.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:10.646 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:44:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:44:10] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T05:44:10.661 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:10.692 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:10.807 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:44:11.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:10 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:11.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:10 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:11.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:10 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:11.744 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:11.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:11.849 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:11.878 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:11.921 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:11.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:11.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:11.982 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:44:12.025 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:12.067 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:12.234 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:11 smithi008 ceph-mon[184418]: pgmap v713: 99 pgs: 1 active+clean+scrubbing, 5 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 21194/187827 objects degraded (11.284%); 5778/187827 objects misplaced (3.076%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:44:12.234 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3012216504' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:44:12.235 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:11 smithi008 ceph-mon[191227]: pgmap v713: 99 pgs: 1 active+clean+scrubbing, 5 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 21194/187827 objects degraded (11.284%); 5778/187827 objects misplaced (3.076%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:44:12.235 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3012216504' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:44:12.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:11 smithi191 ceph-mon[161669]: pgmap v713: 99 pgs: 1 active+clean+scrubbing, 5 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 21194/187827 objects degraded (11.284%); 5778/187827 objects misplaced (3.076%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:44:12.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3012216504' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:44:12.353 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:12.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:12.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:12.459 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:12.495 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:12.868 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:44:12.868 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:44:12.981 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:12.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:12.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:13.086 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:13.124 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:13.198 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:13.198 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[184418]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.198 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[184418]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.198 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3012216504' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:44:13.198 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[184418]: osdmap e242: 8 total, 8 up, 8 in 2023-10-09T05:44:13.198 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[184418]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.199 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[184418]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.199 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[184418]: pgmap v715: 99 pgs: 1 active+clean+scrubbing, 5 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 MiB/s wr, 770 op/s; 21063/185505 objects degraded (11.354%); 5566/185505 objects misplaced (3.000%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T05:44:13.199 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 21194/187827 objects degraded (11.284%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:13.199 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:13.199 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[191227]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.199 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[191227]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.199 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3012216504' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:44:13.199 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[191227]: osdmap e242: 8 total, 8 up, 8 in 2023-10-09T05:44:13.199 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[191227]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.200 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[191227]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.200 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[191227]: pgmap v715: 99 pgs: 1 active+clean+scrubbing, 5 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 MiB/s wr, 770 op/s; 21063/185505 objects degraded (11.354%); 5566/185505 objects misplaced (3.000%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T05:44:13.200 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:12 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 21194/187827 objects degraded (11.284%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:13.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:12 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:12 smithi191 ceph-mon[161669]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:12 smithi191 ceph-mon[161669]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3012216504' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:44:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:12 smithi191 ceph-mon[161669]: osdmap e242: 8 total, 8 up, 8 in 2023-10-09T05:44:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:12 smithi191 ceph-mon[161669]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:12 smithi191 ceph-mon[161669]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:12 smithi191 ceph-mon[161669]: pgmap v715: 99 pgs: 1 active+clean+scrubbing, 5 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 MiB/s wr, 770 op/s; 21063/185505 objects degraded (11.354%); 5566/185505 objects misplaced (3.000%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T05:44:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:12 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 21194/187827 objects degraded (11.284%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:13.339 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:13.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:13.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:13.446 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:13.479 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:13.539 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:13.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:13.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:13.642 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:13.684 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:13.751 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:44:13.766 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:13.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:13.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:13.778 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:44:13.804 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:44:13.835 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:44:13.865 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:44:13.870 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:13.894 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:44:13.908 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:44:13.929 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:13.946 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:44:13.986 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:44:13.995 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:13 smithi008 ceph-mon[184418]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.995 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:13 smithi008 ceph-mon[184418]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.995 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:13 smithi008 ceph-mon[184418]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.996 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1535336862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:13.996 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:13.996 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:13 smithi008 ceph-mon[191227]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.996 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:13 smithi008 ceph-mon[191227]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.996 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:13 smithi008 ceph-mon[191227]: failed to encode map e242 with expected crc 2023-10-09T05:44:13.996 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1535336862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:13.996 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:14.012 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:14.013 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:44:14.013 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:44:14.013 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:44:14.013 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:44:14.013 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:44:14.013 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:44:14.013 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:44:14.013 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:44:14.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:14.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:14.120 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:14.164 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:13 smithi191 ceph-mon[161669]: failed to encode map e242 with expected crc 2023-10-09T05:44:14.164 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:13 smithi191 ceph-mon[161669]: failed to encode map e242 with expected crc 2023-10-09T05:44:14.164 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:13 smithi191 ceph-mon[161669]: failed to encode map e242 with expected crc 2023-10-09T05:44:14.164 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1535336862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:14.164 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:14.164 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:14.236 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:14.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:14.354 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:14.356 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:44:14.363 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:44:14.394 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:15.292 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:44:15.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:14 smithi191 ceph-mon[161669]: pgmap v716: 99 pgs: 1 active+clean+scrubbing, 5 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 288 op/s; 20971/183954 objects degraded (11.400%); 5566/183954 objects misplaced (3.026%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T05:44:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:14 smithi191 ceph-mon[161669]: from='client.45518 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:44:15.323 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:14 smithi008 ceph-mon[184418]: pgmap v716: 99 pgs: 1 active+clean+scrubbing, 5 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 288 op/s; 20971/183954 objects degraded (11.400%); 5566/183954 objects misplaced (3.026%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T05:44:15.323 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:14 smithi008 ceph-mon[184418]: from='client.45518 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:44:15.325 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:14 smithi008 ceph-mon[191227]: pgmap v716: 99 pgs: 1 active+clean+scrubbing, 5 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 288 op/s; 20971/183954 objects degraded (11.400%); 5566/183954 objects misplaced (3.026%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T05:44:15.325 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:14 smithi008 ceph-mon[191227]: from='client.45518 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:44:15.387 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:15.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:15.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:15.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:15.521 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:15.578 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:15.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:15.656 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":716,"stamp":"2023-10-09T05:44:14.261175+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4021426123,"num_objects":61318,"num_object_clones":0,"num_object_copies":183954,"num_objects_missing_on_primary":5903,"num_objects_missing":5903,"num_objects_degraded":20971,"num_objects_misplaced":5566,"num_objects_unfound":0,"num_objects_dirty":61318,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":128804,"num_write_kb":6062694,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1933,"num_bytes_recovered":129313300,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":202135,"ondisk_log_size":202135,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15418192,"kb_used_data":12343312,"kb_used_omap":45,"kb_used_meta":3074770,"kb_avail":734543024,"statfs":{"total":767960285184,"available":752172056576,"internally_reserved":0,"allocated":12639551488,"data_stored":12633904439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":3148564771},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,2,8,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-175767552,"num_objects":-2682,"num_object_clones":0,"num_object_copies":-8046,"num_objects_missing_on_primary":5903,"num_objects_missing":5903,"num_objects_degraded":8773,"num_objects_misplaced":1496,"num_objects_unfound":0,"num_objects_dirty":-2682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2682,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":745,"num_bytes_recovered":49152544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.300702"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":223,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.968176+0000","last_change":"2023-10-09T05:31:02.552118+0000","last_active":"2023-10-09T05:44:03.968176+0000","last_peered":"2023-10-09T05:44:03.968176+0000","last_clean":"2023-10-09T05:44:03.968176+0000","last_became_active":"2023-10-09T05:31:02.551800+0000","last_became_peered":"2023-10-09T05:31:02.551800+0000","last_unstale":"2023-10-09T05:44:03.968176+0000","last_undegraded":"2023-10-09T05:44:03.968176+0000","last_fullsized":"2023-10-09T05:44:03.968176+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":214,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.960054+0000","last_change":"2023-10-09T05:31:32.868031+0000","last_active":"2023-10-09T05:44:03.960054+0000","last_peered":"2023-10-09T05:44:03.960054+0000","last_clean":"2023-10-09T05:44:03.960054+0000","last_became_active":"2023-10-09T05:31:32.867688+0000","last_became_peered":"2023-10-09T05:31:32.867688+0000","last_unstale":"2023-10-09T05:44:03.960054+0000","last_undegraded":"2023-10-09T05:44:03.960054+0000","last_fullsized":"2023-10-09T05:44:03.960054+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:56.475318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":189,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.984511+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:44:04.984511+0000","last_peered":"2023-10-09T05:44:04.984511+0000","last_clean":"2023-10-09T05:44:04.984511+0000","last_became_active":"2023-10-09T05:31:32.868936+0000","last_became_peered":"2023-10-09T05:31:32.868936+0000","last_unstale":"2023-10-09T05:44:04.984511+0000","last_undegraded":"2023-10-09T05:44:04.984511+0000","last_fullsized":"2023-10-09T05:44:04.984511+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:42.858580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":229,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.968752+0000","last_change":"2023-10-09T05:43:19.889677+0000","last_active":"2023-10-09T05:44:04.968752+0000","last_peered":"2023-10-09T05:44:04.968752+0000","last_clean":"2023-10-09T05:44:04.968752+0000","last_became_active":"2023-10-09T05:43:19.889320+0000","last_became_peered":"2023-10-09T05:43:19.889320+0000","last_unstale":"2023-10-09T05:44:04.968752+0000","last_undegraded":"2023-10-09T05:44:04.968752+0000","last_fullsized":"2023-10-09T05:44:04.968752+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:57.376656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":230,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.962891+0000","last_change":"2023-10-09T05:31:32.867898+0000","last_active":"2023-10-09T05:44:03.962891+0000","last_peered":"2023-10-09T05:44:03.962891+0000","last_clean":"2023-10-09T05:44:03.962891+0000","last_became_active":"2023-10-09T05:31:32.867571+0000","last_became_peered":"2023-10-09T05:31:32.867571+0000","last_unstale":"2023-10-09T05:44:03.962891+0000","last_undegraded":"2023-10-09T05:44:03.962891+0000","last_fullsized":"2023-10-09T05:44:03.962891+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:40.570522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":206,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.973741+0000","last_change":"2023-10-09T05:30:30.456842+0000","last_active":"2023-10-09T05:44:03.973741+0000","last_peered":"2023-10-09T05:44:03.973741+0000","last_clean":"2023-10-09T05:44:03.973741+0000","last_became_active":"2023-10-09T05:30:30.456491+0000","last_became_peered":"2023-10-09T05:30:30.456491+0000","last_unstale":"2023-10-09T05:44:03.973741+0000","last_undegraded":"2023-10-09T05:44:03.973741+0000","last_fullsized":"2023-10-09T05:44:03.973741+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"241'7928","reported_seq":7974,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:11.589825+0000","last_change":"2023-10-09T05:44:04.042724+0000","last_active":"2023-10-09T05:44:11.589825+0000","last_peered":"2023-10-09T05:44:11.589825+0000","last_clean":"2023-10-09T05:44:11.589825+0000","last_became_active":"2023-10-09T05:44:04.029419+0000","last_became_peered":"2023-10-09T05:44:04.029419+0000","last_unstale":"2023-10-09T05:44:11.589825+0000","last_undegraded":"2023-10-09T05:44:11.589825+0000","last_fullsized":"2023-10-09T05:44:11.589825+0000","mapping_epoch":239,"log_start":"218'500","ondisk_log_start":"218'500","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7428,"log_dups_size":0,"ondisk_log_size":7428,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127500288,"num_objects":1946,"num_object_clones":0,"num_object_copies":5838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3992,"num_write_kb":189984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":194,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.969350+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:44:04.969350+0000","last_peered":"2023-10-09T05:44:04.969350+0000","last_clean":"2023-10-09T05:44:04.969350+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:44:04.969350+0000","last_undegraded":"2023-10-09T05:44:04.969350+0000","last_fullsized":"2023-10-09T05:44:04.969350+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":214,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.962971+0000","last_change":"2023-10-09T05:31:32.870178+0000","last_active":"2023-10-09T05:44:03.962971+0000","last_peered":"2023-10-09T05:44:03.962971+0000","last_clean":"2023-10-09T05:44:03.962971+0000","last_became_active":"2023-10-09T05:31:32.869864+0000","last_became_peered":"2023-10-09T05:31:32.869864+0000","last_unstale":"2023-10-09T05:44:03.962971+0000","last_undegraded":"2023-10-09T05:44:03.962971+0000","last_fullsized":"2023-10-09T05:44:03.962971+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:19.401146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"241'8131","reported_seq":8175,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:11.606466+0000","last_change":"2023-10-09T05:44:04.045501+0000","last_active":"2023-10-09T05:44:11.606466+0000","last_peered":"2023-10-09T05:44:11.606466+0000","last_clean":"2023-10-09T05:44:11.606466+0000","last_became_active":"2023-10-09T05:44:04.026886+0000","last_became_peered":"2023-10-09T05:44:04.026886+0000","last_unstale":"2023-10-09T05:44:11.606466+0000","last_undegraded":"2023-10-09T05:44:11.606466+0000","last_fullsized":"2023-10-09T05:44:11.606466+0000","mapping_epoch":239,"log_start":"218'700","ondisk_log_start":"218'700","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7431,"log_dups_size":0,"ondisk_log_size":7431,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129171456,"num_objects":1971,"num_object_clones":0,"num_object_copies":5913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4093,"num_write_kb":194048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":241,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.969601+0000","last_change":"2023-10-09T05:43:19.886014+0000","last_active":"2023-10-09T05:44:03.969601+0000","last_peered":"2023-10-09T05:44:03.969601+0000","last_clean":"2023-10-09T05:44:03.969601+0000","last_became_active":"2023-10-09T05:43:19.885659+0000","last_became_peered":"2023-10-09T05:43:19.885659+0000","last_unstale":"2023-10-09T05:44:03.969601+0000","last_undegraded":"2023-10-09T05:44:03.969601+0000","last_fullsized":"2023-10-09T05:44:03.969601+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":237,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.959698+0000","last_change":"2023-10-09T05:43:19.883115+0000","last_active":"2023-10-09T05:44:03.959698+0000","last_peered":"2023-10-09T05:44:03.959698+0000","last_clean":"2023-10-09T05:44:03.959698+0000","last_became_active":"2023-10-09T05:43:19.882674+0000","last_became_peered":"2023-10-09T05:43:19.882674+0000","last_unstale":"2023-10-09T05:44:03.959698+0000","last_undegraded":"2023-10-09T05:44:03.959698+0000","last_fullsized":"2023-10-09T05:44:03.959698+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:30.479481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"241'7793","reported_seq":7836,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:09.781799+0000","last_change":"2023-10-09T05:44:04.043289+0000","last_active":"2023-10-09T05:44:09.781799+0000","last_peered":"2023-10-09T05:44:09.781799+0000","last_clean":"2023-10-09T05:44:09.781799+0000","last_became_active":"2023-10-09T05:44:04.032393+0000","last_became_peered":"2023-10-09T05:44:04.032393+0000","last_unstale":"2023-10-09T05:44:09.781799+0000","last_undegraded":"2023-10-09T05:44:09.781799+0000","last_fullsized":"2023-10-09T05:44:09.781799+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7793,"log_dups_size":0,"ondisk_log_size":7793,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123863040,"num_objects":1890,"num_object_clones":0,"num_object_copies":5670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3918,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":26,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.993582+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:44:03.993582+0000","last_peered":"2023-10-09T05:44:03.993582+0000","last_clean":"2023-10-09T05:44:03.993582+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:44:03.993582+0000","last_undegraded":"2023-10-09T05:44:03.993582+0000","last_fullsized":"2023-10-09T05:44:03.993582+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":189,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.984685+0000","last_change":"2023-10-09T05:31:02.548148+0000","last_active":"2023-10-09T05:44:04.984685+0000","last_peered":"2023-10-09T05:44:04.984685+0000","last_clean":"2023-10-09T05:44:04.984685+0000","last_became_active":"2023-10-09T05:31:02.547748+0000","last_became_peered":"2023-10-09T05:31:02.547748+0000","last_unstale":"2023-10-09T05:44:04.984685+0000","last_undegraded":"2023-10-09T05:44:04.984685+0000","last_fullsized":"2023-10-09T05:44:04.984685+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:30.763018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"241'7808","reported_seq":7853,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:10.002649+0000","last_change":"2023-10-09T05:44:04.030193+0000","last_active":"2023-10-09T05:44:10.002649+0000","last_peered":"2023-10-09T05:44:10.002649+0000","last_clean":"2023-10-09T05:44:10.002649+0000","last_became_active":"2023-10-09T05:44:04.017624+0000","last_became_peered":"2023-10-09T05:44:04.017624+0000","last_unstale":"2023-10-09T05:44:10.002649+0000","last_undegraded":"2023-10-09T05:44:10.002649+0000","last_fullsized":"2023-10-09T05:44:10.002649+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7808,"log_dups_size":0,"ondisk_log_size":7808,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126418944,"num_objects":1929,"num_object_clones":0,"num_object_copies":5787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3930,"num_write_kb":187488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":267,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.969013+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:44:03.969013+0000","last_peered":"2023-10-09T05:44:03.969013+0000","last_clean":"2023-10-09T05:44:03.969013+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:44:03.969013+0000","last_undegraded":"2023-10-09T05:44:03.969013+0000","last_fullsized":"2023-10-09T05:44:03.969013+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":206,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.968508+0000","last_change":"2023-10-09T05:30:30.461165+0000","last_active":"2023-10-09T05:44:03.968508+0000","last_peered":"2023-10-09T05:44:03.968508+0000","last_clean":"2023-10-09T05:44:03.968508+0000","last_became_active":"2023-10-09T05:30:30.460790+0000","last_became_peered":"2023-10-09T05:30:30.460790+0000","last_unstale":"2023-10-09T05:44:03.968508+0000","last_undegraded":"2023-10-09T05:44:03.968508+0000","last_fullsized":"2023-10-09T05:44:03.968508+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"241'7812","reported_seq":7856,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:10.583426+0000","last_change":"2023-10-09T05:44:04.058002+0000","last_active":"2023-10-09T05:44:10.583426+0000","last_peered":"2023-10-09T05:44:10.583426+0000","last_clean":"2023-10-09T05:44:10.583426+0000","last_became_active":"2023-10-09T05:44:04.046680+0000","last_became_peered":"2023-10-09T05:44:04.046680+0000","last_unstale":"2023-10-09T05:44:10.583426+0000","last_undegraded":"2023-10-09T05:44:10.583426+0000","last_fullsized":"2023-10-09T05:44:10.583426+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7812,"log_dups_size":0,"ondisk_log_size":7812,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125075456,"num_objects":1909,"num_object_clones":0,"num_object_copies":5727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3929,"num_write_kb":186784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":26,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.990125+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:44:03.990125+0000","last_peered":"2023-10-09T05:44:03.990125+0000","last_clean":"2023-10-09T05:44:03.990125+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:44:03.990125+0000","last_undegraded":"2023-10-09T05:44:03.990125+0000","last_fullsized":"2023-10-09T05:44:03.990125+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":247,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.959522+0000","last_change":"2023-10-09T05:43:19.882836+0000","last_active":"2023-10-09T05:44:03.959522+0000","last_peered":"2023-10-09T05:44:03.959522+0000","last_clean":"2023-10-09T05:44:03.959522+0000","last_became_active":"2023-10-09T05:43:19.882415+0000","last_became_peered":"2023-10-09T05:43:19.882415+0000","last_unstale":"2023-10-09T05:44:03.959522+0000","last_undegraded":"2023-10-09T05:44:03.959522+0000","last_fullsized":"2023-10-09T05:44:03.959522+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:20.794814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"242'8038","reported_seq":8264,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:44:13.269406+0000","last_change":"2023-10-09T05:44:04.155256+0000","last_active":"2023-10-09T05:44:13.269406+0000","last_peered":"2023-10-09T05:44:13.269406+0000","last_clean":"2023-10-09T05:43:18.711607+0000","last_became_active":"2023-10-09T05:44:04.113448+0000","last_became_peered":"2023-10-09T05:44:04.113448+0000","last_unstale":"2023-10-09T05:44:13.269406+0000","last_undegraded":"2023-10-09T05:44:04.097200+0000","last_fullsized":"2023-10-09T05:44:13.269406+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8038,"log_dups_size":0,"ondisk_log_size":8038,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125435904,"num_objects":1914,"num_object_clones":0,"num_object_copies":5742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4061,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":5505024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1917},{"shards":"0,3,7","objects":-3}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":228,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.966551+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:44:04.966551+0000","last_peered":"2023-10-09T05:44:04.966551+0000","last_clean":"2023-10-09T05:44:04.966551+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:44:04.966551+0000","last_undegraded":"2023-10-09T05:44:04.966551+0000","last_fullsized":"2023-10-09T05:44:04.966551+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":240,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.970785+0000","last_change":"2023-10-09T05:43:19.886058+0000","last_active":"2023-10-09T05:44:03.970785+0000","last_peered":"2023-10-09T05:44:03.970785+0000","last_clean":"2023-10-09T05:44:03.970785+0000","last_became_active":"2023-10-09T05:43:19.885713+0000","last_became_peered":"2023-10-09T05:43:19.885713+0000","last_unstale":"2023-10-09T05:44:03.970785+0000","last_undegraded":"2023-10-09T05:44:03.970785+0000","last_fullsized":"2023-10-09T05:44:03.970785+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"241'8029","reported_seq":8140,"reported_epoch":241,"state":"active+recovering","last_fresh":"2023-10-09T05:44:09.768891+0000","last_change":"2023-10-09T05:44:07.104371+0000","last_active":"2023-10-09T05:44:09.768891+0000","last_peered":"2023-10-09T05:44:09.768891+0000","last_clean":"2023-10-09T05:43:18.738931+0000","last_became_active":"2023-10-09T05:44:05.020084+0000","last_became_peered":"2023-10-09T05:44:05.020084+0000","last_unstale":"2023-10-09T05:44:09.768891+0000","last_undegraded":"2023-10-09T05:44:09.768891+0000","last_fullsized":"2023-10-09T05:44:09.768891+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8029,"log_dups_size":0,"ondisk_log_size":8029,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127500288,"num_objects":1946,"num_object_clones":0,"num_object_copies":5838,"num_objects_missing_on_primary":2913,"num_objects_missing":2913,"num_objects_degraded":0,"num_objects_misplaced":1931,"num_objects_unfound":0,"num_objects_dirty":1946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4028,"num_write_kb":191136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1703936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1931},{"shards":"0,1,2,7","objects":8},{"shards":"0,2,7","objects":7}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":20,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.989744+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:44:03.989744+0000","last_peered":"2023-10-09T05:44:03.989744+0000","last_clean":"2023-10-09T05:44:03.989744+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:44:03.989744+0000","last_undegraded":"2023-10-09T05:44:03.989744+0000","last_fullsized":"2023-10-09T05:44:03.989744+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":227,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.968816+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:44:04.968816+0000","last_peered":"2023-10-09T05:44:04.968816+0000","last_clean":"2023-10-09T05:44:04.968816+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:44:04.968816+0000","last_undegraded":"2023-10-09T05:44:04.968816+0000","last_fullsized":"2023-10-09T05:44:04.968816+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"242'7918","reported_seq":7966,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-10-09T05:44:13.313338+0000","last_change":"2023-10-09T05:44:04.027585+0000","last_active":"2023-10-09T05:44:13.313338+0000","last_peered":"2023-10-09T05:44:13.313338+0000","last_clean":"2023-10-09T05:44:13.313338+0000","last_became_active":"2023-10-09T05:44:04.015891+0000","last_became_peered":"2023-10-09T05:44:04.015891+0000","last_unstale":"2023-10-09T05:44:13.313338+0000","last_undegraded":"2023-10-09T05:44:13.313338+0000","last_fullsized":"2023-10-09T05:44:13.313338+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7918,"log_dups_size":0,"ondisk_log_size":7918,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123764736,"num_objects":1889,"num_object_clones":0,"num_object_copies":5667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3996,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":215,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.970974+0000","last_change":"2023-10-09T05:29:59.475086+0000","last_active":"2023-10-09T05:44:03.970974+0000","last_peered":"2023-10-09T05:44:03.970974+0000","last_clean":"2023-10-09T05:44:03.970974+0000","last_became_active":"2023-10-09T05:29:59.474656+0000","last_became_peered":"2023-10-09T05:29:59.474656+0000","last_unstale":"2023-10-09T05:44:03.970974+0000","last_undegraded":"2023-10-09T05:44:03.970974+0000","last_fullsized":"2023-10-09T05:44:03.970974+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":247,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.961645+0000","last_change":"2023-10-09T05:43:19.883642+0000","last_active":"2023-10-09T05:44:03.961645+0000","last_peered":"2023-10-09T05:44:03.961645+0000","last_clean":"2023-10-09T05:44:03.961645+0000","last_became_active":"2023-10-09T05:43:19.883419+0000","last_became_peered":"2023-10-09T05:43:19.883419+0000","last_unstale":"2023-10-09T05:44:03.961645+0000","last_undegraded":"2023-10-09T05:44:03.961645+0000","last_fullsized":"2023-10-09T05:44:03.961645+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:13.488569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"242'7822","reported_seq":7860,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-10-09T05:44:13.986340+0000","last_change":"2023-10-09T05:41:29.078278+0000","last_active":"2023-10-09T05:44:13.986340+0000","last_peered":"2023-10-09T05:44:13.986340+0000","last_clean":"2023-10-09T05:44:13.986340+0000","last_became_active":"2023-10-09T05:41:29.077562+0000","last_became_peered":"2023-10-09T05:41:29.077562+0000","last_unstale":"2023-10-09T05:44:13.986340+0000","last_undegraded":"2023-10-09T05:44:13.986340+0000","last_fullsized":"2023-10-09T05:44:13.986340+0000","mapping_epoch":213,"log_start":"218'851","ondisk_log_start":"218'851","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":6971,"log_dups_size":0,"ondisk_log_size":6971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:29.784837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240648192,"num_objects":3672,"num_object_clones":0,"num_object_copies":11016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7822,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":24,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.990461+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:44:03.990461+0000","last_peered":"2023-10-09T05:44:03.990461+0000","last_clean":"2023-10-09T05:44:03.990461+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:44:03.990461+0000","last_undegraded":"2023-10-09T05:44:03.990461+0000","last_fullsized":"2023-10-09T05:44:03.990461+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":225,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.970106+0000","last_change":"2023-10-09T05:43:19.885929+0000","last_active":"2023-10-09T05:44:03.970106+0000","last_peered":"2023-10-09T05:44:03.970106+0000","last_clean":"2023-10-09T05:44:03.970106+0000","last_became_active":"2023-10-09T05:43:19.885542+0000","last_became_peered":"2023-10-09T05:43:19.885542+0000","last_unstale":"2023-10-09T05:44:03.970106+0000","last_undegraded":"2023-10-09T05:44:03.970106+0000","last_fullsized":"2023-10-09T05:44:03.970106+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"241'7862","reported_seq":7899,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:11.636644+0000","last_change":"2023-10-09T05:41:29.081067+0000","last_active":"2023-10-09T05:44:11.636644+0000","last_peered":"2023-10-09T05:44:11.636644+0000","last_clean":"2023-10-09T05:44:11.636644+0000","last_became_active":"2023-10-09T05:41:29.080704+0000","last_became_peered":"2023-10-09T05:41:29.080704+0000","last_unstale":"2023-10-09T05:44:11.636644+0000","last_undegraded":"2023-10-09T05:44:11.636644+0000","last_fullsized":"2023-10-09T05:44:11.636644+0000","mapping_epoch":213,"log_start":"221'1594","ondisk_log_start":"221'1594","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":6268,"log_dups_size":0,"ondisk_log_size":6268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:29.483205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246677504,"num_objects":3764,"num_object_clones":0,"num_object_copies":11292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7862,"num_write_kb":372032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":24,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.994082+0000","last_change":"2023-10-09T05:43:20.538857+0000","last_active":"2023-10-09T05:44:03.994082+0000","last_peered":"2023-10-09T05:44:03.994082+0000","last_clean":"2023-10-09T05:44:03.994082+0000","last_became_active":"2023-10-09T05:43:19.889408+0000","last_became_peered":"2023-10-09T05:43:19.889408+0000","last_unstale":"2023-10-09T05:44:03.994082+0000","last_undegraded":"2023-10-09T05:44:03.994082+0000","last_fullsized":"2023-10-09T05:44:03.994082+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":230,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.968348+0000","last_change":"2023-10-09T05:41:44.881327+0000","last_active":"2023-10-09T05:44:03.968348+0000","last_peered":"2023-10-09T05:44:03.968348+0000","last_clean":"2023-10-09T05:44:03.968348+0000","last_became_active":"2023-10-09T05:41:44.881044+0000","last_became_peered":"2023-10-09T05:41:44.881044+0000","last_unstale":"2023-10-09T05:44:03.968348+0000","last_undegraded":"2023-10-09T05:44:03.968348+0000","last_fullsized":"2023-10-09T05:44:03.968348+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"242'7852","reported_seq":7904,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-10-09T05:44:13.417304+0000","last_change":"2023-10-09T05:44:04.044555+0000","last_active":"2023-10-09T05:44:13.417304+0000","last_peered":"2023-10-09T05:44:13.417304+0000","last_clean":"2023-10-09T05:44:13.417304+0000","last_became_active":"2023-10-09T05:44:04.010531+0000","last_became_peered":"2023-10-09T05:44:04.010531+0000","last_unstale":"2023-10-09T05:44:13.417304+0000","last_undegraded":"2023-10-09T05:44:13.417304+0000","last_fullsized":"2023-10-09T05:44:13.417304+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7852,"log_dups_size":0,"ondisk_log_size":7852,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124125184,"num_objects":1894,"num_object_clones":0,"num_object_copies":5682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3968,"num_write_kb":187584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":24,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.993485+0000","last_change":"2023-10-09T05:43:20.148955+0000","last_active":"2023-10-09T05:44:03.993485+0000","last_peered":"2023-10-09T05:44:03.993485+0000","last_clean":"2023-10-09T05:44:03.993485+0000","last_became_active":"2023-10-09T05:43:19.884767+0000","last_became_peered":"2023-10-09T05:43:19.884767+0000","last_unstale":"2023-10-09T05:44:03.993485+0000","last_undegraded":"2023-10-09T05:44:03.993485+0000","last_fullsized":"2023-10-09T05:44:03.993485+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":248,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.973656+0000","last_change":"2023-10-09T05:43:19.888083+0000","last_active":"2023-10-09T05:44:03.973656+0000","last_peered":"2023-10-09T05:44:03.973656+0000","last_clean":"2023-10-09T05:44:03.973656+0000","last_became_active":"2023-10-09T05:43:19.887781+0000","last_became_peered":"2023-10-09T05:43:19.887781+0000","last_unstale":"2023-10-09T05:44:03.973656+0000","last_undegraded":"2023-10-09T05:44:03.973656+0000","last_fullsized":"2023-10-09T05:44:03.973656+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"242'7977","reported_seq":10396,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:44:12.085396+0000","last_change":"2023-10-09T05:44:05.021873+0000","last_active":"2023-10-09T05:44:12.085396+0000","last_peered":"2023-10-09T05:44:12.085396+0000","last_clean":"2023-10-09T05:43:18.869044+0000","last_became_active":"2023-10-09T05:44:04.975737+0000","last_became_peered":"2023-10-09T05:44:04.975737+0000","last_unstale":"2023-10-09T05:44:12.085396+0000","last_undegraded":"2023-10-09T05:44:04.966355+0000","last_fullsized":"2023-10-09T05:44:12.085396+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7977,"log_dups_size":0,"ondisk_log_size":7977,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127401984,"num_objects":1944,"num_object_clones":0,"num_object_copies":5832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2345,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4025,"num_write_kb":191008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":627,"num_bytes_recovered":41091072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":1913},{"shards":"2,4,7","objects":31}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":208,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.970086+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:44:03.970086+0000","last_peered":"2023-10-09T05:44:03.970086+0000","last_clean":"2023-10-09T05:44:03.970086+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:44:03.970086+0000","last_undegraded":"2023-10-09T05:44:03.970086+0000","last_fullsized":"2023-10-09T05:44:03.970086+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":195,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.968452+0000","last_change":"2023-10-09T05:30:30.459793+0000","last_active":"2023-10-09T05:44:04.968452+0000","last_peered":"2023-10-09T05:44:04.968452+0000","last_clean":"2023-10-09T05:44:04.968452+0000","last_became_active":"2023-10-09T05:30:30.459445+0000","last_became_peered":"2023-10-09T05:30:30.459445+0000","last_unstale":"2023-10-09T05:44:04.968452+0000","last_undegraded":"2023-10-09T05:44:04.968452+0000","last_fullsized":"2023-10-09T05:44:04.968452+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:47.471761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"242'8173","reported_seq":8219,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:44:13.505516+0000","last_change":"2023-10-09T05:43:21.226422+0000","last_active":"2023-10-09T05:44:13.505516+0000","last_peered":"2023-10-09T05:44:13.505516+0000","last_clean":"2023-10-09T05:43:18.871019+0000","last_became_active":"2023-10-09T05:43:21.208517+0000","last_became_peered":"2023-10-09T05:43:21.208517+0000","last_unstale":"2023-10-09T05:44:13.505516+0000","last_undegraded":"2023-10-09T05:43:20.928538+0000","last_fullsized":"2023-10-09T05:43:20.877574+0000","mapping_epoch":231,"log_start":"218'584","ondisk_log_start":"218'584","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7589,"log_dups_size":0,"ondisk_log_size":7589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:20.409497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248840192,"num_objects":3797,"num_object_clones":0,"num_object_copies":11391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5985,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8173,"num_write_kb":383040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3","objects":-1723},{"shards":"0,3,4","objects":3797}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":212,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.960508+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:44:03.960508+0000","last_peered":"2023-10-09T05:44:03.960508+0000","last_clean":"2023-10-09T05:44:03.960508+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:44:03.960508+0000","last_undegraded":"2023-10-09T05:44:03.960508+0000","last_fullsized":"2023-10-09T05:44:03.960508+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":206,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.968961+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:44:03.968961+0000","last_peered":"2023-10-09T05:44:03.968961+0000","last_clean":"2023-10-09T05:44:03.968961+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:44:03.968961+0000","last_undegraded":"2023-10-09T05:44:03.968961+0000","last_fullsized":"2023-10-09T05:44:03.968961+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"242'8172","reported_seq":8211,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-10-09T05:44:12.109206+0000","last_change":"2023-10-09T05:41:29.078791+0000","last_active":"2023-10-09T05:44:12.109206+0000","last_peered":"2023-10-09T05:44:12.109206+0000","last_clean":"2023-10-09T05:44:12.109206+0000","last_became_active":"2023-10-09T05:41:29.078491+0000","last_became_peered":"2023-10-09T05:41:29.078491+0000","last_unstale":"2023-10-09T05:44:12.109206+0000","last_undegraded":"2023-10-09T05:44:12.109206+0000","last_fullsized":"2023-10-09T05:44:12.109206+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8172,"log_dups_size":0,"ondisk_log_size":8172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:37.817282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256376832,"num_objects":3912,"num_object_clones":0,"num_object_copies":11736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8172,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":207,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.972709+0000","last_change":"2023-10-09T05:30:30.459163+0000","last_active":"2023-10-09T05:44:03.972709+0000","last_peered":"2023-10-09T05:44:03.972709+0000","last_clean":"2023-10-09T05:44:03.972709+0000","last_became_active":"2023-10-09T05:30:30.458861+0000","last_became_peered":"2023-10-09T05:30:30.458861+0000","last_unstale":"2023-10-09T05:44:03.972709+0000","last_undegraded":"2023-10-09T05:44:03.972709+0000","last_fullsized":"2023-10-09T05:44:03.972709+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":221,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.968399+0000","last_change":"2023-10-09T05:31:02.549716+0000","last_active":"2023-10-09T05:44:03.968399+0000","last_peered":"2023-10-09T05:44:03.968399+0000","last_clean":"2023-10-09T05:44:03.968399+0000","last_became_active":"2023-10-09T05:31:02.549391+0000","last_became_peered":"2023-10-09T05:31:02.549391+0000","last_unstale":"2023-10-09T05:44:03.968399+0000","last_undegraded":"2023-10-09T05:44:03.968399+0000","last_fullsized":"2023-10-09T05:44:03.968399+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"241'7889","reported_seq":7925,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:10.779346+0000","last_change":"2023-10-09T05:41:29.078645+0000","last_active":"2023-10-09T05:44:10.779346+0000","last_peered":"2023-10-09T05:44:10.779346+0000","last_clean":"2023-10-09T05:44:10.779346+0000","last_became_active":"2023-10-09T05:41:29.078355+0000","last_became_peered":"2023-10-09T05:41:29.078355+0000","last_unstale":"2023-10-09T05:44:10.779346+0000","last_undegraded":"2023-10-09T05:44:10.779346+0000","last_fullsized":"2023-10-09T05:44:10.779346+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7889,"log_dups_size":0,"ondisk_log_size":7889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252051480,"num_objects":3847,"num_object_clones":0,"num_object_copies":11541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3847,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7889,"num_write_kb":375489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":240,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.968109+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:44:03.968109+0000","last_peered":"2023-10-09T05:44:03.968109+0000","last_clean":"2023-10-09T05:44:03.968109+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:44:03.968109+0000","last_undegraded":"2023-10-09T05:44:03.968109+0000","last_fullsized":"2023-10-09T05:44:03.968109+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":215,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.968811+0000","last_change":"2023-10-09T05:31:02.552960+0000","last_active":"2023-10-09T05:44:03.968811+0000","last_peered":"2023-10-09T05:44:03.968811+0000","last_clean":"2023-10-09T05:44:03.968811+0000","last_became_active":"2023-10-09T05:31:02.552655+0000","last_became_peered":"2023-10-09T05:31:02.552655+0000","last_unstale":"2023-10-09T05:44:03.968811+0000","last_undegraded":"2023-10-09T05:44:03.968811+0000","last_fullsized":"2023-10-09T05:44:03.968811+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"241'7884","reported_seq":8619,"reported_epoch":241,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:44:10.853946+0000","last_change":"2023-10-09T05:44:03.247331+0000","last_active":"2023-10-09T05:44:10.853946+0000","last_peered":"2023-10-09T05:44:10.853946+0000","last_clean":"2023-10-09T05:43:18.861565+0000","last_became_active":"2023-10-09T05:43:21.207790+0000","last_became_peered":"2023-10-09T05:43:21.207790+0000","last_unstale":"2023-10-09T05:44:10.853946+0000","last_undegraded":"2023-10-09T05:43:21.070384+0000","last_fullsized":"2023-10-09T05:43:20.885426+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7884,"log_dups_size":0,"ondisk_log_size":7884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252837888,"num_objects":3858,"num_object_clones":0,"num_object_copies":11574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1466,"num_objects_misplaced":3635,"num_objects_unfound":0,"num_objects_dirty":3858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7884,"num_write_kb":375744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":345,"num_bytes_recovered":22544384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3],"avail_no_missing":["6","2","3"],"object_location_counts":[{"shards":"2,3,6","objects":3635},{"shards":"3,6","objects":-1243}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":248,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.962810+0000","last_change":"2023-10-09T05:43:19.883213+0000","last_active":"2023-10-09T05:44:03.962810+0000","last_peered":"2023-10-09T05:44:03.962810+0000","last_clean":"2023-10-09T05:44:03.962810+0000","last_became_active":"2023-10-09T05:43:19.882914+0000","last_became_peered":"2023-10-09T05:43:19.882914+0000","last_unstale":"2023-10-09T05:44:03.962810+0000","last_undegraded":"2023-10-09T05:44:03.962810+0000","last_fullsized":"2023-10-09T05:44:03.962810+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.768555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":190,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.985239+0000","last_change":"2023-10-09T05:31:32.869220+0000","last_active":"2023-10-09T05:44:04.985239+0000","last_peered":"2023-10-09T05:44:04.985239+0000","last_clean":"2023-10-09T05:44:04.985239+0000","last_became_active":"2023-10-09T05:31:32.868943+0000","last_became_peered":"2023-10-09T05:31:32.868943+0000","last_unstale":"2023-10-09T05:44:04.985239+0000","last_undegraded":"2023-10-09T05:44:04.985239+0000","last_fullsized":"2023-10-09T05:44:04.985239+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.974274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"241'7790","reported_seq":7834,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:09.411780+0000","last_change":"2023-10-09T05:44:04.007126+0000","last_active":"2023-10-09T05:44:09.411780+0000","last_peered":"2023-10-09T05:44:09.411780+0000","last_clean":"2023-10-09T05:44:09.411780+0000","last_became_active":"2023-10-09T05:44:03.989206+0000","last_became_peered":"2023-10-09T05:44:03.989206+0000","last_unstale":"2023-10-09T05:44:09.411780+0000","last_undegraded":"2023-10-09T05:44:09.411780+0000","last_fullsized":"2023-10-09T05:44:09.411780+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7790,"log_dups_size":0,"ondisk_log_size":7790,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124059648,"num_objects":1893,"num_object_clones":0,"num_object_copies":5679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3915,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":855,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.984762+0000","last_change":"2023-10-09T05:43:22.965549+0000","last_active":"2023-10-09T05:44:04.984762+0000","last_peered":"2023-10-09T05:44:04.984762+0000","last_clean":"2023-10-09T05:44:04.984762+0000","last_became_active":"2023-10-09T05:43:22.964989+0000","last_became_peered":"2023-10-09T05:43:22.964989+0000","last_unstale":"2023-10-09T05:44:04.984762+0000","last_undegraded":"2023-10-09T05:44:04.984762+0000","last_fullsized":"2023-10-09T05:44:04.984762+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:51.536899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":223,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.970697+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:44:03.970697+0000","last_peered":"2023-10-09T05:44:03.970697+0000","last_clean":"2023-10-09T05:44:03.970697+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:44:03.970697+0000","last_undegraded":"2023-10-09T05:44:03.970697+0000","last_fullsized":"2023-10-09T05:44:03.970697+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":175,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.984763+0000","last_change":"2023-10-09T05:30:30.461909+0000","last_active":"2023-10-09T05:44:04.984763+0000","last_peered":"2023-10-09T05:44:04.984763+0000","last_clean":"2023-10-09T05:44:04.984763+0000","last_became_active":"2023-10-09T05:30:30.460963+0000","last_became_peered":"2023-10-09T05:30:30.460963+0000","last_unstale":"2023-10-09T05:44:04.984763+0000","last_undegraded":"2023-10-09T05:44:04.984763+0000","last_fullsized":"2023-10-09T05:44:04.984763+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.037016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"241'7820","reported_seq":7865,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:10.648978+0000","last_change":"2023-10-09T05:44:04.046557+0000","last_active":"2023-10-09T05:44:10.648978+0000","last_peered":"2023-10-09T05:44:10.648978+0000","last_clean":"2023-10-09T05:44:10.648978+0000","last_became_active":"2023-10-09T05:44:04.025781+0000","last_became_peered":"2023-10-09T05:44:04.025781+0000","last_unstale":"2023-10-09T05:44:10.648978+0000","last_undegraded":"2023-10-09T05:44:10.648978+0000","last_fullsized":"2023-10-09T05:44:10.648978+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7820,"log_dups_size":0,"ondisk_log_size":7820,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124551168,"num_objects":1900,"num_object_clones":0,"num_object_copies":5700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3936,"num_write_kb":186784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":677,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.970332+0000","last_change":"2023-10-09T05:30:30.465967+0000","last_active":"2023-10-09T05:44:03.970332+0000","last_peered":"2023-10-09T05:44:03.970332+0000","last_clean":"2023-10-09T05:44:03.970332+0000","last_became_active":"2023-10-09T05:30:30.465430+0000","last_became_peered":"2023-10-09T05:30:30.465430+0000","last_unstale":"2023-10-09T05:44:03.970332+0000","last_undegraded":"2023-10-09T05:44:03.970332+0000","last_fullsized":"2023-10-09T05:44:03.970332+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":207,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.968606+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:44:03.968606+0000","last_peered":"2023-10-09T05:44:03.968606+0000","last_clean":"2023-10-09T05:44:03.968606+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:44:03.968606+0000","last_undegraded":"2023-10-09T05:44:03.968606+0000","last_fullsized":"2023-10-09T05:44:03.968606+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":220,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.970234+0000","last_change":"2023-10-09T05:31:32.866540+0000","last_active":"2023-10-09T05:44:03.970234+0000","last_peered":"2023-10-09T05:44:03.970234+0000","last_clean":"2023-10-09T05:44:03.970234+0000","last_became_active":"2023-10-09T05:31:32.865896+0000","last_became_peered":"2023-10-09T05:31:32.865896+0000","last_unstale":"2023-10-09T05:44:03.970234+0000","last_undegraded":"2023-10-09T05:44:03.970234+0000","last_fullsized":"2023-10-09T05:44:03.970234+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"242'7925","reported_seq":7973,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-10-09T05:44:13.142897+0000","last_change":"2023-10-09T05:44:03.990111+0000","last_active":"2023-10-09T05:44:13.142897+0000","last_peered":"2023-10-09T05:44:13.142897+0000","last_clean":"2023-10-09T05:44:13.142897+0000","last_became_active":"2023-10-09T05:44:03.976601+0000","last_became_peered":"2023-10-09T05:44:03.976601+0000","last_unstale":"2023-10-09T05:44:13.142897+0000","last_undegraded":"2023-10-09T05:44:13.142897+0000","last_fullsized":"2023-10-09T05:44:13.142897+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7925,"log_dups_size":0,"ondisk_log_size":7925,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123305984,"num_objects":1881,"num_object_clones":0,"num_object_copies":5643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4002,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":558,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.964711+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:44:04.964711+0000","last_peered":"2023-10-09T05:44:04.964711+0000","last_clean":"2023-10-09T05:44:04.964711+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:44:04.964711+0000","last_undegraded":"2023-10-09T05:44:04.964711+0000","last_fullsized":"2023-10-09T05:44:04.964711+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":190,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.984659+0000","last_change":"2023-10-09T05:31:02.548063+0000","last_active":"2023-10-09T05:44:04.984659+0000","last_peered":"2023-10-09T05:44:04.984659+0000","last_clean":"2023-10-09T05:44:04.984659+0000","last_became_active":"2023-10-09T05:31:02.547677+0000","last_became_peered":"2023-10-09T05:31:02.547677+0000","last_unstale":"2023-10-09T05:44:04.984659+0000","last_undegraded":"2023-10-09T05:44:04.984659+0000","last_fullsized":"2023-10-09T05:44:04.984659+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:05.205644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":220,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.985413+0000","last_change":"2023-10-09T05:30:30.457353+0000","last_active":"2023-10-09T05:44:04.985413+0000","last_peered":"2023-10-09T05:44:04.985413+0000","last_clean":"2023-10-09T05:44:04.985413+0000","last_became_active":"2023-10-09T05:30:30.456863+0000","last_became_peered":"2023-10-09T05:30:30.456863+0000","last_unstale":"2023-10-09T05:44:04.985413+0000","last_undegraded":"2023-10-09T05:44:04.985413+0000","last_fullsized":"2023-10-09T05:44:04.985413+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:15.233004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"242'8036","reported_seq":8262,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:44:13.478446+0000","last_change":"2023-10-09T05:44:05.057485+0000","last_active":"2023-10-09T05:44:13.478446+0000","last_peered":"2023-10-09T05:44:13.478446+0000","last_clean":"2023-10-09T05:43:18.711607+0000","last_became_active":"2023-10-09T05:44:05.023661+0000","last_became_peered":"2023-10-09T05:44:05.023661+0000","last_unstale":"2023-10-09T05:44:13.478446+0000","last_undegraded":"2023-10-09T05:44:05.020403+0000","last_fullsized":"2023-10-09T05:44:13.478446+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8036,"log_dups_size":0,"ondisk_log_size":8036,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125566976,"num_objects":1916,"num_object_clones":0,"num_object_copies":5748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2885,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4059,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":5373952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1913},{"shards":"0,3,7","objects":3}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":887,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.972454+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:44:03.972454+0000","last_peered":"2023-10-09T05:44:03.972454+0000","last_clean":"2023-10-09T05:44:03.972454+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:44:03.972454+0000","last_undegraded":"2023-10-09T05:44:03.972454+0000","last_fullsized":"2023-10-09T05:44:03.972454+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":391,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.972868+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:44:04.972868+0000","last_peered":"2023-10-09T05:44:04.972868+0000","last_clean":"2023-10-09T05:44:04.972868+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:44:04.972868+0000","last_undegraded":"2023-10-09T05:44:04.972868+0000","last_fullsized":"2023-10-09T05:44:04.972868+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":206,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.960862+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:44:03.960862+0000","last_peered":"2023-10-09T05:44:03.960862+0000","last_clean":"2023-10-09T05:44:03.960862+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:44:03.960862+0000","last_undegraded":"2023-10-09T05:44:03.960862+0000","last_fullsized":"2023-10-09T05:44:03.960862+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":222,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.973270+0000","last_change":"2023-10-09T05:43:19.888977+0000","last_active":"2023-10-09T05:44:03.973270+0000","last_peered":"2023-10-09T05:44:03.973270+0000","last_clean":"2023-10-09T05:44:03.973270+0000","last_became_active":"2023-10-09T05:43:19.888644+0000","last_became_peered":"2023-10-09T05:43:19.888644+0000","last_unstale":"2023-10-09T05:44:03.973270+0000","last_undegraded":"2023-10-09T05:44:03.973270+0000","last_fullsized":"2023-10-09T05:44:03.973270+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"241'8034","reported_seq":8151,"reported_epoch":241,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:44:09.846939+0000","last_change":"2023-10-09T05:44:07.019753+0000","last_active":"2023-10-09T05:44:09.846939+0000","last_peered":"2023-10-09T05:44:09.846939+0000","last_clean":"2023-10-09T05:43:18.738931+0000","last_became_active":"2023-10-09T05:44:06.984836+0000","last_became_peered":"2023-10-09T05:44:06.984836+0000","last_unstale":"2023-10-09T05:44:09.846939+0000","last_undegraded":"2023-10-09T05:44:04.978402+0000","last_fullsized":"2023-10-09T05:44:09.846939+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8034,"log_dups_size":0,"ondisk_log_size":8034,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127172608,"num_objects":1940,"num_object_clones":0,"num_object_copies":5820,"num_objects_missing_on_primary":2990,"num_objects_missing":2990,"num_objects_degraded":2990,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4032,"num_write_kb":191136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1942},{"shards":"0,1,2,7","objects":4},{"shards":"0,2,7","objects":-6}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":685,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.995589+0000","last_change":"2023-10-09T05:43:22.954273+0000","last_active":"2023-10-09T05:44:03.995589+0000","last_peered":"2023-10-09T05:44:03.995589+0000","last_clean":"2023-10-09T05:44:03.995589+0000","last_became_active":"2023-10-09T05:43:22.953880+0000","last_became_peered":"2023-10-09T05:43:22.953880+0000","last_unstale":"2023-10-09T05:44:03.995589+0000","last_undegraded":"2023-10-09T05:44:03.995589+0000","last_fullsized":"2023-10-09T05:44:03.995589+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":195,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.966546+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:44:04.966546+0000","last_peered":"2023-10-09T05:44:04.966546+0000","last_clean":"2023-10-09T05:44:04.966546+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:44:04.966546+0000","last_undegraded":"2023-10-09T05:44:04.966546+0000","last_fullsized":"2023-10-09T05:44:04.966546+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":210,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.968538+0000","last_change":"2023-10-09T05:31:02.545553+0000","last_active":"2023-10-09T05:44:04.968538+0000","last_peered":"2023-10-09T05:44:04.968538+0000","last_clean":"2023-10-09T05:44:04.968538+0000","last_became_active":"2023-10-09T05:31:02.545045+0000","last_became_peered":"2023-10-09T05:31:02.545045+0000","last_unstale":"2023-10-09T05:44:04.968538+0000","last_undegraded":"2023-10-09T05:44:04.968538+0000","last_fullsized":"2023-10-09T05:44:04.968538+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:39.824772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"241'7789","reported_seq":7842,"reported_epoch":241,"state":"active+clean+scrubbing","last_fresh":"2023-10-09T05:44:10.055726+0000","last_change":"2023-10-09T05:44:10.062524+0000","last_active":"2023-10-09T05:44:10.055726+0000","last_peered":"2023-10-09T05:44:10.055726+0000","last_clean":"2023-10-09T05:44:10.055726+0000","last_became_active":"2023-10-09T05:44:03.993350+0000","last_became_peered":"2023-10-09T05:44:03.993350+0000","last_unstale":"2023-10-09T05:44:10.055726+0000","last_undegraded":"2023-10-09T05:44:10.055726+0000","last_fullsized":"2023-10-09T05:44:10.055726+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7789,"log_dups_size":0,"ondisk_log_size":7789,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127664128,"num_objects":1948,"num_object_clones":0,"num_object_copies":5844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3911,"num_write_kb":187488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":437,"reported_epoch":241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:44:04.966852+0000","last_change":"2023-10-09T05:43:20.934237+0000","last_active":"2023-10-09T05:44:04.966852+0000","last_peered":"2023-10-09T05:44:04.966852+0000","last_clean":"2023-10-09T05:43:08.905133+0000","last_became_active":"2023-10-09T05:43:20.932614+0000","last_became_peered":"2023-10-09T05:43:20.932614+0000","last_unstale":"2023-10-09T05:44:04.966852+0000","last_undegraded":"2023-10-09T05:43:20.878705+0000","last_fullsized":"2023-10-09T05:43:20.878227+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:53.541315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":190,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.985001+0000","last_change":"2023-10-09T05:31:02.546679+0000","last_active":"2023-10-09T05:44:04.985001+0000","last_peered":"2023-10-09T05:44:04.985001+0000","last_clean":"2023-10-09T05:44:04.985001+0000","last_became_active":"2023-10-09T05:31:02.546380+0000","last_became_peered":"2023-10-09T05:31:02.546380+0000","last_unstale":"2023-10-09T05:44:04.985001+0000","last_undegraded":"2023-10-09T05:44:04.985001+0000","last_fullsized":"2023-10-09T05:44:04.985001+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.820289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":213,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.968615+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:44:04.968615+0000","last_peered":"2023-10-09T05:44:04.968615+0000","last_clean":"2023-10-09T05:44:04.968615+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:44:04.968615+0000","last_undegraded":"2023-10-09T05:44:04.968615+0000","last_fullsized":"2023-10-09T05:44:04.968615+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"241'7945","reported_seq":7991,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:11.551357+0000","last_change":"2023-10-09T05:44:04.001523+0000","last_active":"2023-10-09T05:44:11.551357+0000","last_peered":"2023-10-09T05:44:11.551357+0000","last_clean":"2023-10-09T05:44:11.551357+0000","last_became_active":"2023-10-09T05:44:03.990530+0000","last_became_peered":"2023-10-09T05:44:03.990530+0000","last_unstale":"2023-10-09T05:44:11.551357+0000","last_undegraded":"2023-10-09T05:44:11.551357+0000","last_fullsized":"2023-10-09T05:44:11.551357+0000","mapping_epoch":213,"log_start":"218'500","ondisk_log_start":"218'500","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7445,"log_dups_size":0,"ondisk_log_size":7445,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126386176,"num_objects":1928,"num_object_clones":0,"num_object_copies":5784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4008,"num_write_kb":189984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":253,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.961559+0000","last_change":"2023-10-09T05:43:20.063203+0000","last_active":"2023-10-09T05:44:03.961559+0000","last_peered":"2023-10-09T05:44:03.961559+0000","last_clean":"2023-10-09T05:44:03.961559+0000","last_became_active":"2023-10-09T05:43:19.885257+0000","last_became_peered":"2023-10-09T05:43:19.885257+0000","last_unstale":"2023-10-09T05:44:03.961559+0000","last_undegraded":"2023-10-09T05:44:03.961559+0000","last_fullsized":"2023-10-09T05:44:03.961559+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:36.424359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":631,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.984920+0000","last_change":"2023-10-09T05:30:30.457953+0000","last_active":"2023-10-09T05:44:04.984920+0000","last_peered":"2023-10-09T05:44:04.984920+0000","last_clean":"2023-10-09T05:44:04.984920+0000","last_became_active":"2023-10-09T05:30:30.457324+0000","last_became_peered":"2023-10-09T05:30:30.457324+0000","last_unstale":"2023-10-09T05:44:04.984920+0000","last_undegraded":"2023-10-09T05:44:04.984920+0000","last_fullsized":"2023-10-09T05:44:04.984920+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:00.751032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":243,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.963049+0000","last_change":"2023-10-09T05:43:19.886969+0000","last_active":"2023-10-09T05:44:03.963049+0000","last_peered":"2023-10-09T05:44:03.963049+0000","last_clean":"2023-10-09T05:44:03.963049+0000","last_became_active":"2023-10-09T05:43:19.885111+0000","last_became_peered":"2023-10-09T05:43:19.885111+0000","last_unstale":"2023-10-09T05:44:03.963049+0000","last_undegraded":"2023-10-09T05:44:03.963049+0000","last_fullsized":"2023-10-09T05:44:03.963049+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:26.890192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"241'8146","reported_seq":8190,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:11.726061+0000","last_change":"2023-10-09T05:44:04.034371+0000","last_active":"2023-10-09T05:44:11.726061+0000","last_peered":"2023-10-09T05:44:11.726061+0000","last_clean":"2023-10-09T05:44:11.726061+0000","last_became_active":"2023-10-09T05:44:04.009599+0000","last_became_peered":"2023-10-09T05:44:04.009599+0000","last_unstale":"2023-10-09T05:44:11.726061+0000","last_undegraded":"2023-10-09T05:44:11.726061+0000","last_fullsized":"2023-10-09T05:44:11.726061+0000","mapping_epoch":213,"log_start":"218'700","ondisk_log_start":"218'700","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7446,"log_dups_size":0,"ondisk_log_size":7446,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128188416,"num_objects":1956,"num_object_clones":0,"num_object_copies":5868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4108,"num_write_kb":194048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":776,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.973670+0000","last_change":"2023-10-09T05:43:22.945599+0000","last_active":"2023-10-09T05:44:03.973670+0000","last_peered":"2023-10-09T05:44:03.973670+0000","last_clean":"2023-10-09T05:44:03.973670+0000","last_became_active":"2023-10-09T05:43:22.945167+0000","last_became_peered":"2023-10-09T05:43:22.945167+0000","last_unstale":"2023-10-09T05:44:03.973670+0000","last_undegraded":"2023-10-09T05:44:03.973670+0000","last_fullsized":"2023-10-09T05:44:03.973670+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":198,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.968620+0000","last_change":"2023-10-09T05:30:30.458888+0000","last_active":"2023-10-09T05:44:04.968620+0000","last_peered":"2023-10-09T05:44:04.968620+0000","last_clean":"2023-10-09T05:44:04.968620+0000","last_became_active":"2023-10-09T05:30:30.458152+0000","last_became_peered":"2023-10-09T05:30:30.458152+0000","last_unstale":"2023-10-09T05:44:04.968620+0000","last_undegraded":"2023-10-09T05:44:04.968620+0000","last_fullsized":"2023-10-09T05:44:04.968620+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:50.289626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":191,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.968197+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:44:03.968197+0000","last_peered":"2023-10-09T05:44:03.968197+0000","last_clean":"2023-10-09T05:44:03.968197+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:44:03.968197+0000","last_undegraded":"2023-10-09T05:44:03.968197+0000","last_fullsized":"2023-10-09T05:44:03.968197+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":215,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.968540+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:44:04.968540+0000","last_peered":"2023-10-09T05:44:04.968540+0000","last_clean":"2023-10-09T05:44:04.968540+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:44:04.968540+0000","last_undegraded":"2023-10-09T05:44:04.968540+0000","last_fullsized":"2023-10-09T05:44:04.968540+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":244,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.964690+0000","last_change":"2023-10-09T05:41:44.887519+0000","last_active":"2023-10-09T05:44:04.964690+0000","last_peered":"2023-10-09T05:44:04.964690+0000","last_clean":"2023-10-09T05:44:04.964690+0000","last_became_active":"2023-10-09T05:41:44.887047+0000","last_became_peered":"2023-10-09T05:41:44.887047+0000","last_unstale":"2023-10-09T05:44:04.964690+0000","last_undegraded":"2023-10-09T05:44:04.964690+0000","last_fullsized":"2023-10-09T05:44:04.964690+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:49.475913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"242'7982","reported_seq":10410,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:44:12.056941+0000","last_change":"2023-10-09T05:44:04.148373+0000","last_active":"2023-10-09T05:44:12.056941+0000","last_peered":"2023-10-09T05:44:12.056941+0000","last_clean":"2023-10-09T05:43:18.869044+0000","last_became_active":"2023-10-09T05:44:04.104935+0000","last_became_peered":"2023-10-09T05:44:04.104935+0000","last_unstale":"2023-10-09T05:44:12.056941+0000","last_undegraded":"2023-10-09T05:44:04.094988+0000","last_fullsized":"2023-10-09T05:44:12.056941+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7982,"log_dups_size":0,"ondisk_log_size":7982,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127074304,"num_objects":1939,"num_object_clones":0,"num_object_copies":5817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2365,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4030,"num_write_kb":191008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":632,"num_bytes_recovered":41418752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":1970},{"shards":"2,4,7","objects":-31}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":210,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.970015+0000","last_change":"2023-10-09T05:30:30.461174+0000","last_active":"2023-10-09T05:44:03.970015+0000","last_peered":"2023-10-09T05:44:03.970015+0000","last_clean":"2023-10-09T05:44:03.970015+0000","last_became_active":"2023-10-09T05:30:30.460809+0000","last_became_peered":"2023-10-09T05:30:30.460809+0000","last_unstale":"2023-10-09T05:44:03.970015+0000","last_undegraded":"2023-10-09T05:44:03.970015+0000","last_fullsized":"2023-10-09T05:44:03.970015+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":221,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.968650+0000","last_change":"2023-10-09T05:31:32.869382+0000","last_active":"2023-10-09T05:44:03.968650+0000","last_peered":"2023-10-09T05:44:03.968650+0000","last_clean":"2023-10-09T05:44:03.968650+0000","last_became_active":"2023-10-09T05:31:32.869126+0000","last_became_peered":"2023-10-09T05:31:32.869126+0000","last_unstale":"2023-10-09T05:44:03.968650+0000","last_undegraded":"2023-10-09T05:44:03.968650+0000","last_fullsized":"2023-10-09T05:44:03.968650+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"242'7839","reported_seq":7886,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-10-09T05:44:13.302203+0000","last_change":"2023-10-09T05:44:04.044151+0000","last_active":"2023-10-09T05:44:13.302203+0000","last_peered":"2023-10-09T05:44:13.302203+0000","last_clean":"2023-10-09T05:44:13.302203+0000","last_became_active":"2023-10-09T05:44:04.035827+0000","last_became_peered":"2023-10-09T05:44:04.035827+0000","last_unstale":"2023-10-09T05:44:13.302203+0000","last_undegraded":"2023-10-09T05:44:13.302203+0000","last_fullsized":"2023-10-09T05:44:13.302203+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7839,"log_dups_size":0,"ondisk_log_size":7839,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124977152,"num_objects":1907,"num_object_clones":0,"num_object_copies":5721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3955,"num_write_kb":187584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":195,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.970608+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:44:03.970608+0000","last_peered":"2023-10-09T05:44:03.970608+0000","last_clean":"2023-10-09T05:44:03.970608+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:44:03.970608+0000","last_undegraded":"2023-10-09T05:44:03.970608+0000","last_fullsized":"2023-10-09T05:44:03.970608+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":221,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.967983+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:44:03.967983+0000","last_peered":"2023-10-09T05:44:03.967983+0000","last_clean":"2023-10-09T05:44:03.967983+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:44:03.967983+0000","last_undegraded":"2023-10-09T05:44:03.967983+0000","last_fullsized":"2023-10-09T05:44:03.967983+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":221,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.960845+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:44:03.960845+0000","last_peered":"2023-10-09T05:44:03.960845+0000","last_clean":"2023-10-09T05:44:03.960845+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:44:03.960845+0000","last_undegraded":"2023-10-09T05:44:03.960845+0000","last_fullsized":"2023-10-09T05:44:03.960845+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":233,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.959650+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:44:03.959650+0000","last_peered":"2023-10-09T05:44:03.959650+0000","last_clean":"2023-10-09T05:44:03.959650+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:44:03.959650+0000","last_undegraded":"2023-10-09T05:44:03.959650+0000","last_fullsized":"2023-10-09T05:44:03.959650+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":198,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-10-09T05:44:04.969439+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:44:04.969439+0000","last_peered":"2023-10-09T05:44:04.969439+0000","last_clean":"2023-10-09T05:44:04.969439+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:44:04.969439+0000","last_undegraded":"2023-10-09T05:44:04.969439+0000","last_fullsized":"2023-10-09T05:44:04.969439+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":206,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-10-09T05:44:03.967918+0000","last_change":"2023-10-09T05:30:30.457703+0000","last_active":"2023-10-09T05:44:03.967918+0000","last_peered":"2023-10-09T05:44:03.967918+0000","last_clean":"2023-10-09T05:44:03.967918+0000","last_became_active":"2023-10-09T05:30:30.457377+0000","last_became_peered":"2023-10-09T05:30:30.457377+0000","last_unstale":"2023-10-09T05:44:03.967918+0000","last_undegraded":"2023-10-09T05:44:03.967918+0000","last_fullsized":"2023-10-09T05:44:03.967918+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":4016635928,"num_objects":61290,"num_object_clones":0,"num_object_copies":183870,"num_objects_missing_on_primary":5903,"num_objects_missing":5903,"num_objects_degraded":20957,"num_objects_misplaced":5566,"num_objects_unfound":0,"num_objects_dirty":61290,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":127698,"num_write_kb":6047553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1832,"num_bytes_recovered":119734272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12586987520,"data_stored":12586975304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":200965,"ondisk_log_size":200965,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846707,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":342152,"kb_used_data":19648,"kb_used_omap":14,"kb_used_meta":322481,"kb_avail":93403000,"statfs":{"total":95995035648,"available":95644672000,"internally_reserved":0,"allocated":20119552,"data_stored":19566631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":330221402},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,2,8,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:43:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":3.149,"15min":5.6699999999999999},"min":{"1min":0.26300000000000001,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":4.3979999999999997,"5min":113.018,"15min":220.77500000000001},"last":2.3519999999999999},{"interface":"front","average":{"1min":1.264,"5min":3.1869999999999998,"15min":6.2679999999999998},"min":{"1min":0.307,"5min":0.29499999999999998,"15min":0.22700000000000001},"max":{"1min":4.2640000000000002,"5min":113.005,"15min":220.78700000000001},"last":0.76400000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:43:53 2023","interfaces":[{"interface":"back","average":{"1min":5.9249999999999998,"5min":3.8940000000000001,"15min":3.0990000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":83.575000000000003,"5min":83.575000000000003,"15min":190.68100000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":5.9870000000000001,"5min":3.8210000000000002,"15min":2.8759999999999999},"min":{"1min":0.32100000000000001,"5min":0.255,"15min":0.255},"max":{"1min":87.516999999999996,"5min":87.516999999999996,"15min":191.09200000000001},"last":2.2890000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:43:53 2023","interfaces":[{"interface":"back","average":{"1min":2.169,"5min":1.9359999999999999,"15min":4.6139999999999999},"min":{"1min":0.36199999999999999,"5min":0.313,"15min":0.23899999999999999},"max":{"1min":7.9349999999999996,"5min":10.09,"15min":78.716999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":2.4119999999999999,"5min":2.177,"15min":4.5529999999999999},"min":{"1min":0.33400000000000002,"5min":0.28100000000000003,"15min":0.27000000000000002},"max":{"1min":7.391,"5min":17.811,"15min":78.768000000000001},"last":2.3250000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:43:53 2023","interfaces":[{"interface":"back","average":{"1min":2.903,"5min":6.4900000000000002,"15min":3.0499999999999998},"min":{"1min":0.35999999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.574999999999999,"5min":185.17099999999999,"15min":185.17099999999999},"last":64.558999999999997},{"interface":"front","average":{"1min":2.3210000000000002,"5min":6.4749999999999996,"15min":3.6200000000000001},"min":{"1min":0.34799999999999998,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":4.7969999999999997,"5min":183.35499999999999,"15min":183.35499999999999},"last":64.5}]},{"osd":4,"last update":"Mon Oct 9 05:43:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.54800000000000004,"15min":1.276},"min":{"1min":0.30199999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.3620000000000001,"5min":4.3620000000000001,"15min":12.789},"last":2.3820000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61299999999999999,"15min":1.4750000000000001},"min":{"1min":0.315,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":3.4969999999999999,"5min":4.3419999999999996,"15min":9.0470000000000006},"last":0.74099999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:43:53 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.73399999999999999,"15min":1.4419999999999999},"min":{"1min":0.35599999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.0220000000000002,"5min":4.4429999999999996,"15min":10.813000000000001},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.73999999999999999,"15min":1.792},"min":{"1min":0.33400000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.9500000000000002,"5min":4.6159999999999997,"15min":10.321},"last":0.64700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:43:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.66000000000000003,"15min":1.482},"min":{"1min":0.32900000000000001,"5min":0.318,"15min":0.318},"max":{"1min":2.3079999999999998,"5min":4.468,"15min":9.2530000000000001},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.76800000000000002,"15min":1.4019999999999999},"min":{"1min":0.34699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.2279999999999998,"5min":4.649,"15min":8.2680000000000007},"last":0.81999999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626733,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1533248,"kb_used_data":1227832,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92211904,"statfs":{"total":95995035648,"available":94424989696,"internally_reserved":0,"allocated":1257299968,"data_stored":1256754871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:43:30 2023","interfaces":[{"interface":"back","average":{"1min":15.058,"5min":7.9409999999999998,"15min":4.2859999999999996},"min":{"1min":0.249,"5min":0.214,"15min":0.214},"max":{"1min":162.08799999999999,"5min":162.08799999999999,"15min":162.08799999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":12.91,"5min":7.2599999999999998,"15min":4},"min":{"1min":0.255,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":161.90899999999999,"5min":161.90899999999999,"15min":161.90899999999999},"last":0.316}]},{"osd":1,"last update":"Mon Oct 9 05:43:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":7.766,"15min":3.7850000000000001},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":5.625,"5min":228.24299999999999,"15min":228.24299999999999},"last":0.374},{"interface":"front","average":{"1min":1.595,"5min":7.6159999999999997,"15min":3.8140000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":5.6109999999999998,"5min":228.22499999999999,"15min":228.22499999999999},"last":0.65400000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:43:30 2023","interfaces":[{"interface":"back","average":{"1min":6.782,"5min":6.5839999999999996,"15min":3.8929999999999998},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":77.233000000000004,"5min":115.40600000000001,"15min":115.40600000000001},"last":0.72699999999999998},{"interface":"front","average":{"1min":6.5060000000000002,"5min":6.7789999999999999,"15min":3.9140000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":77.254999999999995,"5min":115.297,"15min":115.297},"last":0.53100000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:43:56 2023","interfaces":[{"interface":"back","average":{"1min":7.093,"5min":7.0650000000000004,"15min":3.661},"min":{"1min":0.33200000000000002,"5min":0.307,"15min":0.26800000000000002},"max":{"1min":77.222999999999999,"5min":115.244,"15min":115.244},"last":0.40000000000000002},{"interface":"front","average":{"1min":7.5300000000000002,"5min":6.9429999999999996,"15min":3.7000000000000002},"min":{"1min":0.38800000000000001,"5min":0.33200000000000002,"15min":0.30299999999999999},"max":{"1min":77.418999999999997,"5min":115.34999999999999,"15min":115.34999999999999},"last":0.623}]},{"osd":4,"last update":"Mon Oct 9 05:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":1.431,"15min":1.726},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":2.73,"5min":115.214,"15min":125.64},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.0299999999999998,"5min":1.7729999999999999,"15min":1.71},"min":{"1min":0.29299999999999998,"5min":0.26000000000000001,"15min":0.254},"max":{"1min":13.609,"5min":115.22799999999999,"15min":125.625},"last":0.39000000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:44:09 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":2.2189999999999999,"15min":1.9079999999999999},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":4.851,"5min":115.316,"15min":127.35899999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.1819999999999999,"5min":2.161,"15min":1.7769999999999999},"min":{"1min":0.318,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.9420000000000002,"5min":115.30500000000001,"15min":128.93799999999999},"last":3.2650000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":1.7130000000000001,"15min":2.3140000000000001},"min":{"1min":0.33000000000000002,"5min":0.32500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3470000000000004,"5min":115.514,"15min":115.514},"last":0.69999999999999996},{"interface":"front","average":{"1min":1.149,"5min":1.7909999999999999,"15min":1.744},"min":{"1min":0.34300000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.085,"5min":116.53100000000001,"15min":116.53100000000001},"last":3.524}]}]},{"osd":1,"up_from":93,"seq":399431958695,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2211844,"kb_used_data":1805564,"kb_used_omap":7,"kb_used_meta":406264,"kb_avail":91533308,"statfs":{"total":95995035648,"available":93730107392,"internally_reserved":0,"allocated":1848897536,"data_stored":1848065596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":416014862},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:43:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4129999999999998,"5min":5.1760000000000002,"15min":3.7440000000000002},"min":{"1min":0.28199999999999997,"5min":0.20300000000000001,"15min":0.185},"max":{"1min":11.35,"5min":82.459999999999994,"15min":207.858},"last":0.30199999999999999},{"interface":"front","average":{"1min":3.0720000000000001,"5min":5.5060000000000002,"15min":4.2489999999999997},"min":{"1min":0.219,"5min":0.182,"15min":0.182},"max":{"1min":11.526,"5min":80.004000000000005,"15min":206.97200000000001},"last":0.318}]},{"osd":2,"last update":"Mon Oct 9 05:43:15 2023","interfaces":[{"interface":"back","average":{"1min":7.6900000000000004,"5min":4.9370000000000003,"15min":4.0220000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.184},"max":{"1min":99.381,"5min":99.381,"15min":101.367},"last":0.31},{"interface":"front","average":{"1min":6.6139999999999999,"5min":4.5659999999999998,"15min":3.7400000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":99.861000000000004,"5min":99.861000000000004,"15min":99.861000000000004},"last":0.23100000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:43:55 2023","interfaces":[{"interface":"back","average":{"1min":2.395,"5min":9.4329999999999998,"15min":4.3490000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":9.1579999999999995,"5min":162.33699999999999,"15min":162.33699999999999},"last":0.27200000000000002},{"interface":"front","average":{"1min":3.6030000000000002,"5min":9.468,"15min":4.6399999999999997},"min":{"1min":0.34999999999999998,"5min":0.26900000000000002,"15min":0.21299999999999999},"max":{"1min":10.339,"5min":162.376,"15min":162.376},"last":0.34599999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:43:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":2.4390000000000001,"15min":2.4319999999999999},"min":{"1min":0.375,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.493,"5min":42.106999999999999,"15min":141.32900000000001},"last":0.376},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.3780000000000001,"15min":1.988},"min":{"1min":0.40200000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":11.446,"5min":40.475999999999999,"15min":141.15899999999999},"last":0.38300000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:43:55 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":2.3340000000000001,"15min":2.3599999999999999},"min":{"1min":0.434,"5min":0.307,"15min":0.307},"max":{"1min":8.907,"5min":40.633000000000003,"15min":235.18700000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.7719999999999998,"5min":2.516,"15min":2.7949999999999999},"min":{"1min":0.41199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.394,"5min":42.078000000000003,"15min":235.79300000000001},"last":0.40200000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:43:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5339999999999998,"5min":3.5990000000000002,"15min":2.0569999999999999},"min":{"1min":0.39500000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.9469999999999992,"5min":103.60899999999999,"15min":103.60899999999999},"last":101.295},{"interface":"front","average":{"1min":2.133,"5min":3.3639999999999999,"15min":2.286},"min":{"1min":0.40000000000000002,"5min":0.378,"15min":0.33400000000000002},"max":{"1min":9.8109999999999999,"5min":103.679,"15min":103.679},"last":101.27}]},{"osd":7,"last update":"Mon Oct 9 05:43:55 2023","interfaces":[{"interface":"back","average":{"1min":1.518,"5min":2.3500000000000001,"15min":2.3809999999999998},"min":{"1min":0.37,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.4089999999999998,"5min":42.204000000000001,"15min":126.68300000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.931,"5min":2.5019999999999998,"15min":2.4969999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.34399999999999997},"max":{"1min":11.395,"5min":42.228999999999999,"15min":126.738},"last":0.441}]}]},{"osd":0,"up_from":88,"seq":377957122221,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2393332,"kb_used_data":1955244,"kb_used_omap":8,"kb_used_meta":438071,"kb_avail":91351820,"statfs":{"total":95995035648,"available":93544263680,"internally_reserved":0,"allocated":2002169856,"data_stored":2001292735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":448585480},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:43:55 2023","interfaces":[{"interface":"back","average":{"1min":2.8500000000000001,"5min":2.3980000000000001,"15min":1.605},"min":{"1min":0.20599999999999999,"5min":0.20200000000000001,"15min":0.17199999999999999},"max":{"1min":13.797000000000001,"5min":17.760000000000002,"15min":17.760000000000002},"last":0.251},{"interface":"front","average":{"1min":3.8999999999999999,"5min":3.4809999999999999,"15min":2.0950000000000002},"min":{"1min":0.26400000000000001,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":13.816000000000001,"5min":26.163,"15min":26.163},"last":0.57499999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:43:23 2023","interfaces":[{"interface":"back","average":{"1min":4.7670000000000003,"5min":4.702,"15min":3.1320000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.214},"max":{"1min":13.302,"5min":92.230000000000004,"15min":92.230000000000004},"last":0.59599999999999997},{"interface":"front","average":{"1min":4.1230000000000002,"5min":4.7539999999999996,"15min":2.9790000000000001},"min":{"1min":0.26800000000000002,"5min":0.222,"15min":0.218},"max":{"1min":16.402999999999999,"5min":92.247,"15min":92.247},"last":0.28000000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:43:55 2023","interfaces":[{"interface":"back","average":{"1min":3.4700000000000002,"5min":7.0270000000000001,"15min":3.524},"min":{"1min":0.29299999999999998,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":14.807,"5min":123.024,"15min":123.024},"last":0.64700000000000002},{"interface":"front","average":{"1min":4.0140000000000002,"5min":6.3479999999999999,"15min":3.371},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":14.85,"5min":99.162000000000006,"15min":99.162000000000006},"last":0.48799999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:43:23 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.47,"15min":1.395},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.35,"5min":11.35,"15min":69.082999999999998},"last":0.374},{"interface":"front","average":{"1min":1.974,"5min":2.2829999999999999,"15min":1.827},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.30399999999999999},"max":{"1min":6.508,"5min":9.5190000000000001,"15min":69.102000000000004},"last":0.69799999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:43:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":6.6639999999999997,"15min":3.0569999999999999},"min":{"1min":0.35699999999999998,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":6.7750000000000004,"5min":281.733,"15min":281.733},"last":0.622},{"interface":"front","average":{"1min":3.395,"5min":6.7850000000000001,"15min":3.117},"min":{"1min":0.31,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.552,"5min":282.95400000000001,"15min":282.95400000000001},"last":0.56799999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:43:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":2.4420000000000002,"15min":1.6379999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":6.625,"5min":11.664,"15min":11.664},"last":1.7509999999999999},{"interface":"front","average":{"1min":3.4180000000000001,"5min":2.633,"15min":1.7509999999999999},"min":{"1min":0.38800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":12.587999999999999,"5min":12.587999999999999,"15min":12.587999999999999},"last":0.53900000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:43:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8919999999999999,"5min":1.72,"15min":1.486},"min":{"1min":0.41999999999999998,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":7.1159999999999997,"5min":11.631,"15min":11.631},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.3159999999999998,"5min":2.4420000000000002,"15min":2.3460000000000001},"min":{"1min":0.44800000000000001,"5min":0.42599999999999999,"15min":0.41399999999999998},"max":{"1min":6.9580000000000002,"5min":9.6329999999999991,"15min":11.068},"last":0.67100000000000004}]}]},{"osd":2,"up_from":98,"seq":420906795168,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2346404,"kb_used_data":1840860,"kb_used_omap":11,"kb_used_meta":505524,"kb_avail":91398748,"statfs":{"total":95995035648,"available":93592317952,"internally_reserved":0,"allocated":1885040640,"data_stored":1884111910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":517657173},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:43:18 2023","interfaces":[{"interface":"back","average":{"1min":4.7080000000000002,"5min":3.5619999999999998,"15min":2.1280000000000001},"min":{"1min":0.23699999999999999,"5min":0.17799999999999999,"15min":0.17199999999999999},"max":{"1min":53.709000000000003,"5min":53.709000000000003,"15min":53.709000000000003},"last":0.23999999999999999},{"interface":"front","average":{"1min":6.7439999999999998,"5min":4.4059999999999997,"15min":2.286},"min":{"1min":0.23899999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":53.664999999999999,"5min":53.664999999999999,"15min":53.664999999999999},"last":0.32500000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:43:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7810000000000001,"5min":5.5430000000000001,"15min":2.919},"min":{"1min":0.29099999999999998,"5min":0.23100000000000001,"15min":0.20699999999999999},"max":{"1min":10.066000000000001,"5min":98.575000000000003,"15min":98.575000000000003},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.9019999999999999,"5min":4.5199999999999996,"15min":2.4550000000000001},"min":{"1min":0.32100000000000001,"5min":0.21299999999999999,"15min":0.20000000000000001},"max":{"1min":5.9429999999999996,"5min":90.135000000000005,"15min":98.397000000000006},"last":2.0129999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:44:04 2023","interfaces":[{"interface":"back","average":{"1min":3.8999999999999999,"5min":8.6029999999999998,"15min":4.0199999999999996},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.24199999999999999},"max":{"1min":13.111000000000001,"5min":150.65899999999999,"15min":150.65899999999999},"last":0.307},{"interface":"front","average":{"1min":3.9180000000000001,"5min":7.8700000000000001,"15min":3.931},"min":{"1min":0.35199999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":16.356999999999999,"5min":150.851,"15min":150.851},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:43:18 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":2.1600000000000001,"15min":1.8500000000000001},"min":{"1min":0.42099999999999999,"5min":0.29099999999999998,"15min":0.27700000000000002},"max":{"1min":5.8739999999999997,"5min":14.382,"15min":115.84999999999999},"last":0.372},{"interface":"front","average":{"1min":0.92800000000000005,"5min":2.1120000000000001,"15min":2.0099999999999998},"min":{"1min":0.42699999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.9910000000000001,"5min":13.815,"15min":116.149},"last":0.50600000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:43:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2370000000000001,"5min":2.5209999999999999,"15min":1.8440000000000001},"min":{"1min":0.46800000000000003,"5min":0.36699999999999999,"15min":0.312},"max":{"1min":10.023,"5min":13.712999999999999,"15min":24.597999999999999},"last":4.524},{"interface":"front","average":{"1min":1.714,"5min":2.1030000000000002,"15min":1.593},"min":{"1min":0.44600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.915,"5min":14.426,"15min":23.919},"last":0.45500000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:43:18 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":2.1859999999999999,"15min":1.6890000000000001},"min":{"1min":0.39600000000000002,"5min":0.35599999999999998,"15min":0.35299999999999998},"max":{"1min":2.9380000000000002,"5min":13.839,"15min":20.503},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.2879999999999998,"5min":2.476,"15min":1.6990000000000001},"min":{"1min":0.52600000000000002,"5min":0.36799999999999999,"15min":0.34499999999999997},"max":{"1min":10.097,"5min":13.757,"15min":13.757},"last":0.56999999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:44:04 2023","interfaces":[{"interface":"back","average":{"1min":1.8100000000000001,"5min":2.1240000000000001,"15min":2.5099999999999998},"min":{"1min":0.53500000000000003,"5min":0.37,"15min":0.37},"max":{"1min":6.9779999999999998,"5min":13.865,"15min":24.030999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":3.2160000000000002,"5min":2.5550000000000002,"15min":2.2599999999999998},"min":{"1min":0.47599999999999998,"5min":0.42399999999999999,"15min":0.41599999999999998},"max":{"1min":13.061999999999999,"5min":13.781000000000001,"15min":24.178000000000001},"last":5.5449999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631642,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3055484,"kb_used_data":2561076,"kb_used_omap":4,"kb_used_meta":494395,"kb_avail":90689668,"statfs":{"total":95995035648,"available":92866220032,"internally_reserved":0,"allocated":2622541824,"data_stored":2621709438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":506261086},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:43:54 2023","interfaces":[{"interface":"back","average":{"1min":8.0939999999999994,"5min":5.8479999999999999,"15min":3.0129999999999999},"min":{"1min":0.24299999999999999,"5min":0.184,"15min":0.183},"max":{"1min":106.95999999999999,"5min":167.82400000000001,"15min":167.82400000000001},"last":2.419},{"interface":"front","average":{"1min":9.8190000000000008,"5min":7.173,"15min":3.5619999999999998},"min":{"1min":0.32000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":107.348,"5min":167.89599999999999,"15min":167.89599999999999},"last":1.702}]},{"osd":1,"last update":"Mon Oct 9 05:43:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5289999999999999,"5min":3.181,"15min":2.395},"min":{"1min":0.27000000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":9.3339999999999996,"5min":48.052,"15min":83.921999999999997},"last":0.77100000000000002},{"interface":"front","average":{"1min":3.379,"5min":3.4620000000000002,"15min":2.4409999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":12.023999999999999,"5min":13.551,"15min":83.707999999999998},"last":0.89800000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:43:54 2023","interfaces":[{"interface":"back","average":{"1min":3.476,"5min":5.867,"15min":4.0540000000000003},"min":{"1min":0.28599999999999998,"5min":0.26500000000000001,"15min":0.215},"max":{"1min":13.1,"5min":190.43299999999999,"15min":190.43299999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":2.919,"5min":6.7489999999999997,"15min":4.423},"min":{"1min":0.27200000000000002,"5min":0.214,"15min":0.214},"max":{"1min":7.5170000000000003,"5min":190.267,"15min":190.267},"last":0.33800000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:43:54 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":3.105,"15min":2.1640000000000001},"min":{"1min":0.38,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.9969999999999999,"5min":51.591999999999999,"15min":51.591999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.0779999999999998,"5min":2.1560000000000001,"15min":1.7130000000000001},"min":{"1min":0.35299999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.75,"5min":26.774999999999999,"15min":45.735999999999997},"last":0.44}]},{"osd":5,"last update":"Mon Oct 9 05:43:54 2023","interfaces":[{"interface":"back","average":{"1min":2.2170000000000001,"5min":2.6589999999999998,"15min":1.8380000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.0839999999999996,"5min":13.114000000000001,"15min":47.948999999999998},"last":3.4079999999999999},{"interface":"front","average":{"1min":2.2480000000000002,"5min":2.359,"15min":1.8240000000000001},"min":{"1min":0.45300000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.8010000000000002,"5min":26.728000000000002,"15min":47.968000000000004},"last":6.0030000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:43:54 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":2.2360000000000002,"15min":1.6970000000000001},"min":{"1min":0.434,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.8250000000000002,"5min":26.847999999999999,"15min":26.847999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":2.5859999999999999,"5min":2.7839999999999998,"15min":1.704},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":7.0309999999999997,"5min":13.058,"15min":13.058},"last":0.93400000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:44:00 2023","interfaces":[{"interface":"back","average":{"1min":2.613,"5min":2.6349999999999998,"15min":2.4079999999999999},"min":{"1min":0.41299999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":7.0359999999999996,"5min":26.024999999999999,"15min":26.024999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":2.7930000000000001,"5min":2.7970000000000002,"15min":2.3340000000000001},"min":{"1min":0.46200000000000002,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":7.0609999999999999,"5min":25.995999999999999,"15min":25.995999999999999},"last":0.61199999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019189,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1896640,"kb_used_data":1595320,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91848512,"statfs":{"total":95995035648,"available":94052876288,"internally_reserved":0,"allocated":1633607680,"data_stored":1633062337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:43:30 2023","interfaces":[{"interface":"back","average":{"1min":5.8680000000000003,"5min":3.4089999999999998,"15min":2.1030000000000002},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.214},"max":{"1min":73.384,"5min":100.504,"15min":100.504},"last":0.252},{"interface":"front","average":{"1min":5.6050000000000004,"5min":3.786,"15min":2.3599999999999999},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.23200000000000001},"max":{"1min":71.884,"5min":100.717,"15min":100.717},"last":1.157}]},{"osd":1,"last update":"Mon Oct 9 05:44:03 2023","interfaces":[{"interface":"back","average":{"1min":2.4700000000000002,"5min":4.6959999999999997,"15min":2.4039999999999999},"min":{"1min":0.33700000000000002,"5min":0.26200000000000001,"15min":0.24299999999999999},"max":{"1min":10.180999999999999,"5min":187.369,"15min":187.369},"last":0.76800000000000002},{"interface":"front","average":{"1min":2.4430000000000001,"5min":4.9729999999999999,"15min":2.5310000000000001},"min":{"1min":0.44400000000000001,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":10.189,"5min":187.35300000000001,"15min":187.35300000000001},"last":1.2210000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:43:22 2023","interfaces":[{"interface":"back","average":{"1min":5.7320000000000002,"5min":4.4169999999999998,"15min":4.3929999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":71.748000000000005,"5min":100.148,"15min":196.93700000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":5.4530000000000003,"5min":4.2729999999999997,"15min":4.2969999999999997},"min":{"1min":0.33800000000000002,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":72.096000000000004,"5min":100.086,"15min":197.08099999999999},"last":0.39100000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:43:48 2023","interfaces":[{"interface":"back","average":{"1min":2.516,"5min":4.8250000000000002,"15min":3.9249999999999998},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":9.5129999999999999,"5min":72.391999999999996,"15min":211.48699999999999},"last":1.2470000000000001},{"interface":"front","average":{"1min":2.54,"5min":4.6299999999999999,"15min":3.79},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.309},"max":{"1min":12.795999999999999,"5min":71.721999999999994,"15min":211.31100000000001},"last":0.32500000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:43:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":0.60599999999999998,"15min":1.5820000000000001},"min":{"1min":0.29399999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.1320000000000001,"5min":3.5259999999999998,"15min":113.35899999999999},"last":1.1930000000000001},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.58599999999999997,"15min":1.522},"min":{"1min":0.35899999999999999,"5min":0.317,"15min":0.30599999999999999},"max":{"1min":3.1179999999999999,"5min":3.3849999999999998,"15min":113.32299999999999},"last":0.35199999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:44:03 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.54800000000000004,"15min":0.71899999999999997},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":3.3969999999999998,"5min":3.4390000000000001,"15min":11.944000000000001},"last":2.379},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.72899999999999998,"15min":0.879},"min":{"1min":0.33500000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.7120000000000002,"5min":7.6500000000000004,"15min":10.356},"last":8.5269999999999992}]},{"osd":7,"last update":"Mon Oct 9 05:44:03 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.64600000000000002,"15min":2.0129999999999999},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.312},"max":{"1min":2.7709999999999999,"5min":4.5819999999999999,"15min":30.393999999999998},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.69899999999999995,"15min":2.2429999999999999},"min":{"1min":0.372,"5min":0.34899999999999998,"15min":0.32300000000000001},"max":{"1min":3.3050000000000002,"5min":4.5999999999999996,"15min":30.334},"last":0.69399999999999995}]}]},{"osd":5,"up_from":36,"seq":154618822961,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1639088,"kb_used_data":1337768,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92106064,"statfs":{"total":95995035648,"available":94316609536,"internally_reserved":0,"allocated":1369874432,"data_stored":1369340921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:43:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":1.6930000000000001,"15min":3.1309999999999998},"min":{"1min":0.253,"5min":0.217,"15min":0.19400000000000001},"max":{"1min":9.2159999999999993,"5min":10.712999999999999,"15min":165.90199999999999},"last":0.255},{"interface":"front","average":{"1min":2.222,"5min":1.639,"15min":3.1909999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.215},"max":{"1min":8.2539999999999996,"5min":8.6099999999999994,"15min":165.96700000000001},"last":0.503}]},{"osd":1,"last update":"Mon Oct 9 05:43:52 2023","interfaces":[{"interface":"back","average":{"1min":5.1449999999999996,"5min":4,"15min":2.5640000000000001},"min":{"1min":0.26500000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":54.898000000000003,"5min":71.209999999999994,"15min":87.769000000000005},"last":0.32600000000000001},{"interface":"front","average":{"1min":4.673,"5min":4.5119999999999996,"15min":2.6549999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.24399999999999999},"max":{"1min":52.512,"5min":71.192999999999998,"15min":87.864000000000004},"last":0.248}]},{"osd":2,"last update":"Mon Oct 9 05:43:29 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":3.847,"15min":4.2949999999999999},"min":{"1min":0.32800000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":31.114000000000001,"5min":73.311999999999998,"15min":290.697},"last":1.153},{"interface":"front","average":{"1min":3.3380000000000001,"5min":3.6349999999999998,"15min":4.1829999999999998},"min":{"1min":0.378,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":33.875999999999998,"5min":73.280000000000001,"15min":290.786},"last":0.35999999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:43:52 2023","interfaces":[{"interface":"back","average":{"1min":4.109,"5min":6.6200000000000001,"15min":3.359},"min":{"1min":0.34899999999999998,"5min":0.30399999999999999,"15min":0.28799999999999998},"max":{"1min":30.545000000000002,"5min":95.236999999999995,"15min":95.236999999999995},"last":1.405},{"interface":"front","average":{"1min":4.859,"5min":6.46,"15min":3.2250000000000001},"min":{"1min":0.372,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":33.478000000000002,"5min":93.471000000000004,"15min":93.471000000000004},"last":0.46600000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:43:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":0.69199999999999995,"15min":0.76200000000000001},"min":{"1min":0.31900000000000001,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":6.2610000000000001,"5min":6.2610000000000001,"15min":9.3510000000000009},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.28,"5min":0.68600000000000005,"15min":0.99299999999999999},"min":{"1min":0.308,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":9.282,"5min":9.282,"15min":14.236000000000001},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:44:09 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":1.0600000000000001,"15min":1.4159999999999999},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.26600000000000001},"max":{"1min":9.2959999999999994,"5min":27.146000000000001,"15min":95.010999999999996},"last":0.442},{"interface":"front","average":{"1min":1.4410000000000001,"5min":1.1519999999999999,"15min":1.331},"min":{"1min":0.38700000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.5410000000000004,"5min":27.215,"15min":94.194000000000003},"last":0.51200000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:44:09 2023","interfaces":[{"interface":"back","average":{"1min":1.776,"5min":0.97699999999999998,"15min":1.405},"min":{"1min":0.36499999999999999,"5min":0.34399999999999997,"15min":0.28999999999999998},"max":{"1min":9.1460000000000008,"5min":9.1460000000000008,"15min":9.9849999999999994},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.585,"5min":0.90700000000000003,"15min":1.5629999999999999},"min":{"1min":0.377,"5min":0.308,"15min":0.308},"max":{"1min":9.3100000000000005,"5min":9.3100000000000005,"15min":14.432},"last":0.495}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1993019392,"data_stored":1993015320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1840844800,"data_stored":1840840728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1876033536,"data_stored":1876033536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2618359808,"data_stored":2618359808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1630470144,"data_stored":1630470144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1367670784,"data_stored":1367670784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1246433280,"data_stored":1246429208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":14155776,"data_stored":14155776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:44:15.664 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 21 2023-10-09T05:44:15.684 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:15.715 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:15.771 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:15.776 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:15.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:15.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:15.884 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:15.920 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:15.929 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:16.114 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:16.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:16.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:16.218 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:16.257 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:16.959 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:16.978 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:16.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:16.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:17.082 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:17.132 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:17.316 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:17 smithi008 ceph-mon[191227]: pgmap v717: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 20919/181986 objects degraded (11.495%); 5309/181986 objects misplaced (2.917%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T05:44:17.623 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:44:17.648 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:17 smithi008 ceph-mon[184418]: pgmap v717: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 20919/181986 objects degraded (11.495%); 5309/181986 objects misplaced (2.917%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T05:44:17.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:17 smithi191 ceph-mon[161669]: pgmap v717: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 20919/181986 objects degraded (11.495%); 5309/181986 objects misplaced (2.917%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T05:44:18.052 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:18.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:18.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:18.156 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:18.214 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:18.404 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:18.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:18.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:18.510 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:18.550 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:18.578 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20919/181986 objects degraded (11.495%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:18.579 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20919/181986 objects degraded (11.495%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:18.614 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:18.622 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20919/181986 objects degraded (11.495%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:18.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:18.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:18.724 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:18.783 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:18.855 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:44:18.855 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:44:18.855 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T05:44:18.855 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:44:18.861 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:18.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:18.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:18.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:19.001 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:19.062 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:19.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:19.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:19.171 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:19.204 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:19.255 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:19.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:19.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:19.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:19.394 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:19.457 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:19.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:19.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:19.567 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:19.599 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:19.901 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:44:20.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:19 smithi008 ceph-mon[184418]: pgmap v718: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 302 op/s; 20800/180264 objects degraded (11.539%); 5309/180264 objects misplaced (2.945%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T05:44:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:19 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1296355668' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:20.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:19 smithi008 ceph-mon[191227]: pgmap v718: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 302 op/s; 20800/180264 objects degraded (11.539%); 5309/180264 objects misplaced (2.945%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T05:44:20.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:19 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1296355668' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:20.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:19 smithi191 ceph-mon[161669]: pgmap v718: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 302 op/s; 20800/180264 objects degraded (11.539%); 5309/180264 objects misplaced (2.945%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T05:44:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:19 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1296355668' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:20.521 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:20.557 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:44:20.590 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:20.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:20.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:20.696 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:20.724 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:20.899 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:44:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:44:20] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-09T05:44:21.180 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:21.180 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[184418]: failed to encode map e243 with expected crc 2023-10-09T05:44:21.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[184418]: failed to encode map e243 with expected crc 2023-10-09T05:44:21.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:44:21.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[184418]: osdmap e243: 8 total, 8 up, 8 in 2023-10-09T05:44:21.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[184418]: failed to encode map e243 with expected crc 2023-10-09T05:44:21.181 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:21.181 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[191227]: failed to encode map e243 with expected crc 2023-10-09T05:44:21.181 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[191227]: failed to encode map e243 with expected crc 2023-10-09T05:44:21.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:44:21.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[191227]: osdmap e243: 8 total, 8 up, 8 in 2023-10-09T05:44:21.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:20 smithi008 ceph-mon[191227]: failed to encode map e243 with expected crc 2023-10-09T05:44:21.244 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:21.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:21.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:20 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:20 smithi191 ceph-mon[161669]: failed to encode map e243 with expected crc 2023-10-09T05:44:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:20 smithi191 ceph-mon[161669]: failed to encode map e243 with expected crc 2023-10-09T05:44:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:20 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:44:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:20 smithi191 ceph-mon[161669]: osdmap e243: 8 total, 8 up, 8 in 2023-10-09T05:44:21.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:20 smithi191 ceph-mon[161669]: failed to encode map e243 with expected crc 2023-10-09T05:44:21.349 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:21.392 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:21.558 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:21.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:21.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:21.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:21.699 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:21.781 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:21.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:21.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:21.889 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:21.934 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:22.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:21 smithi191 ceph-mon[161669]: pgmap v720: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 357 op/s; 20612/177336 objects degraded (11.623%); 5244/177336 objects misplaced (2.957%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T05:44:22.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:21 smithi008 ceph-mon[184418]: pgmap v720: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 357 op/s; 20612/177336 objects degraded (11.623%); 5244/177336 objects misplaced (2.957%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T05:44:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:21 smithi008 ceph-mon[191227]: pgmap v720: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 357 op/s; 20612/177336 objects degraded (11.623%); 5244/177336 objects misplaced (2.957%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T05:44:22.561 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:44:22.592 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:22.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:22.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:22.697 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:22.733 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:22.790 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:22.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:22.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:22.895 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:22.922 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:22.998 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:23.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:22 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1593453793' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:44:23.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:22 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20612/177336 objects degraded (11.623%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:23.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:23.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:23.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:23.150 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:23.203 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:23.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:23.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:23.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:22 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1593453793' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:44:23.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:22 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20612/177336 objects degraded (11.623%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:23.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:22 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1593453793' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:44:23.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:22 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20612/177336 objects degraded (11.623%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:44:23.290 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:23.297 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:44:23.309 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:23.327 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:44:23.401 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:23.713 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:23.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:23.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:23.771 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:44:23.771 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:44:23.771 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:44:23.783 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:44:23.804 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:44:23.818 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:23.832 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:44:23.857 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:23.858 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:44:23.890 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:44:23.893 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:44:23.894 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:44:23.894 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:44:23.894 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:44:23.894 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:44:23.894 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:44:23.894 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:44:23.894 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:44:23.922 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:23.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:23.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:24.026 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:24.052 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:23 smithi008 ceph-mon[184418]: pgmap v721: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 20517/174591 objects degraded (11.751%); 5014/174591 objects misplaced (2.872%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T05:44:24.052 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2140449787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:24.052 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:23 smithi008 ceph-mon[191227]: pgmap v721: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 20517/174591 objects degraded (11.751%); 5014/174591 objects misplaced (2.872%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T05:44:24.052 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2140449787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:24.054 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:24.112 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:24.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:24.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:24.216 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:24.248 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:23 smithi191 ceph-mon[161669]: pgmap v721: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 20517/174591 objects degraded (11.751%); 5014/174591 objects misplaced (2.872%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T05:44:24.248 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2140449787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:24.252 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:24.310 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:24.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:24.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:24.418 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:24.446 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:25.185 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:25.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:25.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:25.288 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:25.317 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:25.390 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:25.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:25.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:25.502 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:25.530 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:25.620 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:25.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:25.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:25.654 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:25.725 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:25.754 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:25.817 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:25.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:25.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:25.925 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:25.959 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:26.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:25 smithi191 ceph-mon[161669]: pgmap v722: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 369 op/s; 20408/172875 objects degraded (11.805%); 5014/172875 objects misplaced (2.900%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:44:26.169 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:25 smithi008 ceph-mon[184418]: pgmap v722: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 369 op/s; 20408/172875 objects degraded (11.805%); 5014/172875 objects misplaced (2.900%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:44:26.169 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:25 smithi008 ceph-mon[191227]: pgmap v722: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 369 op/s; 20408/172875 objects degraded (11.805%); 5014/172875 objects misplaced (2.900%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:44:26.762 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:44:26.763 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:44:26.774 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:26.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:26.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:26.879 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:26.919 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:27.314 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:27.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:27.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:27.407 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:44:27.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:27.453 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:27.528 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:27.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:27.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:27.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:27.677 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:27.764 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:27.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:27.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:27.802 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:44:27.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:27.911 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:27.966 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:27.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:27.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:28.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:27 smithi191 ceph-mon[161669]: pgmap v723: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 20348/170949 objects degraded (11.903%); 4720/170949 objects misplaced (2.761%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:27 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20348/170949 objects degraded (11.903%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:28.071 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:28.104 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:28.162 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:28.163 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:28.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:28.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:28.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:27 smithi008 ceph-mon[184418]: pgmap v723: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 20348/170949 objects degraded (11.903%); 4720/170949 objects misplaced (2.761%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:28.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:27 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20348/170949 objects degraded (11.903%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:28.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:27 smithi008 ceph-mon[191227]: pgmap v723: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 20348/170949 objects degraded (11.903%); 4720/170949 objects misplaced (2.761%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:28.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:27 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20348/170949 objects degraded (11.903%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:28.269 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:28.329 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:44:28.329 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:44:28.329 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:44:28.330 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 1.000000 2023-10-09T05:44:28.330 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T05:44:28.330 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:28.489 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:28.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:28.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:28.604 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:28.642 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:28.651 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:28.920 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3052408627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:44:28.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:28 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:44:28.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:28 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:28.931 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:44:29.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3052408627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:44:29.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:28 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:44:29.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:28 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:29.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3052408627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:44:29.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:28 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:44:29.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:28 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:29.599 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:29.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:29.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:29.706 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:29.744 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:29 smithi008 ceph-mon[184418]: pgmap v724: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 20240/169344 objects degraded (11.952%); 4720/169344 objects misplaced (2.787%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:44:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:29 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:29 smithi008 ceph-mon[184418]: failed to encode map e244 with expected crc 2023-10-09T05:44:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:29 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:44:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:29 smithi008 ceph-mon[184418]: osdmap e244: 8 total, 8 up, 8 in 2023-10-09T05:44:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:29 smithi008 ceph-mon[191227]: pgmap v724: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 20240/169344 objects degraded (11.952%); 4720/169344 objects misplaced (2.787%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:44:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:29 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:29 smithi008 ceph-mon[191227]: failed to encode map e244 with expected crc 2023-10-09T05:44:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:29 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:44:30.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:29 smithi008 ceph-mon[191227]: osdmap e244: 8 total, 8 up, 8 in 2023-10-09T05:44:30.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:29 smithi191 ceph-mon[161669]: pgmap v724: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 20240/169344 objects degraded (11.952%); 4720/169344 objects misplaced (2.787%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:44:30.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:29 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:30.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:29 smithi191 ceph-mon[161669]: failed to encode map e244 with expected crc 2023-10-09T05:44:30.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:29 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:44:30.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:29 smithi191 ceph-mon[161669]: osdmap e244: 8 total, 8 up, 8 in 2023-10-09T05:44:30.383 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:30.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:30.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:30.489 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:30.532 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:30.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:44:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:44:30] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T05:44:30.787 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:30.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:30.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:30.860 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:30.894 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:30.898 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:44:30.928 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:44:30.941 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:30.964 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:44:31.001 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:44:31.029 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:44:31.054 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:44:31.061 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:31.065 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:30 smithi008 ceph-mon[184418]: failed to encode map e244 with expected crc 2023-10-09T05:44:31.065 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:30 smithi008 ceph-mon[184418]: failed to encode map e244 with expected crc 2023-10-09T05:44:31.066 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2480337731' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:31.066 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:30 smithi008 ceph-mon[191227]: failed to encode map e244 with expected crc 2023-10-09T05:44:31.066 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:30 smithi008 ceph-mon[191227]: failed to encode map e244 with expected crc 2023-10-09T05:44:31.066 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2480337731' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:31.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:31.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:31.078 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:44:31.097 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:44:31.099 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:44:31.099 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:44:31.099 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:44:31.099 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:44:31.099 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:44:31.099 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:44:31.099 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:44:31.099 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:44:31.168 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:31.194 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:30 smithi191 ceph-mon[161669]: failed to encode map e244 with expected crc 2023-10-09T05:44:31.194 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:30 smithi191 ceph-mon[161669]: failed to encode map e244 with expected crc 2023-10-09T05:44:31.194 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2480337731' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:31.197 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:31.250 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:31.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:31.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:31.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:31.385 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:31.445 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:31.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:31.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:31.550 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:31.584 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:31.875 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:31.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:31.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:31.979 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:32.006 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:32.069 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:32.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:32.176 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:32.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:31 smithi008 ceph-mon[184418]: pgmap v726: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 20080/166872 objects degraded (12.033%); 4664/166872 objects misplaced (2.795%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:44:32.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:31 smithi008 ceph-mon[191227]: pgmap v726: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 20080/166872 objects degraded (12.033%); 4664/166872 objects misplaced (2.795%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:44:32.210 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:32.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:31 smithi191 ceph-mon[161669]: pgmap v726: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 20080/166872 objects degraded (12.033%); 4664/166872 objects misplaced (2.795%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T05:44:32.504 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:44:32.669 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:32.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:32.773 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:32.833 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:33.097 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:32 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:33.097 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:32 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20080/166872 objects degraded (12.033%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:33.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:32 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:33.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:32 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20080/166872 objects degraded (12.033%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:33.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:32 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:33.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:32 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20080/166872 objects degraded (12.033%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:33.538 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:44:33.538 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:44:33.538 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:44:33.538 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:44:33.538 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.538 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140594 6.9 6:972114e9:::benchmark_data_smithi008_295510_object52231:head [delete] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-10-09T05:44:33.539 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.801636+0000", 2023-10-09T05:44:33.539 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 269.59841966200003, 2023-10-09T05:44:33.539 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.041270468, 2023-10-09T05:44:33.539 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.539 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.539 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.539 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:33.539 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:33.539 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140594 2023-10-09T05:44:33.539 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.540 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.540 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.540 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.540 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.801636+0000", 2023-10-09T05:44:33.540 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.540 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.540 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.540 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.540 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.801636+0000", 2023-10-09T05:44:33.540 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.540 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.541 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.541 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.801635+0000", 2023-10-09T05:44:33.541 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:33.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.541 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.541 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.801642+0000", 2023-10-09T05:44:33.541 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7689999999999996e-06 2023-10-09T05:44:33.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.542 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.825379+0000", 2023-10-09T05:44:33.542 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.023737011999999998 2023-10-09T05:44:33.542 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.542 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.825394+0000", 2023-10-09T05:44:33.542 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4861999999999999e-05 2023-10-09T05:44:33.542 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.543 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.543 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.543 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.835389+0000", 2023-10-09T05:44:33.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0099948799999999994 2023-10-09T05:44:33.543 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.543 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.543 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.543 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.835401+0000", 2023-10-09T05:44:33.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1549000000000001e-05 2023-10-09T05:44:33.543 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.543 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.544 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.544 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.826729+0000", 2023-10-09T05:44:33.544 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99132818599999994 2023-10-09T05:44:33.544 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.544 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.544 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.544 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.826827+0000", 2023-10-09T05:44:33.544 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7775999999999994e-05 2023-10-09T05:44:33.544 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.544 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.545 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:33.545 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.826945+0000", 2023-10-09T05:44:33.545 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000118439 2023-10-09T05:44:33.545 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.545 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.545 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.545 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.830778+0000", 2023-10-09T05:44:33.545 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038329269999999999 2023-10-09T05:44:33.545 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.545 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.545 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.546 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842771+0000", 2023-10-09T05:44:33.546 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011992505000000001 2023-10-09T05:44:33.546 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.546 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.546 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.546 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842811+0000", 2023-10-09T05:44:33.546 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0772000000000002e-05 2023-10-09T05:44:33.546 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.546 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.546 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.547 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842865+0000", 2023-10-09T05:44:33.547 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3745000000000001e-05 2023-10-09T05:44:33.547 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.547 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.547 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.547 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842907+0000", 2023-10-09T05:44:33.547 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1482e-05 2023-10-09T05:44:33.547 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.547 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.547 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.548 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.548 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.548 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140616 6.9 6:92cdb345:::benchmark_data_smithi008_295510_object52253:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:44:33.548 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.868684+0000", 2023-10-09T05:44:33.548 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 269.53137132699999, 2023-10-09T05:44:33.548 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97444081699999996, 2023-10-09T05:44:33.548 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.548 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.549 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.549 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:33.549 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:33.549 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140616 2023-10-09T05:44:33.549 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.549 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.549 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.549 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868684+0000", 2023-10-09T05:44:33.549 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.550 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.550 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.550 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.550 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868684+0000", 2023-10-09T05:44:33.550 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.550 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.550 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.550 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.550 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868683+0000", 2023-10-09T05:44:33.550 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:33.551 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.551 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.551 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.551 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868690+0000", 2023-10-09T05:44:33.551 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.849e-06 2023-10-09T05:44:33.551 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.551 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.551 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.551 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868702+0000", 2023-10-09T05:44:33.551 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2642e-05 2023-10-09T05:44:33.551 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868711+0000", 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1200000000000002e-06 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868775+0000", 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4764999999999994e-05 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.552 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.868787+0000", 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1617000000000001e-05 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.830809+0000", 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96202192200000003 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.830938+0000", 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012940799999999999 2023-10-09T05:44:33.553 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.831034+0000", 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5957000000000003e-05 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842752+0000", 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011717455 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.554 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.842996+0000", 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00024453700000000002 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843074+0000", 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8015000000000004e-05 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843102+0000", 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7381000000000001e-05 2023-10-09T05:44:33.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.557 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.558 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.558 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843125+0000", 2023-10-09T05:44:33.558 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3686e-05 2023-10-09T05:44:33.558 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.558 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.559 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.559 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.559 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.559 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140627 6.9 6:9a23699b:::benchmark_data_smithi008_295510_object52264:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:44:33.559 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.891296+0000", 2023-10-09T05:44:33.559 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 269.50875931899998, 2023-10-09T05:44:33.559 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.95198831100000003, 2023-10-09T05:44:33.560 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.560 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.560 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.560 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:33.560 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:33.560 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140627 2023-10-09T05:44:33.560 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.561 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.561 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.561 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.561 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891296+0000", 2023-10-09T05:44:33.561 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.561 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.562 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.562 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.562 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891296+0000", 2023-10-09T05:44:33.562 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.563 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.563 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.563 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.563 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891294+0000", 2023-10-09T05:44:33.563 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:33.563 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.564 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.564 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.564 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891305+0000", 2023-10-09T05:44:33.564 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1089e-05 2023-10-09T05:44:33.564 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.564 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.564 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.565 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891322+0000", 2023-10-09T05:44:33.565 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6232e-05 2023-10-09T05:44:33.565 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.565 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.565 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.565 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891333+0000", 2023-10-09T05:44:33.565 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1722999999999999e-05 2023-10-09T05:44:33.566 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.566 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.566 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.566 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891404+0000", 2023-10-09T05:44:33.566 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0164000000000001e-05 2023-10-09T05:44:33.567 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.567 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.567 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.567 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.891415+0000", 2023-10-09T05:44:33.567 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1926000000000001e-05 2023-10-09T05:44:33.567 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.568 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.568 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.568 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.834610+0000", 2023-10-09T05:44:33.569 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94319403899999998 2023-10-09T05:44:33.570 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.570 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.570 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.570 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.834746+0000", 2023-10-09T05:44:33.570 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013667 2023-10-09T05:44:33.570 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.570 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.570 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:33.570 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.834874+0000", 2023-10-09T05:44:33.570 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012732499999999999 2023-10-09T05:44:33.570 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.571 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.571 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.571 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843048+0000", 2023-10-09T05:44:33.571 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081748810000000002 2023-10-09T05:44:33.571 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.571 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.571 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.571 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843142+0000", 2023-10-09T05:44:33.571 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3560999999999994e-05 2023-10-09T05:44:33.571 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.571 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.572 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.572 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843196+0000", 2023-10-09T05:44:33.572 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4468000000000001e-05 2023-10-09T05:44:33.572 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.572 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.572 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.572 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843248+0000", 2023-10-09T05:44:33.572 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1266e-05 2023-10-09T05:44:33.572 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.572 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843285+0000", 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7061000000000003e-05 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140639 6.9 6:91830589:::benchmark_data_smithi008_295510_object52276:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.911940+0000", 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 269.48811542800001, 2023-10-09T05:44:33.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94121215400000002, 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140639 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911940+0000", 2023-10-09T05:44:33.574 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911940+0000", 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911940+0000", 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911943+0000", 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7889999999999999e-06 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911949+0000", 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6769999999999998e-06 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.576 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.911955+0000", 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.015e-06 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.912210+0000", 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000254338 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.912220+0000", 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0635999999999999e-05 2023-10-09T05:44:33.577 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.578 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.839273+0000", 2023-10-09T05:44:33.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92705263999999998 2023-10-09T05:44:33.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.578 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.839367+0000", 2023-10-09T05:44:33.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4503e-05 2023-10-09T05:44:33.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.839438+0000", 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0965999999999998e-05 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843330+0000", 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038915070000000002 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853009+0000", 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0096787590000000003 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853053+0000", 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4736e-05 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853105+0000", 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1403999999999998e-05 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.581 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.581 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853152+0000", 2023-10-09T05:44:33.581 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7645999999999997e-05 2023-10-09T05:44:33.581 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.581 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.581 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.581 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.581 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.581 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140645 6.9 6:9412362f:::benchmark_data_smithi008_295510_object52282:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:44:33.581 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.933367+0000", 2023-10-09T05:44:33.581 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 269.46668838099998, 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93559094200000004, 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140645 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.582 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933367+0000", 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933367+0000", 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933365+0000", 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933375+0000", 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6840000000000008e-06 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933396+0000", 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1433999999999998e-05 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933408+0000", 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1389999999999999e-05 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933515+0000", 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000107681 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.933528+0000", 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2191000000000001e-05 2023-10-09T05:44:33.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843353+0000", 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90982495399999996 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843428+0000", 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5394000000000004e-05 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.843498+0000", 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9961000000000003e-05 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853172+0000", 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0096736459999999993 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868740+0000", 2023-10-09T05:44:33.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015568584999999999 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868872+0000", 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013186599999999999 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868919+0000", 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7134e-05 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868958+0000", 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9171000000000002e-05 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140663 6.9 6:9f610fbc:::benchmark_data_smithi008_295510_object52300:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.969184+0000", 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 269.430871653, 2023-10-09T05:44:33.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89992256199999998, 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140663 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969184+0000", 2023-10-09T05:44:33.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969184+0000", 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969182+0000", 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969193+0000", 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0937e-05 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969215+0000", 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2320999999999999e-05 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969226+0000", 2023-10-09T05:44:33.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0558e-05 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969298+0000", 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2253000000000001e-05 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.969312+0000", 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3594999999999999e-05 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.848260+0000", 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.87894824199999999 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.848390+0000", 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013013200000000001 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:33.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.848515+0000", 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012543100000000001 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.857837+0000", 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0093214249999999995 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868779+0000", 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010942062000000001 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.596 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.596 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.869018+0000", 2023-10-09T05:44:33.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00023956199999999999 2023-10-09T05:44:33.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.596 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.596 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.869067+0000", 2023-10-09T05:44:33.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8427000000000002e-05 2023-10-09T05:44:33.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.596 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.869107+0000", 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9813e-05 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:140674 6.9 6:9fbec6cb:::benchmark_data_smithi008_295510_object52311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:03.981500+0000", 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 269.41855551499998, 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.91512313499999998, 2023-10-09T05:44:33.597 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 140674 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981500+0000", 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981500+0000", 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981498+0000", 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.599 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981508+0000", 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7890000000000006e-06 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981528+0000", 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0392999999999999e-05 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981538+0000", 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0149999999999999e-05 2023-10-09T05:44:33.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981604+0000", 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5709000000000004e-05 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:03.981615+0000", 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1090999999999999e-05 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.601 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853195+0000", 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.87157961399999995 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853323+0000", 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012834899999999999 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.853450+0000", 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012675300000000001 2023-10-09T05:44:33.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868789+0000", 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015338991999999999 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.868838+0000", 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9543999999999999e-05 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896521+0000", 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.027682469000000001 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896573+0000", 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1724000000000001e-05 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:04.896623+0000", 2023-10-09T05:44:33.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0781000000000001e-05 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102132 7.6 7:6028800a:::benchmark_data_smithi008_354649_object7636:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.878610+0000", 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 74.521445851999999, 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3895780630000001, 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.605 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102132 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878610+0000", 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878610+0000", 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878608+0000", 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878615+0000", 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6379999999999998e-06 2023-10-09T05:44:33.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878633+0000", 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7983999999999999e-05 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878642+0000", 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5200000000000003e-06 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.879024+0000", 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00038146000000000002 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.879140+0000", 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000116616 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.879378+0000", 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00023714499999999999 2023-10-09T05:44:33.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887275+0000", 2023-10-09T05:44:33.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078973129999999996 2023-10-09T05:44:33.610 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905106+0000", 2023-10-09T05:44:33.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017830736 2023-10-09T05:44:33.610 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905119+0000", 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3549e-05 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877502+0000", 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97238259000000005 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877512+0000", 2023-10-09T05:44:33.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0355e-05 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.239697+0000", 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3621850200000001 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.239848+0000", 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015051299999999999 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268057+0000", 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028209238000000001 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268188+0000", 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013106500000000001 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102135 7.6 7:616fc77f:::benchmark_data_smithi008_354649_object7639:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.878660+0000", 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 74.521395764999994, 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4043974380000002, 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102135 2023-10-09T05:44:33.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878660+0000", 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878660+0000", 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878659+0000", 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878663+0000", 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.242e-06 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878670+0000", 2023-10-09T05:44:33.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0779999999999998e-06 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878676+0000", 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9320000000000001e-06 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.883274+0000", 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045982369999999998 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.883418+0000", 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014399399999999999 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.883540+0000", 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000122225 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.618 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887783+0000", 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042433180000000003 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905157+0000", 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017373588999999998 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905174+0000", 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6509999999999999e-05 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877554+0000", 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97238062300000006 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877564+0000", 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6269999999999995e-06 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243556+0000", 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3659919819999999 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243708+0000", 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015254099999999999 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.282943+0000", 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.039234640000000001 2023-10-09T05:44:33.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283057+0000", 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114479 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102138 7.6 7:69d2abd0:::benchmark_data_smithi008_354649_object7642:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:44:33.622 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 74.521374550999994, 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3933153759999999, 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102138 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878684+0000", 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9849999999999998e-06 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878689+0000", 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6910000000000002e-06 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878697+0000", 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2899999999999997e-06 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887233+0000", 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0085366939999999992 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op must wait for map", 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887240+0000", 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0169999999999999e-06 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905215+0000", 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017974716000000002 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905223+0000", 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7360000000000001e-06 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877592+0000", 2023-10-09T05:44:33.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97236955700000005 2023-10-09T05:44:33.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877601+0000", 2023-10-09T05:44:33.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2449999999999998e-06 2023-10-09T05:44:33.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243738+0000", 2023-10-09T05:44:33.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.366137529 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243861+0000", 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012260400000000001 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.244012+0000", 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015064899999999999 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.257726+0000", 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013714637999999999 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268231+0000", 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010504434999999999 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268269+0000", 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7939999999999999e-05 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268316+0000", 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7679000000000002e-05 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.271997+0000", 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036803859999999999 2023-10-09T05:44:33.631 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102144 7.6 7:6ad058ac:::benchmark_data_smithi008_354649_object7648:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.899418+0000", 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 74.500637800000007, 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3874670130000002, 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.632 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102144 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899418+0000", 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899418+0000", 2023-10-09T05:44:33.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899417+0000", 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899422+0000", 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7659999999999998e-06 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899430+0000", 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6100000000000006e-06 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899438+0000", 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0760000000000003e-06 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905297+0000", 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058583560000000003 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905305+0000", 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0090000000000007e-06 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877625+0000", 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.972320462 2023-10-09T05:44:33.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877633+0000", 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8339999999999999e-06 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.250892+0000", 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.373259207 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.251058+0000", 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016594400000000001 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.251234+0000", 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000175453 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268021+0000", 2023-10-09T05:44:33.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016787834000000001 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283099+0000", 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015077174 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283181+0000", 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2218000000000005e-05 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283226+0000", 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5181e-05 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.286885+0000", 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036590279999999999 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102153 7.6 7:672a3339:::benchmark_data_smithi008_354649_object7657:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.906435+0000", 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 74.493620380999999, 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3840327299999999, 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102153 2023-10-09T05:44:33.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906435+0000", 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906435+0000", 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906433+0000", 2023-10-09T05:44:33.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:33.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906441+0000", 2023-10-09T05:44:33.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4569999999999999e-06 2023-10-09T05:44:33.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906458+0000", 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7399000000000001e-05 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906470+0000", 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2092e-05 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906521+0000", 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0983999999999998e-05 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906533+0000", 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1977000000000001e-05 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877706+0000", 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97117302299999997 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877717+0000", 2023-10-09T05:44:33.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0254e-05 2023-10-09T05:44:33.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.257761+0000", 2023-10-09T05:44:33.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3800444709999999 2023-10-09T05:44:33.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.257940+0000", 2023-10-09T05:44:33.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000179037 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.258090+0000", 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000150161 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268184+0000", 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010093352999999999 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283137+0000", 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014953404999999999 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.286959+0000", 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038219339999999999 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.287005+0000", 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.549e-05 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.290468+0000", 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034635989999999999 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.649 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102170 7.f 7:f2cf7216:::benchmark_data_smithi008_354649_object7674:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.931713+0000", 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 74.468343020999995, 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.30407954, 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102170 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931713+0000", 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931713+0000", 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931711+0000", 2023-10-09T05:44:33.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931717+0000", 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4029999999999998e-06 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931726+0000", 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7409999999999998e-06 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931731+0000", 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3310000000000003e-06 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931790+0000", 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9153000000000001e-05 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931804+0000", 2023-10-09T05:44:33.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4075e-05 2023-10-09T05:44:33.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.876813+0000", 2023-10-09T05:44:33.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94500846400000005 2023-10-09T05:44:33.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.876824+0000", 2023-10-09T05:44:33.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.184e-05 2023-10-09T05:44:33.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.222418+0000", 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3455932580000001 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.222572+0000", 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000154277 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:44:33.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.222670+0000", 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8070000000000001e-05 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230124+0000", 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007453515 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230200+0000", 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6130000000000005e-05 2023-10-09T05:44:33.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.231464+0000", 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0012646619999999999 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.231507+0000", 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2215000000000002e-05 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.235792+0000", 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042857040000000004 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102149 7.5 7:a4fc93e5:::benchmark_data_smithi008_354649_object7653:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e231)", 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:19.870929+0000", 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 73.529126348000005, 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.387032467, 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102149 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870929+0000", 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870929+0000", 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870927+0000", 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870938+0000", 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0886e-05 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870958+0000", 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9826e-05 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870968+0000", 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8220000000000002e-06 2023-10-09T05:44:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.871028+0000", 2023-10-09T05:44:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9914999999999998e-05 2023-10-09T05:44:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op must wait for map", 2023-10-09T05:44:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.871035+0000", 2023-10-09T05:44:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4850000000000003e-06 2023-10-09T05:44:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.878208+0000", 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0071734060000000002 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.878217+0000", 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7299999999999994e-06 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.240613+0000", 2023-10-09T05:44:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3623961600000001 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243975+0000", 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033618929999999999 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.244130+0000", 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015506600000000001 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.254399+0000", 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.01026909 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.254418+0000", 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8729999999999999e-05 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.254469+0000", 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1286000000000003e-05 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.254527+0000", 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7803000000000002e-05 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.257962+0000", 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034347330000000001 2023-10-09T05:44:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102151 7.5 7:a063756d:::benchmark_data_smithi008_354649_object7655:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e231)", 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:19.870989+0000", 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 73.529067022000007, 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.399485726, 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102151 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870989+0000", 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870989+0000", 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870988+0000", 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870992+0000", 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6069999999999999e-06 2023-10-09T05:44:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870999+0000", 2023-10-09T05:44:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4399999999999999e-06 2023-10-09T05:44:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.871005+0000", 2023-10-09T05:44:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9030000000000001e-06 2023-10-09T05:44:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.871111+0000", 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010532899999999999 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.871120+0000", 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4700000000000008e-06 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.880924+0000", 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0098035310000000007 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.880934+0000", 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0684e-05 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.251116+0000", 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3701822880000001 2023-10-09T05:44:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.251264+0000", 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014727800000000001 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,7", 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.251402+0000", 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000138021 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.261774+0000", 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010372484 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.265468+0000", 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00369346 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.266407+0000", 2023-10-09T05:44:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00093907699999999997 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.266455+0000", 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7718e-05 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.270474+0000", 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040199609999999998 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125678 7.6 7:63397418:::benchmark_data_smithi008_354649_object31182:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.999075+0000", 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 30.400981229999999, 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0649156319999999, 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125678 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999075+0000", 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999075+0000", 2023-10-09T05:44:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999073+0000", 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999080+0000", 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3900000000000004e-06 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999095+0000", 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4596999999999999e-05 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999108+0000", 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2767e-05 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006313+0000", 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0072051479999999998 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op must wait for map", 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006322+0000", 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6799999999999999e-06 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.383274+0000", 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37695257599999998 2023-10-09T05:44:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.383295+0000", 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0871999999999999e-05 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.969340+0000", 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.58604520999999998 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.969352+0000", 2023-10-09T05:44:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1914e-05 2023-10-09T05:44:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.033375+0000", 2023-10-09T05:44:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0640223849999999 2023-10-09T05:44:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:44:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.033498+0000", 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000123614 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.063936+0000", 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030438004000000001 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.063990+0000", 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3983000000000001e-05 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125681 7.19 7:9ae10a2c:::benchmark_data_smithi008_354649_object31185:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.999127+0000", 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 30.400928381, 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.044541599, 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125681 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999127+0000", 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999127+0000", 2023-10-09T05:44:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999127+0000", 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999130+0000", 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.382e-06 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999138+0000", 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7389999999999999e-06 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999145+0000", 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7070000000000003e-06 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.494281+0000", 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.49513664899999998 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.494291+0000", 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4739999999999994e-06 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.043589+0000", 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.54929776399999997 2023-10-09T05:44:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.043669+0000", 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0393000000000001e-05 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125687 7.9 7:949be08c:::benchmark_data_smithi008_354649_object31191:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.999151+0000", 2023-10-09T05:44:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 30.400904429000001, 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0448235699999999, 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125687 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999151+0000", 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999151+0000", 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999151+0000", 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999154+0000", 2023-10-09T05:44:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2389999999999998e-06 2023-10-09T05:44:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999198+0000", 2023-10-09T05:44:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3470000000000002e-05 2023-10-09T05:44:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999206+0000", 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8509999999999999e-06 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.005680+0000", 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064743550000000002 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op must wait for map", 2023-10-09T05:44:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.005689+0000", 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6400000000000003e-06 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.494360+0000", 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48867183800000003 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.494368+0000", 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6660000000000008e-06 2023-10-09T05:44:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.022404+0000", 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.52803634200000005 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.022576+0000", 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017137200000000001 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.043916+0000", 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021340650999999999 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.043975+0000", 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8510000000000001e-05 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125695 7.6 7:62e65488:::benchmark_data_smithi008_354649_object31199:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T05:44:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:03.513022+0000", 2023-10-09T05:44:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 29.887033918, 2023-10-09T05:44:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.567767433, 2023-10-09T05:44:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125695 2023-10-09T05:44:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.513022+0000", 2023-10-09T05:44:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.513022+0000", 2023-10-09T05:44:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.513020+0000", 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.513028+0000", 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7800000000000001e-06 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.513043+0000", 2023-10-09T05:44:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4704e-05 2023-10-09T05:44:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.513053+0000", 2023-10-09T05:44:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9760000000000007e-06 2023-10-09T05:44:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.515242+0000", 2023-10-09T05:44:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021888910000000001 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.515256+0000", 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4157e-05 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.969375+0000", 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.454119198 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.969397+0000", 2023-10-09T05:44:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1535e-05 2023-10-09T05:44:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.064005+0000", 2023-10-09T05:44:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.094608636 2023-10-09T05:44:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:44:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.064424+0000", 2023-10-09T05:44:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00041908800000000002 2023-10-09T05:44:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.068925+0000", 2023-10-09T05:44:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045003949999999999 2023-10-09T05:44:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.068989+0000", 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4727000000000006e-05 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.069327+0000", 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00033761799999999998 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.073230+0000", 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039025359999999999 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.073245+0000", 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5841999999999998e-05 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.079329+0000", 2023-10-09T05:44:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0060834560000000001 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.079372+0000", 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2939999999999999e-05 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.080789+0000", 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014174140000000001 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125703 7.6 7:6236fd7c:::benchmark_data_smithi008_354649_object31207:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:04.049531+0000", 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 29.350524443000001, 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.067367433, 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125703 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049531+0000", 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049531+0000", 2023-10-09T05:44:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049529+0000", 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049537+0000", 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7800000000000001e-06 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049554+0000", 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7130999999999999e-05 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049563+0000", 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.602e-06 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049659+0000", 2023-10-09T05:44:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6582000000000005e-05 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049673+0000", 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3178e-05 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.070752+0000", 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.02107978 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.071153+0000", 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00040059500000000001 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.106057+0000", 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.034904042000000003 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.106130+0000", 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3085999999999998e-05 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.106432+0000", 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00030198699999999998 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.108692+0000", 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002259961 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.109523+0000", 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00083126999999999995 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.115580+0000", 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0060562990000000002 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.115649+0000", 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9030999999999997e-05 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.116899+0000", 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0012502030000000001 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:33.715 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:44:33.762 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:33.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:33.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:33.866 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:33.896 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:33.942 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:33.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:33.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:34.049 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:34.077 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:33 smithi008 ceph-mon[184418]: pgmap v727: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 19975/164208 objects degraded (12.164%); 4408/164208 objects misplaced (2.684%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:34.077 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:33 smithi008 ceph-mon[191227]: pgmap v727: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 19975/164208 objects degraded (12.164%); 4408/164208 objects misplaced (2.684%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:34.080 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:33 smithi191 ceph-mon[161669]: pgmap v727: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 19975/164208 objects degraded (12.164%); 4408/164208 objects misplaced (2.684%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:34.479 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:34.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:34.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:34.562 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:44:34.587 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:34.646 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:34.650 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:44:34.763 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:34.805 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:34.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:34.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:34.931 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:34.995 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:35.077 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:35.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:35.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:35.150 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:34 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:44:35.150 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:34 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:44:35.184 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:35.218 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:34 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:44:35.222 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:35.359 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:35.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:35.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:35.472 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:35.514 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:35.654 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:35.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:35.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:35.757 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:35.786 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:35.963 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:35.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:35.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:35.986 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T05:44:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:35 smithi191 ceph-mon[161669]: pgmap v728: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 19865/162618 objects degraded (12.216%); 4408/162618 objects misplaced (2.711%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:35 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1487154002' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:35 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:36.080 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:36.113 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:36.338 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:35 smithi008 ceph-mon[191227]: pgmap v728: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 19865/162618 objects degraded (12.216%); 4408/162618 objects misplaced (2.711%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:36.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:35 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:36.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1487154002' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:36.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:35 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:36.341 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:35 smithi008 ceph-mon[184418]: pgmap v728: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 19865/162618 objects degraded (12.216%); 4408/162618 objects misplaced (2.711%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:36.341 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:35 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:36.341 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1487154002' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:36.341 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:35 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:44:36.448 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:36.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:36.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:36.551 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:36.584 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:36.651 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:36.652 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:36.990 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:36.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:36.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:37.096 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:37.129 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:36 smithi008 ceph-mon[184418]: failed to encode map e245 with expected crc 2023-10-09T05:44:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:36 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:44:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:36 smithi008 ceph-mon[184418]: osdmap e245: 8 total, 8 up, 8 in 2023-10-09T05:44:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:36 smithi008 ceph-mon[184418]: failed to encode map e245 with expected crc 2023-10-09T05:44:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:36 smithi008 ceph-mon[184418]: failed to encode map e245 with expected crc 2023-10-09T05:44:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:36 smithi008 ceph-mon[191227]: failed to encode map e245 with expected crc 2023-10-09T05:44:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:36 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:44:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:36 smithi008 ceph-mon[191227]: osdmap e245: 8 total, 8 up, 8 in 2023-10-09T05:44:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:36 smithi008 ceph-mon[191227]: failed to encode map e245 with expected crc 2023-10-09T05:44:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:36 smithi008 ceph-mon[191227]: failed to encode map e245 with expected crc 2023-10-09T05:44:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:36 smithi191 ceph-mon[161669]: failed to encode map e245 with expected crc 2023-10-09T05:44:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:36 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:44:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:36 smithi191 ceph-mon[161669]: osdmap e245: 8 total, 8 up, 8 in 2023-10-09T05:44:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:36 smithi191 ceph-mon[161669]: failed to encode map e245 with expected crc 2023-10-09T05:44:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:36 smithi191 ceph-mon[161669]: failed to encode map e245 with expected crc 2023-10-09T05:44:38.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:37 smithi191 ceph-mon[161669]: pgmap v730: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 19813/160665 objects degraded (12.332%); 4077/160665 objects misplaced (2.538%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T05:44:38.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:37 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19813/160665 objects degraded (12.332%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:37 smithi008 ceph-mon[184418]: pgmap v730: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 19813/160665 objects degraded (12.332%); 4077/160665 objects misplaced (2.538%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T05:44:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:37 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19813/160665 objects degraded (12.332%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:38.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:37 smithi008 ceph-mon[191227]: pgmap v730: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 19813/160665 objects degraded (12.332%); 4077/160665 objects misplaced (2.538%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T05:44:38.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:37 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19813/160665 objects degraded (12.332%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:39.620 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:39.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:39.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:39.726 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:39.764 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:39.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[184418]: pgmap v731: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 19697/159060 objects degraded (12.383%); 4077/159060 objects misplaced (2.563%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/823677178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:39.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[191227]: pgmap v731: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 19697/159060 objects degraded (12.383%); 4077/159060 objects misplaced (2.563%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:39.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/823677178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:39.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:39 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:39.997 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:44:40.029 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:40.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:40.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:39 smithi191 ceph-mon[161669]: pgmap v731: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 19697/159060 objects degraded (12.383%); 4077/159060 objects misplaced (2.563%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:44:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:39 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:39 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:39 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/823677178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:39 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:44:40.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:40.184 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:40.248 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:40.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:40.354 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:40.384 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:40.478 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:40.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:40.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:40.595 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:40.654 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:40.733 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:40.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:40.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:40.797 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:44:40.798 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:44:40.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:40.798 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:138094 7.10 e246/239 7:0d387400:::benchmark_data_smithi008_354649_object43598:head v 246'8141, mlcod=246'8139)", 2023-10-09T05:44:40.798 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:40.773118+0000", 2023-10-09T05:44:40.798 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0019774739999999999, 2023-10-09T05:44:40.798 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002002233, 2023-10-09T05:44:40.798 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:40.798 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:44:40.798 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:40.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:40.798 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.773118+0000", 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.773118+0000", 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.773117+0000", 2023-10-09T05:44:40.799 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.773129+0000", 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1307e-05 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.773152+0000", 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3606999999999999e-05 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:40.800 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:40.801 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.773177+0000", 2023-10-09T05:44:40.801 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4375e-05 2023-10-09T05:44:40.801 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:40.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:40.801 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:40.801 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.773248+0000", 2023-10-09T05:44:40.801 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1309000000000005e-05 2023-10-09T05:44:40.801 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:40.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:40.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:40.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.773274+0000", 2023-10-09T05:44:40.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5967000000000001e-05 2023-10-09T05:44:40.802 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:40.802 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:40.802 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:40.802 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:40.802 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:44:40.802 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 1 2023-10-09T05:44:40.802 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:44:40.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:40 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:40.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:40 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:44:40.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:40 smithi191 ceph-mon[161669]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:40 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:44:40.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:40 smithi191 ceph-mon[161669]: osdmap e246: 8 total, 8 up, 8 in 2023-10-09T05:44:40.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:40 smithi191 ceph-mon[161669]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:40 smithi191 ceph-mon[161669]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:40 smithi191 ceph-mon[161669]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:40.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:44:40.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[184418]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:44:40.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[184418]: osdmap e246: 8 total, 8 up, 8 in 2023-10-09T05:44:40.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[184418]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[184418]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[184418]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.805 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:44:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:44:40] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T05:44:40.805 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:40.805 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:44:40.805 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[191227]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.805 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:44:40.805 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[191227]: osdmap e246: 8 total, 8 up, 8 in 2023-10-09T05:44:40.805 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[191227]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.805 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[191227]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.806 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:40 smithi008 ceph-mon[191227]: failed to encode map e246 with expected crc 2023-10-09T05:44:40.841 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:40.894 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:40.972 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:40.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:40.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:41.087 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:41.129 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:41.200 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:41.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:41.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:41.337 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:41.385 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:41.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:41.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:41.492 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:41.548 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:41.868 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:41.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:41.942 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:44:41.942 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:44:41.943 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:44:41.943 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.008314 2023-10-09T05:44:41.944 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0.008313750946760456 2023-10-09T05:44:41.974 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:42.006 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:42.077 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:42.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:42.116 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:41 smithi008 ceph-mon[184418]: pgmap v733: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 19515/156417 objects degraded (12.476%); 4025/156417 objects misplaced (2.573%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T05:44:42.116 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:41 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:42.117 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:41 smithi008 ceph-mon[191227]: pgmap v733: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 19515/156417 objects degraded (12.476%); 4025/156417 objects misplaced (2.573%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T05:44:42.117 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:41 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:42.118 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:44:42.156 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:44:42.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:42.204 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:44:42.228 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:42.248 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:44:42.290 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:44:42.297 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:42.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:41 smithi191 ceph-mon[161669]: pgmap v733: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 19515/156417 objects degraded (12.476%); 4025/156417 objects misplaced (2.573%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T05:44:42.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:41 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:42.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:42.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:42.322 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:44:42.361 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:44:42.391 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:44:42.405 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:42.428 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:44:42.439 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:44:42.439 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:44:42.439 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:44:42.439 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:44:42.440 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:44:42.440 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:44:42.440 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:44:42.440 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:44:42.460 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:43.092 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:44:43.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:42 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2192641504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:42 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19515/156417 objects degraded (12.476%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:43.401 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:43.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:42 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:43.403 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2192641504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:43.403 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:42 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19515/156417 objects degraded (12.476%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:43.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:42 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:43.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2192641504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:43.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:42 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19515/156417 objects degraded (12.476%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:43.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:43.505 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:43.542 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:43.611 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:43.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:43.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:43.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:43.749 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:43.811 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:43.817 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:43.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:43.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:43.922 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:43.958 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:44.005 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:44.018 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:44.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:44.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:44.124 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:44.149 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:43 smithi191 ceph-mon[161669]: pgmap v734: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 19363/153237 objects degraded (12.636%); 3790/153237 objects misplaced (2.473%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T05:44:44.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:44.150 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:44.152 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:44.204 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:44.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:44.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:44.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:43 smithi008 ceph-mon[184418]: pgmap v734: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 19363/153237 objects degraded (12.636%); 3790/153237 objects misplaced (2.473%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T05:44:44.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:43 smithi008 ceph-mon[191227]: pgmap v734: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 19363/153237 objects degraded (12.636%); 3790/153237 objects misplaced (2.473%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T05:44:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:44.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:44.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:44.345 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:44.406 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:44.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:44.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:44.514 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:44.554 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:44.562 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:44.612 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:44.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:44.720 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:44.761 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:45.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:44 smithi191 ceph-mon[161669]: pgmap v735: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 19220/151302 objects degraded (12.703%); 3790/151302 objects misplaced (2.505%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:44:45.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:44 smithi008 ceph-mon[184418]: pgmap v735: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 19220/151302 objects degraded (12.703%); 3790/151302 objects misplaced (2.505%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:44:45.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:44 smithi008 ceph-mon[191227]: pgmap v735: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 19220/151302 objects degraded (12.703%); 3790/151302 objects misplaced (2.505%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:44:45.478 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:45.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:45.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:45.584 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:45.619 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:45.681 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:45.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:45.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:45.788 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:45.791 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:44:45.833 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:45.999 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:45 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:45.999 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:46.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:46.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:46.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:46.137 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:46.203 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:46.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:46.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:45 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:46.310 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:46.337 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:46.389 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:46.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:46.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:46.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:46 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:46.496 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:46.540 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:46.623 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:46.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:46.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:46.735 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:46.795 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:47.071 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:47 smithi008 ceph-mon[184418]: pgmap v736: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 19170/149322 objects degraded (12.838%); 3437/149322 objects misplaced (2.302%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T05:44:47.071 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:47 smithi008 ceph-mon[191227]: pgmap v736: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 19170/149322 objects degraded (12.838%); 3437/149322 objects misplaced (2.302%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T05:44:47.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:47 smithi191 ceph-mon[161669]: pgmap v736: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 19170/149322 objects degraded (12.838%); 3437/149322 objects misplaced (2.302%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T05:44:48.049 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:48.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:48.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:48.155 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:48.189 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:48.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:48 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19170/149322 objects degraded (12.838%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:48.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:48 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:48.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:48.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19170/149322 objects degraded (12.838%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:48.246 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:48.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:48.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:48 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19170/149322 objects degraded (12.838%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:48.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:48.397 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:48.450 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:48.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:48.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:48.510 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:48.530 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:44:48.530 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:44:48.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:48.596 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:48.662 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:48.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:48.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:48.771 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:48.804 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:48.867 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:48.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:48.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:48.974 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:49.005 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:49.081 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:49.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:49.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:49.192 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:44:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[184418]: 7.0 scrub starts 2023-10-09T05:44:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[184418]: pgmap v737: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 19067/147684 objects degraded (12.911%); 3437/147684 objects misplaced (2.327%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T05:44:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/240863956' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.008313750946760456}]: dispatch 2023-10-09T05:44:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.008313750946760456}]: dispatch 2023-10-09T05:44:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[191227]: 7.0 scrub starts 2023-10-09T05:44:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[191227]: pgmap v737: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 19067/147684 objects degraded (12.911%); 3437/147684 objects misplaced (2.327%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T05:44:49.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:49.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:49.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:49.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/240863956' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.008313750946760456}]: dispatch 2023-10-09T05:44:49.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:49 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.008313750946760456}]: dispatch 2023-10-09T05:44:49.239 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:49.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:49 smithi191 ceph-mon[161669]: 7.0 scrub starts 2023-10-09T05:44:49.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:49 smithi191 ceph-mon[161669]: pgmap v737: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 19067/147684 objects degraded (12.911%); 3437/147684 objects misplaced (2.327%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T05:44:49.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:49.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:49.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:49.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/240863956' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.008313750946760456}]: dispatch 2023-10-09T05:44:49.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:49 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.008313750946760456}]: dispatch 2023-10-09T05:44:49.477 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:44:49.478 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.6 primary-affinity to 0.00831375 (85442) 2023-10-09T05:44:50.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1159466671' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:44:50.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:44:50.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[184418]: failed to encode map e247 with expected crc 2023-10-09T05:44:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.008313750946760456}]': finished 2023-10-09T05:44:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:44:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[184418]: osdmap e247: 8 total, 8 up, 8 in 2023-10-09T05:44:50.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1159466671' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:44:50.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:44:50.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:50.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[191227]: failed to encode map e247 with expected crc 2023-10-09T05:44:50.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.008313750946760456}]': finished 2023-10-09T05:44:50.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:44:50.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:50 smithi008 ceph-mon[191227]: osdmap e247: 8 total, 8 up, 8 in 2023-10-09T05:44:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1159466671' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:44:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:50 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:44:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:50 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:44:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:50 smithi191 ceph-mon[161669]: failed to encode map e247 with expected crc 2023-10-09T05:44:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:50 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.008313750946760456}]': finished 2023-10-09T05:44:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:50 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:44:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:50 smithi191 ceph-mon[161669]: osdmap e247: 8 total, 8 up, 8 in 2023-10-09T05:44:50.846 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:44:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:44:50] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T05:44:51.046 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:44:51.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:51.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:44:51.150 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:51.178 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:51 smithi008 ceph-mon[184418]: failed to encode map e247 with expected crc 2023-10-09T05:44:51.178 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:51 smithi008 ceph-mon[184418]: failed to encode map e247 with expected crc 2023-10-09T05:44:51.178 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:51 smithi008 ceph-mon[184418]: pgmap v739: 99 pgs: 4 peering, 5 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 18914/144963 objects degraded (13.047%); 3375/144963 objects misplaced (2.328%); 7.5 MiB/s, 125 objects/s recovering 2023-10-09T05:44:51.178 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:51 smithi008 ceph-mon[184418]: osdmap e248: 8 total, 8 up, 8 in 2023-10-09T05:44:51.178 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:51 smithi008 ceph-mon[184418]: failed to encode map e248 with expected crc 2023-10-09T05:44:51.179 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:51 smithi008 ceph-mon[191227]: failed to encode map e247 with expected crc 2023-10-09T05:44:51.179 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:51 smithi008 ceph-mon[191227]: failed to encode map e247 with expected crc 2023-10-09T05:44:51.179 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:51 smithi008 ceph-mon[191227]: pgmap v739: 99 pgs: 4 peering, 5 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 18914/144963 objects degraded (13.047%); 3375/144963 objects misplaced (2.328%); 7.5 MiB/s, 125 objects/s recovering 2023-10-09T05:44:51.179 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:51 smithi008 ceph-mon[191227]: osdmap e248: 8 total, 8 up, 8 in 2023-10-09T05:44:51.179 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:51 smithi008 ceph-mon[191227]: failed to encode map e248 with expected crc 2023-10-09T05:44:51.181 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:51 smithi191 ceph-mon[161669]: failed to encode map e247 with expected crc 2023-10-09T05:44:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:51 smithi191 ceph-mon[161669]: failed to encode map e247 with expected crc 2023-10-09T05:44:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:51 smithi191 ceph-mon[161669]: pgmap v739: 99 pgs: 4 peering, 5 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 18914/144963 objects degraded (13.047%); 3375/144963 objects misplaced (2.328%); 7.5 MiB/s, 125 objects/s recovering 2023-10-09T05:44:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:51 smithi191 ceph-mon[161669]: osdmap e248: 8 total, 8 up, 8 in 2023-10-09T05:44:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:51 smithi191 ceph-mon[161669]: failed to encode map e248 with expected crc 2023-10-09T05:44:51.604 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:44:51.838 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:51.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:51.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:51.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:51.976 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:52.040 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:52.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:52.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:52.146 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:52.191 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[184418]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[184418]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[184418]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[184418]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[184418]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:44:52.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[184418]: osdmap e249: 8 total, 8 up, 8 in 2023-10-09T05:44:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[191227]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[191227]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[191227]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[191227]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[191227]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:44:52.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:52 smithi008 ceph-mon[191227]: osdmap e249: 8 total, 8 up, 8 in 2023-10-09T05:44:52.258 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:52.266 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:52 smithi191 ceph-mon[161669]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.267 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:52 smithi191 ceph-mon[161669]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.267 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:52 smithi191 ceph-mon[161669]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.267 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:52 smithi191 ceph-mon[161669]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.267 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:52 smithi191 ceph-mon[161669]: failed to encode map e248 with expected crc 2023-10-09T05:44:52.267 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:52 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:44:52.267 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:52 smithi191 ceph-mon[161669]: osdmap e249: 8 total, 8 up, 8 in 2023-10-09T05:44:52.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:52.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:52.369 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:52.397 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:52.461 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:52.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:52.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:52.567 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:52.598 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:53.081 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:53.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:53.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:53.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:53.215 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:53.269 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:53.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:53.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:53.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:53 smithi191 ceph-mon[161669]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:53 smithi191 ceph-mon[161669]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:53 smithi191 ceph-mon[161669]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:53 smithi191 ceph-mon[161669]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:53 smithi191 ceph-mon[161669]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:53 smithi191 ceph-mon[161669]: pgmap v742: 99 pgs: 4 peering, 5 active+recovery_wait+degraded, 86 active+clean, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 19042/143226 objects degraded (13.295%); 2592/143226 objects misplaced (1.810%); 9.8 MiB/s, 166 objects/s recovering 2023-10-09T05:44:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 18914/144963 objects degraded (13.047%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:53.325 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:53.361 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:44:53.377 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:53.388 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:44:53.405 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:53.413 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:44:53.436 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:44:53.459 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:44:53.461 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:53.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[191227]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[191227]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[191227]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[191227]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[191227]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[191227]: pgmap v742: 99 pgs: 4 peering, 5 active+recovery_wait+degraded, 86 active+clean, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 19042/143226 objects degraded (13.295%); 2592/143226 objects misplaced (1.810%); 9.8 MiB/s, 166 objects/s recovering 2023-10-09T05:44:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 18914/144963 objects degraded (13.047%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[184418]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[184418]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[184418]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[184418]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[184418]: failed to encode map e249 with expected crc 2023-10-09T05:44:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[184418]: pgmap v742: 99 pgs: 4 peering, 5 active+recovery_wait+degraded, 86 active+clean, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 19042/143226 objects degraded (13.295%); 2592/143226 objects misplaced (1.810%); 9.8 MiB/s, 166 objects/s recovering 2023-10-09T05:44:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 18914/144963 objects degraded (13.047%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:44:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:53.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:53.480 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:44:53.506 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:44:53.533 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:44:53.539 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:44:53.539 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:44:53.539 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:44:53.540 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:44:53.540 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:44:53.540 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:44:53.540 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:44:53.540 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:44:53.573 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:53.619 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:53.948 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:44:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:54 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/396184465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:54.311 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:54.315 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:54 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/396184465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:54.316 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:54 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/396184465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:44:54.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:54.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:54.418 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:54.458 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:55.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:55 smithi191 ceph-mon[161669]: pgmap v743: 99 pgs: 4 peering, 5 active+recovery_wait+degraded, 86 active+clean, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 376 op/s; 18966/140910 objects degraded (13.460%); 2592/140910 objects misplaced (1.839%); 9.5 MiB/s, 161 objects/s recovering 2023-10-09T05:44:55.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:55 smithi008 ceph-mon[184418]: pgmap v743: 99 pgs: 4 peering, 5 active+recovery_wait+degraded, 86 active+clean, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 376 op/s; 18966/140910 objects degraded (13.460%); 2592/140910 objects misplaced (1.839%); 9.5 MiB/s, 161 objects/s recovering 2023-10-09T05:44:55.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:55 smithi008 ceph-mon[191227]: pgmap v743: 99 pgs: 4 peering, 5 active+recovery_wait+degraded, 86 active+clean, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 376 op/s; 18966/140910 objects degraded (13.460%); 2592/140910 objects misplaced (1.839%); 9.5 MiB/s, 161 objects/s recovering 2023-10-09T05:44:55.523 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:55.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:55.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:55.657 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:55.710 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:55.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:55.815 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:55.845 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:55.903 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:55.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:55.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:55.977 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:56.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:56.044 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:56.106 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:56.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:56.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:56.211 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:56.242 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:56.296 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:56.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:56.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:56.403 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:56.450 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:56.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:56 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T05:44:56.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:56 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T05:44:56.510 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:44:56.510 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:44:56.510 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:44:56.510 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:44:56.510 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.510 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4412 6.4 6:2c929489:::benchmark_data_smithi008_295510_object4411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:44:56.511 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:09.108561+0000", 2023-10-09T05:44:56.511 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 467.39939451800001, 2023-10-09T05:44:56.511 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30418963199999999, 2023-10-09T05:44:56.511 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.511 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.511 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.511 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:56.511 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:56.511 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4412 2023-10-09T05:44:56.511 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108561+0000", 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108561+0000", 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.512 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108559+0000", 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108583+0000", 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3278e-05 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.513 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108598+0000", 2023-10-09T05:44:56.514 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4980999999999999e-05 2023-10-09T05:44:56.514 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.514 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.514 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.514 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108605+0000", 2023-10-09T05:44:56.514 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4880000000000001e-06 2023-10-09T05:44:56.514 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.514 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.514 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.514 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108653+0000", 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8251000000000003e-05 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108787+0000", 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000133879 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.108927+0000", 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000139949 2023-10-09T05:44:56.515 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111786+0000", 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002858855 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.114484+0000", 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026978729999999999 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.516 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412641+0000", 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29815728699999999 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412707+0000", 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5531999999999995e-05 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412750+0000", 2023-10-09T05:44:56.517 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3729000000000001e-05 2023-10-09T05:44:56.518 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.518 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.518 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.518 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.518 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.518 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4420 6.4 6:26d4e1d5:::benchmark_data_smithi008_295510_object4419:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:44:56.518 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:09.111673+0000", 2023-10-09T05:44:56.518 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 467.39628276299999, 2023-10-09T05:44:56.519 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30119500100000002, 2023-10-09T05:44:56.519 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.519 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.519 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.519 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:56.519 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:56.519 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4420 2023-10-09T05:44:56.519 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.519 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.519 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.520 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.520 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111673+0000", 2023-10-09T05:44:56.520 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.520 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.520 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.520 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.520 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111673+0000", 2023-10-09T05:44:56.520 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.520 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.520 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111671+0000", 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111697+0000", 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5582999999999999e-05 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.521 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111712+0000", 2023-10-09T05:44:56.522 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5546999999999999e-05 2023-10-09T05:44:56.522 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.522 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.522 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.522 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111720+0000", 2023-10-09T05:44:56.522 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8970000000000008e-06 2023-10-09T05:44:56.522 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.522 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.522 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.522 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111810+0000", 2023-10-09T05:44:56.523 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0062000000000006e-05 2023-10-09T05:44:56.523 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.523 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.523 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.523 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.111951+0000", 2023-10-09T05:44:56.523 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000140662 2023-10-09T05:44:56.523 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.524 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.524 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:44:56.524 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.112099+0000", 2023-10-09T05:44:56.524 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014773400000000001 2023-10-09T05:44:56.524 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.524 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.524 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.524 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.113038+0000", 2023-10-09T05:44:56.524 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00093915700000000003 2023-10-09T05:44:56.524 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.524 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.114491+0000", 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001453378 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412787+0000", 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29829549100000002 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412833+0000", 2023-10-09T05:44:56.525 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5770999999999999e-05 2023-10-09T05:44:56.526 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.526 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.526 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.526 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412868+0000", 2023-10-09T05:44:56.526 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5008000000000001e-05 2023-10-09T05:44:56.526 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.526 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.526 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.526 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.526 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.526 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:4428 6.4 6:22fa3692:::benchmark_data_smithi008_295510_object4427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:44:56.527 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:37:09.116232+0000", 2023-10-09T05:44:56.527 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 467.39172326599999, 2023-10-09T05:44:56.527 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29675727800000001, 2023-10-09T05:44:56.527 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.527 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.527 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.527 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:56.527 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:56.527 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 4428 2023-10-09T05:44:56.528 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.529 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.529 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.529 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.529 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116232+0000", 2023-10-09T05:44:56.529 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.529 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.529 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.529 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.529 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116232+0000", 2023-10-09T05:44:56.529 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.530 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.530 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.530 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.530 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116231+0000", 2023-10-09T05:44:56.530 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:56.530 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.530 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.530 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.530 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116251+0000", 2023-10-09T05:44:56.530 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0611999999999998e-05 2023-10-09T05:44:56.530 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.531 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.531 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.531 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116265+0000", 2023-10-09T05:44:56.531 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3195e-05 2023-10-09T05:44:56.531 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.531 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.531 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.532 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116271+0000", 2023-10-09T05:44:56.532 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5230000000000001e-06 2023-10-09T05:44:56.532 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.532 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.532 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.532 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116312+0000", 2023-10-09T05:44:56.532 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0831000000000003e-05 2023-10-09T05:44:56.532 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.532 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.532 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.532 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116415+0000", 2023-10-09T05:44:56.533 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010352399999999999 2023-10-09T05:44:56.533 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.533 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.533 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:44:56.533 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.116514+0000", 2023-10-09T05:44:56.533 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8212000000000001e-05 2023-10-09T05:44:56.533 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.533 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.533 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.533 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.117005+0000", 2023-10-09T05:44:56.533 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000491516 2023-10-09T05:44:56.534 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.534 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.534 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.534 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.117207+0000", 2023-10-09T05:44:56.534 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000201659 2023-10-09T05:44:56.534 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.534 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.534 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.534 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412905+0000", 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29569794399999999 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412954+0000", 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9382000000000003e-05 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:37:09.412989+0000", 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5240000000000001e-05 2023-10-09T05:44:56.535 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.536 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.536 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.536 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.536 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.536 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:142240 6.4 6:2cab24fa:::benchmark_data_smithi008_295510_object53877:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:44:56.536 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:07.422273+0000", 2023-10-09T05:44:56.536 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 289.08568229999997, 2023-10-09T05:44:56.536 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.424140515, 2023-10-09T05:44:56.536 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.536 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.536 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 142240 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422273+0000", 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.537 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.538 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422273+0000", 2023-10-09T05:44:56.538 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.538 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.538 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.538 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.538 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422271+0000", 2023-10-09T05:44:56.538 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:56.538 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.538 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.538 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.539 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422280+0000", 2023-10-09T05:44:56.539 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8589999999999994e-06 2023-10-09T05:44:56.539 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.539 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.539 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.539 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422300+0000", 2023-10-09T05:44:56.539 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.982e-05 2023-10-09T05:44:56.539 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.539 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.540 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.540 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422310+0000", 2023-10-09T05:44:56.540 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7329999999999998e-06 2023-10-09T05:44:56.540 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.540 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.540 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.540 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422354+0000", 2023-10-09T05:44:56.540 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4317999999999998e-05 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422530+0000", 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017590800000000001 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.422682+0000", 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015153599999999999 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426323+0000", 2023-10-09T05:44:56.542 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036415879999999999 2023-10-09T05:44:56.542 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.542 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.436458+0000", 2023-10-09T05:44:56.542 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010134364 2023-10-09T05:44:56.542 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.542 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.543 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.543 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846310+0000", 2023-10-09T05:44:56.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.40985222199999999 2023-10-09T05:44:56.543 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.543 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.543 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.543 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846362+0000", 2023-10-09T05:44:56.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2284999999999998e-05 2023-10-09T05:44:56.543 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.543 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.544 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.544 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846414+0000", 2023-10-09T05:44:56.544 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1547e-05 2023-10-09T05:44:56.544 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.544 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.544 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.544 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.544 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.545 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:142244 6.4 6:252372f6:::benchmark_data_smithi008_295510_object53881:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:44:56.545 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:07.426444+0000", 2023-10-09T05:44:56.545 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 289.08151178399999, 2023-10-09T05:44:56.545 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.42009002099999998, 2023-10-09T05:44:56.545 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.545 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.545 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.545 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:56.545 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:56.546 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 142244 2023-10-09T05:44:56.546 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.546 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.546 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.546 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.546 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426444+0000", 2023-10-09T05:44:56.546 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.546 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.546 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.546 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.546 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426444+0000", 2023-10-09T05:44:56.547 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.547 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.547 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.547 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.547 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426442+0000", 2023-10-09T05:44:56.547 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:56.547 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.547 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.547 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.547 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426450+0000", 2023-10-09T05:44:56.547 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8180000000000004e-06 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426467+0000", 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7550999999999998e-05 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426478+0000", 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0356e-05 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.548 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.549 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426525+0000", 2023-10-09T05:44:56.549 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7141000000000001e-05 2023-10-09T05:44:56.549 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.549 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.549 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.426671+0000", 2023-10-09T05:44:56.549 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014611999999999999 2023-10-09T05:44:56.549 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.549 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:44:56.550 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.430321+0000", 2023-10-09T05:44:56.550 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036499219999999999 2023-10-09T05:44:56.550 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.550 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.550 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.550 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.436469+0000", 2023-10-09T05:44:56.550 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061478349999999999 2023-10-09T05:44:56.550 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.550 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.550 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.550 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.438906+0000", 2023-10-09T05:44:56.551 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0024374570000000001 2023-10-09T05:44:56.551 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.551 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.551 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.551 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846452+0000", 2023-10-09T05:44:56.551 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.40754571000000001 2023-10-09T05:44:56.551 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.551 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.551 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.551 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846494+0000", 2023-10-09T05:44:56.551 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1936e-05 2023-10-09T05:44:56.552 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.552 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.552 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.552 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846534+0000", 2023-10-09T05:44:56.552 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.998e-05 2023-10-09T05:44:56.552 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.552 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.552 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.552 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.553 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.553 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:142256 6.4 6:2549d186:::benchmark_data_smithi008_295510_object53893:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:44:56.553 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:07.449511+0000", 2023-10-09T05:44:56.553 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 289.05844413400001, 2023-10-09T05:44:56.554 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.39714472299999998, 2023-10-09T05:44:56.554 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.554 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.554 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 142256 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449511+0000", 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.555 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449511+0000", 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449510+0000", 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449517+0000", 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2269999999999996e-06 2023-10-09T05:44:56.556 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449533+0000", 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6172e-05 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449544+0000", 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0431999999999999e-05 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.557 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.558 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449595+0000", 2023-10-09T05:44:56.558 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0905000000000001e-05 2023-10-09T05:44:56.558 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.558 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.558 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.558 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449734+0000", 2023-10-09T05:44:56.558 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000139784 2023-10-09T05:44:56.558 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.558 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.558 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:44:56.558 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.449866+0000", 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013137399999999999 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.453118+0000", 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032525430000000001 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.458251+0000", 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051326599999999998 2023-10-09T05:44:56.559 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.560 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.560 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.560 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846581+0000", 2023-10-09T05:44:56.560 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.38833016799999998 2023-10-09T05:44:56.560 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.560 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.560 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.560 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846623+0000", 2023-10-09T05:44:56.560 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2038e-05 2023-10-09T05:44:56.560 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.560 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846656+0000", 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2858000000000003e-05 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45029.0:142258 6.4 6:283fbab6:::benchmark_data_smithi008_295510_object53895:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:40:07.454512+0000", 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 289.05344342299998, 2023-10-09T05:44:56.561 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.39222012299999998, 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45029", 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 142258 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454512+0000", 2023-10-09T05:44:56.562 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454512+0000", 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454511+0000", 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.563 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454517+0000", 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2840000000000003e-06 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454525+0000", 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6299999999999998e-06 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.564 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454530+0000", 2023-10-09T05:44:56.565 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0350000000000001e-06 2023-10-09T05:44:56.565 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.565 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.565 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.565 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454570+0000", 2023-10-09T05:44:56.565 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0723999999999998e-05 2023-10-09T05:44:56.565 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.565 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.565 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.565 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454696+0000", 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012594599999999999 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.454779+0000", 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2871e-05 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.458263+0000", 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034837380000000001 2023-10-09T05:44:56.566 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.567 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.567 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.567 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.459986+0000", 2023-10-09T05:44:56.567 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017235970000000001 2023-10-09T05:44:56.567 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.567 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.567 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.567 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846684+0000", 2023-10-09T05:44:56.567 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.38669726799999998 2023-10-09T05:44:56.567 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.567 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846709+0000", 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4811e-05 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:40:07.846732+0000", 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3544e-05 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.568 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:19325 7.b 7:d8dcac00:::benchmark_data_smithi008_354649_object19324:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:41:46.644133+0000", 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 189.863821992, 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27924601399999999, 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 19325 2023-10-09T05:44:56.569 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644133+0000", 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644133+0000", 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.570 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644132+0000", 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644155+0000", 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3040999999999998e-05 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644178+0000", 2023-10-09T05:44:56.571 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2901e-05 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644186+0000", 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7489999999999998e-06 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.645061+0000", 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00087493800000000002 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.572 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.645205+0000", 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014464400000000001 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.645338+0000", 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000133048 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.573 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647110+0000", 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001771665 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647117+0000", 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1890000000000001e-06 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923280+0000", 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27616340700000003 2023-10-09T05:44:56.574 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923332+0000", 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1152000000000001e-05 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923379+0000", 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7787999999999997e-05 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.575 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.576 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:19331 7.b 7:de69860c:::benchmark_data_smithi008_354649_object19330:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-09T05:44:56.576 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:41:46.644206+0000", 2023-10-09T05:44:56.576 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 189.863749708, 2023-10-09T05:44:56.576 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27929858400000002, 2023-10-09T05:44:56.576 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.576 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.576 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.576 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:56.576 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:56.577 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 19331 2023-10-09T05:44:56.577 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.577 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.577 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.577 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644206+0000", 2023-10-09T05:44:56.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.578 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644206+0000", 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644205+0000", 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644228+0000", 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3442e-05 2023-10-09T05:44:56.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644237+0000", 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0700000000000007e-06 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.644241+0000", 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7230000000000004e-06 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.646074+0000", 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001833213 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.646216+0000", 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000141555 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.646349+0000", 2023-10-09T05:44:56.581 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000133492 2023-10-09T05:44:56.582 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.582 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.582 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.582 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923238+0000", 2023-10-09T05:44:56.582 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27688896699999999 2023-10-09T05:44:56.582 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.582 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.582 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.582 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923394+0000", 2023-10-09T05:44:56.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000155859 2023-10-09T05:44:56.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923430+0000", 2023-10-09T05:44:56.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5364000000000002e-05 2023-10-09T05:44:56.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923472+0000", 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1999999999999998e-05 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923504+0000", 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2574999999999999e-05 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:19328 7.9 e220/213 7:91090d84:::benchmark_data_smithi008_354649_object19327:head v 220'1177, mlcod=220'1174)", 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:41:46.646587+0000", 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 189.861368277, 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.276629554, 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.646587+0000", 2023-10-09T05:44:56.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.586 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.646587+0000", 2023-10-09T05:44:56.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.586 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.646586+0000", 2023-10-09T05:44:56.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:44:56.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.646609+0000", 2023-10-09T05:44:56.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3005000000000001e-05 2023-10-09T05:44:56.587 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.588 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.646619+0000", 2023-10-09T05:44:56.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0023999999999999e-05 2023-10-09T05:44:56.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.588 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.588 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.646626+0000", 2023-10-09T05:44:56.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0940000000000002e-06 2023-10-09T05:44:56.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.588 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.588 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648067+0000", 2023-10-09T05:44:56.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014402760000000001 2023-10-09T05:44:56.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.589 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.589 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648100+0000", 2023-10-09T05:44:56.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3769999999999997e-05 2023-10-09T05:44:56.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.589 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.589 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649286+0000", 2023-10-09T05:44:56.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001185321 2023-10-09T05:44:56.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.590 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.590 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923217+0000", 2023-10-09T05:44:56.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27393088100000001 2023-10-09T05:44:56.590 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.590 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.590 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.590 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:19334 7.2 e220/213 7:4c082a4e:::benchmark_data_smithi008_354649_object19333:head v 220'1238, mlcod=220'1236)", 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:41:46.647172+0000", 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 189.86078374900001, 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27604569499999998, 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647172+0000", 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647172+0000", 2023-10-09T05:44:56.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647171+0000", 2023-10-09T05:44:56.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:56.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647195+0000", 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3822e-05 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647204+0000", 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4190000000000008e-06 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647210+0000", 2023-10-09T05:44:56.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7119999999999997e-06 2023-10-09T05:44:56.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647286+0000", 2023-10-09T05:44:56.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5724999999999997e-05 2023-10-09T05:44:56.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647305+0000", 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9598000000000001e-05 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923177+0000", 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27587202599999999 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923217+0000", 2023-10-09T05:44:56.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9969999999999998e-05 2023-10-09T05:44:56.596 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.596 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.596 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.596 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:19341 7.b 7:d7f6eb84:::benchmark_data_smithi008_354649_object19340:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:41:46.647563+0000", 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 189.860392873, 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.28031963599999998, 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 19341 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.597 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647563+0000", 2023-10-09T05:44:56.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.598 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647563+0000", 2023-10-09T05:44:56.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.598 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.599 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647561+0000", 2023-10-09T05:44:56.599 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:56.599 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.599 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.599 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647592+0000", 2023-10-09T05:44:56.599 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1835999999999998e-05 2023-10-09T05:44:56.599 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.600 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.600 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647606+0000", 2023-10-09T05:44:56.600 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3789e-05 2023-10-09T05:44:56.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.600 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.600 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.647615+0000", 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8019999999999998e-06 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923538+0000", 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27592325200000001 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923710+0000", 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017188899999999999 2023-10-09T05:44:56.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.602 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:44:56.602 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923876+0000", 2023-10-09T05:44:56.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016603700000000001 2023-10-09T05:44:56.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.602 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.602 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.927659+0000", 2023-10-09T05:44:56.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037831380000000001 2023-10-09T05:44:56.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.927714+0000", 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4147999999999997e-05 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.927795+0000", 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1866000000000005e-05 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.927846+0000", 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0249e-05 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.927882+0000", 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6504000000000002e-05 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.604 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:19336 7.d e220/213 7:bca1948b:::benchmark_data_smithi008_354649_object19335:head v 220'1201, mlcod=220'1199)", 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:41:46.648764+0000", 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 189.859191391, 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27761422600000002, 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648764+0000", 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648764+0000", 2023-10-09T05:44:56.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648763+0000", 2023-10-09T05:44:56.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:56.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648799+0000", 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6016000000000002e-05 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648811+0000", 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.171e-05 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648818+0000", 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1749999999999999e-06 2023-10-09T05:44:56.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648882+0000", 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4158999999999993e-05 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648901+0000", 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8564e-05 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.926347+0000", 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.277445736 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.926378+0000", 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1542999999999999e-05 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.610 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45365.0:19331 7.b e220/213 ondisk, result = 0)", 2023-10-09T05:44:56.610 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:41:46.648834+0000", 2023-10-09T05:44:56.610 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 189.85912136799999, 2023-10-09T05:44:56.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27441149100000001, 2023-10-09T05:44:56.610 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.610 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:44:56.610 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648834+0000", 2023-10-09T05:44:56.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648834+0000", 2023-10-09T05:44:56.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648834+0000", 2023-10-09T05:44:56.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:56.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648837+0000", 2023-10-09T05:44:56.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.18e-06 2023-10-09T05:44:56.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648844+0000", 2023-10-09T05:44:56.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7139999999999996e-06 2023-10-09T05:44:56.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.648848+0000", 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8860000000000003e-06 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923219+0000", 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27437028000000002 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923231+0000", 2023-10-09T05:44:56.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1796e-05 2023-10-09T05:44:56.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923246+0000", 2023-10-09T05:44:56.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4983999999999999e-05 2023-10-09T05:44:56.614 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.614 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.614 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:19342 7.b 7:d7cfbd66:::benchmark_data_smithi008_354649_object19341:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:41:46.649024+0000", 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 189.85893137400001, 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27896845599999998, 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 19342 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.615 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649024+0000", 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649024+0000", 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649023+0000", 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649056+0000", 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3285999999999999e-05 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923196+0000", 2023-10-09T05:44:56.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27414010500000002 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923209+0000", 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2737e-05 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.924714+0000", 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001505585 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.924828+0000", 2023-10-09T05:44:56.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114085 2023-10-09T05:44:56.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:44:56.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.924965+0000", 2023-10-09T05:44:56.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013684600000000001 2023-10-09T05:44:56.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.927721+0000", 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027557900000000001 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.927764+0000", 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2935000000000001e-05 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.927920+0000", 2023-10-09T05:44:56.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015626999999999999 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.927959+0000", 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8549e-05 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.927992+0000", 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3748000000000003e-05 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.621 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.622 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:19327 7.5 e220/213 7:a2316b9b:::benchmark_data_smithi008_354649_object19326:head v 220'1269, mlcod=220'1267)", 2023-10-09T05:44:56.622 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:41:46.649096+0000", 2023-10-09T05:44:56.622 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 189.85885970499999, 2023-10-09T05:44:56.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27730912499999999, 2023-10-09T05:44:56.622 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.622 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.622 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649096+0000", 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649096+0000", 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649095+0000", 2023-10-09T05:44:56.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649121+0000", 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5729999999999999e-05 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923186+0000", 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27406492199999999 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923193+0000", 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2919999999999998e-06 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923265+0000", 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2052999999999996e-05 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923291+0000", 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5514e-05 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.926381+0000", 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030906530000000001 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.926405+0000", 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3461000000000001e-05 2023-10-09T05:44:56.626 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45365.0:19325 7.b e220/213 ondisk, result = 0)", 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:41:46.649182+0000", 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 189.85877324, 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27421342300000001, 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649182+0000", 2023-10-09T05:44:56.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649182+0000", 2023-10-09T05:44:56.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649182+0000", 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.649187+0000", 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5249999999999996e-06 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923157+0000", 2023-10-09T05:44:56.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27396956300000003 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923188+0000", 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1569000000000002e-05 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923268+0000", 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9340999999999998e-05 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923275+0000", 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2620000000000001e-06 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:41:46.923396+0000", 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000120758 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:125679 7.2 e238/213 7:49f6cf6f:::benchmark_data_smithi008_354649_object31183:head v 238'8076, mlcod=238'8072)", 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.960230+0000", 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 53.547724891000001, 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33872840100000001, 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.960230+0000", 2023-10-09T05:44:56.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.960230+0000", 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.960230+0000", 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967296 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.960236+0000", 2023-10-09T05:44:56.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4099999999999999e-06 2023-10-09T05:44:56.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.960794+0000", 2023-10-09T05:44:56.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00055790899999999999 2023-10-09T05:44:56.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.960803+0000", 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.91e-06 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.960890+0000", 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6230000000000006e-05 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.960913+0000", 2023-10-09T05:44:56.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3501000000000001e-05 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.298959+0000", 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33804567200000002 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:137197 7.b 7:da92626f:::benchmark_data_smithi008_354649_object42701:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-09T05:44:56.636 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:38.686989+0000", 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 17.820966326000001, 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019332947, 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 137197 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.686989+0000", 2023-10-09T05:44:56.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.686989+0000", 2023-10-09T05:44:56.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.686987+0000", 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.686996+0000", 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6600000000000001e-06 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.687017+0000", 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0673999999999999e-05 2023-10-09T05:44:56.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.687028+0000", 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1032999999999999e-05 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.691434+0000", 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004405969 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.691590+0000", 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015627300000000001 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.691751+0000", 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016158700000000001 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.696780+0000", 2023-10-09T05:44:56.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050286630000000001 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.702492+0000", 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0057116320000000003 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.706128+0000", 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036363630000000001 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.706203+0000", 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4571000000000001e-05 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:38.706322+0000", 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011936500000000001 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.644 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:137949 7.b 7:d23a4fdd:::benchmark_data_smithi008_354649_object43453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-09T05:44:56.644 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:40.511292+0000", 2023-10-09T05:44:56.644 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 15.996663493, 2023-10-09T05:44:56.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019738309999999998, 2023-10-09T05:44:56.644 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:44:56.644 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:44:56.644 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:44:56.644 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:44:56.644 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:44:56.644 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 137949 2023-10-09T05:44:56.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.511292+0000", 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.511292+0000", 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:44:56.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.511290+0000", 2023-10-09T05:44:56.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:44:56.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:44:56.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.511297+0000", 2023-10-09T05:44:56.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0299999999999996e-06 2023-10-09T05:44:56.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.511314+0000", 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6912999999999999e-05 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.511324+0000", 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0096e-05 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.511383+0000", 2023-10-09T05:44:56.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8609000000000001e-05 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.511508+0000", 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012501100000000001 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.511637+0000", 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000129386 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.515356+0000", 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037183799999999999 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.518415+0000", 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030591609999999999 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.530933+0000", 2023-10-09T05:44:56.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012517891999999999 2023-10-09T05:44:56.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:44:56.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.530989+0000", 2023-10-09T05:44:56.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5964000000000002e-05 2023-10-09T05:44:56.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:44:56.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:44:56.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:44:56.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:40.531030+0000", 2023-10-09T05:44:56.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1446999999999998e-05 2023-10-09T05:44:56.650 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.651 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.651 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.651 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:44:56.651 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:44:56.651 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:44:56.652 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:56 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T05:44:56.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:56.653 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:44:56.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:44:56.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:56.791 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:56.851 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:56.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:56.955 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:44:56.993 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:57.300 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:44:57.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:57.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:44:57.403 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:57.436 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:57.501 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:57.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:57.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:57.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:57.642 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:57 smithi191 ceph-mon[161669]: pgmap v744: 99 pgs: 5 active+recovery_wait+degraded, 91 active+clean, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 294 op/s; 18900/139989 objects degraded (13.501%); 2540/139989 objects misplaced (1.814%); 8.3 MiB/s, 142 objects/s recovering 2023-10-09T05:44:57.645 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:57.694 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:57.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:57.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:57.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:57 smithi008 ceph-mon[184418]: pgmap v744: 99 pgs: 5 active+recovery_wait+degraded, 91 active+clean, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 294 op/s; 18900/139989 objects degraded (13.501%); 2540/139989 objects misplaced (1.814%); 8.3 MiB/s, 142 objects/s recovering 2023-10-09T05:44:57.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:57 smithi008 ceph-mon[191227]: pgmap v744: 99 pgs: 5 active+recovery_wait+degraded, 91 active+clean, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 294 op/s; 18900/139989 objects degraded (13.501%); 2540/139989 objects misplaced (1.814%); 8.3 MiB/s, 142 objects/s recovering 2023-10-09T05:44:57.801 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:57.833 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:57.862 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:44:57.886 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:57.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:57.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:57.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:58.029 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:58.080 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:58.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:58.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:58.156 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:44:58.156 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:44:58.156 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T05:44:58.156 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:44:58.156 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:44:58.156 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T05:44:58.185 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:58.233 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:58.286 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:58.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:58.391 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:44:58.432 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:58.519 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:44:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 18900/139989 objects degraded (13.501%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:44:58.627 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:44:58.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 18900/139989 objects degraded (13.501%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:44:58.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 18900/139989 objects degraded (13.501%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:44:58.811 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:44:58.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:58.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:44:58.895 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:58.925 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:44:58.960 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:59.225 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:44:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:59.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:44:59.332 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:44:59.369 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:59.429 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:44:59.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:59.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:44:59.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:44:59.567 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:59.624 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:44:59.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:59.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:44:59.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:44:59.767 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:44:59.810 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:44:59.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:44:59.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:44:59.904 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:59 smithi008 ceph-mon[191227]: pgmap v745: 99 pgs: 5 active+recovery_wait+degraded, 91 active+clean, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 18849/137967 objects degraded (13.662%); 1651/137967 objects misplaced (1.197%); 14 MiB/s, 226 objects/s recovering 2023-10-09T05:44:59.904 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:44:59.904 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:59.914 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:44:59.919 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:44:59.958 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:00.018 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:00.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:00.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:00.124 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:00.152 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:00.201 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:00.203 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:59 smithi191 ceph-mon[161669]: pgmap v745: 99 pgs: 5 active+recovery_wait+degraded, 91 active+clean, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 18849/137967 objects degraded (13.662%); 1651/137967 objects misplaced (1.197%); 14 MiB/s, 226 objects/s recovering 2023-10-09T05:45:00.204 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:00.204 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:00.204 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2679311854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:00.204 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:44:59 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:00.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:00.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:00.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2679311854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:00.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:44:59 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:59 smithi008 ceph-mon[184418]: pgmap v745: 99 pgs: 5 active+recovery_wait+degraded, 91 active+clean, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 18849/137967 objects degraded (13.662%); 1651/137967 objects misplaced (1.197%); 14 MiB/s, 226 objects/s recovering 2023-10-09T05:45:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2679311854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:44:59 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:00.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:00.332 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:00.376 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:00.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:00.480 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:00.510 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:00.905 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:45:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:45:00] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.33.4" 2023-10-09T05:45:00.905 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:00.905 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[191227]: failed to encode map e250 with expected crc 2023-10-09T05:45:00.905 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[191227]: failed to encode map e250 with expected crc 2023-10-09T05:45:00.905 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:45:00.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[191227]: osdmap e250: 8 total, 8 up, 8 in 2023-10-09T05:45:00.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[191227]: failed to encode map e250 with expected crc 2023-10-09T05:45:00.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:00.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[184418]: failed to encode map e250 with expected crc 2023-10-09T05:45:00.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[184418]: failed to encode map e250 with expected crc 2023-10-09T05:45:00.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:45:00.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[184418]: osdmap e250: 8 total, 8 up, 8 in 2023-10-09T05:45:00.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[184418]: failed to encode map e250 with expected crc 2023-10-09T05:45:00.907 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[184418]: failed to encode map e250 with expected crc 2023-10-09T05:45:01.173 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:01.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:01.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:01.176 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:00 smithi008 ceph-mon[191227]: failed to encode map e250 with expected crc 2023-10-09T05:45:01.210 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:00 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:01.211 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:00 smithi191 ceph-mon[161669]: failed to encode map e250 with expected crc 2023-10-09T05:45:01.211 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:00 smithi191 ceph-mon[161669]: failed to encode map e250 with expected crc 2023-10-09T05:45:01.211 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:00 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:45:01.211 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:00 smithi191 ceph-mon[161669]: osdmap e250: 8 total, 8 up, 8 in 2023-10-09T05:45:01.211 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:00 smithi191 ceph-mon[161669]: failed to encode map e250 with expected crc 2023-10-09T05:45:01.212 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:00 smithi191 ceph-mon[161669]: failed to encode map e250 with expected crc 2023-10-09T05:45:01.277 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:01.314 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:01.369 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:01.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:01.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:01.469 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:45:01.469 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "" 2023-10-09T05:45:01.469 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:45:01.475 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:01.514 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:01.695 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:01.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:01.803 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:01.816 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:01.832 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:02.043 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:02.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:02.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:02.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:01 smithi191 ceph-mon[161669]: pgmap v747: 99 pgs: 5 active+recovery_wait+degraded, 91 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 18693/134898 objects degraded (13.857%); 1594/134898 objects misplaced (1.182%); 14 MiB/s, 229 objects/s recovering 2023-10-09T05:45:02.102 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:01 smithi008 ceph-mon[184418]: pgmap v747: 99 pgs: 5 active+recovery_wait+degraded, 91 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 18693/134898 objects degraded (13.857%); 1594/134898 objects misplaced (1.182%); 14 MiB/s, 229 objects/s recovering 2023-10-09T05:45:02.104 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:01 smithi008 ceph-mon[191227]: pgmap v747: 99 pgs: 5 active+recovery_wait+degraded, 91 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 18693/134898 objects degraded (13.857%); 1594/134898 objects misplaced (1.182%); 14 MiB/s, 229 objects/s recovering 2023-10-09T05:45:02.150 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:02.206 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:02.351 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:02.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:02.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:02.368 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:45:02.399 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:45:02.436 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:45:02.463 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:02.464 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:45:02.490 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:45:02.497 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:02.522 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:45:02.551 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:45:02.557 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:02.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:02.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:02.578 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:45:02.580 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:45:02.580 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:45:02.580 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:45:02.580 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:45:02.580 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:45:02.580 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:45:02.581 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:45:02.581 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:45:02.666 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:02.720 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:02.916 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:02 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2975302855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:02.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:02 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 18693/134898 objects degraded (13.857%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:03.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:02 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2975302855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:03.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:02 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 18693/134898 objects degraded (13.857%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:02 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2975302855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:02 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 18693/134898 objects degraded (13.857%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:03.715 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:03.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:03.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:03.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:03.850 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:03.854 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:45:03.902 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:03.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:03.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:04.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:04.037 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:04.088 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:04.099 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:03 smithi191 ceph-mon[161669]: pgmap v748: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 18641/132930 objects degraded (14.023%); 1362/132930 objects misplaced (1.025%); 10 MiB/s, 201 objects/s recovering 2023-10-09T05:45:04.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:04.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:04.187 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:03 smithi008 ceph-mon[184418]: pgmap v748: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 18641/132930 objects degraded (14.023%); 1362/132930 objects misplaced (1.025%); 10 MiB/s, 201 objects/s recovering 2023-10-09T05:45:04.188 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:03 smithi008 ceph-mon[191227]: pgmap v748: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 18641/132930 objects degraded (14.023%); 1362/132930 objects misplaced (1.025%); 10 MiB/s, 201 objects/s recovering 2023-10-09T05:45:04.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:04.230 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:04.289 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:04.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:04.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:04.395 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:04.426 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:04.478 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:04.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:04.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:04.584 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:04.614 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:04.665 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:04.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:04.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:04.772 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:04.805 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:04.943 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:04.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:04.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:05.049 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:05.080 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:05.432 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:05.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:05.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:05.541 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:05.577 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:05.639 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:05.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:05.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:05.746 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:05.785 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:05.854 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:05.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:05.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:05.906 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:05.919 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:05 smithi008 ceph-mon[184418]: pgmap v749: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 15321/130617 objects degraded (11.730%); 3923/130617 objects misplaced (3.003%); 11 MiB/s, 223 objects/s recovering 2023-10-09T05:45:05.943 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:45:05.944 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:45:05.965 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:05.996 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:06.049 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:06.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:06.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:06.154 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:06.181 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:05 smithi191 ceph-mon[161669]: pgmap v749: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 15321/130617 objects degraded (11.730%); 3923/130617 objects misplaced (3.003%); 11 MiB/s, 223 objects/s recovering 2023-10-09T05:45:06.183 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:06.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:05 smithi008 ceph-mon[191227]: pgmap v749: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 15321/130617 objects degraded (11.730%); 3923/130617 objects misplaced (3.003%); 11 MiB/s, 223 objects/s recovering 2023-10-09T05:45:06.241 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:06.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:06.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:06.348 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:06.379 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:07.111 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:45:07.111 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:45:07.112 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:45:07.112 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T05:45:07.196 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:07.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:07.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:07.229 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:45:07.300 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:07.347 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:07.434 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:45:07.654 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:07.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:07.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:07.758 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:07.792 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:08.144 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:07 smithi008 ceph-mon[184418]: pgmap v750: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 15262/129645 objects degraded (11.772%); 3867/129645 objects misplaced (2.983%); 12 MiB/s, 223 objects/s recovering 2023-10-09T05:45:08.144 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:07 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15262/129645 objects degraded (11.772%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:08.144 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:07 smithi008 ceph-mon[191227]: pgmap v750: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 15262/129645 objects degraded (11.772%); 3867/129645 objects misplaced (2.983%); 12 MiB/s, 223 objects/s recovering 2023-10-09T05:45:08.144 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:07 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15262/129645 objects degraded (11.772%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:08.213 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:08.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:08.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:07 smithi191 ceph-mon[161669]: pgmap v750: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 15262/129645 objects degraded (11.772%); 3867/129645 objects misplaced (2.983%); 12 MiB/s, 223 objects/s recovering 2023-10-09T05:45:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:07 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15262/129645 objects degraded (11.772%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:08.318 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:08.355 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:08.408 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:08.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:08.511 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:08.656 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:08.722 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:08.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:08.828 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:08.860 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:08.909 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:08.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:08.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:09.014 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:09.043 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:09.065 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:09.094 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:09.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:09.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:09.184 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:09.199 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:09.245 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:09.369 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:09.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:09.381 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:09.475 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:09.501 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:09.548 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:09.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:09.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:09.653 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:09.680 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:09.730 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:09.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:09.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:09.835 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:09.930 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:10.051 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:09 smithi008 ceph-mon[184418]: pgmap v751: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 15220/127746 objects degraded (11.914%); 3867/127746 objects misplaced (3.027%); 5.9 MiB/s, 134 objects/s recovering 2023-10-09T05:45:10.051 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:09 smithi008 ceph-mon[191227]: pgmap v751: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 15220/127746 objects degraded (11.914%); 3867/127746 objects misplaced (3.027%); 5.9 MiB/s, 134 objects/s recovering 2023-10-09T05:45:10.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:09 smithi191 ceph-mon[161669]: pgmap v751: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 15220/127746 objects degraded (11.914%); 3867/127746 objects misplaced (3.027%); 5.9 MiB/s, 134 objects/s recovering 2023-10-09T05:45:10.404 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:10.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:10.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:10.509 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:10.618 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:10.694 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:45:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:45:10] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.33.4" 2023-10-09T05:45:10.943 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:45:10.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:10 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/486857093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:45:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:10 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:45:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:10 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/486857093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:45:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:10 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:45:11.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:10 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/486857093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:45:11.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:10 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:45:12.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[184418]: pgmap v752: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 359 op/s; 15071/124665 objects degraded (12.089%); 3585/124665 objects misplaced (2.876%); 7.7 MiB/s, 161 objects/s recovering 2023-10-09T05:45:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:45:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[184418]: osdmap e251: 8 total, 8 up, 8 in 2023-10-09T05:45:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[184418]: failed to encode map e251 with expected crc 2023-10-09T05:45:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[184418]: failed to encode map e251 with expected crc 2023-10-09T05:45:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[184418]: failed to encode map e251 with expected crc 2023-10-09T05:45:12.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[191227]: pgmap v752: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 359 op/s; 15071/124665 objects degraded (12.089%); 3585/124665 objects misplaced (2.876%); 7.7 MiB/s, 161 objects/s recovering 2023-10-09T05:45:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:45:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[191227]: osdmap e251: 8 total, 8 up, 8 in 2023-10-09T05:45:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[191227]: failed to encode map e251 with expected crc 2023-10-09T05:45:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[191227]: failed to encode map e251 with expected crc 2023-10-09T05:45:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:11 smithi008 ceph-mon[191227]: failed to encode map e251 with expected crc 2023-10-09T05:45:12.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:11 smithi191 ceph-mon[161669]: pgmap v752: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 359 op/s; 15071/124665 objects degraded (12.089%); 3585/124665 objects misplaced (2.876%); 7.7 MiB/s, 161 objects/s recovering 2023-10-09T05:45:12.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:11 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:12.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:11 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:45:12.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:11 smithi191 ceph-mon[161669]: osdmap e251: 8 total, 8 up, 8 in 2023-10-09T05:45:12.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:11 smithi191 ceph-mon[161669]: failed to encode map e251 with expected crc 2023-10-09T05:45:12.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:11 smithi191 ceph-mon[161669]: failed to encode map e251 with expected crc 2023-10-09T05:45:12.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:11 smithi191 ceph-mon[161669]: failed to encode map e251 with expected crc 2023-10-09T05:45:13.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:12 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15071/124665 objects degraded (12.089%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:12 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15071/124665 objects degraded (12.089%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:12 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15071/124665 objects degraded (12.089%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:14.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:13 smithi008 ceph-mon[184418]: pgmap v754: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 15011/122730 objects degraded (12.231%); 3585/122730 objects misplaced (2.921%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T05:45:14.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:14.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:13 smithi008 ceph-mon[191227]: pgmap v754: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 15011/122730 objects degraded (12.231%); 3585/122730 objects misplaced (2.921%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T05:45:14.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:14.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:13 smithi191 ceph-mon[161669]: pgmap v754: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 15011/122730 objects degraded (12.231%); 3585/122730 objects misplaced (2.921%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T05:45:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:14.681 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:14.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:14.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:14.793 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:14.827 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:14.895 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:14.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:14.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:15.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:15.043 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:15.099 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:15.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:15.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:15.205 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:15.283 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:15.316 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:45:15.345 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:45:15.377 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:45:15.406 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:45:15.431 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:45:15.456 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:45:15.485 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:45:15.511 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:45:15.512 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:45:15.512 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:45:15.512 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:45:15.512 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:45:15.513 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:45:15.513 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:45:15.513 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:45:15.513 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:45:15.889 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:45:15.889 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:45:15.889 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:45:15.889 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:45:15.889 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:45:15.924 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:15.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:16.029 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:16.057 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:16.110 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:16.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:16.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:16.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:15 smithi008 ceph-mon[191227]: pgmap v755: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 14913/120315 objects degraded (12.395%); 3354/120315 objects misplaced (2.788%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:45:16.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/800449642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:16.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:15 smithi008 ceph-mon[184418]: pgmap v755: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 14913/120315 objects degraded (12.395%); 3354/120315 objects misplaced (2.788%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:45:16.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/800449642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:16.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:16 smithi008 conmon[184394]: 2023-10-09T05:45:16.067+0000 7f8a66521700 -1 mon.a@0(leader).osd e251 definitely_dead 0 2023-10-09T05:45:16.217 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:16.269 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:16.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:15 smithi191 ceph-mon[161669]: pgmap v755: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 14913/120315 objects degraded (12.395%); 3354/120315 objects misplaced (2.788%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:45:16.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/800449642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:16.855 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:45:16.991 INFO:teuthology.orchestra.run.smithi008.stderr:marked out osd.1. 2023-10-09T05:45:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:16 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/579926805' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:45:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:16 smithi008 ceph-mon[184418]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:45:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:16 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/579926805' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:45:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:16 smithi008 ceph-mon[191227]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:45:17.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:16 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/579926805' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:45:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:16 smithi191 ceph-mon[161669]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:45:17.844 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:17.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:17.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:17.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:17.959 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:17 smithi191 ceph-mon[161669]: pgmap v756: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 14861/119199 objects degraded (12.467%); 3297/119199 objects misplaced (2.766%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:45:17.959 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:17 smithi191 ceph-mon[161669]: failed to encode map e252 with expected crc 2023-10-09T05:45:17.959 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/579926805' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:45:17.959 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:17 smithi191 ceph-mon[161669]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T05:45:17.959 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:17 smithi191 ceph-mon[161669]: failed to encode map e252 with expected crc 2023-10-09T05:45:17.959 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:17 smithi191 ceph-mon[161669]: failed to encode map e252 with expected crc 2023-10-09T05:45:17.959 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:17 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14861/119199 objects degraded (12.467%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:17.976 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:18.019 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:18.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:18.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:18.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:18.150 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:18.193 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:18.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:18.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:18.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[191227]: pgmap v756: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 14861/119199 objects degraded (12.467%); 3297/119199 objects misplaced (2.766%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:45:18.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[191227]: failed to encode map e252 with expected crc 2023-10-09T05:45:18.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/579926805' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:45:18.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[191227]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T05:45:18.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[191227]: failed to encode map e252 with expected crc 2023-10-09T05:45:18.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[191227]: failed to encode map e252 with expected crc 2023-10-09T05:45:18.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14861/119199 objects degraded (12.467%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:18.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[191227]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T05:45:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[184418]: pgmap v756: 99 pgs: 5 active+recovery_wait+degraded, 92 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 14861/119199 objects degraded (12.467%); 3297/119199 objects misplaced (2.766%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:45:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[184418]: failed to encode map e252 with expected crc 2023-10-09T05:45:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/579926805' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:45:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[184418]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T05:45:18.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[184418]: failed to encode map e252 with expected crc 2023-10-09T05:45:18.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[184418]: failed to encode map e252 with expected crc 2023-10-09T05:45:18.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14861/119199 objects degraded (12.467%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:18.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:17 smithi008 ceph-mon[184418]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T05:45:18.299 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:18.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:17 smithi191 ceph-mon[161669]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T05:45:18.325 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:18.367 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:18.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:18.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:18.474 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:18.503 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:19.176 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:19.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:19.193 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[184418]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.193 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[184418]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.193 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[184418]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.193 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[184418]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[184418]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[184418]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[184418]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[184418]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[184418]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T05:45:19.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[191227]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[191227]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[191227]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[191227]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.196 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[191227]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.196 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[191227]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.196 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[191227]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.196 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[191227]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.196 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:18 smithi008 ceph-mon[191227]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T05:45:19.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:19.298 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:18 smithi191 ceph-mon[161669]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:18 smithi191 ceph-mon[161669]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:18 smithi191 ceph-mon[161669]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:18 smithi191 ceph-mon[161669]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:18 smithi191 ceph-mon[161669]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:18 smithi191 ceph-mon[161669]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:18 smithi191 ceph-mon[161669]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:18 smithi191 ceph-mon[161669]: failed to encode map e253 with expected crc 2023-10-09T05:45:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:18 smithi191 ceph-mon[161669]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T05:45:19.359 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:45:19.434 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:19.559 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:19.587 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:19.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:19.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:19.695 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:19.746 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:19.936 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:19.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:19.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:20.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:20.074 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:20.146 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:20.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:20.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:20.258 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:20.286 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: pgmap v759: 99 pgs: 5 peering, 5 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.4 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 14803/116955 objects degraded (12.657%); 3297/116955 objects misplaced (2.819%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:45:20.286 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: 7.1 scrub starts 2023-10-09T05:45:20.286 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.286 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.286 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.286 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.286 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.286 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: 7.2 starting backfill to osd.7 from (0'0,0'0] MAX to 251'8909 2023-10-09T05:45:20.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: 7.12 starting backfill to osd.7 from (0'0,0'0] MAX to 251'8853 2023-10-09T05:45:20.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: 7.d starting backfill to osd.3 from (0'0,0'0] MAX to 251'9307 2023-10-09T05:45:20.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 251'9177 2023-10-09T05:45:20.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:19 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:45:20.288 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:20.332 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: pgmap v759: 99 pgs: 5 peering, 5 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.4 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 14803/116955 objects degraded (12.657%); 3297/116955 objects misplaced (2.819%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:45:20.332 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: 7.1 scrub starts 2023-10-09T05:45:20.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: 7.2 starting backfill to osd.7 from (0'0,0'0] MAX to 251'8909 2023-10-09T05:45:20.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: 7.12 starting backfill to osd.7 from (0'0,0'0] MAX to 251'8853 2023-10-09T05:45:20.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: 7.d starting backfill to osd.3 from (0'0,0'0] MAX to 251'9307 2023-10-09T05:45:20.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 251'9177 2023-10-09T05:45:20.334 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:45:20.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: pgmap v759: 99 pgs: 5 peering, 5 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped; 2.4 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 14803/116955 objects degraded (12.657%); 3297/116955 objects misplaced (2.819%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:45:20.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: 7.1 scrub starts 2023-10-09T05:45:20.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: 7.2 starting backfill to osd.7 from (0'0,0'0] MAX to 251'8909 2023-10-09T05:45:20.335 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: failed to encode map e254 with expected crc 2023-10-09T05:45:20.335 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: 7.12 starting backfill to osd.7 from (0'0,0'0] MAX to 251'8853 2023-10-09T05:45:20.335 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: 7.d starting backfill to osd.3 from (0'0,0'0] MAX to 251'9307 2023-10-09T05:45:20.335 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 251'9177 2023-10-09T05:45:20.335 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:19 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:45:20.343 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:20.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:20.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:20.450 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:20.492 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:20.558 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:20.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:20.664 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:20.702 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:45:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:45:20] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.33.4" 2023-10-09T05:45:20.707 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:20.774 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:20.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:20.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:20.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:20.912 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:20.914 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:45:20.973 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:20.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:20.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:21.081 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:21.108 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:21.163 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:21.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:21.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:21.267 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:21.319 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:21.672 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:21.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:21.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:21.775 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:21.805 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:21.860 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:21.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:21.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:21.967 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:22.019 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:22.137 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:22.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:22.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:22.161 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:22.243 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:22.252 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:21 smithi008 ceph-mon[184418]: pgmap v761: 99 pgs: 6 remapped+peering, 5 peering, 7 active+recovery_wait+degraded, 75 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 25034/114534 objects degraded (21.857%); 2283/114534 objects misplaced (1.993%); 5.9 MiB/s, 91 objects/s recovering 2023-10-09T05:45:22.252 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:21 smithi008 ceph-mon[191227]: pgmap v761: 99 pgs: 6 remapped+peering, 5 peering, 7 active+recovery_wait+degraded, 75 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 25034/114534 objects degraded (21.857%); 2283/114534 objects misplaced (1.993%); 5.9 MiB/s, 91 objects/s recovering 2023-10-09T05:45:22.269 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:21 smithi191 ceph-mon[161669]: pgmap v761: 99 pgs: 6 remapped+peering, 5 peering, 7 active+recovery_wait+degraded, 75 active+clean, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 25034/114534 objects degraded (21.857%); 2283/114534 objects misplaced (1.993%); 5.9 MiB/s, 91 objects/s recovering 2023-10-09T05:45:22.270 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:22.324 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:22.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:22.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:22.430 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:22.452 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:22.470 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:22.601 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:22.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:22.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:22.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:22.734 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:22.792 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:22.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:22.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:22.899 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:22.942 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:22.990 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:22 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1323878697' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:22.990 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:22 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 25034/114534 objects degraded (21.857%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:23.016 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:45:23.110 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:23.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:23.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:22 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1323878697' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:23.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:22 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 25034/114534 objects degraded (21.857%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:23.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:22 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1323878697' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:23.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:22 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 25034/114534 objects degraded (21.857%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:23.217 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:23.254 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:24.294 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:45:24.294 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:45:24.294 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T05:45:24.294 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:45:24.294 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:45:24.295 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T05:45:24.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:23 smithi191 ceph-mon[161669]: pgmap v762: 99 pgs: 6 remapped+peering, 6 active+recovery_wait+degraded, 78 active+clean, 1 active+recovering, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 23372/113049 objects degraded (20.674%); 5245/113049 objects misplaced (4.640%); 11 MiB/s, 176 objects/s recovering 2023-10-09T05:45:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:23 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:45:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:23 smithi191 ceph-mon[161669]: failed to encode map e255 with expected crc 2023-10-09T05:45:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:23 smithi191 ceph-mon[161669]: failed to encode map e255 with expected crc 2023-10-09T05:45:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:23 smithi191 ceph-mon[161669]: failed to encode map e255 with expected crc 2023-10-09T05:45:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1323878697' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:45:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:23 smithi191 ceph-mon[161669]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T05:45:24.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[184418]: pgmap v762: 99 pgs: 6 remapped+peering, 6 active+recovery_wait+degraded, 78 active+clean, 1 active+recovering, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 23372/113049 objects degraded (20.674%); 5245/113049 objects misplaced (4.640%); 11 MiB/s, 176 objects/s recovering 2023-10-09T05:45:24.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:45:24.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[184418]: failed to encode map e255 with expected crc 2023-10-09T05:45:24.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[184418]: failed to encode map e255 with expected crc 2023-10-09T05:45:24.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[184418]: failed to encode map e255 with expected crc 2023-10-09T05:45:24.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1323878697' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:45:24.326 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[184418]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T05:45:24.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[191227]: pgmap v762: 99 pgs: 6 remapped+peering, 6 active+recovery_wait+degraded, 78 active+clean, 1 active+recovering, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 23372/113049 objects degraded (20.674%); 5245/113049 objects misplaced (4.640%); 11 MiB/s, 176 objects/s recovering 2023-10-09T05:45:24.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:45:24.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[191227]: failed to encode map e255 with expected crc 2023-10-09T05:45:24.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[191227]: failed to encode map e255 with expected crc 2023-10-09T05:45:24.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[191227]: failed to encode map e255 with expected crc 2023-10-09T05:45:24.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1323878697' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:45:24.327 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:23 smithi008 ceph-mon[191227]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T05:45:25.030 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:25.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:25.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:25.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:25.171 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:25.246 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:25.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:25.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:25.290 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:25 smithi008 ceph-mon[191227]: pgmap v764: 99 pgs: 6 active+recovery_wait+degraded, 78 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 35011/111096 objects degraded (31.514%); 7556/111096 objects misplaced (6.801%); 11 MiB/s, 175 objects/s recovering 2023-10-09T05:45:25.290 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:25 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:45:25.290 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:25 smithi008 ceph-mon[184418]: pgmap v764: 99 pgs: 6 active+recovery_wait+degraded, 78 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 35011/111096 objects degraded (31.514%); 7556/111096 objects misplaced (6.801%); 11 MiB/s, 175 objects/s recovering 2023-10-09T05:45:25.291 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:25 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:45:25.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:25 smithi191 ceph-mon[161669]: pgmap v764: 99 pgs: 6 active+recovery_wait+degraded, 78 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 35011/111096 objects degraded (31.514%); 7556/111096 objects misplaced (6.801%); 11 MiB/s, 175 objects/s recovering 2023-10-09T05:45:25.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:25 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:45:25.357 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:25.392 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:26.057 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:26.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:26.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:26.162 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:26.198 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:26.259 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:26.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:26.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:26.369 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:26.408 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:26.509 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:26.621 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:45:26.638 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:26.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:26.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:26.655 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:45:26.689 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:45:26.719 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:45:26.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:26.745 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:45:26.772 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:45:26.774 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:26.800 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:45:26.827 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:45:26.834 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:45:26.834 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:45:26.834 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:45:26.834 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:45:26.835 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:45:26.835 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:45:26.835 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:45:26.835 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:45:26.845 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:26.852 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:45:26.853 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:45:26.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:26.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:26.955 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:26.985 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:27.036 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:27.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:27.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:27.141 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:27.186 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:27.317 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:45:27.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:27 smithi008 ceph-mon[191227]: pgmap v765: 99 pgs: 6 active+recovery_wait+degraded, 78 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 34896/109749 objects degraded (31.796%); 7450/109749 objects misplaced (6.788%); 9.8 MiB/s, 154 objects/s recovering 2023-10-09T05:45:27.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/638872660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:27.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:27 smithi008 ceph-mon[184418]: pgmap v765: 99 pgs: 6 active+recovery_wait+degraded, 78 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 34896/109749 objects degraded (31.796%); 7450/109749 objects misplaced (6.788%); 9.8 MiB/s, 154 objects/s recovering 2023-10-09T05:45:27.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/638872660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:27.504 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:27.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:27.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:27.608 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:27.636 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:27 smithi191 ceph-mon[161669]: pgmap v765: 99 pgs: 6 active+recovery_wait+degraded, 78 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 300 op/s; 34896/109749 objects degraded (31.796%); 7450/109749 objects misplaced (6.788%); 9.8 MiB/s, 154 objects/s recovering 2023-10-09T05:45:27.636 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/638872660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:27.640 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:27.702 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:27.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:27.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:27.810 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:27.845 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:27.889 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:27.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:27.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:27.995 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:28.065 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:28.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 34896/109749 objects degraded (31.796%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:28.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 34896/109749 objects degraded (31.796%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 34896/109749 objects degraded (31.796%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:28.997 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:29.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:29.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:29.103 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:29.159 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:29.164 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:45:29.289 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:29.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:29.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:29.395 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:29.463 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:29.560 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:29.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:29.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:29.668 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:29.709 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:30.031 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:30.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:30.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:30.136 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:30.160 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:29 smithi191 ceph-mon[161669]: pgmap v766: 99 pgs: 6 active+recovery_wait+degraded, 79 active+clean, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 279 op/s; 34896/107670 objects degraded (32.410%); 6712/107670 objects misplaced (6.234%); 10 MiB/s, 209 objects/s recovering 2023-10-09T05:45:30.160 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:30.160 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:30.161 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:30.162 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:30.212 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:29 smithi008 ceph-mon[184418]: pgmap v766: 99 pgs: 6 active+recovery_wait+degraded, 79 active+clean, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 279 op/s; 34896/107670 objects degraded (32.410%); 6712/107670 objects misplaced (6.234%); 10 MiB/s, 209 objects/s recovering 2023-10-09T05:45:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:29 smithi008 ceph-mon[191227]: pgmap v766: 99 pgs: 6 active+recovery_wait+degraded, 79 active+clean, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 279 op/s; 34896/107670 objects degraded (32.410%); 6712/107670 objects misplaced (6.234%); 10 MiB/s, 209 objects/s recovering 2023-10-09T05:45:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:30.219 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:30.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:30.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:30.321 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:30.352 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:30.418 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:30.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:30.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:30.528 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:30.567 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:30.619 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:30.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:30.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:30.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:45:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:45:30] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T05:45:30.724 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:30.732 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:45:30.732 INFO:teuthology.orchestra.run.smithi008.stdout: "success": "" 2023-10-09T05:45:30.733 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:45:30.778 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:31.096 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:45:32.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:31 smithi008 ceph-mon[184418]: pgmap v767: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 32605/103899 objects degraded (31.381%); 7345/103899 objects misplaced (7.069%); 12 MiB/s, 230 objects/s recovering 2023-10-09T05:45:32.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:31 smithi008 ceph-mon[184418]: 7.8 scrub starts 2023-10-09T05:45:32.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:31 smithi008 ceph-mon[191227]: pgmap v767: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 32605/103899 objects degraded (31.381%); 7345/103899 objects misplaced (7.069%); 12 MiB/s, 230 objects/s recovering 2023-10-09T05:45:32.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:31 smithi008 ceph-mon[191227]: 7.8 scrub starts 2023-10-09T05:45:32.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:31 smithi191 ceph-mon[161669]: pgmap v767: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 32605/103899 objects degraded (31.381%); 7345/103899 objects misplaced (7.069%); 12 MiB/s, 230 objects/s recovering 2023-10-09T05:45:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:31 smithi191 ceph-mon[161669]: 7.8 scrub starts 2023-10-09T05:45:32.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:32 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 32605/103899 objects degraded (31.381%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:32.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:32 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 32605/103899 objects degraded (31.381%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:33.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:32 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 32605/103899 objects degraded (31.381%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:33.348 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:33.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:33.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:33.452 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:33.490 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:33.548 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:33.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:33.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:33.655 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:33.679 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:33.739 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:33.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:33.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:33.846 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:33.886 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:33.945 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:33.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:33.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:33 smithi191 ceph-mon[161669]: pgmap v768: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 32605/101649 objects degraded (32.076%); 7226/101649 objects misplaced (7.109%); 7.8 MiB/s, 168 objects/s recovering 2023-10-09T05:45:34.053 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:34.096 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:34.198 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:33 smithi008 ceph-mon[184418]: pgmap v768: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 32605/101649 objects degraded (32.076%); 7226/101649 objects misplaced (7.109%); 7.8 MiB/s, 168 objects/s recovering 2023-10-09T05:45:34.198 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:33 smithi008 ceph-mon[191227]: pgmap v768: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 32605/101649 objects degraded (32.076%); 7226/101649 objects misplaced (7.109%); 7.8 MiB/s, 168 objects/s recovering 2023-10-09T05:45:34.213 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:34.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:34.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:34.321 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:34.353 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:34.389 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:34.407 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:34.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:34.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:34.512 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:34.555 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:34.754 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:45:34.754 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:45:34.883 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:34.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:34.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:34.987 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:35.041 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:35.533 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:35.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:35.637 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:35.668 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:35.925 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:35 smithi008 ceph-mon[191227]: pgmap v769: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 32532/99552 objects degraded (32.678%); 6405/99552 objects misplaced (6.434%); 11 MiB/s, 218 objects/s recovering 2023-10-09T05:45:35.926 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:35 smithi008 ceph-mon[191227]: 7.18 scrub starts 2023-10-09T05:45:35.926 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2396788283' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:45:35.959 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:45:36.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:35 smithi008 ceph-mon[184418]: pgmap v769: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 32532/99552 objects degraded (32.678%); 6405/99552 objects misplaced (6.434%); 11 MiB/s, 218 objects/s recovering 2023-10-09T05:45:36.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:35 smithi008 ceph-mon[184418]: 7.18 scrub starts 2023-10-09T05:45:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2396788283' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:45:36.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:35 smithi191 ceph-mon[161669]: pgmap v769: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 32532/99552 objects degraded (32.678%); 6405/99552 objects misplaced (6.434%); 11 MiB/s, 218 objects/s recovering 2023-10-09T05:45:36.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:35 smithi191 ceph-mon[161669]: 7.18 scrub starts 2023-10-09T05:45:36.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2396788283' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:45:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[184418]: failed to encode map e256 with expected crc 2023-10-09T05:45:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[184418]: failed to encode map e256 with expected crc 2023-10-09T05:45:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[184418]: failed to encode map e256 with expected crc 2023-10-09T05:45:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2396788283' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:45:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[184418]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T05:45:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[191227]: failed to encode map e256 with expected crc 2023-10-09T05:45:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[191227]: failed to encode map e256 with expected crc 2023-10-09T05:45:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[191227]: failed to encode map e256 with expected crc 2023-10-09T05:45:37.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2396788283' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:45:37.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:36 smithi008 ceph-mon[191227]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T05:45:37.278 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:37.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:37.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:37.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:36 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:36 smithi191 ceph-mon[161669]: failed to encode map e256 with expected crc 2023-10-09T05:45:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:36 smithi191 ceph-mon[161669]: failed to encode map e256 with expected crc 2023-10-09T05:45:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:36 smithi191 ceph-mon[161669]: failed to encode map e256 with expected crc 2023-10-09T05:45:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2396788283' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:45:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:36 smithi191 ceph-mon[161669]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T05:45:37.382 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:37.415 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:38.052 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:45:38.091 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:38.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:38.195 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:37 smithi008 ceph-mon[191227]: pgmap v771: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 382 op/s; 32422/98277 objects degraded (32.990%); 6302/98277 objects misplaced (6.412%); 12 MiB/s, 235 objects/s recovering 2023-10-09T05:45:38.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:37 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 32422/98277 objects degraded (32.990%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:38.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:37 smithi008 ceph-mon[184418]: pgmap v771: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 382 op/s; 32422/98277 objects degraded (32.990%); 6302/98277 objects misplaced (6.412%); 12 MiB/s, 235 objects/s recovering 2023-10-09T05:45:38.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:37 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 32422/98277 objects degraded (32.990%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:38.226 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:37 smithi191 ceph-mon[161669]: pgmap v771: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 382 op/s; 32422/98277 objects degraded (32.990%); 6302/98277 objects misplaced (6.412%); 12 MiB/s, 235 objects/s recovering 2023-10-09T05:45:38.226 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:37 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 32422/98277 objects degraded (32.990%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:38.229 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:38.288 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:38.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:38.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:38.395 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:38.433 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:38.581 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:45:38.581 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:45:38.581 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T05:45:38.581 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:45:38.668 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:38.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:38.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:38.774 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:38.822 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:38.904 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:38.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:38.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:38.912 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:38.936 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:45:38.967 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:45:39.005 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:45:39.011 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:39.046 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:45:39.061 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:39.079 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:45:39.113 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:45:39.148 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:45:39.180 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:45:39.181 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:45:39.181 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:45:39.182 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:45:39.182 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:45:39.182 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:45:39.182 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:45:39.182 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:45:39.182 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:45:39.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:38 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/738594817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:39.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:38 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/738594817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:39.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:38 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/738594817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:39.302 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:39.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:39.407 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:39.451 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:39.666 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:39.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:39.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:39.772 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:39.806 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:39.878 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:39.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:39.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:39.984 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:40.017 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:40.106 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:45:40.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:39 smithi008 ceph-mon[184418]: pgmap v772: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 32422/96294 objects degraded (33.670%); 6189/96294 objects misplaced (6.427%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:45:40.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:39 smithi008 ceph-mon[191227]: pgmap v772: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 32422/96294 objects degraded (33.670%); 6189/96294 objects misplaced (6.427%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:45:40.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:39 smithi191 ceph-mon[161669]: pgmap v772: 99 pgs: 5 active+recovery_wait+degraded, 79 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 32422/96294 objects degraded (33.670%); 6189/96294 objects misplaced (6.427%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:45:40.578 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:40.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:40.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:40.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:40.713 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:40.771 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:40.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:40.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:40.879 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:40.910 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:45:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:45:40] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-10-09T05:45:40.914 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:41.491 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:41.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:41.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:41.596 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:41.682 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:41.975 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:42.066 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:42.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:42.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:42.183 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:42.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:41 smithi008 ceph-mon[184418]: pgmap v773: 99 pgs: 5 active+recovery_wait+degraded, 80 active+clean, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 352 op/s; 32243/93309 objects degraded (34.555%); 5841/93309 objects misplaced (6.260%); 7.7 MiB/s, 223 objects/s recovering 2023-10-09T05:45:42.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:41 smithi008 ceph-mon[191227]: pgmap v773: 99 pgs: 5 active+recovery_wait+degraded, 80 active+clean, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 352 op/s; 32243/93309 objects degraded (34.555%); 5841/93309 objects misplaced (6.260%); 7.7 MiB/s, 223 objects/s recovering 2023-10-09T05:45:42.240 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:42.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:41 smithi191 ceph-mon[161669]: pgmap v773: 99 pgs: 5 active+recovery_wait+degraded, 80 active+clean, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 352 op/s; 32243/93309 objects degraded (34.555%); 5841/93309 objects misplaced (6.260%); 7.7 MiB/s, 223 objects/s recovering 2023-10-09T05:45:42.387 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:42.545 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:42.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:42.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:42.652 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:42.704 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:42.820 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:45:42.821 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:45:42.821 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:45:42.821 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:45:42.821 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:45:43.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:42 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 32243/93309 objects degraded (34.555%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:43.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:42 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 32243/93309 objects degraded (34.555%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:42 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 32243/93309 objects degraded (34.555%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:44.113 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:45:44.185 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:44.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:44.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:44.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:43 smithi008 ceph-mon[184418]: pgmap v774: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 347 op/s; 28662/91233 objects degraded (31.416%); 7103/91233 objects misplaced (7.786%); 9.7 MiB/s, 255 objects/s recovering 2023-10-09T05:45:44.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:44.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:43 smithi008 ceph-mon[191227]: pgmap v774: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 347 op/s; 28662/91233 objects degraded (31.416%); 7103/91233 objects misplaced (7.786%); 9.7 MiB/s, 255 objects/s recovering 2023-10-09T05:45:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:44.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:44.294 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:44.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:43 smithi191 ceph-mon[161669]: pgmap v774: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 347 op/s; 28662/91233 objects degraded (31.416%); 7103/91233 objects misplaced (7.786%); 9.7 MiB/s, 255 objects/s recovering 2023-10-09T05:45:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:44.366 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:44.548 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:44.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:44.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:44.652 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:44.681 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:45.262 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:45:45.495 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:45.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:45.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:45.602 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:45.638 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:45.713 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:45.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:45.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:45.819 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:45.854 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:45.949 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:45 smithi191 ceph-mon[161669]: pgmap v775: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 28602/89085 objects degraded (32.106%); 6985/89085 objects misplaced (7.841%); 5.2 MiB/s, 182 objects/s recovering 2023-10-09T05:45:46.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:45 smithi008 ceph-mon[184418]: pgmap v775: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 28602/89085 objects degraded (32.106%); 6985/89085 objects misplaced (7.841%); 5.2 MiB/s, 182 objects/s recovering 2023-10-09T05:45:46.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:45 smithi008 ceph-mon[191227]: pgmap v775: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 28602/89085 objects degraded (32.106%); 6985/89085 objects misplaced (7.841%); 5.2 MiB/s, 182 objects/s recovering 2023-10-09T05:45:46.408 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:46.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:46.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:46.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:46.545 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:46.601 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:46.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:46.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:46.710 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:46.757 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:46.905 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:46.941 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:46.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:46.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:47.049 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:47.077 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:47.134 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:47.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:47.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:47.189 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:47.241 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:47.273 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:47.287 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:45:47.290 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:45:47.329 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:47.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:47.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:47.437 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:47.467 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:47.523 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:47.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:47.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:47.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:47.654 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:47.701 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:47.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:47.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:47.807 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:47.838 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:47.903 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:47.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:47.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:47.985 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:45:48.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:48.048 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:48.104 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:48.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:48.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:48.210 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:48.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:47 smithi008 ceph-mon[184418]: pgmap v776: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 337 op/s; 28489/87771 objects degraded (32.458%); 6877/87771 objects misplaced (7.835%); 5.0 MiB/s, 176 objects/s recovering 2023-10-09T05:45:48.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2740424617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:48.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:47 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:48.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:47 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 28489/87771 objects degraded (32.458%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:48.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:47 smithi008 ceph-mon[191227]: pgmap v776: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 337 op/s; 28489/87771 objects degraded (32.458%); 6877/87771 objects misplaced (7.835%); 5.0 MiB/s, 176 objects/s recovering 2023-10-09T05:45:48.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2740424617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:48.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:47 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:48.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:47 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 28489/87771 objects degraded (32.458%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:48.234 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:47 smithi191 ceph-mon[161669]: pgmap v776: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 337 op/s; 28489/87771 objects degraded (32.458%); 6877/87771 objects misplaced (7.835%); 5.0 MiB/s, 176 objects/s recovering 2023-10-09T05:45:48.234 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2740424617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:48.234 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:47 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:45:48.234 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:47 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 28489/87771 objects degraded (32.458%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:48.237 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:48.282 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:48.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:48.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:48.387 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:48.486 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:49.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[184418]: from='client.45602 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:45:49.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[184418]: failed to encode map e257 with expected crc 2023-10-09T05:45:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[184418]: failed to encode map e257 with expected crc 2023-10-09T05:45:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:45:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[184418]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T05:45:49.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:49.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[191227]: from='client.45602 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:45:49.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:49.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[191227]: failed to encode map e257 with expected crc 2023-10-09T05:45:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[191227]: failed to encode map e257 with expected crc 2023-10-09T05:45:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:45:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[191227]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T05:45:49.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:48 smithi191 ceph-mon[161669]: from='client.45602 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:45:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:48 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:45:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:48 smithi191 ceph-mon[161669]: failed to encode map e257 with expected crc 2023-10-09T05:45:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:48 smithi191 ceph-mon[161669]: failed to encode map e257 with expected crc 2023-10-09T05:45:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:48 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:45:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:48 smithi191 ceph-mon[161669]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T05:45:49.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:49.305 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:49.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:49.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:49.409 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:49.619 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:49 smithi191 ceph-mon[161669]: pgmap v778: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 28489/85605 objects degraded (33.280%); 6266/85605 objects misplaced (7.320%); 8.1 MiB/s, 230 objects/s recovering 2023-10-09T05:45:50.344 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:49 smithi008 ceph-mon[184418]: pgmap v778: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 28489/85605 objects degraded (33.280%); 6266/85605 objects misplaced (7.320%); 8.1 MiB/s, 230 objects/s recovering 2023-10-09T05:45:50.345 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:49 smithi008 ceph-mon[191227]: pgmap v778: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 28489/85605 objects degraded (33.280%); 6266/85605 objects misplaced (7.320%); 8.1 MiB/s, 230 objects/s recovering 2023-10-09T05:45:50.366 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:50.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:50.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:50.430 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":776,"stamp":"2023-10-09T05:45:46.295438+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1920276427,"num_objects":29257,"num_object_clones":0,"num_object_copies":87771,"num_objects_missing_on_primary":3824,"num_objects_missing":3824,"num_objects_degraded":28489,"num_objects_misplaced":6877,"num_objects_unfound":0,"num_objects_dirty":29257,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":160865,"num_write_kb":6062694,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15742,"num_bytes_recovered":911092244,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216770,"ondisk_log_size":216770,"up":297,"acting":287,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8690796,"kb_used_data":5935092,"kb_used_omap":38,"kb_used_meta":2755545,"kb_avail":647525268,"statfs":{"total":671965249536,"available":663065874432,"internally_reserved":0,"allocated":6077534208,"data_stored":6072486379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39147,"internal_metadata":2821678869},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,0,0,0,2,1,1,6,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-229507072,"num_objects":-3502,"num_object_clones":0,"num_object_copies":-10506,"num_objects_missing_on_primary":-233,"num_objects_missing":-233,"num_objects_degraded":-3933,"num_objects_misplaced":575,"num_objects_unfound":0,"num_objects_dirty":-3502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3502,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1829,"num_bytes_recovered":54067200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.374883"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":247,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.002045+0000","last_change":"2023-10-09T05:45:17.927530+0000","last_active":"2023-10-09T05:45:23.002045+0000","last_peered":"2023-10-09T05:45:23.002045+0000","last_clean":"2023-10-09T05:45:23.002045+0000","last_became_active":"2023-10-09T05:45:17.926963+0000","last_became_peered":"2023-10-09T05:45:17.926963+0000","last_unstale":"2023-10-09T05:45:23.002045+0000","last_undegraded":"2023-10-09T05:45:23.002045+0000","last_fullsized":"2023-10-09T05:45:23.002045+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":238,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.008266+0000","last_change":"2023-10-09T05:45:17.925436+0000","last_active":"2023-10-09T05:45:23.008266+0000","last_peered":"2023-10-09T05:45:23.008266+0000","last_clean":"2023-10-09T05:45:23.008266+0000","last_became_active":"2023-10-09T05:45:17.925064+0000","last_became_peered":"2023-10-09T05:45:17.925064+0000","last_unstale":"2023-10-09T05:45:23.008266+0000","last_undegraded":"2023-10-09T05:45:23.008266+0000","last_fullsized":"2023-10-09T05:45:23.008266+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:14.651551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":53,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.007271+0000","last_change":"2023-10-09T05:45:17.917244+0000","last_active":"2023-10-09T05:45:23.007271+0000","last_peered":"2023-10-09T05:45:23.007271+0000","last_clean":"2023-10-09T05:45:23.007271+0000","last_became_active":"2023-10-09T05:45:17.917012+0000","last_became_peered":"2023-10-09T05:45:17.917012+0000","last_unstale":"2023-10-09T05:45:23.007271+0000","last_undegraded":"2023-10-09T05:45:23.007271+0000","last_fullsized":"2023-10-09T05:45:23.007271+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:25.765620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":243,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.011109+0000","last_change":"2023-10-09T05:43:19.889677+0000","last_active":"2023-10-09T05:45:23.011109+0000","last_peered":"2023-10-09T05:45:23.011109+0000","last_clean":"2023-10-09T05:45:23.011109+0000","last_became_active":"2023-10-09T05:43:19.889320+0000","last_became_peered":"2023-10-09T05:43:19.889320+0000","last_unstale":"2023-10-09T05:45:23.011109+0000","last_undegraded":"2023-10-09T05:45:23.011109+0000","last_fullsized":"2023-10-09T05:45:23.011109+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:57.376656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":254,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.007131+0000","last_change":"2023-10-09T05:45:17.937090+0000","last_active":"2023-10-09T05:45:23.007131+0000","last_peered":"2023-10-09T05:45:23.007131+0000","last_clean":"2023-10-09T05:45:23.007131+0000","last_became_active":"2023-10-09T05:45:17.936778+0000","last_became_peered":"2023-10-09T05:45:17.936778+0000","last_unstale":"2023-10-09T05:45:23.007131+0000","last_undegraded":"2023-10-09T05:45:23.007131+0000","last_fullsized":"2023-10-09T05:45:23.007131+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:12.815984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":80,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001717+0000","last_change":"2023-10-09T05:45:17.925251+0000","last_active":"2023-10-09T05:45:23.001717+0000","last_peered":"2023-10-09T05:45:23.001717+0000","last_clean":"2023-10-09T05:45:23.001717+0000","last_became_active":"2023-10-09T05:45:17.924982+0000","last_became_peered":"2023-10-09T05:45:17.924982+0000","last_unstale":"2023-10-09T05:45:23.001717+0000","last_undegraded":"2023-10-09T05:45:23.001717+0000","last_fullsized":"2023-10-09T05:45:23.001717+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"256'8945","reported_seq":9006,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:41.623803+0000","last_change":"2023-10-09T05:44:04.042724+0000","last_active":"2023-10-09T05:45:41.623803+0000","last_peered":"2023-10-09T05:45:41.623803+0000","last_clean":"2023-10-09T05:45:41.623803+0000","last_became_active":"2023-10-09T05:44:04.029419+0000","last_became_peered":"2023-10-09T05:44:04.029419+0000","last_unstale":"2023-10-09T05:45:41.623803+0000","last_undegraded":"2023-10-09T05:45:41.623803+0000","last_fullsized":"2023-10-09T05:45:41.623803+0000","mapping_epoch":239,"log_start":"218'500","ondisk_log_start":"218'500","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8445,"log_dups_size":0,"ondisk_log_size":8445,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60850176,"num_objects":929,"num_object_clones":0,"num_object_copies":2787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5009,"num_write_kb":189984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":209,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.013414+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:45:23.013414+0000","last_peered":"2023-10-09T05:45:23.013414+0000","last_clean":"2023-10-09T05:45:23.013414+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:45:23.013414+0000","last_undegraded":"2023-10-09T05:45:23.013414+0000","last_fullsized":"2023-10-09T05:45:23.013414+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":239,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:31.806205+0000","last_change":"2023-10-09T05:45:17.922085+0000","last_active":"2023-10-09T05:45:31.806205+0000","last_peered":"2023-10-09T05:45:31.806205+0000","last_clean":"2023-10-09T05:45:31.806205+0000","last_became_active":"2023-10-09T05:45:17.921640+0000","last_became_peered":"2023-10-09T05:45:17.921640+0000","last_unstale":"2023-10-09T05:45:31.806205+0000","last_undegraded":"2023-10-09T05:45:31.806205+0000","last_fullsized":"2023-10-09T05:45:31.806205+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:06.899516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"256'9123","reported_seq":9191,"reported_epoch":256,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:45:41.684888+0000","last_change":"2023-10-09T05:45:19.059245+0000","last_active":"2023-10-09T05:45:41.684888+0000","last_peered":"2023-10-09T05:45:41.684888+0000","last_clean":"2023-10-09T05:45:16.924358+0000","last_became_active":"2023-10-09T05:45:19.022328+0000","last_became_peered":"2023-10-09T05:45:19.022328+0000","last_unstale":"2023-10-09T05:45:41.684888+0000","last_undegraded":"2023-10-09T05:45:41.684888+0000","last_fullsized":"2023-10-09T05:45:41.684888+0000","mapping_epoch":253,"log_start":"218'700","ondisk_log_start":"218'700","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8423,"log_dups_size":0,"ondisk_log_size":8423,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64159744,"num_objects":979,"num_object_clones":0,"num_object_copies":2937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":979,"num_objects_unfound":0,"num_objects_dirty":979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5085,"num_write_kb":194048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,1],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":979}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":265,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.002081+0000","last_change":"2023-10-09T05:45:17.915552+0000","last_active":"2023-10-09T05:45:23.002081+0000","last_peered":"2023-10-09T05:45:23.002081+0000","last_clean":"2023-10-09T05:45:23.002081+0000","last_became_active":"2023-10-09T05:45:17.913215+0000","last_became_peered":"2023-10-09T05:45:17.913215+0000","last_unstale":"2023-10-09T05:45:23.002081+0000","last_undegraded":"2023-10-09T05:45:23.002081+0000","last_fullsized":"2023-10-09T05:45:23.002081+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":253,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:31.806232+0000","last_change":"2023-10-09T05:43:19.883115+0000","last_active":"2023-10-09T05:45:31.806232+0000","last_peered":"2023-10-09T05:45:31.806232+0000","last_clean":"2023-10-09T05:45:31.806232+0000","last_became_active":"2023-10-09T05:43:19.882674+0000","last_became_peered":"2023-10-09T05:43:19.882674+0000","last_unstale":"2023-10-09T05:45:31.806232+0000","last_undegraded":"2023-10-09T05:45:31.806232+0000","last_fullsized":"2023-10-09T05:45:31.806232+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:30.479481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"256'8862","reported_seq":8920,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:44.897349+0000","last_change":"2023-10-09T05:44:04.043289+0000","last_active":"2023-10-09T05:45:44.897349+0000","last_peered":"2023-10-09T05:45:44.897349+0000","last_clean":"2023-10-09T05:45:44.897349+0000","last_became_active":"2023-10-09T05:44:04.032393+0000","last_became_peered":"2023-10-09T05:44:04.032393+0000","last_unstale":"2023-10-09T05:45:44.897349+0000","last_undegraded":"2023-10-09T05:45:44.897349+0000","last_fullsized":"2023-10-09T05:45:44.897349+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8862,"log_dups_size":0,"ondisk_log_size":8862,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53805056,"num_objects":821,"num_object_clones":0,"num_object_copies":2463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4987,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":41,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.023015+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:45:23.023015+0000","last_peered":"2023-10-09T05:45:23.023015+0000","last_clean":"2023-10-09T05:45:23.023015+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:45:23.023015+0000","last_undegraded":"2023-10-09T05:45:23.023015+0000","last_fullsized":"2023-10-09T05:45:23.023015+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.003646+0000","last_change":"2023-10-09T05:45:17.924621+0000","last_active":"2023-10-09T05:45:23.003646+0000","last_peered":"2023-10-09T05:45:23.003646+0000","last_clean":"2023-10-09T05:45:23.003646+0000","last_became_active":"2023-10-09T05:45:17.924313+0000","last_became_peered":"2023-10-09T05:45:17.924313+0000","last_unstale":"2023-10-09T05:45:23.003646+0000","last_undegraded":"2023-10-09T05:45:23.003646+0000","last_fullsized":"2023-10-09T05:45:23.003646+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:37.602060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.10","version":"256'8836","reported_seq":8904,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:45:44.965103+0000","last_change":"2023-10-09T05:45:19.272351+0000","last_active":"2023-10-09T05:45:44.965103+0000","last_peered":"2023-10-09T05:45:44.965103+0000","last_clean":"2023-10-09T05:45:16.956686+0000","last_became_active":"2023-10-09T05:45:19.244940+0000","last_became_peered":"2023-10-09T05:45:19.244940+0000","last_unstale":"2023-10-09T05:45:44.965103+0000","last_undegraded":"2023-10-09T05:45:18.965723+0000","last_fullsized":"2023-10-09T05:45:18.943261+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8836,"log_dups_size":0,"ondisk_log_size":8836,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59047936,"num_objects":901,"num_object_clones":0,"num_object_copies":2703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2906,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4958,"num_write_kb":187488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":890},{"shards":"3,5","objects":-278}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":282,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.002774+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:45:23.002774+0000","last_peered":"2023-10-09T05:45:23.002774+0000","last_clean":"2023-10-09T05:45:23.002774+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:45:23.002774+0000","last_undegraded":"2023-10-09T05:45:23.002774+0000","last_fullsized":"2023-10-09T05:45:23.002774+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":230,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.002295+0000","last_change":"2023-10-09T05:45:17.925631+0000","last_active":"2023-10-09T05:45:23.002295+0000","last_peered":"2023-10-09T05:45:23.002295+0000","last_clean":"2023-10-09T05:45:23.002295+0000","last_became_active":"2023-10-09T05:45:17.925142+0000","last_became_peered":"2023-10-09T05:45:17.925142+0000","last_unstale":"2023-10-09T05:45:23.002295+0000","last_undegraded":"2023-10-09T05:45:23.002295+0000","last_fullsized":"2023-10-09T05:45:23.002295+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"256'8832","reported_seq":8899,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:43.590094+0000","last_change":"2023-10-09T05:44:50.515353+0000","last_active":"2023-10-09T05:45:43.590094+0000","last_peered":"2023-10-09T05:45:43.590094+0000","last_clean":"2023-10-09T05:45:43.590094+0000","last_became_active":"2023-10-09T05:44:50.499263+0000","last_became_peered":"2023-10-09T05:44:50.499263+0000","last_unstale":"2023-10-09T05:45:43.590094+0000","last_undegraded":"2023-10-09T05:45:43.590094+0000","last_fullsized":"2023-10-09T05:45:43.590094+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8832,"log_dups_size":0,"ondisk_log_size":8832,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58228736,"num_objects":889,"num_object_clones":0,"num_object_copies":2667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4949,"num_write_kb":186784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":41,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.003529+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:45:23.003529+0000","last_peered":"2023-10-09T05:45:23.003529+0000","last_clean":"2023-10-09T05:45:23.003529+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:45:23.003529+0000","last_undegraded":"2023-10-09T05:45:23.003529+0000","last_fullsized":"2023-10-09T05:45:23.003529+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":272,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:31.806299+0000","last_change":"2023-10-09T05:45:17.920528+0000","last_active":"2023-10-09T05:45:31.806299+0000","last_peered":"2023-10-09T05:45:31.806299+0000","last_clean":"2023-10-09T05:45:31.806299+0000","last_became_active":"2023-10-09T05:45:17.920250+0000","last_became_peered":"2023-10-09T05:45:17.920250+0000","last_unstale":"2023-10-09T05:45:31.806299+0000","last_undegraded":"2023-10-09T05:45:31.806299+0000","last_fullsized":"2023-10-09T05:45:31.806299+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:08.059709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"256'9076","reported_seq":15162,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:43.667640+0000","last_change":"2023-10-09T05:45:38.011075+0000","last_active":"2023-10-09T05:45:43.667640+0000","last_peered":"2023-10-09T05:45:43.667640+0000","last_clean":"2023-10-09T05:45:43.667640+0000","last_became_active":"2023-10-09T05:44:04.113448+0000","last_became_peered":"2023-10-09T05:44:04.113448+0000","last_unstale":"2023-10-09T05:45:43.667640+0000","last_undegraded":"2023-10-09T05:45:43.667640+0000","last_fullsized":"2023-10-09T05:45:43.667640+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9076,"log_dups_size":0,"ondisk_log_size":9076,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57409536,"num_objects":876,"num_object_clones":0,"num_object_copies":2628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5099,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3006,"num_bytes_recovered":131137536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":242,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.013267+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:45:23.013267+0000","last_peered":"2023-10-09T05:45:23.013267+0000","last_clean":"2023-10-09T05:45:23.013267+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:45:23.013267+0000","last_undegraded":"2023-10-09T05:45:23.013267+0000","last_fullsized":"2023-10-09T05:45:23.013267+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":255,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001401+0000","last_change":"2023-10-09T05:43:19.886058+0000","last_active":"2023-10-09T05:45:23.001401+0000","last_peered":"2023-10-09T05:45:23.001401+0000","last_clean":"2023-10-09T05:45:23.001401+0000","last_became_active":"2023-10-09T05:43:19.885713+0000","last_became_peered":"2023-10-09T05:43:19.885713+0000","last_unstale":"2023-10-09T05:45:23.001401+0000","last_undegraded":"2023-10-09T05:45:23.001401+0000","last_fullsized":"2023-10-09T05:45:23.001401+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"256'9094","reported_seq":11349,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:45:44.977640+0000","last_change":"2023-10-09T05:45:17.932003+0000","last_active":"2023-10-09T05:45:44.977640+0000","last_peered":"2023-10-09T05:45:44.977640+0000","last_clean":"2023-10-09T05:43:18.738931+0000","last_became_active":"2023-10-09T05:44:05.020084+0000","last_became_peered":"2023-10-09T05:44:05.020084+0000","last_unstale":"2023-10-09T05:45:44.977640+0000","last_undegraded":"2023-10-09T05:45:17.927587+0000","last_fullsized":"2023-10-09T05:45:44.977640+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9094,"log_dups_size":0,"ondisk_log_size":9094,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57704448,"num_objects":881,"num_object_clones":0,"num_object_copies":2643,"num_objects_missing_on_primary":1851,"num_objects_missing":1851,"num_objects_degraded":1851,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5093,"num_write_kb":191136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1091,"num_bytes_recovered":71499776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":869},{"shards":"0,1,2,7","objects":5},{"shards":"0,2,7","objects":7}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":35,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.004367+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:45:23.004367+0000","last_peered":"2023-10-09T05:45:23.004367+0000","last_clean":"2023-10-09T05:45:23.004367+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:45:23.004367+0000","last_undegraded":"2023-10-09T05:45:23.004367+0000","last_fullsized":"2023-10-09T05:45:23.004367+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":242,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.011201+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:45:23.011201+0000","last_peered":"2023-10-09T05:45:23.011201+0000","last_clean":"2023-10-09T05:45:23.011201+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:45:23.011201+0000","last_undegraded":"2023-10-09T05:45:23.011201+0000","last_fullsized":"2023-10-09T05:45:23.011201+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"256'8905","reported_seq":8975,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:45:43.406304+0000","last_change":"2023-10-09T05:45:19.213860+0000","last_active":"2023-10-09T05:45:43.406304+0000","last_peered":"2023-10-09T05:45:43.406304+0000","last_clean":"2023-10-09T05:45:16.956792+0000","last_became_active":"2023-10-09T05:45:19.185275+0000","last_became_peered":"2023-10-09T05:45:19.185275+0000","last_unstale":"2023-10-09T05:45:43.406304+0000","last_undegraded":"2023-10-09T05:45:18.955431+0000","last_fullsized":"2023-10-09T05:45:18.914423+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8905,"log_dups_size":0,"ondisk_log_size":8905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59080704,"num_objects":902,"num_object_clones":0,"num_object_copies":2706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2959,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4983,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":903},{"shards":"0,4","objects":-286}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":30,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.010483+0000","last_change":"2023-10-09T05:44:50.483928+0000","last_active":"2023-10-09T05:45:23.010483+0000","last_peered":"2023-10-09T05:45:23.010483+0000","last_clean":"2023-10-09T05:45:23.010483+0000","last_became_active":"2023-10-09T05:44:50.483595+0000","last_became_peered":"2023-10-09T05:44:50.483595+0000","last_unstale":"2023-10-09T05:45:23.010483+0000","last_undegraded":"2023-10-09T05:45:23.010483+0000","last_fullsized":"2023-10-09T05:45:23.010483+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:21.100193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":271,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.008671+0000","last_change":"2023-10-09T05:45:17.919936+0000","last_active":"2023-10-09T05:45:23.008671+0000","last_peered":"2023-10-09T05:45:23.008671+0000","last_clean":"2023-10-09T05:45:23.008671+0000","last_became_active":"2023-10-09T05:45:17.919518+0000","last_became_peered":"2023-10-09T05:45:17.919518+0000","last_unstale":"2023-10-09T05:45:23.008671+0000","last_undegraded":"2023-10-09T05:45:23.008671+0000","last_fullsized":"2023-10-09T05:45:23.008671+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:24.340590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"256'9744","reported_seq":9803,"reported_epoch":256,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:45:45.066202+0000","last_change":"2023-10-09T05:45:19.152449+0000","last_active":"2023-10-09T05:45:45.066202+0000","last_peered":"2023-10-09T05:45:45.066202+0000","last_clean":"2023-10-09T05:45:16.917055+0000","last_became_active":"2023-10-09T05:45:19.101335+0000","last_became_peered":"2023-10-09T05:45:19.101335+0000","last_unstale":"2023-10-09T05:45:45.066202+0000","last_undegraded":"2023-10-09T05:45:45.066202+0000","last_fullsized":"2023-10-09T05:45:45.066202+0000","mapping_epoch":253,"log_start":"221'1951","ondisk_log_start":"221'1951","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":7793,"log_dups_size":0,"ondisk_log_size":7793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114688000,"num_objects":1750,"num_object_clones":0,"num_object_copies":5250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1750,"num_objects_unfound":0,"num_objects_dirty":1750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9744,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1750}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":39,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.003711+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:45:23.003711+0000","last_peered":"2023-10-09T05:45:23.003711+0000","last_clean":"2023-10-09T05:45:23.003711+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:45:23.003711+0000","last_undegraded":"2023-10-09T05:45:23.003711+0000","last_fullsized":"2023-10-09T05:45:23.003711+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":240,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.002751+0000","last_change":"2023-10-09T05:43:19.885929+0000","last_active":"2023-10-09T05:45:23.002751+0000","last_peered":"2023-10-09T05:45:23.002751+0000","last_clean":"2023-10-09T05:45:23.002751+0000","last_became_active":"2023-10-09T05:43:19.885542+0000","last_became_peered":"2023-10-09T05:43:19.885542+0000","last_unstale":"2023-10-09T05:45:23.002751+0000","last_undegraded":"2023-10-09T05:45:23.002751+0000","last_fullsized":"2023-10-09T05:45:23.002751+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"256'9784","reported_seq":9836,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:41.690649+0000","last_change":"2023-10-09T05:41:29.081067+0000","last_active":"2023-10-09T05:45:41.690649+0000","last_peered":"2023-10-09T05:45:41.690649+0000","last_clean":"2023-10-09T05:45:41.690649+0000","last_became_active":"2023-10-09T05:41:29.080704+0000","last_became_peered":"2023-10-09T05:41:29.080704+0000","last_unstale":"2023-10-09T05:45:41.690649+0000","last_undegraded":"2023-10-09T05:45:41.690649+0000","last_fullsized":"2023-10-09T05:45:41.690649+0000","mapping_epoch":213,"log_start":"226'4394","ondisk_log_start":"226'4394","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5390,"log_dups_size":0,"ondisk_log_size":5390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:29.483205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120717312,"num_objects":1842,"num_object_clones":0,"num_object_copies":5526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9784,"num_write_kb":372032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":52,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.003595+0000","last_change":"2023-10-09T05:45:18.061225+0000","last_active":"2023-10-09T05:45:23.003595+0000","last_peered":"2023-10-09T05:45:23.003595+0000","last_clean":"2023-10-09T05:45:23.003595+0000","last_became_active":"2023-10-09T05:45:17.923689+0000","last_became_peered":"2023-10-09T05:45:17.923689+0000","last_unstale":"2023-10-09T05:45:23.003595+0000","last_undegraded":"2023-10-09T05:45:23.003595+0000","last_fullsized":"2023-10-09T05:45:23.003595+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":254,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001320+0000","last_change":"2023-10-09T05:45:17.919668+0000","last_active":"2023-10-09T05:45:23.001320+0000","last_peered":"2023-10-09T05:45:23.001320+0000","last_clean":"2023-10-09T05:45:23.001320+0000","last_became_active":"2023-10-09T05:45:17.919322+0000","last_became_peered":"2023-10-09T05:45:17.919322+0000","last_unstale":"2023-10-09T05:45:23.001320+0000","last_undegraded":"2023-10-09T05:45:23.001320+0000","last_fullsized":"2023-10-09T05:45:23.001320+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"256'8832","reported_seq":8906,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:45:43.603839+0000","last_change":"2023-10-09T05:45:19.120012+0000","last_active":"2023-10-09T05:45:43.603839+0000","last_peered":"2023-10-09T05:45:43.603839+0000","last_clean":"2023-10-09T05:45:16.881440+0000","last_became_active":"2023-10-09T05:45:19.096430+0000","last_became_peered":"2023-10-09T05:45:19.096430+0000","last_unstale":"2023-10-09T05:45:43.603839+0000","last_undegraded":"2023-10-09T05:45:18.961142+0000","last_fullsized":"2023-10-09T05:45:18.913777+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8832,"log_dups_size":0,"ondisk_log_size":8832,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59899904,"num_objects":914,"num_object_clones":0,"num_object_copies":2742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2949,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4948,"num_write_kb":187584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":954},{"shards":"0,4","objects":-340}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":39,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.022960+0000","last_change":"2023-10-09T05:43:20.148955+0000","last_active":"2023-10-09T05:45:23.022960+0000","last_peered":"2023-10-09T05:45:23.022960+0000","last_clean":"2023-10-09T05:45:23.022960+0000","last_became_active":"2023-10-09T05:43:19.884767+0000","last_became_peered":"2023-10-09T05:43:19.884767+0000","last_unstale":"2023-10-09T05:45:23.022960+0000","last_undegraded":"2023-10-09T05:45:23.022960+0000","last_fullsized":"2023-10-09T05:45:23.022960+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":246,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.003974+0000","last_change":"2023-10-09T05:45:17.923093+0000","last_active":"2023-10-09T05:45:23.003974+0000","last_peered":"2023-10-09T05:45:23.003974+0000","last_clean":"2023-10-09T05:45:23.003974+0000","last_became_active":"2023-10-09T05:45:17.922756+0000","last_became_peered":"2023-10-09T05:45:17.922756+0000","last_unstale":"2023-10-09T05:45:23.003974+0000","last_undegraded":"2023-10-09T05:45:23.003974+0000","last_fullsized":"2023-10-09T05:45:23.003974+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.8","version":"256'8948","reported_seq":16076,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:42.043860+0000","last_change":"2023-10-09T05:45:31.254465+0000","last_active":"2023-10-09T05:45:42.043860+0000","last_peered":"2023-10-09T05:45:42.043860+0000","last_clean":"2023-10-09T05:45:42.043860+0000","last_became_active":"2023-10-09T05:44:04.975737+0000","last_became_peered":"2023-10-09T05:44:04.975737+0000","last_unstale":"2023-10-09T05:45:42.043860+0000","last_undegraded":"2023-10-09T05:45:42.043860+0000","last_fullsized":"2023-10-09T05:45:42.043860+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8948,"log_dups_size":0,"ondisk_log_size":8948,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63766528,"num_objects":973,"num_object_clones":0,"num_object_copies":2919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4996,"num_write_kb":191008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2972,"num_bytes_recovered":166461440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":223,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.000996+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:45:23.000996+0000","last_peered":"2023-10-09T05:45:23.000996+0000","last_clean":"2023-10-09T05:45:23.000996+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:45:23.000996+0000","last_undegraded":"2023-10-09T05:45:23.000996+0000","last_fullsized":"2023-10-09T05:45:23.000996+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":219,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:35.935774+0000","last_change":"2023-10-09T05:45:17.930153+0000","last_active":"2023-10-09T05:45:35.935774+0000","last_peered":"2023-10-09T05:45:35.935774+0000","last_clean":"2023-10-09T05:45:35.935774+0000","last_became_active":"2023-10-09T05:45:17.929831+0000","last_became_peered":"2023-10-09T05:45:17.929831+0000","last_unstale":"2023-10-09T05:45:35.935774+0000","last_undegraded":"2023-10-09T05:45:35.935774+0000","last_fullsized":"2023-10-09T05:45:35.935774+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:24.746849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"256'10145","reported_seq":11944,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:45:43.466278+0000","last_change":"2023-10-09T05:45:17.936204+0000","last_active":"2023-10-09T05:45:43.466278+0000","last_peered":"2023-10-09T05:45:43.466278+0000","last_clean":"2023-10-09T05:43:18.871019+0000","last_became_active":"2023-10-09T05:43:21.208517+0000","last_became_peered":"2023-10-09T05:43:21.208517+0000","last_unstale":"2023-10-09T05:45:43.466278+0000","last_undegraded":"2023-10-09T05:43:20.928538+0000","last_fullsized":"2023-10-09T05:43:20.877574+0000","mapping_epoch":231,"log_start":"225'3984","ondisk_log_start":"225'3984","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:20.409497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119603200,"num_objects":1825,"num_object_clones":0,"num_object_copies":5475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5117,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10145,"num_write_kb":383040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":868,"num_bytes_recovered":56885248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3","objects":-2827},{"shards":"0,3,4","objects":1825}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":228,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.008125+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:45:23.008125+0000","last_peered":"2023-10-09T05:45:23.008125+0000","last_clean":"2023-10-09T05:45:23.008125+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:45:23.008125+0000","last_undegraded":"2023-10-09T05:45:23.008125+0000","last_fullsized":"2023-10-09T05:45:23.008125+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":221,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.002683+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:45:23.002683+0000","last_peered":"2023-10-09T05:45:23.002683+0000","last_clean":"2023-10-09T05:45:23.002683+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:45:23.002683+0000","last_undegraded":"2023-10-09T05:45:23.002683+0000","last_fullsized":"2023-10-09T05:45:23.002683+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"256'10202","reported_seq":10255,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:42.016189+0000","last_change":"2023-10-09T05:41:29.078791+0000","last_active":"2023-10-09T05:45:42.016189+0000","last_peered":"2023-10-09T05:45:42.016189+0000","last_clean":"2023-10-09T05:45:42.016189+0000","last_became_active":"2023-10-09T05:41:29.078491+0000","last_became_peered":"2023-10-09T05:41:29.078491+0000","last_unstale":"2023-10-09T05:45:42.016189+0000","last_undegraded":"2023-10-09T05:45:42.016189+0000","last_fullsized":"2023-10-09T05:45:42.016189+0000","mapping_epoch":213,"log_start":"221'1948","ondisk_log_start":"221'1948","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8254,"log_dups_size":0,"ondisk_log_size":8254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:37.817282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123338752,"num_objects":1882,"num_object_clones":0,"num_object_copies":5646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10202,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":81,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001955+0000","last_change":"2023-10-09T05:45:17.921547+0000","last_active":"2023-10-09T05:45:23.001955+0000","last_peered":"2023-10-09T05:45:23.001955+0000","last_clean":"2023-10-09T05:45:23.001955+0000","last_became_active":"2023-10-09T05:45:17.921221+0000","last_became_peered":"2023-10-09T05:45:17.921221+0000","last_unstale":"2023-10-09T05:45:23.001955+0000","last_undegraded":"2023-10-09T05:45:23.001955+0000","last_fullsized":"2023-10-09T05:45:23.001955+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":245,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001554+0000","last_change":"2023-10-09T05:45:17.923692+0000","last_active":"2023-10-09T05:45:23.001554+0000","last_peered":"2023-10-09T05:45:23.001554+0000","last_clean":"2023-10-09T05:45:23.001554+0000","last_became_active":"2023-10-09T05:45:17.917743+0000","last_became_peered":"2023-10-09T05:45:17.917743+0000","last_unstale":"2023-10-09T05:45:23.001554+0000","last_undegraded":"2023-10-09T05:45:23.001554+0000","last_fullsized":"2023-10-09T05:45:23.001554+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"256'9863","reported_seq":9930,"reported_epoch":256,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:45:43.599336+0000","last_change":"2023-10-09T05:45:19.165033+0000","last_active":"2023-10-09T05:45:43.599336+0000","last_peered":"2023-10-09T05:45:43.599336+0000","last_clean":"2023-10-09T05:45:16.912524+0000","last_became_active":"2023-10-09T05:45:19.102874+0000","last_became_peered":"2023-10-09T05:45:19.102874+0000","last_unstale":"2023-10-09T05:45:43.599336+0000","last_undegraded":"2023-10-09T05:45:43.599336+0000","last_fullsized":"2023-10-09T05:45:43.599336+0000","mapping_epoch":253,"log_start":"224'3685","ondisk_log_start":"224'3685","created":213,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":6178,"log_dups_size":0,"ondisk_log_size":6178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:25.137080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122683416,"num_objects":1873,"num_object_clones":0,"num_object_copies":5619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1873,"num_objects_unfound":0,"num_objects_dirty":1873,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9863,"num_write_kb":375489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1873}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":255,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001066+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:45:23.001066+0000","last_peered":"2023-10-09T05:45:23.001066+0000","last_clean":"2023-10-09T05:45:23.001066+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:45:23.001066+0000","last_undegraded":"2023-10-09T05:45:23.001066+0000","last_fullsized":"2023-10-09T05:45:23.001066+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":239,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.000905+0000","last_change":"2023-10-09T05:45:17.922009+0000","last_active":"2023-10-09T05:45:23.000905+0000","last_peered":"2023-10-09T05:45:23.000905+0000","last_clean":"2023-10-09T05:45:23.000905+0000","last_became_active":"2023-10-09T05:45:17.921636+0000","last_became_peered":"2023-10-09T05:45:17.921636+0000","last_unstale":"2023-10-09T05:45:23.000905+0000","last_undegraded":"2023-10-09T05:45:23.000905+0000","last_fullsized":"2023-10-09T05:45:23.000905+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"256'9865","reported_seq":15155,"reported_epoch":256,"state":"active+recovering","last_fresh":"2023-10-09T05:45:41.740129+0000","last_change":"2023-10-09T05:45:38.021539+0000","last_active":"2023-10-09T05:45:41.740129+0000","last_peered":"2023-10-09T05:45:41.740129+0000","last_clean":"2023-10-09T05:43:18.861565+0000","last_became_active":"2023-10-09T05:44:51.482290+0000","last_became_peered":"2023-10-09T05:44:51.482290+0000","last_unstale":"2023-10-09T05:45:41.740129+0000","last_undegraded":"2023-10-09T05:45:41.740129+0000","last_fullsized":"2023-10-09T05:45:41.740129+0000","mapping_epoch":248,"log_start":"226'4493","ondisk_log_start":"226'4493","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5372,"log_dups_size":0,"ondisk_log_size":5372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:27.252412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123011072,"num_objects":1877,"num_object_clones":0,"num_object_copies":5631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1369,"num_objects_unfound":0,"num_objects_dirty":1877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9865,"num_write_kb":375744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2610,"num_bytes_recovered":170983424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"2,3,6","objects":1369},{"shards":"3,6,7","objects":508}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":273,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.007092+0000","last_change":"2023-10-09T05:45:17.922614+0000","last_active":"2023-10-09T05:45:23.007092+0000","last_peered":"2023-10-09T05:45:23.007092+0000","last_clean":"2023-10-09T05:45:23.007092+0000","last_became_active":"2023-10-09T05:45:17.922292+0000","last_became_peered":"2023-10-09T05:45:17.922292+0000","last_unstale":"2023-10-09T05:45:23.007092+0000","last_undegraded":"2023-10-09T05:45:23.007092+0000","last_fullsized":"2023-10-09T05:45:23.007092+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:33.044557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":53,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.008266+0000","last_change":"2023-10-09T05:45:17.921701+0000","last_active":"2023-10-09T05:45:23.008266+0000","last_peered":"2023-10-09T05:45:23.008266+0000","last_clean":"2023-10-09T05:45:23.008266+0000","last_became_active":"2023-10-09T05:45:17.921374+0000","last_became_peered":"2023-10-09T05:45:17.921374+0000","last_unstale":"2023-10-09T05:45:23.008266+0000","last_undegraded":"2023-10-09T05:45:23.008266+0000","last_fullsized":"2023-10-09T05:45:23.008266+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:06.490584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"256'8813","reported_seq":8874,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:44.982339+0000","last_change":"2023-10-09T05:45:18.379765+0000","last_active":"2023-10-09T05:45:44.982339+0000","last_peered":"2023-10-09T05:45:44.982339+0000","last_clean":"2023-10-09T05:45:44.982339+0000","last_became_active":"2023-10-09T05:44:03.989206+0000","last_became_peered":"2023-10-09T05:44:03.989206+0000","last_unstale":"2023-10-09T05:45:44.982339+0000","last_undegraded":"2023-10-09T05:45:44.982339+0000","last_fullsized":"2023-10-09T05:45:44.982339+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8813,"log_dups_size":0,"ondisk_log_size":8813,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57016320,"num_objects":870,"num_object_clones":0,"num_object_copies":2610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4938,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":581,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:45:23.012892+0000","last_change":"2023-10-09T05:45:18.999640+0000","last_active":"2023-10-09T05:45:23.012892+0000","last_peered":"2023-10-09T05:45:23.012892+0000","last_clean":"2023-10-09T05:22:40.584723+0000","last_became_active":"2023-10-09T05:45:18.999259+0000","last_became_peered":"2023-10-09T05:45:18.999259+0000","last_unstale":"2023-10-09T05:45:23.012892+0000","last_undegraded":"2023-10-09T05:45:18.956731+0000","last_fullsized":"2023-10-09T05:45:18.956171+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:23.344079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":238,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001341+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:45:23.001341+0000","last_peered":"2023-10-09T05:45:23.001341+0000","last_clean":"2023-10-09T05:45:23.001341+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:45:23.001341+0000","last_undegraded":"2023-10-09T05:45:23.001341+0000","last_fullsized":"2023-10-09T05:45:23.001341+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.002005+0000","last_change":"2023-10-09T05:45:17.926027+0000","last_active":"2023-10-09T05:45:23.002005+0000","last_peered":"2023-10-09T05:45:23.002005+0000","last_clean":"2023-10-09T05:45:23.002005+0000","last_became_active":"2023-10-09T05:45:17.925620+0000","last_became_peered":"2023-10-09T05:45:17.925620+0000","last_unstale":"2023-10-09T05:45:23.002005+0000","last_undegraded":"2023-10-09T05:45:23.002005+0000","last_fullsized":"2023-10-09T05:45:23.002005+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"256'8807","reported_seq":8875,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:43.520923+0000","last_change":"2023-10-09T05:44:50.502440+0000","last_active":"2023-10-09T05:45:43.520923+0000","last_peered":"2023-10-09T05:45:43.520923+0000","last_clean":"2023-10-09T05:45:43.520923+0000","last_became_active":"2023-10-09T05:44:50.481365+0000","last_became_peered":"2023-10-09T05:44:50.481365+0000","last_unstale":"2023-10-09T05:45:43.520923+0000","last_undegraded":"2023-10-09T05:45:43.520923+0000","last_fullsized":"2023-10-09T05:45:43.520923+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8807,"log_dups_size":0,"ondisk_log_size":8807,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59867136,"num_objects":913,"num_object_clones":0,"num_object_copies":2739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4923,"num_write_kb":186784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":699,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:45:23.002236+0000","last_change":"2023-10-09T05:45:17.925652+0000","last_active":"2023-10-09T05:45:23.002236+0000","last_peered":"2023-10-09T05:45:23.002236+0000","last_clean":"2023-10-09T05:45:10.956605+0000","last_became_active":"2023-10-09T05:45:17.925130+0000","last_became_peered":"2023-10-09T05:45:17.925130+0000","last_unstale":"2023-10-09T05:45:23.002236+0000","last_undegraded":"2023-10-09T05:45:17.905235+0000","last_fullsized":"2023-10-09T05:45:23.002236+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":222,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.002382+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:45:23.002382+0000","last_peered":"2023-10-09T05:45:23.002382+0000","last_clean":"2023-10-09T05:45:23.002382+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:45:23.002382+0000","last_undegraded":"2023-10-09T05:45:23.002382+0000","last_fullsized":"2023-10-09T05:45:23.002382+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":89,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.004032+0000","last_change":"2023-10-09T05:45:17.919819+0000","last_active":"2023-10-09T05:45:23.004032+0000","last_peered":"2023-10-09T05:45:23.004032+0000","last_clean":"2023-10-09T05:45:23.004032+0000","last_became_active":"2023-10-09T05:45:17.919447+0000","last_became_peered":"2023-10-09T05:45:17.919447+0000","last_unstale":"2023-10-09T05:45:23.004032+0000","last_undegraded":"2023-10-09T05:45:23.004032+0000","last_fullsized":"2023-10-09T05:45:23.004032+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:18.280449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"256'8889","reported_seq":8960,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:45:43.429398+0000","last_change":"2023-10-09T05:45:19.212798+0000","last_active":"2023-10-09T05:45:43.429398+0000","last_peered":"2023-10-09T05:45:43.429398+0000","last_clean":"2023-10-09T05:45:16.697360+0000","last_became_active":"2023-10-09T05:45:19.185185+0000","last_became_peered":"2023-10-09T05:45:19.185185+0000","last_unstale":"2023-10-09T05:45:43.429398+0000","last_undegraded":"2023-10-09T05:45:18.969077+0000","last_fullsized":"2023-10-09T05:45:18.916434+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8889,"log_dups_size":0,"ondisk_log_size":8889,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60129280,"num_objects":917,"num_object_clones":0,"num_object_copies":2751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2925,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4966,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":916},{"shards":"0,4","objects":-295}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":573,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:35.948099+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:45:35.948099+0000","last_peered":"2023-10-09T05:45:35.948099+0000","last_clean":"2023-10-09T05:45:35.948099+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:45:35.948099+0000","last_undegraded":"2023-10-09T05:45:35.948099+0000","last_fullsized":"2023-10-09T05:45:35.948099+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.003359+0000","last_change":"2023-10-09T05:45:17.921491+0000","last_active":"2023-10-09T05:45:23.003359+0000","last_peered":"2023-10-09T05:45:23.003359+0000","last_clean":"2023-10-09T05:45:23.003359+0000","last_became_active":"2023-10-09T05:45:17.921145+0000","last_became_peered":"2023-10-09T05:45:17.921145+0000","last_unstale":"2023-10-09T05:45:23.003359+0000","last_undegraded":"2023-10-09T05:45:23.003359+0000","last_fullsized":"2023-10-09T05:45:23.003359+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:45.903609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":73,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001647+0000","last_change":"2023-10-09T05:45:17.921021+0000","last_active":"2023-10-09T05:45:23.001647+0000","last_peered":"2023-10-09T05:45:23.001647+0000","last_clean":"2023-10-09T05:45:23.001647+0000","last_became_active":"2023-10-09T05:45:17.920667+0000","last_became_peered":"2023-10-09T05:45:17.920667+0000","last_unstale":"2023-10-09T05:45:23.001647+0000","last_undegraded":"2023-10-09T05:45:23.001647+0000","last_fullsized":"2023-10-09T05:45:23.001647+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"256'9018","reported_seq":11222,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:45:43.651424+0000","last_change":"2023-10-09T05:44:05.057485+0000","last_active":"2023-10-09T05:45:43.651424+0000","last_peered":"2023-10-09T05:45:43.651424+0000","last_clean":"2023-10-09T05:43:18.711607+0000","last_became_active":"2023-10-09T05:44:05.023661+0000","last_became_peered":"2023-10-09T05:44:05.023661+0000","last_unstale":"2023-10-09T05:45:43.651424+0000","last_undegraded":"2023-10-09T05:44:05.020403+0000","last_fullsized":"2023-10-09T05:45:43.651424+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9018,"log_dups_size":0,"ondisk_log_size":9018,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61210624,"num_objects":934,"num_object_clones":0,"num_object_copies":2802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1903,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5041,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1066,"num_bytes_recovered":69730304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":931},{"shards":"0,3,7","objects":3}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":902,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001840+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:45:23.001840+0000","last_peered":"2023-10-09T05:45:23.001840+0000","last_clean":"2023-10-09T05:45:23.001840+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:45:23.001840+0000","last_undegraded":"2023-10-09T05:45:23.001840+0000","last_fullsized":"2023-10-09T05:45:23.001840+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":406,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.003103+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:45:23.003103+0000","last_peered":"2023-10-09T05:45:23.003103+0000","last_clean":"2023-10-09T05:45:23.003103+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:45:23.003103+0000","last_undegraded":"2023-10-09T05:45:23.003103+0000","last_fullsized":"2023-10-09T05:45:23.003103+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":221,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.007218+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:45:23.007218+0000","last_peered":"2023-10-09T05:45:23.007218+0000","last_clean":"2023-10-09T05:45:23.007218+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:45:23.007218+0000","last_undegraded":"2023-10-09T05:45:23.007218+0000","last_fullsized":"2023-10-09T05:45:23.007218+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":223,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001737+0000","last_change":"2023-10-09T05:44:50.495966+0000","last_active":"2023-10-09T05:45:23.001737+0000","last_peered":"2023-10-09T05:45:23.001737+0000","last_clean":"2023-10-09T05:45:23.001737+0000","last_became_active":"2023-10-09T05:44:50.495628+0000","last_became_peered":"2023-10-09T05:44:50.495628+0000","last_unstale":"2023-10-09T05:45:23.001737+0000","last_undegraded":"2023-10-09T05:45:23.001737+0000","last_fullsized":"2023-10-09T05:45:23.001737+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"256'9048","reported_seq":11210,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:45:45.064594+0000","last_change":"2023-10-09T05:44:07.019753+0000","last_active":"2023-10-09T05:45:45.064594+0000","last_peered":"2023-10-09T05:45:45.064594+0000","last_clean":"2023-10-09T05:43:18.738931+0000","last_became_active":"2023-10-09T05:44:06.984836+0000","last_became_peered":"2023-10-09T05:44:06.984836+0000","last_unstale":"2023-10-09T05:45:45.064594+0000","last_undegraded":"2023-10-09T05:44:04.978402+0000","last_fullsized":"2023-10-09T05:45:45.064594+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9048,"log_dups_size":0,"ondisk_log_size":9048,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60719104,"num_objects":926,"num_object_clones":0,"num_object_copies":2778,"num_objects_missing_on_primary":1973,"num_objects_missing":1973,"num_objects_degraded":1973,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5046,"num_write_kb":191136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1045,"num_bytes_recovered":68485120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":925},{"shards":"0,1,2,7","objects":8},{"shards":"0,2,7","objects":-7}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":708,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:45:23.003200+0000","last_change":"2023-10-09T05:45:18.933861+0000","last_active":"2023-10-09T05:45:23.003200+0000","last_peered":"2023-10-09T05:45:23.003200+0000","last_clean":"2023-10-09T05:45:10.937727+0000","last_became_active":"2023-10-09T05:45:18.933370+0000","last_became_peered":"2023-10-09T05:45:18.933370+0000","last_unstale":"2023-10-09T05:45:23.003200+0000","last_undegraded":"2023-10-09T05:45:18.913825+0000","last_fullsized":"2023-10-09T05:45:18.913230+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":210,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.014761+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:45:23.014761+0000","last_peered":"2023-10-09T05:45:23.014761+0000","last_clean":"2023-10-09T05:45:23.014761+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:45:23.014761+0000","last_undegraded":"2023-10-09T05:45:23.014761+0000","last_fullsized":"2023-10-09T05:45:23.014761+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":234,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.010880+0000","last_change":"2023-10-09T05:45:17.931710+0000","last_active":"2023-10-09T05:45:23.010880+0000","last_peered":"2023-10-09T05:45:23.010880+0000","last_clean":"2023-10-09T05:45:23.010880+0000","last_became_active":"2023-10-09T05:45:17.931290+0000","last_became_peered":"2023-10-09T05:45:17.931290+0000","last_unstale":"2023-10-09T05:45:23.010880+0000","last_undegraded":"2023-10-09T05:45:23.010880+0000","last_fullsized":"2023-10-09T05:45:23.010880+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:50.091125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"256'8863","reported_seq":8958,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:45:44.920787+0000","last_change":"2023-10-09T05:45:19.272383+0000","last_active":"2023-10-09T05:45:44.920787+0000","last_peered":"2023-10-09T05:45:44.920787+0000","last_clean":"2023-10-09T05:45:16.901376+0000","last_became_active":"2023-10-09T05:45:19.243948+0000","last_became_peered":"2023-10-09T05:45:19.243948+0000","last_unstale":"2023-10-09T05:45:44.920787+0000","last_undegraded":"2023-10-09T05:45:18.942714+0000","last_fullsized":"2023-10-09T05:45:18.916529+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8863,"log_dups_size":0,"ondisk_log_size":8863,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57278464,"num_objects":874,"num_object_clones":0,"num_object_copies":2622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2953,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4985,"num_write_kb":187488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":885},{"shards":"3,5","objects":-316}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":439,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.023064+0000","last_change":"2023-10-09T05:44:51.489799+0000","last_active":"2023-10-09T05:45:23.023064+0000","last_peered":"2023-10-09T05:45:23.023064+0000","last_clean":"2023-10-09T05:45:23.023064+0000","last_became_active":"2023-10-09T05:44:51.489301+0000","last_became_peered":"2023-10-09T05:44:51.489301+0000","last_unstale":"2023-10-09T05:45:23.023064+0000","last_undegraded":"2023-10-09T05:45:23.023064+0000","last_fullsized":"2023-10-09T05:45:23.023064+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.004119+0000","last_change":"2023-10-09T05:45:17.926001+0000","last_active":"2023-10-09T05:45:23.004119+0000","last_peered":"2023-10-09T05:45:23.004119+0000","last_clean":"2023-10-09T05:45:23.004119+0000","last_became_active":"2023-10-09T05:45:17.925682+0000","last_became_peered":"2023-10-09T05:45:17.925682+0000","last_unstale":"2023-10-09T05:45:23.004119+0000","last_undegraded":"2023-10-09T05:45:23.004119+0000","last_fullsized":"2023-10-09T05:45:23.004119+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:25.779220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":228,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.011028+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:45:23.011028+0000","last_peered":"2023-10-09T05:45:23.011028+0000","last_clean":"2023-10-09T05:45:23.011028+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:45:23.011028+0000","last_undegraded":"2023-10-09T05:45:23.011028+0000","last_fullsized":"2023-10-09T05:45:23.011028+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"256'8956","reported_seq":9018,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:41.657081+0000","last_change":"2023-10-09T05:44:04.001523+0000","last_active":"2023-10-09T05:45:41.657081+0000","last_peered":"2023-10-09T05:45:41.657081+0000","last_clean":"2023-10-09T05:45:41.657081+0000","last_became_active":"2023-10-09T05:44:03.990530+0000","last_became_peered":"2023-10-09T05:44:03.990530+0000","last_unstale":"2023-10-09T05:45:41.657081+0000","last_undegraded":"2023-10-09T05:45:41.657081+0000","last_fullsized":"2023-10-09T05:45:41.657081+0000","mapping_epoch":213,"log_start":"218'500","ondisk_log_start":"218'500","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8456,"log_dups_size":0,"ondisk_log_size":8456,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60129280,"num_objects":917,"num_object_clones":0,"num_object_copies":2751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5019,"num_write_kb":189984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":269,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.008710+0000","last_change":"2023-10-09T05:43:20.063203+0000","last_active":"2023-10-09T05:45:23.008710+0000","last_peered":"2023-10-09T05:45:23.008710+0000","last_clean":"2023-10-09T05:45:23.008710+0000","last_became_active":"2023-10-09T05:43:19.885257+0000","last_became_peered":"2023-10-09T05:43:19.885257+0000","last_unstale":"2023-10-09T05:45:23.008710+0000","last_undegraded":"2023-10-09T05:45:23.008710+0000","last_fullsized":"2023-10-09T05:45:23.008710+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:36.424359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":470,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:45:23.001027+0000","last_change":"2023-10-09T05:45:18.928989+0000","last_active":"2023-10-09T05:45:23.001027+0000","last_peered":"2023-10-09T05:45:23.001027+0000","last_clean":"2023-10-09T05:22:39.598843+0000","last_became_active":"2023-10-09T05:45:18.928677+0000","last_became_peered":"2023-10-09T05:45:18.928677+0000","last_unstale":"2023-10-09T05:45:23.001027+0000","last_undegraded":"2023-10-09T05:45:18.915184+0000","last_fullsized":"2023-10-09T05:45:18.914623+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":259,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:31.806217+0000","last_change":"2023-10-09T05:43:19.886969+0000","last_active":"2023-10-09T05:45:31.806217+0000","last_peered":"2023-10-09T05:45:31.806217+0000","last_clean":"2023-10-09T05:45:31.806217+0000","last_became_active":"2023-10-09T05:43:19.885111+0000","last_became_peered":"2023-10-09T05:43:19.885111+0000","last_unstale":"2023-10-09T05:45:31.806217+0000","last_undegraded":"2023-10-09T05:45:31.806217+0000","last_fullsized":"2023-10-09T05:45:31.806217+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:26.890192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"256'9196","reported_seq":9265,"reported_epoch":256,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:45:41.691587+0000","last_change":"2023-10-09T05:45:19.054812+0000","last_active":"2023-10-09T05:45:41.691587+0000","last_peered":"2023-10-09T05:45:41.691587+0000","last_clean":"2023-10-09T05:45:16.944353+0000","last_became_active":"2023-10-09T05:45:19.012069+0000","last_became_peered":"2023-10-09T05:45:19.012069+0000","last_unstale":"2023-10-09T05:45:41.691587+0000","last_undegraded":"2023-10-09T05:45:41.691587+0000","last_fullsized":"2023-10-09T05:45:41.691587+0000","mapping_epoch":253,"log_start":"218'700","ondisk_log_start":"218'700","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8496,"log_dups_size":0,"ondisk_log_size":8496,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59375616,"num_objects":906,"num_object_clones":0,"num_object_copies":2718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":906,"num_objects_unfound":0,"num_objects_dirty":906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5158,"num_write_kb":194048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,1],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":906}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":753,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.003847+0000","last_change":"2023-10-09T05:44:50.482211+0000","last_active":"2023-10-09T05:45:23.003847+0000","last_peered":"2023-10-09T05:45:23.003847+0000","last_clean":"2023-10-09T05:45:23.003847+0000","last_became_active":"2023-10-09T05:44:50.481749+0000","last_became_peered":"2023-10-09T05:44:50.481749+0000","last_unstale":"2023-10-09T05:45:23.003847+0000","last_undegraded":"2023-10-09T05:45:23.003847+0000","last_fullsized":"2023-10-09T05:45:23.003847+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":226,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.010972+0000","last_change":"2023-10-09T05:45:17.975802+0000","last_active":"2023-10-09T05:45:23.010972+0000","last_peered":"2023-10-09T05:45:23.010972+0000","last_clean":"2023-10-09T05:45:23.010972+0000","last_became_active":"2023-10-09T05:45:17.931496+0000","last_became_peered":"2023-10-09T05:45:17.931496+0000","last_unstale":"2023-10-09T05:45:23.010972+0000","last_undegraded":"2023-10-09T05:45:23.010972+0000","last_fullsized":"2023-10-09T05:45:23.010972+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:51.419126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":206,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001209+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:45:23.001209+0000","last_peered":"2023-10-09T05:45:23.001209+0000","last_clean":"2023-10-09T05:45:23.001209+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:45:23.001209+0000","last_undegraded":"2023-10-09T05:45:23.001209+0000","last_fullsized":"2023-10-09T05:45:23.001209+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":230,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.010944+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:45:23.010944+0000","last_peered":"2023-10-09T05:45:23.010944+0000","last_clean":"2023-10-09T05:45:23.010944+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:45:23.010944+0000","last_undegraded":"2023-10-09T05:45:23.010944+0000","last_fullsized":"2023-10-09T05:45:23.010944+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":267,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.004229+0000","last_change":"2023-10-09T05:45:17.924914+0000","last_active":"2023-10-09T05:45:23.004229+0000","last_peered":"2023-10-09T05:45:23.004229+0000","last_clean":"2023-10-09T05:45:23.004229+0000","last_became_active":"2023-10-09T05:45:17.924406+0000","last_became_peered":"2023-10-09T05:45:17.924406+0000","last_unstale":"2023-10-09T05:45:23.004229+0000","last_undegraded":"2023-10-09T05:45:23.004229+0000","last_fullsized":"2023-10-09T05:45:23.004229+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:35.316832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"256'9004","reported_seq":16181,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T05:45:41.997090+0000","last_change":"2023-10-09T05:45:35.805746+0000","last_active":"2023-10-09T05:45:41.997090+0000","last_peered":"2023-10-09T05:45:41.997090+0000","last_clean":"2023-10-09T05:45:41.997090+0000","last_became_active":"2023-10-09T05:44:04.104935+0000","last_became_peered":"2023-10-09T05:44:04.104935+0000","last_unstale":"2023-10-09T05:45:41.997090+0000","last_undegraded":"2023-10-09T05:45:41.997090+0000","last_fullsized":"2023-10-09T05:45:41.997090+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9004,"log_dups_size":0,"ondisk_log_size":9004,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60096512,"num_objects":917,"num_object_clones":0,"num_object_copies":2751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5052,"num_write_kb":191008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2997,"num_bytes_recovered":170524672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":238,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.000926+0000","last_change":"2023-10-09T05:45:17.971642+0000","last_active":"2023-10-09T05:45:23.000926+0000","last_peered":"2023-10-09T05:45:23.000926+0000","last_clean":"2023-10-09T05:45:23.000926+0000","last_became_active":"2023-10-09T05:45:17.917031+0000","last_became_peered":"2023-10-09T05:45:17.917031+0000","last_unstale":"2023-10-09T05:45:23.000926+0000","last_undegraded":"2023-10-09T05:45:23.000926+0000","last_fullsized":"2023-10-09T05:45:23.000926+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":245,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.002378+0000","last_change":"2023-10-09T05:45:17.925441+0000","last_active":"2023-10-09T05:45:23.002378+0000","last_peered":"2023-10-09T05:45:23.002378+0000","last_clean":"2023-10-09T05:45:23.002378+0000","last_became_active":"2023-10-09T05:45:17.924653+0000","last_became_peered":"2023-10-09T05:45:17.924653+0000","last_unstale":"2023-10-09T05:45:23.002378+0000","last_undegraded":"2023-10-09T05:45:23.002378+0000","last_fullsized":"2023-10-09T05:45:23.002378+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"256'8805","reported_seq":8874,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:45:43.126613+0000","last_change":"2023-10-09T05:45:19.119065+0000","last_active":"2023-10-09T05:45:43.126613+0000","last_peered":"2023-10-09T05:45:43.126613+0000","last_clean":"2023-10-09T05:45:16.938440+0000","last_became_active":"2023-10-09T05:45:19.095658+0000","last_became_peered":"2023-10-09T05:45:19.095658+0000","last_unstale":"2023-10-09T05:45:43.126613+0000","last_undegraded":"2023-10-09T05:45:18.969330+0000","last_fullsized":"2023-10-09T05:45:18.914061+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8805,"log_dups_size":0,"ondisk_log_size":8805,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61669376,"num_objects":941,"num_object_clones":0,"num_object_copies":2823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2913,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4921,"num_write_kb":187584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":901},{"shards":"0,4","objects":-230}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":210,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001266+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:45:23.001266+0000","last_peered":"2023-10-09T05:45:23.001266+0000","last_clean":"2023-10-09T05:45:23.001266+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:45:23.001266+0000","last_undegraded":"2023-10-09T05:45:23.001266+0000","last_fullsized":"2023-10-09T05:45:23.001266+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":236,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.001871+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:45:23.001871+0000","last_peered":"2023-10-09T05:45:23.001871+0000","last_clean":"2023-10-09T05:45:23.001871+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:45:23.001871+0000","last_undegraded":"2023-10-09T05:45:23.001871+0000","last_fullsized":"2023-10-09T05:45:23.001871+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":237,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:31.806314+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:45:31.806314+0000","last_peered":"2023-10-09T05:45:31.806314+0000","last_clean":"2023-10-09T05:45:31.806314+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:45:31.806314+0000","last_undegraded":"2023-10-09T05:45:31.806314+0000","last_fullsized":"2023-10-09T05:45:31.806314+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":248,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.007081+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:45:23.007081+0000","last_peered":"2023-10-09T05:45:23.007081+0000","last_clean":"2023-10-09T05:45:23.007081+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:45:23.007081+0000","last_undegraded":"2023-10-09T05:45:23.007081+0000","last_fullsized":"2023-10-09T05:45:23.007081+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":213,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.013467+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:45:23.013467+0000","last_peered":"2023-10-09T05:45:23.013467+0000","last_clean":"2023-10-09T05:45:23.013467+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:45:23.013467+0000","last_undegraded":"2023-10-09T05:45:23.013467+0000","last_fullsized":"2023-10-09T05:45:23.013467+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":230,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T05:45:23.000821+0000","last_change":"2023-10-09T05:45:17.920323+0000","last_active":"2023-10-09T05:45:23.000821+0000","last_peered":"2023-10-09T05:45:23.000821+0000","last_clean":"2023-10-09T05:45:23.000821+0000","last_became_active":"2023-10-09T05:45:17.919994+0000","last_became_peered":"2023-10-09T05:45:17.919994+0000","last_unstale":"2023-10-09T05:45:23.000821+0000","last_undegraded":"2023-10-09T05:45:23.000821+0000","last_fullsized":"2023-10-09T05:45:23.000821+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":1915486232,"num_objects":29229,"num_object_clones":0,"num_object_copies":87687,"num_objects_missing_on_primary":3824,"num_objects_missing":3824,"num_objects_degraded":28449,"num_objects_misplaced":6877,"num_objects_unfound":0,"num_objects_dirty":29229,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":159759,"num_write_kb":6047553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15656,"num_bytes_recovered":905707520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7205695488,"data_stored":7205683272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":215600,"ondisk_log_size":215600,"up":78,"acting":71,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":40,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846726,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":545040,"kb_used_data":226568,"kb_used_omap":14,"kb_used_meta":318385,"kb_avail":93200112,"statfs":{"total":95995035648,"available":95436914688,"internally_reserved":0,"allocated":232005632,"data_stored":231433871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":326027098},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,2,1,1,6,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:44:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":3.1259999999999999,"15min":5.2560000000000002},"min":{"1min":0.29499999999999998,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":2.6339999999999999,"5min":113.018,"15min":220.77500000000001},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":3.0470000000000002,"15min":5.7489999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":2.9399999999999999,"5min":113.005,"15min":220.78700000000001},"last":0.42899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:44:54 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":3.734,"15min":3.056},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":7.7729999999999997,"5min":83.575000000000003,"15min":190.68100000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.2310000000000001,"5min":3.7210000000000001,"15min":2.8740000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":2.698,"5min":87.516999999999996,"15min":191.09200000000001},"last":0.45000000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:44:54 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":1.7969999999999999,"15min":4.2400000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":7.8170000000000002,"5min":10.09,"15min":78.716999999999999},"last":2.2240000000000002},{"interface":"front","average":{"1min":1.23,"5min":1.8260000000000001,"15min":4.1820000000000004},"min":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27000000000000002},"max":{"1min":2.9529999999999998,"5min":14.15,"15min":78.768000000000001},"last":0.373}]},{"osd":3,"last update":"Mon Oct 9 05:44:54 2023","interfaces":[{"interface":"back","average":{"1min":4.923,"5min":7.0110000000000001,"15min":3.3250000000000002},"min":{"1min":0.36299999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":64.558999999999997,"5min":185.17099999999999,"15min":185.17099999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":4.4720000000000004,"5min":6.8380000000000001,"15min":3.782},"min":{"1min":0.34399999999999997,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":64.5,"5min":183.35499999999999,"15min":183.35499999999999},"last":0.54500000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:44:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.65500000000000003,"15min":1.2130000000000001},"min":{"1min":0.28299999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.7389999999999999,"5min":4.3620000000000001,"15min":12.789},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.628,"15min":1.3700000000000001},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":3.0169999999999999,"5min":4.3419999999999996,"15min":9.0470000000000006},"last":0.47199999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:44:54 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.753,"15min":1.3620000000000001},"min":{"1min":0.379,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.1459999999999999,"5min":4.4429999999999996,"15min":10.813000000000001},"last":0.498},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.77600000000000002,"15min":1.6639999999999999},"min":{"1min":0.41599999999999998,"5min":0.30399999999999999,"15min":0.28299999999999997},"max":{"1min":2.504,"5min":4.6159999999999997,"15min":10.321},"last":0.41999999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:44:54 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.67100000000000004,"15min":1.383},"min":{"1min":0.41799999999999998,"5min":0.32900000000000001,"15min":0.318},"max":{"1min":2.9929999999999999,"5min":4.468,"15min":9.2530000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.036,"5min":0.83499999999999996,"15min":1.333},"min":{"1min":0.38100000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.3700000000000001,"5min":4.649,"15min":8.2680000000000007},"last":0.38800000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626752,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":874568,"kb_used_data":573248,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92870584,"statfs":{"total":95995035648,"available":95099478016,"internally_reserved":0,"allocated":587005952,"data_stored":586442015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:34 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":4.649,"15min":4.4909999999999997},"min":{"1min":0.23400000000000001,"5min":0.22800000000000001,"15min":0.214},"max":{"1min":2.5,"5min":162.08799999999999,"15min":162.08799999999999},"last":0.31},{"interface":"front","average":{"1min":0.86899999999999999,"5min":4.4669999999999996,"15min":4.2949999999999999},"min":{"1min":0.25800000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.4009999999999998,"5min":161.90899999999999,"15min":161.90899999999999},"last":3.9729999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:44:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":5.2240000000000002,"15min":3.8260000000000001},"min":{"1min":0.27300000000000002,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":6.5279999999999996,"5min":228.24299999999999,"15min":228.24299999999999},"last":3.6869999999999998},{"interface":"front","average":{"1min":0.75900000000000001,"5min":4.9909999999999997,"15min":3.8260000000000001},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":3.6749999999999998,"5min":228.22499999999999,"15min":228.22499999999999},"last":3.6749999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:45:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":4.3339999999999996,"15min":4.093},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.3029999999999999,"5min":87.703000000000003,"15min":115.40600000000001},"last":3.8199999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":4.4980000000000002,"15min":4.1420000000000003},"min":{"1min":0.308,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":2.9049999999999998,"5min":87.689999999999998,"15min":115.297},"last":3.8359999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:44:57 2023","interfaces":[{"interface":"back","average":{"1min":4.4800000000000004,"5min":5.4889999999999999,"15min":3.8889999999999998},"min":{"1min":0.32900000000000001,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":77.578999999999994,"5min":99.543999999999997,"15min":115.244},"last":2.2290000000000001},{"interface":"front","average":{"1min":4.6200000000000001,"5min":5.3929999999999998,"15min":3.9329999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":77.516999999999996,"5min":98.677999999999997,"15min":115.34999999999999},"last":4.0179999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.51100000000000001,"15min":1.7330000000000001},"min":{"1min":0.309,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":2.3620000000000001,"5min":2.73,"15min":125.64},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.1659999999999999,"5min":0.94999999999999996,"15min":1.7549999999999999},"min":{"1min":0.35799999999999998,"5min":0.26000000000000001,"15min":0.254},"max":{"1min":4.0839999999999996,"5min":13.609,"15min":125.625},"last":0.38900000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.86499999999999999,"15min":1.9379999999999999},"min":{"1min":0.35899999999999999,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":6.141,"5min":6.4690000000000003,"15min":127.35899999999999},"last":0.376},{"interface":"front","average":{"1min":1.8979999999999999,"5min":1.0209999999999999,"15min":1.869},"min":{"1min":0.374,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":14.250999999999999,"5min":14.250999999999999,"15min":128.93799999999999},"last":0.51800000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:44:57 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.77600000000000002,"15min":2.165},"min":{"1min":0.375,"5min":0.32500000000000001,"15min":0.26500000000000001},"max":{"1min":2.8069999999999999,"5min":7.319,"15min":115.514},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.8400000000000001,"5min":1.0289999999999999,"15min":1.7270000000000001},"min":{"1min":0.40300000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":14.300000000000001,"5min":14.300000000000001,"15min":116.53100000000001},"last":0.54100000000000004}]}]},{"osd":1,"up_from":0,"seq":399431958713,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":88,"seq":377957122239,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1436372,"kb_used_data":947340,"kb_used_omap":8,"kb_used_meta":489015,"kb_avail":92308780,"statfs":{"total":95995035648,"available":94524190720,"internally_reserved":0,"allocated":970076160,"data_stored":969151986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":500752136},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:44:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":2.1299999999999999,"15min":1.6539999999999999},"min":{"1min":0.23300000000000001,"5min":0.20200000000000001,"15min":0.17199999999999999},"max":{"1min":14.198,"5min":17.760000000000002,"15min":17.760000000000002},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.84799999999999998,"5min":2.9630000000000001,"15min":2.1219999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":7.4829999999999997,"5min":26.163,"15min":26.163},"last":0.34000000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":3.5910000000000002,"15min":3.278},"min":{"1min":0.24199999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.5349999999999999,"5min":92.230000000000004,"15min":92.230000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.48999999999999999,"5min":3.4940000000000002,"15min":3.1440000000000001},"min":{"1min":0.26800000000000002,"5min":0.222,"15min":0.218},"max":{"1min":1.4279999999999999,"5min":92.247,"15min":92.247},"last":0.38700000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:44:56 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":6.5330000000000004,"15min":3.605},"min":{"1min":0.246,"5min":0.246,"15min":0.22800000000000001},"max":{"1min":22.658999999999999,"5min":123.024,"15min":123.024},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.8029999999999999,"5min":5.8410000000000002,"15min":3.464},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":22.713999999999999,"5min":99.162000000000006,"15min":99.162000000000006},"last":0.48699999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.2649999999999999,"15min":1.399},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.381,"5min":11.35,"15min":69.082999999999998},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.825,"15min":1.8819999999999999},"min":{"1min":0.36299999999999999,"5min":0.33500000000000002,"15min":0.30399999999999999},"max":{"1min":1.905,"5min":7.9240000000000004,"15min":69.102000000000004},"last":0.46300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":2.024,"15min":3.1840000000000002},"min":{"1min":0.372,"5min":0.34799999999999998,"15min":0.30199999999999999},"max":{"1min":3.5089999999999999,"5min":9.6219999999999999,"15min":281.733},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.80000000000000004,"5min":2.0779999999999998,"15min":3.2040000000000002},"min":{"1min":0.38,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.0369999999999999,"5min":12.552,"15min":282.95400000000001},"last":0.52700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":2.0529999999999999,"15min":1.7769999999999999},"min":{"1min":0.41399999999999998,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":4.5010000000000003,"5min":7.9969999999999999,"15min":11.664},"last":0.437},{"interface":"front","average":{"1min":1.2050000000000001,"5min":2.1200000000000001,"15min":1.867},"min":{"1min":0.38600000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.4390000000000001,"5min":12.587999999999999,"15min":12.587999999999999},"last":0.56000000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:44:56 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.5,"15min":1.458},"min":{"1min":0.379,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":6.5140000000000002,"5min":11.631,"15min":11.631},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.9830000000000001,"15min":2.2530000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":6.4939999999999998,"5min":9.6329999999999991,"15min":11.068},"last":0.502}]}]},{"osd":2,"up_from":98,"seq":420906795186,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1599956,"kb_used_data":1072268,"kb_used_omap":11,"kb_used_meta":527668,"kb_avail":92145196,"statfs":{"total":95995035648,"available":94356680704,"internally_reserved":0,"allocated":1098002432,"data_stored":1097019145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":540332629},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":13,"5min":4.9630000000000001,"15min":3.0659999999999998},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":162.285,"5min":162.285,"15min":162.285},"last":0.5},{"interface":"front","average":{"1min":13.404,"5min":5.9039999999999999,"15min":3.359},"min":{"1min":0.24399999999999999,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":162.643,"5min":162.643,"15min":162.643},"last":0.54300000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":2.8039999999999998,"15min":3.1179999999999999},"min":{"1min":0.24099999999999999,"5min":0.23100000000000001,"15min":0.20699999999999999},"max":{"1min":2.4289999999999998,"5min":24.265000000000001,"15min":98.575000000000003},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":2.0640000000000001,"15min":2.5640000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":2.0859999999999999,"5min":18.882999999999999,"15min":98.397000000000006},"last":2.1389999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:45:09 2023","interfaces":[{"interface":"back","average":{"1min":13.132,"5min":8.1069999999999993,"15min":4.8710000000000004},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":159.35499999999999,"5min":159.35499999999999,"15min":159.35499999999999},"last":2.1099999999999999},{"interface":"front","average":{"1min":13.404,"5min":6.992,"15min":4.798},"min":{"1min":0.32900000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":159.196,"5min":159.196,"15min":159.196},"last":0.52800000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.74,"15min":1.903},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27700000000000002},"max":{"1min":2.161,"5min":14.382,"15min":115.84999999999999},"last":2.1640000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":1.2609999999999999,"15min":2.056},"min":{"1min":0.309,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.974,"5min":13.815,"15min":116.149},"last":0.66200000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.986,"15min":1.972},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.312},"max":{"1min":2.4260000000000002,"5min":13.712999999999999,"15min":24.597999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.706,"15min":1.6579999999999999},"min":{"1min":0.38400000000000001,"5min":0.35499999999999998,"15min":0.26000000000000001},"max":{"1min":2.2890000000000001,"5min":14.426,"15min":23.919},"last":4.7359999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.5229999999999999,"15min":1.804},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":14.317,"5min":14.317,"15min":20.503},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.6299999999999999,"5min":2.1850000000000001,"15min":1.901},"min":{"1min":0.38900000000000001,"5min":0.36799999999999999,"15min":0.34499999999999997},"max":{"1min":14.436999999999999,"5min":14.436999999999999,"15min":14.436999999999999},"last":0.51700000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:45:09 2023","interfaces":[{"interface":"back","average":{"1min":3.871,"5min":2.2389999999999999,"15min":2.6000000000000001},"min":{"1min":0.39000000000000001,"5min":0.37,"15min":0.37},"max":{"1min":51.701999999999998,"5min":51.701999999999998,"15min":51.701999999999998},"last":0.56100000000000005},{"interface":"front","average":{"1min":4.1859999999999999,"5min":2.786,"15min":2.423},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":51.716000000000001,"5min":51.716000000000001,"15min":51.716000000000001},"last":0.75}]}]},{"osd":3,"up_from":103,"seq":442381631660,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1884716,"kb_used_data":1380452,"kb_used_omap":4,"kb_used_meta":504251,"kb_avail":91860436,"statfs":{"total":95995035648,"available":94065086464,"internally_reserved":0,"allocated":1413582848,"data_stored":1412694712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":516353630},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:44:55 2023","interfaces":[{"interface":"back","average":{"1min":7.7569999999999997,"5min":4.5229999999999997,"15min":3.504},"min":{"1min":0.23499999999999999,"5min":0.184,"15min":0.183},"max":{"1min":149.565,"5min":149.565,"15min":167.82400000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":7.9980000000000002,"5min":5.5869999999999997,"15min":4.0700000000000003},"min":{"1min":0.23400000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":149.291,"5min":149.291,"15min":167.89599999999999},"last":0.23200000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:44:55 2023","interfaces":[{"interface":"back","average":{"1min":7.484,"5min":4.0339999999999998,"15min":2.867},"min":{"1min":0.28699999999999998,"5min":0.26000000000000001,"15min":0.22600000000000001},"max":{"1min":149.32499999999999,"5min":149.32499999999999,"15min":149.32499999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":8.1630000000000003,"5min":4.3029999999999999,"15min":2.96},"min":{"1min":0.27000000000000002,"5min":0.19,"15min":0.19},"max":{"1min":149.34299999999999,"5min":149.34299999999999,"15min":149.34299999999999},"last":0.28199999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:44:55 2023","interfaces":[{"interface":"back","average":{"1min":7.806,"5min":4.173,"15min":4.5460000000000003},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.215},"max":{"1min":150.006,"5min":150.006,"15min":190.43299999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":7.8739999999999997,"5min":4.1070000000000002,"15min":4.9180000000000001},"min":{"1min":0.27000000000000002,"5min":0.214,"15min":0.214},"max":{"1min":150.65100000000001,"5min":150.65100000000001,"15min":190.267},"last":0.56499999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:44:55 2023","interfaces":[{"interface":"back","average":{"1min":7.8630000000000004,"5min":3.3260000000000001,"15min":2.649},"min":{"1min":0.35099999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":149.84700000000001,"5min":149.84700000000001,"15min":149.84700000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":7.5,"5min":3.0720000000000001,"15min":2.1699999999999999},"min":{"1min":0.32800000000000001,"5min":0.315,"15min":0.27400000000000002},"max":{"1min":149.74700000000001,"5min":149.74700000000001,"15min":149.74700000000001},"last":0.47699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:44:55 2023","interfaces":[{"interface":"back","average":{"1min":7.7309999999999999,"5min":3.6000000000000001,"15min":2.3109999999999999},"min":{"1min":0.374,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":149.43000000000001,"5min":149.43000000000001,"15min":149.43000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":8.0150000000000006,"5min":3.3290000000000002,"15min":2.3140000000000001},"min":{"1min":0.372,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":149.69300000000001,"5min":149.69300000000001,"15min":149.69300000000001},"last":0.53800000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:44:55 2023","interfaces":[{"interface":"back","average":{"1min":8.0280000000000005,"5min":3.2719999999999998,"15min":2.1869999999999998},"min":{"1min":0.35699999999999998,"5min":0.33900000000000002,"15min":0.33500000000000002},"max":{"1min":149.72399999999999,"5min":149.72399999999999,"15min":149.72399999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":8.0649999999999995,"5min":3.8319999999999999,"15min":2.2000000000000002},"min":{"1min":0.34599999999999997,"5min":0.307,"15min":0.307},"max":{"1min":149.589,"5min":149.589,"15min":149.589},"last":1.54}]},{"osd":7,"last update":"Mon Oct 9 05:45:05 2023","interfaces":[{"interface":"back","average":{"1min":6.9050000000000002,"5min":3.2240000000000002,"15min":2.718},"min":{"1min":0.41599999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":149.77199999999999,"5min":149.77199999999999,"15min":149.77199999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":6.9480000000000004,"5min":3.375,"15min":2.6669999999999998},"min":{"1min":0.39300000000000002,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":149.80799999999999,"5min":149.80799999999999,"15min":149.80799999999999},"last":0.52500000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019208,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1240584,"kb_used_data":926976,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92504568,"statfs":{"total":95995035648,"available":94724677632,"internally_reserved":0,"allocated":949223424,"data_stored":948659241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":3.2160000000000002,"15min":2.2160000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.214},"max":{"1min":26.478000000000002,"5min":100.504,"15min":100.504},"last":1.9410000000000001},{"interface":"front","average":{"1min":1.9350000000000001,"5min":3.4239999999999999,"15min":2.5369999999999999},"min":{"1min":0.247,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":26.454000000000001,"5min":100.717,"15min":100.717},"last":0.29199999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:45:05 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":4.4509999999999996,"15min":2.4359999999999999},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.24299999999999999},"max":{"1min":2.992,"5min":187.369,"15min":187.369},"last":2.4740000000000002},{"interface":"front","average":{"1min":0.83799999999999997,"5min":4.6070000000000002,"15min":2.5499999999999998},"min":{"1min":0.32700000000000001,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":3.734,"5min":187.35300000000001,"15min":187.35300000000001},"last":2.4830000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:45:23 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":3.6829999999999998,"15min":4.5309999999999997},"min":{"1min":0.27800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.7129999999999992,"5min":100.148,"15min":196.93700000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.4079999999999999,"5min":3.964,"15min":4.4939999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.28100000000000003},"max":{"1min":9.7219999999999995,"5min":100.086,"15min":197.08099999999999},"last":0.39200000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:44:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":3.9300000000000002,"15min":3.9670000000000001},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":4.3390000000000004,"5min":72.391999999999996,"15min":211.48699999999999},"last":2.5910000000000002},{"interface":"front","average":{"1min":1.4490000000000001,"5min":3.7469999999999999,"15min":3.8460000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.309},"max":{"1min":4.9980000000000002,"5min":71.721999999999994,"15min":211.31100000000001},"last":0.45400000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:44:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.71699999999999997,"15min":1.633},"min":{"1min":0.30399999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.6739999999999999,"5min":3.6739999999999999,"15min":113.35899999999999},"last":0.437},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.621,"15min":1.54},"min":{"1min":0.35199999999999998,"5min":0.317,"15min":0.30599999999999999},"max":{"1min":3.4209999999999998,"5min":3.4209999999999998,"15min":113.32299999999999},"last":0.41599999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.55100000000000005,"15min":0.72999999999999998},"min":{"1min":0.35099999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":2.379,"5min":3.3969999999999998,"15min":11.944000000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.6259999999999999,"5min":0.78700000000000003,"15min":0.95599999999999996},"min":{"1min":0.34799999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.5269999999999992,"5min":8.5269999999999992,"15min":10.356},"last":0.47599999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.63100000000000001,"15min":1.827},"min":{"1min":0.379,"5min":0.31900000000000001,"15min":0.312},"max":{"1min":2.0230000000000001,"5min":3.415,"15min":30.393999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.73699999999999999,"15min":2.056},"min":{"1min":0.35599999999999998,"5min":0.34899999999999998,"15min":0.32300000000000001},"max":{"1min":5.7160000000000002,"5min":5.7160000000000002,"15min":30.334},"last":0.46400000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822980,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1109560,"kb_used_data":808240,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92635592,"statfs":{"total":95995035648,"available":94858846208,"internally_reserved":0,"allocated":827637760,"data_stored":827085409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.611,"15min":3.29},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.19400000000000001},"max":{"1min":9.9049999999999994,"5min":12.904999999999999,"15min":165.90199999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.111,"5min":1.782,"15min":3.399},"min":{"1min":0.28299999999999997,"5min":0.27000000000000002,"15min":0.215},"max":{"1min":10.210000000000001,"5min":17.077000000000002,"15min":165.96700000000001},"last":0.45800000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:44:55 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":3.1269999999999998,"15min":2.5950000000000002},"min":{"1min":0.27700000000000002,"5min":0.26500000000000001,"15min":0.23999999999999999},"max":{"1min":3.5990000000000002,"5min":71.209999999999994,"15min":87.769000000000005},"last":0.314},{"interface":"front","average":{"1min":0.79900000000000004,"5min":3.238,"15min":2.6709999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":2.3660000000000001,"5min":71.192999999999998,"15min":87.864000000000004},"last":0.49099999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:45:36 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":1.8480000000000001,"15min":4.4329999999999998},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.9510000000000001,"5min":31.114000000000001,"15min":290.697},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.83499999999999996,"5min":2.1629999999999998,"15min":4.3129999999999997},"min":{"1min":0.35599999999999998,"5min":0.32700000000000001,"15min":0.29899999999999999},"max":{"1min":2.9239999999999999,"5min":33.875999999999998,"15min":290.786},"last":0.372}]},{"osd":3,"last update":"Mon Oct 9 05:44:55 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":3.6349999999999998,"15min":3.4390000000000001},"min":{"1min":0.35699999999999998,"5min":0.30399999999999999,"15min":0.28799999999999998},"max":{"1min":11.206,"5min":85.007999999999996,"15min":95.236999999999995},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.80100000000000005,"5min":3.375,"15min":3.2400000000000002},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.29699999999999999},"max":{"1min":3.5830000000000002,"5min":83.147999999999996,"15min":93.471000000000004},"last":1.589}]},{"osd":4,"last update":"Mon Oct 9 05:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.73899999999999999,"15min":0.77500000000000002},"min":{"1min":0.32700000000000001,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":2.0230000000000001,"5min":6.2610000000000001,"15min":9.3510000000000009},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.73499999999999999,"15min":1.006},"min":{"1min":0.32100000000000001,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":3.6349999999999998,"5min":9.282,"15min":14.236000000000001},"last":0.64900000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:45:10 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.78300000000000003,"15min":1.429},"min":{"1min":0.32600000000000001,"5min":0.316,"15min":0.26600000000000001},"max":{"1min":3.6629999999999998,"5min":9.2959999999999994,"15min":95.010999999999996},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.87,"15min":1.343},"min":{"1min":0.32100000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.8380000000000001,"5min":4.6600000000000001,"15min":94.194000000000003},"last":0.46899999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:45:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.073,"15min":1.3560000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28999999999999998},"max":{"1min":5.8920000000000003,"5min":9.1460000000000008,"15min":9.9849999999999994},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.95599999999999996,"15min":1.4730000000000001},"min":{"1min":0.39200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":3.6890000000000001,"5min":9.3100000000000005,"15min":14.432},"last":0.81200000000000006}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":960499712,"data_stored":960495640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1185222656,"data_stored":1185218584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1089667072,"data_stored":1089667072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1403912192,"data_stored":1403912192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":944766976,"data_stored":944766976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":823984128,"data_stored":823984128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":572657664,"data_stored":572653592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":224985088,"data_stored":224985088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:45:50.437 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 21 2023-10-09T05:45:50.476 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:50.536 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:50.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:45:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:45:50] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.33.4" 2023-10-09T05:45:52.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:51 smithi008 ceph-mon[184418]: pgmap v779: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 28319/82329 objects degraded (34.397%); 6047/82329 objects misplaced (7.345%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T05:45:52.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:51 smithi008 ceph-mon[191227]: pgmap v779: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 28319/82329 objects degraded (34.397%); 6047/82329 objects misplaced (7.345%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T05:45:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:51 smithi191 ceph-mon[161669]: pgmap v779: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 28319/82329 objects degraded (34.397%); 6047/82329 objects misplaced (7.345%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T05:45:52.536 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:52.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:52.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:52.642 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:52.693 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:52.900 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:52.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:52.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:53.004 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:53.038 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:53.086 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:53.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:53.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:53.140 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:45:53.167 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:45:53.191 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:45:53.191 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:53.220 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:45:53.235 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:53.256 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:45:53.293 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:45:53.295 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:53.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:53 smithi191 ceph-mon[161669]: pgmap v780: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 28319/80493 objects degraded (35.182%); 5399/80493 objects misplaced (6.707%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:45:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 28319/82329 objects degraded (34.397%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3874559602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:53.327 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:53.329 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:45:53.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:53.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:53.366 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:45:53.370 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:45:53.370 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:45:53.370 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:45:53.371 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:45:53.371 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:45:53.371 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:45:53.371 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:45:53.371 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:45:53.398 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:45:53.399 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:45:53.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:53.478 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:53.534 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:53.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:53.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:53.641 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:53.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:53.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:53.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:53 smithi008 ceph-mon[184418]: pgmap v780: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 28319/80493 objects degraded (35.182%); 5399/80493 objects misplaced (6.707%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:45:53.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 28319/82329 objects degraded (34.397%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:53.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3874559602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:53.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:53.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:45:53.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:53 smithi008 ceph-mon[191227]: pgmap v780: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 28319/80493 objects degraded (35.182%); 5399/80493 objects misplaced (6.707%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:45:53.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 28319/82329 objects degraded (34.397%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:53.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3874559602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:45:53.693 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:54.652 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:45:55.208 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:55.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:55.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:55.316 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:45:55.346 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:55.397 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:45:55.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:55.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:45:55.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:55 smithi008 ceph-mon[184418]: pgmap v781: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 28269/78456 objects degraded (36.032%); 5280/78456 objects misplaced (6.730%); 8.4 MiB/s, 135 objects/s recovering 2023-10-09T05:45:55.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:55 smithi008 ceph-mon[191227]: pgmap v781: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 28269/78456 objects degraded (36.032%); 5280/78456 objects misplaced (6.730%); 8.4 MiB/s, 135 objects/s recovering 2023-10-09T05:45:55.506 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:55.536 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:55.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:55 smithi191 ceph-mon[161669]: pgmap v781: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 28269/78456 objects degraded (36.032%); 5280/78456 objects misplaced (6.730%); 8.4 MiB/s, 135 objects/s recovering 2023-10-09T05:45:56.110 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:56.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:56.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:56.217 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:45:56.254 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:56.522 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:45:56.587 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:45:56.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:56.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:45:56.692 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:45:56.728 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:56.997 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:45:57.517 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:57 smithi191 ceph-mon[161669]: pgmap v782: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 28153/77019 objects degraded (36.553%); 5145/77019 objects misplaced (6.680%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:45:57.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:57 smithi008 ceph-mon[184418]: pgmap v782: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 28153/77019 objects degraded (36.553%); 5145/77019 objects misplaced (6.680%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:45:57.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:57 smithi008 ceph-mon[191227]: pgmap v782: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 28153/77019 objects degraded (36.553%); 5145/77019 objects misplaced (6.680%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:45:57.790 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:45:57.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:57.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:45:57.895 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:57.935 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:58.122 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:58.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:58.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:58.226 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:58.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:45:58.265 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:58.317 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:45:58.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:58.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:45:58.333 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:45:58.374 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 28153/77019 objects degraded (36.553%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:58.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 28153/77019 objects degraded (36.553%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:58.421 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:45:58.452 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:58.506 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:45:58.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:58.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:45:58.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 28153/77019 objects degraded (36.553%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:45:58.611 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:45:58.642 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:58.700 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:45:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:58.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:45:58.804 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:45:58.856 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:59.011 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:45:59.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:59.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:45:59.043 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:45:59.119 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:59.165 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:59.363 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:59 smithi008 ceph-mon[184418]: pgmap v783: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 28153/74964 objects degraded (37.555%); 4665/74964 objects misplaced (6.223%); 7.4 MiB/s, 157 objects/s recovering 2023-10-09T05:45:59.364 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:59.364 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1943166370' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:45:59.364 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:45:59 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:45:59.529 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:45:59.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:45:59.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:45:59.605 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:59 smithi191 ceph-mon[161669]: pgmap v783: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 28153/74964 objects degraded (37.555%); 4665/74964 objects misplaced (6.223%); 7.4 MiB/s, 157 objects/s recovering 2023-10-09T05:45:59.605 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:59.605 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1943166370' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:45:59.606 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:45:59 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:45:59.634 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:45:59.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:59 smithi008 ceph-mon[191227]: pgmap v783: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 28153/74964 objects degraded (37.555%); 4665/74964 objects misplaced (6.223%); 7.4 MiB/s, 157 objects/s recovering 2023-10-09T05:45:59.665 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:59.665 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1943166370' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:45:59.665 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:45:59 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T05:45:59.667 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:45:59.853 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:45:59.853 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:46:00.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:46:00.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:46:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:46:00] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T05:46:01.004 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:46:01.074 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:01.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:01.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:01.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:01.193 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:46:01.219 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:01.244 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:01.281 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:01.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:01.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:01.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:01 smithi008 ceph-mon[191227]: pgmap v784: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 382 op/s; 27986/71838 objects degraded (38.957%); 4437/71838 objects misplaced (6.176%); 7.3 MiB/s, 150 objects/s recovering 2023-10-09T05:46:01.385 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:01.419 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:01.485 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:01.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:01.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:01.490 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:01 smithi191 ceph-mon[161669]: pgmap v784: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 382 op/s; 27986/71838 objects degraded (38.957%); 4437/71838 objects misplaced (6.176%); 7.3 MiB/s, 150 objects/s recovering 2023-10-09T05:46:01.591 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:01.630 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:01 smithi008 ceph-mon[184418]: pgmap v784: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 382 op/s; 27986/71838 objects degraded (38.957%); 4437/71838 objects misplaced (6.176%); 7.3 MiB/s, 150 objects/s recovering 2023-10-09T05:46:01.635 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:01.718 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:01.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:01.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:01.824 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:01.889 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:02.174 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:02.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:02.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:02.312 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:02.374 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:02.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:02.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:02.383 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:46:02.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:02 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3393558560' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:02.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:02 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:02.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:02 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3393558560' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:02.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:02 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:02.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:02.509 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:02.559 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:02.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:02.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:02.578 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:46:02.578 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:46:02.605 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:02 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3393558560' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:02.605 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:02 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:02.663 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:02.700 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:03.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:46:03.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:03 smithi191 ceph-mon[161669]: pgmap v785: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 340 op/s; 27986/70053 objects degraded (39.950%); 4359/70053 objects misplaced (6.222%); 6.4 MiB/s, 211 objects/s recovering 2023-10-09T05:46:03.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:03 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:03.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:03 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:46:03.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:03 smithi191 ceph-mon[161669]: failed to encode map e258 with expected crc 2023-10-09T05:46:03.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:03 smithi191 ceph-mon[161669]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T05:46:03.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:03 smithi191 ceph-mon[161669]: failed to encode map e258 with expected crc 2023-10-09T05:46:03.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:03 smithi191 ceph-mon[161669]: 7.8 scrub starts 2023-10-09T05:46:03.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 27986/70053 objects degraded (39.950%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:03.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[184418]: pgmap v785: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 340 op/s; 27986/70053 objects degraded (39.950%); 4359/70053 objects misplaced (6.222%); 6.4 MiB/s, 211 objects/s recovering 2023-10-09T05:46:03.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:03.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:46:03.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[184418]: failed to encode map e258 with expected crc 2023-10-09T05:46:03.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[184418]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T05:46:03.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[184418]: failed to encode map e258 with expected crc 2023-10-09T05:46:03.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[184418]: 7.8 scrub starts 2023-10-09T05:46:03.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 27986/70053 objects degraded (39.950%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:03.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[191227]: pgmap v785: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 340 op/s; 27986/70053 objects degraded (39.950%); 4359/70053 objects misplaced (6.222%); 6.4 MiB/s, 211 objects/s recovering 2023-10-09T05:46:03.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:03.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:46:03.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[191227]: failed to encode map e258 with expected crc 2023-10-09T05:46:03.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[191227]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T05:46:03.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[191227]: failed to encode map e258 with expected crc 2023-10-09T05:46:03.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[191227]: 7.8 scrub starts 2023-10-09T05:46:03.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 27986/70053 objects degraded (39.950%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:03.772 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:46:04.185 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:04.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:04.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:04.290 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:04.323 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:04.394 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:04.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:04.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:04.499 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:04.535 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:04.602 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:04.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:04.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:04.709 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:04.758 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:04.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:04.861 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:04.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:04.922 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:46:04.945 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:46:04.963 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:04.971 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:46:04.996 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:46:05.007 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:05.024 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:46:05.058 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:46:05.089 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:46:05.106 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:05.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:05.117 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:46:05.125 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:46:05.125 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:46:05.125 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:46:05.126 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:46:05.126 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:46:05.126 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:46:05.126 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:46:05.126 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:46:05.209 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:05.245 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:05.368 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:05.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:05.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:05.447 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:46:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:46:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:46:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:46:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:124880 7.1 7:8e25ac43:::benchmark_data_smithi008_354649_object30384:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:46:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:01.555270+0000", 2023-10-09T05:46:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 123.88967076599999, 2023-10-09T05:46:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15951778999999999, 2023-10-09T05:46:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124880 2023-10-09T05:46:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.555270+0000", 2023-10-09T05:46:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.555270+0000", 2023-10-09T05:46:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.555268+0000", 2023-10-09T05:46:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0569999999999999e-06 2023-10-09T05:46:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.555277+0000", 2023-10-09T05:46:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4911e-05 2023-10-09T05:46:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.555292+0000", 2023-10-09T05:46:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1879999999999996e-06 2023-10-09T05:46:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.555299+0000", 2023-10-09T05:46:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2851e-05 2023-10-09T05:46:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.555342+0000", 2023-10-09T05:46:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119379 2023-10-09T05:46:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.555461+0000", 2023-10-09T05:46:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000138963 2023-10-09T05:46:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.555600+0000", 2023-10-09T05:46:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0043765849999999997 2023-10-09T05:46:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.559977+0000", 2023-10-09T05:46:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0067732499999999998 2023-10-09T05:46:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.566750+0000", 2023-10-09T05:46:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14796221000000001 2023-10-09T05:46:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.714712+0000", 2023-10-09T05:46:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5052000000000003e-05 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.714747+0000", 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0108000000000003e-05 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.714787+0000", 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15951778999999999 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:124883 7.1 7:8c84c58e:::benchmark_data_smithi008_354649_object30387:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:46:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:01.561483+0000", 2023-10-09T05:46:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 123.88345791, 2023-10-09T05:46:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15341919500000001, 2023-10-09T05:46:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124883 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.561483+0000", 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.561483+0000", 2023-10-09T05:46:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.561481+0000", 2023-10-09T05:46:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3469999999999999e-06 2023-10-09T05:46:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.561489+0000", 2023-10-09T05:46:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2231e-05 2023-10-09T05:46:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.561502+0000", 2023-10-09T05:46:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4500000000000001e-06 2023-10-09T05:46:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.561508+0000", 2023-10-09T05:46:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3587000000000002e-05 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.561542+0000", 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8283999999999996e-05 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.561640+0000", 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7243999999999993e-05 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.561727+0000", 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031489640000000002 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.564876+0000", 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001902466 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.566779+0000", 2023-10-09T05:46:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.148045019 2023-10-09T05:46:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.714824+0000", 2023-10-09T05:46:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1680000000000001e-05 2023-10-09T05:46:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.714865+0000", 2023-10-09T05:46:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.65e-05 2023-10-09T05:46:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.714902+0000", 2023-10-09T05:46:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15341919500000001 2023-10-09T05:46:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125677 7.1 7:8932b233:::benchmark_data_smithi008_354649_object31181:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:46:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.953115+0000", 2023-10-09T05:46:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 122.491825202, 2023-10-09T05:46:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27913971500000001, 2023-10-09T05:46:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:46:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125677 2023-10-09T05:46:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953115+0000", 2023-10-09T05:46:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953115+0000", 2023-10-09T05:46:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953113+0000", 2023-10-09T05:46:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6590000000000002e-06 2023-10-09T05:46:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953123+0000", 2023-10-09T05:46:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4805e-05 2023-10-09T05:46:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953138+0000", 2023-10-09T05:46:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0509999999999999e-06 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953145+0000", 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7503000000000002e-05 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953202+0000", 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014179000000000001 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953344+0000", 2023-10-09T05:46:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015102799999999999 2023-10-09T05:46:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953495+0000", 2023-10-09T05:46:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0043468650000000001 2023-10-09T05:46:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.957842+0000", 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3988999999999997e-05 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.957936+0000", 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27430302200000001 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232239+0000", 2023-10-09T05:46:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5869e-05 2023-10-09T05:46:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232255+0000", 2023-10-09T05:46:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27913971500000001 2023-10-09T05:46:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125688 7.1 7.eda1af1 (undecoded) ondisk+write+known_if_redirected e238)", 2023-10-09T05:46:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.992303+0000", 2023-10-09T05:46:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 122.452637869, 2023-10-09T05:46:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24012315200000001, 2023-10-09T05:46:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:46:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125688 2023-10-09T05:46:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992303+0000", 2023-10-09T05:46:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992303+0000", 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999971 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992300+0000", 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4301000000000001e-05 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992314+0000", 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2853e-05 2023-10-09T05:46:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992337+0000", 2023-10-09T05:46:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1476e-05 2023-10-09T05:46:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992348+0000", 2023-10-09T05:46:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.239966718 2023-10-09T05:46:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232315+0000", 2023-10-09T05:46:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000110808 2023-10-09T05:46:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232426+0000", 2023-10-09T05:46:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24012315200000001 2023-10-09T05:46:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125677 7.11 7:8932b233:::benchmark_data_smithi008_354649_object31181:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 122.447487878, 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0454934010000001, 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125677 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:46:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:46:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999967 2023-10-09T05:46:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997449+0000", 2023-10-09T05:46:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3732e-05 2023-10-09T05:46:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997463+0000", 2023-10-09T05:46:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9613999999999999e-05 2023-10-09T05:46:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997483+0000", 2023-10-09T05:46:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0620000000000007e-06 2023-10-09T05:46:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997492+0000", 2023-10-09T05:46:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.234801074 2023-10-09T05:46:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.488 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.488 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.488 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232293+0000", 2023-10-09T05:46:05.488 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2490000000000001e-06 2023-10-09T05:46:05.488 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232301+0000", 2023-10-09T05:46:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81054634999999997 2023-10-09T05:46:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042848+0000", 2023-10-09T05:46:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8449999999999994e-05 2023-10-09T05:46:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042946+0000", 2023-10-09T05:46:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0454934010000001 2023-10-09T05:46:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125688 7.11 7:8f585b70:::benchmark_data_smithi008_354649_object31192:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:46:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:46:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 122.447440309, 2023-10-09T05:46:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0556018709999999, 2023-10-09T05:46:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125688 2023-10-09T05:46:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:46:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:46:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:46:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:46:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2820000000000001e-06 2023-10-09T05:46:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997503+0000", 2023-10-09T05:46:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0820000000000004e-06 2023-10-09T05:46:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997509+0000", 2023-10-09T05:46:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0659999999999997e-06 2023-10-09T05:46:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997513+0000", 2023-10-09T05:46:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.234809199 2023-10-09T05:46:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232322+0000", 2023-10-09T05:46:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6849999999999998e-06 2023-10-09T05:46:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232328+0000", 2023-10-09T05:46:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81122250699999998 2023-10-09T05:46:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043550+0000", 2023-10-09T05:46:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114354 2023-10-09T05:46:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043665+0000", 2023-10-09T05:46:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00033335100000000001 2023-10-09T05:46:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043998+0000", 2023-10-09T05:46:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001666088 2023-10-09T05:46:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.045664+0000", 2023-10-09T05:46:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041383430000000001 2023-10-09T05:46:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.049803+0000", 2023-10-09T05:46:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032002139999999998 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053003+0000", 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7229999999999999e-05 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053060+0000", 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1964e-05 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053102+0000", 2023-10-09T05:46:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0556018709999999 2023-10-09T05:46:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45365.0:125687 7.9 e238/213)", 2023-10-09T05:46:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:03.006653+0000", 2023-10-09T05:46:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 122.438287368, 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.225546943, 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.006653+0000", 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.503 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.006653+0000", 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.006651+0000", 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8607e-05 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.006669+0000", 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2160000000000001e-05 2023-10-09T05:46:05.504 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.006691+0000", 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2140999999999999e-05 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.006704+0000", 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22547577799999999 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.505 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232179+0000", 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0699e-05 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232200+0000", 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.225546943 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.506 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.507 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:135920 7.1 7:8276760d:::benchmark_data_smithi008_354649_object41424:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-09T05:46:05.507 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:35.184886+0000", 2023-10-09T05:46:05.507 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 90.260054178999994, 2023-10-09T05:46:05.507 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013395374, 2023-10-09T05:46:05.507 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.507 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.507 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.507 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.507 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.507 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135920 2023-10-09T05:46:05.507 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.184886+0000", 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.184886+0000", 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.184884+0000", 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1771e-05 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.184896+0000", 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9143999999999998e-05 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.184915+0000", 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0369999999999995e-06 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.184923+0000", 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6415999999999997e-05 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.184969+0000", 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012864699999999999 2023-10-09T05:46:05.510 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.511 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.511 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.511 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.185098+0000", 2023-10-09T05:46:05.511 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00039998699999999998 2023-10-09T05:46:05.511 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.511 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.511 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.511 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.185498+0000", 2023-10-09T05:46:05.511 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002070512 2023-10-09T05:46:05.511 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.511 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.512 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.512 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.187568+0000", 2023-10-09T05:46:05.512 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038103070000000002 2023-10-09T05:46:05.512 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.512 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.512 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.512 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.191379+0000", 2023-10-09T05:46:05.512 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068014470000000004 2023-10-09T05:46:05.512 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.512 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.512 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.513 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.198180+0000", 2023-10-09T05:46:05.513 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6978000000000002e-05 2023-10-09T05:46:05.513 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.513 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.513 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.513 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.198237+0000", 2023-10-09T05:46:05.513 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4561000000000002e-05 2023-10-09T05:46:05.513 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.514 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.514 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.514 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.198282+0000", 2023-10-09T05:46:05.514 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013395374 2023-10-09T05:46:05.514 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.515 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.515 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.515 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.515 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.515 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:145025 7.1 7:8243f52c:::benchmark_data_smithi008_354649_object50529:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-10-09T05:46:05.516 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:01.084410+0000", 2023-10-09T05:46:05.516 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 64.360530451000002, 2023-10-09T05:46:05.516 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.044882155, 2023-10-09T05:46:05.516 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.516 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.516 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.516 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.517 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.517 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 145025 2023-10-09T05:46:05.517 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.517 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.517 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.517 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.517 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.084410+0000", 2023-10-09T05:46:05.517 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.518 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.518 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.518 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.518 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.084410+0000", 2023-10-09T05:46:05.518 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:05.518 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.518 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.519 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.519 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.084409+0000", 2023-10-09T05:46:05.519 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1419999999999998e-06 2023-10-09T05:46:05.519 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.519 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.519 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.519 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.084415+0000", 2023-10-09T05:46:05.520 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2016e-05 2023-10-09T05:46:05.520 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.520 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.520 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.520 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.084427+0000", 2023-10-09T05:46:05.520 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2909999999999998e-06 2023-10-09T05:46:05.520 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.520 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.520 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.521 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.084432+0000", 2023-10-09T05:46:05.521 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1053000000000002e-05 2023-10-09T05:46:05.521 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.521 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.521 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.521 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.084463+0000", 2023-10-09T05:46:05.521 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1718000000000005e-05 2023-10-09T05:46:05.521 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.521 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.521 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.521 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.084555+0000", 2023-10-09T05:46:05.522 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000370534 2023-10-09T05:46:05.522 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.522 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.522 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.522 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.084925+0000", 2023-10-09T05:46:05.522 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016391260000000001 2023-10-09T05:46:05.522 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.522 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.522 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.523 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.086565+0000", 2023-10-09T05:46:05.523 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001070035 2023-10-09T05:46:05.523 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.523 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.523 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.523 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.087635+0000", 2023-10-09T05:46:05.523 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.041550525999999997 2023-10-09T05:46:05.523 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.523 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.523 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.523 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.129185+0000", 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3987999999999998e-05 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.129249+0000", 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3080000000000001e-05 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:01.129292+0000", 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.044882155 2023-10-09T05:46:05.524 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150768 7.1 7:845e1b23:::benchmark_data_smithi008_354649_object56272:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:19.173620+0000", 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 46.271320690000003, 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016151927999999999, 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.525 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.526 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.526 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150768 2023-10-09T05:46:05.526 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.526 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.526 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.526 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.526 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.173620+0000", 2023-10-09T05:46:05.526 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.526 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.526 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.526 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.527 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.173620+0000", 2023-10-09T05:46:05.527 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:05.527 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.527 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.527 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.527 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.173618+0000", 2023-10-09T05:46:05.527 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1022000000000001e-05 2023-10-09T05:46:05.527 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.527 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.527 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.527 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.173629+0000", 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1625e-05 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.173651+0000", 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7230000000000004e-06 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.173658+0000", 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2576000000000005e-05 2023-10-09T05:46:05.528 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.173721+0000", 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017468400000000001 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.173896+0000", 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00057691000000000003 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.530 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.174472+0000", 2023-10-09T05:46:05.530 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026352739999999999 2023-10-09T05:46:05.530 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.530 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.530 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.530 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.177108+0000", 2023-10-09T05:46:05.530 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001773948 2023-10-09T05:46:05.530 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.530 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.530 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.530 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.178882+0000", 2023-10-09T05:46:05.531 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010817986 2023-10-09T05:46:05.531 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.531 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.531 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.531 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189700+0000", 2023-10-09T05:46:05.531 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4913000000000002e-05 2023-10-09T05:46:05.531 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.531 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.532 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.532 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189745+0000", 2023-10-09T05:46:05.532 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7103000000000001e-05 2023-10-09T05:46:05.532 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.532 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189772+0000", 2023-10-09T05:46:05.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016151927999999999 2023-10-09T05:46:05.533 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.533 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.533 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.534 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.534 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150820 7.1 7:858b74ee:::benchmark_data_smithi008_354649_object56324:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-10-09T05:46:05.534 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:19.277408+0000", 2023-10-09T05:46:05.534 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 46.167532500999997, 2023-10-09T05:46:05.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013573847, 2023-10-09T05:46:05.535 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.535 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.535 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.535 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.535 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.535 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150820 2023-10-09T05:46:05.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.536 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277408+0000", 2023-10-09T05:46:05.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277408+0000", 2023-10-09T05:46:05.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:05.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277406+0000", 2023-10-09T05:46:05.538 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7800000000000006e-06 2023-10-09T05:46:05.538 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.538 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.538 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.538 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277415+0000", 2023-10-09T05:46:05.538 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8309999999999999e-05 2023-10-09T05:46:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277433+0000", 2023-10-09T05:46:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4460000000000001e-06 2023-10-09T05:46:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277442+0000", 2023-10-09T05:46:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5409999999999999e-05 2023-10-09T05:46:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277507+0000", 2023-10-09T05:46:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012991700000000001 2023-10-09T05:46:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277637+0000", 2023-10-09T05:46:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00049892700000000003 2023-10-09T05:46:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.278136+0000", 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004505981 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.282642+0000", 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001957477 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.284599+0000", 2023-10-09T05:46:05.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063005350000000003 2023-10-09T05:46:05.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.290900+0000", 2023-10-09T05:46:05.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6424999999999997e-05 2023-10-09T05:46:05.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.290946+0000", 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5404999999999997e-05 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.290982+0000", 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013573847 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.543 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150821 7.11 7:8b2629d6:::benchmark_data_smithi008_354649_object56325:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-10-09T05:46:05.544 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:19.277460+0000", 2023-10-09T05:46:05.544 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 46.167480306999998, 2023-10-09T05:46:05.544 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013346757000000001, 2023-10-09T05:46:05.544 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.544 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.544 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.544 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.544 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.544 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150821 2023-10-09T05:46:05.544 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.544 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277460+0000", 2023-10-09T05:46:05.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277460+0000", 2023-10-09T05:46:05.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:46:05.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.546 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.546 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277460+0000", 2023-10-09T05:46:05.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8469999999999998e-06 2023-10-09T05:46:05.546 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.546 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.546 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.546 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277463+0000", 2023-10-09T05:46:05.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5597999999999999e-05 2023-10-09T05:46:05.546 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.546 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.546 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277479+0000", 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1104000000000001e-05 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277490+0000", 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6019999999999997e-05 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277526+0000", 2023-10-09T05:46:05.547 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013052699999999999 2023-10-09T05:46:05.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.548 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.548 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.277657+0000", 2023-10-09T05:46:05.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00040716400000000001 2023-10-09T05:46:05.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.548 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.548 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.278064+0000", 2023-10-09T05:46:05.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045626699999999996 2023-10-09T05:46:05.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.549 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.282626+0000", 2023-10-09T05:46:05.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2640000000000002e-06 2023-10-09T05:46:05.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.549 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.282634+0000", 2023-10-09T05:46:05.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080679180000000003 2023-10-09T05:46:05.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.290702+0000", 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1388000000000002e-05 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.290763+0000", 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3835999999999998e-05 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.290807+0000", 2023-10-09T05:46:05.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013346757000000001 2023-10-09T05:46:05.553 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.553 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.553 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.553 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.553 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150849 7.11 7:8af65dfc:::benchmark_data_smithi008_354649_object56353:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-10-09T05:46:05.553 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:19.322210+0000", 2023-10-09T05:46:05.554 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 46.122730582999999, 2023-10-09T05:46:05.554 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013873292000000001, 2023-10-09T05:46:05.554 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.554 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.554 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.554 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.554 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.554 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150849 2023-10-09T05:46:05.554 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.554 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.322210+0000", 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.322210+0000", 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.555 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.322207+0000", 2023-10-09T05:46:05.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2850999999999999e-05 2023-10-09T05:46:05.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.556 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.556 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.322220+0000", 2023-10-09T05:46:05.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2909e-05 2023-10-09T05:46:05.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.557 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.322243+0000", 2023-10-09T05:46:05.557 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8310000000000007e-06 2023-10-09T05:46:05.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.322252+0000", 2023-10-09T05:46:05.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012243099999999999 2023-10-09T05:46:05.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.323476+0000", 2023-10-09T05:46:05.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000110062 2023-10-09T05:46:05.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.323586+0000", 2023-10-09T05:46:05.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00038109999999999999 2023-10-09T05:46:05.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.323967+0000", 2023-10-09T05:46:05.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001950647 2023-10-09T05:46:05.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.561 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.561 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.325918+0000", 2023-10-09T05:46:05.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00078153700000000001 2023-10-09T05:46:05.561 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.561 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.326700+0000", 2023-10-09T05:46:05.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0092860470000000004 2023-10-09T05:46:05.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.562 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.335986+0000", 2023-10-09T05:46:05.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5600000000000003e-05 2023-10-09T05:46:05.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.336041+0000", 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1921000000000001e-05 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.336083+0000", 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013873292000000001 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.563 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.564 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150856 7.11 7:8840fb8e:::benchmark_data_smithi008_354649_object56360:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-10-09T05:46:05.564 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:19.329335+0000", 2023-10-09T05:46:05.564 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 46.115605930999997, 2023-10-09T05:46:05.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014918015, 2023-10-09T05:46:05.564 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.564 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.564 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.564 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150856 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.329335+0000", 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.329335+0000", 2023-10-09T05:46:05.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:05.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.329333+0000", 2023-10-09T05:46:05.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1029999999999998e-06 2023-10-09T05:46:05.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.329339+0000", 2023-10-09T05:46:05.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3217999999999999e-05 2023-10-09T05:46:05.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.568 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.329353+0000", 2023-10-09T05:46:05.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2859999999999999e-06 2023-10-09T05:46:05.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.329358+0000", 2023-10-09T05:46:05.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4573000000000003e-05 2023-10-09T05:46:05.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.329392+0000", 2023-10-09T05:46:05.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0385999999999998e-05 2023-10-09T05:46:05.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.329483+0000", 2023-10-09T05:46:05.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000328258 2023-10-09T05:46:05.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.571 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.571 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.329811+0000", 2023-10-09T05:46:05.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015004770000000001 2023-10-09T05:46:05.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.331312+0000", 2023-10-09T05:46:05.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00080768500000000004 2023-10-09T05:46:05.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.332119+0000", 2023-10-09T05:46:05.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012039395 2023-10-09T05:46:05.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.344159+0000", 2023-10-09T05:46:05.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8395e-05 2023-10-09T05:46:05.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.344207+0000", 2023-10-09T05:46:05.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5439999999999999e-05 2023-10-09T05:46:05.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.344253+0000", 2023-10-09T05:46:05.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014918015 2023-10-09T05:46:05.576 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.576 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.576 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.577 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150861 7.1 7:82139618:::benchmark_data_smithi008_354649_object56365:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-10-09T05:46:05.577 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:19.343292+0000", 2023-10-09T05:46:05.577 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 46.101647964999998, 2023-10-09T05:46:05.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018973871999999999, 2023-10-09T05:46:05.577 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.577 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.577 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.578 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.578 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.578 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150861 2023-10-09T05:46:05.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.578 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.343292+0000", 2023-10-09T05:46:05.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.343292+0000", 2023-10-09T05:46:05.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:05.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.343291+0000", 2023-10-09T05:46:05.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9280000000000005e-06 2023-10-09T05:46:05.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.343300+0000", 2023-10-09T05:46:05.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0013999999999998e-05 2023-10-09T05:46:05.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.343320+0000", 2023-10-09T05:46:05.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0380000000000002e-06 2023-10-09T05:46:05.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.343328+0000", 2023-10-09T05:46:05.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0649999999999998e-05 2023-10-09T05:46:05.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.343378+0000", 2023-10-09T05:46:05.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013286999999999999 2023-10-09T05:46:05.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.343511+0000", 2023-10-09T05:46:05.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00052771000000000003 2023-10-09T05:46:05.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.344039+0000", 2023-10-09T05:46:05.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028487830000000001 2023-10-09T05:46:05.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.346888+0000", 2023-10-09T05:46:05.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002600513 2023-10-09T05:46:05.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.349488+0000", 2023-10-09T05:46:05.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012686513 2023-10-09T05:46:05.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.587 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.362175+0000", 2023-10-09T05:46:05.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9497999999999999e-05 2023-10-09T05:46:05.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.587 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.362224+0000", 2023-10-09T05:46:05.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2314999999999997e-05 2023-10-09T05:46:05.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.362266+0000", 2023-10-09T05:46:05.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018973871999999999 2023-10-09T05:46:05.589 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.589 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.589 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.589 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150945 7.1 7:80f629ca:::benchmark_data_smithi008_354649_object56449:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-10-09T05:46:05.589 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:19.447518+0000", 2023-10-09T05:46:05.590 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 45.997422514, 2023-10-09T05:46:05.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019899511000000002, 2023-10-09T05:46:05.590 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.590 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.590 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.590 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.590 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.591 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150945 2023-10-09T05:46:05.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.591 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.447518+0000", 2023-10-09T05:46:05.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.447518+0000", 2023-10-09T05:46:05.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:05.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.447516+0000", 2023-10-09T05:46:05.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0710000000000001e-06 2023-10-09T05:46:05.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.447525+0000", 2023-10-09T05:46:05.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7096999999999999e-05 2023-10-09T05:46:05.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.447543+0000", 2023-10-09T05:46:05.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8730000000000001e-06 2023-10-09T05:46:05.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.447549+0000", 2023-10-09T05:46:05.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.202e-05 2023-10-09T05:46:05.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.447601+0000", 2023-10-09T05:46:05.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012465499999999999 2023-10-09T05:46:05.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.447726+0000", 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00046921000000000002 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.448195+0000", 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001914065 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.450109+0000", 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002790304 2023-10-09T05:46:05.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.452900+0000", 2023-10-09T05:46:05.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014421544 2023-10-09T05:46:05.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.467321+0000", 2023-10-09T05:46:05.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5393000000000003e-05 2023-10-09T05:46:05.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.467377+0000", 2023-10-09T05:46:05.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0852000000000001e-05 2023-10-09T05:46:05.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.467417+0000", 2023-10-09T05:46:05.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019899511000000002 2023-10-09T05:46:05.600 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.600 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.600 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.600 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:159355 7.1 7:857df81f:::benchmark_data_smithi008_354649_object64859:head [delete] snapc 0=[] ondisk+write+known_if_redirected e256)", 2023-10-09T05:46:05.600 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:42.147485+0000", 2023-10-09T05:46:05.600 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.297455803999998, 2023-10-09T05:46:05.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014204672999999999, 2023-10-09T05:46:05.600 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.601 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.601 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.601 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.601 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.601 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 159355 2023-10-09T05:46:05.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.601 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.147485+0000", 2023-10-09T05:46:05.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.147485+0000", 2023-10-09T05:46:05.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:05.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.147483+0000", 2023-10-09T05:46:05.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2499999999999995e-06 2023-10-09T05:46:05.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.147492+0000", 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7929999999999999e-05 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.147510+0000", 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9290000000000001e-06 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.603 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.147517+0000", 2023-10-09T05:46:05.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5311e-05 2023-10-09T05:46:05.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.147562+0000", 2023-10-09T05:46:05.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010749800000000001 2023-10-09T05:46:05.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.147670+0000", 2023-10-09T05:46:05.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00043082100000000001 2023-10-09T05:46:05.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.148101+0000", 2023-10-09T05:46:05.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002395014 2023-10-09T05:46:05.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.150496+0000", 2023-10-09T05:46:05.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00029811200000000001 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.150794+0000", 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010787115999999999 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.161581+0000", 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3044000000000002e-05 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.161644+0000", 2023-10-09T05:46:05.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5277e-05 2023-10-09T05:46:05.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:42.161689+0000", 2023-10-09T05:46:05.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014204672999999999 2023-10-09T05:46:05.607 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.607 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.607 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:160576 7.11 7:884161db:::benchmark_data_smithi008_354649_object66080:head [delete] snapc 0=[] ondisk+write+known_if_redirected e256)", 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:45.437984+0000", 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 20.006956335000002, 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018519104000000002, 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.608 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 160576 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.437984+0000", 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.437984+0000", 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:05.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.437982+0000", 2023-10-09T05:46:05.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5009999999999993e-06 2023-10-09T05:46:05.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.437992+0000", 2023-10-09T05:46:05.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7765999999999999e-05 2023-10-09T05:46:05.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.438010+0000", 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7150000000000004e-06 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.438018+0000", 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1969999999999999e-05 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.438070+0000", 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000143304 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.438214+0000", 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00045828999999999998 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.438672+0000", 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025306460000000001 2023-10-09T05:46:05.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.441202+0000", 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001898416 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.443101+0000", 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013324271 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.456425+0000", 2023-10-09T05:46:05.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5367999999999997e-05 2023-10-09T05:46:05.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.456461+0000", 2023-10-09T05:46:05.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2718e-05 2023-10-09T05:46:05.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:45.456503+0000", 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018519104000000002 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:161602 7.11 7:8f99fb30:::benchmark_data_smithi008_354649_object67106:head [delete] snapc 0=[] ondisk+write+known_if_redirected e257)", 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:48.380971+0000", 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 17.063969917000001, 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.071117995000000003, 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.615 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 161602 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.380971+0000", 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.380971+0000", 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.380969+0000", 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3790000000000003e-06 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.380976+0000", 2023-10-09T05:46:05.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4600000000000001e-05 2023-10-09T05:46:05.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.380990+0000", 2023-10-09T05:46:05.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3380000000000001e-06 2023-10-09T05:46:05.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.380997+0000", 2023-10-09T05:46:05.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3752999999999999e-05 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.381040+0000", 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7157999999999995e-05 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.381117+0000", 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00030821899999999998 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.381426+0000", 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021177729999999999 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.383543+0000", 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000368895 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.383912+0000", 2023-10-09T05:46:05.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.068101079999999994 2023-10-09T05:46:05.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.452013+0000", 2023-10-09T05:46:05.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5966000000000001e-05 2023-10-09T05:46:05.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.452049+0000", 2023-10-09T05:46:05.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9171999999999997e-05 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:48.452089+0000", 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.071117995000000003 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:162110 7.11 7:8c69bf99:::benchmark_data_smithi008_354649_object67614:head [delete] snapc 0=[] ondisk+write+known_if_redirected e257)", 2023-10-09T05:46:05.622 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:49.934619+0000", 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 15.510321525, 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015067380999999999, 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 162110 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:05.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.934619+0000", 2023-10-09T05:46:05.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:05.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:05.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.934619+0000", 2023-10-09T05:46:05.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:05.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:05.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.934617+0000", 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.109e-06 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.934624+0000", 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5031000000000001e-05 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.934639+0000", 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1790000000000002e-06 2023-10-09T05:46:05.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.934646+0000", 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6938999999999999e-05 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.934693+0000", 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013533499999999999 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:05.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.934829+0000", 2023-10-09T05:46:05.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000461452 2023-10-09T05:46:05.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:46:05.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.935290+0000", 2023-10-09T05:46:05.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022696019999999999 2023-10-09T05:46:05.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:05.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.937560+0000", 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003482307 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.941042+0000", 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0085382370000000006 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.949580+0000", 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2885999999999994e-05 2023-10-09T05:46:05.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.949643+0000", 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3192000000000003e-05 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:49.949686+0000", 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015067380999999999 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.629 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.630 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:05.630 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:05.630 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:46:05.630 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:05.631 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:05 smithi008 ceph-mon[184418]: pgmap v787: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 27929/67962 objects degraded (41.095%); 4233/67962 objects misplaced (6.228%); 4.3 MiB/s, 200 objects/s recovering 2023-10-09T05:46:05.631 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:05 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1861775829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:05.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:05 smithi191 ceph-mon[161669]: pgmap v787: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 27929/67962 objects degraded (41.095%); 4233/67962 objects misplaced (6.228%); 4.3 MiB/s, 200 objects/s recovering 2023-10-09T05:46:05.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:05 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1861775829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:05.632 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:05 smithi008 ceph-mon[191227]: pgmap v787: 99 pgs: 4 active+recovery_wait+degraded, 80 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 27929/67962 objects degraded (41.095%); 4233/67962 objects misplaced (6.228%); 4.3 MiB/s, 200 objects/s recovering 2023-10-09T05:46:05.632 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:05 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1861775829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:05.663 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:06.029 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:06.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:06.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:06.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:06.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:46:06.168 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:06.194 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:46:06.194 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:46:06.194 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2023-10-09T05:46:06.194 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:46:06.225 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:06.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:06.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:06.334 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:06.352 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:46:06.388 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:06.592 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:46:06.693 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:06.695 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:06 smithi008 ceph-mon[184418]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T05:46:06.695 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:06 smithi008 ceph-mon[184418]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.695 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:06 smithi008 ceph-mon[184418]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.695 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:06 smithi008 ceph-mon[184418]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.696 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:06 smithi008 ceph-mon[184418]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:06.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:06.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:06 smithi008 ceph-mon[191227]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T05:46:06.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:06 smithi008 ceph-mon[191227]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:06 smithi008 ceph-mon[191227]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:06 smithi008 ceph-mon[191227]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.698 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:06 smithi008 ceph-mon[191227]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.797 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:06.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:06 smithi191 ceph-mon[161669]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T05:46:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:06 smithi191 ceph-mon[161669]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:06 smithi191 ceph-mon[161669]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:06 smithi191 ceph-mon[161669]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:06 smithi191 ceph-mon[161669]: failed to encode map e259 with expected crc 2023-10-09T05:46:06.842 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:06.939 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:06.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:06.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:07.047 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:07.087 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:07.150 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:07.196 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:07.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:07.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:07.238 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:07.300 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:07.337 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:07.398 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:07 smithi191 ceph-mon[161669]: pgmap v789: 99 pgs: 3 active+recovery_wait+degraded, 81 active+clean, 1 active+undersized+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 27797/66597 objects degraded (41.739%); 4117/66597 objects misplaced (6.182%); 2.6 MiB/s, 158 objects/s recovering 2023-10-09T05:46:07.398 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:07 smithi191 ceph-mon[161669]: failed to encode map e260 with expected crc 2023-10-09T05:46:07.398 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:07 smithi191 ceph-mon[161669]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T05:46:07.398 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:07 smithi191 ceph-mon[161669]: failed to encode map e260 with expected crc 2023-10-09T05:46:07.398 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:07 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:07.398 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:07 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:07.418 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:07.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:07.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:07.530 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:07.561 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:07.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[184418]: pgmap v789: 99 pgs: 3 active+recovery_wait+degraded, 81 active+clean, 1 active+undersized+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 27797/66597 objects degraded (41.739%); 4117/66597 objects misplaced (6.182%); 2.6 MiB/s, 158 objects/s recovering 2023-10-09T05:46:07.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[184418]: failed to encode map e260 with expected crc 2023-10-09T05:46:07.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[184418]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T05:46:07.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[184418]: failed to encode map e260 with expected crc 2023-10-09T05:46:07.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:07.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:07.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[191227]: pgmap v789: 99 pgs: 3 active+recovery_wait+degraded, 81 active+clean, 1 active+undersized+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 27797/66597 objects degraded (41.739%); 4117/66597 objects misplaced (6.182%); 2.6 MiB/s, 158 objects/s recovering 2023-10-09T05:46:07.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[191227]: failed to encode map e260 with expected crc 2023-10-09T05:46:07.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[191227]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T05:46:07.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[191227]: failed to encode map e260 with expected crc 2023-10-09T05:46:07.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:07.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:07 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:07.724 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:07.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:07.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:07.828 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:07.860 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:08.158 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:46:08.158 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:46:08.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.158 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:168318 7.b 7:d745b472:::benchmark_data_smithi008_354649_object73822:head [delete] snapc 0=[] ondisk+write+known_if_redirected e260)", 2023-10-09T05:46:08.158 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:08.152367+0000", 2023-10-09T05:46:08.158 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0054860799999999999, 2023-10-09T05:46:08.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005508092, 2023-10-09T05:46:08.158 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 168318 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.152367+0000", 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:08.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.152367+0000", 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.152365+0000", 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0997e-05 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:08.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.152376+0000", 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4280000000000001e-05 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.152400+0000", 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.005e-05 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.152410+0000", 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1660000000000003e-05 2023-10-09T05:46:08.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.152472+0000", 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000156966 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.152629+0000", 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016463799999999999 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.152793+0000", 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004248964 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157042+0000", 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0207000000000003e-05 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157083+0000", 2023-10-09T05:46:08.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0074000000000003e-05 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157123+0000", 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047555519999999997 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:168319 7.b 7:d1907901:::benchmark_data_smithi008_354649_object73823:head [delete] snapc 0=[] ondisk+write+known_if_redirected e260)", 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:08.157220+0000", 2023-10-09T05:46:08.164 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.00063314399999999998, 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00072208799999999996, 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 168319 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:08.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157220+0000", 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157220+0000", 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157219+0000", 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6559999999999992e-06 2023-10-09T05:46:08.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157226+0000", 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2979999999999992e-06 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157235+0000", 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1200000000000004e-06 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157239+0000", 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5407000000000002e-05 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157274+0000", 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8030999999999997e-05 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157372+0000", 2023-10-09T05:46:08.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5823999999999996e-05 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157468+0000", 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00024792999999999999 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.45365.0:168318 7.b e260/253)", 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:08.157311+0000", 2023-10-09T05:46:08.169 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.00054221100000000002, 2023-10-09T05:46:08.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00067495999999999995, 2023-10-09T05:46:08.170 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:08.170 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:46:08.170 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:08.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:08.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157311+0000", 2023-10-09T05:46:08.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:08.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157311+0000", 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157310+0000", 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.933e-06 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157315+0000", 2023-10-09T05:46:08.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0227999999999999e-05 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157325+0000", 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.733e-06 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.157331+0000", 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0225e-05 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:08.172 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:08.173 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:08.173 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:46:08.173 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 3 2023-10-09T05:46:08.173 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:46:08.223 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:08.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:08.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:08.328 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:08.396 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:08.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:08 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:08.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:08 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:08.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:08 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:08.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 27797/66597 objects degraded (41.739%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:08 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:08 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:08 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 27797/66597 objects degraded (41.739%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:08.794 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:08 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:08.794 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:08 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:08.794 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:08 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:08.794 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 27797/66597 objects degraded (41.739%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:08.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:46:08.885 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:46:09.339 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:46:10.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:09 smithi008 ceph-mon[184418]: pgmap v791: 99 pgs: 3 active+recovery_wait+degraded, 82 active+clean, 1 active+undersized+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 27797/64809 objects degraded (42.891%); 3858/64809 objects misplaced (5.953%); 3.6 MiB/s, 156 objects/s recovering 2023-10-09T05:46:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:09 smithi008 ceph-mon[191227]: pgmap v791: 99 pgs: 3 active+recovery_wait+degraded, 82 active+clean, 1 active+undersized+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 27797/64809 objects degraded (42.891%); 3858/64809 objects misplaced (5.953%); 3.6 MiB/s, 156 objects/s recovering 2023-10-09T05:46:10.217 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:10.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:10.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:09 smithi191 ceph-mon[161669]: pgmap v791: 99 pgs: 3 active+recovery_wait+degraded, 82 active+clean, 1 active+undersized+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 27797/64809 objects degraded (42.891%); 3858/64809 objects misplaced (5.953%); 3.6 MiB/s, 156 objects/s recovering 2023-10-09T05:46:10.322 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:10.365 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:10.462 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:10.505 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:46:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:46:10] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.33.4" 2023-10-09T05:46:10.653 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:10.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:10.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:10.759 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:10.795 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:10.967 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:46:10.968 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:46:10.968 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:46:10.968 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:46:10.968 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:46:11.133 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:10 smithi008 ceph-mon[184418]: 7.1 scrub starts 2023-10-09T05:46:11.136 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:10 smithi008 ceph-mon[191227]: 7.1 scrub starts 2023-10-09T05:46:11.177 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:11.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:11.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:11.282 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:11.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:10 smithi191 ceph-mon[161669]: 7.1 scrub starts 2023-10-09T05:46:11.310 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:11.356 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:11.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:11.436 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:46:11.439 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:46:11.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:11.506 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:11.707 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:11.713 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:46:11.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:11.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:11.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:11.854 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:11.910 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:11.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:11.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:11.937 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:46:12.017 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:12.048 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:12.113 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:12.119 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:11 smithi191 ceph-mon[161669]: pgmap v792: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 351 op/s; 25997/61695 objects degraded (42.138%); 3799/61695 objects misplaced (6.158%); 8.0 MiB/s, 202 objects/s recovering 2023-10-09T05:46:12.119 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3006079943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:12.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:12.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:12.158 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:46:12.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:11 smithi008 ceph-mon[184418]: pgmap v792: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 351 op/s; 25997/61695 objects degraded (42.138%); 3799/61695 objects misplaced (6.158%); 8.0 MiB/s, 202 objects/s recovering 2023-10-09T05:46:12.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3006079943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:12.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:11 smithi008 ceph-mon[191227]: pgmap v792: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 351 op/s; 25997/61695 objects degraded (42.138%); 3799/61695 objects misplaced (6.158%); 8.0 MiB/s, 202 objects/s recovering 2023-10-09T05:46:12.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3006079943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:12.220 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:12.258 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:12.567 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:12.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:12.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:12.671 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:12.703 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:12.759 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:12.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:12.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:12.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:12.902 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:12.943 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:12.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:12.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:13.046 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:13.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:12 smithi191 ceph-mon[161669]: from='client.55558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:46:13.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:12 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:46:13.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:12 smithi191 ceph-mon[161669]: failed to encode map e261 with expected crc 2023-10-09T05:46:13.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:12 smithi191 ceph-mon[161669]: failed to encode map e261 with expected crc 2023-10-09T05:46:13.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3006079943' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:46:13.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:12 smithi191 ceph-mon[161669]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T05:46:13.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:12 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 25997/61695 objects degraded (42.138%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:13.081 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:13.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[184418]: from='client.55558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:46:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:46:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[184418]: failed to encode map e261 with expected crc 2023-10-09T05:46:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[184418]: failed to encode map e261 with expected crc 2023-10-09T05:46:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3006079943' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:46:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[184418]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T05:46:13.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 25997/61695 objects degraded (42.138%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[191227]: from='client.55558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:46:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:46:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[191227]: failed to encode map e261 with expected crc 2023-10-09T05:46:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[191227]: failed to encode map e261 with expected crc 2023-10-09T05:46:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3006079943' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:46:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[191227]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T05:46:13.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:12 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 25997/61695 objects degraded (42.138%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:13.303 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:13.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:13.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:13.370 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":792,"stamp":"2023-10-09T05:46:10.305639+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1350637515,"num_objects":20565,"num_object_clones":0,"num_object_copies":61695,"num_objects_missing_on_primary":3253,"num_objects_missing":3253,"num_objects_degraded":25997,"num_objects_misplaced":3799,"num_objects_unfound":0,"num_objects_dirty":20565,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":169557,"num_write_kb":6062694,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20536,"num_bytes_recovered":1100163623,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222862,"ondisk_log_size":222862,"up":297,"acting":288,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7433108,"kb_used_data":4665116,"kb_used_omap":38,"kb_used_meta":2767833,"kb_avail":648782956,"statfs":{"total":671965249536,"available":664353746944,"internally_reserved":0,"allocated":4777078784,"data_stored":4771942603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39147,"internal_metadata":2834261781},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,1,0,1,2,3,7],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-182583296,"num_objects":-2786,"num_object_clones":0,"num_object_copies":-8358,"num_objects_missing_on_primary":-229,"num_objects_missing":-229,"num_objects_degraded":-1989,"num_objects_misplaced":-560,"num_objects_unfound":0,"num_objects_dirty":-2786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2786,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1608,"num_bytes_recovered":66256915,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.926243"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":251,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.389567+0000","last_change":"2023-10-09T05:45:17.927530+0000","last_active":"2023-10-09T05:46:05.389567+0000","last_peered":"2023-10-09T05:46:05.389567+0000","last_clean":"2023-10-09T05:46:05.389567+0000","last_became_active":"2023-10-09T05:45:17.926963+0000","last_became_peered":"2023-10-09T05:45:17.926963+0000","last_unstale":"2023-10-09T05:46:05.389567+0000","last_undegraded":"2023-10-09T05:46:05.389567+0000","last_fullsized":"2023-10-09T05:46:05.389567+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":242,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.382846+0000","last_change":"2023-10-09T05:45:17.925436+0000","last_active":"2023-10-09T05:46:05.382846+0000","last_peered":"2023-10-09T05:46:05.382846+0000","last_clean":"2023-10-09T05:46:05.382846+0000","last_became_active":"2023-10-09T05:45:17.925064+0000","last_became_peered":"2023-10-09T05:45:17.925064+0000","last_unstale":"2023-10-09T05:46:05.382846+0000","last_undegraded":"2023-10-09T05:46:05.382846+0000","last_fullsized":"2023-10-09T05:46:05.382846+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:14.651551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":57,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.381963+0000","last_change":"2023-10-09T05:45:17.917244+0000","last_active":"2023-10-09T05:46:05.381963+0000","last_peered":"2023-10-09T05:46:05.381963+0000","last_clean":"2023-10-09T05:46:05.381963+0000","last_became_active":"2023-10-09T05:45:17.917012+0000","last_became_peered":"2023-10-09T05:45:17.917012+0000","last_unstale":"2023-10-09T05:46:05.381963+0000","last_undegraded":"2023-10-09T05:46:05.381963+0000","last_fullsized":"2023-10-09T05:46:05.381963+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:25.765620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":248,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.389428+0000","last_change":"2023-10-09T05:43:19.889677+0000","last_active":"2023-10-09T05:46:05.389428+0000","last_peered":"2023-10-09T05:46:05.389428+0000","last_clean":"2023-10-09T05:46:05.389428+0000","last_became_active":"2023-10-09T05:43:19.889320+0000","last_became_peered":"2023-10-09T05:43:19.889320+0000","last_unstale":"2023-10-09T05:46:05.389428+0000","last_undegraded":"2023-10-09T05:46:05.389428+0000","last_fullsized":"2023-10-09T05:46:05.389428+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:57.376656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":259,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.382631+0000","last_change":"2023-10-09T05:45:17.937090+0000","last_active":"2023-10-09T05:46:05.382631+0000","last_peered":"2023-10-09T05:46:05.382631+0000","last_clean":"2023-10-09T05:46:05.382631+0000","last_became_active":"2023-10-09T05:45:17.936778+0000","last_became_peered":"2023-10-09T05:45:17.936778+0000","last_unstale":"2023-10-09T05:46:05.382631+0000","last_undegraded":"2023-10-09T05:46:05.382631+0000","last_fullsized":"2023-10-09T05:46:05.382631+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:12.815984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":84,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.392830+0000","last_change":"2023-10-09T05:45:17.925251+0000","last_active":"2023-10-09T05:46:05.392830+0000","last_peered":"2023-10-09T05:46:05.392830+0000","last_clean":"2023-10-09T05:46:05.392830+0000","last_became_active":"2023-10-09T05:45:17.924982+0000","last_became_peered":"2023-10-09T05:45:17.924982+0000","last_unstale":"2023-10-09T05:46:05.392830+0000","last_undegraded":"2023-10-09T05:46:05.392830+0000","last_fullsized":"2023-10-09T05:46:05.392830+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"260'9219","reported_seq":9284,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:06.576600+0000","last_change":"2023-10-09T05:44:04.042724+0000","last_active":"2023-10-09T05:46:06.576600+0000","last_peered":"2023-10-09T05:46:06.576600+0000","last_clean":"2023-10-09T05:46:06.576600+0000","last_became_active":"2023-10-09T05:44:04.029419+0000","last_became_peered":"2023-10-09T05:44:04.029419+0000","last_unstale":"2023-10-09T05:46:06.576600+0000","last_undegraded":"2023-10-09T05:46:06.576600+0000","last_fullsized":"2023-10-09T05:46:06.576600+0000","mapping_epoch":239,"log_start":"218'500","ondisk_log_start":"218'500","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8719,"log_dups_size":0,"ondisk_log_size":8719,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42893312,"num_objects":655,"num_object_clones":0,"num_object_copies":1965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5283,"num_write_kb":189984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":213,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.390932+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:46:05.390932+0000","last_peered":"2023-10-09T05:46:05.390932+0000","last_clean":"2023-10-09T05:46:05.390932+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:46:05.390932+0000","last_undegraded":"2023-10-09T05:46:05.390932+0000","last_fullsized":"2023-10-09T05:46:05.390932+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":243,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.382705+0000","last_change":"2023-10-09T05:45:17.922085+0000","last_active":"2023-10-09T05:46:05.382705+0000","last_peered":"2023-10-09T05:46:05.382705+0000","last_clean":"2023-10-09T05:46:05.382705+0000","last_became_active":"2023-10-09T05:45:17.921640+0000","last_became_peered":"2023-10-09T05:45:17.921640+0000","last_unstale":"2023-10-09T05:46:05.382705+0000","last_undegraded":"2023-10-09T05:46:05.382705+0000","last_fullsized":"2023-10-09T05:46:05.382705+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:06.899516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"260'9382","reported_seq":9455,"reported_epoch":260,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:46:06.778304+0000","last_change":"2023-10-09T05:45:19.059245+0000","last_active":"2023-10-09T05:46:06.778304+0000","last_peered":"2023-10-09T05:46:06.778304+0000","last_clean":"2023-10-09T05:45:16.924358+0000","last_became_active":"2023-10-09T05:45:19.022328+0000","last_became_peered":"2023-10-09T05:45:19.022328+0000","last_unstale":"2023-10-09T05:46:06.778304+0000","last_undegraded":"2023-10-09T05:46:06.778304+0000","last_fullsized":"2023-10-09T05:46:06.778304+0000","mapping_epoch":253,"log_start":"218'700","ondisk_log_start":"218'700","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8682,"log_dups_size":0,"ondisk_log_size":8682,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47185920,"num_objects":720,"num_object_clones":0,"num_object_copies":2160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":720,"num_objects_unfound":0,"num_objects_dirty":720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5344,"num_write_kb":194048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,1],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":720}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":269,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.392878+0000","last_change":"2023-10-09T05:45:17.915552+0000","last_active":"2023-10-09T05:46:05.392878+0000","last_peered":"2023-10-09T05:46:05.392878+0000","last_clean":"2023-10-09T05:46:05.392878+0000","last_became_active":"2023-10-09T05:45:17.913215+0000","last_became_peered":"2023-10-09T05:45:17.913215+0000","last_unstale":"2023-10-09T05:46:05.392878+0000","last_undegraded":"2023-10-09T05:46:05.392878+0000","last_fullsized":"2023-10-09T05:46:05.392878+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":257,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.381887+0000","last_change":"2023-10-09T05:43:19.883115+0000","last_active":"2023-10-09T05:46:05.381887+0000","last_peered":"2023-10-09T05:46:05.381887+0000","last_clean":"2023-10-09T05:46:05.381887+0000","last_became_active":"2023-10-09T05:43:19.882674+0000","last_became_peered":"2023-10-09T05:43:19.882674+0000","last_unstale":"2023-10-09T05:46:05.381887+0000","last_undegraded":"2023-10-09T05:46:05.381887+0000","last_fullsized":"2023-10-09T05:46:05.381887+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:30.479481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"260'9138","reported_seq":9200,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:09.897237+0000","last_change":"2023-10-09T05:44:04.043289+0000","last_active":"2023-10-09T05:46:09.897237+0000","last_peered":"2023-10-09T05:46:09.897237+0000","last_clean":"2023-10-09T05:46:09.897237+0000","last_became_active":"2023-10-09T05:44:04.032393+0000","last_became_peered":"2023-10-09T05:44:04.032393+0000","last_unstale":"2023-10-09T05:46:09.897237+0000","last_undegraded":"2023-10-09T05:46:09.897237+0000","last_fullsized":"2023-10-09T05:46:09.897237+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9138,"log_dups_size":0,"ondisk_log_size":9138,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35717120,"num_objects":545,"num_object_clones":0,"num_object_copies":1635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5263,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":45,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.398912+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:46:05.398912+0000","last_peered":"2023-10-09T05:46:05.398912+0000","last_clean":"2023-10-09T05:46:05.398912+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:46:05.398912+0000","last_undegraded":"2023-10-09T05:46:05.398912+0000","last_fullsized":"2023-10-09T05:46:05.398912+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":52,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.395656+0000","last_change":"2023-10-09T05:45:17.924621+0000","last_active":"2023-10-09T05:46:05.395656+0000","last_peered":"2023-10-09T05:46:05.395656+0000","last_clean":"2023-10-09T05:46:05.395656+0000","last_became_active":"2023-10-09T05:45:17.924313+0000","last_became_peered":"2023-10-09T05:45:17.924313+0000","last_unstale":"2023-10-09T05:46:05.395656+0000","last_undegraded":"2023-10-09T05:46:05.395656+0000","last_fullsized":"2023-10-09T05:46:05.395656+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:37.602060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.10","version":"260'9110","reported_seq":9182,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:09.722238+0000","last_change":"2023-10-09T05:45:19.272351+0000","last_active":"2023-10-09T05:46:09.722238+0000","last_peered":"2023-10-09T05:46:09.722238+0000","last_clean":"2023-10-09T05:45:16.956686+0000","last_became_active":"2023-10-09T05:45:19.244940+0000","last_became_peered":"2023-10-09T05:45:19.244940+0000","last_unstale":"2023-10-09T05:46:09.722238+0000","last_undegraded":"2023-10-09T05:45:18.965723+0000","last_fullsized":"2023-10-09T05:45:18.943261+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9110,"log_dups_size":0,"ondisk_log_size":9110,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41091072,"num_objects":627,"num_object_clones":0,"num_object_copies":1881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2906,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5232,"num_write_kb":187488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":616},{"shards":"3,5","objects":-552}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":286,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.390067+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:46:05.390067+0000","last_peered":"2023-10-09T05:46:05.390067+0000","last_clean":"2023-10-09T05:46:05.390067+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:46:05.390067+0000","last_undegraded":"2023-10-09T05:46:05.390067+0000","last_fullsized":"2023-10-09T05:46:05.390067+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":234,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.391127+0000","last_change":"2023-10-09T05:45:17.925631+0000","last_active":"2023-10-09T05:46:05.391127+0000","last_peered":"2023-10-09T05:46:05.391127+0000","last_clean":"2023-10-09T05:46:05.391127+0000","last_became_active":"2023-10-09T05:45:17.925142+0000","last_became_peered":"2023-10-09T05:45:17.925142+0000","last_unstale":"2023-10-09T05:46:05.391127+0000","last_undegraded":"2023-10-09T05:46:05.391127+0000","last_fullsized":"2023-10-09T05:46:05.391127+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"260'9098","reported_seq":9169,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:08.666879+0000","last_change":"2023-10-09T05:44:50.515353+0000","last_active":"2023-10-09T05:46:08.666879+0000","last_peered":"2023-10-09T05:46:08.666879+0000","last_clean":"2023-10-09T05:46:08.666879+0000","last_became_active":"2023-10-09T05:44:50.499263+0000","last_became_peered":"2023-10-09T05:44:50.499263+0000","last_unstale":"2023-10-09T05:46:08.666879+0000","last_undegraded":"2023-10-09T05:46:08.666879+0000","last_fullsized":"2023-10-09T05:46:08.666879+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9098,"log_dups_size":0,"ondisk_log_size":9098,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40796160,"num_objects":623,"num_object_clones":0,"num_object_copies":1869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5215,"num_write_kb":186784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":45,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.394019+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:46:05.394019+0000","last_peered":"2023-10-09T05:46:05.394019+0000","last_clean":"2023-10-09T05:46:05.394019+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:46:05.394019+0000","last_undegraded":"2023-10-09T05:46:05.394019+0000","last_fullsized":"2023-10-09T05:46:05.394019+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":276,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.381696+0000","last_change":"2023-10-09T05:45:17.920528+0000","last_active":"2023-10-09T05:46:05.381696+0000","last_peered":"2023-10-09T05:46:05.381696+0000","last_clean":"2023-10-09T05:46:05.381696+0000","last_became_active":"2023-10-09T05:45:17.920250+0000","last_became_peered":"2023-10-09T05:45:17.920250+0000","last_unstale":"2023-10-09T05:46:05.381696+0000","last_undegraded":"2023-10-09T05:46:05.381696+0000","last_fullsized":"2023-10-09T05:46:05.381696+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:08.059709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"260'9334","reported_seq":15424,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:08.682609+0000","last_change":"2023-10-09T05:45:38.011075+0000","last_active":"2023-10-09T05:46:08.682609+0000","last_peered":"2023-10-09T05:46:08.682609+0000","last_clean":"2023-10-09T05:46:08.682609+0000","last_became_active":"2023-10-09T05:44:04.113448+0000","last_became_peered":"2023-10-09T05:44:04.113448+0000","last_unstale":"2023-10-09T05:46:08.682609+0000","last_undegraded":"2023-10-09T05:46:08.682609+0000","last_fullsized":"2023-10-09T05:46:08.682609+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9334,"log_dups_size":0,"ondisk_log_size":9334,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40501248,"num_objects":618,"num_object_clones":0,"num_object_copies":1854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5357,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3006,"num_bytes_recovered":131137536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":246,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.385227+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:46:05.385227+0000","last_peered":"2023-10-09T05:46:05.385227+0000","last_clean":"2023-10-09T05:46:05.385227+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:46:05.385227+0000","last_undegraded":"2023-10-09T05:46:05.385227+0000","last_fullsized":"2023-10-09T05:46:05.385227+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":259,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.392664+0000","last_change":"2023-10-09T05:43:19.886058+0000","last_active":"2023-10-09T05:46:05.392664+0000","last_peered":"2023-10-09T05:46:05.392664+0000","last_clean":"2023-10-09T05:46:05.392664+0000","last_became_active":"2023-10-09T05:43:19.885713+0000","last_became_peered":"2023-10-09T05:43:19.885713+0000","last_unstale":"2023-10-09T05:46:05.392664+0000","last_undegraded":"2023-10-09T05:46:05.392664+0000","last_fullsized":"2023-10-09T05:46:05.392664+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"260'9365","reported_seq":12168,"reported_epoch":260,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:46:10.036837+0000","last_change":"2023-10-09T05:45:17.932003+0000","last_active":"2023-10-09T05:46:10.036837+0000","last_peered":"2023-10-09T05:46:10.036837+0000","last_clean":"2023-10-09T05:43:18.738931+0000","last_became_active":"2023-10-09T05:44:05.020084+0000","last_became_peered":"2023-10-09T05:44:05.020084+0000","last_unstale":"2023-10-09T05:46:10.036837+0000","last_undegraded":"2023-10-09T05:45:17.927587+0000","last_fullsized":"2023-10-09T05:46:10.036837+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9365,"log_dups_size":0,"ondisk_log_size":9365,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39944192,"num_objects":610,"num_object_clones":0,"num_object_copies":1830,"num_objects_missing_on_primary":1578,"num_objects_missing":1578,"num_objects_degraded":1578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5364,"num_write_kb":191136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1362,"num_bytes_recovered":89260032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":596},{"shards":"0,1,2,7","objects":7},{"shards":"0,2,7","objects":7}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":39,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.393958+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:46:05.393958+0000","last_peered":"2023-10-09T05:46:05.393958+0000","last_clean":"2023-10-09T05:46:05.393958+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:46:05.393958+0000","last_undegraded":"2023-10-09T05:46:05.393958+0000","last_fullsized":"2023-10-09T05:46:05.393958+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":246,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.389506+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:46:05.389506+0000","last_peered":"2023-10-09T05:46:05.389506+0000","last_clean":"2023-10-09T05:46:05.389506+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:46:05.389506+0000","last_undegraded":"2023-10-09T05:46:05.389506+0000","last_fullsized":"2023-10-09T05:46:05.389506+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"260'9140","reported_seq":9214,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:08.647197+0000","last_change":"2023-10-09T05:45:19.213860+0000","last_active":"2023-10-09T05:46:08.647197+0000","last_peered":"2023-10-09T05:46:08.647197+0000","last_clean":"2023-10-09T05:45:16.956792+0000","last_became_active":"2023-10-09T05:45:19.185275+0000","last_became_peered":"2023-10-09T05:45:19.185275+0000","last_unstale":"2023-10-09T05:46:08.647197+0000","last_undegraded":"2023-10-09T05:45:18.955431+0000","last_fullsized":"2023-10-09T05:45:18.914423+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9140,"log_dups_size":0,"ondisk_log_size":9140,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43679744,"num_objects":667,"num_object_clones":0,"num_object_copies":2001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2959,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5218,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":668},{"shards":"0,4","objects":-521}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":34,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.384792+0000","last_change":"2023-10-09T05:44:50.483928+0000","last_active":"2023-10-09T05:46:05.384792+0000","last_peered":"2023-10-09T05:46:05.384792+0000","last_clean":"2023-10-09T05:46:05.384792+0000","last_became_active":"2023-10-09T05:44:50.483595+0000","last_became_peered":"2023-10-09T05:44:50.483595+0000","last_unstale":"2023-10-09T05:46:05.384792+0000","last_undegraded":"2023-10-09T05:46:05.384792+0000","last_fullsized":"2023-10-09T05:46:05.384792+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:21.100193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":276,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.384364+0000","last_change":"2023-10-09T05:45:17.919936+0000","last_active":"2023-10-09T05:46:05.384364+0000","last_peered":"2023-10-09T05:46:05.384364+0000","last_clean":"2023-10-09T05:46:05.384364+0000","last_became_active":"2023-10-09T05:45:17.919518+0000","last_became_peered":"2023-10-09T05:45:17.919518+0000","last_unstale":"2023-10-09T05:46:05.384364+0000","last_undegraded":"2023-10-09T05:46:05.384364+0000","last_fullsized":"2023-10-09T05:46:05.384364+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:24.340590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"260'10314","reported_seq":10377,"reported_epoch":260,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:46:10.001555+0000","last_change":"2023-10-09T05:45:19.152449+0000","last_active":"2023-10-09T05:46:10.001555+0000","last_peered":"2023-10-09T05:46:10.001555+0000","last_clean":"2023-10-09T05:45:16.917055+0000","last_became_active":"2023-10-09T05:45:19.101335+0000","last_became_peered":"2023-10-09T05:45:19.101335+0000","last_unstale":"2023-10-09T05:46:10.001555+0000","last_undegraded":"2023-10-09T05:46:10.001555+0000","last_fullsized":"2023-10-09T05:46:10.001555+0000","mapping_epoch":253,"log_start":"221'1951","ondisk_log_start":"221'1951","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8363,"log_dups_size":0,"ondisk_log_size":8363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77332480,"num_objects":1180,"num_object_clones":0,"num_object_copies":3540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1180,"num_objects_unfound":0,"num_objects_dirty":1180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10314,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1180}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":43,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.398957+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:46:05.398957+0000","last_peered":"2023-10-09T05:46:05.398957+0000","last_clean":"2023-10-09T05:46:05.398957+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:46:05.398957+0000","last_undegraded":"2023-10-09T05:46:05.398957+0000","last_fullsized":"2023-10-09T05:46:05.398957+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":244,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.395955+0000","last_change":"2023-10-09T05:43:19.885929+0000","last_active":"2023-10-09T05:46:05.395955+0000","last_peered":"2023-10-09T05:46:05.395955+0000","last_clean":"2023-10-09T05:46:05.395955+0000","last_became_active":"2023-10-09T05:43:19.885542+0000","last_became_peered":"2023-10-09T05:43:19.885542+0000","last_unstale":"2023-10-09T05:46:05.395955+0000","last_undegraded":"2023-10-09T05:46:05.395955+0000","last_fullsized":"2023-10-09T05:46:05.395955+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"260'10316","reported_seq":10373,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:06.759529+0000","last_change":"2023-10-09T05:41:29.081067+0000","last_active":"2023-10-09T05:46:06.759529+0000","last_peered":"2023-10-09T05:46:06.759529+0000","last_clean":"2023-10-09T05:46:06.759529+0000","last_became_active":"2023-10-09T05:41:29.080704+0000","last_became_peered":"2023-10-09T05:41:29.080704+0000","last_unstale":"2023-10-09T05:46:06.759529+0000","last_undegraded":"2023-10-09T05:46:06.759529+0000","last_fullsized":"2023-10-09T05:46:06.759529+0000","mapping_epoch":213,"log_start":"227'4894","ondisk_log_start":"227'4894","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5422,"log_dups_size":0,"ondisk_log_size":5422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:29.483205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85852160,"num_objects":1310,"num_object_clones":0,"num_object_copies":3930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10316,"num_write_kb":372032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":56,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.398426+0000","last_change":"2023-10-09T05:45:18.061225+0000","last_active":"2023-10-09T05:46:05.398426+0000","last_peered":"2023-10-09T05:46:05.398426+0000","last_clean":"2023-10-09T05:46:05.398426+0000","last_became_active":"2023-10-09T05:45:17.923689+0000","last_became_peered":"2023-10-09T05:45:17.923689+0000","last_unstale":"2023-10-09T05:46:05.398426+0000","last_undegraded":"2023-10-09T05:46:05.398426+0000","last_fullsized":"2023-10-09T05:46:05.398426+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":258,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.391095+0000","last_change":"2023-10-09T05:45:17.919668+0000","last_active":"2023-10-09T05:46:05.391095+0000","last_peered":"2023-10-09T05:46:05.391095+0000","last_clean":"2023-10-09T05:46:05.391095+0000","last_became_active":"2023-10-09T05:45:17.919322+0000","last_became_peered":"2023-10-09T05:45:17.919322+0000","last_unstale":"2023-10-09T05:46:05.391095+0000","last_undegraded":"2023-10-09T05:46:05.391095+0000","last_fullsized":"2023-10-09T05:46:05.391095+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"260'9107","reported_seq":9186,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:08.714322+0000","last_change":"2023-10-09T05:45:19.120012+0000","last_active":"2023-10-09T05:46:08.714322+0000","last_peered":"2023-10-09T05:46:08.714322+0000","last_clean":"2023-10-09T05:45:16.881440+0000","last_became_active":"2023-10-09T05:45:19.096430+0000","last_became_peered":"2023-10-09T05:45:19.096430+0000","last_unstale":"2023-10-09T05:46:08.714322+0000","last_undegraded":"2023-10-09T05:45:18.961142+0000","last_fullsized":"2023-10-09T05:45:18.913777+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9107,"log_dups_size":0,"ondisk_log_size":9107,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41877504,"num_objects":639,"num_object_clones":0,"num_object_copies":1917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2949,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5223,"num_write_kb":187584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":679},{"shards":"0,4","objects":-615}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":43,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.398812+0000","last_change":"2023-10-09T05:43:20.148955+0000","last_active":"2023-10-09T05:46:05.398812+0000","last_peered":"2023-10-09T05:46:05.398812+0000","last_clean":"2023-10-09T05:46:05.398812+0000","last_became_active":"2023-10-09T05:43:19.884767+0000","last_became_peered":"2023-10-09T05:43:19.884767+0000","last_unstale":"2023-10-09T05:46:05.398812+0000","last_undegraded":"2023-10-09T05:46:05.398812+0000","last_fullsized":"2023-10-09T05:46:05.398812+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":250,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.394647+0000","last_change":"2023-10-09T05:45:17.923093+0000","last_active":"2023-10-09T05:46:05.394647+0000","last_peered":"2023-10-09T05:46:05.394647+0000","last_clean":"2023-10-09T05:46:05.394647+0000","last_became_active":"2023-10-09T05:45:17.922756+0000","last_became_peered":"2023-10-09T05:45:17.922756+0000","last_unstale":"2023-10-09T05:46:05.394647+0000","last_undegraded":"2023-10-09T05:46:05.394647+0000","last_fullsized":"2023-10-09T05:46:05.394647+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.8","version":"260'9219","reported_seq":16353,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:06.860443+0000","last_change":"2023-10-09T05:46:03.115826+0000","last_active":"2023-10-09T05:46:06.860443+0000","last_peered":"2023-10-09T05:46:06.860443+0000","last_clean":"2023-10-09T05:46:06.860443+0000","last_became_active":"2023-10-09T05:44:04.975737+0000","last_became_peered":"2023-10-09T05:44:04.975737+0000","last_unstale":"2023-10-09T05:46:06.860443+0000","last_undegraded":"2023-10-09T05:46:06.860443+0000","last_fullsized":"2023-10-09T05:46:06.860443+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9219,"log_dups_size":0,"ondisk_log_size":9219,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46006272,"num_objects":702,"num_object_clones":0,"num_object_copies":2106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5267,"num_write_kb":191008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2972,"num_bytes_recovered":166461440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":227,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.392353+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:46:05.392353+0000","last_peered":"2023-10-09T05:46:05.392353+0000","last_clean":"2023-10-09T05:46:05.392353+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:46:05.392353+0000","last_undegraded":"2023-10-09T05:46:05.392353+0000","last_fullsized":"2023-10-09T05:46:05.392353+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":223,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.385309+0000","last_change":"2023-10-09T05:45:17.930153+0000","last_active":"2023-10-09T05:46:05.385309+0000","last_peered":"2023-10-09T05:46:05.385309+0000","last_clean":"2023-10-09T05:46:05.385309+0000","last_became_active":"2023-10-09T05:45:17.929831+0000","last_became_peered":"2023-10-09T05:45:17.929831+0000","last_unstale":"2023-10-09T05:46:05.385309+0000","last_undegraded":"2023-10-09T05:46:05.385309+0000","last_fullsized":"2023-10-09T05:46:05.385309+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:24.746849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"260'10713","reported_seq":12517,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:08.689209+0000","last_change":"2023-10-09T05:45:17.936204+0000","last_active":"2023-10-09T05:46:08.689209+0000","last_peered":"2023-10-09T05:46:08.689209+0000","last_clean":"2023-10-09T05:43:18.871019+0000","last_became_active":"2023-10-09T05:43:21.208517+0000","last_became_peered":"2023-10-09T05:43:21.208517+0000","last_unstale":"2023-10-09T05:46:08.689209+0000","last_undegraded":"2023-10-09T05:43:20.928538+0000","last_fullsized":"2023-10-09T05:43:20.877574+0000","mapping_epoch":231,"log_start":"226'4584","ondisk_log_start":"226'4584","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":6129,"log_dups_size":0,"ondisk_log_size":6129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:20.409497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82378752,"num_objects":1257,"num_object_clones":0,"num_object_copies":3771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5117,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10713,"num_write_kb":383040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":868,"num_bytes_recovered":56885248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3","objects":-3395},{"shards":"0,3,4","objects":1257}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":232,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.383957+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:46:05.383957+0000","last_peered":"2023-10-09T05:46:05.383957+0000","last_clean":"2023-10-09T05:46:05.383957+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:46:05.383957+0000","last_undegraded":"2023-10-09T05:46:05.383957+0000","last_fullsized":"2023-10-09T05:46:05.383957+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":225,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.389964+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:46:05.389964+0000","last_peered":"2023-10-09T05:46:05.389964+0000","last_clean":"2023-10-09T05:46:05.389964+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:46:05.389964+0000","last_undegraded":"2023-10-09T05:46:05.389964+0000","last_fullsized":"2023-10-09T05:46:05.389964+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"260'10729","reported_seq":10787,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:07.057013+0000","last_change":"2023-10-09T05:41:29.078791+0000","last_active":"2023-10-09T05:46:07.057013+0000","last_peered":"2023-10-09T05:46:07.057013+0000","last_clean":"2023-10-09T05:46:07.057013+0000","last_became_active":"2023-10-09T05:41:29.078491+0000","last_became_peered":"2023-10-09T05:41:29.078491+0000","last_unstale":"2023-10-09T05:46:07.057013+0000","last_undegraded":"2023-10-09T05:46:07.057013+0000","last_fullsized":"2023-10-09T05:46:07.057013+0000","mapping_epoch":213,"log_start":"221'2348","ondisk_log_start":"221'2348","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8381,"log_dups_size":0,"ondisk_log_size":8381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:37.817282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88801280,"num_objects":1355,"num_object_clones":0,"num_object_copies":4065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10729,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":85,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.389480+0000","last_change":"2023-10-09T05:45:17.921547+0000","last_active":"2023-10-09T05:46:05.389480+0000","last_peered":"2023-10-09T05:46:05.389480+0000","last_clean":"2023-10-09T05:46:05.389480+0000","last_became_active":"2023-10-09T05:45:17.921221+0000","last_became_peered":"2023-10-09T05:45:17.921221+0000","last_unstale":"2023-10-09T05:46:05.389480+0000","last_undegraded":"2023-10-09T05:46:05.389480+0000","last_fullsized":"2023-10-09T05:46:05.389480+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":249,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.390007+0000","last_change":"2023-10-09T05:45:17.923692+0000","last_active":"2023-10-09T05:46:05.390007+0000","last_peered":"2023-10-09T05:46:05.390007+0000","last_clean":"2023-10-09T05:46:05.390007+0000","last_became_active":"2023-10-09T05:45:17.917743+0000","last_became_peered":"2023-10-09T05:45:17.917743+0000","last_unstale":"2023-10-09T05:46:05.390007+0000","last_undegraded":"2023-10-09T05:46:05.390007+0000","last_fullsized":"2023-10-09T05:46:05.390007+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"260'10453","reported_seq":10524,"reported_epoch":260,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:46:08.785850+0000","last_change":"2023-10-09T05:45:19.165033+0000","last_active":"2023-10-09T05:46:08.785850+0000","last_peered":"2023-10-09T05:46:08.785850+0000","last_clean":"2023-10-09T05:45:16.912524+0000","last_became_active":"2023-10-09T05:45:19.102874+0000","last_became_peered":"2023-10-09T05:45:19.102874+0000","last_unstale":"2023-10-09T05:46:08.785850+0000","last_undegraded":"2023-10-09T05:46:08.785850+0000","last_fullsized":"2023-10-09T05:46:08.785850+0000","mapping_epoch":253,"log_start":"225'4285","ondisk_log_start":"225'4285","created":213,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":6168,"log_dups_size":0,"ondisk_log_size":6168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:25.137080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84017176,"num_objects":1283,"num_object_clones":0,"num_object_copies":3849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1283,"num_objects_unfound":0,"num_objects_dirty":1283,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10453,"num_write_kb":375489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1283}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":259,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.390755+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:46:05.390755+0000","last_peered":"2023-10-09T05:46:05.390755+0000","last_clean":"2023-10-09T05:46:05.390755+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:46:05.390755+0000","last_undegraded":"2023-10-09T05:46:05.390755+0000","last_fullsized":"2023-10-09T05:46:05.390755+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":243,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.390220+0000","last_change":"2023-10-09T05:45:17.922009+0000","last_active":"2023-10-09T05:46:05.390220+0000","last_peered":"2023-10-09T05:46:05.390220+0000","last_clean":"2023-10-09T05:46:05.390220+0000","last_became_active":"2023-10-09T05:45:17.921636+0000","last_became_peered":"2023-10-09T05:45:17.921636+0000","last_unstale":"2023-10-09T05:46:05.390220+0000","last_undegraded":"2023-10-09T05:46:05.390220+0000","last_fullsized":"2023-10-09T05:46:05.390220+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"260'10393","reported_seq":22210,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:06.801573+0000","last_change":"2023-10-09T05:46:04.564946+0000","last_active":"2023-10-09T05:46:06.801573+0000","last_peered":"2023-10-09T05:46:06.801573+0000","last_clean":"2023-10-09T05:46:06.801573+0000","last_became_active":"2023-10-09T05:44:51.482290+0000","last_became_peered":"2023-10-09T05:44:51.482290+0000","last_unstale":"2023-10-09T05:46:06.801573+0000","last_undegraded":"2023-10-09T05:46:06.801573+0000","last_fullsized":"2023-10-09T05:46:06.801573+0000","mapping_epoch":248,"log_start":"227'4993","ondisk_log_start":"227'4993","created":213,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5400,"log_dups_size":0,"ondisk_log_size":5400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:27.252412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88408064,"num_objects":1349,"num_object_clones":0,"num_object_copies":4047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10393,"num_write_kb":375744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5870,"num_bytes_recovered":260702208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":277,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.382546+0000","last_change":"2023-10-09T05:45:17.922614+0000","last_active":"2023-10-09T05:46:05.382546+0000","last_peered":"2023-10-09T05:46:05.382546+0000","last_clean":"2023-10-09T05:46:05.382546+0000","last_became_active":"2023-10-09T05:45:17.922292+0000","last_became_peered":"2023-10-09T05:45:17.922292+0000","last_unstale":"2023-10-09T05:46:05.382546+0000","last_undegraded":"2023-10-09T05:46:05.382546+0000","last_fullsized":"2023-10-09T05:46:05.382546+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:33.044557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":57,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.384225+0000","last_change":"2023-10-09T05:45:17.921701+0000","last_active":"2023-10-09T05:46:05.384225+0000","last_peered":"2023-10-09T05:46:05.384225+0000","last_clean":"2023-10-09T05:46:05.384225+0000","last_became_active":"2023-10-09T05:45:17.921374+0000","last_became_peered":"2023-10-09T05:45:17.921374+0000","last_unstale":"2023-10-09T05:46:05.384225+0000","last_undegraded":"2023-10-09T05:46:05.384225+0000","last_fullsized":"2023-10-09T05:46:05.384225+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:06.490584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"260'9075","reported_seq":9142,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:09.857409+0000","last_change":"2023-10-09T05:46:09.541090+0000","last_active":"2023-10-09T05:46:09.857409+0000","last_peered":"2023-10-09T05:46:09.857409+0000","last_clean":"2023-10-09T05:46:09.857409+0000","last_became_active":"2023-10-09T05:44:03.989206+0000","last_became_peered":"2023-10-09T05:44:03.989206+0000","last_unstale":"2023-10-09T05:46:09.857409+0000","last_undegraded":"2023-10-09T05:46:09.857409+0000","last_fullsized":"2023-10-09T05:46:09.857409+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9075,"log_dups_size":0,"ondisk_log_size":9075,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39845888,"num_objects":608,"num_object_clones":0,"num_object_copies":1824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5200,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":585,"reported_epoch":259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:05.388594+0000","last_change":"2023-10-09T05:45:18.999640+0000","last_active":"2023-10-09T05:46:05.388594+0000","last_peered":"2023-10-09T05:46:05.388594+0000","last_clean":"2023-10-09T05:22:40.584723+0000","last_became_active":"2023-10-09T05:45:18.999259+0000","last_became_peered":"2023-10-09T05:45:18.999259+0000","last_unstale":"2023-10-09T05:46:05.388594+0000","last_undegraded":"2023-10-09T05:45:18.956731+0000","last_fullsized":"2023-10-09T05:45:18.956171+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:23.344079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":242,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.392602+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:46:05.392602+0000","last_peered":"2023-10-09T05:46:05.392602+0000","last_clean":"2023-10-09T05:46:05.392602+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:46:05.392602+0000","last_undegraded":"2023-10-09T05:46:05.392602+0000","last_fullsized":"2023-10-09T05:46:05.392602+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.392194+0000","last_change":"2023-10-09T05:45:17.926027+0000","last_active":"2023-10-09T05:46:05.392194+0000","last_peered":"2023-10-09T05:46:05.392194+0000","last_clean":"2023-10-09T05:46:05.392194+0000","last_became_active":"2023-10-09T05:45:17.925620+0000","last_became_peered":"2023-10-09T05:45:17.925620+0000","last_unstale":"2023-10-09T05:46:05.392194+0000","last_undegraded":"2023-10-09T05:46:05.392194+0000","last_fullsized":"2023-10-09T05:46:05.392194+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"260'9106","reported_seq":9178,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:08.659507+0000","last_change":"2023-10-09T05:44:50.502440+0000","last_active":"2023-10-09T05:46:08.659507+0000","last_peered":"2023-10-09T05:46:08.659507+0000","last_clean":"2023-10-09T05:46:08.659507+0000","last_became_active":"2023-10-09T05:44:50.481365+0000","last_became_peered":"2023-10-09T05:44:50.481365+0000","last_unstale":"2023-10-09T05:46:08.659507+0000","last_undegraded":"2023-10-09T05:46:08.659507+0000","last_fullsized":"2023-10-09T05:46:08.659507+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9106,"log_dups_size":0,"ondisk_log_size":9106,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40271872,"num_objects":614,"num_object_clones":0,"num_object_copies":1842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5222,"num_write_kb":186784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":723,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.392395+0000","last_change":"2023-10-09T05:46:04.573476+0000","last_active":"2023-10-09T05:46:05.392395+0000","last_peered":"2023-10-09T05:46:05.392395+0000","last_clean":"2023-10-09T05:46:05.392395+0000","last_became_active":"2023-10-09T05:45:17.925130+0000","last_became_peered":"2023-10-09T05:45:17.925130+0000","last_unstale":"2023-10-09T05:46:05.392395+0000","last_undegraded":"2023-10-09T05:46:05.392395+0000","last_fullsized":"2023-10-09T05:46:05.392395+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":226,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.391266+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:46:05.391266+0000","last_peered":"2023-10-09T05:46:05.391266+0000","last_clean":"2023-10-09T05:46:05.391266+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:46:05.391266+0000","last_undegraded":"2023-10-09T05:46:05.391266+0000","last_fullsized":"2023-10-09T05:46:05.391266+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":93,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.395775+0000","last_change":"2023-10-09T05:45:17.919819+0000","last_active":"2023-10-09T05:46:05.395775+0000","last_peered":"2023-10-09T05:46:05.395775+0000","last_clean":"2023-10-09T05:46:05.395775+0000","last_became_active":"2023-10-09T05:45:17.919447+0000","last_became_peered":"2023-10-09T05:45:17.919447+0000","last_unstale":"2023-10-09T05:46:05.395775+0000","last_undegraded":"2023-10-09T05:46:05.395775+0000","last_fullsized":"2023-10-09T05:46:05.395775+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:18.280449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"260'9151","reported_seq":9226,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:08.666098+0000","last_change":"2023-10-09T05:45:19.212798+0000","last_active":"2023-10-09T05:46:08.666098+0000","last_peered":"2023-10-09T05:46:08.666098+0000","last_clean":"2023-10-09T05:45:16.697360+0000","last_became_active":"2023-10-09T05:45:19.185185+0000","last_became_peered":"2023-10-09T05:45:19.185185+0000","last_unstale":"2023-10-09T05:46:08.666098+0000","last_undegraded":"2023-10-09T05:45:18.969077+0000","last_fullsized":"2023-10-09T05:45:18.916434+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9151,"log_dups_size":0,"ondisk_log_size":9151,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42958848,"num_objects":655,"num_object_clones":0,"num_object_copies":1965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2925,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5228,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":654},{"shards":"0,4","objects":-557}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":577,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.395482+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:46:05.395482+0000","last_peered":"2023-10-09T05:46:05.395482+0000","last_clean":"2023-10-09T05:46:05.395482+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:46:05.395482+0000","last_undegraded":"2023-10-09T05:46:05.395482+0000","last_fullsized":"2023-10-09T05:46:05.395482+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":52,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.398206+0000","last_change":"2023-10-09T05:45:17.921491+0000","last_active":"2023-10-09T05:46:05.398206+0000","last_peered":"2023-10-09T05:46:05.398206+0000","last_clean":"2023-10-09T05:46:05.398206+0000","last_became_active":"2023-10-09T05:45:17.921145+0000","last_became_peered":"2023-10-09T05:45:17.921145+0000","last_unstale":"2023-10-09T05:46:05.398206+0000","last_undegraded":"2023-10-09T05:46:05.398206+0000","last_fullsized":"2023-10-09T05:46:05.398206+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:45.903609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.392803+0000","last_change":"2023-10-09T05:45:17.921021+0000","last_active":"2023-10-09T05:46:05.392803+0000","last_peered":"2023-10-09T05:46:05.392803+0000","last_clean":"2023-10-09T05:46:05.392803+0000","last_became_active":"2023-10-09T05:45:17.920667+0000","last_became_peered":"2023-10-09T05:45:17.920667+0000","last_unstale":"2023-10-09T05:46:05.392803+0000","last_undegraded":"2023-10-09T05:46:05.392803+0000","last_fullsized":"2023-10-09T05:46:05.392803+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"260'9283","reported_seq":13039,"reported_epoch":260,"state":"active+recovering","last_fresh":"2023-10-09T05:46:08.794506+0000","last_change":"2023-10-09T05:46:05.010803+0000","last_active":"2023-10-09T05:46:08.794506+0000","last_peered":"2023-10-09T05:46:08.794506+0000","last_clean":"2023-10-09T05:43:18.711607+0000","last_became_active":"2023-10-09T05:44:05.023661+0000","last_became_peered":"2023-10-09T05:44:05.023661+0000","last_unstale":"2023-10-09T05:46:08.794506+0000","last_undegraded":"2023-10-09T05:46:08.794506+0000","last_fullsized":"2023-10-09T05:46:08.794506+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9283,"log_dups_size":0,"ondisk_log_size":9283,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43843584,"num_objects":669,"num_object_clones":0,"num_object_copies":2007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":158,"num_objects_unfound":0,"num_objects_dirty":669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5306,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1839,"num_bytes_recovered":120389632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":158},{"shards":"0,3,7","objects":511}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":906,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.392446+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:46:05.392446+0000","last_peered":"2023-10-09T05:46:05.392446+0000","last_clean":"2023-10-09T05:46:05.392446+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:46:05.392446+0000","last_undegraded":"2023-10-09T05:46:05.392446+0000","last_fullsized":"2023-10-09T05:46:05.392446+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":410,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.393730+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:46:05.393730+0000","last_peered":"2023-10-09T05:46:05.393730+0000","last_clean":"2023-10-09T05:46:05.393730+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:46:05.393730+0000","last_undegraded":"2023-10-09T05:46:05.393730+0000","last_fullsized":"2023-10-09T05:46:05.393730+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":226,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:06.410364+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:46:06.410364+0000","last_peered":"2023-10-09T05:46:06.410364+0000","last_clean":"2023-10-09T05:46:06.410364+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:46:06.410364+0000","last_undegraded":"2023-10-09T05:46:06.410364+0000","last_fullsized":"2023-10-09T05:46:06.410364+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":227,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.390219+0000","last_change":"2023-10-09T05:44:50.495966+0000","last_active":"2023-10-09T05:46:05.390219+0000","last_peered":"2023-10-09T05:46:05.390219+0000","last_clean":"2023-10-09T05:46:05.390219+0000","last_became_active":"2023-10-09T05:44:50.495628+0000","last_became_peered":"2023-10-09T05:44:50.495628+0000","last_unstale":"2023-10-09T05:46:05.390219+0000","last_undegraded":"2023-10-09T05:46:05.390219+0000","last_fullsized":"2023-10-09T05:46:05.390219+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"260'9347","reported_seq":12110,"reported_epoch":260,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:46:10.061074+0000","last_change":"2023-10-09T05:44:07.019753+0000","last_active":"2023-10-09T05:46:10.061074+0000","last_peered":"2023-10-09T05:46:10.061074+0000","last_clean":"2023-10-09T05:43:18.738931+0000","last_became_active":"2023-10-09T05:44:06.984836+0000","last_became_peered":"2023-10-09T05:44:06.984836+0000","last_unstale":"2023-10-09T05:46:10.061074+0000","last_undegraded":"2023-10-09T05:44:04.978402+0000","last_fullsized":"2023-10-09T05:46:10.061074+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9347,"log_dups_size":0,"ondisk_log_size":9347,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41123840,"num_objects":627,"num_object_clones":0,"num_object_copies":1881,"num_objects_missing_on_primary":1675,"num_objects_missing":1675,"num_objects_degraded":1675,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5345,"num_write_kb":191136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1344,"num_bytes_recovered":88080384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":627},{"shards":"0,1,2,7","objects":7},{"shards":"0,2,7","objects":-7}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":712,"reported_epoch":259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:05.393603+0000","last_change":"2023-10-09T05:45:18.933861+0000","last_active":"2023-10-09T05:46:05.393603+0000","last_peered":"2023-10-09T05:46:05.393603+0000","last_clean":"2023-10-09T05:45:10.937727+0000","last_became_active":"2023-10-09T05:45:18.933370+0000","last_became_peered":"2023-10-09T05:45:18.933370+0000","last_unstale":"2023-10-09T05:46:05.393603+0000","last_undegraded":"2023-10-09T05:45:18.913825+0000","last_fullsized":"2023-10-09T05:45:18.913230+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":214,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.385997+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:46:05.385997+0000","last_peered":"2023-10-09T05:46:05.385997+0000","last_clean":"2023-10-09T05:46:05.385997+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:46:05.385997+0000","last_undegraded":"2023-10-09T05:46:05.385997+0000","last_fullsized":"2023-10-09T05:46:05.385997+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":238,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.389197+0000","last_change":"2023-10-09T05:45:17.931710+0000","last_active":"2023-10-09T05:46:05.389197+0000","last_peered":"2023-10-09T05:46:05.389197+0000","last_clean":"2023-10-09T05:46:05.389197+0000","last_became_active":"2023-10-09T05:45:17.931290+0000","last_became_peered":"2023-10-09T05:45:17.931290+0000","last_unstale":"2023-10-09T05:46:05.389197+0000","last_undegraded":"2023-10-09T05:46:05.389197+0000","last_fullsized":"2023-10-09T05:46:05.389197+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:50.091125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"260'9131","reported_seq":9230,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:09.912974+0000","last_change":"2023-10-09T05:45:19.272383+0000","last_active":"2023-10-09T05:46:09.912974+0000","last_peered":"2023-10-09T05:46:09.912974+0000","last_clean":"2023-10-09T05:45:16.901376+0000","last_became_active":"2023-10-09T05:45:19.243948+0000","last_became_peered":"2023-10-09T05:45:19.243948+0000","last_unstale":"2023-10-09T05:46:09.912974+0000","last_undegraded":"2023-10-09T05:45:18.942714+0000","last_fullsized":"2023-10-09T05:45:18.916529+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9131,"log_dups_size":0,"ondisk_log_size":9131,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39714816,"num_objects":606,"num_object_clones":0,"num_object_copies":1818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2953,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5253,"num_write_kb":187488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":617},{"shards":"3,5","objects":-584}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":443,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.399002+0000","last_change":"2023-10-09T05:44:51.489799+0000","last_active":"2023-10-09T05:46:05.399002+0000","last_peered":"2023-10-09T05:46:05.399002+0000","last_clean":"2023-10-09T05:46:05.399002+0000","last_became_active":"2023-10-09T05:44:51.489301+0000","last_became_peered":"2023-10-09T05:44:51.489301+0000","last_unstale":"2023-10-09T05:46:05.399002+0000","last_undegraded":"2023-10-09T05:46:05.399002+0000","last_fullsized":"2023-10-09T05:46:05.399002+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":52,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.395852+0000","last_change":"2023-10-09T05:45:17.926001+0000","last_active":"2023-10-09T05:46:05.395852+0000","last_peered":"2023-10-09T05:46:05.395852+0000","last_clean":"2023-10-09T05:46:05.395852+0000","last_became_active":"2023-10-09T05:45:17.925682+0000","last_became_peered":"2023-10-09T05:45:17.925682+0000","last_unstale":"2023-10-09T05:46:05.395852+0000","last_undegraded":"2023-10-09T05:46:05.395852+0000","last_fullsized":"2023-10-09T05:46:05.395852+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:25.779220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":232,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.385517+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:46:05.385517+0000","last_peered":"2023-10-09T05:46:05.385517+0000","last_clean":"2023-10-09T05:46:05.385517+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:46:05.385517+0000","last_undegraded":"2023-10-09T05:46:05.385517+0000","last_fullsized":"2023-10-09T05:46:05.385517+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"260'9207","reported_seq":9273,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:06.752896+0000","last_change":"2023-10-09T05:44:04.001523+0000","last_active":"2023-10-09T05:46:06.752896+0000","last_peered":"2023-10-09T05:46:06.752896+0000","last_clean":"2023-10-09T05:46:06.752896+0000","last_became_active":"2023-10-09T05:44:03.990530+0000","last_became_peered":"2023-10-09T05:44:03.990530+0000","last_unstale":"2023-10-09T05:46:06.752896+0000","last_undegraded":"2023-10-09T05:46:06.752896+0000","last_fullsized":"2023-10-09T05:46:06.752896+0000","mapping_epoch":213,"log_start":"218'500","ondisk_log_start":"218'500","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8707,"log_dups_size":0,"ondisk_log_size":8707,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43679744,"num_objects":666,"num_object_clones":0,"num_object_copies":1998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5270,"num_write_kb":189984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":273,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.384319+0000","last_change":"2023-10-09T05:43:20.063203+0000","last_active":"2023-10-09T05:46:05.384319+0000","last_peered":"2023-10-09T05:46:05.384319+0000","last_clean":"2023-10-09T05:46:05.384319+0000","last_became_active":"2023-10-09T05:43:19.885257+0000","last_became_peered":"2023-10-09T05:43:19.885257+0000","last_unstale":"2023-10-09T05:46:05.384319+0000","last_undegraded":"2023-10-09T05:46:05.384319+0000","last_fullsized":"2023-10-09T05:46:05.384319+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:36.424359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":505,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:06.414199+0000","last_change":"2023-10-09T05:46:06.414199+0000","last_active":"2023-10-09T05:46:06.414199+0000","last_peered":"2023-10-09T05:46:06.414199+0000","last_clean":"2023-10-09T05:46:06.414199+0000","last_became_active":"2023-10-09T05:46:06.413755+0000","last_became_peered":"2023-10-09T05:46:06.413755+0000","last_unstale":"2023-10-09T05:46:06.414199+0000","last_undegraded":"2023-10-09T05:46:06.414199+0000","last_fullsized":"2023-10-09T05:46:06.414199+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":263,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.382780+0000","last_change":"2023-10-09T05:43:19.886969+0000","last_active":"2023-10-09T05:46:05.382780+0000","last_peered":"2023-10-09T05:46:05.382780+0000","last_clean":"2023-10-09T05:46:05.382780+0000","last_became_active":"2023-10-09T05:43:19.885111+0000","last_became_peered":"2023-10-09T05:43:19.885111+0000","last_unstale":"2023-10-09T05:46:05.382780+0000","last_undegraded":"2023-10-09T05:46:05.382780+0000","last_fullsized":"2023-10-09T05:46:05.382780+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:26.890192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"260'9477","reported_seq":9899,"reported_epoch":260,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:46:06.638313+0000","last_change":"2023-10-09T05:46:05.005306+0000","last_active":"2023-10-09T05:46:06.638313+0000","last_peered":"2023-10-09T05:46:06.638313+0000","last_clean":"2023-10-09T05:45:16.944353+0000","last_became_active":"2023-10-09T05:45:19.012069+0000","last_became_peered":"2023-10-09T05:45:19.012069+0000","last_unstale":"2023-10-09T05:46:06.638313+0000","last_undegraded":"2023-10-09T05:46:06.638313+0000","last_fullsized":"2023-10-09T05:46:06.638313+0000","mapping_epoch":253,"log_start":"218'700","ondisk_log_start":"218'700","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8777,"log_dups_size":0,"ondisk_log_size":8777,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40960000,"num_objects":625,"num_object_clones":0,"num_object_copies":1875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":458,"num_objects_unfound":0,"num_objects_dirty":625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5439,"num_write_kb":194048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":173,"num_bytes_recovered":11337728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,1],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":625}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":757,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.399094+0000","last_change":"2023-10-09T05:44:50.482211+0000","last_active":"2023-10-09T05:46:05.399094+0000","last_peered":"2023-10-09T05:46:05.399094+0000","last_clean":"2023-10-09T05:46:05.399094+0000","last_became_active":"2023-10-09T05:44:50.481749+0000","last_became_peered":"2023-10-09T05:44:50.481749+0000","last_unstale":"2023-10-09T05:46:05.399094+0000","last_undegraded":"2023-10-09T05:46:05.399094+0000","last_fullsized":"2023-10-09T05:46:05.399094+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":230,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.389286+0000","last_change":"2023-10-09T05:45:17.975802+0000","last_active":"2023-10-09T05:46:05.389286+0000","last_peered":"2023-10-09T05:46:05.389286+0000","last_clean":"2023-10-09T05:46:05.389286+0000","last_became_active":"2023-10-09T05:45:17.931496+0000","last_became_peered":"2023-10-09T05:45:17.931496+0000","last_unstale":"2023-10-09T05:46:05.389286+0000","last_undegraded":"2023-10-09T05:46:05.389286+0000","last_fullsized":"2023-10-09T05:46:05.389286+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:51.419126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":210,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.390906+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:46:05.390906+0000","last_peered":"2023-10-09T05:46:05.390906+0000","last_clean":"2023-10-09T05:46:05.390906+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:46:05.390906+0000","last_undegraded":"2023-10-09T05:46:05.390906+0000","last_fullsized":"2023-10-09T05:46:05.390906+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":234,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.385449+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:46:05.385449+0000","last_peered":"2023-10-09T05:46:05.385449+0000","last_clean":"2023-10-09T05:46:05.385449+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:46:05.385449+0000","last_undegraded":"2023-10-09T05:46:05.385449+0000","last_fullsized":"2023-10-09T05:46:05.385449+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":272,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.395920+0000","last_change":"2023-10-09T05:45:17.924914+0000","last_active":"2023-10-09T05:46:05.395920+0000","last_peered":"2023-10-09T05:46:05.395920+0000","last_clean":"2023-10-09T05:46:05.395920+0000","last_became_active":"2023-10-09T05:45:17.924406+0000","last_became_peered":"2023-10-09T05:45:17.924406+0000","last_unstale":"2023-10-09T05:46:05.395920+0000","last_undegraded":"2023-10-09T05:46:05.395920+0000","last_fullsized":"2023-10-09T05:46:05.395920+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:35.316832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"260'9291","reported_seq":16472,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-09T05:46:07.038630+0000","last_change":"2023-10-09T05:45:35.805746+0000","last_active":"2023-10-09T05:46:07.038630+0000","last_peered":"2023-10-09T05:46:07.038630+0000","last_clean":"2023-10-09T05:46:07.038630+0000","last_became_active":"2023-10-09T05:44:04.104935+0000","last_became_peered":"2023-10-09T05:44:04.104935+0000","last_unstale":"2023-10-09T05:46:07.038630+0000","last_undegraded":"2023-10-09T05:46:07.038630+0000","last_fullsized":"2023-10-09T05:46:07.038630+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9291,"log_dups_size":0,"ondisk_log_size":9291,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41287680,"num_objects":630,"num_object_clones":0,"num_object_copies":1890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5339,"num_write_kb":191008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2997,"num_bytes_recovered":170524672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":242,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.392256+0000","last_change":"2023-10-09T05:45:17.971642+0000","last_active":"2023-10-09T05:46:05.392256+0000","last_peered":"2023-10-09T05:46:05.392256+0000","last_clean":"2023-10-09T05:46:05.392256+0000","last_became_active":"2023-10-09T05:45:17.917031+0000","last_became_peered":"2023-10-09T05:45:17.917031+0000","last_unstale":"2023-10-09T05:46:05.392256+0000","last_undegraded":"2023-10-09T05:46:05.392256+0000","last_fullsized":"2023-10-09T05:46:05.392256+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":249,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.391325+0000","last_change":"2023-10-09T05:45:17.925441+0000","last_active":"2023-10-09T05:46:05.391325+0000","last_peered":"2023-10-09T05:46:05.391325+0000","last_clean":"2023-10-09T05:46:05.391325+0000","last_became_active":"2023-10-09T05:45:17.924653+0000","last_became_peered":"2023-10-09T05:45:17.924653+0000","last_unstale":"2023-10-09T05:46:05.391325+0000","last_undegraded":"2023-10-09T05:46:05.391325+0000","last_fullsized":"2023-10-09T05:46:05.391325+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"260'9049","reported_seq":9122,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:08.661406+0000","last_change":"2023-10-09T05:45:19.119065+0000","last_active":"2023-10-09T05:46:08.661406+0000","last_peered":"2023-10-09T05:46:08.661406+0000","last_clean":"2023-10-09T05:45:16.938440+0000","last_became_active":"2023-10-09T05:45:19.095658+0000","last_became_peered":"2023-10-09T05:45:19.095658+0000","last_unstale":"2023-10-09T05:46:08.661406+0000","last_undegraded":"2023-10-09T05:45:18.969330+0000","last_fullsized":"2023-10-09T05:45:18.914061+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9049,"log_dups_size":0,"ondisk_log_size":9049,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45678592,"num_objects":697,"num_object_clones":0,"num_object_copies":2091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2913,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5165,"num_write_kb":187584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":657},{"shards":"0,4","objects":-474}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":214,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.392532+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:46:05.392532+0000","last_peered":"2023-10-09T05:46:05.392532+0000","last_clean":"2023-10-09T05:46:05.392532+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:46:05.392532+0000","last_undegraded":"2023-10-09T05:46:05.392532+0000","last_fullsized":"2023-10-09T05:46:05.392532+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":240,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.389384+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:46:05.389384+0000","last_peered":"2023-10-09T05:46:05.389384+0000","last_clean":"2023-10-09T05:46:05.389384+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:46:05.389384+0000","last_undegraded":"2023-10-09T05:46:05.389384+0000","last_fullsized":"2023-10-09T05:46:05.389384+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":241,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.383374+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:46:05.383374+0000","last_peered":"2023-10-09T05:46:05.383374+0000","last_clean":"2023-10-09T05:46:05.383374+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:46:05.383374+0000","last_undegraded":"2023-10-09T05:46:05.383374+0000","last_fullsized":"2023-10-09T05:46:05.383374+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":253,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.381819+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:46:05.381819+0000","last_peered":"2023-10-09T05:46:05.381819+0000","last_clean":"2023-10-09T05:46:05.381819+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:46:05.381819+0000","last_undegraded":"2023-10-09T05:46:05.381819+0000","last_fullsized":"2023-10-09T05:46:05.381819+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":217,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.391011+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:46:05.391011+0000","last_peered":"2023-10-09T05:46:05.391011+0000","last_clean":"2023-10-09T05:46:05.391011+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:46:05.391011+0000","last_undegraded":"2023-10-09T05:46:05.391011+0000","last_fullsized":"2023-10-09T05:46:05.391011+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":234,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-09T05:46:05.390827+0000","last_change":"2023-10-09T05:45:17.920323+0000","last_active":"2023-10-09T05:46:05.390827+0000","last_peered":"2023-10-09T05:46:05.390827+0000","last_clean":"2023-10-09T05:46:05.390827+0000","last_became_active":"2023-10-09T05:45:17.919994+0000","last_became_peered":"2023-10-09T05:45:17.919994+0000","last_unstale":"2023-10-09T05:46:05.390827+0000","last_undegraded":"2023-10-09T05:46:05.390827+0000","last_fullsized":"2023-10-09T05:46:05.390827+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":1345847320,"num_objects":20537,"num_object_clones":0,"num_object_copies":61611,"num_objects_missing_on_primary":3253,"num_objects_missing":3253,"num_objects_degraded":25975,"num_objects_misplaced":3799,"num_objects_unfound":0,"num_objects_dirty":20537,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":168451,"num_write_kb":6047553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20432,"num_bytes_recovered":1094778880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5791494144,"data_stored":5791481928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":221692,"ondisk_log_size":221692,"up":78,"acting":71,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846731,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":575672,"kb_used_data":261360,"kb_used_omap":14,"kb_used_meta":314289,"kb_avail":93169480,"statfs":{"total":95995035648,"available":95405547520,"internally_reserved":0,"allocated":267632640,"data_stored":267052673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":321832794},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,0,1,2,3,7],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":11.387,"5min":5.1150000000000002,"15min":5.5229999999999997},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":110.123,"5min":113.018,"15min":220.77500000000001},"last":0.27500000000000002},{"interface":"front","average":{"1min":11.412000000000001,"5min":5.0330000000000004,"15min":5.9400000000000004},"min":{"1min":0.27100000000000002,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":110.13800000000001,"5min":113.005,"15min":220.78700000000001},"last":0.622}]},{"osd":1,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":3.3250000000000002,"15min":2.9740000000000002},"min":{"1min":0.27000000000000002,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":0.72399999999999998,"5min":83.575000000000003,"15min":190.68100000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.70699999999999996,"5min":3.331,"15min":2.8370000000000002},"min":{"1min":0.26100000000000001,"5min":0.255,"15min":0.255},"max":{"1min":4.5510000000000002,"5min":87.516999999999996,"15min":191.09200000000001},"last":0.38700000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":1.28,"15min":3.8210000000000002},"min":{"1min":0.29599999999999999,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":2.2240000000000002,"5min":7.9349999999999996,"15min":78.716999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.369,"15min":3.7749999999999999},"min":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.27000000000000002},"max":{"1min":2.3079999999999998,"5min":7.391,"15min":78.768000000000001},"last":0.32500000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":2.577,"5min":4.2539999999999996,"15min":3.4430000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":36.576000000000001,"5min":108.90300000000001,"15min":185.17099999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.5670000000000002,"5min":4.2400000000000002,"15min":3.8159999999999998},"min":{"1min":0.33000000000000002,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":36.624000000000002,"5min":109.08499999999999,"15min":183.35499999999999},"last":0.65400000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.64600000000000002,"15min":1.1180000000000001},"min":{"1min":0.29199999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.0630000000000002,"5min":4.3620000000000001,"15min":12.789},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.63,"15min":1.2569999999999999},"min":{"1min":0.312,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":2.5680000000000001,"5min":4.3419999999999996,"15min":9.0470000000000006},"last":0.68100000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.77900000000000003,"15min":1.2809999999999999},"min":{"1min":0.33100000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.2569999999999997,"5min":4.4429999999999996,"15min":10.813000000000001},"last":4.5170000000000003},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.74199999999999999,"15min":1.512},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.28299999999999997},"max":{"1min":0.748,"5min":3.9500000000000002,"15min":10.321},"last":4.7649999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.65400000000000003,"15min":1.274},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.318},"max":{"1min":1.3280000000000001,"5min":4.468,"15min":9.2530000000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.79100000000000004,"15min":1.2290000000000001},"min":{"1min":0.38800000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.91800000000000004,"5min":4.2279999999999998,"15min":8.2680000000000007},"last":0.45200000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626756,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":739752,"kb_used_data":426144,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":93005400,"statfs":{"total":95995035648,"available":95237529600,"internally_reserved":0,"allocated":436371456,"data_stored":435799331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:34 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":4.649,"15min":4.4909999999999997},"min":{"1min":0.23400000000000001,"5min":0.22800000000000001,"15min":0.214},"max":{"1min":2.5,"5min":162.08799999999999,"15min":162.08799999999999},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.86899999999999999,"5min":4.4669999999999996,"15min":4.2949999999999999},"min":{"1min":0.25800000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.4009999999999998,"5min":161.90899999999999,"15min":161.90899999999999},"last":0.36199999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":3.9470000000000001,"15min":3.8479999999999999},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":3.8279999999999998,"5min":228.24299999999999,"15min":228.24299999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.76500000000000001,"5min":3.8180000000000001,"15min":3.839},"min":{"1min":0.318,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":3.8199999999999998,"5min":228.22499999999999,"15min":228.22499999999999},"last":0.48099999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:45:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":4.3339999999999996,"15min":4.093},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.3029999999999999,"5min":87.703000000000003,"15min":115.40600000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":4.4980000000000002,"15min":4.1420000000000003},"min":{"1min":0.308,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":2.9049999999999998,"5min":87.689999999999998,"15min":115.297},"last":0.45800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":3.3420000000000001,"15min":3.8809999999999998},"min":{"1min":0.371,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":4.0019999999999998,"5min":77.578999999999994,"15min":115.244},"last":0.31},{"interface":"front","average":{"1min":1.0329999999999999,"5min":3.4529999999999998,"15min":3.927},"min":{"1min":0.36499999999999999,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":4.0179999999999998,"5min":77.516999999999996,"15min":115.34999999999999},"last":0.44600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.51100000000000001,"15min":1.7330000000000001},"min":{"1min":0.309,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":2.3620000000000001,"5min":2.73,"15min":125.64},"last":0.378},{"interface":"front","average":{"1min":1.1659999999999999,"5min":0.94999999999999996,"15min":1.7549999999999999},"min":{"1min":0.35799999999999998,"5min":0.26000000000000001,"15min":0.254},"max":{"1min":4.0839999999999996,"5min":13.609,"15min":125.625},"last":0.40300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.86499999999999999,"15min":1.9379999999999999},"min":{"1min":0.35899999999999999,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":6.141,"5min":6.4690000000000003,"15min":127.35899999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.8979999999999999,"5min":1.0209999999999999,"15min":1.869},"min":{"1min":0.374,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":14.250999999999999,"5min":14.250999999999999,"15min":128.93799999999999},"last":0.46899999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.754,"15min":1.9930000000000001},"min":{"1min":0.40799999999999997,"5min":0.32500000000000001,"15min":0.26500000000000001},"max":{"1min":0.97599999999999998,"5min":7.319,"15min":115.514},"last":0.499},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.0760000000000001,"15min":1.6639999999999999},"min":{"1min":0.41199999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.032,"5min":14.300000000000001,"15min":116.53100000000001},"last":0.435}]}]},{"osd":1,"up_from":0,"seq":399431958718,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":88,"seq":377957122244,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1161284,"kb_used_data":672252,"kb_used_omap":8,"kb_used_meta":489015,"kb_avail":92583868,"statfs":{"total":95995035648,"available":94805880832,"internally_reserved":0,"allocated":688386048,"data_stored":687445768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":500752136},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:45:56 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":1.5189999999999999,"15min":1.657},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":1.4670000000000001,"5min":14.198,"15min":17.760000000000002},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.747,"5min":1.9059999999999999,"15min":2.1429999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":5.8780000000000001,"5min":13.816000000000001,"15min":26.163},"last":0.40799999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":3.5910000000000002,"15min":3.278},"min":{"1min":0.24199999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.5349999999999999,"5min":92.230000000000004,"15min":92.230000000000004},"last":2.831},{"interface":"front","average":{"1min":0.48999999999999999,"5min":3.4940000000000002,"15min":3.1440000000000001},"min":{"1min":0.26800000000000002,"5min":0.222,"15min":0.218},"max":{"1min":1.4279999999999999,"5min":92.247,"15min":92.247},"last":2.6739999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:45:56 2023","interfaces":[{"interface":"back","average":{"1min":3.177,"5min":3.4689999999999999,"15min":3.79},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":55.328000000000003,"5min":79.159000000000006,"15min":123.024},"last":0.372},{"interface":"front","average":{"1min":3.3759999999999999,"5min":3.9500000000000002,"15min":3.6629999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.222},"max":{"1min":55.292000000000002,"5min":79.353999999999999,"15min":99.162000000000006},"last":2.8580000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.2649999999999999,"15min":1.399},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.381,"5min":11.35,"15min":69.082999999999998},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.825,"15min":1.8819999999999999},"min":{"1min":0.36299999999999999,"5min":0.33500000000000002,"15min":0.30399999999999999},"max":{"1min":1.905,"5min":7.9240000000000004,"15min":69.102000000000004},"last":0.438}]},{"osd":5,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":2.024,"15min":3.1840000000000002},"min":{"1min":0.372,"5min":0.34799999999999998,"15min":0.30199999999999999},"max":{"1min":3.5089999999999999,"5min":9.6219999999999999,"15min":281.733},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.80000000000000004,"5min":2.0779999999999998,"15min":3.2040000000000002},"min":{"1min":0.38,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.0369999999999999,"5min":12.552,"15min":282.95400000000001},"last":2.9020000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":2.0529999999999999,"15min":1.7769999999999999},"min":{"1min":0.41399999999999998,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":4.5010000000000003,"5min":7.9969999999999999,"15min":11.664},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.2050000000000001,"5min":2.1200000000000001,"15min":1.867},"min":{"1min":0.38600000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.4390000000000001,"5min":12.587999999999999,"15min":12.587999999999999},"last":2.9319999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:45:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.2150000000000001,"15min":1.407},"min":{"1min":0.36199999999999999,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":1.5649999999999999,"5min":11.631,"15min":11.631},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.482,"15min":2.1400000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.9850000000000001,"5min":7.2190000000000003,"15min":11.068},"last":0.58699999999999997}]}]},{"osd":2,"up_from":98,"seq":420906795191,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1387084,"kb_used_data":859396,"kb_used_omap":11,"kb_used_meta":527668,"kb_avail":92358068,"statfs":{"total":95995035648,"available":94574661632,"internally_reserved":0,"allocated":880021504,"data_stored":879014640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":540332629},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":13,"5min":4.9630000000000001,"15min":3.0659999999999998},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":162.285,"5min":162.285,"15min":162.285},"last":0.31},{"interface":"front","average":{"1min":13.404,"5min":5.9039999999999999,"15min":3.359},"min":{"1min":0.24399999999999999,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":162.643,"5min":162.643,"15min":162.643},"last":0.70099999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":2.8039999999999998,"15min":3.1179999999999999},"min":{"1min":0.24099999999999999,"5min":0.23100000000000001,"15min":0.20699999999999999},"max":{"1min":2.4289999999999998,"5min":24.265000000000001,"15min":98.575000000000003},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":2.0640000000000001,"15min":2.5640000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":2.0859999999999999,"5min":18.882999999999999,"15min":98.397000000000006},"last":0.85599999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:45:09 2023","interfaces":[{"interface":"back","average":{"1min":13.132,"5min":8.1069999999999993,"15min":4.8710000000000004},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":159.35499999999999,"5min":159.35499999999999,"15min":159.35499999999999},"last":0.82799999999999996},{"interface":"front","average":{"1min":13.404,"5min":6.992,"15min":4.798},"min":{"1min":0.32900000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":159.196,"5min":159.196,"15min":159.196},"last":0.39500000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.74,"15min":1.903},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27700000000000002},"max":{"1min":2.161,"5min":14.382,"15min":115.84999999999999},"last":1.7709999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":1.2609999999999999,"15min":2.056},"min":{"1min":0.309,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.974,"5min":13.815,"15min":116.149},"last":0.44}]},{"osd":5,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.986,"15min":1.972},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.312},"max":{"1min":2.4260000000000002,"5min":13.712999999999999,"15min":24.597999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.706,"15min":1.6579999999999999},"min":{"1min":0.38400000000000001,"5min":0.35499999999999998,"15min":0.26000000000000001},"max":{"1min":2.2890000000000001,"5min":14.426,"15min":23.919},"last":1.931}]},{"osd":6,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.5229999999999999,"15min":1.804},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":14.317,"5min":14.317,"15min":20.503},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.6299999999999999,"5min":2.1850000000000001,"15min":1.901},"min":{"1min":0.38900000000000001,"5min":0.36799999999999999,"15min":0.34499999999999997},"max":{"1min":14.436999999999999,"5min":14.436999999999999,"15min":14.436999999999999},"last":1.6499999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:45:09 2023","interfaces":[{"interface":"back","average":{"1min":3.871,"5min":2.2389999999999999,"15min":2.6000000000000001},"min":{"1min":0.39000000000000001,"5min":0.37,"15min":0.37},"max":{"1min":51.701999999999998,"5min":51.701999999999998,"15min":51.701999999999998},"last":0.72299999999999998},{"interface":"front","average":{"1min":4.1859999999999999,"5min":2.786,"15min":2.423},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":51.716000000000001,"5min":51.716000000000001,"15min":51.716000000000001},"last":0.501}]}]},{"osd":3,"up_from":103,"seq":442381631665,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1523604,"kb_used_data":1015180,"kb_used_omap":4,"kb_used_meta":508347,"kb_avail":92221548,"statfs":{"total":95995035648,"available":94434865152,"internally_reserved":0,"allocated":1039544320,"data_stored":1038640321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":520547934},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":2.8980000000000001,"5min":4.3289999999999997,"15min":3.6699999999999999},"min":{"1min":0.21199999999999999,"5min":0.184,"15min":0.183},"max":{"1min":46.411000000000001,"5min":149.565,"15min":167.82400000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.8100000000000001,"5min":5.024,"15min":4.2320000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.20899999999999999},"max":{"1min":46.387999999999998,"5min":149.291,"15min":167.89599999999999},"last":0.249}]},{"osd":1,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":3.0329999999999999,"15min":2.9020000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22600000000000001},"max":{"1min":4.2960000000000003,"5min":149.32499999999999,"15min":149.32499999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.034,"5min":3.3839999999999999,"15min":3.004},"min":{"1min":0.28000000000000003,"5min":0.19,"15min":0.19},"max":{"1min":4.4450000000000003,"5min":149.34299999999999,"15min":149.34299999999999},"last":1.615}]},{"osd":2,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":3.2949999999999999,"15min":4.5839999999999996},"min":{"1min":0.32200000000000001,"5min":0.27000000000000002,"15min":0.215},"max":{"1min":3.0920000000000001,"5min":150.006,"15min":190.43299999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.80600000000000005,"5min":3.1930000000000001,"15min":4.9429999999999996},"min":{"1min":0.27200000000000002,"5min":0.214,"15min":0.214},"max":{"1min":2.6120000000000001,"5min":150.65100000000001,"15min":190.267},"last":0.40500000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":2.6819999999999999,"15min":2.661},"min":{"1min":0.32400000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.4729999999999999,"5min":149.84700000000001,"15min":149.84700000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.626,"5min":2.5710000000000002,"15min":2.1680000000000001},"min":{"1min":0.378,"5min":0.315,"15min":0.27400000000000002},"max":{"1min":1.869,"5min":149.74700000000001,"15min":149.74700000000001},"last":0.53600000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":2.867,"15min":2.3300000000000001},"min":{"1min":0.35599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.3840000000000003,"5min":149.43000000000001,"15min":149.43000000000001},"last":1.698},{"interface":"front","average":{"1min":0.66200000000000003,"5min":2.8279999999999998,"15min":2.3119999999999998},"min":{"1min":0.42699999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.897,"5min":149.69300000000001,"15min":149.69300000000001},"last":0.55300000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":2.7330000000000001,"15min":2.1909999999999998},"min":{"1min":0.36099999999999999,"5min":0.33900000000000002,"15min":0.33500000000000002},"max":{"1min":3.3620000000000001,"5min":149.72399999999999,"15min":149.72399999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.0289999999999999,"5min":3.0630000000000002,"15min":2.2280000000000002},"min":{"1min":0.40300000000000002,"5min":0.307,"15min":0.307},"max":{"1min":4.4870000000000001,"5min":149.589,"15min":149.589},"last":1.675}]},{"osd":7,"last update":"Mon Oct 9 05:45:05 2023","interfaces":[{"interface":"back","average":{"1min":6.9050000000000002,"5min":3.2240000000000002,"15min":2.718},"min":{"1min":0.41599999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":149.77199999999999,"5min":149.77199999999999,"15min":149.77199999999999},"last":10.398},{"interface":"front","average":{"1min":6.9480000000000004,"5min":3.375,"15min":2.6669999999999998},"min":{"1min":0.39300000000000002,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":149.80799999999999,"5min":149.80799999999999,"15min":149.80799999999999},"last":10.619}]}]},{"osd":4,"up_from":30,"seq":128849019213,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1065968,"kb_used_data":760552,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92679184,"statfs":{"total":95995035648,"available":94903484416,"internally_reserved":0,"allocated":778805248,"data_stored":778232859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":3.2160000000000002,"15min":2.2160000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.214},"max":{"1min":26.478000000000002,"5min":100.504,"15min":100.504},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.9350000000000001,"5min":3.4239999999999999,"15min":2.5369999999999999},"min":{"1min":0.247,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":26.454000000000001,"5min":100.717,"15min":100.717},"last":0.34899999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":4.2539999999999996,"15min":2.4489999999999998},"min":{"1min":0.28699999999999998,"5min":0.26200000000000001,"15min":0.24299999999999999},"max":{"1min":2.944,"5min":187.369,"15min":187.369},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.752,"5min":4.2590000000000003,"15min":2.5630000000000002},"min":{"1min":0.31900000000000001,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":2.4830000000000001,"5min":187.35300000000001,"15min":187.35300000000001},"last":0.32100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:45:23 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":3.6829999999999998,"15min":4.5309999999999997},"min":{"1min":0.27800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.7129999999999992,"5min":100.148,"15min":196.93700000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.4079999999999999,"5min":3.964,"15min":4.4939999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.28100000000000003},"max":{"1min":9.7219999999999995,"5min":100.086,"15min":197.08099999999999},"last":0.65300000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":3.2349999999999999,"15min":3.9889999999999999},"min":{"1min":0.34399999999999997,"5min":0.33800000000000002,"15min":0.30199999999999999},"max":{"1min":3.1709999999999998,"5min":72.391999999999996,"15min":211.48699999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.51600000000000001,"5min":3.2080000000000002,"15min":3.839},"min":{"1min":0.312,"5min":0.312,"15min":0.309},"max":{"1min":1.1819999999999999,"5min":71.721999999999994,"15min":211.31100000000001},"last":0.67800000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.67800000000000005,"15min":1.6359999999999999},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.968,"5min":3.6739999999999999,"15min":113.35899999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.60199999999999998,"15min":1.5369999999999999},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.30599999999999999},"max":{"1min":1.788,"5min":3.4209999999999998,"15min":113.32299999999999},"last":0.71199999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56999999999999995,"15min":0.73199999999999998},"min":{"1min":0.29699999999999999,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":2.089,"5min":3.3969999999999998,"15min":11.944000000000001},"last":0.621},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.84599999999999997,"15min":0.97799999999999998},"min":{"1min":0.40300000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.4000000000000004,"5min":8.5269999999999992,"15min":10.356},"last":0.48099999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.59299999999999997,"15min":1.637},"min":{"1min":0.34300000000000003,"5min":0.33300000000000002,"15min":0.312},"max":{"1min":1.008,"5min":2.7709999999999999,"15min":30.393999999999998},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.72699999999999998,"15min":1.8440000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":2.9820000000000002,"5min":5.7160000000000002,"15min":30.334},"last":0.49299999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822985,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":979744,"kb_used_data":670232,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92765408,"statfs":{"total":95995035648,"available":94991777792,"internally_reserved":0,"allocated":686317568,"data_stored":685757011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.611,"15min":3.29},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.19400000000000001},"max":{"1min":9.9049999999999994,"5min":12.904999999999999,"15min":165.90199999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.111,"5min":1.782,"15min":3.399},"min":{"1min":0.28299999999999997,"5min":0.27000000000000002,"15min":0.215},"max":{"1min":10.210000000000001,"5min":17.077000000000002,"15min":165.96700000000001},"last":0.40799999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.8300000000000001,"15min":2.5950000000000002},"min":{"1min":0.27700000000000002,"5min":0.26500000000000001,"15min":0.23999999999999999},"max":{"1min":3.2879999999999998,"5min":54.898000000000003,"15min":87.769000000000005},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.7290000000000001,"15min":2.6720000000000002},"min":{"1min":0.316,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":1.7989999999999999,"5min":52.512,"15min":87.864000000000004},"last":5.601}]},{"osd":2,"last update":"Mon Oct 9 05:45:36 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":1.8480000000000001,"15min":4.4329999999999998},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.9510000000000001,"5min":31.114000000000001,"15min":290.697},"last":1.3420000000000001},{"interface":"front","average":{"1min":0.83499999999999996,"5min":2.1629999999999998,"15min":4.3129999999999997},"min":{"1min":0.35599999999999998,"5min":0.32700000000000001,"15min":0.29899999999999999},"max":{"1min":2.9239999999999999,"5min":33.875999999999998,"15min":290.786},"last":0.61199999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":2.3820000000000001,"15min":3.488},"min":{"1min":0.39100000000000001,"5min":0.30399999999999999,"15min":0.28799999999999998},"max":{"1min":10.640000000000001,"5min":36.395000000000003,"15min":95.236999999999995},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.3,"5min":2.2909999999999999,"15min":3.2869999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.29699999999999999},"max":{"1min":9.9600000000000009,"5min":36.406999999999996,"15min":93.471000000000004},"last":1.296}]},{"osd":4,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.64500000000000002,"15min":0.77900000000000003},"min":{"1min":0.29599999999999999,"5min":0.28499999999999998,"15min":0.23300000000000001},"max":{"1min":1.6180000000000001,"5min":6.2610000000000001,"15min":9.3510000000000009},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.62,"5min":0.72199999999999998,"15min":1.01},"min":{"1min":0.34300000000000003,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":1.726,"5min":9.282,"15min":14.236000000000001},"last":0.42599999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:45:10 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.78300000000000003,"15min":1.429},"min":{"1min":0.32600000000000001,"5min":0.316,"15min":0.26600000000000001},"max":{"1min":3.6629999999999998,"5min":9.2959999999999994,"15min":95.010999999999996},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.87,"15min":1.343},"min":{"1min":0.32100000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.8380000000000001,"5min":4.6600000000000001,"15min":94.194000000000003},"last":0.38400000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:45:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.073,"15min":1.3560000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28999999999999998},"max":{"1min":5.8920000000000003,"5min":9.1460000000000008,"15min":9.9849999999999994},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.95599999999999996,"15min":1.4730000000000001},"min":{"1min":0.39200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":3.6890000000000001,"5min":9.3100000000000005,"15min":14.432},"last":0.53400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":675352576,"data_stored":675348504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1074860032,"data_stored":1074855960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":871890944,"data_stored":871890944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1028128768,"data_stored":1028128768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":772276224,"data_stored":772276224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":682033152,"data_stored":682033152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":426971136,"data_stored":426967064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":259981312,"data_stored":259981312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:46:13.377 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_2 pgp_num 26 2023-10-09T05:46:13.411 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:13.445 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:13.560 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:13.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:13.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:13.691 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:13.742 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:13.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:13.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:13.790 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:46:13.791 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:46:13.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:46:13.791 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:46:13.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.791 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4048 6.a 6:51c21166:::benchmark_data_smithi008_295510_object4047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:46:13.791 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.480144+0000", 2023-10-09T05:46:13.791 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 545.30593902600003, 2023-10-09T05:46:13.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48056587499999998, 2023-10-09T05:46:13.791 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.791 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4048 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480144+0000", 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480144+0000", 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480143+0000", 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7996e-05 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480181+0000", 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0966e-05 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480192+0000", 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6390000000000003e-06 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480199+0000", 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017538499999999999 2023-10-09T05:46:13.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480374+0000", 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014747999999999999 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480521+0000", 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020043799999999999 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.795 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.480722+0000", 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00076789800000000004 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.481490+0000", 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00070511100000000002 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.482195+0000", 2023-10-09T05:46:13.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.478412911 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.960608+0000", 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5300000000000002e-05 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.960663+0000", 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6805999999999999e-05 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.960710+0000", 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48056587499999998 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4066 6.a 6:51b3cfb1:::benchmark_data_smithi008_295510_object4065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.483733+0000", 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 545.30234964500005, 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47826635200000001, 2023-10-09T05:46:13.798 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4066 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483733+0000", 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483733+0000", 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483732+0000", 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3579999999999995e-05 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483826+0000", 2023-10-09T05:46:13.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.503e-05 2023-10-09T05:46:13.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483841+0000", 2023-10-09T05:46:13.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4710000000000004e-06 2023-10-09T05:46:13.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483847+0000", 2023-10-09T05:46:13.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8745999999999997e-05 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.483896+0000", 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013626099999999999 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484032+0000", 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016087400000000001 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:46:13.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484193+0000", 2023-10-09T05:46:13.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060899000000000001 2023-10-09T05:46:13.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.803 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484802+0000", 2023-10-09T05:46:13.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001503536 2023-10-09T05:46:13.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.803 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486306+0000", 2023-10-09T05:46:13.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.475630736 2023-10-09T05:46:13.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.961936+0000", 2023-10-09T05:46:13.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1003000000000001e-05 2023-10-09T05:46:13.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.961967+0000", 2023-10-09T05:46:13.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2406000000000001e-05 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962000+0000", 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47826635200000001 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4069 6.d 6:be699dd1:::benchmark_data_smithi008_295510_object4068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.484947+0000", 2023-10-09T05:46:13.805 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 545.30113619300005, 2023-10-09T05:46:13.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47790969900000002, 2023-10-09T05:46:13.806 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.806 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.806 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.806 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:13.806 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:13.806 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4069 2023-10-09T05:46:13.808 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.808 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484947+0000", 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484947+0000", 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.484945+0000", 2023-10-09T05:46:13.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019272500000000001 2023-10-09T05:46:13.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485138+0000", 2023-10-09T05:46:13.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1980999999999999e-05 2023-10-09T05:46:13.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485150+0000", 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2870000000000004e-06 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485155+0000", 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2120999999999998e-05 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485207+0000", 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117552 2023-10-09T05:46:13.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485325+0000", 2023-10-09T05:46:13.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000151662 2023-10-09T05:46:13.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:46:13.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485477+0000", 2023-10-09T05:46:13.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011680009999999999 2023-10-09T05:46:13.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486645+0000", 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6749999999999999e-05 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486681+0000", 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47611776500000003 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962799+0000", 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4666999999999999e-05 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962834+0000", 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2668999999999999e-05 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.962857+0000", 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47790969900000002 2023-10-09T05:46:13.814 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.815 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.815 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.815 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4072 6.d 6:b4da5f17:::benchmark_data_smithi008_295510_object4071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:46:13.815 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.485343+0000", 2023-10-09T05:46:13.815 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 545.30073974499999, 2023-10-09T05:46:13.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47815177399999997, 2023-10-09T05:46:13.815 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.815 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.815 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4072 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485343+0000", 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485343+0000", 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485342+0000", 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9259000000000006e-05 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485442+0000", 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0434999999999999e-05 2023-10-09T05:46:13.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485452+0000", 2023-10-09T05:46:13.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2230000000000004e-06 2023-10-09T05:46:13.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485458+0000", 2023-10-09T05:46:13.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000491495 2023-10-09T05:46:13.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.485950+0000", 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013497300000000001 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486085+0000", 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018227900000000001 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:46:13.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486267+0000", 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00061199699999999998 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.486879+0000", 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00083038800000000005 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.487709+0000", 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47569748899999997 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.821 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.821 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963407+0000", 2023-10-09T05:46:13.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8826000000000003e-05 2023-10-09T05:46:13.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.821 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.821 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.821 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963456+0000", 2023-10-09T05:46:13.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9415000000000001e-05 2023-10-09T05:46:13.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.821 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.821 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963495+0000", 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47815177399999997 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4084 6.d 6:b4676de6:::benchmark_data_smithi008_295510_object4083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.489711+0000", 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 545.296372432, 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47387225700000002, 2023-10-09T05:46:13.822 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.823 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.823 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.823 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:13.823 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:13.823 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4084 2023-10-09T05:46:13.823 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.823 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.823 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.823 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.823 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489711+0000", 2023-10-09T05:46:13.823 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.824 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.824 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489711+0000", 2023-10-09T05:46:13.824 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:46:13.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.824 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.824 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489710+0000", 2023-10-09T05:46:13.824 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7155000000000003e-05 2023-10-09T05:46:13.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489777+0000", 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4321000000000001e-05 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489791+0000", 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6220000000000003e-06 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489798+0000", 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.515e-05 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489843+0000", 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6768000000000006e-05 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.489940+0000", 2023-10-09T05:46:13.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000134702 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.490074+0000", 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000556044 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.490630+0000", 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00032459399999999999 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.490955+0000", 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47257032300000001 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963525+0000", 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8180000000000001e-05 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963554+0000", 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9314000000000001e-05 2023-10-09T05:46:13.828 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963583+0000", 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47387225700000002 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:4097 6.a 6:56ff3694:::benchmark_data_smithi008_295510_object4096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:08.498925+0000", 2023-10-09T05:46:13.829 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 545.28715817099999, 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46416822400000002, 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 4097 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.830 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498925+0000", 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498925+0000", 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498923+0000", 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8837e-05 2023-10-09T05:46:13.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498972+0000", 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7210999999999998e-05 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498989+0000", 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1210000000000003e-06 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.498995+0000", 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0176000000000001e-05 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499056+0000", 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016777900000000001 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499223+0000", 2023-10-09T05:46:13.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000203718 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499427+0000", 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00054037499999999995 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.499967+0000", 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000180817 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.500148+0000", 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46288928400000001 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963038+0000", 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3631999999999999e-05 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963071+0000", 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1931999999999999e-05 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:08.963093+0000", 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46416822400000002 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.836 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:142206 6.d 6:b4d13e7d:::benchmark_data_smithi008_295510_object53843:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.366419+0000", 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 366.41966368999999, 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45887450299999999, 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 142206 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.837 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366419+0000", 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366419+0000", 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366418+0000", 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9809999999999997e-06 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366425+0000", 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1507e-05 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366436+0000", 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6210000000000001e-06 2023-10-09T05:46:13.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.840 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.840 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366442+0000", 2023-10-09T05:46:13.840 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8337999999999997e-05 2023-10-09T05:46:13.840 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.840 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.840 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366490+0000", 2023-10-09T05:46:13.840 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000129345 2023-10-09T05:46:13.840 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.840 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366620+0000", 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012503699999999999 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366745+0000", 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041122060000000002 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.841 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.370857+0000", 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45407035200000001 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824927+0000", 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000271425 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.825199+0000", 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0005000000000001e-05 2023-10-09T05:46:13.842 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.843 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.843 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.843 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.825239+0000", 2023-10-09T05:46:13.843 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5108999999999997e-05 2023-10-09T05:46:13.843 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.843 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.843 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.843 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.825294+0000", 2023-10-09T05:46:13.843 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45887450299999999 2023-10-09T05:46:13.843 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.843 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:142208 6.a 6:5453517f:::benchmark_data_smithi008_295510_object53845:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.366458+0000", 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 366.41962514199997, 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46771891900000001, 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:13.844 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 142208 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366458+0000", 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366458+0000", 2023-10-09T05:46:13.845 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366457+0000", 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5499999999999999e-06 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366461+0000", 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4989999999999999e-06 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.846 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366467+0000", 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1790000000000001e-06 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366472+0000", 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4431000000000002e-05 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366516+0000", 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8264999999999996e-05 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366604+0000", 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5539999999999998e-05 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366690+0000", 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46729868499999999 2023-10-09T05:46:13.848 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.833989+0000", 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0900999999999999e-05 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834039+0000", 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9656000000000001e-05 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834079+0000", 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5481e-05 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834135+0000", 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2240000000000002e-05 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834177+0000", 2023-10-09T05:46:13.850 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46771891900000001 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:142210 6.a 6:59802c8c:::benchmark_data_smithi008_295510_object53847:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.366485+0000", 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 366.41959785900002, 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46783791200000002, 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.851 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 142210 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366485+0000", 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366485+0000", 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366485+0000", 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2109999999999998e-06 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366488+0000", 2023-10-09T05:46:13.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7570000000000004e-06 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366493+0000", 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0410000000000002e-06 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.366496+0000", 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047554600000000004 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371251+0000", 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012267699999999999 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371374+0000", 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011798600000000001 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:46:13.855 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371492+0000", 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46255569000000002 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834047+0000", 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000167675 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834215+0000", 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8153e-05 2023-10-09T05:46:13.856 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834243+0000", 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0781000000000001e-05 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834284+0000", 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8974000000000003e-05 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.857 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.858 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834323+0000", 2023-10-09T05:46:13.858 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46783791200000002 2023-10-09T05:46:13.858 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.858 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.858 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.858 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.858 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.858 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:142213 6.a 6:5635753f:::benchmark_data_smithi008_295510_object53850:head [delete] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T05:46:13.858 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.371323+0000", 2023-10-09T05:46:13.858 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 366.41475983100003, 2023-10-09T05:46:13.858 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46515832600000001, 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 142213 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371323+0000", 2023-10-09T05:46:13.859 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371323+0000", 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371322+0000", 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9589999999999995e-06 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.860 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371330+0000", 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3706000000000001e-05 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371343+0000", 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6529999999999994e-06 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371351+0000", 2023-10-09T05:46:13.861 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45795592800000001 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.829307+0000", 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000141691 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.829449+0000", 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000160099 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.862 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.829609+0000", 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047785500000000003 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834387+0000", 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00086802500000000002 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.863 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.835255+0000", 2023-10-09T05:46:13.864 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011420340000000001 2023-10-09T05:46:13.864 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.864 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.864 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.864 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.836397+0000", 2023-10-09T05:46:13.864 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7500000000000003e-05 2023-10-09T05:46:13.864 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.864 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.864 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.864 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.836445+0000", 2023-10-09T05:46:13.864 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6659000000000003e-05 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.836482+0000", 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46515832600000001 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.45029.0:142208 6.a e198/162 ondisk, result = 0)", 2023-10-09T05:46:13.865 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.371518+0000", 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 366.41456532900003, 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46253101000000002, 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371518+0000", 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.866 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371518+0000", 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371517+0000", 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7789999999999998e-06 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371523+0000", 2023-10-09T05:46:13.867 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0664e-05 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371534+0000", 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1079999999999998e-06 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.371540+0000", 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46248180700000002 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.868 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834022+0000", 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3530000000000001e-06 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834031+0000", 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7370000000000001e-05 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.869 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834049+0000", 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.46253101000000002 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.45029.0:142206 6.d e198/162 ondisk, result = 0)", 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.373159+0000", 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 366.41292386499998, 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45177083299999998, 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.870 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.373159+0000", 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.373159+0000", 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.871 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.373158+0000", 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1810000000000001e-06 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.373166+0000", 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45168935799999999 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.872 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824855+0000", 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1770000000000001e-05 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824867+0000", 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1879999999999999e-05 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824909+0000", 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7610000000000002e-06 2023-10-09T05:46:13.873 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.874 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.874 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.874 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824919+0000", 2023-10-09T05:46:13.874 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1137e-05 2023-10-09T05:46:13.874 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.874 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.874 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.874 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824930+0000", 2023-10-09T05:46:13.874 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45177083299999998 2023-10-09T05:46:13.874 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.874 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.45029.0:142210 6.a e198/162 ondisk, result = 0)", 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:40:07.375554+0000", 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 366.41052948399999, 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45866283400000002, 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.875 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.375554+0000", 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.375554+0000", 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.375553+0000", 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4059999999999999e-06 2023-10-09T05:46:13.876 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.375560+0000", 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.44929598900000001 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824856+0000", 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0549e-05 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.877 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.878 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.824867+0000", 2023-10-09T05:46:13.878 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.009337382 2023-10-09T05:46:13.878 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.878 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.878 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.878 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834204+0000", 2023-10-09T05:46:13.878 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6239999999999999e-06 2023-10-09T05:46:13.878 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.878 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.878 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.878 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834210+0000", 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7750000000000004e-06 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:40:07.834216+0000", 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45866283400000002 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.879 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125694 7.10 7:0c5ab723:::benchmark_data_smithi008_354649_object31198:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:03.091387+0000", 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 130.69469556300001, 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95149025600000003, 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125694 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.880 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091387+0000", 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091387+0000", 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999967 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091384+0000", 2023-10-09T05:46:13.881 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1436e-05 2023-10-09T05:46:13.882 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.882 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.882 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.882 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091396+0000", 2023-10-09T05:46:13.882 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3645999999999999e-05 2023-10-09T05:46:13.882 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.882 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.882 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.882 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091409+0000", 2023-10-09T05:46:13.882 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4099999999999997e-06 2023-10-09T05:46:13.882 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091419+0000", 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010047699999999999 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091519+0000", 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6649999999999998e-06 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.883 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091529+0000", 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93579876100000003 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.027328+0000", 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013571199999999999 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.027463+0000", 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015890500000000001 2023-10-09T05:46:13.884 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.885 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:46:13.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.027622+0000", 2023-10-09T05:46:13.885 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027023300000000002 2023-10-09T05:46:13.885 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.885 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.030325+0000", 2023-10-09T05:46:13.885 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040089269999999998 2023-10-09T05:46:13.885 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.034334+0000", 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.008445625 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042779+0000", 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8933e-05 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.886 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042838+0000", 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9582999999999997e-05 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042878+0000", 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95149025600000003 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.887 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150727 7.b 7:ddb4ebd7:::benchmark_data_smithi008_354649_object56231:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e252)", 2023-10-09T05:46:13.888 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:16.972532+0000", 2023-10-09T05:46:13.888 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 56.813551027000003, 2023-10-09T05:46:13.888 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.216839464, 2023-10-09T05:46:13.888 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.888 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.888 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.888 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:13.888 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:13.888 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150727 2023-10-09T05:46:13.888 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.888 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972532+0000", 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972532+0000", 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.889 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.890 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972530+0000", 2023-10-09T05:46:13.890 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8669999999999994e-06 2023-10-09T05:46:13.890 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.890 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.890 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.890 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972538+0000", 2023-10-09T05:46:13.890 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3001e-05 2023-10-09T05:46:13.890 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.890 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.890 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.890 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972551+0000", 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5217e-05 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972566+0000", 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002708668 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.975275+0000", 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9929999999999998e-06 2023-10-09T05:46:13.891 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.975282+0000", 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93068869399999998 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.905971+0000", 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0430000000000008e-06 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.905979+0000", 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.232000741 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.137980+0000", 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021446479999999999 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.140124+0000", 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6557000000000004e-05 2023-10-09T05:46:13.893 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.894 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.894 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:46:13.894 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.140221+0000", 2023-10-09T05:46:13.894 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011995643 2023-10-09T05:46:13.894 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.894 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.894 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.894 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.152216+0000", 2023-10-09T05:46:13.894 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00300789 2023-10-09T05:46:13.894 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.894 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.155224+0000", 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015419295 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.170644+0000", 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018450558999999998 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189094+0000", 2023-10-09T05:46:13.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119319 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189213+0000", 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000158014 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189371+0000", 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.216839464 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.896 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150733 7.0 7:0389df5e:::benchmark_data_smithi008_354649_object56237:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:17.005416+0000", 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 56.780666635000003, 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2781442140000001, 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:13.897 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:13.898 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150733 2023-10-09T05:46:13.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.898 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.898 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.898 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005416+0000", 2023-10-09T05:46:13.898 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.898 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.898 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005416+0000", 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005415+0000", 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0110000000000003e-06 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005422+0000", 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3579e-05 2023-10-09T05:46:13.899 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005435+0000", 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7229999999999998e-06 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005442+0000", 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8405000000000001e-05 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.900 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005501+0000", 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4299999999999995e-06 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005510+0000", 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.902713443 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.908223+0000", 2023-10-09T05:46:13.901 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7690000000000008e-06 2023-10-09T05:46:13.902 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.902 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.902 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.902 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.908233+0000", 2023-10-09T05:46:13.902 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3617464159999999 2023-10-09T05:46:13.902 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.902 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.902 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.902 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.269980+0000", 2023-10-09T05:46:13.902 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119249 2023-10-09T05:46:13.902 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.270099+0000", 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112638 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.270212+0000", 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059159390000000003 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.903 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.904 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.276127+0000", 2023-10-09T05:46:13.904 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033336210000000002 2023-10-09T05:46:13.904 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.904 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.904 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.904 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.279461+0000", 2023-10-09T05:46:13.904 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002913522 2023-10-09T05:46:13.904 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.904 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.904 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.904 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.282375+0000", 2023-10-09T05:46:13.905 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012422099999999999 2023-10-09T05:46:13.905 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.905 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.905 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.905 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.282499+0000", 2023-10-09T05:46:13.905 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00106174 2023-10-09T05:46:13.905 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.905 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.905 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.905 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.283561+0000", 2023-10-09T05:46:13.905 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2781442140000001 2023-10-09T05:46:13.906 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.906 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.906 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.906 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.906 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.906 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150737 7.b 7:d87fa168:::benchmark_data_smithi008_354649_object56241:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:46:13.906 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:17.110860+0000", 2023-10-09T05:46:13.906 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 56.675223297000002, 2023-10-09T05:46:13.906 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0787747959999998, 2023-10-09T05:46:13.906 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.906 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150737 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110860+0000", 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.907 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110860+0000", 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110858+0000", 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9359999999999994e-06 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.908 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110868+0000", 2023-10-09T05:46:13.909 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9113000000000001e-05 2023-10-09T05:46:13.909 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.909 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.909 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.909 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110887+0000", 2023-10-09T05:46:13.909 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4679999999999992e-06 2023-10-09T05:46:13.909 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.909 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.909 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.909 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110895+0000", 2023-10-09T05:46:13.909 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3297e-05 2023-10-09T05:46:13.910 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.910 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.910 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.910 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110969+0000", 2023-10-09T05:46:13.910 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3549e-05 2023-10-09T05:46:13.910 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.910 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.910 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.910 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110982+0000", 2023-10-09T05:46:13.910 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.79500997399999995 2023-10-09T05:46:13.910 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.905992+0000", 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1529999999999999e-06 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.905996+0000", 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.237875888 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.143872+0000", 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0003837 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.144256+0000", 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015668100000000001 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.144412+0000", 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0092940639999999995 2023-10-09T05:46:13.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.153707+0000", 2023-10-09T05:46:13.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001565924 2023-10-09T05:46:13.913 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.155272+0000", 2023-10-09T05:46:13.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.027123054000000001 2023-10-09T05:46:13.913 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.914 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.914 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.182396+0000", 2023-10-09T05:46:13.914 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070547509999999997 2023-10-09T05:46:13.914 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.914 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.914 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.914 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189450+0000", 2023-10-09T05:46:13.914 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3298000000000003e-05 2023-10-09T05:46:13.914 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.914 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.914 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189514+0000", 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120955 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189635+0000", 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0787747959999998 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.915 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150750 7.b 7:d9e0951c:::benchmark_data_smithi008_354649_object56254:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:17.325970+0000", 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 56.460112531999997, 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8639324529999999, 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150750 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.916 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.325970+0000", 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.325970+0000", 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.917 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.918 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.325969+0000", 2023-10-09T05:46:13.918 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8060000000000001e-06 2023-10-09T05:46:13.918 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.918 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.918 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.918 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.325978+0000", 2023-10-09T05:46:13.918 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8423e-05 2023-10-09T05:46:13.918 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.918 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.918 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.918 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.325996+0000", 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7100000000000001e-06 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.326003+0000", 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1128000000000002e-05 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.326064+0000", 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0229e-05 2023-10-09T05:46:13.919 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.326074+0000", 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.57993254100000002 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906007+0000", 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8789999999999996e-06 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.920 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.921 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906010+0000", 2023-10-09T05:46:13.921 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.24214676 2023-10-09T05:46:13.921 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.921 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.921 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.921 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.148157+0000", 2023-10-09T05:46:13.921 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012305199999999999 2023-10-09T05:46:13.921 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.921 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.921 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.921 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.148280+0000", 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000149951 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.148430+0000", 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006522707 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.154953+0000", 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00036319500000000002 2023-10-09T05:46:13.922 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.155316+0000", 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.027118198999999999 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.182434+0000", 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073679979999999997 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.923 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189802+0000", 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1600999999999998e-05 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189834+0000", 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8996000000000001e-05 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189903+0000", 2023-10-09T05:46:13.924 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8639324529999999 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150752 7.10 7:089e0e87:::benchmark_data_smithi008_354649_object56256:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:17.410069+0000", 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 56.376013772, 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.881545647, 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.925 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.926 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:13.926 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:13.926 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150752 2023-10-09T05:46:13.926 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.926 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.926 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.926 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.926 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410069+0000", 2023-10-09T05:46:13.926 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.926 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.926 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410069+0000", 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410068+0000", 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5140000000000001e-06 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.927 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410076+0000", 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7410000000000001e-05 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410093+0000", 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0040000000000002e-06 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410100+0000", 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3235000000000001e-05 2023-10-09T05:46:13.928 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410154+0000", 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7259999999999997e-06 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410163+0000", 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.49602615999999999 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.929 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906190+0000", 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.164e-06 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906195+0000", 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.36729938 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.273494+0000", 2023-10-09T05:46:13.930 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016964899999999999 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.273664+0000", 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015241199999999999 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.273816+0000", 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0056057640000000001 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.931 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.279422+0000", 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079792330000000005 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.287401+0000", 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031066990000000001 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.932 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.290508+0000", 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9672999999999997e-05 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.290558+0000", 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00105737 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.291615+0000", 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.881545647 2023-10-09T05:46:13.933 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150753 7.10 7:0ce0167e:::benchmark_data_smithi008_354649_object56257:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:17.452910+0000", 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 56.333172867999998, 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8600227089999999, 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:13.934 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150753 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452910+0000", 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:13.935 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452910+0000", 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452909+0000", 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4999999999999999e-06 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452917+0000", 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6939000000000002e-05 2023-10-09T05:46:13.936 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.937 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.937 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:13.937 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452934+0000", 2023-10-09T05:46:13.937 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2430000000000002e-06 2023-10-09T05:46:13.937 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.937 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.937 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:13.937 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452940+0000", 2023-10-09T05:46:13.937 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6143000000000001e-05 2023-10-09T05:46:13.937 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.937 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.938 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.938 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452996+0000", 2023-10-09T05:46:13.938 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7030000000000001e-06 2023-10-09T05:46:13.938 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.938 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.938 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.938 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.453006+0000", 2023-10-09T05:46:13.938 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45320798200000001 2023-10-09T05:46:13.938 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.938 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.938 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906214+0000", 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0080000000000002e-06 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906219+0000", 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3699190080000001 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.276138+0000", 2023-10-09T05:46:13.939 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000113307 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.276251+0000", 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014198699999999999 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.276393+0000", 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071874319999999997 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.940 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.941 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:13.941 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.283581+0000", 2023-10-09T05:46:13.941 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038658339999999999 2023-10-09T05:46:13.941 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.941 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.941 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.941 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.287447+0000", 2023-10-09T05:46:13.941 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024229746 2023-10-09T05:46:13.941 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.941 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.941 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.311676+0000", 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6548e-05 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.311753+0000", 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001179905 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.312933+0000", 2023-10-09T05:46:13.942 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8600227089999999 2023-10-09T05:46:13.943 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.943 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.943 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.943 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:13.943 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:13.943 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:46:13.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:13.980 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:14.030 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:14.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:13 smithi191 ceph-mon[161669]: pgmap v794: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 336 op/s; 25997/59613 objects degraded (43.610%); 3707/59613 objects misplaced (6.218%); 7.6 MiB/s, 205 objects/s recovering 2023-10-09T05:46:14.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:14.034 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:14.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:14.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:14.135 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:14.168 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:14.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:13 smithi008 ceph-mon[184418]: pgmap v794: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 336 op/s; 25997/59613 objects degraded (43.610%); 3707/59613 objects misplaced (6.218%); 7.6 MiB/s, 205 objects/s recovering 2023-10-09T05:46:14.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:14.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:14.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:13 smithi008 ceph-mon[191227]: pgmap v794: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 336 op/s; 25997/59613 objects degraded (43.610%); 3707/59613 objects misplaced (6.218%); 7.6 MiB/s, 205 objects/s recovering 2023-10-09T05:46:14.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:14.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:14.330 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:14.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:14.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:14.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:14.443 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:14.483 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:14.488 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:46:14.562 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:14.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:14.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:14.579 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:46:14.606 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:46:14.635 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:46:14.664 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:46:14.671 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:14.694 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:46:14.716 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:14.723 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:46:14.757 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:46:14.788 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:46:14.790 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:46:14.790 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:46:14.790 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:46:14.790 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:46:14.791 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:46:14.791 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:46:14.791 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:46:14.791 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:46:14.798 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:14.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:14.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:14.906 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:14.947 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:14.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:14 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2678463105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:14.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:14 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2678463105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:15.027 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:46:15.029 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:15.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:15.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:15.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:15.283 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:14 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2678463105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:15.291 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:15.379 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:15.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:15.491 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:15.551 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:15.668 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:15.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:15.775 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:15.809 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:16.167 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:46:16.193 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:15 smithi008 ceph-mon[184418]: pgmap v795: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 25997/57720 objects degraded (45.040%); 3437/57720 objects misplaced (5.955%); 7.8 MiB/s, 300 objects/s recovering 2023-10-09T05:46:16.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:15 smithi008 ceph-mon[184418]: 7.1 scrub starts 2023-10-09T05:46:16.194 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:15 smithi008 ceph-mon[191227]: pgmap v795: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 25997/57720 objects degraded (45.040%); 3437/57720 objects misplaced (5.955%); 7.8 MiB/s, 300 objects/s recovering 2023-10-09T05:46:16.194 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:15 smithi008 ceph-mon[191227]: 7.1 scrub starts 2023-10-09T05:46:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:15 smithi191 ceph-mon[161669]: pgmap v795: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 25997/57720 objects degraded (45.040%); 3437/57720 objects misplaced (5.955%); 7.8 MiB/s, 300 objects/s recovering 2023-10-09T05:46:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:15 smithi191 ceph-mon[161669]: 7.1 scrub starts 2023-10-09T05:46:16.587 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:16.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:16.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:16.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:16.747 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:16.849 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:16.892 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:16.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:16.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:16.997 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:17.049 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:17.221 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:46:17.221 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:46:17.317 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:17.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:17.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:17.424 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:17.461 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:17.674 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:17.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:17.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:17.778 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:17.810 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:17.869 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:17.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:17.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:17.925 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 7 pgp_num to 26 2023-10-09T05:46:17.951 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[191227]: pgmap v796: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 338 op/s; 23573/56541 objects degraded (41.692%); 3852/56541 objects misplaced (6.813%); 7.4 MiB/s, 260 objects/s recovering 2023-10-09T05:46:17.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[191227]: 7.1 scrub starts 2023-10-09T05:46:17.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[191227]: failed to encode map e262 with expected crc 2023-10-09T05:46:17.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[191227]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T05:46:17.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[191227]: failed to encode map e262 with expected crc 2023-10-09T05:46:17.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[191227]: failed to encode map e262 with expected crc 2023-10-09T05:46:17.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2116610639' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:46:17.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23573/56541 objects degraded (41.692%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:17.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2116610639' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:46:17.953 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[191227]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T05:46:17.975 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:18.004 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:18.052 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:18.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:18.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:18.156 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:18.201 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:18.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[184418]: pgmap v796: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 338 op/s; 23573/56541 objects degraded (41.692%); 3852/56541 objects misplaced (6.813%); 7.4 MiB/s, 260 objects/s recovering 2023-10-09T05:46:18.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[184418]: 7.1 scrub starts 2023-10-09T05:46:18.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[184418]: failed to encode map e262 with expected crc 2023-10-09T05:46:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[184418]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T05:46:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[184418]: failed to encode map e262 with expected crc 2023-10-09T05:46:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[184418]: failed to encode map e262 with expected crc 2023-10-09T05:46:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2116610639' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:46:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23573/56541 objects degraded (41.692%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2116610639' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:46:18.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:17 smithi008 ceph-mon[184418]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T05:46:18.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:17 smithi191 ceph-mon[161669]: pgmap v796: 99 pgs: 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 338 op/s; 23573/56541 objects degraded (41.692%); 3852/56541 objects misplaced (6.813%); 7.4 MiB/s, 260 objects/s recovering 2023-10-09T05:46:18.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:17 smithi191 ceph-mon[161669]: 7.1 scrub starts 2023-10-09T05:46:18.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:17 smithi191 ceph-mon[161669]: failed to encode map e262 with expected crc 2023-10-09T05:46:18.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:17 smithi191 ceph-mon[161669]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T05:46:18.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:17 smithi191 ceph-mon[161669]: failed to encode map e262 with expected crc 2023-10-09T05:46:18.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:17 smithi191 ceph-mon[161669]: failed to encode map e262 with expected crc 2023-10-09T05:46:18.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2116610639' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:46:18.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:17 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23573/56541 objects degraded (41.692%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:46:18.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2116610639' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:46:18.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:17 smithi191 ceph-mon[161669]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T05:46:18.534 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:46:18.620 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:18.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:18.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:18.704 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:46:18.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:18.756 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:18.846 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:18.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:18.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:18.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:18 smithi008 ceph-mon[191227]: failed to encode map e263 with expected crc 2023-10-09T05:46:18.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:18 smithi008 ceph-mon[191227]: failed to encode map e263 with expected crc 2023-10-09T05:46:18.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:18 smithi008 ceph-mon[191227]: failed to encode map e263 with expected crc 2023-10-09T05:46:18.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:18 smithi008 ceph-mon[191227]: failed to encode map e263 with expected crc 2023-10-09T05:46:18.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:18 smithi008 ceph-mon[191227]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T05:46:18.957 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:19.017 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:19.056 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:46:19.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:18 smithi008 ceph-mon[184418]: failed to encode map e263 with expected crc 2023-10-09T05:46:19.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:18 smithi008 ceph-mon[184418]: failed to encode map e263 with expected crc 2023-10-09T05:46:19.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:18 smithi008 ceph-mon[184418]: failed to encode map e263 with expected crc 2023-10-09T05:46:19.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:18 smithi008 ceph-mon[184418]: failed to encode map e263 with expected crc 2023-10-09T05:46:19.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:18 smithi008 ceph-mon[184418]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T05:46:19.234 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:19.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:19.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:19.305 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:18 smithi191 ceph-mon[161669]: failed to encode map e263 with expected crc 2023-10-09T05:46:19.305 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:18 smithi191 ceph-mon[161669]: failed to encode map e263 with expected crc 2023-10-09T05:46:19.305 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:18 smithi191 ceph-mon[161669]: failed to encode map e263 with expected crc 2023-10-09T05:46:19.306 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:18 smithi191 ceph-mon[161669]: failed to encode map e263 with expected crc 2023-10-09T05:46:19.306 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:18 smithi191 ceph-mon[161669]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T05:46:19.345 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:19.404 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:19.587 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:19.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:19.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:19.694 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:19.700 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:19.752 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:19.808 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:19.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:19.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:19.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:19.943 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:19.994 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:19.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:19.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:20.101 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:20.131 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:20.153 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:19 smithi008 ceph-mon[184418]: pgmap v799: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 23573/54441 objects degraded (43.300%); 3171/54441 objects misplaced (5.825%); 9.0 MiB/s, 274 objects/s recovering 2023-10-09T05:46:20.153 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:19 smithi008 ceph-mon[184418]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.153 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:19 smithi008 ceph-mon[184418]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.153 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:19 smithi008 ceph-mon[184418]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.154 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:19 smithi008 ceph-mon[184418]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:19 smithi008 ceph-mon[191227]: pgmap v799: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 23573/54441 objects degraded (43.300%); 3171/54441 objects misplaced (5.825%); 9.0 MiB/s, 274 objects/s recovering 2023-10-09T05:46:20.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:19 smithi008 ceph-mon[191227]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:19 smithi008 ceph-mon[191227]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:19 smithi008 ceph-mon[191227]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.154 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:19 smithi008 ceph-mon[191227]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.185 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:20.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:20.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:20.289 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:20.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:19 smithi191 ceph-mon[161669]: pgmap v799: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 23573/54441 objects degraded (43.300%); 3171/54441 objects misplaced (5.825%); 9.0 MiB/s, 274 objects/s recovering 2023-10-09T05:46:20.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:19 smithi191 ceph-mon[161669]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:19 smithi191 ceph-mon[161669]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:19 smithi191 ceph-mon[161669]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:19 smithi191 ceph-mon[161669]: failed to encode map e264 with expected crc 2023-10-09T05:46:20.320 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:20.396 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:20.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:20.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:20.502 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:20.547 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:20.730 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:20.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:20.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:20.839 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:20.876 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:46:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:46:20] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.33.4" 2023-10-09T05:46:20.881 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:20.975 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:46:21.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:20 smithi191 ceph-mon[161669]: 7.1 scrub starts 2023-10-09T05:46:21.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:20 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2531332497' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:46:21.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:20 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:46:21.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:20 smithi008 ceph-mon[184418]: 7.1 scrub starts 2023-10-09T05:46:21.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:20 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2531332497' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:46:21.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:20 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:46:21.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:20 smithi008 ceph-mon[191227]: 7.1 scrub starts 2023-10-09T05:46:21.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:20 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2531332497' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:46:21.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:20 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:46:21.264 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:46:21.265 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:46:21.805 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:21.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:21.910 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:21.944 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:22.004 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:22.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:22.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:22.108 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:22.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[184418]: pgmap v801: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 23472/51060 objects degraded (45.969%); 2430/51060 objects misplaced (4.759%); 14 MiB/s, 253 objects/s recovering 2023-10-09T05:46:22.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[184418]: 7.1 scrub starts 2023-10-09T05:46:22.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:22.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[184418]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:46:22.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[184418]: osdmap e265: 8 total, 8 up, 7 in 2023-10-09T05:46:22.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[184418]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[184418]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[184418]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.125 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[191227]: pgmap v801: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 23472/51060 objects degraded (45.969%); 2430/51060 objects misplaced (4.759%); 14 MiB/s, 253 objects/s recovering 2023-10-09T05:46:22.125 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[191227]: 7.1 scrub starts 2023-10-09T05:46:22.125 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:22.125 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[191227]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.125 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:46:22.125 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[191227]: osdmap e265: 8 total, 8 up, 7 in 2023-10-09T05:46:22.125 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[191227]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.126 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[191227]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.126 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:21 smithi008 ceph-mon[191227]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.151 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:21 smithi191 ceph-mon[161669]: pgmap v801: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 23472/51060 objects degraded (45.969%); 2430/51060 objects misplaced (4.759%); 14 MiB/s, 253 objects/s recovering 2023-10-09T05:46:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:21 smithi191 ceph-mon[161669]: 7.1 scrub starts 2023-10-09T05:46:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:21 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:21 smithi191 ceph-mon[161669]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:21 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:46:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:21 smithi191 ceph-mon[161669]: osdmap e265: 8 total, 8 up, 7 in 2023-10-09T05:46:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:21 smithi191 ceph-mon[161669]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:21 smithi191 ceph-mon[161669]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:21 smithi191 ceph-mon[161669]: failed to encode map e265 with expected crc 2023-10-09T05:46:22.310 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:22.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:22.369 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:46:22.388 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:46:22.411 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:46:22.416 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:22.434 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:46:22.453 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:22.457 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:46:22.484 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:46:22.504 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:46:22.511 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:22.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:22.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:22.525 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:46:22.526 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:46:22.526 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:46:22.526 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:46:22.527 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:46:22.527 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:46:22.527 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:46:22.527 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:46:22.527 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:46:22.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:46:22.615 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:22.664 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:22.805 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:22.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:22.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:22.917 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:22.953 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:23.020 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:23.023 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:22 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4238150453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:23.023 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:22 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23472/51060 objects degraded (45.969%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:23.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:22 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4238150453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:23.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:22 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23472/51060 objects degraded (45.969%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:23.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:23.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:23.066 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:46:23.125 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:23.158 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:22 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4238150453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:22 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23472/51060 objects degraded (45.969%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:23.446 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:23.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:23.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:23.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:23.580 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:23.629 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:23.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:23.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:23.734 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:23.765 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:23.821 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:23.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:23.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:23.876 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:46:23.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:23.960 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:24.017 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:24.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:24.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:24.121 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:24.151 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:24.198 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:24.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:24.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:23 smithi008 ceph-mon[184418]: pgmap v803: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1023 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 23472/48981 objects degraded (47.921%); 2025/48981 objects misplaced (4.134%); 12 MiB/s, 216 objects/s recovering 2023-10-09T05:46:24.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:23 smithi008 ceph-mon[191227]: pgmap v803: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1023 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 23472/48981 objects degraded (47.921%); 2025/48981 objects misplaced (4.134%); 12 MiB/s, 216 objects/s recovering 2023-10-09T05:46:24.262 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:46:24.262 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:46:24.262 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2023-10-09T05:46:24.263 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:46:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:23 smithi191 ceph-mon[161669]: pgmap v803: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 1023 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 23472/48981 objects degraded (47.921%); 2025/48981 objects misplaced (4.134%); 12 MiB/s, 216 objects/s recovering 2023-10-09T05:46:24.301 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:24.304 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:24.337 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:24.470 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:24.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:24.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:24.579 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:24.636 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:24.722 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:24.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:24.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:24.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:24.868 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:24.998 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:25.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:25.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:25.108 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:25.148 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:25.210 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:25.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:25.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:25.251 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:46:25.251 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:57816 6.5 6:a6f014e8:::benchmark_data_smithi008_295510_object57815:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:53.723556+0000", 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 511.52500288300001, 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.293916552, 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:25.252 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:25.253 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 57816 2023-10-09T05:46:25.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.253 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723556+0000", 2023-10-09T05:46:25.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723556+0000", 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723555+0000", 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4774000000000003e-05 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723589+0000", 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8880000000000003e-06 2023-10-09T05:46:25.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723598+0000", 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6620000000000002e-06 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.723604+0000", 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0074562639999999998 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731060+0000", 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000178142 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731238+0000", 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018982800000000001 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731428+0000", 2023-10-09T05:46:25.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0034443920000000001 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.734872+0000", 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028075729999999998 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.737680+0000", 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27808630400000001 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.015766+0000", 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9630999999999998e-05 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.015836+0000", 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001636813 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.017473+0000", 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.293916552 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:57823 6.5 6:a9e36634:::benchmark_data_smithi008_295510_object57822:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:37:53.731562+0000", 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 511.51699692900002, 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28707806800000002, 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.259 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 57823 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731562+0000", 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731562+0000", 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731561+0000", 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.126e-05 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731612+0000", 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5167999999999999e-05 2023-10-09T05:46:25.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731627+0000", 2023-10-09T05:46:25.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1280000000000002e-06 2023-10-09T05:46:25.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.731634+0000", 2023-10-09T05:46:25.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001613838 2023-10-09T05:46:25.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.733248+0000", 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013990199999999999 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.733388+0000", 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011707800000000001 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:46:25.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.733505+0000", 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014731880000000001 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.734978+0000", 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027389850000000002 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:53.737717+0000", 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27980261000000001 2023-10-09T05:46:25.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.017520+0000", 2023-10-09T05:46:25.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4610999999999997e-05 2023-10-09T05:46:25.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.017565+0000", 2023-10-09T05:46:25.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010756380000000001 2023-10-09T05:46:25.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:37:54.018640+0000", 2023-10-09T05:46:25.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28707806800000002 2023-10-09T05:46:25.266 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.266 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.266 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.266 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:72554 6.3 6:caefc39f:::benchmark_data_smithi008_295510_object72553:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-10-09T05:46:25.266 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:38:13.303447+0000", 2023-10-09T05:46:25.266 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 491.94511226700001, 2023-10-09T05:46:25.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30038207500000003, 2023-10-09T05:46:25.267 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.267 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.267 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.267 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:25.267 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:25.267 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72554 2023-10-09T05:46:25.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.267 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303447+0000", 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303447+0000", 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303445+0000", 2023-10-09T05:46:25.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8564000000000002e-05 2023-10-09T05:46:25.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.269 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303524+0000", 2023-10-09T05:46:25.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.117e-05 2023-10-09T05:46:25.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.269 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303535+0000", 2023-10-09T05:46:25.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.516e-06 2023-10-09T05:46:25.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.303542+0000", 2023-10-09T05:46:25.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050977210000000004 2023-10-09T05:46:25.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.308639+0000", 2023-10-09T05:46:25.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016793 2023-10-09T05:46:25.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.308807+0000", 2023-10-09T05:46:25.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013890300000000001 2023-10-09T05:46:25.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:46:25.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.308946+0000", 2023-10-09T05:46:25.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021140360000000001 2023-10-09T05:46:25.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.311060+0000", 2023-10-09T05:46:25.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013415409999999999 2023-10-09T05:46:25.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.312402+0000", 2023-10-09T05:46:25.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.289153361 2023-10-09T05:46:25.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.601555+0000", 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8233999999999997e-05 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.601633+0000", 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021954909999999999 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:13.603829+0000", 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30038207500000003 2023-10-09T05:46:25.273 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:74957 6.3 6:ca97a06f:::benchmark_data_smithi008_295510_object74956:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e181)", 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:38:17.013260+0000", 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 488.23529903500003, 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.256926671, 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.274 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:25.275 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:25.275 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 74957 2023-10-09T05:46:25.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.275 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013260+0000", 2023-10-09T05:46:25.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013260+0000", 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013259+0000", 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.71e-05 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013306+0000", 2023-10-09T05:46:25.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1833e-05 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013318+0000", 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8119999999999996e-06 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.013323+0000", 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039195150000000002 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.017243+0000", 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017195099999999999 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.017415+0000", 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000214391 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:46:25.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.017629+0000", 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048030879999999996 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.022432+0000", 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029784350000000002 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.025411+0000", 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24323214100000001 2023-10-09T05:46:25.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.268643+0000", 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8260000000000002e-05 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.268691+0000", 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014955210000000001 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.270187+0000", 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.256926671 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:74962 6.3 6:c979a0e9:::benchmark_data_smithi008_295510_object74961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e181)", 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:38:17.015632+0000", 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 488.232927104, 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25564477200000002, 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.281 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 74962 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015632+0000", 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015632+0000", 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015631+0000", 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4298999999999997e-05 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015675+0000", 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3551e-05 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015689+0000", 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0249999999999999e-06 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.015695+0000", 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040153250000000001 2023-10-09T05:46:25.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.019710+0000", 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118621 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.019828+0000", 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000193387 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:46:25.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.020022+0000", 2023-10-09T05:46:25.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024983150000000001 2023-10-09T05:46:25.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.022520+0000", 2023-10-09T05:46:25.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0064664869999999999 2023-10-09T05:46:25.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.028987+0000", 2023-10-09T05:46:25.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24126114400000001 2023-10-09T05:46:25.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.270248+0000", 2023-10-09T05:46:25.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9090000000000004e-05 2023-10-09T05:46:25.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.270317+0000", 2023-10-09T05:46:25.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00095988599999999996 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:17.271277+0000", 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25564477200000002 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:91608 6.5 6:a934938f:::benchmark_data_smithi008_295510_object3245:head [delete] snapc 0=[] ondisk+write+known_if_redirected e184)", 2023-10-09T05:46:25.288 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:38:41.229295+0000", 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 464.01926366599997, 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24228119000000001, 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91608 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229295+0000", 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229295+0000", 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229295+0000", 2023-10-09T05:46:25.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3749999999999999e-06 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229298+0000", 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3010000000000003e-06 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229305+0000", 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4959999999999998e-06 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229309+0000", 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122569 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229432+0000", 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017970599999999999 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229611+0000", 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016554100000000001 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.229777+0000", 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063023059999999997 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.236079+0000", 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00062206499999999999 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.236701+0000", 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23290044500000001 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.469602+0000", 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0106999999999999e-05 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.469662+0000", 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019147490000000001 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.471577+0000", 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24228119000000001 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45029.0:91632 6.5 6:a980578b:::benchmark_data_smithi008_295510_object3269:head [delete] snapc 0=[] ondisk+write+known_if_redirected e184)", 2023-10-09T05:46:25.295 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:38:41.249264+0000", 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 463.99929505900002, 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23107453999999999, 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45029", 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/814516013", 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91632 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249264+0000", 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249264+0000", 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249263+0000", 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1389999999999998e-06 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249270+0000", 2023-10-09T05:46:25.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3324e-05 2023-10-09T05:46:25.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249283+0000", 2023-10-09T05:46:25.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4700000000000001e-06 2023-10-09T05:46:25.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249289+0000", 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5268000000000001e-05 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249334+0000", 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012688599999999999 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249461+0000", 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013272699999999999 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.249594+0000", 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024026540000000002 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.251996+0000", 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.187454126 2023-10-09T05:46:25.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.439450+0000", 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.039058080000000002 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.478508+0000", 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7931999999999999e-05 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.478586+0000", 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017522530000000001 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:38:41.480339+0000", 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23107453999999999 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.303 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:19336 7.d 7:bca1948b:::benchmark_data_smithi008_354649_object19335:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-09T05:46:25.303 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:46:25.303 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 278.601300659, 2023-10-09T05:46:25.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27934059700000002, 2023-10-09T05:46:25.303 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.303 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.303 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.303 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.303 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.303 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 19336 2023-10-09T05:46:25.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647257+0000", 2023-10-09T05:46:25.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4706000000000003e-05 2023-10-09T05:46:25.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647292+0000", 2023-10-09T05:46:25.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0139999999999999e-05 2023-10-09T05:46:25.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647302+0000", 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4790000000000004e-06 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647308+0000", 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0175000000000001e-05 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647348+0000", 2023-10-09T05:46:25.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3017999999999997e-05 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647431+0000", 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7472000000000005e-05 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647528+0000", 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00087520700000000002 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.648404+0000", 2023-10-09T05:46:25.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091703300000000004 2023-10-09T05:46:25.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.649321+0000", 2023-10-09T05:46:25.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27717746799999998 2023-10-09T05:46:25.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926498+0000", 2023-10-09T05:46:25.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2379000000000001e-05 2023-10-09T05:46:25.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926550+0000", 2023-10-09T05:46:25.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8653000000000003e-05 2023-10-09T05:46:25.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926599+0000", 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27934059700000002 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:52521 7.c 7:3208f079:::benchmark_data_smithi008_354649_object52520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 250.647630881, 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.253878241, 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.310 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52521 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600927+0000", 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114644 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601041+0000", 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3705e-05 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601055+0000", 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3670000000000002e-06 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601062+0000", 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6796000000000003e-05 2023-10-09T05:46:25.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601118+0000", 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000151423 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601270+0000", 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000187224 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601457+0000", 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018848820000000001 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.603342+0000", 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017959000000000001 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.603521+0000", 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25118068700000001 2023-10-09T05:46:25.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854702+0000", 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0417000000000001e-05 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854763+0000", 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3959999999999999e-05 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854806+0000", 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.253878241 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72589 7.d 7:bac7991c:::benchmark_data_smithi008_354649_object72588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 227.790129955, 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.258363538, 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.317 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72589 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.318 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8671000000000004e-05 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458527+0000", 2023-10-09T05:46:25.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2957e-05 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458540+0000", 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6649999999999997e-06 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458547+0000", 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2287999999999997e-05 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458609+0000", 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012157100000000001 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458731+0000", 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000152602 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458883+0000", 2023-10-09T05:46:25.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026998410000000001 2023-10-09T05:46:25.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.461583+0000", 2023-10-09T05:46:25.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0065822210000000001 2023-10-09T05:46:25.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.468166+0000", 2023-10-09T05:46:25.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24850773600000001 2023-10-09T05:46:25.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716673+0000", 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7136000000000002e-05 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716740+0000", 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2272999999999999e-05 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716793+0000", 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.258363538 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72593 7.d 7:ba3e3325:::benchmark_data_smithi008_354649_object72592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 227.782655291, 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25827438800000002, 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.324 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72593 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.325 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465902+0000", 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000110436 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466013+0000", 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0246e-05 2023-10-09T05:46:25.326 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466023+0000", 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7699999999999998e-06 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466029+0000", 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0343999999999999e-05 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.327 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466069+0000", 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4201000000000002e-05 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466153+0000", 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3833999999999996e-05 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466247+0000", 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020428099999999999 2023-10-09T05:46:25.328 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.468290+0000", 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011037670000000001 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.469394+0000", 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25468252499999999 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.329 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.330 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724076+0000", 2023-10-09T05:46:25.330 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0541000000000003e-05 2023-10-09T05:46:25.330 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.330 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.330 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.330 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724127+0000", 2023-10-09T05:46:25.330 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1499000000000003e-05 2023-10-09T05:46:25.330 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.330 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.330 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.330 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724178+0000", 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25827438800000002 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72601 7.d 7:b4df4171:::benchmark_data_smithi008_354649_object72600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 227.77671533700001, 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25693990900000002, 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.331 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.332 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.332 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.332 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.332 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72601 2023-10-09T05:46:25.332 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.332 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.332 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.332 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.332 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:46:25.332 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.332 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471843+0000", 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1391000000000003e-05 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471894+0000", 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3217000000000001e-05 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471907+0000", 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4e-06 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471913+0000", 2023-10-09T05:46:25.334 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6798999999999998e-05 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471950+0000", 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6098000000000001e-05 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.472036+0000", 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010641099999999999 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.335 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.472142+0000", 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001463415 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.473605+0000", 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00083725200000000003 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.474443+0000", 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25426129400000003 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728704+0000", 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5178e-05 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728749+0000", 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4474999999999998e-05 2023-10-09T05:46:25.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728784+0000", 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25693990900000002 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102136 7.c 7:3c9f4178:::benchmark_data_smithi008_354649_object7640:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:46:25.338 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 186.373014635, 2023-10-09T05:46:25.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3695118229999999, 2023-10-09T05:46:25.339 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.339 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:46:25.339 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.339 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.339 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.339 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102136 2023-10-09T05:46:25.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.339 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875543+0000", 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0234999999999999e-05 2023-10-09T05:46:25.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875553+0000", 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6123000000000001e-05 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875569+0000", 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7334e-05 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875586+0000", 2023-10-09T05:46:25.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3238999999999997e-05 2023-10-09T05:46:25.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875630+0000", 2023-10-09T05:46:25.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120889 2023-10-09T05:46:25.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875751+0000", 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000121965 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875873+0000", 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068513569999999998 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.882724+0000", 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.155e-05 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.343 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.882735+0000", 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99459824299999999 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877334+0000", 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.508e-05 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:25.344 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877349+0000", 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.345701239 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.223050+0000", 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016460699999999999 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.223215+0000", 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021743302999999999 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.345 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.346 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.346 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.244958+0000", 2023-10-09T05:46:25.346 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8379000000000001e-05 2023-10-09T05:46:25.346 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.346 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.346 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.346 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245056+0000", 2023-10-09T05:46:25.346 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3695118229999999 2023-10-09T05:46:25.346 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.346 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102150 7.c 7:30e13689:::benchmark_data_smithi008_354649_object7654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 186.34395881699999, 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3456080319999999, 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.347 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102150 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:25.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904598+0000", 2023-10-09T05:46:25.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9720000000000003e-06 2023-10-09T05:46:25.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904605+0000", 2023-10-09T05:46:25.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3104e-05 2023-10-09T05:46:25.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904619+0000", 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7699999999999998e-06 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904624+0000", 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5915000000000003e-05 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904680+0000", 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1524e-05 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904692+0000", 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97280887699999996 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877501+0000", 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9029999999999993e-06 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877510+0000", 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3503837970000001 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.227893+0000", 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017629700000000001 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.228070+0000", 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019175 2023-10-09T05:46:25.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:46:25.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.228261+0000", 2023-10-09T05:46:25.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086207179999999994 2023-10-09T05:46:25.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.236882+0000", 2023-10-09T05:46:25.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0081705159999999992 2023-10-09T05:46:25.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245053+0000", 2023-10-09T05:46:25.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4658000000000003e-05 2023-10-09T05:46:25.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245117+0000", 2023-10-09T05:46:25.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2007000000000004e-05 2023-10-09T05:46:25.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245189+0000", 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005019061 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250208+0000", 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3456080319999999 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.356 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102158 7.c 7:380582de:::benchmark_data_smithi008_354649_object7662:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:46:25.356 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:46:25.356 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 186.33922291499999, 2023-10-09T05:46:25.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3657197050000001, 2023-10-09T05:46:25.356 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.356 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.356 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.356 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.356 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.356 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102158 2023-10-09T05:46:25.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.357 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:46:25.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:46:25.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:25.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909335+0000", 2023-10-09T05:46:25.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1000000000000003e-06 2023-10-09T05:46:25.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909340+0000", 2023-10-09T05:46:25.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9660000000000002e-06 2023-10-09T05:46:25.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909349+0000", 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3199999999999999e-06 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909354+0000", 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2012999999999999e-05 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909396+0000", 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4160000000000001e-06 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909403+0000", 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96813495800000005 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877538+0000", 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5560000000000008e-06 2023-10-09T05:46:25.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877546+0000", 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3727139369999999 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250260+0000", 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017051999999999999 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250431+0000", 2023-10-09T05:46:25.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019215399999999999 2023-10-09T05:46:25.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:46:25.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250623+0000", 2023-10-09T05:46:25.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010646828000000001 2023-10-09T05:46:25.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261270+0000", 2023-10-09T05:46:25.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6761999999999995e-05 2023-10-09T05:46:25.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261337+0000", 2023-10-09T05:46:25.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087281960000000006 2023-10-09T05:46:25.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.270065+0000", 2023-10-09T05:46:25.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9771000000000006e-05 2023-10-09T05:46:25.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.364 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.364 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.364 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.270144+0000", 2023-10-09T05:46:25.364 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0049114320000000003 2023-10-09T05:46:25.364 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.364 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.364 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.364 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275056+0000", 2023-10-09T05:46:25.364 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3657197050000001 2023-10-09T05:46:25.364 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.364 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.365 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.365 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.365 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102172 7.c 7:3ca77be9:::benchmark_data_smithi008_354649_object7676:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:46:25.365 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:46:25.365 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 186.312485845, 2023-10-09T05:46:25.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3439011609999998, 2023-10-09T05:46:25.365 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.365 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.365 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.365 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.366 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.366 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102172 2023-10-09T05:46:25.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.366 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:46:25.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999971 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936071+0000", 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4219999999999999e-05 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936085+0000", 2023-10-09T05:46:25.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3510000000000003e-05 2023-10-09T05:46:25.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936118+0000", 2023-10-09T05:46:25.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3056e-05 2023-10-09T05:46:25.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936131+0000", 2023-10-09T05:46:25.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013862499999999999 2023-10-09T05:46:25.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936270+0000", 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3222999999999997e-05 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936303+0000", 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.941830422 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.878134+0000", 2023-10-09T05:46:25.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3155e-05 2023-10-09T05:46:25.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:46:25.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.878147+0000", 2023-10-09T05:46:25.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.375972827 2023-10-09T05:46:25.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254120+0000", 2023-10-09T05:46:25.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00031237700000000002 2023-10-09T05:46:25.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254432+0000", 2023-10-09T05:46:25.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000179364 2023-10-09T05:46:25.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:46:25.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254611+0000", 2023-10-09T05:46:25.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0066698180000000001 2023-10-09T05:46:25.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261281+0000", 2023-10-09T05:46:25.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00092641700000000002 2023-10-09T05:46:25.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.262208+0000", 2023-10-09T05:46:25.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012930142 2023-10-09T05:46:25.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275138+0000", 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1118999999999995e-05 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275209+0000", 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047656299999999999 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.279974+0000", 2023-10-09T05:46:25.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3439011609999998 2023-10-09T05:46:25.374 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.374 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.374 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.374 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:124846 7.c 7:339e95c6:::benchmark_data_smithi008_354649_object30350:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:46:25.374 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:46:25.374 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 143.74461012500001, 2023-10-09T05:46:25.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19787401099999999, 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124846 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8360000000000001e-06 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503951+0000", 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.534e-06 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503954+0000", 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7929999999999999e-06 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503958+0000", 2023-10-09T05:46:25.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070412340000000004 2023-10-09T05:46:25.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.378 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.510999+0000", 2023-10-09T05:46:25.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012243900000000001 2023-10-09T05:46:25.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.378 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.511122+0000", 2023-10-09T05:46:25.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012357500000000001 2023-10-09T05:46:25.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:46:25.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.511245+0000", 2023-10-09T05:46:25.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070869629999999999 2023-10-09T05:46:25.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.518332+0000", 2023-10-09T05:46:25.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010506108 2023-10-09T05:46:25.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.528838+0000", 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070145440000000002 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.535853+0000", 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2738000000000002e-05 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.535905+0000", 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.165917541 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.701823+0000", 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19787401099999999 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.381 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:124870 7.c 7:34cc5040:::benchmark_data_smithi008_354649_object30374:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:46:25.382 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:46:25.382 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 143.712176175, 2023-10-09T05:46:25.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.194346888, 2023-10-09T05:46:25.382 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.382 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.382 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.382 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.382 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.382 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124870 2023-10-09T05:46:25.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.382 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536382+0000", 2023-10-09T05:46:25.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1800000000000001e-06 2023-10-09T05:46:25.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.384 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.384 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536388+0000", 2023-10-09T05:46:25.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8409999999999995e-06 2023-10-09T05:46:25.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.384 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.384 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536398+0000", 2023-10-09T05:46:25.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9701e-05 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536417+0000", 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16904754699999999 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705465+0000", 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016865399999999999 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705634+0000", 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017818700000000001 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705812+0000", 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068761990000000004 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.712688+0000", 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078597960000000005 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.720548+0000", 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059971089999999996 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.726545+0000", 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4085000000000006e-05 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.726619+0000", 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004110832 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.730730+0000", 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.194346888 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.389 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:136135 7.d 7:b8c830a8:::benchmark_data_smithi008_354649_object41639:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-09T05:46:25.389 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:35.638089+0000", 2023-10-09T05:46:25.389 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 109.61046998400001, 2023-10-09T05:46:25.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.252176754, 2023-10-09T05:46:25.389 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.389 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.389 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.389 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.389 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.389 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 136135 2023-10-09T05:46:25.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.390 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638089+0000", 2023-10-09T05:46:25.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638089+0000", 2023-10-09T05:46:25.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:46:25.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.391 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.391 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638087+0000", 2023-10-09T05:46:25.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0289999999999994e-06 2023-10-09T05:46:25.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.391 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.391 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638096+0000", 2023-10-09T05:46:25.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6886000000000001e-05 2023-10-09T05:46:25.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638113+0000", 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0861e-05 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638123+0000", 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8810000000000001e-05 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638182+0000", 2023-10-09T05:46:25.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000136959 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638319+0000", 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015855900000000001 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638478+0000", 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048134650000000003 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643291+0000", 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073579750000000001 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.650649+0000", 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23951030400000001 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.890160+0000", 2023-10-09T05:46:25.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8352e-05 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.890218+0000", 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8004000000000003e-05 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.890266+0000", 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.252176754 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.395 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:136137 7.d 7:b6392fbb:::benchmark_data_smithi008_354649_object41641:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 109.604572834, 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24962479700000001, 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 136137 2023-10-09T05:46:25.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:25.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:46:25.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1309999999999999e-06 2023-10-09T05:46:25.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:25.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643990+0000", 2023-10-09T05:46:25.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4509999999999992e-06 2023-10-09T05:46:25.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643998+0000", 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5480000000000001e-06 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644004+0000", 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1790999999999999e-05 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644056+0000", 2023-10-09T05:46:25.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015365299999999999 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644209+0000", 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017818200000000001 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644387+0000", 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047856310000000003 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.649173+0000", 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015424 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.650715+0000", 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.242803251 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:46:25.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.893519+0000", 2023-10-09T05:46:25.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7497000000000002e-05 2023-10-09T05:46:25.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:46:25.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.893576+0000", 2023-10-09T05:46:25.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4962000000000002e-05 2023-10-09T05:46:25.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:25.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:25.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:46:25.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.893611+0000", 2023-10-09T05:46:25.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24962479700000001 2023-10-09T05:46:25.403 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.403 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.403 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.403 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:25.403 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:25.403 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:46:25.403 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:25.451 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:25.514 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:46:25.793 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:25.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:25.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:25.867 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:25.898 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:25.942 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:26.088 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:26.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:25 smithi008 ceph-mon[184418]: pgmap v804: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 978 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 23472/46827 objects degraded (50.125%); 1907/46827 objects misplaced (4.072%); 9.9 MiB/s, 181 objects/s recovering 2023-10-09T05:46:26.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:25 smithi008 ceph-mon[191227]: pgmap v804: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 978 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 23472/46827 objects degraded (50.125%); 1907/46827 objects misplaced (4.072%); 9.9 MiB/s, 181 objects/s recovering 2023-10-09T05:46:26.251 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:25 smithi191 ceph-mon[161669]: pgmap v804: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 978 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 23472/46827 objects degraded (50.125%); 1907/46827 objects misplaced (4.072%); 9.9 MiB/s, 181 objects/s recovering 2023-10-09T05:46:26.270 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:46:26.289 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:26.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:26.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:26.392 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:26.432 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:26.704 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:46:26.999 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:46:27.190 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3460040626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:27.190 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:26 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:27.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3460040626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:27.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:26 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:27.278 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3460040626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:27.278 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:26 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:27.762 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:27.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:27.866 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:27.908 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[184418]: pgmap v805: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 952 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 23360/45579 objects degraded (51.252%); 1790/45579 objects misplaced (3.927%); 8.9 MiB/s, 250 objects/s recovering 2023-10-09T05:46:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[184418]: failed to encode map e266 with expected crc 2023-10-09T05:46:27.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[184418]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.096 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:28.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:28.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:28.200 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:28.241 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:46:28.241 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[184418]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.241 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[184418]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T05:46:28.241 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[184418]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.241 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[184418]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.241 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23360/45579 objects degraded (51.252%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:28.242 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[191227]: pgmap v805: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 952 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 23360/45579 objects degraded (51.252%); 1790/45579 objects misplaced (3.927%); 8.9 MiB/s, 250 objects/s recovering 2023-10-09T05:46:28.242 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:28.242 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[191227]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.242 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[191227]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.242 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:46:28.243 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[191227]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.243 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[191227]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T05:46:28.243 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[191227]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.243 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[191227]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.243 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:27 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23360/45579 objects degraded (51.252%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:28.252 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:28.273 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:27 smithi191 ceph-mon[161669]: pgmap v805: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 952 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 23360/45579 objects degraded (51.252%); 1790/45579 objects misplaced (3.927%); 8.9 MiB/s, 250 objects/s recovering 2023-10-09T05:46:28.273 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:27 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:28.273 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:27 smithi191 ceph-mon[161669]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.273 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:27 smithi191 ceph-mon[161669]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.273 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:27 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:46:28.273 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:27 smithi191 ceph-mon[161669]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.273 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:27 smithi191 ceph-mon[161669]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T05:46:28.273 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:27 smithi191 ceph-mon[161669]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.273 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:27 smithi191 ceph-mon[161669]: failed to encode map e266 with expected crc 2023-10-09T05:46:28.274 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:27 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23360/45579 objects degraded (51.252%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:28.480 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:28.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:28.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:28.584 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:28.613 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:28.663 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:28.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:28.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:28.756 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:46:28.756 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:46:28.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.757 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175753 7.5 7:a2493aec:::benchmark_data_smithi008_354649_object81257:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.757 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.393237+0000", 2023-10-09T05:46:28.757 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.36087291500000002, 2023-10-09T05:46:28.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.36091128, 2023-10-09T05:46:28.757 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.757 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.757 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.757 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.757 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.757 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175753 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393237+0000", 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393237+0000", 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:28.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393235+0000", 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5310000000000001e-06 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393244+0000", 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6064000000000001e-05 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393260+0000", 2023-10-09T05:46:28.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7100000000000007e-06 2023-10-09T05:46:28.760 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.760 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393268+0000", 2023-10-09T05:46:28.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00036294799999999998 2023-10-09T05:46:28.760 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.760 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393631+0000", 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001124845 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.394755+0000", 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0062333060000000001 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.400989+0000", 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094668499999999995 2023-10-09T05:46:28.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.401935+0000", 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086980949999999994 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175754 7.15 7:a88a5f46:::benchmark_data_smithi008_354649_object81258:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.393288+0000", 2023-10-09T05:46:28.762 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.36082253399999997, 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.36095743200000002, 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175754 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393288+0000", 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393288+0000", 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393287+0000", 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4599999999999999e-06 2023-10-09T05:46:28.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393291+0000", 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9269999999999999e-06 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393297+0000", 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3230000000000003e-06 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.393304+0000", 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00078463700000000003 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.394088+0000", 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00099101300000000001 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.395079+0000", 2023-10-09T05:46:28.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005952761 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.401032+0000", 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001022617 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.402055+0000", 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087671840000000008 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.767 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175706 7.15 7:ad680021:::benchmark_data_smithi008_354649_object81210:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.304455+0000", 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.449655162, 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.44984354599999998, 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.768 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175706 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.304455+0000", 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.304455+0000", 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:28.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.304453+0000", 2023-10-09T05:46:28.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5299999999999996e-06 2023-10-09T05:46:28.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.304462+0000", 2023-10-09T05:46:28.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5328999999999998e-05 2023-10-09T05:46:28.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.304477+0000", 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3430000000000001e-06 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.304483+0000", 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00033891299999999999 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.304822+0000", 2023-10-09T05:46:28.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010174520000000001 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.305840+0000", 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071813939999999998 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.313021+0000", 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011209950000000001 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.772 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.314142+0000", 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0096868960000000004 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175756 7.15 7:ab403ba3:::benchmark_data_smithi008_354649_object81260:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.400009+0000", 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.35410087299999998, 2023-10-09T05:46:28.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.35434557999999999, 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175756 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.400009+0000", 2023-10-09T05:46:28.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.400009+0000", 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.400007+0000", 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8359999999999998e-06 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.400016+0000", 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3417e-05 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.400030+0000", 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9470000000000003e-06 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.400038+0000", 2023-10-09T05:46:28.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023103910000000002 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.402348+0000", 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00099314200000000003 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.403341+0000", 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0056900809999999996 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.777 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.409031+0000", 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094773899999999996 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.409979+0000", 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099697020000000004 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175763 7.15 7:ac46e89b:::benchmark_data_smithi008_354649_object81267:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.426479+0000", 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.32763099200000001, 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.32794799899999999, 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175763 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.779 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.426479+0000", 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.426479+0000", 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.426478+0000", 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5379999999999997e-06 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.426485+0000", 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5495999999999999e-05 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.426501+0000", 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9990000000000002e-06 2023-10-09T05:46:28.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.426508+0000", 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000198228 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.426706+0000", 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00090317100000000003 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.427609+0000", 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0062037610000000003 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.433813+0000", 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011556400000000001 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.434968+0000", 2023-10-09T05:46:28.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0084892289999999992 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175765 7.15 7:af9a168a:::benchmark_data_smithi008_354649_object81269:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.436388+0000", 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.31772251499999998, 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.318126984, 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.784 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175765 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.436388+0000", 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.436388+0000", 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.436386+0000", 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4179999999999996e-06 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.436392+0000", 2023-10-09T05:46:28.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0977e-05 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.436403+0000", 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9880000000000004e-06 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.436408+0000", 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012424800000000001 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.436532+0000", 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00090134199999999996 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.437433+0000", 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0054712670000000001 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.442905+0000", 2023-10-09T05:46:28.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00090888000000000004 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.443813+0000", 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0074257819999999997 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175726 7.15 7:a83a8aa2:::benchmark_data_smithi008_354649_object81230:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.338345+0000", 2023-10-09T05:46:28.789 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.415765681, 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.41626204900000002, 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175726 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.338345+0000", 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.338345+0000", 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.338342+0000", 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3159999999999997e-06 2023-10-09T05:46:28.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.338351+0000", 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7374999999999999e-05 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.338368+0000", 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5159999999999997e-06 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.338376+0000", 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000337867 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.338713+0000", 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000998332 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.339712+0000", 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006690072 2023-10-09T05:46:28.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.346402+0000", 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001010064 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.347412+0000", 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0090673569999999998 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.794 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175810 7.15 7:aa1c11ae:::benchmark_data_smithi008_354649_object81314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.697289+0000", 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.056821386000000002, 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.057395518999999999, 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175810 2023-10-09T05:46:28.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.697289+0000", 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.697289+0000", 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.697286+0000", 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0971999999999999e-05 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.697297+0000", 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8298e-05 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.697316+0000", 2023-10-09T05:46:28.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3610000000000007e-06 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.697325+0000", 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000344774 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.697670+0000", 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094440400000000003 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.698614+0000", 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0058799710000000003 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.704494+0000", 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011785680000000001 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.705673+0000", 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0083838790000000003 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175731 7.5 7:a458e381:::benchmark_data_smithi008_354649_object81235:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.344595+0000", 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.40951532200000001, 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.41017899400000002, 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.800 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175731 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.344595+0000", 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.344595+0000", 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.344592+0000", 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1185e-05 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.344603+0000", 2023-10-09T05:46:28.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6109000000000001e-05 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.344620+0000", 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5399999999999998e-06 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.344627+0000", 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00036072699999999998 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.344988+0000", 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011244860000000001 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.346112+0000", 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073424629999999996 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.353455+0000", 2023-10-09T05:46:28.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012464539999999999 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.354701+0000", 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010106364 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175677 7.5 7:a6c91e24:::benchmark_data_smithi008_354649_object81181:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.805 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.255969+0000", 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.49814087299999998, 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.49888323200000001, 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175677 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.806 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.255969+0000", 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.255969+0000", 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.255968+0000", 2023-10-09T05:46:28.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4240000000000004e-06 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.255975+0000", 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3461e-05 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.255989+0000", 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.708e-06 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.808 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.255996+0000", 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018820299999999999 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.256184+0000", 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001140587 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.257324+0000", 2023-10-09T05:46:28.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099876349999999999 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.267312+0000", 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010821170000000001 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.268394+0000", 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012424846 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.810 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175787 7.15 7:aa4fca70:::benchmark_data_smithi008_354649_object81291:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.539968+0000", 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.21414208900000001, 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21496583899999999, 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.811 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175787 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.539968+0000", 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.539968+0000", 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:28.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.539966+0000", 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9359999999999994e-06 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.539976+0000", 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7081000000000001e-05 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.539993+0000", 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.199e-06 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.540000+0000", 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0003146 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.540315+0000", 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010585449999999999 2023-10-09T05:46:28.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.541374+0000", 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0072754550000000001 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.548649+0000", 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001134936 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.549784+0000", 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0098158829999999992 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:175839 7.5 7:a3a355f5:::benchmark_data_smithi008_354649_object81343:head [delete] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:28.742882+0000", 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.011227772, 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012131937000000001, 2023-10-09T05:46:28.816 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "delayed", 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 175839 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.742882+0000", 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:46:28.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.742882+0000", 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.742881+0000", 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9109999999999999e-06 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:46:28.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.742890+0000", 2023-10-09T05:46:28.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6458999999999999e-05 2023-10-09T05:46:28.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:46:28.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.742906+0000", 2023-10-09T05:46:28.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7390000000000002e-06 2023-10-09T05:46:28.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:46:28.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.742913+0000", 2023-10-09T05:46:28.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000336776 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.743250+0000", 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001220135 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.744470+0000", 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075869240000000001 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.752057+0000", 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010888510000000001 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:28.753145+0000", 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010263036999999999 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.821 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:46:28.822 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:46:28.822 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 12 2023-10-09T05:46:28.822 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:46:28.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:28.863 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:28.977 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:28.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:28.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:29.085 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:29.130 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:29.266 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:29.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:29.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:29.372 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:29.408 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:29.465 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:29.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:29.569 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:29.597 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:29.772 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:29.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:29.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:29.807 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:46:29.809 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:46:29.811 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:46:29.834 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:46:29.835 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:46:29.859 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:46:29.876 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:29.888 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:46:29.915 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:29.918 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:46:29.950 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:46:29.973 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:46:29.984 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:29.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:29.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:29.998 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:46:30.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:46:30.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:46:30.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:46:30.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:46:30.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:46:30.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:46:30.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:46:30.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:46:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:29 smithi191 ceph-mon[161669]: pgmap v807: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 915 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 303 op/s; 23360/43770 objects degraded (53.370%); 1790/43770 objects misplaced (4.090%); 4.1 MiB/s, 155 objects/s recovering 2023-10-09T05:46:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2431415475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:30.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:30.133 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:30.189 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:30.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:30.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:29 smithi008 ceph-mon[184418]: pgmap v807: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 915 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 303 op/s; 23360/43770 objects degraded (53.370%); 1790/43770 objects misplaced (4.090%); 4.1 MiB/s, 155 objects/s recovering 2023-10-09T05:46:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2431415475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:29 smithi008 ceph-mon[191227]: pgmap v807: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 915 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 303 op/s; 23360/43770 objects degraded (53.370%); 1790/43770 objects misplaced (4.090%); 4.1 MiB/s, 155 objects/s recovering 2023-10-09T05:46:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:46:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2431415475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:30.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:30.310 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:46:30.328 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:30.378 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:30.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:30.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:30.430 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:46:30.481 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:30.531 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:30.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:46:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:46:30] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.33.4" 2023-10-09T05:46:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:30 smithi191 ceph-mon[161669]: from='client.45668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:46:31.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:30 smithi008 ceph-mon[184418]: from='client.45668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:46:31.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:30 smithi008 ceph-mon[191227]: from='client.45668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:46:31.455 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":807,"stamp":"2023-10-09T05:46:28.311641+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":959059915,"num_objects":14590,"num_object_clones":0,"num_object_copies":43770,"num_objects_missing_on_primary":2929,"num_objects_missing":2929,"num_objects_degraded":23360,"num_objects_misplaced":1790,"num_objects_unfound":0,"num_objects_dirty":14590,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":175532,"num_write_kb":6062694,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24290,"num_bytes_recovered":1235102247,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":226837,"ondisk_log_size":226837,"up":297,"acting":288,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6384052,"kb_used_data":3630780,"kb_used_omap":38,"kb_used_meta":2753113,"kb_avail":649832012,"statfs":{"total":671965249536,"available":665427980288,"internally_reserved":0,"allocated":3717918720,"data_stored":3712734797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39147,"internal_metadata":2819188501},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,3,0,0,0,0,1,5,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-159252480,"num_objects":-2430,"num_object_clones":0,"num_object_copies":-7290,"num_objects_missing_on_primary":-112,"num_objects_missing":-112,"num_objects_degraded":-112,"num_objects_misplaced":-640,"num_objects_unfound":0,"num_objects_dirty":-2430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2430,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1246,"num_bytes_recovered":34275328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002752"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":256,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930424+0000","last_change":"2023-10-09T05:45:17.927530+0000","last_active":"2023-10-09T05:46:18.930424+0000","last_peered":"2023-10-09T05:46:18.930424+0000","last_clean":"2023-10-09T05:46:18.930424+0000","last_became_active":"2023-10-09T05:45:17.926963+0000","last_became_peered":"2023-10-09T05:45:17.926963+0000","last_unstale":"2023-10-09T05:46:18.930424+0000","last_undegraded":"2023-10-09T05:46:18.930424+0000","last_fullsized":"2023-10-09T05:46:18.930424+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":247,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.925694+0000","last_change":"2023-10-09T05:45:17.925436+0000","last_active":"2023-10-09T05:46:18.925694+0000","last_peered":"2023-10-09T05:46:18.925694+0000","last_clean":"2023-10-09T05:46:18.925694+0000","last_became_active":"2023-10-09T05:45:17.925064+0000","last_became_peered":"2023-10-09T05:45:17.925064+0000","last_unstale":"2023-10-09T05:46:18.925694+0000","last_undegraded":"2023-10-09T05:46:18.925694+0000","last_fullsized":"2023-10-09T05:46:18.925694+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:14.651551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":62,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.925713+0000","last_change":"2023-10-09T05:45:17.917244+0000","last_active":"2023-10-09T05:46:18.925713+0000","last_peered":"2023-10-09T05:46:18.925713+0000","last_clean":"2023-10-09T05:46:18.925713+0000","last_became_active":"2023-10-09T05:45:17.917012+0000","last_became_peered":"2023-10-09T05:45:17.917012+0000","last_unstale":"2023-10-09T05:46:18.925713+0000","last_undegraded":"2023-10-09T05:46:18.925713+0000","last_fullsized":"2023-10-09T05:46:18.925713+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:25.765620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":253,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930394+0000","last_change":"2023-10-09T05:43:19.889677+0000","last_active":"2023-10-09T05:46:18.930394+0000","last_peered":"2023-10-09T05:46:18.930394+0000","last_clean":"2023-10-09T05:46:18.930394+0000","last_became_active":"2023-10-09T05:43:19.889320+0000","last_became_peered":"2023-10-09T05:43:19.889320+0000","last_unstale":"2023-10-09T05:46:18.930394+0000","last_undegraded":"2023-10-09T05:46:18.930394+0000","last_fullsized":"2023-10-09T05:46:18.930394+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:57.376656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":264,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.925501+0000","last_change":"2023-10-09T05:45:17.937090+0000","last_active":"2023-10-09T05:46:18.925501+0000","last_peered":"2023-10-09T05:46:18.925501+0000","last_clean":"2023-10-09T05:46:18.925501+0000","last_became_active":"2023-10-09T05:45:17.936778+0000","last_became_peered":"2023-10-09T05:45:17.936778+0000","last_unstale":"2023-10-09T05:46:18.925501+0000","last_undegraded":"2023-10-09T05:46:18.925501+0000","last_fullsized":"2023-10-09T05:46:18.925501+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:12.815984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":89,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.936079+0000","last_change":"2023-10-09T05:45:17.925251+0000","last_active":"2023-10-09T05:46:18.936079+0000","last_peered":"2023-10-09T05:46:18.936079+0000","last_clean":"2023-10-09T05:46:18.936079+0000","last_became_active":"2023-10-09T05:45:17.924982+0000","last_became_peered":"2023-10-09T05:45:17.924982+0000","last_unstale":"2023-10-09T05:46:18.936079+0000","last_undegraded":"2023-10-09T05:46:18.936079+0000","last_fullsized":"2023-10-09T05:46:18.936079+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"265'9427","reported_seq":9498,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:26.800992+0000","last_change":"2023-10-09T05:44:04.042724+0000","last_active":"2023-10-09T05:46:26.800992+0000","last_peered":"2023-10-09T05:46:26.800992+0000","last_clean":"2023-10-09T05:46:26.800992+0000","last_became_active":"2023-10-09T05:44:04.029419+0000","last_became_peered":"2023-10-09T05:44:04.029419+0000","last_unstale":"2023-10-09T05:46:26.800992+0000","last_undegraded":"2023-10-09T05:46:26.800992+0000","last_fullsized":"2023-10-09T05:46:26.800992+0000","mapping_epoch":239,"log_start":"218'500","ondisk_log_start":"218'500","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8927,"log_dups_size":0,"ondisk_log_size":8927,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29261824,"num_objects":447,"num_object_clones":0,"num_object_copies":1341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5491,"num_write_kb":189984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":218,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934598+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:46:18.934598+0000","last_peered":"2023-10-09T05:46:18.934598+0000","last_clean":"2023-10-09T05:46:18.934598+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:46:18.934598+0000","last_undegraded":"2023-10-09T05:46:18.934598+0000","last_fullsized":"2023-10-09T05:46:18.934598+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":248,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.925587+0000","last_change":"2023-10-09T05:45:17.922085+0000","last_active":"2023-10-09T05:46:18.925587+0000","last_peered":"2023-10-09T05:46:18.925587+0000","last_clean":"2023-10-09T05:46:18.925587+0000","last_became_active":"2023-10-09T05:45:17.921640+0000","last_became_peered":"2023-10-09T05:45:17.921640+0000","last_unstale":"2023-10-09T05:46:18.925587+0000","last_undegraded":"2023-10-09T05:46:18.925587+0000","last_fullsized":"2023-10-09T05:46:18.925587+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:06.899516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"265'9600","reported_seq":10908,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:26.797207+0000","last_change":"2023-10-09T05:46:17.951765+0000","last_active":"2023-10-09T05:46:26.797207+0000","last_peered":"2023-10-09T05:46:26.797207+0000","last_clean":"2023-10-09T05:46:26.797207+0000","last_became_active":"2023-10-09T05:46:17.924568+0000","last_became_peered":"2023-10-09T05:46:17.924568+0000","last_unstale":"2023-10-09T05:46:26.797207+0000","last_undegraded":"2023-10-09T05:46:26.797207+0000","last_fullsized":"2023-10-09T05:46:26.797207+0000","mapping_epoch":262,"log_start":"218'700","ondisk_log_start":"218'700","created":239,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8900,"log_dups_size":0,"ondisk_log_size":8900,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32899072,"num_objects":502,"num_object_clones":0,"num_object_copies":1506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5562,"num_write_kb":194048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":608,"num_bytes_recovered":39845888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":274,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934941+0000","last_change":"2023-10-09T05:45:17.915552+0000","last_active":"2023-10-09T05:46:18.934941+0000","last_peered":"2023-10-09T05:46:18.934941+0000","last_clean":"2023-10-09T05:46:18.934941+0000","last_became_active":"2023-10-09T05:45:17.913215+0000","last_became_peered":"2023-10-09T05:45:17.913215+0000","last_unstale":"2023-10-09T05:46:18.934941+0000","last_undegraded":"2023-10-09T05:46:18.934941+0000","last_fullsized":"2023-10-09T05:46:18.934941+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":262,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.925623+0000","last_change":"2023-10-09T05:43:19.883115+0000","last_active":"2023-10-09T05:46:18.925623+0000","last_peered":"2023-10-09T05:46:18.925623+0000","last_clean":"2023-10-09T05:46:18.925623+0000","last_became_active":"2023-10-09T05:43:19.882674+0000","last_became_peered":"2023-10-09T05:43:19.882674+0000","last_unstale":"2023-10-09T05:46:18.925623+0000","last_undegraded":"2023-10-09T05:46:18.925623+0000","last_fullsized":"2023-10-09T05:46:18.925623+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:30.479481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"265'9300","reported_seq":9367,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:24.883808+0000","last_change":"2023-10-09T05:44:04.043289+0000","last_active":"2023-10-09T05:46:24.883808+0000","last_peered":"2023-10-09T05:46:24.883808+0000","last_clean":"2023-10-09T05:46:24.883808+0000","last_became_active":"2023-10-09T05:44:04.032393+0000","last_became_peered":"2023-10-09T05:44:04.032393+0000","last_unstale":"2023-10-09T05:46:24.883808+0000","last_undegraded":"2023-10-09T05:46:24.883808+0000","last_fullsized":"2023-10-09T05:46:24.883808+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9300,"log_dups_size":0,"ondisk_log_size":9300,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25100288,"num_objects":383,"num_object_clones":0,"num_object_copies":1149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5425,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":50,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.932207+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:46:18.932207+0000","last_peered":"2023-10-09T05:46:18.932207+0000","last_clean":"2023-10-09T05:46:18.932207+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:46:18.932207+0000","last_undegraded":"2023-10-09T05:46:18.932207+0000","last_fullsized":"2023-10-09T05:46:18.932207+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":58,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:20.961824+0000","last_change":"2023-10-09T05:45:17.924621+0000","last_active":"2023-10-09T05:46:20.961824+0000","last_peered":"2023-10-09T05:46:20.961824+0000","last_clean":"2023-10-09T05:46:20.961824+0000","last_became_active":"2023-10-09T05:45:17.924313+0000","last_became_peered":"2023-10-09T05:45:17.924313+0000","last_unstale":"2023-10-09T05:46:20.961824+0000","last_undegraded":"2023-10-09T05:46:20.961824+0000","last_fullsized":"2023-10-09T05:46:20.961824+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:37.602060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.10","version":"265'9269","reported_seq":11965,"reported_epoch":265,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:25.084293+0000","last_change":"2023-10-09T05:46:14.387921+0000","last_active":"2023-10-09T05:46:25.084293+0000","last_peered":"2023-10-09T05:46:25.084293+0000","last_clean":"2023-10-09T05:45:16.956686+0000","last_became_active":"2023-10-09T05:45:19.244940+0000","last_became_peered":"2023-10-09T05:45:19.244940+0000","last_unstale":"2023-10-09T05:46:25.084293+0000","last_undegraded":"2023-10-09T05:45:18.965723+0000","last_fullsized":"2023-10-09T05:45:18.943261+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9269,"log_dups_size":0,"ondisk_log_size":9269,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30670848,"num_objects":468,"num_object_clones":0,"num_object_copies":1404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":593,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5391,"num_write_kb":187488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1309,"num_bytes_recovered":38404096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"3,5","objects":-125}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":291,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.935344+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:46:18.935344+0000","last_peered":"2023-10-09T05:46:18.935344+0000","last_clean":"2023-10-09T05:46:18.935344+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:46:18.935344+0000","last_undegraded":"2023-10-09T05:46:18.935344+0000","last_fullsized":"2023-10-09T05:46:18.935344+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":239,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930759+0000","last_change":"2023-10-09T05:45:17.925631+0000","last_active":"2023-10-09T05:46:18.930759+0000","last_peered":"2023-10-09T05:46:18.930759+0000","last_clean":"2023-10-09T05:46:18.930759+0000","last_became_active":"2023-10-09T05:45:17.925142+0000","last_became_peered":"2023-10-09T05:45:17.925142+0000","last_unstale":"2023-10-09T05:46:18.930759+0000","last_undegraded":"2023-10-09T05:46:18.930759+0000","last_fullsized":"2023-10-09T05:46:18.930759+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"265'9267","reported_seq":9343,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:23.321442+0000","last_change":"2023-10-09T05:44:50.515353+0000","last_active":"2023-10-09T05:46:23.321442+0000","last_peered":"2023-10-09T05:46:23.321442+0000","last_clean":"2023-10-09T05:46:23.321442+0000","last_became_active":"2023-10-09T05:44:50.499263+0000","last_became_peered":"2023-10-09T05:44:50.499263+0000","last_unstale":"2023-10-09T05:46:23.321442+0000","last_undegraded":"2023-10-09T05:46:23.321442+0000","last_fullsized":"2023-10-09T05:46:23.321442+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9267,"log_dups_size":0,"ondisk_log_size":9267,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29720576,"num_objects":454,"num_object_clones":0,"num_object_copies":1362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5384,"num_write_kb":186784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":50,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.928176+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:46:18.928176+0000","last_peered":"2023-10-09T05:46:18.928176+0000","last_clean":"2023-10-09T05:46:18.928176+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:46:18.928176+0000","last_undegraded":"2023-10-09T05:46:18.928176+0000","last_fullsized":"2023-10-09T05:46:18.928176+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":281,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.925500+0000","last_change":"2023-10-09T05:45:17.920528+0000","last_active":"2023-10-09T05:46:18.925500+0000","last_peered":"2023-10-09T05:46:18.925500+0000","last_clean":"2023-10-09T05:46:18.925500+0000","last_became_active":"2023-10-09T05:45:17.920250+0000","last_became_peered":"2023-10-09T05:45:17.920250+0000","last_unstale":"2023-10-09T05:46:18.925500+0000","last_undegraded":"2023-10-09T05:46:18.925500+0000","last_fullsized":"2023-10-09T05:46:18.925500+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:08.059709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"265'9498","reported_seq":15593,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:23.003968+0000","last_change":"2023-10-09T05:45:38.011075+0000","last_active":"2023-10-09T05:46:23.003968+0000","last_peered":"2023-10-09T05:46:23.003968+0000","last_clean":"2023-10-09T05:46:23.003968+0000","last_became_active":"2023-10-09T05:44:04.113448+0000","last_became_peered":"2023-10-09T05:44:04.113448+0000","last_unstale":"2023-10-09T05:46:23.003968+0000","last_undegraded":"2023-10-09T05:46:23.003968+0000","last_fullsized":"2023-10-09T05:46:23.003968+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9498,"log_dups_size":0,"ondisk_log_size":9498,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29753344,"num_objects":454,"num_object_clones":0,"num_object_copies":1362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5521,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3006,"num_bytes_recovered":131137536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":252,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.928154+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:46:18.928154+0000","last_peered":"2023-10-09T05:46:18.928154+0000","last_clean":"2023-10-09T05:46:18.928154+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:46:18.928154+0000","last_undegraded":"2023-10-09T05:46:18.928154+0000","last_fullsized":"2023-10-09T05:46:18.928154+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":264,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934627+0000","last_change":"2023-10-09T05:43:19.886058+0000","last_active":"2023-10-09T05:46:18.934627+0000","last_peered":"2023-10-09T05:46:18.934627+0000","last_clean":"2023-10-09T05:46:18.934627+0000","last_became_active":"2023-10-09T05:43:19.885713+0000","last_became_peered":"2023-10-09T05:43:19.885713+0000","last_unstale":"2023-10-09T05:46:18.934627+0000","last_undegraded":"2023-10-09T05:46:18.934627+0000","last_fullsized":"2023-10-09T05:46:18.934627+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"265'9516","reported_seq":12627,"reported_epoch":265,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:46:25.074879+0000","last_change":"2023-10-09T05:45:17.932003+0000","last_active":"2023-10-09T05:46:25.074879+0000","last_peered":"2023-10-09T05:46:25.074879+0000","last_clean":"2023-10-09T05:43:18.738931+0000","last_became_active":"2023-10-09T05:44:05.020084+0000","last_became_peered":"2023-10-09T05:44:05.020084+0000","last_unstale":"2023-10-09T05:46:25.074879+0000","last_undegraded":"2023-10-09T05:45:17.927587+0000","last_fullsized":"2023-10-09T05:46:25.074879+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9516,"log_dups_size":0,"ondisk_log_size":9516,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30048256,"num_objects":459,"num_object_clones":0,"num_object_copies":1377,"num_objects_missing_on_primary":1427,"num_objects_missing":1427,"num_objects_degraded":1427,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5515,"num_write_kb":191136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1514,"num_bytes_recovered":99221504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":445},{"shards":"0,1,2,7","objects":6},{"shards":"0,2,7","objects":8}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":44,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.935715+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:46:18.935715+0000","last_peered":"2023-10-09T05:46:18.935715+0000","last_clean":"2023-10-09T05:46:18.935715+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:46:18.935715+0000","last_undegraded":"2023-10-09T05:46:18.935715+0000","last_fullsized":"2023-10-09T05:46:18.935715+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":251,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930414+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:46:18.930414+0000","last_peered":"2023-10-09T05:46:18.930414+0000","last_clean":"2023-10-09T05:46:18.930414+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:46:18.930414+0000","last_undegraded":"2023-10-09T05:46:18.930414+0000","last_fullsized":"2023-10-09T05:46:18.930414+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"265'9308","reported_seq":9388,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:23.796449+0000","last_change":"2023-10-09T05:45:19.213860+0000","last_active":"2023-10-09T05:46:23.796449+0000","last_peered":"2023-10-09T05:46:23.796449+0000","last_clean":"2023-10-09T05:45:16.956792+0000","last_became_active":"2023-10-09T05:45:19.185275+0000","last_became_peered":"2023-10-09T05:45:19.185275+0000","last_unstale":"2023-10-09T05:46:23.796449+0000","last_undegraded":"2023-10-09T05:45:18.955431+0000","last_fullsized":"2023-10-09T05:45:18.914423+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9308,"log_dups_size":0,"ondisk_log_size":9308,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32669696,"num_objects":499,"num_object_clones":0,"num_object_copies":1497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2959,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5386,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":500},{"shards":"0,4","objects":-689}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":39,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.927855+0000","last_change":"2023-10-09T05:44:50.483928+0000","last_active":"2023-10-09T05:46:18.927855+0000","last_peered":"2023-10-09T05:46:18.927855+0000","last_clean":"2023-10-09T05:46:18.927855+0000","last_became_active":"2023-10-09T05:44:50.483595+0000","last_became_peered":"2023-10-09T05:44:50.483595+0000","last_unstale":"2023-10-09T05:46:18.927855+0000","last_undegraded":"2023-10-09T05:46:18.927855+0000","last_fullsized":"2023-10-09T05:46:18.927855+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:21.100193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":281,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.926205+0000","last_change":"2023-10-09T05:45:17.919936+0000","last_active":"2023-10-09T05:46:18.926205+0000","last_peered":"2023-10-09T05:46:18.926205+0000","last_clean":"2023-10-09T05:46:18.926205+0000","last_became_active":"2023-10-09T05:45:17.919518+0000","last_became_peered":"2023-10-09T05:45:17.919518+0000","last_unstale":"2023-10-09T05:46:18.926205+0000","last_undegraded":"2023-10-09T05:46:18.926205+0000","last_fullsized":"2023-10-09T05:46:18.926205+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:24.340590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"265'10649","reported_seq":10717,"reported_epoch":265,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:46:25.047296+0000","last_change":"2023-10-09T05:45:19.152449+0000","last_active":"2023-10-09T05:46:25.047296+0000","last_peered":"2023-10-09T05:46:25.047296+0000","last_clean":"2023-10-09T05:45:16.917055+0000","last_became_active":"2023-10-09T05:45:19.101335+0000","last_became_peered":"2023-10-09T05:45:19.101335+0000","last_unstale":"2023-10-09T05:46:25.047296+0000","last_undegraded":"2023-10-09T05:46:25.047296+0000","last_fullsized":"2023-10-09T05:46:25.047296+0000","mapping_epoch":253,"log_start":"221'1951","ondisk_log_start":"221'1951","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8698,"log_dups_size":0,"ondisk_log_size":8698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55377920,"num_objects":845,"num_object_clones":0,"num_object_copies":2535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":845,"num_objects_unfound":0,"num_objects_dirty":845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10649,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":845}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":48,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.935392+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:46:18.935392+0000","last_peered":"2023-10-09T05:46:18.935392+0000","last_clean":"2023-10-09T05:46:18.935392+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:46:18.935392+0000","last_undegraded":"2023-10-09T05:46:18.935392+0000","last_fullsized":"2023-10-09T05:46:18.935392+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":249,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.937511+0000","last_change":"2023-10-09T05:43:19.885929+0000","last_active":"2023-10-09T05:46:18.937511+0000","last_peered":"2023-10-09T05:46:18.937511+0000","last_clean":"2023-10-09T05:46:18.937511+0000","last_became_active":"2023-10-09T05:43:19.885542+0000","last_became_peered":"2023-10-09T05:43:19.885542+0000","last_unstale":"2023-10-09T05:46:18.937511+0000","last_undegraded":"2023-10-09T05:46:18.937511+0000","last_fullsized":"2023-10-09T05:46:18.937511+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"265'10739","reported_seq":10801,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:26.751131+0000","last_change":"2023-10-09T05:41:29.081067+0000","last_active":"2023-10-09T05:46:26.751131+0000","last_peered":"2023-10-09T05:46:26.751131+0000","last_clean":"2023-10-09T05:46:26.751131+0000","last_became_active":"2023-10-09T05:41:29.080704+0000","last_became_peered":"2023-10-09T05:41:29.080704+0000","last_unstale":"2023-10-09T05:46:26.751131+0000","last_undegraded":"2023-10-09T05:46:26.751131+0000","last_fullsized":"2023-10-09T05:46:26.751131+0000","mapping_epoch":213,"log_start":"227'5294","ondisk_log_start":"227'5294","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5445,"log_dups_size":0,"ondisk_log_size":5445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:29.483205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58130432,"num_objects":887,"num_object_clones":0,"num_object_copies":2661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10739,"num_write_kb":372032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":61,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.929107+0000","last_change":"2023-10-09T05:45:18.061225+0000","last_active":"2023-10-09T05:46:18.929107+0000","last_peered":"2023-10-09T05:46:18.929107+0000","last_clean":"2023-10-09T05:46:18.929107+0000","last_became_active":"2023-10-09T05:45:17.923689+0000","last_became_peered":"2023-10-09T05:45:17.923689+0000","last_unstale":"2023-10-09T05:46:18.929107+0000","last_undegraded":"2023-10-09T05:46:18.929107+0000","last_fullsized":"2023-10-09T05:46:18.929107+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":263,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.931253+0000","last_change":"2023-10-09T05:45:17.919668+0000","last_active":"2023-10-09T05:46:18.931253+0000","last_peered":"2023-10-09T05:46:18.931253+0000","last_clean":"2023-10-09T05:46:18.931253+0000","last_became_active":"2023-10-09T05:45:17.919322+0000","last_became_peered":"2023-10-09T05:45:17.919322+0000","last_unstale":"2023-10-09T05:46:18.931253+0000","last_undegraded":"2023-10-09T05:46:18.931253+0000","last_fullsized":"2023-10-09T05:46:18.931253+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"265'9276","reported_seq":9360,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:23.502466+0000","last_change":"2023-10-09T05:45:19.120012+0000","last_active":"2023-10-09T05:46:23.502466+0000","last_peered":"2023-10-09T05:46:23.502466+0000","last_clean":"2023-10-09T05:45:16.881440+0000","last_became_active":"2023-10-09T05:45:19.096430+0000","last_became_peered":"2023-10-09T05:45:19.096430+0000","last_unstale":"2023-10-09T05:46:23.502466+0000","last_undegraded":"2023-10-09T05:45:18.961142+0000","last_fullsized":"2023-10-09T05:45:18.913777+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9276,"log_dups_size":0,"ondisk_log_size":9276,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30801920,"num_objects":470,"num_object_clones":0,"num_object_copies":1410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2949,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5392,"num_write_kb":187584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":510},{"shards":"0,4","objects":-784}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":48,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.932118+0000","last_change":"2023-10-09T05:43:20.148955+0000","last_active":"2023-10-09T05:46:18.932118+0000","last_peered":"2023-10-09T05:46:18.932118+0000","last_clean":"2023-10-09T05:46:18.932118+0000","last_became_active":"2023-10-09T05:43:19.884767+0000","last_became_peered":"2023-10-09T05:43:19.884767+0000","last_unstale":"2023-10-09T05:46:18.932118+0000","last_undegraded":"2023-10-09T05:46:18.932118+0000","last_fullsized":"2023-10-09T05:46:18.932118+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":255,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.928292+0000","last_change":"2023-10-09T05:45:17.923093+0000","last_active":"2023-10-09T05:46:18.928292+0000","last_peered":"2023-10-09T05:46:18.928292+0000","last_clean":"2023-10-09T05:46:18.928292+0000","last_became_active":"2023-10-09T05:45:17.922756+0000","last_became_peered":"2023-10-09T05:45:17.922756+0000","last_unstale":"2023-10-09T05:46:18.928292+0000","last_undegraded":"2023-10-09T05:46:18.928292+0000","last_fullsized":"2023-10-09T05:46:18.928292+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.8","version":"266'9437","reported_seq":16577,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T05:46:27.028623+0000","last_change":"2023-10-09T05:46:03.115826+0000","last_active":"2023-10-09T05:46:27.028623+0000","last_peered":"2023-10-09T05:46:27.028623+0000","last_clean":"2023-10-09T05:46:27.028623+0000","last_became_active":"2023-10-09T05:44:04.975737+0000","last_became_peered":"2023-10-09T05:44:04.975737+0000","last_unstale":"2023-10-09T05:46:27.028623+0000","last_undegraded":"2023-10-09T05:46:27.028623+0000","last_fullsized":"2023-10-09T05:46:27.028623+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9437,"log_dups_size":0,"ondisk_log_size":9437,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31719424,"num_objects":484,"num_object_clones":0,"num_object_copies":1452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5485,"num_write_kb":191008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2972,"num_bytes_recovered":166461440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":232,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934100+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:46:18.934100+0000","last_peered":"2023-10-09T05:46:18.934100+0000","last_clean":"2023-10-09T05:46:18.934100+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:46:18.934100+0000","last_undegraded":"2023-10-09T05:46:18.934100+0000","last_fullsized":"2023-10-09T05:46:18.934100+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":228,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.928364+0000","last_change":"2023-10-09T05:45:17.930153+0000","last_active":"2023-10-09T05:46:18.928364+0000","last_peered":"2023-10-09T05:46:18.928364+0000","last_clean":"2023-10-09T05:46:18.928364+0000","last_became_active":"2023-10-09T05:45:17.929831+0000","last_became_peered":"2023-10-09T05:45:17.929831+0000","last_unstale":"2023-10-09T05:46:18.928364+0000","last_undegraded":"2023-10-09T05:46:18.928364+0000","last_fullsized":"2023-10-09T05:46:18.928364+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:24.746849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"265'11030","reported_seq":12839,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:23.455263+0000","last_change":"2023-10-09T05:45:17.936204+0000","last_active":"2023-10-09T05:46:23.455263+0000","last_peered":"2023-10-09T05:46:23.455263+0000","last_clean":"2023-10-09T05:43:18.871019+0000","last_became_active":"2023-10-09T05:43:21.208517+0000","last_became_peered":"2023-10-09T05:43:21.208517+0000","last_unstale":"2023-10-09T05:46:23.455263+0000","last_undegraded":"2023-10-09T05:43:20.928538+0000","last_fullsized":"2023-10-09T05:43:20.877574+0000","mapping_epoch":231,"log_start":"226'4884","ondisk_log_start":"226'4884","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":6146,"log_dups_size":0,"ondisk_log_size":6146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:20.409497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61603840,"num_objects":940,"num_object_clones":0,"num_object_copies":2820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5117,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11030,"num_write_kb":383040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":868,"num_bytes_recovered":56885248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3","objects":-3712},{"shards":"0,3,4","objects":940}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":237,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.926113+0000","last_change":"2023-10-09T05:31:32.862044+0000","last_active":"2023-10-09T05:46:18.926113+0000","last_peered":"2023-10-09T05:46:18.926113+0000","last_clean":"2023-10-09T05:46:18.926113+0000","last_became_active":"2023-10-09T05:31:32.861706+0000","last_became_peered":"2023-10-09T05:31:32.861706+0000","last_unstale":"2023-10-09T05:46:18.926113+0000","last_undegraded":"2023-10-09T05:46:18.926113+0000","last_fullsized":"2023-10-09T05:46:18.926113+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:11.928581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":230,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.935253+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:46:18.935253+0000","last_peered":"2023-10-09T05:46:18.935253+0000","last_clean":"2023-10-09T05:46:18.935253+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:46:18.935253+0000","last_undegraded":"2023-10-09T05:46:18.935253+0000","last_fullsized":"2023-10-09T05:46:18.935253+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"266'11207","reported_seq":11271,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T05:46:26.984954+0000","last_change":"2023-10-09T05:41:29.078791+0000","last_active":"2023-10-09T05:46:26.984954+0000","last_peered":"2023-10-09T05:46:26.984954+0000","last_clean":"2023-10-09T05:46:26.984954+0000","last_became_active":"2023-10-09T05:41:29.078491+0000","last_became_peered":"2023-10-09T05:41:29.078491+0000","last_unstale":"2023-10-09T05:46:26.984954+0000","last_undegraded":"2023-10-09T05:46:26.984954+0000","last_fullsized":"2023-10-09T05:46:26.984954+0000","mapping_epoch":213,"log_start":"223'2848","ondisk_log_start":"223'2848","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8359,"log_dups_size":0,"ondisk_log_size":8359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:37.817282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57475072,"num_objects":877,"num_object_clones":0,"num_object_copies":2631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11207,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":90,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930372+0000","last_change":"2023-10-09T05:45:17.921547+0000","last_active":"2023-10-09T05:46:18.930372+0000","last_peered":"2023-10-09T05:46:18.930372+0000","last_clean":"2023-10-09T05:46:18.930372+0000","last_became_active":"2023-10-09T05:45:17.921221+0000","last_became_peered":"2023-10-09T05:45:17.921221+0000","last_unstale":"2023-10-09T05:46:18.930372+0000","last_undegraded":"2023-10-09T05:46:18.930372+0000","last_fullsized":"2023-10-09T05:46:18.930372+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":254,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934536+0000","last_change":"2023-10-09T05:45:17.923692+0000","last_active":"2023-10-09T05:46:18.934536+0000","last_peered":"2023-10-09T05:46:18.934536+0000","last_clean":"2023-10-09T05:46:18.934536+0000","last_became_active":"2023-10-09T05:45:17.917743+0000","last_became_peered":"2023-10-09T05:45:17.917743+0000","last_unstale":"2023-10-09T05:46:18.934536+0000","last_undegraded":"2023-10-09T05:46:18.934536+0000","last_fullsized":"2023-10-09T05:46:18.934536+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"265'10791","reported_seq":10867,"reported_epoch":265,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:46:23.736189+0000","last_change":"2023-10-09T05:45:19.165033+0000","last_active":"2023-10-09T05:46:23.736189+0000","last_peered":"2023-10-09T05:46:23.736189+0000","last_clean":"2023-10-09T05:45:16.912524+0000","last_became_active":"2023-10-09T05:45:19.102874+0000","last_became_peered":"2023-10-09T05:45:19.102874+0000","last_unstale":"2023-10-09T05:46:23.736189+0000","last_undegraded":"2023-10-09T05:46:23.736189+0000","last_fullsized":"2023-10-09T05:46:23.736189+0000","mapping_epoch":253,"log_start":"226'4585","ondisk_log_start":"226'4585","created":213,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":6206,"log_dups_size":0,"ondisk_log_size":6206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:25.137080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61866008,"num_objects":945,"num_object_clones":0,"num_object_copies":2835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":945,"num_objects_unfound":0,"num_objects_dirty":945,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10791,"num_write_kb":375489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":945}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":264,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930358+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:46:18.930358+0000","last_peered":"2023-10-09T05:46:18.930358+0000","last_clean":"2023-10-09T05:46:18.930358+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:46:18.930358+0000","last_undegraded":"2023-10-09T05:46:18.930358+0000","last_fullsized":"2023-10-09T05:46:18.930358+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":248,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934941+0000","last_change":"2023-10-09T05:45:17.922009+0000","last_active":"2023-10-09T05:46:18.934941+0000","last_peered":"2023-10-09T05:46:18.934941+0000","last_clean":"2023-10-09T05:46:18.934941+0000","last_became_active":"2023-10-09T05:45:17.921636+0000","last_became_peered":"2023-10-09T05:45:17.921636+0000","last_unstale":"2023-10-09T05:46:18.934941+0000","last_undegraded":"2023-10-09T05:46:18.934941+0000","last_fullsized":"2023-10-09T05:46:18.934941+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"265'10861","reported_seq":22683,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:26.838952+0000","last_change":"2023-10-09T05:46:04.564946+0000","last_active":"2023-10-09T05:46:26.838952+0000","last_peered":"2023-10-09T05:46:26.838952+0000","last_clean":"2023-10-09T05:46:26.838952+0000","last_became_active":"2023-10-09T05:44:51.482290+0000","last_became_peered":"2023-10-09T05:44:51.482290+0000","last_unstale":"2023-10-09T05:46:26.838952+0000","last_undegraded":"2023-10-09T05:46:26.838952+0000","last_fullsized":"2023-10-09T05:46:26.838952+0000","mapping_epoch":248,"log_start":"228'5493","ondisk_log_start":"228'5493","created":213,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":5368,"log_dups_size":0,"ondisk_log_size":5368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:27.252412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57737216,"num_objects":881,"num_object_clones":0,"num_object_copies":2643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10861,"num_write_kb":375744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5870,"num_bytes_recovered":260702208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":282,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.925413+0000","last_change":"2023-10-09T05:45:17.922614+0000","last_active":"2023-10-09T05:46:18.925413+0000","last_peered":"2023-10-09T05:46:18.925413+0000","last_clean":"2023-10-09T05:46:18.925413+0000","last_became_active":"2023-10-09T05:45:17.922292+0000","last_became_peered":"2023-10-09T05:45:17.922292+0000","last_unstale":"2023-10-09T05:46:18.925413+0000","last_undegraded":"2023-10-09T05:46:18.925413+0000","last_fullsized":"2023-10-09T05:46:18.925413+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:33.044557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":62,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.926026+0000","last_change":"2023-10-09T05:45:17.921701+0000","last_active":"2023-10-09T05:46:18.926026+0000","last_peered":"2023-10-09T05:46:18.926026+0000","last_clean":"2023-10-09T05:46:18.926026+0000","last_became_active":"2023-10-09T05:45:17.921374+0000","last_became_peered":"2023-10-09T05:45:17.921374+0000","last_unstale":"2023-10-09T05:46:18.926026+0000","last_undegraded":"2023-10-09T05:46:18.926026+0000","last_fullsized":"2023-10-09T05:46:18.926026+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:06.490584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"265'9212","reported_seq":9292,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:24.826487+0000","last_change":"2023-10-09T05:46:20.505813+0000","last_active":"2023-10-09T05:46:24.826487+0000","last_peered":"2023-10-09T05:46:24.826487+0000","last_clean":"2023-10-09T05:46:24.826487+0000","last_became_active":"2023-10-09T05:44:03.989206+0000","last_became_peered":"2023-10-09T05:44:03.989206+0000","last_unstale":"2023-10-09T05:46:24.826487+0000","last_undegraded":"2023-10-09T05:46:24.826487+0000","last_fullsized":"2023-10-09T05:46:24.826487+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9212,"log_dups_size":0,"ondisk_log_size":9212,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30867456,"num_objects":471,"num_object_clones":0,"num_object_copies":1413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5337,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":590,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:18.934695+0000","last_change":"2023-10-09T05:45:18.999640+0000","last_active":"2023-10-09T05:46:18.934695+0000","last_peered":"2023-10-09T05:46:18.934695+0000","last_clean":"2023-10-09T05:22:40.584723+0000","last_became_active":"2023-10-09T05:45:18.999259+0000","last_became_peered":"2023-10-09T05:45:18.999259+0000","last_unstale":"2023-10-09T05:46:18.934695+0000","last_undegraded":"2023-10-09T05:45:18.956731+0000","last_fullsized":"2023-10-09T05:45:18.956171+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:23.344079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":247,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934539+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:46:18.934539+0000","last_peered":"2023-10-09T05:46:18.934539+0000","last_clean":"2023-10-09T05:46:18.934539+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:46:18.934539+0000","last_undegraded":"2023-10-09T05:46:18.934539+0000","last_fullsized":"2023-10-09T05:46:18.934539+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":37,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934336+0000","last_change":"2023-10-09T05:45:17.926027+0000","last_active":"2023-10-09T05:46:18.934336+0000","last_peered":"2023-10-09T05:46:18.934336+0000","last_clean":"2023-10-09T05:46:18.934336+0000","last_became_active":"2023-10-09T05:45:17.925620+0000","last_became_peered":"2023-10-09T05:45:17.925620+0000","last_unstale":"2023-10-09T05:46:18.934336+0000","last_undegraded":"2023-10-09T05:46:18.934336+0000","last_fullsized":"2023-10-09T05:46:18.934336+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"265'9262","reported_seq":9339,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:23.207913+0000","last_change":"2023-10-09T05:44:50.502440+0000","last_active":"2023-10-09T05:46:23.207913+0000","last_peered":"2023-10-09T05:46:23.207913+0000","last_clean":"2023-10-09T05:46:23.207913+0000","last_became_active":"2023-10-09T05:44:50.481365+0000","last_became_peered":"2023-10-09T05:44:50.481365+0000","last_unstale":"2023-10-09T05:46:23.207913+0000","last_undegraded":"2023-10-09T05:46:23.207913+0000","last_fullsized":"2023-10-09T05:46:23.207913+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9262,"log_dups_size":0,"ondisk_log_size":9262,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30048256,"num_objects":458,"num_object_clones":0,"num_object_copies":1374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5378,"num_write_kb":186784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":728,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934510+0000","last_change":"2023-10-09T05:46:04.573476+0000","last_active":"2023-10-09T05:46:18.934510+0000","last_peered":"2023-10-09T05:46:18.934510+0000","last_clean":"2023-10-09T05:46:18.934510+0000","last_became_active":"2023-10-09T05:45:17.925130+0000","last_became_peered":"2023-10-09T05:45:17.925130+0000","last_unstale":"2023-10-09T05:46:18.934510+0000","last_undegraded":"2023-10-09T05:46:18.934510+0000","last_fullsized":"2023-10-09T05:46:18.934510+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":231,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930884+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:46:18.930884+0000","last_peered":"2023-10-09T05:46:18.930884+0000","last_clean":"2023-10-09T05:46:18.930884+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:46:18.930884+0000","last_undegraded":"2023-10-09T05:46:18.930884+0000","last_fullsized":"2023-10-09T05:46:18.930884+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":99,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:20.963374+0000","last_change":"2023-10-09T05:45:17.919819+0000","last_active":"2023-10-09T05:46:20.963374+0000","last_peered":"2023-10-09T05:46:20.963374+0000","last_clean":"2023-10-09T05:46:20.963374+0000","last_became_active":"2023-10-09T05:45:17.919447+0000","last_became_peered":"2023-10-09T05:45:17.919447+0000","last_unstale":"2023-10-09T05:46:20.963374+0000","last_undegraded":"2023-10-09T05:46:20.963374+0000","last_fullsized":"2023-10-09T05:46:20.963374+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:18.280449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"265'9320","reported_seq":9400,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:23.317946+0000","last_change":"2023-10-09T05:45:19.212798+0000","last_active":"2023-10-09T05:46:23.317946+0000","last_peered":"2023-10-09T05:46:23.317946+0000","last_clean":"2023-10-09T05:45:16.697360+0000","last_became_active":"2023-10-09T05:45:19.185185+0000","last_became_peered":"2023-10-09T05:45:19.185185+0000","last_unstale":"2023-10-09T05:46:23.317946+0000","last_undegraded":"2023-10-09T05:45:18.969077+0000","last_fullsized":"2023-10-09T05:45:18.916434+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9320,"log_dups_size":0,"ondisk_log_size":9320,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31883264,"num_objects":486,"num_object_clones":0,"num_object_copies":1458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2925,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5397,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":485},{"shards":"0,4","objects":-726}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":583,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:20.961403+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:46:20.961403+0000","last_peered":"2023-10-09T05:46:20.961403+0000","last_clean":"2023-10-09T05:46:20.961403+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:46:20.961403+0000","last_undegraded":"2023-10-09T05:46:20.961403+0000","last_fullsized":"2023-10-09T05:46:20.961403+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":58,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:20.962464+0000","last_change":"2023-10-09T05:45:17.921491+0000","last_active":"2023-10-09T05:46:20.962464+0000","last_peered":"2023-10-09T05:46:20.962464+0000","last_clean":"2023-10-09T05:46:20.962464+0000","last_became_active":"2023-10-09T05:45:17.921145+0000","last_became_peered":"2023-10-09T05:45:17.921145+0000","last_unstale":"2023-10-09T05:46:20.962464+0000","last_undegraded":"2023-10-09T05:46:20.962464+0000","last_fullsized":"2023-10-09T05:46:20.962464+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:45.903609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":82,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.936002+0000","last_change":"2023-10-09T05:45:17.921021+0000","last_active":"2023-10-09T05:46:18.936002+0000","last_peered":"2023-10-09T05:46:18.936002+0000","last_clean":"2023-10-09T05:46:18.936002+0000","last_became_active":"2023-10-09T05:45:17.920667+0000","last_became_peered":"2023-10-09T05:45:17.920667+0000","last_unstale":"2023-10-09T05:46:18.936002+0000","last_undegraded":"2023-10-09T05:46:18.936002+0000","last_fullsized":"2023-10-09T05:46:18.936002+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"265'9468","reported_seq":15492,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:23.111788+0000","last_change":"2023-10-09T05:46:14.387758+0000","last_active":"2023-10-09T05:46:23.111788+0000","last_peered":"2023-10-09T05:46:23.111788+0000","last_clean":"2023-10-09T05:46:23.111788+0000","last_became_active":"2023-10-09T05:44:05.023661+0000","last_became_peered":"2023-10-09T05:44:05.023661+0000","last_unstale":"2023-10-09T05:46:23.111788+0000","last_undegraded":"2023-10-09T05:46:23.111788+0000","last_fullsized":"2023-10-09T05:46:23.111788+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9468,"log_dups_size":0,"ondisk_log_size":9468,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31719424,"num_objects":484,"num_object_clones":0,"num_object_copies":1452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5491,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2969,"num_bytes_recovered":130744320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":911,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934978+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:46:18.934978+0000","last_peered":"2023-10-09T05:46:18.934978+0000","last_clean":"2023-10-09T05:46:18.934978+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:46:18.934978+0000","last_undegraded":"2023-10-09T05:46:18.934978+0000","last_fullsized":"2023-10-09T05:46:18.934978+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":415,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.928257+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:46:18.928257+0000","last_peered":"2023-10-09T05:46:18.928257+0000","last_clean":"2023-10-09T05:46:18.928257+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:46:18.928257+0000","last_undegraded":"2023-10-09T05:46:18.928257+0000","last_fullsized":"2023-10-09T05:46:18.928257+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":231,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.925479+0000","last_change":"2023-10-09T05:31:32.864478+0000","last_active":"2023-10-09T05:46:18.925479+0000","last_peered":"2023-10-09T05:46:18.925479+0000","last_clean":"2023-10-09T05:46:18.925479+0000","last_became_active":"2023-10-09T05:31:32.864212+0000","last_became_peered":"2023-10-09T05:31:32.864212+0000","last_unstale":"2023-10-09T05:46:18.925479+0000","last_undegraded":"2023-10-09T05:46:18.925479+0000","last_fullsized":"2023-10-09T05:46:18.925479+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:15.876295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":232,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934786+0000","last_change":"2023-10-09T05:44:50.495966+0000","last_active":"2023-10-09T05:46:18.934786+0000","last_peered":"2023-10-09T05:46:18.934786+0000","last_clean":"2023-10-09T05:46:18.934786+0000","last_became_active":"2023-10-09T05:44:50.495628+0000","last_became_peered":"2023-10-09T05:44:50.495628+0000","last_unstale":"2023-10-09T05:46:18.934786+0000","last_undegraded":"2023-10-09T05:46:18.934786+0000","last_fullsized":"2023-10-09T05:46:18.934786+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"265'9521","reported_seq":12636,"reported_epoch":265,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:46:25.002750+0000","last_change":"2023-10-09T05:44:07.019753+0000","last_active":"2023-10-09T05:46:25.002750+0000","last_peered":"2023-10-09T05:46:25.002750+0000","last_clean":"2023-10-09T05:43:18.738931+0000","last_became_active":"2023-10-09T05:44:06.984836+0000","last_became_peered":"2023-10-09T05:44:06.984836+0000","last_unstale":"2023-10-09T05:46:25.002750+0000","last_undegraded":"2023-10-09T05:44:04.978402+0000","last_fullsized":"2023-10-09T05:46:25.002750+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9521,"log_dups_size":0,"ondisk_log_size":9521,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29720576,"num_objects":453,"num_object_clones":0,"num_object_copies":1359,"num_objects_missing_on_primary":1502,"num_objects_missing":1502,"num_objects_degraded":1502,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5519,"num_write_kb":191136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1518,"num_bytes_recovered":99483648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":454},{"shards":"0,1,2,7","objects":6},{"shards":"0,2,7","objects":-7}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":717,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:18.928422+0000","last_change":"2023-10-09T05:45:18.933861+0000","last_active":"2023-10-09T05:46:18.928422+0000","last_peered":"2023-10-09T05:46:18.928422+0000","last_clean":"2023-10-09T05:45:10.937727+0000","last_became_active":"2023-10-09T05:45:18.933370+0000","last_became_peered":"2023-10-09T05:45:18.933370+0000","last_unstale":"2023-10-09T05:46:18.928422+0000","last_undegraded":"2023-10-09T05:45:18.913825+0000","last_fullsized":"2023-10-09T05:45:18.913230+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":219,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930178+0000","last_change":"2023-10-09T05:31:32.861444+0000","last_active":"2023-10-09T05:46:18.930178+0000","last_peered":"2023-10-09T05:46:18.930178+0000","last_clean":"2023-10-09T05:46:18.930178+0000","last_became_active":"2023-10-09T05:31:32.861155+0000","last_became_peered":"2023-10-09T05:31:32.861155+0000","last_unstale":"2023-10-09T05:46:18.930178+0000","last_undegraded":"2023-10-09T05:46:18.930178+0000","last_fullsized":"2023-10-09T05:46:18.930178+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:56.326468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":243,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.928813+0000","last_change":"2023-10-09T05:45:17.931710+0000","last_active":"2023-10-09T05:46:18.928813+0000","last_peered":"2023-10-09T05:46:18.928813+0000","last_clean":"2023-10-09T05:46:18.928813+0000","last_became_active":"2023-10-09T05:45:17.931290+0000","last_became_peered":"2023-10-09T05:45:17.931290+0000","last_unstale":"2023-10-09T05:46:18.928813+0000","last_undegraded":"2023-10-09T05:46:18.928813+0000","last_fullsized":"2023-10-09T05:46:18.928813+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:50.091125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"265'9284","reported_seq":9388,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:24.960630+0000","last_change":"2023-10-09T05:45:19.272383+0000","last_active":"2023-10-09T05:46:24.960630+0000","last_peered":"2023-10-09T05:46:24.960630+0000","last_clean":"2023-10-09T05:45:16.901376+0000","last_became_active":"2023-10-09T05:45:19.243948+0000","last_became_peered":"2023-10-09T05:45:19.243948+0000","last_unstale":"2023-10-09T05:46:24.960630+0000","last_undegraded":"2023-10-09T05:45:18.942714+0000","last_fullsized":"2023-10-09T05:45:18.916529+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9284,"log_dups_size":0,"ondisk_log_size":9284,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29687808,"num_objects":453,"num_object_clones":0,"num_object_copies":1359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2953,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5406,"num_write_kb":187488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":464},{"shards":"3,5","objects":-737}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":448,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.932292+0000","last_change":"2023-10-09T05:44:51.489799+0000","last_active":"2023-10-09T05:46:18.932292+0000","last_peered":"2023-10-09T05:46:18.932292+0000","last_clean":"2023-10-09T05:46:18.932292+0000","last_became_active":"2023-10-09T05:44:51.489301+0000","last_became_peered":"2023-10-09T05:44:51.489301+0000","last_unstale":"2023-10-09T05:46:18.932292+0000","last_undegraded":"2023-10-09T05:46:18.932292+0000","last_fullsized":"2023-10-09T05:46:18.932292+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":58,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:20.963439+0000","last_change":"2023-10-09T05:45:17.926001+0000","last_active":"2023-10-09T05:46:20.963439+0000","last_peered":"2023-10-09T05:46:20.963439+0000","last_clean":"2023-10-09T05:46:20.963439+0000","last_became_active":"2023-10-09T05:45:17.925682+0000","last_became_peered":"2023-10-09T05:45:17.925682+0000","last_unstale":"2023-10-09T05:46:20.963439+0000","last_undegraded":"2023-10-09T05:46:20.963439+0000","last_fullsized":"2023-10-09T05:46:20.963439+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:25.779220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":237,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.928595+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:46:18.928595+0000","last_peered":"2023-10-09T05:46:18.928595+0000","last_clean":"2023-10-09T05:46:18.928595+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:46:18.928595+0000","last_undegraded":"2023-10-09T05:46:18.928595+0000","last_fullsized":"2023-10-09T05:46:18.928595+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"265'9431","reported_seq":9502,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:26.678691+0000","last_change":"2023-10-09T05:44:04.001523+0000","last_active":"2023-10-09T05:46:26.678691+0000","last_peered":"2023-10-09T05:46:26.678691+0000","last_clean":"2023-10-09T05:46:26.678691+0000","last_became_active":"2023-10-09T05:44:03.990530+0000","last_became_peered":"2023-10-09T05:44:03.990530+0000","last_unstale":"2023-10-09T05:46:26.678691+0000","last_undegraded":"2023-10-09T05:46:26.678691+0000","last_fullsized":"2023-10-09T05:46:26.678691+0000","mapping_epoch":213,"log_start":"218'500","ondisk_log_start":"218'500","created":213,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":8931,"log_dups_size":0,"ondisk_log_size":8931,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28999680,"num_objects":442,"num_object_clones":0,"num_object_copies":1326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5494,"num_write_kb":189984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":278,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.926114+0000","last_change":"2023-10-09T05:43:20.063203+0000","last_active":"2023-10-09T05:46:18.926114+0000","last_peered":"2023-10-09T05:46:18.926114+0000","last_clean":"2023-10-09T05:46:18.926114+0000","last_became_active":"2023-10-09T05:43:19.885257+0000","last_became_peered":"2023-10-09T05:43:19.885257+0000","last_unstale":"2023-10-09T05:46:18.926114+0000","last_undegraded":"2023-10-09T05:46:18.926114+0000","last_fullsized":"2023-10-09T05:46:18.926114+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:36.424359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":510,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.935007+0000","last_change":"2023-10-09T05:46:06.414199+0000","last_active":"2023-10-09T05:46:18.935007+0000","last_peered":"2023-10-09T05:46:18.935007+0000","last_clean":"2023-10-09T05:46:18.935007+0000","last_became_active":"2023-10-09T05:46:06.413755+0000","last_became_peered":"2023-10-09T05:46:06.413755+0000","last_unstale":"2023-10-09T05:46:18.935007+0000","last_undegraded":"2023-10-09T05:46:18.935007+0000","last_fullsized":"2023-10-09T05:46:18.935007+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":268,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.925664+0000","last_change":"2023-10-09T05:43:19.886969+0000","last_active":"2023-10-09T05:46:18.925664+0000","last_peered":"2023-10-09T05:46:18.925664+0000","last_clean":"2023-10-09T05:46:18.925664+0000","last_became_active":"2023-10-09T05:43:19.885111+0000","last_became_peered":"2023-10-09T05:43:19.885111+0000","last_unstale":"2023-10-09T05:46:18.925664+0000","last_undegraded":"2023-10-09T05:46:18.925664+0000","last_fullsized":"2023-10-09T05:46:18.925664+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:26.890192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"265'9710","reported_seq":10913,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:26.728951+0000","last_change":"2023-10-09T05:46:18.956964+0000","last_active":"2023-10-09T05:46:26.728951+0000","last_peered":"2023-10-09T05:46:26.728951+0000","last_clean":"2023-10-09T05:46:26.728951+0000","last_became_active":"2023-10-09T05:46:18.930931+0000","last_became_peered":"2023-10-09T05:46:18.930931+0000","last_unstale":"2023-10-09T05:46:26.728951+0000","last_undegraded":"2023-10-09T05:46:26.728951+0000","last_fullsized":"2023-10-09T05:46:26.728951+0000","mapping_epoch":263,"log_start":"218'700","ondisk_log_start":"218'700","created":213,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9010,"log_dups_size":0,"ondisk_log_size":9010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25690112,"num_objects":392,"num_object_clones":0,"num_object_copies":1176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5672,"num_write_kb":194048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":554,"num_bytes_recovered":36306944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":762,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.935529+0000","last_change":"2023-10-09T05:44:50.482211+0000","last_active":"2023-10-09T05:46:18.935529+0000","last_peered":"2023-10-09T05:46:18.935529+0000","last_clean":"2023-10-09T05:46:18.935529+0000","last_became_active":"2023-10-09T05:44:50.481749+0000","last_became_peered":"2023-10-09T05:44:50.481749+0000","last_unstale":"2023-10-09T05:46:18.935529+0000","last_undegraded":"2023-10-09T05:46:18.935529+0000","last_fullsized":"2023-10-09T05:46:18.935529+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":235,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930308+0000","last_change":"2023-10-09T05:45:17.975802+0000","last_active":"2023-10-09T05:46:18.930308+0000","last_peered":"2023-10-09T05:46:18.930308+0000","last_clean":"2023-10-09T05:46:18.930308+0000","last_became_active":"2023-10-09T05:45:17.931496+0000","last_became_peered":"2023-10-09T05:45:17.931496+0000","last_unstale":"2023-10-09T05:46:18.930308+0000","last_undegraded":"2023-10-09T05:46:18.930308+0000","last_fullsized":"2023-10-09T05:46:18.930308+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:51.419126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":215,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930496+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:46:18.930496+0000","last_peered":"2023-10-09T05:46:18.930496+0000","last_clean":"2023-10-09T05:46:18.930496+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:46:18.930496+0000","last_undegraded":"2023-10-09T05:46:18.930496+0000","last_fullsized":"2023-10-09T05:46:18.930496+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":239,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.928516+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:46:18.928516+0000","last_peered":"2023-10-09T05:46:18.928516+0000","last_clean":"2023-10-09T05:46:18.928516+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:46:18.928516+0000","last_undegraded":"2023-10-09T05:46:18.928516+0000","last_fullsized":"2023-10-09T05:46:18.928516+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":278,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-10-09T05:46:20.963464+0000","last_change":"2023-10-09T05:45:17.924914+0000","last_active":"2023-10-09T05:46:20.963464+0000","last_peered":"2023-10-09T05:46:20.963464+0000","last_clean":"2023-10-09T05:46:20.963464+0000","last_became_active":"2023-10-09T05:45:17.924406+0000","last_became_peered":"2023-10-09T05:45:17.924406+0000","last_unstale":"2023-10-09T05:46:20.963464+0000","last_undegraded":"2023-10-09T05:46:20.963464+0000","last_fullsized":"2023-10-09T05:46:20.963464+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:35.316832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"266'9511","reported_seq":16698,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T05:46:27.063689+0000","last_change":"2023-10-09T05:45:35.805746+0000","last_active":"2023-10-09T05:46:27.063689+0000","last_peered":"2023-10-09T05:46:27.063689+0000","last_clean":"2023-10-09T05:46:27.063689+0000","last_became_active":"2023-10-09T05:44:04.104935+0000","last_became_peered":"2023-10-09T05:44:04.104935+0000","last_unstale":"2023-10-09T05:46:27.063689+0000","last_undegraded":"2023-10-09T05:46:27.063689+0000","last_fullsized":"2023-10-09T05:46:27.063689+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9511,"log_dups_size":0,"ondisk_log_size":9511,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26869760,"num_objects":410,"num_object_clones":0,"num_object_copies":1230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5559,"num_write_kb":191008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2997,"num_bytes_recovered":170524672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":247,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934014+0000","last_change":"2023-10-09T05:45:17.971642+0000","last_active":"2023-10-09T05:46:18.934014+0000","last_peered":"2023-10-09T05:46:18.934014+0000","last_clean":"2023-10-09T05:46:18.934014+0000","last_became_active":"2023-10-09T05:45:17.917031+0000","last_became_peered":"2023-10-09T05:45:17.917031+0000","last_unstale":"2023-10-09T05:46:18.934014+0000","last_undegraded":"2023-10-09T05:46:18.934014+0000","last_fullsized":"2023-10-09T05:46:18.934014+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":254,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930951+0000","last_change":"2023-10-09T05:45:17.925441+0000","last_active":"2023-10-09T05:46:18.930951+0000","last_peered":"2023-10-09T05:46:18.930951+0000","last_clean":"2023-10-09T05:46:18.930951+0000","last_became_active":"2023-10-09T05:45:17.924653+0000","last_became_peered":"2023-10-09T05:45:17.924653+0000","last_unstale":"2023-10-09T05:46:18.930951+0000","last_undegraded":"2023-10-09T05:46:18.930951+0000","last_fullsized":"2023-10-09T05:46:18.930951+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"265'9228","reported_seq":9306,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:46:23.740759+0000","last_change":"2023-10-09T05:45:19.119065+0000","last_active":"2023-10-09T05:46:23.740759+0000","last_peered":"2023-10-09T05:46:23.740759+0000","last_clean":"2023-10-09T05:45:16.938440+0000","last_became_active":"2023-10-09T05:45:19.095658+0000","last_became_peered":"2023-10-09T05:45:19.095658+0000","last_unstale":"2023-10-09T05:46:23.740759+0000","last_undegraded":"2023-10-09T05:45:18.969330+0000","last_fullsized":"2023-10-09T05:45:18.914061+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:41:28.058397+0000","last_clean_scrub_stamp":"2023-10-09T05:41:28.058397+0000","objects_scrubbed":0,"log_size":9228,"log_dups_size":0,"ondisk_log_size":9228,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33947648,"num_objects":518,"num_object_clones":0,"num_object_copies":1554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2913,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5344,"num_write_kb":187584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":478},{"shards":"0,4","objects":-653}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":219,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.933997+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:46:18.933997+0000","last_peered":"2023-10-09T05:46:18.933997+0000","last_clean":"2023-10-09T05:46:18.933997+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:46:18.933997+0000","last_undegraded":"2023-10-09T05:46:18.933997+0000","last_fullsized":"2023-10-09T05:46:18.933997+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":245,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930317+0000","last_change":"2023-10-09T05:31:32.867349+0000","last_active":"2023-10-09T05:46:18.930317+0000","last_peered":"2023-10-09T05:46:18.930317+0000","last_clean":"2023-10-09T05:46:18.930317+0000","last_became_active":"2023-10-09T05:31:32.867102+0000","last_became_peered":"2023-10-09T05:31:32.867102+0000","last_unstale":"2023-10-09T05:46:18.930317+0000","last_undegraded":"2023-10-09T05:46:18.930317+0000","last_fullsized":"2023-10-09T05:46:18.930317+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":246,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.926082+0000","last_change":"2023-10-09T05:31:32.864546+0000","last_active":"2023-10-09T05:46:18.926082+0000","last_peered":"2023-10-09T05:46:18.926082+0000","last_clean":"2023-10-09T05:46:18.926082+0000","last_became_active":"2023-10-09T05:31:32.864235+0000","last_became_peered":"2023-10-09T05:31:32.864235+0000","last_unstale":"2023-10-09T05:46:18.926082+0000","last_undegraded":"2023-10-09T05:46:18.926082+0000","last_fullsized":"2023-10-09T05:46:18.926082+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:33.984282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":258,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.925541+0000","last_change":"2023-10-09T05:31:32.860364+0000","last_active":"2023-10-09T05:46:18.925541+0000","last_peered":"2023-10-09T05:46:18.925541+0000","last_clean":"2023-10-09T05:46:18.925541+0000","last_became_active":"2023-10-09T05:31:32.860014+0000","last_became_peered":"2023-10-09T05:31:32.860014+0000","last_unstale":"2023-10-09T05:46:18.925541+0000","last_undegraded":"2023-10-09T05:46:18.925541+0000","last_fullsized":"2023-10-09T05:46:18.925541+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:03.968474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":222,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.934690+0000","last_change":"2023-10-09T05:31:02.550858+0000","last_active":"2023-10-09T05:46:18.934690+0000","last_peered":"2023-10-09T05:46:18.934690+0000","last_clean":"2023-10-09T05:46:18.934690+0000","last_became_active":"2023-10-09T05:31:02.550500+0000","last_became_peered":"2023-10-09T05:31:02.550500+0000","last_unstale":"2023-10-09T05:46:18.934690+0000","last_undegraded":"2023-10-09T05:46:18.934690+0000","last_fullsized":"2023-10-09T05:46:18.934690+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:19.062170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":239,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-09T05:46:18.930230+0000","last_change":"2023-10-09T05:45:17.920323+0000","last_active":"2023-10-09T05:46:18.930230+0000","last_peered":"2023-10-09T05:46:18.930230+0000","last_clean":"2023-10-09T05:46:18.930230+0000","last_became_active":"2023-10-09T05:45:17.919994+0000","last_became_peered":"2023-10-09T05:45:17.919994+0000","last_unstale":"2023-10-09T05:46:18.930230+0000","last_undegraded":"2023-10-09T05:46:18.930230+0000","last_fullsized":"2023-10-09T05:46:18.930230+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":954269720,"num_objects":14562,"num_object_clones":0,"num_object_copies":43686,"num_objects_missing_on_primary":2929,"num_objects_missing":2929,"num_objects_degraded":23338,"num_objects_misplaced":1790,"num_objects_unfound":0,"num_objects_dirty":14562,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":174426,"num_write_kb":6047553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24186,"num_bytes_recovered":1229717504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4651954176,"data_stored":4651941960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":225667,"ondisk_log_size":225667,"up":78,"acting":71,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846734,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":566560,"kb_used_data":244056,"kb_used_omap":14,"kb_used_meta":322481,"kb_avail":93178592,"statfs":{"total":95995035648,"available":95414878208,"internally_reserved":0,"allocated":249913344,"data_stored":249333056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":330221402},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,0,0,0,0,1,5,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":11.387,"5min":5.1150000000000002,"15min":5.5229999999999997},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":110.123,"5min":113.018,"15min":220.77500000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":11.412000000000001,"5min":5.0330000000000004,"15min":5.9400000000000004},"min":{"1min":0.27100000000000002,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":110.13800000000001,"5min":113.005,"15min":220.78700000000001},"last":0.40699999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":3.3250000000000002,"15min":2.9740000000000002},"min":{"1min":0.27000000000000002,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":0.72399999999999998,"5min":83.575000000000003,"15min":190.68100000000001},"last":3.7269999999999999},{"interface":"front","average":{"1min":0.70699999999999996,"5min":3.331,"15min":2.8370000000000002},"min":{"1min":0.26100000000000001,"5min":0.255,"15min":0.255},"max":{"1min":4.5510000000000002,"5min":87.516999999999996,"15min":191.09200000000001},"last":3.5590000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":1.28,"15min":3.8210000000000002},"min":{"1min":0.29599999999999999,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":2.2240000000000002,"5min":7.9349999999999996,"15min":78.716999999999999},"last":3.8450000000000002},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.369,"15min":3.7749999999999999},"min":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.27000000000000002},"max":{"1min":2.3079999999999998,"5min":7.391,"15min":78.768000000000001},"last":1.552}]},{"osd":3,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":2.577,"5min":4.2539999999999996,"15min":3.4430000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":36.576000000000001,"5min":108.90300000000001,"15min":185.17099999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.5670000000000002,"5min":4.2400000000000002,"15min":3.8159999999999998},"min":{"1min":0.33000000000000002,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":36.624000000000002,"5min":109.08499999999999,"15min":183.35499999999999},"last":4.2530000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.64600000000000002,"15min":1.1180000000000001},"min":{"1min":0.29199999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.0630000000000002,"5min":4.3620000000000001,"15min":12.789},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.63,"15min":1.2569999999999999},"min":{"1min":0.312,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":2.5680000000000001,"5min":4.3419999999999996,"15min":9.0470000000000006},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.77900000000000003,"15min":1.2809999999999999},"min":{"1min":0.33100000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.2569999999999997,"5min":4.4429999999999996,"15min":10.813000000000001},"last":0.503},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.74199999999999999,"15min":1.512},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.28299999999999997},"max":{"1min":0.748,"5min":3.9500000000000002,"15min":10.321},"last":0.68500000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.65400000000000003,"15min":1.274},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.318},"max":{"1min":1.3280000000000001,"5min":4.468,"15min":9.2530000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.79100000000000004,"15min":1.2290000000000001},"min":{"1min":0.38800000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.91800000000000004,"5min":4.2279999999999998,"15min":8.2680000000000007},"last":0.46400000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626760,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":593176,"kb_used_data":287760,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93151976,"statfs":{"total":95995035648,"available":95387623424,"internally_reserved":0,"allocated":294666240,"data_stored":294093776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:34 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":4.649,"15min":4.4909999999999997},"min":{"1min":0.23400000000000001,"5min":0.22800000000000001,"15min":0.214},"max":{"1min":2.5,"5min":162.08799999999999,"15min":162.08799999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.86899999999999999,"5min":4.4669999999999996,"15min":4.2949999999999999},"min":{"1min":0.25800000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.4009999999999998,"5min":161.90899999999999,"15min":161.90899999999999},"last":0.34000000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":3.9470000000000001,"15min":3.8479999999999999},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.25},"max":{"1min":3.8279999999999998,"5min":228.24299999999999,"15min":228.24299999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.76500000000000001,"5min":3.8180000000000001,"15min":3.839},"min":{"1min":0.318,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":3.8199999999999998,"5min":228.22499999999999,"15min":228.22499999999999},"last":0.45600000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:45:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":4.3339999999999996,"15min":4.093},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.3029999999999999,"5min":87.703000000000003,"15min":115.40600000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.64800000000000002,"5min":4.4980000000000002,"15min":4.1420000000000003},"min":{"1min":0.308,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":2.9049999999999998,"5min":87.689999999999998,"15min":115.297},"last":0.35099999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":3.3420000000000001,"15min":3.8809999999999998},"min":{"1min":0.371,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":4.0019999999999998,"5min":77.578999999999994,"15min":115.244},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":3.4529999999999998,"15min":3.927},"min":{"1min":0.36499999999999999,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":4.0179999999999998,"5min":77.516999999999996,"15min":115.34999999999999},"last":0.48499999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53600000000000003,"15min":1.7350000000000001},"min":{"1min":0.33800000000000002,"5min":0.25,"15min":0.23400000000000001},"max":{"1min":1.345,"5min":2.73,"15min":125.64},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.873,"5min":0.98399999999999999,"15min":1.778},"min":{"1min":0.31900000000000001,"5min":0.26000000000000001,"15min":0.254},"max":{"1min":2.8140000000000001,"5min":13.609,"15min":125.625},"last":0.377}]},{"osd":5,"last update":"Mon Oct 9 05:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.79900000000000004,"15min":1.9490000000000001},"min":{"1min":0.35999999999999999,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":3.1899999999999999,"5min":6.141,"15min":127.35899999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.97599999999999998,"15min":1.887},"min":{"1min":0.34699999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.2389999999999999,"5min":14.250999999999999,"15min":128.93799999999999},"last":0.52900000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.754,"15min":1.9930000000000001},"min":{"1min":0.40799999999999997,"5min":0.32500000000000001,"15min":0.26500000000000001},"max":{"1min":0.97599999999999998,"5min":7.319,"15min":115.514},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.0760000000000001,"15min":1.6639999999999999},"min":{"1min":0.41199999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.032,"5min":14.300000000000001,"15min":116.53100000000001},"last":0.55100000000000005}]}]},{"osd":1,"up_from":0,"seq":399431958721,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":88,"seq":377957122247,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":979768,"kb_used_data":494832,"kb_used_omap":8,"kb_used_meta":484919,"kb_avail":92765384,"statfs":{"total":95995035648,"available":94991753216,"internally_reserved":0,"allocated":506707968,"data_stored":505755848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":496557832},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:45:56 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":1.5189999999999999,"15min":1.657},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":1.4670000000000001,"5min":14.198,"15min":17.760000000000002},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.747,"5min":1.9059999999999999,"15min":2.1429999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":5.8780000000000001,"5min":13.816000000000001,"15min":26.163},"last":0.29899999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":3.5910000000000002,"15min":3.278},"min":{"1min":0.24199999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.5349999999999999,"5min":92.230000000000004,"15min":92.230000000000004},"last":0.249},{"interface":"front","average":{"1min":0.48999999999999999,"5min":3.4940000000000002,"15min":3.1440000000000001},"min":{"1min":0.26800000000000002,"5min":0.222,"15min":0.218},"max":{"1min":1.4279999999999999,"5min":92.247,"15min":92.247},"last":0.437}]},{"osd":3,"last update":"Mon Oct 9 05:45:56 2023","interfaces":[{"interface":"back","average":{"1min":3.177,"5min":3.4689999999999999,"15min":3.79},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":55.328000000000003,"5min":79.159000000000006,"15min":123.024},"last":0.34399999999999997},{"interface":"front","average":{"1min":3.3759999999999999,"5min":3.9500000000000002,"15min":3.6629999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.222},"max":{"1min":55.292000000000002,"5min":79.353999999999999,"15min":99.162000000000006},"last":0.33000000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.2649999999999999,"15min":1.399},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.381,"5min":11.35,"15min":69.082999999999998},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.825,"15min":1.8819999999999999},"min":{"1min":0.36299999999999999,"5min":0.33500000000000002,"15min":0.30399999999999999},"max":{"1min":1.905,"5min":7.9240000000000004,"15min":69.102000000000004},"last":0.41099999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":2.024,"15min":3.1840000000000002},"min":{"1min":0.372,"5min":0.34799999999999998,"15min":0.30199999999999999},"max":{"1min":3.5089999999999999,"5min":9.6219999999999999,"15min":281.733},"last":0.38},{"interface":"front","average":{"1min":0.80000000000000004,"5min":2.0779999999999998,"15min":3.2040000000000002},"min":{"1min":0.38,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.0369999999999999,"5min":12.552,"15min":282.95400000000001},"last":0.45900000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:45:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":2.0529999999999999,"15min":1.7769999999999999},"min":{"1min":0.41399999999999998,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":4.5010000000000003,"5min":7.9969999999999999,"15min":11.664},"last":0.627},{"interface":"front","average":{"1min":1.2050000000000001,"5min":2.1200000000000001,"15min":1.867},"min":{"1min":0.38600000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.4390000000000001,"5min":12.587999999999999,"15min":12.587999999999999},"last":0.56100000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:45:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.2150000000000001,"15min":1.407},"min":{"1min":0.36199999999999999,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":1.5649999999999999,"5min":11.631,"15min":11.631},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.482,"15min":2.1400000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.9850000000000001,"5min":7.2190000000000003,"15min":11.068},"last":0.51000000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795195,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1215956,"kb_used_data":678412,"kb_used_omap":11,"kb_used_meta":537524,"kb_avail":92529196,"statfs":{"total":95995035648,"available":94749896704,"internally_reserved":0,"allocated":694693888,"data_stored":693663913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":550425173},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":13,"5min":4.9630000000000001,"15min":3.0659999999999998},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":162.285,"5min":162.285,"15min":162.285},"last":0.36899999999999999},{"interface":"front","average":{"1min":13.404,"5min":5.9039999999999999,"15min":3.359},"min":{"1min":0.24399999999999999,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":162.643,"5min":162.643,"15min":162.643},"last":1.829}]},{"osd":1,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":2.8039999999999998,"15min":3.1179999999999999},"min":{"1min":0.24099999999999999,"5min":0.23100000000000001,"15min":0.20699999999999999},"max":{"1min":2.4289999999999998,"5min":24.265000000000001,"15min":98.575000000000003},"last":1.8049999999999999},{"interface":"front","average":{"1min":0.52800000000000002,"5min":2.0640000000000001,"15min":2.5640000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":2.0859999999999999,"5min":18.882999999999999,"15min":98.397000000000006},"last":0.27400000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":7.077,"15min":4.8840000000000003},"min":{"1min":0.26900000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":2.1099999999999999,"5min":159.35499999999999,"15min":159.35499999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.63700000000000001,"5min":5.7949999999999999,"15min":4.8140000000000001},"min":{"1min":0.31,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.4409999999999998,"5min":159.196,"15min":159.196},"last":0.52900000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.74,"15min":1.903},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27700000000000002},"max":{"1min":2.161,"5min":14.382,"15min":115.84999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":1.2609999999999999,"15min":2.056},"min":{"1min":0.309,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.974,"5min":13.815,"15min":116.149},"last":0.58599999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.986,"15min":1.972},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.312},"max":{"1min":2.4260000000000002,"5min":13.712999999999999,"15min":24.597999999999999},"last":2.0649999999999999},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.706,"15min":1.6579999999999999},"min":{"1min":0.38400000000000001,"5min":0.35499999999999998,"15min":0.26000000000000001},"max":{"1min":2.2890000000000001,"5min":14.426,"15min":23.919},"last":0.39100000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:45:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.5229999999999999,"15min":1.804},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":14.317,"5min":14.317,"15min":20.503},"last":0.629},{"interface":"front","average":{"1min":1.6299999999999999,"5min":2.1850000000000001,"15min":1.901},"min":{"1min":0.38900000000000001,"5min":0.36799999999999999,"15min":0.34499999999999997},"max":{"1min":14.436999999999999,"5min":14.436999999999999,"15min":14.436999999999999},"last":2.0019999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.6799999999999999,"15min":2.4849999999999999},"min":{"1min":0.45000000000000001,"5min":0.37,"15min":0.37},"max":{"1min":2.4319999999999999,"5min":51.701999999999998,"15min":51.701999999999998},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.77400000000000002,"5min":2.2400000000000002,"15min":2.3599999999999999},"min":{"1min":0.39200000000000002,"5min":0.373,"15min":0.373},"max":{"1min":2.3119999999999998,"5min":51.716000000000001,"15min":51.716000000000001},"last":2.0870000000000002}]}]},{"osd":3,"up_from":103,"seq":442381631669,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195088,"kb_used_data":686728,"kb_used_omap":4,"kb_used_meta":508347,"kb_avail":92550064,"statfs":{"total":95995035648,"available":94771265536,"internally_reserved":0,"allocated":703209472,"data_stored":702293984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":520547934},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":2.8980000000000001,"5min":4.3289999999999997,"15min":3.6699999999999999},"min":{"1min":0.21199999999999999,"5min":0.184,"15min":0.183},"max":{"1min":46.411000000000001,"5min":149.565,"15min":167.82400000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":2.8100000000000001,"5min":5.024,"15min":4.2320000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.20899999999999999},"max":{"1min":46.387999999999998,"5min":149.291,"15min":167.89599999999999},"last":0.28100000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":3.0329999999999999,"15min":2.9020000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22600000000000001},"max":{"1min":4.2960000000000003,"5min":149.32499999999999,"15min":149.32499999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.034,"5min":3.3839999999999999,"15min":3.004},"min":{"1min":0.28000000000000003,"5min":0.19,"15min":0.19},"max":{"1min":4.4450000000000003,"5min":149.34299999999999,"15min":149.34299999999999},"last":0.89700000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":3.2949999999999999,"15min":4.5839999999999996},"min":{"1min":0.32200000000000001,"5min":0.27000000000000002,"15min":0.215},"max":{"1min":3.0920000000000001,"5min":150.006,"15min":190.43299999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.80600000000000005,"5min":3.1930000000000001,"15min":4.9429999999999996},"min":{"1min":0.27200000000000002,"5min":0.214,"15min":0.214},"max":{"1min":2.6120000000000001,"5min":150.65100000000001,"15min":190.267},"last":0.437}]},{"osd":4,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":2.6819999999999999,"15min":2.661},"min":{"1min":0.32400000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.4729999999999999,"5min":149.84700000000001,"15min":149.84700000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.626,"5min":2.5710000000000002,"15min":2.1680000000000001},"min":{"1min":0.378,"5min":0.315,"15min":0.27400000000000002},"max":{"1min":1.869,"5min":149.74700000000001,"15min":149.74700000000001},"last":0.59299999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":2.867,"15min":2.3300000000000001},"min":{"1min":0.35599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.3840000000000003,"5min":149.43000000000001,"15min":149.43000000000001},"last":1.002},{"interface":"front","average":{"1min":0.66200000000000003,"5min":2.8279999999999998,"15min":2.3119999999999998},"min":{"1min":0.42699999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.897,"5min":149.69300000000001,"15min":149.69300000000001},"last":0.53900000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":2.7330000000000001,"15min":2.1909999999999998},"min":{"1min":0.36099999999999999,"5min":0.33900000000000002,"15min":0.33500000000000002},"max":{"1min":3.3620000000000001,"5min":149.72399999999999,"15min":149.72399999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.0289999999999999,"5min":3.0630000000000002,"15min":2.2280000000000002},"min":{"1min":0.40300000000000002,"5min":0.307,"15min":0.307},"max":{"1min":4.4870000000000001,"5min":149.589,"15min":149.589},"last":1.028}]},{"osd":7,"last update":"Mon Oct 9 05:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":2.6760000000000002,"15min":2.653},"min":{"1min":0.39400000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.398,"5min":149.77199999999999,"15min":149.77199999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.3620000000000001,"5min":2.7930000000000001,"15min":2.6269999999999998},"min":{"1min":0.42599999999999999,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":10.619,"5min":149.80799999999999,"15min":149.80799999999999},"last":0.60499999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019216,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":952536,"kb_used_data":655312,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92792616,"statfs":{"total":95995035648,"available":95019638784,"internally_reserved":0,"allocated":671039488,"data_stored":670466778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":3.2160000000000002,"15min":2.2160000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.214},"max":{"1min":26.478000000000002,"5min":100.504,"15min":100.504},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.9350000000000001,"5min":3.4239999999999999,"15min":2.5369999999999999},"min":{"1min":0.247,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":26.454000000000001,"5min":100.717,"15min":100.717},"last":0.27000000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":4.2539999999999996,"15min":2.4489999999999998},"min":{"1min":0.28699999999999998,"5min":0.26200000000000001,"15min":0.24299999999999999},"max":{"1min":2.944,"5min":187.369,"15min":187.369},"last":5.806},{"interface":"front","average":{"1min":0.752,"5min":4.2590000000000003,"15min":2.5630000000000002},"min":{"1min":0.31900000000000001,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":2.4830000000000001,"5min":187.35300000000001,"15min":187.35300000000001},"last":6.2859999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:45:23 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":3.6829999999999998,"15min":4.5309999999999997},"min":{"1min":0.27800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.7129999999999992,"5min":100.148,"15min":196.93700000000001},"last":1.7769999999999999},{"interface":"front","average":{"1min":1.4079999999999999,"5min":3.964,"15min":4.4939999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.28100000000000003},"max":{"1min":9.7219999999999995,"5min":100.086,"15min":197.08099999999999},"last":1.53}]},{"osd":3,"last update":"Mon Oct 9 05:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":3.2349999999999999,"15min":3.9889999999999999},"min":{"1min":0.34399999999999997,"5min":0.33800000000000002,"15min":0.30199999999999999},"max":{"1min":3.1709999999999998,"5min":72.391999999999996,"15min":211.48699999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.51600000000000001,"5min":3.2080000000000002,"15min":3.839},"min":{"1min":0.312,"5min":0.312,"15min":0.309},"max":{"1min":1.1819999999999999,"5min":71.721999999999994,"15min":211.31100000000001},"last":1.867}]},{"osd":5,"last update":"Mon Oct 9 05:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.67800000000000005,"15min":1.6359999999999999},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.968,"5min":3.6739999999999999,"15min":113.35899999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.60199999999999998,"15min":1.5369999999999999},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.30599999999999999},"max":{"1min":1.788,"5min":3.4209999999999998,"15min":113.32299999999999},"last":0.53400000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56999999999999995,"15min":0.73199999999999998},"min":{"1min":0.29699999999999999,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":2.089,"5min":3.3969999999999998,"15min":11.944000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.84599999999999997,"15min":0.97799999999999998},"min":{"1min":0.40300000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.4000000000000004,"5min":8.5269999999999992,"15min":10.356},"last":0.54800000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.59299999999999997,"15min":1.637},"min":{"1min":0.34300000000000003,"5min":0.33300000000000002,"15min":0.312},"max":{"1min":1.008,"5min":2.7709999999999999,"15min":30.393999999999998},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.72699999999999998,"15min":1.8440000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":2.9820000000000002,"5min":5.7160000000000002,"15min":30.334},"last":0.51800000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822988,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":880968,"kb_used_data":583680,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92864184,"statfs":{"total":95995035648,"available":95092924416,"internally_reserved":0,"allocated":597688320,"data_stored":597127442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.611,"15min":3.29},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.19400000000000001},"max":{"1min":9.9049999999999994,"5min":12.904999999999999,"15min":165.90199999999999},"last":0.26900000000000002},{"interface":"front","average":{"1min":1.111,"5min":1.782,"15min":3.399},"min":{"1min":0.28299999999999997,"5min":0.27000000000000002,"15min":0.215},"max":{"1min":10.210000000000001,"5min":17.077000000000002,"15min":165.96700000000001},"last":0.254}]},{"osd":1,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.8300000000000001,"15min":2.5950000000000002},"min":{"1min":0.27700000000000002,"5min":0.26500000000000001,"15min":0.23999999999999999},"max":{"1min":3.2879999999999998,"5min":54.898000000000003,"15min":87.769000000000005},"last":3.5870000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.7290000000000001,"15min":2.6720000000000002},"min":{"1min":0.316,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":1.7989999999999999,"5min":52.512,"15min":87.864000000000004},"last":0.31}]},{"osd":2,"last update":"Mon Oct 9 05:45:36 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":1.8480000000000001,"15min":4.4329999999999998},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.9510000000000001,"5min":31.114000000000001,"15min":290.697},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.83499999999999996,"5min":2.1629999999999998,"15min":4.3129999999999997},"min":{"1min":0.35599999999999998,"5min":0.32700000000000001,"15min":0.29899999999999999},"max":{"1min":2.9239999999999999,"5min":33.875999999999998,"15min":290.786},"last":0.45300000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":2.3820000000000001,"15min":3.488},"min":{"1min":0.39100000000000001,"5min":0.30399999999999999,"15min":0.28799999999999998},"max":{"1min":10.640000000000001,"5min":36.395000000000003,"15min":95.236999999999995},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.3,"5min":2.2909999999999999,"15min":3.2869999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.29699999999999999},"max":{"1min":9.9600000000000009,"5min":36.406999999999996,"15min":93.471000000000004},"last":0.34799999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.64500000000000002,"15min":0.77900000000000003},"min":{"1min":0.29599999999999999,"5min":0.28499999999999998,"15min":0.23300000000000001},"max":{"1min":1.6180000000000001,"5min":6.2610000000000001,"15min":9.3510000000000009},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.72199999999999998,"15min":1.01},"min":{"1min":0.34300000000000003,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":1.726,"5min":9.282,"15min":14.236000000000001},"last":0.46500000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.78000000000000003,"15min":1.4399999999999999},"min":{"1min":0.32300000000000001,"5min":0.316,"15min":0.26600000000000001},"max":{"1min":1.9670000000000001,"5min":9.2959999999999994,"15min":95.010999999999996},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.626,"5min":0.79900000000000004,"15min":1.3460000000000001},"min":{"1min":0.36799999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.3879999999999999,"5min":4.5410000000000004,"15min":94.194000000000003},"last":0.35599999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.89000000000000001,"15min":1.2809999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.1699999999999999,"5min":9.1460000000000008,"15min":9.9849999999999994},"last":3.2719999999999998},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.877,"15min":1.3859999999999999},"min":{"1min":0.33600000000000002,"5min":0.308,"15min":0.308},"max":{"1min":2.2189999999999999,"5min":9.3100000000000005,"15min":14.432},"last":0.41099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":497160192,"data_stored":497156120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":993595392,"data_stored":993591320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":686948352,"data_stored":686948352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":692322304,"data_stored":692322304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":663289856,"data_stored":663289856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593756160,"data_stored":593756160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":283447296,"data_stored":283443224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":241434624,"data_stored":241434624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:46:31.462 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:46:31.462 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_2 pg_num 36 2023-10-09T05:46:31.505 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:31.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:31.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:31.616 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:31.655 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:31.714 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:31.857 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:31.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:31.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:31.916 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:31 smithi191 ceph-mon[161669]: pgmap v808: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 838 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 23250/40077 objects degraded (58.013%); 1538/40077 objects misplaced (3.838%); 4.2 MiB/s, 223 objects/s recovering 2023-10-09T05:46:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:31 smithi008 ceph-mon[184418]: pgmap v808: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 838 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 23250/40077 objects degraded (58.013%); 1538/40077 objects misplaced (3.838%); 4.2 MiB/s, 223 objects/s recovering 2023-10-09T05:46:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:31 smithi008 ceph-mon[191227]: pgmap v808: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 838 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 23250/40077 objects degraded (58.013%); 1538/40077 objects misplaced (3.838%); 4.2 MiB/s, 223 objects/s recovering 2023-10-09T05:46:31.967 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:32.008 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:32.087 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:32.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:32.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:32.152 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:46:32.152 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:46:32.195 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:32.233 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:32.400 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:32.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:32.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:32.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:32.537 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:32.580 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:32.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:32.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:32.683 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:32.724 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:33.010 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:33.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:33.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:33.020 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:46:33.080 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:33.117 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:33.155 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:33.208 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:33.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:33.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:33.314 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:33.349 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:33.445 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:33.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:33.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:33.459 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:46:33.498 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:46:33.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:33.585 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:33.689 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:33.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:33.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:33.794 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:33.838 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:34.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:33 smithi008 ceph-mon[184418]: pgmap v809: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 787 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 378 op/s; 23250/37623 objects degraded (61.797%); 1538/37623 objects misplaced (4.088%); 1.4 MiB/s, 168 objects/s recovering 2023-10-09T05:46:34.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23250/37623 objects degraded (61.797%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:33 smithi008 ceph-mon[191227]: pgmap v809: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 787 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 378 op/s; 23250/37623 objects degraded (61.797%); 1538/37623 objects misplaced (4.088%); 1.4 MiB/s, 168 objects/s recovering 2023-10-09T05:46:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23250/37623 objects degraded (61.797%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:33 smithi191 ceph-mon[161669]: pgmap v809: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 787 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 378 op/s; 23250/37623 objects degraded (61.797%); 1538/37623 objects misplaced (4.088%); 1.4 MiB/s, 168 objects/s recovering 2023-10-09T05:46:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23250/37623 objects degraded (61.797%), 11 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:34.716 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:34.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:34.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:34.800 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:34.821 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:34.852 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:34.929 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:34.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:34.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:34.943 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:46:34.981 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1023042806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:34.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:34 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:34.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1023042806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:34.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:34 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:35.035 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:35.070 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:35.126 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:35.128 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:46:35.128 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:46:35.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:35.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:35.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:35.256 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1023042806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:35.256 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:34 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:46:35.259 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:35.359 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:35.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:35.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:35.463 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:35.490 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:35.532 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:35.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:35.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:35.744 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:35.770 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:35.884 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:46:35.897 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:35.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:35.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:35.962 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 7 pg_num to 36 2023-10-09T05:46:36.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:36.047 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:36.117 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:36.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:36.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:36.133 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:35 smithi191 ceph-mon[161669]: pgmap v810: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 744 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 23250/35568 objects degraded (65.368%); 1439/35568 objects misplaced (4.046%); 1.4 MiB/s, 168 objects/s recovering 2023-10-09T05:46:36.133 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:35 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:36.133 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:35 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:46:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:35 smithi191 ceph-mon[161669]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T05:46:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:35 smithi191 ceph-mon[161669]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:35 smithi191 ceph-mon[161669]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:35 smithi191 ceph-mon[161669]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:35 smithi191 ceph-mon[161669]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.135 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:35 smithi191 ceph-mon[161669]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.135 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/75006332' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:46:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[184418]: pgmap v810: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 744 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 23250/35568 objects degraded (65.368%); 1439/35568 objects misplaced (4.046%); 1.4 MiB/s, 168 objects/s recovering 2023-10-09T05:46:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:46:36.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[184418]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T05:46:36.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[184418]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[184418]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[184418]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[184418]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[184418]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/75006332' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:46:36.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[191227]: pgmap v810: 99 pgs: 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 744 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 23250/35568 objects degraded (65.368%); 1439/35568 objects misplaced (4.046%); 1.4 MiB/s, 168 objects/s recovering 2023-10-09T05:46:36.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:46:36.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:46:36.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[191227]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T05:46:36.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[191227]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[191227]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[191227]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[191227]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[191227]: failed to encode map e267 with expected crc 2023-10-09T05:46:36.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/75006332' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:46:36.234 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:36.266 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:36.317 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:36.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:36.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:36.460 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:36.511 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:36.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:36.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:36.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:36.649 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:36.706 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:36.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:36.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:36.815 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:36.850 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:36.917 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:36.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:36.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:37.024 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:36 smithi191 ceph-mon[161669]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:36 smithi191 ceph-mon[161669]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/75006332' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:46:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:36 smithi191 ceph-mon[161669]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T05:46:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:36 smithi191 ceph-mon[161669]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:36 smithi191 ceph-mon[161669]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.054 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[184418]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[184418]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/75006332' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:46:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[184418]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T05:46:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[184418]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[184418]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[191227]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[191227]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/75006332' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:46:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[191227]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T05:46:37.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[191227]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:36 smithi008 ceph-mon[191227]: failed to encode map e268 with expected crc 2023-10-09T05:46:37.643 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:37.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:37.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:37.747 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:37.776 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:37.825 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:37.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:37.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:37.932 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:37.979 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:38.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:37 smithi191 ceph-mon[161669]: pgmap v813: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 86 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 716 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 21160/34224 objects degraded (61.828%); 1680/34224 objects misplaced (4.909%); 1.7 MiB/s, 227 objects/s recovering 2023-10-09T05:46:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:37 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 21160/34224 objects degraded (61.828%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:38.066 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:38.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:37 smithi008 ceph-mon[184418]: pgmap v813: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 86 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 716 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 21160/34224 objects degraded (61.828%); 1680/34224 objects misplaced (4.909%); 1.7 MiB/s, 227 objects/s recovering 2023-10-09T05:46:38.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:37 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 21160/34224 objects degraded (61.828%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:38.073 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:37 smithi008 ceph-mon[191227]: pgmap v813: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 86 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 716 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 397 op/s; 21160/34224 objects degraded (61.828%); 1680/34224 objects misplaced (4.909%); 1.7 MiB/s, 227 objects/s recovering 2023-10-09T05:46:38.073 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:37 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 21160/34224 objects degraded (61.828%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:38.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:38.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:38.174 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:38.220 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:38.284 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:38.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:38.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:38.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:38.422 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:38.485 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:38.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:38.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:38.591 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:38.629 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:38.706 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:38.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:38.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:38.816 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:38.872 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:38.952 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:38.953 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:38.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:38.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:39.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:38 smithi191 ceph-mon[161669]: 7.1 scrub starts 2023-10-09T05:46:39.066 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:39.103 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:39.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:38 smithi008 ceph-mon[184418]: 7.1 scrub starts 2023-10-09T05:46:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:38 smithi008 ceph-mon[191227]: 7.1 scrub starts 2023-10-09T05:46:39.276 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:39.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:39.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:39.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:39.407 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:39.467 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:39.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:39.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:39.498 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:46:39.577 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:39.606 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:39.660 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:39.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:39.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:39.765 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:39.832 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:40.112 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:46:40.112 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:46:40.112 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:46:40.112 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:46:40.112 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:46:40.113 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:46:40.113 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:46:40.121 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:46:40.122 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:46:40.123 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:46:40.123 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:46:40.123 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:46:40.123 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:46:40.123 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:46:40.123 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:46:40.123 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:46:40.146 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:39 smithi008 ceph-mon[184418]: pgmap v814: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 86 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 675 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 21160/32253 objects degraded (65.606%); 1680/32253 objects misplaced (5.209%); 912 KiB/s, 122 objects/s recovering 2023-10-09T05:46:40.146 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:39 smithi008 ceph-mon[184418]: 7.1 scrub starts 2023-10-09T05:46:40.146 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2954762597' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:40.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:39 smithi008 ceph-mon[191227]: pgmap v814: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 86 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 675 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 21160/32253 objects degraded (65.606%); 1680/32253 objects misplaced (5.209%); 912 KiB/s, 122 objects/s recovering 2023-10-09T05:46:40.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:39 smithi008 ceph-mon[191227]: 7.1 scrub starts 2023-10-09T05:46:40.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2954762597' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:40.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:39 smithi191 ceph-mon[161669]: pgmap v814: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 86 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 675 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 21160/32253 objects degraded (65.606%); 1680/32253 objects misplaced (5.209%); 912 KiB/s, 122 objects/s recovering 2023-10-09T05:46:40.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:39 smithi191 ceph-mon[161669]: 7.1 scrub starts 2023-10-09T05:46:40.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2954762597' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:40.951 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:46:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:46:40] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.33.4" 2023-10-09T05:46:41.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:46:41.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:46:41.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:46:41.677 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:41.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:41.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:41.781 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:41.810 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:42.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[184418]: pgmap v815: 99 pgs: 1 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 609 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 355 op/s; 19955/29094 objects degraded (68.588%); 1817/29094 objects misplaced (6.245%); 3.9 MiB/s, 170 objects/s recovering 2023-10-09T05:46:42.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[184418]: failed to encode map e269 with expected crc 2023-10-09T05:46:42.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:46:42.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[184418]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T05:46:42.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[184418]: failed to encode map e269 with expected crc 2023-10-09T05:46:42.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[184418]: failed to encode map e269 with expected crc 2023-10-09T05:46:42.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[191227]: pgmap v815: 99 pgs: 1 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 609 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 355 op/s; 19955/29094 objects degraded (68.588%); 1817/29094 objects misplaced (6.245%); 3.9 MiB/s, 170 objects/s recovering 2023-10-09T05:46:42.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[191227]: failed to encode map e269 with expected crc 2023-10-09T05:46:42.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:46:42.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[191227]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T05:46:42.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[191227]: failed to encode map e269 with expected crc 2023-10-09T05:46:42.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:41 smithi008 ceph-mon[191227]: failed to encode map e269 with expected crc 2023-10-09T05:46:42.294 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:42.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:42.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:42.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:41 smithi191 ceph-mon[161669]: pgmap v815: 99 pgs: 1 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 609 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 355 op/s; 19955/29094 objects degraded (68.588%); 1817/29094 objects misplaced (6.245%); 3.9 MiB/s, 170 objects/s recovering 2023-10-09T05:46:42.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:41 smithi191 ceph-mon[161669]: failed to encode map e269 with expected crc 2023-10-09T05:46:42.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:41 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:46:42.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:41 smithi191 ceph-mon[161669]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T05:46:42.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:41 smithi191 ceph-mon[161669]: failed to encode map e269 with expected crc 2023-10-09T05:46:42.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:41 smithi191 ceph-mon[161669]: failed to encode map e269 with expected crc 2023-10-09T05:46:42.398 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:42.425 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:42.471 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:42.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:42.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:42.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:42.603 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:42.651 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:42.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:42.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:42.716 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:42.760 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:42.787 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:46:42.787 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:46:42.788 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T05:46:42.788 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T05:46:42.800 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:42.839 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:46:42.840 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:46:42.840 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.840 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180271 7.1d 7.9ab3575d (undecoded) ondisk+write+known_if_redirected e269)", 2023-10-09T05:46:42.840 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:46:42.840 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 1.709288591, 2023-10-09T05:46:42.840 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7093106470000001, 2023-10-09T05:46:42.840 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:46:42.840 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:46:42.840 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180271 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:46:42.841 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:46:42.842 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.842 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.842 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.842 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:46:42.842 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129509+0000", 2023-10-09T05:46:42.842 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:46:42.842 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.842 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.842 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:46:42.842 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129525+0000", 2023-10-09T05:46:42.842 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5430999999999998e-05 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129553+0000", 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8175e-05 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129568+0000", 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5267000000000001e-05 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.843 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.333120+0000", 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20355221500000001 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.333134+0000", 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4073000000000001e-05 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.844 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180353 7.1d 7.4c39729d (undecoded) ondisk+write+known_if_redirected e270)", 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.161196+0000", 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.67760507199999997, 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.67767608199999996, 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180353 2023-10-09T05:46:42.845 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.846 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:46:42.846 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.846 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:46:42.846 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161196+0000", 2023-10-09T05:46:42.846 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.846 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.846 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.846 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:46:42.846 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161196+0000", 2023-10-09T05:46:42.846 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.846 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161194+0000", 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161202+0000", 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6799999999999993e-06 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.847 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161216+0000", 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3688e-05 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161224+0000", 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4319999999999999e-06 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161368+0000", 2023-10-09T05:46:42.848 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014366800000000001 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161380+0000", 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2149e-05 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180395 7.1f 7.98c439ff (undecoded) ondisk+write+known_if_redirected e270)", 2023-10-09T05:46:42.849 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.436247+0000", 2023-10-09T05:46:42.850 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.40255439500000001, 2023-10-09T05:46:42.850 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.40266991499999999, 2023-10-09T05:46:42.850 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:46:42.850 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:46:42.850 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:46:42.850 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:46:42.850 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:42.850 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180395 2023-10-09T05:46:42.850 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.850 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:46:42.850 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.436247+0000", 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.436247+0000", 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:46:42.851 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.436245+0000", 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.436251+0000", 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9340000000000003e-06 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.436260+0000", 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1390000000000001e-06 2023-10-09T05:46:42.852 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.436266+0000", 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5260000000000001e-06 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.436318+0000", 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2311000000000001e-05 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.853 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:46:42.854 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.436329+0000", 2023-10-09T05:46:42.854 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0519999999999999e-05 2023-10-09T05:46:42.854 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.854 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:46:42.854 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.854 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.854 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.854 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180357 7.1d 7.88db79dd (undecoded) ondisk+write+known_if_redirected e270)", 2023-10-09T05:46:42.854 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.164256+0000", 2023-10-09T05:46:42.854 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.67454512099999997, 2023-10-09T05:46:42.854 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.67470130800000006, 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180357 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.164256+0000", 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.855 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.164256+0000", 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.164255+0000", 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.856 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.164258+0000", 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9179999999999998e-06 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.164265+0000", 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8480000000000003e-06 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.164271+0000", 2023-10-09T05:46:42.857 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6049999999999997e-06 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.164649+0000", 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00037862900000000001 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.164657+0000", 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.515e-06 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.858 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180280 7.1d 7.a44ee59d (undecoded) ondisk+write+known_if_redirected e269)", 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 1.476940881, 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.477136456, 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:46:42.859 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180280 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.860 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361859+0000", 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361865+0000", 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9800000000000003e-06 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:46:42.861 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361876+0000", 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1178999999999999e-05 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361882+0000", 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0070000000000003e-06 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361924+0000", 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2243999999999997e-05 2023-10-09T05:46:42.862 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361931+0000", 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.097e-06 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180305 7.1d 7.bf364bd (undecoded) ondisk+write+known_if_redirected e270)", 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.064410+0000", 2023-10-09T05:46:42.863 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.77439137800000002, 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.77462496599999997, 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180305 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064410+0000", 2023-10-09T05:46:42.864 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064410+0000", 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064408+0000", 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.865 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064413+0000", 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6400000000000002e-06 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064423+0000", 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7489999999999993e-06 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:46:42.866 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064430+0000", 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8399999999999997e-06 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064529+0000", 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9314000000000005e-05 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064537+0000", 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0450000000000001e-06 2023-10-09T05:46:42.867 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180323 7.1d 7.26699f7d (undecoded) ondisk+write+known_if_redirected e270)", 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.107250+0000", 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.73155107399999997, 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73182259000000005, 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:46:42.868 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180323 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107250+0000", 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107250+0000", 2023-10-09T05:46:42.869 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.870 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.870 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.870 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:46:42.870 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107248+0000", 2023-10-09T05:46:42.870 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:42.870 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.870 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.870 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:46:42.870 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107255+0000", 2023-10-09T05:46:42.870 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.708e-06 2023-10-09T05:46:42.870 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107267+0000", 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2282e-05 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107275+0000", 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8979999999999996e-06 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.871 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107338+0000", 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3078999999999997e-05 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107348+0000", 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9119999999999993e-06 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.872 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.873 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180369 7.1d 7.82d60c1d (undecoded) ondisk+write+known_if_redirected e270)", 2023-10-09T05:46:42.873 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.184014+0000", 2023-10-09T05:46:42.873 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.65478681299999997, 2023-10-09T05:46:42.873 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.65509789799999996, 2023-10-09T05:46:42.873 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:46:42.873 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:46:42.873 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:46:42.873 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:46:42.873 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:42.873 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180369 2023-10-09T05:46:42.873 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.184014+0000", 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.184014+0000", 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.874 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.184013+0000", 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.184018+0000", 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1490000000000002e-06 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:46:42.875 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.184031+0000", 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.27e-05 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.184039+0000", 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6910000000000003e-06 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.184093+0000", 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3263000000000001e-05 2023-10-09T05:46:42.876 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.184102+0000", 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6369999999999994e-06 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180292 7.1f 7.2273d13f (undecoded) ondisk+write+known_if_redirected e269)", 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:46:42.877 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 1.3113149260000001, 2023-10-09T05:46:42.878 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3116644210000001, 2023-10-09T05:46:42.878 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:46:42.878 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:46:42.878 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:46:42.878 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:46:42.878 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:46:42.878 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180292 2023-10-09T05:46:42.878 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.878 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:46:42.878 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.878 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527485+0000", 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:46:42.879 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527491+0000", 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.139e-06 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527504+0000", 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2964e-05 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.880 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527512+0000", 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2230000000000007e-06 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527553+0000", 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1266000000000001e-05 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527560+0000", 2023-10-09T05:46:42.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6599999999999998e-06 2023-10-09T05:46:42.882 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.882 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:46:42.882 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.882 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:46:42.882 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:46:42.882 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 9 2023-10-09T05:46:42.882 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:46:42.886 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:42.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:42.991 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:43.046 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:43.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: 7.13 scrub starts 2023-10-09T05:46:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T05:46:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: 7.d continuing backfill to osd.3 from (227'4985,268'11192] MIN to 268'11192 2023-10-09T05:46:43.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: 7.b continuing backfill to osd.2 from (221'1951,268'10978] MIN to 268'10978 2023-10-09T05:46:43.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19955/29094 objects degraded (68.588%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:43.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[184418]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T05:46:43.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: 7.13 scrub starts 2023-10-09T05:46:43.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T05:46:43.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: 7.d continuing backfill to osd.3 from (227'4985,268'11192] MIN to 268'11192 2023-10-09T05:46:43.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: 7.b continuing backfill to osd.2 from (221'1951,268'10978] MIN to 268'10978 2023-10-09T05:46:43.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19955/29094 objects degraded (68.588%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:43.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:42 smithi008 ceph-mon[191227]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T05:46:43.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: 7.13 scrub starts 2023-10-09T05:46:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T05:46:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: failed to encode map e270 with expected crc 2023-10-09T05:46:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: 7.d continuing backfill to osd.3 from (227'4985,268'11192] MIN to 268'11192 2023-10-09T05:46:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: 7.b continuing backfill to osd.2 from (221'1951,268'10978] MIN to 268'10978 2023-10-09T05:46:43.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19955/29094 objects degraded (68.588%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:43.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:42 smithi191 ceph-mon[161669]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T05:46:43.355 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:43.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:43.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:43.459 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:43.489 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: pgmap v818: 109 pgs: 1 active+clean+scrubbing, 4 peering, 9 unknown, 1 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 573 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 19955/27372 objects degraded (72.903%); 1817/27372 objects misplaced (6.638%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:46:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:44.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:43 smithi191 ceph-mon[161669]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T05:46:44.383 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:46:44.424 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: pgmap v818: 109 pgs: 1 active+clean+scrubbing, 4 peering, 9 unknown, 1 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 573 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 19955/27372 objects degraded (72.903%); 1817/27372 objects misplaced (6.638%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:46:44.424 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.424 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.424 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.424 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.424 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.424 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.425 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.425 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.425 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.425 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.425 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:44.425 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[184418]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T05:46:44.426 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: pgmap v818: 109 pgs: 1 active+clean+scrubbing, 4 peering, 9 unknown, 1 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 573 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 19955/27372 objects degraded (72.903%); 1817/27372 objects misplaced (6.638%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:46:44.426 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.426 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.426 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.426 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.427 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.427 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.427 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.427 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.427 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.427 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: failed to encode map e271 with expected crc 2023-10-09T05:46:44.427 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:44.427 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:43 smithi008 ceph-mon[191227]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T05:46:44.664 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:44.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:44.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:44.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:44.800 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:44.804 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:46:44.850 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:44.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:44.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:44.956 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:45.010 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:44 smithi191 ceph-mon[161669]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:44 smithi191 ceph-mon[161669]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:44 smithi191 ceph-mon[161669]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:44 smithi191 ceph-mon[161669]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:44 smithi191 ceph-mon[161669]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:44 smithi191 ceph-mon[161669]: 7.1d continuing backfill to osd.3 from (227'4985,268'11192] MIN to 268'11192 2023-10-09T05:46:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:44 smithi191 ceph-mon[161669]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:44 smithi191 ceph-mon[161669]: 7.1b continuing backfill to osd.2 from (221'1951,268'10978] MIN to 268'10978 2023-10-09T05:46:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:44 smithi191 ceph-mon[161669]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.144 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:45.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:45.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:45.252 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:45.286 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:45.341 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:45.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:45.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:45.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:45.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[184418]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[184418]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[184418]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[184418]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[184418]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[184418]: 7.1d continuing backfill to osd.3 from (227'4985,268'11192] MIN to 268'11192 2023-10-09T05:46:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[184418]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[184418]: 7.1b continuing backfill to osd.2 from (221'1951,268'10978] MIN to 268'10978 2023-10-09T05:46:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[184418]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[191227]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[191227]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[191227]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[191227]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[191227]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[191227]: 7.1d continuing backfill to osd.3 from (227'4985,268'11192] MIN to 268'11192 2023-10-09T05:46:45.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[191227]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[191227]: 7.1b continuing backfill to osd.2 from (221'1951,268'10978] MIN to 268'10978 2023-10-09T05:46:45.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:44 smithi008 ceph-mon[191227]: failed to encode map e272 with expected crc 2023-10-09T05:46:45.479 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:45.538 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:45.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:45.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:45.643 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:45.676 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:46.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:45 smithi191 ceph-mon[161669]: pgmap v821: 109 pgs: 2 remapped+peering, 1 active+clean+scrubbing, 5 peering, 7 unknown, 1 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 551 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 16336/26334 objects degraded (62.034%); 1074/26334 objects misplaced (4.078%); 6.4 MiB/s, 130 objects/s recovering 2023-10-09T05:46:46.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:45 smithi008 ceph-mon[184418]: pgmap v821: 109 pgs: 2 remapped+peering, 1 active+clean+scrubbing, 5 peering, 7 unknown, 1 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 551 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 16336/26334 objects degraded (62.034%); 1074/26334 objects misplaced (4.078%); 6.4 MiB/s, 130 objects/s recovering 2023-10-09T05:46:46.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:45 smithi008 ceph-mon[191227]: pgmap v821: 109 pgs: 2 remapped+peering, 1 active+clean+scrubbing, 5 peering, 7 unknown, 1 active+recovery_wait+degraded, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 551 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 16336/26334 objects degraded (62.034%); 1074/26334 objects misplaced (4.078%); 6.4 MiB/s, 130 objects/s recovering 2023-10-09T05:46:46.711 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:46.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:46.815 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:46.840 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:46.884 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:46.890 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:46.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:46.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:46.997 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:47.035 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:47.059 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:47.293 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:47.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:47.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:47.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:47.425 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:47.474 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:47.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:47.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:47.558 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:47.579 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:47.606 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:47.655 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:47.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:47.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:47.764 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:47.796 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:47.856 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:47.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:47.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:47.962 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:47.992 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:46:47.994 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:48.014 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:47 smithi008 ceph-mon[184418]: pgmap v822: 109 pgs: 2 remapped+peering, 1 active+clean+scrubbing, 5 peering, 4 unknown, 3 active+recovery_wait+degraded, 84 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 536 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 217 op/s; 16126/25608 objects degraded (62.973%); 743/25608 objects misplaced (2.901%); 7.6 MiB/s, 278 objects/s recovering 2023-10-09T05:46:48.014 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1884471274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:48.015 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:47 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:48.015 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:47 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16126/25608 objects degraded (62.973%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:48.015 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:47 smithi008 ceph-mon[191227]: pgmap v822: 109 pgs: 2 remapped+peering, 1 active+clean+scrubbing, 5 peering, 4 unknown, 3 active+recovery_wait+degraded, 84 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 536 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 217 op/s; 16126/25608 objects degraded (62.973%); 743/25608 objects misplaced (2.901%); 7.6 MiB/s, 278 objects/s recovering 2023-10-09T05:46:48.015 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1884471274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:48.015 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:47 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:48.015 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:47 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16126/25608 objects degraded (62.973%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:48.048 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:48.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:47 smithi191 ceph-mon[161669]: pgmap v822: 109 pgs: 2 remapped+peering, 1 active+clean+scrubbing, 5 peering, 4 unknown, 3 active+recovery_wait+degraded, 84 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 536 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 217 op/s; 16126/25608 objects degraded (62.973%); 743/25608 objects misplaced (2.901%); 7.6 MiB/s, 278 objects/s recovering 2023-10-09T05:46:48.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1884471274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:47 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:46:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:47 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16126/25608 objects degraded (62.973%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:48.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:48.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:48.156 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:48.181 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:48.236 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:48.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:48.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:48.343 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:48.382 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:48.646 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:48.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:48.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:48.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:48.778 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:48.831 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:48.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:48.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:48.937 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:48.967 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:49.026 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:49.032 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:48 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:46:49.032 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:48 smithi191 ceph-mon[161669]: failed to encode map e273 with expected crc 2023-10-09T05:46:49.032 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:48 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:46:49.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:48 smithi191 ceph-mon[161669]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T05:46:49.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:48 smithi191 ceph-mon[161669]: failed to encode map e273 with expected crc 2023-10-09T05:46:49.033 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:48 smithi191 ceph-mon[161669]: failed to encode map e273 with expected crc 2023-10-09T05:46:49.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:49.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:49.134 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:49.168 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:49.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:46:49.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[184418]: failed to encode map e273 with expected crc 2023-10-09T05:46:49.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:46:49.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[184418]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T05:46:49.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[184418]: failed to encode map e273 with expected crc 2023-10-09T05:46:49.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[184418]: failed to encode map e273 with expected crc 2023-10-09T05:46:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:46:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[191227]: failed to encode map e273 with expected crc 2023-10-09T05:46:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:46:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[191227]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T05:46:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[191227]: failed to encode map e273 with expected crc 2023-10-09T05:46:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:48 smithi008 ceph-mon[191227]: failed to encode map e273 with expected crc 2023-10-09T05:46:50.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:49 smithi008 ceph-mon[184418]: pgmap v824: 109 pgs: 2 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 93 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 507 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 16126/24183 objects degraded (66.683%); 743/24183 objects misplaced (3.072%); 6.8 MiB/s, 248 objects/s recovering 2023-10-09T05:46:50.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:49 smithi008 ceph-mon[191227]: pgmap v824: 109 pgs: 2 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 93 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 507 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 16126/24183 objects degraded (66.683%); 743/24183 objects misplaced (3.072%); 6.8 MiB/s, 248 objects/s recovering 2023-10-09T05:46:50.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:49 smithi191 ceph-mon[161669]: pgmap v824: 109 pgs: 2 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 93 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 507 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 16126/24183 objects degraded (66.683%); 743/24183 objects misplaced (3.072%); 6.8 MiB/s, 248 objects/s recovering 2023-10-09T05:46:50.891 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:50.975 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:46:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:46:50] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-09T05:46:50.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:50.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:51.078 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:51.111 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:51.170 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:51.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:51.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:51.275 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:51.304 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:51.354 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:51.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:51.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:51.459 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:51.568 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:51.840 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:51.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:51.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:51.944 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:52.004 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:52.105 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:51 smithi008 ceph-mon[184418]: 7.12 scrub starts 2023-10-09T05:46:52.106 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:51 smithi008 ceph-mon[184418]: pgmap v825: 109 pgs: 2 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 439 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 17157/20916 objects degraded (82.028%); 968/20916 objects misplaced (4.628%); 6.6 MiB/s, 409 objects/s recovering 2023-10-09T05:46:52.106 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:51 smithi008 ceph-mon[191227]: 7.12 scrub starts 2023-10-09T05:46:52.106 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:51 smithi008 ceph-mon[191227]: pgmap v825: 109 pgs: 2 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 439 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 17157/20916 objects degraded (82.028%); 968/20916 objects misplaced (4.628%); 6.6 MiB/s, 409 objects/s recovering 2023-10-09T05:46:52.128 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:52.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:52.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:52.233 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:52.261 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:51 smithi191 ceph-mon[161669]: 7.12 scrub starts 2023-10-09T05:46:52.261 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:51 smithi191 ceph-mon[161669]: pgmap v825: 109 pgs: 2 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 439 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 17157/20916 objects degraded (82.028%); 968/20916 objects misplaced (4.628%); 6.6 MiB/s, 409 objects/s recovering 2023-10-09T05:46:52.263 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:52.312 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:52.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:52.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:52.417 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:52.453 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:52.730 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:52.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:52.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:52.834 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:52.861 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:52.917 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:52.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:52.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:53.027 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:46:53.060 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:52 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 17157/20916 objects degraded (82.028%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:53.408 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:52 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 17157/20916 objects degraded (82.028%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:53.409 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:52 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 17157/20916 objects degraded (82.028%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:54.126 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:53 smithi008 ceph-mon[184418]: pgmap v826: 109 pgs: 2 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 396 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 17157/18897 objects degraded (90.792%); 968/18897 objects misplaced (5.123%); 2.8 MiB/s, 298 objects/s recovering 2023-10-09T05:46:54.127 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:53 smithi008 ceph-mon[191227]: pgmap v826: 109 pgs: 2 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 396 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 17157/18897 objects degraded (90.792%); 968/18897 objects misplaced (5.123%); 2.8 MiB/s, 298 objects/s recovering 2023-10-09T05:46:54.152 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:46:54.155 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:46:54.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:54.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:46:54.261 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:54.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:53 smithi191 ceph-mon[161669]: pgmap v826: 109 pgs: 2 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 396 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 17157/18897 objects degraded (90.792%); 968/18897 objects misplaced (5.123%); 2.8 MiB/s, 298 objects/s recovering 2023-10-09T05:46:54.322 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:54.337 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:46:54.369 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:46:54.403 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:54.404 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:46:54.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:54.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:54.440 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:46:54.481 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:46:54.509 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:54.522 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:46:54.555 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:46:54.565 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:54.587 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:46:54.588 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:46:54.588 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:46:54.588 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:46:54.588 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:46:54.588 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:46:54.588 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:46:54.588 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:46:54.588 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:46:54.903 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:54.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:55.007 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:55.012 INFO:teuthology.orchestra.run.smithi008.stdout:moved 22 / 327 (6.72783%) 2023-10-09T05:46:55.012 INFO:teuthology.orchestra.run.smithi008.stdout:avg 46.7143 2023-10-09T05:46:55.012 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 8.76216 -> 10.9507 (expected baseline 6.32778) 2023-10-09T05:46:55.012 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.5 with 31 -> 24 pgs (0.663609 -> 0.513761 * mean) 2023-10-09T05:46:55.012 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.3 with 62 -> 57 pgs (1.32722 -> 1.22018 * mean) 2023-10-09T05:46:55.013 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:46:55.013 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T05:46:55.013 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 3 2023-10-09T05:46:55.013 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:46:55.013 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0079 2023-10-09T05:46:55.013 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0087 2023-10-09T05:46:55.013 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 weight 1.0000 -> 0.7324 2023-10-09T05:46:55.013 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2 weight 1.0000 -> 0.7670 2023-10-09T05:46:55.013 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 1.0000 -> 0.8699 2023-10-09T05:46:55.016 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:55 smithi008 ceph-mon[191227]: 7.c deep-scrub starts 2023-10-09T05:46:55.016 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:55 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/154178115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:55.016 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:55 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'50263','3':'57011','5':'47997'}"}]: dispatch 2023-10-09T05:46:55.035 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:46:55.035 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:46:55.051 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:55.097 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:55.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:55.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:55.202 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:55.228 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:55.275 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:55.280 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:55 smithi191 ceph-mon[161669]: 7.c deep-scrub starts 2023-10-09T05:46:55.280 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:55 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/154178115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:55.280 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:55 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'50263','3':'57011','5':'47997'}"}]: dispatch 2023-10-09T05:46:55.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:55.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:55.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:55.409 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:55.453 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:55.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:55.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:55.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:55 smithi008 ceph-mon[184418]: 7.c deep-scrub starts 2023-10-09T05:46:55.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:55 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/154178115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:46:55.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:55 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'50263','3':'57011','5':'47997'}"}]: dispatch 2023-10-09T05:46:55.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:55.586 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:55.629 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:55.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:55.734 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:46:55.759 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:55.805 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:46:55.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:55.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:46:55.913 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:55.940 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:55.987 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:55.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:55.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:56.092 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:46:56.157 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:56.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:56 smithi191 ceph-mon[161669]: pgmap v827: 109 pgs: 2 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 356 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 16722/16956 objects degraded (98.620%); 865/16956 objects misplaced (5.101%); 2.3 MiB/s, 373 objects/s recovering 2023-10-09T05:46:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:56 smithi191 ceph-mon[161669]: from='client.45704 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:56 smithi191 ceph-mon[161669]: failed to encode map e274 with expected crc 2023-10-09T05:46:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:56 smithi191 ceph-mon[161669]: failed to encode map e274 with expected crc 2023-10-09T05:46:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'50263','3':'57011','5':'47997'}"}]': finished 2023-10-09T05:46:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:56 smithi191 ceph-mon[161669]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T05:46:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:56 smithi191 ceph-mon[161669]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T05:46:56.337 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[184418]: pgmap v827: 109 pgs: 2 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 356 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 16722/16956 objects degraded (98.620%); 865/16956 objects misplaced (5.101%); 2.3 MiB/s, 373 objects/s recovering 2023-10-09T05:46:56.338 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[184418]: from='client.45704 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:56.338 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[184418]: failed to encode map e274 with expected crc 2023-10-09T05:46:56.338 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[184418]: failed to encode map e274 with expected crc 2023-10-09T05:46:56.338 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'50263','3':'57011','5':'47997'}"}]': finished 2023-10-09T05:46:56.338 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[184418]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T05:46:56.338 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[184418]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T05:46:56.338 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[191227]: pgmap v827: 109 pgs: 2 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 356 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 16722/16956 objects degraded (98.620%); 865/16956 objects misplaced (5.101%); 2.3 MiB/s, 373 objects/s recovering 2023-10-09T05:46:56.338 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[191227]: from='client.45704 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:56.338 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[191227]: failed to encode map e274 with expected crc 2023-10-09T05:46:56.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[191227]: failed to encode map e274 with expected crc 2023-10-09T05:46:56.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'50263','3':'57011','5':'47997'}"}]': finished 2023-10-09T05:46:56.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[191227]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T05:46:56.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:56 smithi008 ceph-mon[191227]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T05:46:56.707 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:46:56.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:56.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:46:56.811 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:56.968 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:57.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: 7.3 scrub starts 2023-10-09T05:46:57.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: 7.b starting backfill to osd.0 from (0'0,0'0] MAX to 273'11102 2023-10-09T05:46:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: 7.1b starting backfill to osd.0 from (0'0,0'0] MAX to 273'11108 2023-10-09T05:46:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: 7.b starting backfill to osd.4 from (0'0,0'0] MAX to 273'11102 2023-10-09T05:46:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: 7.1b starting backfill to osd.4 from (0'0,0'0] MAX to 273'11108 2023-10-09T05:46:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: pgmap v830: 109 pgs: 2 remapped+peering, 9 peering, 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 330 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 16639/15684 objects degraded (106.089%); 444/15684 objects misplaced (2.831%); 2.2 MiB/s, 364 objects/s recovering 2023-10-09T05:46:57.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:57 smithi191 ceph-mon[161669]: osdmap e276: 8 total, 8 up, 7 in 2023-10-09T05:46:57.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: 7.3 scrub starts 2023-10-09T05:46:57.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: 7.b starting backfill to osd.0 from (0'0,0'0] MAX to 273'11102 2023-10-09T05:46:57.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: 7.1b starting backfill to osd.0 from (0'0,0'0] MAX to 273'11108 2023-10-09T05:46:57.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: 7.b starting backfill to osd.4 from (0'0,0'0] MAX to 273'11102 2023-10-09T05:46:57.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: 7.1b starting backfill to osd.4 from (0'0,0'0] MAX to 273'11108 2023-10-09T05:46:57.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: pgmap v830: 109 pgs: 2 remapped+peering, 9 peering, 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 330 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 16639/15684 objects degraded (106.089%); 444/15684 objects misplaced (2.831%); 2.2 MiB/s, 364 objects/s recovering 2023-10-09T05:46:57.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[184418]: osdmap e276: 8 total, 8 up, 7 in 2023-10-09T05:46:57.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: 7.3 scrub starts 2023-10-09T05:46:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: failed to encode map e275 with expected crc 2023-10-09T05:46:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: 7.b starting backfill to osd.0 from (0'0,0'0] MAX to 273'11102 2023-10-09T05:46:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: 7.1b starting backfill to osd.0 from (0'0,0'0] MAX to 273'11108 2023-10-09T05:46:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: 7.b starting backfill to osd.4 from (0'0,0'0] MAX to 273'11102 2023-10-09T05:46:57.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: 7.1b starting backfill to osd.4 from (0'0,0'0] MAX to 273'11108 2023-10-09T05:46:57.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: pgmap v830: 109 pgs: 2 remapped+peering, 9 peering, 2 active+recovery_wait+degraded, 86 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 330 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 16639/15684 objects degraded (106.089%); 444/15684 objects misplaced (2.831%); 2.2 MiB/s, 364 objects/s recovering 2023-10-09T05:46:57.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:57 smithi008 ceph-mon[191227]: osdmap e276: 8 total, 8 up, 7 in 2023-10-09T05:46:57.690 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:57.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:57.796 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:46:57.843 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:57.925 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:46:57.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:57.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:46:58.037 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:58.080 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:58.242 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:46:58.243 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T05:46:58.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:58 smithi191 ceph-mon[161669]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:58 smithi191 ceph-mon[161669]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:58 smithi191 ceph-mon[161669]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:58 smithi191 ceph-mon[161669]: 7.16 starting backfill to osd.4 from (0'0,0'0] MAX to 273'9808 2023-10-09T05:46:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:58 smithi191 ceph-mon[161669]: 7.6 starting backfill to osd.4 from (0'0,0'0] MAX to 273'9804 2023-10-09T05:46:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:58 smithi191 ceph-mon[161669]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:58 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:46:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:58 smithi191 ceph-mon[161669]: 7.1e starting backfill to osd.3 from (0'0,0'0] MAX to 273'11650 2023-10-09T05:46:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:58 smithi191 ceph-mon[161669]: 7.e starting backfill to osd.3 from (0'0,0'0] MAX to 273'11660 2023-10-09T05:46:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16639/15684 objects degraded (106.089%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:58.303 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[184418]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[184418]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[184418]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[184418]: 7.16 starting backfill to osd.4 from (0'0,0'0] MAX to 273'9808 2023-10-09T05:46:58.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[184418]: 7.6 starting backfill to osd.4 from (0'0,0'0] MAX to 273'9804 2023-10-09T05:46:58.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[184418]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:46:58.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[184418]: 7.1e starting backfill to osd.3 from (0'0,0'0] MAX to 273'11650 2023-10-09T05:46:58.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[184418]: 7.e starting backfill to osd.3 from (0'0,0'0] MAX to 273'11660 2023-10-09T05:46:58.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16639/15684 objects degraded (106.089%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:58.304 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[191227]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[191227]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[191227]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[191227]: 7.16 starting backfill to osd.4 from (0'0,0'0] MAX to 273'9808 2023-10-09T05:46:58.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[191227]: 7.6 starting backfill to osd.4 from (0'0,0'0] MAX to 273'9804 2023-10-09T05:46:58.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[191227]: failed to encode map e276 with expected crc 2023-10-09T05:46:58.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:46:58.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[191227]: 7.1e starting backfill to osd.3 from (0'0,0'0] MAX to 273'11650 2023-10-09T05:46:58.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[191227]: 7.e starting backfill to osd.3 from (0'0,0'0] MAX to 273'11660 2023-10-09T05:46:58.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16639/15684 objects degraded (106.089%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:46:58.324 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:46:58.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:58.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:46:58.337 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:46:58.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:46:58.434 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:58.461 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:58.519 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:46:58.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:58.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:46:58.625 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:58.661 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:58.722 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:58.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:58.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:58.830 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:46:58.870 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:58.937 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:46:58.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:58.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:46:59.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:46:59.071 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:46:59.123 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:46:59.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:46:59.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:46:59.134 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:46:59.229 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:46:59.264 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:59 smithi008 ceph-mon[184418]: pgmap v832: 109 pgs: 1 activating+degraded+remapped, 1 activating+remapped, 2 remapped+peering, 9 peering, 2 active+recovery_wait+degraded, 84 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 303 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 248 op/s; 16640/14427 objects degraded (115.339%); 742/14427 objects misplaced (5.143%); 1.5 MiB/s, 229 objects/s recovering 2023-10-09T05:47:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:46:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:59 smithi008 ceph-mon[191227]: pgmap v832: 109 pgs: 1 activating+degraded+remapped, 1 activating+remapped, 2 remapped+peering, 9 peering, 2 active+recovery_wait+degraded, 84 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 303 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 248 op/s; 16640/14427 objects degraded (115.339%); 742/14427 objects misplaced (5.143%); 1.5 MiB/s, 229 objects/s recovering 2023-10-09T05:47:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:46:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:59 smithi191 ceph-mon[161669]: pgmap v832: 109 pgs: 1 activating+degraded+remapped, 1 activating+remapped, 2 remapped+peering, 9 peering, 2 active+recovery_wait+degraded, 84 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 303 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 248 op/s; 16640/14427 objects degraded (115.339%); 742/14427 objects misplaced (5.143%); 1.5 MiB/s, 229 objects/s recovering 2023-10-09T05:47:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:46:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:00.724 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:00.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:00.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:00.792 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:00.799 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:47:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:47:00] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.33.4" 2023-10-09T05:47:00.829 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:00.837 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:00.865 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:00.921 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:00.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:00.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:01.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:01.056 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:01.119 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:01.123 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:00 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T05:47:01.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:01.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:01.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:00 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T05:47:01.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:00 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T05:47:01.224 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:01.261 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:01.325 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:01.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:01.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:01.397 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:01.430 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:01.461 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:01.518 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:01.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:01.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:01.625 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:01.663 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:02.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:01 smithi008 ceph-mon[184418]: pgmap v833: 109 pgs: 2 active+recovery_wait+degraded, 1 activating+degraded+remapped, 1 activating+remapped, 89 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 249 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 19586/11820 objects degraded (165.702%); 1132/11820 objects misplaced (9.577%); 5.9 MiB/s, 353 objects/s recovering 2023-10-09T05:47:02.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:01 smithi008 ceph-mon[191227]: pgmap v833: 109 pgs: 2 active+recovery_wait+degraded, 1 activating+degraded+remapped, 1 activating+remapped, 89 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 249 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 19586/11820 objects degraded (165.702%); 1132/11820 objects misplaced (9.577%); 5.9 MiB/s, 353 objects/s recovering 2023-10-09T05:47:02.277 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:02.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:02.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:02.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:01 smithi191 ceph-mon[161669]: pgmap v833: 109 pgs: 2 active+recovery_wait+degraded, 1 activating+degraded+remapped, 1 activating+remapped, 89 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 249 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 19586/11820 objects degraded (165.702%); 1132/11820 objects misplaced (9.577%); 5.9 MiB/s, 353 objects/s recovering 2023-10-09T05:47:02.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:02.409 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:02.467 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:02.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:02.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:02.574 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:02.609 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:02.670 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:02.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:02.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:02.778 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:02.809 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:02.865 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:02.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:02.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:02.958 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:02 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19586/11820 objects degraded (165.702%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:02.970 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:03.000 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:03.048 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:03.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:03.154 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:03.207 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:03.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:02 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19586/11820 objects degraded (165.702%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:03.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:02 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19586/11820 objects degraded (165.702%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:04.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:03 smithi008 ceph-mon[184418]: pgmap v834: 109 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 197 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 346 op/s; 19585/9342 objects degraded (209.645%); 979/9342 objects misplaced (10.480%); 4.8 MiB/s, 289 objects/s recovering 2023-10-09T05:47:04.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:03 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/249624534' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:47:04.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:03 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:47:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:03 smithi008 ceph-mon[191227]: pgmap v834: 109 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 197 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 346 op/s; 19585/9342 objects degraded (209.645%); 979/9342 objects misplaced (10.480%); 4.8 MiB/s, 289 objects/s recovering 2023-10-09T05:47:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:03 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/249624534' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:47:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:03 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:47:04.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:03 smithi191 ceph-mon[161669]: pgmap v834: 109 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 197 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 346 op/s; 19585/9342 objects degraded (209.645%); 979/9342 objects misplaced (10.480%); 4.8 MiB/s, 289 objects/s recovering 2023-10-09T05:47:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:03 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/249624534' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:47:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:03 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:47:04.591 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:47:05.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:47:05.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[184418]: failed to encode map e277 with expected crc 2023-10-09T05:47:05.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[184418]: failed to encode map e277 with expected crc 2023-10-09T05:47:05.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:47:05.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[184418]: osdmap e277: 8 total, 8 up, 7 in 2023-10-09T05:47:05.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[184418]: failed to encode map e277 with expected crc 2023-10-09T05:47:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:47:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[191227]: failed to encode map e277 with expected crc 2023-10-09T05:47:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[191227]: failed to encode map e277 with expected crc 2023-10-09T05:47:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:47:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[191227]: osdmap e277: 8 total, 8 up, 7 in 2023-10-09T05:47:05.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:04 smithi008 ceph-mon[191227]: failed to encode map e277 with expected crc 2023-10-09T05:47:05.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:04 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:47:05.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:04 smithi191 ceph-mon[161669]: failed to encode map e277 with expected crc 2023-10-09T05:47:05.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:04 smithi191 ceph-mon[161669]: failed to encode map e277 with expected crc 2023-10-09T05:47:05.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:04 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:47:05.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:04 smithi191 ceph-mon[161669]: osdmap e277: 8 total, 8 up, 7 in 2023-10-09T05:47:05.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:04 smithi191 ceph-mon[161669]: failed to encode map e277 with expected crc 2023-10-09T05:47:05.419 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:05.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:05.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:05.525 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:05.568 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:06.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:05 smithi191 ceph-mon[161669]: pgmap v836: 109 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 7 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 150 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 19585/7080 objects degraded (276.624%); 720/7080 objects misplaced (10.169%); 3.3 MiB/s, 337 objects/s recovering 2023-10-09T05:47:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:05 smithi008 ceph-mon[184418]: pgmap v836: 109 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 7 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 150 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 19585/7080 objects degraded (276.624%); 720/7080 objects misplaced (10.169%); 3.3 MiB/s, 337 objects/s recovering 2023-10-09T05:47:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:05 smithi008 ceph-mon[191227]: pgmap v836: 109 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 7 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 150 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 19585/7080 objects degraded (276.624%); 720/7080 objects misplaced (10.169%); 3.3 MiB/s, 337 objects/s recovering 2023-10-09T05:47:06.567 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:06.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:06.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:06.670 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:06.721 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:07.296 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:47:07.296 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:47:07.800 INFO:teuthology.orchestra.run.smithi008.stdout:moved 28 / 327 (8.56269%) 2023-10-09T05:47:07.800 INFO:teuthology.orchestra.run.smithi008.stdout:avg 46.7143 2023-10-09T05:47:07.800 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 10.9507 -> 14.3797 (expected baseline 6.32778) 2023-10-09T05:47:07.800 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.5 with 24 -> 22 pgs (0.513761 -> 0.470948 * mean) 2023-10-09T05:47:07.801 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.3 with 57 -> 59 pgs (1.22018 -> 1.263 * mean) 2023-10-09T05:47:07.801 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:47:07.801 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T05:47:07.801 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 1 2023-10-09T05:47:07.801 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:47:07.801 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0072 2023-10-09T05:47:07.801 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0079 2023-10-09T05:47:07.801 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 weight 1.0000 -> 0.6945 2023-10-09T05:47:07.801 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 weight 0.7324 -> 0.5491 2023-10-09T05:47:07.801 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2 weight 0.7670 -> 0.6138 2023-10-09T05:47:07.802 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T05:47:07.832 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:07.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:07.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:07.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:07.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:47:07 smithi008 conmon[177229]: 2023-10-09T05:47:07.712+0000 7f020285b700 -1 mgr.server reply reply (3) No such process no change 2023-10-09T05:47:07.967 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:08.019 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:08.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:08.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:08.125 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:08.157 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:07 smithi191 ceph-mon[161669]: pgmap v837: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 120 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 329 op/s; 19458/5616 objects degraded (346.474%); 468/5616 objects misplaced (8.333%); 3.6 MiB/s, 354 objects/s recovering 2023-10-09T05:47:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:07 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:47:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:07 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19458/5616 objects degraded (346.474%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:08.424 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Removed 94493 objects 2023-10-09T05:47:08.424 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Clean up completed and total clean up time :241.014 2023-10-09T05:47:08.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:07 smithi008 ceph-mon[191227]: pgmap v837: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 120 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 329 op/s; 19458/5616 objects degraded (346.474%); 468/5616 objects misplaced (8.333%); 3.6 MiB/s, 354 objects/s recovering 2023-10-09T05:47:08.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:07 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:47:08.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:07 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19458/5616 objects degraded (346.474%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:07 smithi008 ceph-mon[184418]: pgmap v837: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 120 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 329 op/s; 19458/5616 objects degraded (346.474%); 468/5616 objects misplaced (8.333%); 3.6 MiB/s, 354 objects/s recovering 2023-10-09T05:47:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:07 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:47:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:07 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19458/5616 objects degraded (346.474%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:08.773 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:47:09.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:08 smithi008 ceph-mon[184418]: from='client.55639 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:08 smithi008 ceph-mon[191227]: from='client.55639 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:08 smithi191 ceph-mon[161669]: from='client.55639 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:09.931 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:09.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:09.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:10.035 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:47:10.073 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:10.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:09 smithi008 ceph-mon[184418]: pgmap v838: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 76 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 19458/3525 objects degraded (552.000%); 341/3525 objects misplaced (9.674%); 3.4 MiB/s, 329 objects/s recovering 2023-10-09T05:47:10.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:09 smithi008 ceph-mon[184418]: failed to encode map e278 with expected crc 2023-10-09T05:47:10.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:09 smithi008 ceph-mon[184418]: failed to encode map e278 with expected crc 2023-10-09T05:47:10.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:09 smithi008 ceph-mon[184418]: osdmap e278: 8 total, 8 up, 7 in 2023-10-09T05:47:10.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:09 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:09 smithi008 ceph-mon[191227]: pgmap v838: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 76 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 19458/3525 objects degraded (552.000%); 341/3525 objects misplaced (9.674%); 3.4 MiB/s, 329 objects/s recovering 2023-10-09T05:47:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:09 smithi008 ceph-mon[191227]: failed to encode map e278 with expected crc 2023-10-09T05:47:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:09 smithi008 ceph-mon[191227]: failed to encode map e278 with expected crc 2023-10-09T05:47:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:09 smithi008 ceph-mon[191227]: osdmap e278: 8 total, 8 up, 7 in 2023-10-09T05:47:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:09 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:10.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:09 smithi191 ceph-mon[161669]: pgmap v838: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 90 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 76 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 19458/3525 objects degraded (552.000%); 341/3525 objects misplaced (9.674%); 3.4 MiB/s, 329 objects/s recovering 2023-10-09T05:47:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:09 smithi191 ceph-mon[161669]: failed to encode map e278 with expected crc 2023-10-09T05:47:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:09 smithi191 ceph-mon[161669]: failed to encode map e278 with expected crc 2023-10-09T05:47:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:09 smithi191 ceph-mon[161669]: osdmap e278: 8 total, 8 up, 7 in 2023-10-09T05:47:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:09 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:10.677 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:47:10.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:47:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:47:10] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.33.4" 2023-10-09T05:47:11.005 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:47:11.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:11.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:47:11.044 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T05:47:11.045 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T05:47:11.114 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:11.168 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:11.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:10 smithi191 ceph-mon[161669]: failed to encode map e278 with expected crc 2023-10-09T05:47:11.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:10 smithi191 ceph-mon[161669]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:10 smithi191 ceph-mon[161669]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:10 smithi191 ceph-mon[161669]: osdmap e279: 8 total, 8 up, 7 in 2023-10-09T05:47:11.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:10 smithi191 ceph-mon[161669]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:10 smithi191 ceph-mon[161669]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:10 smithi191 ceph-mon[161669]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:10 smithi191 ceph-mon[161669]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:10 smithi191 ceph-mon[161669]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:10 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:11.329 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:11.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:11.335 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[184418]: failed to encode map e278 with expected crc 2023-10-09T05:47:11.335 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[184418]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.335 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[184418]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.335 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[184418]: osdmap e279: 8 total, 8 up, 7 in 2023-10-09T05:47:11.335 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[184418]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.336 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[184418]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.336 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[184418]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.336 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[184418]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.336 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[184418]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.336 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:11.336 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[191227]: failed to encode map e278 with expected crc 2023-10-09T05:47:11.336 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[191227]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.336 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[191227]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.337 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[191227]: osdmap e279: 8 total, 8 up, 7 in 2023-10-09T05:47:11.337 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[191227]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.337 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[191227]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.337 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[191227]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.337 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[191227]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.337 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[191227]: failed to encode map e279 with expected crc 2023-10-09T05:47:11.337 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:10 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:11.378 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:11.407 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:47:11.429 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:11.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:11.445 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:47:11.465 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:11.479 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:47:11.511 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:11.513 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:47:11.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:11.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:11.538 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:47:11.558 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:47:11.576 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:47:11.594 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:47:11.596 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:47:11.596 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:47:11.596 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:47:11.596 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:47:11.596 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:47:11.596 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:47:11.596 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:47:11.596 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:47:11.617 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:47:11.648 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:12.051 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:47:12.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:12.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:47:12.157 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:12.184 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:12.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[184418]: pgmap v841: 109 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 91 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 26 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 344 op/s; 19107/1110 objects degraded (1721.351%); 38/1110 objects misplaced (3.423%); 1.9 MiB/s, 278 objects/s recovering 2023-10-09T05:47:12.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[184418]: failed to encode map e279 with expected crc 2023-10-09T05:47:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[184418]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[184418]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[184418]: osdmap e280: 8 total, 8 up, 7 in 2023-10-09T05:47:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[184418]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[184418]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:12.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1551071537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:12.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[191227]: pgmap v841: 109 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 91 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 26 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 344 op/s; 19107/1110 objects degraded (1721.351%); 38/1110 objects misplaced (3.423%); 1.9 MiB/s, 278 objects/s recovering 2023-10-09T05:47:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[191227]: failed to encode map e279 with expected crc 2023-10-09T05:47:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[191227]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[191227]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[191227]: osdmap e280: 8 total, 8 up, 7 in 2023-10-09T05:47:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[191227]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[191227]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:12.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1551071537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:12.229 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:12.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:12 smithi191 ceph-mon[161669]: pgmap v841: 109 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 91 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 26 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 344 op/s; 19107/1110 objects degraded (1721.351%); 38/1110 objects misplaced (3.423%); 1.9 MiB/s, 278 objects/s recovering 2023-10-09T05:47:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:12 smithi191 ceph-mon[161669]: failed to encode map e279 with expected crc 2023-10-09T05:47:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:12 smithi191 ceph-mon[161669]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:12 smithi191 ceph-mon[161669]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:12 smithi191 ceph-mon[161669]: osdmap e280: 8 total, 8 up, 7 in 2023-10-09T05:47:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:12 smithi191 ceph-mon[161669]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:12 smithi191 ceph-mon[161669]: failed to encode map e280 with expected crc 2023-10-09T05:47:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:12 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:12 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1551071537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:12.335 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:12.361 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:12.405 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:12.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:12.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:12.512 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:47:12.543 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:12.952 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T05:47:12.953 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T05:47:13.094 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:47:13.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:13.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:47:13.198 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:13.252 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:13.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:13 smithi191 ceph-mon[161669]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:13 smithi191 ceph-mon[161669]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T05:47:13.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:13 smithi191 ceph-mon[161669]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:13 smithi191 ceph-mon[161669]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:13 smithi191 ceph-mon[161669]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19107/1110 objects degraded (1721.351%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:13.304 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:13.309 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[184418]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.309 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[184418]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T05:47:13.309 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[184418]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[184418]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[184418]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19107/1110 objects degraded (1721.351%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:13.310 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[191227]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[191227]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T05:47:13.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[191227]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[191227]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[191227]: failed to encode map e281 with expected crc 2023-10-09T05:47:13.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19107/1110 objects degraded (1721.351%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:13.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:13.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:13.412 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:13.439 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:13.498 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:13.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:13.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:13.603 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:13.631 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:14.022 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:14 smithi008 ceph-mon[184418]: pgmap v844: 109 pgs: 2 active+recovery_wait+degraded, 93 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 276 op/s; 19107/858 objects degraded (2226.923%); 20/858 objects misplaced (2.331%); 1.2 MiB/s, 293 objects/s recovering 2023-10-09T05:47:14.023 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:14.023 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:14.024 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:14 smithi008 ceph-mon[191227]: pgmap v844: 109 pgs: 2 active+recovery_wait+degraded, 93 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 276 op/s; 19107/858 objects degraded (2226.923%); 20/858 objects misplaced (2.331%); 1.2 MiB/s, 293 objects/s recovering 2023-10-09T05:47:14.024 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:14.024 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:14.145 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:14.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:14.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:14.201 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:14.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:14.274 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:14 smithi191 ceph-mon[161669]: pgmap v844: 109 pgs: 2 active+recovery_wait+degraded, 93 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 276 op/s; 19107/858 objects degraded (2226.923%); 20/858 objects misplaced (2.331%); 1.2 MiB/s, 293 objects/s recovering 2023-10-09T05:47:14.274 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:14.274 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:14.276 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:14.319 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:14.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:14.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:14.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:14.453 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:14.497 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:14.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:14.603 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:14.628 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:14.672 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:14.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:14.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:14.778 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:14.804 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:14.825 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:47:14.825 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:47:14.825 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T05:47:14.825 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:47:14.829 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:14.848 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:14.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:14.955 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:47:14.993 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:15.202 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:47:15.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:15.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:47:15.304 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:47:15.333 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:15.959 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:47:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[184418]: pgmap v845: 109 pgs: 2 active+recovery_wait+degraded, 94 active+clean, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 79 op/s; 18332/858 objects degraded (2136.597%); 20/858 objects misplaced (2.331%); 920 KiB/s, 160 objects/s recovering 2023-10-09T05:47:16.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[184418]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[184418]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[184418]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T05:47:16.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[184418]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[184418]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/888264312' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:47:16.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:47:16.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[191227]: pgmap v845: 109 pgs: 2 active+recovery_wait+degraded, 94 active+clean, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 79 op/s; 18332/858 objects degraded (2136.597%); 20/858 objects misplaced (2.331%); 920 KiB/s, 160 objects/s recovering 2023-10-09T05:47:16.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[191227]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[191227]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[191227]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T05:47:16.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[191227]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[191227]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/888264312' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:47:16.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:15 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:47:16.233 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:47:16.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:15 smithi191 ceph-mon[161669]: pgmap v845: 109 pgs: 2 active+recovery_wait+degraded, 94 active+clean, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 79 op/s; 18332/858 objects degraded (2136.597%); 20/858 objects misplaced (2.331%); 920 KiB/s, 160 objects/s recovering 2023-10-09T05:47:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:15 smithi191 ceph-mon[161669]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:15 smithi191 ceph-mon[161669]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:15 smithi191 ceph-mon[161669]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T05:47:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:15 smithi191 ceph-mon[161669]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:15 smithi191 ceph-mon[161669]: failed to encode map e282 with expected crc 2023-10-09T05:47:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/888264312' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:47:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:15 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:47:17.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:47:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[184418]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[184418]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:47:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[184418]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T05:47:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[184418]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[184418]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:47:17.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[191227]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[191227]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:47:17.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[191227]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T05:47:17.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[191227]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:16 smithi008 ceph-mon[191227]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:16 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:47:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:16 smithi191 ceph-mon[161669]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:16 smithi191 ceph-mon[161669]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:16 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:47:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:16 smithi191 ceph-mon[161669]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T05:47:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:16 smithi191 ceph-mon[161669]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:16 smithi191 ceph-mon[161669]: failed to encode map e283 with expected crc 2023-10-09T05:47:17.742 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:47:17.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:17.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:47:17.846 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:17.874 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:17.920 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:17.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:17.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:18.025 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:18.051 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:18.096 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:18.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:18.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:18.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:17 smithi008 ceph-mon[184418]: pgmap v848: 109 pgs: 1 peering, 1 active+recovery_wait+degraded, 94 active+clean, 1 active+recovering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 26 op/s; 15819/858 objects degraded (1843.706%); 20/858 objects misplaced (2.331%); 0 B/s, 425 objects/s recovering 2023-10-09T05:47:18.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:17 smithi008 ceph-mon[191227]: pgmap v848: 109 pgs: 1 peering, 1 active+recovery_wait+degraded, 94 active+clean, 1 active+recovering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 26 op/s; 15819/858 objects degraded (1843.706%); 20/858 objects misplaced (2.331%); 0 B/s, 425 objects/s recovering 2023-10-09T05:47:18.224 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:17 smithi191 ceph-mon[161669]: pgmap v848: 109 pgs: 1 peering, 1 active+recovery_wait+degraded, 94 active+clean, 1 active+recovering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 26 op/s; 15819/858 objects degraded (1843.706%); 20/858 objects misplaced (2.331%); 0 B/s, 425 objects/s recovering 2023-10-09T05:47:18.227 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:18.274 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:18.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:18.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:18.379 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:18.416 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:18.636 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:18.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:18.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:18.739 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:18.768 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:19.121 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15819/858 objects degraded (1843.706%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:19.122 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15819/858 objects degraded (1843.706%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:19.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15819/858 objects degraded (1843.706%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:19.740 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:47:19.877 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:19.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:19.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:19.981 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:19.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:19 smithi008 ceph-mon[184418]: pgmap v849: 109 pgs: 1 peering, 1 active+recovery_wait+degraded, 94 active+clean, 1 active+recovering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 15819/858 objects degraded (1843.706%); 20/858 objects misplaced (2.331%); 0 B/s, 358 objects/s recovering 2023-10-09T05:47:19.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:19 smithi008 ceph-mon[191227]: pgmap v849: 109 pgs: 1 peering, 1 active+recovery_wait+degraded, 94 active+clean, 1 active+recovering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 15819/858 objects degraded (1843.706%); 20/858 objects misplaced (2.331%); 0 B/s, 358 objects/s recovering 2023-10-09T05:47:20.008 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:20.053 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:20.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:20.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:20.159 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:20.202 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:20.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:19 smithi191 ceph-mon[161669]: pgmap v849: 109 pgs: 1 peering, 1 active+recovery_wait+degraded, 94 active+clean, 1 active+recovering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 15819/858 objects degraded (1843.706%); 20/858 objects misplaced (2.331%); 0 B/s, 358 objects/s recovering 2023-10-09T05:47:20.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:47:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:47:20] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.33.4" 2023-10-09T05:47:22.060 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:47:22.118 INFO:teuthology.orchestra.run.smithi008.stdout:moved 0 / 327 (0%) 2023-10-09T05:47:22.118 INFO:teuthology.orchestra.run.smithi008.stdout:avg 46.7143 2023-10-09T05:47:22.118 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 10.9507 -> 10.9507 (expected baseline 6.32778) 2023-10-09T05:47:22.118 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.5 with 24 -> 24 pgs (0.513761 -> 0.513761 * mean) 2023-10-09T05:47:22.118 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.3 with 57 -> 57 pgs (1.22018 -> 1.22018 * mean) 2023-10-09T05:47:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:47:22.119 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T05:47:22.119 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 0.05 2023-10-09T05:47:22.119 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:47:22.119 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0066 2023-10-09T05:47:22.119 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0658 2023-10-09T05:47:22.121 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T05:47:22.147 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:22.148 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:22.172 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:22.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:22.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:22.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:21 smithi008 ceph-mon[184418]: pgmap v850: 109 pgs: 1 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 14048/858 objects degraded (1637.296%); 20/858 objects misplaced (2.331%); 0 B/s, 442 objects/s recovering 2023-10-09T05:47:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:21 smithi008 ceph-mon[191227]: pgmap v850: 109 pgs: 1 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 14048/858 objects degraded (1637.296%); 20/858 objects misplaced (2.331%); 0 B/s, 442 objects/s recovering 2023-10-09T05:47:22.278 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:22.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:21 smithi191 ceph-mon[161669]: pgmap v850: 109 pgs: 1 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 14048/858 objects degraded (1637.296%); 20/858 objects misplaced (2.331%); 0 B/s, 442 objects/s recovering 2023-10-09T05:47:22.305 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:22.351 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:22.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:22.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:22.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:22.482 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:22.526 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:22.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:22.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:22.635 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:22.665 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:22.943 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_2' removed 2023-10-09T05:47:22.950 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:22 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3135718953' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:47:22.951 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:22 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14048/858 objects degraded (1637.296%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:23.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:22 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3135718953' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:47:23.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:22 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14048/858 objects degraded (1637.296%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:23.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:22 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3135718953' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:47:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:22 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14048/858 objects degraded (1637.296%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:47:23.716 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:23.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:23.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:23.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:23.847 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:23.893 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:23.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:23.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:23.956 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T05:47:23.956 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-10-09T05:47:23.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[184418]: from='client.35778 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:23.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[184418]: pgmap v851: 109 pgs: 1 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 14048/858 objects degraded (1637.296%); 20/858 objects misplaced (2.331%); 0 B/s, 345 objects/s recovering 2023-10-09T05:47:23.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[184418]: failed to encode map e284 with expected crc 2023-10-09T05:47:23.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[184418]: failed to encode map e284 with expected crc 2023-10-09T05:47:23.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3135718953' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:47:23.983 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[184418]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T05:47:23.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[191227]: from='client.35778 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:23.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[191227]: pgmap v851: 109 pgs: 1 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 14048/858 objects degraded (1637.296%); 20/858 objects misplaced (2.331%); 0 B/s, 345 objects/s recovering 2023-10-09T05:47:23.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[191227]: failed to encode map e284 with expected crc 2023-10-09T05:47:23.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[191227]: failed to encode map e284 with expected crc 2023-10-09T05:47:23.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3135718953' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:47:23.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:23 smithi008 ceph-mon[191227]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T05:47:23.999 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:47:24.037 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:24.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:23 smithi191 ceph-mon[161669]: from='client.35778 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:23 smithi191 ceph-mon[161669]: pgmap v851: 109 pgs: 1 active+recovery_wait+degraded, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 14048/858 objects degraded (1637.296%); 20/858 objects misplaced (2.331%); 0 B/s, 345 objects/s recovering 2023-10-09T05:47:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:23 smithi191 ceph-mon[161669]: failed to encode map e284 with expected crc 2023-10-09T05:47:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:23 smithi191 ceph-mon[161669]: failed to encode map e284 with expected crc 2023-10-09T05:47:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3135718953' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:47:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:23 smithi191 ceph-mon[161669]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T05:47:25.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:24 smithi008 ceph-mon[184418]: failed to encode map e284 with expected crc 2023-10-09T05:47:25.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:24 smithi008 ceph-mon[191227]: failed to encode map e284 with expected crc 2023-10-09T05:47:25.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:24 smithi191 ceph-mon[161669]: failed to encode map e284 with expected crc 2023-10-09T05:47:25.735 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:47:25.736 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:47:25.736 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T05:47:25.736 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T05:47:26.042 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:25 smithi008 ceph-mon[184418]: pgmap v853: 73 pgs: 73 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:26.042 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:25 smithi008 ceph-mon[184418]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14048/858 objects degraded (1637.296%), 7 pgs degraded, 2 pgs undersized) 2023-10-09T05:47:26.042 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:25 smithi008 ceph-mon[191227]: pgmap v853: 73 pgs: 73 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:26.042 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:25 smithi008 ceph-mon[191227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14048/858 objects degraded (1637.296%), 7 pgs degraded, 2 pgs undersized) 2023-10-09T05:47:26.078 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:47:26.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:26.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:47:26.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:26.211 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:25 smithi191 ceph-mon[161669]: pgmap v853: 73 pgs: 73 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:26.211 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:25 smithi191 ceph-mon[161669]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14048/858 objects degraded (1637.296%), 7 pgs degraded, 2 pgs undersized) 2023-10-09T05:47:26.214 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:26.264 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:26.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:26.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:26.371 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:26.409 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:26.636 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:26.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:26.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:26.721 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:26.722 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:26.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:26.774 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:26.823 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:26.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:26.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:26.930 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:47:26.961 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:27.054 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:27.374 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:47:27.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:27.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:47:27.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:27.511 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:27.559 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:27.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:27.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:27.618 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:47:27.618 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:47:27.618 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:47:27.619 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:47:27.619 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:47:27.664 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:47:27.702 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:28.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:27 smithi008 ceph-mon[184418]: pgmap v854: 73 pgs: 73 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3512999451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:27 smithi008 ceph-mon[191227]: pgmap v854: 73 pgs: 73 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3512999451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:28.280 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:47:28.280 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:47:28.280 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:47:28.280 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:47:28.280 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:47:28.280 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:47:28.281 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:47:28.281 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:47:28.281 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:47:28.281 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:47:28.281 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:47:28.281 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:47:28.281 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:47:28.281 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:47:28.281 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:47:28.281 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:47:28.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:27 smithi191 ceph-mon[161669]: pgmap v854: 73 pgs: 73 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3512999451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:28.893 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:47:30.020 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:47:30.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:30.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:47:30.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:30.154 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:29 smithi191 ceph-mon[161669]: pgmap v855: 73 pgs: 73 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:30.154 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:30.154 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:30.154 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:30.157 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:30.204 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:30.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:30.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:29 smithi008 ceph-mon[184418]: pgmap v855: 73 pgs: 73 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:29 smithi008 ceph-mon[191227]: pgmap v855: 73 pgs: 73 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:30.310 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:30.338 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:30.383 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:30.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:30.490 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:30.517 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:30.558 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:30.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:30.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:30.664 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:47:30.690 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:47:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:47:30] "GET /metrics HTTP/1.1" 200 34195 "" "Prometheus/2.33.4" 2023-10-09T05:47:30.694 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:31.231 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:47:31.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:31.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:47:31.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:31.368 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:31.416 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:31.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:31.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:31.524 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:47:31.552 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:31.922 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:47:31.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:31.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:47:32.056 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:32.088 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:32.135 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:32.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:32.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:32.169 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:31 smithi008 ceph-mon[184418]: pgmap v856: 73 pgs: 73 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:32.170 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:32.170 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:32.170 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:31 smithi008 ceph-mon[191227]: pgmap v856: 73 pgs: 73 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:32.170 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:32.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:32.240 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:32.267 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:31 smithi191 ceph-mon[161669]: pgmap v856: 73 pgs: 73 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:32.268 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:32.268 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:32.270 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:32.316 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:32.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:32.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:32.387 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:32.422 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:47:32.458 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:32.548 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:47:32.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:32.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:47:32.652 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:32.681 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:32.726 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:32.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:32.833 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:32.863 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:32.915 INFO:teuthology.orchestra.run.smithi008.stdout:moved 5 / 219 (2.28311%) 2023-10-09T05:47:32.915 INFO:teuthology.orchestra.run.smithi008.stdout:avg 31.2857 2023-10-09T05:47:32.915 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 7.18559 -> 7.62916 (expected baseline 5.17845) 2023-10-09T05:47:32.916 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.5 with 19 -> 17 pgs (0.607306 -> 0.543379 * mean) 2023-10-09T05:47:32.916 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.4 with 37 -> 35 pgs (1.18265 -> 1.11872 * mean) 2023-10-09T05:47:32.916 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:47:32.916 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T05:47:32.916 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 0.05 2023-10-09T05:47:32.916 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:47:32.916 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0063 2023-10-09T05:47:32.916 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0070 2023-10-09T05:47:32.916 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 weight 1.0000 -> 0.9500 2023-10-09T05:47:32.916 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 weight 0.7324 -> 0.6824 2023-10-09T05:47:32.916 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 0.8699 -> 0.8925 2023-10-09T05:47:32.919 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:47:32.921 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_3' created 2023-10-09T05:47:32.964 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:32.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:32.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:32 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2439213526' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:47:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:32 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:47:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:47:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:47:33.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:33.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:32 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1374158629' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T05:47:33.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'58494','4':'62260','5':'44721'}"}]: dispatch 2023-10-09T05:47:33.068 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:33.094 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:33.136 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:33.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:33.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:33.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2439213526' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:47:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:47:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:47:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:47:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1374158629' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T05:47:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'58494','4':'62260','5':'44721'}"}]: dispatch 2023-10-09T05:47:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2439213526' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:47:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:47:33.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:47:33.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:47:33.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:33.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1374158629' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T05:47:33.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'58494','4':'62260','5':'44721'}"}]: dispatch 2023-10-09T05:47:33.243 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:33.272 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:34.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: pgmap v857: 73 pgs: 73 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:34.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:34.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:47:34.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:47:34.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1374158629' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T05:47:34.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'58494','4':'62260','5':'44721'}"}]': finished 2023-10-09T05:47:34.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T05:47:34.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[184418]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: pgmap v857: 73 pgs: 73 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:34.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:34.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:47:34.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:47:34.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1374158629' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T05:47:34.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'58494','4':'62260','5':'44721'}"}]': finished 2023-10-09T05:47:34.188 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T05:47:34.188 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.188 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.188 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.188 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.188 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.188 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:33 smithi008 ceph-mon[191227]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.189 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:47:34.189 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T05:47:34.201 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:47:34.225 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:34.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: pgmap v857: 73 pgs: 73 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:47:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1374158629' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'58494','4':'62260','5':'44721'}"}]': finished 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:33 smithi191 ceph-mon[161669]: failed to encode map e285 with expected crc 2023-10-09T05:47:34.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:34.362 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:34.410 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:34.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:34.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:34.516 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:34.545 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:34.916 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[191227]: failed to encode map e286 with expected crc 2023-10-09T05:47:34.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[191227]: failed to encode map e286 with expected crc 2023-10-09T05:47:34.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[191227]: failed to encode map e286 with expected crc 2023-10-09T05:47:34.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[191227]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T05:47:34.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[191227]: failed to encode map e286 with expected crc 2023-10-09T05:47:34.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[191227]: failed to encode map e286 with expected crc 2023-10-09T05:47:34.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[191227]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.174 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[184418]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.174 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[184418]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.175 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[184418]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.175 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[184418]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T05:47:35.175 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[184418]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.175 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[184418]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.175 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:34 smithi008 ceph-mon[184418]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.282 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:35.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:35.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:34 smithi191 ceph-mon[161669]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:34 smithi191 ceph-mon[161669]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:34 smithi191 ceph-mon[161669]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:34 smithi191 ceph-mon[161669]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T05:47:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:34 smithi191 ceph-mon[161669]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:34 smithi191 ceph-mon[161669]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:34 smithi191 ceph-mon[161669]: failed to encode map e286 with expected crc 2023-10-09T05:47:35.339 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T05:47:35.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:35.417 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:35.465 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:35.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:35.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:35.549 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:35.572 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:35.600 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:35.621 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:35.644 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:35.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:35.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:35.650 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:47:35.651 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:47:35.749 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:35.782 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:35.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:35 smithi008 ceph-mon[184418]: pgmap v860: 89 pgs: 1 peering, 5 creating+peering, 11 unknown, 72 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:35.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:35 smithi008 ceph-mon[184418]: failed to encode map e287 with expected crc 2023-10-09T05:47:35.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:35 smithi008 ceph-mon[184418]: failed to encode map e287 with expected crc 2023-10-09T05:47:35.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:35 smithi008 ceph-mon[184418]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T05:47:35.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:35 smithi008 ceph-mon[191227]: pgmap v860: 89 pgs: 1 peering, 5 creating+peering, 11 unknown, 72 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:35.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:35 smithi008 ceph-mon[191227]: failed to encode map e287 with expected crc 2023-10-09T05:47:35.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:35 smithi008 ceph-mon[191227]: failed to encode map e287 with expected crc 2023-10-09T05:47:35.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:35 smithi008 ceph-mon[191227]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T05:47:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:35 smithi191 ceph-mon[161669]: pgmap v860: 89 pgs: 1 peering, 5 creating+peering, 11 unknown, 72 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:35 smithi191 ceph-mon[161669]: failed to encode map e287 with expected crc 2023-10-09T05:47:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:35 smithi191 ceph-mon[161669]: failed to encode map e287 with expected crc 2023-10-09T05:47:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:35 smithi191 ceph-mon[161669]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T05:47:36.568 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:36.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:36.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:36.672 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:36.704 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:36.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:36 smithi008 ceph-mon[191227]: failed to encode map e287 with expected crc 2023-10-09T05:47:37.174 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:37.175 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:36 smithi008 ceph-mon[184418]: failed to encode map e287 with expected crc 2023-10-09T05:47:37.176 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:47:37.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:37.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:37.265 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:37.278 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:37.288 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:47:37.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:36 smithi191 ceph-mon[161669]: failed to encode map e287 with expected crc 2023-10-09T05:47:37.309 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:37.357 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:37.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:37.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:37.463 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:37.503 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:37.555 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:37.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:37.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:37.659 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:47:37.689 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:38.180 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:37 smithi008 ceph-mon[184418]: pgmap v862: 89 pgs: 1 peering, 5 creating+peering, 5 unknown, 78 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:38.180 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:37 smithi008 ceph-mon[191227]: pgmap v862: 89 pgs: 1 peering, 5 creating+peering, 5 unknown, 78 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:38.254 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:47:38.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:38.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:47:38.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:37 smithi191 ceph-mon[161669]: pgmap v862: 89 pgs: 1 peering, 5 creating+peering, 5 unknown, 78 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:38.360 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:38.391 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:39.161 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:47:39.161 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:47:39.161 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:47:39.161 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:47:39.161 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:47:39.161 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:47:39.161 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:47:39.161 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:47:39.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:47:39.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:47:39.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:47:39.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:47:39.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:47:39.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:47:39.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:47:39.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:47:39.447 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:39.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:39.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:39.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:39.583 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:39.632 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:39.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:39.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:39.739 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:39.767 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:39.812 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:39.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:39.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:39.919 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:39.952 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:40.007 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:40.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:40.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:40.014 INFO:teuthology.orchestra.run.smithi008.stdout:moved 1 / 267 (0.374532%) 2023-10-09T05:47:40.014 INFO:teuthology.orchestra.run.smithi008.stdout:avg 38.1429 2023-10-09T05:47:40.014 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 8.00765 -> 8.11398 (expected baseline 5.71786) 2023-10-09T05:47:40.014 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.5 with 23 -> 23 pgs (0.602996 -> 0.602996 * mean) 2023-10-09T05:47:40.014 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.3 with 46 -> 47 pgs (1.20599 -> 1.23221 * mean) 2023-10-09T05:47:40.014 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:47:40.014 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T05:47:40.014 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 1 2023-10-09T05:47:40.014 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:47:40.014 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0062 2023-10-09T05:47:40.015 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0623 2023-10-09T05:47:40.015 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 0.8925 -> 0.9151 2023-10-09T05:47:40.016 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T05:47:40.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:40.139 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:40.181 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:40.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:40.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:39 smithi008 ceph-mon[184418]: pgmap v863: 89 pgs: 1 peering, 5 creating+peering, 83 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1774455801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:40.215 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:47:40 smithi008 conmon[177229]: 2023-10-09T05:47:40.012+0000 7f020285b700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-10-09T05:47:40.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:39 smithi008 ceph-mon[191227]: pgmap v863: 89 pgs: 1 peering, 5 creating+peering, 83 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:40.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1774455801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:40.288 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:40.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:39 smithi191 ceph-mon[161669]: pgmap v863: 89 pgs: 1 peering, 5 creating+peering, 83 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:40.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1774455801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:40.317 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:40.414 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:40.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:40.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:40.518 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:47:40.546 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:40.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:47:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:47:40] "GET /metrics HTTP/1.1" 200 34195 "" "Prometheus/2.33.4" 2023-10-09T05:47:40.882 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:47:40.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:40.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:47:40.935 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:40.953 INFO:teuthology.orchestra.run.smithi008.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T05:47:40.986 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:40.988 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:47:41.023 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:41.164 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:40 smithi008 ceph-mon[184418]: from='client.35820 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:41.164 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:40 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3619536550' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:47:41.165 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:40 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:47:41.165 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:40 smithi008 ceph-mon[191227]: from='client.35820 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:41.165 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:40 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3619536550' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:47:41.165 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:40 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:47:41.221 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:47:41.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:41.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:47:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:40 smithi191 ceph-mon[161669]: from='client.35820 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:47:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:40 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3619536550' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:47:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:40 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:47:41.325 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:41.357 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:41.406 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:41.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:41.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:41.511 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:41.541 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:41.619 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:41.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:41.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:41.723 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:47:41.750 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:41.846 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:47:41.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:41.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:47:41.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:41.980 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:42.028 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:42.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:42.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:42.093 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:41 smithi008 ceph-mon[184418]: pgmap v864: 89 pgs: 89 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:42.094 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:41 smithi008 ceph-mon[184418]: failed to encode map e288 with expected crc 2023-10-09T05:47:42.094 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:41 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:47:42.094 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:41 smithi008 ceph-mon[184418]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T05:47:42.094 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:41 smithi008 ceph-mon[184418]: failed to encode map e288 with expected crc 2023-10-09T05:47:42.094 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:41 smithi008 ceph-mon[191227]: pgmap v864: 89 pgs: 89 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:42.094 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:41 smithi008 ceph-mon[191227]: failed to encode map e288 with expected crc 2023-10-09T05:47:42.094 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:41 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:47:42.094 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:41 smithi008 ceph-mon[191227]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T05:47:42.095 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:41 smithi008 ceph-mon[191227]: failed to encode map e288 with expected crc 2023-10-09T05:47:42.134 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:42.161 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:42.204 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:42.208 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:41 smithi191 ceph-mon[161669]: pgmap v864: 89 pgs: 89 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:42.208 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:41 smithi191 ceph-mon[161669]: failed to encode map e288 with expected crc 2023-10-09T05:47:42.208 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:41 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:47:42.208 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:41 smithi191 ceph-mon[161669]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T05:47:42.208 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:41 smithi191 ceph-mon[161669]: failed to encode map e288 with expected crc 2023-10-09T05:47:42.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:42.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:42.310 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:42.340 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:42.840 DEBUG:teuthology.orchestra.run.smithi008:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T05:47:42.843 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:47:42.843 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:47:42.908 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:47:42.945 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:42.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:42.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:43.049 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:43.076 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:43.111 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:hints = 1 2023-10-09T05:47:43.111 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:47:43.111 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object prefix: benchmark_data_smithi008_432048 2023-10-09T05:47:43.112 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:47:43.112 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:47:43.365 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:43.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:43.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:43.470 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:43.518 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:43.587 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:43.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:43.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:43.694 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:43.757 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:43.907 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:43.935 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:43.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:43.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:43.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:43 smithi008 ceph-mon[191227]: pgmap v866: 89 pgs: 89 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:43.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 5]}]: dispatch 2023-10-09T05:47:43.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:43 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:44.042 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:44.077 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:44.112 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 1 16 3074 3058 191.115 191.125 0.00520839 0.0052154 2023-10-09T05:47:44.145 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:44.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:44.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:44.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:43 smithi008 ceph-mon[184418]: pgmap v866: 89 pgs: 89 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 5]}]: dispatch 2023-10-09T05:47:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:43 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:44.252 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:43 smithi191 ceph-mon[161669]: pgmap v866: 89 pgs: 89 active+clean; 4.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T05:47:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 5]}]: dispatch 2023-10-09T05:47:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:43 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:44.324 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:44.414 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:47:44.414 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:47:44.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:47:44.414 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:3852 8.d e289/285 8:bd3c44a0:::benchmark_data_smithi008_432048_object3851:head v 289'249, mlcod=289'247)", 2023-10-09T05:47:44.414 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:47:44.401367+0000", 2023-10-09T05:47:44.414 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0028084080000000001, 2023-10-09T05:47:44.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028396039999999999, 2023-10-09T05:47:44.415 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:47:44.415 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:47:44.415 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:47:44.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:47:44.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:47:44.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:47:44.401367+0000", 2023-10-09T05:47:44.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:47:44.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:47:44.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:47:44.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:47:44.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:47:44.401367+0000", 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:47:44.401359+0000", 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999919 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:47:44.401454+0000", 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5159000000000002e-05 2023-10-09T05:47:44.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:47:44.401470+0000", 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6458000000000001e-05 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:47:44.401479+0000", 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.704e-06 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:47:44.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:47:44.401526+0000", 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6539000000000003e-05 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:47:44.401548+0000", 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1963e-05 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:47:44.418 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:47:44.419 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 1 2023-10-09T05:47:44.419 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:47:44.441 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:44.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:44.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:44.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:44.591 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:44.682 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:44.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:44.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:44.788 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:47:44.816 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:44.986 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:47:44.990 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: failed to encode map e289 with expected crc 2023-10-09T05:47:44.990 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 5]}]': finished 2023-10-09T05:47:44.990 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T05:47:44.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: failed to encode map e289 with expected crc 2023-10-09T05:47:44.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: failed to encode map e289 with expected crc 2023-10-09T05:47:44.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: failed to encode map e289 with expected crc 2023-10-09T05:47:44.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: failed to encode map e289 with expected crc 2023-10-09T05:47:44.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:44.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2659150834' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:47:44.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:47:44.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:47:44.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2659150834' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:47:44.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:44 smithi191 ceph-mon[161669]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T05:47:45.112 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 2 15 5590 5575 174.203 157.312 0.00239202 0.00572876 2023-10-09T05:47:45.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: failed to encode map e289 with expected crc 2023-10-09T05:47:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 5]}]': finished 2023-10-09T05:47:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T05:47:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: failed to encode map e289 with expected crc 2023-10-09T05:47:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: failed to encode map e289 with expected crc 2023-10-09T05:47:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: failed to encode map e289 with expected crc 2023-10-09T05:47:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: failed to encode map e289 with expected crc 2023-10-09T05:47:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:45.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2659150834' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:47:45.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:47:45.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:47:45.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2659150834' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:47:45.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[184418]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T05:47:45.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: failed to encode map e289 with expected crc 2023-10-09T05:47:45.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 5]}]': finished 2023-10-09T05:47:45.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T05:47:45.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: failed to encode map e289 with expected crc 2023-10-09T05:47:45.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: failed to encode map e289 with expected crc 2023-10-09T05:47:45.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: failed to encode map e289 with expected crc 2023-10-09T05:47:45.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: failed to encode map e289 with expected crc 2023-10-09T05:47:45.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:45.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2659150834' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:47:45.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:47:45.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:47:45.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2659150834' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:47:45.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:44 smithi008 ceph-mon[191227]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T05:47:45.937 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:47:45.938 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:47:45.938 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T05:47:45.938 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:47:45.938 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:47:45.938 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T05:47:46.112 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 3 16 8937 8921 185.836 209.125 0.00288464 0.00537335 2023-10-09T05:47:46.134 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:47:46.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:46.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:47:46.240 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:47:46.295 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[184418]: pgmap v868: 89 pgs: 89 active+clean; 53 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 6.0 MiB/s wr, 96 op/s 2023-10-09T05:47:46.295 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[184418]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.295 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[184418]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.296 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[184418]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.296 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[184418]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.296 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[184418]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.296 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[191227]: pgmap v868: 89 pgs: 89 active+clean; 53 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 6.0 MiB/s wr, 96 op/s 2023-10-09T05:47:46.297 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[191227]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.297 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[191227]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.297 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[191227]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.297 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[191227]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.297 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:45 smithi008 ceph-mon[191227]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.299 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:45 smithi191 ceph-mon[161669]: pgmap v868: 89 pgs: 89 active+clean; 53 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 6.0 MiB/s wr, 96 op/s 2023-10-09T05:47:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:45 smithi191 ceph-mon[161669]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:45 smithi191 ceph-mon[161669]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:45 smithi191 ceph-mon[161669]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:45 smithi191 ceph-mon[161669]: failed to encode map e290 with expected crc 2023-10-09T05:47:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:45 smithi191 ceph-mon[161669]: failed to encode map e290 with expected crc 2023-10-09T05:47:47.112 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 4 16 12112 12096 188.982 198.438 0.00373593 0.00528501 2023-10-09T05:47:48.112 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 5 16 14674 14658 183.207 160.125 0.00144366 0.00540575 2023-10-09T05:47:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:47 smithi191 ceph-mon[161669]: pgmap v870: 89 pgs: 89 active+clean; 179 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 29 MiB/s wr, 463 op/s 2023-10-09T05:47:48.340 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:47 smithi008 ceph-mon[184418]: pgmap v870: 89 pgs: 89 active+clean; 179 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 29 MiB/s wr, 463 op/s 2023-10-09T05:47:48.341 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:47 smithi008 ceph-mon[191227]: pgmap v870: 89 pgs: 89 active+clean; 179 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 29 MiB/s wr, 463 op/s 2023-10-09T05:47:48.926 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:47:48.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:48.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:47:49.032 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:49.112 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 6 16 17548 17532 182.607 179.625 0.00689221 0.00547006 2023-10-09T05:47:49.179 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:49.309 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:49.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:49.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:49.366 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:47:49.416 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:49.458 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:49.525 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:49.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:49.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:49.632 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:47:49.688 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:49.700 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:49.815 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:47:49.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:49.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:47:49.925 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:47:49.981 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:50.094 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:47:50.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:50.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:47:50.112 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 7 16 19635 19619 175.152 130.438 0.00590499 0.00570264 2023-10-09T05:47:50.171 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:47:50.195 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:47:50.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:50.233 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:47:50.248 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:50.270 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:47:50.279 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:49 smithi008 ceph-mon[191227]: pgmap v871: 89 pgs: 89 active+clean; 403 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s 2023-10-09T05:47:50.280 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:49 smithi008 ceph-mon[184418]: pgmap v871: 89 pgs: 89 active+clean; 403 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s 2023-10-09T05:47:50.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:49 smithi191 ceph-mon[161669]: pgmap v871: 89 pgs: 89 active+clean; 403 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 54 MiB/s wr, 862 op/s 2023-10-09T05:47:50.310 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:47:50.351 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:47:50.375 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:50.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:50.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:50.388 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:47:50.430 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:47:50.432 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:47:50.433 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:47:50.433 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:47:50.433 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:47:50.433 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:47:50.433 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:47:50.433 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:47:50.433 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:47:50.486 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:50.541 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:47:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:47:50] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.33.4" 2023-10-09T05:47:50.548 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:50.655 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:50.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:50.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:50.763 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:47:50.821 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:51.113 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 8 16 22041 22025 172.053 150.375 0.00655317 0.00580676 2023-10-09T05:47:51.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:51 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/801985976' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:51.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:51 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/801985976' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:51.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:51 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/801985976' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:47:51.525 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:47:52.113 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 9 16 24054 24038 166.914 125.812 0.00558211 0.00598561 2023-10-09T05:47:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:52 smithi191 ceph-mon[161669]: pgmap v872: 89 pgs: 89 active+clean; 1015 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T05:47:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:52 smithi008 ceph-mon[184418]: pgmap v872: 89 pgs: 89 active+clean; 1015 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T05:47:52.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:52 smithi008 ceph-mon[191227]: pgmap v872: 89 pgs: 89 active+clean; 1015 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T05:47:52.918 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:52.935 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:47:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:52.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:47:53.050 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:53.114 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 10 16 25839 25823 161.357 111.562 0.00750371 0.00619125 2023-10-09T05:47:53.117 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:53.438 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:47:53.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:53.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:47:53.533 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:53.540 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:53.707 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:53.814 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:47:53.814 INFO:teuthology.orchestra.run.smithi008.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:47:53.814 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:47:53.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:53.880 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:53.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:53.981 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:47:54.033 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:54.114 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 11 16 27837 27821 158.04 124.875 0.00684476 0.00632159 2023-10-09T05:47:54.203 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:47:54.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:54.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:47:54.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:54 smithi008 ceph-mon[184418]: pgmap v873: 89 pgs: 89 active+clean; 1.2 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T05:47:54.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:54 smithi008 ceph-mon[191227]: pgmap v873: 89 pgs: 89 active+clean; 1.2 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T05:47:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:54 smithi191 ceph-mon[161669]: pgmap v873: 89 pgs: 89 active+clean; 1.2 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T05:47:54.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:54.371 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:54.460 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:54.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:54.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:54.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:54.622 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:54.723 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:54.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:54.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:54.831 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:47:54.898 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:54.973 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:47:55.114 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 12 16 29954 29938 155.895 132.312 0.0096414 0.00640985 2023-10-09T05:47:55.994 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:47:55.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:55.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:47:56.098 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:47:56.114 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 13 16 31774 31758 152.652 113.75 0.00198926 0.00651857 2023-10-09T05:47:56.124 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:56 smithi191 ceph-mon[161669]: pgmap v874: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-09T05:47:56.410 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:47:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:56.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:47:56.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:56 smithi008 ceph-mon[184418]: pgmap v874: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-09T05:47:56.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:56 smithi008 ceph-mon[191227]: pgmap v874: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-09T05:47:56.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:56.546 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:56.894 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:56.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:56.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:57.003 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:47:57.053 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:57.114 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 14 16 33418 33402 149.087 102.75 0.00787432 0.00670318 2023-10-09T05:47:57.381 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:47:57.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:57.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:47:57.488 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:57.499 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:57.533 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:57.605 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:57.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:57.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:57.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:47:57.765 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:57.823 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:47:57.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:47:57.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:47:57.927 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:47:58.008 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:47:58.028 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:58 smithi008 ceph-mon[184418]: pgmap v875: 89 pgs: 89 active+clean; 1.7 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 145 MiB/s wr, 2.31k op/s 2023-10-09T05:47:58.114 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 15 16 35037 35021 145.893 101.188 0.011057 0.00684842 2023-10-09T05:47:58.288 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:47:58.289 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:47:58.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:58 smithi191 ceph-mon[161669]: pgmap v875: 89 pgs: 89 active+clean; 1.7 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 145 MiB/s wr, 2.31k op/s 2023-10-09T05:47:58.324 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:58 smithi008 ceph-mon[191227]: pgmap v875: 89 pgs: 89 active+clean; 1.7 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 145 MiB/s wr, 2.31k op/s 2023-10-09T05:47:59.065 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:47:59.115 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 16 16 36751 36735 143.47 107.125 0.00573441 0.00696369 2023-10-09T05:47:59.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:59 smithi191 ceph-mon[161669]: pgmap v876: 89 pgs: 89 active+clean; 1.8 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T05:47:59.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2068004237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:47:59.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:59.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:47:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:59.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:59 smithi008 ceph-mon[184418]: pgmap v876: 89 pgs: 89 active+clean; 1.8 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T05:47:59.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2068004237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:47:59.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:47:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:59.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:59 smithi008 ceph-mon[191227]: pgmap v876: 89 pgs: 89 active+clean; 1.8 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T05:47:59.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2068004237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:47:59.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:47:59.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:47:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:00.115 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 17 16 38185 38169 140.302 89.625 0.00328666 0.00709582 2023-10-09T05:48:00.144 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:00.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:00.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[191227]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2068004237' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:48:00.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[191227]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T05:48:00.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[191227]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.148 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[191227]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.148 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[191227]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.148 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[191227]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:00.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[184418]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2068004237' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:48:00.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[184418]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T05:48:00.148 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[184418]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[184418]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[184418]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.149 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:00 smithi008 ceph-mon[184418]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:00.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:00.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:00.298 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:00 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:00.298 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:00 smithi191 ceph-mon[161669]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.298 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2068004237' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:48:00.298 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:00 smithi191 ceph-mon[161669]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T05:48:00.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:00 smithi191 ceph-mon[161669]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:00 smithi191 ceph-mon[161669]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:00 smithi191 ceph-mon[161669]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:00 smithi191 ceph-mon[161669]: failed to encode map e291 with expected crc 2023-10-09T05:48:00.312 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:00.400 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:00.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:00.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:00.506 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:00.549 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:00.591 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:00.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:00.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:00.698 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:00.756 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:00.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:48:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:48:00] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.33.4" 2023-10-09T05:48:01.115 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 18 16 39597 39581 137.41 88.25 0.0121249 0.00727213 2023-10-09T05:48:01.202 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:01.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:01.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:01.290 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:01 smithi008 ceph-mon[184418]: pgmap v878: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T05:48:01.290 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:01 smithi008 ceph-mon[191227]: pgmap v878: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T05:48:01.292 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:48:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:01 smithi191 ceph-mon[161669]: pgmap v878: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T05:48:01.311 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:01.367 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:01.443 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:01.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:01.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:01.549 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:01.621 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:01.724 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:01.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:01.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:01.832 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:01.856 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:48:01.885 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:01.897 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:48:01.943 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:48:01.992 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:48:02.023 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:02.027 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:48:02.059 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:02.068 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:48:02.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:02.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:02.105 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:48:02.115 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 19 16 41148 41132 135.28 96.9375 0.00735831 0.00738593 2023-10-09T05:48:02.144 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:48:02.146 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:48:02.146 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:48:02.146 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:48:02.147 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:48:02.147 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:48:02.147 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:48:02.147 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:48:02.147 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:48:02.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:02.177 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:02 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1270324441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:02.177 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:02 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1270324441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:02.210 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:48:02.211 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:48:02.211 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:48:02.211 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 1.000000 2023-10-09T05:48:02.211 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T05:48:02.223 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:02.323 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:02.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:02 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1270324441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:02.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:02.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:02.434 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:02.489 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:03.115 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:48:03.116 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:48:03.115980+0000 min lat: 0.00124024 max lat: 0.160396 avg lat: 0.00747501 2023-10-09T05:48:03.116 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:48:03.116 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 20 16 42715 42699 133.412 97.9375 0.00224923 0.00747501 2023-10-09T05:48:03.695 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:03.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:03.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:03.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:03.832 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:03.930 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:03.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:03.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:04.038 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:04.115 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 21 16 44181 44165 131.422 91.625 0.00920404 0.00760436 2023-10-09T05:48:04.122 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:04.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:03 smithi008 ceph-mon[184418]: pgmap v879: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:48:04.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:03 smithi008 ceph-mon[191227]: pgmap v879: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:48:04.269 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:04.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:04.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:03 smithi191 ceph-mon[161669]: pgmap v879: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T05:48:04.376 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:04.406 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:04.448 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:04.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:04.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:04.554 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:04.582 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:04.626 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:04.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:04.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:04.732 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:04.792 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:04.901 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:04.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:04.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:04.979 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:04.984 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:05.010 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:05.085 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:05.115 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 22 16 45382 45366 128.858 75.0625 0.00563988 0.00775298 2023-10-09T05:48:05.222 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:05.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:05.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:05.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:05.377 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:05.472 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:05.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:05.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:05.580 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:05.648 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:05.837 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:48:06.053 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:06.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:06.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:06.116 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 23 16 46869 46853 127.296 92.9375 0.0127568 0.00784792 2023-10-09T05:48:06.159 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:06.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:05 smithi008 ceph-mon[184418]: pgmap v880: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-09T05:48:06.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:05 smithi008 ceph-mon[191227]: pgmap v880: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-09T05:48:06.243 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:06.289 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:48:06.289 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:48:06.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:48:06.289 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:48:06.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.289 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102141 7.8 7:1ebffa85:::benchmark_data_smithi008_354649_object7645:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:48:06.290 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.894031+0000", 2023-10-09T05:48:06.290 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 287.24738908400002, 2023-10-09T05:48:06.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3312663869999999, 2023-10-09T05:48:06.290 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.290 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.290 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.290 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.290 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.290 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102141 2023-10-09T05:48:06.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.290 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894031+0000", 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894031+0000", 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894030+0000", 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894035+0000", 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8479999999999997e-06 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894046+0000", 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0501e-05 2023-10-09T05:48:06.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894055+0000", 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0860000000000008e-06 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894107+0000", 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2182000000000003e-05 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894119+0000", 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1722000000000001e-05 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870208+0000", 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97608879900000001 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870222+0000", 2023-10-09T05:48:06.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3732e-05 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.209616+0000", 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3393948500000001 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.209842+0000", 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00022551799999999999 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.210051+0000", 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020889199999999999 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.221116+0000", 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011064981 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.221206+0000", 2023-10-09T05:48:06.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0245000000000001e-05 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.221293+0000", 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7059999999999997e-05 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.221361+0000", 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7506e-05 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.298 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.298 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.225298+0000", 2023-10-09T05:48:06.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039369940000000001 2023-10-09T05:48:06.298 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.298 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.298 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.298 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.298 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102155 7.8 7:1c0eb140:::benchmark_data_smithi008_354649_object7659:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:48:06.298 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.907631+0000", 2023-10-09T05:48:06.298 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 287.23378960399998, 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.330968441, 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102155 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907631+0000", 2023-10-09T05:48:06.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907631+0000", 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907630+0000", 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907635+0000", 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.152e-06 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907646+0000", 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1117e-05 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907651+0000", 2023-10-09T05:48:06.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5770000000000001e-06 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907702+0000", 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0269999999999998e-05 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907715+0000", 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2850000000000001e-05 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870248+0000", 2023-10-09T05:48:06.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96253387899999998 2023-10-09T05:48:06.303 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.303 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.303 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870257+0000", 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3550000000000005e-06 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.217936+0000", 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3476795529999999 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.218069+0000", 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013219300000000001 2023-10-09T05:48:06.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.218202+0000", 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013337599999999999 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.229249+0000", 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011046554 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.233501+0000", 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042527809999999997 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.234614+0000", 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001112273 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.234665+0000", 2023-10-09T05:48:06.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1879999999999998e-05 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.238599+0000", 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039336279999999998 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102163 7.8 7:18e716b9:::benchmark_data_smithi008_354649_object7667:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.920286+0000", 2023-10-09T05:48:06.307 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 287.22113393000001, 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.322594327, 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102163 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920286+0000", 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920286+0000", 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920285+0000", 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920291+0000", 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7899999999999996e-06 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920302+0000", 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1046000000000001e-05 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920310+0000", 2023-10-09T05:48:06.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6930000000000007e-06 2023-10-09T05:48:06.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920371+0000", 2023-10-09T05:48:06.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1540999999999999e-05 2023-10-09T05:48:06.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920382+0000", 2023-10-09T05:48:06.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0910000000000001e-05 2023-10-09T05:48:06.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870283+0000", 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94990104200000003 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870291+0000", 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1130000000000001e-06 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.225356+0000", 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.355064152 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.225545+0000", 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000189061 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.225732+0000", 2023-10-09T05:48:06.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018689299999999999 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.233511+0000", 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0077791470000000001 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.238664+0000", 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051530389999999999 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.238929+0000", 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00026558199999999999 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.238994+0000", 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4844999999999993e-05 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.242881+0000", 2023-10-09T05:48:06.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038864920000000001 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102168 7.8 7:17f42d0f:::benchmark_data_smithi008_354649_object7672:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.928352+0000", 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 287.21306845999999, 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3191062200000001, 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.316 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.317 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.317 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.317 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102168 2023-10-09T05:48:06.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.317 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928352+0000", 2023-10-09T05:48:06.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928352+0000", 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928350+0000", 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928359+0000", 2023-10-09T05:48:06.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3429999999999996e-06 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928376+0000", 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7161999999999999e-05 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928386+0000", 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0054000000000001e-05 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928449+0000", 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2954999999999996e-05 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928461+0000", 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2351000000000001e-05 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870577+0000", 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94211601300000003 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870588+0000", 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0668e-05 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230146+0000", 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3595585649999999 2023-10-09T05:48:06.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230346+0000", 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000200001 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230526+0000", 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017919 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.322 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.323 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.233655+0000", 2023-10-09T05:48:06.323 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031297579999999998 2023-10-09T05:48:06.323 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.323 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.323 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.323 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.238696+0000", 2023-10-09T05:48:06.323 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050402520000000003 2023-10-09T05:48:06.323 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.323 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.323 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.323 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.242968+0000", 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042724520000000004 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243043+0000", 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5434999999999999e-05 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.247458+0000", 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0044145460000000001 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.324 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:111928 7.e 7:7b8ee2cb:::benchmark_data_smithi008_354649_object17432:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:37.829725+0000", 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 268.31169513399999, 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27588755799999998, 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.325 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 111928 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.829725+0000", 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.829725+0000", 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.829723+0000", 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.829733+0000", 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5510000000000005e-06 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.829749+0000", 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5744999999999999e-05 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.829757+0000", 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6060000000000003e-06 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.829811+0000", 2023-10-09T05:48:06.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3207000000000001e-05 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.829935+0000", 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000124855 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.830043+0000", 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000107755 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.837252+0000", 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0072088880000000001 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.837310+0000", 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7859999999999998e-05 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:38.105534+0000", 2023-10-09T05:48:06.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26822458599999999 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:38.105578+0000", 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3442999999999997e-05 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:38.105613+0000", 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4758000000000002e-05 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.331 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:111930 7.e 7:7f415922:::benchmark_data_smithi008_354649_object17434:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:37.830286+0000", 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 268.31113446900002, 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27542427200000003, 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.332 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 111930 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.830286+0000", 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.830286+0000", 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.333 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.830285+0000", 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.830290+0000", 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.544e-06 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.334 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.830298+0000", 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4600000000000003e-06 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.830304+0000", 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4489999999999998e-06 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.833631+0000", 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033273830000000002 2023-10-09T05:48:06.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.833739+0000", 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000107817 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.833813+0000", 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4189000000000004e-05 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.837393+0000", 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035801909999999999 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.839408+0000", 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020147210000000001 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:38.105650+0000", 2023-10-09T05:48:06.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26624176599999999 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:38.105678+0000", 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7917999999999999e-05 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:38.105710+0000", 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2537999999999999e-05 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.338 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:111934 7.8 7:11e9502d:::benchmark_data_smithi008_354649_object17438:head [delete] snapc 0=[] ondisk+write+known_if_redirected e235)", 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:37.837605+0000", 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 268.30381513200001, 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.265973395, 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.339 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 111934 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.837605+0000", 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.837605+0000", 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.837604+0000", 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.837610+0000", 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.122e-06 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.837622+0000", 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1718e-05 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.837629+0000", 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4460000000000004e-06 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.842804+0000", 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051745200000000002 2023-10-09T05:48:06.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.842861+0000", 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7364999999999997e-05 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.842933+0000", 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2256999999999996e-05 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.343 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.844385+0000", 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001451763 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.846951+0000", 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025656440000000002 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.860128+0000", 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013176973 2023-10-09T05:48:06.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.345 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.345 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:37.860165+0000", 2023-10-09T05:48:06.345 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7421e-05 2023-10-09T05:48:06.345 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.345 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.345 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:38.103579+0000", 2023-10-09T05:48:06.345 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24341344200000001 2023-10-09T05:48:06.345 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.345 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:122824 7.8 7:1c9459b6:::benchmark_data_smithi008_354649_object28328:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:57.631372+0000", 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 248.51004848900001, 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26740687299999999, 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.346 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 122824 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.631372+0000", 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.631372+0000", 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.631371+0000", 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.631374+0000", 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6639999999999998e-06 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.631380+0000", 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6210000000000001e-06 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.631383+0000", 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1319999999999998e-06 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.643430+0000", 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012046714 2023-10-09T05:48:06.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.643544+0000", 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114518 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.643699+0000", 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000155079 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.660426+0000", 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016726857000000001 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.666782+0000", 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006355622 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.895037+0000", 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.22825522500000001 2023-10-09T05:48:06.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.352 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.895098+0000", 2023-10-09T05:48:06.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1198999999999995e-05 2023-10-09T05:48:06.352 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.352 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:57.898779+0000", 2023-10-09T05:48:06.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003680628 2023-10-09T05:48:06.352 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.352 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125676 7.8 7:15209edd:::benchmark_data_smithi008_354649_object31180:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.953092+0000", 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 243.18832778800001, 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33119336100000002, 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.353 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125676 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953092+0000", 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953092+0000", 2023-10-09T05:48:06.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953091+0000", 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953100+0000", 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9600000000000006e-06 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953119+0000", 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9273999999999999e-05 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953130+0000", 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0744000000000001e-05 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953218+0000", 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8712e-05 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953359+0000", 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014026800000000001 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.284269+0000", 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33090979500000001 2023-10-09T05:48:06.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.358 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.358 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.358 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.284286+0000", 2023-10-09T05:48:06.358 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7306999999999999e-05 2023-10-09T05:48:06.358 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.358 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.358 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.358 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.358 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.358 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:125684 7.3 e238/213 7:c9988d38:::benchmark_data_smithi008_354649_object31188:head v 238'7873, mlcod=238'7871)", 2023-10-09T05:48:06.358 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.980604+0000", 2023-10-09T05:48:06.359 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 243.160816689, 2023-10-09T05:48:06.359 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.34659507299999998, 2023-10-09T05:48:06.359 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.359 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:48:06.359 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.359 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.359 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.359 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980604+0000", 2023-10-09T05:48:06.359 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980604+0000", 2023-10-09T05:48:06.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980602+0000", 2023-10-09T05:48:06.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:06.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980617+0000", 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4684999999999999e-05 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980632+0000", 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4992e-05 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980644+0000", 2023-10-09T05:48:06.362 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1753000000000001e-05 2023-10-09T05:48:06.362 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980861+0000", 2023-10-09T05:48:06.362 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00021713400000000001 2023-10-09T05:48:06.362 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980886+0000", 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5148999999999999e-05 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.327199+0000", 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.34631276100000002 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.363 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125676 7.8 7:15209edd:::benchmark_data_smithi008_354649_object31180:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:03.006382+0000", 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 243.13503836300001, 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0270797059999999, 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125676 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.364 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.365 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006382+0000", 2023-10-09T05:48:06.365 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.365 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.365 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006382+0000", 2023-10-09T05:48:06.365 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.365 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006379+0000", 2023-10-09T05:48:06.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:48:06.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.366 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.366 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006392+0000", 2023-10-09T05:48:06.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2278e-05 2023-10-09T05:48:06.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.366 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.366 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006411+0000", 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8771e-05 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006423+0000", 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2140000000000001e-05 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.298138+0000", 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29171578399999998 2023-10-09T05:48:06.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.298155+0000", 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6248000000000001e-05 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.958770+0000", 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.66061571799999996 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.958788+0000", 2023-10-09T05:48:06.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7986e-05 2023-10-09T05:48:06.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.990377+0000", 2023-10-09T05:48:06.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0315884870000001 2023-10-09T05:48:06.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:48:06.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.990742+0000", 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00036464899999999998 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.025159+0000", 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.034417784999999999 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.025291+0000", 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013127599999999999 2023-10-09T05:48:06.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:48:06.371 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.025442+0000", 2023-10-09T05:48:06.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015114300000000001 2023-10-09T05:48:06.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.371 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.028098+0000", 2023-10-09T05:48:06.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026560030000000001 2023-10-09T05:48:06.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.028113+0000", 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4803999999999999e-05 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.031883+0000", 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037704729999999999 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.031952+0000", 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8750000000000004e-05 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.033462+0000", 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001509809 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.373 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125690 7.18 7:1f975c2f:::benchmark_data_smithi008_354649_object31194:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:03.043212+0000", 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 243.098208701, 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1173635340000001, 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125690 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043212+0000", 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043212+0000", 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043209+0000", 2023-10-09T05:48:06.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:48:06.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.376 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.376 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043218+0000", 2023-10-09T05:48:06.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.081e-06 2023-10-09T05:48:06.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.376 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.376 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043237+0000", 2023-10-09T05:48:06.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8473000000000001e-05 2023-10-09T05:48:06.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043249+0000", 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2850999999999999e-05 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.298392+0000", 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.25514280299999997 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.298400+0000", 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7619999999999995e-06 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.117212+0000", 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.81881223800000003 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.118138+0000", 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000925796 2023-10-09T05:48:06.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.151854+0000", 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.033715429999999998 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.151931+0000", 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7076999999999994e-05 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.152013+0000", 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2369999999999999e-05 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.154638+0000", 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026251180000000001 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.154925+0000", 2023-10-09T05:48:06.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00028725099999999998 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.158589+0000", 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003663805 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.158660+0000", 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0753999999999994e-05 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.160575+0000", 2023-10-09T05:48:06.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00191519 2023-10-09T05:48:06.382 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.382 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.382 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.382 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125699 7.8 7:12a3ca9b:::benchmark_data_smithi008_354649_object31203:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T05:48:06.382 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:04.049537+0000", 2023-10-09T05:48:06.383 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 242.09188284699999, 2023-10-09T05:48:06.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99354653900000001, 2023-10-09T05:48:06.383 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.383 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.383 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.383 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.383 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.383 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125699 2023-10-09T05:48:06.383 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.383 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.383 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049537+0000", 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049537+0000", 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.384 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049536+0000", 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049542+0000", 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9699999999999996e-06 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049552+0000", 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0336999999999999e-05 2023-10-09T05:48:06.385 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.386 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049561+0000", 2023-10-09T05:48:06.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5199999999999997e-06 2023-10-09T05:48:06.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.386 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.080267+0000", 2023-10-09T05:48:06.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030705823 2023-10-09T05:48:06.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.080279+0000", 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2414e-05 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.027631+0000", 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94735189399999997 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:48:06.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.027981+0000", 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00034967100000000002 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.035589+0000", 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007608498 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.035698+0000", 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000108607 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:48:06.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.035835+0000", 2023-10-09T05:48:06.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013765199999999999 2023-10-09T05:48:06.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.041951+0000", 2023-10-09T05:48:06.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061161460000000003 2023-10-09T05:48:06.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.041966+0000", 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4367e-05 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.042023+0000", 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6901999999999998e-05 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.042087+0000", 2023-10-09T05:48:06.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4338000000000006e-05 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.043084+0000", 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00099690400000000006 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125747 7.8 7:1254c147:::benchmark_data_smithi008_354649_object31251:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T05:48:06.391 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:04.185506+0000", 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 241.95591450000001, 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85855084100000001, 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125747 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.392 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185506+0000", 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185506+0000", 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185504+0000", 2023-10-09T05:48:06.393 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185511+0000", 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5050000000000004e-06 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185527+0000", 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6192e-05 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185536+0000", 2023-10-09T05:48:06.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8340000000000005e-06 2023-10-09T05:48:06.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.186726+0000", 2023-10-09T05:48:06.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011901629999999999 2023-10-09T05:48:06.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.186737+0000", 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0543999999999999e-05 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.028847+0000", 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.84211075000000002 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.029239+0000", 2023-10-09T05:48:06.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00039186899999999998 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.039583+0000", 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010344183999999999 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.039701+0000", 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011815300000000001 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.039851+0000", 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000149414 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.043440+0000", 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035890679999999999 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.043469+0000", 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8659000000000001e-05 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.043474+0000", 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2780000000000001e-06 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.043504+0000", 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9754e-05 2023-10-09T05:48:06.399 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.044057+0000", 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00055292199999999996 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:136134 7.e 7:73de093b:::benchmark_data_smithi008_354649_object41638:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:35.638088+0000", 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 210.50333217599999, 2023-10-09T05:48:06.400 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.245445416, 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 136134 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.638088+0000", 2023-10-09T05:48:06.401 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.638088+0000", 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.638087+0000", 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.638092+0000", 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2399999999999998e-06 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.638103+0000", 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.061e-05 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.638110+0000", 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5220000000000001e-06 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.638155+0000", 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4802999999999998e-05 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.638282+0000", 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000126997 2023-10-09T05:48:06.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:48:06.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.638388+0000", 2023-10-09T05:48:06.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010545800000000001 2023-10-09T05:48:06.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.642822+0000", 2023-10-09T05:48:06.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0044342720000000004 2023-10-09T05:48:06.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.643980+0000", 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011580290000000001 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.883443+0000", 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23946313499999999 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.883493+0000", 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0170000000000002e-05 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:35.883533+0000", 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0204000000000003e-05 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.407 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:137496 7.18 7:1a5a2ba4:::benchmark_data_smithi008_354649_object43000:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-09T05:48:06.408 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:39.361241+0000", 2023-10-09T05:48:06.408 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 206.780179656, 2023-10-09T05:48:06.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.21243556199999999, 2023-10-09T05:48:06.408 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.408 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.408 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.408 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.408 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.408 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 137496 2023-10-09T05:48:06.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.408 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.361241+0000", 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.361241+0000", 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.361239+0000", 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.361245+0000", 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0190000000000003e-06 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.361255+0000", 2023-10-09T05:48:06.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6649999999999998e-06 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.361261+0000", 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2040000000000002e-06 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.361470+0000", 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000208743 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.412 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:48:06.412 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.361758+0000", 2023-10-09T05:48:06.412 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00028791 2023-10-09T05:48:06.412 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.412 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.412 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.412 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.376537+0000", 2023-10-09T05:48:06.412 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014779092000000001 2023-10-09T05:48:06.412 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.412 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.412 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.376631+0000", 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4712999999999999e-05 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.376754+0000", 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000122466 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.379262+0000", 2023-10-09T05:48:06.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025077480000000002 2023-10-09T05:48:06.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.571785+0000", 2023-10-09T05:48:06.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19252365699999999 2023-10-09T05:48:06.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.571833+0000", 2023-10-09T05:48:06.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7681e-05 2023-10-09T05:48:06.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.571892+0000", 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9072e-05 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.573676+0000", 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001784117 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.415 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:137504 7.3 e245/239 7:c780b69d:::benchmark_data_smithi008_354649_object43008:head v 245'8242, mlcod=245'8240)", 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:39.381012+0000", 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 206.760408321, 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19560680499999999, 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.381012+0000", 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.381012+0000", 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.381011+0000", 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:06.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.381017+0000", 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6489999999999996e-06 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.571716+0000", 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19069926300000001 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.571730+0000", 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3713999999999999e-05 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.571791+0000", 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1079000000000003e-05 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.571815+0000", 2023-10-09T05:48:06.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3578999999999999e-05 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.576570+0000", 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0047551069999999997 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:39.576619+0000", 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8918999999999997e-05 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.420 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180268 7.e 7.51e4fa9e (undecoded) ondisk+write+known_if_redirected e268)", 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.012770+0000", 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 85.128650139000001, 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27216674200000002, 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.421 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180268 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.012770+0000", 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.012770+0000", 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.423 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.012768+0000", 2023-10-09T05:48:06.423 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:48:06.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.423 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.012780+0000", 2023-10-09T05:48:06.423 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2061e-05 2023-10-09T05:48:06.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.425 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.012799+0000", 2023-10-09T05:48:06.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8717999999999999e-05 2023-10-09T05:48:06.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.425 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.012811+0000", 2023-10-09T05:48:06.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2255999999999999e-05 2023-10-09T05:48:06.426 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.426 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.426 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.284848+0000", 2023-10-09T05:48:06.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27203716300000003 2023-10-09T05:48:06.426 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.426 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.426 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.284937+0000", 2023-10-09T05:48:06.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8624999999999999e-05 2023-10-09T05:48:06.426 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.427 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.427 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.427 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180268 7.1e 7:795f278a:::benchmark_data_smithi008_354649_object85772:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e269)", 2023-10-09T05:48:06.427 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.024498+0000", 2023-10-09T05:48:06.427 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 85.116922527, 2023-10-09T05:48:06.427 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99404307599999997, 2023-10-09T05:48:06.427 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.427 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.427 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180268 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024498+0000", 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024498+0000", 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024496+0000", 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024505+0000", 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2900000000000008e-06 2023-10-09T05:48:06.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024528+0000", 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3096e-05 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024540+0000", 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1959e-05 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.430 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.285107+0000", 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26056715899999999 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.285116+0000", 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5250000000000005e-06 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.016113+0000", 2023-10-09T05:48:06.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73099665800000002 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.016255+0000", 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014244400000000001 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.016443+0000", 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018830400000000001 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.432 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.433 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.017705+0000", 2023-10-09T05:48:06.433 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0012614029999999999 2023-10-09T05:48:06.433 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.433 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.433 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.433 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018393+0000", 2023-10-09T05:48:06.433 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00068864900000000005 2023-10-09T05:48:06.433 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.433 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.433 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.434 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018481+0000", 2023-10-09T05:48:06.434 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7276999999999998e-05 2023-10-09T05:48:06.434 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018513+0000", 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1922999999999999e-05 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018541+0000", 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8174000000000002e-05 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.435 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.436 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:44623 8.a 8:52e77fe6:::benchmark_data_smithi008_432048_object44622:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-10-09T05:48:06.436 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:04.385412+0000", 2023-10-09T05:48:06.436 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 1.7560084489999999, 2023-10-09T05:48:06.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24223594200000001, 2023-10-09T05:48:06.436 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:06.436 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:06.436 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:06.436 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:48:06.436 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:06.437 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 44623 2023-10-09T05:48:06.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.437 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:06.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:06.437 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.385412+0000", 2023-10-09T05:48:06.437 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:06.437 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.385412+0000", 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.385410+0000", 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.385430+0000", 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9655000000000001e-05 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.385443+0000", 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2717999999999999e-05 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.385449+0000", 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2899999999999999e-06 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:06.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.385485+0000", 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6236999999999999e-05 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.385596+0000", 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011072 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.385689+0000", 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3041999999999995e-05 2023-10-09T05:48:06.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.387452+0000", 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00176318 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.389408+0000", 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019560319999999999 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:06.441 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.627559+0000", 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23815105 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.627607+0000", 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7278999999999999e-05 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:04.627648+0000", 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1044000000000002e-05 2023-10-09T05:48:06.442 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.443 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.443 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.443 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:06.443 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:06.443 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:48:06.443 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:05 smithi191 ceph-mon[161669]: pgmap v880: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-09T05:48:07.116 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 24 16 48492 48476 126.219 101.438 0.00476203 0.00791721 2023-10-09T05:48:07.670 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:07.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:07.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:07.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:07.802 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:07.909 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:07.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:07.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:08.020 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:08.070 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:07 smithi191 ceph-mon[161669]: pgmap v881: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:48:08.076 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:08.117 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 25 16 49377 49361 123.375 55.3125 0.019437 0.00809997 2023-10-09T05:48:08.172 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:08.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:08.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:08.280 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:08.312 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:07 smithi008 ceph-mon[184418]: pgmap v881: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:48:08.320 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:07 smithi008 ceph-mon[191227]: pgmap v881: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T05:48:08.348 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:48:08.364 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:08.580 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:08.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:08.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:08.627 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:08.687 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:08.747 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:08.889 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:08.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:08.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:09.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:09.059 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:09.117 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 26 16 50703 50687 121.817 82.875 0.0137656 0.00820453 2023-10-09T05:48:09.164 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:09.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:09.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:09.274 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:09.359 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:09.535 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:09.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:09.641 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:09.726 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:09.758 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:09.931 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:09.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:09.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:09.996 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T05:48:10.015 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:48:10.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:10.111 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:10.117 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 27 16 51963 51947 120.222 78.75 0.00917643 0.00831299 2023-10-09T05:48:10.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:09 smithi008 ceph-mon[184418]: pgmap v882: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T05:48:10.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:09 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3171402834' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:10.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:09 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:10.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:09 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1749178589' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:48:10.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:09 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:48:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:09 smithi008 ceph-mon[191227]: pgmap v882: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T05:48:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:09 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3171402834' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:09 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:09 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1749178589' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:48:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:09 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:48:10.233 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:10.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:10.244 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:09 smithi191 ceph-mon[161669]: pgmap v882: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T05:48:10.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:09 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3171402834' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:10.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:09 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:10.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:09 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1749178589' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:48:10.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:09 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:48:10.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:10.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:10.411 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:10.516 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:10.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:10.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:10.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:10.693 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:10.793 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:10.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:10.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:10.900 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:10.954 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:10.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:48:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:48:10] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.33.4" 2023-10-09T05:48:11.001 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:11.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:11.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:10 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:10 smithi191 ceph-mon[161669]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:10 smithi191 ceph-mon[161669]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:10 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:48:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:10 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:48:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:10 smithi191 ceph-mon[161669]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T05:48:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:10 smithi191 ceph-mon[161669]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:10 smithi191 ceph-mon[161669]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:11.117 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 28 16 53243 53227 118.785 80 0.00903809 0.00841307 2023-10-09T05:48:11.163 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:11.253 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:11.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:11.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:11.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:11.431 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:11.433 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:11.433 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[184418]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.433 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[184418]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.433 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:48:11.433 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:48:11.433 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[184418]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T05:48:11.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[184418]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[184418]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.434 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:11.434 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[191227]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.434 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[191227]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.434 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:48:11.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:48:11.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[191227]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T05:48:11.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[191227]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:10 smithi008 ceph-mon[191227]: failed to encode map e292 with expected crc 2023-10-09T05:48:11.511 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:11.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:11.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:11.616 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:11.666 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:11.717 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:11.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:11.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:11.827 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:11.888 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:11.987 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:11 smithi191 ceph-mon[161669]: pgmap v884: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:48:12.118 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 29 16 54538 54522 117.48 80.9375 0.00740718 0.00850727 2023-10-09T05:48:12.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:11 smithi008 ceph-mon[184418]: pgmap v884: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:48:12.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:11 smithi008 ceph-mon[191227]: pgmap v884: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:48:12.979 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:12.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:12.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:13.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:13.118 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 30 16 55709 55693 116.004 73.1875 0.0067562 0.00861571 2023-10-09T05:48:13.158 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:13.256 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:13.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:13.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:13.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:13.423 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:13.544 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:13.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:13.654 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:13.710 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:13.873 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:13.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:13.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:13.983 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:14.073 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:14.108 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:13 smithi008 ceph-mon[191227]: pgmap v885: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:48:14.108 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:14.108 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:14.112 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:13 smithi008 ceph-mon[184418]: pgmap v885: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:48:14.112 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:14.112 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:14.118 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 31 16 57106 57090 115.078 87.3125 0.0125429 0.00868383 2023-10-09T05:48:14.285 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:14.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:14.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:14.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:13 smithi191 ceph-mon[161669]: pgmap v885: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-09T05:48:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:14.390 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:14.422 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:48:14.438 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:14.463 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:48:14.501 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:48:14.525 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:14.530 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:48:14.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:14.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:14.572 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:48:14.605 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:48:14.605 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:48:14.609 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:48:14.632 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:14.648 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:48:14.678 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:48:14.684 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:48:14.684 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:48:14.684 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:48:14.684 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:48:14.685 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:48:14.685 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:48:14.685 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:48:14.685 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:48:14.689 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:14.778 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:14.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:14.888 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:14.959 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:15.118 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 32 16 58441 58425 114.089 83.4375 0.0111707 0.00876093 2023-10-09T05:48:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/252555142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/252555142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:15.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/252555142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:15.351 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:15.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:15.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:15.455 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:15.482 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:15.527 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:15.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:15.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:15.591 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:48:15.630 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:15.656 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:15.716 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:15.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:15.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:15.823 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:15.848 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:15.889 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:15.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:15.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:15.992 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:16.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:16 smithi191 ceph-mon[161669]: pgmap v886: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T05:48:16.110 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:16.118 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 33 16 59292 59276 112.244 53.1875 0.00981701 0.00890471 2023-10-09T05:48:16.425 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:16.426 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:16 smithi008 ceph-mon[191227]: pgmap v886: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T05:48:16.429 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:16 smithi008 ceph-mon[184418]: pgmap v886: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T05:48:16.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:16.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:16.514 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:48:16.536 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:16.614 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:16.751 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:16.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:16.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:16.763 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:16.857 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:16.939 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:17.094 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:17.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:17.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:17.118 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 34 16 60565 60549 111.282 79.5625 0.0104739 0.00898088 2023-10-09T05:48:17.205 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:17.250 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:17.396 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:17.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:17.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:17.505 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:17.558 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:17.644 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:17.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:17.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:17.823 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:17.876 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:17.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:17.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:17.913 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:48:17.913 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:48:17.913 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:48:17.913 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.000000 2023-10-09T05:48:17.913 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 0 2023-10-09T05:48:17.984 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:18.010 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:18.092 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:18.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:18.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:18.102 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:18 smithi191 ceph-mon[161669]: pgmap v887: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T05:48:18.119 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 35 16 61658 61642 110.052 68.3125 0.0133878 0.00908134 2023-10-09T05:48:18.203 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:18.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:18 smithi008 ceph-mon[184418]: pgmap v887: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T05:48:18.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:18 smithi008 ceph-mon[191227]: pgmap v887: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T05:48:18.263 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:18.377 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:18.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:18.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:18.462 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:48:18.473 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:18.487 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:18.592 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:18.932 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:18.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:18.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:19.042 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:19.104 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:19.118 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:19.124 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 36 16 62882 62866 109.11 76.5 0.0110574 0.00915982 2023-10-09T05:48:19.173 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:19.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:19.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:19.285 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:19.364 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:19.650 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:19.687 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:19.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:19.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:19.795 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:19.864 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:20.031 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:48:20.127 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 37 16 63953 63937 107.956 66.9375 0.0169219 0.00925757 2023-10-09T05:48:20.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:20 smithi008 ceph-mon[191227]: pgmap v888: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:48:20.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:20 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1632441682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:48:20.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:20 smithi008 ceph-mon[184418]: pgmap v888: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:48:20.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:20 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1632441682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:48:20.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:20 smithi191 ceph-mon[161669]: pgmap v888: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T05:48:20.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:20 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1632441682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:48:20.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:48:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:48:20] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-10-09T05:48:21.129 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 38 16 65124 65108 107.036 73.1875 0.0121853 0.0093379 2023-10-09T05:48:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:21 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:21 smithi191 ceph-mon[161669]: failed to encode map e293 with expected crc 2023-10-09T05:48:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:21 smithi191 ceph-mon[161669]: failed to encode map e293 with expected crc 2023-10-09T05:48:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:21 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1632441682' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:48:21.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:21 smithi191 ceph-mon[161669]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T05:48:21.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:21 smithi191 ceph-mon[161669]: failed to encode map e293 with expected crc 2023-10-09T05:48:21.303 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:21.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[184418]: failed to encode map e293 with expected crc 2023-10-09T05:48:21.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[184418]: failed to encode map e293 with expected crc 2023-10-09T05:48:21.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1632441682' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:48:21.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[184418]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T05:48:21.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[184418]: failed to encode map e293 with expected crc 2023-10-09T05:48:21.304 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:21.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[191227]: failed to encode map e293 with expected crc 2023-10-09T05:48:21.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[191227]: failed to encode map e293 with expected crc 2023-10-09T05:48:21.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1632441682' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:48:21.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[191227]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T05:48:21.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:21 smithi008 ceph-mon[191227]: failed to encode map e293 with expected crc 2023-10-09T05:48:21.305 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:21.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:21.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:21.413 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:21.462 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:22.129 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 39 16 66005 65989 105.703 55.0625 0.00908354 0.00945572 2023-10-09T05:48:22.191 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:22 smithi008 ceph-mon[184418]: pgmap v890: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T05:48:22.194 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:22 smithi008 ceph-mon[191227]: pgmap v890: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T05:48:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:22 smithi191 ceph-mon[161669]: pgmap v890: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T05:48:22.502 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:22.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:22.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:22.612 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:22.672 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:22.789 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:22.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:22.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:22.895 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:22.946 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:23.130 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:48:23.130527+0000 min lat: 0.00124024 max lat: 0.275386 avg lat: 0.00952073 2023-10-09T05:48:23.130 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:48:23.130 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 40 16 67235 67219 104.983 76.875 0.015218 0.00952073 2023-10-09T05:48:23.249 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:23.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:23.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:23.360 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:23.437 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:23.599 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:23.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:23.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:23.659 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:48:23.659 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:48:23.706 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:23.768 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:24.091 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.3 primary-affinity to 0 (802) 2023-10-09T05:48:24.130 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 41 16 68316 68300 104.07 67.5625 0.00312822 0.0095997 2023-10-09T05:48:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:23 smithi191 ceph-mon[161669]: pgmap v891: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T05:48:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/239725563' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T05:48:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:23 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T05:48:24.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:23 smithi008 ceph-mon[184418]: pgmap v891: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T05:48:24.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/239725563' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T05:48:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:23 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T05:48:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:23 smithi008 ceph-mon[191227]: pgmap v891: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T05:48:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/239725563' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T05:48:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:23 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-09T05:48:24.618 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:48:24.970 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:24.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:24.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:25.073 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:25.130 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 42 16 68372 68356 101.676 3.5 0.0078286 0.00983027 2023-10-09T05:48:25.137 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:25.239 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:25.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:25.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:25.349 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:25.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[184418]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[184418]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T05:48:25.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[184418]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T05:48:25.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[184418]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[184418]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[184418]: pgmap v893: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T05:48:25.397 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[191227]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[191227]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T05:48:25.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[191227]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T05:48:25.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[191227]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[191227]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:25 smithi008 ceph-mon[191227]: pgmap v893: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T05:48:25.406 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:25.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:25 smithi191 ceph-mon[161669]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:25 smithi191 ceph-mon[161669]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:25 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-09T05:48:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:25 smithi191 ceph-mon[161669]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T05:48:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:25 smithi191 ceph-mon[161669]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:25 smithi191 ceph-mon[161669]: failed to encode map e294 with expected crc 2023-10-09T05:48:25.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:25 smithi191 ceph-mon[161669]: pgmap v893: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T05:48:25.616 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:25.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:25.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:25.665 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:48:25.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:25.788 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:25.872 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:25.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:25.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:25.985 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:26.087 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:26.087 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:26.130 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 43 16 69517 69501 100.976 71.5625 0.0154706 0.00989716 2023-10-09T05:48:26.206 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:48:26.217 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:26.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:26.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:26.244 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:48:26.275 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:48:26.310 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:48:26.326 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:26.344 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:48:26.382 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[184418]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.382 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[184418]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.382 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[184418]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.382 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[184418]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.382 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[184418]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T05:48:26.383 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[184418]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.383 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[184418]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.383 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[184418]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.383 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[184418]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.383 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[184418]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.383 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[191227]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.383 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[191227]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.383 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[191227]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.383 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[191227]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.383 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[191227]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T05:48:26.384 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[191227]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.384 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[191227]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.384 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[191227]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.384 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[191227]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.384 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:26 smithi008 ceph-mon[191227]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.389 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:26.391 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:48:26.423 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:48:26.458 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:48:26.460 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:48:26.460 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:48:26.460 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:48:26.460 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:48:26.460 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:48:26.460 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:48:26.460 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:48:26.460 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:48:26.492 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:26.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:26 smithi191 ceph-mon[161669]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:26 smithi191 ceph-mon[161669]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:26 smithi191 ceph-mon[161669]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:26 smithi191 ceph-mon[161669]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:26 smithi191 ceph-mon[161669]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T05:48:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:26 smithi191 ceph-mon[161669]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:26 smithi191 ceph-mon[161669]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:26 smithi191 ceph-mon[161669]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:26 smithi191 ceph-mon[161669]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:26 smithi191 ceph-mon[161669]: failed to encode map e295 with expected crc 2023-10-09T05:48:26.599 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:26.671 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:26.770 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:26.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:26.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:26.889 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:26.918 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:27.084 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1938086923' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:27.084 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:27 smithi008 ceph-mon[184418]: pgmap v895: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T05:48:27.132 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 44 16 70537 70521 100.126 63.75 0.0205351 0.00998266 2023-10-09T05:48:27.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1938086923' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:27.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:27 smithi008 ceph-mon[191227]: pgmap v895: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T05:48:27.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1938086923' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:27 smithi191 ceph-mon[161669]: pgmap v895: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T05:48:27.981 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:27.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:27.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:28.094 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:28.133 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 45 16 71570 71554 99.3343 64.5625 0.00673731 0.0100631 2023-10-09T05:48:28.181 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:28.219 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:28.314 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:28.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:28.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:28.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:28.505 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:28.616 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:28.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:28.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:28.729 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:28.793 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:29.000 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:29.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:29.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:29.107 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:29.134 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 46 16 72705 72689 98.7171 70.9375 0.00997696 0.0101255 2023-10-09T05:48:29.179 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:29.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:28 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:29.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:28 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:29.280 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:29.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:29.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:29.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:28 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:29.391 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:29.418 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:29.463 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:29.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:29.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:29.566 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:29.594 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:29.660 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:29.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:29.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:29.768 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:29.835 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:29.949 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:29.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:29.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:29.978 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:48:30.065 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:30.125 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:30.133 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 47 16 73486 73470 97.655 48.8125 0.0182555 0.0102351 2023-10-09T05:48:30.221 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:30.222 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:48:30.232 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:29 smithi008 ceph-mon[191227]: pgmap v896: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-09T05:48:30.232 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2986296281' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:30.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:30.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:30.234 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:29 smithi008 ceph-mon[184418]: pgmap v896: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-09T05:48:30.234 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2986296281' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:30.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:29 smithi191 ceph-mon[161669]: pgmap v896: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-09T05:48:30.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2986296281' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:30.334 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:30.400 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:30.491 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:48:30.492 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:48:30.492 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-10-09T05:48:30.492 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:48:30.509 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:48:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:48:30] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-10-09T05:48:30.528 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:30.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:30.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:30.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:30.707 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:30.784 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:30.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:30.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:30.893 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:31.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:31 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:48:31.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:31 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:48:31.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:31 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2986296281' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:48:31.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:31 smithi191 ceph-mon[161669]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T05:48:31.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:31 smithi191 ceph-mon[161669]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:31 smithi191 ceph-mon[161669]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.009 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:31.133 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 48 16 74531 74515 96.9813 65.3125 0.0190413 0.0103063 2023-10-09T05:48:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:48:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:48:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2986296281' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:48:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[184418]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T05:48:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[184418]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[184418]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[184418]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[184418]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[184418]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[184418]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:48:31.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:48:31.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2986296281' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:48:31.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[191227]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T05:48:31.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[191227]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[191227]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[191227]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[191227]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[191227]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:31 smithi008 ceph-mon[191227]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:31 smithi191 ceph-mon[161669]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:31 smithi191 ceph-mon[161669]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:31 smithi191 ceph-mon[161669]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:31 smithi191 ceph-mon[161669]: failed to encode map e296 with expected crc 2023-10-09T05:48:31.312 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:31.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:31.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:31.366 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:48:31.366 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:48:31.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:48:31.367 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:48:31.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.367 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125668 7.2 7:4690f46d:::benchmark_data_smithi008_354649_object31172:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:48:31.367 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.998068+0000", 2023-10-09T05:48:31.367 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 268.36446272500001, 2023-10-09T05:48:31.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0285217099999999, 2023-10-09T05:48:31.368 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.368 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:48:31.368 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.368 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:31.368 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:31.368 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125668 2023-10-09T05:48:31.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.369 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998068+0000", 2023-10-09T05:48:31.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998068+0000", 2023-10-09T05:48:31.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998064+0000", 2023-10-09T05:48:31.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999962 2023-10-09T05:48:31.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.371 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998079+0000", 2023-10-09T05:48:31.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4626e-05 2023-10-09T05:48:31.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998114+0000", 2023-10-09T05:48:31.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5065999999999999e-05 2023-10-09T05:48:31.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998134+0000", 2023-10-09T05:48:31.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9709000000000001e-05 2023-10-09T05:48:31.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.508948+0000", 2023-10-09T05:48:31.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.51081447700000004 2023-10-09T05:48:31.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:31.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.508964+0000", 2023-10-09T05:48:31.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5758999999999999e-05 2023-10-09T05:48:31.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.374 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.026497+0000", 2023-10-09T05:48:31.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.51753330099999995 2023-10-09T05:48:31.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.026590+0000", 2023-10-09T05:48:31.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2484000000000004e-05 2023-10-09T05:48:31.375 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.375 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.375 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.376 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125675 7.12 7:4e04343f:::benchmark_data_smithi008_354649_object31179:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:48:31.376 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.998149+0000", 2023-10-09T05:48:31.376 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 268.36438228700001, 2023-10-09T05:48:31.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0470999430000001, 2023-10-09T05:48:31.376 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.376 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:48:31.376 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.377 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:31.377 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:31.377 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125675 2023-10-09T05:48:31.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.377 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998149+0000", 2023-10-09T05:48:31.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998149+0000", 2023-10-09T05:48:31.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998148+0000", 2023-10-09T05:48:31.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:31.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998153+0000", 2023-10-09T05:48:31.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0549999999999999e-06 2023-10-09T05:48:31.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998181+0000", 2023-10-09T05:48:31.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8314e-05 2023-10-09T05:48:31.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998188+0000", 2023-10-09T05:48:31.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9929999999999998e-06 2023-10-09T05:48:31.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.505618+0000", 2023-10-09T05:48:31.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.50742975800000001 2023-10-09T05:48:31.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:31.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.505627+0000", 2023-10-09T05:48:31.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7269999999999996e-06 2023-10-09T05:48:31.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.383 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.045159+0000", 2023-10-09T05:48:31.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.53953261399999997 2023-10-09T05:48:31.383 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.383 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.383 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.383 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.045249+0000", 2023-10-09T05:48:31.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9115000000000003e-05 2023-10-09T05:48:31.383 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.384 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.384 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.384 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125679 7.12 7:49f6cf6f:::benchmark_data_smithi008_354649_object31183:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:48:31.384 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.998193+0000", 2023-10-09T05:48:31.384 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 268.36433748500002, 2023-10-09T05:48:31.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0474672460000001, 2023-10-09T05:48:31.385 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.385 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:48:31.385 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.385 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:31.385 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:31.385 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125679 2023-10-09T05:48:31.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.386 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.386 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998193+0000", 2023-10-09T05:48:31.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998193+0000", 2023-10-09T05:48:31.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998193+0000", 2023-10-09T05:48:31.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:31.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998197+0000", 2023-10-09T05:48:31.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7340000000000002e-06 2023-10-09T05:48:31.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998204+0000", 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6380000000000004e-06 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998209+0000", 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0629999999999996e-06 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.505647+0000", 2023-10-09T05:48:31.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.50743739700000001 2023-10-09T05:48:31.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:31.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.505654+0000", 2023-10-09T05:48:31.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2250000000000003e-06 2023-10-09T05:48:31.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.045592+0000", 2023-10-09T05:48:31.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.53993788600000003 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.045661+0000", 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8689999999999995e-05 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125684 7.13 7:c9988d38:::benchmark_data_smithi008_354649_object31188:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:48:31.391 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.998214+0000", 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 268.36431700499998, 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.044232475, 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125684 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.392 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.393 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998214+0000", 2023-10-09T05:48:31.393 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.393 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.393 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.393 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998214+0000", 2023-10-09T05:48:31.393 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998214+0000", 2023-10-09T05:48:31.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:31.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998217+0000", 2023-10-09T05:48:31.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0539999999999999e-06 2023-10-09T05:48:31.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998222+0000", 2023-10-09T05:48:31.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5489999999999997e-06 2023-10-09T05:48:31.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.998228+0000", 2023-10-09T05:48:31.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7119999999999997e-06 2023-10-09T05:48:31.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.530142+0000", 2023-10-09T05:48:31.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.53191393399999998 2023-10-09T05:48:31.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:31.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.530149+0000", 2023-10-09T05:48:31.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8800000000000002e-06 2023-10-09T05:48:31.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.042325+0000", 2023-10-09T05:48:31.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.51217680600000004 2023-10-09T05:48:31.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.399 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.399 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.042446+0000", 2023-10-09T05:48:31.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012087 2023-10-09T05:48:31.399 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.399 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.399 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.399 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.400 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.400 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125693 7.13 7:c832cc4f:::benchmark_data_smithi008_354649_object31197:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T05:48:31.400 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:03.070133+0000", 2023-10-09T05:48:31.400 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 268.29239816799998, 2023-10-09T05:48:31.400 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97788652200000004, 2023-10-09T05:48:31.400 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.400 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:31.400 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.401 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:31.401 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:31.401 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125693 2023-10-09T05:48:31.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.401 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.401 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.401 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.401 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.070133+0000", 2023-10-09T05:48:31.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.402 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.070133+0000", 2023-10-09T05:48:31.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.070131+0000", 2023-10-09T05:48:31.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:48:31.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.070142+0000", 2023-10-09T05:48:31.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1124999999999999e-05 2023-10-09T05:48:31.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.070194+0000", 2023-10-09T05:48:31.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2710000000000002e-05 2023-10-09T05:48:31.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.070209+0000", 2023-10-09T05:48:31.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4564e-05 2023-10-09T05:48:31.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.530173+0000", 2023-10-09T05:48:31.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.45996387700000002 2023-10-09T05:48:31.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:31.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.530177+0000", 2023-10-09T05:48:31.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1010000000000002e-06 2023-10-09T05:48:31.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.042955+0000", 2023-10-09T05:48:31.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.51277840799999996 2023-10-09T05:48:31.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.043075+0000", 2023-10-09T05:48:31.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000119292 2023-10-09T05:48:31.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.408 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.408 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:48:31.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.043424+0000", 2023-10-09T05:48:31.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00034893400000000002 2023-10-09T05:48:31.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:31.409 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.045470+0000", 2023-10-09T05:48:31.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020467160000000001 2023-10-09T05:48:31.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.047893+0000", 2023-10-09T05:48:31.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002422558 2023-10-09T05:48:31.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.047926+0000", 2023-10-09T05:48:31.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3325999999999999e-05 2023-10-09T05:48:31.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.047983+0000", 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6669000000000002e-05 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.048019+0000", 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6263000000000002e-05 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150740 7.2 7:447bd200:::benchmark_data_smithi008_354649_object56244:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.196216+0000", 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 194.16631466499999, 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.908238267, 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150740 2023-10-09T05:48:31.412 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.196216+0000", 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.196216+0000", 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.196215+0000", 2023-10-09T05:48:31.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:31.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.196224+0000", 2023-10-09T05:48:31.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7359999999999996e-06 2023-10-09T05:48:31.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.196243+0000", 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9142000000000001e-05 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.196252+0000", 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8419999999999994e-06 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.196304+0000", 2023-10-09T05:48:31.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1823999999999997e-05 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.196315+0000", 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1085e-05 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.914591+0000", 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.71827620400000003 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.914607+0000", 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5930000000000002e-05 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.055012+0000", 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1404043210000001 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.055173+0000", 2023-10-09T05:48:31.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016130300000000001 2023-10-09T05:48:31.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T05:48:31.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.055715+0000", 2023-10-09T05:48:31.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00054167099999999997 2023-10-09T05:48:31.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.068247+0000", 2023-10-09T05:48:31.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012532099 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.068695+0000", 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00044808700000000001 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.069418+0000", 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00072340600000000001 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.420 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.104262+0000", 2023-10-09T05:48:31.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.034843543999999997 2023-10-09T05:48:31.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.420 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:31.420 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.104332+0000", 2023-10-09T05:48:31.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0168999999999999e-05 2023-10-09T05:48:31.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.421 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.104454+0000", 2023-10-09T05:48:31.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012248000000000001 2023-10-09T05:48:31.421 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.421 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.421 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.421 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.421 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.421 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180269 7.2 7:4013466a:::benchmark_data_smithi008_354649_object85773:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e269)", 2023-10-09T05:48:31.421 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.024398+0000", 2023-10-09T05:48:31.421 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 110.338132351, 2023-10-09T05:48:31.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.060562156, 2023-10-09T05:48:31.422 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.422 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:48:31.422 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.422 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:31.422 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:31.422 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180269 2023-10-09T05:48:31.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.423 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.423 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024398+0000", 2023-10-09T05:48:31.423 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.424 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024398+0000", 2023-10-09T05:48:31.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.424 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.424 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.424 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024396+0000", 2023-10-09T05:48:31.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:48:31.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.425 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024410+0000", 2023-10-09T05:48:31.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3794e-05 2023-10-09T05:48:31.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.426 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024438+0000", 2023-10-09T05:48:31.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8130999999999999e-05 2023-10-09T05:48:31.426 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.426 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.426 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024452+0000", 2023-10-09T05:48:31.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3961000000000001e-05 2023-10-09T05:48:31.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.427 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.427 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.027291+0000", 2023-10-09T05:48:31.427 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028387809999999999 2023-10-09T05:48:31.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.427 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op must wait for map", 2023-10-09T05:48:31.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.027299+0000", 2023-10-09T05:48:31.428 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7220000000000005e-06 2023-10-09T05:48:31.428 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.428 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.428 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.342330+0000", 2023-10-09T05:48:31.428 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.31503056600000001 2023-10-09T05:48:31.428 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:31.429 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.342338+0000", 2023-10-09T05:48:31.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1640000000000001e-06 2023-10-09T05:48:31.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.429 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.080141+0000", 2023-10-09T05:48:31.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73780304500000005 2023-10-09T05:48:31.430 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.430 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:48:31.430 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.080575+0000", 2023-10-09T05:48:31.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00043348900000000001 2023-10-09T05:48:31.430 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.431 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.431 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.084916+0000", 2023-10-09T05:48:31.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0043410280000000002 2023-10-09T05:48:31.431 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.431 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.431 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.431 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.084961+0000", 2023-10-09T05:48:31.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4997000000000003e-05 2023-10-09T05:48:31.431 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.432 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.432 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.432 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.432 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.432 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180273 7.c 7:356754b2:::benchmark_data_smithi008_354649_object85777:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:48:31.432 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.204573+0000", 2023-10-09T05:48:31.432 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 110.15795811700001, 2023-10-09T05:48:31.433 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89913223099999995, 2023-10-09T05:48:31.433 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.433 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:31.433 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.433 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:31.433 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:31.433 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180273 2023-10-09T05:48:31.433 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.433 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.433 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.433 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.434 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.204573+0000", 2023-10-09T05:48:31.434 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.434 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.434 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.434 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.434 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.204573+0000", 2023-10-09T05:48:31.434 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.434 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.204571+0000", 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.204577+0000", 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1990000000000003e-06 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.204588+0000", 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1599e-05 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.204601+0000", 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2693e-05 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.293158+0000", 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.088557302000000004 2023-10-09T05:48:31.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.293187+0000", 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8240000000000001e-05 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.058075+0000", 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.76488788699999999 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.437 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064005+0000", 2023-10-09T05:48:31.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0059305790000000001 2023-10-09T05:48:31.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.438 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:48:31.438 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064533+0000", 2023-10-09T05:48:31.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052799499999999996 2023-10-09T05:48:31.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:31.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.068589+0000", 2023-10-09T05:48:31.439 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040557850000000001 2023-10-09T05:48:31.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.068679+0000", 2023-10-09T05:48:31.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0394000000000002e-05 2023-10-09T05:48:31.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.440 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.440 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.103614+0000", 2023-10-09T05:48:31.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.034934262000000001 2023-10-09T05:48:31.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.441 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.441 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:31.441 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.103665+0000", 2023-10-09T05:48:31.441 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1891999999999997e-05 2023-10-09T05:48:31.441 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.441 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.441 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.441 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.103705+0000", 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9478999999999999e-05 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180274 7.a 7:54a1720d:::benchmark_data_smithi008_354649_object85778:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.277264+0000", 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 110.085266722, 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.80384711900000005, 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.442 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180274 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.277264+0000", 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.277264+0000", 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.277262+0000", 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.277274+0000", 2023-10-09T05:48:31.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2177e-05 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.277299+0000", 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5680000000000001e-05 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.277313+0000", 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3888e-05 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.342295+0000", 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.064981509000000007 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.342305+0000", 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0135999999999999e-05 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.446 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.060662+0000", 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.71835686300000001 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.060772+0000", 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011032 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,5", 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.061116+0000", 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00034352699999999999 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.447 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.063620+0000", 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025041669999999999 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.081002+0000", 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017382221999999999 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.448 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.081034+0000", 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1374999999999997e-05 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.081074+0000", 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0485000000000003e-05 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.081111+0000", 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7255999999999997e-05 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.449 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:54616 8.0 8:0f017be8:::benchmark_data_smithi008_432048_object54615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:12.235172+0000", 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 19.127359160000001, 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17737821300000001, 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:48:31.450 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:31.451 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54616 2023-10-09T05:48:31.453 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.453 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.453 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.453 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.453 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235172+0000", 2023-10-09T05:48:31.454 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.454 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.454 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.454 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.454 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235172+0000", 2023-10-09T05:48:31.454 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.454 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.455 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.455 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.455 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235171+0000", 2023-10-09T05:48:31.455 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:31.455 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.455 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.455 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.455 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235206+0000", 2023-10-09T05:48:31.455 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4891000000000002e-05 2023-10-09T05:48:31.456 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.456 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.456 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.456 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235214+0000", 2023-10-09T05:48:31.456 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1289999999999996e-06 2023-10-09T05:48:31.456 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.456 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.457 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.457 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235220+0000", 2023-10-09T05:48:31.457 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1380000000000004e-06 2023-10-09T05:48:31.457 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.457 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.457 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.457 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.243441+0000", 2023-10-09T05:48:31.457 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0082204099999999992 2023-10-09T05:48:31.457 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.458 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.458 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.458 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.243572+0000", 2023-10-09T05:48:31.458 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000130949 2023-10-09T05:48:31.458 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.458 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.458 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:48:31.458 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.243706+0000", 2023-10-09T05:48:31.458 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000134757 2023-10-09T05:48:31.458 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.458 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.247941+0000", 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042343820000000001 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.254859+0000", 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0069184650000000004 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.459 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412456+0000", 2023-10-09T05:48:31.460 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.157596341 2023-10-09T05:48:31.460 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.460 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.460 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:31.460 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412507+0000", 2023-10-09T05:48:31.460 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1532000000000001e-05 2023-10-09T05:48:31.460 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.460 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.460 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.460 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412550+0000", 2023-10-09T05:48:31.461 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2796999999999997e-05 2023-10-09T05:48:31.461 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.461 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.461 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.461 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.461 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.461 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:54623 8.0 8:0f7285ba:::benchmark_data_smithi008_432048_object54622:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-10-09T05:48:31.462 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:12.235344+0000", 2023-10-09T05:48:31.462 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 19.127186460000001, 2023-10-09T05:48:31.462 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17731562300000001, 2023-10-09T05:48:31.462 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.462 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:31.462 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.462 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:48:31.463 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:31.463 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54623 2023-10-09T05:48:31.463 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.463 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.463 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.463 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.463 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235344+0000", 2023-10-09T05:48:31.463 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.464 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.464 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.464 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.464 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235344+0000", 2023-10-09T05:48:31.464 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.464 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.464 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.464 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.465 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235344+0000", 2023-10-09T05:48:31.465 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:31.465 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.465 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.465 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.465 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235367+0000", 2023-10-09T05:48:31.465 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3713000000000001e-05 2023-10-09T05:48:31.465 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.466 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.466 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.466 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235375+0000", 2023-10-09T05:48:31.466 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6070000000000002e-06 2023-10-09T05:48:31.466 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.466 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.466 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.466 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.235381+0000", 2023-10-09T05:48:31.467 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0719999999999996e-06 2023-10-09T05:48:31.467 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.467 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.467 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.467 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.245672+0000", 2023-10-09T05:48:31.467 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010291079 2023-10-09T05:48:31.467 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.467 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.468 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.468 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.245816+0000", 2023-10-09T05:48:31.468 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000143993 2023-10-09T05:48:31.468 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.468 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.468 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:48:31.468 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.245968+0000", 2023-10-09T05:48:31.468 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000151378 2023-10-09T05:48:31.469 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.469 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.469 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:31.469 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.249733+0000", 2023-10-09T05:48:31.469 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037650380000000001 2023-10-09T05:48:31.469 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.469 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.469 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.470 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.254899+0000", 2023-10-09T05:48:31.470 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051664049999999998 2023-10-09T05:48:31.470 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.470 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.470 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.470 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412588+0000", 2023-10-09T05:48:31.470 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.157688723 2023-10-09T05:48:31.471 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.471 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.471 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:31.471 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412626+0000", 2023-10-09T05:48:31.471 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8084999999999999e-05 2023-10-09T05:48:31.471 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.471 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.471 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.472 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412660+0000", 2023-10-09T05:48:31.472 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4153000000000003e-05 2023-10-09T05:48:31.472 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.472 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.472 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.472 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.472 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.472 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:54647 8.0 8:07f9ee24:::benchmark_data_smithi008_432048_object54646:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-10-09T05:48:31.473 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:12.255045+0000", 2023-10-09T05:48:31.473 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 19.107485659999998, 2023-10-09T05:48:31.473 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15998806400000001, 2023-10-09T05:48:31.473 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.473 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:31.473 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.473 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:48:31.474 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:31.474 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54647 2023-10-09T05:48:31.474 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.474 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.474 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.474 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.474 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255045+0000", 2023-10-09T05:48:31.474 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.475 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.475 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.475 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.475 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255045+0000", 2023-10-09T05:48:31.475 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.475 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.475 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.476 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.476 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255044+0000", 2023-10-09T05:48:31.476 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:31.476 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.476 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.476 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.476 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255065+0000", 2023-10-09T05:48:31.476 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0735999999999999e-05 2023-10-09T05:48:31.477 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.477 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.477 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.477 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255073+0000", 2023-10-09T05:48:31.477 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5299999999999996e-06 2023-10-09T05:48:31.477 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.477 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.478 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.478 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255078+0000", 2023-10-09T05:48:31.478 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4029999999999998e-06 2023-10-09T05:48:31.478 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.478 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.478 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.478 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255111+0000", 2023-10-09T05:48:31.478 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2963999999999998e-05 2023-10-09T05:48:31.479 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.479 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.479 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.479 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255259+0000", 2023-10-09T05:48:31.479 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000147777 2023-10-09T05:48:31.479 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.479 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.479 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:48:31.480 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255355+0000", 2023-10-09T05:48:31.480 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5317999999999998e-05 2023-10-09T05:48:31.480 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.480 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.480 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.480 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412703+0000", 2023-10-09T05:48:31.480 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15734835799999999 2023-10-09T05:48:31.481 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.481 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.481 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.481 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412733+0000", 2023-10-09T05:48:31.481 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9992e-05 2023-10-09T05:48:31.481 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.481 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.481 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:31.482 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.414952+0000", 2023-10-09T05:48:31.482 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002219258 2023-10-09T05:48:31.482 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.482 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.482 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:31.482 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.414995+0000", 2023-10-09T05:48:31.482 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3296e-05 2023-10-09T05:48:31.482 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.483 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.483 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.483 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.415033+0000", 2023-10-09T05:48:31.483 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7836000000000002e-05 2023-10-09T05:48:31.483 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.483 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.483 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.484 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.484 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.484 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45782.0:54616 8.0 e292/285 ondisk, result = 0)", 2023-10-09T05:48:31.484 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:12.255201+0000", 2023-10-09T05:48:31.484 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 19.107330256000001, 2023-10-09T05:48:31.484 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15735623300000001, 2023-10-09T05:48:31.484 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.484 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:48:31.485 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.485 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.485 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.485 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255201+0000", 2023-10-09T05:48:31.485 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.485 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.485 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.485 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255201+0000", 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255200+0000", 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255205+0000", 2023-10-09T05:48:31.486 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0560000000000004e-06 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255214+0000", 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.037e-06 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255220+0000", 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.088e-06 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.487 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412442+0000", 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15722208900000001 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412450+0000", 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3830000000000003e-06 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412557+0000", 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010721 2023-10-09T05:48:31.488 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45782.0:54623 8.0 e292/285 ondisk, result = 0)", 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:12.255230+0000", 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 19.107301122999999, 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15744122499999999, 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.489 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255230+0000", 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255230+0000", 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.490 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255229+0000", 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255233+0000", 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1039999999999998e-06 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255238+0000", 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.84e-06 2023-10-09T05:48:31.491 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.255243+0000", 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.369e-06 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412576+0000", 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15733344899999999 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.492 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.493 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412582+0000", 2023-10-09T05:48:31.493 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0900000000000001e-06 2023-10-09T05:48:31.493 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.493 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.493 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.493 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412671+0000", 2023-10-09T05:48:31.494 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8685999999999998e-05 2023-10-09T05:48:31.494 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.494 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.494 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.494 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.494 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.494 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45782.0:54647 8.0 e292/285 ondisk, result = 0)", 2023-10-09T05:48:31.494 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:12.258298+0000", 2023-10-09T05:48:31.494 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 19.104232922000001, 2023-10-09T05:48:31.494 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15440715599999999, 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.258298+0000", 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.258298+0000", 2023-10-09T05:48:31.495 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.258297+0000", 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.258302+0000", 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9559999999999996e-06 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.496 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.410434+0000", 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.1521314 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.410445+0000", 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1776999999999999e-05 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412692+0000", 2023-10-09T05:48:31.497 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022467239999999999 2023-10-09T05:48:31.498 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.498 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.498 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.498 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412698+0000", 2023-10-09T05:48:31.498 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0480000000000004e-06 2023-10-09T05:48:31.498 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.498 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.498 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.498 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412705+0000", 2023-10-09T05:48:31.498 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.002e-06 2023-10-09T05:48:31.498 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:54651 8.0 8:0e38e4ff:::benchmark_data_smithi008_432048_object54650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:12.258499+0000", 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 19.104032271000001, 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.162369238, 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:48:31.499 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:48:31.500 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:31.500 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 54651 2023-10-09T05:48:31.500 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.500 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.500 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.500 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.500 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.258499+0000", 2023-10-09T05:48:31.500 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.500 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.500 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.500 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.258499+0000", 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.258497+0000", 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.258517+0000", 2023-10-09T05:48:31.501 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0024999999999999e-05 2023-10-09T05:48:31.502 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.502 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.502 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.502 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.410427+0000", 2023-10-09T05:48:31.502 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.151909607 2023-10-09T05:48:31.502 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.502 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.503 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.503 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.410448+0000", 2023-10-09T05:48:31.503 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0823999999999999e-05 2023-10-09T05:48:31.503 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.503 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.503 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.503 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412754+0000", 2023-10-09T05:48:31.503 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023057609999999999 2023-10-09T05:48:31.504 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.504 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.504 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.504 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412886+0000", 2023-10-09T05:48:31.504 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013227000000000001 2023-10-09T05:48:31.504 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.504 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.504 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:48:31.505 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.413022+0000", 2023-10-09T05:48:31.505 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013564999999999999 2023-10-09T05:48:31.505 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.505 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.505 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:48:31.505 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.417486+0000", 2023-10-09T05:48:31.505 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0044642470000000002 2023-10-09T05:48:31.506 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.506 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.506 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.506 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.417653+0000", 2023-10-09T05:48:31.506 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000167338 2023-10-09T05:48:31.506 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.506 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.420786+0000", 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031326610000000001 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.420833+0000", 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6808999999999999e-05 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.507 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.420868+0000", 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5228999999999998e-05 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45782.0:54647 8.0 e292/285 ondisk, result = 0)", 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:12.259974+0000", 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 19.102557231999999, 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15276094600000001, 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:48:31.508 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.259974+0000", 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.259974+0000", 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.509 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.259973+0000", 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.259977+0000", 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6120000000000001e-06 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.410453+0000", 2023-10-09T05:48:31.510 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.150475585 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.410461+0000", 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7379999999999994e-06 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412723+0000", 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002262853 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.511 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.512 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.512 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412729+0000", 2023-10-09T05:48:31.512 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1420000000000001e-06 2023-10-09T05:48:31.512 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.512 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.512 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.512 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:12.412735+0000", 2023-10-09T05:48:31.512 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9789999999999998e-06 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:70307 8.c e295/285 8:3bf22b17:::benchmark_data_smithi008_432048_object70306:head v 295'4515, mlcod=295'4513)", 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:26.869666+0000", 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.4928649739999997, 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.10846813399999999, 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:31.513 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.869666+0000", 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.869666+0000", 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.514 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.515 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.869665+0000", 2023-10-09T05:48:31.515 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:31.515 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.515 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.515 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.515 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.869685+0000", 2023-10-09T05:48:31.515 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.942e-05 2023-10-09T05:48:31.515 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.516 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.516 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.516 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.869693+0000", 2023-10-09T05:48:31.516 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6400000000000003e-06 2023-10-09T05:48:31.516 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.516 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.516 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.517 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.869699+0000", 2023-10-09T05:48:31.517 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3460000000000001e-06 2023-10-09T05:48:31.517 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.517 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.517 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.517 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.869744+0000", 2023-10-09T05:48:31.517 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5549e-05 2023-10-09T05:48:31.518 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.518 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.518 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.518 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.869762+0000", 2023-10-09T05:48:31.518 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8088e-05 2023-10-09T05:48:31.518 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.518 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.519 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:31.519 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.978092+0000", 2023-10-09T05:48:31.519 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.108330015 2023-10-09T05:48:31.519 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.519 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.519 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.519 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.978134+0000", 2023-10-09T05:48:31.519 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1764000000000002e-05 2023-10-09T05:48:31.520 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.520 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.520 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.520 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.520 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.520 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:70306 8.b e295/294 8:d9035811:::benchmark_data_smithi008_432048_object70305:head v 295'4485, mlcod=295'4483)", 2023-10-09T05:48:31.521 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:26.870712+0000", 2023-10-09T05:48:31.521 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.4918188709999995, 2023-10-09T05:48:31.521 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.116338807, 2023-10-09T05:48:31.521 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.521 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:31.521 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.521 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.522 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.522 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870712+0000", 2023-10-09T05:48:31.522 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.522 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.522 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.522 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.523 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870712+0000", 2023-10-09T05:48:31.523 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870711+0000", 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870741+0000", 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9853999999999999e-05 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.524 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870751+0000", 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0309000000000001e-05 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870758+0000", 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5610000000000004e-06 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.525 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870805+0000", 2023-10-09T05:48:31.526 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7243999999999997e-05 2023-10-09T05:48:31.526 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.526 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.526 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.526 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.978069+0000", 2023-10-09T05:48:31.526 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.107263845 2023-10-09T05:48:31.526 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.526 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.527 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:31.527 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.987003+0000", 2023-10-09T05:48:31.527 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0089340510000000001 2023-10-09T05:48:31.527 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.527 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.527 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.527 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.987051+0000", 2023-10-09T05:48:31.527 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7698000000000003e-05 2023-10-09T05:48:31.527 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.527 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.527 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:70308 8.f e295/294 8:f5c36ee3:::benchmark_data_smithi008_432048_object70307:head v 295'4295, mlcod=295'4292)", 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:26.870773+0000", 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.4917576009999998, 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.114593713, 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870773+0000", 2023-10-09T05:48:31.528 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870773+0000", 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870773+0000", 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.529 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.530 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:48:31.530 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870799+0000", 2023-10-09T05:48:31.530 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6296999999999999e-05 2023-10-09T05:48:31.530 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.530 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.530 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:48:31.530 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870806+0000", 2023-10-09T05:48:31.530 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7710000000000001e-06 2023-10-09T05:48:31.530 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.530 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.530 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.870811+0000", 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8779999999999997e-06 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.978134+0000", 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.10732346199999999 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.978169+0000", 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4479e-05 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.531 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.532 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:48:31.532 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.985335+0000", 2023-10-09T05:48:31.532 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0071665820000000003 2023-10-09T05:48:31.532 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:48:31.532 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:48:31.532 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:48:31.532 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:26.985367+0000", 2023-10-09T05:48:31.532 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1535999999999998e-05 2023-10-09T05:48:31.532 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.532 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.532 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.533 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:48:31.533 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:48:31.533 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:48:31.533 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:31.534 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:31.603 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:31.662 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:31.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:31.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:31.767 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:31.793 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:31.835 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:31.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:31.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:31.943 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:31.975 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:32.133 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 49 16 75298 75282 95.9808 47.9375 0.00822215 0.0104144 2023-10-09T05:48:32.293 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:32.298 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:32 smithi008 ceph-mon[184418]: pgmap v898: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:48:32.299 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:32 smithi008 ceph-mon[191227]: pgmap v898: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:48:32.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:32.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:32.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:32 smithi191 ceph-mon[161669]: pgmap v898: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:48:32.401 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:32.421 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:32.454 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:32.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:48:32.542 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:32.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:32.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:32.655 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:32.712 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:32.801 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:32.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:32.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:32.889 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:48:32.914 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:32.956 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:33.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:33 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:33.044 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:33.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:33.134 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 50 16 76345 76329 95.37 65.4375 0.00848032 0.01048 2023-10-09T05:48:33.161 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:33.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:33 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:33 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:33.236 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:33.934 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:33.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:33.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:34.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:34.088 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:34.129 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:34.133 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 51 16 77135 77119 94.4683 49.375 0.00314368 0.0105684 2023-10-09T05:48:34.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:34.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:34.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:34 smithi008 ceph-mon[184418]: pgmap v899: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-09T05:48:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:34 smithi008 ceph-mon[191227]: pgmap v899: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-09T05:48:34.237 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:34.280 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:34.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:34 smithi191 ceph-mon[161669]: pgmap v899: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-09T05:48:34.616 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:34.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:34.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:34.723 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:34.797 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:35.134 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 52 16 78200 78184 93.9317 66.5625 0.0152672 0.010641 2023-10-09T05:48:35.316 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:48:35.351 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:48:35.355 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:35.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:35.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:35.397 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:35.399 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:48:35.447 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:48:35.469 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:35.499 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:48:35.513 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:48:35.525 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:35.545 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:48:35.586 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:48:35.623 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:48:35.625 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:48:35.625 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:48:35.625 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:48:35.626 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:48:35.626 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:48:35.626 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:48:35.626 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:48:35.626 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:48:35.634 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:35.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:35.741 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:35.836 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:35.864 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:48:35.876 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:48:36.135 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 53 16 79072 79056 93.186 54.5 0.0166672 0.0107247 2023-10-09T05:48:36.194 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:36 smithi191 ceph-mon[161669]: pgmap v900: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s 2023-10-09T05:48:36.194 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3462529328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:36.194 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:36 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:36.194 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:36 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:36.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:36 smithi008 ceph-mon[184418]: pgmap v900: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s 2023-10-09T05:48:36.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3462529328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:36 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:36 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:36 smithi008 ceph-mon[191227]: pgmap v900: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s 2023-10-09T05:48:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3462529328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:36 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:36.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:36 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:36.257 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:36.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:36.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:36.369 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:36.406 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:37.135 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 54 16 79935 79919 92.4593 53.9375 0.00877595 0.0108104 2023-10-09T05:48:37.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:37 smithi008 ceph-mon[184418]: from='client.35901 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:48:37.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:37 smithi008 ceph-mon[184418]: pgmap v901: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T05:48:37.366 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:37 smithi008 ceph-mon[191227]: from='client.35901 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:48:37.366 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:37 smithi008 ceph-mon[191227]: pgmap v901: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T05:48:37.492 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:37 smithi191 ceph-mon[161669]: from='client.35901 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:48:37.492 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:37 smithi191 ceph-mon[161669]: pgmap v901: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-09T05:48:37.602 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:37.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:37.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:37.707 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:37.762 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:37.809 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:48:37.809 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:48:38.019 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":900,"stamp":"2023-10-09T05:48:34.358959+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4896069555,"num_objects":74663,"num_object_clones":0,"num_object_copies":223989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74663,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":75741,"num_write_kb":4791781,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":132,"num_bytes_recovered":5385367,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75805,"ondisk_log_size":75805,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18396952,"kb_used_data":14427552,"kb_used_omap":38,"kb_used_meta":3969305,"kb_avail":637819112,"statfs":{"total":671965249536,"available":653126770688,"internally_reserved":0,"allocated":14773813248,"data_stored":14768160905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39147,"internal_metadata":4064569109},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,6,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":574816256,"num_objects":8771,"num_object_clones":0,"num_object_copies":26313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8771,"num_write_kb":561344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.304367"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":287,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.064062+0000","last_change":"2023-10-09T05:45:17.927530+0000","last_active":"2023-10-09T05:48:25.064062+0000","last_peered":"2023-10-09T05:48:25.064062+0000","last_clean":"2023-10-09T05:48:25.064062+0000","last_became_active":"2023-10-09T05:45:17.926963+0000","last_became_peered":"2023-10-09T05:45:17.926963+0000","last_unstale":"2023-10-09T05:48:25.064062+0000","last_undegraded":"2023-10-09T05:48:25.064062+0000","last_fullsized":"2023-10-09T05:48:25.064062+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":239,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.071248+0000","last_change":"2023-10-09T05:48:25.069466+0000","last_active":"2023-10-09T05:48:25.071248+0000","last_peered":"2023-10-09T05:48:25.071248+0000","last_clean":"2023-10-09T05:48:25.071248+0000","last_became_active":"2023-10-09T05:48:25.069167+0000","last_became_peered":"2023-10-09T05:48:25.069167+0000","last_unstale":"2023-10-09T05:48:25.071248+0000","last_undegraded":"2023-10-09T05:48:25.071248+0000","last_fullsized":"2023-10-09T05:48:25.071248+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":53,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.071658+0000","last_change":"2023-10-09T05:48:25.071096+0000","last_active":"2023-10-09T05:48:25.071658+0000","last_peered":"2023-10-09T05:48:25.071658+0000","last_clean":"2023-10-09T05:48:25.071658+0000","last_became_active":"2023-10-09T05:48:25.070794+0000","last_became_peered":"2023-10-09T05:48:25.070794+0000","last_unstale":"2023-10-09T05:48:25.071658+0000","last_undegraded":"2023-10-09T05:48:25.071658+0000","last_fullsized":"2023-10-09T05:48:25.071658+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":294,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.058213+0000","last_change":"2023-10-09T05:46:56.027800+0000","last_active":"2023-10-09T05:48:25.058213+0000","last_peered":"2023-10-09T05:48:25.058213+0000","last_clean":"2023-10-09T05:48:25.058213+0000","last_became_active":"2023-10-09T05:46:56.027372+0000","last_became_peered":"2023-10-09T05:46:56.027372+0000","last_unstale":"2023-10-09T05:48:25.058213+0000","last_undegraded":"2023-10-09T05:48:25.058213+0000","last_fullsized":"2023-10-09T05:48:25.058213+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:47.111072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":255,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:28.879919+0000","last_change":"2023-10-09T05:48:25.061900+0000","last_active":"2023-10-09T05:48:28.879919+0000","last_peered":"2023-10-09T05:48:28.879919+0000","last_clean":"2023-10-09T05:48:28.879919+0000","last_became_active":"2023-10-09T05:48:25.061568+0000","last_became_peered":"2023-10-09T05:48:25.061568+0000","last_unstale":"2023-10-09T05:48:28.879919+0000","last_undegraded":"2023-10-09T05:48:28.879919+0000","last_fullsized":"2023-10-09T05:48:28.879919+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:03.250730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":120,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.062380+0000","last_change":"2023-10-09T05:45:17.925251+0000","last_active":"2023-10-09T05:48:25.062380+0000","last_peered":"2023-10-09T05:48:25.062380+0000","last_clean":"2023-10-09T05:48:25.062380+0000","last_became_active":"2023-10-09T05:45:17.924982+0000","last_became_peered":"2023-10-09T05:45:17.924982+0000","last_unstale":"2023-10-09T05:48:25.062380+0000","last_undegraded":"2023-10-09T05:48:25.062380+0000","last_fullsized":"2023-10-09T05:48:25.062380+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":250,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.056914+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:48:25.056914+0000","last_peered":"2023-10-09T05:48:25.056914+0000","last_clean":"2023-10-09T05:48:25.056914+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:48:25.056914+0000","last_undegraded":"2023-10-09T05:48:25.056914+0000","last_fullsized":"2023-10-09T05:48:25.056914+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":239,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.081930+0000","last_change":"2023-10-09T05:48:25.069506+0000","last_active":"2023-10-09T05:48:25.081930+0000","last_peered":"2023-10-09T05:48:25.081930+0000","last_clean":"2023-10-09T05:48:25.081930+0000","last_became_active":"2023-10-09T05:48:25.069232+0000","last_became_peered":"2023-10-09T05:48:25.069232+0000","last_unstale":"2023-10-09T05:48:25.081930+0000","last_undegraded":"2023-10-09T05:48:25.081930+0000","last_fullsized":"2023-10-09T05:48:25.081930+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":306,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.061991+0000","last_change":"2023-10-09T05:47:33.925724+0000","last_active":"2023-10-09T05:48:25.061991+0000","last_peered":"2023-10-09T05:48:25.061991+0000","last_clean":"2023-10-09T05:48:25.061991+0000","last_became_active":"2023-10-09T05:47:33.925304+0000","last_became_peered":"2023-10-09T05:47:33.925304+0000","last_unstale":"2023-10-09T05:48:25.061991+0000","last_undegraded":"2023-10-09T05:48:25.061991+0000","last_fullsized":"2023-10-09T05:48:25.061991+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":231,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.068703+0000","last_change":"2023-10-09T05:48:25.068328+0000","last_active":"2023-10-09T05:48:25.068703+0000","last_peered":"2023-10-09T05:48:25.068703+0000","last_clean":"2023-10-09T05:48:25.068703+0000","last_became_active":"2023-10-09T05:48:25.068094+0000","last_became_peered":"2023-10-09T05:48:25.068094+0000","last_unstale":"2023-10-09T05:48:25.068703+0000","last_undegraded":"2023-10-09T05:48:25.068703+0000","last_fullsized":"2023-10-09T05:48:25.068703+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":81,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.061888+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:48:25.061888+0000","last_peered":"2023-10-09T05:48:25.061888+0000","last_clean":"2023-10-09T05:48:25.061888+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:48:25.061888+0000","last_undegraded":"2023-10-09T05:48:25.061888+0000","last_fullsized":"2023-10-09T05:48:25.061888+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":89,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.060840+0000","last_change":"2023-10-09T05:45:17.924621+0000","last_active":"2023-10-09T05:48:25.060840+0000","last_peered":"2023-10-09T05:48:25.060840+0000","last_clean":"2023-10-09T05:48:25.060840+0000","last_became_active":"2023-10-09T05:45:17.924313+0000","last_became_peered":"2023-10-09T05:45:17.924313+0000","last_unstale":"2023-10-09T05:48:25.060840+0000","last_undegraded":"2023-10-09T05:48:25.060840+0000","last_fullsized":"2023-10-09T05:48:25.060840+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:37.602060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":322,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.065821+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:48:25.065821+0000","last_peered":"2023-10-09T05:48:25.065821+0000","last_clean":"2023-10-09T05:48:25.065821+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:48:25.065821+0000","last_undegraded":"2023-10-09T05:48:25.065821+0000","last_fullsized":"2023-10-09T05:48:25.065821+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":270,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.065488+0000","last_change":"2023-10-09T05:45:17.925631+0000","last_active":"2023-10-09T05:48:25.065488+0000","last_peered":"2023-10-09T05:48:25.065488+0000","last_clean":"2023-10-09T05:48:25.065488+0000","last_became_active":"2023-10-09T05:45:17.925142+0000","last_became_peered":"2023-10-09T05:45:17.925142+0000","last_unstale":"2023-10-09T05:48:25.065488+0000","last_undegraded":"2023-10-09T05:48:25.065488+0000","last_fullsized":"2023-10-09T05:48:25.065488+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":81,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.061534+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:48:25.061534+0000","last_peered":"2023-10-09T05:48:25.061534+0000","last_clean":"2023-10-09T05:48:25.061534+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:48:25.061534+0000","last_undegraded":"2023-10-09T05:48:25.061534+0000","last_fullsized":"2023-10-09T05:48:25.061534+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":272,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.067058+0000","last_change":"2023-10-09T05:48:25.065222+0000","last_active":"2023-10-09T05:48:25.067058+0000","last_peered":"2023-10-09T05:48:25.067058+0000","last_clean":"2023-10-09T05:48:25.067058+0000","last_became_active":"2023-10-09T05:48:25.064901+0000","last_became_peered":"2023-10-09T05:48:25.064901+0000","last_unstale":"2023-10-09T05:48:25.067058+0000","last_undegraded":"2023-10-09T05:48:25.067058+0000","last_fullsized":"2023-10-09T05:48:25.067058+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":283,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.057633+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:48:25.057633+0000","last_peered":"2023-10-09T05:48:25.057633+0000","last_clean":"2023-10-09T05:48:25.057633+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:48:25.057633+0000","last_undegraded":"2023-10-09T05:48:25.057633+0000","last_fullsized":"2023-10-09T05:48:25.057633+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":254,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.061608+0000","last_change":"2023-10-09T05:46:56.019914+0000","last_active":"2023-10-09T05:48:25.061608+0000","last_peered":"2023-10-09T05:48:25.061608+0000","last_clean":"2023-10-09T05:48:25.061608+0000","last_became_active":"2023-10-09T05:46:56.019487+0000","last_became_peered":"2023-10-09T05:46:56.019487+0000","last_unstale":"2023-10-09T05:48:25.061608+0000","last_undegraded":"2023-10-09T05:48:25.061608+0000","last_fullsized":"2023-10-09T05:48:25.061608+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":75,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.063258+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:48:25.063258+0000","last_peered":"2023-10-09T05:48:25.063258+0000","last_clean":"2023-10-09T05:48:25.063258+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:48:25.063258+0000","last_undegraded":"2023-10-09T05:48:25.063258+0000","last_fullsized":"2023-10-09T05:48:25.063258+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":283,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.058291+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:48:25.058291+0000","last_peered":"2023-10-09T05:48:25.058291+0000","last_clean":"2023-10-09T05:48:25.058291+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:48:25.058291+0000","last_undegraded":"2023-10-09T05:48:25.058291+0000","last_fullsized":"2023-10-09T05:48:25.058291+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":90,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.056180+0000","last_change":"2023-10-09T05:46:56.194588+0000","last_active":"2023-10-09T05:48:25.056180+0000","last_peered":"2023-10-09T05:48:25.056180+0000","last_clean":"2023-10-09T05:48:25.056180+0000","last_became_active":"2023-10-09T05:46:56.018480+0000","last_became_peered":"2023-10-09T05:46:56.018480+0000","last_unstale":"2023-10-09T05:48:25.056180+0000","last_undegraded":"2023-10-09T05:48:25.056180+0000","last_fullsized":"2023-10-09T05:48:25.056180+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:39.150417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":292,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.067246+0000","last_change":"2023-10-09T05:48:25.066354+0000","last_active":"2023-10-09T05:48:25.067246+0000","last_peered":"2023-10-09T05:48:25.067246+0000","last_clean":"2023-10-09T05:48:25.067246+0000","last_became_active":"2023-10-09T05:48:25.065854+0000","last_became_peered":"2023-10-09T05:48:25.065854+0000","last_unstale":"2023-10-09T05:48:25.067246+0000","last_undegraded":"2023-10-09T05:48:25.067246+0000","last_fullsized":"2023-10-09T05:48:25.067246+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"296'4561","reported_seq":4580,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:30.097637+0000","last_change":"2023-10-09T05:47:33.926554+0000","last_active":"2023-10-09T05:48:30.097637+0000","last_peered":"2023-10-09T05:48:30.097637+0000","last_clean":"2023-10-09T05:48:30.097637+0000","last_became_active":"2023-10-09T05:47:33.926057+0000","last_became_peered":"2023-10-09T05:47:33.926057+0000","last_unstale":"2023-10-09T05:48:30.097637+0000","last_undegraded":"2023-10-09T05:48:30.097637+0000","last_fullsized":"2023-10-09T05:48:30.097637+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4561,"log_dups_size":0,"ondisk_log_size":4561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298909696,"num_objects":4561,"num_object_clones":0,"num_object_copies":13683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4561,"num_write_kb":291904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":79,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.061956+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:48:25.061956+0000","last_peered":"2023-10-09T05:48:25.061956+0000","last_clean":"2023-10-09T05:48:25.061956+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:48:25.061956+0000","last_undegraded":"2023-10-09T05:48:25.061956+0000","last_fullsized":"2023-10-09T05:48:25.061956+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":239,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.062206+0000","last_change":"2023-10-09T05:46:56.021669+0000","last_active":"2023-10-09T05:48:25.062206+0000","last_peered":"2023-10-09T05:48:25.062206+0000","last_clean":"2023-10-09T05:48:25.062206+0000","last_became_active":"2023-10-09T05:46:56.021306+0000","last_became_peered":"2023-10-09T05:46:56.021306+0000","last_unstale":"2023-10-09T05:48:25.062206+0000","last_undegraded":"2023-10-09T05:48:25.062206+0000","last_fullsized":"2023-10-09T05:48:25.062206+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"296'4572","reported_seq":4591,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:30.066726+0000","last_change":"2023-10-09T05:47:33.924487+0000","last_active":"2023-10-09T05:48:30.066726+0000","last_peered":"2023-10-09T05:48:30.066726+0000","last_clean":"2023-10-09T05:48:30.066726+0000","last_became_active":"2023-10-09T05:47:33.924029+0000","last_became_peered":"2023-10-09T05:47:33.924029+0000","last_unstale":"2023-10-09T05:48:30.066726+0000","last_undegraded":"2023-10-09T05:48:30.066726+0000","last_fullsized":"2023-10-09T05:48:30.066726+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4572,"log_dups_size":0,"ondisk_log_size":4572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299630592,"num_objects":4572,"num_object_clones":0,"num_object_copies":13716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4572,"num_write_kb":292608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":92,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.062084+0000","last_change":"2023-10-09T05:45:18.061225+0000","last_active":"2023-10-09T05:48:25.062084+0000","last_peered":"2023-10-09T05:48:25.062084+0000","last_clean":"2023-10-09T05:48:25.062084+0000","last_became_active":"2023-10-09T05:45:17.923689+0000","last_became_peered":"2023-10-09T05:45:17.923689+0000","last_unstale":"2023-10-09T05:48:25.062084+0000","last_undegraded":"2023-10-09T05:48:25.062084+0000","last_fullsized":"2023-10-09T05:48:25.062084+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":303,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.064159+0000","last_change":"2023-10-09T05:47:44.967957+0000","last_active":"2023-10-09T05:48:25.064159+0000","last_peered":"2023-10-09T05:48:25.064159+0000","last_clean":"2023-10-09T05:48:25.064159+0000","last_became_active":"2023-10-09T05:47:44.967620+0000","last_became_peered":"2023-10-09T05:47:44.967620+0000","last_unstale":"2023-10-09T05:48:25.064159+0000","last_undegraded":"2023-10-09T05:48:25.064159+0000","last_fullsized":"2023-10-09T05:48:25.064159+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"296'4757","reported_seq":4777,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:33.806971+0000","last_change":"2023-10-09T05:47:33.924395+0000","last_active":"2023-10-09T05:48:33.806971+0000","last_peered":"2023-10-09T05:48:33.806971+0000","last_clean":"2023-10-09T05:48:33.806971+0000","last_became_active":"2023-10-09T05:47:33.924062+0000","last_became_peered":"2023-10-09T05:47:33.924062+0000","last_unstale":"2023-10-09T05:48:33.806971+0000","last_undegraded":"2023-10-09T05:48:33.806971+0000","last_fullsized":"2023-10-09T05:48:33.806971+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4757,"log_dups_size":0,"ondisk_log_size":4757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:53.244643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311754752,"num_objects":4757,"num_object_clones":0,"num_object_copies":14271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4757,"num_write_kb":304448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":92,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.061717+0000","last_change":"2023-10-09T05:46:56.190043+0000","last_active":"2023-10-09T05:48:25.061717+0000","last_peered":"2023-10-09T05:48:25.061717+0000","last_clean":"2023-10-09T05:48:25.061717+0000","last_became_active":"2023-10-09T05:46:56.019072+0000","last_became_peered":"2023-10-09T05:46:56.019072+0000","last_unstale":"2023-10-09T05:48:25.061717+0000","last_undegraded":"2023-10-09T05:48:25.061717+0000","last_fullsized":"2023-10-09T05:48:25.061717+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":286,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.061182+0000","last_change":"2023-10-09T05:45:17.923093+0000","last_active":"2023-10-09T05:48:25.061182+0000","last_peered":"2023-10-09T05:48:25.061182+0000","last_clean":"2023-10-09T05:48:25.061182+0000","last_became_active":"2023-10-09T05:45:17.922756+0000","last_became_peered":"2023-10-09T05:45:17.922756+0000","last_unstale":"2023-10-09T05:48:25.061182+0000","last_undegraded":"2023-10-09T05:48:25.061182+0000","last_fullsized":"2023-10-09T05:48:25.061182+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.7","version":"296'4719","reported_seq":4745,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:30.048697+0000","last_change":"2023-10-09T05:48:25.074647+0000","last_active":"2023-10-09T05:48:30.048697+0000","last_peered":"2023-10-09T05:48:30.048697+0000","last_clean":"2023-10-09T05:48:30.048697+0000","last_became_active":"2023-10-09T05:48:25.069969+0000","last_became_peered":"2023-10-09T05:48:25.069969+0000","last_unstale":"2023-10-09T05:48:30.048697+0000","last_undegraded":"2023-10-09T05:48:30.048697+0000","last_fullsized":"2023-10-09T05:48:30.048697+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4719,"log_dups_size":0,"ondisk_log_size":4719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309264384,"num_objects":4719,"num_object_clones":0,"num_object_copies":14157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4719,"num_write_kb":302016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":263,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.062425+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:48:25.062425+0000","last_peered":"2023-10-09T05:48:25.062425+0000","last_clean":"2023-10-09T05:48:25.062425+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:48:25.062425+0000","last_undegraded":"2023-10-09T05:48:25.062425+0000","last_fullsized":"2023-10-09T05:48:25.062425+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":260,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.056065+0000","last_change":"2023-10-09T05:45:17.930153+0000","last_active":"2023-10-09T05:48:25.056065+0000","last_peered":"2023-10-09T05:48:25.056065+0000","last_clean":"2023-10-09T05:48:25.056065+0000","last_became_active":"2023-10-09T05:45:17.929831+0000","last_became_peered":"2023-10-09T05:45:17.929831+0000","last_unstale":"2023-10-09T05:48:25.056065+0000","last_undegraded":"2023-10-09T05:48:25.056065+0000","last_fullsized":"2023-10-09T05:48:25.056065+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:24.746849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"296'4607","reported_seq":4633,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:30.087342+0000","last_change":"2023-10-09T05:48:25.082204+0000","last_active":"2023-10-09T05:48:30.087342+0000","last_peered":"2023-10-09T05:48:30.087342+0000","last_clean":"2023-10-09T05:48:30.087342+0000","last_became_active":"2023-10-09T05:48:25.069221+0000","last_became_peered":"2023-10-09T05:48:25.069221+0000","last_unstale":"2023-10-09T05:48:30.087342+0000","last_undegraded":"2023-10-09T05:48:30.087342+0000","last_fullsized":"2023-10-09T05:48:30.087342+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4607,"log_dups_size":0,"ondisk_log_size":4607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301924352,"num_objects":4607,"num_object_clones":0,"num_object_copies":13821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4607,"num_write_kb":294848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":32,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:28.880015+0000","last_change":"2023-10-09T05:48:25.067945+0000","last_active":"2023-10-09T05:48:28.880015+0000","last_peered":"2023-10-09T05:48:28.880015+0000","last_clean":"2023-10-09T05:48:28.880015+0000","last_became_active":"2023-10-09T05:48:25.067721+0000","last_became_peered":"2023-10-09T05:48:25.067721+0000","last_unstale":"2023-10-09T05:48:28.880015+0000","last_undegraded":"2023-10-09T05:48:28.880015+0000","last_fullsized":"2023-10-09T05:48:28.880015+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:58.655544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":261,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.065731+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:48:25.065731+0000","last_peered":"2023-10-09T05:48:25.065731+0000","last_clean":"2023-10-09T05:48:25.065731+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:48:25.065731+0000","last_undegraded":"2023-10-09T05:48:25.065731+0000","last_fullsized":"2023-10-09T05:48:25.065731+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"296'4794","reported_seq":4814,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:33.870821+0000","last_change":"2023-10-09T05:47:33.925817+0000","last_active":"2023-10-09T05:48:33.870821+0000","last_peered":"2023-10-09T05:48:33.870821+0000","last_clean":"2023-10-09T05:48:33.870821+0000","last_became_active":"2023-10-09T05:47:33.924908+0000","last_became_peered":"2023-10-09T05:47:33.924908+0000","last_unstale":"2023-10-09T05:48:33.870821+0000","last_undegraded":"2023-10-09T05:48:33.870821+0000","last_fullsized":"2023-10-09T05:48:33.870821+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4794,"log_dups_size":0,"ondisk_log_size":4794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:16.620805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314179584,"num_objects":4794,"num_object_clones":0,"num_object_copies":14382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4794,"num_write_kb":306816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":121,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.064416+0000","last_change":"2023-10-09T05:45:17.921547+0000","last_active":"2023-10-09T05:48:25.064416+0000","last_peered":"2023-10-09T05:48:25.064416+0000","last_clean":"2023-10-09T05:48:25.064416+0000","last_became_active":"2023-10-09T05:45:17.921221+0000","last_became_peered":"2023-10-09T05:45:17.921221+0000","last_unstale":"2023-10-09T05:48:25.064416+0000","last_undegraded":"2023-10-09T05:48:25.064416+0000","last_fullsized":"2023-10-09T05:48:25.064416+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":285,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.066295+0000","last_change":"2023-10-09T05:45:17.923692+0000","last_active":"2023-10-09T05:48:25.066295+0000","last_peered":"2023-10-09T05:48:25.066295+0000","last_clean":"2023-10-09T05:48:25.066295+0000","last_became_active":"2023-10-09T05:45:17.917743+0000","last_became_peered":"2023-10-09T05:45:17.917743+0000","last_unstale":"2023-10-09T05:48:25.066295+0000","last_undegraded":"2023-10-09T05:48:25.066295+0000","last_fullsized":"2023-10-09T05:48:25.066295+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"296'4484","reported_seq":4503,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:30.049462+0000","last_change":"2023-10-09T05:47:33.924497+0000","last_active":"2023-10-09T05:48:30.049462+0000","last_peered":"2023-10-09T05:48:30.049462+0000","last_clean":"2023-10-09T05:48:30.049462+0000","last_became_active":"2023-10-09T05:47:33.924148+0000","last_became_peered":"2023-10-09T05:47:33.924148+0000","last_unstale":"2023-10-09T05:48:30.049462+0000","last_undegraded":"2023-10-09T05:48:30.049462+0000","last_fullsized":"2023-10-09T05:48:30.049462+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4484,"log_dups_size":0,"ondisk_log_size":4484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293863424,"num_objects":4484,"num_object_clones":0,"num_object_copies":13452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4484,"num_write_kb":286976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":295,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.063848+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:48:25.063848+0000","last_peered":"2023-10-09T05:48:25.063848+0000","last_clean":"2023-10-09T05:48:25.063848+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:48:25.063848+0000","last_undegraded":"2023-10-09T05:48:25.063848+0000","last_fullsized":"2023-10-09T05:48:25.063848+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":279,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.066086+0000","last_change":"2023-10-09T05:45:17.922009+0000","last_active":"2023-10-09T05:48:25.066086+0000","last_peered":"2023-10-09T05:48:25.066086+0000","last_clean":"2023-10-09T05:48:25.066086+0000","last_became_active":"2023-10-09T05:45:17.921636+0000","last_became_peered":"2023-10-09T05:45:17.921636+0000","last_unstale":"2023-10-09T05:48:25.066086+0000","last_undegraded":"2023-10-09T05:48:25.066086+0000","last_fullsized":"2023-10-09T05:48:25.066086+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"296'4558","reported_seq":4577,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:30.072635+0000","last_change":"2023-10-09T05:47:33.926505+0000","last_active":"2023-10-09T05:48:30.072635+0000","last_peered":"2023-10-09T05:48:30.072635+0000","last_clean":"2023-10-09T05:48:30.072635+0000","last_became_active":"2023-10-09T05:47:33.926163+0000","last_became_peered":"2023-10-09T05:47:33.926163+0000","last_unstale":"2023-10-09T05:48:30.072635+0000","last_undegraded":"2023-10-09T05:48:30.072635+0000","last_fullsized":"2023-10-09T05:48:30.072635+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4558,"log_dups_size":0,"ondisk_log_size":4558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298713088,"num_objects":4558,"num_object_clones":0,"num_object_copies":13674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4558,"num_write_kb":291712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":274,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.071374+0000","last_change":"2023-10-09T05:48:25.069827+0000","last_active":"2023-10-09T05:48:25.071374+0000","last_peered":"2023-10-09T05:48:25.071374+0000","last_clean":"2023-10-09T05:48:25.071374+0000","last_became_active":"2023-10-09T05:48:25.069514+0000","last_became_peered":"2023-10-09T05:48:25.069514+0000","last_unstale":"2023-10-09T05:48:25.071374+0000","last_undegraded":"2023-10-09T05:48:25.071374+0000","last_fullsized":"2023-10-09T05:48:25.071374+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":53,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.084100+0000","last_change":"2023-10-09T05:48:25.074478+0000","last_active":"2023-10-09T05:48:25.084100+0000","last_peered":"2023-10-09T05:48:25.084100+0000","last_clean":"2023-10-09T05:48:25.084100+0000","last_became_active":"2023-10-09T05:48:25.074071+0000","last_became_peered":"2023-10-09T05:48:25.074071+0000","last_unstale":"2023-10-09T05:48:25.084100+0000","last_undegraded":"2023-10-09T05:48:25.084100+0000","last_fullsized":"2023-10-09T05:48:25.084100+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.e","version":"296'4828","reported_seq":4848,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:33.867315+0000","last_change":"2023-10-09T05:47:33.927196+0000","last_active":"2023-10-09T05:48:33.867315+0000","last_peered":"2023-10-09T05:48:33.867315+0000","last_clean":"2023-10-09T05:48:33.867315+0000","last_became_active":"2023-10-09T05:47:33.926965+0000","last_became_peered":"2023-10-09T05:47:33.926965+0000","last_unstale":"2023-10-09T05:48:33.867315+0000","last_undegraded":"2023-10-09T05:48:33.867315+0000","last_fullsized":"2023-10-09T05:48:33.867315+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4828,"log_dups_size":0,"ondisk_log_size":4828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:17.414280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316407808,"num_objects":4828,"num_object_clones":0,"num_object_copies":14484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4828,"num_write_kb":308992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":657,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.058366+0000","last_change":"2023-10-09T05:46:43.948925+0000","last_active":"2023-10-09T05:48:25.058366+0000","last_peered":"2023-10-09T05:48:25.058366+0000","last_clean":"2023-10-09T05:48:25.058366+0000","last_became_active":"2023-10-09T05:46:43.943804+0000","last_became_peered":"2023-10-09T05:46:43.943804+0000","last_unstale":"2023-10-09T05:48:25.058366+0000","last_undegraded":"2023-10-09T05:48:25.058366+0000","last_fullsized":"2023-10-09T05:48:25.058366+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:37.381415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":278,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.062520+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:48:25.062520+0000","last_peered":"2023-10-09T05:48:25.062520+0000","last_clean":"2023-10-09T05:48:25.062520+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:48:25.062520+0000","last_undegraded":"2023-10-09T05:48:25.062520+0000","last_fullsized":"2023-10-09T05:48:25.062520+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":49,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.064158+0000","last_change":"2023-10-09T05:46:56.021018+0000","last_active":"2023-10-09T05:48:25.064158+0000","last_peered":"2023-10-09T05:48:25.064158+0000","last_clean":"2023-10-09T05:48:25.064158+0000","last_became_active":"2023-10-09T05:46:56.020650+0000","last_became_peered":"2023-10-09T05:46:56.020650+0000","last_unstale":"2023-10-09T05:48:25.064158+0000","last_undegraded":"2023-10-09T05:48:25.064158+0000","last_fullsized":"2023-10-09T05:48:25.064158+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"296'4559","reported_seq":4576,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:30.095985+0000","last_change":"2023-10-09T05:47:33.924772+0000","last_active":"2023-10-09T05:48:30.095985+0000","last_peered":"2023-10-09T05:48:30.095985+0000","last_clean":"2023-10-09T05:48:30.095985+0000","last_became_active":"2023-10-09T05:47:33.924435+0000","last_became_peered":"2023-10-09T05:47:33.924435+0000","last_unstale":"2023-10-09T05:48:30.095985+0000","last_undegraded":"2023-10-09T05:48:30.095985+0000","last_fullsized":"2023-10-09T05:48:30.095985+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4559,"log_dups_size":0,"ondisk_log_size":4559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298778624,"num_objects":4559,"num_object_clones":0,"num_object_copies":13677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4559,"num_write_kb":291776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":759,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.063016+0000","last_change":"2023-10-09T05:46:04.573476+0000","last_active":"2023-10-09T05:48:25.063016+0000","last_peered":"2023-10-09T05:48:25.063016+0000","last_clean":"2023-10-09T05:48:25.063016+0000","last_became_active":"2023-10-09T05:45:17.925130+0000","last_became_peered":"2023-10-09T05:45:17.925130+0000","last_unstale":"2023-10-09T05:48:25.063016+0000","last_undegraded":"2023-10-09T05:48:25.063016+0000","last_fullsized":"2023-10-09T05:48:25.063016+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":262,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.065793+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:48:25.065793+0000","last_peered":"2023-10-09T05:48:25.065793+0000","last_clean":"2023-10-09T05:48:25.065793+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:48:25.065793+0000","last_undegraded":"2023-10-09T05:48:25.065793+0000","last_fullsized":"2023-10-09T05:48:25.065793+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":130,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.060947+0000","last_change":"2023-10-09T05:45:17.919819+0000","last_active":"2023-10-09T05:48:25.060947+0000","last_peered":"2023-10-09T05:48:25.060947+0000","last_clean":"2023-10-09T05:48:25.060947+0000","last_became_active":"2023-10-09T05:45:17.919447+0000","last_became_peered":"2023-10-09T05:45:17.919447+0000","last_unstale":"2023-10-09T05:48:25.060947+0000","last_undegraded":"2023-10-09T05:48:25.060947+0000","last_fullsized":"2023-10-09T05:48:25.060947+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:18.280449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"296'4678","reported_seq":4706,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:29.998805+0000","last_change":"2023-10-09T05:48:25.088786+0000","last_active":"2023-10-09T05:48:29.998805+0000","last_peered":"2023-10-09T05:48:29.998805+0000","last_clean":"2023-10-09T05:48:29.998805+0000","last_became_active":"2023-10-09T05:48:25.069118+0000","last_became_peered":"2023-10-09T05:48:25.069118+0000","last_unstale":"2023-10-09T05:48:29.998805+0000","last_undegraded":"2023-10-09T05:48:29.998805+0000","last_fullsized":"2023-10-09T05:48:29.998805+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4678,"log_dups_size":0,"ondisk_log_size":4678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306577408,"num_objects":4678,"num_object_clones":0,"num_object_copies":14034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4678,"num_write_kb":299392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":614,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.060665+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:48:25.060665+0000","last_peered":"2023-10-09T05:48:25.060665+0000","last_clean":"2023-10-09T05:48:25.060665+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:48:25.060665+0000","last_undegraded":"2023-10-09T05:48:25.060665+0000","last_fullsized":"2023-10-09T05:48:25.060665+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":89,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.060296+0000","last_change":"2023-10-09T05:45:17.921491+0000","last_active":"2023-10-09T05:48:25.060296+0000","last_peered":"2023-10-09T05:48:25.060296+0000","last_clean":"2023-10-09T05:48:25.060296+0000","last_became_active":"2023-10-09T05:45:17.921145+0000","last_became_peered":"2023-10-09T05:45:17.921145+0000","last_unstale":"2023-10-09T05:48:25.060296+0000","last_undegraded":"2023-10-09T05:48:25.060296+0000","last_fullsized":"2023-10-09T05:48:25.060296+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:45.903609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":113,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.062396+0000","last_change":"2023-10-09T05:45:17.921021+0000","last_active":"2023-10-09T05:48:25.062396+0000","last_peered":"2023-10-09T05:48:25.062396+0000","last_clean":"2023-10-09T05:48:25.062396+0000","last_became_active":"2023-10-09T05:45:17.920667+0000","last_became_peered":"2023-10-09T05:45:17.920667+0000","last_unstale":"2023-10-09T05:48:25.062396+0000","last_undegraded":"2023-10-09T05:48:25.062396+0000","last_fullsized":"2023-10-09T05:48:25.062396+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"296'4858","reported_seq":4878,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:33.845355+0000","last_change":"2023-10-09T05:47:33.926003+0000","last_active":"2023-10-09T05:48:33.845355+0000","last_peered":"2023-10-09T05:48:33.845355+0000","last_clean":"2023-10-09T05:48:33.845355+0000","last_became_active":"2023-10-09T05:47:33.925296+0000","last_became_peered":"2023-10-09T05:47:33.925296+0000","last_unstale":"2023-10-09T05:48:33.845355+0000","last_undegraded":"2023-10-09T05:48:33.845355+0000","last_fullsized":"2023-10-09T05:48:33.845355+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4858,"log_dups_size":0,"ondisk_log_size":4858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:53.364286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318373888,"num_objects":4858,"num_object_clones":0,"num_object_copies":14574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4858,"num_write_kb":310912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":942,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.062360+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:48:25.062360+0000","last_peered":"2023-10-09T05:48:25.062360+0000","last_clean":"2023-10-09T05:48:25.062360+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:48:25.062360+0000","last_undegraded":"2023-10-09T05:48:25.062360+0000","last_fullsized":"2023-10-09T05:48:25.062360+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":446,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.061760+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:48:25.061760+0000","last_peered":"2023-10-09T05:48:25.061760+0000","last_clean":"2023-10-09T05:48:25.061760+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:48:25.061760+0000","last_undegraded":"2023-10-09T05:48:25.061760+0000","last_fullsized":"2023-10-09T05:48:25.061760+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":68,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.072636+0000","last_change":"2023-10-09T05:48:25.068759+0000","last_active":"2023-10-09T05:48:25.072636+0000","last_peered":"2023-10-09T05:48:25.072636+0000","last_clean":"2023-10-09T05:48:25.072636+0000","last_became_active":"2023-10-09T05:48:25.068358+0000","last_became_peered":"2023-10-09T05:48:25.068358+0000","last_unstale":"2023-10-09T05:48:25.072636+0000","last_undegraded":"2023-10-09T05:48:25.072636+0000","last_fullsized":"2023-10-09T05:48:25.072636+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":263,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.066590+0000","last_change":"2023-10-09T05:44:50.495966+0000","last_active":"2023-10-09T05:48:25.066590+0000","last_peered":"2023-10-09T05:48:25.066590+0000","last_clean":"2023-10-09T05:48:25.066590+0000","last_became_active":"2023-10-09T05:44:50.495628+0000","last_became_peered":"2023-10-09T05:44:50.495628+0000","last_unstale":"2023-10-09T05:48:25.066590+0000","last_undegraded":"2023-10-09T05:48:25.066590+0000","last_fullsized":"2023-10-09T05:48:25.066590+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"296'4706","reported_seq":4726,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:32.233221+0000","last_change":"2023-10-09T05:47:33.922023+0000","last_active":"2023-10-09T05:48:32.233221+0000","last_peered":"2023-10-09T05:48:32.233221+0000","last_clean":"2023-10-09T05:48:32.233221+0000","last_became_active":"2023-10-09T05:47:33.921780+0000","last_became_peered":"2023-10-09T05:47:33.921780+0000","last_unstale":"2023-10-09T05:48:32.233221+0000","last_undegraded":"2023-10-09T05:48:32.233221+0000","last_fullsized":"2023-10-09T05:48:32.233221+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4706,"log_dups_size":0,"ondisk_log_size":4706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:53.017735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308412416,"num_objects":4706,"num_object_clones":0,"num_object_copies":14118,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4706,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4706,"num_write_kb":301184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":779,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.061800+0000","last_change":"2023-10-09T05:47:10.000767+0000","last_active":"2023-10-09T05:48:25.061800+0000","last_peered":"2023-10-09T05:48:25.061800+0000","last_clean":"2023-10-09T05:48:25.061800+0000","last_became_active":"2023-10-09T05:47:10.000259+0000","last_became_peered":"2023-10-09T05:47:10.000259+0000","last_unstale":"2023-10-09T05:48:25.061800+0000","last_undegraded":"2023-10-09T05:48:25.061800+0000","last_fullsized":"2023-10-09T05:48:25.061800+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":269,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.057680+0000","last_change":"2023-10-09T05:47:33.926192+0000","last_active":"2023-10-09T05:48:25.057680+0000","last_peered":"2023-10-09T05:48:25.057680+0000","last_clean":"2023-10-09T05:48:25.057680+0000","last_became_active":"2023-10-09T05:47:33.925588+0000","last_became_peered":"2023-10-09T05:47:33.925588+0000","last_unstale":"2023-10-09T05:48:25.057680+0000","last_undegraded":"2023-10-09T05:48:25.057680+0000","last_fullsized":"2023-10-09T05:48:25.057680+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:29.495701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":275,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.058026+0000","last_change":"2023-10-09T05:45:17.931710+0000","last_active":"2023-10-09T05:48:25.058026+0000","last_peered":"2023-10-09T05:48:25.058026+0000","last_clean":"2023-10-09T05:48:25.058026+0000","last_became_active":"2023-10-09T05:45:17.931290+0000","last_became_peered":"2023-10-09T05:45:17.931290+0000","last_unstale":"2023-10-09T05:48:25.058026+0000","last_undegraded":"2023-10-09T05:48:25.058026+0000","last_fullsized":"2023-10-09T05:48:25.058026+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:50.091125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"296'4497","reported_seq":4525,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:30.046679+0000","last_change":"2023-10-09T05:48:25.074441+0000","last_active":"2023-10-09T05:48:30.046679+0000","last_peered":"2023-10-09T05:48:30.046679+0000","last_clean":"2023-10-09T05:48:30.046679+0000","last_became_active":"2023-10-09T05:48:25.069907+0000","last_became_peered":"2023-10-09T05:48:25.069907+0000","last_unstale":"2023-10-09T05:48:30.046679+0000","last_undegraded":"2023-10-09T05:48:30.046679+0000","last_fullsized":"2023-10-09T05:48:30.046679+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4497,"log_dups_size":0,"ondisk_log_size":4497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294715392,"num_objects":4497,"num_object_clones":0,"num_object_copies":13491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4497,"num_write_kb":287808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":479,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.061970+0000","last_change":"2023-10-09T05:44:51.489799+0000","last_active":"2023-10-09T05:48:25.061970+0000","last_peered":"2023-10-09T05:48:25.061970+0000","last_clean":"2023-10-09T05:48:25.061970+0000","last_became_active":"2023-10-09T05:44:51.489301+0000","last_became_peered":"2023-10-09T05:44:51.489301+0000","last_unstale":"2023-10-09T05:48:25.061970+0000","last_undegraded":"2023-10-09T05:48:25.061970+0000","last_fullsized":"2023-10-09T05:48:25.061970+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":98,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.061037+0000","last_change":"2023-10-09T05:47:33.923152+0000","last_active":"2023-10-09T05:48:25.061037+0000","last_peered":"2023-10-09T05:48:25.061037+0000","last_clean":"2023-10-09T05:48:25.061037+0000","last_became_active":"2023-10-09T05:47:33.922832+0000","last_became_peered":"2023-10-09T05:47:33.922832+0000","last_unstale":"2023-10-09T05:48:25.061037+0000","last_undegraded":"2023-10-09T05:48:25.061037+0000","last_fullsized":"2023-10-09T05:48:25.061037+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:24.298932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":269,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.056762+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:48:25.056762+0000","last_peered":"2023-10-09T05:48:25.056762+0000","last_clean":"2023-10-09T05:48:25.056762+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:48:25.056762+0000","last_undegraded":"2023-10-09T05:48:25.056762+0000","last_fullsized":"2023-10-09T05:48:25.056762+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"296'4926","reported_seq":4945,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:33.870390+0000","last_change":"2023-10-09T05:47:33.923368+0000","last_active":"2023-10-09T05:48:33.870390+0000","last_peered":"2023-10-09T05:48:33.870390+0000","last_clean":"2023-10-09T05:48:33.870390+0000","last_became_active":"2023-10-09T05:47:33.923068+0000","last_became_peered":"2023-10-09T05:47:33.923068+0000","last_unstale":"2023-10-09T05:48:33.870390+0000","last_undegraded":"2023-10-09T05:48:33.870390+0000","last_fullsized":"2023-10-09T05:48:33.870390+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4926,"log_dups_size":0,"ondisk_log_size":4926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:48.468565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322830336,"num_objects":4926,"num_object_clones":0,"num_object_copies":14778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4926,"num_write_kb":315264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":243,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.067225+0000","last_change":"2023-10-09T05:48:25.066163+0000","last_active":"2023-10-09T05:48:25.067225+0000","last_peered":"2023-10-09T05:48:25.067225+0000","last_clean":"2023-10-09T05:48:25.067225+0000","last_became_active":"2023-10-09T05:48:25.065145+0000","last_became_peered":"2023-10-09T05:48:25.065145+0000","last_unstale":"2023-10-09T05:48:25.067225+0000","last_undegraded":"2023-10-09T05:48:25.067225+0000","last_fullsized":"2023-10-09T05:48:25.067225+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":541,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.066092+0000","last_change":"2023-10-09T05:46:06.414199+0000","last_active":"2023-10-09T05:48:25.066092+0000","last_peered":"2023-10-09T05:48:25.066092+0000","last_clean":"2023-10-09T05:48:25.066092+0000","last_became_active":"2023-10-09T05:46:06.413755+0000","last_became_peered":"2023-10-09T05:46:06.413755+0000","last_unstale":"2023-10-09T05:48:25.066092+0000","last_undegraded":"2023-10-09T05:48:25.066092+0000","last_fullsized":"2023-10-09T05:48:25.066092+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":236,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.068202+0000","last_change":"2023-10-09T05:48:25.067326+0000","last_active":"2023-10-09T05:48:25.068202+0000","last_peered":"2023-10-09T05:48:25.068202+0000","last_clean":"2023-10-09T05:48:25.068202+0000","last_became_active":"2023-10-09T05:48:25.066977+0000","last_became_peered":"2023-10-09T05:48:25.066977+0000","last_unstale":"2023-10-09T05:48:25.068202+0000","last_undegraded":"2023-10-09T05:48:25.068202+0000","last_fullsized":"2023-10-09T05:48:25.068202+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"296'4531","reported_seq":4550,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:48:30.080996+0000","last_change":"2023-10-09T05:47:33.926483+0000","last_active":"2023-10-09T05:48:30.080996+0000","last_peered":"2023-10-09T05:48:30.080996+0000","last_clean":"2023-10-09T05:48:30.080996+0000","last_became_active":"2023-10-09T05:47:33.926006+0000","last_became_peered":"2023-10-09T05:47:33.926006+0000","last_unstale":"2023-10-09T05:48:30.080996+0000","last_undegraded":"2023-10-09T05:48:30.080996+0000","last_fullsized":"2023-10-09T05:48:30.080996+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":4531,"log_dups_size":0,"ondisk_log_size":4531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296943616,"num_objects":4531,"num_object_clones":0,"num_object_copies":13593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4531,"num_write_kb":289984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":793,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.062060+0000","last_change":"2023-10-09T05:44:50.482211+0000","last_active":"2023-10-09T05:48:25.062060+0000","last_peered":"2023-10-09T05:48:25.062060+0000","last_clean":"2023-10-09T05:48:25.062060+0000","last_became_active":"2023-10-09T05:44:50.481749+0000","last_became_peered":"2023-10-09T05:44:50.481749+0000","last_unstale":"2023-10-09T05:48:25.062060+0000","last_undegraded":"2023-10-09T05:48:25.062060+0000","last_fullsized":"2023-10-09T05:48:25.062060+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":267,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.058121+0000","last_change":"2023-10-09T05:45:17.975802+0000","last_active":"2023-10-09T05:48:25.058121+0000","last_peered":"2023-10-09T05:48:25.058121+0000","last_clean":"2023-10-09T05:48:25.058121+0000","last_became_active":"2023-10-09T05:45:17.931496+0000","last_became_peered":"2023-10-09T05:45:17.931496+0000","last_unstale":"2023-10-09T05:48:25.058121+0000","last_undegraded":"2023-10-09T05:48:25.058121+0000","last_fullsized":"2023-10-09T05:48:25.058121+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:51.419126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":246,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.063930+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:48:25.063930+0000","last_peered":"2023-10-09T05:48:25.063930+0000","last_clean":"2023-10-09T05:48:25.063930+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:48:25.063930+0000","last_undegraded":"2023-10-09T05:48:25.063930+0000","last_fullsized":"2023-10-09T05:48:25.063930+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":271,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.056675+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:48:25.056675+0000","last_peered":"2023-10-09T05:48:25.056675+0000","last_clean":"2023-10-09T05:48:25.056675+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:48:25.056675+0000","last_undegraded":"2023-10-09T05:48:25.056675+0000","last_fullsized":"2023-10-09T05:48:25.056675+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":270,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:28.880066+0000","last_change":"2023-10-09T05:48:25.069355+0000","last_active":"2023-10-09T05:48:28.880066+0000","last_peered":"2023-10-09T05:48:28.880066+0000","last_clean":"2023-10-09T05:48:28.880066+0000","last_became_active":"2023-10-09T05:48:25.069015+0000","last_became_peered":"2023-10-09T05:48:25.069015+0000","last_unstale":"2023-10-09T05:48:28.880066+0000","last_undegraded":"2023-10-09T05:48:28.880066+0000","last_fullsized":"2023-10-09T05:48:28.880066+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:22.057261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":278,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.062328+0000","last_change":"2023-10-09T05:45:17.971642+0000","last_active":"2023-10-09T05:48:25.062328+0000","last_peered":"2023-10-09T05:48:25.062328+0000","last_clean":"2023-10-09T05:48:25.062328+0000","last_became_active":"2023-10-09T05:45:17.917031+0000","last_became_peered":"2023-10-09T05:45:17.917031+0000","last_unstale":"2023-10-09T05:48:25.062328+0000","last_undegraded":"2023-10-09T05:48:25.062328+0000","last_fullsized":"2023-10-09T05:48:25.062328+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":285,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.065875+0000","last_change":"2023-10-09T05:45:17.925441+0000","last_active":"2023-10-09T05:48:25.065875+0000","last_peered":"2023-10-09T05:48:25.065875+0000","last_clean":"2023-10-09T05:48:25.065875+0000","last_became_active":"2023-10-09T05:45:17.924653+0000","last_became_peered":"2023-10-09T05:45:17.924653+0000","last_unstale":"2023-10-09T05:48:25.065875+0000","last_undegraded":"2023-10-09T05:48:25.065875+0000","last_fullsized":"2023-10-09T05:48:25.065875+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":250,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.062474+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:48:25.062474+0000","last_peered":"2023-10-09T05:48:25.062474+0000","last_clean":"2023-10-09T05:48:25.062474+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:48:25.062474+0000","last_undegraded":"2023-10-09T05:48:25.062474+0000","last_fullsized":"2023-10-09T05:48:25.062474+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":91,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:28.879979+0000","last_change":"2023-10-09T05:48:25.062693+0000","last_active":"2023-10-09T05:48:28.879979+0000","last_peered":"2023-10-09T05:48:28.879979+0000","last_clean":"2023-10-09T05:48:28.879979+0000","last_became_active":"2023-10-09T05:48:25.062334+0000","last_became_peered":"2023-10-09T05:48:25.062334+0000","last_unstale":"2023-10-09T05:48:28.879979+0000","last_undegraded":"2023-10-09T05:48:28.879979+0000","last_fullsized":"2023-10-09T05:48:28.879979+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:33:15.975841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":13,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:27.234937+0000","last_change":"2023-10-09T05:48:25.069406+0000","last_active":"2023-10-09T05:48:27.234937+0000","last_peered":"2023-10-09T05:48:27.234937+0000","last_clean":"2023-10-09T05:48:27.234937+0000","last_became_active":"2023-10-09T05:48:25.069064+0000","last_became_peered":"2023-10-09T05:48:25.069064+0000","last_unstale":"2023-10-09T05:48:27.234937+0000","last_undegraded":"2023-10-09T05:48:27.234937+0000","last_fullsized":"2023-10-09T05:48:27.234937+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.567858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":96,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:27.229644+0000","last_change":"2023-10-09T05:48:25.063854+0000","last_active":"2023-10-09T05:48:27.229644+0000","last_peered":"2023-10-09T05:48:27.229644+0000","last_clean":"2023-10-09T05:48:27.229644+0000","last_became_active":"2023-10-09T05:48:25.063579+0000","last_became_peered":"2023-10-09T05:48:25.063579+0000","last_unstale":"2023-10-09T05:48:27.229644+0000","last_undegraded":"2023-10-09T05:48:27.229644+0000","last_fullsized":"2023-10-09T05:48:27.229644+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:02.389789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":269,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.057004+0000","last_change":"2023-10-09T05:46:56.197243+0000","last_active":"2023-10-09T05:48:25.057004+0000","last_peered":"2023-10-09T05:48:25.057004+0000","last_clean":"2023-10-09T05:48:25.057004+0000","last_became_active":"2023-10-09T05:46:56.022978+0000","last_became_peered":"2023-10-09T05:46:56.022978+0000","last_unstale":"2023-10-09T05:48:25.057004+0000","last_undegraded":"2023-10-09T05:48:25.057004+0000","last_fullsized":"2023-10-09T05:48:25.057004+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:08.905002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":270,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:48:25.064842+0000","last_change":"2023-10-09T05:45:17.920323+0000","last_active":"2023-10-09T05:48:25.064842+0000","last_peered":"2023-10-09T05:48:25.064842+0000","last_clean":"2023-10-09T05:48:25.064842+0000","last_became_active":"2023-10-09T05:45:17.919994+0000","last_became_peered":"2023-10-09T05:45:17.919994+0000","last_unstale":"2023-10-09T05:48:25.064842+0000","last_undegraded":"2023-10-09T05:48:25.064842+0000","last_fullsized":"2023-10-09T05:48:25.064842+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":10308,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4891279360,"num_objects":74635,"num_object_clones":0,"num_object_copies":223905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":74635,"num_write_kb":4776640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14663680000,"data_stored":14663680000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74635,"ondisk_log_size":74635,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846759,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1791080,"kb_used_data":1464480,"kb_used_omap":14,"kb_used_meta":326577,"kb_avail":91954072,"statfs":{"total":95995035648,"available":94160969728,"internally_reserved":0,"allocated":1499627520,"data_stored":1499017329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":334415706},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":3.0390000000000001,"15min":4.6200000000000001},"min":{"1min":0.224,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":2.7469999999999999,"5min":110.123,"15min":220.77500000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.76400000000000001,"5min":2.9830000000000001,"15min":4.899},"min":{"1min":0.25900000000000001,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":2.6920000000000002,"5min":110.13800000000001,"15min":220.78700000000001},"last":0.52900000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.792,"15min":2.8690000000000002},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":2.8010000000000002,"5min":83.575000000000003,"15min":190.68100000000001},"last":0.504},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.907,"15min":2.7759999999999998},"min":{"1min":0.28699999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.2240000000000002,"5min":87.516999999999996,"15min":191.09200000000001},"last":0.51900000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.1779999999999999,"15min":3.0329999999999999},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":2.8260000000000001,"5min":7.9349999999999996,"15min":78.716999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.21,"15min":2.9830000000000001},"min":{"1min":0.32100000000000001,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":2.8399999999999999,"5min":7.391,"15min":78.768000000000001},"last":0.41499999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":2.8010000000000002,"5min":2.8010000000000002,"15min":3.577},"min":{"1min":0.38,"5min":0.315,"15min":0.315},"max":{"1min":42.683999999999997,"5min":64.558999999999997,"15min":185.17099999999999},"last":50.865000000000002},{"interface":"front","average":{"1min":2.6880000000000002,"5min":2.6480000000000001,"15min":3.802},"min":{"1min":0.40899999999999997,"5min":0.33000000000000002,"15min":0.29799999999999999},"max":{"1min":43.283999999999999,"5min":64.5,"15min":183.35499999999999},"last":50.808}]},{"osd":4,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.69599999999999995,"15min":0.93899999999999995},"min":{"1min":0.314,"5min":0.28299999999999997,"15min":0.25700000000000001},"max":{"1min":3.5419999999999998,"5min":4.3620000000000001,"15min":12.789},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.78100000000000003,"5min":0.70699999999999996,"15min":1.0700000000000001},"min":{"1min":0.314,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":2.7160000000000002,"5min":3.9169999999999998,"15min":9.0470000000000006},"last":0.36499999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.92100000000000004,"15min":1.139},"min":{"1min":0.34200000000000003,"5min":0.33100000000000002,"15min":0.26900000000000002},"max":{"1min":2.6709999999999998,"5min":5.4950000000000001,"15min":10.813000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.80800000000000005,"15min":1.2649999999999999},"min":{"1min":0.34200000000000003,"5min":0.33400000000000002,"15min":0.28299999999999997},"max":{"1min":2.7679999999999998,"5min":4.7649999999999997,"15min":10.321},"last":0.58099999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.71599999999999997,"15min":1.0940000000000001},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.318},"max":{"1min":2.6429999999999998,"5min":3.7229999999999999,"15min":9.2530000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.77800000000000002,"15min":1.0469999999999999},"min":{"1min":0.378,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":2.7250000000000001,"5min":4.2279999999999998,"15min":8.2680000000000007},"last":0.56200000000000006}]}]},{"osd":6,"up_from":42,"seq":180388626785,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2673448,"kb_used_data":2083744,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":91071704,"statfs":{"total":95995035648,"available":93257424896,"internally_reserved":0,"allocated":2133753856,"data_stored":2133151637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:47:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":4.2350000000000003,"15min":4.5449999999999999},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.633,"5min":162.08799999999999,"15min":162.08799999999999},"last":1.2649999999999999},{"interface":"front","average":{"1min":0.66400000000000003,"5min":4.2229999999999999,"15min":4.3700000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.7869999999999999,"5min":161.90899999999999,"15min":161.90899999999999},"last":0.313}]},{"osd":1,"last update":"Mon Oct 9 05:48:03 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.03,"15min":3.8119999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":0.85599999999999998,"5min":6.5279999999999996,"15min":228.24299999999999},"last":1.2789999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.871,"15min":3.794},"min":{"1min":0.38700000000000001,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":1.6899999999999999,"5min":5.6109999999999998,"15min":228.22499999999999},"last":0.99199999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:47:35 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":2.6659999999999999,"15min":4.0430000000000001},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.9470000000000001,"5min":77.233000000000004,"15min":115.40600000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.57099999999999995,"5min":2.7229999999999999,"15min":4.0960000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":2.1070000000000002,"5min":77.254999999999995,"15min":115.297},"last":1.232}]},{"osd":3,"last update":"Mon Oct 9 05:48:03 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":2.782,"15min":3.8319999999999999},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":2.1629999999999998,"5min":77.578999999999994,"15min":115.244},"last":1.2589999999999999},{"interface":"front","average":{"1min":0.67900000000000005,"5min":2.9399999999999999,"15min":3.879},"min":{"1min":0.35499999999999998,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":1.667,"5min":77.516999999999996,"15min":115.34999999999999},"last":0.55000000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":0.70999999999999996,"15min":1.7749999999999999},"min":{"1min":0.34699999999999998,"5min":0.27100000000000002,"15min":0.23400000000000001},"max":{"1min":4.7359999999999998,"5min":4.7359999999999998,"15min":125.64},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.306,"5min":1.22,"15min":1.8440000000000001},"min":{"1min":0.35799999999999998,"5min":0.246,"15min":0.246},"max":{"1min":5.5999999999999996,"5min":13.609,"15min":125.625},"last":1.6950000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:48:22 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.84899999999999998,"15min":1.958},"min":{"1min":0.38,"5min":0.28999999999999998,"15min":0.27600000000000002},"max":{"1min":1.7410000000000001,"5min":6.141,"15min":127.35899999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.212,"5min":1.161,"15min":1.9359999999999999},"min":{"1min":0.40799999999999997,"5min":0.28699999999999998,"15min":0.27500000000000002},"max":{"1min":3.6960000000000002,"5min":14.250999999999999,"15min":128.93799999999999},"last":0.48399999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:48:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.70799999999999996,"15min":1.6519999999999999},"min":{"1min":0.40600000000000003,"5min":0.33000000000000002,"15min":0.26500000000000001},"max":{"1min":1.5469999999999999,"5min":4.3470000000000004,"15min":115.514},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":1.097,"15min":1.4750000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":0.94899999999999995,"5min":14.300000000000001,"15min":116.53100000000001},"last":0.51400000000000001}]}]},{"osd":1,"up_from":0,"seq":399431958747,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":88,"seq":377957122273,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2394828,"kb_used_data":1833604,"kb_used_omap":8,"kb_used_meta":561207,"kb_avail":91350324,"statfs":{"total":95995035648,"available":93542731776,"internally_reserved":0,"allocated":1877610496,"data_stored":1876564197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":574676744},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:48:02 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":1.143,"15min":1.694},"min":{"1min":0.23000000000000001,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":1.383,"5min":14.198,"15min":17.760000000000002},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":1.417,"15min":2.1949999999999998},"min":{"1min":0.29899999999999999,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":1.8660000000000001,"5min":13.816000000000001,"15min":26.163},"last":0.28000000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":1.411,"15min":3.4740000000000002},"min":{"1min":0.219,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.0199999999999996,"5min":19.405999999999999,"15min":92.230000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.2569999999999999,"15min":3.2570000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.218},"max":{"1min":4.8879999999999999,"5min":16.407,"15min":92.247},"last":0.36499999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:48:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":2.1659999999999999,"15min":3.9079999999999999},"min":{"1min":0.29399999999999998,"5min":0.218,"15min":0.218},"max":{"1min":5.2119999999999997,"5min":55.328000000000003,"15min":123.024},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.66400000000000003,"5min":2.379,"15min":3.79},"min":{"1min":0.29599999999999999,"5min":0.26800000000000002,"15min":0.222},"max":{"1min":2.403,"5min":55.292000000000002,"15min":99.162000000000006},"last":0.55900000000000005}]},{"osd":4,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.83899999999999997,"15min":1.452},"min":{"1min":0.32500000000000001,"5min":0.29899999999999999,"15min":0.25700000000000001},"max":{"1min":3.8570000000000002,"5min":10.084,"15min":69.082999999999998},"last":2.6579999999999999},{"interface":"front","average":{"1min":1,"5min":1.1559999999999999,"15min":1.9790000000000001},"min":{"1min":0.41999999999999998,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":2.4460000000000002,"5min":19.638999999999999,"15min":69.102000000000004},"last":5.4930000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.345,"15min":3.286},"min":{"1min":0.48599999999999999,"5min":0.34599999999999997,"15min":0.30199999999999999},"max":{"1min":2.8079999999999998,"5min":19.547000000000001,"15min":281.733},"last":1.4570000000000001},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.353,"15min":3.343},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":4.3890000000000002,"5min":19.463999999999999,"15min":282.95400000000001},"last":1.7809999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.4159999999999999,"15min":1.8819999999999999},"min":{"1min":0.46300000000000002,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":2.895,"5min":19.579000000000001,"15min":19.579000000000001},"last":1.6830000000000001},{"interface":"front","average":{"1min":1.234,"5min":1.4950000000000001,"15min":2.0219999999999998},"min":{"1min":0.42299999999999999,"5min":0.34899999999999998,"15min":0.32500000000000001},"max":{"1min":4.4370000000000003,"5min":19.497,"15min":19.497},"last":0.75900000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:48:02 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.258,"15min":1.425},"min":{"1min":0.436,"5min":0.36199999999999999,"15min":0.33900000000000002},"max":{"1min":1.6180000000000001,"5min":25.378,"15min":25.378},"last":3.8220000000000001},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.544,"15min":2.069},"min":{"1min":0.436,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.6000000000000001,"5min":25.436,"15min":25.436},"last":5.556}]}]},{"osd":2,"up_from":98,"seq":420906795220,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2767044,"kb_used_data":2115516,"kb_used_omap":11,"kb_used_meta":651508,"kb_avail":90978108,"statfs":{"total":95995035648,"available":93161582592,"internally_reserved":0,"allocated":2166288384,"data_stored":2165130930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":667144789},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:47:33 2023","interfaces":[{"interface":"back","average":{"1min":2.6859999999999999,"5min":5.25,"15min":3.4689999999999999},"min":{"1min":0.217,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":42.914999999999999,"5min":162.285,"15min":162.285},"last":3.3799999999999999},{"interface":"front","average":{"1min":2.6970000000000001,"5min":6.1050000000000004,"15min":3.766},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19600000000000001},"max":{"1min":42.972999999999999,"5min":162.643,"15min":162.643},"last":2.6909999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:47:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":1.54,"15min":3.145},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":1.637,"5min":16.050999999999998,"15min":98.575000000000003},"last":2.7759999999999998},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.105,"15min":2.5950000000000002},"min":{"1min":0.253,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":2.6859999999999999,"5min":7.7149999999999999,"15min":98.397000000000006},"last":5.4470000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":3.879,"15min":4.9530000000000003},"min":{"1min":0.32100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1859999999999999,"5min":159.35499999999999,"15min":159.35499999999999},"last":5.3719999999999999},{"interface":"front","average":{"1min":0.98199999999999998,"5min":3.9279999999999999,"15min":4.8719999999999999},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.0780000000000003,"5min":159.196,"15min":159.196},"last":0.875}]},{"osd":4,"last update":"Mon Oct 9 05:47:33 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":1.004,"15min":1.9199999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27700000000000002},"max":{"1min":2.5739999999999998,"5min":7.1120000000000001,"15min":115.84999999999999},"last":5.5279999999999996},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.85899999999999999,"15min":2.0630000000000002},"min":{"1min":0.31,"5min":0.309,"15min":0.28699999999999998},"max":{"1min":2.9870000000000001,"5min":7.0359999999999996,"15min":116.149},"last":0.63900000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:47:33 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.403,"15min":1.9910000000000001},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.312},"max":{"1min":2.984,"5min":13.035,"15min":24.597999999999999},"last":3.1240000000000001},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.111,"15min":1.6879999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.26000000000000001},"max":{"1min":2.915,"5min":7.1829999999999998,"15min":23.919},"last":5.4850000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:47:33 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":1.119,"15min":1.8340000000000001},"min":{"1min":0.34999999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.0910000000000002,"5min":14.317,"15min":20.503},"last":1.4710000000000001},{"interface":"front","average":{"1min":0.74299999999999999,"5min":1.655,"15min":1.9159999999999999},"min":{"1min":0.40600000000000003,"5min":0.38900000000000001,"15min":0.34499999999999997},"max":{"1min":2.7850000000000001,"5min":14.436999999999999,"15min":14.436999999999999},"last":7.1230000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.675,"15min":2.2749999999999999},"min":{"1min":0.41299999999999998,"5min":0.38300000000000001,"15min":0.37},"max":{"1min":4.8899999999999997,"5min":51.701999999999998,"15min":51.701999999999998},"last":6.3330000000000002},{"interface":"front","average":{"1min":0.97799999999999998,"5min":2.0409999999999999,"15min":2.2639999999999998},"min":{"1min":0.44900000000000001,"5min":0.373,"15min":0.373},"max":{"1min":3.3519999999999999,"5min":51.716000000000001,"15min":51.716000000000001},"last":8.7430000000000003}]}]},{"osd":3,"up_from":103,"seq":442381631694,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3082572,"kb_used_data":2421636,"kb_used_omap":4,"kb_used_meta":660923,"kb_avail":90662580,"statfs":{"total":95995035648,"available":92838481920,"internally_reserved":0,"allocated":2479755264,"data_stored":2478708452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":676785758},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.105,"5min":4.5289999999999999,"15min":3.8769999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.183},"max":{"1min":45.731000000000002,"5min":149.565,"15min":167.82400000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":2.9710000000000001,"5min":4.8819999999999997,"15min":4.4340000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":45.877000000000002,"5min":149.291,"15min":167.89599999999999},"last":6.3019999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":2.4820000000000002,"15min":2.9470000000000001},"min":{"1min":0.29199999999999998,"5min":0.247,"15min":0.22600000000000001},"max":{"1min":6.3730000000000002,"5min":149.32499999999999,"15min":149.32499999999999},"last":1.875},{"interface":"front","average":{"1min":0.68500000000000005,"5min":2.903,"15min":3.0830000000000002},"min":{"1min":0.24299999999999999,"5min":0.19,"15min":0.19},"max":{"1min":5.5540000000000003,"5min":149.34299999999999,"15min":149.34299999999999},"last":6.6299999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":2.758,"15min":4.6280000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.215},"max":{"1min":6.2110000000000003,"5min":150.006,"15min":190.43299999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.85099999999999998,"5min":2.6840000000000002,"15min":5.0049999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.944,"5min":150.65100000000001,"15min":190.267},"last":0.28899999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":2.528,"15min":2.7029999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":2.794,"5min":149.84700000000001,"15min":149.84700000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.82899999999999996,"5min":2.3900000000000001,"15min":2.1989999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.27400000000000002},"max":{"1min":6.1559999999999997,"5min":149.74700000000001,"15min":149.74700000000001},"last":0.57799999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":2.6419999999999999,"15min":2.3999999999999999},"min":{"1min":0.36099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.024,"5min":149.43000000000001,"15min":149.43000000000001},"last":6.5170000000000003},{"interface":"front","average":{"1min":0.90100000000000002,"5min":2.5880000000000001,"15min":2.3559999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":6.0899999999999999,"5min":149.69300000000001,"15min":149.69300000000001},"last":0.51600000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":2.5510000000000002,"15min":2.2210000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.33500000000000002},"max":{"1min":6.1299999999999999,"5min":149.72399999999999,"15min":149.72399999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.0169999999999999,"5min":2.7869999999999999,"15min":2.2949999999999999},"min":{"1min":0.33200000000000002,"5min":0.307,"15min":0.307},"max":{"1min":6.048,"5min":149.589,"15min":149.589},"last":5.2519999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":2.5169999999999999,"15min":2.4700000000000002},"min":{"1min":0.41499999999999998,"5min":0.35699999999999998,"15min":0.33700000000000002},"max":{"1min":2.8260000000000001,"5min":149.77199999999999,"15min":149.77199999999999},"last":0.435},{"interface":"front","average":{"1min":0.90400000000000003,"5min":2.6920000000000002,"15min":2.5219999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.35699999999999998},"max":{"1min":3.194,"5min":149.80799999999999,"15min":149.80799999999999},"last":0.48699999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019241,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3318688,"kb_used_data":2733080,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":90426464,"statfs":{"total":95995035648,"available":92596699136,"internally_reserved":0,"allocated":2798673920,"data_stored":2798071307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:47:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":2.1709999999999998,"15min":2.2959999999999998},"min":{"1min":0.22600000000000001,"5min":0.219,"15min":0.214},"max":{"1min":11.140000000000001,"5min":73.384,"15min":100.504},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.95599999999999996,"5min":2.3319999999999999,"15min":2.617},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":10.983000000000001,"5min":71.884,"15min":100.717},"last":3.7149999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:48:09 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.2150000000000001,"15min":2.5009999999999999},"min":{"1min":0.29099999999999998,"5min":0.26800000000000002,"15min":0.24299999999999999},"max":{"1min":3.8759999999999999,"5min":10.180999999999999,"15min":187.369},"last":8.6349999999999998},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.1639999999999999,"15min":2.6059999999999999},"min":{"1min":0.42699999999999999,"5min":0.307,"15min":0.26600000000000001},"max":{"1min":3.0089999999999999,"5min":10.189,"15min":187.35300000000001},"last":3.7589999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:47:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":2.0169999999999999,"15min":4.5460000000000003},"min":{"1min":0.29299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.923,"5min":71.748000000000005,"15min":196.93700000000001},"last":8.5920000000000005},{"interface":"front","average":{"1min":0.60799999999999998,"5min":2.1520000000000001,"15min":4.4950000000000001},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.28100000000000003},"max":{"1min":2.8769999999999998,"5min":72.096000000000004,"15min":197.08099999999999},"last":0.52600000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:47:52 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":2.5310000000000001,"15min":4.452},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.77900000000000003,"5min":123.59,"15min":211.48699999999999},"last":3.6459999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":2.5369999999999999,"15min":4.3029999999999999},"min":{"1min":0.372,"5min":0.312,"15min":0.309},"max":{"1min":2.2519999999999998,"5min":123.566,"15min":211.31100000000001},"last":0.503}]},{"osd":5,"last update":"Mon Oct 9 05:47:52 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.71999999999999997,"15min":1.643},"min":{"1min":0.314,"5min":0.27800000000000002,"15min":0.24099999999999999},"max":{"1min":0.73399999999999999,"5min":3.6739999999999999,"15min":113.35899999999999},"last":3.6880000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.64600000000000002,"15min":1.54},"min":{"1min":0.35099999999999998,"5min":0.32800000000000001,"15min":0.30599999999999999},"max":{"1min":2.0169999999999999,"5min":3.4209999999999998,"15min":113.32299999999999},"last":0.84399999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:48:09 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.63500000000000001,"15min":0.73999999999999999},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.28100000000000003},"max":{"1min":2.081,"5min":3.3969999999999998,"15min":11.944000000000001},"last":0.871},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.90700000000000003,"15min":0.98899999999999999},"min":{"1min":0.34999999999999998,"5min":0.32800000000000001,"15min":0.28399999999999997},"max":{"1min":2.2789999999999999,"5min":8.5269999999999992,"15min":10.356},"last":4.3600000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:48:09 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":1.615,"15min":1.603},"min":{"1min":0.36799999999999999,"5min":0.34300000000000003,"15min":0.312},"max":{"1min":2.9980000000000002,"5min":96.131,"15min":96.131},"last":3.7330000000000001},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.7769999999999999,"15min":1.7609999999999999},"min":{"1min":0.443,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":3.9329999999999998,"5min":96.465000000000003,"15min":96.465000000000003},"last":3.972}]}]},{"osd":5,"up_from":36,"seq":154618823013,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2369292,"kb_used_data":1775492,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":91375860,"statfs":{"total":95995035648,"available":93568880640,"internally_reserved":0,"allocated":1818103808,"data_stored":1817517053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:47:37 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":2.851,"15min":3.7989999999999999},"min":{"1min":0.23799999999999999,"5min":0.23300000000000001,"15min":0.19400000000000001},"max":{"1min":3.1930000000000001,"5min":96.858000000000004,"15min":165.90199999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.72699999999999998,"5min":3.0880000000000001,"15min":3.9249999999999998},"min":{"1min":0.27000000000000002,"5min":0.254,"15min":0.215},"max":{"1min":3.2149999999999999,"5min":97.555999999999997,"15min":165.96700000000001},"last":0.56899999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:48:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.6679999999999999,"15min":2.6299999999999999},"min":{"1min":0.30099999999999999,"5min":0.26500000000000001,"15min":0.23999999999999999},"max":{"1min":3.266,"5min":54.898000000000003,"15min":87.769000000000005},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.5109999999999999,"15min":2.6949999999999998},"min":{"1min":0.35399999999999998,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":3.3380000000000001,"5min":52.512,"15min":87.864000000000004},"last":0.59199999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:47:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.6200000000000001,"15min":4.4420000000000002},"min":{"1min":0.29799999999999999,"5min":0.28100000000000003,"15min":0.26700000000000002},"max":{"1min":3.319,"5min":31.114000000000001,"15min":290.697},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.621,"5min":1.5600000000000001,"15min":4.3300000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":1.6180000000000001,"5min":33.875999999999998,"15min":290.786},"last":0.55500000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:48:00 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":2.2810000000000001,"15min":3.6880000000000002},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":4.1769999999999996,"5min":53.243000000000002,"15min":95.236999999999995},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.018,"5min":2.2599999999999998,"15min":3.4980000000000002},"min":{"1min":0.38500000000000001,"5min":0.34599999999999997,"15min":0.29699999999999999},"max":{"1min":4.1260000000000003,"5min":53.259999999999998,"15min":93.471000000000004},"last":0.49199999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:48:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.69699999999999995,"15min":0.78500000000000003},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.23300000000000001},"max":{"1min":3.3090000000000002,"5min":6.2610000000000001,"15min":9.3510000000000009},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.79300000000000004,"15min":1.0249999999999999},"min":{"1min":0.34300000000000003,"5min":0.27000000000000002,"15min":0.23400000000000001},"max":{"1min":3.3769999999999998,"5min":9.282,"15min":14.236000000000001},"last":0.35299999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.83199999999999996,"15min":1.458},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.26600000000000001},"max":{"1min":3.8580000000000001,"5min":9.2959999999999994,"15min":95.010999999999996},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.81100000000000005,"15min":1.353},"min":{"1min":0.34399999999999997,"5min":0.32100000000000001,"15min":0.27300000000000002},"max":{"1min":3.0499999999999998,"5min":4.5410000000000004,"15min":94.194000000000003},"last":0.41299999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.997,"15min":1.1599999999999999},"min":{"1min":0.33700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.6710000000000003,"5min":9.1460000000000008,"15min":9.9849999999999994},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.96299999999999997,"15min":1.212},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.308},"max":{"1min":3.9740000000000002,"5min":9.3100000000000005,"15min":14.432},"last":0.439}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1890254848,"data_stored":1890254848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2161901568,"data_stored":2161901568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2480734208,"data_stored":2480734208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2706636800,"data_stored":2706636800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1788084224,"data_stored":1788084224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2132148224,"data_stored":2132148224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1503920128,"data_stored":1503920128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:48:38.021 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:38.026 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:48:38.026 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 26 2023-10-09T05:48:38.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:38.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:38.128 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:38.135 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 55 16 80902 80886 91.8772 60.4375 0.0113792 0.0108793 2023-10-09T05:48:38.199 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:38.819 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:38.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:38.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:38.913 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:48:38.927 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:38.948 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:39.007 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:39.135 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 56 16 81742 81726 91.1742 52.5 0.0114501 0.0109626 2023-10-09T05:48:39.476 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:39.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:39.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:39.580 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:39.659 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:39.783 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:48:39.814 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:39.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:39.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:39.848 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:48:39.923 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:39.974 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:39.983 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:48:40.024 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:40.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:40.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:40.127 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:39 smithi008 ceph-mon[184418]: pgmap v902: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T05:48:40.128 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3356198608' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:48:40.128 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:39 smithi008 ceph-mon[191227]: pgmap v902: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T05:48:40.128 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3356198608' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:48:40.128 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:40.135 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 57 16 82500 82484 90.4059 47.375 0.00582792 0.011021 2023-10-09T05:48:40.155 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:40.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:39 smithi191 ceph-mon[161669]: pgmap v902: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T05:48:40.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3356198608' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:48:40.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:48:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:48:40] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-10-09T05:48:40.970 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:40.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:40.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:41.079 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:41.138 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 58 16 83383 83367 89.7942 55.1875 0.0169487 0.011132 2023-10-09T05:48:41.151 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:41.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:40 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:41.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:40 smithi191 ceph-mon[161669]: failed to encode map e297 with expected crc 2023-10-09T05:48:41.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:40 smithi191 ceph-mon[161669]: failed to encode map e297 with expected crc 2023-10-09T05:48:41.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:40 smithi191 ceph-mon[161669]: failed to encode map e297 with expected crc 2023-10-09T05:48:41.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:40 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3356198608' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:48:41.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:40 smithi191 ceph-mon[161669]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T05:48:41.393 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:41.393 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[184418]: failed to encode map e297 with expected crc 2023-10-09T05:48:41.393 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[184418]: failed to encode map e297 with expected crc 2023-10-09T05:48:41.393 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[184418]: failed to encode map e297 with expected crc 2023-10-09T05:48:41.393 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3356198608' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:48:41.393 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[184418]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T05:48:41.394 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:41.394 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[191227]: failed to encode map e297 with expected crc 2023-10-09T05:48:41.394 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[191227]: failed to encode map e297 with expected crc 2023-10-09T05:48:41.394 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[191227]: failed to encode map e297 with expected crc 2023-10-09T05:48:41.394 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3356198608' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:48:41.394 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:40 smithi008 ceph-mon[191227]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T05:48:41.487 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:41.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:41.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:41.593 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:41.677 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:41.810 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:42.139 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 59 16 84241 84225 89.1798 53.625 0.0168896 0.0112075 2023-10-09T05:48:42.153 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:42.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:42.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:42.266 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:42.292 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:41 smithi008 ceph-mon[184418]: pgmap v904: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-09T05:48:42.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:42 smithi008 ceph-mon[191227]: pgmap v904: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-09T05:48:42.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:41 smithi191 ceph-mon[161669]: pgmap v904: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-09T05:48:42.354 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:42.420 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:48:42.420 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:48:42.876 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:42.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:42.999 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:43.010 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 8 pg_num to 26 2023-10-09T05:48:43.034 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:43.139 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:48:43.140277+0000 min lat: 0.00124024 max lat: 1.00922 avg lat: 0.0112952 2023-10-09T05:48:43.139 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:48:43.140 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 60 16 85008 84992 88.4926 47.9375 0.0132333 0.0112952 2023-10-09T05:48:43.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4227455388' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:48:43.398 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4227455388' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:48:43.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4227455388' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:48:43.400 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:43.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:43.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:43.431 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:48:43.451 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:48:43.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:43.539 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:43.653 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:43.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:43.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:43.789 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:43.834 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:44.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:44 smithi191 ceph-mon[161669]: pgmap v905: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s 2023-10-09T05:48:44.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:44 smithi191 ceph-mon[161669]: failed to encode map e298 with expected crc 2023-10-09T05:48:44.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4227455388' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:48:44.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:44 smithi191 ceph-mon[161669]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T05:48:44.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:44 smithi191 ceph-mon[161669]: failed to encode map e298 with expected crc 2023-10-09T05:48:44.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:44 smithi191 ceph-mon[161669]: failed to encode map e298 with expected crc 2023-10-09T05:48:44.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:44.124 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:44.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[184418]: pgmap v905: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s 2023-10-09T05:48:44.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[184418]: failed to encode map e298 with expected crc 2023-10-09T05:48:44.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4227455388' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:48:44.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[184418]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T05:48:44.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[184418]: failed to encode map e298 with expected crc 2023-10-09T05:48:44.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[184418]: failed to encode map e298 with expected crc 2023-10-09T05:48:44.131 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:44.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[191227]: pgmap v905: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s 2023-10-09T05:48:44.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[191227]: failed to encode map e298 with expected crc 2023-10-09T05:48:44.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4227455388' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:48:44.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[191227]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T05:48:44.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[191227]: failed to encode map e298 with expected crc 2023-10-09T05:48:44.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[191227]: failed to encode map e298 with expected crc 2023-10-09T05:48:44.132 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:44.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:44.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:44.141 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 61 16 85878 85862 87.9334 54.375 0.0130167 0.0113671 2023-10-09T05:48:44.240 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:44.321 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:44.336 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:44.502 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:44.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:44.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:44.607 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:44.668 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:48:44.677 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:44.773 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:48:44.805 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:48:44.816 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:44.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:44.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:44.839 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:48:44.879 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:48:44.908 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:48:44.921 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:44.952 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:48:44.978 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:48:44.980 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:45.024 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:48:45.036 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:48:45.036 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:48:45.036 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:48:45.036 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:48:45.036 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:48:45.036 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:48:45.036 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:48:45.036 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:48:45.109 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:45 smithi008 ceph-mon[191227]: failed to encode map e298 with expected crc 2023-10-09T05:48:45.109 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:48:45.109 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/27475639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:45.111 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:45 smithi008 ceph-mon[184418]: failed to encode map e298 with expected crc 2023-10-09T05:48:45.111 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:48:45.111 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/27475639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:45.149 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 62 16 86720 86704 87.364 52.625 0.0634795 0.0114276 2023-10-09T05:48:45.306 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:45 smithi191 ceph-mon[161669]: failed to encode map e298 with expected crc 2023-10-09T05:48:45.306 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:48:45.306 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/27475639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:46.140 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 63 16 86722 86706 85.9798 0.125 0.0164385 0.0114306 2023-10-09T05:48:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:46 smithi191 ceph-mon[161669]: pgmap v907: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 754 op/s 2023-10-09T05:48:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:46 smithi191 ceph-mon[161669]: failed to encode map e299 with expected crc 2023-10-09T05:48:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:46 smithi191 ceph-mon[161669]: failed to encode map e299 with expected crc 2023-10-09T05:48:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:48:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:46 smithi191 ceph-mon[161669]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T05:48:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:46 smithi191 ceph-mon[161669]: failed to encode map e299 with expected crc 2023-10-09T05:48:46.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:46.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:46.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[184418]: pgmap v907: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 754 op/s 2023-10-09T05:48:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[184418]: failed to encode map e299 with expected crc 2023-10-09T05:48:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[184418]: failed to encode map e299 with expected crc 2023-10-09T05:48:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:48:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[184418]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T05:48:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[184418]: failed to encode map e299 with expected crc 2023-10-09T05:48:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[191227]: pgmap v907: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 754 op/s 2023-10-09T05:48:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[191227]: failed to encode map e299 with expected crc 2023-10-09T05:48:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[191227]: failed to encode map e299 with expected crc 2023-10-09T05:48:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:48:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[191227]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T05:48:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[191227]: failed to encode map e299 with expected crc 2023-10-09T05:48:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:46.540 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:46.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:46.647 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:46.756 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:46.839 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:46.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:46.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:47.014 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:47.094 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:48:47.094 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:48:47.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:48:47.094 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:48:47.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.094 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125677 7.1 7:8932b233:::benchmark_data_smithi008_354649_object31181:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:48:47.094 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.953115+0000", 2023-10-09T05:48:47.095 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 284.13542644900002, 2023-10-09T05:48:47.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27913971500000001, 2023-10-09T05:48:47.095 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.095 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:48:47.095 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.095 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:48:47.095 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:47.095 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125677 2023-10-09T05:48:47.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.095 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953115+0000", 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953115+0000", 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953113+0000", 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6590000000000002e-06 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953123+0000", 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4805e-05 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953138+0000", 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0509999999999999e-06 2023-10-09T05:48:47.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953145+0000", 2023-10-09T05:48:47.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7503000000000002e-05 2023-10-09T05:48:47.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953202+0000", 2023-10-09T05:48:47.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014179000000000001 2023-10-09T05:48:47.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953344+0000", 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015102799999999999 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.953495+0000", 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0043468650000000001 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.957842+0000", 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3988999999999997e-05 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.957936+0000", 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27430302200000001 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232239+0000", 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5869e-05 2023-10-09T05:48:47.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232255+0000", 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27913971500000001 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125688 7.1 7.eda1af1 (undecoded) ondisk+write+known_if_redirected e238)", 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.992303+0000", 2023-10-09T05:48:47.101 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 284.09623911599999, 2023-10-09T05:48:47.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24012315200000001, 2023-10-09T05:48:47.102 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.102 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:48:47.102 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.102 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:48:47.102 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:47.102 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125688 2023-10-09T05:48:47.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.102 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992303+0000", 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992303+0000", 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999971 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992300+0000", 2023-10-09T05:48:47.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4301000000000001e-05 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992314+0000", 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2853e-05 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992337+0000", 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1476e-05 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.992348+0000", 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.239966718 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232315+0000", 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000110808 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232426+0000", 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24012315200000001 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125677 7.11 7:8932b233:::benchmark_data_smithi008_354649_object31181:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 284.091089125, 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0454934010000001, 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.106 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125677 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999967 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997449+0000", 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3732e-05 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997463+0000", 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9613999999999999e-05 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997483+0000", 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0620000000000007e-06 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997492+0000", 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.234801074 2023-10-09T05:48:47.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232293+0000", 2023-10-09T05:48:47.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2490000000000001e-06 2023-10-09T05:48:47.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:48:47.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232301+0000", 2023-10-09T05:48:47.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81054634999999997 2023-10-09T05:48:47.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042848+0000", 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8449999999999994e-05 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042946+0000", 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0454934010000001 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125688 7.11 7:8f585b70:::benchmark_data_smithi008_354649_object31192:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 284.09104155599999, 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0556018709999999, 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125688 2023-10-09T05:48:47.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:48:47.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2820000000000001e-06 2023-10-09T05:48:47.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997503+0000", 2023-10-09T05:48:47.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0820000000000004e-06 2023-10-09T05:48:47.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997509+0000", 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0659999999999997e-06 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997513+0000", 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.234809199 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232322+0000", 2023-10-09T05:48:47.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6849999999999998e-06 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232328+0000", 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81122250699999998 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043550+0000", 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114354 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043665+0000", 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00033335100000000001 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043998+0000", 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001666088 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.045664+0000", 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041383430000000001 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.049803+0000", 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032002139999999998 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053003+0000", 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7229999999999999e-05 2023-10-09T05:48:47.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053060+0000", 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1964e-05 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053102+0000", 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0556018709999999 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.119 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180297 7.21 7:846cd9a9:::benchmark_data_smithi008_354649_object85801:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:41.587040+0000", 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 125.501502055, 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48169166200000002, 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:47.120 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180297 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587040+0000", 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587040+0000", 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:47.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587038+0000", 2023-10-09T05:48:47.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6429999999999995e-06 2023-10-09T05:48:47.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587046+0000", 2023-10-09T05:48:47.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4168999999999999e-05 2023-10-09T05:48:47.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587060+0000", 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9569999999999999e-06 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587066+0000", 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7617000000000003e-05 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587104+0000", 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1820000000000003e-06 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587110+0000", 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47181123800000002 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.058921+0000", 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117278 2023-10-09T05:48:47.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.059038+0000", 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000192509 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.059231+0000", 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001372269 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060603+0000", 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2033000000000003e-05 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060635+0000", 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079930550000000006 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.068628+0000", 2023-10-09T05:48:47.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9888e-05 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.068688+0000", 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3340999999999997e-05 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.068731+0000", 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48169166200000002 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.127 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.128 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.128 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180300 7.1 7:807d12ef:::benchmark_data_smithi008_354649_object85804:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:48:47.128 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:41.743813+0000", 2023-10-09T05:48:47.128 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 125.344728284, 2023-10-09T05:48:47.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.316588969, 2023-10-09T05:48:47.128 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.128 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:47.128 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.128 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180300 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743813+0000", 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743813+0000", 2023-10-09T05:48:47.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:48:47.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743812+0000", 2023-10-09T05:48:47.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2349999999999999e-06 2023-10-09T05:48:47.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743820+0000", 2023-10-09T05:48:47.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5903999999999999e-05 2023-10-09T05:48:47.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743836+0000", 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0990000000000004e-06 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743842+0000", 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5695000000000001e-05 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743897+0000", 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4329999999999993e-06 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743907+0000", 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29237399200000003 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.036281+0000", 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9270000000000004e-05 2023-10-09T05:48:47.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.036370+0000", 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028330299999999999 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.036653+0000", 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020744069999999999 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.038728+0000", 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024384929999999999 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.041166+0000", 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019156355999999999 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060323+0000", 2023-10-09T05:48:47.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2685999999999997e-05 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060375+0000", 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6947000000000001e-05 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060402+0000", 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.316588969 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:44625 8.2 8:4808ba14:::benchmark_data_smithi008_432048_object44624:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:04.386356+0000", 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 42.702185221999997, 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24626183099999999, 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 44625 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.386356+0000", 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.386356+0000", 2023-10-09T05:48:47.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.386355+0000", 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9884999999999997e-05 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.386445+0000", 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1384e-05 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.386456+0000", 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5980000000000003e-06 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.386466+0000", 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00099656399999999991 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.387462+0000", 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012104600000000001 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.387583+0000", 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2998999999999995e-05 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.387676+0000", 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001735215 2023-10-09T05:48:47.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.389412+0000", 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00022731499999999999 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.389639+0000", 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24290198900000001 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.632541+0000", 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1311999999999999e-05 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.632592+0000", 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6088000000000002e-05 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:04.632618+0000", 2023-10-09T05:48:47.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24626183099999999 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:76527 8.2 8:42bbc742:::benchmark_data_smithi008_432048_object76526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:33.307549+0000", 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 13.780992204, 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23370227800000001, 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:47.143 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 76527 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.307549+0000", 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.307549+0000", 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.307549+0000", 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2515000000000002e-05 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.307591+0000", 2023-10-09T05:48:47.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4599999999999992e-06 2023-10-09T05:48:47.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.307601+0000", 2023-10-09T05:48:47.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5979999999999999e-06 2023-10-09T05:48:47.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.307605+0000", 2023-10-09T05:48:47.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8626999999999999e-05 2023-10-09T05:48:47.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.307634+0000", 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6204000000000003e-05 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.307720+0000", 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102172 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.307822+0000", 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079912219999999992 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.315814+0000", 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0999999999999998e-06 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.315821+0000", 2023-10-09T05:48:47.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.225331421 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541152+0000", 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4440000000000001e-05 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541207+0000", 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5158999999999999e-05 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541252+0000", 2023-10-09T05:48:47.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23370227800000001 2023-10-09T05:48:47.150 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.150 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.150 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.150 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:76528 8.2 8:447a487f:::benchmark_data_smithi008_432048_object76527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-09T05:48:47.150 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:33.308374+0000", 2023-10-09T05:48:47.150 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 13.780167461, 2023-10-09T05:48:47.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.232985633, 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 76528 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308374+0000", 2023-10-09T05:48:47.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308374+0000", 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308373+0000", 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3009000000000006e-05 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308436+0000", 2023-10-09T05:48:47.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1027e-05 2023-10-09T05:48:47.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308447+0000", 2023-10-09T05:48:47.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0290000000000002e-06 2023-10-09T05:48:47.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308453+0000", 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020373740000000002 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.310491+0000", 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000137143 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.310628+0000", 2023-10-09T05:48:47.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000162645 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.310790+0000", 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0091487749999999996 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.319939+0000", 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2655e-05 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.319952+0000", 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22133517699999999 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541287+0000", 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1468999999999999e-05 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:48:47.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541328+0000", 2023-10-09T05:48:47.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1449999999999999e-05 2023-10-09T05:48:47.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541360+0000", 2023-10-09T05:48:47.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.232985633 2023-10-09T05:48:47.157 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.157 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.157 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.158 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:76529 8.2 8:4e38f992:::benchmark_data_smithi008_432048_object76528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-09T05:48:47.158 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:33.308551+0000", 2023-10-09T05:48:47.158 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 13.779990504000001, 2023-10-09T05:48:47.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23291133999999999, 2023-10-09T05:48:47.158 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.158 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:47.158 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.158 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.158 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.158 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 76529 2023-10-09T05:48:47.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308551+0000", 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308551+0000", 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308550+0000", 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7762000000000006e-05 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308618+0000", 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8709999999999997e-06 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308626+0000", 2023-10-09T05:48:47.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9690000000000002e-06 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308631+0000", 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047434799999999996 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.313375+0000", 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000145933 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.313520+0000", 2023-10-09T05:48:47.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000181538 2023-10-09T05:48:47.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:48:47.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.313702+0000", 2023-10-09T05:48:47.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063051519999999996 2023-10-09T05:48:47.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.320007+0000", 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7524000000000001e-05 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.320055+0000", 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22133666599999999 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541391+0000", 2023-10-09T05:48:47.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9186000000000001e-05 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541431+0000", 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2004000000000001e-05 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541463+0000", 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23291133999999999 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.164 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:76531 8.2 8:46bfad9d:::benchmark_data_smithi008_432048_object76530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:33.308884+0000", 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 13.779657239000001, 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23268060099999999, 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.165 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 76531 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308884+0000", 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308884+0000", 2023-10-09T05:48:47.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.308884+0000", 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013892 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.309022+0000", 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8199999999999992e-06 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.309032+0000", 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8880000000000002e-06 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.309038+0000", 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006797857 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.315836+0000", 2023-10-09T05:48:47.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8207999999999994e-05 2023-10-09T05:48:47.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.315914+0000", 2023-10-09T05:48:47.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3530999999999996e-05 2023-10-09T05:48:47.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:48:47.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.316008+0000", 2023-10-09T05:48:47.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004039478 2023-10-09T05:48:47.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.320047+0000", 2023-10-09T05:48:47.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5506999999999999e-05 2023-10-09T05:48:47.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.320063+0000", 2023-10-09T05:48:47.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22143027900000001 2023-10-09T05:48:47.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541493+0000", 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9798000000000001e-05 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541533+0000", 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2268999999999999e-05 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541565+0000", 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23268060099999999 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:76534 8.2 8:4b815165:::benchmark_data_smithi008_432048_object76533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:33.313890+0000", 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 13.774651287999999, 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.227781448, 2023-10-09T05:48:47.172 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 76534 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.313890+0000", 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.313890+0000", 2023-10-09T05:48:47.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:47.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.313889+0000", 2023-10-09T05:48:47.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1606e-05 2023-10-09T05:48:47.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.313941+0000", 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6157000000000001e-05 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.313957+0000", 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1099999999999997e-06 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.313964+0000", 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003889539 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.317853+0000", 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011566900000000001 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.317969+0000", 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015900699999999999 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.318128+0000", 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019489590000000001 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.320077+0000", 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000225293 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.320302+0000", 2023-10-09T05:48:47.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.2212973 2023-10-09T05:48:47.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541600+0000", 2023-10-09T05:48:47.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9542000000000002e-05 2023-10-09T05:48:47.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:48:47.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541639+0000", 2023-10-09T05:48:47.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2746000000000001e-05 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:33.541672+0000", 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.227781448 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86697 8.5 8:a67798ab:::benchmark_data_smithi008_432048_object86696:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.034059+0000", 2023-10-09T05:48:47.179 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 2.0544828220000002, 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24198027899999999, 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86697 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.034059+0000", 2023-10-09T05:48:47.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.034059+0000", 2023-10-09T05:48:47.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:47.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.034058+0000", 2023-10-09T05:48:47.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5463000000000002e-05 2023-10-09T05:48:47.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.034094+0000", 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4510000000000003e-06 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.034101+0000", 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6399999999999996e-06 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.034106+0000", 2023-10-09T05:48:47.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6807999999999999e-05 2023-10-09T05:48:47.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.034203+0000", 2023-10-09T05:48:47.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000139596 2023-10-09T05:48:47.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.034342+0000", 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000222444 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.034565+0000", 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027129400000000001 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.037278+0000", 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019242987 2023-10-09T05:48:47.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.056521+0000", 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21950552200000001 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276026+0000", 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2999999999999999e-05 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276039+0000", 2023-10-09T05:48:47.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24198027899999999 2023-10-09T05:48:47.186 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.186 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.186 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.186 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86704 8.5 8:af611326:::benchmark_data_smithi008_432048_object86703:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-09T05:48:47.186 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.038824+0000", 2023-10-09T05:48:47.186 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 2.0497181539999998, 2023-10-09T05:48:47.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23725903300000001, 2023-10-09T05:48:47.186 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86704 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.038824+0000", 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.038824+0000", 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.038822+0000", 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8389000000000001e-05 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.038861+0000", 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6390000000000004e-06 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.038870+0000", 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0660000000000003e-06 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.038875+0000", 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7701000000000003e-05 2023-10-09T05:48:47.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.038923+0000", 2023-10-09T05:48:47.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120417 2023-10-09T05:48:47.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.039043+0000", 2023-10-09T05:48:47.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015650199999999999 2023-10-09T05:48:47.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.039200+0000", 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059938650000000001 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.045194+0000", 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011416727999999999 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.056610+0000", 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.219465191 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276076+0000", 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9519999999999997e-06 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276083+0000", 2023-10-09T05:48:47.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23725903300000001 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86708 8.5 8:ad569a26:::benchmark_data_smithi008_432048_object86707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.047914+0000", 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 2.040628061, 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22820432600000001, 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:48:47.193 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.194 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.194 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.194 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86708 2023-10-09T05:48:47.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.194 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.047914+0000", 2023-10-09T05:48:47.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.047914+0000", 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.047912+0000", 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4719999999999999e-05 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.047956+0000", 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5407999999999999e-05 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.047972+0000", 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5660000000000001e-06 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.047979+0000", 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118854 2023-10-09T05:48:47.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.048098+0000", 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016355699999999999 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.048262+0000", 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019245199999999999 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:48:47.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.048454+0000", 2023-10-09T05:48:47.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075154219999999999 2023-10-09T05:48:47.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.055970+0000", 2023-10-09T05:48:47.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00067676599999999998 2023-10-09T05:48:47.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.056646+0000", 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21946423800000001 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276111+0000", 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.374e-06 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276118+0000", 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22820432600000001 2023-10-09T05:48:47.199 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86697 8.5 8:a67798ab:::benchmark_data_smithi008_432048_object86696:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088415+0000", 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 2.0001268529999998, 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.168776316, 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86697 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088415+0000", 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088415+0000", 2023-10-09T05:48:47.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:48:47.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088413+0000", 2023-10-09T05:48:47.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6736999999999995e-05 2023-10-09T05:48:47.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088479+0000", 2023-10-09T05:48:47.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0219000000000001e-05 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088500+0000", 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1593000000000001e-05 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088511+0000", 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017065599999999999 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088682+0000", 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1679999999999993e-06 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088691+0000", 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18748368300000001 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276175+0000", 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1679999999999993e-06 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276184+0000", 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98094864800000003 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.257133+0000", 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8631000000000002e-05 2023-10-09T05:48:47.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.257191+0000", 2023-10-09T05:48:47.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.168776316 2023-10-09T05:48:47.206 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.206 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.206 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.206 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86704 8.15 8:af611326:::benchmark_data_smithi008_432048_object86703:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:48:47.206 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088533+0000", 2023-10-09T05:48:47.207 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 2.0000089409999999, 2023-10-09T05:48:47.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1725390840000001, 2023-10-09T05:48:47.207 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.207 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:48:47.207 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.207 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.207 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.207 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86704 2023-10-09T05:48:47.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.207 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088533+0000", 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088533+0000", 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088532+0000", 2023-10-09T05:48:47.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3013000000000002e-05 2023-10-09T05:48:47.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088585+0000", 2023-10-09T05:48:47.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5350000000000004e-06 2023-10-09T05:48:47.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088594+0000", 2023-10-09T05:48:47.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3850000000000001e-06 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088599+0000", 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19036776799999999 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.278967+0000", 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1251e-05 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.278978+0000", 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98199783900000004 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.260976+0000", 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5806999999999999e-05 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261072+0000", 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1725390840000001 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86708 8.15 8:ad569a26:::benchmark_data_smithi008_432048_object86707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.999936328, 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1769074829999999, 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.212 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:48:47.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.213 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86708 2023-10-09T05:48:47.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.213 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:48:47.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6990000000000002e-05 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088652+0000", 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5610000000000004e-06 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088658+0000", 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7790000000000003e-06 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088663+0000", 2023-10-09T05:48:47.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19033387900000001 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.278997+0000", 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0989999999999998e-06 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.279002+0000", 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98210046799999995 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261102+0000", 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094320799999999996 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.262046+0000", 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003411837 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265458+0000", 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5321000000000001e-05 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265513+0000", 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1769074829999999 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86714 8.15 8:a990e10e:::benchmark_data_smithi008_432048_object86713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088669+0000", 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.999872756, 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.177276395, 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86714 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088669+0000", 2023-10-09T05:48:47.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088669+0000", 2023-10-09T05:48:47.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:48:47.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088668+0000", 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1603999999999997e-05 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088710+0000", 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8530000000000003e-06 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088717+0000", 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7299999999999996e-06 2023-10-09T05:48:47.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088722+0000", 2023-10-09T05:48:47.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.190296668 2023-10-09T05:48:47.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.279018+0000", 2023-10-09T05:48:47.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.271e-06 2023-10-09T05:48:47.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.279024+0000", 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98307463399999995 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.262098+0000", 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00063494799999999998 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:48:47.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.262733+0000", 2023-10-09T05:48:47.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031703159999999998 2023-10-09T05:48:47.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265903+0000", 2023-10-09T05:48:47.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1888999999999998e-05 2023-10-09T05:48:47.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265945+0000", 2023-10-09T05:48:47.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.177276395 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86722 8.12 8:4898e42c:::benchmark_data_smithi008_432048_object86721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.349501+0000", 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.7390409529999999, 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91716482799999999, 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:48:47.225 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86722 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349501+0000", 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:47.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349501+0000", 2023-10-09T05:48:47.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999971 2023-10-09T05:48:47.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:47.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349498+0000", 2023-10-09T05:48:47.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0645999999999997e-05 2023-10-09T05:48:47.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:47.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349539+0000", 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.594e-05 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349554+0000", 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2979999999999997e-06 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349564+0000", 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001176355 2023-10-09T05:48:47.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.350740+0000", 2023-10-09T05:48:47.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0907000000000001e-05 2023-10-09T05:48:47.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:48:47.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.350751+0000", 2023-10-09T05:48:47.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91074052900000002 2023-10-09T05:48:47.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:47.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261492+0000", 2023-10-09T05:48:47.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000151316 2023-10-09T05:48:47.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:47.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261643+0000", 2023-10-09T05:48:47.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021199000000000001 2023-10-09T05:48:47.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261855+0000", 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046603080000000002 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266515+0000", 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.63e-06 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266522+0000", 2023-10-09T05:48:47.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0194e-05 2023-10-09T05:48:47.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:47.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266582+0000", 2023-10-09T05:48:47.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8968999999999998e-05 2023-10-09T05:48:47.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:48:47.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266631+0000", 2023-10-09T05:48:47.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4625000000000001e-05 2023-10-09T05:48:47.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:47.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:47.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:48:47.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266666+0000", 2023-10-09T05:48:47.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91716482799999999 2023-10-09T05:48:47.235 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.235 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.235 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.235 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:47.235 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:47.235 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:48:47.236 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 64 16 87809 87793 85.6979 67.9375 0.00975404 0.0116644 2023-10-09T05:48:47.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:47.237 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:47.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:47 smithi191 ceph-mon[161669]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:47 smithi191 ceph-mon[161669]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:47 smithi191 ceph-mon[161669]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:47 smithi191 ceph-mon[161669]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T05:48:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:47 smithi191 ceph-mon[161669]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:47 smithi191 ceph-mon[161669]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:47 smithi191 ceph-mon[161669]: pgmap v910: 99 pgs: 7 peering, 10 unknown, 82 active+clean; 4.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail 2023-10-09T05:48:47.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:47.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:47.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:47.395 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:47.478 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:47.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:47.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:47.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:47.608 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:47.649 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:47.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:47.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:47.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[184418]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[184418]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[184418]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[184418]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T05:48:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[184418]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[184418]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[184418]: pgmap v910: 99 pgs: 7 peering, 10 unknown, 82 active+clean; 4.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail 2023-10-09T05:48:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:47.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[191227]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[191227]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[191227]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[191227]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T05:48:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[191227]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[191227]: failed to encode map e300 with expected crc 2023-10-09T05:48:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[191227]: pgmap v910: 99 pgs: 7 peering, 10 unknown, 82 active+clean; 4.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail 2023-10-09T05:48:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:47.752 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:47.795 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:47.897 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:47.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:47.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:48.002 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:48.064 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:48.140 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 65 15 88715 88700 85.2517 56.6875 0.0173811 0.0117254 2023-10-09T05:48:48.153 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:48.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:48.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:48.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:48:48.256 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:48:48.257 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:48.345 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:48.439 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:48 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/934601477' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:48.440 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:48 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:48.440 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:48.440 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:48.440 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:48.440 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:48 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/934601477' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:48.440 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:48 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:48.440 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:48.440 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:48.441 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:48.462 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:48.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:48.473 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:48 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/934601477' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:48.473 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:48 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:48:48.473 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:48.473 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:48.473 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:48:48.573 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:48.619 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:48.666 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:48:48.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:48.846 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:48.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:48.852 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:48:48.947 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:48.973 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:49.114 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:49.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:49.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:48:49.140 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 66 16 89788 89772 84.9751 67 0.010617 0.0117634 2023-10-09T05:48:49.157 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:48:49.157 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:48:49.157 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T05:48:49.157 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T05:48:49.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:49.241 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:49 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:49.241 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:49 smithi191 ceph-mon[161669]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.241 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:49 smithi191 ceph-mon[161669]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.242 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:49 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:48:49.242 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:49 smithi191 ceph-mon[161669]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T05:48:49.242 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:49 smithi191 ceph-mon[161669]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.242 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:49 smithi191 ceph-mon[161669]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.242 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:49 smithi191 ceph-mon[161669]: pgmap v912: 99 pgs: 7 peering, 10 unknown, 82 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail 2023-10-09T05:48:49.289 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:49.495 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:49.495 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[184418]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.495 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[184418]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.495 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:48:49.496 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[184418]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T05:48:49.496 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[184418]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.496 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[184418]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.496 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[184418]: pgmap v912: 99 pgs: 7 peering, 10 unknown, 82 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail 2023-10-09T05:48:49.496 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:49.496 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[191227]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.496 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[191227]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.497 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:48:49.497 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[191227]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T05:48:49.497 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[191227]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.497 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[191227]: failed to encode map e301 with expected crc 2023-10-09T05:48:49.497 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:49 smithi008 ceph-mon[191227]: pgmap v912: 99 pgs: 7 peering, 10 unknown, 82 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail 2023-10-09T05:48:49.502 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:49.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:49.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:49.606 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:49.659 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:49.671 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:49.742 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:49.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:49.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:49.850 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:49.865 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:49.881 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:49.984 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:49.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:49.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:50.090 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:50.137 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:50.140 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 67 16 90661 90645 84.5213 54.5625 0.0150607 0.0118263 2023-10-09T05:48:50.422 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:50 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T05:48:50.422 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:50 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T05:48:50.425 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:50 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T05:48:50.477 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:50.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:50.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:50.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:48:50.638 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:50.685 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:48:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:48:50] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.33.4" 2023-10-09T05:48:50.781 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:48:50.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:50.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:48:50.885 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:48:50.929 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:50.957 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:48:50.957 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:48:50.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.957 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:91412 8.13 8:c81c9ed3:::benchmark_data_smithi008_432048_object91411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e302)", 2023-10-09T05:48:50.957 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:50.951011+0000", 2023-10-09T05:48:50.957 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0058200220000000002, 2023-10-09T05:48:50.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0058449510000000001, 2023-10-09T05:48:50.957 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:48:50.957 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91412 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.951011+0000", 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:50.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.951011+0000", 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.951009+0000", 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8815e-05 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:48:50.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.951048+0000", 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2908000000000001e-05 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.951061+0000", 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0209999999999999e-06 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.951066+0000", 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4820000000000001e-05 2023-10-09T05:48:50.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:50.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:48:50.961 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.951111+0000", 2023-10-09T05:48:50.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6712999999999994e-05 2023-10-09T05:48:50.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:50.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:48:50.961 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.951208+0000", 2023-10-09T05:48:50.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000107521 2023-10-09T05:48:50.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:50.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.951315+0000", 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015965720000000001 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.952912+0000", 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00032699699999999999 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:48:50.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.953239+0000", 2023-10-09T05:48:50.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022279940000000001 2023-10-09T05:48:50.963 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:50.963 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:48:50.963 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:50.963 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:48:50.963 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:48:50.963 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 1 2023-10-09T05:48:50.963 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:48:51.140 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 68 16 91543 91527 84.0891 55.125 0.0035601 0.0118178 2023-10-09T05:48:51.427 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:51 smithi191 ceph-mon[161669]: pgmap v913: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T05:48:51.498 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:48:51.524 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:51 smithi008 ceph-mon[184418]: pgmap v913: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T05:48:51.524 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:51 smithi008 ceph-mon[184418]: failed to encode map e302 with expected crc 2023-10-09T05:48:51.524 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:51 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T05:48:51.524 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:51 smithi008 ceph-mon[184418]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T05:48:51.524 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:51 smithi008 ceph-mon[184418]: failed to encode map e302 with expected crc 2023-10-09T05:48:51.524 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:51 smithi008 ceph-mon[191227]: pgmap v913: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T05:48:51.524 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:51 smithi008 ceph-mon[191227]: failed to encode map e302 with expected crc 2023-10-09T05:48:51.525 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:51 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T05:48:51.525 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:51 smithi008 ceph-mon[191227]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T05:48:51.525 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:51 smithi008 ceph-mon[191227]: failed to encode map e302 with expected crc 2023-10-09T05:48:51.550 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:48:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:51.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:48:51.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:51.680 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:51 smithi191 ceph-mon[161669]: failed to encode map e302 with expected crc 2023-10-09T05:48:51.680 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:51 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T05:48:51.680 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:51 smithi191 ceph-mon[161669]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T05:48:51.680 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:51 smithi191 ceph-mon[161669]: failed to encode map e302 with expected crc 2023-10-09T05:48:51.683 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:51.800 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:51.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:51.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:51.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:48:51.903 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:51.933 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:52.025 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:52.140 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 69 16 91568 91552 82.8934 1.5625 0.00337969 0.0118153 2023-10-09T05:48:52.150 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:52.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:52.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:52.194 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:48:52.255 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:52.284 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:52.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:52 smithi008 ceph-mon[191227]: failed to encode map e303 with expected crc 2023-10-09T05:48:52.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:52 smithi008 ceph-mon[191227]: failed to encode map e303 with expected crc 2023-10-09T05:48:52.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:52 smithi008 ceph-mon[191227]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T05:48:52.693 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:52.696 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:52 smithi008 ceph-mon[184418]: failed to encode map e303 with expected crc 2023-10-09T05:48:52.696 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:52 smithi008 ceph-mon[184418]: failed to encode map e303 with expected crc 2023-10-09T05:48:52.696 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:52 smithi008 ceph-mon[184418]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T05:48:52.696 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:52 smithi008 conmon[184394]: 2023-10-09T05:48:52.578+0000 7f8a66521700 -1 mon.a@0(leader).osd e304 definitely_dead 0 2023-10-09T05:48:52.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:52.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:52.760 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:52 smithi191 ceph-mon[161669]: failed to encode map e303 with expected crc 2023-10-09T05:48:52.760 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:52 smithi191 ceph-mon[161669]: failed to encode map e303 with expected crc 2023-10-09T05:48:52.760 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:52 smithi191 ceph-mon[161669]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T05:48:52.763 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:48:52.785 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:48:52.798 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:52.814 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:48:52.839 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:52.845 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:48:52.993 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:48:53.036 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:48:53.053 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:53.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:53.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:53.078 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:48:53.110 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:48:53.116 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:48:53.116 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:48:53.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:48:53.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:48:53.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:48:53.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:48:53.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:48:53.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:48:53.140 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 70 16 91808 91792 81.9238 15 0.0144154 0.0122021 2023-10-09T05:48:53.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:53.216 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:53.333 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:53.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:53.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:53.442 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:53.447 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[184418]: pgmap v916: 99 pgs: 1 remapped+peering, 98 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 242 MiB/s wr, 3.88k op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T05:48:53.447 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[184418]: failed to encode map e304 with expected crc 2023-10-09T05:48:53.448 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[184418]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T05:48:53.448 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[184418]: failed to encode map e304 with expected crc 2023-10-09T05:48:53.448 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2607572490' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:48:53.448 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/849890723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:53.448 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[191227]: pgmap v916: 99 pgs: 1 remapped+peering, 98 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 242 MiB/s wr, 3.88k op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T05:48:53.448 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[191227]: failed to encode map e304 with expected crc 2023-10-09T05:48:53.448 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[191227]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T05:48:53.448 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[191227]: failed to encode map e304 with expected crc 2023-10-09T05:48:53.449 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2607572490' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:48:53.449 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/849890723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:53.489 INFO:teuthology.orchestra.run.smithi008.stderr:marked in osd.1. 2023-10-09T05:48:53.509 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:53 smithi191 ceph-mon[161669]: pgmap v916: 99 pgs: 1 remapped+peering, 98 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 242 MiB/s wr, 3.88k op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T05:48:53.509 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:53 smithi191 ceph-mon[161669]: failed to encode map e304 with expected crc 2023-10-09T05:48:53.509 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:53 smithi191 ceph-mon[161669]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T05:48:53.509 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:53 smithi191 ceph-mon[161669]: failed to encode map e304 with expected crc 2023-10-09T05:48:53.510 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2607572490' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:48:53.510 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/849890723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:48:53.514 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:53.559 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:53.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:53.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:53.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:53.692 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:53.840 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:53.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:53.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:53.867 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:48:53.867 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:48:53.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:53.972 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:54.068 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:54.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:54.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:54.140 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 71 16 92204 92188 81.1188 24.75 0.00390077 0.0122113 2023-10-09T05:48:54.172 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:54.204 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:54.691 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:48:54.717 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:54 smithi191 ceph-mon[161669]: failed to encode map e304 with expected crc 2023-10-09T05:48:54.717 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:54 smithi191 ceph-mon[161669]: failed to encode map e305 with expected crc 2023-10-09T05:48:54.717 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:54 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2607572490' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:48:54.717 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:54 smithi191 ceph-mon[161669]: osdmap e305: 8 total, 8 up, 8 in 2023-10-09T05:48:54.717 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:54 smithi191 ceph-mon[161669]: failed to encode map e305 with expected crc 2023-10-09T05:48:54.717 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:54 smithi191 ceph-mon[161669]: failed to encode map e305 with expected crc 2023-10-09T05:48:54.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[184418]: failed to encode map e304 with expected crc 2023-10-09T05:48:54.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[184418]: failed to encode map e305 with expected crc 2023-10-09T05:48:54.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2607572490' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:48:54.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[184418]: osdmap e305: 8 total, 8 up, 8 in 2023-10-09T05:48:54.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[184418]: failed to encode map e305 with expected crc 2023-10-09T05:48:54.918 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[184418]: failed to encode map e305 with expected crc 2023-10-09T05:48:54.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[191227]: failed to encode map e304 with expected crc 2023-10-09T05:48:54.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[191227]: failed to encode map e305 with expected crc 2023-10-09T05:48:54.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2607572490' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:48:54.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[191227]: osdmap e305: 8 total, 8 up, 8 in 2023-10-09T05:48:54.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[191227]: failed to encode map e305 with expected crc 2023-10-09T05:48:54.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:54 smithi008 ceph-mon[191227]: failed to encode map e305 with expected crc 2023-10-09T05:48:55.048 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:48:55.140 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 72 16 92204 92188 79.9925 0 - 0.0122113 2023-10-09T05:48:55.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:55 smithi191 ceph-mon[161669]: pgmap v919: 99 pgs: 19 peering, 11 unknown, 1 remapped+peering, 68 active+clean; 3.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail 2023-10-09T05:48:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:55 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 12 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2023-10-09T05:48:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:55 smithi191 ceph-mon[161669]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:55 smithi191 ceph-mon[161669]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:55 smithi191 ceph-mon[161669]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:55 smithi191 ceph-mon[161669]: osdmap e306: 8 total, 8 up, 8 in 2023-10-09T05:48:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:55 smithi191 ceph-mon[161669]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:55 smithi191 ceph-mon[161669]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.842 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:55.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:55.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:55.857 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[184418]: pgmap v919: 99 pgs: 19 peering, 11 unknown, 1 remapped+peering, 68 active+clean; 3.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail 2023-10-09T05:48:55.857 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 12 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2023-10-09T05:48:55.858 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[184418]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.858 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[184418]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.858 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[184418]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.858 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[184418]: osdmap e306: 8 total, 8 up, 8 in 2023-10-09T05:48:55.858 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[184418]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.858 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[184418]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.858 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[191227]: pgmap v919: 99 pgs: 19 peering, 11 unknown, 1 remapped+peering, 68 active+clean; 3.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail 2023-10-09T05:48:55.858 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 12 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2023-10-09T05:48:55.858 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[191227]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.859 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[191227]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.859 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[191227]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.859 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[191227]: osdmap e306: 8 total, 8 up, 8 in 2023-10-09T05:48:55.859 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[191227]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.859 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:55 smithi008 ceph-mon[191227]: failed to encode map e306 with expected crc 2023-10-09T05:48:55.952 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:55.984 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:56.097 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T05:48:56.141 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 73 16 92293 92277 78.9722 2.78125 0.0225295 0.0126566 2023-10-09T05:48:56.142 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:56.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:56.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:56.247 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:56.334 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:56.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:56.520 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:56.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:56.602 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:48:56.602 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:48:56.622 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:56.627 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:56 smithi191 ceph-mon[161669]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.628 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:56 smithi191 ceph-mon[161669]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.628 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:56 smithi191 ceph-mon[161669]: osdmap e307: 8 total, 8 up, 8 in 2023-10-09T05:48:56.628 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:56 smithi191 ceph-mon[161669]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.628 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:56 smithi191 ceph-mon[161669]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.628 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:56 smithi191 ceph-mon[161669]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.628 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:56 smithi191 ceph-mon[161669]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.628 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:56 smithi191 ceph-mon[161669]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.666 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:56.879 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:56.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:56.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[184418]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[184418]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[184418]: osdmap e307: 8 total, 8 up, 8 in 2023-10-09T05:48:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[184418]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[184418]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[184418]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[184418]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[184418]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[191227]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[191227]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[191227]: osdmap e307: 8 total, 8 up, 8 in 2023-10-09T05:48:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[191227]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[191227]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[191227]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[191227]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:56 smithi008 ceph-mon[191227]: failed to encode map e307 with expected crc 2023-10-09T05:48:56.983 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:48:57.019 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:57.142 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 74 16 92889 92873 78.4085 37.25 0.0156711 0.0127481 2023-10-09T05:48:57.154 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:48:57.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:57.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:48:57.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:48:57.324 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:57.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:48:57.390 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:48:57.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:57.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:48:57.495 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:57.516 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:48:57.616 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:57.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:57 smithi191 ceph-mon[161669]: failed to encode map e307 with expected crc 2023-10-09T05:48:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:57 smithi191 ceph-mon[161669]: pgmap v922: 99 pgs: 19 peering, 10 unknown, 4 remapped+peering, 66 active+clean; 3.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 4 B/s, 0 objects/s recovering 2023-10-09T05:48:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:57 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/500172137' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:48:57.810 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:57 smithi008 ceph-mon[191227]: failed to encode map e307 with expected crc 2023-10-09T05:48:57.810 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:57 smithi008 ceph-mon[191227]: pgmap v922: 99 pgs: 19 peering, 10 unknown, 4 remapped+peering, 66 active+clean; 3.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 4 B/s, 0 objects/s recovering 2023-10-09T05:48:57.811 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:57 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/500172137' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:48:57.811 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:57 smithi008 ceph-mon[184418]: failed to encode map e307 with expected crc 2023-10-09T05:48:57.811 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:57 smithi008 ceph-mon[184418]: pgmap v922: 99 pgs: 19 peering, 10 unknown, 4 remapped+peering, 66 active+clean; 3.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 4 B/s, 0 objects/s recovering 2023-10-09T05:48:57.811 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:57 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/500172137' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:48:57.817 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:48:57.828 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:57.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:57.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:57.937 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:48:57.958 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:48:58.033 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:58.142 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 75 16 93634 93618 77.984 46.5625 0.0101528 0.0128167 2023-10-09T05:48:58.158 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:48:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:58.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:48:58.270 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:58.307 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:58.481 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:58.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:58.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:58.588 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:48:58.619 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:58.667 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:48:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:58.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:48:58.770 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:48:58.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:58 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:58.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/500172137' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:48:58.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:58 smithi191 ceph-mon[161669]: osdmap e308: 8 total, 8 up, 8 in 2023-10-09T05:48:58.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:58 smithi191 ceph-mon[161669]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:58 smithi191 ceph-mon[161669]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:58 smithi191 ceph-mon[161669]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:58 smithi191 ceph-mon[161669]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.818 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:58.818 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/500172137' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:48:58.818 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[184418]: osdmap e308: 8 total, 8 up, 8 in 2023-10-09T05:48:58.818 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[184418]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.819 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[184418]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.819 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[184418]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.819 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[184418]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.819 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:48:58.819 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/500172137' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:48:58.819 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[191227]: osdmap e308: 8 total, 8 up, 8 in 2023-10-09T05:48:58.819 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[191227]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.819 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[191227]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.819 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[191227]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.820 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:58 smithi008 ceph-mon[191227]: failed to encode map e308 with expected crc 2023-10-09T05:48:58.822 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:59.142 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 76 16 94267 94251 77.4785 39.5625 0.0102178 0.0129014 2023-10-09T05:48:59.616 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:48:59.616 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:48:59.766 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:48:59.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:59.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:48:59.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:48:59.914 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:48:59.938 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:59.958 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:48:59.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:48:59.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:00.062 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:00.107 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:59 smithi008 ceph-mon[184418]: pgmap v924: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 19 peering, 3 unknown, 3 remapped+peering, 66 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 20165/259962 objects degraded (7.757%); 8159/259962 objects misplaced (3.139%); 3.3 MiB/s, 50 objects/s recovering 2023-10-09T05:49:00.107 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:00.107 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:00.107 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:59 smithi008 ceph-mon[184418]: Health check failed: Degraded data redundancy: 20165/259962 objects degraded (7.757%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:00.107 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:48:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:00.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:59 smithi008 ceph-mon[191227]: pgmap v924: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 19 peering, 3 unknown, 3 remapped+peering, 66 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 20165/259962 objects degraded (7.757%); 8159/259962 objects misplaced (3.139%); 3.3 MiB/s, 50 objects/s recovering 2023-10-09T05:49:00.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:00.107 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:00.108 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:59 smithi008 ceph-mon[191227]: Health check failed: Degraded data redundancy: 20165/259962 objects degraded (7.757%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:00.108 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:48:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:00.111 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:00.142 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 77 16 94917 94901 76.9995 40.625 0.0050554 0.0129792 2023-10-09T05:49:00.269 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:00.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:00.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:59 smithi191 ceph-mon[161669]: pgmap v924: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 19 peering, 3 unknown, 3 remapped+peering, 66 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 20165/259962 objects degraded (7.757%); 8159/259962 objects misplaced (3.139%); 3.3 MiB/s, 50 objects/s recovering 2023-10-09T05:49:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:59 smithi191 ceph-mon[161669]: Health check failed: Degraded data redundancy: 20165/259962 objects degraded (7.757%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:48:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:00.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:49:00.377 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:00.466 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:00.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:49:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:49:00] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.33.4" 2023-10-09T05:49:00.848 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:00.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:00.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:00.870 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:49:00.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:49:00.992 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:01.097 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:49:01.098 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:49:01.098 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-10-09T05:49:01.098 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:49:01.113 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:49:01.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:01.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:49:01.142 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 78 16 95678 95662 76.6222 47.5625 0.0155242 0.0130452 2023-10-09T05:49:01.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:00 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 12 pgs inactive, 12 pgs peering) 2023-10-09T05:49:01.201 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:00 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 12 pgs inactive, 12 pgs peering) 2023-10-09T05:49:01.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:49:01.252 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:00 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 12 pgs inactive, 12 pgs peering) 2023-10-09T05:49:01.255 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:01.418 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:49:01.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:01.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:01.522 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:01.556 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:01.875 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:49:02.073 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:02.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:02.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:02.142 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 79 16 96264 96248 76.116 36.625 0.012783 0.0131316 2023-10-09T05:49:02.195 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:49:02.212 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:01 smithi008 ceph-mon[184418]: pgmap v925: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 410 KiB/s rd, 355 MiB/s wr, 5.96k op/s; 34600/282378 objects degraded (12.253%); 8159/282378 objects misplaced (2.889%); 2.7 MiB/s, 46 objects/s recovering 2023-10-09T05:49:02.212 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:01 smithi008 ceph-mon[191227]: pgmap v925: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 410 KiB/s rd, 355 MiB/s wr, 5.96k op/s; 34600/282378 objects degraded (12.253%); 8159/282378 objects misplaced (2.889%); 2.7 MiB/s, 46 objects/s recovering 2023-10-09T05:49:02.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:01 smithi191 ceph-mon[161669]: pgmap v925: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 410 KiB/s rd, 355 MiB/s wr, 5.96k op/s; 34600/282378 objects degraded (12.253%); 8159/282378 objects misplaced (2.889%); 2.7 MiB/s, 46 objects/s recovering 2023-10-09T05:49:02.235 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:02.299 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:49:02.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:02.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:02.390 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:02.403 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:02.489 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:49:02.489 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:49:02.511 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:02.652 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:02.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:02.763 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:02.800 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:02.845 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:02.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:02.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:02.948 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:02.986 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:03.143 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:49:03.143456+0000 min lat: 0.00124024 max lat: 2.58525 avg lat: 0.0132234 2023-10-09T05:49:03.143 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:49:03.143 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 80 16 96810 96794 75.5912 34.125 0.0199994 0.0132234 2023-10-09T05:49:03.182 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:49:03.235 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:49:03.272 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:49:03.278 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:49:03.281 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:03.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:03.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:03.336 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:49:03.381 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:49:03.388 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:03.439 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:49:03.469 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:49:03.492 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:03.500 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:49:03.536 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:49:03.536 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:49:03.536 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:49:03.537 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:49:03.537 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:49:03.537 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:49:03.537 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:49:03.537 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:49:03.643 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:03.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:03.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:03.747 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:49:03.753 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:03.824 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:04.144 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 81 16 97515 97499 75.2008 44.0625 0.00276281 0.0132927 2023-10-09T05:49:04.283 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:04 smithi191 ceph-mon[161669]: pgmap v926: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 312 KiB/s rd, 277 MiB/s wr, 4.64k op/s; 34600/285222 objects degraded (12.131%); 7787/285222 objects misplaced (2.730%); 3.5 MiB/s, 59 objects/s recovering 2023-10-09T05:49:04.284 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/721380397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:04.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:03 smithi008 ceph-mon[184418]: pgmap v926: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 312 KiB/s rd, 277 MiB/s wr, 4.64k op/s; 34600/285222 objects degraded (12.131%); 7787/285222 objects misplaced (2.730%); 3.5 MiB/s, 59 objects/s recovering 2023-10-09T05:49:04.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:03 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/721380397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:03 smithi008 ceph-mon[191227]: pgmap v926: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 312 KiB/s rd, 277 MiB/s wr, 4.64k op/s; 34600/285222 objects degraded (12.131%); 7787/285222 objects misplaced (2.730%); 3.5 MiB/s, 59 objects/s recovering 2023-10-09T05:49:04.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:03 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/721380397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:04.731 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:04.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:04.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:04.840 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:49:04.877 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:04.941 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:49:04.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:04.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:49:05.045 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:05.056 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:05 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 34600/288294 objects degraded (12.002%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:05.058 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:05 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 34600/288294 objects degraded (12.002%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:05.124 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:05 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 34600/288294 objects degraded (12.002%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:05.134 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:05.144 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 82 16 98024 98008 74.6718 31.8125 0.0208265 0.0133859 2023-10-09T05:49:05.364 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:49:05.365 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:49:05.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:49:05.365 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:49:05.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.365 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:19336 7.d 7:bca1948b:::benchmark_data_smithi008_354649_object19335:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-09T05:49:05.365 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:49:05.365 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 438.71471953899999, 2023-10-09T05:49:05.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27934059700000002, 2023-10-09T05:49:05.365 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.365 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 19336 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647257+0000", 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4706000000000003e-05 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647292+0000", 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0139999999999999e-05 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647302+0000", 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4790000000000004e-06 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647308+0000", 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0175000000000001e-05 2023-10-09T05:49:05.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647348+0000", 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3017999999999997e-05 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647431+0000", 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7472000000000005e-05 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647528+0000", 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00087520700000000002 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.648404+0000", 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091703300000000004 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.649321+0000", 2023-10-09T05:49:05.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27717746799999998 2023-10-09T05:49:05.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926498+0000", 2023-10-09T05:49:05.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2379000000000001e-05 2023-10-09T05:49:05.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926550+0000", 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8653000000000003e-05 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926599+0000", 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27934059700000002 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:52521 7.c 7:3208f079:::benchmark_data_smithi008_354649_object52520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 410.76104976099998, 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.253878241, 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52521 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.373 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600927+0000", 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114644 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601041+0000", 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3705e-05 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601055+0000", 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3670000000000002e-06 2023-10-09T05:49:05.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601062+0000", 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6796000000000003e-05 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601118+0000", 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000151423 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601270+0000", 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000187224 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601457+0000", 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018848820000000001 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.603342+0000", 2023-10-09T05:49:05.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017959000000000001 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.603521+0000", 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25118068700000001 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854702+0000", 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0417000000000001e-05 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854763+0000", 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3959999999999999e-05 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854806+0000", 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.253878241 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72589 7.d 7:bac7991c:::benchmark_data_smithi008_354649_object72588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 387.90354883499998, 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.258363538, 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72589 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.380 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8671000000000004e-05 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458527+0000", 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2957e-05 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458540+0000", 2023-10-09T05:49:05.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6649999999999997e-06 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458547+0000", 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2287999999999997e-05 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458609+0000", 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012157100000000001 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458731+0000", 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000152602 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458883+0000", 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026998410000000001 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.461583+0000", 2023-10-09T05:49:05.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0065822210000000001 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.468166+0000", 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24850773600000001 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716673+0000", 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7136000000000002e-05 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716740+0000", 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2272999999999999e-05 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716793+0000", 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.258363538 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72593 7.d 7:ba3e3325:::benchmark_data_smithi008_354649_object72592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 387.89607417100001, 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25827438800000002, 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72593 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.387 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465902+0000", 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000110436 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466013+0000", 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0246e-05 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466023+0000", 2023-10-09T05:49:05.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7699999999999998e-06 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466029+0000", 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0343999999999999e-05 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466069+0000", 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4201000000000002e-05 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466153+0000", 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3833999999999996e-05 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466247+0000", 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020428099999999999 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.391 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.468290+0000", 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011037670000000001 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.469394+0000", 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25468252499999999 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724076+0000", 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0541000000000003e-05 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724127+0000", 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1499000000000003e-05 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724178+0000", 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25827438800000002 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.394 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72601 7.d 7:b4df4171:::benchmark_data_smithi008_354649_object72600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:49:05.394 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:49:05.394 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 387.89013421700002, 2023-10-09T05:49:05.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25693990900000002, 2023-10-09T05:49:05.394 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.394 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.394 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.394 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.394 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.394 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72601 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471843+0000", 2023-10-09T05:49:05.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1391000000000003e-05 2023-10-09T05:49:05.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471894+0000", 2023-10-09T05:49:05.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3217000000000001e-05 2023-10-09T05:49:05.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471907+0000", 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4e-06 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471913+0000", 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6798999999999998e-05 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471950+0000", 2023-10-09T05:49:05.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6098000000000001e-05 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.472036+0000", 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010641099999999999 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.472142+0000", 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001463415 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.473605+0000", 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00083725200000000003 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.474443+0000", 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25426129400000003 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728704+0000", 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5178e-05 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728749+0000", 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4474999999999998e-05 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728784+0000", 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25693990900000002 2023-10-09T05:49:05.400 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102136 7.c 7:3c9f4178:::benchmark_data_smithi008_354649_object7640:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 346.48643351499999, 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3695118229999999, 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.401 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102136 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875543+0000", 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0234999999999999e-05 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875553+0000", 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6123000000000001e-05 2023-10-09T05:49:05.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875569+0000", 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7334e-05 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875586+0000", 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3238999999999997e-05 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.404 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875630+0000", 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120889 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875751+0000", 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000121965 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875873+0000", 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068513569999999998 2023-10-09T05:49:05.405 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.882724+0000", 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.155e-05 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.882735+0000", 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99459824299999999 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877334+0000", 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.508e-05 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877349+0000", 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.345701239 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.223050+0000", 2023-10-09T05:49:05.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016460699999999999 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.223215+0000", 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021743302999999999 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.244958+0000", 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8379000000000001e-05 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.408 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245056+0000", 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3695118229999999 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102150 7.c 7:30e13689:::benchmark_data_smithi008_354649_object7654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 346.45737769700003, 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3456080319999999, 2023-10-09T05:49:05.409 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102150 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.410 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904598+0000", 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9720000000000003e-06 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.411 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.412 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904605+0000", 2023-10-09T05:49:05.412 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3104e-05 2023-10-09T05:49:05.412 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.412 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.412 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.412 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904619+0000", 2023-10-09T05:49:05.412 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7699999999999998e-06 2023-10-09T05:49:05.412 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.412 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.412 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.412 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904624+0000", 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5915000000000003e-05 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904680+0000", 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1524e-05 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904692+0000", 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97280887699999996 2023-10-09T05:49:05.413 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.414 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.414 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.414 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877501+0000", 2023-10-09T05:49:05.414 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9029999999999993e-06 2023-10-09T05:49:05.414 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.414 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.414 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:05.414 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877510+0000", 2023-10-09T05:49:05.414 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3503837970000001 2023-10-09T05:49:05.414 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.414 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.415 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.415 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.227893+0000", 2023-10-09T05:49:05.415 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017629700000000001 2023-10-09T05:49:05.415 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.415 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.415 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.415 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.228070+0000", 2023-10-09T05:49:05.415 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019175 2023-10-09T05:49:05.415 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.415 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.415 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.228261+0000", 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086207179999999994 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.236882+0000", 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0081705159999999992 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245053+0000", 2023-10-09T05:49:05.416 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4658000000000003e-05 2023-10-09T05:49:05.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.417 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.417 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.417 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245117+0000", 2023-10-09T05:49:05.417 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2007000000000004e-05 2023-10-09T05:49:05.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.417 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.417 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.417 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245189+0000", 2023-10-09T05:49:05.417 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005019061 2023-10-09T05:49:05.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250208+0000", 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3456080319999999 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102158 7.c 7:380582de:::benchmark_data_smithi008_354649_object7662:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:49:05.418 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 346.45264179499998, 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3657197050000001, 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102158 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.419 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909335+0000", 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1000000000000003e-06 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.420 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909340+0000", 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9660000000000002e-06 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909349+0000", 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3199999999999999e-06 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.421 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909354+0000", 2023-10-09T05:49:05.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2012999999999999e-05 2023-10-09T05:49:05.422 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.422 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.422 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.422 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909396+0000", 2023-10-09T05:49:05.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4160000000000001e-06 2023-10-09T05:49:05.422 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.422 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.422 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:05.422 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909403+0000", 2023-10-09T05:49:05.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96813495800000005 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877538+0000", 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5560000000000008e-06 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877546+0000", 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3727139369999999 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.423 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250260+0000", 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017051999999999999 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250431+0000", 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019215399999999999 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:49:05.424 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250623+0000", 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010646828000000001 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261270+0000", 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6761999999999995e-05 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261337+0000", 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087281960000000006 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.426 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.270065+0000", 2023-10-09T05:49:05.426 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9771000000000006e-05 2023-10-09T05:49:05.426 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.426 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.426 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.270144+0000", 2023-10-09T05:49:05.426 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0049114320000000003 2023-10-09T05:49:05.426 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.426 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275056+0000", 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3657197050000001 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102172 7.c 7:3ca77be9:::benchmark_data_smithi008_354649_object7676:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 346.42590472500001, 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3439011609999998, 2023-10-09T05:49:05.427 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102172 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.429 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.429 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:49:05.429 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999971 2023-10-09T05:49:05.429 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.429 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.429 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936071+0000", 2023-10-09T05:49:05.429 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4219999999999999e-05 2023-10-09T05:49:05.429 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936085+0000", 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3510000000000003e-05 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936118+0000", 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3056e-05 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.430 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936131+0000", 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013862499999999999 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936270+0000", 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3222999999999997e-05 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936303+0000", 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.941830422 2023-10-09T05:49:05.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.878134+0000", 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3155e-05 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.878147+0000", 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.375972827 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.432 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254120+0000", 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00031237700000000002 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254432+0000", 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000179364 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254611+0000", 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0066698180000000001 2023-10-09T05:49:05.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261281+0000", 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00092641700000000002 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.262208+0000", 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012930142 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.434 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275138+0000", 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1118999999999995e-05 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275209+0000", 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047656299999999999 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.279974+0000", 2023-10-09T05:49:05.435 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3439011609999998 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:124846 7.c 7:339e95c6:::benchmark_data_smithi008_354649_object30350:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 303.85802900499999, 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19787401099999999, 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.436 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124846 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.437 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.438 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:49:05.438 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:49:05.438 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.438 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.438 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.438 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:49:05.438 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8360000000000001e-06 2023-10-09T05:49:05.438 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.438 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.438 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.438 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503951+0000", 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.534e-06 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503954+0000", 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7929999999999999e-06 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503958+0000", 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070412340000000004 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.439 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.510999+0000", 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012243900000000001 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.511122+0000", 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012357500000000001 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:49:05.440 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.511245+0000", 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070869629999999999 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.518332+0000", 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010506108 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.528838+0000", 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070145440000000002 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.441 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.535853+0000", 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2738000000000002e-05 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.535905+0000", 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.165917541 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.701823+0000", 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19787401099999999 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:124870 7.c 7:34cc5040:::benchmark_data_smithi008_354649_object30374:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 303.82559505500001, 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.194346888, 2023-10-09T05:49:05.443 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124870 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.444 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536382+0000", 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1800000000000001e-06 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.445 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536388+0000", 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8409999999999995e-06 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536398+0000", 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9701e-05 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536417+0000", 2023-10-09T05:49:05.446 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16904754699999999 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705465+0000", 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016865399999999999 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705634+0000", 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017818700000000001 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.447 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705812+0000", 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068761990000000004 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.712688+0000", 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078597960000000005 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.448 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.720548+0000", 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059971089999999996 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.726545+0000", 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4085000000000006e-05 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.726619+0000", 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004110832 2023-10-09T05:49:05.449 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.730730+0000", 2023-10-09T05:49:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.194346888 2023-10-09T05:49:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:136135 7.d 7:b8c830a8:::benchmark_data_smithi008_354649_object41639:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-09T05:49:05.450 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:35.638089+0000", 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 269.723888864, 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.252176754, 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 136135 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.451 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638089+0000", 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638089+0000", 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638087+0000", 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0289999999999994e-06 2023-10-09T05:49:05.452 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638096+0000", 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6886000000000001e-05 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638113+0000", 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0861e-05 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.453 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638123+0000", 2023-10-09T05:49:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8810000000000001e-05 2023-10-09T05:49:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638182+0000", 2023-10-09T05:49:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000136959 2023-10-09T05:49:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.454 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638319+0000", 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015855900000000001 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638478+0000", 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048134650000000003 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643291+0000", 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073579750000000001 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.455 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.650649+0000", 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23951030400000001 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.890160+0000", 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8352e-05 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.456 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.890218+0000", 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8004000000000003e-05 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.890266+0000", 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.252176754 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.457 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:136137 7.d 7:b6392fbb:::benchmark_data_smithi008_354649_object41641:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 269.71799171399999, 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24962479700000001, 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 136137 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.458 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.459 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1309999999999999e-06 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643990+0000", 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4509999999999992e-06 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643998+0000", 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5480000000000001e-06 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.460 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644004+0000", 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1790999999999999e-05 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644056+0000", 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015365299999999999 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.461 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644209+0000", 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017818200000000001 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644387+0000", 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047856310000000003 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.649173+0000", 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015424 2023-10-09T05:49:05.462 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.650715+0000", 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.242803251 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.893519+0000", 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7497000000000002e-05 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.463 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.893576+0000", 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4962000000000002e-05 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.893611+0000", 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24962479700000001 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.464 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:41724 8.0 e291/285 8:01b70071:::benchmark_data_smithi008_432048_object41723:head v 291'2644, mlcod=291'2641)", 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:02.471292+0000", 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 62.890686158999998, 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030552701000000002, 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471292+0000", 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.465 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471292+0000", 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471291+0000", 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6661000000000004e-05 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471357+0000", 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7370000000000001e-06 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471367+0000", 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7799999999999997e-06 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471373+0000", 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00049471499999999996 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471867+0000", 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8739000000000001e-05 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471886+0000", 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018492409999999999 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.468 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.473735+0000", 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028109080000000002 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.501845+0000", 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030552701000000002 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:41716 8.f e291/285 8:faa82907:::benchmark_data_smithi008_432048_object41715:head v 291'2545, mlcod=291'2542)", 2023-10-09T05:49:05.469 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:02.471382+0000", 2023-10-09T05:49:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 62.890595597999997, 2023-10-09T05:49:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030548492999999999, 2023-10-09T05:49:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471382+0000", 2023-10-09T05:49:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.470 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471382+0000", 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471382+0000", 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2911999999999999e-05 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471415+0000", 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5379999999999999e-06 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471421+0000", 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9489999999999998e-06 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471425+0000", 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6309000000000001e-05 2023-10-09T05:49:05.472 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471462+0000", 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8779000000000003e-05 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471501+0000", 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022499960000000002 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.473751+0000", 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028180394000000001 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.501931+0000", 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030548492999999999 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:71979 8.f e295/294 8:f088e1a0:::benchmark_data_smithi008_432048_object71978:head v 295'4395, mlcod=295'4393)", 2023-10-09T05:49:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:28.507415+0000", 2023-10-09T05:49:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 36.854562524000002, 2023-10-09T05:49:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023385482999999999, 2023-10-09T05:49:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507415+0000", 2023-10-09T05:49:05.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507415+0000", 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507415+0000", 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5426999999999999e-05 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.476 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507440+0000", 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8349999999999998e-06 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507447+0000", 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5120000000000002e-06 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507452+0000", 2023-10-09T05:49:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029621090000000001 2023-10-09T05:49:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.510414+0000", 2023-10-09T05:49:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1447999999999999e-05 2023-10-09T05:49:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.510435+0000", 2023-10-09T05:49:05.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020330933999999998 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.530766+0000", 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4652e-05 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.530801+0000", 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023385482999999999 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.479 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:71988 8.f e295/294 8:f2a7b732:::benchmark_data_smithi008_432048_object71987:head v 295'4396, mlcod=295'4393)", 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:28.508864+0000", 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 36.853114402000003, 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021963251, 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.480 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.508864+0000", 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.508864+0000", 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.508862+0000", 2023-10-09T05:49:05.481 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9734999999999999e-05 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.508892+0000", 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7425999999999999e-05 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.508909+0000", 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3300000000000001e-06 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.482 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.508917+0000", 2023-10-09T05:49:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039989320000000002 2023-10-09T05:49:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.512916+0000", 2023-10-09T05:49:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5528999999999999e-05 2023-10-09T05:49:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.483 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.512941+0000", 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017864781999999999 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.530806+0000", 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0996e-05 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.530827+0000", 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021963251 2023-10-09T05:49:05.484 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:75678 8.5 e296/285 8:a9f05736:::benchmark_data_smithi008_432048_object75677:head v 296'4715, mlcod=296'4713)", 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:32.466634+0000", 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 32.895344153000003, 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021519693999999999, 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.485 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:32.466634+0000", 2023-10-09T05:49:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:32.466634+0000", 2023-10-09T05:49:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.486 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:32.466632+0000", 2023-10-09T05:49:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5773999999999999e-05 2023-10-09T05:49:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.487 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:32.466678+0000", 2023-10-09T05:49:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1891000000000001e-05 2023-10-09T05:49:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:32.466700+0000", 2023-10-09T05:49:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1409999999999994e-06 2023-10-09T05:49:05.489 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:32.466709+0000", 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000388007 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:32.467097+0000", 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1690000000000001e-05 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.490 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:32.467119+0000", 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020993396000000001 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:32.488112+0000", 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1289999999999999e-05 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:32.488154+0000", 2023-10-09T05:49:05.491 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021519693999999999 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:86717 8.6 e300/299 8:65a0d5c6:::benchmark_data_smithi008_432048_object86716:head v 300'5376, mlcod=0'0)", 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:46.222066+0000", 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 19.139912469999999, 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022574397, 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.492 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222066+0000", 2023-10-09T05:49:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222066+0000", 2023-10-09T05:49:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:49:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.493 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222065+0000", 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1554000000000001e-05 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222117+0000", 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1640000000000004e-06 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222124+0000", 2023-10-09T05:49:05.494 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0900000000000005e-06 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222132+0000", 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0091579690000000002 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.231290+0000", 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5908999999999999e-05 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.231316+0000", 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0132868 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.244603+0000", 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7187000000000002e-05 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.496 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.244640+0000", 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022574397 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:89017 8.10 e301/299 8:0cd3ebd7:::benchmark_data_smithi008_432048_object89016:head v 301'5542, mlcod=301'5540)", 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:48.392752+0000", 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.969225910999999, 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022788323999999999, 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:05.497 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392752+0000", 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392752+0000", 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392752+0000", 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2337e-05 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392774+0000", 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0094e-05 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392784+0000", 2023-10-09T05:49:05.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.357e-06 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392789+0000", 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038184220000000001 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.396607+0000", 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0106e-05 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.396627+0000", 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018869139 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.415496+0000", 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4217e-05 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.415540+0000", 2023-10-09T05:49:05.501 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022788323999999999 2023-10-09T05:49:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:05.502 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:05.502 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:49:05.549 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:05.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:05.559 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:05.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:05.661 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:05.749 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:05.954 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:05.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:05.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:06.063 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:06.144 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 83 16 98706 98690 74.2857 42.625 0.00854426 0.0134551 2023-10-09T05:49:06.157 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:06.200 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:06 smithi008 ceph-mon[184418]: pgmap v927: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 277 KiB/s rd, 230 MiB/s wr, 3.86k op/s; 34600/288294 objects degraded (12.002%); 7787/288294 objects misplaced (2.701%); 3.1 MiB/s, 51 objects/s recovering 2023-10-09T05:49:06.201 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:06 smithi008 ceph-mon[191227]: pgmap v927: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 277 KiB/s rd, 230 MiB/s wr, 3.86k op/s; 34600/288294 objects degraded (12.002%); 7787/288294 objects misplaced (2.701%); 3.1 MiB/s, 51 objects/s recovering 2023-10-09T05:49:06.221 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:49:06.305 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:06.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:06.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:06.417 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:06.525 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:06.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:06 smithi191 ceph-mon[161669]: pgmap v927: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 277 KiB/s rd, 230 MiB/s wr, 3.86k op/s; 34600/288294 objects degraded (12.002%); 7787/288294 objects misplaced (2.701%); 3.1 MiB/s, 51 objects/s recovering 2023-10-09T05:49:06.658 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:49:06.751 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:06.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:06.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:06.827 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:49:06.848 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:49:06.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:06.912 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:06.990 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:06.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:06.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:07.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:07.147 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:07.149 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 84 16 99351 99335 73.8767 40.3125 0.0230341 0.0135301 2023-10-09T05:49:07.260 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:07.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:07.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:07.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:07.389 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:07.431 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:07.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:07.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:07.535 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:07.564 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:07.596 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:49:08.150 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 85 16 99942 99926 73.4422 36.9375 0.0118692 0.0136114 2023-10-09T05:49:08.262 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:49:08.262 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:49:08.262 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T05:49:08.262 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:49:08.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:08 smithi191 ceph-mon[161669]: pgmap v928: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 246 KiB/s rd, 212 MiB/s wr, 3.57k op/s; 34600/292140 objects degraded (11.844%); 7787/292140 objects misplaced (2.666%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:49:08.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:08 smithi191 ceph-mon[161669]: from='client.45875 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:49:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:08 smithi008 ceph-mon[184418]: pgmap v928: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 246 KiB/s rd, 212 MiB/s wr, 3.57k op/s; 34600/292140 objects degraded (11.844%); 7787/292140 objects misplaced (2.666%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:49:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:08 smithi008 ceph-mon[184418]: from='client.45875 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:49:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:08 smithi008 ceph-mon[191227]: pgmap v928: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 246 KiB/s rd, 212 MiB/s wr, 3.57k op/s; 34600/292140 objects degraded (11.844%); 7787/292140 objects misplaced (2.666%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T05:49:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:08 smithi008 ceph-mon[191227]: from='client.45875 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:49:09.032 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:09.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:09.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:09.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:49:09.140 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:09.149 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":928,"stamp":"2023-10-09T05:49:06.374909+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6384850867,"num_objects":97380,"num_object_clones":0,"num_object_copies":292140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34600,"num_objects_misplaced":7787,"num_objects_unfound":0,"num_objects_dirty":97380,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":98458,"num_write_kb":6245669,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":581,"num_bytes_recovered":34745514,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":152748,"ondisk_log_size":152748,"up":297,"acting":285,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":312,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23572048,"kb_used_data":18787344,"kb_used_omap":45,"kb_used_meta":4784594,"kb_avail":726389168,"statfs":{"total":767960285184,"available":743822508032,"internally_reserved":0,"allocated":19238240256,"data_stored":19231372562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":4899424547},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,3,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2226520064,"num_objects":33975,"num_object_clones":0,"num_object_copies":101925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34600,"num_objects_misplaced":7787,"num_objects_unfound":0,"num_objects_dirty":33975,"num_whiteouts":0,"num_read":1468,"num_read_kb":2463,"num_write":34219,"num_write_kb":2175047,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":457,"num_bytes_recovered":29097984,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003651"},"pg_stats":[{"pgid":"8.11","version":"308'5742","reported_seq":5788,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:03.838480+0000","last_change":"2023-10-09T05:48:55.843352+0000","last_active":"2023-10-09T05:49:03.838480+0000","last_peered":"2023-10-09T05:49:03.838480+0000","last_clean":"2023-10-09T05:48:53.444054+0000","last_became_active":"2023-10-09T05:48:55.837353+0000","last_became_peered":"2023-10-09T05:48:55.837353+0000","last_unstale":"2023-10-09T05:49:03.838480+0000","last_undegraded":"2023-10-09T05:48:55.496526+0000","last_fullsized":"2023-10-09T05:48:55.477252+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5742,"log_dups_size":0,"ondisk_log_size":5742,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199426048,"num_objects":3043,"num_object_clones":0,"num_object_copies":9129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2852,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3043,"num_write_kb":194752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":191},{"shards":"0,2,5","objects":2852}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":308,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.482987+0000","last_change":"2023-10-09T05:48:54.494513+0000","last_active":"2023-10-09T05:48:55.482987+0000","last_peered":"2023-10-09T05:48:55.482987+0000","last_clean":"2023-10-09T05:48:55.482987+0000","last_became_active":"2023-10-09T05:48:54.493510+0000","last_became_peered":"2023-10-09T05:48:54.493510+0000","last_unstale":"2023-10-09T05:48:55.482987+0000","last_undegraded":"2023-10-09T05:48:55.482987+0000","last_fullsized":"2023-10-09T05:48:55.482987+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.546981+0000","last_change":"2023-10-09T05:48:54.492024+0000","last_active":"2023-10-09T05:48:55.546981+0000","last_peered":"2023-10-09T05:48:55.546981+0000","last_clean":"2023-10-09T05:48:55.546981+0000","last_became_active":"2023-10-09T05:48:54.491690+0000","last_became_peered":"2023-10-09T05:48:54.491690+0000","last_unstale":"2023-10-09T05:48:55.546981+0000","last_undegraded":"2023-10-09T05:48:55.546981+0000","last_fullsized":"2023-10-09T05:48:55.546981+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:41.936752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.12","version":"308'5675","reported_seq":5715,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:05.144971+0000","last_change":"2023-10-09T05:48:46.263477+0000","last_active":"2023-10-09T05:49:05.144971+0000","last_peered":"2023-10-09T05:49:05.144971+0000","last_clean":"2023-10-09T05:49:05.144971+0000","last_became_active":"2023-10-09T05:48:46.259060+0000","last_became_peered":"2023-10-09T05:48:46.259060+0000","last_unstale":"2023-10-09T05:49:05.144971+0000","last_undegraded":"2023-10-09T05:49:05.144971+0000","last_fullsized":"2023-10-09T05:49:05.144971+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5675,"log_dups_size":0,"ondisk_log_size":5675,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197099520,"num_objects":3008,"num_object_clones":0,"num_object_copies":9024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3008,"num_write_kb":192480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.832761+0000","last_change":"2023-10-09T05:48:54.491563+0000","last_active":"2023-10-09T05:48:55.832761+0000","last_peered":"2023-10-09T05:48:55.832761+0000","last_clean":"2023-10-09T05:48:55.832761+0000","last_became_active":"2023-10-09T05:48:54.491227+0000","last_became_peered":"2023-10-09T05:48:54.491227+0000","last_unstale":"2023-10-09T05:48:55.832761+0000","last_undegraded":"2023-10-09T05:48:55.832761+0000","last_fullsized":"2023-10-09T05:48:55.832761+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:37.990305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":315,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476484+0000","last_change":"2023-10-09T05:48:54.493724+0000","last_active":"2023-10-09T05:48:55.476484+0000","last_peered":"2023-10-09T05:48:55.476484+0000","last_clean":"2023-10-09T05:48:55.476484+0000","last_became_active":"2023-10-09T05:48:54.493380+0000","last_became_peered":"2023-10-09T05:48:54.493380+0000","last_unstale":"2023-10-09T05:48:55.476484+0000","last_undegraded":"2023-10-09T05:48:55.476484+0000","last_fullsized":"2023-10-09T05:48:55.476484+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:05.264766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"308'5730","reported_seq":5774,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:05.225404+0000","last_change":"2023-10-09T05:48:55.978875+0000","last_active":"2023-10-09T05:49:05.225404+0000","last_peered":"2023-10-09T05:49:05.225404+0000","last_clean":"2023-10-09T05:48:53.465296+0000","last_became_active":"2023-10-09T05:48:55.972707+0000","last_became_peered":"2023-10-09T05:48:55.972707+0000","last_unstale":"2023-10-09T05:49:05.225404+0000","last_undegraded":"2023-10-09T05:48:55.487364+0000","last_fullsized":"2023-10-09T05:48:55.475509+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199163904,"num_objects":3039,"num_object_clones":0,"num_object_copies":9117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2813,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3039,"num_write_kb":194496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[5,6],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5,6","objects":2812},{"shards":"5,6","objects":226}],"blocked_by":[],"up_primary":1,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":276,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476344+0000","last_change":"2023-10-09T05:48:54.492664+0000","last_active":"2023-10-09T05:48:55.476344+0000","last_peered":"2023-10-09T05:48:55.476344+0000","last_clean":"2023-10-09T05:48:55.476344+0000","last_became_active":"2023-10-09T05:48:54.492299+0000","last_became_peered":"2023-10-09T05:48:54.492299+0000","last_unstale":"2023-10-09T05:48:55.476344+0000","last_undegraded":"2023-10-09T05:48:55.476344+0000","last_fullsized":"2023-10-09T05:48:55.476344+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:43.198424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":141,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476435+0000","last_change":"2023-10-09T05:48:54.492447+0000","last_active":"2023-10-09T05:48:55.476435+0000","last_peered":"2023-10-09T05:48:55.476435+0000","last_clean":"2023-10-09T05:48:55.476435+0000","last_became_active":"2023-10-09T05:48:54.492089+0000","last_became_peered":"2023-10-09T05:48:54.492089+0000","last_unstale":"2023-10-09T05:48:55.476435+0000","last_undegraded":"2023-10-09T05:48:55.476435+0000","last_fullsized":"2023-10-09T05:48:55.476435+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":262,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476695+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:48:55.476695+0000","last_peered":"2023-10-09T05:48:55.476695+0000","last_clean":"2023-10-09T05:48:55.476695+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:48:55.476695+0000","last_undegraded":"2023-10-09T05:48:55.476695+0000","last_fullsized":"2023-10-09T05:48:55.476695+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.809076+0000","last_change":"2023-10-09T05:48:54.486629+0000","last_active":"2023-10-09T05:48:55.809076+0000","last_peered":"2023-10-09T05:48:55.809076+0000","last_clean":"2023-10-09T05:48:55.809076+0000","last_became_active":"2023-10-09T05:48:54.486107+0000","last_became_peered":"2023-10-09T05:48:54.486107+0000","last_unstale":"2023-10-09T05:48:55.809076+0000","last_undegraded":"2023-10-09T05:48:55.809076+0000","last_fullsized":"2023-10-09T05:48:55.809076+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:58.679726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":327,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477955+0000","last_change":"2023-10-09T05:48:54.491950+0000","last_active":"2023-10-09T05:48:55.477955+0000","last_peered":"2023-10-09T05:48:55.477955+0000","last_clean":"2023-10-09T05:48:55.477955+0000","last_became_active":"2023-10-09T05:48:54.491559+0000","last_became_peered":"2023-10-09T05:48:54.491559+0000","last_unstale":"2023-10-09T05:48:55.477955+0000","last_undegraded":"2023-10-09T05:48:55.477955+0000","last_fullsized":"2023-10-09T05:48:55.477955+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":243,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.478777+0000","last_change":"2023-10-09T05:48:25.068328+0000","last_active":"2023-10-09T05:48:55.478777+0000","last_peered":"2023-10-09T05:48:55.478777+0000","last_clean":"2023-10-09T05:48:55.478777+0000","last_became_active":"2023-10-09T05:48:25.068094+0000","last_became_peered":"2023-10-09T05:48:25.068094+0000","last_unstale":"2023-10-09T05:48:55.478777+0000","last_undegraded":"2023-10-09T05:48:55.478777+0000","last_fullsized":"2023-10-09T05:48:55.478777+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":93,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477580+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:48:55.477580+0000","last_peered":"2023-10-09T05:48:55.477580+0000","last_clean":"2023-10-09T05:48:55.477580+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:48:55.477580+0000","last_undegraded":"2023-10-09T05:48:55.477580+0000","last_fullsized":"2023-10-09T05:48:55.477580+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.931321+0000","last_change":"2023-10-09T05:48:54.490345+0000","last_active":"2023-10-09T05:48:55.931321+0000","last_peered":"2023-10-09T05:48:55.931321+0000","last_clean":"2023-10-09T05:48:55.931321+0000","last_became_active":"2023-10-09T05:48:54.490000+0000","last_became_peered":"2023-10-09T05:48:54.490000+0000","last_unstale":"2023-10-09T05:48:55.931321+0000","last_undegraded":"2023-10-09T05:48:55.931321+0000","last_fullsized":"2023-10-09T05:48:55.931321+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:02.457608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":334,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.483188+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:48:55.483188+0000","last_peered":"2023-10-09T05:48:55.483188+0000","last_clean":"2023-10-09T05:48:55.483188+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:48:55.483188+0000","last_undegraded":"2023-10-09T05:48:55.483188+0000","last_fullsized":"2023-10-09T05:48:55.483188+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":291,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.514280+0000","last_change":"2023-10-09T05:48:54.492082+0000","last_active":"2023-10-09T05:48:55.514280+0000","last_peered":"2023-10-09T05:48:55.514280+0000","last_clean":"2023-10-09T05:48:55.514280+0000","last_became_active":"2023-10-09T05:48:54.491753+0000","last_became_peered":"2023-10-09T05:48:54.491753+0000","last_unstale":"2023-10-09T05:48:55.514280+0000","last_undegraded":"2023-10-09T05:48:55.514280+0000","last_fullsized":"2023-10-09T05:48:55.514280+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"308'5729","reported_seq":5771,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:05.224902+0000","last_change":"2023-10-09T05:48:46.314063+0000","last_active":"2023-10-09T05:49:05.224902+0000","last_peered":"2023-10-09T05:49:05.224902+0000","last_clean":"2023-10-09T05:49:05.224902+0000","last_became_active":"2023-10-09T05:48:46.261937+0000","last_became_peered":"2023-10-09T05:48:46.261937+0000","last_unstale":"2023-10-09T05:49:05.224902+0000","last_undegraded":"2023-10-09T05:49:05.224902+0000","last_fullsized":"2023-10-09T05:49:05.224902+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199688192,"num_objects":3047,"num_object_clones":0,"num_object_copies":9141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3047,"num_write_kb":195008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":93,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477414+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:48:55.477414+0000","last_peered":"2023-10-09T05:48:55.477414+0000","last_clean":"2023-10-09T05:48:55.477414+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:48:55.477414+0000","last_undegraded":"2023-10-09T05:48:55.477414+0000","last_fullsized":"2023-10-09T05:48:55.477414+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.607869+0000","last_change":"2023-10-09T05:48:54.492572+0000","last_active":"2023-10-09T05:48:55.607869+0000","last_peered":"2023-10-09T05:48:55.607869+0000","last_clean":"2023-10-09T05:48:55.607869+0000","last_became_active":"2023-10-09T05:48:54.491827+0000","last_became_peered":"2023-10-09T05:48:54.491827+0000","last_unstale":"2023-10-09T05:48:55.607869+0000","last_undegraded":"2023-10-09T05:48:55.607869+0000","last_fullsized":"2023-10-09T05:48:55.607869+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:33.185508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.19","version":"308'5822","reported_seq":5864,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:03.709039+0000","last_change":"2023-10-09T05:48:46.264369+0000","last_active":"2023-10-09T05:49:03.709039+0000","last_peered":"2023-10-09T05:49:03.709039+0000","last_clean":"2023-10-09T05:49:03.709039+0000","last_became_active":"2023-10-09T05:48:46.259158+0000","last_became_peered":"2023-10-09T05:48:46.259158+0000","last_unstale":"2023-10-09T05:49:03.709039+0000","last_undegraded":"2023-10-09T05:49:03.709039+0000","last_fullsized":"2023-10-09T05:49:03.709039+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5822,"log_dups_size":0,"ondisk_log_size":5822,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201457664,"num_objects":3074,"num_object_clones":0,"num_object_copies":9222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3074,"num_write_kb":196736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":296,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477045+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:48:55.477045+0000","last_peered":"2023-10-09T05:48:55.477045+0000","last_clean":"2023-10-09T05:48:55.477045+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:48:55.477045+0000","last_undegraded":"2023-10-09T05:48:55.477045+0000","last_fullsized":"2023-10-09T05:48:55.477045+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":266,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477308+0000","last_change":"2023-10-09T05:46:56.019914+0000","last_active":"2023-10-09T05:48:55.477308+0000","last_peered":"2023-10-09T05:48:55.477308+0000","last_clean":"2023-10-09T05:48:55.477308+0000","last_became_active":"2023-10-09T05:46:56.019487+0000","last_became_peered":"2023-10-09T05:46:56.019487+0000","last_unstale":"2023-10-09T05:48:55.477308+0000","last_undegraded":"2023-10-09T05:48:55.477308+0000","last_fullsized":"2023-10-09T05:48:55.477308+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":87,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.478832+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:48:55.478832+0000","last_peered":"2023-10-09T05:48:55.478832+0000","last_clean":"2023-10-09T05:48:55.478832+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:48:55.478832+0000","last_undegraded":"2023-10-09T05:48:55.478832+0000","last_fullsized":"2023-10-09T05:48:55.478832+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":295,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476518+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:48:55.476518+0000","last_peered":"2023-10-09T05:48:55.476518+0000","last_clean":"2023-10-09T05:48:55.476518+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:48:55.476518+0000","last_undegraded":"2023-10-09T05:48:55.476518+0000","last_fullsized":"2023-10-09T05:48:55.476518+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":102,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476053+0000","last_change":"2023-10-09T05:46:56.194588+0000","last_active":"2023-10-09T05:48:55.476053+0000","last_peered":"2023-10-09T05:48:55.476053+0000","last_clean":"2023-10-09T05:48:55.476053+0000","last_became_active":"2023-10-09T05:46:56.018480+0000","last_became_peered":"2023-10-09T05:46:56.018480+0000","last_unstale":"2023-10-09T05:48:55.476053+0000","last_undegraded":"2023-10-09T05:48:55.476053+0000","last_fullsized":"2023-10-09T05:48:55.476053+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:39.150417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":313,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477248+0000","last_change":"2023-10-09T05:48:54.492012+0000","last_active":"2023-10-09T05:48:55.477248+0000","last_peered":"2023-10-09T05:48:55.477248+0000","last_clean":"2023-10-09T05:48:55.477248+0000","last_became_active":"2023-10-09T05:48:54.491644+0000","last_became_peered":"2023-10-09T05:48:54.491644+0000","last_unstale":"2023-10-09T05:48:55.477248+0000","last_undegraded":"2023-10-09T05:48:55.477248+0000","last_fullsized":"2023-10-09T05:48:55.477248+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"308'5829","reported_seq":5868,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:05.059828+0000","last_change":"2023-10-09T05:48:46.259028+0000","last_active":"2023-10-09T05:49:05.059828+0000","last_peered":"2023-10-09T05:49:05.059828+0000","last_clean":"2023-10-09T05:49:05.059828+0000","last_became_active":"2023-10-09T05:48:46.256441+0000","last_became_peered":"2023-10-09T05:48:46.256441+0000","last_unstale":"2023-10-09T05:49:05.059828+0000","last_undegraded":"2023-10-09T05:49:05.059828+0000","last_fullsized":"2023-10-09T05:49:05.059828+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5829,"log_dups_size":0,"ondisk_log_size":5829,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203259904,"num_objects":3101,"num_object_clones":0,"num_object_copies":9303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3101,"num_write_kb":198496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":91,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477198+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:48:55.477198+0000","last_peered":"2023-10-09T05:48:55.477198+0000","last_clean":"2023-10-09T05:48:55.477198+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:48:55.477198+0000","last_undegraded":"2023-10-09T05:48:55.477198+0000","last_fullsized":"2023-10-09T05:48:55.477198+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.478076+0000","last_change":"2023-10-09T05:46:56.021669+0000","last_active":"2023-10-09T05:48:55.478076+0000","last_peered":"2023-10-09T05:48:55.478076+0000","last_clean":"2023-10-09T05:48:55.478076+0000","last_became_active":"2023-10-09T05:46:56.021306+0000","last_became_peered":"2023-10-09T05:46:56.021306+0000","last_unstale":"2023-10-09T05:48:55.478076+0000","last_undegraded":"2023-10-09T05:48:55.478076+0000","last_fullsized":"2023-10-09T05:48:55.478076+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"308'5765","reported_seq":5802,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:05.193341+0000","last_change":"2023-10-09T05:48:46.258066+0000","last_active":"2023-10-09T05:49:05.193341+0000","last_peered":"2023-10-09T05:49:05.193341+0000","last_clean":"2023-10-09T05:49:05.193341+0000","last_became_active":"2023-10-09T05:48:46.251759+0000","last_became_peered":"2023-10-09T05:48:46.251759+0000","last_unstale":"2023-10-09T05:49:05.193341+0000","last_undegraded":"2023-10-09T05:49:05.193341+0000","last_fullsized":"2023-10-09T05:49:05.193341+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5765,"log_dups_size":0,"ondisk_log_size":5765,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201555968,"num_objects":3075,"num_object_clones":0,"num_object_copies":9225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3075,"num_write_kb":196832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":117,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.478045+0000","last_change":"2023-10-09T05:48:54.539425+0000","last_active":"2023-10-09T05:48:55.478045+0000","last_peered":"2023-10-09T05:48:55.478045+0000","last_clean":"2023-10-09T05:48:55.478045+0000","last_became_active":"2023-10-09T05:48:54.486401+0000","last_became_peered":"2023-10-09T05:48:54.486401+0000","last_unstale":"2023-10-09T05:48:55.478045+0000","last_undegraded":"2023-10-09T05:48:55.478045+0000","last_fullsized":"2023-10-09T05:48:55.478045+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":324,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.482692+0000","last_change":"2023-10-09T05:48:54.492534+0000","last_active":"2023-10-09T05:48:55.482692+0000","last_peered":"2023-10-09T05:48:55.482692+0000","last_clean":"2023-10-09T05:48:55.482692+0000","last_became_active":"2023-10-09T05:48:54.492172+0000","last_became_peered":"2023-10-09T05:48:54.492172+0000","last_unstale":"2023-10-09T05:48:55.482692+0000","last_undegraded":"2023-10-09T05:48:55.482692+0000","last_fullsized":"2023-10-09T05:48:55.482692+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"308'5678","reported_seq":5720,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:03.767929+0000","last_change":"2023-10-09T05:48:46.220278+0000","last_active":"2023-10-09T05:49:03.767929+0000","last_peered":"2023-10-09T05:49:03.767929+0000","last_clean":"2023-10-09T05:49:03.767929+0000","last_became_active":"2023-10-09T05:48:46.215626+0000","last_became_peered":"2023-10-09T05:48:46.215626+0000","last_unstale":"2023-10-09T05:49:03.767929+0000","last_undegraded":"2023-10-09T05:49:03.767929+0000","last_fullsized":"2023-10-09T05:49:03.767929+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5678,"log_dups_size":0,"ondisk_log_size":5678,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195985408,"num_objects":2990,"num_object_clones":0,"num_object_copies":8970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2990,"num_write_kb":191392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":104,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477423+0000","last_change":"2023-10-09T05:46:56.190043+0000","last_active":"2023-10-09T05:48:55.477423+0000","last_peered":"2023-10-09T05:48:55.477423+0000","last_clean":"2023-10-09T05:48:55.477423+0000","last_became_active":"2023-10-09T05:46:56.019072+0000","last_became_peered":"2023-10-09T05:46:56.019072+0000","last_unstale":"2023-10-09T05:48:55.477423+0000","last_undegraded":"2023-10-09T05:48:55.477423+0000","last_fullsized":"2023-10-09T05:48:55.477423+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.938953+0000","last_change":"2023-10-09T05:48:54.489089+0000","last_active":"2023-10-09T05:48:55.938953+0000","last_peered":"2023-10-09T05:48:55.938953+0000","last_clean":"2023-10-09T05:48:55.938953+0000","last_became_active":"2023-10-09T05:48:54.488817+0000","last_became_peered":"2023-10-09T05:48:54.488817+0000","last_unstale":"2023-10-09T05:48:55.938953+0000","last_undegraded":"2023-10-09T05:48:55.938953+0000","last_fullsized":"2023-10-09T05:48:55.938953+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:40.555711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"308'5823","reported_seq":5875,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:02.022447+0000","last_change":"2023-10-09T05:48:56.022926+0000","last_active":"2023-10-09T05:49:02.022447+0000","last_peered":"2023-10-09T05:49:02.022447+0000","last_clean":"2023-10-09T05:48:53.410221+0000","last_became_active":"2023-10-09T05:48:56.013961+0000","last_became_peered":"2023-10-09T05:48:56.013961+0000","last_unstale":"2023-10-09T05:49:02.022447+0000","last_undegraded":"2023-10-09T05:48:55.541735+0000","last_fullsized":"2023-10-09T05:48:55.521053+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5823,"log_dups_size":0,"ondisk_log_size":5823,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199163904,"num_objects":3039,"num_object_clones":0,"num_object_copies":9117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2925,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3039,"num_write_kb":194496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2925},{"shards":"3,6","objects":114}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":275,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476238+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:48:55.476238+0000","last_peered":"2023-10-09T05:48:55.476238+0000","last_clean":"2023-10-09T05:48:55.476238+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:48:55.476238+0000","last_undegraded":"2023-10-09T05:48:55.476238+0000","last_fullsized":"2023-10-09T05:48:55.476238+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":281,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476385+0000","last_change":"2023-10-09T05:48:54.495800+0000","last_active":"2023-10-09T05:48:55.476385+0000","last_peered":"2023-10-09T05:48:55.476385+0000","last_clean":"2023-10-09T05:48:55.476385+0000","last_became_active":"2023-10-09T05:48:54.495540+0000","last_became_peered":"2023-10-09T05:48:54.495540+0000","last_unstale":"2023-10-09T05:48:55.476385+0000","last_undegraded":"2023-10-09T05:48:55.476385+0000","last_fullsized":"2023-10-09T05:48:55.476385+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:26.903669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"308'5784","reported_seq":5838,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:02.060794+0000","last_change":"2023-10-09T05:48:55.832264+0000","last_active":"2023-10-09T05:49:02.060794+0000","last_peered":"2023-10-09T05:49:02.060794+0000","last_clean":"2023-10-09T05:48:53.429181+0000","last_became_active":"2023-10-09T05:48:55.826708+0000","last_became_peered":"2023-10-09T05:48:55.826708+0000","last_unstale":"2023-10-09T05:49:02.060794+0000","last_undegraded":"2023-10-09T05:48:55.501215+0000","last_fullsized":"2023-10-09T05:48:55.480116+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5784,"log_dups_size":0,"ondisk_log_size":5784,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199688192,"num_objects":3047,"num_object_clones":0,"num_object_copies":9141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2892,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3047,"num_write_kb":195008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4","objects":155},{"shards":"3,4,6","objects":2892}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":44,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476486+0000","last_change":"2023-10-09T05:48:25.067945+0000","last_active":"2023-10-09T05:48:55.476486+0000","last_peered":"2023-10-09T05:48:55.476486+0000","last_clean":"2023-10-09T05:48:55.476486+0000","last_became_active":"2023-10-09T05:48:25.067721+0000","last_became_peered":"2023-10-09T05:48:25.067721+0000","last_unstale":"2023-10-09T05:48:55.476486+0000","last_undegraded":"2023-10-09T05:48:55.476486+0000","last_fullsized":"2023-10-09T05:48:55.476486+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:58.655544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":273,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.483096+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:48:55.483096+0000","last_peered":"2023-10-09T05:48:55.483096+0000","last_clean":"2023-10-09T05:48:55.483096+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:48:55.483096+0000","last_undegraded":"2023-10-09T05:48:55.483096+0000","last_fullsized":"2023-10-09T05:48:55.483096+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"308'5724","reported_seq":5771,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:03.804669+0000","last_change":"2023-10-09T05:48:55.854752+0000","last_active":"2023-10-09T05:49:03.804669+0000","last_peered":"2023-10-09T05:49:03.804669+0000","last_clean":"2023-10-09T05:48:53.421353+0000","last_became_active":"2023-10-09T05:48:55.847882+0000","last_became_peered":"2023-10-09T05:48:55.847882+0000","last_unstale":"2023-10-09T05:49:03.804669+0000","last_undegraded":"2023-10-09T05:48:55.499754+0000","last_fullsized":"2023-10-09T05:48:55.476049+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5724,"log_dups_size":0,"ondisk_log_size":5724,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198246400,"num_objects":3025,"num_object_clones":0,"num_object_copies":9075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2913,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3025,"num_write_kb":193600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":112},{"shards":"0,2,5","objects":2913}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":142,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.482934+0000","last_change":"2023-10-09T05:48:54.494715+0000","last_active":"2023-10-09T05:48:55.482934+0000","last_peered":"2023-10-09T05:48:55.482934+0000","last_clean":"2023-10-09T05:48:55.482934+0000","last_became_active":"2023-10-09T05:48:54.493988+0000","last_became_peered":"2023-10-09T05:48:54.493988+0000","last_unstale":"2023-10-09T05:48:55.482934+0000","last_undegraded":"2023-10-09T05:48:55.482934+0000","last_fullsized":"2023-10-09T05:48:55.482934+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":306,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.482975+0000","last_change":"2023-10-09T05:48:54.493193+0000","last_active":"2023-10-09T05:48:55.482975+0000","last_peered":"2023-10-09T05:48:55.482975+0000","last_clean":"2023-10-09T05:48:55.482975+0000","last_became_active":"2023-10-09T05:48:54.492886+0000","last_became_peered":"2023-10-09T05:48:54.492886+0000","last_unstale":"2023-10-09T05:48:55.482975+0000","last_undegraded":"2023-10-09T05:48:55.482975+0000","last_fullsized":"2023-10-09T05:48:55.482975+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"308'5696","reported_seq":5737,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:05.226929+0000","last_change":"2023-10-09T05:48:46.222212+0000","last_active":"2023-10-09T05:49:05.226929+0000","last_peered":"2023-10-09T05:49:05.226929+0000","last_clean":"2023-10-09T05:49:05.226929+0000","last_became_active":"2023-10-09T05:48:46.219241+0000","last_became_peered":"2023-10-09T05:48:46.219241+0000","last_unstale":"2023-10-09T05:49:05.226929+0000","last_undegraded":"2023-10-09T05:49:05.226929+0000","last_fullsized":"2023-10-09T05:49:05.226929+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5696,"log_dups_size":0,"ondisk_log_size":5696,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198475776,"num_objects":3028,"num_object_clones":0,"num_object_copies":9084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3028,"num_write_kb":193824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":307,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.482584+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:48:55.482584+0000","last_peered":"2023-10-09T05:48:55.482584+0000","last_clean":"2023-10-09T05:48:55.482584+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:48:55.482584+0000","last_undegraded":"2023-10-09T05:48:55.482584+0000","last_fullsized":"2023-10-09T05:48:55.482584+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":300,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.483056+0000","last_change":"2023-10-09T05:48:54.492708+0000","last_active":"2023-10-09T05:48:55.483056+0000","last_peered":"2023-10-09T05:48:55.483056+0000","last_clean":"2023-10-09T05:48:55.483056+0000","last_became_active":"2023-10-09T05:48:54.492382+0000","last_became_peered":"2023-10-09T05:48:54.492382+0000","last_unstale":"2023-10-09T05:48:55.483056+0000","last_undegraded":"2023-10-09T05:48:55.483056+0000","last_fullsized":"2023-10-09T05:48:55.483056+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"308'5681","reported_seq":5727,"reported_epoch":308,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T05:49:02.216479+0000","last_change":"2023-10-09T05:48:55.981040+0000","last_active":"2023-10-09T05:49:02.216479+0000","last_peered":"2023-10-09T05:49:02.216479+0000","last_clean":"2023-10-09T05:48:53.427903+0000","last_became_active":"2023-10-09T05:48:55.975010+0000","last_became_peered":"2023-10-09T05:48:55.975010+0000","last_unstale":"2023-10-09T05:49:02.216479+0000","last_undegraded":"2023-10-09T05:49:02.216479+0000","last_fullsized":"2023-10-09T05:49:02.216479+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5681,"log_dups_size":0,"ondisk_log_size":5681,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195952640,"num_objects":2990,"num_object_clones":0,"num_object_copies":8970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2881,"num_objects_unfound":0,"num_objects_dirty":2990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2990,"num_write_kb":191360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[2,5,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":109}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.809332+0000","last_change":"2023-10-09T05:48:54.489891+0000","last_active":"2023-10-09T05:48:55.809332+0000","last_peered":"2023-10-09T05:48:55.809332+0000","last_clean":"2023-10-09T05:48:55.809332+0000","last_became_active":"2023-10-09T05:48:54.489570+0000","last_became_peered":"2023-10-09T05:48:54.489570+0000","last_unstale":"2023-10-09T05:48:55.809332+0000","last_undegraded":"2023-10-09T05:48:55.809332+0000","last_fullsized":"2023-10-09T05:48:55.809332+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:41.194371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.926194+0000","last_change":"2023-10-09T05:48:54.488131+0000","last_active":"2023-10-09T05:48:55.926194+0000","last_peered":"2023-10-09T05:48:55.926194+0000","last_clean":"2023-10-09T05:48:55.926194+0000","last_became_active":"2023-10-09T05:48:54.487748+0000","last_became_peered":"2023-10-09T05:48:54.487748+0000","last_unstale":"2023-10-09T05:48:55.926194+0000","last_undegraded":"2023-10-09T05:48:55.926194+0000","last_fullsized":"2023-10-09T05:48:55.926194+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:05.047252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"308'6110","reported_seq":6142,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:03.819397+0000","last_change":"2023-10-09T05:47:33.927196+0000","last_active":"2023-10-09T05:49:03.819397+0000","last_peered":"2023-10-09T05:49:03.819397+0000","last_clean":"2023-10-09T05:49:03.819397+0000","last_became_active":"2023-10-09T05:47:33.926965+0000","last_became_peered":"2023-10-09T05:47:33.926965+0000","last_unstale":"2023-10-09T05:49:03.819397+0000","last_undegraded":"2023-10-09T05:49:03.819397+0000","last_fullsized":"2023-10-09T05:49:03.819397+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6110,"log_dups_size":0,"ondisk_log_size":6110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:17.414280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400424960,"num_objects":6110,"num_object_clones":0,"num_object_copies":18330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6110,"num_write_kb":391040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":674,"reported_epoch":307,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:48:55.609226+0000","last_change":"2023-10-09T05:48:55.609226+0000","last_active":"2023-10-09T05:48:55.609226+0000","last_peered":"2023-10-09T05:48:55.609226+0000","last_clean":"2023-10-09T05:48:52.714079+0000","last_became_active":"2023-10-09T05:48:55.608475+0000","last_became_peered":"2023-10-09T05:48:55.608475+0000","last_unstale":"2023-10-09T05:48:55.609226+0000","last_undegraded":"2023-10-09T05:48:55.476148+0000","last_fullsized":"2023-10-09T05:48:55.475626+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:14.257300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":290,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476177+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:48:55.476177+0000","last_peered":"2023-10-09T05:48:55.476177+0000","last_clean":"2023-10-09T05:48:55.476177+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:48:55.476177+0000","last_undegraded":"2023-10-09T05:48:55.476177+0000","last_fullsized":"2023-10-09T05:48:55.476177+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.938864+0000","last_change":"2023-10-09T05:48:54.489994+0000","last_active":"2023-10-09T05:48:55.938864+0000","last_peered":"2023-10-09T05:48:55.938864+0000","last_clean":"2023-10-09T05:48:55.938864+0000","last_became_active":"2023-10-09T05:48:54.489701+0000","last_became_peered":"2023-10-09T05:48:54.489701+0000","last_unstale":"2023-10-09T05:48:55.938864+0000","last_undegraded":"2023-10-09T05:48:55.938864+0000","last_fullsized":"2023-10-09T05:48:55.938864+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:10.442883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"308'5709","reported_seq":5748,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:05.185621+0000","last_change":"2023-10-09T05:48:46.259058+0000","last_active":"2023-10-09T05:49:05.185621+0000","last_peered":"2023-10-09T05:49:05.185621+0000","last_clean":"2023-10-09T05:49:05.185621+0000","last_became_active":"2023-10-09T05:48:46.247029+0000","last_became_peered":"2023-10-09T05:48:46.247029+0000","last_unstale":"2023-10-09T05:49:05.185621+0000","last_undegraded":"2023-10-09T05:49:05.185621+0000","last_fullsized":"2023-10-09T05:49:05.185621+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198377472,"num_objects":3027,"num_object_clones":0,"num_object_copies":9081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3027,"num_write_kb":193728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":798,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476164+0000","last_change":"2023-10-09T05:48:54.592642+0000","last_active":"2023-10-09T05:48:55.476164+0000","last_peered":"2023-10-09T05:48:55.476164+0000","last_clean":"2023-10-09T05:48:55.476164+0000","last_became_active":"2023-10-09T05:48:54.491917+0000","last_became_peered":"2023-10-09T05:48:54.491917+0000","last_unstale":"2023-10-09T05:48:55.476164+0000","last_undegraded":"2023-10-09T05:48:55.476164+0000","last_fullsized":"2023-10-09T05:48:55.476164+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":274,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.514338+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:48:55.514338+0000","last_peered":"2023-10-09T05:48:55.514338+0000","last_clean":"2023-10-09T05:48:55.514338+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:48:55.514338+0000","last_undegraded":"2023-10-09T05:48:55.514338+0000","last_fullsized":"2023-10-09T05:48:55.514338+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.607619+0000","last_change":"2023-10-09T05:48:54.492183+0000","last_active":"2023-10-09T05:48:55.607619+0000","last_peered":"2023-10-09T05:48:55.607619+0000","last_clean":"2023-10-09T05:48:55.607619+0000","last_became_active":"2023-10-09T05:48:54.491382+0000","last_became_peered":"2023-10-09T05:48:54.491382+0000","last_unstale":"2023-10-09T05:48:55.607619+0000","last_undegraded":"2023-10-09T05:48:55.607619+0000","last_fullsized":"2023-10-09T05:48:55.607619+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:06.256308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"308'6217","reported_seq":6264,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:05.205702+0000","last_change":"2023-10-09T05:48:56.003601+0000","last_active":"2023-10-09T05:49:05.205702+0000","last_peered":"2023-10-09T05:49:05.205702+0000","last_clean":"2023-10-09T05:48:53.462109+0000","last_became_active":"2023-10-09T05:48:55.976639+0000","last_became_peered":"2023-10-09T05:48:55.976639+0000","last_unstale":"2023-10-09T05:49:05.205702+0000","last_undegraded":"2023-10-09T05:48:55.512858+0000","last_fullsized":"2023-10-09T05:48:55.482514+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6217,"log_dups_size":0,"ondisk_log_size":6217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407437312,"num_objects":6217,"num_object_clones":0,"num_object_copies":18651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5835,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6217,"num_write_kb":397888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":5835},{"shards":"3,4","objects":382}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":626,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.472758+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:48:55.472758+0000","last_peered":"2023-10-09T05:48:55.472758+0000","last_clean":"2023-10-09T05:48:55.472758+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:48:55.472758+0000","last_undegraded":"2023-10-09T05:48:55.472758+0000","last_fullsized":"2023-10-09T05:48:55.472758+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.809192+0000","last_change":"2023-10-09T05:48:54.490716+0000","last_active":"2023-10-09T05:48:55.809192+0000","last_peered":"2023-10-09T05:48:55.809192+0000","last_clean":"2023-10-09T05:48:55.809192+0000","last_became_active":"2023-10-09T05:48:54.490350+0000","last_became_peered":"2023-10-09T05:48:54.490350+0000","last_unstale":"2023-10-09T05:48:55.809192+0000","last_undegraded":"2023-10-09T05:48:55.809192+0000","last_fullsized":"2023-10-09T05:48:55.809192+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:20.190860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.926110+0000","last_change":"2023-10-09T05:48:54.492185+0000","last_active":"2023-10-09T05:48:55.926110+0000","last_peered":"2023-10-09T05:48:55.926110+0000","last_clean":"2023-10-09T05:48:55.926110+0000","last_became_active":"2023-10-09T05:48:54.491885+0000","last_became_peered":"2023-10-09T05:48:54.491885+0000","last_unstale":"2023-10-09T05:48:55.926110+0000","last_undegraded":"2023-10-09T05:48:55.926110+0000","last_fullsized":"2023-10-09T05:48:55.926110+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:26.045180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"308'5824","reported_seq":5866,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:03.825472+0000","last_change":"2023-10-09T05:48:46.220263+0000","last_active":"2023-10-09T05:49:03.825472+0000","last_peered":"2023-10-09T05:49:03.825472+0000","last_clean":"2023-10-09T05:49:03.825472+0000","last_became_active":"2023-10-09T05:48:46.215666+0000","last_became_peered":"2023-10-09T05:48:46.215666+0000","last_unstale":"2023-10-09T05:49:03.825472+0000","last_undegraded":"2023-10-09T05:49:03.825472+0000","last_fullsized":"2023-10-09T05:49:03.825472+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201588736,"num_objects":3076,"num_object_clones":0,"num_object_copies":9228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3076,"num_write_kb":196864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":954,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.488247+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:48:55.488247+0000","last_peered":"2023-10-09T05:48:55.488247+0000","last_clean":"2023-10-09T05:48:55.488247+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:48:55.488247+0000","last_undegraded":"2023-10-09T05:48:55.488247+0000","last_fullsized":"2023-10-09T05:48:55.488247+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":458,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477082+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:48:55.477082+0000","last_peered":"2023-10-09T05:48:55.477082+0000","last_clean":"2023-10-09T05:48:55.477082+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:48:55.477082+0000","last_undegraded":"2023-10-09T05:48:55.477082+0000","last_fullsized":"2023-10-09T05:48:55.477082+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":80,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476085+0000","last_change":"2023-10-09T05:48:25.068759+0000","last_active":"2023-10-09T05:48:55.476085+0000","last_peered":"2023-10-09T05:48:55.476085+0000","last_clean":"2023-10-09T05:48:55.476085+0000","last_became_active":"2023-10-09T05:48:25.068358+0000","last_became_peered":"2023-10-09T05:48:25.068358+0000","last_unstale":"2023-10-09T05:48:55.476085+0000","last_undegraded":"2023-10-09T05:48:55.476085+0000","last_fullsized":"2023-10-09T05:48:55.476085+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":275,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.483272+0000","last_change":"2023-10-09T05:44:50.495966+0000","last_active":"2023-10-09T05:48:55.483272+0000","last_peered":"2023-10-09T05:48:55.483272+0000","last_clean":"2023-10-09T05:48:55.483272+0000","last_became_active":"2023-10-09T05:44:50.495628+0000","last_became_peered":"2023-10-09T05:44:50.495628+0000","last_unstale":"2023-10-09T05:48:55.483272+0000","last_undegraded":"2023-10-09T05:48:55.483272+0000","last_fullsized":"2023-10-09T05:48:55.483272+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"308'5992","reported_seq":6029,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:02.164283+0000","last_change":"2023-10-09T05:48:55.974254+0000","last_active":"2023-10-09T05:49:02.164283+0000","last_peered":"2023-10-09T05:49:02.164283+0000","last_clean":"2023-10-09T05:48:53.473854+0000","last_became_active":"2023-10-09T05:48:55.957470+0000","last_became_peered":"2023-10-09T05:48:55.957470+0000","last_unstale":"2023-10-09T05:49:02.164283+0000","last_undegraded":"2023-10-09T05:48:55.523025+0000","last_fullsized":"2023-10-09T05:48:55.472698+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5992,"log_dups_size":0,"ondisk_log_size":5992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:23.468317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392691712,"num_objects":5992,"num_object_clones":0,"num_object_copies":17976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5725,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5992,"num_write_kb":383488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[2,3],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3","objects":267},{"shards":"2,3,4","objects":5725}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":799,"reported_epoch":307,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:48:55.810113+0000","last_change":"2023-10-09T05:48:55.810113+0000","last_active":"2023-10-09T05:48:55.810113+0000","last_peered":"2023-10-09T05:48:55.810113+0000","last_clean":"2023-10-09T05:48:52.442052+0000","last_became_active":"2023-10-09T05:48:55.809751+0000","last_became_peered":"2023-10-09T05:48:55.809751+0000","last_unstale":"2023-10-09T05:48:55.810113+0000","last_undegraded":"2023-10-09T05:48:55.477697+0000","last_fullsized":"2023-10-09T05:48:55.476822+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,0],"avail_no_missing":["7","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":282,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.501378+0000","last_change":"2023-10-09T05:47:33.926192+0000","last_active":"2023-10-09T05:48:55.501378+0000","last_peered":"2023-10-09T05:48:55.501378+0000","last_clean":"2023-10-09T05:48:55.501378+0000","last_became_active":"2023-10-09T05:47:33.925588+0000","last_became_peered":"2023-10-09T05:47:33.925588+0000","last_unstale":"2023-10-09T05:48:55.501378+0000","last_undegraded":"2023-10-09T05:48:55.501378+0000","last_fullsized":"2023-10-09T05:48:55.501378+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:29.495701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":296,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476279+0000","last_change":"2023-10-09T05:48:54.489115+0000","last_active":"2023-10-09T05:48:55.476279+0000","last_peered":"2023-10-09T05:48:55.476279+0000","last_clean":"2023-10-09T05:48:55.476279+0000","last_became_active":"2023-10-09T05:48:54.488783+0000","last_became_peered":"2023-10-09T05:48:54.488783+0000","last_unstale":"2023-10-09T05:48:55.476279+0000","last_undegraded":"2023-10-09T05:48:55.476279+0000","last_fullsized":"2023-10-09T05:48:55.476279+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:13.193778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"308'5884","reported_seq":5928,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:02.048955+0000","last_change":"2023-10-09T05:48:56.022929+0000","last_active":"2023-10-09T05:49:02.048955+0000","last_peered":"2023-10-09T05:49:02.048955+0000","last_clean":"2023-10-09T05:48:53.467734+0000","last_became_active":"2023-10-09T05:48:56.009202+0000","last_became_peered":"2023-10-09T05:48:56.009202+0000","last_unstale":"2023-10-09T05:49:02.048955+0000","last_undegraded":"2023-10-09T05:48:55.537860+0000","last_fullsized":"2023-10-09T05:48:55.479952+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5884,"log_dups_size":0,"ondisk_log_size":5884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:31.901862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385613824,"num_objects":5884,"num_object_clones":0,"num_object_copies":17652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5637,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5884,"num_write_kb":376576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4","objects":247},{"shards":"3,4,6","objects":5637}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":491,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477664+0000","last_change":"2023-10-09T05:44:51.489799+0000","last_active":"2023-10-09T05:48:55.477664+0000","last_peered":"2023-10-09T05:48:55.477664+0000","last_clean":"2023-10-09T05:48:55.477664+0000","last_became_active":"2023-10-09T05:44:51.489301+0000","last_became_peered":"2023-10-09T05:44:51.489301+0000","last_unstale":"2023-10-09T05:48:55.477664+0000","last_undegraded":"2023-10-09T05:48:55.477664+0000","last_fullsized":"2023-10-09T05:48:55.477664+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.607706+0000","last_change":"2023-10-09T05:48:54.491374+0000","last_active":"2023-10-09T05:48:55.607706+0000","last_peered":"2023-10-09T05:48:55.607706+0000","last_clean":"2023-10-09T05:48:55.607706+0000","last_became_active":"2023-10-09T05:48:54.490889+0000","last_became_peered":"2023-10-09T05:48:54.490889+0000","last_unstale":"2023-10-09T05:48:55.607706+0000","last_undegraded":"2023-10-09T05:48:55.607706+0000","last_fullsized":"2023-10-09T05:48:55.607706+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:34.755791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":282,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476660+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:48:55.476660+0000","last_peered":"2023-10-09T05:48:55.476660+0000","last_clean":"2023-10-09T05:48:55.476660+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:48:55.476660+0000","last_undegraded":"2023-10-09T05:48:55.476660+0000","last_fullsized":"2023-10-09T05:48:55.476660+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"308'6216","reported_seq":6248,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:03.853190+0000","last_change":"2023-10-09T05:47:33.923368+0000","last_active":"2023-10-09T05:49:03.853190+0000","last_peered":"2023-10-09T05:49:03.853190+0000","last_clean":"2023-10-09T05:49:03.853190+0000","last_became_active":"2023-10-09T05:47:33.923068+0000","last_became_peered":"2023-10-09T05:47:33.923068+0000","last_unstale":"2023-10-09T05:49:03.853190+0000","last_undegraded":"2023-10-09T05:49:03.853190+0000","last_fullsized":"2023-10-09T05:49:03.853190+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6216,"log_dups_size":0,"ondisk_log_size":6216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:48.468565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407371776,"num_objects":6216,"num_object_clones":0,"num_object_copies":18648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6216,"num_write_kb":397824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":255,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477330+0000","last_change":"2023-10-09T05:48:25.066163+0000","last_active":"2023-10-09T05:48:55.477330+0000","last_peered":"2023-10-09T05:48:55.477330+0000","last_clean":"2023-10-09T05:48:55.477330+0000","last_became_active":"2023-10-09T05:48:25.065145+0000","last_became_peered":"2023-10-09T05:48:25.065145+0000","last_unstale":"2023-10-09T05:48:55.477330+0000","last_undegraded":"2023-10-09T05:48:55.477330+0000","last_fullsized":"2023-10-09T05:48:55.477330+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":473,"reported_epoch":307,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T05:48:57.079265+0000","last_change":"2023-10-09T05:48:55.619562+0000","last_active":"2023-10-09T05:48:57.079265+0000","last_peered":"2023-10-09T05:48:57.079265+0000","last_clean":"2023-10-09T05:22:39.598843+0000","last_became_active":"2023-10-09T05:48:55.618332+0000","last_became_peered":"2023-10-09T05:48:55.618332+0000","last_unstale":"2023-10-09T05:48:57.079265+0000","last_undegraded":"2023-10-09T05:48:57.079265+0000","last_fullsized":"2023-10-09T05:48:57.079265+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:01.870836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[3,6,4],"avail_no_missing":["3","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":248,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477994+0000","last_change":"2023-10-09T05:48:25.067326+0000","last_active":"2023-10-09T05:48:55.477994+0000","last_peered":"2023-10-09T05:48:55.477994+0000","last_clean":"2023-10-09T05:48:55.477994+0000","last_became_active":"2023-10-09T05:48:25.066977+0000","last_became_peered":"2023-10-09T05:48:25.066977+0000","last_unstale":"2023-10-09T05:48:55.477994+0000","last_undegraded":"2023-10-09T05:48:55.477994+0000","last_fullsized":"2023-10-09T05:48:55.477994+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"308'6044","reported_seq":6075,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:05.213564+0000","last_change":"2023-10-09T05:47:33.926483+0000","last_active":"2023-10-09T05:49:05.213564+0000","last_peered":"2023-10-09T05:49:05.213564+0000","last_clean":"2023-10-09T05:49:05.213564+0000","last_became_active":"2023-10-09T05:47:33.926006+0000","last_became_peered":"2023-10-09T05:47:33.926006+0000","last_unstale":"2023-10-09T05:49:05.213564+0000","last_undegraded":"2023-10-09T05:49:05.213564+0000","last_fullsized":"2023-10-09T05:49:05.213564+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6044,"log_dups_size":0,"ondisk_log_size":6044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396099584,"num_objects":6044,"num_object_clones":0,"num_object_copies":18132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6044,"num_write_kb":386816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":805,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.477266+0000","last_change":"2023-10-09T05:44:50.482211+0000","last_active":"2023-10-09T05:48:55.477266+0000","last_peered":"2023-10-09T05:48:55.477266+0000","last_clean":"2023-10-09T05:48:55.477266+0000","last_became_active":"2023-10-09T05:44:50.481749+0000","last_became_peered":"2023-10-09T05:44:50.481749+0000","last_unstale":"2023-10-09T05:48:55.477266+0000","last_undegraded":"2023-10-09T05:48:55.477266+0000","last_fullsized":"2023-10-09T05:48:55.477266+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":292,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476435+0000","last_change":"2023-10-09T05:48:54.585691+0000","last_active":"2023-10-09T05:48:55.476435+0000","last_peered":"2023-10-09T05:48:55.476435+0000","last_clean":"2023-10-09T05:48:55.476435+0000","last_became_active":"2023-10-09T05:48:54.498259+0000","last_became_peered":"2023-10-09T05:48:54.498259+0000","last_unstale":"2023-10-09T05:48:55.476435+0000","last_undegraded":"2023-10-09T05:48:55.476435+0000","last_fullsized":"2023-10-09T05:48:55.476435+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.684612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":258,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.482600+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:48:55.482600+0000","last_peered":"2023-10-09T05:48:55.482600+0000","last_clean":"2023-10-09T05:48:55.482600+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:48:55.482600+0000","last_undegraded":"2023-10-09T05:48:55.482600+0000","last_fullsized":"2023-10-09T05:48:55.482600+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"308'5779","reported_seq":7169,"reported_epoch":308,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:49:02.086511+0000","last_change":"2023-10-09T05:48:54.551295+0000","last_active":"2023-10-09T05:49:02.086511+0000","last_peered":"2023-10-09T05:49:02.086511+0000","last_clean":"2023-10-09T05:48:50.362911+0000","last_became_active":"2023-10-09T05:48:52.776305+0000","last_became_peered":"2023-10-09T05:48:52.776305+0000","last_unstale":"2023-10-09T05:49:02.086511+0000","last_undegraded":"2023-10-09T05:49:02.086511+0000","last_fullsized":"2023-10-09T05:48:52.436423+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5779,"log_dups_size":0,"ondisk_log_size":5779,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199360512,"num_objects":3042,"num_object_clones":0,"num_object_copies":9126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4896,"num_objects_unfound":0,"num_objects_dirty":3042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3042,"num_write_kb":194688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":444,"num_bytes_recovered":29097984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4","objects":594},{"shards":"3,4,6","objects":2448}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":283,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476581+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:48:55.476581+0000","last_peered":"2023-10-09T05:48:55.476581+0000","last_clean":"2023-10-09T05:48:55.476581+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:48:55.476581+0000","last_undegraded":"2023-10-09T05:48:55.476581+0000","last_fullsized":"2023-10-09T05:48:55.476581+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.607790+0000","last_change":"2023-10-09T05:48:54.492394+0000","last_active":"2023-10-09T05:48:55.607790+0000","last_peered":"2023-10-09T05:48:55.607790+0000","last_clean":"2023-10-09T05:48:55.607790+0000","last_became_active":"2023-10-09T05:48:54.491682+0000","last_became_peered":"2023-10-09T05:48:54.491682+0000","last_unstale":"2023-10-09T05:48:55.607790+0000","last_undegraded":"2023-10-09T05:48:55.607790+0000","last_fullsized":"2023-10-09T05:48:55.607790+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:28.838164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.17","version":"308'5861","reported_seq":5913,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:01.952095+0000","last_change":"2023-10-09T05:48:56.019001+0000","last_active":"2023-10-09T05:49:01.952095+0000","last_peered":"2023-10-09T05:49:01.952095+0000","last_clean":"2023-10-09T05:48:53.462142+0000","last_became_active":"2023-10-09T05:48:56.004223+0000","last_became_peered":"2023-10-09T05:48:56.004223+0000","last_unstale":"2023-10-09T05:49:01.952095+0000","last_undegraded":"2023-10-09T05:48:55.514193+0000","last_fullsized":"2023-10-09T05:48:55.479940+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5861,"log_dups_size":0,"ondisk_log_size":5861,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201654272,"num_objects":3077,"num_object_clones":0,"num_object_copies":9231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2986,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3077,"num_write_kb":196928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2986},{"shards":"3,6","objects":91}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":303,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476040+0000","last_change":"2023-10-09T05:48:54.583404+0000","last_active":"2023-10-09T05:48:55.476040+0000","last_peered":"2023-10-09T05:48:55.476040+0000","last_clean":"2023-10-09T05:48:55.476040+0000","last_became_active":"2023-10-09T05:48:54.492565+0000","last_became_peered":"2023-10-09T05:48:54.492565+0000","last_unstale":"2023-10-09T05:48:55.476040+0000","last_undegraded":"2023-10-09T05:48:55.476040+0000","last_fullsized":"2023-10-09T05:48:55.476040+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":306,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.514383+0000","last_change":"2023-10-09T05:48:54.493079+0000","last_active":"2023-10-09T05:48:55.514383+0000","last_peered":"2023-10-09T05:48:55.514383+0000","last_clean":"2023-10-09T05:48:55.514383+0000","last_became_active":"2023-10-09T05:48:54.492013+0000","last_became_peered":"2023-10-09T05:48:54.492013+0000","last_unstale":"2023-10-09T05:48:55.514383+0000","last_undegraded":"2023-10-09T05:48:55.514383+0000","last_fullsized":"2023-10-09T05:48:55.514383+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"308'5721","reported_seq":5763,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:03.845849+0000","last_change":"2023-10-09T05:48:46.260747+0000","last_active":"2023-10-09T05:49:03.845849+0000","last_peered":"2023-10-09T05:49:03.845849+0000","last_clean":"2023-10-09T05:49:03.845849+0000","last_became_active":"2023-10-09T05:48:46.258900+0000","last_became_peered":"2023-10-09T05:48:46.258900+0000","last_unstale":"2023-10-09T05:49:03.845849+0000","last_undegraded":"2023-10-09T05:49:03.845849+0000","last_fullsized":"2023-10-09T05:49:03.845849+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5721,"log_dups_size":0,"ondisk_log_size":5721,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198803456,"num_objects":3034,"num_object_clones":0,"num_object_copies":9102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3034,"num_write_kb":194144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":262,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476122+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:48:55.476122+0000","last_peered":"2023-10-09T05:48:55.476122+0000","last_clean":"2023-10-09T05:48:55.476122+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:48:55.476122+0000","last_undegraded":"2023-10-09T05:48:55.476122+0000","last_fullsized":"2023-10-09T05:48:55.476122+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":103,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476974+0000","last_change":"2023-10-09T05:48:25.062693+0000","last_active":"2023-10-09T05:48:55.476974+0000","last_peered":"2023-10-09T05:48:55.476974+0000","last_clean":"2023-10-09T05:48:55.476974+0000","last_became_active":"2023-10-09T05:48:25.062334+0000","last_became_peered":"2023-10-09T05:48:25.062334+0000","last_unstale":"2023-10-09T05:48:55.476974+0000","last_undegraded":"2023-10-09T05:48:55.476974+0000","last_fullsized":"2023-10-09T05:48:55.476974+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:33:15.975841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.15","version":"308'5719","reported_seq":5761,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:05.242346+0000","last_change":"2023-10-09T05:48:46.308705+0000","last_active":"2023-10-09T05:49:05.242346+0000","last_peered":"2023-10-09T05:49:05.242346+0000","last_clean":"2023-10-09T05:49:05.242346+0000","last_became_active":"2023-10-09T05:48:46.258676+0000","last_became_peered":"2023-10-09T05:48:46.258676+0000","last_unstale":"2023-10-09T05:49:05.242346+0000","last_undegraded":"2023-10-09T05:49:05.242346+0000","last_fullsized":"2023-10-09T05:49:05.242346+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5719,"log_dups_size":0,"ondisk_log_size":5719,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198541312,"num_objects":3030,"num_object_clones":0,"num_object_copies":9090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3030,"num_write_kb":193888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":25,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.473577+0000","last_change":"2023-10-09T05:48:25.069406+0000","last_active":"2023-10-09T05:48:55.473577+0000","last_peered":"2023-10-09T05:48:55.473577+0000","last_clean":"2023-10-09T05:48:55.473577+0000","last_became_active":"2023-10-09T05:48:25.069064+0000","last_became_peered":"2023-10-09T05:48:25.069064+0000","last_unstale":"2023-10-09T05:48:55.473577+0000","last_undegraded":"2023-10-09T05:48:55.473577+0000","last_fullsized":"2023-10-09T05:48:55.473577+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.567858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":108,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.472680+0000","last_change":"2023-10-09T05:48:25.063854+0000","last_active":"2023-10-09T05:48:55.472680+0000","last_peered":"2023-10-09T05:48:55.472680+0000","last_clean":"2023-10-09T05:48:55.472680+0000","last_became_active":"2023-10-09T05:48:25.063579+0000","last_became_peered":"2023-10-09T05:48:25.063579+0000","last_unstale":"2023-10-09T05:48:55.472680+0000","last_undegraded":"2023-10-09T05:48:55.472680+0000","last_fullsized":"2023-10-09T05:48:55.472680+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:02.389789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.14","version":"308'5824","reported_seq":5866,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T05:49:05.210831+0000","last_change":"2023-10-09T05:48:46.325697+0000","last_active":"2023-10-09T05:49:05.210831+0000","last_peered":"2023-10-09T05:49:05.210831+0000","last_clean":"2023-10-09T05:49:05.210831+0000","last_became_active":"2023-10-09T05:48:46.268207+0000","last_became_peered":"2023-10-09T05:48:46.268207+0000","last_unstale":"2023-10-09T05:49:05.210831+0000","last_undegraded":"2023-10-09T05:49:05.210831+0000","last_fullsized":"2023-10-09T05:49:05.210831+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202932224,"num_objects":3097,"num_object_clones":0,"num_object_copies":9291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3097,"num_write_kb":198176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":282,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.476783+0000","last_change":"2023-10-09T05:46:56.197243+0000","last_active":"2023-10-09T05:48:55.476783+0000","last_peered":"2023-10-09T05:48:55.476783+0000","last_clean":"2023-10-09T05:48:55.476783+0000","last_became_active":"2023-10-09T05:46:56.022978+0000","last_became_peered":"2023-10-09T05:46:56.022978+0000","last_unstale":"2023-10-09T05:48:55.476783+0000","last_undegraded":"2023-10-09T05:48:55.476783+0000","last_fullsized":"2023-10-09T05:48:55.476783+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:08.905002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":291,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T05:48:55.482586+0000","last_change":"2023-10-09T05:48:54.492580+0000","last_active":"2023-10-09T05:48:55.482586+0000","last_peered":"2023-10-09T05:48:55.482586+0000","last_clean":"2023-10-09T05:48:55.482586+0000","last_became_active":"2023-10-09T05:48:54.492259+0000","last_became_peered":"2023-10-09T05:48:54.492259+0000","last_unstale":"2023-10-09T05:48:55.482586+0000","last_undegraded":"2023-10-09T05:48:55.482586+0000","last_fullsized":"2023-10-09T05:48:55.482586+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":10308,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":6380060672,"num_objects":97352,"num_object_clones":0,"num_object_copies":292056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34578,"num_objects_misplaced":7777,"num_objects_unfound":0,"num_objects_dirty":97352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":97352,"num_write_kb":6230528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":448,"num_bytes_recovered":29360128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19260309504,"data_stored":19260309504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":151578,"ondisk_log_size":151578,"up":78,"acting":68,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846766,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2320412,"kb_used_data":1997908,"kb_used_omap":14,"kb_used_meta":322481,"kb_avail":91424740,"statfs":{"total":95995035648,"available":93618933760,"internally_reserved":0,"allocated":2045857792,"data_stored":2045231684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":330221402},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":3.0390000000000001,"15min":4.6200000000000001},"min":{"1min":0.224,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":2.7469999999999999,"5min":110.123,"15min":220.77500000000001},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":2.9830000000000001,"15min":4.899},"min":{"1min":0.25900000000000001,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":2.6920000000000002,"5min":110.13800000000001,"15min":220.78700000000001},"last":0.376}]},{"osd":1,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.792,"15min":2.8690000000000002},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":2.8010000000000002,"5min":83.575000000000003,"15min":190.68100000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.907,"15min":2.7759999999999998},"min":{"1min":0.28699999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.2240000000000002,"5min":87.516999999999996,"15min":191.09200000000001},"last":0.29999999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.1779999999999999,"15min":3.0329999999999999},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":2.8260000000000001,"5min":7.9349999999999996,"15min":78.716999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.21,"15min":2.9830000000000001},"min":{"1min":0.32100000000000001,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":2.8399999999999999,"5min":7.391,"15min":78.768000000000001},"last":0.44}]},{"osd":3,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":2.8010000000000002,"5min":2.8010000000000002,"15min":3.577},"min":{"1min":0.38,"5min":0.315,"15min":0.315},"max":{"1min":42.683999999999997,"5min":64.558999999999997,"15min":185.17099999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":2.6880000000000002,"5min":2.6480000000000001,"15min":3.802},"min":{"1min":0.40899999999999997,"5min":0.33000000000000002,"15min":0.29799999999999999},"max":{"1min":43.283999999999999,"5min":64.5,"15min":183.35499999999999},"last":0.57199999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.69599999999999995,"15min":0.93899999999999995},"min":{"1min":0.314,"5min":0.28299999999999997,"15min":0.25700000000000001},"max":{"1min":3.5419999999999998,"5min":4.3620000000000001,"15min":12.789},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.78100000000000003,"5min":0.70699999999999996,"15min":1.0700000000000001},"min":{"1min":0.314,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":2.7160000000000002,"5min":3.9169999999999998,"15min":9.0470000000000006},"last":0.432}]},{"osd":5,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.92100000000000004,"15min":1.139},"min":{"1min":0.34200000000000003,"5min":0.33100000000000002,"15min":0.26900000000000002},"max":{"1min":2.6709999999999998,"5min":5.4950000000000001,"15min":10.813000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.80800000000000005,"15min":1.2649999999999999},"min":{"1min":0.34200000000000003,"5min":0.33400000000000002,"15min":0.28299999999999997},"max":{"1min":2.7679999999999998,"5min":4.7649999999999997,"15min":10.321},"last":0.38500000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.71599999999999997,"15min":1.0940000000000001},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.318},"max":{"1min":2.6429999999999998,"5min":3.7229999999999999,"15min":9.2530000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.77800000000000002,"15min":1.0469999999999999},"min":{"1min":0.378,"5min":0.34699999999999998,"15min":0.30599999999999999},"max":{"1min":2.7250000000000001,"5min":4.2279999999999998,"15min":8.2680000000000007},"last":0.40600000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626792,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3300308,"kb_used_data":2710604,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":90444844,"statfs":{"total":95995035648,"available":92615520256,"internally_reserved":0,"allocated":2775658496,"data_stored":2775040742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:35 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.4970000000000001,"15min":4.5999999999999996},"min":{"1min":0.32900000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.0570000000000004,"5min":67.349999999999994,"15min":162.08799999999999},"last":19.204000000000001},{"interface":"front","average":{"1min":1.397,"5min":1.9199999999999999,"15min":4.423},"min":{"1min":0.313,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.7650000000000001,"5min":67.206000000000003,"15min":161.90899999999999},"last":19.263999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":0.96699999999999997,"15min":3.8029999999999999},"min":{"1min":0.33000000000000002,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":4.9779999999999998,"5min":6.5279999999999996,"15min":228.24299999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.4370000000000001,"5min":0.83999999999999997,"15min":3.7799999999999998},"min":{"1min":0.39200000000000002,"5min":0.312,"15min":0.252},"max":{"1min":5.1829999999999998,"5min":5.1829999999999998,"15min":228.22499999999999},"last":0.626}]},{"osd":2,"last update":"Mon Oct 9 05:48:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.5209999999999999,"15min":3.9860000000000002},"min":{"1min":0.36599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.5430000000000001,"5min":72.450999999999993,"15min":115.40600000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.379,"5min":1.698,"15min":4.0599999999999996},"min":{"1min":0.33500000000000002,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":4.9459999999999997,"5min":72.466999999999999,"15min":115.297},"last":0.61699999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":1.8440000000000001,"15min":3.8420000000000001},"min":{"1min":0.33300000000000002,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":19.52,"5min":77.578999999999994,"15min":115.244},"last":19.52},{"interface":"front","average":{"1min":2.589,"5min":1.952,"15min":3.8959999999999999},"min":{"1min":0.33800000000000002,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":19.231999999999999,"5min":77.516999999999996,"15min":115.34999999999999},"last":19.231999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":0.70999999999999996,"15min":1.7749999999999999},"min":{"1min":0.34699999999999998,"5min":0.27100000000000002,"15min":0.23400000000000001},"max":{"1min":4.7359999999999998,"5min":4.7359999999999998,"15min":125.64},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.306,"5min":1.22,"15min":1.8440000000000001},"min":{"1min":0.35799999999999998,"5min":0.246,"15min":0.246},"max":{"1min":5.5999999999999996,"5min":13.609,"15min":125.625},"last":0.35799999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:48:22 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.84899999999999998,"15min":1.958},"min":{"1min":0.38,"5min":0.28999999999999998,"15min":0.27600000000000002},"max":{"1min":1.7410000000000001,"5min":6.141,"15min":127.35899999999999},"last":0.318},{"interface":"front","average":{"1min":1.212,"5min":1.161,"15min":1.9359999999999999},"min":{"1min":0.40799999999999997,"5min":0.28699999999999998,"15min":0.27500000000000002},"max":{"1min":3.6960000000000002,"5min":14.250999999999999,"15min":128.93799999999999},"last":0.40600000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":0.77400000000000002,"15min":1.522},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26500000000000001},"max":{"1min":5.2279999999999998,"5min":5.2279999999999998,"15min":115.514},"last":0.375},{"interface":"front","average":{"1min":2.0499999999999998,"5min":1.2769999999999999,"15min":1.472},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":6.2450000000000001,"5min":14.300000000000001,"15min":116.53100000000001},"last":0.47199999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958753,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":581896,"kb_used_data":130048,"kb_used_omap":7,"kb_used_meta":451832,"kb_avail":93163256,"statfs":{"total":95995035648,"available":95399174144,"internally_reserved":0,"allocated":133169152,"data_stored":132153466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":462676494},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":2.7429999999999999,"15min":4.5010000000000003},"min":{"1min":0.224,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":3.7890000000000001,"5min":61.222999999999999,"15min":207.858},"last":3.7890000000000001},{"interface":"front","average":{"1min":0.70899999999999996,"5min":2.956,"15min":5.0540000000000003},"min":{"1min":0.27000000000000002,"5min":0.23300000000000001,"15min":0.182},"max":{"1min":3.9060000000000001,"5min":61.366999999999997,"15min":206.97200000000001},"last":3.9060000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:48:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":2.29,"15min":4.657},"min":{"1min":0.25800000000000001,"5min":0.19800000000000001,"15min":0.184},"max":{"1min":11.914,"5min":68.878,"15min":101.367},"last":2.0470000000000002},{"interface":"front","average":{"1min":1.1459999999999999,"5min":2.387,"15min":4.3860000000000001},"min":{"1min":0.29799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":5.9260000000000002,"5min":68.831999999999994,"15min":99.861000000000004},"last":0.66800000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.6240000000000001,"15min":4.6929999999999996},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":7.5599999999999996,"5min":27.489999999999998,"15min":162.33699999999999},"last":2.7949999999999999},{"interface":"front","average":{"1min":1.125,"5min":1.448,"15min":4.9349999999999996},"min":{"1min":0.316,"5min":0.26000000000000001,"15min":0.21299999999999999},"max":{"1min":5.9550000000000001,"5min":27.788,"15min":162.376},"last":0.74099999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":1.242,"15min":2.597},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":29.605,"5min":29.605,"15min":141.32900000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.605,"5min":1.034,"15min":2.1040000000000001},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":29.515999999999998,"5min":29.515999999999998,"15min":141.15899999999999},"last":0.70499999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":3.2519999999999998,"5min":1.591,"15min":2.653},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":32.774000000000001,"5min":32.774000000000001,"15min":235.18700000000001},"last":2.0800000000000001},{"interface":"front","average":{"1min":3.02,"5min":1.385,"15min":2.9399999999999999},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":36.976999999999997,"5min":36.976999999999997,"15min":235.79300000000001},"last":0.499}]},{"osd":6,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":2.3690000000000002,"15min":2.5960000000000001},"min":{"1min":0.38700000000000001,"5min":0.316,"15min":0.316},"max":{"1min":8.4870000000000001,"5min":101.295,"15min":103.60899999999999},"last":2.1070000000000002},{"interface":"front","average":{"1min":1.22,"5min":2.1110000000000002,"15min":2.7410000000000001},"min":{"1min":0.5,"5min":0.38900000000000001,"15min":0.33400000000000002},"max":{"1min":3.8559999999999999,"5min":101.27,"15min":103.679},"last":0.53200000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":3.238,"5min":1.355,"15min":2.1669999999999998},"min":{"1min":0.44700000000000001,"5min":0.375,"15min":0.35999999999999999},"max":{"1min":35.021000000000001,"5min":35.021000000000001,"15min":126.68300000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":3.6150000000000002,"5min":1.4350000000000001,"15min":2.282},"min":{"1min":0.44600000000000001,"5min":0.39700000000000002,"15min":0.34399999999999997},"max":{"1min":35.493000000000002,"5min":35.493000000000002,"15min":126.738},"last":0.81899999999999995}]}]},{"osd":0,"up_from":88,"seq":377957122279,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2953364,"kb_used_data":2342092,"kb_used_omap":8,"kb_used_meta":611255,"kb_avail":90791788,"statfs":{"total":95995035648,"available":92970790912,"internally_reserved":0,"allocated":2398302208,"data_stored":2397216068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":625925896},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.75600000000000001,"15min":1.6830000000000001},"min":{"1min":0.248,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":4.2560000000000002,"5min":14.198,"15min":17.760000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.1000000000000001,"5min":0.85699999999999998,"15min":2.2069999999999999},"min":{"1min":0.28000000000000003,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":2.9990000000000001,"5min":8.2680000000000007,"15min":26.163},"last":0.313}]},{"osd":2,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":1.411,"15min":3.4740000000000002},"min":{"1min":0.219,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.0199999999999996,"5min":19.405999999999999,"15min":92.230000000000004},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.2569999999999999,"15min":3.2570000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.218},"max":{"1min":4.8879999999999999,"5min":16.407,"15min":92.247},"last":0.375}]},{"osd":3,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.6659999999999999,"15min":3.9089999999999998},"min":{"1min":0.32800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":3.4790000000000001,"5min":55.328000000000003,"15min":123.024},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.429,"5min":1.8620000000000001,"15min":3.7829999999999999},"min":{"1min":0.34699999999999998,"5min":0.26800000000000002,"15min":0.222},"max":{"1min":6.4470000000000001,"5min":55.292000000000002,"15min":99.162000000000006},"last":0.34699999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.83899999999999997,"15min":1.452},"min":{"1min":0.32500000000000001,"5min":0.29899999999999999,"15min":0.25700000000000001},"max":{"1min":3.8570000000000002,"5min":10.084,"15min":69.082999999999998},"last":40.976999999999997},{"interface":"front","average":{"1min":1,"5min":1.1559999999999999,"15min":1.9790000000000001},"min":{"1min":0.41999999999999998,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":2.4460000000000002,"5min":19.638999999999999,"15min":69.102000000000004},"last":41.002000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.345,"15min":3.286},"min":{"1min":0.48599999999999999,"5min":0.34599999999999997,"15min":0.30199999999999999},"max":{"1min":2.8079999999999998,"5min":19.547000000000001,"15min":281.733},"last":43.107999999999997},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.353,"15min":3.343},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":4.3890000000000002,"5min":19.463999999999999,"15min":282.95400000000001},"last":41.969000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.4159999999999999,"15min":1.8819999999999999},"min":{"1min":0.46300000000000002,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":2.895,"5min":19.579000000000001,"15min":19.579000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.234,"5min":1.4950000000000001,"15min":2.0219999999999998},"min":{"1min":0.42299999999999999,"5min":0.34899999999999998,"15min":0.32500000000000001},"max":{"1min":4.4370000000000003,"5min":19.497,"15min":19.497},"last":0.58499999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.1859999999999999,"15min":1.4379999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.7220000000000004,"5min":25.378,"15min":25.378},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.55,"5min":1.391,"15min":2.0129999999999999},"min":{"1min":0.48399999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":5.556,"5min":25.436,"15min":25.436},"last":0.626}]}]},{"osd":2,"up_from":98,"seq":420906795226,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3327388,"kb_used_data":2675860,"kb_used_omap":11,"kb_used_meta":651508,"kb_avail":90417764,"statfs":{"total":95995035648,"available":92587790336,"internally_reserved":0,"allocated":2740080640,"data_stored":2738868184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":667144789},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":4.5069999999999997,"15min":3.5009999999999999},"min":{"1min":0.22900000000000001,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":4.415,"5min":162.285,"15min":162.285},"last":2.5209999999999999},{"interface":"front","average":{"1min":0.91400000000000003,"5min":4.9390000000000001,"15min":3.7989999999999999},"min":{"1min":0.26400000000000001,"5min":0.22600000000000001,"15min":0.19600000000000001},"max":{"1min":3.0249999999999999,"5min":162.643,"15min":162.643},"last":1.3380000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:48:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.206,"15min":3.1829999999999998},"min":{"1min":0.28399999999999997,"5min":0.193,"15min":0.193},"max":{"1min":3.7469999999999999,"5min":16.050999999999998,"15min":98.575000000000003},"last":6.6970000000000001},{"interface":"front","average":{"1min":1.103,"5min":0.94499999999999995,"15min":2.6389999999999998},"min":{"1min":0.24199999999999999,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":5.4470000000000001,"5min":7.7149999999999999,"15min":98.397000000000006},"last":4.7590000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":3.879,"15min":4.9530000000000003},"min":{"1min":0.32100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1859999999999999,"5min":159.35499999999999,"15min":159.35499999999999},"last":2.6190000000000002},{"interface":"front","average":{"1min":0.98199999999999998,"5min":3.9279999999999999,"15min":4.8719999999999999},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.0780000000000003,"5min":159.196,"15min":159.196},"last":2.585}]},{"osd":4,"last update":"Mon Oct 9 05:48:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":0.90500000000000003,"15min":1.9550000000000001},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.27700000000000002},"max":{"1min":5.5279999999999996,"5min":7.1120000000000001,"15min":115.84999999999999},"last":2.6480000000000001},{"interface":"front","average":{"1min":1.625,"5min":0.998,"15min":2.1200000000000001},"min":{"1min":0.36199999999999999,"5min":0.309,"15min":0.28699999999999998},"max":{"1min":8.7739999999999991,"5min":8.7739999999999991,"15min":116.149},"last":2.4260000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:48:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.252,"15min":2.0470000000000002},"min":{"1min":0.38900000000000001,"5min":0.33900000000000002,"15min":0.312},"max":{"1min":4.1269999999999998,"5min":13.035,"15min":24.597999999999999},"last":1.385},{"interface":"front","average":{"1min":1.278,"5min":1.0229999999999999,"15min":1.7270000000000001},"min":{"1min":0.39100000000000001,"5min":0.30499999999999999,"15min":0.26000000000000001},"max":{"1min":5.4850000000000003,"5min":7.1829999999999998,"15min":23.919},"last":3.1890000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:48:37 2023","interfaces":[{"interface":"back","average":{"1min":2.419,"5min":1.417,"15min":1.9490000000000001},"min":{"1min":0.46000000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":25.818999999999999,"5min":25.818999999999999,"15min":25.818999999999999},"last":2.4660000000000002},{"interface":"front","average":{"1min":2.657,"5min":1.7290000000000001,"15min":2.048},"min":{"1min":0.46400000000000002,"5min":0.38900000000000001,"15min":0.34499999999999997},"max":{"1min":24.481000000000002,"5min":24.481000000000002,"15min":24.481000000000002},"last":1.4219999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.675,"15min":2.2749999999999999},"min":{"1min":0.41299999999999998,"5min":0.38300000000000001,"15min":0.37},"max":{"1min":4.8899999999999997,"5min":51.701999999999998,"15min":51.701999999999998},"last":2.5550000000000002},{"interface":"front","average":{"1min":0.97799999999999998,"5min":2.0409999999999999,"15min":2.2639999999999998},"min":{"1min":0.44900000000000001,"5min":0.373,"15min":0.373},"max":{"1min":3.3519999999999999,"5min":51.716000000000001,"15min":51.716000000000001},"last":1.4510000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631700,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3759380,"kb_used_data":3092684,"kb_used_omap":4,"kb_used_meta":666683,"kb_avail":89985772,"statfs":{"total":95995035648,"available":92145430528,"internally_reserved":0,"allocated":3166908416,"data_stored":3165820068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":682683998},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.105,"5min":4.5289999999999999,"15min":3.8769999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.183},"max":{"1min":45.731000000000002,"5min":149.565,"15min":167.82400000000001},"last":1.044},{"interface":"front","average":{"1min":2.9710000000000001,"5min":4.8819999999999997,"15min":4.4340000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":45.877000000000002,"5min":149.291,"15min":167.89599999999999},"last":0.40100000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":2.4820000000000002,"15min":2.9470000000000001},"min":{"1min":0.29199999999999998,"5min":0.247,"15min":0.22600000000000001},"max":{"1min":6.3730000000000002,"5min":149.32499999999999,"15min":149.32499999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.68500000000000005,"5min":2.903,"15min":3.0830000000000002},"min":{"1min":0.24299999999999999,"5min":0.19,"15min":0.19},"max":{"1min":5.5540000000000003,"5min":149.34299999999999,"15min":149.34299999999999},"last":1.8779999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":2.758,"15min":4.6280000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.215},"max":{"1min":6.2110000000000003,"5min":150.006,"15min":190.43299999999999},"last":0.94299999999999995},{"interface":"front","average":{"1min":0.85099999999999998,"5min":2.6840000000000002,"15min":5.0049999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.944,"5min":150.65100000000001,"15min":190.267},"last":1.343}]},{"osd":4,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":2.528,"15min":2.7029999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":2.794,"5min":149.84700000000001,"15min":149.84700000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.82899999999999996,"5min":2.3900000000000001,"15min":2.1989999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.27400000000000002},"max":{"1min":6.1559999999999997,"5min":149.74700000000001,"15min":149.74700000000001},"last":0.86699999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":2.6419999999999999,"15min":2.3999999999999999},"min":{"1min":0.36099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.024,"5min":149.43000000000001,"15min":149.43000000000001},"last":1.919},{"interface":"front","average":{"1min":0.90100000000000002,"5min":2.5880000000000001,"15min":2.3559999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":6.0899999999999999,"5min":149.69300000000001,"15min":149.69300000000001},"last":0.90200000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:48:01 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":2.5510000000000002,"15min":2.2210000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.33500000000000002},"max":{"1min":6.1299999999999999,"5min":149.72399999999999,"15min":149.72399999999999},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.0169999999999999,"5min":2.7869999999999999,"15min":2.2949999999999999},"min":{"1min":0.33200000000000002,"5min":0.307,"15min":0.307},"max":{"1min":6.048,"5min":149.589,"15min":149.589},"last":1.9450000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":2.5169999999999999,"15min":2.4700000000000002},"min":{"1min":0.41499999999999998,"5min":0.35699999999999998,"15min":0.33700000000000002},"max":{"1min":2.8260000000000001,"5min":149.77199999999999,"15min":149.77199999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.90400000000000003,"5min":2.6920000000000002,"15min":2.5219999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.35699999999999998},"max":{"1min":3.194,"5min":149.80799999999999,"15min":149.80799999999999},"last":0.67800000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019248,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4386260,"kb_used_data":3484812,"kb_used_omap":0,"kb_used_meta":901440,"kb_avail":89358892,"statfs":{"total":95995035648,"available":91503505408,"internally_reserved":0,"allocated":3568447488,"data_stored":3567828958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923074560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:42 2023","interfaces":[{"interface":"back","average":{"1min":3.407,"5min":1.679,"15min":2.488},"min":{"1min":0.34300000000000003,"5min":0.219,"15min":0.214},"max":{"1min":16.984999999999999,"5min":26.478000000000002,"15min":100.504},"last":0.40500000000000003},{"interface":"front","average":{"1min":3.1400000000000001,"5min":1.839,"15min":2.7909999999999999},"min":{"1min":0.32500000000000001,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":18.928000000000001,"5min":26.454000000000001,"15min":100.717},"last":0.41599999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:48:09 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.2150000000000001,"15min":2.5009999999999999},"min":{"1min":0.29099999999999998,"5min":0.26800000000000002,"15min":0.24299999999999999},"max":{"1min":3.8759999999999999,"5min":10.180999999999999,"15min":187.369},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.1639999999999999,"15min":2.6059999999999999},"min":{"1min":0.42699999999999999,"5min":0.307,"15min":0.26600000000000001},"max":{"1min":3.0089999999999999,"5min":10.189,"15min":187.35300000000001},"last":0.438}]},{"osd":2,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":1.419,"15min":4.694},"min":{"1min":0.42899999999999999,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":17.335999999999999,"5min":17.335999999999999,"15min":196.93700000000001},"last":0.378},{"interface":"front","average":{"1min":2.1320000000000001,"5min":1.4870000000000001,"15min":4.5999999999999996},"min":{"1min":0.311,"5min":0.29799999999999999,"15min":0.28100000000000003},"max":{"1min":16.795000000000002,"5min":16.795000000000002,"15min":197.08099999999999},"last":0.36799999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:48:53 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":2.5369999999999999,"15min":4.5679999999999996},"min":{"1min":0.35399999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":16.667000000000002,"5min":123.59,"15min":211.48699999999999},"last":30.111999999999998},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.5680000000000001,"15min":4.4100000000000001},"min":{"1min":0.439,"5min":0.312,"15min":0.311},"max":{"1min":16.530999999999999,"5min":123.566,"15min":211.31100000000001},"last":30.709}]},{"osd":5,"last update":"Mon Oct 9 05:48:53 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":1.173,"15min":1.794},"min":{"1min":0.27900000000000003,"5min":0.27800000000000002,"15min":0.24099999999999999},"max":{"1min":12.34,"5min":12.34,"15min":113.35899999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.6619999999999999,"5min":1.04,"15min":1.6699999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":10.191000000000001,"5min":10.191000000000001,"15min":113.32299999999999},"last":0.42499999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:48:09 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.63500000000000001,"15min":0.73999999999999999},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.28100000000000003},"max":{"1min":2.081,"5min":3.3969999999999998,"15min":11.944000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.90700000000000003,"15min":0.98899999999999999},"min":{"1min":0.34999999999999998,"5min":0.32800000000000001,"15min":0.28399999999999997},"max":{"1min":2.2789999999999999,"5min":8.5269999999999992,"15min":10.356},"last":0.34899999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:48:09 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":1.615,"15min":1.603},"min":{"1min":0.36799999999999999,"5min":0.34300000000000003,"15min":0.312},"max":{"1min":2.9980000000000002,"5min":96.131,"15min":96.131},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.7769999999999999,"15min":1.7609999999999999},"min":{"1min":0.443,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":3.9329999999999998,"5min":96.465000000000003,"15min":96.465000000000003},"last":0.44400000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823020,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2943040,"kb_used_data":2353336,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":90802112,"statfs":{"total":95995035648,"available":92981362688,"internally_reserved":0,"allocated":2409816064,"data_stored":2409213392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,3,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":2.6989999999999998,"15min":3.847},"min":{"1min":0.23499999999999999,"5min":0.23300000000000001,"15min":0.19400000000000001},"max":{"1min":6.7679999999999998,"5min":96.858000000000004,"15min":165.90199999999999},"last":0.312},{"interface":"front","average":{"1min":0.83799999999999997,"5min":2.8109999999999999,"15min":3.9380000000000002},"min":{"1min":0.28799999999999998,"5min":0.254,"15min":0.215},"max":{"1min":3.4500000000000002,"5min":97.555999999999997,"15min":165.96700000000001},"last":1.8}]},{"osd":1,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":1.0129999999999999,"15min":2.7029999999999998},"min":{"1min":0.35299999999999998,"5min":0.27000000000000002,"15min":0.23999999999999999},"max":{"1min":8.6799999999999997,"5min":8.6799999999999997,"15min":87.769000000000005},"last":2.488},{"interface":"front","average":{"1min":1.1080000000000001,"5min":0.79800000000000004,"15min":2.6970000000000001},"min":{"1min":0.29099999999999998,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":3.4870000000000001,"5min":5.601,"15min":87.864000000000004},"last":1.776}]},{"osd":2,"last update":"Mon Oct 9 05:48:40 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.244,"15min":4.5190000000000001},"min":{"1min":0.29699999999999999,"5min":0.28100000000000003,"15min":0.26700000000000002},"max":{"1min":6.8710000000000004,"5min":10.722,"15min":290.697},"last":1.052},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.2090000000000001,"15min":4.399},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":8.5030000000000001,"5min":8.5030000000000001,"15min":290.786},"last":2.5209999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":1.9079999999999999,"15min":3.7690000000000001},"min":{"1min":0.432,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":8.5350000000000001,"5min":53.243000000000002,"15min":95.236999999999995},"last":2.577},{"interface":"front","average":{"1min":1.9830000000000001,"5min":1.6850000000000001,"15min":3.5720000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29699999999999999},"max":{"1min":6.8339999999999996,"5min":53.259999999999998,"15min":93.471000000000004},"last":1.145}]},{"osd":4,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":8.9939999999999998,"5min":2.266,"15min":1.351},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":130.81,"5min":130.81,"15min":130.81},"last":1.704},{"interface":"front","average":{"1min":9.8119999999999994,"5min":2.4990000000000001,"15min":1.6439999999999999},"min":{"1min":0.33500000000000002,"5min":0.27000000000000002,"15min":0.254},"max":{"1min":133.803,"5min":133.803,"15min":133.803},"last":0.33500000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.83199999999999996,"15min":1.458},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.26600000000000001},"max":{"1min":3.8580000000000001,"5min":9.2959999999999994,"15min":95.010999999999996},"last":2.415},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.81100000000000005,"15min":1.353},"min":{"1min":0.34399999999999997,"5min":0.32100000000000001,"15min":0.27300000000000002},"max":{"1min":3.0499999999999998,"5min":4.5410000000000004,"15min":94.194000000000003},"last":3.1400000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:48:17 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.997,"15min":1.1599999999999999},"min":{"1min":0.33700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.6710000000000003,"5min":9.1460000000000008,"15min":9.9849999999999994},"last":3.1240000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.96299999999999997,"15min":1.212},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.308},"max":{"1min":3.9740000000000002,"5min":9.3100000000000005,"15min":14.432},"last":0.44700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2395275264,"data_stored":2395275264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":141623296,"data_stored":141623296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2735538176,"data_stored":2735538176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3168927744,"data_stored":3168927744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3575185408,"data_stored":3575185408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2422734848,"data_stored":2422734848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2769354752,"data_stored":2769354752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2051670016,"data_stored":2051670016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:49:09.157 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:49:09.157 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 36 2023-10-09T05:49:09.157 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 86 16 100561 100545 73.0382 38.6875 0.0426159 0.0136859 2023-10-09T05:49:09.267 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:09.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:09 smithi008 ceph-mon[191227]: pgmap v929: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 147 KiB/s rd, 67 MiB/s wr, 1.15k op/s; 34600/294801 objects degraded (11.737%); 7459/294801 objects misplaced (2.530%); 2.0 MiB/s, 33 objects/s recovering 2023-10-09T05:49:09.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:09 smithi008 ceph-mon[184418]: pgmap v929: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 147 KiB/s rd, 67 MiB/s wr, 1.15k op/s; 34600/294801 objects degraded (11.737%); 7459/294801 objects misplaced (2.530%); 2.0 MiB/s, 33 objects/s recovering 2023-10-09T05:49:09.486 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:09.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:09.533 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:49:09.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:09 smithi191 ceph-mon[161669]: pgmap v929: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 147 KiB/s rd, 67 MiB/s wr, 1.15k op/s; 34600/294801 objects degraded (11.737%); 7459/294801 objects misplaced (2.530%); 2.0 MiB/s, 33 objects/s recovering 2023-10-09T05:49:09.621 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:09.703 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:09.729 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:09.890 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:09.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:09.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:10.007 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:10.046 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:10.110 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:10.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:10.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:10.151 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 87 16 101123 101107 72.6013 35.125 0.0198041 0.0137677 2023-10-09T05:49:10.216 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:10.244 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:10.287 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:10.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:10.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:10.390 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:10.462 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:10.503 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:10.571 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:49:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:10 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:10 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/933603072' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:10 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:10.716 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:49:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:49:10] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.33.4" 2023-10-09T05:49:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:10 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:10 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/933603072' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:10 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:10.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:10 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:10.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:10 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/933603072' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:10.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:10 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:11.154 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 88 16 101738 101722 72.2132 38.4375 0.0142635 0.0138422 2023-10-09T05:49:11.161 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:49:11.161 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:49:11.162 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:49:11.162 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:49:11.162 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:49:11.520 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:11.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:11.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:11.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:11.680 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:11 smithi191 ceph-mon[161669]: pgmap v930: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 133 KiB/s rd, 72 MiB/s wr, 1.22k op/s; 34600/301302 objects degraded (11.483%); 7459/301302 objects misplaced (2.476%); 1.8 MiB/s, 30 objects/s recovering 2023-10-09T05:49:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:11 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 34600/301302 objects degraded (11.483%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:11 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:49:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:11 smithi191 ceph-mon[161669]: failed to encode map e309 with expected crc 2023-10-09T05:49:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:11 smithi191 ceph-mon[161669]: failed to encode map e309 with expected crc 2023-10-09T05:49:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:11 smithi191 ceph-mon[161669]: failed to encode map e309 with expected crc 2023-10-09T05:49:11.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:11 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:49:11.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:11 smithi191 ceph-mon[161669]: osdmap e309: 8 total, 8 up, 8 in 2023-10-09T05:49:11.887 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:11.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:11.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:11.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[184418]: pgmap v930: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 133 KiB/s rd, 72 MiB/s wr, 1.22k op/s; 34600/301302 objects degraded (11.483%); 7459/301302 objects misplaced (2.476%); 1.8 MiB/s, 30 objects/s recovering 2023-10-09T05:49:11.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 34600/301302 objects degraded (11.483%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:11.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:49:11.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[184418]: failed to encode map e309 with expected crc 2023-10-09T05:49:11.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[184418]: failed to encode map e309 with expected crc 2023-10-09T05:49:11.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[184418]: failed to encode map e309 with expected crc 2023-10-09T05:49:11.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:49:11.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[184418]: osdmap e309: 8 total, 8 up, 8 in 2023-10-09T05:49:11.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[191227]: pgmap v930: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 133 KiB/s rd, 72 MiB/s wr, 1.22k op/s; 34600/301302 objects degraded (11.483%); 7459/301302 objects misplaced (2.476%); 1.8 MiB/s, 30 objects/s recovering 2023-10-09T05:49:11.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 34600/301302 objects degraded (11.483%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:11.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:49:11.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[191227]: failed to encode map e309 with expected crc 2023-10-09T05:49:11.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[191227]: failed to encode map e309 with expected crc 2023-10-09T05:49:11.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[191227]: failed to encode map e309 with expected crc 2023-10-09T05:49:11.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:49:11.970 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:11 smithi008 ceph-mon[191227]: osdmap e309: 8 total, 8 up, 8 in 2023-10-09T05:49:11.992 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:12.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:49:12.090 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:12.157 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 89 16 102400 102384 71.8621 41.375 0.0168217 0.0139106 2023-10-09T05:49:12.436 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:12.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:12.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:12.528 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:49:12.543 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:12.585 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:13.157 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total time run: 90.0189 2023-10-09T05:49:13.158 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total writes made: 102939 2023-10-09T05:49:13.158 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Write size: 65536 2023-10-09T05:49:13.158 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object size: 65536 2023-10-09T05:49:13.158 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Bandwidth (MB/sec): 71.4704 2023-10-09T05:49:13.158 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Bandwidth: 42.641 2023-10-09T05:49:13.158 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max bandwidth (MB/sec): 209.125 2023-10-09T05:49:13.158 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min bandwidth (MB/sec): 0 2023-10-09T05:49:13.158 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average IOPS: 1143 2023-10-09T05:49:13.158 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev IOPS: 682.266 2023-10-09T05:49:13.158 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max IOPS: 3346 2023-10-09T05:49:13.158 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min IOPS: 0 2023-10-09T05:49:13.159 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average Latency(s): 0.0139871 2023-10-09T05:49:13.159 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Latency(s): 0.0479986 2023-10-09T05:49:13.159 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max latency(s): 2.58525 2023-10-09T05:49:13.159 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min latency(s): 0.00124024 2023-10-09T05:49:13.163 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:49:13.339 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:13.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:13.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:13.443 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:13.481 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:13.707 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:13.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:13.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:13.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:49:13.861 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:13.928 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:49:13.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:13.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:14.032 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:49:14.073 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:14.141 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:49:14.143 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:13 smithi191 ceph-mon[161669]: pgmap v932: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 621 op/s; 34600/303876 objects degraded (11.386%); 7081/303876 objects misplaced (2.330%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:14.143 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:14.144 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:14.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:14.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:14.170 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:49:14.171 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:49:14.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:49:14.171 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:49:14.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.171 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102149 7.5 7.a7c93f25 (undecoded) ondisk+write+known_if_redirected e230)", 2023-10-09T05:49:14.171 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.906659+0000", 2023-10-09T05:49:14.171 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 355.26153554699999, 2023-10-09T05:49:14.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97026822800000001, 2023-10-09T05:49:14.171 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.171 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102149 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906659+0000", 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906659+0000", 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906658+0000", 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0549999999999996e-06 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906664+0000", 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4589999999999998e-06 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906672+0000", 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1719999999999998e-06 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906678+0000", 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6041999999999997e-05 2023-10-09T05:49:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906724+0000", 2023-10-09T05:49:14.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8980000000000001e-06 2023-10-09T05:49:14.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:14.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906730+0000", 2023-10-09T05:49:14.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97018501300000004 2023-10-09T05:49:14.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.876915+0000", 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2931999999999999e-05 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.876928+0000", 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97026822800000001 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102151 7.5 7.b6aec605 (undecoded) ondisk+write+known_if_redirected e230)", 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.906691+0000", 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 355.26150401000001, 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97026954700000001, 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102151 2023-10-09T05:49:14.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.178 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906691+0000", 2023-10-09T05:49:14.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906691+0000", 2023-10-09T05:49:14.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:49:14.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906690+0000", 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9209999999999999e-06 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906692+0000", 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6110000000000004e-06 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906697+0000", 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9569999999999998e-06 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.906700+0000", 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00046790699999999999 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.907168+0000", 2023-10-09T05:49:14.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.967e-06 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.907173+0000", 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96977922400000005 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.876952+0000", 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3059999999999998e-06 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.876960+0000", 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97026954700000001 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:151472 7.5 7:a05f3fd6:::benchmark_data_smithi008_354649_object56976:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:20.476351+0000", 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 233.69184393800001, 2023-10-09T05:49:14.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78401668599999996, 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 151472 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:20.476351+0000", 2023-10-09T05:49:14.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:20.476351+0000", 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:20.476349+0000", 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1339999999999998e-06 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:20.476356+0000", 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4843e-05 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:20.476371+0000", 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1829999999999997e-06 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:20.476378+0000", 2023-10-09T05:49:14.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00049835099999999996 2023-10-09T05:49:14.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:20.476877+0000", 2023-10-09T05:49:14.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001857051 2023-10-09T05:49:14.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:49:14.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:20.478734+0000", 2023-10-09T05:49:14.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0095145349999999993 2023-10-09T05:49:14.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:20.488248+0000", 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022567360000000001 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:20.490505+0000", 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.53851103 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:21.029016+0000", 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014924820000000001 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:21.030509+0000", 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038179469999999999 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:21.034327+0000", 2023-10-09T05:49:14.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003639635 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:21.037966+0000", 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000176799 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:21.038143+0000", 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.216753904 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:21.254897+0000", 2023-10-09T05:49:14.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00096272700000000003 2023-10-09T05:49:14.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:14.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:21.255860+0000", 2023-10-09T05:49:14.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045079810000000003 2023-10-09T05:49:14.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:21.260368+0000", 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78401668599999996 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:152437 7.5 7:a716a07b:::benchmark_data_smithi008_354649_object57941:head [delete] snapc 0=[] ondisk+write+known_if_redirected e255)", 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:23.417001+0000", 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 230.75119402799999, 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78378357499999995, 2023-10-09T05:49:14.191 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 152437 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:23.417001+0000", 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:23.417001+0000", 2023-10-09T05:49:14.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:49:14.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:23.417000+0000", 2023-10-09T05:49:14.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1250000000000001e-06 2023-10-09T05:49:14.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:23.417005+0000", 2023-10-09T05:49:14.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8900000000000007e-06 2023-10-09T05:49:14.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:23.417013+0000", 2023-10-09T05:49:14.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4510000000000002e-06 2023-10-09T05:49:14.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:23.417017+0000", 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003839196 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:23.420856+0000", 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001989045 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:23.422845+0000", 2023-10-09T05:49:14.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0088288329999999995 2023-10-09T05:49:14.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:23.431674+0000", 2023-10-09T05:49:14.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001840969 2023-10-09T05:49:14.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:49:14.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:23.433515+0000", 2023-10-09T05:49:14.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.60425496000000001 2023-10-09T05:49:14.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:24.037770+0000", 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014752820000000001 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:24.039246+0000", 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042135649999999998 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:24.043459+0000", 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0036059220000000001 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:24.047065+0000", 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000179142 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:24.047244+0000", 2023-10-09T05:49:14.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14769047199999999 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:24.194935+0000", 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001079867 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:24.196015+0000", 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004769868 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:24.200784+0000", 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78378357499999995 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:168225 7.5 7:a7a4fac5:::benchmark_data_smithi008_354649_object73729:head [delete] snapc 0=[] ondisk+write+known_if_redirected e260)", 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:07.753660+0000", 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 186.41453448799999, 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78279890699999999, 2023-10-09T05:49:14.200 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 168225 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:07.753660+0000", 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:07.753660+0000", 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:07.753658+0000", 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8899999999999996e-06 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:07.753667+0000", 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5179e-05 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:07.753682+0000", 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2629999999999997e-06 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:07.753690+0000", 2023-10-09T05:49:14.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023531619999999998 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:07.756043+0000", 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012574039999999999 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:07.757300+0000", 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0074923489999999997 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:07.764793+0000", 2023-10-09T05:49:14.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014174229999999999 2023-10-09T05:49:14.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:49:14.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:07.766210+0000", 2023-10-09T05:49:14.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.62151965499999995 2023-10-09T05:49:14.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.387730+0000", 2023-10-09T05:49:14.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00096840100000000003 2023-10-09T05:49:14.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:14.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.388698+0000", 2023-10-09T05:49:14.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002401285 2023-10-09T05:49:14.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:49:14.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.391099+0000", 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13259913600000001 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.523699+0000", 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000346261 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.524045+0000", 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0093718210000000007 2023-10-09T05:49:14.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.533417+0000", 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00059318399999999996 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.534010+0000", 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024494429999999999 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:08.536459+0000", 2023-10-09T05:49:14.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78279890699999999 2023-10-09T05:49:14.209 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.209 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.209 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.209 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:169305 7.15 7:aab54f87:::benchmark_data_smithi008_354649_object74809:head [delete] snapc 0=[] ondisk+write+known_if_redirected e260)", 2023-10-09T05:49:14.209 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:10.619417+0000", 2023-10-09T05:49:14.209 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 183.548777484, 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78880979799999995, 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 169305 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:10.619417+0000", 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:10.619417+0000", 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:10.619416+0000", 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3950000000000001e-06 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:10.619424+0000", 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8658999999999999e-05 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:10.619443+0000", 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2500000000000003e-06 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:10.619449+0000", 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0089946599999999998 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:10.628444+0000", 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001194093 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:10.629638+0000", 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0096541300000000004 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:10.639292+0000", 2023-10-09T05:49:14.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001395458 2023-10-09T05:49:14.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:49:14.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:10.640687+0000", 2023-10-09T05:49:14.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.73439651100000003 2023-10-09T05:49:14.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.375084+0000", 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010067850000000001 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.376091+0000", 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003160024 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.379251+0000", 2023-10-09T05:49:14.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059699560000000002 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.385221+0000", 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079371779999999996 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.393158+0000", 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011845477 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.405003+0000", 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00078377799999999995 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.405787+0000", 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024401470000000001 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.408227+0000", 2023-10-09T05:49:14.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78880979799999995 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:169746 7.5 7:a494acde:::benchmark_data_smithi008_354649_object75250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e260)", 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:11.752343+0000", 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 182.41585147699999, 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.77680863499999997, 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:14.218 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 169746 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.752343+0000", 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.752343+0000", 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.752343+0000", 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6770000000000003e-06 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.752347+0000", 2023-10-09T05:49:14.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6199999999999999e-06 2023-10-09T05:49:14.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.752355+0000", 2023-10-09T05:49:14.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3329999999999998e-06 2023-10-09T05:49:14.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.752359+0000", 2023-10-09T05:49:14.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017955987 2023-10-09T05:49:14.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.770315+0000", 2023-10-09T05:49:14.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011316690000000001 2023-10-09T05:49:14.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:49:14.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.771447+0000", 2023-10-09T05:49:14.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020487037999999999 2023-10-09T05:49:14.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.791934+0000", 2023-10-09T05:49:14.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015147450000000001 2023-10-09T05:49:14.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:49:14.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:11.793449+0000", 2023-10-09T05:49:14.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.72039848399999995 2023-10-09T05:49:14.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:12.513847+0000", 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010393970000000001 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:12.514887+0000", 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028669630000000002 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:49:14.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:12.517754+0000", 2023-10-09T05:49:14.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0036295939999999999 2023-10-09T05:49:14.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:12.521383+0000", 2023-10-09T05:49:14.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000438804 2023-10-09T05:49:14.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:12.521822+0000", 2023-10-09T05:49:14.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041232400000000002 2023-10-09T05:49:14.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:14.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:12.525945+0000", 2023-10-09T05:49:14.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00061150600000000005 2023-10-09T05:49:14.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:14.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:12.526557+0000", 2023-10-09T05:49:14.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025952950000000001 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:12.529152+0000", 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.77680863499999997 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:173879 7.5 7:a40c6a3c:::benchmark_data_smithi008_354649_object79383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e265)", 2023-10-09T05:49:14.227 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:23.511983+0000", 2023-10-09T05:49:14.228 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 170.65621212299999, 2023-10-09T05:49:14.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.84901140900000005, 2023-10-09T05:49:14.228 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.228 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:14.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:14.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:14.228 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 173879 2023-10-09T05:49:14.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.228 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.511983+0000", 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.511983+0000", 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.511981+0000", 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.961e-06 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.511991+0000", 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2801e-05 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.512004+0000", 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2050000000000001e-06 2023-10-09T05:49:14.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.512009+0000", 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013003 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.512139+0000", 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001002523 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.513142+0000", 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014497052 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.527639+0000", 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011893450000000001 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.528828+0000", 2023-10-09T05:49:14.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.63258417899999997 2023-10-09T05:49:14.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.161412+0000", 2023-10-09T05:49:14.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00093681099999999996 2023-10-09T05:49:14.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:14.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.162349+0000", 2023-10-09T05:49:14.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023430159999999998 2023-10-09T05:49:14.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.164692+0000", 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19346987900000001 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.358162+0000", 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00033277799999999999 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.358495+0000", 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00031056699999999998 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.358805+0000", 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00047197299999999999 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.359277+0000", 2023-10-09T05:49:14.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017168249999999999 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.360994+0000", 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.84901140900000005 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:173884 7.5 7:a3c8607e:::benchmark_data_smithi008_354649_object79388:head [delete] snapc 0=[] ondisk+write+known_if_redirected e265)", 2023-10-09T05:49:14.236 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:23.546393+0000", 2023-10-09T05:49:14.237 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 170.621801906, 2023-10-09T05:49:14.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.82706812399999996, 2023-10-09T05:49:14.237 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.237 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:14.237 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.237 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:14.237 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:14.237 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 173884 2023-10-09T05:49:14.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.237 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.546393+0000", 2023-10-09T05:49:14.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.546393+0000", 2023-10-09T05:49:14.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:14.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.546391+0000", 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0084e-05 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.546401+0000", 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7671000000000001e-05 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.546419+0000", 2023-10-09T05:49:14.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8449999999999992e-06 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.546428+0000", 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00039401900000000001 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.546822+0000", 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013104799999999999 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.548132+0000", 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0062526860000000004 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.554385+0000", 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001207423 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:49:14.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.555592+0000", 2023-10-09T05:49:14.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.79731427300000002 2023-10-09T05:49:14.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.352907+0000", 2023-10-09T05:49:14.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00077657099999999999 2023-10-09T05:49:14.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:14.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.353683+0000", 2023-10-09T05:49:14.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002002392 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.355686+0000", 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0054019849999999998 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.361088+0000", 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099078259999999998 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.370995+0000", 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00030762399999999999 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.371303+0000", 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00046250700000000002 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:14.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.371766+0000", 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016955060000000001 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.373461+0000", 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.82706812399999996 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.245 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:173885 7.5 7:a2ecb4b6:::benchmark_data_smithi008_354649_object79389:head [delete] snapc 0=[] ondisk+write+known_if_redirected e265)", 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:23.566447+0000", 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 170.60174826100001, 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81011298600000003, 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 173885 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.566447+0000", 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.566447+0000", 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.566445+0000", 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.365e-06 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.566452+0000", 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.202e-05 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.566464+0000", 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8199999999999997e-06 2023-10-09T05:49:14.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.566471+0000", 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002235913 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.568707+0000", 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00098311199999999996 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.569690+0000", 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006154301 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.575845+0000", 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00097567900000000002 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:23.576820+0000", 2023-10-09T05:49:14.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78694242800000003 2023-10-09T05:49:14.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.363763+0000", 2023-10-09T05:49:14.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000965046 2023-10-09T05:49:14.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:14.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.364728+0000", 2023-10-09T05:49:14.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023994369999999999 2023-10-09T05:49:14.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:49:14.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.367127+0000", 2023-10-09T05:49:14.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0064279699999999999 2023-10-09T05:49:14.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:14.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.373555+0000", 2023-10-09T05:49:14.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00032063099999999999 2023-10-09T05:49:14.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.373876+0000", 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00027289099999999999 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.374149+0000", 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00044113399999999998 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:14.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.374590+0000", 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019696240000000001 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:24.376560+0000", 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81011298600000003 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.254 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180148 7.15 7:accd2834:::benchmark_data_smithi008_354649_object85652:head [delete] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:40.563216+0000", 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 153.60497871499999, 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.77686454500000002, 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180148 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:40.563216+0000", 2023-10-09T05:49:14.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:40.563216+0000", 2023-10-09T05:49:14.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:14.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:40.563214+0000", 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8290000000000003e-06 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:40.563224+0000", 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.221e-05 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:40.563236+0000", 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7560000000000002e-06 2023-10-09T05:49:14.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:40.563243+0000", 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028538800000000003 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:40.563529+0000", 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091988499999999995 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:40.564448+0000", 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0057101390000000004 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:40.570159+0000", 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00070893400000000004 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:40.570867+0000", 2023-10-09T05:49:14.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.60790681899999999 2023-10-09T05:49:14.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.178774+0000", 2023-10-09T05:49:14.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00066010400000000001 2023-10-09T05:49:14.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:49:14.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.179434+0000", 2023-10-09T05:49:14.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019722849999999998 2023-10-09T05:49:14.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.181407+0000", 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01274141 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.194148+0000", 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.082610875 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.276759+0000", 2023-10-09T05:49:14.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0616576 2023-10-09T05:49:14.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:14.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.338417+0000", 2023-10-09T05:49:14.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00034302399999999999 2023-10-09T05:49:14.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:49:14.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.338760+0000", 2023-10-09T05:49:14.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013210699999999999 2023-10-09T05:49:14.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.340081+0000", 2023-10-09T05:49:14.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.77686454500000002 2023-10-09T05:49:14.263 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.263 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.263 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:91074 8.10 8.faaa5db0 (undecoded) ondisk+write+known_if_redirected e302)", 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:50.463178+0000", 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.705017038000001, 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96300553600000005, 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91074 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.264 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.463178+0000", 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.463178+0000", 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.463176+0000", 2023-10-09T05:49:14.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4476e-05 2023-10-09T05:49:14.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.463210+0000", 2023-10-09T05:49:14.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.119e-05 2023-10-09T05:49:14.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.463221+0000", 2023-10-09T05:49:14.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1010000000000003e-06 2023-10-09T05:49:14.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.267 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.463229+0000", 2023-10-09T05:49:14.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4323000000000001e-05 2023-10-09T05:49:14.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.267 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.463283+0000", 2023-10-09T05:49:14.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9160000000000001e-06 2023-10-09T05:49:14.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:14.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.463291+0000", 2023-10-09T05:49:14.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96287819399999997 2023-10-09T05:49:14.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426169+0000", 2023-10-09T05:49:14.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4324999999999999e-05 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426183+0000", 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96300553600000005 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:91088 8.10 8.d166ddb0 (undecoded) ondisk+write+known_if_redirected e302)", 2023-10-09T05:49:14.269 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:50.465379+0000", 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.702815440999998, 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96083468400000005, 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91088 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.465379+0000", 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.465379+0000", 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.465378+0000", 2023-10-09T05:49:14.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6711999999999993e-05 2023-10-09T05:49:14.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.465455+0000", 2023-10-09T05:49:14.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4161e-05 2023-10-09T05:49:14.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.465469+0000", 2023-10-09T05:49:14.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0990000000000001e-06 2023-10-09T05:49:14.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.465476+0000", 2023-10-09T05:49:14.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1134999999999997e-05 2023-10-09T05:49:14.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.465527+0000", 2023-10-09T05:49:14.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4099999999999997e-06 2023-10-09T05:49:14.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:14.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.465537+0000", 2023-10-09T05:49:14.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.960670684 2023-10-09T05:49:14.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426207+0000", 2023-10-09T05:49:14.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8460000000000002e-06 2023-10-09T05:49:14.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426214+0000", 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96083468400000005 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:91089 8.10 8.87a01870 (undecoded) ondisk+write+known_if_redirected e302)", 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:50.467905+0000", 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.700289640000001, 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95833757900000005, 2023-10-09T05:49:14.275 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91089 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.467905+0000", 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.467905+0000", 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.467903+0000", 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012945799999999999 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.468033+0000", 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0999999999999999e-05 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.468054+0000", 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1359999999999997e-06 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.468062+0000", 2023-10-09T05:49:14.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5432000000000001e-05 2023-10-09T05:49:14.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.468117+0000", 2023-10-09T05:49:14.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1499e-05 2023-10-09T05:49:14.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:14.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.468129+0000", 2023-10-09T05:49:14.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95810690200000004 2023-10-09T05:49:14.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426236+0000", 2023-10-09T05:49:14.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0010000000000004e-06 2023-10-09T05:49:14.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426243+0000", 2023-10-09T05:49:14.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95833757900000005 2023-10-09T05:49:14.280 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.280 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.281 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.281 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:91095 8.10 8.68451ab0 (undecoded) ondisk+write+known_if_redirected e302)", 2023-10-09T05:49:14.281 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:50.473484+0000", 2023-10-09T05:49:14.281 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.694710428, 2023-10-09T05:49:14.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95278616699999996, 2023-10-09T05:49:14.281 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.281 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:14.281 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.281 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:49:14.282 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:49:14.282 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91095 2023-10-09T05:49:14.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.282 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.473484+0000", 2023-10-09T05:49:14.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.473484+0000", 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.473482+0000", 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7296000000000003e-05 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.473530+0000", 2023-10-09T05:49:14.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3263e-05 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.473543+0000", 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.404e-06 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.473549+0000", 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6474000000000001e-05 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.473616+0000", 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8250000000000005e-06 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.473624+0000", 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95264034600000003 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426264+0000", 2023-10-09T05:49:14.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5769999999999999e-06 2023-10-09T05:49:14.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426271+0000", 2023-10-09T05:49:14.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95278616699999996 2023-10-09T05:49:14.286 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.286 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.286 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:91120 8.10 8.6d40a9f0 (undecoded) ondisk+write+known_if_redirected e302)", 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:50.493186+0000", 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.675009039999999, 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93311329799999998, 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91120 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.287 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.493186+0000", 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.493186+0000", 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.493184+0000", 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8563999999999998e-05 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.493233+0000", 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7059999999999999e-05 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.493250+0000", 2023-10-09T05:49:14.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2849999999999997e-06 2023-10-09T05:49:14.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.493257+0000", 2023-10-09T05:49:14.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7899999999999998e-05 2023-10-09T05:49:14.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.493315+0000", 2023-10-09T05:49:14.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7200000000000001e-06 2023-10-09T05:49:14.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:14.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.493325+0000", 2023-10-09T05:49:14.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93296693500000005 2023-10-09T05:49:14.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426292+0000", 2023-10-09T05:49:14.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3119999999999996e-06 2023-10-09T05:49:14.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426299+0000", 2023-10-09T05:49:14.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93311329799999998 2023-10-09T05:49:14.294 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.294 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.294 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.294 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:91141 8.10 8.2ae00ef0 (undecoded) ondisk+write+known_if_redirected e302)", 2023-10-09T05:49:14.294 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:50.507337+0000", 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.660858319999999, 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91898893100000001, 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91141 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.507337+0000", 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.507337+0000", 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.507334+0000", 2023-10-09T05:49:14.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0859000000000003e-05 2023-10-09T05:49:14.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.507375+0000", 2023-10-09T05:49:14.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3244999999999999e-05 2023-10-09T05:49:14.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.507398+0000", 2023-10-09T05:49:14.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4519999999999997e-06 2023-10-09T05:49:14.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.507407+0000", 2023-10-09T05:49:14.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2339999999999997e-05 2023-10-09T05:49:14.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.507459+0000", 2023-10-09T05:49:14.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1620000000000008e-06 2023-10-09T05:49:14.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:14.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.507467+0000", 2023-10-09T05:49:14.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.918851643 2023-10-09T05:49:14.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426319+0000", 2023-10-09T05:49:14.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.511e-06 2023-10-09T05:49:14.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.300 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.300 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426325+0000", 2023-10-09T05:49:14.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91898893100000001 2023-10-09T05:49:14.300 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.300 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.300 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.300 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:91178 8.10 8.3c006990 (undecoded) ondisk+write+known_if_redirected e302)", 2023-10-09T05:49:14.301 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:50.537385+0000", 2023-10-09T05:49:14.301 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.630810219000001, 2023-10-09T05:49:14.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88896838899999997, 2023-10-09T05:49:14.301 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.301 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:14.301 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.301 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:49:14.301 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:49:14.301 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91178 2023-10-09T05:49:14.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.301 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.537385+0000", 2023-10-09T05:49:14.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.537385+0000", 2023-10-09T05:49:14.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999971 2023-10-09T05:49:14.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.537382+0000", 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2488000000000003e-05 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.537434+0000", 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0999e-05 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.537455+0000", 2023-10-09T05:49:14.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.038e-06 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.537463+0000", 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1551999999999996e-05 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.537525+0000", 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0236e-05 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:14.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.537535+0000", 2023-10-09T05:49:14.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88881151800000002 2023-10-09T05:49:14.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426346+0000", 2023-10-09T05:49:14.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5780000000000004e-06 2023-10-09T05:49:14.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.306 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426353+0000", 2023-10-09T05:49:14.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88896838899999997 2023-10-09T05:49:14.306 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.306 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.306 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.306 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:91251 8.10 8.77ef2c30 (undecoded) ondisk+write+known_if_redirected e302)", 2023-10-09T05:49:14.306 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:50.604854+0000", 2023-10-09T05:49:14.306 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.563340839999999, 2023-10-09T05:49:14.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.821527165, 2023-10-09T05:49:14.307 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.307 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:14.307 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.307 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:49:14.307 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:49:14.307 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91251 2023-10-09T05:49:14.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.307 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.604854+0000", 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.604854+0000", 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.604852+0000", 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5701999999999997e-05 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.604898+0000", 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9105000000000001e-05 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.604917+0000", 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6040000000000004e-06 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.604924+0000", 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4461000000000005e-05 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.604989+0000", 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8213999999999999e-05 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.605007+0000", 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.82136753900000004 2023-10-09T05:49:14.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426375+0000", 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4949999999999996e-06 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426381+0000", 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.821527165 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.311 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:91272 8.10 8.e52a6d70 (undecoded) ondisk+write+known_if_redirected e302)", 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:50.624294+0000", 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.543900974, 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.80211618399999995, 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:49:14.312 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 91272 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.624294+0000", 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.624294+0000", 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999943 2023-10-09T05:49:14.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.624288+0000", 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9549000000000003e-05 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.624338+0000", 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4911e-05 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.624353+0000", 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9120000000000001e-06 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.624359+0000", 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4262999999999999e-05 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.624414+0000", 2023-10-09T05:49:14.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8939999999999993e-06 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.624422+0000", 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.80198127100000005 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426403+0000", 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2010000000000002e-06 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:49:14.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:49:14.317 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:49:14.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:51.426410+0000", 2023-10-09T05:49:14.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.80211618399999995 2023-10-09T05:49:14.317 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.317 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.317 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.317 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:49:14.317 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:49:14.317 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:49:14.317 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:14.318 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:13 smithi008 ceph-mon[191227]: pgmap v932: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 621 op/s; 34600/303876 objects degraded (11.386%); 7081/303876 objects misplaced (2.330%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:14.319 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:14.319 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:14.319 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:13 smithi008 ceph-mon[184418]: pgmap v932: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 621 op/s; 34600/303876 objects degraded (11.386%); 7081/303876 objects misplaced (2.330%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:14.319 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:14.319 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:14.357 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:14.412 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:49:14.413 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:49:14.419 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:14.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:14.449 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:49:14.489 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:49:14.523 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:14.534 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:49:14.582 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:49:14.613 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:49:14.641 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:14.646 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:49:14.679 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:49:14.680 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:49:14.680 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:49:14.680 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:49:14.680 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:49:14.680 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:49:14.680 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:49:14.680 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:49:14.973 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:49:15.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2146124555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:15.326 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:49:15.443 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2146124555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:15.443 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2146124555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:15.523 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:15.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:15.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:15.626 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:15.661 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:15.721 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:15.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:15.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:15.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:15.865 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:15.919 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:15.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:15.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:16.027 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:16.105 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:16 smithi008 ceph-mon[184418]: pgmap v933: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 38 MiB/s wr, 617 op/s; 34600/305985 objects degraded (11.308%); 7081/305985 objects misplaced (2.314%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:16.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:16 smithi008 ceph-mon[191227]: pgmap v933: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 38 MiB/s wr, 617 op/s; 34600/305985 objects degraded (11.308%); 7081/305985 objects misplaced (2.314%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:16 smithi191 ceph-mon[161669]: pgmap v933: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 38 MiB/s wr, 617 op/s; 34600/305985 objects degraded (11.308%); 7081/305985 objects misplaced (2.314%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:16.425 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:16.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:16.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:16.533 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:16.593 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:16.617 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:16.675 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:16.740 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:16.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:16.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:16.845 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:16.882 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:16.937 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:16.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:16.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:17.042 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:17.148 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:17.237 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:17 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 34600/306279 objects degraded (11.297%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:17.243 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:17 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 34600/306279 objects degraded (11.297%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:17.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:17 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 34600/306279 objects degraded (11.297%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:17.435 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:17.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:17.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:17.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:49:17.585 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:17.652 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:49:17.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:17.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:17.760 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:17.796 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:17.855 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:17.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:17.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:17.964 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:17.993 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:18.056 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:18.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:18.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:18.122 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 8 pg_num to 36 2023-10-09T05:49:18.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:18 smithi008 ceph-mon[184418]: pgmap v934: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 614 op/s; 34600/306279 objects degraded (11.297%); 7081/306279 objects misplaced (2.312%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:18.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3085821032' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:49:18.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:49:18.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:18 smithi008 ceph-mon[191227]: pgmap v934: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 614 op/s; 34600/306279 objects degraded (11.297%); 7081/306279 objects misplaced (2.312%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:18.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3085821032' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:49:18.153 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:49:18.167 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:18.236 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:18.272 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:49:18.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:18 smithi191 ceph-mon[161669]: pgmap v934: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 614 op/s; 34600/306279 objects degraded (11.297%); 7081/306279 objects misplaced (2.312%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:18.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3085821032' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:49:18.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:49:19.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[191227]: failed to encode map e310 with expected crc 2023-10-09T05:49:19.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:49:19.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[191227]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T05:49:19.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[191227]: failed to encode map e310 with expected crc 2023-10-09T05:49:19.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[191227]: failed to encode map e310 with expected crc 2023-10-09T05:49:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:49:19.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[184418]: failed to encode map e310 with expected crc 2023-10-09T05:49:19.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:49:19.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[184418]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T05:49:19.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[184418]: failed to encode map e310 with expected crc 2023-10-09T05:49:19.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[184418]: failed to encode map e310 with expected crc 2023-10-09T05:49:19.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:19 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:49:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:19 smithi191 ceph-mon[161669]: failed to encode map e310 with expected crc 2023-10-09T05:49:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:49:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:19 smithi191 ceph-mon[161669]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T05:49:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:19 smithi191 ceph-mon[161669]: failed to encode map e310 with expected crc 2023-10-09T05:49:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:19 smithi191 ceph-mon[161669]: failed to encode map e310 with expected crc 2023-10-09T05:49:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:19 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:49:19.763 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:19.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:19.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:19.868 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:19.894 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:19.939 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:19.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:19.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:20.046 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:49:20.072 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:20.124 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:49:20.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:20.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:49:20.233 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:20.261 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:20.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:20 smithi191 ceph-mon[161669]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 496 op/s; 34600/304572 objects degraded (11.360%); 6657/304572 objects misplaced (2.186%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:49:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:20 smithi191 ceph-mon[161669]: failed to encode map e311 with expected crc 2023-10-09T05:49:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:20 smithi191 ceph-mon[161669]: failed to encode map e311 with expected crc 2023-10-09T05:49:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:49:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:20 smithi191 ceph-mon[161669]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T05:49:20.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:20 smithi008 ceph-mon[184418]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 496 op/s; 34600/304572 objects degraded (11.360%); 6657/304572 objects misplaced (2.186%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:49:20.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:20 smithi008 ceph-mon[184418]: failed to encode map e311 with expected crc 2023-10-09T05:49:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:20 smithi008 ceph-mon[184418]: failed to encode map e311 with expected crc 2023-10-09T05:49:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:49:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:20 smithi008 ceph-mon[184418]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T05:49:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:20 smithi008 ceph-mon[191227]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 85 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 496 op/s; 34600/304572 objects degraded (11.360%); 6657/304572 objects misplaced (2.186%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:49:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:20 smithi008 ceph-mon[191227]: failed to encode map e311 with expected crc 2023-10-09T05:49:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:20 smithi008 ceph-mon[191227]: failed to encode map e311 with expected crc 2023-10-09T05:49:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:49:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:20 smithi008 ceph-mon[191227]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T05:49:20.767 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:49:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:49:20] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.33.4" 2023-10-09T05:49:20.873 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:20.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:20.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:20.978 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:21.011 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:21.052 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:21.120 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[184418]: failed to encode map e311 with expected crc 2023-10-09T05:49:21.120 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[184418]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.120 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[184418]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.120 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[184418]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T05:49:21.120 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[184418]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.120 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[184418]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.120 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[184418]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.121 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[184418]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.121 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[184418]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.121 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[184418]: pgmap v939: 109 pgs: 1 activating, 1 peering, 7 unknown, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 87 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 170 B/s rd, 125 MiB/s wr, 2.63k op/s; 31696/330729 objects degraded (9.584%); 6657/330729 objects misplaced (2.013%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:21.122 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[191227]: failed to encode map e311 with expected crc 2023-10-09T05:49:21.122 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[191227]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.122 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[191227]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.122 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[191227]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T05:49:21.122 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[191227]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.122 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[191227]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.122 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[191227]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.122 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[191227]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.122 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[191227]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.122 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:21 smithi008 ceph-mon[191227]: pgmap v939: 109 pgs: 1 activating, 1 peering, 7 unknown, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 87 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 170 B/s rd, 125 MiB/s wr, 2.63k op/s; 31696/330729 objects degraded (9.584%); 6657/330729 objects misplaced (2.013%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:21.219 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:21.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:21.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:21.351 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:21.511 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:21.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:21 smithi191 ceph-mon[161669]: failed to encode map e311 with expected crc 2023-10-09T05:49:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:21 smithi191 ceph-mon[161669]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:21 smithi191 ceph-mon[161669]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:21 smithi191 ceph-mon[161669]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T05:49:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:21 smithi191 ceph-mon[161669]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:21 smithi191 ceph-mon[161669]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:21 smithi191 ceph-mon[161669]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:21 smithi191 ceph-mon[161669]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:21 smithi191 ceph-mon[161669]: failed to encode map e312 with expected crc 2023-10-09T05:49:21.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:21 smithi191 ceph-mon[161669]: pgmap v939: 109 pgs: 1 activating, 1 peering, 7 unknown, 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 87 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 170 B/s rd, 125 MiB/s wr, 2.63k op/s; 31696/330729 objects degraded (9.584%); 6657/330729 objects misplaced (2.013%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:49:21.710 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:21.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:21.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:21.837 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:49:21.864 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:21.907 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:49:21.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:21.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:49:21.923 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:49:21.924 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:49:22.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:49:22.042 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:22.086 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:49:22.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:22.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:49:22.193 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:22.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:22 smithi008 ceph-mon[184418]: failed to encode map e313 with expected crc 2023-10-09T05:49:22.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:22 smithi008 ceph-mon[184418]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T05:49:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:22 smithi008 ceph-mon[184418]: failed to encode map e313 with expected crc 2023-10-09T05:49:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:22 smithi008 ceph-mon[184418]: failed to encode map e313 with expected crc 2023-10-09T05:49:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:22 smithi008 ceph-mon[191227]: failed to encode map e313 with expected crc 2023-10-09T05:49:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:22 smithi008 ceph-mon[191227]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T05:49:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:22 smithi008 ceph-mon[191227]: failed to encode map e313 with expected crc 2023-10-09T05:49:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:22 smithi008 ceph-mon[191227]: failed to encode map e313 with expected crc 2023-10-09T05:49:22.219 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:22.261 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:22.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:22.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:22.366 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:22.402 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:22.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:22 smithi191 ceph-mon[161669]: failed to encode map e313 with expected crc 2023-10-09T05:49:22.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:22 smithi191 ceph-mon[161669]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T05:49:22.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:22 smithi191 ceph-mon[161669]: failed to encode map e313 with expected crc 2023-10-09T05:49:22.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:22 smithi191 ceph-mon[161669]: failed to encode map e313 with expected crc 2023-10-09T05:49:23.139 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:49:23.168 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:23.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:23.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:23.272 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:23.305 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:23.361 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:23.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:23.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:23.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[184418]: failed to encode map e313 with expected crc 2023-10-09T05:49:23.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[184418]: failed to encode map e313 with expected crc 2023-10-09T05:49:23.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[184418]: failed to encode map e313 with expected crc 2023-10-09T05:49:23.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1887321956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:49:23.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:49:23.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[184418]: pgmap v941: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 peering, 7 unknown, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 87 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 163 MiB/s wr, 3.21k op/s; 31696/330357 objects degraded (9.594%); 5931/330357 objects misplaced (1.795%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:49:23.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 31696/330729 objects degraded (9.584%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:23.465 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:23.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[191227]: failed to encode map e313 with expected crc 2023-10-09T05:49:23.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[191227]: failed to encode map e313 with expected crc 2023-10-09T05:49:23.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[191227]: failed to encode map e313 with expected crc 2023-10-09T05:49:23.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1887321956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:49:23.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:49:23.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[191227]: pgmap v941: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 peering, 7 unknown, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 87 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 163 MiB/s wr, 3.21k op/s; 31696/330357 objects degraded (9.594%); 5931/330357 objects misplaced (1.795%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:49:23.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 31696/330729 objects degraded (9.584%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:23.505 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:23 smithi191 ceph-mon[161669]: failed to encode map e313 with expected crc 2023-10-09T05:49:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:23 smithi191 ceph-mon[161669]: failed to encode map e313 with expected crc 2023-10-09T05:49:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:23 smithi191 ceph-mon[161669]: failed to encode map e313 with expected crc 2023-10-09T05:49:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1887321956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:49:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:23 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:49:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:23 smithi191 ceph-mon[161669]: pgmap v941: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 peering, 7 unknown, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 87 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 163 MiB/s wr, 3.21k op/s; 31696/330357 objects degraded (9.594%); 5931/330357 objects misplaced (1.795%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:49:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 31696/330729 objects degraded (9.584%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:24.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:24 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:49:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:24 smithi008 ceph-mon[191227]: failed to encode map e314 with expected crc 2023-10-09T05:49:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:24 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:49:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:24 smithi008 ceph-mon[191227]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T05:49:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:24 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:49:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:24 smithi008 ceph-mon[184418]: failed to encode map e314 with expected crc 2023-10-09T05:49:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:24 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:49:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:24 smithi008 ceph-mon[184418]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T05:49:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:24 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:49:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:24 smithi191 ceph-mon[161669]: failed to encode map e314 with expected crc 2023-10-09T05:49:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:24 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:49:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:24 smithi191 ceph-mon[161669]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T05:49:24.828 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:24.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:24.939 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:24.975 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:25.136 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:25 smithi008 ceph-mon[191227]: failed to encode map e314 with expected crc 2023-10-09T05:49:25.136 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:25 smithi008 ceph-mon[191227]: pgmap v943: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 peering, 5 unknown, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 88 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 133 MiB/s wr, 2.43k op/s; 31696/330147 objects degraded (9.601%); 5931/330147 objects misplaced (1.796%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:49:25.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:25 smithi008 ceph-mon[184418]: failed to encode map e314 with expected crc 2023-10-09T05:49:25.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:25 smithi008 ceph-mon[184418]: pgmap v943: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 peering, 5 unknown, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 88 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 133 MiB/s wr, 2.43k op/s; 31696/330147 objects degraded (9.601%); 5931/330147 objects misplaced (1.796%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:49:25.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:25 smithi191 ceph-mon[161669]: failed to encode map e314 with expected crc 2023-10-09T05:49:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:25 smithi191 ceph-mon[161669]: pgmap v943: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 peering, 5 unknown, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 88 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 133 MiB/s wr, 2.43k op/s; 31696/330147 objects degraded (9.601%); 5931/330147 objects misplaced (1.796%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:49:25.621 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:49:25.762 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:25.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:25.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:25.878 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:49:25.878 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:49:25.878 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T05:49:25.878 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:49:25.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:49:25.908 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:25.953 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:49:25.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:25.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:49:26.059 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:26.086 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:26.132 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:26.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:26.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:26.240 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:26.267 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:26.316 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:26.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:26.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:26.423 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:26.536 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:26.543 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:26.610 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:49:26.661 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:49:26.683 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:26.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:26.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:26.703 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:49:26.756 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:49:26.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:49:26.802 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:26.813 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:49:26.827 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:26.862 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:49:26.884 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:49:26.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:26.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:26.901 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:49:26.963 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:49:26.979 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:49:26.979 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:49:26.979 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:49:26.979 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:49:26.979 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:49:26.979 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:49:26.979 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:49:26.979 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:49:26.989 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:27.038 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:27.415 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:27.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:27.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:27.521 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:27.549 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:27.599 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:27.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:27.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:27.707 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:27.764 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:27 smithi008 ceph-mon[184418]: pgmap v944: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37379/296415 objects degraded (12.610%); 3050/296415 objects misplaced (1.029%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:49:27.764 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1541558067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:27.765 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:27 smithi008 ceph-mon[191227]: pgmap v944: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37379/296415 objects degraded (12.610%); 3050/296415 objects misplaced (1.029%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:49:27.765 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1541558067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:27.771 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:27 smithi191 ceph-mon[161669]: pgmap v944: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37379/296415 objects degraded (12.610%); 3050/296415 objects misplaced (1.029%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T05:49:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1541558067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:27.846 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:49:28.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 37379/296415 objects degraded (12.610%), 18 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:28.857 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 37379/296415 objects degraded (12.610%), 18 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:28.860 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 37379/296415 objects degraded (12.610%), 18 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:29.243 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:29.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:29.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:29.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:29.386 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:29.434 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:29.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:29.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:29.541 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:29.608 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:29.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:29 smithi008 ceph-mon[191227]: pgmap v945: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37379/296199 objects degraded (12.620%); 2084/296199 objects misplaced (0.704%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T05:49:29.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:29.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:29 smithi008 ceph-mon[184418]: pgmap v945: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37379/296199 objects degraded (12.620%); 2084/296199 objects misplaced (0.704%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T05:49:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:30.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:29 smithi191 ceph-mon[161669]: pgmap v945: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37379/296199 objects degraded (12.620%); 2084/296199 objects misplaced (0.704%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T05:49:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:30.764 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:49:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:49:30] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.33.4" 2023-10-09T05:49:30.834 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:30.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:30.945 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:30.971 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:31.019 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:31.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:31.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:31.124 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:31.172 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:31.214 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:31.307 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:49:31.307 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:49:31.307 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:49:31.307 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:49:31.308 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:49:31.585 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:31.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:31.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:31.690 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:31.773 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:32.119 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:49:32.129 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:49:32.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:31 smithi008 ceph-mon[184418]: pgmap v946: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37050/293148 objects degraded (12.639%); 2084/293148 objects misplaced (0.711%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:49:32.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:31 smithi008 ceph-mon[191227]: pgmap v946: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37050/293148 objects degraded (12.639%); 2084/293148 objects misplaced (0.711%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:49:32.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:31 smithi191 ceph-mon[161669]: pgmap v946: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37050/293148 objects degraded (12.639%); 2084/293148 objects misplaced (0.711%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:49:32.795 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:49:33.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:32 smithi008 ceph-mon[184418]: from='client.55807 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:49:33.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:32 smithi008 ceph-mon[191227]: from='client.55807 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:49:33.233 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:33.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:33.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:33.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:32 smithi191 ceph-mon[161669]: from='client.55807 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:49:33.338 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:33.390 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:33.940 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:49:34.036 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:34.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:34.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:34.150 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:33 smithi008 ceph-mon[184418]: pgmap v947: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37050/292857 objects degraded (12.651%); 784/292857 objects misplaced (0.268%); 9.6 MiB/s, 155 objects/s recovering 2023-10-09T05:49:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 37050/292857 objects degraded (12.651%), 18 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:33 smithi008 ceph-mon[191227]: pgmap v947: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37050/292857 objects degraded (12.651%); 784/292857 objects misplaced (0.268%); 9.6 MiB/s, 155 objects/s recovering 2023-10-09T05:49:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 37050/292857 objects degraded (12.651%), 18 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:34.239 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:34.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:33 smithi191 ceph-mon[161669]: pgmap v947: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37050/292857 objects degraded (12.651%); 784/292857 objects misplaced (0.268%); 9.6 MiB/s, 155 objects/s recovering 2023-10-09T05:49:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 37050/292857 objects degraded (12.651%), 18 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:34.649 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":946,"stamp":"2023-10-09T05:49:30.396183+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6406805451,"num_objects":97716,"num_object_clones":0,"num_object_copies":293148,"num_objects_missing_on_primary":19734,"num_objects_missing":19734,"num_objects_degraded":37050,"num_objects_misplaced":2084,"num_objects_unfound":0,"num_objects_dirty":97716,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":109298,"num_write_kb":6603238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2414,"num_bytes_recovered":154283178,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":227471,"ondisk_log_size":227471,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":348,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":25080248,"kb_used_data":19285688,"kb_used_omap":45,"kb_used_meta":5794386,"kb_avail":724880968,"statfs":{"total":767960285184,"available":742278111232,"internally_reserved":0,"allocated":19748544512,"data_stored":19741509033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":5933451555},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1,3,5,3,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-820871168,"num_objects":-12527,"num_object_clones":0,"num_object_copies":-37581,"num_objects_missing_on_primary":19734,"num_objects_missing":19734,"num_objects_degraded":5354,"num_objects_misplaced":-4573,"num_objects_unfound":0,"num_objects_dirty":-12527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-9867,"num_write_kb":-716512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1268,"num_bytes_recovered":82509824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.278226"},"pg_stats":[{"pgid":"8.23","version":"314'6046","reported_seq":6154,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:29.151390+0000","last_change":"2023-10-09T05:49:21.579854+0000","last_active":"2023-10-09T05:49:29.151390+0000","last_peered":"2023-10-09T05:49:29.151390+0000","last_clean":"2023-10-09T05:48:53.427903+0000","last_became_active":"2023-10-09T05:49:21.560913+0000","last_became_peered":"2023-10-09T05:49:21.560913+0000","last_unstale":"2023-10-09T05:49:29.151390+0000","last_undegraded":"2023-10-09T05:49:21.500088+0000","last_fullsized":"2023-10-09T05:49:29.151390+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6046,"log_dups_size":0,"ondisk_log_size":6046,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99450880,"num_objects":1518,"num_object_clones":0,"num_object_copies":4554,"num_objects_missing_on_primary":1409,"num_objects_missing":1409,"num_objects_degraded":1409,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1690,"num_write_kb":102624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":1572864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":172},{"shards":"2,5,6","objects":1346}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.22","version":"314'5997","reported_seq":6053,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:29.415358+0000","last_change":"2023-10-09T05:49:20.140728+0000","last_active":"2023-10-09T05:49:29.415358+0000","last_peered":"2023-10-09T05:49:29.415358+0000","last_clean":"2023-10-09T05:49:29.415358+0000","last_became_active":"2023-10-09T05:49:20.132812+0000","last_became_peered":"2023-10-09T05:49:20.132812+0000","last_unstale":"2023-10-09T05:49:29.415358+0000","last_undegraded":"2023-10-09T05:49:29.415358+0000","last_fullsized":"2023-10-09T05:49:29.415358+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5997,"log_dups_size":0,"ondisk_log_size":5997,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98189312,"num_objects":1498,"num_object_clones":0,"num_object_copies":4494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1675,"num_write_kb":101552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.21","version":"314'6054","reported_seq":6152,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:29.014997+0000","last_change":"2023-10-09T05:49:21.991103+0000","last_active":"2023-10-09T05:49:29.014997+0000","last_peered":"2023-10-09T05:49:29.014997+0000","last_clean":"2023-10-09T05:48:53.421353+0000","last_became_active":"2023-10-09T05:49:21.956287+0000","last_became_peered":"2023-10-09T05:49:21.956287+0000","last_unstale":"2023-10-09T05:49:29.014997+0000","last_undegraded":"2023-10-09T05:49:21.126622+0000","last_fullsized":"2023-10-09T05:49:29.014997+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6054,"log_dups_size":0,"ondisk_log_size":6054,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99352576,"num_objects":1516,"num_object_clones":0,"num_object_copies":4548,"num_objects_missing_on_primary":1450,"num_objects_missing":1450,"num_objects_degraded":1450,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1687,"num_write_kb":102496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":1245184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,1,2","objects":138},{"shards":"0,2,5","objects":1378}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.20","version":"314'6133","reported_seq":6247,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:29.473350+0000","last_change":"2023-10-09T05:49:21.912294+0000","last_active":"2023-10-09T05:49:29.473350+0000","last_peered":"2023-10-09T05:49:29.473350+0000","last_clean":"2023-10-09T05:48:53.429181+0000","last_became_active":"2023-10-09T05:49:21.894029+0000","last_became_peered":"2023-10-09T05:49:21.894029+0000","last_unstale":"2023-10-09T05:49:29.473350+0000","last_undegraded":"2023-10-09T05:49:21.127313+0000","last_fullsized":"2023-10-09T05:49:29.473350+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6133,"log_dups_size":0,"ondisk_log_size":6133,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101416960,"num_objects":1548,"num_object_clones":0,"num_object_copies":4644,"num_objects_missing_on_primary":1391,"num_objects_missing":1391,"num_objects_degraded":1391,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1710,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":1507328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,4","objects":211},{"shards":"3,4,6","objects":1337}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.11","version":"314'6077","reported_seq":6129,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:28.826269+0000","last_change":"2023-10-09T05:48:55.843352+0000","last_active":"2023-10-09T05:49:28.826269+0000","last_peered":"2023-10-09T05:49:28.826269+0000","last_clean":"2023-10-09T05:48:53.444054+0000","last_became_active":"2023-10-09T05:48:55.837353+0000","last_became_peered":"2023-10-09T05:48:55.837353+0000","last_unstale":"2023-10-09T05:49:28.826269+0000","last_undegraded":"2023-10-09T05:48:55.496526+0000","last_fullsized":"2023-10-09T05:48:55.477252+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6077,"log_dups_size":0,"ondisk_log_size":6077,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199884800,"num_objects":3050,"num_object_clones":0,"num_object_copies":9150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2852,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3378,"num_write_kb":205696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":198},{"shards":"0,2,5","objects":2688}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":314,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136569+0000","last_change":"2023-10-09T05:48:54.494513+0000","last_active":"2023-10-09T05:49:21.136569+0000","last_peered":"2023-10-09T05:49:21.136569+0000","last_clean":"2023-10-09T05:49:21.136569+0000","last_became_active":"2023-10-09T05:48:54.493510+0000","last_became_peered":"2023-10-09T05:48:54.493510+0000","last_unstale":"2023-10-09T05:49:21.136569+0000","last_undegraded":"2023-10-09T05:49:21.136569+0000","last_fullsized":"2023-10-09T05:49:21.136569+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123272+0000","last_change":"2023-10-09T05:48:54.492024+0000","last_active":"2023-10-09T05:49:21.123272+0000","last_peered":"2023-10-09T05:49:21.123272+0000","last_clean":"2023-10-09T05:49:21.123272+0000","last_became_active":"2023-10-09T05:48:54.491690+0000","last_became_peered":"2023-10-09T05:48:54.491690+0000","last_unstale":"2023-10-09T05:49:21.123272+0000","last_undegraded":"2023-10-09T05:49:21.123272+0000","last_fullsized":"2023-10-09T05:49:21.123272+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:41.936752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.12","version":"314'6012","reported_seq":6058,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:30.257796+0000","last_change":"2023-10-09T05:48:46.263477+0000","last_active":"2023-10-09T05:49:30.257796+0000","last_peered":"2023-10-09T05:49:30.257796+0000","last_clean":"2023-10-09T05:49:30.257796+0000","last_became_active":"2023-10-09T05:48:46.259060+0000","last_became_peered":"2023-10-09T05:48:46.259060+0000","last_unstale":"2023-10-09T05:49:30.257796+0000","last_undegraded":"2023-10-09T05:49:30.257796+0000","last_fullsized":"2023-10-09T05:49:30.257796+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6012,"log_dups_size":0,"ondisk_log_size":6012,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194936832,"num_objects":2975,"num_object_clones":0,"num_object_copies":8925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3345,"num_write_kb":202208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.131519+0000","last_change":"2023-10-09T05:48:54.491563+0000","last_active":"2023-10-09T05:49:21.131519+0000","last_peered":"2023-10-09T05:49:21.131519+0000","last_clean":"2023-10-09T05:49:21.131519+0000","last_became_active":"2023-10-09T05:48:54.491227+0000","last_became_peered":"2023-10-09T05:48:54.491227+0000","last_unstale":"2023-10-09T05:49:21.131519+0000","last_undegraded":"2023-10-09T05:49:21.131519+0000","last_fullsized":"2023-10-09T05:49:21.131519+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:37.990305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":321,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.159508+0000","last_change":"2023-10-09T05:48:54.493724+0000","last_active":"2023-10-09T05:49:21.159508+0000","last_peered":"2023-10-09T05:49:21.159508+0000","last_clean":"2023-10-09T05:49:21.159508+0000","last_became_active":"2023-10-09T05:48:54.493380+0000","last_became_peered":"2023-10-09T05:48:54.493380+0000","last_unstale":"2023-10-09T05:49:21.159508+0000","last_undegraded":"2023-10-09T05:49:21.159508+0000","last_fullsized":"2023-10-09T05:49:21.159508+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:05.264766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"314'6061","reported_seq":6111,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:29.832035+0000","last_change":"2023-10-09T05:48:55.978875+0000","last_active":"2023-10-09T05:49:29.832035+0000","last_peered":"2023-10-09T05:49:29.832035+0000","last_clean":"2023-10-09T05:48:53.465296+0000","last_became_active":"2023-10-09T05:48:55.972707+0000","last_became_peered":"2023-10-09T05:48:55.972707+0000","last_unstale":"2023-10-09T05:49:29.832035+0000","last_undegraded":"2023-10-09T05:48:55.487364+0000","last_fullsized":"2023-10-09T05:48:55.475509+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6061,"log_dups_size":0,"ondisk_log_size":6061,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197656576,"num_objects":3016,"num_object_clones":0,"num_object_copies":9048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2813,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3370,"num_write_kb":204352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[5,6],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5,6","objects":2635},{"shards":"5,6","objects":203}],"blocked_by":[],"up_primary":1,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":282,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.151102+0000","last_change":"2023-10-09T05:48:54.492664+0000","last_active":"2023-10-09T05:49:21.151102+0000","last_peered":"2023-10-09T05:49:21.151102+0000","last_clean":"2023-10-09T05:49:21.151102+0000","last_became_active":"2023-10-09T05:48:54.492299+0000","last_became_peered":"2023-10-09T05:48:54.492299+0000","last_unstale":"2023-10-09T05:49:21.151102+0000","last_undegraded":"2023-10-09T05:49:21.151102+0000","last_fullsized":"2023-10-09T05:49:21.151102+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:43.198424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":147,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.142424+0000","last_change":"2023-10-09T05:48:54.492447+0000","last_active":"2023-10-09T05:49:21.142424+0000","last_peered":"2023-10-09T05:49:21.142424+0000","last_clean":"2023-10-09T05:49:21.142424+0000","last_became_active":"2023-10-09T05:48:54.492089+0000","last_became_peered":"2023-10-09T05:48:54.492089+0000","last_unstale":"2023-10-09T05:49:21.142424+0000","last_undegraded":"2023-10-09T05:49:21.142424+0000","last_fullsized":"2023-10-09T05:49:21.142424+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.1c","version":"314'6903","reported_seq":6951,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:28.572772+0000","last_change":"2023-10-09T05:49:20.384940+0000","last_active":"2023-10-09T05:49:28.572772+0000","last_peered":"2023-10-09T05:49:28.572772+0000","last_clean":"2023-10-09T05:49:28.572772+0000","last_became_active":"2023-10-09T05:49:20.380945+0000","last_became_peered":"2023-10-09T05:49:20.380945+0000","last_unstale":"2023-10-09T05:49:28.572772+0000","last_undegraded":"2023-10-09T05:49:28.572772+0000","last_fullsized":"2023-10-09T05:49:28.572772+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6903,"log_dups_size":0,"ondisk_log_size":6903,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203882496,"num_objects":3111,"num_object_clones":0,"num_object_copies":9333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3471,"num_write_kb":210624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":269,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.157814+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:49:21.157814+0000","last_peered":"2023-10-09T05:49:21.157814+0000","last_clean":"2023-10-09T05:49:21.157814+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:49:21.157814+0000","last_undegraded":"2023-10-09T05:49:21.157814+0000","last_fullsized":"2023-10-09T05:49:21.157814+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.132127+0000","last_change":"2023-10-09T05:48:54.486629+0000","last_active":"2023-10-09T05:49:21.132127+0000","last_peered":"2023-10-09T05:49:21.132127+0000","last_clean":"2023-10-09T05:49:21.132127+0000","last_became_active":"2023-10-09T05:48:54.486107+0000","last_became_peered":"2023-10-09T05:48:54.486107+0000","last_unstale":"2023-10-09T05:49:21.132127+0000","last_undegraded":"2023-10-09T05:49:21.132127+0000","last_fullsized":"2023-10-09T05:49:21.132127+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:58.679726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1d","version":"314'6655","reported_seq":6702,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:30.312802+0000","last_change":"2023-10-09T05:49:20.157594+0000","last_active":"2023-10-09T05:49:30.312802+0000","last_peered":"2023-10-09T05:49:30.312802+0000","last_clean":"2023-10-09T05:49:30.312802+0000","last_became_active":"2023-10-09T05:49:20.153063+0000","last_became_peered":"2023-10-09T05:49:20.153063+0000","last_unstale":"2023-10-09T05:49:30.312802+0000","last_undegraded":"2023-10-09T05:49:30.312802+0000","last_fullsized":"2023-10-09T05:49:30.312802+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6655,"log_dups_size":0,"ondisk_log_size":6655,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197427212,"num_objects":3013,"num_object_clones":0,"num_object_copies":9039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3013,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3349,"num_write_kb":203553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":333,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123827+0000","last_change":"2023-10-09T05:48:54.491950+0000","last_active":"2023-10-09T05:49:21.123827+0000","last_peered":"2023-10-09T05:49:21.123827+0000","last_clean":"2023-10-09T05:49:21.123827+0000","last_became_active":"2023-10-09T05:48:54.491559+0000","last_became_peered":"2023-10-09T05:48:54.491559+0000","last_unstale":"2023-10-09T05:49:21.123827+0000","last_undegraded":"2023-10-09T05:49:21.123827+0000","last_fullsized":"2023-10-09T05:49:21.123827+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":249,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123198+0000","last_change":"2023-10-09T05:48:25.068328+0000","last_active":"2023-10-09T05:49:21.123198+0000","last_peered":"2023-10-09T05:49:21.123198+0000","last_clean":"2023-10-09T05:49:21.123198+0000","last_became_active":"2023-10-09T05:48:25.068094+0000","last_became_peered":"2023-10-09T05:48:25.068094+0000","last_unstale":"2023-10-09T05:49:21.123198+0000","last_undegraded":"2023-10-09T05:49:21.123198+0000","last_fullsized":"2023-10-09T05:49:21.123198+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.1e","version":"314'6735","reported_seq":6782,"reported_epoch":314,"state":"active+recovery_wait","last_fresh":"2023-10-09T05:49:28.826256+0000","last_change":"2023-10-09T05:49:21.159312+0000","last_active":"2023-10-09T05:49:28.826256+0000","last_peered":"2023-10-09T05:49:28.826256+0000","last_clean":"2023-10-09T05:49:19.070876+0000","last_became_active":"2023-10-09T05:49:20.168491+0000","last_became_peered":"2023-10-09T05:49:20.168491+0000","last_unstale":"2023-10-09T05:49:28.826256+0000","last_undegraded":"2023-10-09T05:49:28.826256+0000","last_fullsized":"2023-10-09T05:49:28.826256+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6735,"log_dups_size":0,"ondisk_log_size":6735,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201523200,"num_objects":3075,"num_object_clones":0,"num_object_copies":9225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3388,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":3075}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":99,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123653+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:49:21.123653+0000","last_peered":"2023-10-09T05:49:21.123653+0000","last_clean":"2023-10-09T05:49:21.123653+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:49:21.123653+0000","last_undegraded":"2023-10-09T05:49:21.123653+0000","last_fullsized":"2023-10-09T05:49:21.123653+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.134474+0000","last_change":"2023-10-09T05:48:54.490345+0000","last_active":"2023-10-09T05:49:21.134474+0000","last_peered":"2023-10-09T05:49:21.134474+0000","last_clean":"2023-10-09T05:49:21.134474+0000","last_became_active":"2023-10-09T05:48:54.490000+0000","last_became_peered":"2023-10-09T05:48:54.490000+0000","last_unstale":"2023-10-09T05:49:21.134474+0000","last_undegraded":"2023-10-09T05:49:21.134474+0000","last_fullsized":"2023-10-09T05:49:21.134474+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:02.457608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1f","version":"314'6601","reported_seq":6739,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:29.601780+0000","last_change":"2023-10-09T05:49:23.718403+0000","last_active":"2023-10-09T05:49:29.601780+0000","last_peered":"2023-10-09T05:49:29.601780+0000","last_clean":"2023-10-09T05:48:53.467734+0000","last_became_active":"2023-10-09T05:49:23.645737+0000","last_became_peered":"2023-10-09T05:49:23.645737+0000","last_unstale":"2023-10-09T05:49:29.601780+0000","last_undegraded":"2023-10-09T05:49:21.161481+0000","last_fullsized":"2023-10-09T05:49:29.601780+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196837376,"num_objects":3004,"num_object_clones":0,"num_object_copies":9012,"num_objects_missing_on_primary":2777,"num_objects_missing":2777,"num_objects_degraded":2777,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3318,"num_write_kb":202272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2293760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,4","objects":336},{"shards":"1,3,4,6","objects":7},{"shards":"3,4,6","objects":2661}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":340,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136591+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:49:21.136591+0000","last_peered":"2023-10-09T05:49:21.136591+0000","last_clean":"2023-10-09T05:49:21.136591+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:49:21.136591+0000","last_undegraded":"2023-10-09T05:49:21.136591+0000","last_fullsized":"2023-10-09T05:49:21.136591+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":297,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.142078+0000","last_change":"2023-10-09T05:48:54.492082+0000","last_active":"2023-10-09T05:49:21.142078+0000","last_peered":"2023-10-09T05:49:21.142078+0000","last_clean":"2023-10-09T05:49:21.142078+0000","last_became_active":"2023-10-09T05:48:54.491753+0000","last_became_peered":"2023-10-09T05:48:54.491753+0000","last_unstale":"2023-10-09T05:49:21.142078+0000","last_undegraded":"2023-10-09T05:49:21.142078+0000","last_fullsized":"2023-10-09T05:49:21.142078+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"314'6035","reported_seq":6083,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:29.992983+0000","last_change":"2023-10-09T05:48:46.314063+0000","last_active":"2023-10-09T05:49:29.992983+0000","last_peered":"2023-10-09T05:49:29.992983+0000","last_clean":"2023-10-09T05:49:29.992983+0000","last_became_active":"2023-10-09T05:48:46.261937+0000","last_became_peered":"2023-10-09T05:48:46.261937+0000","last_unstale":"2023-10-09T05:49:29.992983+0000","last_undegraded":"2023-10-09T05:49:29.992983+0000","last_fullsized":"2023-10-09T05:49:29.992983+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6035,"log_dups_size":0,"ondisk_log_size":6035,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198508544,"num_objects":3029,"num_object_clones":0,"num_object_copies":9087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3353,"num_write_kb":204224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":99,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123375+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:49:21.123375+0000","last_peered":"2023-10-09T05:49:21.123375+0000","last_clean":"2023-10-09T05:49:21.123375+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:49:21.123375+0000","last_undegraded":"2023-10-09T05:49:21.123375+0000","last_fullsized":"2023-10-09T05:49:21.123375+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.122055+0000","last_change":"2023-10-09T05:48:54.492572+0000","last_active":"2023-10-09T05:49:21.122055+0000","last_peered":"2023-10-09T05:49:21.122055+0000","last_clean":"2023-10-09T05:49:21.122055+0000","last_became_active":"2023-10-09T05:48:54.491827+0000","last_became_peered":"2023-10-09T05:48:54.491827+0000","last_unstale":"2023-10-09T05:49:21.122055+0000","last_undegraded":"2023-10-09T05:49:21.122055+0000","last_fullsized":"2023-10-09T05:49:21.122055+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:33.185508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.19","version":"314'6163","reported_seq":6211,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:28.781081+0000","last_change":"2023-10-09T05:48:46.264369+0000","last_active":"2023-10-09T05:49:28.781081+0000","last_peered":"2023-10-09T05:49:28.781081+0000","last_clean":"2023-10-09T05:49:28.781081+0000","last_became_active":"2023-10-09T05:48:46.259158+0000","last_became_peered":"2023-10-09T05:48:46.259158+0000","last_unstale":"2023-10-09T05:49:28.781081+0000","last_undegraded":"2023-10-09T05:49:28.781081+0000","last_fullsized":"2023-10-09T05:49:28.781081+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6163,"log_dups_size":0,"ondisk_log_size":6163,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203227136,"num_objects":3101,"num_object_clones":0,"num_object_copies":9303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3415,"num_write_kb":208512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":302,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.157325+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:49:21.157325+0000","last_peered":"2023-10-09T05:49:21.157325+0000","last_clean":"2023-10-09T05:49:21.157325+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:49:21.157325+0000","last_undegraded":"2023-10-09T05:49:21.157325+0000","last_fullsized":"2023-10-09T05:49:21.157325+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":272,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123551+0000","last_change":"2023-10-09T05:46:56.019914+0000","last_active":"2023-10-09T05:49:21.123551+0000","last_peered":"2023-10-09T05:49:21.123551+0000","last_clean":"2023-10-09T05:49:21.123551+0000","last_became_active":"2023-10-09T05:46:56.019487+0000","last_became_peered":"2023-10-09T05:46:56.019487+0000","last_unstale":"2023-10-09T05:49:21.123551+0000","last_undegraded":"2023-10-09T05:49:21.123551+0000","last_fullsized":"2023-10-09T05:49:21.123551+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.1a","version":"314'6682","reported_seq":6830,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:29.519134+0000","last_change":"2023-10-09T05:49:23.419200+0000","last_active":"2023-10-09T05:49:29.519134+0000","last_peered":"2023-10-09T05:49:29.519134+0000","last_clean":"2023-10-09T05:48:53.473854+0000","last_became_active":"2023-10-09T05:49:23.397583+0000","last_became_peered":"2023-10-09T05:49:23.397583+0000","last_unstale":"2023-10-09T05:49:29.519134+0000","last_undegraded":"2023-10-09T05:49:21.157095+0000","last_fullsized":"2023-10-09T05:49:29.519134+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6682,"log_dups_size":0,"ondisk_log_size":6682,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199098368,"num_objects":3038,"num_object_clones":0,"num_object_copies":9114,"num_objects_missing_on_primary":2788,"num_objects_missing":2788,"num_objects_degraded":2788,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3365,"num_write_kb":204896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3145728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2,3","objects":362},{"shards":"2,3,4","objects":2676}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":93,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123256+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:49:21.123256+0000","last_peered":"2023-10-09T05:49:21.123256+0000","last_clean":"2023-10-09T05:49:21.123256+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:49:21.123256+0000","last_undegraded":"2023-10-09T05:49:21.123256+0000","last_fullsized":"2023-10-09T05:49:21.123256+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":302,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.159571+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:49:21.159571+0000","last_peered":"2023-10-09T05:49:21.159571+0000","last_clean":"2023-10-09T05:49:21.159571+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:49:21.159571+0000","last_undegraded":"2023-10-09T05:49:21.159571+0000","last_fullsized":"2023-10-09T05:49:21.159571+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1b","version":"314'6828","reported_seq":7009,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:30.290454+0000","last_change":"2023-10-09T05:49:20.414092+0000","last_active":"2023-10-09T05:49:30.290454+0000","last_peered":"2023-10-09T05:49:30.290454+0000","last_clean":"2023-10-09T05:48:53.462109+0000","last_became_active":"2023-10-09T05:49:20.374568+0000","last_became_peered":"2023-10-09T05:49:20.374568+0000","last_unstale":"2023-10-09T05:49:30.290454+0000","last_undegraded":"2023-10-09T05:49:20.366455+0000","last_fullsized":"2023-10-09T05:49:30.290454+0000","mapping_epoch":311,"log_start":"289'200","ondisk_log_start":"289'200","created":311,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6628,"log_dups_size":0,"ondisk_log_size":6628,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201228288,"num_objects":3071,"num_object_clones":0,"num_object_copies":9213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2870,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3444,"num_write_kb":208448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":3866624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"1,3,4","objects":314},{"shards":"2,3,4","objects":2757}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":108,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.150944+0000","last_change":"2023-10-09T05:46:56.194588+0000","last_active":"2023-10-09T05:49:21.150944+0000","last_peered":"2023-10-09T05:49:21.150944+0000","last_clean":"2023-10-09T05:49:21.150944+0000","last_became_active":"2023-10-09T05:46:56.018480+0000","last_became_peered":"2023-10-09T05:46:56.018480+0000","last_unstale":"2023-10-09T05:49:21.150944+0000","last_undegraded":"2023-10-09T05:49:21.150944+0000","last_fullsized":"2023-10-09T05:49:21.150944+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:39.150417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":319,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123816+0000","last_change":"2023-10-09T05:48:54.492012+0000","last_active":"2023-10-09T05:49:21.123816+0000","last_peered":"2023-10-09T05:49:21.123816+0000","last_clean":"2023-10-09T05:49:21.123816+0000","last_became_active":"2023-10-09T05:48:54.491644+0000","last_became_peered":"2023-10-09T05:48:54.491644+0000","last_unstale":"2023-10-09T05:49:21.123816+0000","last_undegraded":"2023-10-09T05:49:21.123816+0000","last_fullsized":"2023-10-09T05:49:21.123816+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"314'6105","reported_seq":6150,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:30.272117+0000","last_change":"2023-10-09T05:48:46.259028+0000","last_active":"2023-10-09T05:49:30.272117+0000","last_peered":"2023-10-09T05:49:30.272117+0000","last_clean":"2023-10-09T05:49:30.272117+0000","last_became_active":"2023-10-09T05:48:46.256441+0000","last_became_peered":"2023-10-09T05:48:46.256441+0000","last_unstale":"2023-10-09T05:49:30.272117+0000","last_undegraded":"2023-10-09T05:49:30.272117+0000","last_fullsized":"2023-10-09T05:49:30.272117+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6105,"log_dups_size":0,"ondisk_log_size":6105,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201687040,"num_objects":3077,"num_object_clones":0,"num_object_copies":9231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3377,"num_write_kb":206560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":97,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123829+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:49:21.123829+0000","last_peered":"2023-10-09T05:49:21.123829+0000","last_clean":"2023-10-09T05:49:21.123829+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:49:21.123829+0000","last_undegraded":"2023-10-09T05:49:21.123829+0000","last_fullsized":"2023-10-09T05:49:21.123829+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":257,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123583+0000","last_change":"2023-10-09T05:46:56.021669+0000","last_active":"2023-10-09T05:49:21.123583+0000","last_peered":"2023-10-09T05:49:21.123583+0000","last_clean":"2023-10-09T05:49:21.123583+0000","last_became_active":"2023-10-09T05:46:56.021306+0000","last_became_peered":"2023-10-09T05:46:56.021306+0000","last_unstale":"2023-10-09T05:49:21.123583+0000","last_undegraded":"2023-10-09T05:49:21.123583+0000","last_fullsized":"2023-10-09T05:49:21.123583+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"314'6080","reported_seq":6123,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:29.971768+0000","last_change":"2023-10-09T05:48:46.258066+0000","last_active":"2023-10-09T05:49:29.971768+0000","last_peered":"2023-10-09T05:49:29.971768+0000","last_clean":"2023-10-09T05:49:29.971768+0000","last_became_active":"2023-10-09T05:48:46.251759+0000","last_became_peered":"2023-10-09T05:48:46.251759+0000","last_unstale":"2023-10-09T05:49:29.971768+0000","last_undegraded":"2023-10-09T05:49:29.971768+0000","last_fullsized":"2023-10-09T05:49:29.971768+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6080,"log_dups_size":0,"ondisk_log_size":6080,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199786496,"num_objects":3048,"num_object_clones":0,"num_object_copies":9144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3390,"num_write_kb":206048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":123,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123891+0000","last_change":"2023-10-09T05:48:54.539425+0000","last_active":"2023-10-09T05:49:21.123891+0000","last_peered":"2023-10-09T05:49:21.123891+0000","last_clean":"2023-10-09T05:49:21.123891+0000","last_became_active":"2023-10-09T05:48:54.486401+0000","last_became_peered":"2023-10-09T05:48:54.486401+0000","last_unstale":"2023-10-09T05:49:21.123891+0000","last_undegraded":"2023-10-09T05:49:21.123891+0000","last_fullsized":"2023-10-09T05:49:21.123891+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":330,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136352+0000","last_change":"2023-10-09T05:48:54.492534+0000","last_active":"2023-10-09T05:49:21.136352+0000","last_peered":"2023-10-09T05:49:21.136352+0000","last_clean":"2023-10-09T05:49:21.136352+0000","last_became_active":"2023-10-09T05:48:54.492172+0000","last_became_peered":"2023-10-09T05:48:54.492172+0000","last_unstale":"2023-10-09T05:49:21.136352+0000","last_undegraded":"2023-10-09T05:49:21.136352+0000","last_fullsized":"2023-10-09T05:49:21.136352+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"314'6027","reported_seq":6075,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:28.810566+0000","last_change":"2023-10-09T05:48:46.220278+0000","last_active":"2023-10-09T05:49:28.810566+0000","last_peered":"2023-10-09T05:49:28.810566+0000","last_clean":"2023-10-09T05:49:28.810566+0000","last_became_active":"2023-10-09T05:48:46.215626+0000","last_became_peered":"2023-10-09T05:48:46.215626+0000","last_unstale":"2023-10-09T05:49:28.810566+0000","last_undegraded":"2023-10-09T05:49:28.810566+0000","last_fullsized":"2023-10-09T05:49:28.810566+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6027,"log_dups_size":0,"ondisk_log_size":6027,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197230592,"num_objects":3009,"num_object_clones":0,"num_object_copies":9027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3339,"num_write_kb":203168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":110,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123590+0000","last_change":"2023-10-09T05:46:56.190043+0000","last_active":"2023-10-09T05:49:21.123590+0000","last_peered":"2023-10-09T05:49:21.123590+0000","last_clean":"2023-10-09T05:49:21.123590+0000","last_became_active":"2023-10-09T05:46:56.019072+0000","last_became_peered":"2023-10-09T05:46:56.019072+0000","last_unstale":"2023-10-09T05:49:21.123590+0000","last_undegraded":"2023-10-09T05:49:21.123590+0000","last_fullsized":"2023-10-09T05:49:21.123590+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.169553+0000","last_change":"2023-10-09T05:48:54.489089+0000","last_active":"2023-10-09T05:49:21.169553+0000","last_peered":"2023-10-09T05:49:21.169553+0000","last_clean":"2023-10-09T05:49:21.169553+0000","last_became_active":"2023-10-09T05:48:54.488817+0000","last_became_peered":"2023-10-09T05:48:54.488817+0000","last_unstale":"2023-10-09T05:49:21.169553+0000","last_undegraded":"2023-10-09T05:49:21.169553+0000","last_fullsized":"2023-10-09T05:49:21.169553+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:40.555711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"314'6189","reported_seq":6247,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:26.938790+0000","last_change":"2023-10-09T05:48:56.022926+0000","last_active":"2023-10-09T05:49:26.938790+0000","last_peered":"2023-10-09T05:49:26.938790+0000","last_clean":"2023-10-09T05:48:53.410221+0000","last_became_active":"2023-10-09T05:48:56.013961+0000","last_became_peered":"2023-10-09T05:48:56.013961+0000","last_unstale":"2023-10-09T05:49:26.938790+0000","last_undegraded":"2023-10-09T05:48:55.541735+0000","last_fullsized":"2023-10-09T05:48:55.521053+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6189,"log_dups_size":0,"ondisk_log_size":6189,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202178560,"num_objects":3085,"num_object_clones":0,"num_object_copies":9255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2925,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3405,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2765},{"shards":"3,6","objects":160}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":281,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.140826+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:49:21.140826+0000","last_peered":"2023-10-09T05:49:21.140826+0000","last_clean":"2023-10-09T05:49:21.140826+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:49:21.140826+0000","last_undegraded":"2023-10-09T05:49:21.140826+0000","last_fullsized":"2023-10-09T05:49:21.140826+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":287,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.158065+0000","last_change":"2023-10-09T05:48:54.495800+0000","last_active":"2023-10-09T05:49:21.158065+0000","last_peered":"2023-10-09T05:49:21.158065+0000","last_clean":"2023-10-09T05:49:21.158065+0000","last_became_active":"2023-10-09T05:48:54.495540+0000","last_became_peered":"2023-10-09T05:48:54.495540+0000","last_unstale":"2023-10-09T05:49:21.158065+0000","last_undegraded":"2023-10-09T05:49:21.158065+0000","last_fullsized":"2023-10-09T05:49:21.158065+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:26.903669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"314'6140","reported_seq":6266,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:29.614741+0000","last_change":"2023-10-09T05:49:21.912515+0000","last_active":"2023-10-09T05:49:29.614741+0000","last_peered":"2023-10-09T05:49:29.614741+0000","last_clean":"2023-10-09T05:48:53.429181+0000","last_became_active":"2023-10-09T05:49:21.896767+0000","last_became_peered":"2023-10-09T05:49:21.896767+0000","last_unstale":"2023-10-09T05:49:29.614741+0000","last_undegraded":"2023-10-09T05:49:21.127213+0000","last_fullsized":"2023-10-09T05:49:29.614741+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6140,"log_dups_size":0,"ondisk_log_size":6140,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100958208,"num_objects":1540,"num_object_clones":0,"num_object_copies":4620,"num_objects_missing_on_primary":1448,"num_objects_missing":1448,"num_objects_degraded":1448,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1716,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":1966080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,4","objects":154},{"shards":"3,4,6","objects":1386}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":50,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.158146+0000","last_change":"2023-10-09T05:48:25.067945+0000","last_active":"2023-10-09T05:49:21.158146+0000","last_peered":"2023-10-09T05:49:21.158146+0000","last_clean":"2023-10-09T05:49:21.158146+0000","last_became_active":"2023-10-09T05:48:25.067721+0000","last_became_peered":"2023-10-09T05:48:25.067721+0000","last_unstale":"2023-10-09T05:49:21.158146+0000","last_undegraded":"2023-10-09T05:49:21.158146+0000","last_fullsized":"2023-10-09T05:49:21.158146+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:58.655544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":279,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136494+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:49:21.136494+0000","last_peered":"2023-10-09T05:49:21.136494+0000","last_clean":"2023-10-09T05:49:21.136494+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:49:21.136494+0000","last_undegraded":"2023-10-09T05:49:21.136494+0000","last_fullsized":"2023-10-09T05:49:21.136494+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"314'6056","reported_seq":6156,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:29.302935+0000","last_change":"2023-10-09T05:49:22.000684+0000","last_active":"2023-10-09T05:49:29.302935+0000","last_peered":"2023-10-09T05:49:29.302935+0000","last_clean":"2023-10-09T05:48:53.421353+0000","last_became_active":"2023-10-09T05:49:21.977937+0000","last_became_peered":"2023-10-09T05:49:21.977937+0000","last_unstale":"2023-10-09T05:49:29.302935+0000","last_undegraded":"2023-10-09T05:49:21.129646+0000","last_fullsized":"2023-10-09T05:49:29.302935+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6056,"log_dups_size":0,"ondisk_log_size":6056,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99221504,"num_objects":1514,"num_object_clones":0,"num_object_copies":4542,"num_objects_missing_on_primary":1423,"num_objects_missing":1423,"num_objects_degraded":1423,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1689,"num_write_kb":102496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":1376256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,1,2","objects":152},{"shards":"0,2,5","objects":1362}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":148,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136390+0000","last_change":"2023-10-09T05:48:54.494715+0000","last_active":"2023-10-09T05:49:21.136390+0000","last_peered":"2023-10-09T05:49:21.136390+0000","last_clean":"2023-10-09T05:49:21.136390+0000","last_became_active":"2023-10-09T05:48:54.493988+0000","last_became_peered":"2023-10-09T05:48:54.493988+0000","last_unstale":"2023-10-09T05:49:21.136390+0000","last_undegraded":"2023-10-09T05:49:21.136390+0000","last_fullsized":"2023-10-09T05:49:21.136390+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":312,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136680+0000","last_change":"2023-10-09T05:48:54.493193+0000","last_active":"2023-10-09T05:49:21.136680+0000","last_peered":"2023-10-09T05:49:21.136680+0000","last_clean":"2023-10-09T05:49:21.136680+0000","last_became_active":"2023-10-09T05:48:54.492886+0000","last_became_peered":"2023-10-09T05:48:54.492886+0000","last_unstale":"2023-10-09T05:49:21.136680+0000","last_undegraded":"2023-10-09T05:49:21.136680+0000","last_fullsized":"2023-10-09T05:49:21.136680+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"314'6001","reported_seq":6058,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:30.000559+0000","last_change":"2023-10-09T05:49:20.137476+0000","last_active":"2023-10-09T05:49:30.000559+0000","last_peered":"2023-10-09T05:49:30.000559+0000","last_clean":"2023-10-09T05:49:30.000559+0000","last_became_active":"2023-10-09T05:49:20.132759+0000","last_became_peered":"2023-10-09T05:49:20.132759+0000","last_unstale":"2023-10-09T05:49:30.000559+0000","last_undegraded":"2023-10-09T05:49:30.000559+0000","last_fullsized":"2023-10-09T05:49:30.000559+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6001,"log_dups_size":0,"ondisk_log_size":6001,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97927168,"num_objects":1494,"num_object_clones":0,"num_object_copies":4482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1679,"num_write_kb":101552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":313,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136046+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:49:21.136046+0000","last_peered":"2023-10-09T05:49:21.136046+0000","last_clean":"2023-10-09T05:49:21.136046+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:49:21.136046+0000","last_undegraded":"2023-10-09T05:49:21.136046+0000","last_fullsized":"2023-10-09T05:49:21.136046+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":306,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136833+0000","last_change":"2023-10-09T05:48:54.492708+0000","last_active":"2023-10-09T05:49:21.136833+0000","last_peered":"2023-10-09T05:49:21.136833+0000","last_clean":"2023-10-09T05:49:21.136833+0000","last_became_active":"2023-10-09T05:48:54.492382+0000","last_became_peered":"2023-10-09T05:48:54.492382+0000","last_unstale":"2023-10-09T05:49:21.136833+0000","last_undegraded":"2023-10-09T05:49:21.136833+0000","last_fullsized":"2023-10-09T05:49:21.136833+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"314'6036","reported_seq":6122,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:29.543969+0000","last_change":"2023-10-09T05:49:21.611985+0000","last_active":"2023-10-09T05:49:29.543969+0000","last_peered":"2023-10-09T05:49:29.543969+0000","last_clean":"2023-10-09T05:48:53.427903+0000","last_became_active":"2023-10-09T05:49:21.584088+0000","last_became_peered":"2023-10-09T05:49:21.584088+0000","last_unstale":"2023-10-09T05:49:29.543969+0000","last_undegraded":"2023-10-09T05:49:21.527023+0000","last_fullsized":"2023-10-09T05:49:29.543969+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6036,"log_dups_size":0,"ondisk_log_size":6036,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100106240,"num_objects":1527,"num_object_clones":0,"num_object_copies":4581,"num_objects_missing_on_primary":1434,"num_objects_missing":1434,"num_objects_degraded":1434,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1679,"num_write_kb":102624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":154},{"shards":"2,5,6","objects":1373}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.132389+0000","last_change":"2023-10-09T05:48:54.489891+0000","last_active":"2023-10-09T05:49:21.132389+0000","last_peered":"2023-10-09T05:49:21.132389+0000","last_clean":"2023-10-09T05:49:21.132389+0000","last_became_active":"2023-10-09T05:48:54.489570+0000","last_became_peered":"2023-10-09T05:48:54.489570+0000","last_unstale":"2023-10-09T05:49:21.132389+0000","last_undegraded":"2023-10-09T05:49:21.132389+0000","last_fullsized":"2023-10-09T05:49:21.132389+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:41.194371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.154034+0000","last_change":"2023-10-09T05:48:54.488131+0000","last_active":"2023-10-09T05:49:21.154034+0000","last_peered":"2023-10-09T05:49:21.154034+0000","last_clean":"2023-10-09T05:49:21.154034+0000","last_became_active":"2023-10-09T05:48:54.487748+0000","last_became_peered":"2023-10-09T05:48:54.487748+0000","last_unstale":"2023-10-09T05:49:21.154034+0000","last_undegraded":"2023-10-09T05:49:21.154034+0000","last_fullsized":"2023-10-09T05:49:21.154034+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:05.047252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"314'6728","reported_seq":6775,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:28.547185+0000","last_change":"2023-10-09T05:49:20.163537+0000","last_active":"2023-10-09T05:49:28.547185+0000","last_peered":"2023-10-09T05:49:28.547185+0000","last_clean":"2023-10-09T05:49:28.547185+0000","last_became_active":"2023-10-09T05:49:20.159521+0000","last_became_peered":"2023-10-09T05:49:20.159521+0000","last_unstale":"2023-10-09T05:49:28.547185+0000","last_undegraded":"2023-10-09T05:49:28.547185+0000","last_fullsized":"2023-10-09T05:49:28.547185+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6728,"log_dups_size":0,"ondisk_log_size":6728,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201981952,"num_objects":3082,"num_object_clones":0,"num_object_copies":9246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3381,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":681,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:23.132560+0000","last_change":"2023-10-09T05:48:55.609226+0000","last_active":"2023-10-09T05:49:23.132560+0000","last_peered":"2023-10-09T05:49:23.132560+0000","last_clean":"2023-10-09T05:48:52.714079+0000","last_became_active":"2023-10-09T05:48:55.608475+0000","last_became_peered":"2023-10-09T05:48:55.608475+0000","last_unstale":"2023-10-09T05:49:23.132560+0000","last_undegraded":"2023-10-09T05:48:55.476148+0000","last_fullsized":"2023-10-09T05:48:55.475626+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:14.257300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":296,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.140737+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:49:21.140737+0000","last_peered":"2023-10-09T05:49:21.140737+0000","last_clean":"2023-10-09T05:49:21.140737+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:49:21.140737+0000","last_undegraded":"2023-10-09T05:49:21.140737+0000","last_fullsized":"2023-10-09T05:49:21.140737+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.169456+0000","last_change":"2023-10-09T05:48:54.489994+0000","last_active":"2023-10-09T05:49:21.169456+0000","last_peered":"2023-10-09T05:49:21.169456+0000","last_clean":"2023-10-09T05:49:21.169456+0000","last_became_active":"2023-10-09T05:48:54.489701+0000","last_became_peered":"2023-10-09T05:48:54.489701+0000","last_unstale":"2023-10-09T05:49:21.169456+0000","last_undegraded":"2023-10-09T05:49:21.169456+0000","last_fullsized":"2023-10-09T05:49:21.169456+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:10.442883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"314'5990","reported_seq":6035,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:30.134819+0000","last_change":"2023-10-09T05:48:46.259058+0000","last_active":"2023-10-09T05:49:30.134819+0000","last_peered":"2023-10-09T05:49:30.134819+0000","last_clean":"2023-10-09T05:49:30.134819+0000","last_became_active":"2023-10-09T05:48:46.247029+0000","last_became_peered":"2023-10-09T05:48:46.247029+0000","last_unstale":"2023-10-09T05:49:30.134819+0000","last_undegraded":"2023-10-09T05:49:30.134819+0000","last_fullsized":"2023-10-09T05:49:30.134819+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":5990,"log_dups_size":0,"ondisk_log_size":5990,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197001216,"num_objects":3006,"num_object_clones":0,"num_object_copies":9018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3308,"num_write_kb":202048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":804,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.141383+0000","last_change":"2023-10-09T05:48:54.592642+0000","last_active":"2023-10-09T05:49:21.141383+0000","last_peered":"2023-10-09T05:49:21.141383+0000","last_clean":"2023-10-09T05:49:21.141383+0000","last_became_active":"2023-10-09T05:48:54.491917+0000","last_became_peered":"2023-10-09T05:48:54.491917+0000","last_unstale":"2023-10-09T05:49:21.141383+0000","last_undegraded":"2023-10-09T05:49:21.141383+0000","last_fullsized":"2023-10-09T05:49:21.141383+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":280,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.142170+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:49:21.142170+0000","last_peered":"2023-10-09T05:49:21.142170+0000","last_clean":"2023-10-09T05:49:21.142170+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:49:21.142170+0000","last_undegraded":"2023-10-09T05:49:21.142170+0000","last_fullsized":"2023-10-09T05:49:21.142170+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.121867+0000","last_change":"2023-10-09T05:48:54.492183+0000","last_active":"2023-10-09T05:49:21.121867+0000","last_peered":"2023-10-09T05:49:21.121867+0000","last_clean":"2023-10-09T05:49:21.121867+0000","last_became_active":"2023-10-09T05:48:54.491382+0000","last_became_peered":"2023-10-09T05:48:54.491382+0000","last_unstale":"2023-10-09T05:49:21.121867+0000","last_undegraded":"2023-10-09T05:49:21.121867+0000","last_fullsized":"2023-10-09T05:49:21.121867+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:06.256308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"314'6825","reported_seq":6998,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:30.245336+0000","last_change":"2023-10-09T05:49:21.590522+0000","last_active":"2023-10-09T05:49:30.245336+0000","last_peered":"2023-10-09T05:49:30.245336+0000","last_clean":"2023-10-09T05:48:53.462109+0000","last_became_active":"2023-10-09T05:49:21.586314+0000","last_became_peered":"2023-10-09T05:49:21.586314+0000","last_unstale":"2023-10-09T05:49:30.245336+0000","last_undegraded":"2023-10-09T05:49:21.140419+0000","last_fullsized":"2023-10-09T05:49:30.245336+0000","mapping_epoch":312,"log_start":"289'200","ondisk_log_start":"289'200","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6625,"log_dups_size":0,"ondisk_log_size":6625,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201424896,"num_objects":3073,"num_object_clones":0,"num_object_copies":9219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2848,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3440,"num_write_kb":208448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":3670016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"1,3,4","objects":365},{"shards":"2,3,4","objects":2708}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":632,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136828+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:49:21.136828+0000","last_peered":"2023-10-09T05:49:21.136828+0000","last_clean":"2023-10-09T05:49:21.136828+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:49:21.136828+0000","last_undegraded":"2023-10-09T05:49:21.136828+0000","last_fullsized":"2023-10-09T05:49:21.136828+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.132611+0000","last_change":"2023-10-09T05:48:54.490716+0000","last_active":"2023-10-09T05:49:21.132611+0000","last_peered":"2023-10-09T05:49:21.132611+0000","last_clean":"2023-10-09T05:49:21.132611+0000","last_became_active":"2023-10-09T05:48:54.490350+0000","last_became_peered":"2023-10-09T05:48:54.490350+0000","last_unstale":"2023-10-09T05:49:21.132611+0000","last_undegraded":"2023-10-09T05:49:21.132611+0000","last_fullsized":"2023-10-09T05:49:21.132611+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:20.190860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.153946+0000","last_change":"2023-10-09T05:48:54.492185+0000","last_active":"2023-10-09T05:49:21.153946+0000","last_peered":"2023-10-09T05:49:21.153946+0000","last_clean":"2023-10-09T05:49:21.153946+0000","last_became_active":"2023-10-09T05:48:54.491885+0000","last_became_peered":"2023-10-09T05:48:54.491885+0000","last_unstale":"2023-10-09T05:49:21.153946+0000","last_undegraded":"2023-10-09T05:49:21.153946+0000","last_fullsized":"2023-10-09T05:49:21.153946+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:26.045180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"314'6180","reported_seq":6228,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:28.781221+0000","last_change":"2023-10-09T05:48:46.220263+0000","last_active":"2023-10-09T05:49:28.781221+0000","last_peered":"2023-10-09T05:49:28.781221+0000","last_clean":"2023-10-09T05:49:28.781221+0000","last_became_active":"2023-10-09T05:48:46.215666+0000","last_became_peered":"2023-10-09T05:48:46.215666+0000","last_unstale":"2023-10-09T05:49:28.781221+0000","last_undegraded":"2023-10-09T05:49:28.781221+0000","last_fullsized":"2023-10-09T05:49:28.781221+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6180,"log_dups_size":0,"ondisk_log_size":6180,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203161600,"num_objects":3100,"num_object_clones":0,"num_object_copies":9300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3432,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":960,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.141928+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:49:21.141928+0000","last_peered":"2023-10-09T05:49:21.141928+0000","last_clean":"2023-10-09T05:49:21.141928+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:49:21.141928+0000","last_undegraded":"2023-10-09T05:49:21.141928+0000","last_fullsized":"2023-10-09T05:49:21.141928+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":464,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123193+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:49:21.123193+0000","last_peered":"2023-10-09T05:49:21.123193+0000","last_clean":"2023-10-09T05:49:21.123193+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:49:21.123193+0000","last_undegraded":"2023-10-09T05:49:21.123193+0000","last_fullsized":"2023-10-09T05:49:21.123193+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":86,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.141215+0000","last_change":"2023-10-09T05:48:25.068759+0000","last_active":"2023-10-09T05:49:21.141215+0000","last_peered":"2023-10-09T05:49:21.141215+0000","last_clean":"2023-10-09T05:49:21.141215+0000","last_became_active":"2023-10-09T05:48:25.068358+0000","last_became_peered":"2023-10-09T05:48:25.068358+0000","last_unstale":"2023-10-09T05:49:21.141215+0000","last_undegraded":"2023-10-09T05:49:21.141215+0000","last_fullsized":"2023-10-09T05:49:21.141215+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":281,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136764+0000","last_change":"2023-10-09T05:44:50.495966+0000","last_active":"2023-10-09T05:49:21.136764+0000","last_peered":"2023-10-09T05:49:21.136764+0000","last_clean":"2023-10-09T05:49:21.136764+0000","last_became_active":"2023-10-09T05:44:50.495628+0000","last_became_peered":"2023-10-09T05:44:50.495628+0000","last_unstale":"2023-10-09T05:49:21.136764+0000","last_undegraded":"2023-10-09T05:49:21.136764+0000","last_fullsized":"2023-10-09T05:49:21.136764+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"314'6680","reported_seq":6822,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:29.607684+0000","last_change":"2023-10-09T05:49:23.419408+0000","last_active":"2023-10-09T05:49:29.607684+0000","last_peered":"2023-10-09T05:49:29.607684+0000","last_clean":"2023-10-09T05:48:53.473854+0000","last_became_active":"2023-10-09T05:49:23.402116+0000","last_became_peered":"2023-10-09T05:49:23.402116+0000","last_unstale":"2023-10-09T05:49:29.607684+0000","last_undegraded":"2023-10-09T05:49:21.176409+0000","last_fullsized":"2023-10-09T05:49:29.607684+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6680,"log_dups_size":0,"ondisk_log_size":6680,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199229440,"num_objects":3040,"num_object_clones":0,"num_object_copies":9120,"num_objects_missing_on_primary":2839,"num_objects_missing":2839,"num_objects_degraded":2839,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3363,"num_write_kb":204896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":3014656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2,3","objects":316},{"shards":"2,3,4","objects":2724}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":805,"reported_epoch":313,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:21.123487+0000","last_change":"2023-10-09T05:48:55.810113+0000","last_active":"2023-10-09T05:49:21.123487+0000","last_peered":"2023-10-09T05:49:21.123487+0000","last_clean":"2023-10-09T05:48:52.442052+0000","last_became_active":"2023-10-09T05:48:55.809751+0000","last_became_peered":"2023-10-09T05:48:55.809751+0000","last_unstale":"2023-10-09T05:49:21.123487+0000","last_undegraded":"2023-10-09T05:48:55.477697+0000","last_fullsized":"2023-10-09T05:48:55.476822+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,0],"avail_no_missing":["7","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":288,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.157833+0000","last_change":"2023-10-09T05:47:33.926192+0000","last_active":"2023-10-09T05:49:21.157833+0000","last_peered":"2023-10-09T05:49:21.157833+0000","last_clean":"2023-10-09T05:49:21.157833+0000","last_became_active":"2023-10-09T05:47:33.925588+0000","last_became_peered":"2023-10-09T05:47:33.925588+0000","last_unstale":"2023-10-09T05:49:21.157833+0000","last_undegraded":"2023-10-09T05:49:21.157833+0000","last_fullsized":"2023-10-09T05:49:21.157833+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:29.495701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":303,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:23.132976+0000","last_change":"2023-10-09T05:48:54.489115+0000","last_active":"2023-10-09T05:49:23.132976+0000","last_peered":"2023-10-09T05:49:23.132976+0000","last_clean":"2023-10-09T05:49:23.132976+0000","last_became_active":"2023-10-09T05:48:54.488783+0000","last_became_peered":"2023-10-09T05:48:54.488783+0000","last_unstale":"2023-10-09T05:49:23.132976+0000","last_undegraded":"2023-10-09T05:49:23.132976+0000","last_fullsized":"2023-10-09T05:49:23.132976+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:13.193778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"314'6604","reported_seq":6738,"reported_epoch":314,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:29.594137+0000","last_change":"2023-10-09T05:49:23.636172+0000","last_active":"2023-10-09T05:49:29.594137+0000","last_peered":"2023-10-09T05:49:29.594137+0000","last_clean":"2023-10-09T05:48:53.467734+0000","last_became_active":"2023-10-09T05:49:22.482315+0000","last_became_peered":"2023-10-09T05:49:22.482315+0000","last_unstale":"2023-10-09T05:49:29.594137+0000","last_undegraded":"2023-10-09T05:49:21.143637+0000","last_fullsized":"2023-10-09T05:49:29.594137+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6604,"log_dups_size":0,"ondisk_log_size":6604,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196640768,"num_objects":3000,"num_object_clones":0,"num_object_copies":9000,"num_objects_missing_on_primary":2775,"num_objects_missing":2775,"num_objects_degraded":2775,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3320,"num_write_kb":202272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":2490368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,4","objects":350},{"shards":"1,3,4,6","objects":3},{"shards":"3,4,6","objects":2647}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":497,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123714+0000","last_change":"2023-10-09T05:44:51.489799+0000","last_active":"2023-10-09T05:49:21.123714+0000","last_peered":"2023-10-09T05:49:21.123714+0000","last_clean":"2023-10-09T05:49:21.123714+0000","last_became_active":"2023-10-09T05:44:51.489301+0000","last_became_peered":"2023-10-09T05:44:51.489301+0000","last_unstale":"2023-10-09T05:49:21.123714+0000","last_undegraded":"2023-10-09T05:49:21.123714+0000","last_fullsized":"2023-10-09T05:49:21.123714+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.121951+0000","last_change":"2023-10-09T05:48:54.491374+0000","last_active":"2023-10-09T05:49:21.121951+0000","last_peered":"2023-10-09T05:49:21.121951+0000","last_clean":"2023-10-09T05:49:21.121951+0000","last_became_active":"2023-10-09T05:48:54.490889+0000","last_became_peered":"2023-10-09T05:48:54.490889+0000","last_unstale":"2023-10-09T05:49:21.121951+0000","last_undegraded":"2023-10-09T05:49:21.121951+0000","last_fullsized":"2023-10-09T05:49:21.121951+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:34.755791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":288,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.158347+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:49:21.158347+0000","last_peered":"2023-10-09T05:49:21.158347+0000","last_clean":"2023-10-09T05:49:21.158347+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:49:21.158347+0000","last_undegraded":"2023-10-09T05:49:21.158347+0000","last_fullsized":"2023-10-09T05:49:21.158347+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"314'6896","reported_seq":6944,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:28.799373+0000","last_change":"2023-10-09T05:49:20.373182+0000","last_active":"2023-10-09T05:49:28.799373+0000","last_peered":"2023-10-09T05:49:28.799373+0000","last_clean":"2023-10-09T05:49:28.799373+0000","last_became_active":"2023-10-09T05:49:20.351645+0000","last_became_peered":"2023-10-09T05:49:20.351645+0000","last_unstale":"2023-10-09T05:49:28.799373+0000","last_undegraded":"2023-10-09T05:49:28.799373+0000","last_fullsized":"2023-10-09T05:49:28.799373+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6896,"log_dups_size":0,"ondisk_log_size":6896,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204341248,"num_objects":3118,"num_object_clones":0,"num_object_copies":9354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3464,"num_write_kb":210624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":261,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123928+0000","last_change":"2023-10-09T05:48:25.066163+0000","last_active":"2023-10-09T05:49:21.123928+0000","last_peered":"2023-10-09T05:49:21.123928+0000","last_clean":"2023-10-09T05:49:21.123928+0000","last_became_active":"2023-10-09T05:48:25.065145+0000","last_became_peered":"2023-10-09T05:48:25.065145+0000","last_unstale":"2023-10-09T05:49:21.123928+0000","last_undegraded":"2023-10-09T05:49:21.123928+0000","last_fullsized":"2023-10-09T05:49:21.123928+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":479,"reported_epoch":313,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T05:49:21.136862+0000","last_change":"2023-10-09T05:48:55.619562+0000","last_active":"2023-10-09T05:49:21.136862+0000","last_peered":"2023-10-09T05:49:21.136862+0000","last_clean":"2023-10-09T05:22:39.598843+0000","last_became_active":"2023-10-09T05:48:55.618332+0000","last_became_peered":"2023-10-09T05:48:55.618332+0000","last_unstale":"2023-10-09T05:49:21.136862+0000","last_undegraded":"2023-10-09T05:49:21.136862+0000","last_fullsized":"2023-10-09T05:49:21.136862+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:01.870836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[3,6,4],"avail_no_missing":["3","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":254,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123493+0000","last_change":"2023-10-09T05:48:25.067326+0000","last_active":"2023-10-09T05:49:21.123493+0000","last_peered":"2023-10-09T05:49:21.123493+0000","last_clean":"2023-10-09T05:49:21.123493+0000","last_became_active":"2023-10-09T05:48:25.066977+0000","last_became_peered":"2023-10-09T05:48:25.066977+0000","last_unstale":"2023-10-09T05:49:21.123493+0000","last_undegraded":"2023-10-09T05:49:21.123493+0000","last_fullsized":"2023-10-09T05:49:21.123493+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"314'6664","reported_seq":6712,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:30.303621+0000","last_change":"2023-10-09T05:49:20.153015+0000","last_active":"2023-10-09T05:49:30.303621+0000","last_peered":"2023-10-09T05:49:30.303621+0000","last_clean":"2023-10-09T05:49:30.303621+0000","last_became_active":"2023-10-09T05:49:20.148237+0000","last_became_peered":"2023-10-09T05:49:20.148237+0000","last_unstale":"2023-10-09T05:49:30.303621+0000","last_undegraded":"2023-10-09T05:49:30.303621+0000","last_fullsized":"2023-10-09T05:49:30.303621+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6664,"log_dups_size":0,"ondisk_log_size":6664,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196837388,"num_objects":3004,"num_object_clones":0,"num_object_copies":9012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3358,"num_write_kb":203552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":811,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.123897+0000","last_change":"2023-10-09T05:44:50.482211+0000","last_active":"2023-10-09T05:49:21.123897+0000","last_peered":"2023-10-09T05:49:21.123897+0000","last_clean":"2023-10-09T05:49:21.123897+0000","last_became_active":"2023-10-09T05:44:50.481749+0000","last_became_peered":"2023-10-09T05:44:50.481749+0000","last_unstale":"2023-10-09T05:49:21.123897+0000","last_undegraded":"2023-10-09T05:49:21.123897+0000","last_fullsized":"2023-10-09T05:49:21.123897+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":299,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:23.133246+0000","last_change":"2023-10-09T05:48:54.585691+0000","last_active":"2023-10-09T05:49:23.133246+0000","last_peered":"2023-10-09T05:49:23.133246+0000","last_clean":"2023-10-09T05:49:23.133246+0000","last_became_active":"2023-10-09T05:48:54.498259+0000","last_became_peered":"2023-10-09T05:48:54.498259+0000","last_unstale":"2023-10-09T05:49:23.133246+0000","last_undegraded":"2023-10-09T05:49:23.133246+0000","last_fullsized":"2023-10-09T05:49:23.133246+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.684612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":264,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136032+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:49:21.136032+0000","last_peered":"2023-10-09T05:49:21.136032+0000","last_clean":"2023-10-09T05:49:21.136032+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:49:21.136032+0000","last_undegraded":"2023-10-09T05:49:21.136032+0000","last_fullsized":"2023-10-09T05:49:21.136032+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"314'6127","reported_seq":11755,"reported_epoch":314,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:49:27.149429+0000","last_change":"2023-10-09T05:48:54.551295+0000","last_active":"2023-10-09T05:49:27.149429+0000","last_peered":"2023-10-09T05:49:27.149429+0000","last_clean":"2023-10-09T05:48:50.362911+0000","last_became_active":"2023-10-09T05:48:52.776305+0000","last_became_peered":"2023-10-09T05:48:52.776305+0000","last_unstale":"2023-10-09T05:49:27.149429+0000","last_undegraded":"2023-10-09T05:49:27.149429+0000","last_fullsized":"2023-10-09T05:48:52.436423+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6127,"log_dups_size":0,"ondisk_log_size":6127,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204210176,"num_objects":3116,"num_object_clones":0,"num_object_copies":9348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2074,"num_objects_unfound":0,"num_objects_dirty":3116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3390,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1855,"num_bytes_recovered":121569280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4","objects":2079},{"shards":"3,4,6","objects":1037}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":289,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.158240+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:49:21.158240+0000","last_peered":"2023-10-09T05:49:21.158240+0000","last_clean":"2023-10-09T05:49:21.158240+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:49:21.158240+0000","last_undegraded":"2023-10-09T05:49:21.158240+0000","last_fullsized":"2023-10-09T05:49:21.158240+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":19,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.122014+0000","last_change":"2023-10-09T05:48:54.492394+0000","last_active":"2023-10-09T05:49:21.122014+0000","last_peered":"2023-10-09T05:49:21.122014+0000","last_clean":"2023-10-09T05:49:21.122014+0000","last_became_active":"2023-10-09T05:48:54.491682+0000","last_became_peered":"2023-10-09T05:48:54.491682+0000","last_unstale":"2023-10-09T05:49:21.122014+0000","last_undegraded":"2023-10-09T05:49:21.122014+0000","last_fullsized":"2023-10-09T05:49:21.122014+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:28.838164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.17","version":"314'6216","reported_seq":6274,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:26.788740+0000","last_change":"2023-10-09T05:48:56.019001+0000","last_active":"2023-10-09T05:49:26.788740+0000","last_peered":"2023-10-09T05:49:26.788740+0000","last_clean":"2023-10-09T05:48:53.462142+0000","last_became_active":"2023-10-09T05:48:56.004223+0000","last_became_peered":"2023-10-09T05:48:56.004223+0000","last_unstale":"2023-10-09T05:49:26.788740+0000","last_undegraded":"2023-10-09T05:48:55.514193+0000","last_fullsized":"2023-10-09T05:48:55.479940+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6216,"log_dups_size":0,"ondisk_log_size":6216,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205783040,"num_objects":3140,"num_object_clones":0,"num_object_copies":9420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2986,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3432,"num_write_kb":210304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2840},{"shards":"3,6","objects":154}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":309,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.140741+0000","last_change":"2023-10-09T05:48:54.583404+0000","last_active":"2023-10-09T05:49:21.140741+0000","last_peered":"2023-10-09T05:49:21.140741+0000","last_clean":"2023-10-09T05:49:21.140741+0000","last_became_active":"2023-10-09T05:48:54.492565+0000","last_became_peered":"2023-10-09T05:48:54.492565+0000","last_unstale":"2023-10-09T05:49:21.140741+0000","last_undegraded":"2023-10-09T05:49:21.140741+0000","last_fullsized":"2023-10-09T05:49:21.140741+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":312,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.142176+0000","last_change":"2023-10-09T05:48:54.493079+0000","last_active":"2023-10-09T05:49:21.142176+0000","last_peered":"2023-10-09T05:49:21.142176+0000","last_clean":"2023-10-09T05:49:21.142176+0000","last_became_active":"2023-10-09T05:48:54.492013+0000","last_became_peered":"2023-10-09T05:48:54.492013+0000","last_unstale":"2023-10-09T05:49:21.142176+0000","last_undegraded":"2023-10-09T05:49:21.142176+0000","last_fullsized":"2023-10-09T05:49:21.142176+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"314'6049","reported_seq":6097,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:28.673681+0000","last_change":"2023-10-09T05:48:46.260747+0000","last_active":"2023-10-09T05:49:28.673681+0000","last_peered":"2023-10-09T05:49:28.673681+0000","last_clean":"2023-10-09T05:49:28.673681+0000","last_became_active":"2023-10-09T05:48:46.258900+0000","last_became_peered":"2023-10-09T05:48:46.258900+0000","last_unstale":"2023-10-09T05:49:28.673681+0000","last_undegraded":"2023-10-09T05:49:28.673681+0000","last_fullsized":"2023-10-09T05:49:28.673681+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6049,"log_dups_size":0,"ondisk_log_size":6049,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199852032,"num_objects":3050,"num_object_clones":0,"num_object_copies":9150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3362,"num_write_kb":205152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":268,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.140667+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:49:21.140667+0000","last_peered":"2023-10-09T05:49:21.140667+0000","last_clean":"2023-10-09T05:49:21.140667+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:49:21.140667+0000","last_undegraded":"2023-10-09T05:49:21.140667+0000","last_fullsized":"2023-10-09T05:49:21.140667+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":109,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.157231+0000","last_change":"2023-10-09T05:48:25.062693+0000","last_active":"2023-10-09T05:49:21.157231+0000","last_peered":"2023-10-09T05:49:21.157231+0000","last_clean":"2023-10-09T05:49:21.157231+0000","last_became_active":"2023-10-09T05:48:25.062334+0000","last_became_peered":"2023-10-09T05:48:25.062334+0000","last_unstale":"2023-10-09T05:49:21.157231+0000","last_undegraded":"2023-10-09T05:49:21.157231+0000","last_fullsized":"2023-10-09T05:49:21.157231+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:33:15.975841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.15","version":"314'6015","reported_seq":6063,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:30.138763+0000","last_change":"2023-10-09T05:48:46.308705+0000","last_active":"2023-10-09T05:49:30.138763+0000","last_peered":"2023-10-09T05:49:30.138763+0000","last_clean":"2023-10-09T05:49:30.138763+0000","last_became_active":"2023-10-09T05:48:46.258676+0000","last_became_peered":"2023-10-09T05:48:46.258676+0000","last_unstale":"2023-10-09T05:49:30.138763+0000","last_undegraded":"2023-10-09T05:49:30.138763+0000","last_fullsized":"2023-10-09T05:49:30.138763+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6015,"log_dups_size":0,"ondisk_log_size":6015,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196837376,"num_objects":3004,"num_object_clones":0,"num_object_copies":9012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3326,"num_write_kb":202528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":31,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.138529+0000","last_change":"2023-10-09T05:48:25.069406+0000","last_active":"2023-10-09T05:49:21.138529+0000","last_peered":"2023-10-09T05:49:21.138529+0000","last_clean":"2023-10-09T05:49:21.138529+0000","last_became_active":"2023-10-09T05:48:25.069064+0000","last_became_peered":"2023-10-09T05:48:25.069064+0000","last_unstale":"2023-10-09T05:49:21.138529+0000","last_undegraded":"2023-10-09T05:49:21.138529+0000","last_fullsized":"2023-10-09T05:49:21.138529+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.567858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.136749+0000","last_change":"2023-10-09T05:48:25.063854+0000","last_active":"2023-10-09T05:49:21.136749+0000","last_peered":"2023-10-09T05:49:21.136749+0000","last_clean":"2023-10-09T05:49:21.136749+0000","last_became_active":"2023-10-09T05:48:25.063579+0000","last_became_peered":"2023-10-09T05:48:25.063579+0000","last_unstale":"2023-10-09T05:49:21.136749+0000","last_undegraded":"2023-10-09T05:49:21.136749+0000","last_fullsized":"2023-10-09T05:49:21.136749+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:02.389789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.14","version":"314'6111","reported_seq":6159,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T05:49:30.280901+0000","last_change":"2023-10-09T05:48:46.325697+0000","last_active":"2023-10-09T05:49:30.280901+0000","last_peered":"2023-10-09T05:49:30.280901+0000","last_clean":"2023-10-09T05:49:30.280901+0000","last_became_active":"2023-10-09T05:48:46.268207+0000","last_became_peered":"2023-10-09T05:48:46.268207+0000","last_unstale":"2023-10-09T05:49:30.280901+0000","last_undegraded":"2023-10-09T05:49:30.280901+0000","last_fullsized":"2023-10-09T05:49:30.280901+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6111,"log_dups_size":0,"ondisk_log_size":6111,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202997760,"num_objects":3098,"num_object_clones":0,"num_object_copies":9294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3384,"num_write_kb":207392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":288,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.157908+0000","last_change":"2023-10-09T05:46:56.197243+0000","last_active":"2023-10-09T05:49:21.157908+0000","last_peered":"2023-10-09T05:49:21.157908+0000","last_clean":"2023-10-09T05:49:21.157908+0000","last_became_active":"2023-10-09T05:46:56.022978+0000","last_became_peered":"2023-10-09T05:46:56.022978+0000","last_unstale":"2023-10-09T05:49:21.157908+0000","last_undegraded":"2023-10-09T05:49:21.157908+0000","last_fullsized":"2023-10-09T05:49:21.157908+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:08.905002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":297,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.137112+0000","last_change":"2023-10-09T05:48:54.492580+0000","last_active":"2023-10-09T05:49:21.137112+0000","last_peered":"2023-10-09T05:49:21.137112+0000","last_clean":"2023-10-09T05:49:21.137112+0000","last_became_active":"2023-10-09T05:48:54.492259+0000","last_became_peered":"2023-10-09T05:48:54.492259+0000","last_unstale":"2023-10-09T05:49:21.137112+0000","last_undegraded":"2023-10-09T05:49:21.137112+0000","last_fullsized":"2023-10-09T05:49:21.137112+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":10308,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":36,"stat_sum":{"num_bytes":6402015256,"num_objects":97688,"num_object_clones":0,"num_object_copies":293064,"num_objects_missing_on_primary":19734,"num_objects_missing":19734,"num_objects_degraded":37028,"num_objects_misplaced":2074,"num_objects_unfound":0,"num_objects_dirty":97688,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":108192,"num_write_kb":6588097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2281,"num_bytes_recovered":148897792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19705638912,"data_stored":19705626696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":226301,"ondisk_log_size":226301,"up":108,"acting":103,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846771,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3024348,"kb_used_data":2102164,"kb_used_omap":14,"kb_used_meta":922097,"kb_avail":90720804,"statfs":{"total":95995035648,"available":92898103296,"internally_reserved":0,"allocated":2152615936,"data_stored":2151973134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":944228186},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":5.4740000000000002,"5min":3.851,"15min":4.4930000000000003},"min":{"1min":0.23400000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":79.307000000000002,"5min":110.123,"15min":220.77500000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":5.4269999999999996,"5min":3.8159999999999998,"15min":4.6909999999999998},"min":{"1min":0.29699999999999999,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":80.269999999999996,"5min":110.13800000000001,"15min":220.78700000000001},"last":0.51600000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.77300000000000002,"15min":2.8170000000000002},"min":{"1min":0.27800000000000002,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":2.8330000000000002,"5min":7.7729999999999997,"15min":190.68100000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.90100000000000002,"15min":2.7549999999999999},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.255},"max":{"1min":4.2789999999999999,"5min":4.5510000000000002,"15min":191.09200000000001},"last":0.30399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":1.014,"15min":2.6659999999999999},"min":{"1min":0.29499999999999998,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":4.6200000000000001,"5min":7.8170000000000002,"15min":78.716999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.0469999999999999,"5min":0.93700000000000006,"15min":2.5990000000000002},"min":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":3.5510000000000002,"5min":3.5510000000000002,"15min":78.768000000000001},"last":0.46800000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":8.1210000000000004,"5min":3.8450000000000002,"15min":4.0650000000000004},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":69.066000000000003,"5min":69.066000000000003,"15min":185.17099999999999},"last":0.44},{"interface":"front","average":{"1min":8.0099999999999998,"5min":3.786,"15min":4.2000000000000002},"min":{"1min":0.36399999999999999,"5min":0.33000000000000002,"15min":0.29799999999999999},"max":{"1min":69.055999999999997,"5min":69.055999999999997,"15min":183.35499999999999},"last":0.501}]},{"osd":4,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.77300000000000002,"15min":0.86899999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":3.9990000000000001,"5min":3.9990000000000001,"15min":12.789},"last":0.376},{"interface":"front","average":{"1min":1.4039999999999999,"5min":0.86599999999999999,"15min":1.0129999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":9.7829999999999995,"5min":9.7829999999999995,"15min":9.7829999999999995},"last":0.45500000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":0.96299999999999997,"15min":1.075},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":4.2949999999999999,"5min":5.4950000000000001,"15min":10.813000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.3260000000000001,"5min":0.90300000000000002,"15min":1.1679999999999999},"min":{"1min":0.371,"5min":0.34200000000000003,"15min":0.28299999999999997},"max":{"1min":4.1520000000000001,"5min":4.7649999999999997,"15min":10.321},"last":0.39400000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.79000000000000004,"15min":1.0109999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.318},"max":{"1min":4.3259999999999996,"5min":4.3259999999999996,"15min":9.2530000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.726,"5min":0.94499999999999995,"15min":1.024},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.30599999999999999},"max":{"1min":8.0670000000000002,"5min":8.0670000000000002,"15min":8.2680000000000007},"last":0.432}]}]},{"osd":6,"up_from":42,"seq":180388626796,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3309776,"kb_used_data":2711880,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":90435376,"statfs":{"total":95995035648,"available":92605825024,"internally_reserved":0,"allocated":2776965120,"data_stored":2776334744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:35 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.4970000000000001,"15min":4.5999999999999996},"min":{"1min":0.32900000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.0570000000000004,"5min":67.349999999999994,"15min":162.08799999999999},"last":42.869},{"interface":"front","average":{"1min":1.397,"5min":1.9199999999999999,"15min":4.423},"min":{"1min":0.313,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.7650000000000001,"5min":67.206000000000003,"15min":161.90899999999999},"last":42.892000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":0.96699999999999997,"15min":3.8029999999999999},"min":{"1min":0.33000000000000002,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":4.9779999999999998,"5min":6.5279999999999996,"15min":228.24299999999999},"last":48.729999999999997},{"interface":"front","average":{"1min":1.4370000000000001,"5min":0.83999999999999997,"15min":3.7799999999999998},"min":{"1min":0.39200000000000002,"5min":0.312,"15min":0.252},"max":{"1min":5.1829999999999998,"5min":5.1829999999999998,"15min":228.22499999999999},"last":48.741999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:48:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.5209999999999999,"15min":3.9860000000000002},"min":{"1min":0.36599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.5430000000000001,"5min":72.450999999999993,"15min":115.40600000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.379,"5min":1.698,"15min":4.0599999999999996},"min":{"1min":0.33500000000000002,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":4.9459999999999997,"5min":72.466999999999999,"15min":115.297},"last":0.40600000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":1.8440000000000001,"15min":3.8420000000000001},"min":{"1min":0.33300000000000002,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":19.52,"5min":77.578999999999994,"15min":115.244},"last":0.372},{"interface":"front","average":{"1min":2.589,"5min":1.952,"15min":3.8959999999999999},"min":{"1min":0.33800000000000002,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":19.231999999999999,"5min":77.516999999999996,"15min":115.34999999999999},"last":0.42199999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:49:24 2023","interfaces":[{"interface":"back","average":{"1min":15.987,"5min":3.7519999999999998,"15min":2.774},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24099999999999999},"max":{"1min":257.375,"5min":257.375,"15min":257.375},"last":0.39100000000000001},{"interface":"front","average":{"1min":16.145,"5min":4.0430000000000001,"15min":2.8620000000000001},"min":{"1min":0.29099999999999998,"5min":0.246,"15min":0.246},"max":{"1min":257.61200000000002,"5min":257.61200000000002,"15min":257.61200000000002},"last":0.34799999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:49:24 2023","interfaces":[{"interface":"back","average":{"1min":16.251999999999999,"5min":3.8570000000000002,"15min":2.9820000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":256.05700000000002,"5min":256.05700000000002,"15min":256.05700000000002},"last":0.439},{"interface":"front","average":{"1min":16.359000000000002,"5min":4.1959999999999997,"15min":2.956},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.27500000000000002},"max":{"1min":256.08499999999998,"5min":256.08499999999998,"15min":256.08499999999998},"last":0.80400000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":0.77400000000000002,"15min":1.522},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26500000000000001},"max":{"1min":5.2279999999999998,"5min":5.2279999999999998,"15min":115.514},"last":0.54300000000000004},{"interface":"front","average":{"1min":2.0499999999999998,"5min":1.2769999999999999,"15min":1.472},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":6.2450000000000001,"5min":14.300000000000001,"15min":116.53100000000001},"last":0.47899999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958758,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":771280,"kb_used_data":283336,"kb_used_omap":7,"kb_used_meta":487928,"kb_avail":92973872,"statfs":{"total":95995035648,"available":95205244928,"internally_reserved":0,"allocated":290136064,"data_stored":289096809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":499638798},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1,3,5,3,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":2.7429999999999999,"15min":4.5010000000000003},"min":{"1min":0.224,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":3.7890000000000001,"5min":61.222999999999999,"15min":207.858},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.70899999999999996,"5min":2.956,"15min":5.0540000000000003},"min":{"1min":0.27000000000000002,"5min":0.23300000000000001,"15min":0.182},"max":{"1min":3.9060000000000001,"5min":61.366999999999997,"15min":206.97200000000001},"last":0.30199999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:49:27 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":1.6080000000000001,"15min":4.6379999999999999},"min":{"1min":0.24199999999999999,"5min":0.19800000000000001,"15min":0.184},"max":{"1min":10.210000000000001,"5min":17.768999999999998,"15min":101.367},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.3580000000000001,"5min":1.4410000000000001,"15min":4.4249999999999998},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":16.702999999999999,"5min":17.718,"15min":99.861000000000004},"last":0.51400000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.6240000000000001,"15min":4.6929999999999996},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":7.5599999999999996,"5min":27.489999999999998,"15min":162.33699999999999},"last":0.505},{"interface":"front","average":{"1min":1.125,"5min":1.448,"15min":4.9349999999999996},"min":{"1min":0.316,"5min":0.26000000000000001,"15min":0.21299999999999999},"max":{"1min":5.9550000000000001,"5min":27.788,"15min":162.376},"last":0.61599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":1.242,"15min":2.597},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":29.605,"5min":29.605,"15min":141.32900000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.605,"5min":1.034,"15min":2.1040000000000001},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":29.515999999999998,"5min":29.515999999999998,"15min":141.15899999999999},"last":0.58799999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":3.2519999999999998,"5min":1.591,"15min":2.653},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":32.774000000000001,"5min":32.774000000000001,"15min":235.18700000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":3.02,"5min":1.385,"15min":2.9399999999999999},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":36.976999999999997,"5min":36.976999999999997,"15min":235.79300000000001},"last":0.34499999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":2.3690000000000002,"15min":2.5960000000000001},"min":{"1min":0.38700000000000001,"5min":0.316,"15min":0.316},"max":{"1min":8.4870000000000001,"5min":101.295,"15min":103.60899999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.22,"5min":2.1110000000000002,"15min":2.7410000000000001},"min":{"1min":0.5,"5min":0.38900000000000001,"15min":0.33400000000000002},"max":{"1min":3.8559999999999999,"5min":101.27,"15min":103.679},"last":0.42899999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":3.238,"5min":1.355,"15min":2.1669999999999998},"min":{"1min":0.44700000000000001,"5min":0.375,"15min":0.35999999999999999},"max":{"1min":35.021000000000001,"5min":35.021000000000001,"15min":126.68300000000001},"last":0.81399999999999995},{"interface":"front","average":{"1min":3.6150000000000002,"5min":1.4350000000000001,"15min":2.282},"min":{"1min":0.44600000000000001,"5min":0.39700000000000002,"15min":0.34399999999999997},"max":{"1min":35.493000000000002,"5min":35.493000000000002,"15min":126.738},"last":0.84499999999999997}]}]},{"osd":0,"up_from":88,"seq":377957122284,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2984804,"kb_used_data":2359580,"kb_used_omap":8,"kb_used_meta":625207,"kb_avail":90760348,"statfs":{"total":95995035648,"available":92938596352,"internally_reserved":0,"allocated":2416209920,"data_stored":2415092905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":640212744},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.75600000000000001,"15min":1.6830000000000001},"min":{"1min":0.248,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":4.2560000000000002,"5min":14.198,"15min":17.760000000000002},"last":2.0209999999999999},{"interface":"front","average":{"1min":1.1000000000000001,"5min":0.85699999999999998,"15min":2.2069999999999999},"min":{"1min":0.28000000000000003,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":2.9990000000000001,"5min":8.2680000000000007,"15min":26.163},"last":2.0430000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":1.411,"15min":3.4740000000000002},"min":{"1min":0.219,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.0199999999999996,"5min":19.405999999999999,"15min":92.230000000000004},"last":1.45},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.2569999999999999,"15min":3.2570000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.218},"max":{"1min":4.8879999999999999,"5min":16.407,"15min":92.247},"last":0.33600000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.6659999999999999,"15min":3.9089999999999998},"min":{"1min":0.32800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":3.4790000000000001,"5min":55.328000000000003,"15min":123.024},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.429,"5min":1.8620000000000001,"15min":3.7829999999999999},"min":{"1min":0.34699999999999998,"5min":0.26800000000000002,"15min":0.222},"max":{"1min":6.4470000000000001,"5min":55.292000000000002,"15min":99.162000000000006},"last":1.5129999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.83899999999999997,"15min":1.452},"min":{"1min":0.32500000000000001,"5min":0.29899999999999999,"15min":0.25700000000000001},"max":{"1min":3.8570000000000002,"5min":10.084,"15min":69.082999999999998},"last":0.56799999999999995},{"interface":"front","average":{"1min":1,"5min":1.1559999999999999,"15min":1.9790000000000001},"min":{"1min":0.41999999999999998,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":2.4460000000000002,"5min":19.638999999999999,"15min":69.102000000000004},"last":0.58199999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.345,"15min":3.286},"min":{"1min":0.48599999999999999,"5min":0.34599999999999997,"15min":0.30199999999999999},"max":{"1min":2.8079999999999998,"5min":19.547000000000001,"15min":281.733},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.353,"15min":3.343},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":4.3890000000000002,"5min":19.463999999999999,"15min":282.95400000000001},"last":1.407}]},{"osd":6,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.4159999999999999,"15min":1.8819999999999999},"min":{"1min":0.46300000000000002,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":2.895,"5min":19.579000000000001,"15min":19.579000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.234,"5min":1.4950000000000001,"15min":2.0219999999999998},"min":{"1min":0.42299999999999999,"5min":0.34899999999999998,"15min":0.32500000000000001},"max":{"1min":4.4370000000000003,"5min":19.497,"15min":19.497},"last":1.4790000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.1859999999999999,"15min":1.4379999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.7220000000000004,"5min":25.378,"15min":25.378},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.55,"5min":1.391,"15min":2.0129999999999999},"min":{"1min":0.48399999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":5.556,"5min":25.436,"15min":25.436},"last":0.495}]}]},{"osd":2,"up_from":98,"seq":420906795231,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3410992,"kb_used_data":2727464,"kb_used_omap":11,"kb_used_meta":683508,"kb_avail":90334160,"statfs":{"total":95995035648,"available":92502179840,"internally_reserved":0,"allocated":2792923136,"data_stored":2791675733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":699912789},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:37 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":4.5069999999999997,"15min":3.5009999999999999},"min":{"1min":0.22900000000000001,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":4.415,"5min":162.285,"15min":162.285},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.91400000000000003,"5min":4.9390000000000001,"15min":3.7989999999999999},"min":{"1min":0.26400000000000001,"5min":0.22600000000000001,"15min":0.19600000000000001},"max":{"1min":3.0249999999999999,"5min":162.643,"15min":162.643},"last":0.26300000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:48:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.206,"15min":3.1829999999999998},"min":{"1min":0.28399999999999997,"5min":0.193,"15min":0.193},"max":{"1min":3.7469999999999999,"5min":16.050999999999998,"15min":98.575000000000003},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.103,"5min":0.94499999999999995,"15min":2.6389999999999998},"min":{"1min":0.24199999999999999,"5min":0.20999999999999999,"15min":0.20000000000000001},"max":{"1min":5.4470000000000001,"5min":7.7149999999999999,"15min":98.397000000000006},"last":0.58899999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:49:15 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":3.4670000000000001,"15min":5.0250000000000004},"min":{"1min":0.30399999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.6349999999999998,"5min":159.35499999999999,"15min":159.35499999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.7749999999999999,"5min":3.4990000000000001,"15min":4.9000000000000004},"min":{"1min":0.31900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.4710000000000001,"5min":159.196,"15min":159.196},"last":0.86599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:48:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":0.90500000000000003,"15min":1.9550000000000001},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.27700000000000002},"max":{"1min":5.5279999999999996,"5min":7.1120000000000001,"15min":115.84999999999999},"last":0.504},{"interface":"front","average":{"1min":1.625,"5min":0.998,"15min":2.1200000000000001},"min":{"1min":0.36199999999999999,"5min":0.309,"15min":0.28699999999999998},"max":{"1min":8.7739999999999991,"5min":8.7739999999999991,"15min":116.149},"last":0.374}]},{"osd":5,"last update":"Mon Oct 9 05:48:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.252,"15min":2.0470000000000002},"min":{"1min":0.38900000000000001,"5min":0.33900000000000002,"15min":0.312},"max":{"1min":4.1269999999999998,"5min":13.035,"15min":24.597999999999999},"last":0.83899999999999997},{"interface":"front","average":{"1min":1.278,"5min":1.0229999999999999,"15min":1.7270000000000001},"min":{"1min":0.39100000000000001,"5min":0.30499999999999999,"15min":0.26000000000000001},"max":{"1min":5.4850000000000003,"5min":7.1829999999999998,"15min":23.919},"last":0.55900000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:48:37 2023","interfaces":[{"interface":"back","average":{"1min":2.419,"5min":1.417,"15min":1.9490000000000001},"min":{"1min":0.46000000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":25.818999999999999,"5min":25.818999999999999,"15min":25.818999999999999},"last":0.437},{"interface":"front","average":{"1min":2.657,"5min":1.7290000000000001,"15min":2.048},"min":{"1min":0.46400000000000002,"5min":0.38900000000000001,"15min":0.34499999999999997},"max":{"1min":24.481000000000002,"5min":24.481000000000002,"15min":24.481000000000002},"last":0.89100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:49:15 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":1.6679999999999999,"15min":2.2250000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.3330000000000002,"5min":51.701999999999998,"15min":51.701999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":2.8450000000000002,"5min":1.9670000000000001,"15min":2.339},"min":{"1min":0.40899999999999997,"5min":0.373,"15min":0.373},"max":{"1min":8.7430000000000003,"5min":51.716000000000001,"15min":51.716000000000001},"last":0.92500000000000004}]}]},{"osd":3,"up_from":103,"seq":442381631705,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3841048,"kb_used_data":3163408,"kb_used_omap":4,"kb_used_meta":677627,"kb_avail":89904104,"statfs":{"total":95995035648,"available":92061802496,"internally_reserved":0,"allocated":3239329792,"data_stored":3238221682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":693890654},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":4.6680000000000001,"5min":3.8439999999999999,"15min":4.1260000000000003},"min":{"1min":0.253,"5min":0.188,"15min":0.183},"max":{"1min":47.173000000000002,"5min":149.565,"15min":167.82400000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":5.3040000000000003,"5min":3.9790000000000001,"15min":4.7069999999999999},"min":{"1min":0.21199999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":47.908000000000001,"5min":149.291,"15min":167.89599999999999},"last":6.2969999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":2.4380000000000002,"15min":3.0600000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.22600000000000001},"max":{"1min":7.984,"5min":149.32499999999999,"15min":149.32499999999999},"last":14.153},{"interface":"front","average":{"1min":2.7149999999999999,"5min":2.77,"15min":3.2320000000000002},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.19},"max":{"1min":11.547000000000001,"5min":149.34299999999999,"15min":149.34299999999999},"last":6.3940000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":2.4630000000000001,"15min":4.2949999999999999},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.215},"max":{"1min":8.0009999999999994,"5min":150.006,"15min":190.43299999999999},"last":4.8479999999999999},{"interface":"front","average":{"1min":1.597,"5min":2.4199999999999999,"15min":4.6020000000000003},"min":{"1min":0.221,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.8280000000000003,"5min":150.65100000000001,"15min":190.267},"last":7.593}]},{"osd":4,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":2.4329999999999998,"15min":2.6299999999999999},"min":{"1min":0.34899999999999998,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":5.883,"5min":149.84700000000001,"15min":149.84700000000001},"last":6.1630000000000003},{"interface":"front","average":{"1min":2.052,"5min":2.3849999999999998,"15min":2.266},"min":{"1min":0.49199999999999999,"5min":0.32100000000000001,"15min":0.27400000000000002},"max":{"1min":7.9249999999999998,"5min":149.74700000000001,"15min":149.74700000000001},"last":0.69699999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":3.117,"5min":2.8220000000000001,"15min":2.54},"min":{"1min":0.46700000000000003,"5min":0.35399999999999998,"15min":0.28199999999999997},"max":{"1min":11.6,"5min":149.43000000000001,"15min":149.43000000000001},"last":6.3630000000000004},{"interface":"front","average":{"1min":2.7090000000000001,"5min":2.6800000000000002,"15min":2.4710000000000001},"min":{"1min":0.51600000000000001,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":7.8689999999999998,"5min":149.69300000000001,"15min":149.69300000000001},"last":0.72899999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":2.4889999999999999,"15min":2.258},"min":{"1min":0.38700000000000001,"5min":0.34499999999999997,"15min":0.33500000000000002},"max":{"1min":7.9009999999999998,"5min":149.72399999999999,"15min":149.72399999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":2.5169999999999999,"5min":2.7730000000000001,"15min":2.4140000000000001},"min":{"1min":0.46500000000000002,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":11.647,"5min":149.589,"15min":149.589},"last":6.4299999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:49:16 2023","interfaces":[{"interface":"back","average":{"1min":2.4009999999999998,"5min":2.4740000000000002,"15min":2.48},"min":{"1min":0.435,"5min":0.35699999999999998,"15min":0.33700000000000002},"max":{"1min":9.3260000000000005,"5min":149.77199999999999,"15min":149.77199999999999},"last":6.1180000000000003},{"interface":"front","average":{"1min":1.9990000000000001,"5min":2.5329999999999999,"15min":2.5249999999999999},"min":{"1min":0.48199999999999998,"5min":0.39100000000000001,"15min":0.35699999999999998},"max":{"1min":5.4299999999999997,"5min":149.80799999999999,"15min":149.80799999999999},"last":6.2080000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019253,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4379472,"kb_used_data":3478024,"kb_used_omap":0,"kb_used_meta":901440,"kb_avail":89365680,"statfs":{"total":95995035648,"available":91510456320,"internally_reserved":0,"allocated":3561496576,"data_stored":3560865424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923074560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:42 2023","interfaces":[{"interface":"back","average":{"1min":3.407,"5min":1.679,"15min":2.488},"min":{"1min":0.34300000000000003,"5min":0.219,"15min":0.214},"max":{"1min":16.984999999999999,"5min":26.478000000000002,"15min":100.504},"last":1.6499999999999999},{"interface":"front","average":{"1min":3.1400000000000001,"5min":1.839,"15min":2.7909999999999999},"min":{"1min":0.32500000000000001,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":18.928000000000001,"5min":26.454000000000001,"15min":100.717},"last":0.29499999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":1.2949999999999999,"15min":2.645},"min":{"1min":0.36599999999999999,"5min":0.26800000000000002,"15min":0.24299999999999999},"max":{"1min":10.406000000000001,"5min":10.406000000000001,"15min":187.369},"last":0.36399999999999999},{"interface":"front","average":{"1min":2.8079999999999998,"5min":1.2370000000000001,"15min":2.726},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26600000000000001},"max":{"1min":12.202999999999999,"5min":12.202999999999999,"15min":187.35300000000001},"last":0.42199999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:48:30 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":1.419,"15min":4.694},"min":{"1min":0.42899999999999999,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":17.335999999999999,"5min":17.335999999999999,"15min":196.93700000000001},"last":0.434},{"interface":"front","average":{"1min":2.1320000000000001,"5min":1.4870000000000001,"15min":4.5999999999999996},"min":{"1min":0.311,"5min":0.29799999999999999,"15min":0.28100000000000003},"max":{"1min":16.795000000000002,"5min":16.795000000000002,"15min":197.08099999999999},"last":0.41499999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:48:53 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":2.5369999999999999,"15min":4.5679999999999996},"min":{"1min":0.35399999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":16.667000000000002,"5min":123.59,"15min":211.48699999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.5680000000000001,"15min":4.4100000000000001},"min":{"1min":0.439,"5min":0.312,"15min":0.311},"max":{"1min":16.530999999999999,"5min":123.566,"15min":211.31100000000001},"last":0.46700000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:48:53 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":1.173,"15min":1.794},"min":{"1min":0.27900000000000003,"5min":0.27800000000000002,"15min":0.24099999999999999},"max":{"1min":12.34,"5min":12.34,"15min":113.35899999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.6619999999999999,"5min":1.04,"15min":1.6699999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":10.191000000000001,"5min":10.191000000000001,"15min":113.32299999999999},"last":0.443}]},{"osd":6,"last update":"Mon Oct 9 05:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9059999999999999,"5min":0.86199999999999999,"15min":0.82199999999999995},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.8889999999999993,"5min":9.8889999999999993,"15min":11.944000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.889,"5min":1.1220000000000001,"15min":1.056},"min":{"1min":0.34899999999999998,"5min":0.32800000000000001,"15min":0.28399999999999997},"max":{"1min":6.0179999999999998,"5min":8.5269999999999992,"15min":10.356},"last":0.45300000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5649999999999999,"5min":1.7789999999999999,"15min":1.4830000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.312},"max":{"1min":5.8650000000000002,"5min":96.131,"15min":96.131},"last":0.377},{"interface":"front","average":{"1min":1.802,"5min":1.978,"15min":1.625},"min":{"1min":0.36399999999999999,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":7.6580000000000004,"5min":96.465000000000003,"15min":96.465000000000003},"last":0.40100000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823025,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3358528,"kb_used_data":2459832,"kb_used_omap":0,"kb_used_meta":898688,"kb_avail":90386624,"statfs":{"total":95995035648,"available":92555902976,"internally_reserved":0,"allocated":2518867968,"data_stored":2518248602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920256512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:48:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":2.6989999999999998,"15min":3.847},"min":{"1min":0.23499999999999999,"5min":0.23300000000000001,"15min":0.19400000000000001},"max":{"1min":6.7679999999999998,"5min":96.858000000000004,"15min":165.90199999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.83799999999999997,"5min":2.8109999999999999,"15min":3.9380000000000002},"min":{"1min":0.28799999999999998,"5min":0.254,"15min":0.215},"max":{"1min":3.4500000000000002,"5min":97.555999999999997,"15min":165.96700000000001},"last":3.3479999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":1.0129999999999999,"15min":2.7029999999999998},"min":{"1min":0.35299999999999998,"5min":0.27000000000000002,"15min":0.23999999999999999},"max":{"1min":8.6799999999999997,"5min":8.6799999999999997,"15min":87.769000000000005},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.1080000000000001,"5min":0.79800000000000004,"15min":2.6970000000000001},"min":{"1min":0.29099999999999998,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":3.4870000000000001,"5min":5.601,"15min":87.864000000000004},"last":2.9609999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:48:40 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.244,"15min":4.5190000000000001},"min":{"1min":0.29699999999999999,"5min":0.28100000000000003,"15min":0.26700000000000002},"max":{"1min":6.8710000000000004,"5min":10.722,"15min":290.697},"last":3.5659999999999998},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.2090000000000001,"15min":4.399},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":8.5030000000000001,"5min":8.5030000000000001,"15min":290.786},"last":0.58999999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":1.9079999999999999,"15min":3.7690000000000001},"min":{"1min":0.432,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":8.5350000000000001,"5min":53.243000000000002,"15min":95.236999999999995},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.9830000000000001,"5min":1.6850000000000001,"15min":3.5720000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29699999999999999},"max":{"1min":6.8339999999999996,"5min":53.259999999999998,"15min":93.471000000000004},"last":3.6179999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":8.9939999999999998,"5min":2.266,"15min":1.351},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":130.81,"5min":130.81,"15min":130.81},"last":0.32800000000000001},{"interface":"front","average":{"1min":9.8119999999999994,"5min":2.4990000000000001,"15min":1.6439999999999999},"min":{"1min":0.33500000000000002,"5min":0.27000000000000002,"15min":0.254},"max":{"1min":133.803,"5min":133.803,"15min":133.803},"last":0.377}]},{"osd":6,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":0.89300000000000002,"15min":1.504},"min":{"1min":0.32600000000000001,"5min":0.32000000000000001,"15min":0.26600000000000001},"max":{"1min":7.8220000000000001,"5min":7.8220000000000001,"15min":95.010999999999996},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.3719999999999999,"5min":0.997,"15min":1.466},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.27300000000000002},"max":{"1min":8.4629999999999992,"5min":8.4629999999999992,"15min":94.194000000000003},"last":0.46100000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.0009999999999999,"15min":1.1639999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.9980000000000002,"5min":6.9980000000000002,"15min":9.9849999999999994},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.2689999999999999,"5min":0.90000000000000002,"15min":1.155},"min":{"1min":0.316,"5min":0.316,"15min":0.308},"max":{"1min":6.8929999999999998,"5min":6.8929999999999998,"15min":14.432},"last":0.44900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2406481920,"data_stored":2406481920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":286851072,"data_stored":286851072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2783514624,"data_stored":2783510552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3235184640,"data_stored":3235184640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3556638720,"data_stored":3556638720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2520322048,"data_stored":2520317976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2768699392,"data_stored":2768699392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2147946496,"data_stored":2147942424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:49:34.657 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:49:34.657 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 31 2023-10-09T05:49:34.825 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:34.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:34.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:34.929 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:35.028 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:35.171 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:35.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:35.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:35.185 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:35.282 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:35.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:35 smithi191 ceph-mon[161669]: failed to encode map e315 with expected crc 2023-10-09T05:49:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:35 smithi191 ceph-mon[161669]: failed to encode map e315 with expected crc 2023-10-09T05:49:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:35 smithi191 ceph-mon[161669]: osdmap e315: 8 total, 8 up, 8 in 2023-10-09T05:49:35.338 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:35 smithi008 ceph-mon[184418]: failed to encode map e315 with expected crc 2023-10-09T05:49:35.338 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:35 smithi008 ceph-mon[184418]: failed to encode map e315 with expected crc 2023-10-09T05:49:35.338 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:35 smithi008 ceph-mon[184418]: osdmap e315: 8 total, 8 up, 8 in 2023-10-09T05:49:35.338 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:35 smithi008 ceph-mon[191227]: failed to encode map e315 with expected crc 2023-10-09T05:49:35.338 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:35 smithi008 ceph-mon[191227]: failed to encode map e315 with expected crc 2023-10-09T05:49:35.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:35 smithi008 ceph-mon[191227]: osdmap e315: 8 total, 8 up, 8 in 2023-10-09T05:49:35.349 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:35.589 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:35.694 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:35.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:35.808 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:35.878 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:36.058 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:36.093 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:36.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:36.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:36.203 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:36.232 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:36.276 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:36.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:36.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:36.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:36 smithi191 ceph-mon[161669]: pgmap v949: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37038/291879 objects degraded (12.690%); 784/291879 objects misplaced (0.269%); 9.6 MiB/s, 156 objects/s recovering 2023-10-09T05:49:36.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:36 smithi191 ceph-mon[161669]: failed to encode map e316 with expected crc 2023-10-09T05:49:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:36 smithi191 ceph-mon[161669]: failed to encode map e316 with expected crc 2023-10-09T05:49:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:36 smithi191 ceph-mon[161669]: osdmap e316: 8 total, 8 up, 8 in 2023-10-09T05:49:36.367 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:36 smithi008 ceph-mon[191227]: pgmap v949: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37038/291879 objects degraded (12.690%); 784/291879 objects misplaced (0.269%); 9.6 MiB/s, 156 objects/s recovering 2023-10-09T05:49:36.367 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:36 smithi008 ceph-mon[191227]: failed to encode map e316 with expected crc 2023-10-09T05:49:36.367 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:36 smithi008 ceph-mon[191227]: failed to encode map e316 with expected crc 2023-10-09T05:49:36.367 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:36 smithi008 ceph-mon[191227]: osdmap e316: 8 total, 8 up, 8 in 2023-10-09T05:49:36.368 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:36 smithi008 ceph-mon[184418]: pgmap v949: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 88 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 37038/291879 objects degraded (12.690%); 784/291879 objects misplaced (0.269%); 9.6 MiB/s, 156 objects/s recovering 2023-10-09T05:49:36.368 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:36 smithi008 ceph-mon[184418]: failed to encode map e316 with expected crc 2023-10-09T05:49:36.368 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:36 smithi008 ceph-mon[184418]: failed to encode map e316 with expected crc 2023-10-09T05:49:36.368 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:36 smithi008 ceph-mon[184418]: osdmap e316: 8 total, 8 up, 8 in 2023-10-09T05:49:36.383 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:36.412 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:36.458 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:36.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:36.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:36.563 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:36.597 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:37.100 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:49:37.153 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:37.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:37.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:37.260 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:37.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:37 smithi008 ceph-mon[184418]: failed to encode map e317 with expected crc 2023-10-09T05:49:37.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:37 smithi008 ceph-mon[184418]: osdmap e317: 8 total, 8 up, 8 in 2023-10-09T05:49:37.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:37 smithi008 ceph-mon[184418]: failed to encode map e317 with expected crc 2023-10-09T05:49:37.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:37 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2525940960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:37.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:37 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:37.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:37 smithi008 ceph-mon[191227]: failed to encode map e317 with expected crc 2023-10-09T05:49:37.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:37 smithi008 ceph-mon[191227]: osdmap e317: 8 total, 8 up, 8 in 2023-10-09T05:49:37.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:37 smithi008 ceph-mon[191227]: failed to encode map e317 with expected crc 2023-10-09T05:49:37.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:37 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2525940960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:37.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:37 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:37.312 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:37.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:37 smithi191 ceph-mon[161669]: failed to encode map e317 with expected crc 2023-10-09T05:49:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:37 smithi191 ceph-mon[161669]: osdmap e317: 8 total, 8 up, 8 in 2023-10-09T05:49:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:37 smithi191 ceph-mon[161669]: failed to encode map e317 with expected crc 2023-10-09T05:49:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:37 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2525940960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:37.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:37 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[184418]: pgmap v952: 109 pgs: 2 peering, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 35341/289749 objects degraded (12.197%); 1341/289749 objects misplaced (0.463%); 14 MiB/s, 222 objects/s recovering 2023-10-09T05:49:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:49:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[184418]: failed to encode map e318 with expected crc 2023-10-09T05:49:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[184418]: failed to encode map e318 with expected crc 2023-10-09T05:49:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:49:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[184418]: osdmap e318: 8 total, 8 up, 8 in 2023-10-09T05:49:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[184418]: failed to encode map e318 with expected crc 2023-10-09T05:49:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 35341/289749 objects degraded (12.197%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[191227]: pgmap v952: 109 pgs: 2 peering, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 35341/289749 objects degraded (12.197%); 1341/289749 objects misplaced (0.463%); 14 MiB/s, 222 objects/s recovering 2023-10-09T05:49:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:49:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[191227]: failed to encode map e318 with expected crc 2023-10-09T05:49:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[191227]: failed to encode map e318 with expected crc 2023-10-09T05:49:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:49:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[191227]: osdmap e318: 8 total, 8 up, 8 in 2023-10-09T05:49:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[191227]: failed to encode map e318 with expected crc 2023-10-09T05:49:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 35341/289749 objects degraded (12.197%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:38.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:38 smithi191 ceph-mon[161669]: pgmap v952: 109 pgs: 2 peering, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 35341/289749 objects degraded (12.197%); 1341/289749 objects misplaced (0.463%); 14 MiB/s, 222 objects/s recovering 2023-10-09T05:49:38.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:38 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:49:38.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:38 smithi191 ceph-mon[161669]: failed to encode map e318 with expected crc 2023-10-09T05:49:38.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:38 smithi191 ceph-mon[161669]: failed to encode map e318 with expected crc 2023-10-09T05:49:38.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:38 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:49:38.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:38 smithi191 ceph-mon[161669]: osdmap e318: 8 total, 8 up, 8 in 2023-10-09T05:49:38.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:38 smithi191 ceph-mon[161669]: failed to encode map e318 with expected crc 2023-10-09T05:49:38.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 35341/289749 objects degraded (12.197%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:38.860 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:38.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:38.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:38.976 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:39.041 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:39.111 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:39.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:39.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:39.216 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:39.242 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:39.286 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:39.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:39.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:39.392 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:39.437 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:39.663 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:39.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:39.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:39.768 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:49:39.795 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:39.840 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:49:39.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:39.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:39.947 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:39.973 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:40.014 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:40.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:40.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:40.123 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:40.152 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:40.198 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:40.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:40.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:40.218 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:49:40.219 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:49:40.219 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:49:40.219 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:49:40.219 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.219 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102132 7.6 7:6028800a:::benchmark_data_smithi008_354649_object7636:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T05:49:40.219 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.878610+0000", 2023-10-09T05:49:40.220 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 381.330300454, 2023-10-09T05:49:40.220 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3895780630000001, 2023-10-09T05:49:40.220 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.220 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:49:40.220 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.220 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.221 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.221 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102132 2023-10-09T05:49:40.221 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.221 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.221 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.221 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.221 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878610+0000", 2023-10-09T05:49:40.222 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.222 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.222 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.222 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.222 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878610+0000", 2023-10-09T05:49:40.223 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.223 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.223 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.223 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.223 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878608+0000", 2023-10-09T05:49:40.223 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:40.224 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.224 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.224 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.224 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878615+0000", 2023-10-09T05:49:40.224 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6379999999999998e-06 2023-10-09T05:49:40.224 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.225 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.225 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.225 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878633+0000", 2023-10-09T05:49:40.225 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7983999999999999e-05 2023-10-09T05:49:40.225 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.225 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.226 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878642+0000", 2023-10-09T05:49:40.226 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5200000000000003e-06 2023-10-09T05:49:40.226 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.226 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.227 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.879024+0000", 2023-10-09T05:49:40.227 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00038146000000000002 2023-10-09T05:49:40.227 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.227 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.227 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.227 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.879140+0000", 2023-10-09T05:49:40.228 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000116616 2023-10-09T05:49:40.228 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.228 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.228 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:49:40.228 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.879378+0000", 2023-10-09T05:49:40.228 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00023714499999999999 2023-10-09T05:49:40.228 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.229 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.229 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.229 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887275+0000", 2023-10-09T05:49:40.229 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078973129999999996 2023-10-09T05:49:40.229 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.229 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.230 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.230 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905106+0000", 2023-10-09T05:49:40.230 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017830736 2023-10-09T05:49:40.230 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.230 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.231 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.231 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905119+0000", 2023-10-09T05:49:40.231 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3549e-05 2023-10-09T05:49:40.231 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.231 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.231 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.232 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877502+0000", 2023-10-09T05:49:40.232 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97238259000000005 2023-10-09T05:49:40.232 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.232 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.232 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.232 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877512+0000", 2023-10-09T05:49:40.233 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0355e-05 2023-10-09T05:49:40.233 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.233 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.233 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.233 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.239697+0000", 2023-10-09T05:49:40.233 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3621850200000001 2023-10-09T05:49:40.234 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.234 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.234 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:49:40.234 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.239848+0000", 2023-10-09T05:49:40.234 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015051299999999999 2023-10-09T05:49:40.234 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.235 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.235 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268057+0000", 2023-10-09T05:49:40.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028209238000000001 2023-10-09T05:49:40.235 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.235 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.235 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268188+0000", 2023-10-09T05:49:40.236 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013106500000000001 2023-10-09T05:49:40.236 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.236 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.236 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.236 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.236 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102135 7.6 7:616fc77f:::benchmark_data_smithi008_354649_object7639:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T05:49:40.236 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.878660+0000", 2023-10-09T05:49:40.237 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 381.33025036700002, 2023-10-09T05:49:40.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4043974380000002, 2023-10-09T05:49:40.237 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.237 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:49:40.237 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.237 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.237 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.237 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102135 2023-10-09T05:49:40.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.238 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.238 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.238 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.238 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878660+0000", 2023-10-09T05:49:40.238 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.239 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.239 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878660+0000", 2023-10-09T05:49:40.239 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.239 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.239 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.239 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878659+0000", 2023-10-09T05:49:40.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:49:40.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.240 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878663+0000", 2023-10-09T05:49:40.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.242e-06 2023-10-09T05:49:40.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878670+0000", 2023-10-09T05:49:40.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0779999999999998e-06 2023-10-09T05:49:40.241 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.241 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878676+0000", 2023-10-09T05:49:40.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9320000000000001e-06 2023-10-09T05:49:40.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.242 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.242 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.883274+0000", 2023-10-09T05:49:40.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045982369999999998 2023-10-09T05:49:40.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.243 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.243 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.883418+0000", 2023-10-09T05:49:40.243 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014399399999999999 2023-10-09T05:49:40.243 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.243 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.243 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:49:40.243 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.883540+0000", 2023-10-09T05:49:40.244 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000122225 2023-10-09T05:49:40.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.244 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.244 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.244 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887783+0000", 2023-10-09T05:49:40.244 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042433180000000003 2023-10-09T05:49:40.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905157+0000", 2023-10-09T05:49:40.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017373588999999998 2023-10-09T05:49:40.245 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905174+0000", 2023-10-09T05:49:40.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6509999999999999e-05 2023-10-09T05:49:40.245 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877554+0000", 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97238062300000006 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877564+0000", 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6269999999999995e-06 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243556+0000", 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3659919819999999 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243708+0000", 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015254099999999999 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.282943+0000", 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.039234640000000001 2023-10-09T05:49:40.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283057+0000", 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114479 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102138 7.6 7:69d2abd0:::benchmark_data_smithi008_354649_object7642:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:49:40.248 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 381.330229153, 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3933153759999999, 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102138 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:49:40.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878684+0000", 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9849999999999998e-06 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878689+0000", 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6910000000000002e-06 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878697+0000", 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2899999999999997e-06 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887233+0000", 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0085366939999999992 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op must wait for map", 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887240+0000", 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0169999999999999e-06 2023-10-09T05:49:40.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.253 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.253 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905215+0000", 2023-10-09T05:49:40.253 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017974716000000002 2023-10-09T05:49:40.253 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.253 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.253 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905223+0000", 2023-10-09T05:49:40.253 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7360000000000001e-06 2023-10-09T05:49:40.253 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877592+0000", 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97236955700000005 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877601+0000", 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2449999999999998e-06 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243738+0000", 2023-10-09T05:49:40.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.366137529 2023-10-09T05:49:40.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243861+0000", 2023-10-09T05:49:40.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012260400000000001 2023-10-09T05:49:40.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:49:40.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.244012+0000", 2023-10-09T05:49:40.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015064899999999999 2023-10-09T05:49:40.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.257726+0000", 2023-10-09T05:49:40.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013714637999999999 2023-10-09T05:49:40.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268231+0000", 2023-10-09T05:49:40.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010504434999999999 2023-10-09T05:49:40.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268269+0000", 2023-10-09T05:49:40.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7939999999999999e-05 2023-10-09T05:49:40.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268316+0000", 2023-10-09T05:49:40.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7679000000000002e-05 2023-10-09T05:49:40.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.271997+0000", 2023-10-09T05:49:40.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036803859999999999 2023-10-09T05:49:40.258 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.258 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.258 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102144 7.6 7:6ad058ac:::benchmark_data_smithi008_354649_object7648:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.899418+0000", 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 381.30949240199999, 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3874670130000002, 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.259 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102144 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899418+0000", 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899418+0000", 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899417+0000", 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899422+0000", 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7659999999999998e-06 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899430+0000", 2023-10-09T05:49:40.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6100000000000006e-06 2023-10-09T05:49:40.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.262 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.262 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899438+0000", 2023-10-09T05:49:40.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0760000000000003e-06 2023-10-09T05:49:40.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.262 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.262 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905297+0000", 2023-10-09T05:49:40.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058583560000000003 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905305+0000", 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0090000000000007e-06 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877625+0000", 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.972320462 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877633+0000", 2023-10-09T05:49:40.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8339999999999999e-06 2023-10-09T05:49:40.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.250892+0000", 2023-10-09T05:49:40.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.373259207 2023-10-09T05:49:40.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.251058+0000", 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016594400000000001 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.251234+0000", 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000175453 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268021+0000", 2023-10-09T05:49:40.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016787834000000001 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283099+0000", 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015077174 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283181+0000", 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2218000000000005e-05 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283226+0000", 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5181e-05 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.286885+0000", 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036590279999999999 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.268 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102153 7.6 7:672a3339:::benchmark_data_smithi008_354649_object7657:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:49:40.268 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.906435+0000", 2023-10-09T05:49:40.268 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 381.30247498300002, 2023-10-09T05:49:40.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3840327299999999, 2023-10-09T05:49:40.268 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.268 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.269 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.269 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.269 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.269 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102153 2023-10-09T05:49:40.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.269 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906435+0000", 2023-10-09T05:49:40.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906435+0000", 2023-10-09T05:49:40.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906433+0000", 2023-10-09T05:49:40.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:40.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.272 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906441+0000", 2023-10-09T05:49:40.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4569999999999999e-06 2023-10-09T05:49:40.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.272 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.272 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906458+0000", 2023-10-09T05:49:40.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7399000000000001e-05 2023-10-09T05:49:40.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906470+0000", 2023-10-09T05:49:40.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2092e-05 2023-10-09T05:49:40.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.274 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906521+0000", 2023-10-09T05:49:40.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0983999999999998e-05 2023-10-09T05:49:40.274 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.274 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.274 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906533+0000", 2023-10-09T05:49:40.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1977000000000001e-05 2023-10-09T05:49:40.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877706+0000", 2023-10-09T05:49:40.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97117302299999997 2023-10-09T05:49:40.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877717+0000", 2023-10-09T05:49:40.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0254e-05 2023-10-09T05:49:40.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.257761+0000", 2023-10-09T05:49:40.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3800444709999999 2023-10-09T05:49:40.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.257940+0000", 2023-10-09T05:49:40.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000179037 2023-10-09T05:49:40.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:49:40.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.258090+0000", 2023-10-09T05:49:40.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000150161 2023-10-09T05:49:40.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268184+0000", 2023-10-09T05:49:40.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010093352999999999 2023-10-09T05:49:40.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283137+0000", 2023-10-09T05:49:40.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014953404999999999 2023-10-09T05:49:40.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.286959+0000", 2023-10-09T05:49:40.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038219339999999999 2023-10-09T05:49:40.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.287005+0000", 2023-10-09T05:49:40.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.549e-05 2023-10-09T05:49:40.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.290468+0000", 2023-10-09T05:49:40.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034635989999999999 2023-10-09T05:49:40.281 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.282 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.282 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.282 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102170 7.f 7:f2cf7216:::benchmark_data_smithi008_354649_object7674:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:49:40.282 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.931713+0000", 2023-10-09T05:49:40.282 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 381.27719762300001, 2023-10-09T05:49:40.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.30407954, 2023-10-09T05:49:40.283 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.283 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.283 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.283 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.283 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.283 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102170 2023-10-09T05:49:40.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.284 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931713+0000", 2023-10-09T05:49:40.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931713+0000", 2023-10-09T05:49:40.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931711+0000", 2023-10-09T05:49:40.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:40.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931717+0000", 2023-10-09T05:49:40.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4029999999999998e-06 2023-10-09T05:49:40.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931726+0000", 2023-10-09T05:49:40.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7409999999999998e-06 2023-10-09T05:49:40.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931731+0000", 2023-10-09T05:49:40.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3310000000000003e-06 2023-10-09T05:49:40.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931790+0000", 2023-10-09T05:49:40.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9153000000000001e-05 2023-10-09T05:49:40.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931804+0000", 2023-10-09T05:49:40.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4075e-05 2023-10-09T05:49:40.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.290 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.876813+0000", 2023-10-09T05:49:40.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94500846400000005 2023-10-09T05:49:40.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.876824+0000", 2023-10-09T05:49:40.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.184e-05 2023-10-09T05:49:40.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.222418+0000", 2023-10-09T05:49:40.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3455932580000001 2023-10-09T05:49:40.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.222572+0000", 2023-10-09T05:49:40.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000154277 2023-10-09T05:49:40.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:49:40.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.222670+0000", 2023-10-09T05:49:40.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8070000000000001e-05 2023-10-09T05:49:40.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230124+0000", 2023-10-09T05:49:40.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007453515 2023-10-09T05:49:40.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230200+0000", 2023-10-09T05:49:40.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6130000000000005e-05 2023-10-09T05:49:40.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.231464+0000", 2023-10-09T05:49:40.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0012646619999999999 2023-10-09T05:49:40.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.231507+0000", 2023-10-09T05:49:40.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2215000000000002e-05 2023-10-09T05:49:40.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.235792+0000", 2023-10-09T05:49:40.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042857040000000004 2023-10-09T05:49:40.296 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.296 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.296 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.297 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125678 7.6 7:63397418:::benchmark_data_smithi008_354649_object31182:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:49:40.297 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.999075+0000", 2023-10-09T05:49:40.297 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 337.20983583200001, 2023-10-09T05:49:40.297 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0649156319999999, 2023-10-09T05:49:40.297 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.297 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:49:40.298 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.298 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.298 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.298 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125678 2023-10-09T05:49:40.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.298 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.298 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.298 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999075+0000", 2023-10-09T05:49:40.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999075+0000", 2023-10-09T05:49:40.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.300 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.300 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999073+0000", 2023-10-09T05:49:40.300 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:40.300 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.300 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999080+0000", 2023-10-09T05:49:40.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3900000000000004e-06 2023-10-09T05:49:40.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.301 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.301 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999095+0000", 2023-10-09T05:49:40.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4596999999999999e-05 2023-10-09T05:49:40.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999108+0000", 2023-10-09T05:49:40.302 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2767e-05 2023-10-09T05:49:40.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006313+0000", 2023-10-09T05:49:40.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0072051479999999998 2023-10-09T05:49:40.303 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.303 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.303 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op must wait for map", 2023-10-09T05:49:40.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006322+0000", 2023-10-09T05:49:40.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6799999999999999e-06 2023-10-09T05:49:40.303 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.303 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.383274+0000", 2023-10-09T05:49:40.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37695257599999998 2023-10-09T05:49:40.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.383295+0000", 2023-10-09T05:49:40.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0871999999999999e-05 2023-10-09T05:49:40.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.969340+0000", 2023-10-09T05:49:40.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.58604520999999998 2023-10-09T05:49:40.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.969352+0000", 2023-10-09T05:49:40.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1914e-05 2023-10-09T05:49:40.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.033375+0000", 2023-10-09T05:49:40.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0640223849999999 2023-10-09T05:49:40.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:49:40.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.033498+0000", 2023-10-09T05:49:40.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000123614 2023-10-09T05:49:40.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.063936+0000", 2023-10-09T05:49:40.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030438004000000001 2023-10-09T05:49:40.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.063990+0000", 2023-10-09T05:49:40.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3983000000000001e-05 2023-10-09T05:49:40.309 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.309 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.309 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.309 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150726 7.4 7:2606d17b:::benchmark_data_smithi008_354649_object56230:head [delete] snapc 0=[] ondisk+write+known_if_redirected e251)", 2023-10-09T05:49:40.309 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:16.967446+0000", 2023-10-09T05:49:40.310 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 263.24146458600001, 2023-10-09T05:49:40.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.266506991, 2023-10-09T05:49:40.310 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.310 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.310 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.310 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.310 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.311 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150726 2023-10-09T05:49:40.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.311 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967446+0000", 2023-10-09T05:49:40.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.312 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967446+0000", 2023-10-09T05:49:40.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.313 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.313 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967445+0000", 2023-10-09T05:49:40.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:49:40.313 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.313 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.313 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967448+0000", 2023-10-09T05:49:40.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.942e-06 2023-10-09T05:49:40.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967454+0000", 2023-10-09T05:49:40.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8379999999999998e-06 2023-10-09T05:49:40.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967460+0000", 2023-10-09T05:49:40.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7620000000000001e-06 2023-10-09T05:49:40.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.984917+0000", 2023-10-09T05:49:40.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017457026 2023-10-09T05:49:40.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.316 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.316 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.984930+0000", 2023-10-09T05:49:40.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3217000000000001e-05 2023-10-09T05:49:40.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.923010+0000", 2023-10-09T05:49:40.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93808012799999996 2023-10-09T05:49:40.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.923019+0000", 2023-10-09T05:49:40.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6230000000000003e-06 2023-10-09T05:49:40.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.216263+0000", 2023-10-09T05:49:40.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2932439330000001 2023-10-09T05:49:40.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.216410+0000", 2023-10-09T05:49:40.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000146861 2023-10-09T05:49:40.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:49:40.322 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.216920+0000", 2023-10-09T05:49:40.322 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00051030300000000004 2023-10-09T05:49:40.322 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.322 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.322 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.323 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.224475+0000", 2023-10-09T05:49:40.323 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0075551230000000004 2023-10-09T05:49:40.323 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.323 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.323 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.323 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.224528+0000", 2023-10-09T05:49:40.323 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.295e-05 2023-10-09T05:49:40.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.233016+0000", 2023-10-09T05:49:40.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0084877560000000008 2023-10-09T05:49:40.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.325 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.325 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.233080+0000", 2023-10-09T05:49:40.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4058000000000002e-05 2023-10-09T05:49:40.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.325 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.325 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.233953+0000", 2023-10-09T05:49:40.326 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00087297799999999995 2023-10-09T05:49:40.326 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.326 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.326 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.327 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150729 7.9 7:91e0d910:::benchmark_data_smithi008_354649_object56233:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:49:40.327 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:16.994744+0000", 2023-10-09T05:49:40.327 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 263.21416669299998, 2023-10-09T05:49:40.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.138625711, 2023-10-09T05:49:40.327 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.327 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.328 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.328 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.328 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.328 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150729 2023-10-09T05:49:40.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.328 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994744+0000", 2023-10-09T05:49:40.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994744+0000", 2023-10-09T05:49:40.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994742+0000", 2023-10-09T05:49:40.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:40.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.331 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994749+0000", 2023-10-09T05:49:40.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.421e-06 2023-10-09T05:49:40.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994762+0000", 2023-10-09T05:49:40.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3101999999999999e-05 2023-10-09T05:49:40.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.332 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994771+0000", 2023-10-09T05:49:40.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0070000000000003e-06 2023-10-09T05:49:40.333 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.333 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.333 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.333 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994839+0000", 2023-10-09T05:49:40.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7821e-05 2023-10-09T05:49:40.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.334 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994853+0000", 2023-10-09T05:49:40.334 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4782e-05 2023-10-09T05:49:40.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917318+0000", 2023-10-09T05:49:40.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92246414300000001 2023-10-09T05:49:40.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917330+0000", 2023-10-09T05:49:40.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2471e-05 2023-10-09T05:49:40.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.116457+0000", 2023-10-09T05:49:40.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.199127415 2023-10-09T05:49:40.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.116616+0000", 2023-10-09T05:49:40.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015812399999999999 2023-10-09T05:49:40.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:49:40.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.117033+0000", 2023-10-09T05:49:40.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00041699599999999999 2023-10-09T05:49:40.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.338 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.338 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.338 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.123489+0000", 2023-10-09T05:49:40.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064567210000000003 2023-10-09T05:49:40.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.339 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.125273+0000", 2023-10-09T05:49:40.339 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001783722 2023-10-09T05:49:40.339 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.339 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.132228+0000", 2023-10-09T05:49:40.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0069553769999999996 2023-10-09T05:49:40.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.132292+0000", 2023-10-09T05:49:40.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3936999999999994e-05 2023-10-09T05:49:40.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.133369+0000", 2023-10-09T05:49:40.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001077103 2023-10-09T05:49:40.341 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.341 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.342 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.342 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150734 7.14 7:2e650744:::benchmark_data_smithi008_354649_object56238:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:49:40.342 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.025678+0000", 2023-10-09T05:49:40.342 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 263.18323227899998, 2023-10-09T05:49:40.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.200712695, 2023-10-09T05:49:40.343 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.343 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.343 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.343 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.343 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.343 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150734 2023-10-09T05:49:40.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.344 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.344 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.344 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025678+0000", 2023-10-09T05:49:40.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.345 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025678+0000", 2023-10-09T05:49:40.345 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.345 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.345 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.346 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025676+0000", 2023-10-09T05:49:40.346 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:40.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.346 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.346 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025686+0000", 2023-10-09T05:49:40.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2490000000000007e-06 2023-10-09T05:49:40.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.347 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.347 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.347 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025700+0000", 2023-10-09T05:49:40.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4002000000000001e-05 2023-10-09T05:49:40.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025708+0000", 2023-10-09T05:49:40.348 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4409999999999993e-06 2023-10-09T05:49:40.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025787+0000", 2023-10-09T05:49:40.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8909e-05 2023-10-09T05:49:40.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025804+0000", 2023-10-09T05:49:40.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6718000000000002e-05 2023-10-09T05:49:40.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922949+0000", 2023-10-09T05:49:40.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89714522100000005 2023-10-09T05:49:40.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922958+0000", 2023-10-09T05:49:40.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3349999999999996e-06 2023-10-09T05:49:40.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.352 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.210519+0000", 2023-10-09T05:49:40.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2875604629999999 2023-10-09T05:49:40.352 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.352 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.210670+0000", 2023-10-09T05:49:40.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015123 2023-10-09T05:49:40.352 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.353 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:49:40.353 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.210978+0000", 2023-10-09T05:49:40.353 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00030860600000000001 2023-10-09T05:49:40.353 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.353 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.353 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.353 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.217348+0000", 2023-10-09T05:49:40.353 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0063698720000000004 2023-10-09T05:49:40.353 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.353 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.353 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.217358+0000", 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3179999999999996e-06 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.225201+0000", 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078434549999999992 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.225268+0000", 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7101999999999996e-05 2023-10-09T05:49:40.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.226391+0000", 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001122556 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150735 7.9 7:92c6da78:::benchmark_data_smithi008_354649_object56239:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.026350+0000", 2023-10-09T05:49:40.355 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 263.18256046499999, 2023-10-09T05:49:40.356 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1077192149999999, 2023-10-09T05:49:40.356 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.356 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.356 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.356 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.356 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.356 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150735 2023-10-09T05:49:40.356 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.356 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.356 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.356 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026350+0000", 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026350+0000", 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026349+0000", 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:49:40.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.358 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.358 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.358 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026354+0000", 2023-10-09T05:49:40.358 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4159999999999997e-06 2023-10-09T05:49:40.358 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.358 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.358 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.358 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026363+0000", 2023-10-09T05:49:40.358 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4760000000000004e-06 2023-10-09T05:49:40.358 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.358 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026368+0000", 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8380000000000001e-06 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026430+0000", 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2142000000000002e-05 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.359 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026444+0000", 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4007e-05 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917360+0000", 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89091560000000003 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917368+0000", 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4489999999999999e-06 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.121241+0000", 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.203872729 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.121389+0000", 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014778299999999999 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:49:40.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.121817+0000", 2023-10-09T05:49:40.362 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00042842200000000003 2023-10-09T05:49:40.362 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.125281+0000", 2023-10-09T05:49:40.362 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034634980000000002 2023-10-09T05:49:40.362 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.126920+0000", 2023-10-09T05:49:40.362 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016398389999999999 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.133435+0000", 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0065145170000000001 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.133481+0000", 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6368000000000001e-05 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.364 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.364 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.134069+0000", 2023-10-09T05:49:40.364 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00058786500000000002 2023-10-09T05:49:40.364 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.364 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.364 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.364 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.364 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.364 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150739 7.d 7:b35c0714:::benchmark_data_smithi008_354649_object56243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:49:40.364 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.167284+0000", 2023-10-09T05:49:40.364 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 263.041626511, 2023-10-09T05:49:40.365 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.022290125, 2023-10-09T05:49:40.365 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.365 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.365 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.365 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.365 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.365 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150739 2023-10-09T05:49:40.365 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.365 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167284+0000", 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167284+0000", 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167282+0000", 2023-10-09T05:49:40.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167290+0000", 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9489999999999996e-06 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167307+0000", 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7193e-05 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167316+0000", 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0899999999999994e-06 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167435+0000", 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000119153 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167447+0000", 2023-10-09T05:49:40.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1272e-05 2023-10-09T05:49:40.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922254+0000", 2023-10-09T05:49:40.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75480757799999998 2023-10-09T05:49:40.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922266+0000", 2023-10-09T05:49:40.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1959e-05 2023-10-09T05:49:40.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.131723+0000", 2023-10-09T05:49:40.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.20945652 2023-10-09T05:49:40.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.135226+0000", 2023-10-09T05:49:40.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003503515 2023-10-09T05:49:40.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.135659+0000", 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00043321599999999999 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.165146+0000", 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.029487005 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.165190+0000", 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3157e-05 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.168957+0000", 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037669999999999999 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.189402+0000", 2023-10-09T05:49:40.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020445218000000001 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.189455+0000", 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3368000000000001e-05 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.189574+0000", 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0001189 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.373 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.374 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.374 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150745 7.14 7:2dacea96:::benchmark_data_smithi008_354649_object56249:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:49:40.374 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.262460+0000", 2023-10-09T05:49:40.374 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 262.94645053300002, 2023-10-09T05:49:40.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9738604209999999, 2023-10-09T05:49:40.374 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.374 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.374 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.374 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150745 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262460+0000", 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262460+0000", 2023-10-09T05:49:40.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262458+0000", 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262466+0000", 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1210000000000007e-06 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262608+0000", 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014194000000000001 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262615+0000", 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2599999999999999e-06 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262665+0000", 2023-10-09T05:49:40.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9876000000000002e-05 2023-10-09T05:49:40.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262673+0000", 2023-10-09T05:49:40.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1780000000000003e-06 2023-10-09T05:49:40.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922980+0000", 2023-10-09T05:49:40.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.66030662399999995 2023-10-09T05:49:40.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922988+0000", 2023-10-09T05:49:40.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3159999999999997e-06 2023-10-09T05:49:40.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.214973+0000", 2023-10-09T05:49:40.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2919844110000001 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.215142+0000", 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016878100000000001 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.215556+0000", 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000414208 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.218997+0000", 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034408960000000001 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.224502+0000", 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0055058310000000001 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.233025+0000", 2023-10-09T05:49:40.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0085221529999999993 2023-10-09T05:49:40.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.235169+0000", 2023-10-09T05:49:40.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002143874 2023-10-09T05:49:40.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.236320+0000", 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001151745 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150749 7.19 7:980a22d1:::benchmark_data_smithi008_354649_object56253:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.300508+0000", 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 262.908402819, 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.833932568, 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.383 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150749 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.300508+0000", 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.300508+0000", 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.300506+0000", 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.385 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.300513+0000", 2023-10-09T05:49:40.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8369999999999999e-06 2023-10-09T05:49:40.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.386 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.300529+0000", 2023-10-09T05:49:40.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6396999999999999e-05 2023-10-09T05:49:40.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.386 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.300537+0000", 2023-10-09T05:49:40.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9659999999999996e-06 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.300594+0000", 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6799999999999998e-05 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.300603+0000", 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4399999999999994e-06 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917406+0000", 2023-10-09T05:49:40.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.61680217599999998 2023-10-09T05:49:40.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917410+0000", 2023-10-09T05:49:40.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0969999999999999e-06 2023-10-09T05:49:40.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.123524+0000", 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.206114892 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.123690+0000", 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016582399999999999 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.124109+0000", 2023-10-09T05:49:40.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00041830900000000001 2023-10-09T05:49:40.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.126160+0000", 2023-10-09T05:49:40.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00205091 2023-10-09T05:49:40.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.127109+0000", 2023-10-09T05:49:40.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00094944500000000004 2023-10-09T05:49:40.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.133429+0000", 2023-10-09T05:49:40.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0063201209999999997 2023-10-09T05:49:40.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.133481+0000", 2023-10-09T05:49:40.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1461000000000001e-05 2023-10-09T05:49:40.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.134440+0000", 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095955699999999999 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180271 7.1d 7:baeacd59:::benchmark_data_smithi008_354649_object85775:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 179.079398056, 2023-10-09T05:49:40.392 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.855218754, 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180271 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:49:40.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129509+0000", 2023-10-09T05:49:40.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:49:40.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129525+0000", 2023-10-09T05:49:40.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5430999999999998e-05 2023-10-09T05:49:40.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129553+0000", 2023-10-09T05:49:40.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8175e-05 2023-10-09T05:49:40.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129568+0000", 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5267000000000001e-05 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.333120+0000", 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20355221500000001 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.333134+0000", 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4073000000000001e-05 2023-10-09T05:49:40.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929537+0000", 2023-10-09T05:49:40.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5964021320000001 2023-10-09T05:49:40.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929548+0000", 2023-10-09T05:49:40.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1759999999999999e-05 2023-10-09T05:49:40.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.974829+0000", 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.045281066 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.974986+0000", 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000156174 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:49:40.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.975339+0000", 2023-10-09T05:49:40.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00035308699999999998 2023-10-09T05:49:40.399 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.399 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.399 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.399 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977566+0000", 2023-10-09T05:49:40.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022271399999999998 2023-10-09T05:49:40.399 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.399 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.399 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.399 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984241+0000", 2023-10-09T05:49:40.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066755160000000003 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984422+0000", 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000180633 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984530+0000", 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010849999999999999 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.400 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984609+0000", 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8506999999999999e-05 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984731+0000", 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012211599999999999 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.402 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180280 7.1d 7:b9a77225:::benchmark_data_smithi008_354649_object85784:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:49:40.402 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:49:40.402 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 178.847050346, 2023-10-09T05:49:40.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6230030800000002, 2023-10-09T05:49:40.402 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.402 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.402 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.402 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.402 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.402 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180280 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361859+0000", 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361865+0000", 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9800000000000003e-06 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361876+0000", 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1178999999999999e-05 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361882+0000", 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0070000000000003e-06 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361924+0000", 2023-10-09T05:49:40.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2243999999999997e-05 2023-10-09T05:49:40.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361931+0000", 2023-10-09T05:49:40.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.097e-06 2023-10-09T05:49:40.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929573+0000", 2023-10-09T05:49:40.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.567641397 2023-10-09T05:49:40.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929580+0000", 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4569999999999999e-06 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977745+0000", 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0481653790000001 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977840+0000", 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4581999999999997e-05 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.978096+0000", 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00025544499999999998 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984291+0000", 2023-10-09T05:49:40.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061956720000000002 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984301+0000", 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7710000000000001e-06 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984457+0000", 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000155885 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984780+0000", 2023-10-09T05:49:40.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00032328700000000001 2023-10-09T05:49:40.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984806+0000", 2023-10-09T05:49:40.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6290000000000001e-05 2023-10-09T05:49:40.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984863+0000", 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6693000000000001e-05 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180292 7.1f 7:fc8bce44:::benchmark_data_smithi008_354649_object85796:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 178.68142439100001, 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4946860599999998, 2023-10-09T05:49:40.411 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180292 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.412 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:49:40.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527485+0000", 2023-10-09T05:49:40.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:49:40.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527491+0000", 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.139e-06 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527504+0000", 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2964e-05 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527512+0000", 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2230000000000007e-06 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527553+0000", 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1266000000000001e-05 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527560+0000", 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6599999999999998e-06 2023-10-09T05:49:40.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929860+0000", 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4023002959999999 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929869+0000", 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9080000000000001e-06 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986248+0000", 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0563782779999999 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986407+0000", 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000159588 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986738+0000", 2023-10-09T05:49:40.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00033078600000000001 2023-10-09T05:49:40.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.015271+0000", 2023-10-09T05:49:40.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028533060999999998 2023-10-09T05:49:40.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.015291+0000", 2023-10-09T05:49:40.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0077000000000001e-05 2023-10-09T05:49:40.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.021759+0000", 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064680570000000001 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.021808+0000", 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8615000000000001e-05 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.022172+0000", 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00036433100000000002 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180305 7.1d 7:bd26cfd0:::benchmark_data_smithi008_354649_object85809:head [delete] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.064410+0000", 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 178.144500843, 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.920595904, 2023-10-09T05:49:40.420 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.421 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.421 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.421 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.421 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.421 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180305 2023-10-09T05:49:40.421 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.421 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.421 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.421 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064410+0000", 2023-10-09T05:49:40.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064410+0000", 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064408+0000", 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064413+0000", 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6400000000000002e-06 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064423+0000", 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7489999999999993e-06 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.423 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064430+0000", 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8399999999999997e-06 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064529+0000", 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9314000000000005e-05 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064537+0000", 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0450000000000001e-06 2023-10-09T05:49:40.424 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929599+0000", 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86506175299999999 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929606+0000", 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0119999999999999e-06 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.979407+0000", 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0498009509999999 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.979540+0000", 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000132836 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.979783+0000", 2023-10-09T05:49:40.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000243329 2023-10-09T05:49:40.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.427 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.427 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984308+0000", 2023-10-09T05:49:40.427 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045243610000000002 2023-10-09T05:49:40.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.427 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.427 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984349+0000", 2023-10-09T05:49:40.428 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:40.428 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:40 smithi008 ceph-mon[184418]: pgmap v954: 109 pgs: 2 peering, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 35341/289509 objects degraded (12.207%); 1341/289509 objects misplaced (0.463%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T05:49:40.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1041999999999998e-05 2023-10-09T05:49:40.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.429 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984496+0000", 2023-10-09T05:49:40.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014723000000000001 2023-10-09T05:49:40.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.430 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984921+0000", 2023-10-09T05:49:40.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00042504100000000003 2023-10-09T05:49:40.430 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.430 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.430 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984948+0000", 2023-10-09T05:49:40.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6665e-05 2023-10-09T05:49:40.430 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.430 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.430 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.985005+0000", 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7825999999999998e-05 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180323 7.1d 7:bef99664:::benchmark_data_smithi008_354649_object85827:head [delete] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.107250+0000", 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 178.10166053899999, 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8812351980000002, 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.431 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180323 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107250+0000", 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.432 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107250+0000", 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107248+0000", 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.433 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107255+0000", 2023-10-09T05:49:40.434 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.708e-06 2023-10-09T05:49:40.434 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.434 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.434 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.434 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107267+0000", 2023-10-09T05:49:40.434 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2282e-05 2023-10-09T05:49:40.434 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.434 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.434 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.434 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107275+0000", 2023-10-09T05:49:40.434 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8979999999999996e-06 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107338+0000", 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3078999999999997e-05 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107348+0000", 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9119999999999993e-06 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929626+0000", 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.82227738500000003 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929632+0000", 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6510000000000004e-06 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.981128+0000", 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.051495222 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.981256+0000", 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000128782 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.981435+0000", 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000179144 2023-10-09T05:49:40.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.438 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.438 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984314+0000", 2023-10-09T05:49:40.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002878598 2023-10-09T05:49:40.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.438 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.438 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984387+0000", 2023-10-09T05:49:40.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2719000000000006e-05 2023-10-09T05:49:40.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.988300+0000", 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039127140000000003 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.988334+0000", 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4264000000000003e-05 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.988376+0000", 2023-10-09T05:49:40.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2225000000000003e-05 2023-10-09T05:49:40.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.440 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.440 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.988485+0000", 2023-10-09T05:49:40.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010901899999999999 2023-10-09T05:49:40.440 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.440 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.440 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.440 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180353 7.1d 7:b94e9c32:::benchmark_data_smithi008_354649_object85857:head [delete] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.161196+0000", 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 178.04771453699999, 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8349015990000002, 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180353 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.441 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:49:40.442 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.442 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:49:40.442 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161196+0000", 2023-10-09T05:49:40.442 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.442 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.442 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.442 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:49:40.442 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161196+0000", 2023-10-09T05:49:40.442 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:49:40.442 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.442 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161194+0000", 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161202+0000", 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6799999999999993e-06 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:49:40.443 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161216+0000", 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3688e-05 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161224+0000", 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4319999999999999e-06 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161368+0000", 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014366800000000001 2023-10-09T05:49:40.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.445 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.445 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161380+0000", 2023-10-09T05:49:40.445 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2149e-05 2023-10-09T05:49:40.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.445 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.445 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929653+0000", 2023-10-09T05:49:40.445 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.76827273799999996 2023-10-09T05:49:40.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.446 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:49:40.446 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929660+0000", 2023-10-09T05:49:40.446 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5000000000000002e-06 2023-10-09T05:49:40.446 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.446 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.446 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:49:40.446 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.985043+0000", 2023-10-09T05:49:40.446 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.055382222 2023-10-09T05:49:40.446 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.446 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.446 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.985137+0000", 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4852000000000006e-05 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.985402+0000", 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000265087 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.992072+0000", 2023-10-09T05:49:40.447 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066693619999999999 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.992108+0000", 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6211000000000003e-05 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.995908+0000", 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037999230000000002 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.448 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.995923+0000", 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4630999999999999e-05 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.995976+0000", 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3701e-05 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.996097+0000", 2023-10-09T05:49:40.449 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012113299999999999 2023-10-09T05:49:40.450 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.450 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.450 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.450 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:49:40.450 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:49:40.450 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:49:40.450 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:40 smithi008 ceph-mon[191227]: pgmap v954: 109 pgs: 2 peering, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 35341/289509 objects degraded (12.207%); 1341/289509 objects misplaced (0.463%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T05:49:40.456 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:49:40.465 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:49:40.503 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:49:40.506 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:49:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:49:40] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.33.4" 2023-10-09T05:49:40.535 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:49:40.545 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:40.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:40 smithi191 ceph-mon[161669]: pgmap v954: 109 pgs: 2 peering, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 89 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 35341/289509 objects degraded (12.207%); 1341/289509 objects misplaced (0.463%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T05:49:40.580 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:49:40.602 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:49:40.651 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:49:40.685 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:40.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:40.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:40.692 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:49:40.697 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:49:40.697 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:49:40.698 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:49:40.698 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:49:40.698 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:49:40.698 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:49:40.698 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:49:40.698 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:49:40.786 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:49:40.792 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:40.857 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:40.907 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:40.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:40.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:40.918 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:49:41.012 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:41.062 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:41.098 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:41 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3308457543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:41.098 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:41 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3187725535' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:49:41.098 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:41 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:49:41.358 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:41 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3308457543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:41.358 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:41 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3187725535' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:49:41.358 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:41 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:49:41.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:41 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3308457543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:41.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:41 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3187725535' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:49:41.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:41 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:49:42.111 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:49:42.118 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:42.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:42.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:42.148 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:49:42.225 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:42.270 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:42.324 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:42.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:42.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:42.428 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:42.461 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:42 smithi008 ceph-mon[184418]: pgmap v955: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 35024/286074 objects degraded (12.243%); 1298/286074 objects misplaced (0.454%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:49:42.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:42 smithi008 ceph-mon[191227]: pgmap v955: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 35024/286074 objects degraded (12.243%); 1298/286074 objects misplaced (0.454%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:49:42.466 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:42.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:42 smithi191 ceph-mon[161669]: pgmap v955: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 35024/286074 objects degraded (12.243%); 1298/286074 objects misplaced (0.454%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:49:42.728 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:42.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:42.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:42.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:49:42.862 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:42.905 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:49:42.905 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:42.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:42.955 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:43.012 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:43.049 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:43.268 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:43.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:43.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:43.370 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 35024/286074 objects degraded (12.243%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:43.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 35024/286074 objects degraded (12.243%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:43.374 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:43.448 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:43.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 35024/286074 objects degraded (12.243%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:43.729 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:43.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:43.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:43.834 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:43.875 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:44.416 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:44 smithi008 ceph-mon[184418]: pgmap v956: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 35024/285804 objects degraded (12.255%); 1298/285804 objects misplaced (0.454%); 6.8 MiB/s, 112 objects/s recovering 2023-10-09T05:49:44.416 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:44.416 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:44 smithi008 ceph-mon[191227]: pgmap v956: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 35024/285804 objects degraded (12.255%); 1298/285804 objects misplaced (0.454%); 6.8 MiB/s, 112 objects/s recovering 2023-10-09T05:49:44.416 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:44.443 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:44.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:44.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:44.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:49:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:44 smithi191 ceph-mon[161669]: pgmap v956: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 35024/285804 objects degraded (12.255%); 1298/285804 objects misplaced (0.454%); 6.8 MiB/s, 112 objects/s recovering 2023-10-09T05:49:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:44.575 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:44.620 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:49:44.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:44.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:44.725 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:44.775 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:49:44.776 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:45.066 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:45.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:45.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:45.175 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:45.290 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:45.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:45 smithi008 ceph-mon[184418]: pgmap v957: 109 pgs: 1 active+recovery_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 35024/284865 objects degraded (12.295%); 1298/284865 objects misplaced (0.456%); 2.7 MiB/s, 44 objects/s recovering 2023-10-09T05:49:45.466 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:45.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:45 smithi008 ceph-mon[191227]: pgmap v957: 109 pgs: 1 active+recovery_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 35024/284865 objects degraded (12.295%); 1298/284865 objects misplaced (0.456%); 2.7 MiB/s, 44 objects/s recovering 2023-10-09T05:49:45.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:45.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:45.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:45 smithi191 ceph-mon[161669]: pgmap v957: 109 pgs: 1 active+recovery_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 35024/284865 objects degraded (12.295%); 1298/284865 objects misplaced (0.456%); 2.7 MiB/s, 44 objects/s recovering 2023-10-09T05:49:45.579 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:45.625 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:45.927 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:45.963 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:45.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:45.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:46.020 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:49:46.020 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:49:46.072 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:49:46.098 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:46.140 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:49:46.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:46.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:49:46.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:49:46.275 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:46.301 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2775149552' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:49:46.301 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:46 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:49:46.334 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:49:46.337 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:49:46.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:46.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:49:46.444 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:46.500 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:46.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2775149552' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:49:46.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:46 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:49:46.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2775149552' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:49:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:46 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:49:47.113 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:49:47.113 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:49:47.113 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-10-09T05:49:47.114 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:49:47.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:49:47.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[184418]: failed to encode map e319 with expected crc 2023-10-09T05:49:47.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[184418]: failed to encode map e319 with expected crc 2023-10-09T05:49:47.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:49:47.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[184418]: osdmap e319: 8 total, 8 up, 8 in 2023-10-09T05:49:47.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[184418]: pgmap v959: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 34718/282732 objects degraded (12.279%); 1286/282732 objects misplaced (0.455%); 4.5 MiB/s, 73 objects/s recovering 2023-10-09T05:49:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:49:47.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[191227]: failed to encode map e319 with expected crc 2023-10-09T05:49:47.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[191227]: failed to encode map e319 with expected crc 2023-10-09T05:49:47.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:49:47.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[191227]: osdmap e319: 8 total, 8 up, 8 in 2023-10-09T05:49:47.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:47 smithi008 ceph-mon[191227]: pgmap v959: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 34718/282732 objects degraded (12.279%); 1286/282732 objects misplaced (0.455%); 4.5 MiB/s, 73 objects/s recovering 2023-10-09T05:49:47.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:47 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:49:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:47 smithi191 ceph-mon[161669]: failed to encode map e319 with expected crc 2023-10-09T05:49:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:47 smithi191 ceph-mon[161669]: failed to encode map e319 with expected crc 2023-10-09T05:49:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:47 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:49:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:47 smithi191 ceph-mon[161669]: osdmap e319: 8 total, 8 up, 8 in 2023-10-09T05:49:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:47 smithi191 ceph-mon[161669]: pgmap v959: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 34718/282732 objects degraded (12.279%); 1286/282732 objects misplaced (0.455%); 4.5 MiB/s, 73 objects/s recovering 2023-10-09T05:49:47.842 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:47.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:47.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:47.957 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:48.044 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:48.057 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:49:48.257 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:48.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:48.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:48.362 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:48.392 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:48 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:49:48.392 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 34718/282732 objects degraded (12.279%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:48.392 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:48.392 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:48 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:49:48.392 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 34718/282732 objects degraded (12.279%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:48.393 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:48.403 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:48.779 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:48.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:48.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:48.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:48 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:49:48.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 34718/282732 objects degraded (12.279%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:48.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:48.894 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:48.979 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:48.982 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:49.030 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:49.202 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:49.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:49.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:49.313 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:49.370 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:49.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:49 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:49:49.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:49 smithi008 ceph-mon[184418]: pgmap v960: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 234 op/s; 34718/282486 objects degraded (12.290%); 1286/282486 objects misplaced (0.455%); 4.2 MiB/s, 69 objects/s recovering 2023-10-09T05:49:49.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:49 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:49:49.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:49 smithi008 ceph-mon[191227]: pgmap v960: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 234 op/s; 34718/282486 objects degraded (12.290%); 1286/282486 objects misplaced (0.455%); 4.2 MiB/s, 69 objects/s recovering 2023-10-09T05:49:49.526 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:49:49.568 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:49.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:49.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:49.574 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:49:49.607 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:49:49.644 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:49:49.674 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:49.676 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:49:49.719 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:49:49.772 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:49.772 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:49:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:49 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:49:49.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:49 smithi191 ceph-mon[161669]: pgmap v960: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 234 op/s; 34718/282486 objects degraded (12.290%); 1286/282486 objects misplaced (0.455%); 4.2 MiB/s, 69 objects/s recovering 2023-10-09T05:49:49.808 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:49:49.810 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:49:49.810 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:49:49.810 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:49:49.810 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:49:49.810 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:49:49.810 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:49:49.810 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:49:49.810 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:49:49.974 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:49:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1763382046' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1763382046' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:50.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1763382046' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:49:50.959 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:49:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:49:50] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.33.4" 2023-10-09T05:49:51.696 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:51.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:51.713 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:51 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:49:51.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:51 smithi008 ceph-mon[184418]: pgmap v961: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 34383/278784 objects degraded (12.333%); 1234/278784 objects misplaced (0.443%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:49:51.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:51 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:51.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:51 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:51.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:51 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:49:51.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:51 smithi008 ceph-mon[191227]: pgmap v961: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 34383/278784 objects degraded (12.333%); 1234/278784 objects misplaced (0.443%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:49:51.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:51 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:51.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:51 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:51.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:51 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:49:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:51 smithi191 ceph-mon[161669]: pgmap v961: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 34383/278784 objects degraded (12.333%); 1234/278784 objects misplaced (0.443%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:49:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:51 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:51 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:51.815 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:49:51.846 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:51.893 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:49:51.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:51.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:49:51.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:52.028 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:52.077 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:52.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:52.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:52.183 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:49:52.210 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:52.253 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:49:52.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:52.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:49:52.360 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:52.390 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:52.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:52 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:49:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:52 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:49:52.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:52 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:49:52.748 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:52.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:52.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:52.852 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:49:52.880 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:52.928 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:49:52.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:52.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:53.034 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:49:53.061 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:53.111 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:49:53.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:53.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:49:53.181 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:53.217 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:53.333 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:49:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:49:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:49:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:49:53.333 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:49:53.340 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:53.694 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:53 smithi008 ceph-mon[184418]: pgmap v962: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 34383/278562 objects degraded (12.343%); 1234/278562 objects misplaced (0.443%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:49:53.695 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 34383/278562 objects degraded (12.343%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:53.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:53 smithi008 ceph-mon[191227]: pgmap v962: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 34383/278562 objects degraded (12.343%); 1234/278562 objects misplaced (0.443%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:49:53.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 34383/278562 objects degraded (12.343%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:53.732 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:53.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:53.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:53 smithi191 ceph-mon[161669]: pgmap v962: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 34383/278562 objects degraded (12.343%); 1234/278562 objects misplaced (0.443%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:49:53.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 34383/278562 objects degraded (12.343%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:53.838 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:49:53.865 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:54.130 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:49:54.488 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:49:54.492 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:49:54.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:54.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:49:54.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:49:54.625 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:54.668 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:49:54.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:54.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:49:54.775 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:49:54.801 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:54.844 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:49:54.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:54.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:49:54.949 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:54.990 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:55.631 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:55.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:55.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:55.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:55.753 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:49:55.755 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:49:55.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:55 smithi191 ceph-mon[161669]: pgmap v963: 109 pgs: 1 active+recovery_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 243 op/s; 34383/277557 objects degraded (12.388%); 1234/277557 objects misplaced (0.445%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:49:55.769 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:55.773 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:55.787 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:55 smithi008 ceph-mon[184418]: pgmap v963: 109 pgs: 1 active+recovery_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 243 op/s; 34383/277557 objects degraded (12.388%); 1234/277557 objects misplaced (0.445%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:49:55.787 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:55 smithi008 ceph-mon[191227]: pgmap v963: 109 pgs: 1 active+recovery_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 243 op/s; 34383/277557 objects degraded (12.388%); 1234/277557 objects misplaced (0.445%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T05:49:55.814 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:55.815 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:55.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:55.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:55.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:49:55.944 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:55.995 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:49:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:56.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:49:56.101 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:56.153 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:56.756 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:56 smithi008 ceph-mon[184418]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:49:56.757 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:56 smithi008 ceph-mon[191227]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:49:56.793 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":963,"stamp":"2023-10-09T05:49:54.415979+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6066214859,"num_objects":92519,"num_object_clones":0,"num_object_copies":277557,"num_objects_missing_on_primary":18684,"num_objects_missing":18684,"num_objects_degraded":34383,"num_objects_misplaced":1234,"num_objects_unfound":0,"num_objects_dirty":92519,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":114495,"num_write_kb":6603238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4830,"num_bytes_recovered":311176362,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":232668,"ondisk_log_size":232668,"up":327,"acting":323,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":347,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24276944,"kb_used_data":18450448,"kb_used_omap":45,"kb_used_meta":5826386,"kb_avail":725684272,"statfs":{"total":767960285184,"available":743100694528,"internally_reserved":0,"allocated":18893258752,"data_stored":18886191445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":5966219555},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2,1,1,2,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-159645696,"num_objects":-2436,"num_object_clones":0,"num_object_copies":-7308,"num_objects_missing_on_primary":-532,"num_objects_missing":-532,"num_objects_degraded":-641,"num_objects_misplaced":-64,"num_objects_unfound":0,"num_objects_dirty":-2436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2436,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003659"},"pg_stats":[{"pgid":"8.23","version":"319'6102","reported_seq":6460,"reported_epoch":319,"state":"active+recovering","last_fresh":"2023-10-09T05:49:49.713862+0000","last_change":"2023-10-09T05:49:33.820438+0000","last_active":"2023-10-09T05:49:49.713862+0000","last_peered":"2023-10-09T05:49:49.713862+0000","last_clean":"2023-10-09T05:48:53.427903+0000","last_became_active":"2023-10-09T05:49:21.560913+0000","last_became_peered":"2023-10-09T05:49:21.560913+0000","last_unstale":"2023-10-09T05:49:49.713862+0000","last_undegraded":"2023-10-09T05:49:49.713862+0000","last_fullsized":"2023-10-09T05:49:49.713862+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6102,"log_dups_size":0,"ondisk_log_size":6102,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95780864,"num_objects":1462,"num_object_clones":0,"num_object_copies":4386,"num_objects_missing_on_primary":1287,"num_objects_missing":1287,"num_objects_degraded":0,"num_objects_misplaced":1224,"num_objects_unfound":0,"num_objects_dirty":1462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1746,"num_write_kb":102624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":146,"num_bytes_recovered":9568256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":238},{"shards":"2,5,6","objects":1224}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.22","version":"319'6064","reported_seq":6125,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:50.064419+0000","last_change":"2023-10-09T05:49:20.140728+0000","last_active":"2023-10-09T05:49:50.064419+0000","last_peered":"2023-10-09T05:49:50.064419+0000","last_clean":"2023-10-09T05:49:50.064419+0000","last_became_active":"2023-10-09T05:49:20.132812+0000","last_became_peered":"2023-10-09T05:49:20.132812+0000","last_unstale":"2023-10-09T05:49:50.064419+0000","last_undegraded":"2023-10-09T05:49:50.064419+0000","last_fullsized":"2023-10-09T05:49:50.064419+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6064,"log_dups_size":0,"ondisk_log_size":6064,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93798400,"num_objects":1431,"num_object_clones":0,"num_object_copies":4293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1742,"num_write_kb":101552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.21","version":"319'6125","reported_seq":6370,"reported_epoch":319,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:49.610030+0000","last_change":"2023-10-09T05:49:21.991103+0000","last_active":"2023-10-09T05:49:49.610030+0000","last_peered":"2023-10-09T05:49:49.610030+0000","last_clean":"2023-10-09T05:48:53.421353+0000","last_became_active":"2023-10-09T05:49:21.956287+0000","last_became_peered":"2023-10-09T05:49:21.956287+0000","last_unstale":"2023-10-09T05:49:49.610030+0000","last_undegraded":"2023-10-09T05:49:21.126622+0000","last_fullsized":"2023-10-09T05:49:49.610030+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6125,"log_dups_size":0,"ondisk_log_size":6125,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94699520,"num_objects":1445,"num_object_clones":0,"num_object_copies":4335,"num_objects_missing_on_primary":1379,"num_objects_missing":1379,"num_objects_degraded":1379,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1758,"num_write_kb":102496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":5898240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,1,2","objects":138},{"shards":"0,2,5","objects":1307}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.20","version":"319'6203","reported_seq":6463,"reported_epoch":319,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:48.470758+0000","last_change":"2023-10-09T05:49:21.912294+0000","last_active":"2023-10-09T05:49:48.470758+0000","last_peered":"2023-10-09T05:49:48.470758+0000","last_clean":"2023-10-09T05:48:53.429181+0000","last_became_active":"2023-10-09T05:49:21.894029+0000","last_became_peered":"2023-10-09T05:49:21.894029+0000","last_unstale":"2023-10-09T05:49:48.470758+0000","last_undegraded":"2023-10-09T05:49:21.127313+0000","last_fullsized":"2023-10-09T05:49:48.470758+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6203,"log_dups_size":0,"ondisk_log_size":6203,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96829440,"num_objects":1478,"num_object_clones":0,"num_object_copies":4434,"num_objects_missing_on_primary":1320,"num_objects_missing":1320,"num_objects_degraded":1320,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1780,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":6094848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,4","objects":211},{"shards":"1,3,4,6","objects":1},{"shards":"3,4,6","objects":1266}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.11","version":"319'6291","reported_seq":6348,"reported_epoch":319,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:54.006999+0000","last_change":"2023-10-09T05:48:55.843352+0000","last_active":"2023-10-09T05:49:54.006999+0000","last_peered":"2023-10-09T05:49:54.006999+0000","last_clean":"2023-10-09T05:48:53.444054+0000","last_became_active":"2023-10-09T05:48:55.837353+0000","last_became_peered":"2023-10-09T05:48:55.837353+0000","last_unstale":"2023-10-09T05:49:54.006999+0000","last_undegraded":"2023-10-09T05:48:55.496526+0000","last_fullsized":"2023-10-09T05:48:55.477252+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6291,"log_dups_size":0,"ondisk_log_size":6291,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185860096,"num_objects":2836,"num_object_clones":0,"num_object_copies":8508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2852,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3592,"num_write_kb":205696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":-16},{"shards":"0,2,5","objects":2474}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":319,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.065308+0000","last_change":"2023-10-09T05:48:54.494513+0000","last_active":"2023-10-09T05:49:37.065308+0000","last_peered":"2023-10-09T05:49:37.065308+0000","last_clean":"2023-10-09T05:49:37.065308+0000","last_became_active":"2023-10-09T05:48:54.493510+0000","last_became_peered":"2023-10-09T05:48:54.493510+0000","last_unstale":"2023-10-09T05:49:37.065308+0000","last_undegraded":"2023-10-09T05:49:37.065308+0000","last_fullsized":"2023-10-09T05:49:37.065308+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.088830+0000","last_change":"2023-10-09T05:48:54.492024+0000","last_active":"2023-10-09T05:49:37.088830+0000","last_peered":"2023-10-09T05:49:37.088830+0000","last_clean":"2023-10-09T05:49:37.088830+0000","last_became_active":"2023-10-09T05:48:54.491690+0000","last_became_peered":"2023-10-09T05:48:54.491690+0000","last_unstale":"2023-10-09T05:49:37.088830+0000","last_undegraded":"2023-10-09T05:49:37.088830+0000","last_fullsized":"2023-10-09T05:49:37.088830+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:41.936752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.12","version":"319'6161","reported_seq":6212,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:50.072188+0000","last_change":"2023-10-09T05:48:46.263477+0000","last_active":"2023-10-09T05:49:50.072188+0000","last_peered":"2023-10-09T05:49:50.072188+0000","last_clean":"2023-10-09T05:49:50.072188+0000","last_became_active":"2023-10-09T05:48:46.259060+0000","last_became_peered":"2023-10-09T05:48:46.259060+0000","last_unstale":"2023-10-09T05:49:50.072188+0000","last_undegraded":"2023-10-09T05:49:50.072188+0000","last_fullsized":"2023-10-09T05:49:50.072188+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185171968,"num_objects":2826,"num_object_clones":0,"num_object_copies":8478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3494,"num_write_kb":202208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.101515+0000","last_change":"2023-10-09T05:48:54.491563+0000","last_active":"2023-10-09T05:49:37.101515+0000","last_peered":"2023-10-09T05:49:37.101515+0000","last_clean":"2023-10-09T05:49:37.101515+0000","last_became_active":"2023-10-09T05:48:54.491227+0000","last_became_peered":"2023-10-09T05:48:54.491227+0000","last_unstale":"2023-10-09T05:49:37.101515+0000","last_undegraded":"2023-10-09T05:49:37.101515+0000","last_fullsized":"2023-10-09T05:49:37.101515+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:37.990305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":326,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.071787+0000","last_change":"2023-10-09T05:48:54.493724+0000","last_active":"2023-10-09T05:49:37.071787+0000","last_peered":"2023-10-09T05:49:37.071787+0000","last_clean":"2023-10-09T05:49:37.071787+0000","last_became_active":"2023-10-09T05:48:54.493380+0000","last_became_peered":"2023-10-09T05:48:54.493380+0000","last_unstale":"2023-10-09T05:49:37.071787+0000","last_undegraded":"2023-10-09T05:49:37.071787+0000","last_fullsized":"2023-10-09T05:49:37.071787+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:05.264766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"319'6201","reported_seq":6256,"reported_epoch":319,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:50.151592+0000","last_change":"2023-10-09T05:48:55.978875+0000","last_active":"2023-10-09T05:49:50.151592+0000","last_peered":"2023-10-09T05:49:50.151592+0000","last_clean":"2023-10-09T05:48:53.465296+0000","last_became_active":"2023-10-09T05:48:55.972707+0000","last_became_peered":"2023-10-09T05:48:55.972707+0000","last_unstale":"2023-10-09T05:49:50.151592+0000","last_undegraded":"2023-10-09T05:48:55.487364+0000","last_fullsized":"2023-10-09T05:48:55.475509+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6201,"log_dups_size":0,"ondisk_log_size":6201,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188481536,"num_objects":2876,"num_object_clones":0,"num_object_copies":8628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2813,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3510,"num_write_kb":204352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[5,6],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5,6","objects":2495},{"shards":"5,6","objects":63}],"blocked_by":[],"up_primary":1,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":287,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.069931+0000","last_change":"2023-10-09T05:48:54.492664+0000","last_active":"2023-10-09T05:49:37.069931+0000","last_peered":"2023-10-09T05:49:37.069931+0000","last_clean":"2023-10-09T05:49:37.069931+0000","last_became_active":"2023-10-09T05:48:54.492299+0000","last_became_peered":"2023-10-09T05:48:54.492299+0000","last_unstale":"2023-10-09T05:49:37.069931+0000","last_undegraded":"2023-10-09T05:49:37.069931+0000","last_fullsized":"2023-10-09T05:49:37.069931+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:43.198424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":152,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.069369+0000","last_change":"2023-10-09T05:48:54.492447+0000","last_active":"2023-10-09T05:49:37.069369+0000","last_peered":"2023-10-09T05:49:37.069369+0000","last_clean":"2023-10-09T05:49:37.069369+0000","last_became_active":"2023-10-09T05:48:54.492089+0000","last_became_peered":"2023-10-09T05:48:54.492089+0000","last_unstale":"2023-10-09T05:49:37.069369+0000","last_undegraded":"2023-10-09T05:49:37.069369+0000","last_fullsized":"2023-10-09T05:49:37.069369+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.1c","version":"319'7107","reported_seq":7160,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:53.725208+0000","last_change":"2023-10-09T05:49:20.384940+0000","last_active":"2023-10-09T05:49:53.725208+0000","last_peered":"2023-10-09T05:49:53.725208+0000","last_clean":"2023-10-09T05:49:53.725208+0000","last_became_active":"2023-10-09T05:49:20.380945+0000","last_became_peered":"2023-10-09T05:49:20.380945+0000","last_unstale":"2023-10-09T05:49:53.725208+0000","last_undegraded":"2023-10-09T05:49:53.725208+0000","last_fullsized":"2023-10-09T05:49:53.725208+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7107,"log_dups_size":0,"ondisk_log_size":7107,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190513152,"num_objects":2907,"num_object_clones":0,"num_object_copies":8721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3675,"num_write_kb":210624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":274,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.071652+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:49:37.071652+0000","last_peered":"2023-10-09T05:49:37.071652+0000","last_clean":"2023-10-09T05:49:37.071652+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:49:37.071652+0000","last_undegraded":"2023-10-09T05:49:37.071652+0000","last_fullsized":"2023-10-09T05:49:37.071652+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.095109+0000","last_change":"2023-10-09T05:48:54.486629+0000","last_active":"2023-10-09T05:49:37.095109+0000","last_peered":"2023-10-09T05:49:37.095109+0000","last_clean":"2023-10-09T05:49:37.095109+0000","last_became_active":"2023-10-09T05:48:54.486107+0000","last_became_peered":"2023-10-09T05:48:54.486107+0000","last_unstale":"2023-10-09T05:49:37.095109+0000","last_undegraded":"2023-10-09T05:49:37.095109+0000","last_fullsized":"2023-10-09T05:49:37.095109+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:58.679726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1d","version":"319'6804","reported_seq":6856,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:50.201217+0000","last_change":"2023-10-09T05:49:20.157594+0000","last_active":"2023-10-09T05:49:50.201217+0000","last_peered":"2023-10-09T05:49:50.201217+0000","last_clean":"2023-10-09T05:49:50.201217+0000","last_became_active":"2023-10-09T05:49:20.153063+0000","last_became_peered":"2023-10-09T05:49:20.153063+0000","last_unstale":"2023-10-09T05:49:50.201217+0000","last_undegraded":"2023-10-09T05:49:50.201217+0000","last_fullsized":"2023-10-09T05:49:50.201217+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6804,"log_dups_size":0,"ondisk_log_size":6804,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187662348,"num_objects":2864,"num_object_clones":0,"num_object_copies":8592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2864,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3498,"num_write_kb":203553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":338,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064152+0000","last_change":"2023-10-09T05:48:54.491950+0000","last_active":"2023-10-09T05:49:37.064152+0000","last_peered":"2023-10-09T05:49:37.064152+0000","last_clean":"2023-10-09T05:49:37.064152+0000","last_became_active":"2023-10-09T05:48:54.491559+0000","last_became_peered":"2023-10-09T05:48:54.491559+0000","last_unstale":"2023-10-09T05:49:37.064152+0000","last_undegraded":"2023-10-09T05:49:37.064152+0000","last_fullsized":"2023-10-09T05:49:37.064152+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":254,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.063651+0000","last_change":"2023-10-09T05:48:25.068328+0000","last_active":"2023-10-09T05:49:37.063651+0000","last_peered":"2023-10-09T05:49:37.063651+0000","last_clean":"2023-10-09T05:49:37.063651+0000","last_became_active":"2023-10-09T05:48:25.068094+0000","last_became_peered":"2023-10-09T05:48:25.068094+0000","last_unstale":"2023-10-09T05:49:37.063651+0000","last_undegraded":"2023-10-09T05:49:37.063651+0000","last_fullsized":"2023-10-09T05:49:37.063651+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.1e","version":"319'6947","reported_seq":6999,"reported_epoch":319,"state":"active+recovery_wait","last_fresh":"2023-10-09T05:49:53.722410+0000","last_change":"2023-10-09T05:49:21.159312+0000","last_active":"2023-10-09T05:49:53.722410+0000","last_peered":"2023-10-09T05:49:53.722410+0000","last_clean":"2023-10-09T05:49:19.070876+0000","last_became_active":"2023-10-09T05:49:20.168491+0000","last_became_peered":"2023-10-09T05:49:20.168491+0000","last_unstale":"2023-10-09T05:49:53.722410+0000","last_undegraded":"2023-10-09T05:49:53.722410+0000","last_fullsized":"2023-10-09T05:49:53.722410+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6947,"log_dups_size":0,"ondisk_log_size":6947,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187629568,"num_objects":2863,"num_object_clones":0,"num_object_copies":8589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3600,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":2863}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":104,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064137+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:49:37.064137+0000","last_peered":"2023-10-09T05:49:37.064137+0000","last_clean":"2023-10-09T05:49:37.064137+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:49:37.064137+0000","last_undegraded":"2023-10-09T05:49:37.064137+0000","last_fullsized":"2023-10-09T05:49:37.064137+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.120160+0000","last_change":"2023-10-09T05:48:54.490345+0000","last_active":"2023-10-09T05:49:37.120160+0000","last_peered":"2023-10-09T05:49:37.120160+0000","last_clean":"2023-10-09T05:49:37.120160+0000","last_became_active":"2023-10-09T05:48:54.490000+0000","last_became_peered":"2023-10-09T05:48:54.490000+0000","last_unstale":"2023-10-09T05:49:37.120160+0000","last_undegraded":"2023-10-09T05:49:37.120160+0000","last_fullsized":"2023-10-09T05:49:37.120160+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:02.457608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1f","version":"319'6749","reported_seq":7185,"reported_epoch":319,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:49.850198+0000","last_change":"2023-10-09T05:49:23.718403+0000","last_active":"2023-10-09T05:49:49.850198+0000","last_peered":"2023-10-09T05:49:49.850198+0000","last_clean":"2023-10-09T05:48:53.467734+0000","last_became_active":"2023-10-09T05:49:23.645737+0000","last_became_peered":"2023-10-09T05:49:23.645737+0000","last_unstale":"2023-10-09T05:49:49.850198+0000","last_undegraded":"2023-10-09T05:49:21.161481+0000","last_fullsized":"2023-10-09T05:49:49.850198+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6749,"log_dups_size":0,"ondisk_log_size":6749,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187138048,"num_objects":2856,"num_object_clones":0,"num_object_copies":8568,"num_objects_missing_on_primary":2631,"num_objects_missing":2631,"num_objects_degraded":2631,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3466,"num_write_kb":202272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":184,"num_bytes_recovered":11927552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,4","objects":335},{"shards":"1,3,4,6","objects":6},{"shards":"3,4,6","objects":2515}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":345,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.065169+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:49:37.065169+0000","last_peered":"2023-10-09T05:49:37.065169+0000","last_clean":"2023-10-09T05:49:37.065169+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:49:37.065169+0000","last_undegraded":"2023-10-09T05:49:37.065169+0000","last_fullsized":"2023-10-09T05:49:37.065169+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":302,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.067624+0000","last_change":"2023-10-09T05:48:54.492082+0000","last_active":"2023-10-09T05:49:37.067624+0000","last_peered":"2023-10-09T05:49:37.067624+0000","last_clean":"2023-10-09T05:49:37.067624+0000","last_became_active":"2023-10-09T05:48:54.491753+0000","last_became_peered":"2023-10-09T05:48:54.491753+0000","last_unstale":"2023-10-09T05:49:37.067624+0000","last_undegraded":"2023-10-09T05:49:37.067624+0000","last_fullsized":"2023-10-09T05:49:37.067624+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"319'6191","reported_seq":6244,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:50.146427+0000","last_change":"2023-10-09T05:48:46.314063+0000","last_active":"2023-10-09T05:49:50.146427+0000","last_peered":"2023-10-09T05:49:50.146427+0000","last_clean":"2023-10-09T05:49:50.146427+0000","last_became_active":"2023-10-09T05:48:46.261937+0000","last_became_peered":"2023-10-09T05:48:46.261937+0000","last_unstale":"2023-10-09T05:49:50.146427+0000","last_undegraded":"2023-10-09T05:49:50.146427+0000","last_fullsized":"2023-10-09T05:49:50.146427+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6191,"log_dups_size":0,"ondisk_log_size":6191,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188284928,"num_objects":2873,"num_object_clones":0,"num_object_copies":8619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3509,"num_write_kb":204224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":104,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.063831+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:49:37.063831+0000","last_peered":"2023-10-09T05:49:37.063831+0000","last_clean":"2023-10-09T05:49:37.063831+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:49:37.063831+0000","last_undegraded":"2023-10-09T05:49:37.063831+0000","last_fullsized":"2023-10-09T05:49:37.063831+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.088673+0000","last_change":"2023-10-09T05:48:54.492572+0000","last_active":"2023-10-09T05:49:37.088673+0000","last_peered":"2023-10-09T05:49:37.088673+0000","last_clean":"2023-10-09T05:49:37.088673+0000","last_became_active":"2023-10-09T05:48:54.491827+0000","last_became_peered":"2023-10-09T05:48:54.491827+0000","last_unstale":"2023-10-09T05:49:37.088673+0000","last_undegraded":"2023-10-09T05:49:37.088673+0000","last_fullsized":"2023-10-09T05:49:37.088673+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:33.185508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.19","version":"319'6337","reported_seq":6390,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:53.650553+0000","last_change":"2023-10-09T05:48:46.264369+0000","last_active":"2023-10-09T05:49:53.650553+0000","last_peered":"2023-10-09T05:49:53.650553+0000","last_clean":"2023-10-09T05:49:53.650553+0000","last_became_active":"2023-10-09T05:48:46.259158+0000","last_became_peered":"2023-10-09T05:48:46.259158+0000","last_unstale":"2023-10-09T05:49:53.650553+0000","last_undegraded":"2023-10-09T05:49:53.650553+0000","last_fullsized":"2023-10-09T05:49:53.650553+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6337,"log_dups_size":0,"ondisk_log_size":6337,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191823872,"num_objects":2927,"num_object_clones":0,"num_object_copies":8781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3589,"num_write_kb":208512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":307,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.070061+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:49:37.070061+0000","last_peered":"2023-10-09T05:49:37.070061+0000","last_clean":"2023-10-09T05:49:37.070061+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:49:37.070061+0000","last_undegraded":"2023-10-09T05:49:37.070061+0000","last_fullsized":"2023-10-09T05:49:37.070061+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":277,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064022+0000","last_change":"2023-10-09T05:46:56.019914+0000","last_active":"2023-10-09T05:49:37.064022+0000","last_peered":"2023-10-09T05:49:37.064022+0000","last_clean":"2023-10-09T05:49:37.064022+0000","last_became_active":"2023-10-09T05:46:56.019487+0000","last_became_peered":"2023-10-09T05:46:56.019487+0000","last_unstale":"2023-10-09T05:49:37.064022+0000","last_undegraded":"2023-10-09T05:49:37.064022+0000","last_fullsized":"2023-10-09T05:49:37.064022+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.1a","version":"319'6828","reported_seq":7275,"reported_epoch":319,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:49.862678+0000","last_change":"2023-10-09T05:49:23.419200+0000","last_active":"2023-10-09T05:49:49.862678+0000","last_peered":"2023-10-09T05:49:49.862678+0000","last_clean":"2023-10-09T05:48:53.473854+0000","last_became_active":"2023-10-09T05:49:23.397583+0000","last_became_peered":"2023-10-09T05:49:23.397583+0000","last_unstale":"2023-10-09T05:49:49.862678+0000","last_undegraded":"2023-10-09T05:49:21.157095+0000","last_fullsized":"2023-10-09T05:49:49.862678+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6828,"log_dups_size":0,"ondisk_log_size":6828,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189530112,"num_objects":2892,"num_object_clones":0,"num_object_copies":8676,"num_objects_missing_on_primary":2641,"num_objects_missing":2641,"num_objects_degraded":2641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3511,"num_write_kb":204896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":197,"num_bytes_recovered":12779520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2,3","objects":363},{"shards":"2,3,4","objects":2529}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":98,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.063784+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:49:37.063784+0000","last_peered":"2023-10-09T05:49:37.063784+0000","last_clean":"2023-10-09T05:49:37.063784+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:49:37.063784+0000","last_undegraded":"2023-10-09T05:49:37.063784+0000","last_fullsized":"2023-10-09T05:49:37.063784+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":307,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.071826+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:49:37.071826+0000","last_peered":"2023-10-09T05:49:37.071826+0000","last_clean":"2023-10-09T05:49:37.071826+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:49:37.071826+0000","last_undegraded":"2023-10-09T05:49:37.071826+0000","last_fullsized":"2023-10-09T05:49:37.071826+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1b","version":"319'6984","reported_seq":7484,"reported_epoch":319,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:50.275384+0000","last_change":"2023-10-09T05:49:20.414092+0000","last_active":"2023-10-09T05:49:50.275384+0000","last_peered":"2023-10-09T05:49:50.275384+0000","last_clean":"2023-10-09T05:48:53.462109+0000","last_became_active":"2023-10-09T05:49:20.374568+0000","last_became_peered":"2023-10-09T05:49:20.374568+0000","last_unstale":"2023-10-09T05:49:50.275384+0000","last_undegraded":"2023-10-09T05:49:20.366455+0000","last_fullsized":"2023-10-09T05:49:50.275384+0000","mapping_epoch":311,"log_start":"289'200","ondisk_log_start":"289'200","created":311,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6784,"log_dups_size":0,"ondisk_log_size":6784,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191004672,"num_objects":2915,"num_object_clones":0,"num_object_copies":8745,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2713,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2915,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3600,"num_write_kb":208448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":218,"num_bytes_recovered":14155776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"1,3,4","objects":315},{"shards":"2,3,4","objects":2600}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":114,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.069823+0000","last_change":"2023-10-09T05:46:56.194588+0000","last_active":"2023-10-09T05:49:37.069823+0000","last_peered":"2023-10-09T05:49:37.069823+0000","last_clean":"2023-10-09T05:49:37.069823+0000","last_became_active":"2023-10-09T05:46:56.018480+0000","last_became_peered":"2023-10-09T05:46:56.018480+0000","last_unstale":"2023-10-09T05:49:37.069823+0000","last_undegraded":"2023-10-09T05:49:37.069823+0000","last_fullsized":"2023-10-09T05:49:37.069823+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:39.150417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":324,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064063+0000","last_change":"2023-10-09T05:48:54.492012+0000","last_active":"2023-10-09T05:49:37.064063+0000","last_peered":"2023-10-09T05:49:37.064063+0000","last_clean":"2023-10-09T05:49:37.064063+0000","last_became_active":"2023-10-09T05:48:54.491644+0000","last_became_peered":"2023-10-09T05:48:54.491644+0000","last_unstale":"2023-10-09T05:49:37.064063+0000","last_undegraded":"2023-10-09T05:49:37.064063+0000","last_fullsized":"2023-10-09T05:49:37.064063+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"319'6249","reported_seq":6299,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:50.187769+0000","last_change":"2023-10-09T05:48:46.259028+0000","last_active":"2023-10-09T05:49:50.187769+0000","last_peered":"2023-10-09T05:49:50.187769+0000","last_clean":"2023-10-09T05:49:50.187769+0000","last_became_active":"2023-10-09T05:48:46.256441+0000","last_became_peered":"2023-10-09T05:48:46.256441+0000","last_unstale":"2023-10-09T05:49:50.187769+0000","last_undegraded":"2023-10-09T05:49:50.187769+0000","last_fullsized":"2023-10-09T05:49:50.187769+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6249,"log_dups_size":0,"ondisk_log_size":6249,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192249856,"num_objects":2933,"num_object_clones":0,"num_object_copies":8799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3521,"num_write_kb":206560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":102,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064071+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:49:37.064071+0000","last_peered":"2023-10-09T05:49:37.064071+0000","last_clean":"2023-10-09T05:49:37.064071+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:49:37.064071+0000","last_undegraded":"2023-10-09T05:49:37.064071+0000","last_fullsized":"2023-10-09T05:49:37.064071+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":262,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.063804+0000","last_change":"2023-10-09T05:46:56.021669+0000","last_active":"2023-10-09T05:49:37.063804+0000","last_peered":"2023-10-09T05:49:37.063804+0000","last_clean":"2023-10-09T05:49:37.063804+0000","last_became_active":"2023-10-09T05:46:56.021306+0000","last_became_peered":"2023-10-09T05:46:56.021306+0000","last_unstale":"2023-10-09T05:49:37.063804+0000","last_undegraded":"2023-10-09T05:49:37.063804+0000","last_fullsized":"2023-10-09T05:49:37.063804+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"319'6231","reported_seq":6279,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:49.315687+0000","last_change":"2023-10-09T05:48:46.258066+0000","last_active":"2023-10-09T05:49:49.315687+0000","last_peered":"2023-10-09T05:49:49.315687+0000","last_clean":"2023-10-09T05:49:49.315687+0000","last_became_active":"2023-10-09T05:48:46.251759+0000","last_became_peered":"2023-10-09T05:48:46.251759+0000","last_unstale":"2023-10-09T05:49:49.315687+0000","last_undegraded":"2023-10-09T05:49:49.315687+0000","last_fullsized":"2023-10-09T05:49:49.315687+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6231,"log_dups_size":0,"ondisk_log_size":6231,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189890560,"num_objects":2897,"num_object_clones":0,"num_object_copies":8691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3541,"num_write_kb":206048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":128,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064222+0000","last_change":"2023-10-09T05:48:54.539425+0000","last_active":"2023-10-09T05:49:37.064222+0000","last_peered":"2023-10-09T05:49:37.064222+0000","last_clean":"2023-10-09T05:49:37.064222+0000","last_became_active":"2023-10-09T05:48:54.486401+0000","last_became_peered":"2023-10-09T05:48:54.486401+0000","last_unstale":"2023-10-09T05:49:37.064222+0000","last_undegraded":"2023-10-09T05:49:37.064222+0000","last_fullsized":"2023-10-09T05:49:37.064222+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":335,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064731+0000","last_change":"2023-10-09T05:48:54.492534+0000","last_active":"2023-10-09T05:49:37.064731+0000","last_peered":"2023-10-09T05:49:37.064731+0000","last_clean":"2023-10-09T05:49:37.064731+0000","last_became_active":"2023-10-09T05:48:54.492172+0000","last_became_peered":"2023-10-09T05:48:54.492172+0000","last_unstale":"2023-10-09T05:49:37.064731+0000","last_undegraded":"2023-10-09T05:49:37.064731+0000","last_fullsized":"2023-10-09T05:49:37.064731+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"319'6200","reported_seq":6254,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:53.719308+0000","last_change":"2023-10-09T05:48:46.220278+0000","last_active":"2023-10-09T05:49:53.719308+0000","last_peered":"2023-10-09T05:49:53.719308+0000","last_clean":"2023-10-09T05:49:53.719308+0000","last_became_active":"2023-10-09T05:48:46.215626+0000","last_became_peered":"2023-10-09T05:48:46.215626+0000","last_unstale":"2023-10-09T05:49:53.719308+0000","last_undegraded":"2023-10-09T05:49:53.719308+0000","last_fullsized":"2023-10-09T05:49:53.719308+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6200,"log_dups_size":0,"ondisk_log_size":6200,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185892864,"num_objects":2836,"num_object_clones":0,"num_object_copies":8508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3512,"num_write_kb":203168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":115,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064052+0000","last_change":"2023-10-09T05:46:56.190043+0000","last_active":"2023-10-09T05:49:37.064052+0000","last_peered":"2023-10-09T05:49:37.064052+0000","last_clean":"2023-10-09T05:49:37.064052+0000","last_became_active":"2023-10-09T05:46:56.019072+0000","last_became_peered":"2023-10-09T05:46:56.019072+0000","last_unstale":"2023-10-09T05:49:37.064052+0000","last_undegraded":"2023-10-09T05:49:37.064052+0000","last_fullsized":"2023-10-09T05:49:37.064052+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.132798+0000","last_change":"2023-10-09T05:48:54.489089+0000","last_active":"2023-10-09T05:49:37.132798+0000","last_peered":"2023-10-09T05:49:37.132798+0000","last_clean":"2023-10-09T05:49:37.132798+0000","last_became_active":"2023-10-09T05:48:54.488817+0000","last_became_peered":"2023-10-09T05:48:54.488817+0000","last_unstale":"2023-10-09T05:49:37.132798+0000","last_undegraded":"2023-10-09T05:49:37.132798+0000","last_fullsized":"2023-10-09T05:49:37.132798+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:40.555711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"319'6372","reported_seq":6436,"reported_epoch":319,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:52.141949+0000","last_change":"2023-10-09T05:48:56.022926+0000","last_active":"2023-10-09T05:49:52.141949+0000","last_peered":"2023-10-09T05:49:52.141949+0000","last_clean":"2023-10-09T05:48:53.410221+0000","last_became_active":"2023-10-09T05:48:56.013961+0000","last_became_peered":"2023-10-09T05:48:56.013961+0000","last_unstale":"2023-10-09T05:49:52.141949+0000","last_undegraded":"2023-10-09T05:48:55.541735+0000","last_fullsized":"2023-10-09T05:48:55.521053+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6372,"log_dups_size":0,"ondisk_log_size":6372,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190185472,"num_objects":2902,"num_object_clones":0,"num_object_copies":8706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2925,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3588,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2582},{"shards":"3,6","objects":-23}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":286,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.069089+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:49:37.069089+0000","last_peered":"2023-10-09T05:49:37.069089+0000","last_clean":"2023-10-09T05:49:37.069089+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:49:37.069089+0000","last_undegraded":"2023-10-09T05:49:37.069089+0000","last_fullsized":"2023-10-09T05:49:37.069089+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":292,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.070984+0000","last_change":"2023-10-09T05:48:54.495800+0000","last_active":"2023-10-09T05:49:37.070984+0000","last_peered":"2023-10-09T05:49:37.070984+0000","last_clean":"2023-10-09T05:49:37.070984+0000","last_became_active":"2023-10-09T05:48:54.495540+0000","last_became_peered":"2023-10-09T05:48:54.495540+0000","last_unstale":"2023-10-09T05:49:37.070984+0000","last_undegraded":"2023-10-09T05:49:37.070984+0000","last_fullsized":"2023-10-09T05:49:37.070984+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:26.903669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"319'6196","reported_seq":6439,"reported_epoch":319,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:49.677619+0000","last_change":"2023-10-09T05:49:21.912515+0000","last_active":"2023-10-09T05:49:49.677619+0000","last_peered":"2023-10-09T05:49:49.677619+0000","last_clean":"2023-10-09T05:48:53.429181+0000","last_became_active":"2023-10-09T05:49:21.896767+0000","last_became_peered":"2023-10-09T05:49:21.896767+0000","last_unstale":"2023-10-09T05:49:49.677619+0000","last_undegraded":"2023-10-09T05:49:21.127213+0000","last_fullsized":"2023-10-09T05:49:49.677619+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6196,"log_dups_size":0,"ondisk_log_size":6196,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97288192,"num_objects":1484,"num_object_clones":0,"num_object_copies":4452,"num_objects_missing_on_primary":1392,"num_objects_missing":1392,"num_objects_degraded":1392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1772,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":86,"num_bytes_recovered":5636096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,4","objects":154},{"shards":"3,4,6","objects":1330}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":55,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.071077+0000","last_change":"2023-10-09T05:48:25.067945+0000","last_active":"2023-10-09T05:49:37.071077+0000","last_peered":"2023-10-09T05:49:37.071077+0000","last_clean":"2023-10-09T05:49:37.071077+0000","last_became_active":"2023-10-09T05:48:25.067721+0000","last_became_peered":"2023-10-09T05:48:25.067721+0000","last_unstale":"2023-10-09T05:49:37.071077+0000","last_undegraded":"2023-10-09T05:49:37.071077+0000","last_fullsized":"2023-10-09T05:49:37.071077+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:58.655544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":284,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.065064+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:49:37.065064+0000","last_peered":"2023-10-09T05:49:37.065064+0000","last_clean":"2023-10-09T05:49:37.065064+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:49:37.065064+0000","last_undegraded":"2023-10-09T05:49:37.065064+0000","last_fullsized":"2023-10-09T05:49:37.065064+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"319'6127","reported_seq":6374,"reported_epoch":319,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:49.299529+0000","last_change":"2023-10-09T05:49:22.000684+0000","last_active":"2023-10-09T05:49:49.299529+0000","last_peered":"2023-10-09T05:49:49.299529+0000","last_clean":"2023-10-09T05:48:53.421353+0000","last_became_active":"2023-10-09T05:49:21.977937+0000","last_became_peered":"2023-10-09T05:49:21.977937+0000","last_unstale":"2023-10-09T05:49:49.299529+0000","last_undegraded":"2023-10-09T05:49:21.129646+0000","last_fullsized":"2023-10-09T05:49:49.299529+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6127,"log_dups_size":0,"ondisk_log_size":6127,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94568448,"num_objects":1443,"num_object_clones":0,"num_object_copies":4329,"num_objects_missing_on_primary":1352,"num_objects_missing":1352,"num_objects_degraded":1352,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1760,"num_write_kb":102496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":6029312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,1,2","objects":152},{"shards":"0,2,5","objects":1291}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":153,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.065126+0000","last_change":"2023-10-09T05:48:54.494715+0000","last_active":"2023-10-09T05:49:37.065126+0000","last_peered":"2023-10-09T05:49:37.065126+0000","last_clean":"2023-10-09T05:49:37.065126+0000","last_became_active":"2023-10-09T05:48:54.493988+0000","last_became_peered":"2023-10-09T05:48:54.493988+0000","last_unstale":"2023-10-09T05:49:37.065126+0000","last_undegraded":"2023-10-09T05:49:37.065126+0000","last_fullsized":"2023-10-09T05:49:37.065126+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":317,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064913+0000","last_change":"2023-10-09T05:48:54.493193+0000","last_active":"2023-10-09T05:49:37.064913+0000","last_peered":"2023-10-09T05:49:37.064913+0000","last_clean":"2023-10-09T05:49:37.064913+0000","last_became_active":"2023-10-09T05:48:54.492886+0000","last_became_peered":"2023-10-09T05:48:54.492886+0000","last_unstale":"2023-10-09T05:49:37.064913+0000","last_undegraded":"2023-10-09T05:49:37.064913+0000","last_fullsized":"2023-10-09T05:49:37.064913+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"319'6079","reported_seq":6141,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:49.797749+0000","last_change":"2023-10-09T05:49:20.137476+0000","last_active":"2023-10-09T05:49:49.797749+0000","last_peered":"2023-10-09T05:49:49.797749+0000","last_clean":"2023-10-09T05:49:49.797749+0000","last_became_active":"2023-10-09T05:49:20.132759+0000","last_became_peered":"2023-10-09T05:49:20.132759+0000","last_unstale":"2023-10-09T05:49:49.797749+0000","last_undegraded":"2023-10-09T05:49:49.797749+0000","last_fullsized":"2023-10-09T05:49:49.797749+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6079,"log_dups_size":0,"ondisk_log_size":6079,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92815360,"num_objects":1416,"num_object_clones":0,"num_object_copies":4248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1757,"num_write_kb":101552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":318,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064601+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:49:37.064601+0000","last_peered":"2023-10-09T05:49:37.064601+0000","last_clean":"2023-10-09T05:49:37.064601+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:49:37.064601+0000","last_undegraded":"2023-10-09T05:49:37.064601+0000","last_fullsized":"2023-10-09T05:49:37.064601+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":311,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.065227+0000","last_change":"2023-10-09T05:48:54.492708+0000","last_active":"2023-10-09T05:49:37.065227+0000","last_peered":"2023-10-09T05:49:37.065227+0000","last_clean":"2023-10-09T05:49:37.065227+0000","last_became_active":"2023-10-09T05:48:54.492382+0000","last_became_peered":"2023-10-09T05:48:54.492382+0000","last_unstale":"2023-10-09T05:49:37.065227+0000","last_undegraded":"2023-10-09T05:49:37.065227+0000","last_fullsized":"2023-10-09T05:49:37.065227+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"319'6113","reported_seq":6360,"reported_epoch":319,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:49.892615+0000","last_change":"2023-10-09T05:49:21.611985+0000","last_active":"2023-10-09T05:49:49.892615+0000","last_peered":"2023-10-09T05:49:49.892615+0000","last_clean":"2023-10-09T05:48:53.427903+0000","last_became_active":"2023-10-09T05:49:21.584088+0000","last_became_peered":"2023-10-09T05:49:21.584088+0000","last_unstale":"2023-10-09T05:49:49.892615+0000","last_undegraded":"2023-10-09T05:49:21.527023+0000","last_fullsized":"2023-10-09T05:49:49.892615+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6113,"log_dups_size":0,"ondisk_log_size":6113,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95059968,"num_objects":1450,"num_object_clones":0,"num_object_copies":4350,"num_objects_missing_on_primary":1356,"num_objects_missing":1356,"num_objects_degraded":1356,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1756,"num_write_kb":102624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":6029312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":155},{"shards":"2,5,6","objects":1295}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.095369+0000","last_change":"2023-10-09T05:48:54.489891+0000","last_active":"2023-10-09T05:49:37.095369+0000","last_peered":"2023-10-09T05:49:37.095369+0000","last_clean":"2023-10-09T05:49:37.095369+0000","last_became_active":"2023-10-09T05:48:54.489570+0000","last_became_peered":"2023-10-09T05:48:54.489570+0000","last_unstale":"2023-10-09T05:49:37.095369+0000","last_undegraded":"2023-10-09T05:49:37.095369+0000","last_fullsized":"2023-10-09T05:49:37.095369+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:41.194371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.148584+0000","last_change":"2023-10-09T05:48:54.488131+0000","last_active":"2023-10-09T05:49:37.148584+0000","last_peered":"2023-10-09T05:49:37.148584+0000","last_clean":"2023-10-09T05:49:37.148584+0000","last_became_active":"2023-10-09T05:48:54.487748+0000","last_became_peered":"2023-10-09T05:48:54.487748+0000","last_unstale":"2023-10-09T05:49:37.148584+0000","last_undegraded":"2023-10-09T05:49:37.148584+0000","last_fullsized":"2023-10-09T05:49:37.148584+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:05.047252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"319'6903","reported_seq":6956,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:53.651746+0000","last_change":"2023-10-09T05:49:20.163537+0000","last_active":"2023-10-09T05:49:53.651746+0000","last_peered":"2023-10-09T05:49:53.651746+0000","last_clean":"2023-10-09T05:49:53.651746+0000","last_became_active":"2023-10-09T05:49:20.159521+0000","last_became_peered":"2023-10-09T05:49:20.159521+0000","last_unstale":"2023-10-09T05:49:53.651746+0000","last_undegraded":"2023-10-09T05:49:53.651746+0000","last_fullsized":"2023-10-09T05:49:53.651746+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6903,"log_dups_size":0,"ondisk_log_size":6903,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190513152,"num_objects":2907,"num_object_clones":0,"num_object_copies":8721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3556,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":681,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.088742+0000","last_change":"2023-10-09T05:49:35.055130+0000","last_active":"2023-10-09T05:49:37.088742+0000","last_peered":"2023-10-09T05:49:37.088742+0000","last_clean":"2023-10-09T05:49:37.088742+0000","last_became_active":"2023-10-09T05:49:35.054450+0000","last_became_peered":"2023-10-09T05:49:35.054450+0000","last_unstale":"2023-10-09T05:49:37.088742+0000","last_undegraded":"2023-10-09T05:49:37.088742+0000","last_fullsized":"2023-10-09T05:49:37.088742+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:29.769983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":301,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.069122+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:49:37.069122+0000","last_peered":"2023-10-09T05:49:37.069122+0000","last_clean":"2023-10-09T05:49:37.069122+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:49:37.069122+0000","last_undegraded":"2023-10-09T05:49:37.069122+0000","last_fullsized":"2023-10-09T05:49:37.069122+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.132709+0000","last_change":"2023-10-09T05:48:54.489994+0000","last_active":"2023-10-09T05:49:37.132709+0000","last_peered":"2023-10-09T05:49:37.132709+0000","last_clean":"2023-10-09T05:49:37.132709+0000","last_became_active":"2023-10-09T05:48:54.489701+0000","last_became_peered":"2023-10-09T05:48:54.489701+0000","last_unstale":"2023-10-09T05:49:37.132709+0000","last_undegraded":"2023-10-09T05:49:37.132709+0000","last_fullsized":"2023-10-09T05:49:37.132709+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:10.442883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"319'6135","reported_seq":6185,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:49.631520+0000","last_change":"2023-10-09T05:48:46.259058+0000","last_active":"2023-10-09T05:49:49.631520+0000","last_peered":"2023-10-09T05:49:49.631520+0000","last_clean":"2023-10-09T05:49:49.631520+0000","last_became_active":"2023-10-09T05:48:46.247029+0000","last_became_peered":"2023-10-09T05:48:46.247029+0000","last_unstale":"2023-10-09T05:49:49.631520+0000","last_undegraded":"2023-10-09T05:49:49.631520+0000","last_fullsized":"2023-10-09T05:49:49.631520+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6135,"log_dups_size":0,"ondisk_log_size":6135,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187498496,"num_objects":2861,"num_object_clones":0,"num_object_copies":8583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3453,"num_write_kb":202048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":809,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.069205+0000","last_change":"2023-10-09T05:48:54.592642+0000","last_active":"2023-10-09T05:49:37.069205+0000","last_peered":"2023-10-09T05:49:37.069205+0000","last_clean":"2023-10-09T05:49:37.069205+0000","last_became_active":"2023-10-09T05:48:54.491917+0000","last_became_peered":"2023-10-09T05:48:54.491917+0000","last_unstale":"2023-10-09T05:49:37.069205+0000","last_undegraded":"2023-10-09T05:49:37.069205+0000","last_fullsized":"2023-10-09T05:49:37.069205+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":285,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.067788+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:49:37.067788+0000","last_peered":"2023-10-09T05:49:37.067788+0000","last_clean":"2023-10-09T05:49:37.067788+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:49:37.067788+0000","last_undegraded":"2023-10-09T05:49:37.067788+0000","last_fullsized":"2023-10-09T05:49:37.067788+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.088471+0000","last_change":"2023-10-09T05:48:54.492183+0000","last_active":"2023-10-09T05:49:37.088471+0000","last_peered":"2023-10-09T05:49:37.088471+0000","last_clean":"2023-10-09T05:49:37.088471+0000","last_became_active":"2023-10-09T05:48:54.491382+0000","last_became_peered":"2023-10-09T05:48:54.491382+0000","last_unstale":"2023-10-09T05:49:37.088471+0000","last_undegraded":"2023-10-09T05:49:37.088471+0000","last_fullsized":"2023-10-09T05:49:37.088471+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:06.256308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"319'6976","reported_seq":7456,"reported_epoch":319,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:50.142227+0000","last_change":"2023-10-09T05:49:21.590522+0000","last_active":"2023-10-09T05:49:50.142227+0000","last_peered":"2023-10-09T05:49:50.142227+0000","last_clean":"2023-10-09T05:48:53.462109+0000","last_became_active":"2023-10-09T05:49:21.586314+0000","last_became_peered":"2023-10-09T05:49:21.586314+0000","last_unstale":"2023-10-09T05:49:50.142227+0000","last_undegraded":"2023-10-09T05:49:21.140419+0000","last_fullsized":"2023-10-09T05:49:50.142227+0000","mapping_epoch":312,"log_start":"289'200","ondisk_log_start":"289'200","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6776,"log_dups_size":0,"ondisk_log_size":6776,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191528960,"num_objects":2922,"num_object_clones":0,"num_object_copies":8766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2697,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3591,"num_write_kb":208448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"1,3,4","objects":365},{"shards":"2,3,4","objects":2557}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":638,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:46.297698+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:49:46.297698+0000","last_peered":"2023-10-09T05:49:46.297698+0000","last_clean":"2023-10-09T05:49:46.297698+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:49:46.297698+0000","last_undegraded":"2023-10-09T05:49:46.297698+0000","last_fullsized":"2023-10-09T05:49:46.297698+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.095223+0000","last_change":"2023-10-09T05:48:54.490716+0000","last_active":"2023-10-09T05:49:37.095223+0000","last_peered":"2023-10-09T05:49:37.095223+0000","last_clean":"2023-10-09T05:49:37.095223+0000","last_became_active":"2023-10-09T05:48:54.490350+0000","last_became_peered":"2023-10-09T05:48:54.490350+0000","last_unstale":"2023-10-09T05:49:37.095223+0000","last_undegraded":"2023-10-09T05:49:37.095223+0000","last_fullsized":"2023-10-09T05:49:37.095223+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:20.190860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.148535+0000","last_change":"2023-10-09T05:48:54.492185+0000","last_active":"2023-10-09T05:49:37.148535+0000","last_peered":"2023-10-09T05:49:37.148535+0000","last_clean":"2023-10-09T05:49:37.148535+0000","last_became_active":"2023-10-09T05:48:54.491885+0000","last_became_peered":"2023-10-09T05:48:54.491885+0000","last_unstale":"2023-10-09T05:49:37.148535+0000","last_undegraded":"2023-10-09T05:49:37.148535+0000","last_fullsized":"2023-10-09T05:49:37.148535+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:26.045180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"319'6356","reported_seq":6410,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:53.656019+0000","last_change":"2023-10-09T05:48:46.220263+0000","last_active":"2023-10-09T05:49:53.656019+0000","last_peered":"2023-10-09T05:49:53.656019+0000","last_clean":"2023-10-09T05:49:53.656019+0000","last_became_active":"2023-10-09T05:48:46.215666+0000","last_became_peered":"2023-10-09T05:48:46.215666+0000","last_unstale":"2023-10-09T05:49:53.656019+0000","last_undegraded":"2023-10-09T05:49:53.656019+0000","last_fullsized":"2023-10-09T05:49:53.656019+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6356,"log_dups_size":0,"ondisk_log_size":6356,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191627264,"num_objects":2924,"num_object_clones":0,"num_object_copies":8772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3608,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":965,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.069633+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:49:37.069633+0000","last_peered":"2023-10-09T05:49:37.069633+0000","last_clean":"2023-10-09T05:49:37.069633+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:49:37.069633+0000","last_undegraded":"2023-10-09T05:49:37.069633+0000","last_fullsized":"2023-10-09T05:49:37.069633+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":469,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.063631+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:49:37.063631+0000","last_peered":"2023-10-09T05:49:37.063631+0000","last_clean":"2023-10-09T05:49:37.063631+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:49:37.063631+0000","last_undegraded":"2023-10-09T05:49:37.063631+0000","last_fullsized":"2023-10-09T05:49:37.063631+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.069065+0000","last_change":"2023-10-09T05:48:25.068759+0000","last_active":"2023-10-09T05:49:37.069065+0000","last_peered":"2023-10-09T05:49:37.069065+0000","last_clean":"2023-10-09T05:49:37.069065+0000","last_became_active":"2023-10-09T05:48:25.068358+0000","last_became_peered":"2023-10-09T05:48:25.068358+0000","last_unstale":"2023-10-09T05:49:37.069065+0000","last_undegraded":"2023-10-09T05:49:37.069065+0000","last_fullsized":"2023-10-09T05:49:37.069065+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":286,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.065087+0000","last_change":"2023-10-09T05:44:50.495966+0000","last_active":"2023-10-09T05:49:37.065087+0000","last_peered":"2023-10-09T05:49:37.065087+0000","last_clean":"2023-10-09T05:49:37.065087+0000","last_became_active":"2023-10-09T05:44:50.495628+0000","last_became_peered":"2023-10-09T05:44:50.495628+0000","last_unstale":"2023-10-09T05:49:37.065087+0000","last_undegraded":"2023-10-09T05:49:37.065087+0000","last_fullsized":"2023-10-09T05:49:37.065087+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"319'6825","reported_seq":7265,"reported_epoch":319,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:49.797183+0000","last_change":"2023-10-09T05:49:23.419408+0000","last_active":"2023-10-09T05:49:49.797183+0000","last_peered":"2023-10-09T05:49:49.797183+0000","last_clean":"2023-10-09T05:48:53.473854+0000","last_became_active":"2023-10-09T05:49:23.402116+0000","last_became_peered":"2023-10-09T05:49:23.402116+0000","last_unstale":"2023-10-09T05:49:49.797183+0000","last_undegraded":"2023-10-09T05:49:21.176409+0000","last_fullsized":"2023-10-09T05:49:49.797183+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6825,"log_dups_size":0,"ondisk_log_size":6825,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189726720,"num_objects":2895,"num_object_clones":0,"num_object_copies":8685,"num_objects_missing_on_primary":2691,"num_objects_missing":2691,"num_objects_degraded":2691,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3508,"num_write_kb":204896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":193,"num_bytes_recovered":12517376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2,3","objects":316},{"shards":"1,2,3,4","objects":3},{"shards":"2,3,4","objects":2576}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":841,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.063690+0000","last_change":"2023-10-09T05:49:36.046279+0000","last_active":"2023-10-09T05:49:37.063690+0000","last_peered":"2023-10-09T05:49:37.063690+0000","last_clean":"2023-10-09T05:49:37.063690+0000","last_became_active":"2023-10-09T05:49:36.045828+0000","last_became_peered":"2023-10-09T05:49:36.045828+0000","last_unstale":"2023-10-09T05:49:37.063690+0000","last_undegraded":"2023-10-09T05:49:37.063690+0000","last_fullsized":"2023-10-09T05:49:37.063690+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":293,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.070099+0000","last_change":"2023-10-09T05:47:33.926192+0000","last_active":"2023-10-09T05:49:37.070099+0000","last_peered":"2023-10-09T05:49:37.070099+0000","last_clean":"2023-10-09T05:49:37.070099+0000","last_became_active":"2023-10-09T05:47:33.925588+0000","last_became_peered":"2023-10-09T05:47:33.925588+0000","last_unstale":"2023-10-09T05:49:37.070099+0000","last_undegraded":"2023-10-09T05:49:37.070099+0000","last_fullsized":"2023-10-09T05:49:37.070099+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:29.495701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":308,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.071479+0000","last_change":"2023-10-09T05:48:54.489115+0000","last_active":"2023-10-09T05:49:37.071479+0000","last_peered":"2023-10-09T05:49:37.071479+0000","last_clean":"2023-10-09T05:49:37.071479+0000","last_became_active":"2023-10-09T05:48:54.488783+0000","last_became_peered":"2023-10-09T05:48:54.488783+0000","last_unstale":"2023-10-09T05:49:37.071479+0000","last_undegraded":"2023-10-09T05:49:37.071479+0000","last_fullsized":"2023-10-09T05:49:37.071479+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:13.193778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"319'6747","reported_seq":7168,"reported_epoch":319,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:49:49.919975+0000","last_change":"2023-10-09T05:49:23.636172+0000","last_active":"2023-10-09T05:49:49.919975+0000","last_peered":"2023-10-09T05:49:49.919975+0000","last_clean":"2023-10-09T05:48:53.467734+0000","last_became_active":"2023-10-09T05:49:22.482315+0000","last_became_peered":"2023-10-09T05:49:22.482315+0000","last_unstale":"2023-10-09T05:49:49.919975+0000","last_undegraded":"2023-10-09T05:49:21.143637+0000","last_fullsized":"2023-10-09T05:49:49.919975+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6747,"log_dups_size":0,"ondisk_log_size":6747,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187269120,"num_objects":2857,"num_object_clones":0,"num_object_copies":8571,"num_objects_missing_on_primary":2635,"num_objects_missing":2635,"num_objects_degraded":2635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3463,"num_write_kb":202272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":180,"num_bytes_recovered":11796480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,4","objects":349},{"shards":"1,3,4,6","objects":1},{"shards":"3,4,6","objects":2507}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":502,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064172+0000","last_change":"2023-10-09T05:44:51.489799+0000","last_active":"2023-10-09T05:49:37.064172+0000","last_peered":"2023-10-09T05:49:37.064172+0000","last_clean":"2023-10-09T05:49:37.064172+0000","last_became_active":"2023-10-09T05:44:51.489301+0000","last_became_peered":"2023-10-09T05:44:51.489301+0000","last_unstale":"2023-10-09T05:49:37.064172+0000","last_undegraded":"2023-10-09T05:49:37.064172+0000","last_fullsized":"2023-10-09T05:49:37.064172+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.088544+0000","last_change":"2023-10-09T05:48:54.491374+0000","last_active":"2023-10-09T05:49:37.088544+0000","last_peered":"2023-10-09T05:49:37.088544+0000","last_clean":"2023-10-09T05:49:37.088544+0000","last_became_active":"2023-10-09T05:48:54.490889+0000","last_became_peered":"2023-10-09T05:48:54.490889+0000","last_unstale":"2023-10-09T05:49:37.088544+0000","last_undegraded":"2023-10-09T05:49:37.088544+0000","last_fullsized":"2023-10-09T05:49:37.088544+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:34.755791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":293,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.071260+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:49:37.071260+0000","last_peered":"2023-10-09T05:49:37.071260+0000","last_clean":"2023-10-09T05:49:37.071260+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:49:37.071260+0000","last_undegraded":"2023-10-09T05:49:37.071260+0000","last_fullsized":"2023-10-09T05:49:37.071260+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"319'7075","reported_seq":7129,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:53.227321+0000","last_change":"2023-10-09T05:49:20.373182+0000","last_active":"2023-10-09T05:49:53.227321+0000","last_peered":"2023-10-09T05:49:53.227321+0000","last_clean":"2023-10-09T05:49:53.227321+0000","last_became_active":"2023-10-09T05:49:20.351645+0000","last_became_peered":"2023-10-09T05:49:20.351645+0000","last_unstale":"2023-10-09T05:49:53.227321+0000","last_undegraded":"2023-10-09T05:49:53.227321+0000","last_fullsized":"2023-10-09T05:49:53.227321+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7075,"log_dups_size":0,"ondisk_log_size":7075,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192610304,"num_objects":2939,"num_object_clones":0,"num_object_copies":8817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3643,"num_write_kb":210624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":266,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064171+0000","last_change":"2023-10-09T05:48:25.066163+0000","last_active":"2023-10-09T05:49:37.064171+0000","last_peered":"2023-10-09T05:49:37.064171+0000","last_clean":"2023-10-09T05:49:37.064171+0000","last_became_active":"2023-10-09T05:48:25.065145+0000","last_became_peered":"2023-10-09T05:48:25.065145+0000","last_unstale":"2023-10-09T05:49:37.064171+0000","last_undegraded":"2023-10-09T05:49:37.064171+0000","last_fullsized":"2023-10-09T05:49:37.064171+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":484,"reported_epoch":318,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T05:49:37.079494+0000","last_change":"2023-10-09T05:48:55.619562+0000","last_active":"2023-10-09T05:49:37.079494+0000","last_peered":"2023-10-09T05:49:37.079494+0000","last_clean":"2023-10-09T05:22:39.598843+0000","last_became_active":"2023-10-09T05:48:55.618332+0000","last_became_peered":"2023-10-09T05:48:55.618332+0000","last_unstale":"2023-10-09T05:49:37.079494+0000","last_undegraded":"2023-10-09T05:49:37.079494+0000","last_fullsized":"2023-10-09T05:49:37.079494+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:01.870836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[3,6,4],"avail_no_missing":["3","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":259,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.063782+0000","last_change":"2023-10-09T05:48:25.067326+0000","last_active":"2023-10-09T05:49:37.063782+0000","last_peered":"2023-10-09T05:49:37.063782+0000","last_clean":"2023-10-09T05:49:37.063782+0000","last_became_active":"2023-10-09T05:48:25.066977+0000","last_became_peered":"2023-10-09T05:48:25.066977+0000","last_unstale":"2023-10-09T05:49:37.063782+0000","last_undegraded":"2023-10-09T05:49:37.063782+0000","last_fullsized":"2023-10-09T05:49:37.063782+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"319'6816","reported_seq":6869,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:50.235520+0000","last_change":"2023-10-09T05:49:20.153015+0000","last_active":"2023-10-09T05:49:50.235520+0000","last_peered":"2023-10-09T05:49:50.235520+0000","last_clean":"2023-10-09T05:49:50.235520+0000","last_became_active":"2023-10-09T05:49:20.148237+0000","last_became_peered":"2023-10-09T05:49:20.148237+0000","last_unstale":"2023-10-09T05:49:50.235520+0000","last_undegraded":"2023-10-09T05:49:50.235520+0000","last_fullsized":"2023-10-09T05:49:50.235520+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6816,"log_dups_size":0,"ondisk_log_size":6816,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186875916,"num_objects":2852,"num_object_clones":0,"num_object_copies":8556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3510,"num_write_kb":203552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":816,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064160+0000","last_change":"2023-10-09T05:44:50.482211+0000","last_active":"2023-10-09T05:49:37.064160+0000","last_peered":"2023-10-09T05:49:37.064160+0000","last_clean":"2023-10-09T05:49:37.064160+0000","last_became_active":"2023-10-09T05:44:50.481749+0000","last_became_peered":"2023-10-09T05:44:50.481749+0000","last_unstale":"2023-10-09T05:49:37.064160+0000","last_undegraded":"2023-10-09T05:49:37.064160+0000","last_fullsized":"2023-10-09T05:49:37.064160+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":304,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.071738+0000","last_change":"2023-10-09T05:48:54.585691+0000","last_active":"2023-10-09T05:49:37.071738+0000","last_peered":"2023-10-09T05:49:37.071738+0000","last_clean":"2023-10-09T05:49:37.071738+0000","last_became_active":"2023-10-09T05:48:54.498259+0000","last_became_peered":"2023-10-09T05:48:54.498259+0000","last_unstale":"2023-10-09T05:49:37.071738+0000","last_undegraded":"2023-10-09T05:49:37.071738+0000","last_fullsized":"2023-10-09T05:49:37.071738+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.684612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":269,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064626+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:49:37.064626+0000","last_peered":"2023-10-09T05:49:37.064626+0000","last_clean":"2023-10-09T05:49:37.064626+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:49:37.064626+0000","last_undegraded":"2023-10-09T05:49:37.064626+0000","last_fullsized":"2023-10-09T05:49:37.064626+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"319'6303","reported_seq":15024,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:49.734165+0000","last_change":"2023-10-09T05:49:49.613404+0000","last_active":"2023-10-09T05:49:49.734165+0000","last_peered":"2023-10-09T05:49:49.734165+0000","last_clean":"2023-10-09T05:49:49.734165+0000","last_became_active":"2023-10-09T05:49:35.024339+0000","last_became_peered":"2023-10-09T05:49:35.024339+0000","last_unstale":"2023-10-09T05:49:49.734165+0000","last_undegraded":"2023-10-09T05:49:49.734165+0000","last_fullsized":"2023-10-09T05:49:49.734165+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6303,"log_dups_size":0,"ondisk_log_size":6303,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192675840,"num_objects":2940,"num_object_clones":0,"num_object_copies":8820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3566,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2892,"num_bytes_recovered":189530112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":294,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.071181+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:49:37.071181+0000","last_peered":"2023-10-09T05:49:37.071181+0000","last_clean":"2023-10-09T05:49:37.071181+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:49:37.071181+0000","last_undegraded":"2023-10-09T05:49:37.071181+0000","last_fullsized":"2023-10-09T05:49:37.071181+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":24,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.088611+0000","last_change":"2023-10-09T05:48:54.492394+0000","last_active":"2023-10-09T05:49:37.088611+0000","last_peered":"2023-10-09T05:49:37.088611+0000","last_clean":"2023-10-09T05:49:37.088611+0000","last_became_active":"2023-10-09T05:48:54.491682+0000","last_became_peered":"2023-10-09T05:48:54.491682+0000","last_unstale":"2023-10-09T05:49:37.088611+0000","last_undegraded":"2023-10-09T05:49:37.088611+0000","last_fullsized":"2023-10-09T05:49:37.088611+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:28.838164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.17","version":"319'6419","reported_seq":6482,"reported_epoch":319,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:49:51.873859+0000","last_change":"2023-10-09T05:48:56.019001+0000","last_active":"2023-10-09T05:49:51.873859+0000","last_peered":"2023-10-09T05:49:51.873859+0000","last_clean":"2023-10-09T05:48:53.462142+0000","last_became_active":"2023-10-09T05:48:56.004223+0000","last_became_peered":"2023-10-09T05:48:56.004223+0000","last_unstale":"2023-10-09T05:49:51.873859+0000","last_undegraded":"2023-10-09T05:48:55.514193+0000","last_fullsized":"2023-10-09T05:48:55.479940+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6419,"log_dups_size":0,"ondisk_log_size":6419,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192479232,"num_objects":2937,"num_object_clones":0,"num_object_copies":8811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2986,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3635,"num_write_kb":210304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2637},{"shards":"3,6","objects":-49}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":314,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.069044+0000","last_change":"2023-10-09T05:48:54.583404+0000","last_active":"2023-10-09T05:49:37.069044+0000","last_peered":"2023-10-09T05:49:37.069044+0000","last_clean":"2023-10-09T05:49:37.069044+0000","last_became_active":"2023-10-09T05:48:54.492565+0000","last_became_peered":"2023-10-09T05:48:54.492565+0000","last_unstale":"2023-10-09T05:49:37.069044+0000","last_undegraded":"2023-10-09T05:49:37.069044+0000","last_fullsized":"2023-10-09T05:49:37.069044+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":317,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.067862+0000","last_change":"2023-10-09T05:48:54.493079+0000","last_active":"2023-10-09T05:49:37.067862+0000","last_peered":"2023-10-09T05:49:37.067862+0000","last_clean":"2023-10-09T05:49:37.067862+0000","last_became_active":"2023-10-09T05:48:54.492013+0000","last_became_peered":"2023-10-09T05:48:54.492013+0000","last_unstale":"2023-10-09T05:49:37.067862+0000","last_undegraded":"2023-10-09T05:49:37.067862+0000","last_fullsized":"2023-10-09T05:49:37.067862+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"319'6234","reported_seq":6287,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:54.018305+0000","last_change":"2023-10-09T05:48:46.260747+0000","last_active":"2023-10-09T05:49:54.018305+0000","last_peered":"2023-10-09T05:49:54.018305+0000","last_clean":"2023-10-09T05:49:54.018305+0000","last_became_active":"2023-10-09T05:48:46.258900+0000","last_became_peered":"2023-10-09T05:48:46.258900+0000","last_unstale":"2023-10-09T05:49:54.018305+0000","last_undegraded":"2023-10-09T05:49:54.018305+0000","last_fullsized":"2023-10-09T05:49:54.018305+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6234,"log_dups_size":0,"ondisk_log_size":6234,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187727872,"num_objects":2865,"num_object_clones":0,"num_object_copies":8595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3547,"num_write_kb":205152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":273,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.069110+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:49:37.069110+0000","last_peered":"2023-10-09T05:49:37.069110+0000","last_clean":"2023-10-09T05:49:37.069110+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:49:37.069110+0000","last_undegraded":"2023-10-09T05:49:37.069110+0000","last_fullsized":"2023-10-09T05:49:37.069110+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":114,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.070050+0000","last_change":"2023-10-09T05:48:25.062693+0000","last_active":"2023-10-09T05:49:37.070050+0000","last_peered":"2023-10-09T05:49:37.070050+0000","last_clean":"2023-10-09T05:49:37.070050+0000","last_became_active":"2023-10-09T05:48:25.062334+0000","last_became_peered":"2023-10-09T05:48:25.062334+0000","last_unstale":"2023-10-09T05:49:37.070050+0000","last_undegraded":"2023-10-09T05:49:37.070050+0000","last_fullsized":"2023-10-09T05:49:37.070050+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:33:15.975841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.15","version":"319'6173","reported_seq":6226,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:50.199943+0000","last_change":"2023-10-09T05:48:46.308705+0000","last_active":"2023-10-09T05:49:50.199943+0000","last_peered":"2023-10-09T05:49:50.199943+0000","last_clean":"2023-10-09T05:49:50.199943+0000","last_became_active":"2023-10-09T05:48:46.258676+0000","last_became_peered":"2023-10-09T05:48:46.258676+0000","last_unstale":"2023-10-09T05:49:50.199943+0000","last_undegraded":"2023-10-09T05:49:50.199943+0000","last_fullsized":"2023-10-09T05:49:50.199943+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6173,"log_dups_size":0,"ondisk_log_size":6173,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186482688,"num_objects":2846,"num_object_clones":0,"num_object_copies":8538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3484,"num_write_kb":202528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":36,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.074954+0000","last_change":"2023-10-09T05:48:25.069406+0000","last_active":"2023-10-09T05:49:37.074954+0000","last_peered":"2023-10-09T05:49:37.074954+0000","last_clean":"2023-10-09T05:49:37.074954+0000","last_became_active":"2023-10-09T05:48:25.069064+0000","last_became_peered":"2023-10-09T05:48:25.069064+0000","last_unstale":"2023-10-09T05:49:37.074954+0000","last_undegraded":"2023-10-09T05:49:37.074954+0000","last_fullsized":"2023-10-09T05:49:37.074954+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.567858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":119,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.075212+0000","last_change":"2023-10-09T05:48:25.063854+0000","last_active":"2023-10-09T05:49:37.075212+0000","last_peered":"2023-10-09T05:49:37.075212+0000","last_clean":"2023-10-09T05:49:37.075212+0000","last_became_active":"2023-10-09T05:48:25.063579+0000","last_became_peered":"2023-10-09T05:48:25.063579+0000","last_unstale":"2023-10-09T05:49:37.075212+0000","last_undegraded":"2023-10-09T05:49:37.075212+0000","last_fullsized":"2023-10-09T05:49:37.075212+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:02.389789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.14","version":"319'6275","reported_seq":6328,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T05:49:49.731343+0000","last_change":"2023-10-09T05:48:46.325697+0000","last_active":"2023-10-09T05:49:49.731343+0000","last_peered":"2023-10-09T05:49:49.731343+0000","last_clean":"2023-10-09T05:49:49.731343+0000","last_became_active":"2023-10-09T05:48:46.268207+0000","last_became_peered":"2023-10-09T05:48:46.268207+0000","last_unstale":"2023-10-09T05:49:49.731343+0000","last_undegraded":"2023-10-09T05:49:49.731343+0000","last_fullsized":"2023-10-09T05:49:49.731343+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6275,"log_dups_size":0,"ondisk_log_size":6275,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192249856,"num_objects":2934,"num_object_clones":0,"num_object_copies":8802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3548,"num_write_kb":207392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":293,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.071754+0000","last_change":"2023-10-09T05:46:56.197243+0000","last_active":"2023-10-09T05:49:37.071754+0000","last_peered":"2023-10-09T05:49:37.071754+0000","last_clean":"2023-10-09T05:49:37.071754+0000","last_became_active":"2023-10-09T05:46:56.022978+0000","last_became_peered":"2023-10-09T05:46:56.022978+0000","last_unstale":"2023-10-09T05:49:37.071754+0000","last_undegraded":"2023-10-09T05:49:37.071754+0000","last_fullsized":"2023-10-09T05:49:37.071754+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:08.905002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":302,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T05:49:37.064639+0000","last_change":"2023-10-09T05:48:54.492580+0000","last_active":"2023-10-09T05:49:37.064639+0000","last_peered":"2023-10-09T05:49:37.064639+0000","last_clean":"2023-10-09T05:49:37.064639+0000","last_became_active":"2023-10-09T05:48:54.492259+0000","last_became_peered":"2023-10-09T05:48:54.492259+0000","last_unstale":"2023-10-09T05:49:37.064639+0000","last_undegraded":"2023-10-09T05:49:37.064639+0000","last_fullsized":"2023-10-09T05:49:37.064639+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":10308,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":36,"stat_sum":{"num_bytes":6061424664,"num_objects":92491,"num_object_clones":0,"num_object_copies":277473,"num_objects_missing_on_primary":18684,"num_objects_missing":18684,"num_objects_degraded":34383,"num_objects_misplaced":1224,"num_objects_unfound":0,"num_objects_dirty":92491,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":113389,"num_write_kb":6588097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4675,"num_bytes_recovered":305790976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18838007808,"data_stored":18837995592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":231498,"ondisk_log_size":231498,"up":108,"acting":104,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846775,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2959300,"kb_used_data":2045372,"kb_used_omap":14,"kb_used_meta":913905,"kb_avail":90785852,"statfs":{"total":95995035648,"available":92964712448,"internally_reserved":0,"allocated":2094460928,"data_stored":2093816990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":935839578},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":5.4740000000000002,"5min":3.851,"15min":4.4930000000000003},"min":{"1min":0.23400000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":79.307000000000002,"5min":110.123,"15min":220.77500000000001},"last":0.438},{"interface":"front","average":{"1min":5.4269999999999996,"5min":3.8159999999999998,"15min":4.6909999999999998},"min":{"1min":0.29699999999999999,"5min":0.245,"15min":0.22700000000000001},"max":{"1min":80.269999999999996,"5min":110.13800000000001,"15min":220.78700000000001},"last":0.42799999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.77300000000000002,"15min":2.8170000000000002},"min":{"1min":0.27800000000000002,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":2.8330000000000002,"5min":7.7729999999999997,"15min":190.68100000000001},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.90100000000000002,"15min":2.7549999999999999},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.255},"max":{"1min":4.2789999999999999,"5min":4.5510000000000002,"15min":191.09200000000001},"last":0.49399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":1.014,"15min":2.6659999999999999},"min":{"1min":0.29499999999999998,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":4.6200000000000001,"5min":7.8170000000000002,"15min":78.716999999999999},"last":1.087},{"interface":"front","average":{"1min":1.0469999999999999,"5min":0.93700000000000006,"15min":2.5990000000000002},"min":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":3.5510000000000002,"5min":3.5510000000000002,"15min":78.768000000000001},"last":8.3049999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":8.1210000000000004,"5min":3.8450000000000002,"15min":4.0650000000000004},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":69.066000000000003,"5min":69.066000000000003,"15min":185.17099999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":8.0099999999999998,"5min":3.786,"15min":4.2000000000000002},"min":{"1min":0.36399999999999999,"5min":0.33000000000000002,"15min":0.29799999999999999},"max":{"1min":69.055999999999997,"5min":69.055999999999997,"15min":183.35499999999999},"last":3.8199999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.77300000000000002,"15min":0.86899999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":3.9990000000000001,"5min":3.9990000000000001,"15min":12.789},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.4039999999999999,"5min":0.86599999999999999,"15min":1.0129999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":9.7829999999999995,"5min":9.7829999999999995,"15min":9.7829999999999995},"last":0.34699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":0.96299999999999997,"15min":1.075},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":4.2949999999999999,"5min":5.4950000000000001,"15min":10.813000000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.3260000000000001,"5min":0.90300000000000002,"15min":1.1679999999999999},"min":{"1min":0.371,"5min":0.34200000000000003,"15min":0.28299999999999997},"max":{"1min":4.1520000000000001,"5min":4.7649999999999997,"15min":10.321},"last":0.40600000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.79000000000000004,"15min":1.0109999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.318},"max":{"1min":4.3259999999999996,"5min":4.3259999999999996,"15min":9.2530000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.726,"5min":0.94499999999999995,"15min":1.024},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.30599999999999999},"max":{"1min":8.0670000000000002,"5min":8.0670000000000002,"15min":8.2680000000000007},"last":0.442}]}]},{"osd":6,"up_from":42,"seq":180388626801,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3156536,"kb_used_data":2566832,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":90588616,"statfs":{"total":95995035648,"available":92762742784,"internally_reserved":0,"allocated":2628435968,"data_stored":2627804456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:38 2023","interfaces":[{"interface":"back","average":{"1min":7.9820000000000002,"5min":2.3620000000000001,"15min":4.8040000000000003},"min":{"1min":0.32500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":42.869,"5min":42.869,"15min":162.08799999999999},"last":0.255},{"interface":"front","average":{"1min":7.4960000000000004,"5min":2.4910000000000001,"15min":4.625},"min":{"1min":0.252,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":42.892000000000003,"5min":42.892000000000003,"15min":161.90899999999999},"last":0.30499999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":0.96699999999999997,"15min":3.8029999999999999},"min":{"1min":0.33000000000000002,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":4.9779999999999998,"5min":6.5279999999999996,"15min":228.24299999999999},"last":2.081},{"interface":"front","average":{"1min":1.4370000000000001,"5min":0.83999999999999997,"15min":3.7799999999999998},"min":{"1min":0.39200000000000002,"5min":0.312,"15min":0.252},"max":{"1min":5.1829999999999998,"5min":5.1829999999999998,"15min":228.22499999999999},"last":2.0699999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:49:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":0.91400000000000003,"15min":3.7890000000000001},"min":{"1min":0.35799999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.1680000000000001,"5min":6.1680000000000001,"15min":115.40600000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.4550000000000001,"5min":0.997,"15min":3.8460000000000001},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":6.1440000000000001,"5min":6.1440000000000001,"15min":115.297},"last":0.44500000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":1.8440000000000001,"15min":3.8420000000000001},"min":{"1min":0.33300000000000002,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":19.52,"5min":77.578999999999994,"15min":115.244},"last":0.54300000000000004},{"interface":"front","average":{"1min":2.589,"5min":1.952,"15min":3.8959999999999999},"min":{"1min":0.33800000000000002,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":19.231999999999999,"5min":77.516999999999996,"15min":115.34999999999999},"last":0.47299999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:49:24 2023","interfaces":[{"interface":"back","average":{"1min":15.987,"5min":3.7519999999999998,"15min":2.774},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24099999999999999},"max":{"1min":257.375,"5min":257.375,"15min":257.375},"last":0.32200000000000001},{"interface":"front","average":{"1min":16.145,"5min":4.0430000000000001,"15min":2.8620000000000001},"min":{"1min":0.29099999999999998,"5min":0.246,"15min":0.246},"max":{"1min":257.61200000000002,"5min":257.61200000000002,"15min":257.61200000000002},"last":0.36899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:49:24 2023","interfaces":[{"interface":"back","average":{"1min":16.251999999999999,"5min":3.8570000000000002,"15min":2.9820000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":256.05700000000002,"5min":256.05700000000002,"15min":256.05700000000002},"last":0.35599999999999998},{"interface":"front","average":{"1min":16.359000000000002,"5min":4.1959999999999997,"15min":2.956},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.27500000000000002},"max":{"1min":256.08499999999998,"5min":256.08499999999998,"15min":256.08499999999998},"last":0.40600000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":0.77400000000000002,"15min":1.522},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26500000000000001},"max":{"1min":5.2279999999999998,"5min":5.2279999999999998,"15min":115.514},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.0499999999999998,"5min":1.2769999999999999,"15min":1.472},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":6.2450000000000001,"5min":14.300000000000001,"15min":116.53100000000001},"last":0.48399999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958762,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":770552,"kb_used_data":286704,"kb_used_omap":7,"kb_used_meta":483832,"kb_avail":92974600,"statfs":{"total":95995035648,"available":95205990400,"internally_reserved":0,"allocated":293584896,"data_stored":292544505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":495444494},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,0,1,2,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":2.7429999999999999,"15min":4.5010000000000003},"min":{"1min":0.224,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":3.7890000000000001,"5min":61.222999999999999,"15min":207.858},"last":4.782},{"interface":"front","average":{"1min":0.70899999999999996,"5min":2.956,"15min":5.0540000000000003},"min":{"1min":0.27000000000000002,"5min":0.23300000000000001,"15min":0.182},"max":{"1min":3.9060000000000001,"5min":61.366999999999997,"15min":206.97200000000001},"last":1.976}]},{"osd":2,"last update":"Mon Oct 9 05:49:27 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":1.6080000000000001,"15min":4.6379999999999999},"min":{"1min":0.24199999999999999,"5min":0.19800000000000001,"15min":0.184},"max":{"1min":10.210000000000001,"5min":17.768999999999998,"15min":101.367},"last":2.1499999999999999},{"interface":"front","average":{"1min":2.3580000000000001,"5min":1.4410000000000001,"15min":4.4249999999999998},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":16.702999999999999,"5min":17.718,"15min":99.861000000000004},"last":4.8040000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.6240000000000001,"15min":4.6929999999999996},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":7.5599999999999996,"5min":27.489999999999998,"15min":162.33699999999999},"last":1.496},{"interface":"front","average":{"1min":1.125,"5min":1.448,"15min":4.9349999999999996},"min":{"1min":0.316,"5min":0.26000000000000001,"15min":0.21299999999999999},"max":{"1min":5.9550000000000001,"5min":27.788,"15min":162.376},"last":6.3879999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":1.242,"15min":2.597},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":29.605,"5min":29.605,"15min":141.32900000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.605,"5min":1.034,"15min":2.1040000000000001},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":29.515999999999998,"5min":29.515999999999998,"15min":141.15899999999999},"last":4.7279999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":3.2519999999999998,"5min":1.591,"15min":2.653},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":32.774000000000001,"5min":32.774000000000001,"15min":235.18700000000001},"last":1.5840000000000001},{"interface":"front","average":{"1min":3.02,"5min":1.385,"15min":2.9399999999999999},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":36.976999999999997,"5min":36.976999999999997,"15min":235.79300000000001},"last":0.442}]},{"osd":6,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":2.3690000000000002,"15min":2.5960000000000001},"min":{"1min":0.38700000000000001,"5min":0.316,"15min":0.316},"max":{"1min":8.4870000000000001,"5min":101.295,"15min":103.60899999999999},"last":1.5389999999999999},{"interface":"front","average":{"1min":1.22,"5min":2.1110000000000002,"15min":2.7410000000000001},"min":{"1min":0.5,"5min":0.38900000000000001,"15min":0.33400000000000002},"max":{"1min":3.8559999999999999,"5min":101.27,"15min":103.679},"last":0.52400000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":3.238,"5min":1.355,"15min":2.1669999999999998},"min":{"1min":0.44700000000000001,"5min":0.375,"15min":0.35999999999999999},"max":{"1min":35.021000000000001,"5min":35.021000000000001,"15min":126.68300000000001},"last":4.6900000000000004},{"interface":"front","average":{"1min":3.6150000000000002,"5min":1.4350000000000001,"15min":2.282},"min":{"1min":0.44600000000000001,"5min":0.39700000000000002,"15min":0.34399999999999997},"max":{"1min":35.493000000000002,"5min":35.493000000000002,"15min":126.738},"last":4.7610000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122289,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2867924,"kb_used_data":2216460,"kb_used_omap":8,"kb_used_meta":651447,"kb_avail":90877228,"statfs":{"total":95995035648,"available":93058281472,"internally_reserved":0,"allocated":2269655040,"data_stored":2268529338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":667082504},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.75600000000000001,"15min":1.6830000000000001},"min":{"1min":0.248,"5min":0.17999999999999999,"15min":0.17199999999999999},"max":{"1min":4.2560000000000002,"5min":14.198,"15min":17.760000000000002},"last":0.26500000000000001},{"interface":"front","average":{"1min":1.1000000000000001,"5min":0.85699999999999998,"15min":2.2069999999999999},"min":{"1min":0.28000000000000003,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":2.9990000000000001,"5min":8.2680000000000007,"15min":26.163},"last":0.28100000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:49:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0209999999999999,"5min":1.5549999999999999,"15min":3.282},"min":{"1min":0.28199999999999997,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":30.257999999999999,"5min":30.257999999999999,"15min":92.230000000000004},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.5960000000000001,"5min":1.23,"15min":3.0739999999999998},"min":{"1min":0.33600000000000002,"5min":0.221,"15min":0.218},"max":{"1min":11.709,"5min":11.709,"15min":92.247},"last":0.308}]},{"osd":3,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.6659999999999999,"15min":3.9089999999999998},"min":{"1min":0.32800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":3.4790000000000001,"5min":55.328000000000003,"15min":123.024},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.429,"5min":1.8620000000000001,"15min":3.7829999999999999},"min":{"1min":0.34699999999999998,"5min":0.26800000000000002,"15min":0.222},"max":{"1min":6.4470000000000001,"5min":55.292000000000002,"15min":99.162000000000006},"last":0.52400000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:49:35 2023","interfaces":[{"interface":"back","average":{"1min":4.6849999999999996,"5min":1.5960000000000001,"15min":1.6899999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.25700000000000001},"max":{"1min":40.976999999999997,"5min":40.976999999999997,"15min":69.082999999999998},"last":0.496},{"interface":"front","average":{"1min":4.1449999999999996,"5min":1.6559999999999999,"15min":2.1459999999999999},"min":{"1min":0.46400000000000002,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":41.002000000000002,"5min":41.002000000000002,"15min":69.102000000000004},"last":0.374}]},{"osd":5,"last update":"Mon Oct 9 05:49:35 2023","interfaces":[{"interface":"back","average":{"1min":4.1920000000000002,"5min":1.7450000000000001,"15min":3.4420000000000002},"min":{"1min":0.36399999999999999,"5min":0.34599999999999997,"15min":0.30199999999999999},"max":{"1min":43.107999999999997,"5min":43.107999999999997,"15min":281.733},"last":0.46800000000000003},{"interface":"front","average":{"1min":4.8769999999999998,"5min":1.9570000000000001,"15min":3.556},"min":{"1min":0.438,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":41.969000000000001,"5min":41.969000000000001,"15min":282.95400000000001},"last":0.45400000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:49:35 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.2549999999999999,"15min":1.8360000000000001},"min":{"1min":0.40999999999999998,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":15.789,"5min":19.579000000000001,"15min":19.579000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.4689999999999999,"5min":1.6080000000000001,"15min":2.0699999999999998},"min":{"1min":0.40699999999999997,"5min":0.36699999999999999,"15min":0.32500000000000001},"max":{"1min":19.710999999999999,"5min":19.710999999999999,"15min":19.710999999999999},"last":0.41399999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.1859999999999999,"15min":1.4379999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.7220000000000004,"5min":25.378,"15min":25.378},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.55,"5min":1.391,"15min":2.0129999999999999},"min":{"1min":0.48399999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":5.556,"5min":25.436,"15min":25.436},"last":0.441}]}]},{"osd":2,"up_from":98,"seq":420906795236,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3298276,"kb_used_data":2606556,"kb_used_omap":11,"kb_used_meta":691700,"kb_avail":90446876,"statfs":{"total":95995035648,"available":92617601024,"internally_reserved":0,"allocated":2669113344,"data_stored":2667853381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":708301397},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":4.548,"15min":3.524},"min":{"1min":0.23699999999999999,"5min":0.17999999999999999,"15min":0.17799999999999999},"max":{"1min":8.2439999999999998,"5min":162.285,"15min":162.285},"last":12.829000000000001},{"interface":"front","average":{"1min":2.2730000000000001,"5min":4.6970000000000001,"15min":3.8330000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":8.6069999999999993,"5min":162.643,"15min":162.643},"last":3.0920000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:49:40 2023","interfaces":[{"interface":"back","average":{"1min":8.75,"5min":2.3330000000000002,"15min":3.6589999999999998},"min":{"1min":0.27400000000000002,"5min":0.193,"15min":0.193},"max":{"1min":125.77500000000001,"5min":125.77500000000001,"15min":125.77500000000001},"last":6.6539999999999999},{"interface":"front","average":{"1min":7.5359999999999996,"5min":2.077,"15min":3.0670000000000002},"min":{"1min":0.25700000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":125.857,"5min":125.857,"15min":125.857},"last":0.55000000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:49:15 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":3.4670000000000001,"15min":5.0250000000000004},"min":{"1min":0.30399999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.6349999999999998,"5min":159.35499999999999,"15min":159.35499999999999},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.7749999999999999,"5min":3.4990000000000001,"15min":4.9000000000000004},"min":{"1min":0.31900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.4710000000000001,"5min":159.196,"15min":159.196},"last":3.9209999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:49:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":0.87,"15min":1.9219999999999999},"min":{"1min":0.39500000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.2809999999999997,"5min":5.5279999999999996,"15min":115.84999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.026,"15min":2.113},"min":{"1min":0.317,"5min":0.309,"15min":0.28699999999999998},"max":{"1min":7.0949999999999998,"5min":8.7739999999999991,"15min":116.149},"last":3.8879999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:49:40 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.1399999999999999,"15min":2.0289999999999999},"min":{"1min":0.38500000000000001,"5min":0.33900000000000002,"15min":0.312},"max":{"1min":8.484,"5min":8.484,"15min":24.597999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.3959999999999999,"5min":1.006,"15min":1.7070000000000001},"min":{"1min":0.39700000000000002,"5min":0.30499999999999999,"15min":0.26000000000000001},"max":{"1min":4.3140000000000001,"5min":5.4850000000000003,"15min":23.919},"last":0.52000000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:49:40 2023","interfaces":[{"interface":"back","average":{"1min":1.484,"5min":1.4239999999999999,"15min":1.9379999999999999},"min":{"1min":0.38600000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.0650000000000004,"5min":25.818999999999999,"15min":25.818999999999999},"last":3.839},{"interface":"front","average":{"1min":1.7749999999999999,"5min":1.5309999999999999,"15min":2.069},"min":{"1min":0.45800000000000002,"5min":0.38900000000000001,"15min":0.36799999999999999},"max":{"1min":8.5739999999999998,"5min":24.481000000000002,"15min":24.481000000000002},"last":0.74199999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:49:15 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":1.6679999999999999,"15min":2.2250000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.3330000000000002,"5min":51.701999999999998,"15min":51.701999999999998},"last":3.952},{"interface":"front","average":{"1min":2.8450000000000002,"5min":1.9670000000000001,"15min":2.339},"min":{"1min":0.40899999999999997,"5min":0.373,"15min":0.373},"max":{"1min":8.7430000000000003,"5min":51.716000000000001,"15min":51.716000000000001},"last":0.71599999999999997}]}]},{"osd":3,"up_from":103,"seq":442381631710,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3689540,"kb_used_data":2981564,"kb_used_omap":4,"kb_used_meta":707963,"kb_avail":90055612,"statfs":{"total":95995035648,"available":92216946688,"internally_reserved":0,"allocated":3053121536,"data_stored":3052008525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":724954718},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":4.6680000000000001,"5min":3.8439999999999999,"15min":4.1260000000000003},"min":{"1min":0.253,"5min":0.188,"15min":0.183},"max":{"1min":47.173000000000002,"5min":149.565,"15min":167.82400000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":5.3040000000000003,"5min":3.9790000000000001,"15min":4.7069999999999999},"min":{"1min":0.21199999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":47.908000000000001,"5min":149.291,"15min":167.89599999999999},"last":0.25700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":2.4380000000000002,"15min":3.0600000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.22600000000000001},"max":{"1min":7.984,"5min":149.32499999999999,"15min":149.32499999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.7149999999999999,"5min":2.77,"15min":3.2320000000000002},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.19},"max":{"1min":11.547000000000001,"5min":149.34299999999999,"15min":149.34299999999999},"last":0.33300000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":2.4630000000000001,"15min":4.2949999999999999},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.215},"max":{"1min":8.0009999999999994,"5min":150.006,"15min":190.43299999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.597,"5min":2.4199999999999999,"15min":4.6020000000000003},"min":{"1min":0.221,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.8280000000000003,"5min":150.65100000000001,"15min":190.267},"last":0.45300000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":2.4329999999999998,"15min":2.6299999999999999},"min":{"1min":0.34899999999999998,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":5.883,"5min":149.84700000000001,"15min":149.84700000000001},"last":0.318},{"interface":"front","average":{"1min":2.052,"5min":2.3849999999999998,"15min":2.266},"min":{"1min":0.49199999999999999,"5min":0.32100000000000001,"15min":0.27400000000000002},"max":{"1min":7.9249999999999998,"5min":149.74700000000001,"15min":149.74700000000001},"last":0.40899999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":3.117,"5min":2.8220000000000001,"15min":2.54},"min":{"1min":0.46700000000000003,"5min":0.35399999999999998,"15min":0.28199999999999997},"max":{"1min":11.6,"5min":149.43000000000001,"15min":149.43000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.7090000000000001,"5min":2.6800000000000002,"15min":2.4710000000000001},"min":{"1min":0.51600000000000001,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":7.8689999999999998,"5min":149.69300000000001,"15min":149.69300000000001},"last":0.38}]},{"osd":6,"last update":"Mon Oct 9 05:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":2.4889999999999999,"15min":2.258},"min":{"1min":0.38700000000000001,"5min":0.34499999999999997,"15min":0.33500000000000002},"max":{"1min":7.9009999999999998,"5min":149.72399999999999,"15min":149.72399999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.5169999999999999,"5min":2.7730000000000001,"15min":2.4140000000000001},"min":{"1min":0.46500000000000002,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":11.647,"5min":149.589,"15min":149.589},"last":0.42499999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:49:16 2023","interfaces":[{"interface":"back","average":{"1min":2.4009999999999998,"5min":2.4740000000000002,"15min":2.48},"min":{"1min":0.435,"5min":0.35699999999999998,"15min":0.33700000000000002},"max":{"1min":9.3260000000000005,"5min":149.77199999999999,"15min":149.77199999999999},"last":0.497},{"interface":"front","average":{"1min":1.9990000000000001,"5min":2.5329999999999999,"15min":2.5249999999999999},"min":{"1min":0.48199999999999998,"5min":0.39100000000000001,"15min":0.35699999999999998},"max":{"1min":5.4299999999999997,"5min":149.80799999999999,"15min":149.80799999999999},"last":0.51700000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019257,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4237816,"kb_used_data":3344560,"kb_used_omap":0,"kb_used_meta":893248,"kb_avail":89507336,"statfs":{"total":95995035648,"available":91655512064,"internally_reserved":0,"allocated":3424829440,"data_stored":3424197152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914685952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.696,"15min":2.391},"min":{"1min":0.219,"5min":0.219,"15min":0.214},"max":{"1min":7.2380000000000004,"5min":26.478000000000002,"15min":100.504},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.3439999999999999,"5min":1.9430000000000001,"15min":2.75},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":14.670999999999999,"5min":26.454000000000001,"15min":100.717},"last":0.48499999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":1.2949999999999999,"15min":2.645},"min":{"1min":0.36599999999999999,"5min":0.26800000000000002,"15min":0.24299999999999999},"max":{"1min":10.406000000000001,"5min":10.406000000000001,"15min":187.369},"last":0.316},{"interface":"front","average":{"1min":2.8079999999999998,"5min":1.2370000000000001,"15min":2.726},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26600000000000001},"max":{"1min":12.202999999999999,"5min":12.202999999999999,"15min":187.35300000000001},"last":0.42199999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:49:31 2023","interfaces":[{"interface":"back","average":{"1min":1.9139999999999999,"5min":1.4770000000000001,"15min":4.5519999999999996},"min":{"1min":0.378,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":7.7220000000000004,"5min":17.335999999999999,"15min":196.93700000000001},"last":1.073},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.3180000000000001,"15min":4.4379999999999997},"min":{"1min":0.313,"5min":0.29799999999999999,"15min":0.28100000000000003},"max":{"1min":7.9710000000000001,"5min":16.795000000000002,"15min":197.08099999999999},"last":0.43099999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:48:53 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":2.5369999999999999,"15min":4.5679999999999996},"min":{"1min":0.35399999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":16.667000000000002,"5min":123.59,"15min":211.48699999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.5680000000000001,"15min":4.4100000000000001},"min":{"1min":0.439,"5min":0.312,"15min":0.311},"max":{"1min":16.530999999999999,"5min":123.566,"15min":211.31100000000001},"last":0.58599999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:48:53 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":1.173,"15min":1.794},"min":{"1min":0.27900000000000003,"5min":0.27800000000000002,"15min":0.24099999999999999},"max":{"1min":12.34,"5min":12.34,"15min":113.35899999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.6619999999999999,"5min":1.04,"15min":1.6699999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":10.191000000000001,"5min":10.191000000000001,"15min":113.32299999999999},"last":0.35899999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9059999999999999,"5min":0.86199999999999999,"15min":0.82199999999999995},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.8889999999999993,"5min":9.8889999999999993,"15min":11.944000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":1.889,"5min":1.1220000000000001,"15min":1.056},"min":{"1min":0.34899999999999998,"5min":0.32800000000000001,"15min":0.28399999999999997},"max":{"1min":6.0179999999999998,"5min":8.5269999999999992,"15min":10.356},"last":0.44700000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5649999999999999,"5min":1.7789999999999999,"15min":1.4830000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.312},"max":{"1min":5.8650000000000002,"5min":96.131,"15min":96.131},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.802,"5min":1.978,"15min":1.625},"min":{"1min":0.36399999999999999,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":7.6580000000000004,"5min":96.465000000000003,"15min":96.465000000000003},"last":0.496}]}]},{"osd":5,"up_from":36,"seq":154618823029,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3297000,"kb_used_data":2402400,"kb_used_omap":0,"kb_used_meta":894592,"kb_avail":90448152,"statfs":{"total":95995035648,"available":92618907648,"internally_reserved":0,"allocated":2460057600,"data_stored":2459437098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916062208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:44 2023","interfaces":[{"interface":"back","average":{"1min":5.5599999999999996,"5min":3.407,"15min":3.8149999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.217},"max":{"1min":85.650000000000006,"5min":96.858000000000004,"15min":165.90199999999999},"last":0.37},{"interface":"front","average":{"1min":6.657,"5min":3.5179999999999998,"15min":3.911},"min":{"1min":0.29399999999999998,"5min":0.254,"15min":0.215},"max":{"1min":85.866,"5min":97.555999999999997,"15min":165.96700000000001},"last":0.58099999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":1.0129999999999999,"15min":2.7029999999999998},"min":{"1min":0.35299999999999998,"5min":0.27000000000000002,"15min":0.23999999999999999},"max":{"1min":8.6799999999999997,"5min":8.6799999999999997,"15min":87.769000000000005},"last":3.5489999999999999},{"interface":"front","average":{"1min":1.1080000000000001,"5min":0.79800000000000004,"15min":2.6970000000000001},"min":{"1min":0.29099999999999998,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":3.4870000000000001,"5min":5.601,"15min":87.864000000000004},"last":7.6020000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:49:44 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.0569999999999999,"15min":4.4530000000000003},"min":{"1min":0.30199999999999999,"5min":0.28100000000000003,"15min":0.26700000000000002},"max":{"1min":9.4979999999999993,"5min":9.4979999999999993,"15min":290.697},"last":0.439},{"interface":"front","average":{"1min":1.7,"5min":1.1000000000000001,"15min":4.3719999999999999},"min":{"1min":0.36499999999999999,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":9.2799999999999994,"5min":9.2799999999999994,"15min":290.786},"last":1.2989999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":1.9079999999999999,"15min":3.7690000000000001},"min":{"1min":0.432,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":8.5350000000000001,"5min":53.243000000000002,"15min":95.236999999999995},"last":6.383},{"interface":"front","average":{"1min":1.9830000000000001,"5min":1.6850000000000001,"15min":3.5720000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29699999999999999},"max":{"1min":6.8339999999999996,"5min":53.259999999999998,"15min":93.471000000000004},"last":3.4369999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":8.9939999999999998,"5min":2.266,"15min":1.351},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":130.81,"5min":130.81,"15min":130.81},"last":0.34399999999999997},{"interface":"front","average":{"1min":9.8119999999999994,"5min":2.4990000000000001,"15min":1.6439999999999999},"min":{"1min":0.33500000000000002,"5min":0.27000000000000002,"15min":0.254},"max":{"1min":133.803,"5min":133.803,"15min":133.803},"last":0.41699999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":0.89300000000000002,"15min":1.504},"min":{"1min":0.32600000000000001,"5min":0.32000000000000001,"15min":0.26600000000000001},"max":{"1min":7.8220000000000001,"5min":7.8220000000000001,"15min":95.010999999999996},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.3719999999999999,"5min":0.997,"15min":1.466},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.27300000000000002},"max":{"1min":8.4629999999999992,"5min":8.4629999999999992,"15min":94.194000000000003},"last":0.35599999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.0009999999999999,"15min":1.1639999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.9980000000000002,"5min":6.9980000000000002,"15min":9.9849999999999994},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.2689999999999999,"5min":0.90000000000000002,"15min":1.155},"min":{"1min":0.316,"5min":0.316,"15min":0.308},"max":{"1min":6.8929999999999998,"5min":6.8929999999999998,"15min":14.432},"last":0.59299999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2259746816,"data_stored":2259746816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":291176448,"data_stored":291176448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2657947648,"data_stored":2657943576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3049193472,"data_stored":3049193472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3420454912,"data_stored":3420454912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2457145344,"data_stored":2457141272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2618032128,"data_stored":2618032128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2084311040,"data_stored":2084306968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:49:56.800 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:49:56.800 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 46 2023-10-09T05:49:56.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:56 smithi191 ceph-mon[161669]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:49:56.907 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:56.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:56.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:57.018 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:49:57.140 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:57.606 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:49:57.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:57.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:49:57.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:57 smithi008 ceph-mon[184418]: pgmap v964: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 34058/274980 objects degraded (12.386%); 1200/274980 objects misplaced (0.436%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:49:57.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:57 smithi008 ceph-mon[191227]: pgmap v964: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 34058/274980 objects degraded (12.386%); 1200/274980 objects misplaced (0.436%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:49:57.723 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:49:57.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:57 smithi191 ceph-mon[161669]: pgmap v964: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 34058/274980 objects degraded (12.386%); 1200/274980 objects misplaced (0.436%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:49:57.832 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:58.533 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:49:58.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:58 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:49:58.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/15833155' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:58.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 34058/274980 objects degraded (12.386%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:49:58.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:58 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:49:58.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/15833155' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:58.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 34058/274980 objects degraded (12.386%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:49:58.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:58 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:49:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/15833155' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:49:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 34058/274980 objects degraded (12.386%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:49:58.804 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:49:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:58.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:49:58.921 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:49:58.973 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:49:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:59 smithi191 ceph-mon[161669]: pgmap v965: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 34058/274758 objects degraded (12.396%); 1200/274758 objects misplaced (0.437%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:49:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:59 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:49:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/15833155' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:49:59.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:59 smithi191 ceph-mon[161669]: osdmap e320: 8 total, 8 up, 8 in 2023-10-09T05:49:59.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:59 smithi191 ceph-mon[161669]: failed to encode map e320 with expected crc 2023-10-09T05:49:59.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:59 smithi191 ceph-mon[161669]: failed to encode map e320 with expected crc 2023-10-09T05:49:59.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:59 smithi191 ceph-mon[161669]: failed to encode map e320 with expected crc 2023-10-09T05:49:59.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:59.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:49:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:59.951 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:49:59.957 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[184418]: pgmap v965: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 34058/274758 objects degraded (12.396%); 1200/274758 objects misplaced (0.437%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:49:59.957 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:49:59.957 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/15833155' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:49:59.957 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[184418]: osdmap e320: 8 total, 8 up, 8 in 2023-10-09T05:49:59.957 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[184418]: failed to encode map e320 with expected crc 2023-10-09T05:49:59.957 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[184418]: failed to encode map e320 with expected crc 2023-10-09T05:49:59.957 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[184418]: failed to encode map e320 with expected crc 2023-10-09T05:49:59.958 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:59.958 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:59.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:49:59.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:49:59.959 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[191227]: pgmap v965: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 34058/274758 objects degraded (12.396%); 1200/274758 objects misplaced (0.437%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:49:59.959 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:49:59.959 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/15833155' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:49:59.959 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[191227]: osdmap e320: 8 total, 8 up, 8 in 2023-10-09T05:49:59.959 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[191227]: failed to encode map e320 with expected crc 2023-10-09T05:49:59.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[191227]: failed to encode map e320 with expected crc 2023-10-09T05:49:59.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[191227]: failed to encode map e320 with expected crc 2023-10-09T05:49:59.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:49:59.960 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:49:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:00.060 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:00.135 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:00.503 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: Health detail: HEALTH_WARN Degraded data redundancy: 34058/274758 objects degraded (12.396%), 15 pgs degraded, 4 pgs undersized 2023-10-09T05:50:00.503 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: [WRN] PG_DEGRADED: Degraded data redundancy: 34058/274758 objects degraded (12.396%), 15 pgs degraded, 4 pgs undersized 2023-10-09T05:50:00.503 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.0 is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.503 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.1 is active+recovery_wait+degraded, acting [1,0,2] 2023-10-09T05:50:00.503 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.3 is active+recovery_wait+degraded, acting [1,5,6] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.7 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.a is active+recovery_wait+degraded, acting [1,2,3] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.b is active+recovery_wait+degraded, acting [4,3,1] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.f is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.11 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,2] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.13 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.17 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.1a is active+recovery_wait+degraded, acting [1,2,3] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.1b is active+recovery_wait+degraded, acting [4,3,1] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.1f is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.20 is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[191227]: pg 8.21 is active+recovery_wait+degraded, acting [1,0,2] 2023-10-09T05:50:00.634 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:00.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:00.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:00.685 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:00.739 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:00.775 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: Health detail: HEALTH_WARN Degraded data redundancy: 34058/274758 objects degraded (12.396%), 15 pgs degraded, 4 pgs undersized 2023-10-09T05:50:00.775 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: [WRN] PG_DEGRADED: Degraded data redundancy: 34058/274758 objects degraded (12.396%), 15 pgs degraded, 4 pgs undersized 2023-10-09T05:50:00.775 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.0 is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.775 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.1 is active+recovery_wait+degraded, acting [1,0,2] 2023-10-09T05:50:00.775 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.3 is active+recovery_wait+degraded, acting [1,5,6] 2023-10-09T05:50:00.775 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.7 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2023-10-09T05:50:00.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.a is active+recovery_wait+degraded, acting [1,2,3] 2023-10-09T05:50:00.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.b is active+recovery_wait+degraded, acting [4,3,1] 2023-10-09T05:50:00.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.f is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.11 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,2] 2023-10-09T05:50:00.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.13 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2023-10-09T05:50:00.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.17 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2023-10-09T05:50:00.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.1a is active+recovery_wait+degraded, acting [1,2,3] 2023-10-09T05:50:00.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.1b is active+recovery_wait+degraded, acting [4,3,1] 2023-10-09T05:50:00.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.1f is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.20 is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:00 smithi008 ceph-mon[184418]: pg 8.21 is active+recovery_wait+degraded, acting [1,0,2] 2023-10-09T05:50:00.777 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:50:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:50:00] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.33.4" 2023-10-09T05:50:00.791 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: Health detail: HEALTH_WARN Degraded data redundancy: 34058/274758 objects degraded (12.396%), 15 pgs degraded, 4 pgs undersized 2023-10-09T05:50:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: [WRN] PG_DEGRADED: Degraded data redundancy: 34058/274758 objects degraded (12.396%), 15 pgs degraded, 4 pgs undersized 2023-10-09T05:50:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.0 is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.1 is active+recovery_wait+degraded, acting [1,0,2] 2023-10-09T05:50:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.3 is active+recovery_wait+degraded, acting [1,5,6] 2023-10-09T05:50:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.7 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2023-10-09T05:50:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.a is active+recovery_wait+degraded, acting [1,2,3] 2023-10-09T05:50:00.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.b is active+recovery_wait+degraded, acting [4,3,1] 2023-10-09T05:50:00.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.f is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.11 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,2] 2023-10-09T05:50:00.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.13 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2023-10-09T05:50:00.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.17 is stuck undersized for 62s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2023-10-09T05:50:00.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.1a is active+recovery_wait+degraded, acting [1,2,3] 2023-10-09T05:50:00.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.1b is active+recovery_wait+degraded, acting [4,3,1] 2023-10-09T05:50:00.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.1f is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.20 is active+recovery_wait+degraded, acting [1,3,4] 2023-10-09T05:50:00.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:00 smithi191 ceph-mon[161669]: pg 8.21 is active+recovery_wait+degraded, acting [1,0,2] 2023-10-09T05:50:01.053 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:01.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:01.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:01.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:01.192 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:01.214 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:50:01.239 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:01.244 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:50:01.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:01.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:01.277 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:50:01.307 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:50:01.326 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:50:01.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:01.368 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:50:01.374 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:01.402 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:50:01.421 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:01.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:01.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:01.438 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:50:01.441 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:50:01.441 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:50:01.441 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:50:01.441 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:50:01.441 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:50:01.441 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:50:01.441 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:50:01.441 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:50:01.528 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:01.602 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:01 smithi008 ceph-mon[191227]: pgmap v967: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 33698/271095 objects degraded (12.430%); 1139/271095 objects misplaced (0.420%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:50:01.602 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:01 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4108332656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:01.603 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:01 smithi008 ceph-mon[184418]: pgmap v967: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 33698/271095 objects degraded (12.430%); 1139/271095 objects misplaced (0.420%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:50:01.604 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:01 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4108332656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:01.608 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:01.748 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:50:01.749 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:50:01.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:01 smithi191 ceph-mon[161669]: pgmap v967: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 33698/271095 objects degraded (12.430%); 1139/271095 objects misplaced (0.420%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:50:01.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:01 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4108332656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:02.335 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:50:02.609 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:02.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:02.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:02.714 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:02.759 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:03.042 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:03 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:50:03.043 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:03.043 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:03 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:03.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:03 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:50:03.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:03.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:03 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:03.123 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:03.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:03.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:03.241 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:03.269 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:03 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:50:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:03 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:03.316 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:03.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:03.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:03.425 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:03.451 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:03.493 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:03.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:03.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:03.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:03.625 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:03.672 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:03.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:03.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:03.778 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:03.829 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:03.978 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:03.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:03.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:04.084 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:04.101 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:04 smithi008 ceph-mon[191227]: pgmap v968: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 33698/270852 objects degraded (12.441%); 1139/270852 objects misplaced (0.421%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:50:04.101 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:04 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 33698/271095 objects degraded (12.430%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:04.101 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:04 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:50:04.101 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:04 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:50:04.101 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:04 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:04.102 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:04 smithi008 ceph-mon[184418]: pgmap v968: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 33698/270852 objects degraded (12.441%); 1139/270852 objects misplaced (0.421%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:50:04.102 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:04 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 33698/271095 objects degraded (12.430%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:04.102 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:04 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:50:04.102 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:04 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:50:04.102 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:04 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:04.146 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:04.261 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:50:04.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:04 smithi191 ceph-mon[161669]: pgmap v968: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 33698/270852 objects degraded (12.441%); 1139/270852 objects misplaced (0.421%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:50:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:04 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 33698/271095 objects degraded (12.430%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:04 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:50:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:04 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:50:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:04 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:04.400 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:04.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:04.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:04.508 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:04.586 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:04.666 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:04.716 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:04.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:04.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:04.824 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:04.914 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:05.020 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:05.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:05.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:05.124 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:05.150 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:05.197 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:05.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:05.305 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:05.377 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:05.520 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:05.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:05.624 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:05.650 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:05.695 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:05.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:05.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:05.801 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:05.854 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:05.917 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:05.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:05.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:06.025 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:06.051 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:06.096 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:06.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:06.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:06.101 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:50:06.111 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 8 pg_num to 46 2023-10-09T05:50:06.146 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:06 smithi008 ceph-mon[184418]: pgmap v969: 109 pgs: 1 active+recovery_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 259 op/s; 33698/269769 objects degraded (12.491%); 1139/269769 objects misplaced (0.422%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:50:06.146 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:06 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:50:06.146 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/184864866' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:50:06.146 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:06 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:50:06.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:06 smithi008 ceph-mon[191227]: pgmap v969: 109 pgs: 1 active+recovery_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 259 op/s; 33698/269769 objects degraded (12.491%); 1139/269769 objects misplaced (0.422%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:50:06.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:06 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:50:06.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/184864866' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:50:06.147 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:06 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:50:06.200 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:06.255 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:06.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:06 smithi191 ceph-mon[161669]: pgmap v969: 109 pgs: 1 active+recovery_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 259 op/s; 33698/269769 objects degraded (12.491%); 1139/269769 objects misplaced (0.422%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:50:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:06 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:50:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/184864866' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:50:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:06 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:50:06.375 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:06.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:06.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:06.480 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:06.507 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:06.977 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:07.010 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:07.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:07.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:07.122 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:07.275 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:07.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:07 smithi008 ceph-mon[191227]: failed to encode map e321 with expected crc 2023-10-09T05:50:07.306 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:07 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "46"}]': finished 2023-10-09T05:50:07.306 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:07 smithi008 ceph-mon[191227]: osdmap e321: 8 total, 8 up, 8 in 2023-10-09T05:50:07.306 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:07 smithi008 ceph-mon[191227]: failed to encode map e321 with expected crc 2023-10-09T05:50:07.306 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:07 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:50:07.308 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:07 smithi008 ceph-mon[184418]: failed to encode map e321 with expected crc 2023-10-09T05:50:07.308 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:07 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "46"}]': finished 2023-10-09T05:50:07.308 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:07 smithi008 ceph-mon[184418]: osdmap e321: 8 total, 8 up, 8 in 2023-10-09T05:50:07.308 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:07 smithi008 ceph-mon[184418]: failed to encode map e321 with expected crc 2023-10-09T05:50:07.309 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:07 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:50:07.327 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:07.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:07.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:07.419 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:07.435 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:07.478 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:07.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:07 smithi191 ceph-mon[161669]: failed to encode map e321 with expected crc 2023-10-09T05:50:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:07 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "46"}]': finished 2023-10-09T05:50:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:07 smithi191 ceph-mon[161669]: osdmap e321: 8 total, 8 up, 8 in 2023-10-09T05:50:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:07 smithi191 ceph-mon[161669]: failed to encode map e321 with expected crc 2023-10-09T05:50:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:07 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:50:07.618 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:07.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:07.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:07.727 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:07.762 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:08.060 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:50:08.396 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: pgmap v971: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 311 op/s; 33349/267288 objects degraded (12.477%); 1122/267288 objects misplaced (0.420%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T05:50:08.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: failed to encode map e322 with expected crc 2023-10-09T05:50:08.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: failed to encode map e322 with expected crc 2023-10-09T05:50:08.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: failed to encode map e322 with expected crc 2023-10-09T05:50:08.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "46"}]': finished 2023-10-09T05:50:08.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: osdmap e322: 8 total, 8 up, 8 in 2023-10-09T05:50:08.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/209370826' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:08.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:08.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 33349/267288 objects degraded (12.477%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:08.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:08.398 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:50:08.398 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[184418]: osdmap e323: 8 total, 8 up, 8 in 2023-10-09T05:50:08.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: pgmap v971: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 311 op/s; 33349/267288 objects degraded (12.477%); 1122/267288 objects misplaced (0.420%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T05:50:08.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: failed to encode map e322 with expected crc 2023-10-09T05:50:08.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: failed to encode map e322 with expected crc 2023-10-09T05:50:08.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: failed to encode map e322 with expected crc 2023-10-09T05:50:08.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "46"}]': finished 2023-10-09T05:50:08.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: osdmap e322: 8 total, 8 up, 8 in 2023-10-09T05:50:08.398 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/209370826' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:08.399 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:08.399 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 33349/267288 objects degraded (12.477%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:08.399 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:08.399 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:50:08.399 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:08 smithi008 ceph-mon[191227]: osdmap e323: 8 total, 8 up, 8 in 2023-10-09T05:50:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: pgmap v971: 109 pgs: 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 311 op/s; 33349/267288 objects degraded (12.477%); 1122/267288 objects misplaced (0.420%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T05:50:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: failed to encode map e322 with expected crc 2023-10-09T05:50:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: failed to encode map e322 with expected crc 2023-10-09T05:50:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: failed to encode map e322 with expected crc 2023-10-09T05:50:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "46"}]': finished 2023-10-09T05:50:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: osdmap e322: 8 total, 8 up, 8 in 2023-10-09T05:50:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/209370826' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 33349/267288 objects degraded (12.477%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:50:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:08 smithi191 ceph-mon[161669]: osdmap e323: 8 total, 8 up, 8 in 2023-10-09T05:50:09.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[184418]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[184418]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[184418]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[184418]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[184418]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[184418]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.194 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[184418]: osdmap e324: 8 total, 8 up, 8 in 2023-10-09T05:50:09.194 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[191227]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[191227]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[191227]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[191227]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[191227]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[191227]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:09 smithi008 ceph-mon[191227]: osdmap e324: 8 total, 8 up, 8 in 2023-10-09T05:50:09.327 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:09.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:09.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:09.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:09.479 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:09 smithi191 ceph-mon[161669]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.479 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:09 smithi191 ceph-mon[161669]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.479 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:09 smithi191 ceph-mon[161669]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.479 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:09 smithi191 ceph-mon[161669]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.479 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:09 smithi191 ceph-mon[161669]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.479 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:09 smithi191 ceph-mon[161669]: failed to encode map e323 with expected crc 2023-10-09T05:50:09.479 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:09 smithi191 ceph-mon[161669]: osdmap e324: 8 total, 8 up, 8 in 2023-10-09T05:50:09.481 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:09.529 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:09.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:09.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:09.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:09.665 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:09.709 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:09.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:09.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:09.814 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:09.840 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:09.883 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:09.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:09.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:09.988 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:10.069 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:10.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:10 smithi008 ceph-mon[184418]: pgmap v974: 119 pgs: 10 unknown, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 33349/267042 objects degraded (12.488%); 1122/267042 objects misplaced (0.420%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:50:10.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:10 smithi008 ceph-mon[184418]: failed to encode map e324 with expected crc 2023-10-09T05:50:10.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:10 smithi008 ceph-mon[184418]: failed to encode map e324 with expected crc 2023-10-09T05:50:10.152 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:10 smithi008 ceph-mon[184418]: failed to encode map e324 with expected crc 2023-10-09T05:50:10.198 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:10.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:10.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:10.306 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:10.332 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:10.375 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:10.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:10.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:10.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:10 smithi008 ceph-mon[191227]: pgmap v974: 119 pgs: 10 unknown, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 33349/267042 objects degraded (12.488%); 1122/267042 objects misplaced (0.420%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:50:10.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:10 smithi008 ceph-mon[191227]: failed to encode map e324 with expected crc 2023-10-09T05:50:10.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:10 smithi008 ceph-mon[191227]: failed to encode map e324 with expected crc 2023-10-09T05:50:10.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:10 smithi008 ceph-mon[191227]: failed to encode map e324 with expected crc 2023-10-09T05:50:10.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:10.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:10 smithi191 ceph-mon[161669]: pgmap v974: 119 pgs: 10 unknown, 1 active+recovering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 91 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 33349/267042 objects degraded (12.488%); 1122/267042 objects misplaced (0.420%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T05:50:10.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:10 smithi191 ceph-mon[161669]: failed to encode map e324 with expected crc 2023-10-09T05:50:10.504 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:10 smithi191 ceph-mon[161669]: failed to encode map e324 with expected crc 2023-10-09T05:50:10.504 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:10 smithi191 ceph-mon[161669]: failed to encode map e324 with expected crc 2023-10-09T05:50:10.506 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:10.548 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:10.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:10.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:10.608 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:50:10.631 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:50:10.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:10.655 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:50:10.675 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:50:10.680 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:10.694 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:50:10.719 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:50:10.723 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:10.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:10.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:10.759 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:50:10.795 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:50:10.822 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:50:10.822 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:50:10.822 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:50:10.823 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:50:10.823 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:50:10.823 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:50:10.823 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:50:10.823 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:50:10.829 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:10.855 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:10.868 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:50:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:50:10] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.33.4" 2023-10-09T05:50:10.905 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:10.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:10.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:10.918 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:50:10.918 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:50:10.918 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T05:50:10.918 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:50:11.010 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:11.134 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:11 smithi008 ceph-mon[184418]: failed to encode map e325 with expected crc 2023-10-09T05:50:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:11 smithi008 ceph-mon[184418]: osdmap e325: 8 total, 8 up, 8 in 2023-10-09T05:50:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:11 smithi008 ceph-mon[184418]: pgmap v977: 119 pgs: 2 unknown, 1 active+recovering, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 95 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 33301/265092 objects degraded (12.562%); 1122/265092 objects misplaced (0.423%); 768 KiB/s, 12 objects/s recovering 2023-10-09T05:50:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2221877793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:11.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:11 smithi008 ceph-mon[191227]: failed to encode map e325 with expected crc 2023-10-09T05:50:11.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:11 smithi008 ceph-mon[191227]: osdmap e325: 8 total, 8 up, 8 in 2023-10-09T05:50:11.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:11 smithi008 ceph-mon[191227]: pgmap v977: 119 pgs: 2 unknown, 1 active+recovering, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 95 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 33301/265092 objects degraded (12.562%); 1122/265092 objects misplaced (0.423%); 768 KiB/s, 12 objects/s recovering 2023-10-09T05:50:11.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2221877793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:11.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:11 smithi191 ceph-mon[161669]: failed to encode map e325 with expected crc 2023-10-09T05:50:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:11 smithi191 ceph-mon[161669]: osdmap e325: 8 total, 8 up, 8 in 2023-10-09T05:50:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:11 smithi191 ceph-mon[161669]: pgmap v977: 119 pgs: 2 unknown, 1 active+recovering, 4 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 95 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 33301/265092 objects degraded (12.562%); 1122/265092 objects misplaced (0.423%); 768 KiB/s, 12 objects/s recovering 2023-10-09T05:50:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2221877793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:11.967 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:11.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:11.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:12.028 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:50:12.028 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:50:12.028 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:50:12.028 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-10-09T05:50:12.028 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0 2023-10-09T05:50:12.036 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:50:12.073 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:12.112 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:12.428 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:12.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:12.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:12.483 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:50:12.534 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:12.613 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:12.752 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:12.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:12.837 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:12.866 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:12.918 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:12.925 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:13.218 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:13.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:13.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:13.324 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:13.409 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:13.770 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:13.791 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:13.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:13 smithi191 ceph-mon[161669]: pgmap v978: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 33112/264399 objects degraded (12.523%); 877/264399 objects misplaced (0.332%); 5.7 MiB/s, 92 objects/s recovering 2023-10-09T05:50:13.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 33112/264399 objects degraded (12.523%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:13.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:13.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:13.813 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:13 smithi008 ceph-mon[184418]: pgmap v978: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 33112/264399 objects degraded (12.523%); 877/264399 objects misplaced (0.332%); 5.7 MiB/s, 92 objects/s recovering 2023-10-09T05:50:13.813 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 33112/264399 objects degraded (12.523%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:13.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:13 smithi008 ceph-mon[191227]: pgmap v978: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 33112/264399 objects degraded (12.523%); 877/264399 objects misplaced (0.332%); 5.7 MiB/s, 92 objects/s recovering 2023-10-09T05:50:13.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 33112/264399 objects degraded (12.523%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:13.914 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:13.964 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:14.405 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:50:14.853 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:14.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:14.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:14.957 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:14.996 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:15.133 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:15.133 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:15.136 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:15.136 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:15.187 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:15.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:15.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:15.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:15.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:15.322 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:15.368 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:15.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:15.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:15.474 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:15.530 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:15.803 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:15.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:15.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:15.922 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:15.949 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:15.994 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:15.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:15.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:16.099 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:16.137 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:15 smithi008 ceph-mon[184418]: pgmap v979: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 33112/263415 objects degraded (12.570%); 877/263415 objects misplaced (0.333%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:50:16.138 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:15 smithi008 ceph-mon[191227]: pgmap v979: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 33112/263415 objects degraded (12.570%); 877/263415 objects misplaced (0.333%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:50:16.149 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:15 smithi191 ceph-mon[161669]: pgmap v979: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 33112/263415 objects degraded (12.570%); 877/263415 objects misplaced (0.333%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:50:16.777 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:16.875 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:16.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:16.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:16.945 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:50:16.982 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:17.028 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:17.109 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:16 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/211975910' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:17.117 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:16 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/211975910' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:17.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:16 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/211975910' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:17.803 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:17.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:17.908 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:17.942 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:17.989 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:17.990 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.1 primary-affinity to 0 (802) 2023-10-09T05:50:17.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:17.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:18.095 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:18.103 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[191227]: pgmap v980: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 32706/260949 objects degraded (12.533%); 858/260949 objects misplaced (0.329%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T05:50:18.103 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:18.103 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[191227]: failed to encode map e326 with expected crc 2023-10-09T05:50:18.103 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[191227]: failed to encode map e326 with expected crc 2023-10-09T05:50:18.103 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/211975910' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:18.104 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[191227]: osdmap e326: 8 total, 8 up, 8 in 2023-10-09T05:50:18.104 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1672240942' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T05:50:18.104 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T05:50:18.104 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[184418]: pgmap v980: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 32706/260949 objects degraded (12.533%); 858/260949 objects misplaced (0.329%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T05:50:18.104 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:18.104 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[184418]: failed to encode map e326 with expected crc 2023-10-09T05:50:18.104 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[184418]: failed to encode map e326 with expected crc 2023-10-09T05:50:18.104 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/211975910' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:18.104 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[184418]: osdmap e326: 8 total, 8 up, 8 in 2023-10-09T05:50:18.104 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1672240942' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T05:50:18.105 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:17 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T05:50:18.120 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:18.163 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:18.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:18.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:18.269 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:18.291 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:17 smithi191 ceph-mon[161669]: pgmap v980: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 32706/260949 objects degraded (12.533%); 858/260949 objects misplaced (0.329%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T05:50:18.292 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:17 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:18.292 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:17 smithi191 ceph-mon[161669]: failed to encode map e326 with expected crc 2023-10-09T05:50:18.292 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:17 smithi191 ceph-mon[161669]: failed to encode map e326 with expected crc 2023-10-09T05:50:18.292 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/211975910' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:18.292 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:17 smithi191 ceph-mon[161669]: osdmap e326: 8 total, 8 up, 8 in 2023-10-09T05:50:18.292 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1672240942' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T05:50:18.292 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:17 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T05:50:18.294 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:18.338 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:18.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:18.444 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:18.470 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:18.514 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:18.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:18.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:18.620 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:18.653 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:18.777 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:50:18.777 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:50:18.777 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:50:18.778 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:50:18.778 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:50:19.003 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:19.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:19.107 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:19.147 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:19.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: failed to encode map e327 with expected crc 2023-10-09T05:50:19.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: failed to encode map e327 with expected crc 2023-10-09T05:50:19.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T05:50:19.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: osdmap e327: 8 total, 8 up, 8 in 2023-10-09T05:50:19.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: failed to encode map e327 with expected crc 2023-10-09T05:50:19.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 32706/260949 objects degraded (12.533%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:19.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: osdmap e328: 8 total, 8 up, 8 in 2023-10-09T05:50:19.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[184418]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: failed to encode map e327 with expected crc 2023-10-09T05:50:19.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: failed to encode map e327 with expected crc 2023-10-09T05:50:19.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T05:50:19.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: osdmap e327: 8 total, 8 up, 8 in 2023-10-09T05:50:19.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: failed to encode map e327 with expected crc 2023-10-09T05:50:19.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 32706/260949 objects degraded (12.533%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:19.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: osdmap e328: 8 total, 8 up, 8 in 2023-10-09T05:50:19.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:18 smithi008 ceph-mon[191227]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: failed to encode map e327 with expected crc 2023-10-09T05:50:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: failed to encode map e327 with expected crc 2023-10-09T05:50:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T05:50:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: osdmap e327: 8 total, 8 up, 8 in 2023-10-09T05:50:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: failed to encode map e327 with expected crc 2023-10-09T05:50:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 32706/260949 objects degraded (12.533%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:50:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: osdmap e328: 8 total, 8 up, 8 in 2023-10-09T05:50:19.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: failed to encode map e328 with expected crc 2023-10-09T05:50:19.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:18 smithi191 ceph-mon[161669]: failed to encode map e328 with expected crc 2023-10-09T05:50:20.004 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:50:20.058 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:50:20.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[184418]: pgmap v984: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 32706/260853 objects degraded (12.538%); 858/260853 objects misplaced (0.329%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:50:20.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[184418]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[184418]: osdmap e329: 8 total, 8 up, 8 in 2023-10-09T05:50:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[184418]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[184418]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[184418]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[184418]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[191227]: pgmap v984: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 32706/260853 objects degraded (12.538%); 858/260853 objects misplaced (0.329%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:50:20.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[191227]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[191227]: osdmap e329: 8 total, 8 up, 8 in 2023-10-09T05:50:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[191227]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[191227]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[191227]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:20 smithi008 ceph-mon[191227]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:20 smithi191 ceph-mon[161669]: pgmap v984: 119 pgs: 1 active+recovering, 4 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 13 active+recovery_wait+degraded, 95 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 32706/260853 objects degraded (12.538%); 858/260853 objects misplaced (0.329%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T05:50:20.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:20 smithi191 ceph-mon[161669]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:20 smithi191 ceph-mon[161669]: osdmap e329: 8 total, 8 up, 8 in 2023-10-09T05:50:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:20 smithi191 ceph-mon[161669]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:20 smithi191 ceph-mon[161669]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:20 smithi191 ceph-mon[161669]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:20 smithi191 ceph-mon[161669]: failed to encode map e329 with expected crc 2023-10-09T05:50:20.323 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:20.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:20.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:20.430 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:20.462 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:20.577 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:50:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:50:20] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.33.4" 2023-10-09T05:50:20.584 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:20.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:20.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:20.690 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:20.716 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:20.717 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:20.759 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:20.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:20.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:20.847 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:50:20.867 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:20.893 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:20.893 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:20.935 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:20.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:21.041 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:21.073 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:21 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4062446575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:21.303 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:21 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4062446575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:21.303 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:21 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4062446575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:21.356 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:50:21.389 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:50:21.434 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:50:21.473 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:50:21.510 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:50:21.542 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:21.544 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:50:21.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:21.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:21.576 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:50:21.581 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:50:21.581 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:50:21.581 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:50:21.581 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:50:21.581 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:50:21.582 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:50:21.582 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:50:21.582 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:50:21.652 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:21.679 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:21.730 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:21.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:21.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:21.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:21.863 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:21.916 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:21.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:21.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:22.022 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:22.036 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:22 smithi191 ceph-mon[161669]: pgmap v986: 119 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 98 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 31093/259281 objects degraded (11.992%); 2321/259281 objects misplaced (0.895%); 1.6 MiB/s, 27 objects/s recovering 2023-10-09T05:50:22.067 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:22.344 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:22 smithi008 ceph-mon[184418]: pgmap v986: 119 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 98 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 31093/259281 objects degraded (11.992%); 2321/259281 objects misplaced (0.895%); 1.6 MiB/s, 27 objects/s recovering 2023-10-09T05:50:22.344 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:22 smithi008 ceph-mon[191227]: pgmap v986: 119 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 98 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 31093/259281 objects degraded (11.992%); 2321/259281 objects misplaced (0.895%); 1.6 MiB/s, 27 objects/s recovering 2023-10-09T05:50:22.384 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:22.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:22.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:22.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:22.517 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:22.560 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:22.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:22.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:22.667 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:22.702 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:23.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 31093/259281 objects degraded (11.992%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:23.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 31093/259281 objects degraded (11.992%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:23.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 31093/259281 objects degraded (11.992%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:23.561 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:23.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:23.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:23.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:23.696 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:23.751 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:23.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:23.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:23.859 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:23.951 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:24.056 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:24.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:24.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:24.095 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:50:24.096 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:50:24.162 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:24.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:24 smithi008 ceph-mon[184418]: pgmap v987: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 98 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 29675/258672 objects degraded (11.472%); 3397/258672 objects misplaced (1.313%); 4.4 MiB/s, 72 objects/s recovering 2023-10-09T05:50:24.206 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:24 smithi008 ceph-mon[191227]: pgmap v987: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 98 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 29675/258672 objects degraded (11.472%); 3397/258672 objects misplaced (1.313%); 4.4 MiB/s, 72 objects/s recovering 2023-10-09T05:50:24.220 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:24.226 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:50:24.227 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:50:24.227 INFO:tasks.thrashosds.thrasher:Reweighting osd 3 to 0.8934425589358884 2023-10-09T05:50:24.227 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 0.8934425589358884 2023-10-09T05:50:24.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:24 smithi191 ceph-mon[161669]: pgmap v987: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 98 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 29675/258672 objects degraded (11.472%); 3397/258672 objects misplaced (1.313%); 4.4 MiB/s, 72 objects/s recovering 2023-10-09T05:50:25.164 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:25.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:25.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:25.267 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:50:25.269 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:25.314 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:25.422 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:25.428 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:25.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:25.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:25.540 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:25.569 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:25.623 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:25.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:25.686 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:50:25.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:25.754 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:25.803 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:25.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:25.909 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:25.964 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:26.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:26 smithi008 ceph-mon[184418]: pgmap v988: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 29615/257625 objects degraded (11.495%); 3397/257625 objects misplaced (1.319%); 4.3 MiB/s, 70 objects/s recovering 2023-10-09T05:50:26.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2018402287' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:26.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:26 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:26.072 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:26 smithi008 ceph-mon[191227]: pgmap v988: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 29615/257625 objects degraded (11.495%); 3397/257625 objects misplaced (1.319%); 4.3 MiB/s, 70 objects/s recovering 2023-10-09T05:50:26.073 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2018402287' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:26.073 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:26 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:26.088 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:50:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:26 smithi191 ceph-mon[161669]: pgmap v988: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 29615/257625 objects degraded (11.495%); 3397/257625 objects misplaced (1.319%); 4.3 MiB/s, 70 objects/s recovering 2023-10-09T05:50:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2018402287' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:26 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:26.606 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:26.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:26.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:26.711 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:26.737 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:26.783 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:26.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:26.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:26.891 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:26.937 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:27.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:27.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[184418]: failed to encode map e330 with expected crc 2023-10-09T05:50:27.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:50:27.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[184418]: osdmap e330: 8 total, 8 up, 8 in 2023-10-09T05:50:27.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[184418]: failed to encode map e330 with expected crc 2023-10-09T05:50:27.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[184418]: failed to encode map e330 with expected crc 2023-10-09T05:50:27.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:27.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[191227]: failed to encode map e330 with expected crc 2023-10-09T05:50:27.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:50:27.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[191227]: osdmap e330: 8 total, 8 up, 8 in 2023-10-09T05:50:27.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[191227]: failed to encode map e330 with expected crc 2023-10-09T05:50:27.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:27 smithi008 ceph-mon[191227]: failed to encode map e330 with expected crc 2023-10-09T05:50:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:27 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:27 smithi191 ceph-mon[161669]: failed to encode map e330 with expected crc 2023-10-09T05:50:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:27 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:50:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:27 smithi191 ceph-mon[161669]: osdmap e330: 8 total, 8 up, 8 in 2023-10-09T05:50:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:27 smithi191 ceph-mon[161669]: failed to encode map e330 with expected crc 2023-10-09T05:50:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:27 smithi191 ceph-mon[161669]: failed to encode map e330 with expected crc 2023-10-09T05:50:27.606 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:27.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:27.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:27.686 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:27.714 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:27.744 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:27.790 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:27.799 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:27.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:27.904 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:27.930 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:27.976 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:27.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:27.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:28.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:28.110 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:28.166 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:28.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:28.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:28.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:28 smithi008 ceph-mon[184418]: pgmap v990: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 29335/254484 objects degraded (11.527%); 3233/254484 objects misplaced (1.270%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T05:50:28.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 29335/254484 objects degraded (11.527%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:28 smithi008 ceph-mon[191227]: pgmap v990: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 29335/254484 objects degraded (11.527%); 3233/254484 objects misplaced (1.270%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T05:50:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 29335/254484 objects degraded (11.527%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:28.271 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:28.299 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:28.344 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:28.348 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:28 smithi191 ceph-mon[161669]: pgmap v990: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 29335/254484 objects degraded (11.527%); 3233/254484 objects misplaced (1.270%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T05:50:28.348 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 29335/254484 objects degraded (11.527%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:28.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:28.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:28.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:28.478 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:28.527 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:28.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:28.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:28.633 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:28.676 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:28.894 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:28.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:28.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:29.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:29.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.3 to 0.893443 (e4b8) 2023-10-09T05:50:29.190 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:29.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4107088484' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8934425589358884}]: dispatch 2023-10-09T05:50:29.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:29 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8934425589358884}]: dispatch 2023-10-09T05:50:29.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:29.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4107088484' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8934425589358884}]: dispatch 2023-10-09T05:50:29.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:29 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8934425589358884}]: dispatch 2023-10-09T05:50:29.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:29.256 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:29.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4107088484' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8934425589358884}]: dispatch 2023-10-09T05:50:29.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:29 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8934425589358884}]: dispatch 2023-10-09T05:50:29.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:29.517 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:50:30.407 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[191227]: pgmap v991: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 29076/253470 objects degraded (11.471%); 3233/253470 objects misplaced (1.275%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T05:50:30.407 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:50:30.407 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[191227]: failed to encode map e331 with expected crc 2023-10-09T05:50:30.407 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[191227]: failed to encode map e331 with expected crc 2023-10-09T05:50:30.408 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.8934425589358884}]': finished 2023-10-09T05:50:30.408 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[191227]: osdmap e331: 8 total, 8 up, 8 in 2023-10-09T05:50:30.408 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[191227]: failed to encode map e331 with expected crc 2023-10-09T05:50:30.408 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[184418]: pgmap v991: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 29076/253470 objects degraded (11.471%); 3233/253470 objects misplaced (1.275%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T05:50:30.408 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:50:30.408 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[184418]: failed to encode map e331 with expected crc 2023-10-09T05:50:30.408 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[184418]: failed to encode map e331 with expected crc 2023-10-09T05:50:30.408 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.8934425589358884}]': finished 2023-10-09T05:50:30.408 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[184418]: osdmap e331: 8 total, 8 up, 8 in 2023-10-09T05:50:30.409 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:30 smithi008 ceph-mon[184418]: failed to encode map e331 with expected crc 2023-10-09T05:50:30.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:30 smithi191 ceph-mon[161669]: pgmap v991: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 29076/253470 objects degraded (11.471%); 3233/253470 objects misplaced (1.275%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T05:50:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:30 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:50:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:30 smithi191 ceph-mon[161669]: failed to encode map e331 with expected crc 2023-10-09T05:50:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:30 smithi191 ceph-mon[161669]: failed to encode map e331 with expected crc 2023-10-09T05:50:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:30 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.8934425589358884}]': finished 2023-10-09T05:50:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:30 smithi191 ceph-mon[161669]: osdmap e331: 8 total, 8 up, 8 in 2023-10-09T05:50:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:30 smithi191 ceph-mon[161669]: failed to encode map e331 with expected crc 2023-10-09T05:50:30.716 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:50:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:50:30] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.33.4" 2023-10-09T05:50:31.234 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:31.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:31.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:31.343 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:31.475 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:31.756 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:31.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:31.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:31.834 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:50:31.834 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:50:31.861 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:31.893 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:31.941 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:31.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:31.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:32.029 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:32.051 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:32.082 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:50:32.099 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:32.125 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:50:32.159 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:50:32.188 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:50:32.217 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:50:32.247 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:50:32.277 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:50:32.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:32 smithi191 ceph-mon[161669]: pgmap v993: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 28826/248532 objects degraded (11.599%); 3156/248532 objects misplaced (1.270%); 8.4 MiB/s, 135 objects/s recovering 2023-10-09T05:50:32.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:32 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3798240889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:32.304 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:50:32.307 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:50:32.307 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:50:32.307 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:50:32.307 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:50:32.307 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:50:32.308 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:50:32.308 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:50:32.308 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:50:32.379 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:32 smithi008 ceph-mon[184418]: pgmap v993: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 28826/248532 objects degraded (11.599%); 3156/248532 objects misplaced (1.270%); 8.4 MiB/s, 135 objects/s recovering 2023-10-09T05:50:32.379 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:32 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3798240889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:32.379 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:32 smithi008 ceph-mon[191227]: pgmap v993: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 28826/248532 objects degraded (11.599%); 3156/248532 objects misplaced (1.270%); 8.4 MiB/s, 135 objects/s recovering 2023-10-09T05:50:32.379 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:32 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3798240889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:33.347 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:33.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:33.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:33.451 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:33.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 28826/248532 objects degraded (11.599%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:33.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 28826/248532 objects degraded (11.599%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:33.492 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:33.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 28826/248532 objects degraded (11.599%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:34.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:34 smithi008 ceph-mon[184418]: pgmap v994: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 28388/246846 objects degraded (11.500%); 3076/246846 objects misplaced (1.246%); 12 MiB/s, 192 objects/s recovering 2023-10-09T05:50:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:34 smithi008 ceph-mon[191227]: pgmap v994: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 28388/246846 objects degraded (11.500%); 3076/246846 objects misplaced (1.246%); 12 MiB/s, 192 objects/s recovering 2023-10-09T05:50:34.320 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:50:34.534 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:34.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:34.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:34.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:34 smithi191 ceph-mon[161669]: pgmap v994: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 28388/246846 objects degraded (11.500%); 3076/246846 objects misplaced (1.246%); 12 MiB/s, 192 objects/s recovering 2023-10-09T05:50:34.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:34.671 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:34.721 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:34.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:34.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:34.825 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:34.866 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:35.156 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:35.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:35.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:35.239 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:35.262 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:35.343 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:35.438 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:35.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:35.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:35.542 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:35.606 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:35.673 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:35.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:35.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:35.779 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:35.808 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:35.863 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:35.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:35.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:35.970 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:36.000 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:36.055 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:36.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:36.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:36.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:36 smithi008 ceph-mon[184418]: pgmap v995: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 28273/244761 objects degraded (11.551%); 3076/244761 objects misplaced (1.257%); 11 MiB/s, 171 objects/s recovering 2023-10-09T05:50:36.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:36 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:50:36.114 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3974192034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:36.130 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:50:36.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:36.202 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:36.266 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:36.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:36.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:36.345 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:50:36.345 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:50:36.345 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T05:50:36.345 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:50:36.345 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:50:36.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T05:50:36.369 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:36 smithi191 ceph-mon[161669]: pgmap v995: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 28273/244761 objects degraded (11.551%); 3076/244761 objects misplaced (1.257%); 11 MiB/s, 171 objects/s recovering 2023-10-09T05:50:36.370 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:36 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:50:36.370 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3974192034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:36.374 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:36.410 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:36 smithi008 ceph-mon[191227]: pgmap v995: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 28273/244761 objects degraded (11.551%); 3076/244761 objects misplaced (1.257%); 11 MiB/s, 171 objects/s recovering 2023-10-09T05:50:36.410 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:36 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:50:36.410 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3974192034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:36.414 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:36.444 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:50:36.475 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:36.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:36.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:36.581 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:36.632 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:36.778 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:50:37.024 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:37.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:37.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:37.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:37.166 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:37 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:50:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:37 smithi008 ceph-mon[184418]: failed to encode map e332 with expected crc 2023-10-09T05:50:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:37 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3974192034' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:37 smithi008 ceph-mon[184418]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T05:50:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:37 smithi008 ceph-mon[184418]: failed to encode map e332 with expected crc 2023-10-09T05:50:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:37 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:50:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:37 smithi008 ceph-mon[191227]: failed to encode map e332 with expected crc 2023-10-09T05:50:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:37 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3974192034' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:37 smithi008 ceph-mon[191227]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T05:50:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:37 smithi008 ceph-mon[191227]: failed to encode map e332 with expected crc 2023-10-09T05:50:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:37 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:50:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:37 smithi191 ceph-mon[161669]: failed to encode map e332 with expected crc 2023-10-09T05:50:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:37 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3974192034' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:37 smithi191 ceph-mon[161669]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T05:50:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:37 smithi191 ceph-mon[161669]: failed to encode map e332 with expected crc 2023-10-09T05:50:37.513 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:37.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:37.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:37.619 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:37.679 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:37.845 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:37.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:37.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:37.951 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:38.008 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:38.066 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:38.104 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:38.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:38.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:38.240 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:38.256 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:50:38.256 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:50:38.256 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:50:38.256 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:50:38.256 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:50:38.282 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:38 smithi008 ceph-mon[184418]: pgmap v997: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 28098/241602 objects degraded (11.630%); 3000/241602 objects misplaced (1.242%); 9.5 MiB/s, 151 objects/s recovering 2023-10-09T05:50:38.283 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:38 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:50:38.283 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 28098/241602 objects degraded (11.630%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:38.283 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:38 smithi008 ceph-mon[191227]: pgmap v997: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 28098/241602 objects degraded (11.630%); 3000/241602 objects misplaced (1.242%); 9.5 MiB/s, 151 objects/s recovering 2023-10-09T05:50:38.283 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:38 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:50:38.283 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 28098/241602 objects degraded (11.630%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:38.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:38 smithi191 ceph-mon[161669]: pgmap v997: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 28098/241602 objects degraded (11.630%); 3000/241602 objects misplaced (1.242%); 9.5 MiB/s, 151 objects/s recovering 2023-10-09T05:50:38.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:38 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:50:38.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 28098/241602 objects degraded (11.630%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:38.489 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:38.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:38.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:38.593 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:38.655 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:39.761 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:50:39.781 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:39.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:39.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:39.887 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:39.915 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:40.083 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:50:40.199 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:40 smithi008 ceph-mon[184418]: pgmap v998: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 27671/240072 objects degraded (11.526%); 2954/240072 objects misplaced (1.230%); 11 MiB/s, 169 objects/s recovering 2023-10-09T05:50:40.199 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:40 smithi008 ceph-mon[191227]: pgmap v998: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 27671/240072 objects degraded (11.526%); 2954/240072 objects misplaced (1.230%); 11 MiB/s, 169 objects/s recovering 2023-10-09T05:50:40.238 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:40.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:40.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:40.345 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:40.374 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:40.430 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:40.432 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:40 smithi191 ceph-mon[161669]: pgmap v998: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 27671/240072 objects degraded (11.526%); 2954/240072 objects misplaced (1.230%); 11 MiB/s, 169 objects/s recovering 2023-10-09T05:50:40.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:40.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:40.533 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:40.568 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:40.621 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:40.624 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:40.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:40.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:40.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:40.759 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:40.836 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:40.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:40.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:40.866 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:50:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:50:40] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.33.4" 2023-10-09T05:50:40.940 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:40.973 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:41.042 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:41.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:41.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:41.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:41.176 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:41.223 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:50:41.259 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:50:41.289 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:41.292 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:50:41.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:41.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:41.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:41 smithi191 ceph-mon[161669]: pgmap v999: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 27374/234528 objects degraded (11.672%); 2883/234528 objects misplaced (1.229%); 11 MiB/s, 172 objects/s recovering 2023-10-09T05:50:41.334 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:50:41.358 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:50:41.358 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "" 2023-10-09T05:50:41.358 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:50:41.360 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:50:41.385 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:50:41.396 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:41.407 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:50:41.422 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:41 smithi008 ceph-mon[184418]: pgmap v999: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 27374/234528 objects degraded (11.672%); 2883/234528 objects misplaced (1.229%); 11 MiB/s, 172 objects/s recovering 2023-10-09T05:50:41.422 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:41 smithi008 ceph-mon[191227]: pgmap v999: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 27374/234528 objects degraded (11.672%); 2883/234528 objects misplaced (1.229%); 11 MiB/s, 172 objects/s recovering 2023-10-09T05:50:41.426 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:41.426 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:50:41.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:50:41.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:50:41.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:50:41.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:50:41.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:50:41.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:50:41.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:50:41.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:50:41.884 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:41.896 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:41.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:41.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:42.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:42.047 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:42.134 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:42.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:42.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:42 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:50:42.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/88568304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:42.239 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:42.268 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:42.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:42 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:50:42.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/88568304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:42.317 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:42.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:42.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:42.424 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:42.454 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:42.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:42 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:50:42.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/88568304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:42.499 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:42.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:42.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:42.605 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:42.659 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:43.135 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:43.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:43.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:43.244 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:43.282 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:43.346 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:43.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:43.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:43.399 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:43 smithi008 ceph-mon[191227]: pgmap v1000: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 26929/232848 objects degraded (11.565%); 2774/232848 objects misplaced (1.191%); 11 MiB/s, 175 objects/s recovering 2023-10-09T05:50:43.400 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 27374/234528 objects degraded (11.672%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:43.401 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:43 smithi008 ceph-mon[184418]: pgmap v1000: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 26929/232848 objects degraded (11.565%); 2774/232848 objects misplaced (1.191%); 11 MiB/s, 175 objects/s recovering 2023-10-09T05:50:43.401 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 27374/234528 objects degraded (11.672%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:43.453 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:43.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:43 smithi191 ceph-mon[161669]: pgmap v1000: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 26929/232848 objects degraded (11.565%); 2774/232848 objects misplaced (1.191%); 11 MiB/s, 175 objects/s recovering 2023-10-09T05:50:43.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 27374/234528 objects degraded (11.672%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:43.484 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:43.548 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:43.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:43.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:43.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:43.680 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:43.729 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:43.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:43.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:43.835 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:43.871 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:43.924 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:43.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:43.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:44.029 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:44.065 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:44.340 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:44.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:44.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:44.447 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:44.491 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:45.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:45.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:45.124 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:45.124 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:45.145 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:45.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:45.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:45.255 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:45.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:45.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:45.336 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:45.401 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:50:45.516 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:45.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:45.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:45.623 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:45.652 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:45.704 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:45.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:45.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:45.808 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:50:45.808 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:50:45.810 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:45.841 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:45.893 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:45.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:45.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:45.943 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:50:45.999 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:46.028 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:46.075 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:46.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:46.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:46.180 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:46.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:45 smithi008 ceph-mon[184418]: pgmap v1001: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 26828/230736 objects degraded (11.627%); 2774/230736 objects misplaced (1.202%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:50:46.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1030136261' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:46.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:45 smithi008 ceph-mon[191227]: pgmap v1001: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 26828/230736 objects degraded (11.627%); 2774/230736 objects misplaced (1.202%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:50:46.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1030136261' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:46.214 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:46.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:45 smithi191 ceph-mon[161669]: pgmap v1001: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 26828/230736 objects degraded (11.627%); 2774/230736 objects misplaced (1.202%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:50:46.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1030136261' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:47.064 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:47.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:47.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:47.169 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:47.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:47.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[184418]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1030136261' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:50:47.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[184418]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T05:50:47.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[184418]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[184418]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[184418]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.210 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:47.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[191227]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1030136261' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:50:47.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[191227]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T05:50:47.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[191227]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[191227]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:46 smithi008 ceph-mon[191227]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.211 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:50:47.211 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:50:47.211 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:50:47.211 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2023-10-09T05:50:47.212 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T05:50:47.213 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:47.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:46 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:46 smithi191 ceph-mon[161669]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1030136261' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:50:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:46 smithi191 ceph-mon[161669]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T05:50:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:46 smithi191 ceph-mon[161669]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:46 smithi191 ceph-mon[161669]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:46 smithi191 ceph-mon[161669]: failed to encode map e333 with expected crc 2023-10-09T05:50:47.431 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:50:47.684 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:47.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:47.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:47.792 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:47.832 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:50:47.878 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:47.903 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:47.919 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:47.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:47 smithi008 ceph-mon[184418]: pgmap v1003: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 26651/227400 objects degraded (11.720%); 2709/227400 objects misplaced (1.191%); 11 MiB/s, 173 objects/s recovering 2023-10-09T05:50:47.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:47 smithi008 ceph-mon[191227]: pgmap v1003: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 26651/227400 objects degraded (11.720%); 2709/227400 objects misplaced (1.191%); 11 MiB/s, 173 objects/s recovering 2023-10-09T05:50:48.004 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:48.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:48.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:48.121 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:48.159 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:48.227 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:48.227 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:47 smithi191 ceph-mon[161669]: pgmap v1003: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 26651/227400 objects degraded (11.720%); 2709/227400 objects misplaced (1.191%); 11 MiB/s, 173 objects/s recovering 2023-10-09T05:50:48.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:48.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:48.329 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:48.368 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:48.477 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:48.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:48.582 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:48.612 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:48.665 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:48.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:48.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:48.769 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:48.797 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:48.850 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:48.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:48.955 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:48.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 26651/227400 objects degraded (11.720%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:49.013 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:49.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 26651/227400 objects degraded (11.720%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:49.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 26651/227400 objects degraded (11.720%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:49.694 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:50:50.027 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:50.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:50.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:50.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:50.169 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:50.201 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:50:50.202 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:50:50.213 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:50.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:50.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:50.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:50 smithi191 ceph-mon[161669]: pgmap v1004: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 26189/225639 objects degraded (11.607%); 2594/225639 objects misplaced (1.150%); 12 MiB/s, 184 objects/s recovering 2023-10-09T05:50:50.317 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:50.342 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:50.370 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:50.424 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:50.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:50.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:50 smithi008 ceph-mon[184418]: pgmap v1004: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 26189/225639 objects degraded (11.607%); 2594/225639 objects misplaced (1.150%); 12 MiB/s, 184 objects/s recovering 2023-10-09T05:50:50.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:50 smithi008 ceph-mon[191227]: pgmap v1004: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 26189/225639 objects degraded (11.607%); 2594/225639 objects misplaced (1.150%); 12 MiB/s, 184 objects/s recovering 2023-10-09T05:50:50.528 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:50.590 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:50.722 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:50.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:50.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:50.831 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:50.861 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:50.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:50:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:50:50] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.33.4" 2023-10-09T05:50:51.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:50:51.009 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:51.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:51.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:51.114 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:51.143 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:51.194 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:51.197 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:50:51.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:51.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:51.230 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:50:51.271 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:50:51.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:51.317 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:50:51.338 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:51.347 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:50:51.381 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:50:51.402 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:51.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:51.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:51.434 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:50:51.455 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:50:51.473 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:50:51.481 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:50:51.482 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:50:51.482 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:50:51.482 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:50:51.482 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:50:51.482 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:50:51.482 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:50:51.482 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:50:51.507 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:51.579 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:52.060 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T05:50:52.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:52 smithi191 ceph-mon[161669]: pgmap v1005: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 25885/219993 objects degraded (11.766%); 2535/219993 objects misplaced (1.152%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:50:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/223926274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/49945986' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:50:52.372 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:52 smithi008 ceph-mon[184418]: pgmap v1005: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 25885/219993 objects degraded (11.766%); 2535/219993 objects misplaced (1.152%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:50:52.372 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/223926274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:52.372 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/49945986' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:50:52.372 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:52 smithi008 ceph-mon[191227]: pgmap v1005: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 25885/219993 objects degraded (11.766%); 2535/219993 objects misplaced (1.152%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:50:52.373 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/223926274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:52.373 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/49945986' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:50:52.835 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:52.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:52.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:52.939 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:52.984 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:53.139 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:50:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:50:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:50:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:50:53.140 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:50:53.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:53 smithi008 ceph-mon[184418]: failed to encode map e334 with expected crc 2023-10-09T05:50:53.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:53 smithi008 ceph-mon[184418]: failed to encode map e334 with expected crc 2023-10-09T05:50:53.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/49945986' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:50:53.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:53 smithi008 ceph-mon[184418]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T05:50:53.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 25885/219993 objects degraded (11.766%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:53 smithi008 ceph-mon[191227]: failed to encode map e334 with expected crc 2023-10-09T05:50:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:53 smithi008 ceph-mon[191227]: failed to encode map e334 with expected crc 2023-10-09T05:50:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/49945986' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:50:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:53 smithi008 ceph-mon[191227]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T05:50:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 25885/219993 objects degraded (11.766%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:53.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:53 smithi191 ceph-mon[161669]: failed to encode map e334 with expected crc 2023-10-09T05:50:53.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:53 smithi191 ceph-mon[161669]: failed to encode map e334 with expected crc 2023-10-09T05:50:53.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/49945986' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:50:53.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:53 smithi191 ceph-mon[161669]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T05:50:53.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 25885/219993 objects degraded (11.766%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:53.541 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:53.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:53.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:53.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:53.789 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:53.864 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:53.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:53.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:53.924 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:50:53.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:54.006 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:54.068 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:54.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:54.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:54.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:54.220 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:54.281 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:54.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:54.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:54.385 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:50:54.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:54.420 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:54 smithi191 ceph-mon[161669]: failed to encode map e334 with expected crc 2023-10-09T05:50:54.420 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:54 smithi191 ceph-mon[161669]: pgmap v1007: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 25452/218379 objects degraded (11.655%); 2427/218379 objects misplaced (1.111%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:50:54.424 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:54.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:54 smithi008 ceph-mon[184418]: failed to encode map e334 with expected crc 2023-10-09T05:50:54.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:54 smithi008 ceph-mon[184418]: pgmap v1007: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 25452/218379 objects degraded (11.655%); 2427/218379 objects misplaced (1.111%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:50:54.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:54 smithi008 ceph-mon[191227]: failed to encode map e334 with expected crc 2023-10-09T05:50:54.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:54 smithi008 ceph-mon[191227]: pgmap v1007: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 25452/218379 objects degraded (11.655%); 2427/218379 objects misplaced (1.111%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:50:54.545 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:54.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:54.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:54.651 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:54.714 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:54.791 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:54.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:54.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:54.896 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:54.931 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:54.998 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:55.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:55.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:55.089 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:50:55.107 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:55.136 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:55.288 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:55.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:55 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1880557597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:55.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:55 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:55.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:55.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:55.297 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:55 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1880557597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:55.297 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:55 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:55.298 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:55 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1880557597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:55.298 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:55 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:55.391 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:55.433 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:55.450 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:50:55.650 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:55.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:55.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:55.757 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:55.784 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:56.089 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:50:56.089 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:50:56.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:50:56.089 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:50:56.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.090 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125677 7.11 7:8932b233:::benchmark_data_smithi008_354649_object31181:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:50:56.090 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:50:56.090 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 413.08839222799998, 2023-10-09T05:50:56.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0454934010000001, 2023-10-09T05:50:56.090 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.090 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:50:56.090 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.090 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:50:56.090 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:50:56.090 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125677 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999967 2023-10-09T05:50:56.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997449+0000", 2023-10-09T05:50:56.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3732e-05 2023-10-09T05:50:56.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997463+0000", 2023-10-09T05:50:56.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9613999999999999e-05 2023-10-09T05:50:56.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997483+0000", 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0620000000000007e-06 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997492+0000", 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.234801074 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232293+0000", 2023-10-09T05:50:56.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2490000000000001e-06 2023-10-09T05:50:56.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:50:56.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232301+0000", 2023-10-09T05:50:56.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81054634999999997 2023-10-09T05:50:56.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042848+0000", 2023-10-09T05:50:56.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8449999999999994e-05 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042946+0000", 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0454934010000001 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125688 7.11 7:8f585b70:::benchmark_data_smithi008_354649_object31192:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:50:56.095 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:50:56.096 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 413.08834465899997, 2023-10-09T05:50:56.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0556018709999999, 2023-10-09T05:50:56.096 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.096 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.096 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.096 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:50:56.096 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:50:56.096 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125688 2023-10-09T05:50:56.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.096 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:50:56.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2820000000000001e-06 2023-10-09T05:50:56.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997503+0000", 2023-10-09T05:50:56.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0820000000000004e-06 2023-10-09T05:50:56.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997509+0000", 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0659999999999997e-06 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997513+0000", 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.234809199 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232322+0000", 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6849999999999998e-06 2023-10-09T05:50:56.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:50:56.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232328+0000", 2023-10-09T05:50:56.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81122250699999998 2023-10-09T05:50:56.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043550+0000", 2023-10-09T05:50:56.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114354 2023-10-09T05:50:56.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043665+0000", 2023-10-09T05:50:56.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00033335100000000001 2023-10-09T05:50:56.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:50:56.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043998+0000", 2023-10-09T05:50:56.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001666088 2023-10-09T05:50:56.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.045664+0000", 2023-10-09T05:50:56.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041383430000000001 2023-10-09T05:50:56.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.049803+0000", 2023-10-09T05:50:56.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032002139999999998 2023-10-09T05:50:56.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053003+0000", 2023-10-09T05:50:56.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7229999999999999e-05 2023-10-09T05:50:56.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053060+0000", 2023-10-09T05:50:56.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1964e-05 2023-10-09T05:50:56.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053102+0000", 2023-10-09T05:50:56.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0556018709999999 2023-10-09T05:50:56.104 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.104 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.104 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.104 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180297 7.21 7:846cd9a9:::benchmark_data_smithi008_354649_object85801:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:50:56.104 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:41.587040+0000", 2023-10-09T05:50:56.104 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 254.49880515800001, 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48169166200000002, 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180297 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587040+0000", 2023-10-09T05:50:56.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587040+0000", 2023-10-09T05:50:56.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:50:56.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587038+0000", 2023-10-09T05:50:56.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6429999999999995e-06 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587046+0000", 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4168999999999999e-05 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587060+0000", 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9569999999999999e-06 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587066+0000", 2023-10-09T05:50:56.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7617000000000003e-05 2023-10-09T05:50:56.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587104+0000", 2023-10-09T05:50:56.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1820000000000003e-06 2023-10-09T05:50:56.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587110+0000", 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47181123800000002 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.058921+0000", 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117278 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.059038+0000", 2023-10-09T05:50:56.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000192509 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.059231+0000", 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001372269 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060603+0000", 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2033000000000003e-05 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060635+0000", 2023-10-09T05:50:56.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079930550000000006 2023-10-09T05:50:56.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.068628+0000", 2023-10-09T05:50:56.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9888e-05 2023-10-09T05:50:56.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.068688+0000", 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3340999999999997e-05 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.068731+0000", 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48169166200000002 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.113 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180300 7.1 7:807d12ef:::benchmark_data_smithi008_354649_object85804:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:50:56.113 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:41.743813+0000", 2023-10-09T05:50:56.113 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 254.34203138699999, 2023-10-09T05:50:56.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.316588969, 2023-10-09T05:50:56.113 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.113 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.113 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.113 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:50:56.113 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:50:56.113 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180300 2023-10-09T05:50:56.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.114 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743813+0000", 2023-10-09T05:50:56.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743813+0000", 2023-10-09T05:50:56.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:50:56.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743812+0000", 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2349999999999999e-06 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743820+0000", 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5903999999999999e-05 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743836+0000", 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0990000000000004e-06 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743842+0000", 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5695000000000001e-05 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743897+0000", 2023-10-09T05:50:56.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4329999999999993e-06 2023-10-09T05:50:56.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:50:56.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743907+0000", 2023-10-09T05:50:56.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29237399200000003 2023-10-09T05:50:56.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.036281+0000", 2023-10-09T05:50:56.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9270000000000004e-05 2023-10-09T05:50:56.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.036370+0000", 2023-10-09T05:50:56.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028330299999999999 2023-10-09T05:50:56.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:50:56.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.036653+0000", 2023-10-09T05:50:56.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020744069999999999 2023-10-09T05:50:56.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.038728+0000", 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024384929999999999 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.041166+0000", 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019156355999999999 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060323+0000", 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2685999999999997e-05 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060375+0000", 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6947000000000001e-05 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060402+0000", 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.316588969 2023-10-09T05:50:56.120 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.121 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.121 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.121 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86697 8.5 8:a67798ab:::benchmark_data_smithi008_432048_object86696:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:50:56.121 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088415+0000", 2023-10-09T05:50:56.121 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 130.99742995599999, 2023-10-09T05:50:56.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.168776316, 2023-10-09T05:50:56.121 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.121 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:50:56.121 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86697 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088415+0000", 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088415+0000", 2023-10-09T05:50:56.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:50:56.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088413+0000", 2023-10-09T05:50:56.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6736999999999995e-05 2023-10-09T05:50:56.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088479+0000", 2023-10-09T05:50:56.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0219000000000001e-05 2023-10-09T05:50:56.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088500+0000", 2023-10-09T05:50:56.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1593000000000001e-05 2023-10-09T05:50:56.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088511+0000", 2023-10-09T05:50:56.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017065599999999999 2023-10-09T05:50:56.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088682+0000", 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1679999999999993e-06 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088691+0000", 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18748368300000001 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276175+0000", 2023-10-09T05:50:56.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1679999999999993e-06 2023-10-09T05:50:56.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:50:56.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276184+0000", 2023-10-09T05:50:56.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98094864800000003 2023-10-09T05:50:56.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.257133+0000", 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8631000000000002e-05 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.257191+0000", 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.168776316 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.127 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86704 8.15 8:af611326:::benchmark_data_smithi008_432048_object86703:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:50:56.128 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088533+0000", 2023-10-09T05:50:56.128 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 130.99731204400001, 2023-10-09T05:50:56.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1725390840000001, 2023-10-09T05:50:56.128 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.128 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:50:56.128 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.128 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.128 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.128 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86704 2023-10-09T05:50:56.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.128 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088533+0000", 2023-10-09T05:50:56.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088533+0000", 2023-10-09T05:50:56.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:50:56.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088532+0000", 2023-10-09T05:50:56.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3013000000000002e-05 2023-10-09T05:50:56.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088585+0000", 2023-10-09T05:50:56.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5350000000000004e-06 2023-10-09T05:50:56.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088594+0000", 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3850000000000001e-06 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088599+0000", 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19036776799999999 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.278967+0000", 2023-10-09T05:50:56.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1251e-05 2023-10-09T05:50:56.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:50:56.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.278978+0000", 2023-10-09T05:50:56.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98199783900000004 2023-10-09T05:50:56.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.260976+0000", 2023-10-09T05:50:56.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5806999999999999e-05 2023-10-09T05:50:56.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261072+0000", 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1725390840000001 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86708 8.15 8:ad569a26:::benchmark_data_smithi008_432048_object86707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:50:56.133 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 130.997239431, 2023-10-09T05:50:56.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1769074829999999, 2023-10-09T05:50:56.134 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.134 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:50:56.134 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.134 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.134 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.134 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86708 2023-10-09T05:50:56.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.134 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:50:56.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:50:56.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:50:56.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6990000000000002e-05 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088652+0000", 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5610000000000004e-06 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088658+0000", 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7790000000000003e-06 2023-10-09T05:50:56.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088663+0000", 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19033387900000001 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.278997+0000", 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0989999999999998e-06 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.279002+0000", 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98210046799999995 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261102+0000", 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094320799999999996 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.262046+0000", 2023-10-09T05:50:56.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003411837 2023-10-09T05:50:56.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265458+0000", 2023-10-09T05:50:56.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5321000000000001e-05 2023-10-09T05:50:56.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265513+0000", 2023-10-09T05:50:56.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1769074829999999 2023-10-09T05:50:56.139 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86714 8.15 8:a990e10e:::benchmark_data_smithi008_432048_object86713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088669+0000", 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 130.99717585900001, 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.177276395, 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.140 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.141 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.141 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86714 2023-10-09T05:50:56.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.141 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088669+0000", 2023-10-09T05:50:56.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088669+0000", 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088668+0000", 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1603999999999997e-05 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088710+0000", 2023-10-09T05:50:56.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8530000000000003e-06 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088717+0000", 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7299999999999996e-06 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088722+0000", 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.190296668 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.279018+0000", 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.271e-06 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.279024+0000", 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98307463399999995 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.262098+0000", 2023-10-09T05:50:56.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00063494799999999998 2023-10-09T05:50:56.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:50:56.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.262733+0000", 2023-10-09T05:50:56.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031703159999999998 2023-10-09T05:50:56.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265903+0000", 2023-10-09T05:50:56.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1888999999999998e-05 2023-10-09T05:50:56.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265945+0000", 2023-10-09T05:50:56.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.177276395 2023-10-09T05:50:56.146 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.146 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.146 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.146 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86722 8.12 8:4898e42c:::benchmark_data_smithi008_432048_object86721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-10-09T05:50:56.146 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.349501+0000", 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 130.73634405600001, 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91716482799999999, 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86722 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349501+0000", 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349501+0000", 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999971 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349498+0000", 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0645999999999997e-05 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349539+0000", 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.594e-05 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349554+0000", 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2979999999999997e-06 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349564+0000", 2023-10-09T05:50:56.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001176355 2023-10-09T05:50:56.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.350740+0000", 2023-10-09T05:50:56.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0907000000000001e-05 2023-10-09T05:50:56.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:50:56.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.350751+0000", 2023-10-09T05:50:56.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91074052900000002 2023-10-09T05:50:56.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261492+0000", 2023-10-09T05:50:56.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000151316 2023-10-09T05:50:56.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261643+0000", 2023-10-09T05:50:56.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021199000000000001 2023-10-09T05:50:56.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261855+0000", 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046603080000000002 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266515+0000", 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.63e-06 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266522+0000", 2023-10-09T05:50:56.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0194e-05 2023-10-09T05:50:56.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266582+0000", 2023-10-09T05:50:56.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8968999999999998e-05 2023-10-09T05:50:56.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266631+0000", 2023-10-09T05:50:56.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4625000000000001e-05 2023-10-09T05:50:56.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266666+0000", 2023-10-09T05:50:56.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91716482799999999 2023-10-09T05:50:56.154 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.154 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.154 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88187 8.b 8:d5bc5a72:::benchmark_data_smithi008_432048_object88186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.469959+0000", 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 128.61588584200001, 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30218297900000002, 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88187 2023-10-09T05:50:56.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.469959+0000", 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.469959+0000", 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.469957+0000", 2023-10-09T05:50:56.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9060000000000002e-05 2023-10-09T05:50:56.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.469997+0000", 2023-10-09T05:50:56.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2267e-05 2023-10-09T05:50:56.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470009+0000", 2023-10-09T05:50:56.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6200000000000004e-06 2023-10-09T05:50:56.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470014+0000", 2023-10-09T05:50:56.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1153000000000003e-05 2023-10-09T05:50:56.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470066+0000", 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000135251 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470201+0000", 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017363400000000001 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470374+0000", 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30140982500000002 2023-10-09T05:50:56.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.771784+0000", 2023-10-09T05:50:56.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9709e-05 2023-10-09T05:50:56.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.771844+0000", 2023-10-09T05:50:56.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020167500000000001 2023-10-09T05:50:56.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772046+0000", 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8795999999999997e-05 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772104+0000", 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.748e-05 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772142+0000", 2023-10-09T05:50:56.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30218297900000002 2023-10-09T05:50:56.162 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.162 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.162 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.162 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88189 8.f 8:f2efcde2:::benchmark_data_smithi008_432048_object88188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:50:56.162 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.470029+0000", 2023-10-09T05:50:56.162 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 128.615815312, 2023-10-09T05:50:56.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30080939000000001, 2023-10-09T05:50:56.162 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88189 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470029+0000", 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470029+0000", 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470029+0000", 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0537000000000002e-05 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470069+0000", 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.289e-06 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470077+0000", 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0610000000000003e-06 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470082+0000", 2023-10-09T05:50:56.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0009999999999997e-05 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470132+0000", 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012962999999999999 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470261+0000", 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000156281 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470418+0000", 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003489214 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473907+0000", 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29485467900000001 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768762+0000", 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019996879999999999 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770761+0000", 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3291000000000003e-05 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770805+0000", 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4267999999999998e-05 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770839+0000", 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30080939000000001 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88193 8.0 8:01ab35b2:::benchmark_data_smithi008_432048_object88192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.470097+0000", 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 128.615747586, 2023-10-09T05:50:56.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.300629545, 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88193 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470097+0000", 2023-10-09T05:50:56.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470097+0000", 2023-10-09T05:50:56.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:50:56.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470097+0000", 2023-10-09T05:50:56.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0299000000000002e-05 2023-10-09T05:50:56.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470137+0000", 2023-10-09T05:50:56.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3930000000000002e-06 2023-10-09T05:50:56.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470144+0000", 2023-10-09T05:50:56.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5410000000000002e-06 2023-10-09T05:50:56.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470149+0000", 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6909000000000001e-05 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470196+0000", 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112024 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470308+0000", 2023-10-09T05:50:56.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011089799999999999 2023-10-09T05:50:56.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:50:56.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470419+0000", 2023-10-09T05:50:56.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015793319999999999 2023-10-09T05:50:56.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.471998+0000", 2023-10-09T05:50:56.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29678586000000001 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768784+0000", 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001849271 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770633+0000", 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1421000000000001e-05 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770685+0000", 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2166000000000003e-05 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770727+0000", 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.300629545 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88195 8.b 8:d594efea:::benchmark_data_smithi008_432048_object88194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.470165+0000", 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 128.615679333, 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.302059788, 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88195 2023-10-09T05:50:56.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.178 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470165+0000", 2023-10-09T05:50:56.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470165+0000", 2023-10-09T05:50:56.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:50:56.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470165+0000", 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6063999999999999e-05 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470201+0000", 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9789999999999996e-06 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470208+0000", 2023-10-09T05:50:56.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7940000000000002e-06 2023-10-09T05:50:56.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470213+0000", 2023-10-09T05:50:56.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030594849999999998 2023-10-09T05:50:56.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473272+0000", 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000137114 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473409+0000", 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018650599999999999 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473596+0000", 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29822002399999997 2023-10-09T05:50:56.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.771816+0000", 2023-10-09T05:50:56.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014296999999999999 2023-10-09T05:50:56.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.771959+0000", 2023-10-09T05:50:56.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019265600000000001 2023-10-09T05:50:56.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772151+0000", 2023-10-09T05:50:56.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9477999999999997e-05 2023-10-09T05:50:56.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772191+0000", 2023-10-09T05:50:56.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4397000000000003e-05 2023-10-09T05:50:56.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772225+0000", 2023-10-09T05:50:56.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.302059788 2023-10-09T05:50:56.184 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.184 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.184 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.184 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88196 8.15 8:ae709537:::benchmark_data_smithi008_432048_object88195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:50:56.184 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.470222+0000", 2023-10-09T05:50:56.184 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 128.615623263, 2023-10-09T05:50:56.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30074758899999998, 2023-10-09T05:50:56.185 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.185 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.185 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.185 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.185 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.185 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88196 2023-10-09T05:50:56.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.185 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470222+0000", 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470222+0000", 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470221+0000", 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4220000000000001e-05 2023-10-09T05:50:56.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470255+0000", 2023-10-09T05:50:56.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.585e-06 2023-10-09T05:50:56.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470264+0000", 2023-10-09T05:50:56.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8439999999999997e-06 2023-10-09T05:50:56.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470269+0000", 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2599000000000002e-05 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470321+0000", 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011573700000000001 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470437+0000", 2023-10-09T05:50:56.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119529 2023-10-09T05:50:56.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:50:56.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470556+0000", 2023-10-09T05:50:56.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001557498 2023-10-09T05:50:56.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.472114+0000", 2023-10-09T05:50:56.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29872052100000002 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770834+0000", 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7939000000000002e-05 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770892+0000", 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.159e-05 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770934+0000", 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5179000000000003e-05 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770969+0000", 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30074758899999998 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88197 8.15 8:adcf56d2:::benchmark_data_smithi008_432048_object88196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.473499+0000", 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 128.61234532399999, 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29885566000000002, 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88197 2023-10-09T05:50:56.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473499+0000", 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473499+0000", 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473498+0000", 2023-10-09T05:50:56.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1958e-05 2023-10-09T05:50:56.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473540+0000", 2023-10-09T05:50:56.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1843e-05 2023-10-09T05:50:56.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473552+0000", 2023-10-09T05:50:56.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3709999999999996e-06 2023-10-09T05:50:56.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473558+0000", 2023-10-09T05:50:56.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00027575100000000002 2023-10-09T05:50:56.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473834+0000", 2023-10-09T05:50:56.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105849 2023-10-09T05:50:56.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473940+0000", 2023-10-09T05:50:56.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014184199999999999 2023-10-09T05:50:56.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:50:56.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.474081+0000", 2023-10-09T05:50:56.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29451698100000001 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768598+0000", 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022665799999999998 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770865+0000", 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014092810000000001 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772274+0000", 2023-10-09T05:50:56.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5215e-05 2023-10-09T05:50:56.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772320+0000", 2023-10-09T05:50:56.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5620000000000001e-05 2023-10-09T05:50:56.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772355+0000", 2023-10-09T05:50:56.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29885566000000002 2023-10-09T05:50:56.199 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.199 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.199 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.199 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88199 8.b 8:d46d9522:::benchmark_data_smithi008_432048_object88198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:50:56.199 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.473580+0000", 2023-10-09T05:50:56.199 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 128.61226509599999, 2023-10-09T05:50:56.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30265360200000002, 2023-10-09T05:50:56.200 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.200 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.200 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.200 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.200 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.200 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88199 2023-10-09T05:50:56.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.200 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473580+0000", 2023-10-09T05:50:56.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473580+0000", 2023-10-09T05:50:56.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:50:56.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473579+0000", 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7914999999999999e-05 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473617+0000", 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3939999999999998e-06 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473624+0000", 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3409999999999996e-06 2023-10-09T05:50:56.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473629+0000", 2023-10-09T05:50:56.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030601980000000001 2023-10-09T05:50:56.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.476689+0000", 2023-10-09T05:50:56.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019625999999999999 2023-10-09T05:50:56.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.476885+0000", 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29183931099999999 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768725+0000", 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038368299999999998 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772561+0000", 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024502909999999998 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.775012+0000", 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001103314 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.776115+0000", 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0850000000000002e-05 2023-10-09T05:50:56.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.776176+0000", 2023-10-09T05:50:56.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7447000000000001e-05 2023-10-09T05:50:56.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.776233+0000", 2023-10-09T05:50:56.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30265360200000002 2023-10-09T05:50:56.206 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.206 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.207 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.207 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88200 8.0 8:0164bb44:::benchmark_data_smithi008_432048_object88199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:50:56.207 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.473639+0000", 2023-10-09T05:50:56.207 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 128.61220563699999, 2023-10-09T05:50:56.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29720032499999999, 2023-10-09T05:50:56.207 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.207 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.207 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.207 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88200 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473639+0000", 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473639+0000", 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473639+0000", 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2280000000000003e-05 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473671+0000", 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5320000000000003e-06 2023-10-09T05:50:56.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473677+0000", 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0439999999999997e-06 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473682+0000", 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5552000000000001e-05 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473717+0000", 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000121168 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473838+0000", 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000156129 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473994+0000", 2023-10-09T05:50:56.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29474761999999999 2023-10-09T05:50:56.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768742+0000", 2023-10-09T05:50:56.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5961e-05 2023-10-09T05:50:56.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768818+0000", 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001946845 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770765+0000", 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2567000000000001e-05 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770807+0000", 2023-10-09T05:50:56.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2126999999999999e-05 2023-10-09T05:50:56.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770839+0000", 2023-10-09T05:50:56.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29720032499999999 2023-10-09T05:50:56.215 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.215 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.215 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.215 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.45782.0:88187 8.b e300/294 ondisk, result = 0)", 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.476620+0000", 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 128.60922499099999, 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29516727199999998, 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.476620+0000", 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.476620+0000", 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.476619+0000", 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1659999999999999e-06 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.476625+0000", 2023-10-09T05:50:56.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2197e-05 2023-10-09T05:50:56.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.476637+0000", 2023-10-09T05:50:56.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7279999999999998e-06 2023-10-09T05:50:56.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.476644+0000", 2023-10-09T05:50:56.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29512570300000002 2023-10-09T05:50:56.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.771770+0000", 2023-10-09T05:50:56.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4749999999999996e-06 2023-10-09T05:50:56.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.771777+0000", 2023-10-09T05:50:56.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9389999999999992e-06 2023-10-09T05:50:56.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.771787+0000", 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29516727199999998 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88202 8.7 8:e091e4b1:::benchmark_data_smithi008_432048_object88201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.478777+0000", 2023-10-09T05:50:56.220 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 128.607068295, 2023-10-09T05:50:56.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.297133535, 2023-10-09T05:50:56.221 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.221 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.221 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.221 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.221 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.221 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88202 2023-10-09T05:50:56.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.221 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.478777+0000", 2023-10-09T05:50:56.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.478777+0000", 2023-10-09T05:50:56.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:50:56.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.478775+0000", 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0946999999999999e-05 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.478826+0000", 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28985588000000001 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768682+0000", 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0759e-05 2023-10-09T05:50:56.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768692+0000", 2023-10-09T05:50:56.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000104724 2023-10-09T05:50:56.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768797+0000", 2023-10-09T05:50:56.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014896399999999999 2023-10-09T05:50:56.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768946+0000", 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016876700000000001 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.769115+0000", 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039946770000000003 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.773109+0000", 2023-10-09T05:50:56.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2729999999999999e-06 2023-10-09T05:50:56.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.773116+0000", 2023-10-09T05:50:56.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002712309 2023-10-09T05:50:56.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.775828+0000", 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6659999999999997e-05 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.775875+0000", 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5407000000000002e-05 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.775910+0000", 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.297133535 2023-10-09T05:50:56.227 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:90582 8.15 8:aa5a2eaf:::benchmark_data_smithi008_432048_object90581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:49.810251+0000", 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 126.275594138, 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29703675200000001, 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:50:56.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:50:56.229 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:50:56.229 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 90582 2023-10-09T05:50:56.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.229 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:50:56.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:50:56.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:49.810251+0000", 2023-10-09T05:50:56.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:50:56.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:50:56.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:49.810251+0000", 2023-10-09T05:50:56.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:50:56.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:50:56.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:49.810249+0000", 2023-10-09T05:50:56.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7704000000000003e-05 2023-10-09T05:50:56.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:50:56.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:49.810297+0000", 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4490000000000005e-06 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:49.810306+0000", 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0150000000000003e-06 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:49.810311+0000", 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001983748 2023-10-09T05:50:56.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:50:56.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:49.812295+0000", 2023-10-09T05:50:56.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27913863500000002 2023-10-09T05:50:56.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:50:56.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.091434+0000", 2023-10-09T05:50:56.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018252399999999999 2023-10-09T05:50:56.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.091616+0000", 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038500299999999999 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.095466+0000", 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045166939999999999 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.099983+0000", 2023-10-09T05:50:56.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071992200000000001 2023-10-09T05:50:56.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:50:56.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.107182+0000", 2023-10-09T05:50:56.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9515999999999997e-05 2023-10-09T05:50:56.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:50:56.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.107242+0000", 2023-10-09T05:50:56.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5623e-05 2023-10-09T05:50:56.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:50:56.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:50:56.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:50:56.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:50.107287+0000", 2023-10-09T05:50:56.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29703675200000001 2023-10-09T05:50:56.235 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.235 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.235 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.235 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:50:56.236 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:50:56.236 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:50:56.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:56.237 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:56.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:56.328 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[184418]: pgmap v1008: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 25326/216207 objects degraded (11.714%); 2427/216207 objects misplaced (1.123%); 12 MiB/s, 188 objects/s recovering 2023-10-09T05:50:56.328 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:56.328 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[184418]: failed to encode map e335 with expected crc 2023-10-09T05:50:56.328 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:56.328 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[184418]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T05:50:56.329 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[184418]: failed to encode map e335 with expected crc 2023-10-09T05:50:56.329 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[184418]: failed to encode map e335 with expected crc 2023-10-09T05:50:56.329 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[191227]: pgmap v1008: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 25326/216207 objects degraded (11.714%); 2427/216207 objects misplaced (1.123%); 12 MiB/s, 188 objects/s recovering 2023-10-09T05:50:56.329 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:56.329 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[191227]: failed to encode map e335 with expected crc 2023-10-09T05:50:56.329 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:56.329 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[191227]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T05:50:56.329 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[191227]: failed to encode map e335 with expected crc 2023-10-09T05:50:56.330 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:56 smithi008 ceph-mon[191227]: failed to encode map e335 with expected crc 2023-10-09T05:50:56.338 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:56.386 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:56.421 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:56.441 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:56.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:56.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:56 smithi191 ceph-mon[161669]: pgmap v1008: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 25326/216207 objects degraded (11.714%); 2427/216207 objects misplaced (1.123%); 12 MiB/s, 188 objects/s recovering 2023-10-09T05:50:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:56 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:56 smithi191 ceph-mon[161669]: failed to encode map e335 with expected crc 2023-10-09T05:50:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:56 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:56 smithi191 ceph-mon[161669]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T05:50:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:56 smithi191 ceph-mon[161669]: failed to encode map e335 with expected crc 2023-10-09T05:50:56.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:56 smithi191 ceph-mon[161669]: failed to encode map e335 with expected crc 2023-10-09T05:50:56.551 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:56.586 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:56.818 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:56.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:56.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:56.876 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:50:56.922 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:56.956 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:57.037 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:57.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:57.147 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:57.184 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:57 smithi191 ceph-mon[161669]: failed to encode map e335 with expected crc 2023-10-09T05:50:57.184 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:57 smithi191 ceph-mon[161669]: failed to encode map e335 with expected crc 2023-10-09T05:50:57.184 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:57 smithi191 ceph-mon[161669]: failed to encode map e335 with expected crc 2023-10-09T05:50:57.189 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:57.246 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:57.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:57.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:57.327 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:50:57.350 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:57.381 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:57 smithi008 ceph-mon[184418]: failed to encode map e335 with expected crc 2023-10-09T05:50:57.381 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:57 smithi008 ceph-mon[184418]: failed to encode map e335 with expected crc 2023-10-09T05:50:57.382 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:57 smithi008 ceph-mon[184418]: failed to encode map e335 with expected crc 2023-10-09T05:50:57.382 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:57 smithi008 ceph-mon[191227]: failed to encode map e335 with expected crc 2023-10-09T05:50:57.382 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:57 smithi008 ceph-mon[191227]: failed to encode map e335 with expected crc 2023-10-09T05:50:57.382 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:57 smithi008 ceph-mon[191227]: failed to encode map e335 with expected crc 2023-10-09T05:50:57.384 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:57.428 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:57.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:57.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:57.532 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:57.560 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:57.615 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:57.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:57.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:57.722 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:57.754 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:58.020 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:58.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:58.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:58.128 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:58.171 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:58.263 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:58 smithi008 ceph-mon[184418]: pgmap v1010: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 25144/212952 objects degraded (11.807%); 2359/212952 objects misplaced (1.108%); 10 MiB/s, 159 objects/s recovering 2023-10-09T05:50:58.263 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 25144/212952 objects degraded (11.807%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:58.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:58 smithi008 ceph-mon[191227]: pgmap v1010: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 25144/212952 objects degraded (11.807%); 2359/212952 objects misplaced (1.108%); 10 MiB/s, 159 objects/s recovering 2023-10-09T05:50:58.263 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 25144/212952 objects degraded (11.807%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:58.296 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:58.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:58.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:58 smithi191 ceph-mon[161669]: pgmap v1010: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 25144/212952 objects degraded (11.807%); 2359/212952 objects misplaced (1.108%); 10 MiB/s, 159 objects/s recovering 2023-10-09T05:50:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 25144/212952 objects degraded (11.807%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:50:58.356 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:50:58.401 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:58.434 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:58.558 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:50:58.584 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:50:58.615 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:50:58.640 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:50:58.640 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T05:50:58.650 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:50:58.666 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:58.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:58.684 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:50:58.725 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:50:58.765 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:50:58.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:58.804 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:58.821 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:50:58.850 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:50:58.850 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:50:58.850 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:50:58.850 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:50:58.851 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:50:58.851 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:50:58.851 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:50:58.851 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:50:58.916 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:58.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:58.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:59.020 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:59.057 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:59.126 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:59.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:59.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:59.200 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:50:59.200 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:50:59.232 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:59.274 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:59.338 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2480115637' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:59.338 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:59.338 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:50:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:59.376 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:59.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:59.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2480115637' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:50:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:59.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2480115637' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:59.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:50:59.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:50:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:59.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:59.511 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:59.566 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:59.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:59.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:59.671 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:59.702 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:59.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:50:59.890 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:59.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:59.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:59.997 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:00.038 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:00.048 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:00.100 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:00.126 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:00.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:00.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:00 smithi008 ceph-mon[184418]: pgmap v1011: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 24708/211305 objects degraded (11.693%); 2182/211305 objects misplaced (1.033%); 12 MiB/s, 191 objects/s recovering 2023-10-09T05:51:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:00 smithi008 ceph-mon[191227]: pgmap v1011: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 24708/211305 objects degraded (11.693%); 2182/211305 objects misplaced (1.033%); 12 MiB/s, 191 objects/s recovering 2023-10-09T05:51:00.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:00.285 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:00.331 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:51:00.354 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:00.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:00.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:00.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:00 smithi191 ceph-mon[161669]: pgmap v1011: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 24708/211305 objects degraded (11.693%); 2182/211305 objects misplaced (1.033%); 12 MiB/s, 191 objects/s recovering 2023-10-09T05:51:00.462 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:00.514 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:00.661 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:51:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:51:00] "GET /metrics HTTP/1.1" 200 36611 "" "Prometheus/2.33.4" 2023-10-09T05:51:00.688 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:00.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:00.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:00.793 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:00.852 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:00.993 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:00.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:01.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:01.100 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:01.129 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:01.939 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:51:01.939 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:51:01.962 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:51:02.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:02 smithi008 ceph-mon[184418]: pgmap v1012: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 24471/206487 objects degraded (11.851%); 2118/206487 objects misplaced (1.026%); 12 MiB/s, 186 objects/s recovering 2023-10-09T05:51:02.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:02 smithi008 ceph-mon[191227]: pgmap v1012: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 24471/206487 objects degraded (11.851%); 2118/206487 objects misplaced (1.026%); 12 MiB/s, 186 objects/s recovering 2023-10-09T05:51:02.397 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:02.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:02.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:02.503 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:02 smithi191 ceph-mon[161669]: pgmap v1012: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 24471/206487 objects degraded (11.851%); 2118/206487 objects misplaced (1.026%); 12 MiB/s, 186 objects/s recovering 2023-10-09T05:51:02.582 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:02.722 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:51:02.794 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:02.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:02.900 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:02.958 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:03.150 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:03.217 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:51:03.276 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:03.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:03.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:03.386 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:03.455 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24471/206487 objects degraded (11.851%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:03.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24471/206487 objects degraded (11.851%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:03.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24471/206487 objects degraded (11.851%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:03.492 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:03.693 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:03.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:03.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:03.809 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:03.864 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:04.126 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:04 smithi191 ceph-mon[161669]: pgmap v1013: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 24063/204909 objects degraded (11.743%); 1987/204909 objects misplaced (0.970%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:51:04.127 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:04 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:04.127 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4133856887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:51:04.127 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:04 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:51:04.127 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2389156578' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:04.127 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:04 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:04.132 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T05:51:04.141 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:51:04.169 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[184418]: pgmap v1013: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 24063/204909 objects degraded (11.743%); 1987/204909 objects misplaced (0.970%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:51:04.169 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:04.169 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4133856887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:51:04.169 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:51:04.169 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2389156578' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:04.169 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:04.170 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[191227]: pgmap v1013: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 24063/204909 objects degraded (11.743%); 1987/204909 objects misplaced (0.970%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:51:04.170 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:04.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4133856887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:51:04.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:51:04.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2389156578' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:04.171 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:04 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:04.277 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:04.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:04.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:04.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:04.421 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:04.716 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:04.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:04.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:04.822 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:04.876 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:05.132 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:05.132 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:51:05.132 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:51:05.132 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[191227]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T05:51:05.132 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[191227]: failed to encode map e336 with expected crc 2023-10-09T05:51:05.133 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[191227]: failed to encode map e336 with expected crc 2023-10-09T05:51:05.133 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[191227]: failed to encode map e336 with expected crc 2023-10-09T05:51:05.133 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[191227]: failed to encode map e336 with expected crc 2023-10-09T05:51:05.259 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:51:05.259 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:51:05.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:05 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:05.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:05 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:51:05.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:05 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:51:05.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:05 smithi191 ceph-mon[161669]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T05:51:05.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:05 smithi191 ceph-mon[161669]: failed to encode map e336 with expected crc 2023-10-09T05:51:05.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:05 smithi191 ceph-mon[161669]: failed to encode map e336 with expected crc 2023-10-09T05:51:05.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:05 smithi191 ceph-mon[161669]: failed to encode map e336 with expected crc 2023-10-09T05:51:05.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:05 smithi191 ceph-mon[161669]: failed to encode map e336 with expected crc 2023-10-09T05:51:05.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:05.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:51:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:51:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[184418]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T05:51:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[184418]: failed to encode map e336 with expected crc 2023-10-09T05:51:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[184418]: failed to encode map e336 with expected crc 2023-10-09T05:51:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[184418]: failed to encode map e336 with expected crc 2023-10-09T05:51:05.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:05 smithi008 ceph-mon[184418]: failed to encode map e336 with expected crc 2023-10-09T05:51:06.335 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:51:06.381 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T05:51:06.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:06 smithi008 ceph-mon[184418]: pgmap v1015: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 23954/202812 objects degraded (11.811%); 1987/202812 objects misplaced (0.980%); 12 MiB/s, 193 objects/s recovering 2023-10-09T05:51:06.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:06 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:51:06.399 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:06 smithi008 ceph-mon[191227]: pgmap v1015: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 23954/202812 objects degraded (11.811%); 1987/202812 objects misplaced (0.980%); 12 MiB/s, 193 objects/s recovering 2023-10-09T05:51:06.399 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:06 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:51:06.476 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:06.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:06.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:06.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:06 smithi191 ceph-mon[161669]: pgmap v1015: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 23954/202812 objects degraded (11.811%); 1987/202812 objects misplaced (0.980%); 12 MiB/s, 193 objects/s recovering 2023-10-09T05:51:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:06 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:51:06.581 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:06.624 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:06.745 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:06.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:06.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:06.849 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:06.882 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:06.930 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:51:07.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:07.033 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:07.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:07.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:07.142 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:07 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:51:07.143 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:07 smithi191 ceph-mon[161669]: failed to encode map e337 with expected crc 2023-10-09T05:51:07.143 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:07 smithi191 ceph-mon[161669]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T05:51:07.143 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:07 smithi191 ceph-mon[161669]: failed to encode map e337 with expected crc 2023-10-09T05:51:07.143 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:07 smithi191 ceph-mon[161669]: failed to encode map e337 with expected crc 2023-10-09T05:51:07.189 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:07.279 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:07.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:07.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:07.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:07.438 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:07 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:51:07.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:07 smithi008 ceph-mon[191227]: failed to encode map e337 with expected crc 2023-10-09T05:51:07.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:07 smithi008 ceph-mon[191227]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T05:51:07.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:07 smithi008 ceph-mon[191227]: failed to encode map e337 with expected crc 2023-10-09T05:51:07.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:07 smithi008 ceph-mon[191227]: failed to encode map e337 with expected crc 2023-10-09T05:51:07.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:07 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:51:07.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:07 smithi008 ceph-mon[184418]: failed to encode map e337 with expected crc 2023-10-09T05:51:07.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:07 smithi008 ceph-mon[184418]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T05:51:07.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:07 smithi008 ceph-mon[184418]: failed to encode map e337 with expected crc 2023-10-09T05:51:07.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:07 smithi008 ceph-mon[184418]: failed to encode map e337 with expected crc 2023-10-09T05:51:07.467 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:51:07.498 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:51:07.530 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:51:07.555 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:07.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:07.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:07.563 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:51:07.594 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:51:07.628 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:51:07.634 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:07.659 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:07.663 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:51:07.701 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:51:07.721 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:51:07.721 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:51:07.721 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:51:07.721 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:51:07.721 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:51:07.721 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:51:07.721 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:51:07.721 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:51:07.726 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:08.092 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:08.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:08.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:08.200 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:08.260 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:08.345 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:08.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:08.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:08.433 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:51:08.450 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[184418]: pgmap v1017: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 23741/199026 objects degraded (11.929%); 1922/199026 objects misplaced (0.966%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:51:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[184418]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[184418]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[184418]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T05:51:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[184418]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[184418]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[184418]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3626162017' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23741/199026 objects degraded (11.929%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:08.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[191227]: pgmap v1017: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 23741/199026 objects degraded (11.929%); 1922/199026 objects misplaced (0.966%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:51:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[191227]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[191227]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[191227]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T05:51:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[191227]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[191227]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[191227]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3626162017' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23741/199026 objects degraded (11.929%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:08.533 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:08 smithi191 ceph-mon[161669]: pgmap v1017: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 23741/199026 objects degraded (11.929%); 1922/199026 objects misplaced (0.966%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:51:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:08 smithi191 ceph-mon[161669]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:08 smithi191 ceph-mon[161669]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:08 smithi191 ceph-mon[161669]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T05:51:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:08 smithi191 ceph-mon[161669]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:08 smithi191 ceph-mon[161669]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:08 smithi191 ceph-mon[161669]: failed to encode map e338 with expected crc 2023-10-09T05:51:08.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3626162017' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:08.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23741/199026 objects degraded (11.929%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:08.785 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:51:08.785 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:51:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:141986 8.15 e338/299 8:abfb68d6:::benchmark_data_smithi008_432048_object39044:head v 338'7063, mlcod=338'7061)", 2023-10-09T05:51:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:51:08.784385+0000", 2023-10-09T05:51:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.00040645200000000002, 2023-10-09T05:51:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00043560099999999998, 2023-10-09T05:51:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:51:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:08.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:08.784385+0000", 2023-10-09T05:51:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:08.784385+0000", 2023-10-09T05:51:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:51:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:08.784384+0000", 2023-10-09T05:51:08.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0660000000000004e-06 2023-10-09T05:51:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:08.784392+0000", 2023-10-09T05:51:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4905e-05 2023-10-09T05:51:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:08.784407+0000", 2023-10-09T05:51:08.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1970000000000001e-06 2023-10-09T05:51:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:08.784413+0000", 2023-10-09T05:51:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3645e-05 2023-10-09T05:51:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:08.784457+0000", 2023-10-09T05:51:08.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9995000000000001e-05 2023-10-09T05:51:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:08.784477+0000", 2023-10-09T05:51:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2e-05 2023-10-09T05:51:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:08.790 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:51:08.791 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 1 2023-10-09T05:51:08.791 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:51:09.078 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:09.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:09.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:09.184 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:09.222 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:09.526 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:09.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:09.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:09.633 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:09.659 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:51:09.694 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:10.241 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:51:10.281 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:10.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:10.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:10.336 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:10.386 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:10.457 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:10 smithi008 ceph-mon[184418]: pgmap v1019: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 23265/197349 objects degraded (11.789%); 1922/197349 objects misplaced (0.974%); 8.9 MiB/s, 143 objects/s recovering 2023-10-09T05:51:10.458 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:10 smithi008 ceph-mon[191227]: pgmap v1019: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 23265/197349 objects degraded (11.789%); 1922/197349 objects misplaced (0.974%); 8.9 MiB/s, 143 objects/s recovering 2023-10-09T05:51:10.468 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:10.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:10 smithi191 ceph-mon[161669]: pgmap v1019: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 23265/197349 objects degraded (11.789%); 1922/197349 objects misplaced (0.974%); 8.9 MiB/s, 143 objects/s recovering 2023-10-09T05:51:10.581 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:10.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:10.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:10.686 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:10.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:51:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:51:10] "GET /metrics HTTP/1.1" 200 36611 "" "Prometheus/2.33.4" 2023-10-09T05:51:10.736 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:11.064 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:11.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:11.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:11.141 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:51:11.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:11.211 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:11.546 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:11.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:11.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:11.651 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:11.680 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:11.919 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:11.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:11.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:12.028 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:12.090 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:12.200 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T05:51:12.221 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:51:12.221 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:51:12.221 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:51:12.221 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:51:12.221 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:51:12.374 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[184418]: pgmap v1020: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 22951/191184 objects degraded (12.005%); 1629/191184 objects misplaced (0.852%); 14 MiB/s, 240 objects/s recovering 2023-10-09T05:51:12.375 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:51:12.375 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3080542785' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:51:12.375 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:51:12.375 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:12.375 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:12.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[191227]: pgmap v1020: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 22951/191184 objects degraded (12.005%); 1629/191184 objects misplaced (0.852%); 14 MiB/s, 240 objects/s recovering 2023-10-09T05:51:12.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:51:12.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3080542785' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:51:12.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:51:12.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:12.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:12 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:12.551 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:12.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:12 smithi191 ceph-mon[161669]: pgmap v1020: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 22951/191184 objects degraded (12.005%); 1629/191184 objects misplaced (0.852%); 14 MiB/s, 240 objects/s recovering 2023-10-09T05:51:12.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:12 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:51:12.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3080542785' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:51:12.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:12 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:51:12.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:12 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:12.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:12 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:12.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:12.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:12.656 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:12.690 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:13.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:51:13.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:13 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:51:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:13 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:51:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:13 smithi191 ceph-mon[161669]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T05:51:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:13 smithi191 ceph-mon[161669]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:13 smithi191 ceph-mon[161669]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:13 smithi191 ceph-mon[161669]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:13 smithi191 ceph-mon[161669]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:13 smithi191 ceph-mon[161669]: pgmap v1022: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 20732/189408 objects degraded (10.946%); 1629/189408 objects misplaced (0.860%); 16 MiB/s, 260 objects/s recovering 2023-10-09T05:51:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 22951/191184 objects degraded (12.005%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:13 smithi191 ceph-mon[161669]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T05:51:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:51:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:51:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[184418]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T05:51:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[184418]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[184418]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[184418]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[184418]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[184418]: pgmap v1022: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 20732/189408 objects degraded (10.946%); 1629/189408 objects misplaced (0.860%); 16 MiB/s, 260 objects/s recovering 2023-10-09T05:51:13.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 22951/191184 objects degraded (12.005%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:13.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[184418]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T05:51:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:51:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:51:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[191227]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T05:51:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[191227]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[191227]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[191227]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[191227]: failed to encode map e339 with expected crc 2023-10-09T05:51:13.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[191227]: pgmap v1022: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 20732/189408 objects degraded (10.946%); 1629/189408 objects misplaced (0.860%); 16 MiB/s, 260 objects/s recovering 2023-10-09T05:51:13.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 22951/191184 objects degraded (12.005%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:13.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:13 smithi008 ceph-mon[191227]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T05:51:13.470 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:51:13.497 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T05:51:13.561 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:13.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:13.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:13.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:13.702 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:13.732 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:13.753 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:13.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:13.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:13.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:13.884 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:14.030 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:14.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:14.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:14.134 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:14.166 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:14 smithi191 ceph-mon[161669]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:14 smithi191 ceph-mon[161669]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:14 smithi191 ceph-mon[161669]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:14 smithi191 ceph-mon[161669]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:14 smithi191 ceph-mon[161669]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:14 smithi191 ceph-mon[161669]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T05:51:14.379 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:14.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:14.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:14.419 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[184418]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.420 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[184418]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.420 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[184418]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.420 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[184418]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.420 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[184418]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.420 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:14.420 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[184418]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T05:51:14.420 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[191227]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.420 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[191227]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.420 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[191227]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.421 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[191227]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.421 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[191227]: failed to encode map e340 with expected crc 2023-10-09T05:51:14.421 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:14.421 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:14 smithi008 ceph-mon[191227]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T05:51:14.487 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:14.523 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:14.632 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:14.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:14.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:14.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:14.778 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:14.898 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:14.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:14.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:15.002 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:15.105 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:15.124 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:51:15.124 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:51:15.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:51:15.124 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:51:15.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.124 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:19336 7.d 7:bca1948b:::benchmark_data_smithi008_354649_object19335:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-09T05:51:15.124 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:51:15.124 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 568.47398783300002, 2023-10-09T05:51:15.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27934059700000002, 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 19336 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:51:15.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647258+0000", 2023-10-09T05:51:15.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:51:15.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647257+0000", 2023-10-09T05:51:15.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4706000000000003e-05 2023-10-09T05:51:15.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647292+0000", 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0139999999999999e-05 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647302+0000", 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4790000000000004e-06 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647308+0000", 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0175000000000001e-05 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647348+0000", 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3017999999999997e-05 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647431+0000", 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7472000000000005e-05 2023-10-09T05:51:15.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:51:15.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.647528+0000", 2023-10-09T05:51:15.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00087520700000000002 2023-10-09T05:51:15.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.648404+0000", 2023-10-09T05:51:15.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091703300000000004 2023-10-09T05:51:15.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.649321+0000", 2023-10-09T05:51:15.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27717746799999998 2023-10-09T05:51:15.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926498+0000", 2023-10-09T05:51:15.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2379000000000001e-05 2023-10-09T05:51:15.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926550+0000", 2023-10-09T05:51:15.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8653000000000003e-05 2023-10-09T05:51:15.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:41:46.926599+0000", 2023-10-09T05:51:15.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27934059700000002 2023-10-09T05:51:15.131 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.131 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.131 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:52521 7.c 7:3208f079:::benchmark_data_smithi008_354649_object52520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e223)", 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 540.52031805499996, 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.253878241, 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52521 2023-10-09T05:51:15.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.133 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:51:15.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600928+0000", 2023-10-09T05:51:15.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:15.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.600927+0000", 2023-10-09T05:51:15.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114644 2023-10-09T05:51:15.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601041+0000", 2023-10-09T05:51:15.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3705e-05 2023-10-09T05:51:15.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601055+0000", 2023-10-09T05:51:15.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3670000000000002e-06 2023-10-09T05:51:15.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601062+0000", 2023-10-09T05:51:15.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6796000000000003e-05 2023-10-09T05:51:15.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601118+0000", 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000151423 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601270+0000", 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000187224 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.601457+0000", 2023-10-09T05:51:15.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018848820000000001 2023-10-09T05:51:15.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.603342+0000", 2023-10-09T05:51:15.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017959000000000001 2023-10-09T05:51:15.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.603521+0000", 2023-10-09T05:51:15.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25118068700000001 2023-10-09T05:51:15.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854702+0000", 2023-10-09T05:51:15.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0417000000000001e-05 2023-10-09T05:51:15.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854763+0000", 2023-10-09T05:51:15.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3959999999999999e-05 2023-10-09T05:51:15.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:14.854806+0000", 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.253878241 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72589 7.d 7:bac7991c:::benchmark_data_smithi008_354649_object72588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 517.66281712900002, 2023-10-09T05:51:15.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.258363538, 2023-10-09T05:51:15.140 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.140 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.140 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.140 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.140 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.140 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72589 2023-10-09T05:51:15.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.140 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:51:15.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:51:15.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:15.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458429+0000", 2023-10-09T05:51:15.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8671000000000004e-05 2023-10-09T05:51:15.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458527+0000", 2023-10-09T05:51:15.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2957e-05 2023-10-09T05:51:15.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458540+0000", 2023-10-09T05:51:15.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6649999999999997e-06 2023-10-09T05:51:15.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458547+0000", 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2287999999999997e-05 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458609+0000", 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012157100000000001 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458731+0000", 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000152602 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.458883+0000", 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026998410000000001 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.461583+0000", 2023-10-09T05:51:15.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0065822210000000001 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.468166+0000", 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24850773600000001 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716673+0000", 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7136000000000002e-05 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716740+0000", 2023-10-09T05:51:15.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2272999999999999e-05 2023-10-09T05:51:15.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.716793+0000", 2023-10-09T05:51:15.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.258363538 2023-10-09T05:51:15.146 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.146 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.146 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72593 7.d 7:ba3e3325:::benchmark_data_smithi008_354649_object72592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 517.65534246499999, 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25827438800000002, 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.147 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72593 2023-10-09T05:51:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:51:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465904+0000", 2023-10-09T05:51:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.465902+0000", 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000110436 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466013+0000", 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0246e-05 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466023+0000", 2023-10-09T05:51:15.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7699999999999998e-06 2023-10-09T05:51:15.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466029+0000", 2023-10-09T05:51:15.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0343999999999999e-05 2023-10-09T05:51:15.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466069+0000", 2023-10-09T05:51:15.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4201000000000002e-05 2023-10-09T05:51:15.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466153+0000", 2023-10-09T05:51:15.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3833999999999996e-05 2023-10-09T05:51:15.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:51:15.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.466247+0000", 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020428099999999999 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.468290+0000", 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011037670000000001 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.469394+0000", 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25468252499999999 2023-10-09T05:51:15.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724076+0000", 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0541000000000003e-05 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724127+0000", 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1499000000000003e-05 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.724178+0000", 2023-10-09T05:51:15.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25827438800000002 2023-10-09T05:51:15.154 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.154 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.154 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.154 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:72601 7.d 7:b4df4171:::benchmark_data_smithi008_354649_object72600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-10-09T05:51:15.154 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:51:15.154 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 517.64940251099995, 2023-10-09T05:51:15.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25693990900000002, 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 72601 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471844+0000", 2023-10-09T05:51:15.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:51:15.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471843+0000", 2023-10-09T05:51:15.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1391000000000003e-05 2023-10-09T05:51:15.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471894+0000", 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3217000000000001e-05 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471907+0000", 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4e-06 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471913+0000", 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6798999999999998e-05 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.471950+0000", 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6098000000000001e-05 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.472036+0000", 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010641099999999999 2023-10-09T05:51:15.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.472142+0000", 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001463415 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.473605+0000", 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00083725200000000003 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.474443+0000", 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25426129400000003 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728704+0000", 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5178e-05 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728749+0000", 2023-10-09T05:51:15.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4474999999999998e-05 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:42:37.728784+0000", 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25693990900000002 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102136 7.c 7:3c9f4178:::benchmark_data_smithi008_354649_object7640:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T05:51:15.161 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:51:15.162 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 476.24570180900002, 2023-10-09T05:51:15.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3695118229999999, 2023-10-09T05:51:15.162 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.162 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:51:15.162 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.162 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.162 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.162 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102136 2023-10-09T05:51:15.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.162 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875544+0000", 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875543+0000", 2023-10-09T05:51:15.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0234999999999999e-05 2023-10-09T05:51:15.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875553+0000", 2023-10-09T05:51:15.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6123000000000001e-05 2023-10-09T05:51:15.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875569+0000", 2023-10-09T05:51:15.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7334e-05 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875586+0000", 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3238999999999997e-05 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875630+0000", 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120889 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875751+0000", 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000121965 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.875873+0000", 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068513569999999998 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.882724+0000", 2023-10-09T05:51:15.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.155e-05 2023-10-09T05:51:15.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:51:15.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.882735+0000", 2023-10-09T05:51:15.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99459824299999999 2023-10-09T05:51:15.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877334+0000", 2023-10-09T05:51:15.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.508e-05 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877349+0000", 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.345701239 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.223050+0000", 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016460699999999999 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.223215+0000", 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021743302999999999 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.244958+0000", 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8379000000000001e-05 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245056+0000", 2023-10-09T05:51:15.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3695118229999999 2023-10-09T05:51:15.170 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.170 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.170 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.170 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102150 7.c 7:30e13689:::benchmark_data_smithi008_354649_object7654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:51:15.170 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:51:15.170 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 476.21664599100001, 2023-10-09T05:51:15.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3456080319999999, 2023-10-09T05:51:15.170 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102150 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904600+0000", 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904598+0000", 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9720000000000003e-06 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904605+0000", 2023-10-09T05:51:15.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3104e-05 2023-10-09T05:51:15.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904619+0000", 2023-10-09T05:51:15.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7699999999999998e-06 2023-10-09T05:51:15.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904624+0000", 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5915000000000003e-05 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904680+0000", 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1524e-05 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.904692+0000", 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97280887699999996 2023-10-09T05:51:15.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877501+0000", 2023-10-09T05:51:15.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9029999999999993e-06 2023-10-09T05:51:15.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:51:15.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877510+0000", 2023-10-09T05:51:15.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3503837970000001 2023-10-09T05:51:15.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.227893+0000", 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017629700000000001 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.228070+0000", 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019175 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:51:15.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.228261+0000", 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086207179999999994 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.236882+0000", 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0081705159999999992 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245053+0000", 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4658000000000003e-05 2023-10-09T05:51:15.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245117+0000", 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2007000000000004e-05 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.245189+0000", 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005019061 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250208+0000", 2023-10-09T05:51:15.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3456080319999999 2023-10-09T05:51:15.179 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.179 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.179 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.179 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102158 7.c 7:380582de:::benchmark_data_smithi008_354649_object7662:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:51:15.179 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:51:15.179 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 476.21191008900001, 2023-10-09T05:51:15.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3657197050000001, 2023-10-09T05:51:15.180 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.180 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.180 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102158 2023-10-09T05:51:15.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.180 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909336+0000", 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909335+0000", 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1000000000000003e-06 2023-10-09T05:51:15.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909340+0000", 2023-10-09T05:51:15.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9660000000000002e-06 2023-10-09T05:51:15.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909349+0000", 2023-10-09T05:51:15.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3199999999999999e-06 2023-10-09T05:51:15.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909354+0000", 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2012999999999999e-05 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909396+0000", 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4160000000000001e-06 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:51:15.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.909403+0000", 2023-10-09T05:51:15.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96813495800000005 2023-10-09T05:51:15.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877538+0000", 2023-10-09T05:51:15.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5560000000000008e-06 2023-10-09T05:51:15.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:51:15.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.877546+0000", 2023-10-09T05:51:15.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3727139369999999 2023-10-09T05:51:15.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250260+0000", 2023-10-09T05:51:15.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017051999999999999 2023-10-09T05:51:15.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250431+0000", 2023-10-09T05:51:15.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019215399999999999 2023-10-09T05:51:15.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:51:15.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.250623+0000", 2023-10-09T05:51:15.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010646828000000001 2023-10-09T05:51:15.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261270+0000", 2023-10-09T05:51:15.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6761999999999995e-05 2023-10-09T05:51:15.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261337+0000", 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087281960000000006 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.270065+0000", 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9771000000000006e-05 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.270144+0000", 2023-10-09T05:51:15.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0049114320000000003 2023-10-09T05:51:15.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275056+0000", 2023-10-09T05:51:15.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3657197050000001 2023-10-09T05:51:15.188 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.188 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.188 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.189 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:102172 7.c 7:3ca77be9:::benchmark_data_smithi008_354649_object7676:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:51:15.189 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:51:15.189 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 476.18517301899999, 2023-10-09T05:51:15.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3439011609999998, 2023-10-09T05:51:15.189 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.189 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.189 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.189 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.189 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.189 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102172 2023-10-09T05:51:15.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936073+0000", 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999971 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936071+0000", 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4219999999999999e-05 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936085+0000", 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3510000000000003e-05 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936118+0000", 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3056e-05 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936131+0000", 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013862499999999999 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936270+0000", 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3222999999999997e-05 2023-10-09T05:51:15.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:18.936303+0000", 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.941830422 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.878134+0000", 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3155e-05 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:19.878147+0000", 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.375972827 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254120+0000", 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00031237700000000002 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254432+0000", 2023-10-09T05:51:15.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000179364 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.254611+0000", 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0066698180000000001 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.261281+0000", 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00092641700000000002 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.262208+0000", 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012930142 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275138+0000", 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1118999999999995e-05 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.275209+0000", 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047656299999999999 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:43:21.279974+0000", 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3439011609999998 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:124846 7.c 7:339e95c6:::benchmark_data_smithi008_354649_object30350:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 433.61729729899997, 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19787401099999999, 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124846 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:51:15.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:51:15.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:15.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503949+0000", 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8360000000000001e-06 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503951+0000", 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.534e-06 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503954+0000", 2023-10-09T05:51:15.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7929999999999999e-06 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.503958+0000", 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070412340000000004 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.510999+0000", 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012243900000000001 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.511122+0000", 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012357500000000001 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.511245+0000", 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070869629999999999 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.518332+0000", 2023-10-09T05:51:15.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010506108 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.528838+0000", 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070145440000000002 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.535853+0000", 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2738000000000002e-05 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.535905+0000", 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.165917541 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.701823+0000", 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19787401099999999 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.205 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:124870 7.c 7:34cc5040:::benchmark_data_smithi008_354649_object30374:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:51:15.205 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:51:15.205 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 433.58486334899999, 2023-10-09T05:51:15.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.194346888, 2023-10-09T05:51:15.205 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.205 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.205 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.205 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.205 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.205 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124870 2023-10-09T05:51:15.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.206 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:51:15.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536383+0000", 2023-10-09T05:51:15.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536382+0000", 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1800000000000001e-06 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536388+0000", 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8409999999999995e-06 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536398+0000", 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9701e-05 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.536417+0000", 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16904754699999999 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705465+0000", 2023-10-09T05:51:15.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016865399999999999 2023-10-09T05:51:15.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705634+0000", 2023-10-09T05:51:15.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017818700000000001 2023-10-09T05:51:15.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:51:15.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.705812+0000", 2023-10-09T05:51:15.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068761990000000004 2023-10-09T05:51:15.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.712688+0000", 2023-10-09T05:51:15.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078597960000000005 2023-10-09T05:51:15.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.720548+0000", 2023-10-09T05:51:15.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059971089999999996 2023-10-09T05:51:15.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.726545+0000", 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4085000000000006e-05 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.726619+0000", 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004110832 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:01.730730+0000", 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.194346888 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:136135 7.d 7:b8c830a8:::benchmark_data_smithi008_354649_object41639:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:35.638089+0000", 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 399.48315715799998, 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.252176754, 2023-10-09T05:51:15.212 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 136135 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638089+0000", 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638089+0000", 2023-10-09T05:51:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:51:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638087+0000", 2023-10-09T05:51:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0289999999999994e-06 2023-10-09T05:51:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638096+0000", 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6886000000000001e-05 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638113+0000", 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0861e-05 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638123+0000", 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8810000000000001e-05 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638182+0000", 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000136959 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638319+0000", 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015855900000000001 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.638478+0000", 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048134650000000003 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643291+0000", 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073579750000000001 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.650649+0000", 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23951030400000001 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.890160+0000", 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8352e-05 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.890218+0000", 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8004000000000003e-05 2023-10-09T05:51:15.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.890266+0000", 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.252176754 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:136137 7.d 7:b6392fbb:::benchmark_data_smithi008_354649_object41641:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 399.47726000799997, 2023-10-09T05:51:15.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24962479700000001, 2023-10-09T05:51:15.220 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.220 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.220 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:51:15.220 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:51:15.220 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:15.220 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 136137 2023-10-09T05:51:15.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.220 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643986+0000", 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1309999999999999e-06 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643990+0000", 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4509999999999992e-06 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.643998+0000", 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5480000000000001e-06 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644004+0000", 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1790999999999999e-05 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644056+0000", 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015365299999999999 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644209+0000", 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017818200000000001 2023-10-09T05:51:15.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.644387+0000", 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047856310000000003 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.649173+0000", 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015424 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.650715+0000", 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.242803251 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.893519+0000", 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7497000000000002e-05 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.893576+0000", 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4962000000000002e-05 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:35.893611+0000", 2023-10-09T05:51:15.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.24962479700000001 2023-10-09T05:51:15.226 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.226 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.226 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.226 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:41724 8.0 e291/285 8:01b70071:::benchmark_data_smithi008_432048_object41723:head v 291'2644, mlcod=291'2641)", 2023-10-09T05:51:15.226 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:02.471292+0000", 2023-10-09T05:51:15.226 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 192.64995445299999, 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030552701000000002, 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471292+0000", 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471292+0000", 2023-10-09T05:51:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471291+0000", 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6661000000000004e-05 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471357+0000", 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7370000000000001e-06 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471367+0000", 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7799999999999997e-06 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471373+0000", 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00049471499999999996 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471867+0000", 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8739000000000001e-05 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471886+0000", 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018492409999999999 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.473735+0000", 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028109080000000002 2023-10-09T05:51:15.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.501845+0000", 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030552701000000002 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:41716 8.f e291/285 8:faa82907:::benchmark_data_smithi008_432048_object41715:head v 291'2545, mlcod=291'2542)", 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:02.471382+0000", 2023-10-09T05:51:15.231 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 192.64986389200001, 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030548492999999999, 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471382+0000", 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471382+0000", 2023-10-09T05:51:15.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471382+0000", 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2911999999999999e-05 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471415+0000", 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5379999999999999e-06 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471421+0000", 2023-10-09T05:51:15.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9489999999999998e-06 2023-10-09T05:51:15.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471425+0000", 2023-10-09T05:51:15.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6309000000000001e-05 2023-10-09T05:51:15.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471462+0000", 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8779000000000003e-05 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.471501+0000", 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022499960000000002 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.473751+0000", 2023-10-09T05:51:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028180394000000001 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:02.501931+0000", 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030548492999999999 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:71979 8.f e295/294 8:f088e1a0:::benchmark_data_smithi008_432048_object71978:head v 295'4395, mlcod=295'4393)", 2023-10-09T05:51:15.236 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:28.507415+0000", 2023-10-09T05:51:15.237 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 166.613830818, 2023-10-09T05:51:15.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023385482999999999, 2023-10-09T05:51:15.237 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.237 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.237 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.237 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.237 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507415+0000", 2023-10-09T05:51:15.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507415+0000", 2023-10-09T05:51:15.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:15.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507415+0000", 2023-10-09T05:51:15.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5426999999999999e-05 2023-10-09T05:51:15.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507440+0000", 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8349999999999998e-06 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507447+0000", 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5120000000000002e-06 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.507452+0000", 2023-10-09T05:51:15.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029621090000000001 2023-10-09T05:51:15.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.510414+0000", 2023-10-09T05:51:15.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1447999999999999e-05 2023-10-09T05:51:15.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.510435+0000", 2023-10-09T05:51:15.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020330933999999998 2023-10-09T05:51:15.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.530766+0000", 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4652e-05 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:28.530801+0000", 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023385482999999999 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.241 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:86717 8.6 e300/299 8:65a0d5c6:::benchmark_data_smithi008_432048_object86716:head v 300'5376, mlcod=0'0)", 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:46.222066+0000", 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 148.89918076399999, 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022574397, 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222066+0000", 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222066+0000", 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222065+0000", 2023-10-09T05:51:15.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1554000000000001e-05 2023-10-09T05:51:15.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222117+0000", 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1640000000000004e-06 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222124+0000", 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0900000000000005e-06 2023-10-09T05:51:15.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.222132+0000", 2023-10-09T05:51:15.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0091579690000000002 2023-10-09T05:51:15.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.231290+0000", 2023-10-09T05:51:15.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5908999999999999e-05 2023-10-09T05:51:15.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.231316+0000", 2023-10-09T05:51:15.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0132868 2023-10-09T05:51:15.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.244603+0000", 2023-10-09T05:51:15.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7187000000000002e-05 2023-10-09T05:51:15.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.244640+0000", 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022574397 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:89017 8.10 e301/299 8:0cd3ebd7:::benchmark_data_smithi008_432048_object89016:head v 301'5542, mlcod=301'5540)", 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:48.392752+0000", 2023-10-09T05:51:15.248 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 146.728494205, 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022788323999999999, 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392752+0000", 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392752+0000", 2023-10-09T05:51:15.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:15.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392752+0000", 2023-10-09T05:51:15.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2337e-05 2023-10-09T05:51:15.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392774+0000", 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0094e-05 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392784+0000", 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.357e-06 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.392789+0000", 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038184220000000001 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.396607+0000", 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0106e-05 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.396627+0000", 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018869139 2023-10-09T05:51:15.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.415496+0000", 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4217e-05 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:48.415540+0000", 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022788323999999999 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.253 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45782.0:133872 8.23 e334/328 8:c6214d4b:::benchmark_data_smithi008_432048_object30930:head v 334'6430, mlcod=291'2350)", 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:50:52.197965+0000", 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 22.923281714000002, 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15881416900000001, 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.197965+0000", 2023-10-09T05:51:15.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.197965+0000", 2023-10-09T05:51:15.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:15.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.197964+0000", 2023-10-09T05:51:15.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8760000000000002e-06 2023-10-09T05:51:15.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.197970+0000", 2023-10-09T05:51:15.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.155744408 2023-10-09T05:51:15.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.353715+0000", 2023-10-09T05:51:15.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1385e-05 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.353726+0000", 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5969e-05 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.353772+0000", 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0598000000000001e-05 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:51:15.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.353803+0000", 2023-10-09T05:51:15.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029507919999999998 2023-10-09T05:51:15.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:51:15.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.356753+0000", 2023-10-09T05:51:15.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5439000000000002e-05 2023-10-09T05:51:15.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.356779+0000", 2023-10-09T05:51:15.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15881416900000001 2023-10-09T05:51:15.259 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.259 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.259 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.259 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "MOSDPGPull(8.1f e333/328 cost 1049576)", 2023-10-09T05:51:15.259 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:50:52.202245+0000", 2023-10-09T05:51:15.259 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 22.919001501, 2023-10-09T05:51:15.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15240483399999999, 2023-10-09T05:51:15.260 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:51:15.260 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:51:15.260 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:51:15.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:51:15.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.202245+0000", 2023-10-09T05:51:15.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:51:15.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:51:15.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.202245+0000", 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.202244+0000", 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.255e-05 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.202257+0000", 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15146111600000001 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:51:15.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.353718+0000", 2023-10-09T05:51:15.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0950000000000002e-06 2023-10-09T05:51:15.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:51:15.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.353727+0000", 2023-10-09T05:51:15.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00024877699999999999 2023-10-09T05:51:15.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:51:15.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.353976+0000", 2023-10-09T05:51:15.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00067411299999999995 2023-10-09T05:51:15.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:51:15.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:51:15.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:51:15.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:52.354650+0000", 2023-10-09T05:51:15.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15240483399999999 2023-10-09T05:51:15.263 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.263 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.263 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.263 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:51:15.264 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:51:15.264 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:51:15.264 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:51:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:51:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[191227]: failed to encode map e341 with expected crc 2023-10-09T05:51:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[191227]: failed to encode map e341 with expected crc 2023-10-09T05:51:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[191227]: failed to encode map e341 with expected crc 2023-10-09T05:51:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[191227]: pgmap v1025: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovery_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 20638/187386 objects degraded (11.014%); 1629/187386 objects misplaced (0.869%); 13 MiB/s, 212 objects/s recovering 2023-10-09T05:51:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2841650846' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:15.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:51:15.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[184418]: failed to encode map e341 with expected crc 2023-10-09T05:51:15.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[184418]: failed to encode map e341 with expected crc 2023-10-09T05:51:15.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[184418]: failed to encode map e341 with expected crc 2023-10-09T05:51:15.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[184418]: pgmap v1025: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovery_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 20638/187386 objects degraded (11.014%); 1629/187386 objects misplaced (0.869%); 13 MiB/s, 212 objects/s recovering 2023-10-09T05:51:15.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2841650846' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:15.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:15 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:15 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:51:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:15 smithi191 ceph-mon[161669]: failed to encode map e341 with expected crc 2023-10-09T05:51:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:15 smithi191 ceph-mon[161669]: failed to encode map e341 with expected crc 2023-10-09T05:51:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:15 smithi191 ceph-mon[161669]: failed to encode map e341 with expected crc 2023-10-09T05:51:15.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:15 smithi191 ceph-mon[161669]: pgmap v1025: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovery_wait, 1 active+recovering, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 20638/187386 objects degraded (11.014%); 1629/187386 objects misplaced (0.869%); 13 MiB/s, 212 objects/s recovering 2023-10-09T05:51:15.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2841650846' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:15.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:15 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:15.930 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:15.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:15.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:15.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:51:16.038 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:16.099 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:16.186 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:16.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:16.200 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:51:16.200 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[184418]: failed to encode map e342 with expected crc 2023-10-09T05:51:16.200 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[184418]: failed to encode map e342 with expected crc 2023-10-09T05:51:16.200 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:16.200 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[184418]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T05:51:16.200 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[184418]: failed to encode map e342 with expected crc 2023-10-09T05:51:16.296 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:16.342 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:16.407 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:16.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:16.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:16.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:51:16.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[191227]: failed to encode map e342 with expected crc 2023-10-09T05:51:16.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[191227]: failed to encode map e342 with expected crc 2023-10-09T05:51:16.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:16.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[191227]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T05:51:16.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:16 smithi008 ceph-mon[191227]: failed to encode map e342 with expected crc 2023-10-09T05:51:16.470 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:51:16.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:16.546 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:16 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:51:16.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:16 smithi191 ceph-mon[161669]: failed to encode map e342 with expected crc 2023-10-09T05:51:16.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:16 smithi191 ceph-mon[161669]: failed to encode map e342 with expected crc 2023-10-09T05:51:16.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:16 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:16.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:16 smithi191 ceph-mon[161669]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T05:51:16.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:16 smithi191 ceph-mon[161669]: failed to encode map e342 with expected crc 2023-10-09T05:51:16.550 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:16.660 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:16.687 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:16.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:16.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:16.793 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:16.830 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:16.907 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:16.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:16.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:17.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:17.052 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:17.261 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:17.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:17.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:17.366 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:17.394 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:17.443 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:17.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:17.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:17.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:17 smithi008 ceph-mon[184418]: 8.10 deep-scrub starts 2023-10-09T05:51:17.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:17 smithi008 ceph-mon[184418]: pgmap v1027: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 20447/184494 objects degraded (11.083%); 1561/184494 objects misplaced (0.846%); 10 MiB/s, 164 objects/s recovering 2023-10-09T05:51:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:17 smithi008 ceph-mon[191227]: 8.10 deep-scrub starts 2023-10-09T05:51:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:17 smithi008 ceph-mon[191227]: pgmap v1027: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 20447/184494 objects degraded (11.083%); 1561/184494 objects misplaced (0.846%); 10 MiB/s, 164 objects/s recovering 2023-10-09T05:51:17.546 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:17.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:17 smithi191 ceph-mon[161669]: 8.10 deep-scrub starts 2023-10-09T05:51:17.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:17 smithi191 ceph-mon[161669]: pgmap v1027: 119 pgs: 1 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 20447/184494 objects degraded (11.083%); 1561/184494 objects misplaced (0.846%); 10 MiB/s, 164 objects/s recovering 2023-10-09T05:51:17.616 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:51:17.635 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:17.676 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:51:17.718 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:51:17.773 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:51:17.781 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:17.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:17.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:17.821 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:51:17.857 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:51:17.896 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:17.901 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:51:17.947 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:51:17.950 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:51:17.950 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:51:17.950 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:51:17.950 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:51:17.951 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:51:17.951 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:51:17.951 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:51:17.951 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:51:17.966 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:18.075 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:18.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:18.188 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:18.244 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:18.305 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:18.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:18.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:18.329 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:51:18.329 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:51:18.360 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3745140847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:18.360 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:18 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:18.360 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:18 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:18.360 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20447/184494 objects degraded (11.083%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:18.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:18.464 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3745140847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:18 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:18.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:18 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:18.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20447/184494 objects degraded (11.083%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:18.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3745140847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:18.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:18 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:18.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:18 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:18.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20447/184494 objects degraded (11.083%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:51:18.519 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:18.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:18.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:18.623 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:18.653 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:18.757 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:51:19.114 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:51:19.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:19 smithi008 ceph-mon[184418]: pgmap v1028: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 27864/182865 objects degraded (15.237%); 1561/182865 objects misplaced (0.854%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:51:19.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:19 smithi008 ceph-mon[191227]: pgmap v1028: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 27864/182865 objects degraded (15.237%); 1561/182865 objects misplaced (0.854%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:51:19.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:19 smithi191 ceph-mon[161669]: pgmap v1028: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 27864/182865 objects degraded (15.237%); 1561/182865 objects misplaced (0.854%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:51:19.600 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:51:19.787 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:19.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:19.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:19.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:19.959 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:20.171 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:20.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:20.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:20.275 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:20.353 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:20.355 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:20.422 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:20.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:20 smithi008 ceph-mon[184418]: 8.10 deep-scrub starts 2023-10-09T05:51:20.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:51:20.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:51:20.424 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:20.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:20.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:20.425 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:20 smithi008 ceph-mon[191227]: 8.10 deep-scrub starts 2023-10-09T05:51:20.425 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:51:20.425 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:51:20.425 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:20.526 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:20.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:20 smithi191 ceph-mon[161669]: 8.10 deep-scrub starts 2023-10-09T05:51:20.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:51:20.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:51:20.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:51:20.571 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:20.711 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:51:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:51:20] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.33.4" 2023-10-09T05:51:20.882 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:20.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:20.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:20.989 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:21.030 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:21.143 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:21.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:21.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:21.239 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T05:51:21.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:21.292 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:21.362 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:21.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:21.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:21.413 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:51:21.413 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:51:21.438 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:21 smithi191 ceph-mon[161669]: pgmap v1029: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 27743/179982 objects degraded (15.414%); 1561/179982 objects misplaced (0.867%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T05:51:21.438 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:21 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3526093520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:51:21.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:21 smithi008 ceph-mon[184418]: pgmap v1029: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 27743/179982 objects degraded (15.414%); 1561/179982 objects misplaced (0.867%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T05:51:21.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:21 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3526093520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:51:21.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:21 smithi008 ceph-mon[191227]: pgmap v1029: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 27743/179982 objects degraded (15.414%); 1561/179982 objects misplaced (0.867%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T05:51:21.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:21 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3526093520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:51:21.468 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:21.552 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:21.819 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:51:22.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:51:22.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:22 smithi191 ceph-mon[161669]: failed to encode map e343 with expected crc 2023-10-09T05:51:22.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:22 smithi191 ceph-mon[161669]: failed to encode map e343 with expected crc 2023-10-09T05:51:22.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:22 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3526093520' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:51:22.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:22 smithi191 ceph-mon[161669]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T05:51:22.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:22 smithi191 ceph-mon[161669]: failed to encode map e343 with expected crc 2023-10-09T05:51:22.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:22 smithi008 ceph-mon[184418]: failed to encode map e343 with expected crc 2023-10-09T05:51:22.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:22 smithi008 ceph-mon[184418]: failed to encode map e343 with expected crc 2023-10-09T05:51:22.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:22 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3526093520' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:51:22.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:22 smithi008 ceph-mon[184418]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T05:51:22.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:22 smithi008 ceph-mon[184418]: failed to encode map e343 with expected crc 2023-10-09T05:51:22.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:22 smithi008 ceph-mon[191227]: failed to encode map e343 with expected crc 2023-10-09T05:51:22.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:22 smithi008 ceph-mon[191227]: failed to encode map e343 with expected crc 2023-10-09T05:51:22.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:22 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3526093520' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:51:22.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:22 smithi008 ceph-mon[191227]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T05:51:22.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:22 smithi008 ceph-mon[191227]: failed to encode map e343 with expected crc 2023-10-09T05:51:22.802 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:51:23.246 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:23.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:23.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:23 smithi191 ceph-mon[161669]: failed to encode map e344 with expected crc 2023-10-09T05:51:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:23 smithi191 ceph-mon[161669]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T05:51:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:23 smithi191 ceph-mon[161669]: failed to encode map e344 with expected crc 2023-10-09T05:51:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:23 smithi191 ceph-mon[161669]: failed to encode map e344 with expected crc 2023-10-09T05:51:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:23 smithi191 ceph-mon[161669]: pgmap v1032: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 26721/174189 objects degraded (15.340%); 1502/174189 objects misplaced (0.862%); 15 MiB/s, 240 objects/s recovering 2023-10-09T05:51:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 27743/179982 objects degraded (15.414%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:23.356 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T05:51:23.356 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:23.403 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:23.468 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:23.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:23.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:23.574 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:23.609 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:23.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[191227]: failed to encode map e344 with expected crc 2023-10-09T05:51:23.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[191227]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T05:51:23.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[191227]: failed to encode map e344 with expected crc 2023-10-09T05:51:23.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[191227]: failed to encode map e344 with expected crc 2023-10-09T05:51:23.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[191227]: pgmap v1032: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 26721/174189 objects degraded (15.340%); 1502/174189 objects misplaced (0.862%); 15 MiB/s, 240 objects/s recovering 2023-10-09T05:51:23.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 27743/179982 objects degraded (15.414%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:23.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[184418]: failed to encode map e344 with expected crc 2023-10-09T05:51:23.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[184418]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T05:51:23.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[184418]: failed to encode map e344 with expected crc 2023-10-09T05:51:23.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[184418]: failed to encode map e344 with expected crc 2023-10-09T05:51:23.708 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[184418]: pgmap v1032: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 26721/174189 objects degraded (15.340%); 1502/174189 objects misplaced (0.862%); 15 MiB/s, 240 objects/s recovering 2023-10-09T05:51:23.709 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 27743/179982 objects degraded (15.414%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:23.720 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:23.850 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:23.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:23.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:23.961 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:24.059 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:24.306 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:24.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:24.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:24.417 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:24.509 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:24.626 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:51:24.627 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:51:24.709 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:24.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:24.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:24.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:24.852 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:24.900 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:24.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:24.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:25.003 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:25.029 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:25.070 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:25.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:25.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:25.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:25.208 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:25.454 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:51:25.462 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:25.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:25.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:25.538 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:25.566 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:25.603 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:25.636 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:51:25.658 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:25.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:25.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:25.763 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:25.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:25 smithi191 ceph-mon[161669]: pgmap v1033: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 26573/171078 objects degraded (15.533%); 1502/171078 objects misplaced (0.878%); 13 MiB/s, 204 objects/s recovering 2023-10-09T05:51:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:25 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/127964458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:25.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:25 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:25.850 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:25 smithi008 ceph-mon[184418]: pgmap v1033: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 26573/171078 objects degraded (15.533%); 1502/171078 objects misplaced (0.878%); 13 MiB/s, 204 objects/s recovering 2023-10-09T05:51:25.850 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:25 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/127964458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:25.851 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:25 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:25.851 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:25 smithi008 ceph-mon[191227]: pgmap v1033: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 26573/171078 objects degraded (15.533%); 1502/171078 objects misplaced (0.878%); 13 MiB/s, 204 objects/s recovering 2023-10-09T05:51:25.851 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:25 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/127964458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:25.851 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:25 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:25.856 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:26.434 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:26.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:26.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:26.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:26.565 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:26.613 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:26.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:26.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:26.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:51:26.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:26.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[184418]: failed to encode map e345 with expected crc 2023-10-09T05:51:26.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:51:26.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[184418]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T05:51:26.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[184418]: failed to encode map e345 with expected crc 2023-10-09T05:51:26.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:51:26.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:26.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[191227]: failed to encode map e345 with expected crc 2023-10-09T05:51:26.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:51:26.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[191227]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T05:51:26.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:26 smithi008 ceph-mon[191227]: failed to encode map e345 with expected crc 2023-10-09T05:51:26.720 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:26.805 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:27.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:26 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:51:27.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:26 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:27.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:26 smithi191 ceph-mon[161669]: failed to encode map e345 with expected crc 2023-10-09T05:51:27.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:26 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:51:27.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:26 smithi191 ceph-mon[161669]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T05:51:27.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:26 smithi191 ceph-mon[161669]: failed to encode map e345 with expected crc 2023-10-09T05:51:27.145 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:27.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:27.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:27.257 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:27.294 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:51:27.326 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:51:27.356 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:27.362 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:51:27.427 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:51:27.465 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:51:27.503 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:51:27.542 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:51:27.586 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:51:27.604 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:51:27.605 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:51:27.605 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:51:27.605 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:51:27.605 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:51:27.605 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:51:27.605 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:51:27.605 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:51:27.612 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:27 smithi008 ceph-mon[184418]: pgmap v1035: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 26241/166155 objects degraded (15.793%); 1394/166155 objects misplaced (0.839%); 16 MiB/s, 248 objects/s recovering 2023-10-09T05:51:27.612 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2923852649' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:27.612 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:27 smithi008 ceph-mon[191227]: pgmap v1035: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 26241/166155 objects degraded (15.793%); 1394/166155 objects misplaced (0.839%); 16 MiB/s, 248 objects/s recovering 2023-10-09T05:51:27.613 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2923852649' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:28.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:27 smithi191 ceph-mon[161669]: pgmap v1035: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 26241/166155 objects degraded (15.793%); 1394/166155 objects misplaced (0.839%); 16 MiB/s, 248 objects/s recovering 2023-10-09T05:51:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2923852649' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:28.603 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:28.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:28.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:28.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:28.767 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:28.836 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:28.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:28.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:28.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:28.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 26241/166155 objects degraded (15.793%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:28.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 26241/166155 objects degraded (15.793%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:28.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 26241/166155 objects degraded (15.793%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:28.985 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:29.054 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:29.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:29.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:29.114 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:51:29.165 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:29.201 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:29.267 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:29.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:29.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:29.380 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:29.417 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:29.438 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:29.484 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:29.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:29.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:29.591 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:29.690 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:29.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:29.865 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:29 smithi008 ceph-mon[184418]: pgmap v1036: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 25472/164052 objects degraded (15.527%); 1394/164052 objects misplaced (0.850%); 15 MiB/s, 240 objects/s recovering 2023-10-09T05:51:29.865 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:29.865 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:29 smithi008 ceph-mon[191227]: pgmap v1036: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 25472/164052 objects degraded (15.527%); 1394/164052 objects misplaced (0.850%); 15 MiB/s, 240 objects/s recovering 2023-10-09T05:51:29.865 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:29.866 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:29.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:29.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:29.997 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:29 smithi191 ceph-mon[161669]: pgmap v1036: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 25472/164052 objects degraded (15.527%); 1394/164052 objects misplaced (0.850%); 15 MiB/s, 240 objects/s recovering 2023-10-09T05:51:29.997 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:29.998 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:30.066 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:30.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:30.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:30.177 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:30.256 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:30.471 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:30.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:30.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:30.578 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:30.607 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:30.663 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T05:51:30.691 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:30.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1437214063' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:51:30.698 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:51:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:51:30] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.33.4" 2023-10-09T05:51:30.698 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1437214063' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:51:30.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:30.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:30.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:30.825 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:30.867 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:30.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1437214063' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:51:30.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:30.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:30.972 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:31.008 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:31.078 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:31.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:31.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:31.184 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:31.254 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:31.475 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:51:31.658 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:31.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:31.773 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:31.812 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:31.875 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:31.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:31.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:31.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[184418]: pgmap v1037: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 25299/160614 objects degraded (15.751%); 1394/160614 objects misplaced (0.868%); 14 MiB/s, 218 objects/s recovering 2023-10-09T05:51:31.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1437214063' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:51:31.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[184418]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T05:51:31.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[184418]: failed to encode map e346 with expected crc 2023-10-09T05:51:31.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[184418]: failed to encode map e346 with expected crc 2023-10-09T05:51:31.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[184418]: failed to encode map e346 with expected crc 2023-10-09T05:51:31.917 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[184418]: failed to encode map e346 with expected crc 2023-10-09T05:51:31.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[191227]: pgmap v1037: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 25299/160614 objects degraded (15.751%); 1394/160614 objects misplaced (0.868%); 14 MiB/s, 218 objects/s recovering 2023-10-09T05:51:31.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1437214063' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:51:31.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[191227]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T05:51:31.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[191227]: failed to encode map e346 with expected crc 2023-10-09T05:51:31.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[191227]: failed to encode map e346 with expected crc 2023-10-09T05:51:31.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[191227]: failed to encode map e346 with expected crc 2023-10-09T05:51:31.918 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:31 smithi008 ceph-mon[191227]: failed to encode map e346 with expected crc 2023-10-09T05:51:31.981 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:32.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:31 smithi191 ceph-mon[161669]: pgmap v1037: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 25299/160614 objects degraded (15.751%); 1394/160614 objects misplaced (0.868%); 14 MiB/s, 218 objects/s recovering 2023-10-09T05:51:32.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:31 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1437214063' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:51:32.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:31 smithi191 ceph-mon[161669]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T05:51:32.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:31 smithi191 ceph-mon[161669]: failed to encode map e346 with expected crc 2023-10-09T05:51:32.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:31 smithi191 ceph-mon[161669]: failed to encode map e346 with expected crc 2023-10-09T05:51:32.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:31 smithi191 ceph-mon[161669]: failed to encode map e346 with expected crc 2023-10-09T05:51:32.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:31 smithi191 ceph-mon[161669]: failed to encode map e346 with expected crc 2023-10-09T05:51:32.008 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:32.075 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:32.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:32.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:32.183 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:32.209 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:32.220 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:32.246 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T05:51:32.276 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:32.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:32.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:32.384 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:32.478 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:32.757 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:32.773 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:32 smithi008 ceph-mon[184418]: failed to encode map e347 with expected crc 2023-10-09T05:51:32.773 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:32 smithi008 ceph-mon[184418]: failed to encode map e347 with expected crc 2023-10-09T05:51:32.773 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:32 smithi008 ceph-mon[184418]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T05:51:32.773 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:32 smithi008 ceph-mon[184418]: failed to encode map e347 with expected crc 2023-10-09T05:51:32.774 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:32.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:32.787 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:32 smithi008 ceph-mon[191227]: failed to encode map e347 with expected crc 2023-10-09T05:51:32.788 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:32 smithi008 ceph-mon[191227]: failed to encode map e347 with expected crc 2023-10-09T05:51:32.788 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:32 smithi008 ceph-mon[191227]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T05:51:32.788 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:32 smithi008 ceph-mon[191227]: failed to encode map e347 with expected crc 2023-10-09T05:51:32.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:32.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:32.919 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:32.934 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:51:32.934 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:51:32.934 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.934 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:156063 8.11 8:8f772ac6:::benchmark_data_smithi008_432048_object53121:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T05:51:32.934 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:32.903014+0000", 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.028583743000000002, 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.02861901, 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 156063 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.935 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:32.936 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.903014+0000", 2023-10-09T05:51:32.936 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:32.936 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.936 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.936 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:32.936 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.903014+0000", 2023-10-09T05:51:32.936 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:32.936 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.936 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.936 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:32.936 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.903012+0000", 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.903021+0000", 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.191e-06 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.903040+0000", 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9646000000000001e-05 2023-10-09T05:51:32.937 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.903052+0000", 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1372000000000001e-05 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.903123+0000", 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1649000000000005e-05 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:51:32.938 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.903639+0000", 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000515995 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:156071 8.6 8:60cb922b:::benchmark_data_smithi008_432048_object53129:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:32.917041+0000", 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.014557192, 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014674089, 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:32.939 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 156071 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.917041+0000", 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.940 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:32.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.917041+0000", 2023-10-09T05:51:32.941 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:32.941 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.941 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:32.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.917039+0000", 2023-10-09T05:51:32.941 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.917044+0000", 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.22e-06 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.917053+0000", 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6640000000000004e-06 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.942 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.917059+0000", 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0100000000000001e-06 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.926535+0000", 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.009475987 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.926697+0000", 2023-10-09T05:51:32.943 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016178300000000001 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.927088+0000", 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00039131700000000002 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.929984+0000", 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002895819 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.944 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.931147+0000", 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011635 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45782.0:156053 8.11 e347/328 ondisk, result = 0)", 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:32.928521+0000", 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0030766249999999999, 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033029689999999998, 2023-10-09T05:51:32.945 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.928521+0000", 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.928521+0000", 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:32.946 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.928521+0000", 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.928524+0000", 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7949999999999999e-06 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:32.947 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.928529+0000", 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6200000000000004e-06 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.928532+0000", 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8059999999999999e-06 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.931390+0000", 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028578689999999999 2023-10-09T05:51:32.948 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.931398+0000", 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1769999999999998e-06 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:156076 8.21 8.ae980161 (undecoded) ondisk+write+known_if_redirected e347)", 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:32.930713+0000", 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.000884777, 2023-10-09T05:51:32.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001166682, 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "queued for pg", 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 156076 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.930713+0000", 2023-10-09T05:51:32.950 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.930713+0000", 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.930711+0000", 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.951 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.930718+0000", 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8120000000000002e-06 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.930734+0000", 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5528000000000001e-05 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.930741+0000", 2023-10-09T05:51:32.952 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7999999999999999e-06 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:156053 8.11 8:8cfb616b:::benchmark_data_smithi008_432048_object53111:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:32.872221+0000", 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.059376426000000003, 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.059716142999999999, 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:32.953 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 156053 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.872221+0000", 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.954 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.872221+0000", 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.872220+0000", 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.872228+0000", 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1829999999999994e-06 2023-10-09T05:51:32.955 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.872249+0000", 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.126e-05 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.872258+0000", 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7439999999999996e-06 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.872372+0000", 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114299 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.872923+0000", 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00055042100000000005 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.914596+0000", 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.041673175999999999 2023-10-09T05:51:32.957 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.914709+0000", 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011283800000000001 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.915203+0000", 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000494594 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.925513+0000", 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010309926000000001 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.926381+0000", 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00086778699999999996 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.931407+0000", 2023-10-09T05:51:32.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050262140000000002 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:32.931460+0000", 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2989999999999999e-05 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 5 2023-10-09T05:51:32.960 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:51:32.996 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:33.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:33.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:33.001 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:32 smithi191 ceph-mon[161669]: failed to encode map e347 with expected crc 2023-10-09T05:51:33.002 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:32 smithi191 ceph-mon[161669]: failed to encode map e347 with expected crc 2023-10-09T05:51:33.002 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:32 smithi191 ceph-mon[161669]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T05:51:33.002 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:32 smithi191 ceph-mon[161669]: failed to encode map e347 with expected crc 2023-10-09T05:51:33.102 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:33.135 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:33.205 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:33.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:33.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:33.314 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:33.387 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:33.514 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:33.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:33.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:33.625 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:33.662 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:33.679 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:33 smithi008 ceph-mon[184418]: pgmap v1040: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24151/154188 objects degraded (15.663%); 1286/154188 objects misplaced (0.834%); 17 MiB/s, 264 objects/s recovering 2023-10-09T05:51:33.679 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24151/154188 objects degraded (15.663%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:33.736 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:33.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:33.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:33.845 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:33.907 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:33.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:33 smithi008 ceph-mon[191227]: pgmap v1040: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24151/154188 objects degraded (15.663%); 1286/154188 objects misplaced (0.834%); 17 MiB/s, 264 objects/s recovering 2023-10-09T05:51:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24151/154188 objects degraded (15.663%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:33 smithi191 ceph-mon[161669]: pgmap v1040: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24151/154188 objects degraded (15.663%); 1286/154188 objects misplaced (0.834%); 17 MiB/s, 264 objects/s recovering 2023-10-09T05:51:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24151/154188 objects degraded (15.663%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:34.517 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:34.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:34.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:34.628 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:34.666 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:35.124 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:35.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:35.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:35.229 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:35.264 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:35.330 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:35.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:35.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:35.438 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:35.492 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:35.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:35 smithi008 ceph-mon[191227]: pgmap v1041: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 23997/151068 objects degraded (15.885%); 1286/151068 objects misplaced (0.851%); 15 MiB/s, 246 objects/s recovering 2023-10-09T05:51:35.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:35 smithi008 ceph-mon[184418]: pgmap v1041: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 23997/151068 objects degraded (15.885%); 1286/151068 objects misplaced (0.851%); 15 MiB/s, 246 objects/s recovering 2023-10-09T05:51:36.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:35 smithi191 ceph-mon[161669]: pgmap v1041: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 23997/151068 objects degraded (15.885%); 1286/151068 objects misplaced (0.851%); 15 MiB/s, 246 objects/s recovering 2023-10-09T05:51:36.152 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:36.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:36.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:36.172 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:51:36.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:36.285 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:36.337 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:36.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:36.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:36.443 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:36.531 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:36.783 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:36.913 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:36.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:36.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:37.034 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:37.088 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:51:37.124 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:37.142 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:51:37.198 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:51:37.238 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:37.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:37.262 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:51:37.310 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:51:37.346 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:51:37.359 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:37.390 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:51:37.429 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:51:37.432 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:51:37.432 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:51:37.432 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:51:37.432 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:51:37.432 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:51:37.432 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:51:37.432 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:51:37.432 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:51:37.446 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:37.504 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:37.755 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:51:37.786 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:37 smithi008 ceph-mon[184418]: pgmap v1042: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 23644/146250 objects degraded (16.167%); 1184/146250 objects misplaced (0.810%); 14 MiB/s, 228 objects/s recovering 2023-10-09T05:51:37.786 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:37 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2479269912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:37.786 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:37 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1484448235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:37.786 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:37 smithi008 ceph-mon[191227]: pgmap v1042: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 23644/146250 objects degraded (16.167%); 1184/146250 objects misplaced (0.810%); 14 MiB/s, 228 objects/s recovering 2023-10-09T05:51:37.786 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:37 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2479269912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:37.786 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:37 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1484448235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:38.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:37 smithi191 ceph-mon[161669]: pgmap v1042: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 3.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 23644/146250 objects degraded (16.167%); 1184/146250 objects misplaced (0.810%); 14 MiB/s, 228 objects/s recovering 2023-10-09T05:51:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:37 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2479269912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:37 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1484448235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:38.815 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:38.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:38.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:38.923 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:38.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[184418]: failed to encode map e348 with expected crc 2023-10-09T05:51:38.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1484448235' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:38.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[184418]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T05:51:38.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[184418]: failed to encode map e348 with expected crc 2023-10-09T05:51:38.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23644/146250 objects degraded (16.167%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[191227]: failed to encode map e348 with expected crc 2023-10-09T05:51:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1484448235' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:38.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[191227]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T05:51:38.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[191227]: failed to encode map e348 with expected crc 2023-10-09T05:51:38.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23644/146250 objects degraded (16.167%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:39.003 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:38 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:38 smithi191 ceph-mon[161669]: failed to encode map e348 with expected crc 2023-10-09T05:51:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:38 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1484448235' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:38 smithi191 ceph-mon[161669]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T05:51:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:38 smithi191 ceph-mon[161669]: failed to encode map e348 with expected crc 2023-10-09T05:51:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23644/146250 objects degraded (16.167%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:40.344 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:40 smithi008 ceph-mon[191227]: pgmap v1044: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 22800/144021 objects degraded (15.831%); 1184/144021 objects misplaced (0.822%); 15 MiB/s, 240 objects/s recovering 2023-10-09T05:51:40.350 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:40 smithi008 ceph-mon[184418]: pgmap v1044: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 22800/144021 objects degraded (15.831%); 1184/144021 objects misplaced (0.822%); 15 MiB/s, 240 objects/s recovering 2023-10-09T05:51:40.355 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:40.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:40.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:40.475 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:40.516 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:40 smithi191 ceph-mon[161669]: pgmap v1044: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 22800/144021 objects degraded (15.831%); 1184/144021 objects misplaced (0.822%); 15 MiB/s, 240 objects/s recovering 2023-10-09T05:51:40.519 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:40.607 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:40.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:40.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:40.638 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:51:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:51:40] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.33.4" 2023-10-09T05:51:40.718 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:40.794 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:41.132 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:41.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:41.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:41.253 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:41.316 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:41.431 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:41.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:41.543 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:41.608 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:41.902 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:41.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:41.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:42.023 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:42.059 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:42.102 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T05:51:42.133 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:42.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:42.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:42.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:42 smithi008 ceph-mon[184418]: pgmap v1045: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 22662/140751 objects degraded (16.101%); 1184/140751 objects misplaced (0.841%); 14 MiB/s, 229 objects/s recovering 2023-10-09T05:51:42.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3882217134' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:51:42.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:42 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:51:42.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:42 smithi008 ceph-mon[191227]: pgmap v1045: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 22662/140751 objects degraded (16.101%); 1184/140751 objects misplaced (0.841%); 14 MiB/s, 229 objects/s recovering 2023-10-09T05:51:42.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3882217134' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:51:42.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:42 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:51:42.243 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:42.270 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:42.327 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:42.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:42 smithi191 ceph-mon[161669]: pgmap v1045: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 99 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 22662/140751 objects degraded (16.101%); 1184/140751 objects misplaced (0.841%); 14 MiB/s, 229 objects/s recovering 2023-10-09T05:51:42.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3882217134' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:51:42.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:42 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:51:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:42.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:42.376 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:51:42.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:42.459 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:42.484 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:51:42.502 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:42.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:42.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:42.611 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:42.642 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:42.877 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:51:42.877 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:51:42.899 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:42.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:42.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:43.002 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:43.031 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:43.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[191227]: failed to encode map e349 with expected crc 2023-10-09T05:51:43.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:51:43.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[191227]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T05:51:43.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[191227]: failed to encode map e349 with expected crc 2023-10-09T05:51:43.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[191227]: failed to encode map e349 with expected crc 2023-10-09T05:51:43.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 22662/140751 objects degraded (16.101%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:43.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[191227]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T05:51:43.358 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[184418]: failed to encode map e349 with expected crc 2023-10-09T05:51:43.358 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:51:43.358 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[184418]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T05:51:43.358 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[184418]: failed to encode map e349 with expected crc 2023-10-09T05:51:43.358 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[184418]: failed to encode map e349 with expected crc 2023-10-09T05:51:43.358 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 22662/140751 objects degraded (16.101%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:43.358 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:43 smithi008 ceph-mon[184418]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T05:51:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:43 smithi191 ceph-mon[161669]: failed to encode map e349 with expected crc 2023-10-09T05:51:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:43 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:51:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:43 smithi191 ceph-mon[161669]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T05:51:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:43 smithi191 ceph-mon[161669]: failed to encode map e349 with expected crc 2023-10-09T05:51:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:43 smithi191 ceph-mon[161669]: failed to encode map e349 with expected crc 2023-10-09T05:51:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 22662/140751 objects degraded (16.101%), 18 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:43 smithi191 ceph-mon[161669]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T05:51:44.161 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:44.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:44.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:44.246 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T05:51:44.266 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:44.323 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:51:44.325 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:44.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[184418]: pgmap v1047: 119 pgs: 5 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 18189/134721 objects degraded (13.501%); 1097/134721 objects misplaced (0.814%); 17 MiB/s, 275 objects/s recovering 2023-10-09T05:51:44.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[184418]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[184418]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[184418]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[184418]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[184418]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[184418]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[184418]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[184418]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.413 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:44.413 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[184418]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T05:51:44.413 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[191227]: pgmap v1047: 119 pgs: 5 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 18189/134721 objects degraded (13.501%); 1097/134721 objects misplaced (0.814%); 17 MiB/s, 275 objects/s recovering 2023-10-09T05:51:44.413 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[191227]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.413 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[191227]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.413 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[191227]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.413 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[191227]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.413 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[191227]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.413 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[191227]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.413 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[191227]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.414 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[191227]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.414 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:44.414 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:44 smithi008 ceph-mon[191227]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T05:51:44.450 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:44.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:44.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:44.533 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:44 smithi191 ceph-mon[161669]: pgmap v1047: 119 pgs: 5 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 18189/134721 objects degraded (13.501%); 1097/134721 objects misplaced (0.814%); 17 MiB/s, 275 objects/s recovering 2023-10-09T05:51:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:44 smithi191 ceph-mon[161669]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:44 smithi191 ceph-mon[161669]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:44 smithi191 ceph-mon[161669]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:44 smithi191 ceph-mon[161669]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:44 smithi191 ceph-mon[161669]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:44 smithi191 ceph-mon[161669]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:44 smithi191 ceph-mon[161669]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:44 smithi191 ceph-mon[161669]: failed to encode map e350 with expected crc 2023-10-09T05:51:44.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:44.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:44 smithi191 ceph-mon[161669]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T05:51:44.563 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:44.600 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:44.600 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:44.664 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:44.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:44.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:44.773 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:44.804 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:44.866 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:44.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:44.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:44.973 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:45.052 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:45 smithi008 ceph-mon[184418]: failed to encode map e351 with expected crc 2023-10-09T05:51:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:45 smithi008 ceph-mon[184418]: failed to encode map e351 with expected crc 2023-10-09T05:51:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:45 smithi008 ceph-mon[184418]: failed to encode map e351 with expected crc 2023-10-09T05:51:45.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:45 smithi008 ceph-mon[191227]: failed to encode map e351 with expected crc 2023-10-09T05:51:45.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:45 smithi008 ceph-mon[191227]: failed to encode map e351 with expected crc 2023-10-09T05:51:45.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:45 smithi008 ceph-mon[191227]: failed to encode map e351 with expected crc 2023-10-09T05:51:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:45 smithi191 ceph-mon[161669]: failed to encode map e351 with expected crc 2023-10-09T05:51:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:45 smithi191 ceph-mon[161669]: failed to encode map e351 with expected crc 2023-10-09T05:51:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:45 smithi191 ceph-mon[161669]: failed to encode map e351 with expected crc 2023-10-09T05:51:45.355 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:45.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:45.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:45.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:45.495 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:45.558 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:45.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:45.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:45.668 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:45.761 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:46 smithi008 ceph-mon[184418]: pgmap v1050: 119 pgs: 3 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovery_wait, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 95 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 24609/132558 objects degraded (18.565%); 1097/132558 objects misplaced (0.828%); 13 MiB/s, 202 objects/s recovering 2023-10-09T05:51:46.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:46 smithi008 ceph-mon[191227]: pgmap v1050: 119 pgs: 3 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovery_wait, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 95 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 24609/132558 objects degraded (18.565%); 1097/132558 objects misplaced (0.828%); 13 MiB/s, 202 objects/s recovering 2023-10-09T05:51:46.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:46 smithi191 ceph-mon[161669]: pgmap v1050: 119 pgs: 3 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovery_wait, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 95 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 24609/132558 objects degraded (18.565%); 1097/132558 objects misplaced (0.828%); 13 MiB/s, 202 objects/s recovering 2023-10-09T05:51:47.149 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:47.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:47.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:47.258 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:47.293 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:47.306 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:47.346 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:47.362 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:47.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:47.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:47.471 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:47.554 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:47.825 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:47.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:47.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:47.937 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:47.981 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:48.050 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:48.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:48.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:48.114 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:48 smithi008 ceph-mon[191227]: pgmap v1051: 119 pgs: 3 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 95 active+clean; 2.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24392/129327 objects degraded (18.861%); 1005/129327 objects misplaced (0.777%); 14 MiB/s, 230 objects/s recovering 2023-10-09T05:51:48.115 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24392/129327 objects degraded (18.861%), 16 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:48.157 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:48.251 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:48 smithi191 ceph-mon[161669]: pgmap v1051: 119 pgs: 3 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 95 active+clean; 2.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24392/129327 objects degraded (18.861%); 1005/129327 objects misplaced (0.777%); 14 MiB/s, 230 objects/s recovering 2023-10-09T05:51:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24392/129327 objects degraded (18.861%), 16 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:48.380 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:48 smithi008 ceph-mon[184418]: pgmap v1051: 119 pgs: 3 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+recovering, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 95 active+clean; 2.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24392/129327 objects degraded (18.861%); 1005/129327 objects misplaced (0.777%); 14 MiB/s, 230 objects/s recovering 2023-10-09T05:51:48.380 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24392/129327 objects degraded (18.861%), 16 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:48.416 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:48.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:48.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:48.473 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:51:48.521 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:48.534 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:51:48.570 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:48.584 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:51:48.641 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:48.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:48.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:48.652 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:51:48.692 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:51:48.745 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:51:48.751 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:48.794 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:51:48.843 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:51:48.847 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:48.863 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:51:48.863 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:51:48.863 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:51:48.863 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:51:48.864 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:51:48.864 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:51:48.864 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:51:48.864 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:51:49.194 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:51:49.209 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/740115770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:49.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/722371444' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:49.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:49 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:49.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/740115770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:49.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/722371444' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:49.212 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:49 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:49.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/740115770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/722371444' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:49 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:50.325 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:50.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:50.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:50.432 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[184418]: pgmap v1052: 119 pgs: 3 activating+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 36689/127920 objects degraded (28.681%); 1005/127920 objects misplaced (0.786%); 13 MiB/s, 206 objects/s recovering 2023-10-09T05:51:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[184418]: failed to encode map e352 with expected crc 2023-10-09T05:51:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:51:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[184418]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T05:51:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[184418]: failed to encode map e352 with expected crc 2023-10-09T05:51:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[191227]: pgmap v1052: 119 pgs: 3 activating+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 36689/127920 objects degraded (28.681%); 1005/127920 objects misplaced (0.786%); 13 MiB/s, 206 objects/s recovering 2023-10-09T05:51:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[191227]: failed to encode map e352 with expected crc 2023-10-09T05:51:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:51:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[191227]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T05:51:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:50 smithi008 ceph-mon[191227]: failed to encode map e352 with expected crc 2023-10-09T05:51:50.473 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:50 smithi191 ceph-mon[161669]: pgmap v1052: 119 pgs: 3 activating+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 36689/127920 objects degraded (28.681%); 1005/127920 objects misplaced (0.786%); 13 MiB/s, 206 objects/s recovering 2023-10-09T05:51:50.474 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:50 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:50.474 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:50 smithi191 ceph-mon[161669]: failed to encode map e352 with expected crc 2023-10-09T05:51:50.474 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:50 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:51:50.474 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:50 smithi191 ceph-mon[161669]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T05:51:50.474 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:50 smithi191 ceph-mon[161669]: failed to encode map e352 with expected crc 2023-10-09T05:51:50.475 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:50.533 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:50.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:50.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:50.641 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:50.699 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:50.768 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:51:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:51:50] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.33.4" 2023-10-09T05:51:50.774 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:50.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:50.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:50.890 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:50.921 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:52.130 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:52.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:52.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:52.234 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:52.270 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:52.404 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:52.407 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:52 smithi191 ceph-mon[161669]: pgmap v1054: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 36547/124554 objects degraded (29.342%); 1005/124554 objects misplaced (0.807%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T05:51:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:52.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:52.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:52 smithi008 ceph-mon[184418]: pgmap v1054: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 36547/124554 objects degraded (29.342%); 1005/124554 objects misplaced (0.807%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T05:51:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:52 smithi008 ceph-mon[191227]: pgmap v1054: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 36547/124554 objects degraded (29.342%); 1005/124554 objects misplaced (0.807%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T05:51:52.509 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:52.585 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:52.810 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:52.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:52.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:52.925 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:52.965 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:53.017 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:53.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:53.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:53.108 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:51:53.109 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:51:53.109 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:51:53.109 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:51:53.109 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.109 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102132 7.6 7:6028800a:::benchmark_data_smithi008_354649_object7636:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T05:51:53.109 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.878610+0000", 2023-10-09T05:51:53.109 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 514.22314321500005, 2023-10-09T05:51:53.109 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3895780630000001, 2023-10-09T05:51:53.109 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.109 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:51:53.109 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.110 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.110 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.110 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102132 2023-10-09T05:51:53.110 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.110 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.110 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.110 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.110 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878610+0000", 2023-10-09T05:51:53.110 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.110 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.110 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878610+0000", 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878608+0000", 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.111 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878615+0000", 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6379999999999998e-06 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878633+0000", 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7983999999999999e-05 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878642+0000", 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5200000000000003e-06 2023-10-09T05:51:53.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.879024+0000", 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00038146000000000002 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.879140+0000", 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000116616 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.113 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.879378+0000", 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00023714499999999999 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887275+0000", 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078973129999999996 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905106+0000", 2023-10-09T05:51:53.114 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017830736 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905119+0000", 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3549e-05 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877502+0000", 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97238259000000005 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877512+0000", 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0355e-05 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.239697+0000", 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3621850200000001 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:51:53.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.239848+0000", 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015051299999999999 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268057+0000", 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028209238000000001 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268188+0000", 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013106500000000001 2023-10-09T05:51:53.117 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.118 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.118 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.118 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102135 7.6 7:616fc77f:::benchmark_data_smithi008_354649_object7639:head [delete] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-10-09T05:51:53.118 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.878660+0000", 2023-10-09T05:51:53.118 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 514.22309312799996, 2023-10-09T05:51:53.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4043974380000002, 2023-10-09T05:51:53.118 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.118 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:51:53.118 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102135 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878660+0000", 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.119 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878660+0000", 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878659+0000", 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878663+0000", 2023-10-09T05:51:53.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.242e-06 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878670+0000", 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0779999999999998e-06 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878676+0000", 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9320000000000001e-06 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.122 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.122 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.883274+0000", 2023-10-09T05:51:53.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045982369999999998 2023-10-09T05:51:53.122 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.122 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.122 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.122 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.883418+0000", 2023-10-09T05:51:53.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014399399999999999 2023-10-09T05:51:53.122 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.122 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.122 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.883540+0000", 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000122225 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887783+0000", 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042433180000000003 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905157+0000", 2023-10-09T05:51:53.123 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017373588999999998 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905174+0000", 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6509999999999999e-05 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877554+0000", 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97238062300000006 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.124 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877564+0000", 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6269999999999995e-06 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243556+0000", 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3659919819999999 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:51:53.125 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243708+0000", 2023-10-09T05:51:53.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015254099999999999 2023-10-09T05:51:53.126 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.126 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.126 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.126 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.282943+0000", 2023-10-09T05:51:53.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.039234640000000001 2023-10-09T05:51:53.126 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.126 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.126 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.126 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283057+0000", 2023-10-09T05:51:53.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114479 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102138 7.6 7:69d2abd0:::benchmark_data_smithi008_354649_object7642:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 514.223071914, 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3933153759999999, 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.127 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102138 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878681+0000", 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878684+0000", 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9849999999999998e-06 2023-10-09T05:51:53.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878689+0000", 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6910000000000002e-06 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.878697+0000", 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2899999999999997e-06 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887233+0000", 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0085366939999999992 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op must wait for map", 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.887240+0000", 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0169999999999999e-06 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905215+0000", 2023-10-09T05:51:53.131 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017974716000000002 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905223+0000", 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7360000000000001e-06 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877592+0000", 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97236955700000005 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.132 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877601+0000", 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2449999999999998e-06 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243738+0000", 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.366137529 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243861+0000", 2023-10-09T05:51:53.134 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012260400000000001 2023-10-09T05:51:53.134 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.134 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.134 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:51:53.134 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.244012+0000", 2023-10-09T05:51:53.134 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015064899999999999 2023-10-09T05:51:53.134 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.134 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.134 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.134 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.257726+0000", 2023-10-09T05:51:53.134 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013714637999999999 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268231+0000", 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010504434999999999 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268269+0000", 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7939999999999999e-05 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268316+0000", 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7679000000000002e-05 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.271997+0000", 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036803859999999999 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.136 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102144 7.6 7:6ad058ac:::benchmark_data_smithi008_354649_object7648:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.899418+0000", 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 514.20233516300004, 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3874670130000002, 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102144 2023-10-09T05:51:53.137 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.138 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.138 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.138 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899418+0000", 2023-10-09T05:51:53.138 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.138 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.138 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.138 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899418+0000", 2023-10-09T05:51:53.138 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.138 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899417+0000", 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899422+0000", 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7659999999999998e-06 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.139 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899430+0000", 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6100000000000006e-06 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.899438+0000", 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0760000000000003e-06 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905297+0000", 2023-10-09T05:51:53.140 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058583560000000003 2023-10-09T05:51:53.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.141 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.141 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.905305+0000", 2023-10-09T05:51:53.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0090000000000007e-06 2023-10-09T05:51:53.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.141 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.141 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877625+0000", 2023-10-09T05:51:53.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.972320462 2023-10-09T05:51:53.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877633+0000", 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8339999999999999e-06 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.250892+0000", 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.373259207 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.251058+0000", 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016594400000000001 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.251234+0000", 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000175453 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268021+0000", 2023-10-09T05:51:53.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016787834000000001 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283099+0000", 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015077174 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283181+0000", 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2218000000000005e-05 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283226+0000", 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5181e-05 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.286885+0000", 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036590279999999999 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.145 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.146 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102153 7.6 7:672a3339:::benchmark_data_smithi008_354649_object7657:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:51:53.146 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.906435+0000", 2023-10-09T05:51:53.146 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 514.19531774400002, 2023-10-09T05:51:53.146 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3840327299999999, 2023-10-09T05:51:53.146 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.146 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.146 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.146 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.146 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.146 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102153 2023-10-09T05:51:53.146 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906435+0000", 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906435+0000", 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.148 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.149 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906433+0000", 2023-10-09T05:51:53.149 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:53.149 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.149 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.149 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.149 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906441+0000", 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4569999999999999e-06 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906458+0000", 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7399000000000001e-05 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906470+0000", 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2092e-05 2023-10-09T05:51:53.150 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906521+0000", 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0983999999999998e-05 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.906533+0000", 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1977000000000001e-05 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.151 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877706+0000", 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97117302299999997 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.877717+0000", 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0254e-05 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.257761+0000", 2023-10-09T05:51:53.152 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3800444709999999 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.257940+0000", 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000179037 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.258090+0000", 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000150161 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.153 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.154 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.268184+0000", 2023-10-09T05:51:53.154 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010093352999999999 2023-10-09T05:51:53.154 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.154 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.154 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.154 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.283137+0000", 2023-10-09T05:51:53.154 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014953404999999999 2023-10-09T05:51:53.154 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.154 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.154 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.154 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.286959+0000", 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038219339999999999 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.287005+0000", 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.549e-05 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.290468+0000", 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034635989999999999 2023-10-09T05:51:53.155 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102170 7.f 7:f2cf7216:::benchmark_data_smithi008_354649_object7674:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.931713+0000", 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 514.170040384, 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.30407954, 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.156 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102170 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931713+0000", 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.157 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931713+0000", 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931711+0000", 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931717+0000", 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4029999999999998e-06 2023-10-09T05:51:53.158 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931726+0000", 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7409999999999998e-06 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931731+0000", 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3310000000000003e-06 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.159 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931790+0000", 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9153000000000001e-05 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.931804+0000", 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4075e-05 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.876813+0000", 2023-10-09T05:51:53.160 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94500846400000005 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.876824+0000", 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.184e-05 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.222418+0000", 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3455932580000001 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.161 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.222572+0000", 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000154277 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.222670+0000", 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8070000000000001e-05 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.162 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230124+0000", 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007453515 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230200+0000", 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6130000000000005e-05 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.231464+0000", 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0012646619999999999 2023-10-09T05:51:53.163 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.231507+0000", 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2215000000000002e-05 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.235792+0000", 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042857040000000004 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.164 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125678 7.6 7:63397418:::benchmark_data_smithi008_354649_object31182:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.999075+0000", 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 470.10267859300001, 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0649156319999999, 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.165 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125678 2023-10-09T05:51:53.166 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.166 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.166 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.166 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.166 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999075+0000", 2023-10-09T05:51:53.166 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.166 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.166 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.166 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999075+0000", 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999073+0000", 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999080+0000", 2023-10-09T05:51:53.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3900000000000004e-06 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999095+0000", 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4596999999999999e-05 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.999108+0000", 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2767e-05 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.168 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.169 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006313+0000", 2023-10-09T05:51:53.169 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0072051479999999998 2023-10-09T05:51:53.169 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.169 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.169 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op must wait for map", 2023-10-09T05:51:53.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006322+0000", 2023-10-09T05:51:53.169 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6799999999999999e-06 2023-10-09T05:51:53.169 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.170 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.170 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.170 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.383274+0000", 2023-10-09T05:51:53.170 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37695257599999998 2023-10-09T05:51:53.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.170 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.170 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.170 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.383295+0000", 2023-10-09T05:51:53.170 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0871999999999999e-05 2023-10-09T05:51:53.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.170 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.171 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.171 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.969340+0000", 2023-10-09T05:51:53.171 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.58604520999999998 2023-10-09T05:51:53.171 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.171 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.171 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.171 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.969352+0000", 2023-10-09T05:51:53.171 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1914e-05 2023-10-09T05:51:53.171 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.171 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.171 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.172 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.033375+0000", 2023-10-09T05:51:53.172 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0640223849999999 2023-10-09T05:51:53.172 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.172 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.172 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:51:53.172 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.033498+0000", 2023-10-09T05:51:53.172 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000123614 2023-10-09T05:51:53.172 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.172 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.063936+0000", 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030438004000000001 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.063990+0000", 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3983000000000001e-05 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.174 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.174 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150726 7.4 7:2606d17b:::benchmark_data_smithi008_354649_object56230:head [delete] snapc 0=[] ondisk+write+known_if_redirected e251)", 2023-10-09T05:51:53.174 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:16.967446+0000", 2023-10-09T05:51:53.174 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 396.134307347, 2023-10-09T05:51:53.174 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.266506991, 2023-10-09T05:51:53.174 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.174 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.174 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150726 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967446+0000", 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.176 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967446+0000", 2023-10-09T05:51:53.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.176 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.176 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.176 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.176 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967445+0000", 2023-10-09T05:51:53.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:53.176 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.176 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.177 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967448+0000", 2023-10-09T05:51:53.177 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.942e-06 2023-10-09T05:51:53.177 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.177 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967454+0000", 2023-10-09T05:51:53.177 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8379999999999998e-06 2023-10-09T05:51:53.177 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.178 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967460+0000", 2023-10-09T05:51:53.178 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7620000000000001e-06 2023-10-09T05:51:53.178 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.178 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.178 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.178 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.984917+0000", 2023-10-09T05:51:53.178 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017457026 2023-10-09T05:51:53.178 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.178 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.179 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.179 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.984930+0000", 2023-10-09T05:51:53.179 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3217000000000001e-05 2023-10-09T05:51:53.179 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.179 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.179 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.179 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.923010+0000", 2023-10-09T05:51:53.179 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93808012799999996 2023-10-09T05:51:53.179 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.180 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.180 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.923019+0000", 2023-10-09T05:51:53.180 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6230000000000003e-06 2023-10-09T05:51:53.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.180 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.180 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.216263+0000", 2023-10-09T05:51:53.180 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2932439330000001 2023-10-09T05:51:53.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.181 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.181 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.216410+0000", 2023-10-09T05:51:53.181 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000146861 2023-10-09T05:51:53.181 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.181 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.181 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:51:53.181 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.216920+0000", 2023-10-09T05:51:53.181 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00051030300000000004 2023-10-09T05:51:53.181 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.182 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.182 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.182 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.224475+0000", 2023-10-09T05:51:53.182 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0075551230000000004 2023-10-09T05:51:53.182 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.182 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.182 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.182 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.224528+0000", 2023-10-09T05:51:53.182 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.295e-05 2023-10-09T05:51:53.182 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.233016+0000", 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0084877560000000008 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.233080+0000", 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4058000000000002e-05 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.183 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.233953+0000", 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00087297799999999995 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150729 7.9 7:91e0d910:::benchmark_data_smithi008_354649_object56233:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:16.994744+0000", 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 396.10700945399998, 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.138625711, 2023-10-09T05:51:53.184 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150729 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994744+0000", 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994744+0000", 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994742+0000", 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994749+0000", 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.421e-06 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994762+0000", 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3101999999999999e-05 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994771+0000", 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0070000000000003e-06 2023-10-09T05:51:53.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994839+0000", 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7821e-05 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994853+0000", 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4782e-05 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917318+0000", 2023-10-09T05:51:53.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92246414300000001 2023-10-09T05:51:53.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917330+0000", 2023-10-09T05:51:53.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2471e-05 2023-10-09T05:51:53.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.116457+0000", 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.199127415 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.116616+0000", 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015812399999999999 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.117033+0000", 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00041699599999999999 2023-10-09T05:51:53.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.123489+0000", 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064567210000000003 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.125273+0000", 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001783722 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.191 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.132228+0000", 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0069553769999999996 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.132292+0000", 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3936999999999994e-05 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.133369+0000", 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001077103 2023-10-09T05:51:53.192 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150734 7.14 7:2e650744:::benchmark_data_smithi008_354649_object56238:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.025678+0000", 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 396.07607503999998, 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.200712695, 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.193 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150734 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025678+0000", 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.194 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025678+0000", 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025676+0000", 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025686+0000", 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2490000000000007e-06 2023-10-09T05:51:53.195 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025700+0000", 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4002000000000001e-05 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025708+0000", 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4409999999999993e-06 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.196 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025787+0000", 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8909e-05 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025804+0000", 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6718000000000002e-05 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922949+0000", 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89714522100000005 2023-10-09T05:51:53.197 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922958+0000", 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3349999999999996e-06 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.210519+0000", 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2875604629999999 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.198 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.210670+0000", 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015123 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.210978+0000", 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00030860600000000001 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.217348+0000", 2023-10-09T05:51:53.199 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0063698720000000004 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.217358+0000", 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3179999999999996e-06 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.225201+0000", 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078434549999999992 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.225268+0000", 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7101999999999996e-05 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.226391+0000", 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001122556 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.201 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.202 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150735 7.9 7:92c6da78:::benchmark_data_smithi008_354649_object56239:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:51:53.202 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.026350+0000", 2023-10-09T05:51:53.202 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 396.07540322599999, 2023-10-09T05:51:53.202 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1077192149999999, 2023-10-09T05:51:53.202 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.202 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.202 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150735 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026350+0000", 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.203 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.204 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026350+0000", 2023-10-09T05:51:53.204 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.204 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.204 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.204 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.204 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026349+0000", 2023-10-09T05:51:53.204 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:51:53.204 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.204 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.204 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.205 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026354+0000", 2023-10-09T05:51:53.205 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4159999999999997e-06 2023-10-09T05:51:53.205 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.205 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.205 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.205 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026363+0000", 2023-10-09T05:51:53.205 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4760000000000004e-06 2023-10-09T05:51:53.205 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.205 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.205 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.205 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026368+0000", 2023-10-09T05:51:53.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8380000000000001e-06 2023-10-09T05:51:53.206 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.206 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.206 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.206 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026430+0000", 2023-10-09T05:51:53.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2142000000000002e-05 2023-10-09T05:51:53.206 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.206 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.206 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.206 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026444+0000", 2023-10-09T05:51:53.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4007e-05 2023-10-09T05:51:53.207 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.207 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.207 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.207 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917360+0000", 2023-10-09T05:51:53.207 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89091560000000003 2023-10-09T05:51:53.207 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.207 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.207 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.207 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917368+0000", 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4489999999999999e-06 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.121241+0000", 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.203872729 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.121389+0000", 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014778299999999999 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.208 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.209 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:51:53.209 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.121817+0000", 2023-10-09T05:51:53.209 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00042842200000000003 2023-10-09T05:51:53.209 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.209 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.209 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.209 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.125281+0000", 2023-10-09T05:51:53.209 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034634980000000002 2023-10-09T05:51:53.209 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.209 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.209 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.126920+0000", 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016398389999999999 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.133435+0000", 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0065145170000000001 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.133481+0000", 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6368000000000001e-05 2023-10-09T05:51:53.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.134069+0000", 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00058786500000000002 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150739 7.d 7:b35c0714:::benchmark_data_smithi008_354649_object56243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.167284+0000", 2023-10-09T05:51:53.211 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 395.934469272, 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.022290125, 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150739 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.212 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167284+0000", 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167284+0000", 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167282+0000", 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.214 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.214 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167290+0000", 2023-10-09T05:51:53.214 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9489999999999996e-06 2023-10-09T05:51:53.214 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.214 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.214 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.214 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167307+0000", 2023-10-09T05:51:53.214 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7193e-05 2023-10-09T05:51:53.214 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.214 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.214 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167316+0000", 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0899999999999994e-06 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167435+0000", 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000119153 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.167447+0000", 2023-10-09T05:51:53.215 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1272e-05 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922254+0000", 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75480757799999998 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922266+0000", 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1959e-05 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.216 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.131723+0000", 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.20945652 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.135226+0000", 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003503515 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.135659+0000", 2023-10-09T05:51:53.217 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00043321599999999999 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.165146+0000", 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.029487005 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.165190+0000", 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3157e-05 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.218 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.168957+0000", 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037669999999999999 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.189402+0000", 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020445218000000001 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.189455+0000", 2023-10-09T05:51:53.219 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3368000000000001e-05 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.189574+0000", 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0001189 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150745 7.14 7:2dacea96:::benchmark_data_smithi008_354649_object56249:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:51:53.220 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.262460+0000", 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 395.83929329400002, 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9738604209999999, 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150745 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.221 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262460+0000", 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262460+0000", 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262458+0000", 2023-10-09T05:51:53.222 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262466+0000", 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1210000000000007e-06 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262608+0000", 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014194000000000001 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.223 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262615+0000", 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2599999999999999e-06 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262665+0000", 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9876000000000002e-05 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.224 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.262673+0000", 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1780000000000003e-06 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922980+0000", 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.66030662399999995 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922988+0000", 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3159999999999997e-06 2023-10-09T05:51:53.225 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.214973+0000", 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2919844110000001 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.215142+0000", 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016878100000000001 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.215556+0000", 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000414208 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.218997+0000", 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034408960000000001 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.224502+0000", 2023-10-09T05:51:53.227 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0055058310000000001 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.233025+0000", 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0085221529999999993 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.235169+0000", 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002143874 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.228 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.236320+0000", 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001151745 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180271 7.1d 7:baeacd59:::benchmark_data_smithi008_354649_object85775:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 311.972240817, 2023-10-09T05:51:53.229 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.855218754, 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180271 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:51:53.230 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129509+0000", 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.231 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129525+0000", 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5430999999999998e-05 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129553+0000", 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8175e-05 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129568+0000", 2023-10-09T05:51:53.232 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5267000000000001e-05 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.333120+0000", 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20355221500000001 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.333134+0000", 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4073000000000001e-05 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.233 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929537+0000", 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5964021320000001 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929548+0000", 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1759999999999999e-05 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.234 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.974829+0000", 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.045281066 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.974986+0000", 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000156174 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.975339+0000", 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00035308699999999998 2023-10-09T05:51:53.235 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977566+0000", 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022271399999999998 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984241+0000", 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066755160000000003 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984422+0000", 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000180633 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984530+0000", 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010849999999999999 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984609+0000", 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8506999999999999e-05 2023-10-09T05:51:53.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984731+0000", 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012211599999999999 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180280 7.1d 7:b9a77225:::benchmark_data_smithi008_354649_object85784:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:51:53.238 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 311.739893107, 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6230030800000002, 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180280 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:51:53.239 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361859+0000", 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361865+0000", 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9800000000000003e-06 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361876+0000", 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1178999999999999e-05 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361882+0000", 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0070000000000003e-06 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361924+0000", 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2243999999999997e-05 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361931+0000", 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.097e-06 2023-10-09T05:51:53.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929573+0000", 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.567641397 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929580+0000", 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4569999999999999e-06 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.243 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977745+0000", 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0481653790000001 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977840+0000", 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4581999999999997e-05 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.978096+0000", 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00025544499999999998 2023-10-09T05:51:53.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984291+0000", 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061956720000000002 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984301+0000", 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7710000000000001e-06 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984457+0000", 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000155885 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984780+0000", 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00032328700000000001 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984806+0000", 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6290000000000001e-05 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984863+0000", 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6693000000000001e-05 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180292 7.1f 7:fc8bce44:::benchmark_data_smithi008_354649_object85796:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 311.574267152, 2023-10-09T05:51:53.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4946860599999998, 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180292 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527485+0000", 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527491+0000", 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.139e-06 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527504+0000", 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2964e-05 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527512+0000", 2023-10-09T05:51:53.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2230000000000007e-06 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527553+0000", 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1266000000000001e-05 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527560+0000", 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6599999999999998e-06 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929860+0000", 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4023002959999999 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929869+0000", 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9080000000000001e-06 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986248+0000", 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0563782779999999 2023-10-09T05:51:53.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986407+0000", 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000159588 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986738+0000", 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00033078600000000001 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.253 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.015271+0000", 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028533060999999998 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.015291+0000", 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0077000000000001e-05 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.021759+0000", 2023-10-09T05:51:53.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064680570000000001 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.021808+0000", 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8615000000000001e-05 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.022172+0000", 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00036433100000000002 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.255 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180305 7.1d 7:bd26cfd0:::benchmark_data_smithi008_354649_object85809:head [delete] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.064410+0000", 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 311.037343604, 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.920595904, 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.256 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180305 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064410+0000", 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064410+0000", 2023-10-09T05:51:53.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064408+0000", 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064413+0000", 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6400000000000002e-06 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064423+0000", 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7489999999999993e-06 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064430+0000", 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8399999999999997e-06 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.259 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064529+0000", 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9314000000000005e-05 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.064537+0000", 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0450000000000001e-06 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929599+0000", 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86506175299999999 2023-10-09T05:51:53.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929606+0000", 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0119999999999999e-06 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.979407+0000", 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0498009509999999 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.979540+0000", 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000132836 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.979783+0000", 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000243329 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984308+0000", 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045243610000000002 2023-10-09T05:51:53.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984349+0000", 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1041999999999998e-05 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984496+0000", 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014723000000000001 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.263 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984921+0000", 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00042504100000000003 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984948+0000", 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6665e-05 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.985005+0000", 2023-10-09T05:51:53.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7825999999999998e-05 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180323 7.1d 7:bef99664:::benchmark_data_smithi008_354649_object85827:head [delete] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.107250+0000", 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 310.99450330000002, 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8812351980000002, 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.265 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180323 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107250+0000", 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107250+0000", 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107248+0000", 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107255+0000", 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.708e-06 2023-10-09T05:51:53.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107267+0000", 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2282e-05 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107275+0000", 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8979999999999996e-06 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107338+0000", 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3078999999999997e-05 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.107348+0000", 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9119999999999993e-06 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929626+0000", 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.82227738500000003 2023-10-09T05:51:53.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929632+0000", 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6510000000000004e-06 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.981128+0000", 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.051495222 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.981256+0000", 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000128782 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.981435+0000", 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000179144 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984314+0000", 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002878598 2023-10-09T05:51:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984387+0000", 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2719000000000006e-05 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.988300+0000", 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039127140000000003 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.988334+0000", 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4264000000000003e-05 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.988376+0000", 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2225000000000003e-05 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.988485+0000", 2023-10-09T05:51:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010901899999999999 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180353 7.1d 7:b94e9c32:::benchmark_data_smithi008_354649_object85857:head [delete] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:42.161196+0000", 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 310.94055729799999, 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8349015990000002, 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180353 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161196+0000", 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161196+0000", 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161194+0000", 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161202+0000", 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6799999999999993e-06 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161216+0000", 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3688e-05 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161224+0000", 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4319999999999999e-06 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161368+0000", 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014366800000000001 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.161380+0000", 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2149e-05 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929653+0000", 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.76827273799999996 2023-10-09T05:51:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929660+0000", 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5000000000000002e-06 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.985043+0000", 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.055382222 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.985137+0000", 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4852000000000006e-05 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.985402+0000", 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000265087 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.992072+0000", 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066693619999999999 2023-10-09T05:51:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.992108+0000", 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6211000000000003e-05 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.995908+0000", 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037999230000000002 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.995923+0000", 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4630999999999999e-05 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.995976+0000", 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3701e-05 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.996097+0000", 2023-10-09T05:51:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012113299999999999 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119615 8.11 8:8e69f363:::benchmark_data_smithi008_432048_object16673:head [delete] snapc 0=[] ondisk+write+known_if_redirected e326)", 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:17.939578+0000", 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 95.162174882000002, 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8574401840000001, 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:51:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119615 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.939578+0000", 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:51:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.939578+0000", 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.939575+0000", 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.939591+0000", 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5444e-05 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.939621+0000", 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9750000000000001e-05 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.939635+0000", 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4674e-05 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.939709+0000", 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3788000000000006e-05 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.939892+0000", 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018249499999999999 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.939999+0000", 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010719500000000001 2023-10-09T05:51:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.943042+0000", 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030430510000000002 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.945029+0000", 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019875700000000001 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.951658+0000", 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066287430000000003 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:17.951674+0000", 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6211000000000001e-05 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.040963+0000", 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.089288359999999997 2023-10-09T05:51:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.040972+0000", 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4700000000000008e-06 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.729597+0000", 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.688625066 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:51:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.729685+0000", 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8168e-05 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.796946+0000", 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.067260600000000004 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.797018+0000", 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2495000000000003e-05 2023-10-09T05:51:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:51:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:51:53.292 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:51:53.292 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 36547/124554 objects degraded (29.342%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:53.385 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 36547/124554 objects degraded (29.342%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:53.385 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 36547/124554 objects degraded (29.342%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:53.387 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:53.503 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:53.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:53.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:53.610 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:53.664 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:53.759 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:53.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:53.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:53.862 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:53.889 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:53.933 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:53.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:53.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:54.037 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:54.061 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:51:54.064 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:54.136 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:54.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:54.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:54.162 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:51:54.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:54 smithi008 ceph-mon[184418]: pgmap v1055: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 35101/118107 objects degraded (29.720%); 921/118107 objects misplaced (0.780%); 15 MiB/s, 235 objects/s recovering 2023-10-09T05:51:54.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:54 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3579136550' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:51:54.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:54 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:51:54.232 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T05:51:54.243 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:54.291 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:54.369 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:54.376 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:54 smithi191 ceph-mon[161669]: pgmap v1055: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 35101/118107 objects degraded (29.720%); 921/118107 objects misplaced (0.780%); 15 MiB/s, 235 objects/s recovering 2023-10-09T05:51:54.376 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:54 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3579136550' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:51:54.376 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:54 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:51:54.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:54.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:54.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:54 smithi008 ceph-mon[191227]: pgmap v1055: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 35101/118107 objects degraded (29.720%); 921/118107 objects misplaced (0.780%); 15 MiB/s, 235 objects/s recovering 2023-10-09T05:51:54.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:54 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3579136550' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:51:54.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:54 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:51:54.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:54.505 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:54.608 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:54.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:54.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:54.715 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:54.747 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:54.829 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:54.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:54.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:54.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:54.988 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:55.000 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:51:55.074 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:55.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:55.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:55.184 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:55.334 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T05:51:55.343 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:55.542 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:55.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:55 smithi191 ceph-mon[161669]: failed to encode map e353 with expected crc 2023-10-09T05:51:55.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:55 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:51:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:55 smithi191 ceph-mon[161669]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T05:51:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:55 smithi191 ceph-mon[161669]: failed to encode map e353 with expected crc 2023-10-09T05:51:55.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:55 smithi191 ceph-mon[161669]: pgmap v1057: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 34959/115422 objects degraded (30.288%); 921/115422 objects misplaced (0.798%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:51:55.557 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:55 smithi008 ceph-mon[191227]: failed to encode map e353 with expected crc 2023-10-09T05:51:55.557 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:55 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:51:55.557 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:55 smithi008 ceph-mon[191227]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T05:51:55.557 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:55 smithi008 ceph-mon[191227]: failed to encode map e353 with expected crc 2023-10-09T05:51:55.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:55 smithi008 ceph-mon[191227]: pgmap v1057: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 34959/115422 objects degraded (30.288%); 921/115422 objects misplaced (0.798%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:51:55.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:55.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:55.569 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:55 smithi008 ceph-mon[184418]: failed to encode map e353 with expected crc 2023-10-09T05:51:55.570 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:55 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:51:55.570 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:55 smithi008 ceph-mon[184418]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T05:51:55.570 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:55 smithi008 ceph-mon[184418]: failed to encode map e353 with expected crc 2023-10-09T05:51:55.570 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:55 smithi008 ceph-mon[184418]: pgmap v1057: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 34959/115422 objects degraded (30.288%); 921/115422 objects misplaced (0.798%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:51:55.599 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:55.632 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:55.659 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:55.774 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:56.054 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:56.237 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:56.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:56.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:56.347 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:56.416 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:56.479 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:56.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:56.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:56.585 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:56.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:56 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:51:56.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:56 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:51:56.742 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:56.748 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:56 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:51:56.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:56.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:56.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:56.896 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:56.963 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:56.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:56.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:57.068 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:57.114 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:57.193 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:57.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:57.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:57.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:57.379 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:57.459 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:57.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:57.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:57.567 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:57.665 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:57.722 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:57.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:57.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:57 smithi191 ceph-mon[161669]: pgmap v1058: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 34667/111087 objects degraded (31.207%); 795/111087 objects misplaced (0.716%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:51:57.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:57.871 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:57.952 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:57.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:57.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:57.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:57 smithi008 ceph-mon[184418]: pgmap v1058: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 34667/111087 objects degraded (31.207%); 795/111087 objects misplaced (0.716%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:51:57.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:57 smithi008 ceph-mon[191227]: pgmap v1058: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 34667/111087 objects degraded (31.207%); 795/111087 objects misplaced (0.716%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:51:58.060 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:58.136 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 34667/111087 objects degraded (31.207%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:58.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 34667/111087 objects degraded (31.207%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:58.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 34667/111087 objects degraded (31.207%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:51:59.292 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:59.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:59.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:59.402 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:59.499 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:59.558 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:59 smithi008 ceph-mon[184418]: pgmap v1059: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 33530/108903 objects degraded (30.789%); 795/108903 objects misplaced (0.730%); 16 MiB/s, 249 objects/s recovering 2023-10-09T05:51:59.558 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:51:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:59.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:59 smithi008 ceph-mon[191227]: pgmap v1059: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 33530/108903 objects degraded (30.789%); 795/108903 objects misplaced (0.730%); 16 MiB/s, 249 objects/s recovering 2023-10-09T05:51:59.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:51:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:59.562 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:59.636 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:59.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:59.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:59.749 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:59.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:59 smithi191 ceph-mon[161669]: pgmap v1059: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 33530/108903 objects degraded (30.789%); 795/108903 objects misplaced (0.730%); 16 MiB/s, 249 objects/s recovering 2023-10-09T05:51:59.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:51:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:59.821 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:00.287 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:00.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:00.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:00.404 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:00.408 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:52:00.449 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:00.451 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:52:00.494 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:52:00.530 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:00.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:00.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:00.556 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:52:00.581 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:52:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:52:00] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-10-09T05:52:00.583 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:00 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:00.583 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/430163569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:00.588 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:00 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:00.589 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/430163569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:00.590 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:52:00.638 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:00.642 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:52:00.680 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:00.692 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:52:00.726 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:52:00.745 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:52:00.745 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:52:00.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:52:00.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:52:00.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:52:00.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:52:00.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:52:00.746 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:52:00.754 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:00.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:00.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:00.763 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:00 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:00.763 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/430163569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:00.773 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:00.773 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:00.863 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:00.997 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:01.215 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:01.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:01.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:01.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:01.363 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:01.463 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:01.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:01.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:01.555 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:52:01.577 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:01.617 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:01.617 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:01 smithi191 ceph-mon[161669]: pgmap v1060: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 33343/105375 objects degraded (31.642%); 795/105375 objects misplaced (0.754%); 15 MiB/s, 237 objects/s recovering 2023-10-09T05:52:01.618 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:01 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3481963226' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:01.742 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:01.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:01.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:01.855 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:01.914 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:01 smithi008 ceph-mon[184418]: pgmap v1060: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 33343/105375 objects degraded (31.642%); 795/105375 objects misplaced (0.754%); 15 MiB/s, 237 objects/s recovering 2023-10-09T05:52:01.914 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:01 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3481963226' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:01.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:01 smithi008 ceph-mon[191227]: pgmap v1060: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 33343/105375 objects degraded (31.642%); 795/105375 objects misplaced (0.754%); 15 MiB/s, 237 objects/s recovering 2023-10-09T05:52:01.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:01 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3481963226' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:01.923 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:02.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:02 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:52:02.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:02 smithi191 ceph-mon[161669]: failed to encode map e354 with expected crc 2023-10-09T05:52:02.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:02 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3481963226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:02.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:02 smithi191 ceph-mon[161669]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T05:52:02.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:02 smithi191 ceph-mon[161669]: failed to encode map e354 with expected crc 2023-10-09T05:52:02.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:02 smithi191 ceph-mon[161669]: failed to encode map e354 with expected crc 2023-10-09T05:52:02.919 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:52:02.919 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[184418]: failed to encode map e354 with expected crc 2023-10-09T05:52:02.920 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3481963226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:02.920 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[184418]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T05:52:02.920 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[184418]: failed to encode map e354 with expected crc 2023-10-09T05:52:02.920 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[184418]: failed to encode map e354 with expected crc 2023-10-09T05:52:02.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:52:02.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[191227]: failed to encode map e354 with expected crc 2023-10-09T05:52:02.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3481963226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:02.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[191227]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T05:52:02.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[191227]: failed to encode map e354 with expected crc 2023-10-09T05:52:02.921 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:02 smithi008 ceph-mon[191227]: failed to encode map e354 with expected crc 2023-10-09T05:52:03.772 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:52:03.787 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:03.792 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:03 smithi008 ceph-mon[191227]: pgmap v1062: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 31887/98724 objects degraded (32.299%); 673/98724 objects misplaced (0.682%); 18 MiB/s, 292 objects/s recovering 2023-10-09T05:52:03.793 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 31887/98724 objects degraded (32.299%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:03.793 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:03 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:03.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:03.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:03.795 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:03 smithi008 ceph-mon[184418]: pgmap v1062: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 31887/98724 objects degraded (32.299%); 673/98724 objects misplaced (0.682%); 18 MiB/s, 292 objects/s recovering 2023-10-09T05:52:03.796 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 31887/98724 objects degraded (32.299%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:03.796 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:03 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:03 smithi191 ceph-mon[161669]: pgmap v1062: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 31887/98724 objects degraded (32.299%); 673/98724 objects misplaced (0.682%); 18 MiB/s, 292 objects/s recovering 2023-10-09T05:52:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 31887/98724 objects degraded (32.299%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:03 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:03.895 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:03.937 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:04.441 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:04.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:04.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:04.559 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:04.646 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:04.965 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:04.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:04.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:05.076 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:05.080 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:05.184 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:05.486 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:05.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:05.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:05.578 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T05:52:05.592 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:05.665 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:05.665 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:52:05.729 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:05 smithi008 ceph-mon[184418]: pgmap v1063: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 31734/95646 objects degraded (33.179%); 673/95646 objects misplaced (0.704%); 15 MiB/s, 243 objects/s recovering 2023-10-09T05:52:05.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:05 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:05.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:05 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/163199796' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:52:05.730 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:05 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:52:05.730 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:05 smithi008 ceph-mon[191227]: pgmap v1063: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 31734/95646 objects degraded (33.179%); 673/95646 objects misplaced (0.704%); 15 MiB/s, 243 objects/s recovering 2023-10-09T05:52:05.730 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:05 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:05.730 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:05 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/163199796' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:52:05.730 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:05 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:52:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:05 smithi191 ceph-mon[161669]: pgmap v1063: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 31734/95646 objects degraded (33.179%); 673/95646 objects misplaced (0.704%); 15 MiB/s, 243 objects/s recovering 2023-10-09T05:52:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:05 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:05 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/163199796' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:52:06.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:05 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:52:06.617 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:52:06.632 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:06.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:06.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:06.782 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:06.843 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:06.852 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:06 smithi191 ceph-mon[161669]: failed to encode map e355 with expected crc 2023-10-09T05:52:06.852 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:06 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:52:06.852 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:06 smithi191 ceph-mon[161669]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T05:52:06.852 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:06 smithi191 ceph-mon[161669]: failed to encode map e355 with expected crc 2023-10-09T05:52:06.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:06.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:06.954 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:06.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:06 smithi008 ceph-mon[191227]: failed to encode map e355 with expected crc 2023-10-09T05:52:06.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:06 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:52:06.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:06 smithi008 ceph-mon[191227]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T05:52:06.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:06 smithi008 ceph-mon[191227]: failed to encode map e355 with expected crc 2023-10-09T05:52:06.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:06 smithi008 ceph-mon[184418]: failed to encode map e355 with expected crc 2023-10-09T05:52:06.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:06 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:52:06.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:06 smithi008 ceph-mon[184418]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T05:52:06.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:06 smithi008 ceph-mon[184418]: failed to encode map e355 with expected crc 2023-10-09T05:52:07.001 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:07.082 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:07.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:07.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:07.193 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:07.258 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:07.609 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T05:52:07.680 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:07.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:07.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:07.790 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:07.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:07 smithi008 ceph-mon[184418]: pgmap v1065: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 31415/90777 objects degraded (34.607%); 568/90777 objects misplaced (0.626%); 14 MiB/s, 228 objects/s recovering 2023-10-09T05:52:07.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:07 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:07.930 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:07 smithi008 ceph-mon[191227]: pgmap v1065: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 31415/90777 objects degraded (34.607%); 568/90777 objects misplaced (0.626%); 14 MiB/s, 228 objects/s recovering 2023-10-09T05:52:07.930 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:07 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:07.930 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:08.040 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:08.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:07 smithi191 ceph-mon[161669]: pgmap v1065: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 31415/90777 objects degraded (34.607%); 568/90777 objects misplaced (0.626%); 14 MiB/s, 228 objects/s recovering 2023-10-09T05:52:08.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:07 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:08.082 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:08.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:08.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:08.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:08.225 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:08.301 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:08.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:08.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:08.408 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:08.459 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:08.470 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:08.470 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:08.504 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:08.533 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:08.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:08.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:08.642 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:08.717 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:08.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 31415/90777 objects degraded (34.607%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:08.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 31415/90777 objects degraded (34.607%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:08.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 31415/90777 objects degraded (34.607%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:09.986 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:09.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:09.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:10.094 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:10.129 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:10.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:10 smithi008 ceph-mon[184418]: pgmap v1066: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 30195/88464 objects degraded (34.133%); 568/88464 objects misplaced (0.642%); 18 MiB/s, 287 objects/s recovering 2023-10-09T05:52:10.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:10 smithi008 ceph-mon[191227]: pgmap v1066: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 30195/88464 objects degraded (34.133%); 568/88464 objects misplaced (0.642%); 18 MiB/s, 287 objects/s recovering 2023-10-09T05:52:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:10 smithi191 ceph-mon[161669]: pgmap v1066: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 30195/88464 objects degraded (34.133%); 568/88464 objects misplaced (0.642%); 18 MiB/s, 287 objects/s recovering 2023-10-09T05:52:10.749 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:52:10.751 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:10.761 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:52:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:52:10] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-10-09T05:52:10.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:10.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:10.863 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:10.963 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:11.159 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:11.206 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:11.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:11.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:11.314 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:11.376 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:11 smithi008 ceph-mon[184418]: 8.10 deep-scrub starts 2023-10-09T05:52:11.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:11 smithi008 ceph-mon[191227]: 8.10 deep-scrub starts 2023-10-09T05:52:11.381 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:11 smithi191 ceph-mon[161669]: 8.10 deep-scrub starts 2023-10-09T05:52:12.001 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:12.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:12.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:12.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:12.159 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:12.256 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:12.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:12.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:12.306 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:12 smithi008 ceph-mon[184418]: pgmap v1067: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 30035/84711 objects degraded (35.456%); 568/84711 objects misplaced (0.671%); 14 MiB/s, 225 objects/s recovering 2023-10-09T05:52:12.307 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:12 smithi008 ceph-mon[191227]: pgmap v1067: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 30035/84711 objects degraded (35.456%); 568/84711 objects misplaced (0.671%); 14 MiB/s, 225 objects/s recovering 2023-10-09T05:52:12.380 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:12.454 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:12.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:12 smithi191 ceph-mon[161669]: pgmap v1067: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 30035/84711 objects degraded (35.456%); 568/84711 objects misplaced (0.671%); 14 MiB/s, 225 objects/s recovering 2023-10-09T05:52:12.758 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:12.851 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:52:12.892 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:52:12.901 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:12.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:12.948 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:52:12.990 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:52:13.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:13.021 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:52:13.060 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:13.067 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:52:13.106 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:52:13.127 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:13.139 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:52:13.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:13.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:13.143 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:52:13.143 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:52:13.143 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:52:13.143 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:52:13.144 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:52:13.144 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:52:13.144 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:52:13.144 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:52:13.173 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2498515353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:13.174 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 30035/84711 objects degraded (35.456%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:13.174 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2498515353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:13.174 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 30035/84711 objects degraded (35.456%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:13.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:13.300 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:13.397 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:13.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:13.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:13.413 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2498515353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:13.413 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 30035/84711 objects degraded (35.456%), 21 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:13.514 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:13.562 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:14.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:14 smithi008 ceph-mon[184418]: pgmap v1068: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 28412/77781 objects degraded (36.528%); 468/77781 objects misplaced (0.602%); 16 MiB/s, 253 objects/s recovering 2023-10-09T05:52:14.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:14 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:14.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:14.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:14.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:14 smithi008 ceph-mon[191227]: pgmap v1068: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 28412/77781 objects degraded (36.528%); 468/77781 objects misplaced (0.602%); 16 MiB/s, 253 objects/s recovering 2023-10-09T05:52:14.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:14 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:14.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:14.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:14.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:14 smithi191 ceph-mon[161669]: pgmap v1068: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 28412/77781 objects degraded (36.528%); 468/77781 objects misplaced (0.602%); 16 MiB/s, 253 objects/s recovering 2023-10-09T05:52:14.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:14 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:14.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:14.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:15.113 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:15.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:15.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:15.216 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:15.262 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:15.338 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:15.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:15.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:15.443 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:15.489 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:15.558 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:15.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:15.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:15.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:15.705 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:15.788 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:15.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:15.901 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:15.972 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:16.159 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:52:16.291 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:16.295 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:16 smithi008 ceph-mon[184418]: pgmap v1069: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 28243/74589 objects degraded (37.865%); 468/74589 objects misplaced (0.627%); 16 MiB/s, 254 objects/s recovering 2023-10-09T05:52:16.295 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:16 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3175675375' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:16.295 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:16 smithi008 ceph-mon[191227]: pgmap v1069: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 28243/74589 objects degraded (37.865%); 468/74589 objects misplaced (0.627%); 16 MiB/s, 254 objects/s recovering 2023-10-09T05:52:16.295 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:16 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3175675375' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:16.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:16.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:16.397 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:16.433 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:16.529 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:16.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:16.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:16.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:16 smithi191 ceph-mon[161669]: pgmap v1069: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 28243/74589 objects degraded (37.865%); 468/74589 objects misplaced (0.627%); 16 MiB/s, 254 objects/s recovering 2023-10-09T05:52:16.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:16 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3175675375' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:16.633 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:16.693 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:17.217 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T05:52:17.280 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:52:17.447 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:17.452 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:17.452 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3175675375' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:52:17.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[184418]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T05:52:17.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[184418]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[184418]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[184418]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[184418]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/999587274' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:52:17.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:52:17.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:17.458 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:17.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3175675375' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:52:17.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[191227]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T05:52:17.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[191227]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[191227]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[191227]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[191227]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/999587274' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:52:17.459 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:17 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:52:17.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:17.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:17 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:17.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3175675375' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:52:17.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:17 smithi191 ceph-mon[161669]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T05:52:17.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:17 smithi191 ceph-mon[161669]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:17 smithi191 ceph-mon[161669]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:17 smithi191 ceph-mon[161669]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:17 smithi191 ceph-mon[161669]: failed to encode map e356 with expected crc 2023-10-09T05:52:17.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/999587274' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:52:17.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:17 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:52:17.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:17.600 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:17.697 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:17.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:17.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:17.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:17.860 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:17.958 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:17.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:17.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:18.068 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:18.100 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:18.145 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:18.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:18.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:18.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:18.274 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:18.331 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:18.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:18.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:18.439 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:18.462 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:18 smithi191 ceph-mon[161669]: pgmap v1071: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 27885/69282 objects degraded (40.249%); 358/69282 objects misplaced (0.517%); 16 MiB/s, 259 objects/s recovering 2023-10-09T05:52:18.462 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:18 smithi191 ceph-mon[161669]: failed to encode map e357 with expected crc 2023-10-09T05:52:18.462 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:18 smithi191 ceph-mon[161669]: failed to encode map e357 with expected crc 2023-10-09T05:52:18.462 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:52:18.462 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:18 smithi191 ceph-mon[161669]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T05:52:18.463 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 27885/69282 objects degraded (40.249%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:18.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[184418]: pgmap v1071: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 27885/69282 objects degraded (40.249%); 358/69282 objects misplaced (0.517%); 16 MiB/s, 259 objects/s recovering 2023-10-09T05:52:18.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[184418]: failed to encode map e357 with expected crc 2023-10-09T05:52:18.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[184418]: failed to encode map e357 with expected crc 2023-10-09T05:52:18.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:52:18.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[184418]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T05:52:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 27885/69282 objects degraded (40.249%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[191227]: pgmap v1071: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 27885/69282 objects degraded (40.249%); 358/69282 objects misplaced (0.517%); 16 MiB/s, 259 objects/s recovering 2023-10-09T05:52:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[191227]: failed to encode map e357 with expected crc 2023-10-09T05:52:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[191227]: failed to encode map e357 with expected crc 2023-10-09T05:52:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:52:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[191227]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T05:52:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 27885/69282 objects degraded (40.249%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:18.466 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:18.507 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:18.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:18.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:18.616 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:18.672 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:18.904 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:18.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:18.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:18.947 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T05:52:18.947 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:52:19.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:19.055 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:19.121 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:19.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:19.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:19.132 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:19.226 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:19.233 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:52:19.233 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:52:19.233 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:52:19.233 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:52:19.233 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.233 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108328 8.1f 8:fb066e62:::benchmark_data_smithi008_432048_object5386:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:52:19.233 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:29.984571+0000", 2023-10-09T05:52:19.233 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 169.22534872599999, 2023-10-09T05:52:19.233 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92517276500000001, 2023-10-09T05:52:19.234 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.234 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.234 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.234 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.234 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.234 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108328 2023-10-09T05:52:19.234 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.234 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.234 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.234 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984571+0000", 2023-10-09T05:52:19.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.235 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.235 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.235 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984571+0000", 2023-10-09T05:52:19.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.235 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.235 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.235 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984570+0000", 2023-10-09T05:52:19.236 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:19.236 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984577+0000", 2023-10-09T05:52:19.236 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4800000000000004e-06 2023-10-09T05:52:19.236 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984588+0000", 2023-10-09T05:52:19.236 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0906e-05 2023-10-09T05:52:19.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.237 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.237 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.237 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984594+0000", 2023-10-09T05:52:19.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3620000000000002e-06 2023-10-09T05:52:19.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.237 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.237 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.237 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.005278+0000", 2023-10-09T05:52:19.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020683634999999999 2023-10-09T05:52:19.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.238 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.238 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.238 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.009272+0000", 2023-10-09T05:52:19.238 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039940679999999999 2023-10-09T05:52:19.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.238 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.238 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.238 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.062179+0000", 2023-10-09T05:52:19.238 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.052906448000000002 2023-10-09T05:52:19.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.239 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.239 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.066018+0000", 2023-10-09T05:52:19.239 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003839496 2023-10-09T05:52:19.239 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.827356+0000", 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.76133771100000003 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.838496+0000", 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011140278 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.856903+0000", 2023-10-09T05:52:19.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018407137 2023-10-09T05:52:19.241 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.241 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.878092+0000", 2023-10-09T05:52:19.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021188335999999999 2023-10-09T05:52:19.241 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.241 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.878887+0000", 2023-10-09T05:52:19.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00079575399999999997 2023-10-09T05:52:19.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.242 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.242 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.891744+0000", 2023-10-09T05:52:19.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012856932 2023-10-09T05:52:19.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.242 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.242 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.895389+0000", 2023-10-09T05:52:19.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003644974 2023-10-09T05:52:19.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.243 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.243 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.243 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.909744+0000", 2023-10-09T05:52:19.243 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014354693 2023-10-09T05:52:19.243 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.243 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.243 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.243 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.243 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.243 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108520 8.1f 8:fc628894:::benchmark_data_smithi008_432048_object5578:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:52:19.243 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:30.895574+0000", 2023-10-09T05:52:19.244 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 168.31434582, 2023-10-09T05:52:19.244 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89911723799999999, 2023-10-09T05:52:19.244 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.244 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.244 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.244 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.244 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.244 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108520 2023-10-09T05:52:19.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.244 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.895574+0000", 2023-10-09T05:52:19.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.245 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.895574+0000", 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.895572+0000", 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.895581+0000", 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2879999999999998e-06 2023-10-09T05:52:19.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.895598+0000", 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6824999999999999e-05 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.895610+0000", 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1705999999999999e-05 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.941961+0000", 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.046350734999999997 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.945872+0000", 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039114950000000001 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.978847+0000", 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.032974919999999998 2023-10-09T05:52:19.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.984919+0000", 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0060720679999999999 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.705736+0000", 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72081663900000004 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.250 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.713135+0000", 2023-10-09T05:52:19.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0073995149999999997 2023-10-09T05:52:19.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.250 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.250 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.723617+0000", 2023-10-09T05:52:19.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010481717 2023-10-09T05:52:19.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.250 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.250 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.750314+0000", 2023-10-09T05:52:19.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.026696428000000001 2023-10-09T05:52:19.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.251 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.775461+0000", 2023-10-09T05:52:19.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.025147804999999999 2023-10-09T05:52:19.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.251 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.779065+0000", 2023-10-09T05:52:19.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036037109999999999 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.783489+0000", 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0044241430000000002 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.794691+0000", 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011202128 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.252 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108574 8.1f 8:fd32a2a1:::benchmark_data_smithi008_432048_object5632:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:31.165420+0000", 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 168.04449998999999, 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89205451700000005, 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.253 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108574 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.165420+0000", 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.165420+0000", 2023-10-09T05:52:19.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.165403+0000", 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999828 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.165432+0000", 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8924e-05 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.165450+0000", 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7859999999999998e-05 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.165460+0000", 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0205e-05 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.216752+0000", 2023-10-09T05:52:19.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.051292152000000001 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.221899+0000", 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051473209999999998 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.270851+0000", 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.048951503 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.275069+0000", 2023-10-09T05:52:19.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042184830000000003 2023-10-09T05:52:19.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.957760+0000", 2023-10-09T05:52:19.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.68269072799999997 2023-10-09T05:52:19.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.259 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.968080+0000", 2023-10-09T05:52:19.259 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010320085 2023-10-09T05:52:19.259 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.259 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.259 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.990672+0000", 2023-10-09T05:52:19.259 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022591547 2023-10-09T05:52:19.259 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.259 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.043240+0000", 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.052568733999999999 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.043798+0000", 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00055759600000000005 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.045104+0000", 2023-10-09T05:52:19.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00130642 2023-10-09T05:52:19.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.048114+0000", 2023-10-09T05:52:19.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030092780000000002 2023-10-09T05:52:19.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.057474+0000", 2023-10-09T05:52:19.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0093606750000000006 2023-10-09T05:52:19.262 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.262 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.262 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.262 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.262 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108591 8.1f 8:f826337a:::benchmark_data_smithi008_432048_object5649:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:52:19.262 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:31.239317+0000", 2023-10-09T05:52:19.262 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 167.970602926, 2023-10-09T05:52:19.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94874395700000003, 2023-10-09T05:52:19.262 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108591 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239317+0000", 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239317+0000", 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239315+0000", 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239323+0000", 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8040000000000002e-06 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239336+0000", 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2957e-05 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239345+0000", 2023-10-09T05:52:19.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9830000000000002e-06 2023-10-09T05:52:19.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.277097+0000", 2023-10-09T05:52:19.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037752218999999997 2023-10-09T05:52:19.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.281903+0000", 2023-10-09T05:52:19.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048058729999999996 2023-10-09T05:52:19.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.310019+0000", 2023-10-09T05:52:19.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028116208 2023-10-09T05:52:19.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.314967+0000", 2023-10-09T05:52:19.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049481849999999999 2023-10-09T05:52:19.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.268 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.108322+0000", 2023-10-09T05:52:19.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.79335536500000003 2023-10-09T05:52:19.268 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.268 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.114934+0000", 2023-10-09T05:52:19.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066115519999999997 2023-10-09T05:52:19.268 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.269 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.143064+0000", 2023-10-09T05:52:19.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028130485 2023-10-09T05:52:19.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.269 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.156587+0000", 2023-10-09T05:52:19.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013522569999999999 2023-10-09T05:52:19.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.159203+0000", 2023-10-09T05:52:19.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026158650000000002 2023-10-09T05:52:19.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.170337+0000", 2023-10-09T05:52:19.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011134412 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.174909+0000", 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045717029999999999 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.188061+0000", 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013151985 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.271 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.272 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.272 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108652 8.f 8:f2073352:::benchmark_data_smithi008_432048_object5710:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:52:19.272 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:31.520988+0000", 2023-10-09T05:52:19.272 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 167.688931895, 2023-10-09T05:52:19.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.91771643800000002, 2023-10-09T05:52:19.272 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.272 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.272 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.272 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.273 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.273 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108652 2023-10-09T05:52:19.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.273 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.520988+0000", 2023-10-09T05:52:19.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.520988+0000", 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.520987+0000", 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.520992+0000", 2023-10-09T05:52:19.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6859999999999997e-06 2023-10-09T05:52:19.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.521004+0000", 2023-10-09T05:52:19.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1970999999999999e-05 2023-10-09T05:52:19.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.521012+0000", 2023-10-09T05:52:19.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4079999999999998e-06 2023-10-09T05:52:19.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.607138+0000", 2023-10-09T05:52:19.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.086126080999999993 2023-10-09T05:52:19.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.610574+0000", 2023-10-09T05:52:19.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034361769999999999 2023-10-09T05:52:19.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.627903+0000", 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017329026000000001 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.633019+0000", 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00511539 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.382248+0000", 2023-10-09T05:52:19.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.74922937499999998 2023-10-09T05:52:19.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.385176+0000", 2023-10-09T05:52:19.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029280840000000001 2023-10-09T05:52:19.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.392493+0000", 2023-10-09T05:52:19.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0073163280000000004 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.421456+0000", 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028963847000000001 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.422926+0000", 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014697480000000001 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.427456+0000", 2023-10-09T05:52:19.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045298379999999996 2023-10-09T05:52:19.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.429589+0000", 2023-10-09T05:52:19.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021326370000000002 2023-10-09T05:52:19.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.438704+0000", 2023-10-09T05:52:19.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0091157500000000006 2023-10-09T05:52:19.281 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.281 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.281 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.281 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108664 8.1f 8:fb1a1b36:::benchmark_data_smithi008_432048_object5722:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:52:19.281 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:31.614117+0000", 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 167.595803039, 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1244621509999999, 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108664 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.282 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614117+0000", 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614117+0000", 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614115+0000", 2023-10-09T05:52:19.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:19.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614125+0000", 2023-10-09T05:52:19.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0227000000000001e-05 2023-10-09T05:52:19.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614141+0000", 2023-10-09T05:52:19.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5631999999999999e-05 2023-10-09T05:52:19.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614152+0000", 2023-10-09T05:52:19.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0627e-05 2023-10-09T05:52:19.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.644859+0000", 2023-10-09T05:52:19.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030707564999999999 2023-10-09T05:52:19.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.650494+0000", 2023-10-09T05:52:19.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0056349290000000003 2023-10-09T05:52:19.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.686762+0000", 2023-10-09T05:52:19.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.036267668000000003 2023-10-09T05:52:19.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.693827+0000", 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070651710000000003 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.512455+0000", 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.81862801299999999 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.517577+0000", 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051220679999999996 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.526918+0000", 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0093413560000000003 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.563730+0000", 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.036811893999999998 2023-10-09T05:52:19.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.726330+0000", 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.16260017500000001 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.726967+0000", 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000636657 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.729816+0000", 2023-10-09T05:52:19.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028490059999999998 2023-10-09T05:52:19.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.290 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.738579+0000", 2023-10-09T05:52:19.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0087629040000000002 2023-10-09T05:52:19.290 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.290 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.290 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.291 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108709 8.1f 8:fc046827:::benchmark_data_smithi008_432048_object5767:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:52:19.291 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:31.757269+0000", 2023-10-09T05:52:19.291 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 167.45265078099999, 2023-10-09T05:52:19.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.998060002, 2023-10-09T05:52:19.291 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.291 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.291 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.291 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108709 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757269+0000", 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757269+0000", 2023-10-09T05:52:19.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757267+0000", 2023-10-09T05:52:19.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:19.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757278+0000", 2023-10-09T05:52:19.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0876000000000001e-05 2023-10-09T05:52:19.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757298+0000", 2023-10-09T05:52:19.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0075999999999999e-05 2023-10-09T05:52:19.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757309+0000", 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0896999999999999e-05 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.817372+0000", 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.060062786999999999 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.828286+0000", 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010914316 2023-10-09T05:52:19.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.896232+0000", 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.067946329 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.901371+0000", 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051384359999999997 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.544517+0000", 2023-10-09T05:52:19.297 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.64314595100000005 2023-10-09T05:52:19.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.548771+0000", 2023-10-09T05:52:19.297 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042545530000000003 2023-10-09T05:52:19.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.556889+0000", 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081172039999999994 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.739031+0000", 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.182142532 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.740311+0000", 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0012802569999999999 2023-10-09T05:52:19.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.741944+0000", 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001632313 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.744193+0000", 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002249418 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.755329+0000", 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011135987999999999 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108836 8.1f 8:fd9739dd:::benchmark_data_smithi008_432048_object5894:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:32.387648+0000", 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 166.82227142400001, 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92865287500000004, 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.300 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.301 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.301 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.301 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.301 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108836 2023-10-09T05:52:19.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.301 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.301 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.301 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387648+0000", 2023-10-09T05:52:19.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387648+0000", 2023-10-09T05:52:19.302 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.303 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387648+0000", 2023-10-09T05:52:19.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:19.303 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.303 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.303 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387651+0000", 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8210000000000002e-06 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387657+0000", 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8000000000000004e-06 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387663+0000", 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0410000000000003e-06 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.439921+0000", 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.052258302 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.455650+0000", 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015728783 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.506375+0000", 2023-10-09T05:52:19.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.050725721000000001 2023-10-09T05:52:19.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.511713+0000", 2023-10-09T05:52:19.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005337713 2023-10-09T05:52:19.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.231814+0000", 2023-10-09T05:52:19.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72010072700000005 2023-10-09T05:52:19.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.238299+0000", 2023-10-09T05:52:19.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064853979999999999 2023-10-09T05:52:19.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.246417+0000", 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.008117839 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.299253+0000", 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.052835676999999998 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.301785+0000", 2023-10-09T05:52:19.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025323110000000002 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.302525+0000", 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00073951699999999997 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.305906+0000", 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003381481 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.316301+0000", 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010395196000000001 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108877 8.f 8:f5ef4290:::benchmark_data_smithi008_432048_object5935:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:32.524792+0000", 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 166.685127905, 2023-10-09T05:52:19.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93690308099999997, 2023-10-09T05:52:19.311 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.311 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.311 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.311 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.311 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.311 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108877 2023-10-09T05:52:19.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.311 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524792+0000", 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524792+0000", 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524790+0000", 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:19.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524798+0000", 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1699999999999997e-06 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524811+0000", 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3185e-05 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.313 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524820+0000", 2023-10-09T05:52:19.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4840000000000004e-06 2023-10-09T05:52:19.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.771712+0000", 2023-10-09T05:52:19.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.246892371 2023-10-09T05:52:19.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.776583+0000", 2023-10-09T05:52:19.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004870854 2023-10-09T05:52:19.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.816435+0000", 2023-10-09T05:52:19.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.039851866999999999 2023-10-09T05:52:19.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.821386+0000", 2023-10-09T05:52:19.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049510520000000001 2023-10-09T05:52:19.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.316 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.316 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.419677+0000", 2023-10-09T05:52:19.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.59829150799999997 2023-10-09T05:52:19.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.316 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.316 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.423867+0000", 2023-10-09T05:52:19.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041896390000000002 2023-10-09T05:52:19.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.431852+0000", 2023-10-09T05:52:19.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0079849550000000002 2023-10-09T05:52:19.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.440239+0000", 2023-10-09T05:52:19.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0083871919999999999 2023-10-09T05:52:19.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.441156+0000", 2023-10-09T05:52:19.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00091646800000000004 2023-10-09T05:52:19.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.451403+0000", 2023-10-09T05:52:19.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010247166 2023-10-09T05:52:19.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.453645+0000", 2023-10-09T05:52:19.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022420090000000001 2023-10-09T05:52:19.320 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.461695+0000", 2023-10-09T05:52:19.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0080501570000000005 2023-10-09T05:52:19.320 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.321 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.321 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.321 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108988 8.1f 8:fb6f108d:::benchmark_data_smithi008_432048_object6046:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:52:19.322 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:33.159282+0000", 2023-10-09T05:52:19.322 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 166.05063837200001, 2023-10-09T05:52:19.322 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92710595600000001, 2023-10-09T05:52:19.322 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.322 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.322 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.322 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.323 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.323 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108988 2023-10-09T05:52:19.323 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.323 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.323 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.323 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.323 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159282+0000", 2023-10-09T05:52:19.323 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.323 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159282+0000", 2023-10-09T05:52:19.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.325 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159279+0000", 2023-10-09T05:52:19.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:19.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.325 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.325 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.325 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159290+0000", 2023-10-09T05:52:19.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0994e-05 2023-10-09T05:52:19.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.325 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159310+0000", 2023-10-09T05:52:19.326 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9641e-05 2023-10-09T05:52:19.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159326+0000", 2023-10-09T05:52:19.326 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5529e-05 2023-10-09T05:52:19.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.327 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.181153+0000", 2023-10-09T05:52:19.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021827215000000001 2023-10-09T05:52:19.327 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.327 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.327 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.188840+0000", 2023-10-09T05:52:19.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0076866929999999997 2023-10-09T05:52:19.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.328 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.209278+0000", 2023-10-09T05:52:19.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020438587000000001 2023-10-09T05:52:19.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.328 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.215354+0000", 2023-10-09T05:52:19.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006075886 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.986664+0000", 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.771309731 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.990016+0000", 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033521359999999999 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.996527+0000", 2023-10-09T05:52:19.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00651148 2023-10-09T05:52:19.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.051138+0000", 2023-10-09T05:52:19.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.054610341999999999 2023-10-09T05:52:19.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.331 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.070103+0000", 2023-10-09T05:52:19.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018965288 2023-10-09T05:52:19.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.331 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.072016+0000", 2023-10-09T05:52:19.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019130620000000001 2023-10-09T05:52:19.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.073818+0000", 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018016029999999999 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.086387+0000", 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012569800000000001 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.333 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:109010 8.1f 8:f8bf16b4:::benchmark_data_smithi008_432048_object6068:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:52:19.333 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:33.326962+0000", 2023-10-09T05:52:19.333 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 165.88295821899999, 2023-10-09T05:52:19.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.91839299399999996, 2023-10-09T05:52:19.333 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.333 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.333 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.333 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.333 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.334 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 109010 2023-10-09T05:52:19.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.334 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.326962+0000", 2023-10-09T05:52:19.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.326962+0000", 2023-10-09T05:52:19.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.326960+0000", 2023-10-09T05:52:19.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:19.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.326966+0000", 2023-10-09T05:52:19.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0140000000000004e-06 2023-10-09T05:52:19.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.326976+0000", 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5599999999999999e-06 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.326982+0000", 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.601e-06 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.354532+0000", 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.027549408000000001 2023-10-09T05:52:19.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.338 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.338 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.338 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.362713+0000", 2023-10-09T05:52:19.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081818370000000008 2023-10-09T05:52:19.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.338 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.338 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.338 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.399138+0000", 2023-10-09T05:52:19.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.036424853 2023-10-09T05:52:19.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.339 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.408469+0000", 2023-10-09T05:52:19.339 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0093310289999999994 2023-10-09T05:52:19.339 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.135583+0000", 2023-10-09T05:52:19.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72711364999999994 2023-10-09T05:52:19.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.144670+0000", 2023-10-09T05:52:19.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0090872009999999996 2023-10-09T05:52:19.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.163025+0000", 2023-10-09T05:52:19.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018355015999999998 2023-10-09T05:52:19.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.189970+0000", 2023-10-09T05:52:19.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.02694442 2023-10-09T05:52:19.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.195267+0000", 2023-10-09T05:52:19.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005297309 2023-10-09T05:52:19.343 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.343 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.343 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.343 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.212907+0000", 2023-10-09T05:52:19.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017639937000000001 2023-10-09T05:52:19.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.344 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.344 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.217698+0000", 2023-10-09T05:52:19.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004791132 2023-10-09T05:52:19.345 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.345 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.345 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.245355+0000", 2023-10-09T05:52:19.345 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.027656716000000001 2023-10-09T05:52:19.345 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.346 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.346 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.346 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:114012 8.1f 8:fc10c192:::benchmark_data_smithi008_432048_object11070:head [delete] snapc 0=[] ondisk+write+known_if_redirected e319)", 2023-10-09T05:52:19.347 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:53.587821+0000", 2023-10-09T05:52:19.347 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 145.622098529, 2023-10-09T05:52:19.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89940384100000004, 2023-10-09T05:52:19.347 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.347 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.347 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.348 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.348 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.348 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 114012 2023-10-09T05:52:19.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.348 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.587821+0000", 2023-10-09T05:52:19.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.587821+0000", 2023-10-09T05:52:19.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.587819+0000", 2023-10-09T05:52:19.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:19.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.352 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.587829+0000", 2023-10-09T05:52:19.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8020000000000003e-06 2023-10-09T05:52:19.352 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.353 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.587843+0000", 2023-10-09T05:52:19.353 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4319e-05 2023-10-09T05:52:19.353 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.353 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.353 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.587854+0000", 2023-10-09T05:52:19.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0132999999999999e-05 2023-10-09T05:52:19.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.645868+0000", 2023-10-09T05:52:19.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.058014780000000002 2023-10-09T05:52:19.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.650117+0000", 2023-10-09T05:52:19.356 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004249142 2023-10-09T05:52:19.356 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.356 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.356 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.703559+0000", 2023-10-09T05:52:19.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.053441334 2023-10-09T05:52:19.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.357 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.357 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.357 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.707418+0000", 2023-10-09T05:52:19.358 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038589760000000001 2023-10-09T05:52:19.358 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.358 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.358 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.358 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.430192+0000", 2023-10-09T05:52:19.359 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72277416299999997 2023-10-09T05:52:19.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.359 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.359 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.359 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.437433+0000", 2023-10-09T05:52:19.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0072415240000000001 2023-10-09T05:52:19.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.444294+0000", 2023-10-09T05:52:19.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068610240000000003 2023-10-09T05:52:19.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.473109+0000", 2023-10-09T05:52:19.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028814986000000001 2023-10-09T05:52:19.362 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.473601+0000", 2023-10-09T05:52:19.362 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00049157400000000005 2023-10-09T05:52:19.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.363 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.363 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.476189+0000", 2023-10-09T05:52:19.363 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025878780000000001 2023-10-09T05:52:19.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.364 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.364 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.478515+0000", 2023-10-09T05:52:19.364 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023258649999999999 2023-10-09T05:52:19.364 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.364 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.364 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.364 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.487225+0000", 2023-10-09T05:52:19.364 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0087104409999999993 2023-10-09T05:52:19.364 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.364 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.365 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.365 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.365 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:114019 8.f 8:f7f5942a:::benchmark_data_smithi008_432048_object11077:head [delete] snapc 0=[] ondisk+write+known_if_redirected e319)", 2023-10-09T05:52:19.365 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:53.608835+0000", 2023-10-09T05:52:19.365 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 145.601084415, 2023-10-09T05:52:19.365 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.98156760300000001, 2023-10-09T05:52:19.365 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.365 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.365 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.366 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.366 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.366 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 114019 2023-10-09T05:52:19.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.366 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.366 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.366 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608835+0000", 2023-10-09T05:52:19.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608835+0000", 2023-10-09T05:52:19.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608834+0000", 2023-10-09T05:52:19.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:19.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608840+0000", 2023-10-09T05:52:19.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8810000000000001e-06 2023-10-09T05:52:19.368 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.368 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608849+0000", 2023-10-09T05:52:19.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.054e-06 2023-10-09T05:52:19.368 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.368 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608855+0000", 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8660000000000002e-06 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.709132+0000", 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.100276884 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.712800+0000", 2023-10-09T05:52:19.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036683029999999999 2023-10-09T05:52:19.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.729698+0000", 2023-10-09T05:52:19.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016897586999999999 2023-10-09T05:52:19.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.732690+0000", 2023-10-09T05:52:19.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029921689999999998 2023-10-09T05:52:19.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.371 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.556908+0000", 2023-10-09T05:52:19.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.82421833499999997 2023-10-09T05:52:19.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.371 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.560831+0000", 2023-10-09T05:52:19.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039229139999999996 2023-10-09T05:52:19.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.568419+0000", 2023-10-09T05:52:19.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0075875789999999997 2023-10-09T05:52:19.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.575101+0000", 2023-10-09T05:52:19.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066826059999999998 2023-10-09T05:52:19.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.582134+0000", 2023-10-09T05:52:19.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070324810000000001 2023-10-09T05:52:19.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.582676+0000", 2023-10-09T05:52:19.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00054164800000000004 2023-10-09T05:52:19.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.374 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.584594+0000", 2023-10-09T05:52:19.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019186209999999999 2023-10-09T05:52:19.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.374 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.590403+0000", 2023-10-09T05:52:19.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005808957 2023-10-09T05:52:19.374 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.374 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.375 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.375 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:114052 8.1f 8:f8538642:::benchmark_data_smithi008_432048_object11110:head [delete] snapc 0=[] ondisk+write+known_if_redirected e319)", 2023-10-09T05:52:19.375 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:53.698035+0000", 2023-10-09T05:52:19.375 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 145.511885105, 2023-10-09T05:52:19.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97790464700000002, 2023-10-09T05:52:19.375 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.375 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.376 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.376 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.376 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.376 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 114052 2023-10-09T05:52:19.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.376 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.376 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.376 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698035+0000", 2023-10-09T05:52:19.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698035+0000", 2023-10-09T05:52:19.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698033+0000", 2023-10-09T05:52:19.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:19.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698041+0000", 2023-10-09T05:52:19.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6939999999999995e-06 2023-10-09T05:52:19.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698053+0000", 2023-10-09T05:52:19.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2361000000000001e-05 2023-10-09T05:52:19.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698063+0000", 2023-10-09T05:52:19.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8199999999999992e-06 2023-10-09T05:52:19.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.711539+0000", 2023-10-09T05:52:19.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013475941 2023-10-09T05:52:19.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.718579+0000", 2023-10-09T05:52:19.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007040073 2023-10-09T05:52:19.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.773633+0000", 2023-10-09T05:52:19.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.055054084000000003 2023-10-09T05:52:19.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.778235+0000", 2023-10-09T05:52:19.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0046020050000000002 2023-10-09T05:52:19.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.624012+0000", 2023-10-09T05:52:19.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.84577629099999996 2023-10-09T05:52:19.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.628102+0000", 2023-10-09T05:52:19.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004090247 2023-10-09T05:52:19.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.383 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.383 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.636818+0000", 2023-10-09T05:52:19.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0087158210000000003 2023-10-09T05:52:19.383 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.383 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.383 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.383 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.664357+0000", 2023-10-09T05:52:19.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.02753889 2023-10-09T05:52:19.383 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.383 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.383 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.384 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.664692+0000", 2023-10-09T05:52:19.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00033567800000000001 2023-10-09T05:52:19.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.384 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.384 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.666599+0000", 2023-10-09T05:52:19.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001906477 2023-10-09T05:52:19.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.385 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.385 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.668728+0000", 2023-10-09T05:52:19.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021296900000000001 2023-10-09T05:52:19.385 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.385 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.385 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.385 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.675939+0000", 2023-10-09T05:52:19.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007211065 2023-10-09T05:52:19.385 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.386 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.386 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.386 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:114053 8.f 8:f0d5fbe3:::benchmark_data_smithi008_432048_object11111:head [delete] snapc 0=[] ondisk+write+known_if_redirected e319)", 2023-10-09T05:52:19.386 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:53.704034+0000", 2023-10-09T05:52:19.386 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 145.50588575500001, 2023-10-09T05:52:19.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.95060431000000001, 2023-10-09T05:52:19.386 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.387 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.387 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.387 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.387 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.387 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 114053 2023-10-09T05:52:19.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.387 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704034+0000", 2023-10-09T05:52:19.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704034+0000", 2023-10-09T05:52:19.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704032+0000", 2023-10-09T05:52:19.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:19.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704042+0000", 2023-10-09T05:52:19.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2909999999999996e-06 2023-10-09T05:52:19.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704058+0000", 2023-10-09T05:52:19.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6708e-05 2023-10-09T05:52:19.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704068+0000", 2023-10-09T05:52:19.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6020000000000006e-06 2023-10-09T05:52:19.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.713094+0000", 2023-10-09T05:52:19.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0090258779999999993 2023-10-09T05:52:19.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.716172+0000", 2023-10-09T05:52:19.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003077766 2023-10-09T05:52:19.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.392 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.745331+0000", 2023-10-09T05:52:19.392 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.029158836 2023-10-09T05:52:19.392 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.392 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.392 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.392 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.748442+0000", 2023-10-09T05:52:19.392 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031115719999999999 2023-10-09T05:52:19.392 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.392 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.392 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.392 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.599175+0000", 2023-10-09T05:52:19.393 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85073269900000004 2023-10-09T05:52:19.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.393 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.393 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.393 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.604454+0000", 2023-10-09T05:52:19.393 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0052789769999999998 2023-10-09T05:52:19.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.393 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.393 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.393 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.611546+0000", 2023-10-09T05:52:19.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070920089999999998 2023-10-09T05:52:19.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.625320+0000", 2023-10-09T05:52:19.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013774036 2023-10-09T05:52:19.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.625757+0000", 2023-10-09T05:52:19.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000436718 2023-10-09T05:52:19.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.645189+0000", 2023-10-09T05:52:19.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.01943224 2023-10-09T05:52:19.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.647314+0000", 2023-10-09T05:52:19.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021256909999999999 2023-10-09T05:52:19.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.654638+0000", 2023-10-09T05:52:19.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0073239840000000004 2023-10-09T05:52:19.398 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.398 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.398 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.398 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:114063 8.1f 8:fc8f46ab:::benchmark_data_smithi008_432048_object11121:head [delete] snapc 0=[] ondisk+write+known_if_redirected e319)", 2023-10-09T05:52:19.398 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:53.731006+0000", 2023-10-09T05:52:19.399 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 145.47891437000001, 2023-10-09T05:52:19.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0338361009999999, 2023-10-09T05:52:19.399 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.399 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.399 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.399 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.399 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.399 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 114063 2023-10-09T05:52:19.400 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.400 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.400 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.400 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.400 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731006+0000", 2023-10-09T05:52:19.400 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.400 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.400 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.400 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.401 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731006+0000", 2023-10-09T05:52:19.401 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.401 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.401 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.401 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731004+0000", 2023-10-09T05:52:19.401 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:19.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.401 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731011+0000", 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.379e-06 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731027+0000", 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5534999999999999e-05 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.402 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731038+0000", 2023-10-09T05:52:19.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1374e-05 2023-10-09T05:52:19.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.792521+0000", 2023-10-09T05:52:19.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.061482556000000001 2023-10-09T05:52:19.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.796234+0000", 2023-10-09T05:52:19.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037128450000000002 2023-10-09T05:52:19.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.040431+0000", 2023-10-09T05:52:19.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24419717599999999 2023-10-09T05:52:19.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.044129+0000", 2023-10-09T05:52:19.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036982790000000001 2023-10-09T05:52:19.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.687635+0000", 2023-10-09T05:52:19.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.64350545100000001 2023-10-09T05:52:19.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.696095+0000", 2023-10-09T05:52:19.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0084602170000000008 2023-10-09T05:52:19.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.711633+0000", 2023-10-09T05:52:19.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015537781000000001 2023-10-09T05:52:19.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.723121+0000", 2023-10-09T05:52:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011488414000000001 2023-10-09T05:52:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.737918+0000", 2023-10-09T05:52:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014796785 2023-10-09T05:52:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.756216+0000", 2023-10-09T05:52:19.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018297979999999998 2023-10-09T05:52:19.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.408 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.408 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.757979+0000", 2023-10-09T05:52:19.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017634949999999999 2023-10-09T05:52:19.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.408 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.408 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.764842+0000", 2023-10-09T05:52:19.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068624489999999996 2023-10-09T05:52:19.409 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.409 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.409 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.409 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:114185 8.1f 8:f801fb94:::benchmark_data_smithi008_432048_object11243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e319)", 2023-10-09T05:52:19.409 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:54.429094+0000", 2023-10-09T05:52:19.409 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 144.78082557900001, 2023-10-09T05:52:19.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97138411000000002, 2023-10-09T05:52:19.409 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.409 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.410 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.410 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.410 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.410 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 114185 2023-10-09T05:52:19.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.410 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429094+0000", 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429094+0000", 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429092+0000", 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:19.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.412 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.412 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.412 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429102+0000", 2023-10-09T05:52:19.412 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3060000000000003e-06 2023-10-09T05:52:19.412 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.412 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.412 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.412 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429117+0000", 2023-10-09T05:52:19.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5579000000000001e-05 2023-10-09T05:52:19.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429127+0000", 2023-10-09T05:52:19.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0037999999999999e-05 2023-10-09T05:52:19.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.518323+0000", 2023-10-09T05:52:19.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.089195351000000006 2023-10-09T05:52:19.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.524972+0000", 2023-10-09T05:52:19.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066490619999999999 2023-10-09T05:52:19.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.566402+0000", 2023-10-09T05:52:19.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.041430283999999998 2023-10-09T05:52:19.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.579147+0000", 2023-10-09T05:52:19.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012745127 2023-10-09T05:52:19.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.303210+0000", 2023-10-09T05:52:19.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72406284799999998 2023-10-09T05:52:19.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.310075+0000", 2023-10-09T05:52:19.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068648809999999998 2023-10-09T05:52:19.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.320914+0000", 2023-10-09T05:52:19.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010838634999999999 2023-10-09T05:52:19.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.368949+0000", 2023-10-09T05:52:19.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.048035080000000001 2023-10-09T05:52:19.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.370199+0000", 2023-10-09T05:52:19.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001250216 2023-10-09T05:52:19.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.385700+0000", 2023-10-09T05:52:19.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015500978 2023-10-09T05:52:19.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.388642+0000", 2023-10-09T05:52:19.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029421830000000001 2023-10-09T05:52:19.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.400478+0000", 2023-10-09T05:52:19.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0118364 2023-10-09T05:52:19.419 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.419 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.419 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.419 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:115721 8.1f 8:fb033480:::benchmark_data_smithi008_432048_object12779:head [delete] snapc 0=[] ondisk+write+known_if_redirected e320)", 2023-10-09T05:52:19.419 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:00.338682+0000", 2023-10-09T05:52:19.419 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 138.87123822500001, 2023-10-09T05:52:19.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90488364600000004, 2023-10-09T05:52:19.420 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.420 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:19.420 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.420 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.420 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.420 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 115721 2023-10-09T05:52:19.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.420 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.421 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:00.338682+0000", 2023-10-09T05:52:19.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.421 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.421 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.421 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:00.338682+0000", 2023-10-09T05:52:19.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.421 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.422 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.422 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:00.338680+0000", 2023-10-09T05:52:19.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:52:19.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.422 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.422 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:00.338690+0000", 2023-10-09T05:52:19.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0543000000000001e-05 2023-10-09T05:52:19.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.423 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:00.338703+0000", 2023-10-09T05:52:19.423 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3271000000000001e-05 2023-10-09T05:52:19.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.423 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:00.338718+0000", 2023-10-09T05:52:19.423 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4999e-05 2023-10-09T05:52:19.424 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.424 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.424 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:00.410059+0000", 2023-10-09T05:52:19.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.071340463000000007 2023-10-09T05:52:19.424 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.424 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.424 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:00.413961+0000", 2023-10-09T05:52:19.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039020790000000001 2023-10-09T05:52:19.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.425 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:00.449228+0000", 2023-10-09T05:52:19.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.035266947999999999 2023-10-09T05:52:19.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.425 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:00.453397+0000", 2023-10-09T05:52:19.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041687060000000003 2023-10-09T05:52:19.426 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.426 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.426 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:01.194033+0000", 2023-10-09T05:52:19.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.74063623700000003 2023-10-09T05:52:19.426 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.426 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.426 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:19.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:01.197062+0000", 2023-10-09T05:52:19.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030290489999999998 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:01.203021+0000", 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0059594330000000001 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:01.232468+0000", 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.029446941000000001 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:01.233630+0000", 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001161781 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:01.234580+0000", 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095027600000000001 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:19.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:01.236192+0000", 2023-10-09T05:52:19.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001611878 2023-10-09T05:52:19.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.429 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:01.243565+0000", 2023-10-09T05:52:19.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0073731949999999999 2023-10-09T05:52:19.429 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.429 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.429 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.430 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119611 8.1 8:835429e4:::benchmark_data_smithi008_432048_object16669:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e328)", 2023-10-09T05:52:19.430 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:18.039479+0000", 2023-10-09T05:52:19.430 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 121.17044044399999, 2023-10-09T05:52:19.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.748108534, 2023-10-09T05:52:19.430 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.430 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:52:19.430 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.430 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.430 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.431 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119611 2023-10-09T05:52:19.431 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.431 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.431 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.431 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.431 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039479+0000", 2023-10-09T05:52:19.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.431 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.432 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.432 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.432 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039479+0000", 2023-10-09T05:52:19.432 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.432 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.432 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.432 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.432 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039479+0000", 2023-10-09T05:52:19.432 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:19.433 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.433 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.433 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.433 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039482+0000", 2023-10-09T05:52:19.433 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4619999999999999e-06 2023-10-09T05:52:19.433 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.434 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.434 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.434 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039488+0000", 2023-10-09T05:52:19.434 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2859999999999996e-06 2023-10-09T05:52:19.434 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.434 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039494+0000", 2023-10-09T05:52:19.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1489999999999999e-06 2023-10-09T05:52:19.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.040837+0000", 2023-10-09T05:52:19.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013427409999999999 2023-10-09T05:52:19.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.436 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:19.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.040848+0000", 2023-10-09T05:52:19.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1238e-05 2023-10-09T05:52:19.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.436 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.765099+0000", 2023-10-09T05:52:19.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7242508669999999 2023-10-09T05:52:19.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.437 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.768458+0000", 2023-10-09T05:52:19.437 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003359169 2023-10-09T05:52:19.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.437 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.785927+0000", 2023-10-09T05:52:19.437 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017468857000000001 2023-10-09T05:52:19.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.787588+0000", 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001661158 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:143739 8.1f 8:f9bff641:::benchmark_data_smithi008_432048_object40797:head [delete] snapc 0=[] ondisk+write+known_if_redirected e338)", 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:12.140403+0000", 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 67.069517344999994, 2023-10-09T05:52:19.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.91763824400000005, 2023-10-09T05:52:19.439 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:19.439 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:52:19.439 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:19.439 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:19.439 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:19.439 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 143739 2023-10-09T05:52:19.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.439 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:19.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:19.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.140403+0000", 2023-10-09T05:52:19.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.440 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.440 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:19.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.140403+0000", 2023-10-09T05:52:19.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:19.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.440 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.440 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:19.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.140401+0000", 2023-10-09T05:52:19.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.140410+0000", 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5750000000000001e-06 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.140423+0000", 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3271999999999999e-05 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.441 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.442 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:19.442 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.140432+0000", 2023-10-09T05:52:19.442 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7490000000000004e-06 2023-10-09T05:52:19.442 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.442 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.442 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.442 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.167500+0000", 2023-10-09T05:52:19.442 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.027068852000000001 2023-10-09T05:52:19.442 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.443 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:52:19.443 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.171126+0000", 2023-10-09T05:52:19.443 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036253420000000001 2023-10-09T05:52:19.443 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.443 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.443 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.206599+0000", 2023-10-09T05:52:19.443 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.035473146999999997 2023-10-09T05:52:19.443 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.444 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:19.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.209415+0000", 2023-10-09T05:52:19.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002816027 2023-10-09T05:52:19.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.444 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.444 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.222430+0000", 2023-10-09T05:52:19.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013014628 2023-10-09T05:52:19.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.445 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:19.445 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:12.222443+0000", 2023-10-09T05:52:19.445 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3601000000000001e-05 2023-10-09T05:52:19.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.445 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:19.445 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:13.058025+0000", 2023-10-09T05:52:19.445 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83558141900000005 2023-10-09T05:52:19.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:19.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:19.446 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:19.446 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:13.058041+0000", 2023-10-09T05:52:19.446 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6232999999999999e-05 2023-10-09T05:52:19.446 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.446 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.446 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.446 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:19.446 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:19.446 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:52:19.447 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:19.523 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:19.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:19.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:19.631 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:52:19.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:19.667 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:19.756 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:19.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:19.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:19.870 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:19.911 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:19.988 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:19.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:19.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:20.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:20.119 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:20.162 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:20.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:20.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:20.268 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:20.291 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:20 smithi191 ceph-mon[161669]: pgmap v1073: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 26713/67038 objects degraded (39.848%); 358/67038 objects misplaced (0.534%); 19 MiB/s, 302 objects/s recovering 2023-10-09T05:52:20.293 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:20.383 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:20.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:20.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:20.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:20 smithi008 ceph-mon[184418]: pgmap v1073: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 26713/67038 objects degraded (39.848%); 358/67038 objects misplaced (0.534%); 19 MiB/s, 302 objects/s recovering 2023-10-09T05:52:20.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:20 smithi008 ceph-mon[191227]: pgmap v1073: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 26713/67038 objects degraded (39.848%); 358/67038 objects misplaced (0.534%); 19 MiB/s, 302 objects/s recovering 2023-10-09T05:52:20.496 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:20.563 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:20.652 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:20.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:20.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:20.768 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:20.782 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:52:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:52:20] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.33.4" 2023-10-09T05:52:20.810 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:52:20.811 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:20.881 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:20.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:20.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:20.994 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:21.106 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:21.186 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:21 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:21.347 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:21 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:21.347 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:21 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:21.390 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:21.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:21.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:21.502 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:21.563 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:22.099 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:22.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:22.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:22.169 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:22.180 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:22.205 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:22.242 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:22.328 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:22.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:22.367 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:22.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:22.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:22 smithi008 ceph-mon[191227]: pgmap v1074: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 26568/63495 objects degraded (41.843%); 358/63495 objects misplaced (0.564%); 11 MiB/s, 178 objects/s recovering 2023-10-09T05:52:22.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:22 smithi008 ceph-mon[184418]: pgmap v1074: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 26568/63495 objects degraded (41.843%); 358/63495 objects misplaced (0.564%); 11 MiB/s, 178 objects/s recovering 2023-10-09T05:52:22.482 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:22 smithi191 ceph-mon[161669]: pgmap v1074: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 26568/63495 objects degraded (41.843%); 358/63495 objects misplaced (0.564%); 11 MiB/s, 178 objects/s recovering 2023-10-09T05:52:22.488 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:22.579 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:22.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:22.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:22.685 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:22.785 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:23.079 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:23.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:23.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:23.194 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:23.261 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:23.354 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:23.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:23.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:23.450 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 26568/63495 objects degraded (41.843%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:23.451 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 26568/63495 objects degraded (41.843%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:23.457 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:23.529 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 26568/63495 objects degraded (41.843%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:24 smithi008 ceph-mon[184418]: pgmap v1075: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 25134/56937 objects degraded (44.144%); 260/56937 objects misplaced (0.457%); 18 MiB/s, 283 objects/s recovering 2023-10-09T05:52:24.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:24 smithi008 ceph-mon[191227]: pgmap v1075: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 25134/56937 objects degraded (44.144%); 260/56937 objects misplaced (0.457%); 18 MiB/s, 283 objects/s recovering 2023-10-09T05:52:24.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:24 smithi191 ceph-mon[161669]: pgmap v1075: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 25134/56937 objects degraded (44.144%); 260/56937 objects misplaced (0.457%); 18 MiB/s, 283 objects/s recovering 2023-10-09T05:52:24.655 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:24.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:24.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:24.773 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:24.859 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:24.934 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:52:24.995 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:52:25.046 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:52:25.089 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:52:25.139 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:52:25.172 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:52:25.208 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:52:25.247 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:52:25.266 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:52:25.266 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:52:25.267 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:52:25.267 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:52:25.267 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:52:25.267 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:52:25.267 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:52:25.267 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:52:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:25 smithi191 ceph-mon[161669]: pgmap v1076: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 24968/53640 objects degraded (46.547%); 260/53640 objects misplaced (0.485%); 15 MiB/s, 235 objects/s recovering 2023-10-09T05:52:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:25 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2610744380' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:25 smithi008 ceph-mon[184418]: pgmap v1076: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 24968/53640 objects degraded (46.547%); 260/53640 objects misplaced (0.485%); 15 MiB/s, 235 objects/s recovering 2023-10-09T05:52:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:25 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2610744380' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:25.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:25 smithi008 ceph-mon[191227]: pgmap v1076: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 24968/53640 objects degraded (46.547%); 260/53640 objects misplaced (0.485%); 15 MiB/s, 235 objects/s recovering 2023-10-09T05:52:25.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:25 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2610744380' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:27.345 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:27.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:27.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:27.454 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:27.542 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:28.047 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:27 smithi008 ceph-mon[184418]: pgmap v1077: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1017 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 24647/48672 objects degraded (50.639%); 158/48672 objects misplaced (0.325%); 15 MiB/s, 239 objects/s recovering 2023-10-09T05:52:28.047 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:28.047 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:28.048 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:27 smithi008 ceph-mon[191227]: pgmap v1077: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1017 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 24647/48672 objects degraded (50.639%); 158/48672 objects misplaced (0.325%); 15 MiB/s, 239 objects/s recovering 2023-10-09T05:52:28.048 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:28.048 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:28.051 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:28.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:28.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:28.165 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:28.240 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:27 smithi191 ceph-mon[161669]: pgmap v1077: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 1017 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 24647/48672 objects degraded (50.639%); 158/48672 objects misplaced (0.325%); 15 MiB/s, 239 objects/s recovering 2023-10-09T05:52:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:28.525 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:28.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:28.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:28.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:28.678 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:28.728 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:28.728 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:28.739 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:28.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:28.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:28.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:28.877 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:28.976 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:28.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:28.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:28.988 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24647/48672 objects degraded (50.639%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:28.988 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2968709159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:28.989 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:28 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:29.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:29.110 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:52:29.145 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24647/48672 objects degraded (50.639%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:29.146 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2968709159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:29.146 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:28 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:29.147 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24647/48672 objects degraded (50.639%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:29.147 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2968709159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:29.147 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:28 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:29.230 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:52:29.232 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:52:29.258 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:29.300 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:29.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:29.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:29.405 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:29.432 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:29.473 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:29.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:29.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:29.579 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:29.625 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:29.688 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:29.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:29.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:29.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:29.844 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:29.918 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:52:29.919 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:29.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:29.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:30.033 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:30.117 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:52:30.157 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:30.183 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[184418]: pgmap v1078: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 968 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 23444/46323 objects degraded (50.610%); 158/46323 objects misplaced (0.341%); 16 MiB/s, 259 objects/s recovering 2023-10-09T05:52:30.183 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:30.183 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:30.183 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:30.184 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[184418]: failed to encode map e358 with expected crc 2023-10-09T05:52:30.184 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:30.184 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[184418]: osdmap e358: 8 total, 8 up, 8 in 2023-10-09T05:52:30.184 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[184418]: failed to encode map e358 with expected crc 2023-10-09T05:52:30.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[191227]: pgmap v1078: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 968 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 23444/46323 objects degraded (50.610%); 158/46323 objects misplaced (0.341%); 16 MiB/s, 259 objects/s recovering 2023-10-09T05:52:30.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:30.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:30.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:30.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[191227]: failed to encode map e358 with expected crc 2023-10-09T05:52:30.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:30.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[191227]: osdmap e358: 8 total, 8 up, 8 in 2023-10-09T05:52:30.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:29 smithi008 ceph-mon[191227]: failed to encode map e358 with expected crc 2023-10-09T05:52:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:29 smithi191 ceph-mon[161669]: pgmap v1078: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 968 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 23444/46323 objects degraded (50.610%); 158/46323 objects misplaced (0.341%); 16 MiB/s, 259 objects/s recovering 2023-10-09T05:52:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:29 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:29 smithi191 ceph-mon[161669]: failed to encode map e358 with expected crc 2023-10-09T05:52:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:29 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:29 smithi191 ceph-mon[161669]: osdmap e358: 8 total, 8 up, 8 in 2023-10-09T05:52:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:29 smithi191 ceph-mon[161669]: failed to encode map e358 with expected crc 2023-10-09T05:52:30.863 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:52:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:52:30] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-10-09T05:52:31.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:30 smithi008 ceph-mon[184418]: from='client.46091 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:52:31.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:30 smithi008 ceph-mon[191227]: from='client.46091 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:52:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:30 smithi191 ceph-mon[161669]: from='client.46091 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:52:31.315 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:31.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:31.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:31.421 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:31.508 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:31.541 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1079,"stamp":"2023-10-09T05:52:29.095926+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1014831051,"num_objects":15441,"num_object_clones":0,"num_object_copies":46323,"num_objects_missing_on_primary":550,"num_objects_missing":550,"num_objects_degraded":23444,"num_objects_misplaced":158,"num_objects_unfound":0,"num_objects_dirty":15441,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":191573,"num_write_kb":6603238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33577,"num_bytes_recovered":2189504282,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":366346,"ondisk_log_size":366346,"up":357,"acting":343,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":398,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12375520,"kb_used_data":5966368,"kb_used_omap":45,"kb_used_meta":6408786,"kb_avail":737585696,"statfs":{"total":767960285184,"available":755287752704,"internally_reserved":0,"allocated":6109560832,"data_stored":6101771991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":6562597155},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,0,5,3,2,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-452526080,"num_objects":-6905,"num_object_clones":0,"num_object_copies":-20715,"num_objects_missing_on_primary":-218,"num_objects_missing":-218,"num_objects_degraded":-3269,"num_objects_misplaced":-200,"num_objects_unfound":0,"num_objects_dirty":-6905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6905,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2408,"num_bytes_recovered":157810688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.597878"},"pg_stats":[{"pgid":"8.2d","version":"357'8089","reported_seq":8188,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.288230+0000","last_change":"2023-10-09T05:50:08.061665+0000","last_active":"2023-10-09T05:52:25.288230+0000","last_peered":"2023-10-09T05:52:25.288230+0000","last_clean":"2023-10-09T05:52:25.288230+0000","last_became_active":"2023-10-09T05:50:08.057591+0000","last_became_peered":"2023-10-09T05:50:08.057591+0000","last_unstale":"2023-10-09T05:52:25.288230+0000","last_undegraded":"2023-10-09T05:52:25.288230+0000","last_fullsized":"2023-10-09T05:52:25.288230+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8089,"log_dups_size":0,"ondisk_log_size":8089,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14663686,"num_objects":224,"num_object_clones":0,"num_object_copies":672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2957,"num_write_kb":101776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.2c","version":"357'8338","reported_seq":8440,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:24.102242+0000","last_change":"2023-10-09T05:50:08.058697+0000","last_active":"2023-10-09T05:52:24.102242+0000","last_peered":"2023-10-09T05:52:24.102242+0000","last_clean":"2023-10-09T05:52:24.102242+0000","last_became_active":"2023-10-09T05:50:08.055366+0000","last_became_peered":"2023-10-09T05:50:08.055366+0000","last_unstale":"2023-10-09T05:52:24.102242+0000","last_undegraded":"2023-10-09T05:52:24.102242+0000","last_fullsized":"2023-10-09T05:52:24.102242+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8338,"log_dups_size":0,"ondisk_log_size":8338,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17203200,"num_objects":263,"num_object_clones":0,"num_object_copies":789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3029,"num_write_kb":105312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.2b","version":"357'8228","reported_seq":11257,"reported_epoch":357,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:25.442442+0000","last_change":"2023-10-09T05:50:08.118923+0000","last_active":"2023-10-09T05:52:25.442442+0000","last_peered":"2023-10-09T05:52:25.442442+0000","last_clean":"2023-10-09T05:48:53.462109+0000","last_became_active":"2023-10-09T05:50:08.113731+0000","last_became_peered":"2023-10-09T05:50:08.113731+0000","last_unstale":"2023-10-09T05:52:25.442442+0000","last_undegraded":"2023-10-09T05:50:08.089561+0000","last_fullsized":"2023-10-09T05:52:25.442442+0000","mapping_epoch":322,"log_start":"289'200","ondisk_log_start":"289'200","created":322,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8028,"log_dups_size":0,"ondisk_log_size":8028,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18038784,"num_objects":275,"num_object_clones":0,"num_object_copies":825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":130,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2982,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1290,"num_bytes_recovered":84508672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"1,3,4","objects":211},{"shards":"2,3,4","objects":64}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2a","version":"357'8135","reported_seq":11721,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:27.304906+0000","last_change":"2023-10-09T05:51:43.298018+0000","last_active":"2023-10-09T05:52:27.304906+0000","last_peered":"2023-10-09T05:52:27.304906+0000","last_clean":"2023-10-09T05:48:53.473854+0000","last_became_active":"2023-10-09T05:51:43.269984+0000","last_became_peered":"2023-10-09T05:51:43.269984+0000","last_unstale":"2023-10-09T05:52:27.304906+0000","last_undegraded":"2023-10-09T05:51:43.115562+0000","last_fullsized":"2023-10-09T05:51:43.062313+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8135,"log_dups_size":0,"ondisk_log_size":8135,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13205504,"num_objects":202,"num_object_clones":0,"num_object_copies":606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1291,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3000,"num_write_kb":102448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1343,"num_bytes_recovered":87916544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2","objects":-38},{"shards":"1,2,3","objects":184},{"shards":"2,3,4","objects":56}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.29","version":"357'7644","reported_seq":7752,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:24.081473+0000","last_change":"2023-10-09T05:51:44.323120+0000","last_active":"2023-10-09T05:52:24.081473+0000","last_peered":"2023-10-09T05:52:24.081473+0000","last_clean":"2023-10-09T05:51:41.995515+0000","last_became_active":"2023-10-09T05:51:44.308533+0000","last_became_peered":"2023-10-09T05:51:44.308533+0000","last_unstale":"2023-10-09T05:52:24.081473+0000","last_undegraded":"2023-10-09T05:51:44.104196+0000","last_fullsized":"2023-10-09T05:51:44.066927+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7644,"log_dups_size":0,"ondisk_log_size":7644,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15499264,"num_objects":237,"num_object_clones":0,"num_object_copies":711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1638,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3030,"num_write_kb":104512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[0,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":259},{"shards":"0,6","objects":-430}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.28","version":"357'7428","reported_seq":7525,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.495554+0000","last_change":"2023-10-09T05:50:08.055391+0000","last_active":"2023-10-09T05:52:25.495554+0000","last_peered":"2023-10-09T05:52:25.495554+0000","last_clean":"2023-10-09T05:52:25.495554+0000","last_became_active":"2023-10-09T05:50:08.052477+0000","last_became_peered":"2023-10-09T05:50:08.052477+0000","last_unstale":"2023-10-09T05:52:25.495554+0000","last_undegraded":"2023-10-09T05:52:25.495554+0000","last_fullsized":"2023-10-09T05:52:25.495554+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7428,"log_dups_size":0,"ondisk_log_size":7428,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13598720,"num_objects":208,"num_object_clones":0,"num_object_copies":624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2950,"num_write_kb":101024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.27","version":"357'7649","reported_seq":10098,"reported_epoch":357,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:26.795583+0000","last_change":"2023-10-09T05:50:08.428915+0000","last_active":"2023-10-09T05:52:26.795583+0000","last_peered":"2023-10-09T05:52:26.795583+0000","last_clean":"2023-10-09T05:48:53.410221+0000","last_became_active":"2023-10-09T05:50:08.415220+0000","last_became_peered":"2023-10-09T05:50:08.415220+0000","last_unstale":"2023-10-09T05:52:26.795583+0000","last_undegraded":"2023-10-09T05:50:08.088678+0000","last_fullsized":"2023-10-09T05:52:26.795583+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7649,"log_dups_size":0,"ondisk_log_size":7649,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14942208,"num_objects":228,"num_object_clones":0,"num_object_copies":684,"num_objects_missing_on_primary":272,"num_objects_missing":272,"num_objects_degraded":272,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3017,"num_write_kb":103840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1169,"num_bytes_recovered":76611584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,6","objects":175},{"shards":"3,4,6","objects":53}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.26","version":"357'7455","reported_seq":7558,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:23.867155+0000","last_change":"2023-10-09T05:50:08.077569+0000","last_active":"2023-10-09T05:52:23.867155+0000","last_peered":"2023-10-09T05:52:23.867155+0000","last_clean":"2023-10-09T05:52:23.867155+0000","last_became_active":"2023-10-09T05:50:08.072764+0000","last_became_peered":"2023-10-09T05:50:08.072764+0000","last_unstale":"2023-10-09T05:52:23.867155+0000","last_undegraded":"2023-10-09T05:52:23.867155+0000","last_fullsized":"2023-10-09T05:52:23.867155+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7455,"log_dups_size":0,"ondisk_log_size":7455,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13844480,"num_objects":211,"num_object_clones":0,"num_object_copies":633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2963,"num_write_kb":101584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.25","version":"357'7478","reported_seq":7573,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.309434+0000","last_change":"2023-10-09T05:50:08.046650+0000","last_active":"2023-10-09T05:52:25.309434+0000","last_peered":"2023-10-09T05:52:25.309434+0000","last_clean":"2023-10-09T05:52:25.309434+0000","last_became_active":"2023-10-09T05:50:08.043790+0000","last_became_peered":"2023-10-09T05:50:08.043790+0000","last_unstale":"2023-10-09T05:52:25.309434+0000","last_undegraded":"2023-10-09T05:52:25.309434+0000","last_fullsized":"2023-10-09T05:52:25.309434+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7478,"log_dups_size":0,"ondisk_log_size":7478,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17743872,"num_objects":271,"num_object_clones":0,"num_object_copies":813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2949,"num_write_kb":103024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.24","version":"357'7527","reported_seq":7623,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.540926+0000","last_change":"2023-10-09T05:50:08.055768+0000","last_active":"2023-10-09T05:52:25.540926+0000","last_peered":"2023-10-09T05:52:25.540926+0000","last_clean":"2023-10-09T05:52:25.540926+0000","last_became_active":"2023-10-09T05:50:08.052904+0000","last_became_peered":"2023-10-09T05:50:08.052904+0000","last_unstale":"2023-10-09T05:52:25.540926+0000","last_undegraded":"2023-10-09T05:52:25.540926+0000","last_fullsized":"2023-10-09T05:52:25.540926+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7527,"log_dups_size":0,"ondisk_log_size":7527,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16171008,"num_objects":247,"num_object_clones":0,"num_object_copies":741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2981,"num_write_kb":103280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.23","version":"357'7326","reported_seq":10046,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:25.461657+0000","last_change":"2023-10-09T05:50:19.098280+0000","last_active":"2023-10-09T05:52:25.461657+0000","last_peered":"2023-10-09T05:52:25.461657+0000","last_clean":"2023-10-09T05:48:53.427903+0000","last_became_active":"2023-10-09T05:50:19.090954+0000","last_became_peered":"2023-10-09T05:50:19.090954+0000","last_unstale":"2023-10-09T05:52:25.461657+0000","last_undegraded":"2023-10-09T05:50:19.046676+0000","last_fullsized":"2023-10-09T05:50:19.045180+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7326,"log_dups_size":0,"ondisk_log_size":7326,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15564800,"num_objects":238,"num_object_clones":0,"num_object_copies":714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":129,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2970,"num_write_kb":102624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1304,"num_bytes_recovered":85458944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"1,5","objects":172},{"shards":"2,5,6","objects":66}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.22","version":"357'7227","reported_seq":7326,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.463897+0000","last_change":"2023-10-09T05:49:20.140728+0000","last_active":"2023-10-09T05:52:25.463897+0000","last_peered":"2023-10-09T05:52:25.463897+0000","last_clean":"2023-10-09T05:52:25.463897+0000","last_became_active":"2023-10-09T05:49:20.132812+0000","last_became_peered":"2023-10-09T05:49:20.132812+0000","last_unstale":"2023-10-09T05:52:25.463897+0000","last_undegraded":"2023-10-09T05:52:25.463897+0000","last_fullsized":"2023-10-09T05:52:25.463897+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7227,"log_dups_size":0,"ondisk_log_size":7227,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17580032,"num_objects":268,"num_object_clones":0,"num_object_copies":804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2905,"num_write_kb":101552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.21","version":"357'7343","reported_seq":10071,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:24.312731+0000","last_change":"2023-10-09T05:50:19.777110+0000","last_active":"2023-10-09T05:52:24.312731+0000","last_peered":"2023-10-09T05:52:24.312731+0000","last_clean":"2023-10-09T05:48:53.421353+0000","last_became_active":"2023-10-09T05:50:19.773817+0000","last_became_peered":"2023-10-09T05:50:19.773817+0000","last_unstale":"2023-10-09T05:52:24.312731+0000","last_undegraded":"2023-10-09T05:50:19.043939+0000","last_fullsized":"2023-10-09T05:50:19.040976+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7343,"log_dups_size":0,"ondisk_log_size":7343,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14876672,"num_objects":227,"num_object_clones":0,"num_object_copies":681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":161,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2976,"num_write_kb":102496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1308,"num_bytes_recovered":85721088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,1","objects":138},{"shards":"0,2,5","objects":89}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.20","version":"357'7422","reported_seq":10625,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:27.265246+0000","last_change":"2023-10-09T05:51:43.230120+0000","last_active":"2023-10-09T05:52:27.265246+0000","last_peered":"2023-10-09T05:52:27.265246+0000","last_clean":"2023-10-09T05:48:53.429181+0000","last_became_active":"2023-10-09T05:51:43.216843+0000","last_became_peered":"2023-10-09T05:51:43.216843+0000","last_unstale":"2023-10-09T05:52:27.265246+0000","last_undegraded":"2023-10-09T05:51:43.129071+0000","last_fullsized":"2023-10-09T05:51:43.055387+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7422,"log_dups_size":0,"ondisk_log_size":7422,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16941056,"num_objects":259,"num_object_clones":0,"num_object_copies":777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1233,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2999,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1312,"num_bytes_recovered":85983232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[3,1],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3","objects":41},{"shards":"1,3,4","objects":170},{"shards":"3,4,6","objects":48}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.11","version":"357'8575","reported_seq":12889,"reported_epoch":357,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:24.312072+0000","last_change":"2023-10-09T05:50:19.729796+0000","last_active":"2023-10-09T05:52:24.312072+0000","last_peered":"2023-10-09T05:52:24.312072+0000","last_clean":"2023-10-09T05:48:53.444054+0000","last_became_active":"2023-10-09T05:50:19.723776+0000","last_became_peered":"2023-10-09T05:50:19.723776+0000","last_unstale":"2023-10-09T05:52:24.312072+0000","last_undegraded":"2023-10-09T05:50:19.039020+0000","last_fullsized":"2023-10-09T05:52:24.312072+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8575,"log_dups_size":0,"ondisk_log_size":8575,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36175872,"num_objects":552,"num_object_clones":0,"num_object_copies":1656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":747,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5876,"num_write_kb":205696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2105,"num_bytes_recovered":137887744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,1,2","objects":363},{"shards":"0,2,5","objects":189}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":357,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.146520+0000","last_change":"2023-10-09T05:48:54.494513+0000","last_active":"2023-10-09T05:52:16.146520+0000","last_peered":"2023-10-09T05:52:16.146520+0000","last_clean":"2023-10-09T05:52:16.146520+0000","last_became_active":"2023-10-09T05:48:54.493510+0000","last_became_peered":"2023-10-09T05:48:54.493510+0000","last_unstale":"2023-10-09T05:52:16.146520+0000","last_undegraded":"2023-10-09T05:52:16.146520+0000","last_fullsized":"2023-10-09T05:52:16.146520+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":287,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.146327+0000","last_change":"2023-10-09T05:50:18.047163+0000","last_active":"2023-10-09T05:52:16.146327+0000","last_peered":"2023-10-09T05:52:16.146327+0000","last_clean":"2023-10-09T05:52:16.146327+0000","last_became_active":"2023-10-09T05:50:18.046872+0000","last_became_peered":"2023-10-09T05:50:18.046872+0000","last_unstale":"2023-10-09T05:52:16.146327+0000","last_undegraded":"2023-10-09T05:52:16.146327+0000","last_fullsized":"2023-10-09T05:52:16.146327+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.12","version":"357'8537","reported_seq":8626,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.373152+0000","last_change":"2023-10-09T05:48:46.263477+0000","last_active":"2023-10-09T05:52:25.373152+0000","last_peered":"2023-10-09T05:52:25.373152+0000","last_clean":"2023-10-09T05:52:25.373152+0000","last_became_active":"2023-10-09T05:48:46.259060+0000","last_became_peered":"2023-10-09T05:48:46.259060+0000","last_unstale":"2023-10-09T05:52:25.373152+0000","last_undegraded":"2023-10-09T05:52:25.373152+0000","last_fullsized":"2023-10-09T05:52:25.373152+0000","mapping_epoch":299,"log_start":"290'1603","ondisk_log_start":"290'1603","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6934,"log_dups_size":0,"ondisk_log_size":6934,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29458432,"num_objects":450,"num_object_clones":0,"num_object_copies":1350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5870,"num_write_kb":202208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":101,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145406+0000","last_change":"2023-10-09T05:50:18.055745+0000","last_active":"2023-10-09T05:52:16.145406+0000","last_peered":"2023-10-09T05:52:16.145406+0000","last_clean":"2023-10-09T05:52:16.145406+0000","last_became_active":"2023-10-09T05:50:18.055524+0000","last_became_peered":"2023-10-09T05:50:18.055524+0000","last_unstale":"2023-10-09T05:52:16.145406+0000","last_undegraded":"2023-10-09T05:52:16.145406+0000","last_fullsized":"2023-10-09T05:52:16.145406+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":375,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.147298+0000","last_change":"2023-10-09T05:51:22.261181+0000","last_active":"2023-10-09T05:52:16.147298+0000","last_peered":"2023-10-09T05:52:16.147298+0000","last_clean":"2023-10-09T05:52:16.147298+0000","last_became_active":"2023-10-09T05:51:22.260808+0000","last_became_peered":"2023-10-09T05:51:22.260808+0000","last_unstale":"2023-10-09T05:52:16.147298+0000","last_undegraded":"2023-10-09T05:52:16.147298+0000","last_fullsized":"2023-10-09T05:52:16.147298+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:17.362302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"357'8554","reported_seq":13038,"reported_epoch":357,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:25.573754+0000","last_change":"2023-10-09T05:50:19.098294+0000","last_active":"2023-10-09T05:52:25.573754+0000","last_peered":"2023-10-09T05:52:25.573754+0000","last_clean":"2023-10-09T05:48:53.465296+0000","last_became_active":"2023-10-09T05:50:19.091214+0000","last_became_peered":"2023-10-09T05:50:19.091214+0000","last_unstale":"2023-10-09T05:52:25.573754+0000","last_undegraded":"2023-10-09T05:50:19.044182+0000","last_fullsized":"2023-10-09T05:52:25.573754+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8554,"log_dups_size":0,"ondisk_log_size":8554,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34275328,"num_objects":523,"num_object_clones":0,"num_object_copies":1569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":622,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5863,"num_write_kb":204352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2191,"num_bytes_recovered":143589376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"1,5,6","objects":381},{"shards":"2,5,6","objects":141},{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":326,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.143627+0000","last_change":"2023-10-09T05:48:54.492664+0000","last_active":"2023-10-09T05:52:16.143627+0000","last_peered":"2023-10-09T05:52:16.143627+0000","last_clean":"2023-10-09T05:52:16.143627+0000","last_became_active":"2023-10-09T05:48:54.492299+0000","last_became_peered":"2023-10-09T05:48:54.492299+0000","last_unstale":"2023-10-09T05:52:16.143627+0000","last_undegraded":"2023-10-09T05:52:16.143627+0000","last_fullsized":"2023-10-09T05:52:16.143627+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:43.198424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":190,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.160494+0000","last_change":"2023-10-09T05:48:54.492447+0000","last_active":"2023-10-09T05:52:16.160494+0000","last_peered":"2023-10-09T05:52:16.160494+0000","last_clean":"2023-10-09T05:52:16.160494+0000","last_became_active":"2023-10-09T05:48:54.492089+0000","last_became_peered":"2023-10-09T05:48:54.492089+0000","last_unstale":"2023-10-09T05:52:16.160494+0000","last_undegraded":"2023-10-09T05:52:16.160494+0000","last_fullsized":"2023-10-09T05:52:16.160494+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.1c","version":"357'9476","reported_seq":9568,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:23.823861+0000","last_change":"2023-10-09T05:49:20.384940+0000","last_active":"2023-10-09T05:52:23.823861+0000","last_peered":"2023-10-09T05:52:23.823861+0000","last_clean":"2023-10-09T05:52:23.823861+0000","last_became_active":"2023-10-09T05:49:20.380945+0000","last_became_peered":"2023-10-09T05:49:20.380945+0000","last_unstale":"2023-10-09T05:52:23.823861+0000","last_undegraded":"2023-10-09T05:52:23.823861+0000","last_fullsized":"2023-10-09T05:52:23.823861+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9476,"log_dups_size":0,"ondisk_log_size":9476,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35258368,"num_objects":538,"num_object_clones":0,"num_object_copies":1614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6044,"num_write_kb":210624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":313,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.143760+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:52:16.143760+0000","last_peered":"2023-10-09T05:52:16.143760+0000","last_clean":"2023-10-09T05:52:16.143760+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:52:16.143760+0000","last_undegraded":"2023-10-09T05:52:16.143760+0000","last_fullsized":"2023-10-09T05:52:16.143760+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":317,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.150410+0000","last_change":"2023-10-09T05:50:18.054081+0000","last_active":"2023-10-09T05:52:16.150410+0000","last_peered":"2023-10-09T05:52:16.150410+0000","last_clean":"2023-10-09T05:52:16.150410+0000","last_became_active":"2023-10-09T05:50:18.053746+0000","last_became_peered":"2023-10-09T05:50:18.053746+0000","last_unstale":"2023-10-09T05:52:16.150410+0000","last_undegraded":"2023-10-09T05:52:16.150410+0000","last_fullsized":"2023-10-09T05:52:16.150410+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:55.758681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1d","version":"357'9195","reported_seq":9285,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.534724+0000","last_change":"2023-10-09T05:49:20.157594+0000","last_active":"2023-10-09T05:52:25.534724+0000","last_peered":"2023-10-09T05:52:25.534724+0000","last_clean":"2023-10-09T05:52:25.534724+0000","last_became_active":"2023-10-09T05:49:20.153063+0000","last_became_peered":"2023-10-09T05:49:20.153063+0000","last_unstale":"2023-10-09T05:52:25.534724+0000","last_undegraded":"2023-10-09T05:52:25.534724+0000","last_fullsized":"2023-10-09T05:52:25.534724+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9195,"log_dups_size":0,"ondisk_log_size":9195,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30965772,"num_objects":473,"num_object_clones":0,"num_object_copies":1419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":473,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5889,"num_write_kb":203553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":17,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.160272+0000","last_change":"2023-10-09T05:51:43.123678+0000","last_active":"2023-10-09T05:52:16.160272+0000","last_peered":"2023-10-09T05:52:16.160272+0000","last_clean":"2023-10-09T05:52:16.160272+0000","last_became_active":"2023-10-09T05:51:43.121947+0000","last_became_peered":"2023-10-09T05:51:43.121947+0000","last_unstale":"2023-10-09T05:52:16.160272+0000","last_undegraded":"2023-10-09T05:52:16.160272+0000","last_fullsized":"2023-10-09T05:52:16.160272+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":292,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157451+0000","last_change":"2023-10-09T05:48:25.068328+0000","last_active":"2023-10-09T05:52:16.157451+0000","last_peered":"2023-10-09T05:52:16.157451+0000","last_clean":"2023-10-09T05:52:16.157451+0000","last_became_active":"2023-10-09T05:48:25.068094+0000","last_became_peered":"2023-10-09T05:48:25.068094+0000","last_unstale":"2023-10-09T05:52:16.157451+0000","last_undegraded":"2023-10-09T05:52:16.157451+0000","last_fullsized":"2023-10-09T05:52:16.157451+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.1e","version":"357'9248","reported_seq":9343,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:24.256226+0000","last_change":"2023-10-09T05:50:17.990359+0000","last_active":"2023-10-09T05:52:24.256226+0000","last_peered":"2023-10-09T05:52:24.256226+0000","last_clean":"2023-10-09T05:52:24.256226+0000","last_became_active":"2023-10-09T05:49:20.168491+0000","last_became_peered":"2023-10-09T05:49:20.168491+0000","last_unstale":"2023-10-09T05:52:24.256226+0000","last_undegraded":"2023-10-09T05:52:24.256226+0000","last_fullsized":"2023-10-09T05:52:24.256226+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9248,"log_dups_size":0,"ondisk_log_size":9248,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36831232,"num_objects":562,"num_object_clones":0,"num_object_copies":1686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5901,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":142,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.158371+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:52:16.158371+0000","last_peered":"2023-10-09T05:52:16.158371+0000","last_clean":"2023-10-09T05:52:16.158371+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:52:16.158371+0000","last_undegraded":"2023-10-09T05:52:16.158371+0000","last_fullsized":"2023-10-09T05:52:16.158371+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":137,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.144733+0000","last_change":"2023-10-09T05:50:18.045973+0000","last_active":"2023-10-09T05:52:16.144733+0000","last_peered":"2023-10-09T05:52:16.144733+0000","last_clean":"2023-10-09T05:52:16.144733+0000","last_became_active":"2023-10-09T05:50:18.045688+0000","last_became_peered":"2023-10-09T05:50:18.045688+0000","last_unstale":"2023-10-09T05:52:16.144733+0000","last_undegraded":"2023-10-09T05:52:16.144733+0000","last_fullsized":"2023-10-09T05:52:16.144733+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:51.105027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.1f","version":"357'9178","reported_seq":15984,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:27.350747+0000","last_change":"2023-10-09T05:51:14.207219+0000","last_active":"2023-10-09T05:52:27.350747+0000","last_peered":"2023-10-09T05:52:27.350747+0000","last_clean":"2023-10-09T05:48:53.467734+0000","last_became_active":"2023-10-09T05:51:14.187967+0000","last_became_peered":"2023-10-09T05:51:14.187967+0000","last_unstale":"2023-10-09T05:52:27.350747+0000","last_undegraded":"2023-10-09T05:51:14.126809+0000","last_fullsized":"2023-10-09T05:51:14.060740+0000","mapping_epoch":340,"log_start":"290'1412","ondisk_log_start":"290'1412","created":311,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7766,"log_dups_size":0,"ondisk_log_size":7766,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27951104,"num_objects":427,"num_object_clones":0,"num_object_copies":1281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1891,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5895,"num_write_kb":202272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2610,"num_bytes_recovered":170852352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[3,1],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3","objects":1},{"shards":"1,3,4","objects":334},{"shards":"3,4,6","objects":92}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":383,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145851+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:52:16.145851+0000","last_peered":"2023-10-09T05:52:16.145851+0000","last_clean":"2023-10-09T05:52:16.145851+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:52:16.145851+0000","last_undegraded":"2023-10-09T05:52:16.145851+0000","last_fullsized":"2023-10-09T05:52:16.145851+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":340,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.146258+0000","last_change":"2023-10-09T05:48:54.492082+0000","last_active":"2023-10-09T05:52:16.146258+0000","last_peered":"2023-10-09T05:52:16.146258+0000","last_clean":"2023-10-09T05:52:16.146258+0000","last_became_active":"2023-10-09T05:48:54.491753+0000","last_became_peered":"2023-10-09T05:48:54.491753+0000","last_unstale":"2023-10-09T05:52:16.146258+0000","last_undegraded":"2023-10-09T05:52:16.146258+0000","last_fullsized":"2023-10-09T05:52:16.146258+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"357'8580","reported_seq":8671,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.544374+0000","last_change":"2023-10-09T05:48:46.314063+0000","last_active":"2023-10-09T05:52:25.544374+0000","last_peered":"2023-10-09T05:52:25.544374+0000","last_clean":"2023-10-09T05:52:25.544374+0000","last_became_active":"2023-10-09T05:48:46.261937+0000","last_became_peered":"2023-10-09T05:48:46.261937+0000","last_unstale":"2023-10-09T05:52:25.544374+0000","last_undegraded":"2023-10-09T05:52:25.544374+0000","last_fullsized":"2023-10-09T05:52:25.544374+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8580,"log_dups_size":0,"ondisk_log_size":8580,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31719424,"num_objects":484,"num_object_clones":0,"num_object_copies":1452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5898,"num_write_kb":204224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":142,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157840+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:52:16.157840+0000","last_peered":"2023-10-09T05:52:16.157840+0000","last_clean":"2023-10-09T05:52:16.157840+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:52:16.157840+0000","last_undegraded":"2023-10-09T05:52:16.157840+0000","last_fullsized":"2023-10-09T05:52:16.157840+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":320,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.158244+0000","last_change":"2023-10-09T05:50:18.039555+0000","last_active":"2023-10-09T05:52:16.158244+0000","last_peered":"2023-10-09T05:52:16.158244+0000","last_clean":"2023-10-09T05:52:16.158244+0000","last_became_active":"2023-10-09T05:50:18.039222+0000","last_became_peered":"2023-10-09T05:50:18.039222+0000","last_unstale":"2023-10-09T05:52:16.158244+0000","last_undegraded":"2023-10-09T05:52:16.158244+0000","last_fullsized":"2023-10-09T05:52:16.158244+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.19","version":"357'8729","reported_seq":8826,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:24.081461+0000","last_change":"2023-10-09T05:51:44.344144+0000","last_active":"2023-10-09T05:52:24.081461+0000","last_peered":"2023-10-09T05:52:24.081461+0000","last_clean":"2023-10-09T05:51:42.004639+0000","last_became_active":"2023-10-09T05:51:44.322844+0000","last_became_peered":"2023-10-09T05:51:44.322844+0000","last_unstale":"2023-10-09T05:52:24.081461+0000","last_undegraded":"2023-10-09T05:51:44.113025+0000","last_fullsized":"2023-10-09T05:51:44.092936+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8729,"log_dups_size":0,"ondisk_log_size":8729,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35061760,"num_objects":535,"num_object_clones":0,"num_object_copies":1605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3236,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5981,"num_write_kb":208512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[0,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":513},{"shards":"0,6","objects":-850}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":346,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145038+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:52:16.145038+0000","last_peered":"2023-10-09T05:52:16.145038+0000","last_clean":"2023-10-09T05:52:16.145038+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:52:16.145038+0000","last_undegraded":"2023-10-09T05:52:16.145038+0000","last_fullsized":"2023-10-09T05:52:16.145038+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":17,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.160397+0000","last_change":"2023-10-09T05:51:43.075341+0000","last_active":"2023-10-09T05:52:16.160397+0000","last_peered":"2023-10-09T05:52:16.160397+0000","last_clean":"2023-10-09T05:52:16.160397+0000","last_became_active":"2023-10-09T05:51:43.075024+0000","last_became_peered":"2023-10-09T05:51:43.075024+0000","last_unstale":"2023-10-09T05:52:16.160397+0000","last_undegraded":"2023-10-09T05:52:16.160397+0000","last_fullsized":"2023-10-09T05:52:16.160397+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.1a","version":"357'9265","reported_seq":15542,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:27.305469+0000","last_change":"2023-10-09T05:51:43.298787+0000","last_active":"2023-10-09T05:52:27.305469+0000","last_peered":"2023-10-09T05:52:27.305469+0000","last_clean":"2023-10-09T05:48:53.473854+0000","last_became_active":"2023-10-09T05:51:43.282023+0000","last_became_peered":"2023-10-09T05:51:43.282023+0000","last_unstale":"2023-10-09T05:52:27.305469+0000","last_undegraded":"2023-10-09T05:51:43.169659+0000","last_fullsized":"2023-10-09T05:51:43.062848+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9265,"log_dups_size":0,"ondisk_log_size":9265,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29818880,"num_objects":455,"num_object_clones":0,"num_object_copies":1365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2462,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5948,"num_write_kb":204896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2633,"num_bytes_recovered":172425216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2","objects":38},{"shards":"1,2,3","objects":324},{"shards":"2,3,4","objects":93}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":136,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157539+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:52:16.157539+0000","last_peered":"2023-10-09T05:52:16.157539+0000","last_clean":"2023-10-09T05:52:16.157539+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:52:16.157539+0000","last_undegraded":"2023-10-09T05:52:16.157539+0000","last_fullsized":"2023-10-09T05:52:16.157539+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":346,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.147375+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:52:16.147375+0000","last_peered":"2023-10-09T05:52:16.147375+0000","last_clean":"2023-10-09T05:52:16.147375+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:52:16.147375+0000","last_undegraded":"2023-10-09T05:52:16.147375+0000","last_fullsized":"2023-10-09T05:52:16.147375+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1b","version":"357'9426","reported_seq":14849,"reported_epoch":357,"state":"active+recovering","last_fresh":"2023-10-09T05:52:25.502030+0000","last_change":"2023-10-09T05:50:18.038965+0000","last_active":"2023-10-09T05:52:25.502030+0000","last_peered":"2023-10-09T05:52:25.502030+0000","last_clean":"2023-10-09T05:48:53.462109+0000","last_became_active":"2023-10-09T05:49:20.374568+0000","last_became_peered":"2023-10-09T05:49:20.374568+0000","last_unstale":"2023-10-09T05:52:25.502030+0000","last_undegraded":"2023-10-09T05:52:25.502030+0000","last_fullsized":"2023-10-09T05:52:25.502030+0000","mapping_epoch":311,"log_start":"290'1930","ondisk_log_start":"290'1930","created":311,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7496,"log_dups_size":0,"ondisk_log_size":7496,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30965760,"num_objects":473,"num_object_clones":0,"num_object_copies":1419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":158,"num_objects_unfound":0,"num_objects_dirty":473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6042,"num_write_kb":208448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2660,"num_bytes_recovered":174194688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"1,3,4","objects":315},{"shards":"2,3,4","objects":158}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":172,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.143468+0000","last_change":"2023-10-09T05:51:43.221482+0000","last_active":"2023-10-09T05:52:16.143468+0000","last_peered":"2023-10-09T05:52:16.143468+0000","last_clean":"2023-10-09T05:52:16.143468+0000","last_became_active":"2023-10-09T05:51:43.072606+0000","last_became_peered":"2023-10-09T05:51:43.072606+0000","last_unstale":"2023-10-09T05:52:16.143468+0000","last_undegraded":"2023-10-09T05:52:16.143468+0000","last_fullsized":"2023-10-09T05:52:16.143468+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:22.097091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":362,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157768+0000","last_change":"2023-10-09T05:48:54.492012+0000","last_active":"2023-10-09T05:52:16.157768+0000","last_peered":"2023-10-09T05:52:16.157768+0000","last_clean":"2023-10-09T05:52:16.157768+0000","last_became_active":"2023-10-09T05:48:54.491644+0000","last_became_peered":"2023-10-09T05:48:54.491644+0000","last_unstale":"2023-10-09T05:52:16.157768+0000","last_undegraded":"2023-10-09T05:52:16.157768+0000","last_fullsized":"2023-10-09T05:52:16.157768+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"357'7517","reported_seq":7618,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.481387+0000","last_change":"2023-10-09T05:50:17.993839+0000","last_active":"2023-10-09T05:52:25.481387+0000","last_peered":"2023-10-09T05:52:25.481387+0000","last_clean":"2023-10-09T05:52:25.481387+0000","last_became_active":"2023-10-09T05:50:08.045912+0000","last_became_peered":"2023-10-09T05:50:08.045912+0000","last_unstale":"2023-10-09T05:52:25.481387+0000","last_undegraded":"2023-10-09T05:52:25.481387+0000","last_fullsized":"2023-10-09T05:52:25.481387+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7517,"log_dups_size":0,"ondisk_log_size":7517,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16826368,"num_objects":256,"num_object_clones":0,"num_object_copies":768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2970,"num_write_kb":103280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":140,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157691+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:52:16.157691+0000","last_peered":"2023-10-09T05:52:16.157691+0000","last_clean":"2023-10-09T05:52:16.157691+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:52:16.157691+0000","last_undegraded":"2023-10-09T05:52:16.157691+0000","last_fullsized":"2023-10-09T05:52:16.157691+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.159939+0000","last_change":"2023-10-09T05:51:43.075609+0000","last_active":"2023-10-09T05:52:16.159939+0000","last_peered":"2023-10-09T05:52:16.159939+0000","last_clean":"2023-10-09T05:52:16.159939+0000","last_became_active":"2023-10-09T05:51:43.075280+0000","last_became_peered":"2023-10-09T05:51:43.075280+0000","last_unstale":"2023-10-09T05:52:16.159939+0000","last_undegraded":"2023-10-09T05:52:16.159939+0000","last_fullsized":"2023-10-09T05:52:16.159939+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"357'7523","reported_seq":7619,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.364735+0000","last_change":"2023-10-09T05:50:08.046359+0000","last_active":"2023-10-09T05:52:25.364735+0000","last_peered":"2023-10-09T05:52:25.364735+0000","last_clean":"2023-10-09T05:52:25.364735+0000","last_became_active":"2023-10-09T05:50:08.041168+0000","last_became_peered":"2023-10-09T05:50:08.041168+0000","last_unstale":"2023-10-09T05:52:25.364735+0000","last_undegraded":"2023-10-09T05:52:25.364735+0000","last_fullsized":"2023-10-09T05:52:25.364735+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7523,"log_dups_size":0,"ondisk_log_size":7523,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14794752,"num_objects":225,"num_object_clones":0,"num_object_copies":675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2993,"num_write_kb":103024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":166,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157998+0000","last_change":"2023-10-09T05:48:54.539425+0000","last_active":"2023-10-09T05:52:16.157998+0000","last_peered":"2023-10-09T05:52:16.157998+0000","last_clean":"2023-10-09T05:52:16.157998+0000","last_became_active":"2023-10-09T05:48:54.486401+0000","last_became_peered":"2023-10-09T05:48:54.486401+0000","last_unstale":"2023-10-09T05:52:16.157998+0000","last_undegraded":"2023-10-09T05:52:16.157998+0000","last_fullsized":"2023-10-09T05:52:16.157998+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":373,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145460+0000","last_change":"2023-10-09T05:48:54.492534+0000","last_active":"2023-10-09T05:52:16.145460+0000","last_peered":"2023-10-09T05:52:16.145460+0000","last_clean":"2023-10-09T05:52:16.145460+0000","last_became_active":"2023-10-09T05:48:54.492172+0000","last_became_peered":"2023-10-09T05:48:54.492172+0000","last_unstale":"2023-10-09T05:52:16.145460+0000","last_undegraded":"2023-10-09T05:52:16.145460+0000","last_fullsized":"2023-10-09T05:52:16.145460+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"357'7400","reported_seq":7503,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:23.593588+0000","last_change":"2023-10-09T05:50:08.052758+0000","last_active":"2023-10-09T05:52:23.593588+0000","last_peered":"2023-10-09T05:52:23.593588+0000","last_clean":"2023-10-09T05:52:23.593588+0000","last_became_active":"2023-10-09T05:50:08.049618+0000","last_became_peered":"2023-10-09T05:50:08.049618+0000","last_unstale":"2023-10-09T05:52:23.593588+0000","last_undegraded":"2023-10-09T05:52:23.593588+0000","last_fullsized":"2023-10-09T05:52:23.593588+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7400,"log_dups_size":0,"ondisk_log_size":7400,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17448960,"num_objects":266,"num_object_clones":0,"num_object_copies":798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2908,"num_write_kb":101584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":166,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.158281+0000","last_change":"2023-10-09T05:51:43.265312+0000","last_active":"2023-10-09T05:52:16.158281+0000","last_peered":"2023-10-09T05:52:16.158281+0000","last_clean":"2023-10-09T05:52:16.158281+0000","last_became_active":"2023-10-09T05:51:43.076237+0000","last_became_peered":"2023-10-09T05:51:43.076237+0000","last_unstale":"2023-10-09T05:52:16.158281+0000","last_undegraded":"2023-10-09T05:52:16.158281+0000","last_fullsized":"2023-10-09T05:52:16.158281+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":334,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157597+0000","last_change":"2023-10-09T05:50:18.039975+0000","last_active":"2023-10-09T05:52:16.157597+0000","last_peered":"2023-10-09T05:52:16.157597+0000","last_clean":"2023-10-09T05:52:16.157597+0000","last_became_active":"2023-10-09T05:50:18.039332+0000","last_became_peered":"2023-10-09T05:50:18.039332+0000","last_unstale":"2023-10-09T05:52:16.157597+0000","last_undegraded":"2023-10-09T05:52:16.157597+0000","last_fullsized":"2023-10-09T05:52:16.157597+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.7","version":"357'7686","reported_seq":10207,"reported_epoch":357,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:26.917695+0000","last_change":"2023-10-09T05:50:09.462704+0000","last_active":"2023-10-09T05:52:26.917695+0000","last_peered":"2023-10-09T05:52:26.917695+0000","last_clean":"2023-10-09T05:48:53.410221+0000","last_became_active":"2023-10-09T05:50:09.449813+0000","last_became_peered":"2023-10-09T05:50:09.449813+0000","last_unstale":"2023-10-09T05:52:26.917695+0000","last_undegraded":"2023-10-09T05:50:09.140916+0000","last_fullsized":"2023-10-09T05:52:26.917695+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7686,"log_dups_size":0,"ondisk_log_size":7686,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12517376,"num_objects":191,"num_object_clones":0,"num_object_copies":573,"num_objects_missing_on_primary":278,"num_objects_missing":278,"num_objects_degraded":278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3054,"num_write_kb":103840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1206,"num_bytes_recovered":79036416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,6","objects":145},{"shards":"3,4,6","objects":46}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":324,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.160418+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:52:16.160418+0000","last_peered":"2023-10-09T05:52:16.160418+0000","last_clean":"2023-10-09T05:52:16.160418+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:52:16.160418+0000","last_undegraded":"2023-10-09T05:52:16.160418+0000","last_fullsized":"2023-10-09T05:52:16.160418+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":331,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.142933+0000","last_change":"2023-10-09T05:48:54.495800+0000","last_active":"2023-10-09T05:52:16.142933+0000","last_peered":"2023-10-09T05:52:16.142933+0000","last_clean":"2023-10-09T05:52:16.142933+0000","last_became_active":"2023-10-09T05:48:54.495540+0000","last_became_peered":"2023-10-09T05:48:54.495540+0000","last_unstale":"2023-10-09T05:52:16.142933+0000","last_undegraded":"2023-10-09T05:52:16.142933+0000","last_fullsized":"2023-10-09T05:52:16.142933+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:26.903669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"357'7470","reported_seq":10788,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:27.225234+0000","last_change":"2023-10-09T05:51:43.197387+0000","last_active":"2023-10-09T05:52:27.225234+0000","last_peered":"2023-10-09T05:52:27.225234+0000","last_clean":"2023-10-09T05:48:53.429181+0000","last_became_active":"2023-10-09T05:51:43.181186+0000","last_became_peered":"2023-10-09T05:51:43.181186+0000","last_unstale":"2023-10-09T05:52:27.225234+0000","last_undegraded":"2023-10-09T05:51:43.114355+0000","last_fullsized":"2023-10-09T05:51:43.055685+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7470,"log_dups_size":0,"ondisk_log_size":7470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13795328,"num_objects":210,"num_object_clones":0,"num_object_copies":630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1299,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3046,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1360,"num_bytes_recovered":89128960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[3,1],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3","objects":-23},{"shards":"1,3,4","objects":177},{"shards":"3,4,6","objects":56}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":94,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.143036+0000","last_change":"2023-10-09T05:48:25.067945+0000","last_active":"2023-10-09T05:52:16.143036+0000","last_peered":"2023-10-09T05:52:16.143036+0000","last_clean":"2023-10-09T05:52:16.143036+0000","last_became_active":"2023-10-09T05:48:25.067721+0000","last_became_peered":"2023-10-09T05:48:25.067721+0000","last_unstale":"2023-10-09T05:52:16.143036+0000","last_undegraded":"2023-10-09T05:52:16.143036+0000","last_fullsized":"2023-10-09T05:52:16.143036+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:58.655544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":322,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145731+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:52:16.145731+0000","last_peered":"2023-10-09T05:52:16.145731+0000","last_clean":"2023-10-09T05:52:16.145731+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:52:16.145731+0000","last_undegraded":"2023-10-09T05:52:16.145731+0000","last_fullsized":"2023-10-09T05:52:16.145731+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"357'7322","reported_seq":10330,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:23.862251+0000","last_change":"2023-10-09T05:51:07.166631+0000","last_active":"2023-10-09T05:52:23.862251+0000","last_peered":"2023-10-09T05:52:23.862251+0000","last_clean":"2023-10-09T05:52:23.862251+0000","last_became_active":"2023-10-09T05:51:07.160634+0000","last_became_peered":"2023-10-09T05:51:07.160634+0000","last_unstale":"2023-10-09T05:52:23.862251+0000","last_undegraded":"2023-10-09T05:52:23.862251+0000","last_fullsized":"2023-10-09T05:52:23.862251+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7322,"log_dups_size":0,"ondisk_log_size":7322,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16252928,"num_objects":248,"num_object_clones":0,"num_object_copies":744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2955,"num_write_kb":102496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":90636288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":191,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.146415+0000","last_change":"2023-10-09T05:48:54.494715+0000","last_active":"2023-10-09T05:52:16.146415+0000","last_peered":"2023-10-09T05:52:16.146415+0000","last_clean":"2023-10-09T05:52:16.146415+0000","last_became_active":"2023-10-09T05:48:54.493988+0000","last_became_peered":"2023-10-09T05:48:54.493988+0000","last_unstale":"2023-10-09T05:52:16.146415+0000","last_undegraded":"2023-10-09T05:52:16.146415+0000","last_fullsized":"2023-10-09T05:52:16.146415+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":355,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145941+0000","last_change":"2023-10-09T05:48:54.493193+0000","last_active":"2023-10-09T05:52:16.145941+0000","last_peered":"2023-10-09T05:52:16.145941+0000","last_clean":"2023-10-09T05:52:16.145941+0000","last_became_active":"2023-10-09T05:48:54.492886+0000","last_became_peered":"2023-10-09T05:48:54.492886+0000","last_unstale":"2023-10-09T05:52:16.145941+0000","last_undegraded":"2023-10-09T05:52:16.145941+0000","last_fullsized":"2023-10-09T05:52:16.145941+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"357'7257","reported_seq":7357,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.413755+0000","last_change":"2023-10-09T05:49:20.137476+0000","last_active":"2023-10-09T05:52:25.413755+0000","last_peered":"2023-10-09T05:52:25.413755+0000","last_clean":"2023-10-09T05:52:25.413755+0000","last_became_active":"2023-10-09T05:49:20.132759+0000","last_became_peered":"2023-10-09T05:49:20.132759+0000","last_unstale":"2023-10-09T05:52:25.413755+0000","last_undegraded":"2023-10-09T05:52:25.413755+0000","last_fullsized":"2023-10-09T05:52:25.413755+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7257,"log_dups_size":0,"ondisk_log_size":7257,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15613952,"num_objects":238,"num_object_clones":0,"num_object_copies":714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2935,"num_write_kb":101552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":356,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145377+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:52:16.145377+0000","last_peered":"2023-10-09T05:52:16.145377+0000","last_clean":"2023-10-09T05:52:16.145377+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:52:16.145377+0000","last_undegraded":"2023-10-09T05:52:16.145377+0000","last_fullsized":"2023-10-09T05:52:16.145377+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":349,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.146223+0000","last_change":"2023-10-09T05:48:54.492708+0000","last_active":"2023-10-09T05:52:16.146223+0000","last_peered":"2023-10-09T05:52:16.146223+0000","last_clean":"2023-10-09T05:52:16.146223+0000","last_became_active":"2023-10-09T05:48:54.492382+0000","last_became_peered":"2023-10-09T05:48:54.492382+0000","last_unstale":"2023-10-09T05:52:16.146223+0000","last_undegraded":"2023-10-09T05:52:16.146223+0000","last_fullsized":"2023-10-09T05:52:16.146223+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"357'7355","reported_seq":10130,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:25.556294+0000","last_change":"2023-10-09T05:50:19.097528+0000","last_active":"2023-10-09T05:52:25.556294+0000","last_peered":"2023-10-09T05:52:25.556294+0000","last_clean":"2023-10-09T05:48:53.427903+0000","last_became_active":"2023-10-09T05:50:19.074460+0000","last_became_peered":"2023-10-09T05:50:19.074460+0000","last_unstale":"2023-10-09T05:52:25.556294+0000","last_undegraded":"2023-10-09T05:50:19.039951+0000","last_fullsized":"2023-10-09T05:50:19.038108+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7355,"log_dups_size":0,"ondisk_log_size":7355,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13664256,"num_objects":208,"num_object_clones":0,"num_object_copies":624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":115,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2998,"num_write_kb":102624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1333,"num_bytes_recovered":87359488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"1,5","objects":154},{"shards":"2,5,6","objects":54}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":322,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157451+0000","last_change":"2023-10-09T05:50:18.048706+0000","last_active":"2023-10-09T05:52:16.157451+0000","last_peered":"2023-10-09T05:52:16.157451+0000","last_clean":"2023-10-09T05:52:16.157451+0000","last_became_active":"2023-10-09T05:50:18.048410+0000","last_became_peered":"2023-10-09T05:50:18.048410+0000","last_unstale":"2023-10-09T05:52:16.157451+0000","last_undegraded":"2023-10-09T05:52:16.157451+0000","last_fullsized":"2023-10-09T05:52:16.157451+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":101,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145921+0000","last_change":"2023-10-09T05:50:18.072866+0000","last_active":"2023-10-09T05:52:16.145921+0000","last_peered":"2023-10-09T05:52:16.145921+0000","last_clean":"2023-10-09T05:52:16.145921+0000","last_became_active":"2023-10-09T05:50:18.072550+0000","last_became_peered":"2023-10-09T05:50:18.072550+0000","last_unstale":"2023-10-09T05:52:16.145921+0000","last_undegraded":"2023-10-09T05:52:16.145921+0000","last_fullsized":"2023-10-09T05:52:16.145921+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.e","version":"357'9325","reported_seq":9417,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:24.072368+0000","last_change":"2023-10-09T05:49:20.163537+0000","last_active":"2023-10-09T05:52:24.072368+0000","last_peered":"2023-10-09T05:52:24.072368+0000","last_clean":"2023-10-09T05:52:24.072368+0000","last_became_active":"2023-10-09T05:49:20.159521+0000","last_became_peered":"2023-10-09T05:49:20.159521+0000","last_unstale":"2023-10-09T05:52:24.072368+0000","last_undegraded":"2023-10-09T05:52:24.072368+0000","last_fullsized":"2023-10-09T05:52:24.072368+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9325,"log_dups_size":0,"ondisk_log_size":9325,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31784960,"num_objects":485,"num_object_clones":0,"num_object_copies":1455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5978,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":747,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145629+0000","last_change":"2023-10-09T05:50:18.055235+0000","last_active":"2023-10-09T05:52:16.145629+0000","last_peered":"2023-10-09T05:52:16.145629+0000","last_clean":"2023-10-09T05:52:16.145629+0000","last_became_active":"2023-10-09T05:50:18.054656+0000","last_became_peered":"2023-10-09T05:50:18.054656+0000","last_unstale":"2023-10-09T05:52:16.145629+0000","last_undegraded":"2023-10-09T05:52:16.145629+0000","last_fullsized":"2023-10-09T05:52:16.145629+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:33.760262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":339,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.160603+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:52:16.160603+0000","last_peered":"2023-10-09T05:52:16.160603+0000","last_clean":"2023-10-09T05:52:16.160603+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:52:16.160603+0000","last_undegraded":"2023-10-09T05:52:16.160603+0000","last_fullsized":"2023-10-09T05:52:16.160603+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":17,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.159554+0000","last_change":"2023-10-09T05:51:43.248409+0000","last_active":"2023-10-09T05:52:16.159554+0000","last_peered":"2023-10-09T05:52:16.159554+0000","last_clean":"2023-10-09T05:52:16.159554+0000","last_became_active":"2023-10-09T05:51:43.154898+0000","last_became_peered":"2023-10-09T05:51:43.154898+0000","last_unstale":"2023-10-09T05:52:16.159554+0000","last_undegraded":"2023-10-09T05:52:16.159554+0000","last_fullsized":"2023-10-09T05:52:16.159554+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"357'7408","reported_seq":7506,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.434533+0000","last_change":"2023-10-09T05:50:08.053391+0000","last_active":"2023-10-09T05:52:25.434533+0000","last_peered":"2023-10-09T05:52:25.434533+0000","last_clean":"2023-10-09T05:52:25.434533+0000","last_became_active":"2023-10-09T05:50:08.050378+0000","last_became_peered":"2023-10-09T05:50:08.050378+0000","last_unstale":"2023-10-09T05:52:25.434533+0000","last_undegraded":"2023-10-09T05:52:25.434533+0000","last_fullsized":"2023-10-09T05:52:25.434533+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7408,"log_dups_size":0,"ondisk_log_size":7408,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14909440,"num_objects":227,"num_object_clones":0,"num_object_copies":681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2929,"num_write_kb":101024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":847,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.160060+0000","last_change":"2023-10-09T05:48:54.592642+0000","last_active":"2023-10-09T05:52:16.160060+0000","last_peered":"2023-10-09T05:52:16.160060+0000","last_clean":"2023-10-09T05:52:16.160060+0000","last_became_active":"2023-10-09T05:48:54.491917+0000","last_became_peered":"2023-10-09T05:48:54.491917+0000","last_unstale":"2023-10-09T05:52:16.160060+0000","last_undegraded":"2023-10-09T05:52:16.160060+0000","last_fullsized":"2023-10-09T05:52:16.160060+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":323,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.146352+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:52:16.146352+0000","last_peered":"2023-10-09T05:52:16.146352+0000","last_clean":"2023-10-09T05:52:16.146352+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:52:16.146352+0000","last_undegraded":"2023-10-09T05:52:16.146352+0000","last_fullsized":"2023-10-09T05:52:16.146352+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":179,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.170617+0000","last_change":"2023-10-09T05:50:18.043775+0000","last_active":"2023-10-09T05:52:17.170617+0000","last_peered":"2023-10-09T05:52:17.170617+0000","last_clean":"2023-10-09T05:52:17.170617+0000","last_became_active":"2023-10-09T05:50:18.043047+0000","last_became_peered":"2023-10-09T05:50:18.043047+0000","last_unstale":"2023-10-09T05:52:17.170617+0000","last_undegraded":"2023-10-09T05:52:17.170617+0000","last_fullsized":"2023-10-09T05:52:17.170617+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:28.456979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"357'8267","reported_seq":11375,"reported_epoch":357,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:25.516087+0000","last_change":"2023-10-09T05:50:08.085412+0000","last_active":"2023-10-09T05:52:25.516087+0000","last_peered":"2023-10-09T05:52:25.516087+0000","last_clean":"2023-10-09T05:48:53.462109+0000","last_became_active":"2023-10-09T05:50:08.080753+0000","last_became_peered":"2023-10-09T05:50:08.080753+0000","last_unstale":"2023-10-09T05:52:25.516087+0000","last_undegraded":"2023-10-09T05:50:08.030939+0000","last_fullsized":"2023-10-09T05:52:25.516087+0000","mapping_epoch":312,"log_start":"289'200","ondisk_log_start":"289'200","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8067,"log_dups_size":0,"ondisk_log_size":8067,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15482880,"num_objects":236,"num_object_clones":0,"num_object_copies":708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":156,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3020,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1328,"num_bytes_recovered":87064576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"1,3,4","objects":154},{"shards":"2,3,4","objects":82}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":677,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.139695+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:52:16.139695+0000","last_peered":"2023-10-09T05:52:16.139695+0000","last_clean":"2023-10-09T05:52:16.139695+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:52:16.139695+0000","last_undegraded":"2023-10-09T05:52:16.139695+0000","last_fullsized":"2023-10-09T05:52:16.139695+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":138,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.170343+0000","last_change":"2023-10-09T05:50:18.063466+0000","last_active":"2023-10-09T05:52:17.170343+0000","last_peered":"2023-10-09T05:52:17.170343+0000","last_clean":"2023-10-09T05:52:17.170343+0000","last_became_active":"2023-10-09T05:50:18.063120+0000","last_became_peered":"2023-10-09T05:50:18.063120+0000","last_unstale":"2023-10-09T05:52:17.170343+0000","last_undegraded":"2023-10-09T05:52:17.170343+0000","last_fullsized":"2023-10-09T05:52:17.170343+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:13.683314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":161,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.160405+0000","last_change":"2023-10-09T05:50:18.069402+0000","last_active":"2023-10-09T05:52:16.160405+0000","last_peered":"2023-10-09T05:52:16.160405+0000","last_clean":"2023-10-09T05:52:16.160405+0000","last_became_active":"2023-10-09T05:50:18.069196+0000","last_became_peered":"2023-10-09T05:50:18.069196+0000","last_unstale":"2023-10-09T05:52:16.160405+0000","last_undegraded":"2023-10-09T05:52:16.160405+0000","last_fullsized":"2023-10-09T05:52:16.160405+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"357'7616","reported_seq":7726,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:23.744290+0000","last_change":"2023-10-09T05:51:44.323830+0000","last_active":"2023-10-09T05:52:23.744290+0000","last_peered":"2023-10-09T05:52:23.744290+0000","last_clean":"2023-10-09T05:51:42.071320+0000","last_became_active":"2023-10-09T05:51:44.311440+0000","last_became_peered":"2023-10-09T05:51:44.311440+0000","last_unstale":"2023-10-09T05:52:23.744290+0000","last_undegraded":"2023-10-09T05:51:44.114363+0000","last_fullsized":"2023-10-09T05:51:44.105936+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7616,"log_dups_size":0,"ondisk_log_size":7616,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17334272,"num_objects":264,"num_object_clones":0,"num_object_copies":792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1649,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3001,"num_write_kb":104512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[0,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":264},{"shards":"0,6","objects":-453}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":1003,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.166523+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:52:16.166523+0000","last_peered":"2023-10-09T05:52:16.166523+0000","last_clean":"2023-10-09T05:52:16.166523+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:52:16.166523+0000","last_undegraded":"2023-10-09T05:52:16.166523+0000","last_fullsized":"2023-10-09T05:52:16.166523+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":507,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157359+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:52:16.157359+0000","last_peered":"2023-10-09T05:52:16.157359+0000","last_clean":"2023-10-09T05:52:16.157359+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:52:16.157359+0000","last_undegraded":"2023-10-09T05:52:16.157359+0000","last_fullsized":"2023-10-09T05:52:16.157359+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":129,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.159787+0000","last_change":"2023-10-09T05:48:25.068759+0000","last_active":"2023-10-09T05:52:16.159787+0000","last_peered":"2023-10-09T05:52:16.159787+0000","last_clean":"2023-10-09T05:52:16.159787+0000","last_became_active":"2023-10-09T05:48:25.068358+0000","last_became_peered":"2023-10-09T05:48:25.068358+0000","last_unstale":"2023-10-09T05:52:16.159787+0000","last_undegraded":"2023-10-09T05:52:16.159787+0000","last_fullsized":"2023-10-09T05:52:16.159787+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":324,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.146239+0000","last_change":"2023-10-09T05:44:50.495966+0000","last_active":"2023-10-09T05:52:16.146239+0000","last_peered":"2023-10-09T05:52:16.146239+0000","last_clean":"2023-10-09T05:52:16.146239+0000","last_became_active":"2023-10-09T05:44:50.495628+0000","last_became_peered":"2023-10-09T05:44:50.495628+0000","last_unstale":"2023-10-09T05:52:16.146239+0000","last_undegraded":"2023-10-09T05:52:16.146239+0000","last_fullsized":"2023-10-09T05:52:16.146239+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"357'8118","reported_seq":11669,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:27.245114+0000","last_change":"2023-10-09T05:51:43.298014+0000","last_active":"2023-10-09T05:52:27.245114+0000","last_peered":"2023-10-09T05:52:27.245114+0000","last_clean":"2023-10-09T05:48:53.473854+0000","last_became_active":"2023-10-09T05:51:43.270086+0000","last_became_peered":"2023-10-09T05:51:43.270086+0000","last_unstale":"2023-10-09T05:52:27.245114+0000","last_undegraded":"2023-10-09T05:51:43.150850+0000","last_fullsized":"2023-10-09T05:51:43.116350+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8118,"log_dups_size":0,"ondisk_log_size":8118,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14319616,"num_objects":218,"num_object_clones":0,"num_object_copies":654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1234,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2983,"num_write_kb":102448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1325,"num_bytes_recovered":86802432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2,3","objects":170},{"shards":"2,3,4","objects":48}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":879,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157432+0000","last_change":"2023-10-09T05:49:36.046279+0000","last_active":"2023-10-09T05:52:16.157432+0000","last_peered":"2023-10-09T05:52:16.157432+0000","last_clean":"2023-10-09T05:52:16.157432+0000","last_became_active":"2023-10-09T05:49:36.045828+0000","last_became_peered":"2023-10-09T05:49:36.045828+0000","last_unstale":"2023-10-09T05:52:16.157432+0000","last_undegraded":"2023-10-09T05:52:16.157432+0000","last_fullsized":"2023-10-09T05:52:16.157432+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":350,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.151582+0000","last_change":"2023-10-09T05:51:31.682830+0000","last_active":"2023-10-09T05:52:16.151582+0000","last_peered":"2023-10-09T05:52:16.151582+0000","last_clean":"2023-10-09T05:52:16.151582+0000","last_became_active":"2023-10-09T05:51:31.682566+0000","last_became_peered":"2023-10-09T05:51:31.682566+0000","last_unstale":"2023-10-09T05:52:16.151582+0000","last_undegraded":"2023-10-09T05:52:16.151582+0000","last_fullsized":"2023-10-09T05:52:16.151582+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:42.916316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":347,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.147026+0000","last_change":"2023-10-09T05:48:54.489115+0000","last_active":"2023-10-09T05:52:16.147026+0000","last_peered":"2023-10-09T05:52:16.147026+0000","last_clean":"2023-10-09T05:52:16.147026+0000","last_became_active":"2023-10-09T05:48:54.488783+0000","last_became_peered":"2023-10-09T05:48:54.488783+0000","last_unstale":"2023-10-09T05:52:16.147026+0000","last_undegraded":"2023-10-09T05:52:16.147026+0000","last_fullsized":"2023-10-09T05:52:16.147026+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:13.193778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"357'9154","reported_seq":15923,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:27.279689+0000","last_change":"2023-10-09T05:51:13.190036+0000","last_active":"2023-10-09T05:52:27.279689+0000","last_peered":"2023-10-09T05:52:27.279689+0000","last_clean":"2023-10-09T05:48:53.467734+0000","last_became_active":"2023-10-09T05:51:13.176990+0000","last_became_peered":"2023-10-09T05:51:13.176990+0000","last_unstale":"2023-10-09T05:52:27.279689+0000","last_undegraded":"2023-10-09T05:51:13.120625+0000","last_fullsized":"2023-10-09T05:51:13.052350+0000","mapping_epoch":339,"log_start":"290'1428","ondisk_log_start":"290'1428","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7726,"log_dups_size":0,"ondisk_log_size":7726,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29523968,"num_objects":450,"num_object_clones":0,"num_object_copies":1350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1905,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5870,"num_write_kb":202272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2582,"num_bytes_recovered":169148416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[3,1],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3","objects":-2},{"shards":"1,3,4","objects":350},{"shards":"3,4,6","objects":101}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":540,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.158407+0000","last_change":"2023-10-09T05:44:51.489799+0000","last_active":"2023-10-09T05:52:16.158407+0000","last_peered":"2023-10-09T05:52:16.158407+0000","last_clean":"2023-10-09T05:52:16.158407+0000","last_became_active":"2023-10-09T05:44:51.489301+0000","last_became_peered":"2023-10-09T05:44:51.489301+0000","last_unstale":"2023-10-09T05:52:16.158407+0000","last_undegraded":"2023-10-09T05:52:16.158407+0000","last_fullsized":"2023-10-09T05:52:16.158407+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":156,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.170753+0000","last_change":"2023-10-09T05:51:43.127835+0000","last_active":"2023-10-09T05:52:17.170753+0000","last_peered":"2023-10-09T05:52:17.170753+0000","last_clean":"2023-10-09T05:52:17.170753+0000","last_became_active":"2023-10-09T05:51:43.127489+0000","last_became_peered":"2023-10-09T05:51:43.127489+0000","last_unstale":"2023-10-09T05:52:17.170753+0000","last_undegraded":"2023-10-09T05:52:17.170753+0000","last_fullsized":"2023-10-09T05:52:17.170753+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:20.191774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":332,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.143219+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:52:16.143219+0000","last_peered":"2023-10-09T05:52:16.143219+0000","last_clean":"2023-10-09T05:52:16.143219+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:52:16.143219+0000","last_undegraded":"2023-10-09T05:52:16.143219+0000","last_fullsized":"2023-10-09T05:52:16.143219+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"357'8350","reported_seq":8452,"reported_epoch":357,"state":"active+recovery_wait","last_fresh":"2023-10-09T05:52:24.100903+0000","last_change":"2023-10-09T05:50:08.162316+0000","last_active":"2023-10-09T05:52:24.100903+0000","last_peered":"2023-10-09T05:52:24.100903+0000","last_clean":"2023-10-09T05:50:06.828344+0000","last_became_active":"2023-10-09T05:50:08.040134+0000","last_became_peered":"2023-10-09T05:50:08.040134+0000","last_unstale":"2023-10-09T05:52:24.100903+0000","last_undegraded":"2023-10-09T05:52:24.100903+0000","last_fullsized":"2023-10-09T05:52:24.100903+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8350,"log_dups_size":0,"ondisk_log_size":8350,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16416768,"num_objects":250,"num_object_clones":0,"num_object_copies":750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3040,"num_write_kb":105312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":250}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":304,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157929+0000","last_change":"2023-10-09T05:48:25.066163+0000","last_active":"2023-10-09T05:52:16.157929+0000","last_peered":"2023-10-09T05:52:16.157929+0000","last_clean":"2023-10-09T05:52:16.157929+0000","last_became_active":"2023-10-09T05:48:25.065145+0000","last_became_peered":"2023-10-09T05:48:25.065145+0000","last_unstale":"2023-10-09T05:52:16.157929+0000","last_undegraded":"2023-10-09T05:52:16.157929+0000","last_fullsized":"2023-10-09T05:52:16.157929+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":587,"reported_epoch":356,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:16.146137+0000","last_change":"2023-10-09T05:50:19.055196+0000","last_active":"2023-10-09T05:52:16.146137+0000","last_peered":"2023-10-09T05:52:16.146137+0000","last_clean":"2023-10-09T05:48:52.524136+0000","last_became_active":"2023-10-09T05:50:19.054850+0000","last_became_peered":"2023-10-09T05:50:19.054850+0000","last_unstale":"2023-10-09T05:52:16.146137+0000","last_undegraded":"2023-10-09T05:50:19.031371+0000","last_fullsized":"2023-10-09T05:52:16.146137+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":["4","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":297,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157531+0000","last_change":"2023-10-09T05:48:25.067326+0000","last_active":"2023-10-09T05:52:16.157531+0000","last_peered":"2023-10-09T05:52:16.157531+0000","last_clean":"2023-10-09T05:52:16.157531+0000","last_became_active":"2023-10-09T05:48:25.066977+0000","last_became_peered":"2023-10-09T05:48:25.066977+0000","last_unstale":"2023-10-09T05:52:16.157531+0000","last_undegraded":"2023-10-09T05:52:16.157531+0000","last_fullsized":"2023-10-09T05:52:16.157531+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"357'8038","reported_seq":8142,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.363628+0000","last_change":"2023-10-09T05:50:17.989037+0000","last_active":"2023-10-09T05:52:25.363628+0000","last_peered":"2023-10-09T05:52:25.363628+0000","last_clean":"2023-10-09T05:52:25.363628+0000","last_became_active":"2023-10-09T05:50:08.041685+0000","last_became_peered":"2023-10-09T05:50:08.041685+0000","last_unstale":"2023-10-09T05:52:25.363628+0000","last_undegraded":"2023-10-09T05:52:25.363628+0000","last_fullsized":"2023-10-09T05:52:25.363628+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8038,"log_dups_size":0,"ondisk_log_size":8038,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18006022,"num_objects":275,"num_object_clones":0,"num_object_copies":825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2906,"num_write_kb":101776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":854,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.157846+0000","last_change":"2023-10-09T05:44:50.482211+0000","last_active":"2023-10-09T05:52:16.157846+0000","last_peered":"2023-10-09T05:52:16.157846+0000","last_clean":"2023-10-09T05:52:16.157846+0000","last_became_active":"2023-10-09T05:44:50.481749+0000","last_became_peered":"2023-10-09T05:44:50.481749+0000","last_unstale":"2023-10-09T05:52:16.157846+0000","last_undegraded":"2023-10-09T05:52:16.157846+0000","last_fullsized":"2023-10-09T05:52:16.157846+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":343,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.147221+0000","last_change":"2023-10-09T05:48:54.585691+0000","last_active":"2023-10-09T05:52:16.147221+0000","last_peered":"2023-10-09T05:52:16.147221+0000","last_clean":"2023-10-09T05:52:16.147221+0000","last_became_active":"2023-10-09T05:48:54.498259+0000","last_became_peered":"2023-10-09T05:48:54.498259+0000","last_unstale":"2023-10-09T05:52:16.147221+0000","last_undegraded":"2023-10-09T05:52:16.147221+0000","last_fullsized":"2023-10-09T05:52:16.147221+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.684612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":307,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145323+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:52:16.145323+0000","last_peered":"2023-10-09T05:52:16.145323+0000","last_clean":"2023-10-09T05:52:16.145323+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:52:16.145323+0000","last_undegraded":"2023-10-09T05:52:16.145323+0000","last_fullsized":"2023-10-09T05:52:16.145323+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"357'8764","reported_seq":17569,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.254575+0000","last_change":"2023-10-09T05:52:13.371424+0000","last_active":"2023-10-09T05:52:25.254575+0000","last_peered":"2023-10-09T05:52:25.254575+0000","last_clean":"2023-10-09T05:52:25.254575+0000","last_became_active":"2023-10-09T05:49:35.024339+0000","last_became_peered":"2023-10-09T05:49:35.024339+0000","last_unstale":"2023-10-09T05:52:25.254575+0000","last_undegraded":"2023-10-09T05:52:25.254575+0000","last_fullsized":"2023-10-09T05:52:25.254575+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8764,"log_dups_size":0,"ondisk_log_size":8764,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31391744,"num_objects":479,"num_object_clones":0,"num_object_copies":1437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6027,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2892,"num_bytes_recovered":189530112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":333,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.143133+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:52:16.143133+0000","last_peered":"2023-10-09T05:52:16.143133+0000","last_clean":"2023-10-09T05:52:16.143133+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:52:16.143133+0000","last_undegraded":"2023-10-09T05:52:16.143133+0000","last_fullsized":"2023-10-09T05:52:16.143133+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":28,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.140150+0000","last_change":"2023-10-09T05:51:13.060067+0000","last_active":"2023-10-09T05:52:16.140150+0000","last_peered":"2023-10-09T05:52:16.140150+0000","last_clean":"2023-10-09T05:52:16.140150+0000","last_became_active":"2023-10-09T05:51:13.059725+0000","last_became_peered":"2023-10-09T05:51:13.059725+0000","last_unstale":"2023-10-09T05:52:16.140150+0000","last_undegraded":"2023-10-09T05:52:16.140150+0000","last_fullsized":"2023-10-09T05:52:16.140150+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:43.328688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"357'8933","reported_seq":9035,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:27.286036+0000","last_change":"2023-10-09T05:48:56.019001+0000","last_active":"2023-10-09T05:52:27.286036+0000","last_peered":"2023-10-09T05:52:27.286036+0000","last_clean":"2023-10-09T05:48:53.462142+0000","last_became_active":"2023-10-09T05:48:56.004223+0000","last_became_peered":"2023-10-09T05:48:56.004223+0000","last_unstale":"2023-10-09T05:52:27.286036+0000","last_undegraded":"2023-10-09T05:48:55.514193+0000","last_fullsized":"2023-10-09T05:48:55.479940+0000","mapping_epoch":306,"log_start":"290'1938","ondisk_log_start":"290'1938","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27721728,"num_objects":423,"num_object_clones":0,"num_object_copies":1269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2986,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6149,"num_write_kb":210304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":123},{"shards":"3,6","objects":-2563}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":352,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.160384+0000","last_change":"2023-10-09T05:48:54.583404+0000","last_active":"2023-10-09T05:52:16.160384+0000","last_peered":"2023-10-09T05:52:16.160384+0000","last_clean":"2023-10-09T05:52:16.160384+0000","last_became_active":"2023-10-09T05:48:54.492565+0000","last_became_peered":"2023-10-09T05:48:54.492565+0000","last_unstale":"2023-10-09T05:52:16.160384+0000","last_undegraded":"2023-10-09T05:52:16.160384+0000","last_fullsized":"2023-10-09T05:52:16.160384+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":355,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.146398+0000","last_change":"2023-10-09T05:48:54.493079+0000","last_active":"2023-10-09T05:52:16.146398+0000","last_peered":"2023-10-09T05:52:16.146398+0000","last_clean":"2023-10-09T05:52:16.146398+0000","last_became_active":"2023-10-09T05:48:54.492013+0000","last_became_peered":"2023-10-09T05:48:54.492013+0000","last_unstale":"2023-10-09T05:52:16.146398+0000","last_undegraded":"2023-10-09T05:52:16.146398+0000","last_fullsized":"2023-10-09T05:52:16.146398+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"357'8568","reported_seq":8660,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:24.129940+0000","last_change":"2023-10-09T05:48:46.260747+0000","last_active":"2023-10-09T05:52:24.129940+0000","last_peered":"2023-10-09T05:52:24.129940+0000","last_clean":"2023-10-09T05:52:24.129940+0000","last_became_active":"2023-10-09T05:48:46.258900+0000","last_became_peered":"2023-10-09T05:48:46.258900+0000","last_unstale":"2023-10-09T05:52:24.129940+0000","last_undegraded":"2023-10-09T05:52:24.129940+0000","last_fullsized":"2023-10-09T05:52:24.129940+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8568,"log_dups_size":0,"ondisk_log_size":8568,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34766848,"num_objects":531,"num_object_clones":0,"num_object_copies":1593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5881,"num_write_kb":205152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":311,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.160608+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:52:16.160608+0000","last_peered":"2023-10-09T05:52:16.160608+0000","last_clean":"2023-10-09T05:52:16.160608+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:52:16.160608+0000","last_undegraded":"2023-10-09T05:52:16.160608+0000","last_fullsized":"2023-10-09T05:52:16.160608+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":20,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145301+0000","last_change":"2023-10-09T05:51:31.668739+0000","last_active":"2023-10-09T05:52:16.145301+0000","last_peered":"2023-10-09T05:52:16.145301+0000","last_clean":"2023-10-09T05:52:16.145301+0000","last_became_active":"2023-10-09T05:51:31.668481+0000","last_became_peered":"2023-10-09T05:51:31.668481+0000","last_unstale":"2023-10-09T05:52:16.145301+0000","last_undegraded":"2023-10-09T05:52:16.145301+0000","last_fullsized":"2023-10-09T05:52:16.145301+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"357'8563","reported_seq":8654,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.356919+0000","last_change":"2023-10-09T05:48:46.308705+0000","last_active":"2023-10-09T05:52:25.356919+0000","last_peered":"2023-10-09T05:52:25.356919+0000","last_clean":"2023-10-09T05:52:25.356919+0000","last_became_active":"2023-10-09T05:48:46.258676+0000","last_became_peered":"2023-10-09T05:48:46.258676+0000","last_unstale":"2023-10-09T05:52:25.356919+0000","last_undegraded":"2023-10-09T05:52:25.356919+0000","last_fullsized":"2023-10-09T05:52:25.356919+0000","mapping_epoch":299,"log_start":"290'1561","ondisk_log_start":"290'1561","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7002,"log_dups_size":0,"ondisk_log_size":7002,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29851648,"num_objects":456,"num_object_clones":0,"num_object_copies":1368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5874,"num_write_kb":202528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":76,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.172351+0000","last_change":"2023-10-09T05:48:25.069406+0000","last_active":"2023-10-09T05:52:17.172351+0000","last_peered":"2023-10-09T05:52:17.172351+0000","last_clean":"2023-10-09T05:52:17.172351+0000","last_became_active":"2023-10-09T05:48:25.069064+0000","last_became_peered":"2023-10-09T05:48:25.069064+0000","last_unstale":"2023-10-09T05:52:17.172351+0000","last_undegraded":"2023-10-09T05:52:17.172351+0000","last_fullsized":"2023-10-09T05:52:17.172351+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.567858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":158,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.139606+0000","last_change":"2023-10-09T05:48:25.063854+0000","last_active":"2023-10-09T05:52:16.139606+0000","last_peered":"2023-10-09T05:52:16.139606+0000","last_clean":"2023-10-09T05:52:16.139606+0000","last_became_active":"2023-10-09T05:48:25.063579+0000","last_became_peered":"2023-10-09T05:48:25.063579+0000","last_unstale":"2023-10-09T05:52:16.139606+0000","last_undegraded":"2023-10-09T05:52:16.139606+0000","last_fullsized":"2023-10-09T05:52:16.139606+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:02.389789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.14","version":"357'8762","reported_seq":8853,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:25.541158+0000","last_change":"2023-10-09T05:48:46.325697+0000","last_active":"2023-10-09T05:52:25.541158+0000","last_peered":"2023-10-09T05:52:25.541158+0000","last_clean":"2023-10-09T05:52:25.541158+0000","last_became_active":"2023-10-09T05:48:46.268207+0000","last_became_peered":"2023-10-09T05:48:46.268207+0000","last_unstale":"2023-10-09T05:52:25.541158+0000","last_undegraded":"2023-10-09T05:52:25.541158+0000","last_fullsized":"2023-10-09T05:52:25.541158+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8762,"log_dups_size":0,"ondisk_log_size":8762,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29261824,"num_objects":447,"num_object_clones":0,"num_object_copies":1341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6035,"num_write_kb":207392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":347,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.144297+0000","last_change":"2023-10-09T05:51:13.071891+0000","last_active":"2023-10-09T05:52:16.144297+0000","last_peered":"2023-10-09T05:52:16.144297+0000","last_clean":"2023-10-09T05:52:16.144297+0000","last_became_active":"2023-10-09T05:51:13.058378+0000","last_became_peered":"2023-10-09T05:51:13.058378+0000","last_unstale":"2023-10-09T05:52:16.144297+0000","last_undegraded":"2023-10-09T05:52:16.144297+0000","last_fullsized":"2023-10-09T05:52:16.144297+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:41.278366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":340,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T05:52:16.145199+0000","last_change":"2023-10-09T05:48:54.492580+0000","last_active":"2023-10-09T05:52:16.145199+0000","last_peered":"2023-10-09T05:52:16.145199+0000","last_clean":"2023-10-09T05:52:16.145199+0000","last_became_active":"2023-10-09T05:48:54.492259+0000","last_became_peered":"2023-10-09T05:48:54.492259+0000","last_unstale":"2023-10-09T05:52:16.145199+0000","last_undegraded":"2023-10-09T05:52:16.145199+0000","last_fullsized":"2023-10-09T05:52:16.145199+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":10932,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":46,"stat_sum":{"num_bytes":1010040856,"num_objects":15413,"num_object_clones":0,"num_object_copies":46239,"num_objects_missing_on_primary":550,"num_objects_missing":550,"num_objects_degraded":23434,"num_objects_misplaced":158,"num_objects_unfound":0,"num_objects_dirty":15413,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":190467,"num_write_kb":6588097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33405,"num_bytes_recovered":2184118272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6020870144,"data_stored":6020857928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":365176,"ondisk_log_size":365176,"up":138,"acting":124,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":132,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846806,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1284500,"kb_used_data":366476,"kb_used_omap":14,"kb_used_meta":918001,"kb_avail":92460652,"statfs":{"total":95995035648,"available":94679707648,"internally_reserved":0,"allocated":375271424,"data_stored":374592045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":940033882},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":12.824999999999999,"5min":4.1820000000000004,"15min":4.3869999999999996},"min":{"1min":0.22800000000000001,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":105.39100000000001,"5min":105.39100000000001,"15min":220.77500000000001},"last":0.26200000000000001},{"interface":"front","average":{"1min":12.695,"5min":4.1100000000000003,"15min":4.359},"min":{"1min":0.29799999999999999,"5min":0.25900000000000001,"15min":0.22700000000000001},"max":{"1min":105.417,"5min":105.417,"15min":220.78700000000001},"last":0.375}]},{"osd":1,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":7.9139999999999997,"5min":2.363,"15min":3.1760000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":114.422,"5min":114.422,"15min":190.68100000000001},"last":2.786},{"interface":"front","average":{"1min":7.8090000000000002,"5min":2.5760000000000001,"15min":3.234},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":111.708,"5min":111.708,"15min":191.09200000000001},"last":0.28899999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":7.2279999999999998,"5min":2.6629999999999998,"15min":2.0590000000000002},"min":{"1min":0.374,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":100.684,"5min":100.684,"15min":100.684},"last":2.8149999999999999},{"interface":"front","average":{"1min":7.4290000000000003,"5min":2.4990000000000001,"15min":2.0129999999999999},"min":{"1min":0.32000000000000001,"5min":0.30499999999999999,"15min":0.28100000000000003},"max":{"1min":100.736,"5min":100.736,"15min":100.736},"last":0.33500000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":8.4090000000000007,"5min":4.3399999999999999,"15min":4.569},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":103.29300000000001,"5min":103.29300000000001,"15min":185.17099999999999},"last":2.718},{"interface":"front","average":{"1min":11.114000000000001,"5min":4.8150000000000004,"15min":4.6909999999999998},"min":{"1min":0.39700000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":103.202,"5min":103.202,"15min":183.35499999999999},"last":0.44}]},{"osd":4,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":9.8770000000000007,"5min":3.4350000000000001,"15min":1.5529999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.25700000000000001},"max":{"1min":159.50299999999999,"5min":159.50299999999999,"15min":159.50299999999999},"last":0.312},{"interface":"front","average":{"1min":10.189,"5min":3.5230000000000001,"15min":1.611},"min":{"1min":0.32600000000000001,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":159.483,"5min":159.483,"15min":159.483},"last":0.41099999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.476,"15min":1.0329999999999999},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.26900000000000002},"max":{"1min":4.1319999999999997,"5min":49.210000000000001,"15min":49.210000000000001},"last":0.79200000000000004},{"interface":"front","average":{"1min":2.246,"5min":1.7749999999999999,"15min":1.101},"min":{"1min":0.34999999999999998,"5min":0.308,"15min":0.28299999999999997},"max":{"1min":9.6739999999999995,"5min":49.274999999999999,"15min":49.274999999999999},"last":2.8700000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":0.90200000000000002,"15min":0.75600000000000001},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.1720000000000006,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.753},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.095,"15min":0.85499999999999998},"min":{"1min":0.313,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.8200000000000003,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":2.843}]}]},{"osd":6,"up_from":42,"seq":180388626832,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1749124,"kb_used_data":857276,"kb_used_omap":0,"kb_used_meta":891840,"kb_avail":91996028,"statfs":{"total":95995035648,"available":94203932672,"internally_reserved":0,"allocated":877850624,"data_stored":877187213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913244160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":2.2250000000000001,"15min":4.1349999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.20999999999999999},"max":{"1min":2.5379999999999998,"5min":42.869,"15min":162.08799999999999},"last":2.3119999999999998},{"interface":"front","average":{"1min":1.071,"5min":2.2639999999999998,"15min":4.0410000000000004},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.2169999999999996,"5min":42.892000000000003,"15min":161.90899999999999},"last":2.7949999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:52:10 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.9359999999999999,"15min":3.867},"min":{"1min":0.27300000000000002,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":9.3469999999999995,"5min":48.729999999999997,"15min":228.24299999999999},"last":1.149},{"interface":"front","average":{"1min":2.274,"5min":2.1699999999999999,"15min":3.8340000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":13.843,"5min":48.741999999999997,"15min":228.22499999999999},"last":0.97599999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":1.014,"15min":3.3849999999999998},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":2.4369999999999998,"5min":6.1680000000000001,"15min":115.40600000000001},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.0960000000000001,"15min":3.5369999999999999},"min":{"1min":0.33600000000000002,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":3.794,"5min":6.1440000000000001,"15min":115.297},"last":2.0840000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:52:10 2023","interfaces":[{"interface":"back","average":{"1min":10.012,"5min":3.4159999999999999,"15min":4.4009999999999998},"min":{"1min":0.35999999999999999,"5min":0.32800000000000001,"15min":0.26800000000000002},"max":{"1min":145.124,"5min":145.124,"15min":145.124},"last":2.2799999999999998},{"interface":"front","average":{"1min":10.629,"5min":3.681,"15min":4.4610000000000003},"min":{"1min":0.37,"5min":0.33800000000000002,"15min":0.30299999999999999},"max":{"1min":145.26599999999999,"5min":145.26599999999999,"15min":145.26599999999999},"last":0.89700000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:51:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":3.7250000000000001,"15min":2.0270000000000001},"min":{"1min":0.253,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":2.2509999999999999,"5min":257.375,"15min":257.375},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.73799999999999999,"5min":3.8599999999999999,"15min":2.1419999999999999},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":2.8849999999999998,"5min":257.61200000000002,"15min":257.61200000000002},"last":1.002}]},{"osd":5,"last update":"Mon Oct 9 05:51:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":3.7250000000000001,"15min":2.3610000000000002},"min":{"1min":0.28699999999999998,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":2.198,"5min":256.05700000000002,"15min":256.05700000000002},"last":0.754},{"interface":"front","average":{"1min":0.79600000000000004,"5min":3.895,"15min":2.3420000000000001},"min":{"1min":0.34899999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"max":{"1min":3.823,"5min":256.08499999999998,"15min":256.08499999999998},"last":1.131}]},{"osd":7,"last update":"Mon Oct 9 05:52:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4850000000000001,"5min":0.88100000000000001,"15min":1.091},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.308},"max":{"1min":7.3120000000000003,"5min":7.3120000000000003,"15min":115.514},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.9430000000000001,"5min":1.202,"15min":1.333},"min":{"1min":0.38300000000000001,"5min":0.308,"15min":0.30599999999999999},"max":{"1min":6.601,"5min":6.601,"15min":116.53100000000001},"last":1.042}]}]},{"osd":1,"up_from":93,"seq":399431958793,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":944824,"kb_used_data":287472,"kb_used_omap":7,"kb_used_meta":657272,"kb_avail":92800328,"statfs":{"total":95995035648,"available":95027535872,"internally_reserved":0,"allocated":294371328,"data_stored":293189970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":673047054},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":4.3159999999999998,"5min":3.2010000000000001,"15min":4.6799999999999997},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":34.058,"5min":93.176000000000002,"15min":207.858},"last":0.217},{"interface":"front","average":{"1min":4.1699999999999999,"5min":3.2130000000000001,"15min":5.1470000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.182},"max":{"1min":34.634,"5min":92.593999999999994,"15min":206.97200000000001},"last":0.40899999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:51:29 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.8799999999999999,"15min":4.218},"min":{"1min":0.252,"5min":0.22600000000000001,"15min":0.19800000000000001},"max":{"1min":5.0789999999999997,"5min":11.914,"15min":101.367},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.6699999999999999,"15min":4.0060000000000002},"min":{"1min":0.24199999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":12.122999999999999,"5min":16.702999999999999,"15min":99.861000000000004},"last":0.36899999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1230000000000002,"5min":3.2530000000000001,"15min":5.2149999999999999},"min":{"1min":0.25900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.053000000000001,"5min":92.353999999999999,"15min":162.33699999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":3.863,"5min":3.1389999999999998,"15min":5.4470000000000001},"min":{"1min":0.36399999999999999,"5min":0.28599999999999998,"15min":0.26000000000000001},"max":{"1min":16.617999999999999,"5min":93.117999999999995,"15min":162.376},"last":0.64300000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":1.484,"15min":2.3839999999999999},"min":{"1min":0.40699999999999997,"5min":0.312,"15min":0.29799999999999999},"max":{"1min":5.2629999999999999,"5min":29.605,"15min":141.32900000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.153,"5min":1.5840000000000001,"15min":2.0369999999999999},"min":{"1min":0.34000000000000002,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":12.044,"5min":29.515999999999998,"15min":141.15899999999999},"last":1.081}]},{"osd":5,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5710000000000002,"5min":2.1469999999999998,"15min":2.5880000000000001},"min":{"1min":0.29399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.1749999999999998,"5min":32.774000000000001,"15min":235.18700000000001},"last":0.77300000000000002},{"interface":"front","average":{"1min":1.734,"5min":1.5589999999999999,"15min":2.742},"min":{"1min":0.41399999999999998,"5min":0.34499999999999997,"15min":0.32200000000000001},"max":{"1min":5.2910000000000004,"5min":36.976999999999997,"15min":235.79300000000001},"last":2.5009999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5910000000000002,"5min":1.8340000000000001,"15min":2.613},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.316},"max":{"1min":7.2169999999999996,"5min":11.837999999999999,"15min":103.60899999999999},"last":8.3209999999999997},{"interface":"front","average":{"1min":1.7010000000000001,"5min":1.208,"15min":2.601},"min":{"1min":0.37,"5min":0.37,"15min":0.33400000000000002},"max":{"1min":5.1900000000000004,"5min":9.641,"15min":103.679},"last":2.4100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1520000000000001,"5min":1.7330000000000001,"15min":2.1059999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.35999999999999999},"max":{"1min":12.093999999999999,"5min":35.021000000000001,"15min":126.68300000000001},"last":5.8170000000000002},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.8879999999999999,"15min":2.1960000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.35499999999999998},"max":{"1min":12.196,"5min":35.493000000000002,"15min":126.738},"last":0.68899999999999995}]}]},{"osd":0,"up_from":88,"seq":377957122319,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1047244,"kb_used_data":401348,"kb_used_omap":8,"kb_used_meta":645879,"kb_avail":92697908,"statfs":{"total":95995035648,"available":94922657792,"internally_reserved":0,"allocated":410980352,"data_stored":409720904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":661380872},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:52:08 2023","interfaces":[{"interface":"back","average":{"1min":3.133,"5min":2.4319999999999999,"15min":1.992},"min":{"1min":0.186,"5min":0.186,"15min":0.17199999999999999},"max":{"1min":18.164999999999999,"5min":63.534999999999997,"15min":63.534999999999997},"last":0.40999999999999998},{"interface":"front","average":{"1min":3.5619999999999998,"5min":2.4129999999999998,"15min":2.4489999999999998},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":16.484000000000002,"5min":63.677999999999997,"15min":63.677999999999997},"last":0.30599999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:51:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":2.7010000000000001,"15min":3.0369999999999999},"min":{"1min":0.33300000000000002,"5min":0.219,"15min":0.20000000000000001},"max":{"1min":9.4390000000000001,"5min":62.252000000000002,"15min":92.230000000000004},"last":0.64300000000000002},{"interface":"front","average":{"1min":2.4180000000000001,"5min":2.1720000000000002,"15min":2.8100000000000001},"min":{"1min":0.255,"5min":0.221,"15min":0.221},"max":{"1min":16.927,"5min":61.470999999999997,"15min":92.247},"last":4.7050000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:52:08 2023","interfaces":[{"interface":"back","average":{"1min":4.8259999999999996,"5min":2.1219999999999999,"15min":3.9870000000000001},"min":{"1min":0.307,"5min":0.253,"15min":0.218},"max":{"1min":59.838999999999999,"5min":59.838999999999999,"15min":123.024},"last":0.57199999999999995},{"interface":"front","average":{"1min":4.6399999999999997,"5min":2.8279999999999998,"15min":4.0220000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":59.859000000000002,"5min":72.632999999999996,"15min":99.162000000000006},"last":0.61499999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:51:38 2023","interfaces":[{"interface":"back","average":{"1min":9.0350000000000001,"5min":3.3420000000000001,"15min":2.109},"min":{"1min":0.439,"5min":0.29699999999999999,"15min":0.25700000000000001},"max":{"1min":149.63999999999999,"5min":149.63999999999999,"15min":149.63999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":8.3290000000000006,"5min":3.3119999999999998,"15min":2.4649999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":149.65899999999999,"5min":149.65899999999999,"15min":149.65899999999999},"last":0.34300000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:51:38 2023","interfaces":[{"interface":"back","average":{"1min":10.771000000000001,"5min":3.8370000000000002,"15min":3.9399999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.317},"max":{"1min":152.20500000000001,"5min":152.20500000000001,"15min":281.733},"last":0.53900000000000003},{"interface":"front","average":{"1min":10.616,"5min":4.5860000000000003,"15min":4.2279999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.30199999999999999},"max":{"1min":154.30600000000001,"5min":154.30600000000001,"15min":282.95400000000001},"last":0.70899999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:51:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.472,"15min":1.76},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":6.6539999999999999,"5min":19.579000000000001,"15min":19.579000000000001},"last":0.502},{"interface":"front","average":{"1min":2.0750000000000002,"5min":1.847,"15min":1.9810000000000001},"min":{"1min":0.42499999999999999,"5min":0.36699999999999999,"15min":0.32500000000000001},"max":{"1min":10.441000000000001,"5min":19.710999999999999,"15min":19.710999999999999},"last":0.66600000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:52:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.504,"15min":1.4770000000000001},"min":{"1min":0.38500000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":7.6040000000000001,"5min":15.933999999999999,"15min":25.378},"last":4.6420000000000003},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.486,"15min":1.861},"min":{"1min":0.39100000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":12.41,"5min":15.641,"15min":25.436},"last":1.393}]}]},{"osd":2,"up_from":98,"seq":420906795267,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1802872,"kb_used_data":1042928,"kb_used_omap":11,"kb_used_meta":759796,"kb_avail":91942280,"statfs":{"total":95995035648,"available":94148894720,"internally_reserved":0,"allocated":1067958272,"data_stored":1066538484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":778031701},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":2.161,"15min":3.5129999999999999},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.17799999999999999},"max":{"1min":7.577,"5min":42.914999999999999,"15min":162.285},"last":0.252},{"interface":"front","average":{"1min":3.2309999999999999,"5min":2.1120000000000001,"15min":3.8679999999999999},"min":{"1min":0.314,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":10.781000000000001,"5min":42.972999999999999,"15min":162.643},"last":0.26700000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:51:45 2023","interfaces":[{"interface":"back","average":{"1min":3.1909999999999998,"5min":3.8500000000000001,"15min":3.7389999999999999},"min":{"1min":0.23699999999999999,"5min":0.193,"15min":0.193},"max":{"1min":15.329000000000001,"5min":125.77500000000001,"15min":125.77500000000001},"last":0.312},{"interface":"front","average":{"1min":2.625,"5min":3.492,"15min":3.085},"min":{"1min":0.26900000000000002,"5min":0.24199999999999999,"15min":0.20999999999999999},"max":{"1min":11.074999999999999,"5min":125.857,"15min":125.857},"last":0.28599999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.855,"15min":4.9240000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.21099999999999999},"max":{"1min":9.5129999999999999,"5min":9.5129999999999999,"15min":159.35499999999999},"last":29.606999999999999},{"interface":"front","average":{"1min":2.7160000000000002,"5min":1.7170000000000001,"15min":4.7110000000000003},"min":{"1min":0.29399999999999998,"5min":0.217,"15min":0.217},"max":{"1min":11.303000000000001,"5min":11.303000000000001,"15min":159.196},"last":29.920999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:51:45 2023","interfaces":[{"interface":"back","average":{"1min":4.915,"5min":1.901,"15min":2.1030000000000002},"min":{"1min":0.32600000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":68.052000000000007,"5min":68.052000000000007,"15min":115.84999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":4.7869999999999999,"5min":2.0529999999999999,"15min":2.2210000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28699999999999998},"max":{"1min":68.727000000000004,"5min":68.727000000000004,"15min":116.149},"last":0.40100000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:51:45 2023","interfaces":[{"interface":"back","average":{"1min":4.7599999999999998,"5min":2.0179999999999998,"15min":2.1429999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":64.825000000000003,"5min":64.825000000000003,"15min":64.825000000000003},"last":0.36099999999999999},{"interface":"front","average":{"1min":5.0629999999999997,"5min":2.0019999999999998,"15min":1.843},"min":{"1min":0.39300000000000002,"5min":0.30499999999999999,"15min":0.26000000000000001},"max":{"1min":64.582999999999998,"5min":64.582999999999998,"15min":64.582999999999998},"last":0.38600000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:51:45 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.5069999999999999,"15min":1.7989999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.32000000000000001},"max":{"1min":9.5069999999999997,"5min":25.818999999999999,"15min":25.818999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.8859999999999999,"5min":1.653,"15min":2.0099999999999998},"min":{"1min":0.34399999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.821,"5min":24.481000000000002,"15min":24.481000000000002},"last":0.41299999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":1.629,"15min":2.0190000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":12.071999999999999,"5min":12.071999999999999,"15min":51.701999999999998},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.2170000000000001,"5min":1.74,"15min":2.2189999999999999},"min":{"1min":0.43099999999999999,"5min":0.378,"15min":0.373},"max":{"1min":6.2679999999999998,"5min":8.7430000000000003,"15min":51.716000000000001},"last":0.44600000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631741,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1598856,"kb_used_data":862912,"kb_used_omap":4,"kb_used_meta":735867,"kb_avail":92146296,"statfs":{"total":95995035648,"available":94357807104,"internally_reserved":0,"allocated":883621888,"data_stored":882364129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":753528414},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,1,1,0,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":2.7879999999999998,"5min":3.2040000000000002,"15min":4.0789999999999997},"min":{"1min":0.218,"5min":0.188,"15min":0.184},"max":{"1min":8.5039999999999996,"5min":47.173000000000002,"15min":167.82400000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.2000000000000002,"5min":3.0329999999999999,"15min":4.4950000000000001},"min":{"1min":0.22900000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":8.4879999999999995,"5min":47.908000000000001,"15min":167.89599999999999},"last":0.63}]},{"osd":1,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":3.7799999999999998,"15min":3.5680000000000001},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":5.9859999999999998,"5min":150.29599999999999,"15min":150.29599999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.893,"5min":4.0460000000000003,"15min":3.746},"min":{"1min":0.216,"5min":0.216,"15min":0.19},"max":{"1min":13.018000000000001,"5min":151.25700000000001,"15min":151.25700000000001},"last":2.4060000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2730000000000001,"5min":1.6120000000000001,"15min":3.669},"min":{"1min":0.32700000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.577,"5min":12.577,"15min":190.43299999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.3969999999999998,"5min":1.7250000000000001,"15min":4.0170000000000003},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.7390000000000008,"5min":9.7390000000000008,"15min":190.267},"last":0.59699999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":2.681,"15min":2.8180000000000001},"min":{"1min":0.31,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":8.0120000000000005,"5min":126.566,"15min":149.84700000000001},"last":12.851000000000001},{"interface":"front","average":{"1min":1.4390000000000001,"5min":3.7949999999999999,"15min":2.8210000000000002},"min":{"1min":0.35999999999999999,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":4.8920000000000003,"5min":126.651,"15min":149.74700000000001},"last":0.73699999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.7529999999999999,"15min":2.3370000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.28199999999999997},"max":{"1min":12.509,"5min":12.509,"15min":149.43000000000001},"last":4.5330000000000004},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.518,"15min":2.1619999999999999},"min":{"1min":0.39900000000000002,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":4.8140000000000001,"5min":7.8689999999999998,"15min":149.69300000000001},"last":4.4969999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":1.373,"15min":2.0680000000000001},"min":{"1min":0.35899999999999999,"5min":0.34499999999999997,"15min":0.33500000000000002},"max":{"1min":4.3360000000000003,"5min":7.9009999999999998,"15min":149.72399999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.5389999999999999,"5min":1.6759999999999999,"15min":2.379},"min":{"1min":0.35899999999999999,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":12.558999999999999,"5min":12.558999999999999,"15min":149.589},"last":0.61699999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":4.8719999999999999,"5min":2.0870000000000002,"15min":2.4769999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":69.277000000000001,"5min":69.277000000000001,"15min":149.77199999999999},"last":3.786},{"interface":"front","average":{"1min":4.4809999999999999,"5min":2.0329999999999999,"15min":2.5870000000000002},"min":{"1min":0.48299999999999998,"5min":0.374,"15min":0.373},"max":{"1min":69.299000000000007,"5min":69.299000000000007,"15min":149.80799999999999},"last":0.90600000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019288,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2322312,"kb_used_data":1424960,"kb_used_omap":0,"kb_used_meta":897344,"kb_avail":91422840,"statfs":{"total":95995035648,"available":93616988160,"internally_reserved":0,"allocated":1459159040,"data_stored":1458491311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918880256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:52 2023","interfaces":[{"interface":"back","average":{"1min":6.6379999999999999,"5min":4.6890000000000001,"15min":3.0910000000000002},"min":{"1min":0.26900000000000002,"5min":0.219,"15min":0.219},"max":{"1min":95.795000000000002,"5min":132.35599999999999,"15min":132.35599999999999},"last":2.621},{"interface":"front","average":{"1min":7.7619999999999996,"5min":5.1719999999999997,"15min":3.4700000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23200000000000001},"max":{"1min":95.784999999999997,"5min":132.501,"15min":132.501},"last":0.92400000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":6.4989999999999997,"5min":3.2890000000000001,"15min":3.0099999999999998},"min":{"1min":0.34999999999999998,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":59.978999999999999,"5min":68.367000000000004,"15min":187.369},"last":4.2800000000000002},{"interface":"front","average":{"1min":6.1360000000000001,"5min":3.2429999999999999,"15min":3.0819999999999999},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.27700000000000002},"max":{"1min":59.991,"5min":68.376000000000005,"15min":187.35300000000001},"last":6.4809999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:51:33 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":1.5449999999999999,"15min":3.0939999999999999},"min":{"1min":0.308,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":6.2910000000000004,"5min":17.335999999999999,"15min":196.93700000000001},"last":2.3839999999999999},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.3280000000000001,"15min":3.0310000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":4.6150000000000002,"5min":16.795000000000002,"15min":197.08099999999999},"last":2.6549999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:52:02 2023","interfaces":[{"interface":"back","average":{"1min":2.343,"5min":1.929,"15min":4.5419999999999998},"min":{"1min":0.32900000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.347,"5min":30.111999999999998,"15min":211.48699999999999},"last":1.903},{"interface":"front","average":{"1min":1.591,"5min":1.7370000000000001,"15min":4.4050000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.312},"max":{"1min":5.601,"5min":30.709,"15min":211.31100000000001},"last":1.9330000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:52:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.099,"15min":1.1919999999999999},"min":{"1min":0.27300000000000002,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":3.1190000000000002,"5min":12.34,"15min":94.646000000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1,"15min":1.107},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.8490000000000002,"5min":10.191000000000001,"15min":94.692999999999998},"last":0.89300000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.93799999999999994,"15min":0.70499999999999996},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.4939999999999998,"5min":9.8889999999999993,"15min":9.8889999999999993},"last":0.872},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.92700000000000005,"15min":0.875},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":2.387,"5min":6.0179999999999998,"15min":8.5269999999999992},"last":0.94499999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":0.88600000000000001,"15min":1.034},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.5810000000000004,"5min":5.8650000000000002,"15min":96.131},"last":0.83999999999999997},{"interface":"front","average":{"1min":1.1459999999999999,"5min":0.995,"15min":1.1399999999999999},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.3159999999999998,"5min":7.6580000000000004,"15min":96.465000000000003},"last":0.38700000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823060,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1625788,"kb_used_data":722996,"kb_used_omap":0,"kb_used_meta":902784,"kb_avail":92119364,"statfs":{"total":95995035648,"available":94330228736,"internally_reserved":0,"allocated":740347904,"data_stored":739687935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924450816},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:50 2023","interfaces":[{"interface":"back","average":{"1min":17.081,"5min":6.6459999999999999,"15min":3.819},"min":{"1min":0.33000000000000002,"5min":0.224,"15min":0.217},"max":{"1min":210.392,"5min":210.392,"15min":210.392},"last":3.0950000000000002},{"interface":"front","average":{"1min":16.922999999999998,"5min":6.8470000000000004,"15min":3.9140000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":208.43600000000001,"5min":208.43600000000001,"15min":208.43600000000001},"last":0.311}]},{"osd":1,"last update":"Mon Oct 9 05:52:11 2023","interfaces":[{"interface":"back","average":{"1min":12.114000000000001,"5min":4.3680000000000003,"15min":3.4969999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.23999999999999999},"max":{"1min":198.35300000000001,"5min":198.35300000000001,"15min":198.35300000000001},"last":7.5439999999999996},{"interface":"front","average":{"1min":11.247999999999999,"5min":4.4119999999999999,"15min":3.637},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":198.32900000000001,"5min":198.32900000000001,"15min":198.32900000000001},"last":5.4240000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:51:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":1.3200000000000001,"15min":2.1819999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":12.551,"5min":12.551,"15min":73.311999999999998},"last":3.0680000000000001},{"interface":"front","average":{"1min":2.6429999999999998,"5min":1.5529999999999999,"15min":2.2080000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":12.573,"5min":12.573,"15min":73.280000000000001},"last":0.438}]},{"osd":3,"last update":"Mon Oct 9 05:52:11 2023","interfaces":[{"interface":"back","average":{"1min":6.0309999999999997,"5min":3.379,"15min":4.2030000000000003},"min":{"1min":0.47399999999999998,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":63.069000000000003,"5min":85.069000000000003,"15min":95.236999999999995},"last":0.54500000000000004},{"interface":"front","average":{"1min":5.1680000000000001,"5min":3.3079999999999998,"15min":4.024},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":63.128,"5min":84.814999999999998,"15min":93.471000000000004},"last":3.0430000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:52:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":2.3330000000000002,"15min":1.2030000000000001},"min":{"1min":0.28000000000000003,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.8460000000000001,"5min":130.81,"15min":130.81},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.4890000000000001,"5min":2.5539999999999998,"15min":1.3},"min":{"1min":0.32300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":6.8330000000000002,"5min":133.803,"15min":133.803},"last":3.1560000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:51:21 2023","interfaces":[{"interface":"back","average":{"1min":4.9660000000000002,"5min":1.6759999999999999,"15min":1.2310000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":71.426000000000002,"5min":71.426000000000002,"15min":71.426000000000002},"last":3.0019999999999998},{"interface":"front","average":{"1min":5.3360000000000003,"5min":1.879,"15min":1.2709999999999999},"min":{"1min":0.35999999999999999,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":71.498999999999995,"5min":71.498999999999995,"15min":71.498999999999995},"last":0.34200000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:51:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":1.2789999999999999,"15min":1.133},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":31.533000000000001,"5min":31.533000000000001,"15min":31.533000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.5640000000000001,"5min":1.1919999999999999,"15min":1.0760000000000001},"min":{"1min":0.38500000000000001,"5min":0.316,"15min":0.308},"max":{"1min":31.855,"5min":31.855,"15min":31.855},"last":3.1240000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":404226048,"data_stored":404226048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":290062336,"data_stored":290062336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1059262464,"data_stored":1059258392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":875888640,"data_stored":875888640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1453326336,"data_stored":1453326336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":722014208,"data_stored":722010136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":865730560,"data_stored":865730560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":350359552,"data_stored":350355480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:52:31.550 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:52:31.663 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:31.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:31.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:31.775 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:31.775 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:31.846 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:31.881 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:32.016 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:32.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:32.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:32.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:32.173 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:32.196 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:31 smithi008 ceph-mon[184418]: pgmap v1080: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 892 MiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 23261/42678 objects degraded (54.503%); 158/42678 objects misplaced (0.370%); 15 MiB/s, 244 objects/s recovering 2023-10-09T05:52:32.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:31 smithi008 ceph-mon[191227]: pgmap v1080: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 892 MiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 23261/42678 objects degraded (54.503%); 158/42678 objects misplaced (0.370%); 15 MiB/s, 244 objects/s recovering 2023-10-09T05:52:32.268 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:32.275 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:31 smithi191 ceph-mon[161669]: pgmap v1080: 119 pgs: 1 active+recovery_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 892 MiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 23261/42678 objects degraded (54.503%); 158/42678 objects misplaced (0.370%); 15 MiB/s, 244 objects/s recovering 2023-10-09T05:52:32.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:32.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:32.377 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:32.446 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:32.520 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:32.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:32.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:32.627 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:32.701 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:33.632 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:52:33.860 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:33.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:33.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:33 smithi008 ceph-mon[184418]: pgmap v1081: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 759 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 21919/36297 objects degraded (60.388%); 38/36297 objects misplaced (0.105%); 15 MiB/s, 241 objects/s recovering 2023-10-09T05:52:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 21919/36297 objects degraded (60.388%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:33.973 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:34.083 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:34.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:33 smithi008 ceph-mon[191227]: pgmap v1081: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 759 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 21919/36297 objects degraded (60.388%); 38/36297 objects misplaced (0.105%); 15 MiB/s, 241 objects/s recovering 2023-10-09T05:52:34.223 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 21919/36297 objects degraded (60.388%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:34.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:33 smithi191 ceph-mon[161669]: pgmap v1081: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 759 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 21919/36297 objects degraded (60.388%); 38/36297 objects misplaced (0.105%); 15 MiB/s, 241 objects/s recovering 2023-10-09T05:52:34.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 21919/36297 objects degraded (60.388%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:34.585 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:34.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:34.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:34.690 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:34.778 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:35.198 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:35.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:35.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:35.309 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:35.366 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:35.450 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:35.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:35.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:35.590 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:35.633 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:35.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:35.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:35.741 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:35.771 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:35.977 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:35 smithi191 ceph-mon[161669]: pgmap v1082: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 759 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 21919/36297 objects degraded (60.388%); 38/36297 objects misplaced (0.105%); 14 MiB/s, 224 objects/s recovering 2023-10-09T05:52:36.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:35 smithi008 ceph-mon[184418]: pgmap v1082: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 759 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 21919/36297 objects degraded (60.388%); 38/36297 objects misplaced (0.105%); 14 MiB/s, 224 objects/s recovering 2023-10-09T05:52:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:35 smithi008 ceph-mon[191227]: pgmap v1082: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 759 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 21919/36297 objects degraded (60.388%); 38/36297 objects misplaced (0.105%); 14 MiB/s, 224 objects/s recovering 2023-10-09T05:52:36.332 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:36.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:36.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:36.350 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:36.406 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:36.444 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:36.486 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:36.536 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:52:36.571 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:52:36.588 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:36.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:36.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:36.614 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:52:36.663 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:52:36.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:36.705 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:52:36.733 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:36.743 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:52:36.779 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:52:36.817 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:52:36.824 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:52:36.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:52:36.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:52:36.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:52:36.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:52:36.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:52:36.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:52:36.825 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:52:36.826 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:36.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:36.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:36.943 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:37.043 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2917196375' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:37.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2917196375' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:37.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2917196375' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:38.273 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:38.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:38.279 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:37 smithi008 ceph-mon[184418]: pgmap v1083: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 556 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 21712/26571 objects degraded (81.713%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:52:38.280 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:38 smithi008 ceph-mon[191227]: pgmap v1083: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 556 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 21712/26571 objects degraded (81.713%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:52:38.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:37 smithi191 ceph-mon[161669]: pgmap v1083: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 556 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 21712/26571 objects degraded (81.713%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:52:38.380 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:38.454 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:39.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:39 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 21712/26571 objects degraded (81.713%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:39.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:39 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 21712/26571 objects degraded (81.713%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:39.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:39 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 21712/26571 objects degraded (81.713%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:39.554 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:52:39.555 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:52:39.556 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:52:39.556 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:52:39.556 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:52:40.093 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:40 smithi008 ceph-mon[184418]: pgmap v1084: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 495 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 21671/23640 objects degraded (91.671%); 9.0 MiB/s, 150 objects/s recovering 2023-10-09T05:52:40.093 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:40.094 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:40.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:40 smithi008 ceph-mon[191227]: pgmap v1084: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 495 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 21671/23640 objects degraded (91.671%); 9.0 MiB/s, 150 objects/s recovering 2023-10-09T05:52:40.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:40.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:40 smithi191 ceph-mon[161669]: pgmap v1084: 119 pgs: 1 active+recovering, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 96 active+clean; 495 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 21671/23640 objects degraded (91.671%); 9.0 MiB/s, 150 objects/s recovering 2023-10-09T05:52:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:40.797 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:40.803 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:52:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:52:40] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-10-09T05:52:40.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:40.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:40.905 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:40.968 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:42.018 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:42.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:42.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:42.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:42.187 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:42.287 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:42.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:42.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:42.395 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:42.443 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:42 smithi191 ceph-mon[161669]: pgmap v1085: 119 pgs: 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 97 active+clean; 398 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 21671/18957 objects degraded (114.317%); 7.9 MiB/s, 131 objects/s recovering 2023-10-09T05:52:42.444 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:42 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:42.444 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:42 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:42.444 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:42 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:42.446 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:42 smithi008 ceph-mon[184418]: pgmap v1085: 119 pgs: 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 97 active+clean; 398 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 21671/18957 objects degraded (114.317%); 7.9 MiB/s, 131 objects/s recovering 2023-10-09T05:52:42.446 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:42 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:42.446 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:42 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:42.446 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:42 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:42.446 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:42 smithi008 ceph-mon[191227]: pgmap v1085: 119 pgs: 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 97 active+clean; 398 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 21671/18957 objects degraded (114.317%); 7.9 MiB/s, 131 objects/s recovering 2023-10-09T05:52:42.446 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:42 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:42.446 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:42 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:42.446 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:42 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:52:42.447 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:42.479 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:52:42.533 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:42.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:42.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:42.635 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:52:42.641 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:42.763 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:43.003 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:43.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:43.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:43.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:43.144 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:43.210 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:43.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:43.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:43.319 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:43.390 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 21671/18957 objects degraded (114.317%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:43.390 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 21671/18957 objects degraded (114.317%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:43.396 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:43.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 21671/18957 objects degraded (114.317%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:44.154 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:44.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:44.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:44.261 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:44.265 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Removed 102939 objects 2023-10-09T05:52:44.265 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Clean up completed and total clean up time :211.102 2023-10-09T05:52:44.313 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:44.432 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:44.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:44 smithi008 ceph-mon[184418]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 217 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 21442/10269 objects degraded (208.803%); 6.5 MiB/s, 132 objects/s recovering 2023-10-09T05:52:44.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:44.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:44 smithi008 ceph-mon[191227]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 217 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 21442/10269 objects degraded (208.803%); 6.5 MiB/s, 132 objects/s recovering 2023-10-09T05:52:44.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:44.467 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:44.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:44 smithi191 ceph-mon[161669]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 217 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 21442/10269 objects degraded (208.803%); 6.5 MiB/s, 132 objects/s recovering 2023-10-09T05:52:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:44.572 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:44.606 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:44.715 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:44.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:44.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:44.777 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:52:44.779 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:52:44.819 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:44.850 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:44.958 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:45.120 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:52:45.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/623932896' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:45.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:45 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:45.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/623932896' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:45.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:45 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/623932896' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:45 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:46.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[184418]: pgmap v1087: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 217 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 21442/10269 objects degraded (208.803%); 1.4 MiB/s, 51 objects/s recovering 2023-10-09T05:52:46.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[184418]: from='client.36183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:52:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[184418]: failed to encode map e359 with expected crc 2023-10-09T05:52:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[184418]: failed to encode map e359 with expected crc 2023-10-09T05:52:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[184418]: failed to encode map e359 with expected crc 2023-10-09T05:52:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:52:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[184418]: osdmap e359: 8 total, 8 up, 8 in 2023-10-09T05:52:46.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:46.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[191227]: pgmap v1087: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 217 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 21442/10269 objects degraded (208.803%); 1.4 MiB/s, 51 objects/s recovering 2023-10-09T05:52:46.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[191227]: from='client.36183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:52:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[191227]: failed to encode map e359 with expected crc 2023-10-09T05:52:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[191227]: failed to encode map e359 with expected crc 2023-10-09T05:52:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[191227]: failed to encode map e359 with expected crc 2023-10-09T05:52:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:52:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[191227]: osdmap e359: 8 total, 8 up, 8 in 2023-10-09T05:52:46.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:46 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:46 smithi191 ceph-mon[161669]: pgmap v1087: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 217 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 21442/10269 objects degraded (208.803%); 1.4 MiB/s, 51 objects/s recovering 2023-10-09T05:52:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:46 smithi191 ceph-mon[161669]: from='client.36183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:52:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:46 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:46 smithi191 ceph-mon[161669]: failed to encode map e359 with expected crc 2023-10-09T05:52:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:46 smithi191 ceph-mon[161669]: failed to encode map e359 with expected crc 2023-10-09T05:52:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:46 smithi191 ceph-mon[161669]: failed to encode map e359 with expected crc 2023-10-09T05:52:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:46 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:52:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:46 smithi191 ceph-mon[161669]: osdmap e359: 8 total, 8 up, 8 in 2023-10-09T05:52:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:46 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:47.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:47 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:47.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:47 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:47 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:47.909 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T05:52:47.910 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T05:52:48.183 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:48.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:48 smithi008 ceph-mon[184418]: pgmap v1089: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 57 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 21442/2601 objects degraded (824.375%); 166 KiB/s, 30 objects/s recovering 2023-10-09T05:52:48.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:48 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:48.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 21442/2601 objects degraded (824.375%), 21 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:48.186 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:48 smithi008 ceph-mon[191227]: pgmap v1089: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 57 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 21442/2601 objects degraded (824.375%); 166 KiB/s, 30 objects/s recovering 2023-10-09T05:52:48.186 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:48 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:48.186 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 21442/2601 objects degraded (824.375%), 21 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:48.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:48.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:48.289 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:48.322 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:48.368 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:48.371 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:48 smithi191 ceph-mon[161669]: pgmap v1089: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 57 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 21442/2601 objects degraded (824.375%); 166 KiB/s, 30 objects/s recovering 2023-10-09T05:52:48.371 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:48 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:48.371 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 21442/2601 objects degraded (824.375%), 21 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:48.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:48.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:48.473 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:48.501 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:48.545 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:48.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:48.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:48.651 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:48.677 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:48.718 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:48.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:48.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:48.796 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1087,"stamp":"2023-10-09T05:52:44.516458+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":227219403,"num_objects":3423,"num_object_clones":0,"num_object_copies":10269,"num_objects_missing_on_primary":451,"num_objects_missing":451,"num_objects_degraded":21442,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3423,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":203603,"num_write_kb":6603238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35356,"num_bytes_recovered":2283679514,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":372511,"ondisk_log_size":372511,"up":357,"acting":343,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":398,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10398596,"kb_used_data":3939652,"kb_used_omap":45,"kb_used_meta":6458834,"kb_avail":739562620,"statfs":{"total":767960285184,"available":757312122880,"internally_reserved":0,"allocated":4034203648,"data_stored":4026378408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":6613846307},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,3,2,1,0,1,8],"upper_bound":256},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-568590336,"num_objects":-8676,"num_object_clones":0,"num_object_copies":-26028,"num_objects_missing_on_primary":-11,"num_objects_missing":-11,"num_objects_degraded":-477,"num_objects_misplaced":-38,"num_objects_unfound":0,"num_objects_dirty":-8676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8688,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":613,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.008213"},"pg_stats":[{"pgid":"8.2d","version":"358'8271","reported_seq":8371,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.222079+0000","last_change":"2023-10-09T05:50:08.061665+0000","last_active":"2023-10-09T05:52:40.222079+0000","last_peered":"2023-10-09T05:52:40.222079+0000","last_clean":"2023-10-09T05:52:40.222079+0000","last_became_active":"2023-10-09T05:50:08.057591+0000","last_became_peered":"2023-10-09T05:50:08.057591+0000","last_unstale":"2023-10-09T05:52:40.222079+0000","last_undegraded":"2023-10-09T05:52:40.222079+0000","last_fullsized":"2023-10-09T05:52:40.222079+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8271,"log_dups_size":0,"ondisk_log_size":8271,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2736134,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3139,"num_write_kb":101776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.2c","version":"358'8538","reported_seq":8641,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:39.465150+0000","last_change":"2023-10-09T05:50:08.058697+0000","last_active":"2023-10-09T05:52:39.465150+0000","last_peered":"2023-10-09T05:52:39.465150+0000","last_clean":"2023-10-09T05:52:39.465150+0000","last_became_active":"2023-10-09T05:50:08.055366+0000","last_became_peered":"2023-10-09T05:50:08.055366+0000","last_unstale":"2023-10-09T05:52:39.465150+0000","last_undegraded":"2023-10-09T05:52:39.465150+0000","last_fullsized":"2023-10-09T05:52:39.465150+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8538,"log_dups_size":0,"ondisk_log_size":8538,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4096000,"num_objects":63,"num_object_clones":0,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3229,"num_write_kb":105312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.2b","version":"358'8402","reported_seq":11560,"reported_epoch":358,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:40.252841+0000","last_change":"2023-10-09T05:50:08.118923+0000","last_active":"2023-10-09T05:52:40.252841+0000","last_peered":"2023-10-09T05:52:40.252841+0000","last_clean":"2023-10-09T05:48:53.462109+0000","last_became_active":"2023-10-09T05:50:08.113731+0000","last_became_peered":"2023-10-09T05:50:08.113731+0000","last_unstale":"2023-10-09T05:52:40.252841+0000","last_undegraded":"2023-10-09T05:50:08.089561+0000","last_fullsized":"2023-10-09T05:52:40.252841+0000","mapping_epoch":322,"log_start":"289'200","ondisk_log_start":"289'200","created":322,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8202,"log_dups_size":0,"ondisk_log_size":8202,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6635520,"num_objects":101,"num_object_clones":0,"num_object_copies":303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":66,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3156,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1354,"num_bytes_recovered":88702976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"1,3,4","objects":101}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2a","version":"358'8347","reported_seq":12103,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:42.315949+0000","last_change":"2023-10-09T05:51:43.298018+0000","last_active":"2023-10-09T05:52:42.315949+0000","last_peered":"2023-10-09T05:52:42.315949+0000","last_clean":"2023-10-09T05:48:53.473854+0000","last_became_active":"2023-10-09T05:51:43.269984+0000","last_became_peered":"2023-10-09T05:51:43.269984+0000","last_unstale":"2023-10-09T05:52:42.315949+0000","last_undegraded":"2023-10-09T05:51:43.115562+0000","last_fullsized":"2023-10-09T05:51:43.062313+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8347,"log_dups_size":0,"ondisk_log_size":8347,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1179,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3212,"num_write_kb":102448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1399,"num_bytes_recovered":91586560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2","objects":-184},{"shards":"1,2,3","objects":28}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.29","version":"358'7838","reported_seq":7947,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:39.560309+0000","last_change":"2023-10-09T05:51:44.323120+0000","last_active":"2023-10-09T05:52:39.560309+0000","last_peered":"2023-10-09T05:52:39.560309+0000","last_clean":"2023-10-09T05:51:41.995515+0000","last_became_active":"2023-10-09T05:51:44.308533+0000","last_became_peered":"2023-10-09T05:51:44.308533+0000","last_unstale":"2023-10-09T05:52:39.560309+0000","last_undegraded":"2023-10-09T05:51:44.104196+0000","last_fullsized":"2023-10-09T05:51:44.066927+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7838,"log_dups_size":0,"ondisk_log_size":7838,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2785280,"num_objects":43,"num_object_clones":0,"num_object_copies":129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1638,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3224,"num_write_kb":104512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[0,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":65},{"shards":"0,6","objects":-624}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.28","version":"358'7605","reported_seq":7703,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.406608+0000","last_change":"2023-10-09T05:50:08.055391+0000","last_active":"2023-10-09T05:52:40.406608+0000","last_peered":"2023-10-09T05:52:40.406608+0000","last_clean":"2023-10-09T05:52:40.406608+0000","last_became_active":"2023-10-09T05:50:08.052477+0000","last_became_peered":"2023-10-09T05:50:08.052477+0000","last_unstale":"2023-10-09T05:52:40.406608+0000","last_undegraded":"2023-10-09T05:52:40.406608+0000","last_fullsized":"2023-10-09T05:52:40.406608+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7605,"log_dups_size":0,"ondisk_log_size":7605,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1998848,"num_objects":31,"num_object_clones":0,"num_object_copies":93,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3127,"num_write_kb":101024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.27","version":"358'7833","reported_seq":10389,"reported_epoch":358,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:41.967328+0000","last_change":"2023-10-09T05:50:08.428915+0000","last_active":"2023-10-09T05:52:41.967328+0000","last_peered":"2023-10-09T05:52:41.967328+0000","last_clean":"2023-10-09T05:48:53.410221+0000","last_became_active":"2023-10-09T05:50:08.415220+0000","last_became_peered":"2023-10-09T05:50:08.415220+0000","last_unstale":"2023-10-09T05:52:41.967328+0000","last_undegraded":"2023-10-09T05:50:08.088678+0000","last_fullsized":"2023-10-09T05:52:41.967328+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7833,"log_dups_size":0,"ondisk_log_size":7833,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2883584,"num_objects":44,"num_object_clones":0,"num_object_copies":132,"num_objects_missing_on_primary":219,"num_objects_missing":219,"num_objects_degraded":219,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":44,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3201,"num_write_kb":103840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1222,"num_bytes_recovered":80084992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,6","objects":44}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.26","version":"358'7636","reported_seq":7740,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:39.506300+0000","last_change":"2023-10-09T05:50:08.077569+0000","last_active":"2023-10-09T05:52:39.506300+0000","last_peered":"2023-10-09T05:52:39.506300+0000","last_clean":"2023-10-09T05:52:39.506300+0000","last_became_active":"2023-10-09T05:50:08.072764+0000","last_became_peered":"2023-10-09T05:50:08.072764+0000","last_unstale":"2023-10-09T05:52:39.506300+0000","last_undegraded":"2023-10-09T05:52:39.506300+0000","last_fullsized":"2023-10-09T05:52:39.506300+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7636,"log_dups_size":0,"ondisk_log_size":7636,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1982464,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3144,"num_write_kb":101584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.25","version":"358'7669","reported_seq":7765,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.135909+0000","last_change":"2023-10-09T05:50:08.046650+0000","last_active":"2023-10-09T05:52:40.135909+0000","last_peered":"2023-10-09T05:52:40.135909+0000","last_clean":"2023-10-09T05:52:40.135909+0000","last_became_active":"2023-10-09T05:50:08.043790+0000","last_became_peered":"2023-10-09T05:50:08.043790+0000","last_unstale":"2023-10-09T05:52:40.135909+0000","last_undegraded":"2023-10-09T05:52:40.135909+0000","last_fullsized":"2023-10-09T05:52:40.135909+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7669,"log_dups_size":0,"ondisk_log_size":7669,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5226496,"num_objects":80,"num_object_clones":0,"num_object_copies":240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3140,"num_write_kb":103024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.24","version":"358'7717","reported_seq":7814,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.104543+0000","last_change":"2023-10-09T05:50:08.055768+0000","last_active":"2023-10-09T05:52:40.104543+0000","last_peered":"2023-10-09T05:52:40.104543+0000","last_clean":"2023-10-09T05:52:40.104543+0000","last_became_active":"2023-10-09T05:50:08.052904+0000","last_became_peered":"2023-10-09T05:50:08.052904+0000","last_unstale":"2023-10-09T05:52:40.104543+0000","last_undegraded":"2023-10-09T05:52:40.104543+0000","last_fullsized":"2023-10-09T05:52:40.104543+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":322,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7717,"log_dups_size":0,"ondisk_log_size":7717,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3719168,"num_objects":57,"num_object_clones":0,"num_object_copies":171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3171,"num_write_kb":103280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.23","version":"358'7516","reported_seq":10369,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:40.439054+0000","last_change":"2023-10-09T05:50:19.098280+0000","last_active":"2023-10-09T05:52:40.439054+0000","last_peered":"2023-10-09T05:52:40.439054+0000","last_clean":"2023-10-09T05:48:53.427903+0000","last_became_active":"2023-10-09T05:50:19.090954+0000","last_became_peered":"2023-10-09T05:50:19.090954+0000","last_unstale":"2023-10-09T05:52:40.439054+0000","last_undegraded":"2023-10-09T05:50:19.046676+0000","last_fullsized":"2023-10-09T05:50:19.045180+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7516,"log_dups_size":0,"ondisk_log_size":7516,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3112960,"num_objects":48,"num_object_clones":0,"num_object_copies":144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":63,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":48,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3160,"num_write_kb":102624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1370,"num_bytes_recovered":89784320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"1,5","objects":48}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.22","version":"358'7411","reported_seq":7511,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.141503+0000","last_change":"2023-10-09T05:49:20.140728+0000","last_active":"2023-10-09T05:52:40.141503+0000","last_peered":"2023-10-09T05:52:40.141503+0000","last_clean":"2023-10-09T05:52:40.141503+0000","last_became_active":"2023-10-09T05:49:20.132812+0000","last_became_peered":"2023-10-09T05:49:20.132812+0000","last_unstale":"2023-10-09T05:52:40.141503+0000","last_undegraded":"2023-10-09T05:52:40.141503+0000","last_fullsized":"2023-10-09T05:52:40.141503+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7411,"log_dups_size":0,"ondisk_log_size":7411,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5521408,"num_objects":84,"num_object_clones":0,"num_object_copies":252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":84,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3089,"num_write_kb":101552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.21","version":"358'7524","reported_seq":10431,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:39.432980+0000","last_change":"2023-10-09T05:50:19.777110+0000","last_active":"2023-10-09T05:52:39.432980+0000","last_peered":"2023-10-09T05:52:39.432980+0000","last_clean":"2023-10-09T05:48:53.421353+0000","last_became_active":"2023-10-09T05:50:19.773817+0000","last_became_peered":"2023-10-09T05:50:19.773817+0000","last_unstale":"2023-10-09T05:52:39.432980+0000","last_undegraded":"2023-10-09T05:50:19.043939+0000","last_fullsized":"2023-10-09T05:50:19.040976+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7524,"log_dups_size":0,"ondisk_log_size":7524,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3014656,"num_objects":46,"num_object_clones":0,"num_object_copies":138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":72,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3157,"num_write_kb":102496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1397,"num_bytes_recovered":91553792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,1","objects":46}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.20","version":"358'7615","reported_seq":10963,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:42.234908+0000","last_change":"2023-10-09T05:51:43.230120+0000","last_active":"2023-10-09T05:52:42.234908+0000","last_peered":"2023-10-09T05:52:42.234908+0000","last_clean":"2023-10-09T05:48:53.429181+0000","last_became_active":"2023-10-09T05:51:43.216843+0000","last_became_peered":"2023-10-09T05:51:43.216843+0000","last_unstale":"2023-10-09T05:52:42.234908+0000","last_undegraded":"2023-10-09T05:51:43.129071+0000","last_fullsized":"2023-10-09T05:51:43.055387+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7615,"log_dups_size":0,"ondisk_log_size":7615,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4292608,"num_objects":66,"num_object_clones":0,"num_object_copies":198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1137,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3192,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1360,"num_bytes_recovered":89128960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[3,1],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3","objects":-104},{"shards":"1,3,4","objects":25}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.11","version":"358'8956","reported_seq":13649,"reported_epoch":358,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:39.526045+0000","last_change":"2023-10-09T05:50:19.729796+0000","last_active":"2023-10-09T05:52:39.526045+0000","last_peered":"2023-10-09T05:52:39.526045+0000","last_clean":"2023-10-09T05:48:53.444054+0000","last_became_active":"2023-10-09T05:50:19.723776+0000","last_became_peered":"2023-10-09T05:50:19.723776+0000","last_unstale":"2023-10-09T05:52:39.526045+0000","last_undegraded":"2023-10-09T05:50:19.039020+0000","last_fullsized":"2023-10-09T05:52:39.526045+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8956,"log_dups_size":0,"ondisk_log_size":8956,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11206656,"num_objects":171,"num_object_clones":0,"num_object_copies":513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":558,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6257,"num_write_kb":205696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2294,"num_bytes_recovered":150274048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,1,2","objects":171}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":358,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181276+0000","last_change":"2023-10-09T05:48:54.494513+0000","last_active":"2023-10-09T05:52:17.181276+0000","last_peered":"2023-10-09T05:52:17.181276+0000","last_clean":"2023-10-09T05:52:17.181276+0000","last_became_active":"2023-10-09T05:48:54.493510+0000","last_became_peered":"2023-10-09T05:48:54.493510+0000","last_unstale":"2023-10-09T05:52:17.181276+0000","last_undegraded":"2023-10-09T05:52:17.181276+0000","last_fullsized":"2023-10-09T05:52:17.181276+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":288,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180855+0000","last_change":"2023-10-09T05:50:18.047163+0000","last_active":"2023-10-09T05:52:17.180855+0000","last_peered":"2023-10-09T05:52:17.180855+0000","last_clean":"2023-10-09T05:52:17.180855+0000","last_became_active":"2023-10-09T05:50:18.046872+0000","last_became_peered":"2023-10-09T05:50:18.046872+0000","last_unstale":"2023-10-09T05:52:17.180855+0000","last_undegraded":"2023-10-09T05:52:17.180855+0000","last_fullsized":"2023-10-09T05:52:17.180855+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.12","version":"358'8902","reported_seq":8992,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.410497+0000","last_change":"2023-10-09T05:48:46.263477+0000","last_active":"2023-10-09T05:52:40.410497+0000","last_peered":"2023-10-09T05:52:40.410497+0000","last_clean":"2023-10-09T05:52:40.410497+0000","last_became_active":"2023-10-09T05:48:46.259060+0000","last_became_peered":"2023-10-09T05:48:46.259060+0000","last_unstale":"2023-10-09T05:52:40.410497+0000","last_undegraded":"2023-10-09T05:52:40.410497+0000","last_fullsized":"2023-10-09T05:52:40.410497+0000","mapping_epoch":299,"log_start":"291'2334","ondisk_log_start":"291'2334","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6568,"log_dups_size":0,"ondisk_log_size":6568,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5537792,"num_objects":85,"num_object_clones":0,"num_object_copies":255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6235,"num_write_kb":202208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":102,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180916+0000","last_change":"2023-10-09T05:50:18.055745+0000","last_active":"2023-10-09T05:52:17.180916+0000","last_peered":"2023-10-09T05:52:17.180916+0000","last_clean":"2023-10-09T05:52:17.180916+0000","last_became_active":"2023-10-09T05:50:18.055524+0000","last_became_peered":"2023-10-09T05:50:18.055524+0000","last_unstale":"2023-10-09T05:52:17.180916+0000","last_undegraded":"2023-10-09T05:52:17.180916+0000","last_fullsized":"2023-10-09T05:52:17.180916+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":376,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181490+0000","last_change":"2023-10-09T05:51:22.261181+0000","last_active":"2023-10-09T05:52:17.181490+0000","last_peered":"2023-10-09T05:52:17.181490+0000","last_clean":"2023-10-09T05:52:17.181490+0000","last_became_active":"2023-10-09T05:51:22.260808+0000","last_became_peered":"2023-10-09T05:51:22.260808+0000","last_unstale":"2023-10-09T05:52:17.181490+0000","last_undegraded":"2023-10-09T05:52:17.181490+0000","last_fullsized":"2023-10-09T05:52:17.181490+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:17.362302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"358'8924","reported_seq":13693,"reported_epoch":358,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:40.468247+0000","last_change":"2023-10-09T05:50:19.098294+0000","last_active":"2023-10-09T05:52:40.468247+0000","last_peered":"2023-10-09T05:52:40.468247+0000","last_clean":"2023-10-09T05:48:53.465296+0000","last_became_active":"2023-10-09T05:50:19.091214+0000","last_became_peered":"2023-10-09T05:50:19.091214+0000","last_unstale":"2023-10-09T05:52:40.468247+0000","last_undegraded":"2023-10-09T05:50:19.044182+0000","last_fullsized":"2023-10-09T05:52:40.468247+0000","mapping_epoch":328,"log_start":"288'147","ondisk_log_start":"288'147","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8777,"log_dups_size":0,"ondisk_log_size":8777,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10027008,"num_objects":153,"num_object_clones":0,"num_object_copies":459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":480,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6233,"num_write_kb":204352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2333,"num_bytes_recovered":152895488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"1,5,6","objects":153}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":327,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.179846+0000","last_change":"2023-10-09T05:48:54.492664+0000","last_active":"2023-10-09T05:52:17.179846+0000","last_peered":"2023-10-09T05:52:17.179846+0000","last_clean":"2023-10-09T05:52:17.179846+0000","last_became_active":"2023-10-09T05:48:54.492299+0000","last_became_peered":"2023-10-09T05:48:54.492299+0000","last_unstale":"2023-10-09T05:52:17.179846+0000","last_undegraded":"2023-10-09T05:52:17.179846+0000","last_fullsized":"2023-10-09T05:52:17.179846+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:43.198424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":191,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.174609+0000","last_change":"2023-10-09T05:48:54.492447+0000","last_active":"2023-10-09T05:52:17.174609+0000","last_peered":"2023-10-09T05:52:17.174609+0000","last_clean":"2023-10-09T05:52:17.174609+0000","last_became_active":"2023-10-09T05:48:54.492089+0000","last_became_peered":"2023-10-09T05:48:54.492089+0000","last_unstale":"2023-10-09T05:52:17.174609+0000","last_undegraded":"2023-10-09T05:52:17.174609+0000","last_fullsized":"2023-10-09T05:52:17.174609+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.1c","version":"358'9846","reported_seq":9939,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:39.582427+0000","last_change":"2023-10-09T05:49:20.384940+0000","last_active":"2023-10-09T05:52:39.582427+0000","last_peered":"2023-10-09T05:52:39.582427+0000","last_clean":"2023-10-09T05:52:39.582427+0000","last_became_active":"2023-10-09T05:49:20.380945+0000","last_became_peered":"2023-10-09T05:49:20.380945+0000","last_unstale":"2023-10-09T05:52:39.582427+0000","last_undegraded":"2023-10-09T05:52:39.582427+0000","last_fullsized":"2023-10-09T05:52:39.582427+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9846,"log_dups_size":0,"ondisk_log_size":9846,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11010048,"num_objects":168,"num_object_clones":0,"num_object_copies":504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6414,"num_write_kb":210624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":314,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180522+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:52:17.180522+0000","last_peered":"2023-10-09T05:52:17.180522+0000","last_clean":"2023-10-09T05:52:17.180522+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:52:17.180522+0000","last_undegraded":"2023-10-09T05:52:17.180522+0000","last_fullsized":"2023-10-09T05:52:17.180522+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":319,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:29.130509+0000","last_change":"2023-10-09T05:50:18.054081+0000","last_active":"2023-10-09T05:52:29.130509+0000","last_peered":"2023-10-09T05:52:29.130509+0000","last_clean":"2023-10-09T05:52:29.130509+0000","last_became_active":"2023-10-09T05:50:18.053746+0000","last_became_peered":"2023-10-09T05:50:18.053746+0000","last_unstale":"2023-10-09T05:52:29.130509+0000","last_undegraded":"2023-10-09T05:52:29.130509+0000","last_fullsized":"2023-10-09T05:52:29.130509+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:55.758681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1d","version":"358'9567","reported_seq":9658,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.222078+0000","last_change":"2023-10-09T05:49:20.157594+0000","last_active":"2023-10-09T05:52:40.222078+0000","last_peered":"2023-10-09T05:52:40.222078+0000","last_clean":"2023-10-09T05:52:40.222078+0000","last_became_active":"2023-10-09T05:49:20.153063+0000","last_became_peered":"2023-10-09T05:49:20.153063+0000","last_unstale":"2023-10-09T05:52:40.222078+0000","last_undegraded":"2023-10-09T05:52:40.222078+0000","last_fullsized":"2023-10-09T05:52:40.222078+0000","mapping_epoch":311,"log_start":"290'339","ondisk_log_start":"290'339","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9228,"log_dups_size":0,"ondisk_log_size":9228,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6586380,"num_objects":101,"num_object_clones":0,"num_object_copies":303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":101,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6261,"num_write_kb":203553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":18,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.174347+0000","last_change":"2023-10-09T05:51:43.123678+0000","last_active":"2023-10-09T05:52:17.174347+0000","last_peered":"2023-10-09T05:52:17.174347+0000","last_clean":"2023-10-09T05:52:17.174347+0000","last_became_active":"2023-10-09T05:51:43.121947+0000","last_became_peered":"2023-10-09T05:51:43.121947+0000","last_unstale":"2023-10-09T05:52:17.174347+0000","last_undegraded":"2023-10-09T05:52:17.174347+0000","last_fullsized":"2023-10-09T05:52:17.174347+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":293,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.176298+0000","last_change":"2023-10-09T05:48:25.068328+0000","last_active":"2023-10-09T05:52:17.176298+0000","last_peered":"2023-10-09T05:52:17.176298+0000","last_clean":"2023-10-09T05:52:17.176298+0000","last_became_active":"2023-10-09T05:48:25.068094+0000","last_became_peered":"2023-10-09T05:48:25.068094+0000","last_unstale":"2023-10-09T05:52:17.176298+0000","last_undegraded":"2023-10-09T05:52:17.176298+0000","last_fullsized":"2023-10-09T05:52:17.176298+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.1e","version":"358'9612","reported_seq":9708,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:39.601517+0000","last_change":"2023-10-09T05:50:17.990359+0000","last_active":"2023-10-09T05:52:39.601517+0000","last_peered":"2023-10-09T05:52:39.601517+0000","last_clean":"2023-10-09T05:52:39.601517+0000","last_became_active":"2023-10-09T05:49:20.168491+0000","last_became_peered":"2023-10-09T05:49:20.168491+0000","last_unstale":"2023-10-09T05:52:39.601517+0000","last_undegraded":"2023-10-09T05:52:39.601517+0000","last_fullsized":"2023-10-09T05:52:39.601517+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9612,"log_dups_size":0,"ondisk_log_size":9612,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12976128,"num_objects":198,"num_object_clones":0,"num_object_copies":594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6265,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":143,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.182038+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:52:17.182038+0000","last_peered":"2023-10-09T05:52:17.182038+0000","last_clean":"2023-10-09T05:52:17.182038+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:52:17.182038+0000","last_undegraded":"2023-10-09T05:52:17.182038+0000","last_fullsized":"2023-10-09T05:52:17.182038+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":139,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:29.116566+0000","last_change":"2023-10-09T05:50:18.045973+0000","last_active":"2023-10-09T05:52:29.116566+0000","last_peered":"2023-10-09T05:52:29.116566+0000","last_clean":"2023-10-09T05:52:29.116566+0000","last_became_active":"2023-10-09T05:50:18.045688+0000","last_became_peered":"2023-10-09T05:50:18.045688+0000","last_unstale":"2023-10-09T05:52:29.116566+0000","last_undegraded":"2023-10-09T05:52:29.116566+0000","last_fullsized":"2023-10-09T05:52:29.116566+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:51.105027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.1f","version":"358'9544","reported_seq":16628,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:42.369647+0000","last_change":"2023-10-09T05:51:14.207219+0000","last_active":"2023-10-09T05:52:42.369647+0000","last_peered":"2023-10-09T05:52:42.369647+0000","last_clean":"2023-10-09T05:48:53.467734+0000","last_became_active":"2023-10-09T05:51:14.187967+0000","last_became_peered":"2023-10-09T05:51:14.187967+0000","last_unstale":"2023-10-09T05:52:42.369647+0000","last_undegraded":"2023-10-09T05:51:14.126809+0000","last_fullsized":"2023-10-09T05:51:14.060740+0000","mapping_epoch":340,"log_start":"290'2131","ondisk_log_start":"290'2131","created":311,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7413,"log_dups_size":0,"ondisk_log_size":7413,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3964928,"num_objects":61,"num_object_clones":0,"num_object_copies":183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1707,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6261,"num_write_kb":202272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2702,"num_bytes_recovered":176881664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[3,1],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3","objects":-274},{"shards":"1,3,4","objects":60}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":384,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.182002+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:52:17.182002+0000","last_peered":"2023-10-09T05:52:17.182002+0000","last_clean":"2023-10-09T05:52:17.182002+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:52:17.182002+0000","last_undegraded":"2023-10-09T05:52:17.182002+0000","last_fullsized":"2023-10-09T05:52:17.182002+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":341,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181610+0000","last_change":"2023-10-09T05:48:54.492082+0000","last_active":"2023-10-09T05:52:17.181610+0000","last_peered":"2023-10-09T05:52:17.181610+0000","last_clean":"2023-10-09T05:52:17.181610+0000","last_became_active":"2023-10-09T05:48:54.491753+0000","last_became_peered":"2023-10-09T05:48:54.491753+0000","last_unstale":"2023-10-09T05:52:17.181610+0000","last_undegraded":"2023-10-09T05:52:17.181610+0000","last_fullsized":"2023-10-09T05:52:17.181610+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"358'8955","reported_seq":9047,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.556370+0000","last_change":"2023-10-09T05:48:46.314063+0000","last_active":"2023-10-09T05:52:40.556370+0000","last_peered":"2023-10-09T05:52:40.556370+0000","last_clean":"2023-10-09T05:52:40.556370+0000","last_became_active":"2023-10-09T05:48:46.261937+0000","last_became_peered":"2023-10-09T05:48:46.261937+0000","last_unstale":"2023-10-09T05:52:40.556370+0000","last_undegraded":"2023-10-09T05:52:40.556370+0000","last_fullsized":"2023-10-09T05:52:40.556370+0000","mapping_epoch":299,"log_start":"289'274","ondisk_log_start":"289'274","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8681,"log_dups_size":0,"ondisk_log_size":8681,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7143424,"num_objects":109,"num_object_clones":0,"num_object_copies":327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6273,"num_write_kb":204224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":143,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.176842+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:52:17.176842+0000","last_peered":"2023-10-09T05:52:17.176842+0000","last_clean":"2023-10-09T05:52:17.176842+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:52:17.176842+0000","last_undegraded":"2023-10-09T05:52:17.176842+0000","last_fullsized":"2023-10-09T05:52:17.176842+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":321,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.177519+0000","last_change":"2023-10-09T05:50:18.039555+0000","last_active":"2023-10-09T05:52:17.177519+0000","last_peered":"2023-10-09T05:52:17.177519+0000","last_clean":"2023-10-09T05:52:17.177519+0000","last_became_active":"2023-10-09T05:50:18.039222+0000","last_became_peered":"2023-10-09T05:50:18.039222+0000","last_unstale":"2023-10-09T05:52:17.177519+0000","last_undegraded":"2023-10-09T05:52:17.177519+0000","last_fullsized":"2023-10-09T05:52:17.177519+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.19","version":"358'9105","reported_seq":9203,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:39.535804+0000","last_change":"2023-10-09T05:51:44.344144+0000","last_active":"2023-10-09T05:52:39.535804+0000","last_peered":"2023-10-09T05:52:39.535804+0000","last_clean":"2023-10-09T05:51:42.004639+0000","last_became_active":"2023-10-09T05:51:44.322844+0000","last_became_peered":"2023-10-09T05:51:44.322844+0000","last_unstale":"2023-10-09T05:52:39.535804+0000","last_undegraded":"2023-10-09T05:51:44.113025+0000","last_fullsized":"2023-10-09T05:51:44.092936+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9105,"log_dups_size":0,"ondisk_log_size":9105,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10420224,"num_objects":159,"num_object_clones":0,"num_object_copies":477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3236,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6357,"num_write_kb":208512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[0,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":137},{"shards":"0,6","objects":-1226}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":347,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180192+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:52:17.180192+0000","last_peered":"2023-10-09T05:52:17.180192+0000","last_clean":"2023-10-09T05:52:17.180192+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:52:17.180192+0000","last_undegraded":"2023-10-09T05:52:17.180192+0000","last_fullsized":"2023-10-09T05:52:17.180192+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":18,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.175729+0000","last_change":"2023-10-09T05:51:43.075341+0000","last_active":"2023-10-09T05:52:17.175729+0000","last_peered":"2023-10-09T05:52:17.175729+0000","last_clean":"2023-10-09T05:52:17.175729+0000","last_became_active":"2023-10-09T05:51:43.075024+0000","last_became_peered":"2023-10-09T05:51:43.075024+0000","last_unstale":"2023-10-09T05:52:17.175729+0000","last_undegraded":"2023-10-09T05:52:17.175729+0000","last_fullsized":"2023-10-09T05:52:17.175729+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.1a","version":"358'9640","reported_seq":16197,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:42.365360+0000","last_change":"2023-10-09T05:51:43.298787+0000","last_active":"2023-10-09T05:52:42.365360+0000","last_peered":"2023-10-09T05:52:42.365360+0000","last_clean":"2023-10-09T05:48:53.473854+0000","last_became_active":"2023-10-09T05:51:43.282023+0000","last_became_peered":"2023-10-09T05:51:43.282023+0000","last_unstale":"2023-10-09T05:52:42.365360+0000","last_undegraded":"2023-10-09T05:51:43.169659+0000","last_fullsized":"2023-10-09T05:51:43.062848+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9640,"log_dups_size":0,"ondisk_log_size":9640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5242880,"num_objects":80,"num_object_clones":0,"num_object_copies":240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6323,"num_write_kb":204896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2726,"num_bytes_recovered":178520064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2","objects":-244},{"shards":"1,2,3","objects":42}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":137,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.176378+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:52:17.176378+0000","last_peered":"2023-10-09T05:52:17.176378+0000","last_clean":"2023-10-09T05:52:17.176378+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:52:17.176378+0000","last_undegraded":"2023-10-09T05:52:17.176378+0000","last_fullsized":"2023-10-09T05:52:17.176378+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":347,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181566+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:52:17.181566+0000","last_peered":"2023-10-09T05:52:17.181566+0000","last_clean":"2023-10-09T05:52:17.181566+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:52:17.181566+0000","last_undegraded":"2023-10-09T05:52:17.181566+0000","last_fullsized":"2023-10-09T05:52:17.181566+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1b","version":"358'9815","reported_seq":15784,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.450145+0000","last_change":"2023-10-09T05:52:37.575008+0000","last_active":"2023-10-09T05:52:40.450145+0000","last_peered":"2023-10-09T05:52:40.450145+0000","last_clean":"2023-10-09T05:52:40.450145+0000","last_became_active":"2023-10-09T05:49:20.374568+0000","last_became_peered":"2023-10-09T05:49:20.374568+0000","last_unstale":"2023-10-09T05:52:40.450145+0000","last_undegraded":"2023-10-09T05:52:40.450145+0000","last_fullsized":"2023-10-09T05:52:40.450145+0000","mapping_epoch":311,"log_start":"291'2714","ondisk_log_start":"291'2714","created":311,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7101,"log_dups_size":0,"ondisk_log_size":7101,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5472256,"num_objects":84,"num_object_clones":0,"num_object_copies":252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":84,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6431,"num_write_kb":208448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2931,"num_bytes_recovered":184549376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":173,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.179773+0000","last_change":"2023-10-09T05:51:43.221482+0000","last_active":"2023-10-09T05:52:17.179773+0000","last_peered":"2023-10-09T05:52:17.179773+0000","last_clean":"2023-10-09T05:52:17.179773+0000","last_became_active":"2023-10-09T05:51:43.072606+0000","last_became_peered":"2023-10-09T05:51:43.072606+0000","last_unstale":"2023-10-09T05:52:17.179773+0000","last_undegraded":"2023-10-09T05:52:17.179773+0000","last_fullsized":"2023-10-09T05:52:17.179773+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:22.097091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":363,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.177926+0000","last_change":"2023-10-09T05:48:54.492012+0000","last_active":"2023-10-09T05:52:17.177926+0000","last_peered":"2023-10-09T05:52:17.177926+0000","last_clean":"2023-10-09T05:52:17.177926+0000","last_became_active":"2023-10-09T05:48:54.491644+0000","last_became_peered":"2023-10-09T05:48:54.491644+0000","last_unstale":"2023-10-09T05:52:17.177926+0000","last_undegraded":"2023-10-09T05:52:17.177926+0000","last_fullsized":"2023-10-09T05:52:17.177926+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"358'7704","reported_seq":7806,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.325689+0000","last_change":"2023-10-09T05:50:17.993839+0000","last_active":"2023-10-09T05:52:40.325689+0000","last_peered":"2023-10-09T05:52:40.325689+0000","last_clean":"2023-10-09T05:52:40.325689+0000","last_became_active":"2023-10-09T05:50:08.045912+0000","last_became_peered":"2023-10-09T05:50:08.045912+0000","last_unstale":"2023-10-09T05:52:40.325689+0000","last_undegraded":"2023-10-09T05:52:40.325689+0000","last_fullsized":"2023-10-09T05:52:40.325689+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7704,"log_dups_size":0,"ondisk_log_size":7704,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4571136,"num_objects":69,"num_object_clones":0,"num_object_copies":207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3157,"num_write_kb":103280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":141,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.177844+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:52:17.177844+0000","last_peered":"2023-10-09T05:52:17.177844+0000","last_clean":"2023-10-09T05:52:17.177844+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:52:17.177844+0000","last_undegraded":"2023-10-09T05:52:17.177844+0000","last_fullsized":"2023-10-09T05:52:17.177844+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":18,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.173991+0000","last_change":"2023-10-09T05:51:43.075609+0000","last_active":"2023-10-09T05:52:17.173991+0000","last_peered":"2023-10-09T05:52:17.173991+0000","last_clean":"2023-10-09T05:52:17.173991+0000","last_became_active":"2023-10-09T05:51:43.075280+0000","last_became_peered":"2023-10-09T05:51:43.075280+0000","last_unstale":"2023-10-09T05:52:17.173991+0000","last_undegraded":"2023-10-09T05:52:17.173991+0000","last_fullsized":"2023-10-09T05:52:17.173991+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"358'7725","reported_seq":7822,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.405659+0000","last_change":"2023-10-09T05:50:08.046359+0000","last_active":"2023-10-09T05:52:40.405659+0000","last_peered":"2023-10-09T05:52:40.405659+0000","last_clean":"2023-10-09T05:52:40.405659+0000","last_became_active":"2023-10-09T05:50:08.041168+0000","last_became_peered":"2023-10-09T05:50:08.041168+0000","last_unstale":"2023-10-09T05:52:40.405659+0000","last_undegraded":"2023-10-09T05:52:40.405659+0000","last_fullsized":"2023-10-09T05:52:40.405659+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7725,"log_dups_size":0,"ondisk_log_size":7725,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1556480,"num_objects":23,"num_object_clones":0,"num_object_copies":69,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3195,"num_write_kb":103024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":167,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.177820+0000","last_change":"2023-10-09T05:48:54.539425+0000","last_active":"2023-10-09T05:52:17.177820+0000","last_peered":"2023-10-09T05:52:17.177820+0000","last_clean":"2023-10-09T05:52:17.177820+0000","last_became_active":"2023-10-09T05:48:54.486401+0000","last_became_peered":"2023-10-09T05:48:54.486401+0000","last_unstale":"2023-10-09T05:52:17.177820+0000","last_undegraded":"2023-10-09T05:52:17.177820+0000","last_fullsized":"2023-10-09T05:52:17.177820+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":374,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180989+0000","last_change":"2023-10-09T05:48:54.492534+0000","last_active":"2023-10-09T05:52:17.180989+0000","last_peered":"2023-10-09T05:52:17.180989+0000","last_clean":"2023-10-09T05:52:17.180989+0000","last_became_active":"2023-10-09T05:48:54.492172+0000","last_became_peered":"2023-10-09T05:48:54.492172+0000","last_unstale":"2023-10-09T05:52:17.180989+0000","last_undegraded":"2023-10-09T05:52:17.180989+0000","last_fullsized":"2023-10-09T05:52:17.180989+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"358'7564","reported_seq":7668,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:39.337846+0000","last_change":"2023-10-09T05:50:08.052758+0000","last_active":"2023-10-09T05:52:39.337846+0000","last_peered":"2023-10-09T05:52:39.337846+0000","last_clean":"2023-10-09T05:52:39.337846+0000","last_became_active":"2023-10-09T05:50:08.049618+0000","last_became_peered":"2023-10-09T05:50:08.049618+0000","last_unstale":"2023-10-09T05:52:39.337846+0000","last_undegraded":"2023-10-09T05:52:39.337846+0000","last_fullsized":"2023-10-09T05:52:39.337846+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7564,"log_dups_size":0,"ondisk_log_size":7564,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6701056,"num_objects":102,"num_object_clones":0,"num_object_copies":306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3072,"num_write_kb":101584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":167,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181881+0000","last_change":"2023-10-09T05:51:43.265312+0000","last_active":"2023-10-09T05:52:17.181881+0000","last_peered":"2023-10-09T05:52:17.181881+0000","last_clean":"2023-10-09T05:52:17.181881+0000","last_became_active":"2023-10-09T05:51:43.076237+0000","last_became_peered":"2023-10-09T05:51:43.076237+0000","last_unstale":"2023-10-09T05:52:17.181881+0000","last_undegraded":"2023-10-09T05:52:17.181881+0000","last_fullsized":"2023-10-09T05:52:17.181881+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":335,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.176516+0000","last_change":"2023-10-09T05:50:18.039975+0000","last_active":"2023-10-09T05:52:17.176516+0000","last_peered":"2023-10-09T05:52:17.176516+0000","last_clean":"2023-10-09T05:52:17.176516+0000","last_became_active":"2023-10-09T05:50:18.039332+0000","last_became_peered":"2023-10-09T05:50:18.039332+0000","last_unstale":"2023-10-09T05:52:17.176516+0000","last_undegraded":"2023-10-09T05:52:17.176516+0000","last_fullsized":"2023-10-09T05:52:17.176516+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.7","version":"358'7841","reported_seq":10455,"reported_epoch":358,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:41.913486+0000","last_change":"2023-10-09T05:50:09.462704+0000","last_active":"2023-10-09T05:52:41.913486+0000","last_peered":"2023-10-09T05:52:41.913486+0000","last_clean":"2023-10-09T05:48:53.410221+0000","last_became_active":"2023-10-09T05:50:09.449813+0000","last_became_peered":"2023-10-09T05:50:09.449813+0000","last_unstale":"2023-10-09T05:52:41.913486+0000","last_undegraded":"2023-10-09T05:50:09.140916+0000","last_fullsized":"2023-10-09T05:52:41.913486+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7841,"log_dups_size":0,"ondisk_log_size":7841,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2359296,"num_objects":36,"num_object_clones":0,"num_object_copies":108,"num_objects_missing_on_primary":232,"num_objects_missing":232,"num_objects_degraded":232,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3209,"num_write_kb":103840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1252,"num_bytes_recovered":82051072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3,6","objects":36}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":325,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.175341+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:52:17.175341+0000","last_peered":"2023-10-09T05:52:17.175341+0000","last_clean":"2023-10-09T05:52:17.175341+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:52:17.175341+0000","last_undegraded":"2023-10-09T05:52:17.175341+0000","last_fullsized":"2023-10-09T05:52:17.175341+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":333,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:29.110245+0000","last_change":"2023-10-09T05:48:54.495800+0000","last_active":"2023-10-09T05:52:29.110245+0000","last_peered":"2023-10-09T05:52:29.110245+0000","last_clean":"2023-10-09T05:52:29.110245+0000","last_became_active":"2023-10-09T05:48:54.495540+0000","last_became_peered":"2023-10-09T05:48:54.495540+0000","last_unstale":"2023-10-09T05:52:29.110245+0000","last_undegraded":"2023-10-09T05:52:29.110245+0000","last_fullsized":"2023-10-09T05:52:29.110245+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:26.903669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"358'7682","reported_seq":11169,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:42.237911+0000","last_change":"2023-10-09T05:51:43.197387+0000","last_active":"2023-10-09T05:52:42.237911+0000","last_peered":"2023-10-09T05:52:42.237911+0000","last_clean":"2023-10-09T05:48:53.429181+0000","last_became_active":"2023-10-09T05:51:43.181186+0000","last_became_peered":"2023-10-09T05:51:43.181186+0000","last_unstale":"2023-10-09T05:52:42.237911+0000","last_undegraded":"2023-10-09T05:51:43.114355+0000","last_fullsized":"2023-10-09T05:51:43.055685+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7682,"log_dups_size":0,"ondisk_log_size":7682,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1187,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3258,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1416,"num_bytes_recovered":92798976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[3,1],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3","objects":-177},{"shards":"1,3,4","objects":21}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":95,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180008+0000","last_change":"2023-10-09T05:48:25.067945+0000","last_active":"2023-10-09T05:52:17.180008+0000","last_peered":"2023-10-09T05:52:17.180008+0000","last_clean":"2023-10-09T05:52:17.180008+0000","last_became_active":"2023-10-09T05:48:25.067721+0000","last_became_peered":"2023-10-09T05:48:25.067721+0000","last_unstale":"2023-10-09T05:52:17.180008+0000","last_undegraded":"2023-10-09T05:52:17.180008+0000","last_fullsized":"2023-10-09T05:52:17.180008+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:58.655544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":323,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181854+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:52:17.181854+0000","last_peered":"2023-10-09T05:52:17.181854+0000","last_clean":"2023-10-09T05:52:17.181854+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:52:17.181854+0000","last_undegraded":"2023-10-09T05:52:17.181854+0000","last_fullsized":"2023-10-09T05:52:17.181854+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"358'7509","reported_seq":10518,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:39.343655+0000","last_change":"2023-10-09T05:51:07.166631+0000","last_active":"2023-10-09T05:52:39.343655+0000","last_peered":"2023-10-09T05:52:39.343655+0000","last_clean":"2023-10-09T05:52:39.343655+0000","last_became_active":"2023-10-09T05:51:07.160634+0000","last_became_peered":"2023-10-09T05:51:07.160634+0000","last_unstale":"2023-10-09T05:52:39.343655+0000","last_undegraded":"2023-10-09T05:52:39.343655+0000","last_fullsized":"2023-10-09T05:52:39.343655+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7509,"log_dups_size":0,"ondisk_log_size":7509,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3997696,"num_objects":61,"num_object_clones":0,"num_object_copies":183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3142,"num_write_kb":102496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":90636288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":192,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181096+0000","last_change":"2023-10-09T05:48:54.494715+0000","last_active":"2023-10-09T05:52:17.181096+0000","last_peered":"2023-10-09T05:52:17.181096+0000","last_clean":"2023-10-09T05:52:17.181096+0000","last_became_active":"2023-10-09T05:48:54.493988+0000","last_became_peered":"2023-10-09T05:48:54.493988+0000","last_unstale":"2023-10-09T05:52:17.181096+0000","last_undegraded":"2023-10-09T05:52:17.181096+0000","last_fullsized":"2023-10-09T05:52:17.181096+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":356,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180487+0000","last_change":"2023-10-09T05:48:54.493193+0000","last_active":"2023-10-09T05:52:17.180487+0000","last_peered":"2023-10-09T05:52:17.180487+0000","last_clean":"2023-10-09T05:52:17.180487+0000","last_became_active":"2023-10-09T05:48:54.492886+0000","last_became_peered":"2023-10-09T05:48:54.492886+0000","last_unstale":"2023-10-09T05:52:17.180487+0000","last_undegraded":"2023-10-09T05:52:17.180487+0000","last_fullsized":"2023-10-09T05:52:17.180487+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"358'7455","reported_seq":7556,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.455334+0000","last_change":"2023-10-09T05:49:20.137476+0000","last_active":"2023-10-09T05:52:40.455334+0000","last_peered":"2023-10-09T05:52:40.455334+0000","last_clean":"2023-10-09T05:52:40.455334+0000","last_became_active":"2023-10-09T05:49:20.132759+0000","last_became_peered":"2023-10-09T05:49:20.132759+0000","last_unstale":"2023-10-09T05:52:40.455334+0000","last_undegraded":"2023-10-09T05:52:40.455334+0000","last_fullsized":"2023-10-09T05:52:40.455334+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7455,"log_dups_size":0,"ondisk_log_size":7455,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2637824,"num_objects":40,"num_object_clones":0,"num_object_copies":120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3133,"num_write_kb":101552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":357,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180765+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:52:17.180765+0000","last_peered":"2023-10-09T05:52:17.180765+0000","last_clean":"2023-10-09T05:52:17.180765+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:52:17.180765+0000","last_undegraded":"2023-10-09T05:52:17.180765+0000","last_fullsized":"2023-10-09T05:52:17.180765+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":350,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.182300+0000","last_change":"2023-10-09T05:48:54.492708+0000","last_active":"2023-10-09T05:52:17.182300+0000","last_peered":"2023-10-09T05:52:17.182300+0000","last_clean":"2023-10-09T05:52:17.182300+0000","last_became_active":"2023-10-09T05:48:54.492382+0000","last_became_peered":"2023-10-09T05:48:54.492382+0000","last_unstale":"2023-10-09T05:52:17.182300+0000","last_undegraded":"2023-10-09T05:52:17.182300+0000","last_fullsized":"2023-10-09T05:52:17.182300+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"358'7529","reported_seq":10413,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:40.588380+0000","last_change":"2023-10-09T05:50:19.097528+0000","last_active":"2023-10-09T05:52:40.588380+0000","last_peered":"2023-10-09T05:52:40.588380+0000","last_clean":"2023-10-09T05:48:53.427903+0000","last_became_active":"2023-10-09T05:50:19.074460+0000","last_became_peered":"2023-10-09T05:50:19.074460+0000","last_unstale":"2023-10-09T05:52:40.588380+0000","last_undegraded":"2023-10-09T05:50:19.039951+0000","last_fullsized":"2023-10-09T05:50:19.038108+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7529,"log_dups_size":0,"ondisk_log_size":7529,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2260992,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":61,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3172,"num_write_kb":102624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1387,"num_bytes_recovered":90898432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"1,5","objects":34}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":323,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.176564+0000","last_change":"2023-10-09T05:50:18.048706+0000","last_active":"2023-10-09T05:52:17.176564+0000","last_peered":"2023-10-09T05:52:17.176564+0000","last_clean":"2023-10-09T05:52:17.176564+0000","last_became_active":"2023-10-09T05:50:18.048410+0000","last_became_peered":"2023-10-09T05:50:18.048410+0000","last_unstale":"2023-10-09T05:52:17.176564+0000","last_undegraded":"2023-10-09T05:52:17.176564+0000","last_fullsized":"2023-10-09T05:52:17.176564+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":102,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181076+0000","last_change":"2023-10-09T05:50:18.072866+0000","last_active":"2023-10-09T05:52:17.181076+0000","last_peered":"2023-10-09T05:52:17.181076+0000","last_clean":"2023-10-09T05:52:17.181076+0000","last_became_active":"2023-10-09T05:50:18.072550+0000","last_became_peered":"2023-10-09T05:50:18.072550+0000","last_unstale":"2023-10-09T05:52:17.181076+0000","last_undegraded":"2023-10-09T05:52:17.181076+0000","last_fullsized":"2023-10-09T05:52:17.181076+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.e","version":"358'9734","reported_seq":9827,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:39.551308+0000","last_change":"2023-10-09T05:49:20.163537+0000","last_active":"2023-10-09T05:52:39.551308+0000","last_peered":"2023-10-09T05:52:39.551308+0000","last_clean":"2023-10-09T05:52:39.551308+0000","last_became_active":"2023-10-09T05:49:20.159521+0000","last_became_peered":"2023-10-09T05:49:20.159521+0000","last_unstale":"2023-10-09T05:52:39.551308+0000","last_undegraded":"2023-10-09T05:52:39.551308+0000","last_fullsized":"2023-10-09T05:52:39.551308+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9734,"log_dups_size":0,"ondisk_log_size":9734,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4980736,"num_objects":76,"num_object_clones":0,"num_object_copies":228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6387,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":748,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180909+0000","last_change":"2023-10-09T05:50:18.055235+0000","last_active":"2023-10-09T05:52:17.180909+0000","last_peered":"2023-10-09T05:52:17.180909+0000","last_clean":"2023-10-09T05:52:17.180909+0000","last_became_active":"2023-10-09T05:50:18.054656+0000","last_became_peered":"2023-10-09T05:50:18.054656+0000","last_unstale":"2023-10-09T05:52:17.180909+0000","last_undegraded":"2023-10-09T05:52:17.180909+0000","last_fullsized":"2023-10-09T05:52:17.180909+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:33.760262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":340,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.176681+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:52:17.176681+0000","last_peered":"2023-10-09T05:52:17.176681+0000","last_clean":"2023-10-09T05:52:17.176681+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:52:17.176681+0000","last_undegraded":"2023-10-09T05:52:17.176681+0000","last_fullsized":"2023-10-09T05:52:17.176681+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":18,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.173722+0000","last_change":"2023-10-09T05:51:43.248409+0000","last_active":"2023-10-09T05:52:17.173722+0000","last_peered":"2023-10-09T05:52:17.173722+0000","last_clean":"2023-10-09T05:52:17.173722+0000","last_became_active":"2023-10-09T05:51:43.154898+0000","last_became_peered":"2023-10-09T05:51:43.154898+0000","last_unstale":"2023-10-09T05:52:17.173722+0000","last_undegraded":"2023-10-09T05:52:17.173722+0000","last_fullsized":"2023-10-09T05:52:17.173722+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"358'7573","reported_seq":7672,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.556408+0000","last_change":"2023-10-09T05:50:08.053391+0000","last_active":"2023-10-09T05:52:40.556408+0000","last_peered":"2023-10-09T05:52:40.556408+0000","last_clean":"2023-10-09T05:52:40.556408+0000","last_became_active":"2023-10-09T05:50:08.050378+0000","last_became_peered":"2023-10-09T05:50:08.050378+0000","last_unstale":"2023-10-09T05:52:40.556408+0000","last_undegraded":"2023-10-09T05:52:40.556408+0000","last_fullsized":"2023-10-09T05:52:40.556408+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7573,"log_dups_size":0,"ondisk_log_size":7573,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4096000,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3094,"num_write_kb":101024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":848,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.175355+0000","last_change":"2023-10-09T05:48:54.592642+0000","last_active":"2023-10-09T05:52:17.175355+0000","last_peered":"2023-10-09T05:52:17.175355+0000","last_clean":"2023-10-09T05:52:17.175355+0000","last_became_active":"2023-10-09T05:48:54.491917+0000","last_became_peered":"2023-10-09T05:48:54.491917+0000","last_unstale":"2023-10-09T05:52:17.175355+0000","last_undegraded":"2023-10-09T05:52:17.175355+0000","last_fullsized":"2023-10-09T05:52:17.175355+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":324,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181695+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:52:17.181695+0000","last_peered":"2023-10-09T05:52:17.181695+0000","last_clean":"2023-10-09T05:52:17.181695+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:52:17.181695+0000","last_undegraded":"2023-10-09T05:52:17.181695+0000","last_fullsized":"2023-10-09T05:52:17.181695+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":180,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:27.404202+0000","last_change":"2023-10-09T05:50:18.043775+0000","last_active":"2023-10-09T05:52:27.404202+0000","last_peered":"2023-10-09T05:52:27.404202+0000","last_clean":"2023-10-09T05:52:27.404202+0000","last_became_active":"2023-10-09T05:50:18.043047+0000","last_became_peered":"2023-10-09T05:50:18.043047+0000","last_unstale":"2023-10-09T05:52:27.404202+0000","last_undegraded":"2023-10-09T05:52:27.404202+0000","last_fullsized":"2023-10-09T05:52:27.404202+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:28.456979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"358'8474","reported_seq":11747,"reported_epoch":358,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:40.230082+0000","last_change":"2023-10-09T05:50:08.085412+0000","last_active":"2023-10-09T05:52:40.230082+0000","last_peered":"2023-10-09T05:52:40.230082+0000","last_clean":"2023-10-09T05:48:53.462109+0000","last_became_active":"2023-10-09T05:50:08.080753+0000","last_became_peered":"2023-10-09T05:50:08.080753+0000","last_unstale":"2023-10-09T05:52:40.230082+0000","last_undegraded":"2023-10-09T05:50:08.030939+0000","last_fullsized":"2023-10-09T05:52:40.230082+0000","mapping_epoch":312,"log_start":"289'200","ondisk_log_start":"289'200","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8274,"log_dups_size":0,"ondisk_log_size":8274,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1916928,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":74,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3227,"num_write_kb":104224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1410,"num_bytes_recovered":92438528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"1,3,4","objects":29}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":678,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.171090+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:52:17.171090+0000","last_peered":"2023-10-09T05:52:17.171090+0000","last_clean":"2023-10-09T05:52:17.171090+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:52:17.171090+0000","last_undegraded":"2023-10-09T05:52:17.171090+0000","last_fullsized":"2023-10-09T05:52:17.171090+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":139,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:22.403662+0000","last_change":"2023-10-09T05:50:18.063466+0000","last_active":"2023-10-09T05:52:22.403662+0000","last_peered":"2023-10-09T05:52:22.403662+0000","last_clean":"2023-10-09T05:52:22.403662+0000","last_became_active":"2023-10-09T05:50:18.063120+0000","last_became_peered":"2023-10-09T05:50:18.063120+0000","last_unstale":"2023-10-09T05:52:22.403662+0000","last_undegraded":"2023-10-09T05:52:22.403662+0000","last_fullsized":"2023-10-09T05:52:22.403662+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:13.683314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":162,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.174532+0000","last_change":"2023-10-09T05:50:18.069402+0000","last_active":"2023-10-09T05:52:17.174532+0000","last_peered":"2023-10-09T05:52:17.174532+0000","last_clean":"2023-10-09T05:52:17.174532+0000","last_became_active":"2023-10-09T05:50:18.069196+0000","last_became_peered":"2023-10-09T05:50:18.069196+0000","last_unstale":"2023-10-09T05:52:17.174532+0000","last_undegraded":"2023-10-09T05:52:17.174532+0000","last_fullsized":"2023-10-09T05:52:17.174532+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"358'7802","reported_seq":7913,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:39.608340+0000","last_change":"2023-10-09T05:51:44.323830+0000","last_active":"2023-10-09T05:52:39.608340+0000","last_peered":"2023-10-09T05:52:39.608340+0000","last_clean":"2023-10-09T05:51:42.071320+0000","last_became_active":"2023-10-09T05:51:44.311440+0000","last_became_peered":"2023-10-09T05:51:44.311440+0000","last_unstale":"2023-10-09T05:52:39.608340+0000","last_undegraded":"2023-10-09T05:51:44.114363+0000","last_fullsized":"2023-10-09T05:51:44.105936+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7802,"log_dups_size":0,"ondisk_log_size":7802,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5144576,"num_objects":78,"num_object_clones":0,"num_object_copies":234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1649,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":78,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3187,"num_write_kb":104512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[0,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":78},{"shards":"0,6","objects":-639}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":1004,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.174805+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:52:17.174805+0000","last_peered":"2023-10-09T05:52:17.174805+0000","last_clean":"2023-10-09T05:52:17.174805+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:52:17.174805+0000","last_undegraded":"2023-10-09T05:52:17.174805+0000","last_fullsized":"2023-10-09T05:52:17.174805+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":508,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.176385+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:52:17.176385+0000","last_peered":"2023-10-09T05:52:17.176385+0000","last_clean":"2023-10-09T05:52:17.176385+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:52:17.176385+0000","last_undegraded":"2023-10-09T05:52:17.176385+0000","last_fullsized":"2023-10-09T05:52:17.176385+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":130,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.175095+0000","last_change":"2023-10-09T05:48:25.068759+0000","last_active":"2023-10-09T05:52:17.175095+0000","last_peered":"2023-10-09T05:52:17.175095+0000","last_clean":"2023-10-09T05:52:17.175095+0000","last_became_active":"2023-10-09T05:48:25.068358+0000","last_became_peered":"2023-10-09T05:48:25.068358+0000","last_unstale":"2023-10-09T05:52:17.175095+0000","last_undegraded":"2023-10-09T05:52:17.175095+0000","last_fullsized":"2023-10-09T05:52:17.175095+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":325,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180784+0000","last_change":"2023-10-09T05:44:50.495966+0000","last_active":"2023-10-09T05:52:17.180784+0000","last_peered":"2023-10-09T05:52:17.180784+0000","last_clean":"2023-10-09T05:52:17.180784+0000","last_became_active":"2023-10-09T05:44:50.495628+0000","last_became_peered":"2023-10-09T05:44:50.495628+0000","last_unstale":"2023-10-09T05:52:17.180784+0000","last_undegraded":"2023-10-09T05:52:17.180784+0000","last_fullsized":"2023-10-09T05:52:17.180784+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"358'8314","reported_seq":12010,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:42.308980+0000","last_change":"2023-10-09T05:51:43.298014+0000","last_active":"2023-10-09T05:52:42.308980+0000","last_peered":"2023-10-09T05:52:42.308980+0000","last_clean":"2023-10-09T05:48:53.473854+0000","last_became_active":"2023-10-09T05:51:43.270086+0000","last_became_peered":"2023-10-09T05:51:43.270086+0000","last_unstale":"2023-10-09T05:52:42.308980+0000","last_undegraded":"2023-10-09T05:51:43.150850+0000","last_fullsized":"2023-10-09T05:51:43.116350+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8314,"log_dups_size":0,"ondisk_log_size":8314,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1474560,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1138,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3179,"num_write_kb":102448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1373,"num_bytes_recovered":89948160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2","objects":-148},{"shards":"1,2,3","objects":22}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":880,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.176473+0000","last_change":"2023-10-09T05:49:36.046279+0000","last_active":"2023-10-09T05:52:17.176473+0000","last_peered":"2023-10-09T05:52:17.176473+0000","last_clean":"2023-10-09T05:52:17.176473+0000","last_became_active":"2023-10-09T05:49:36.045828+0000","last_became_peered":"2023-10-09T05:49:36.045828+0000","last_unstale":"2023-10-09T05:52:17.176473+0000","last_undegraded":"2023-10-09T05:52:17.176473+0000","last_fullsized":"2023-10-09T05:52:17.176473+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":351,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.185083+0000","last_change":"2023-10-09T05:51:31.682830+0000","last_active":"2023-10-09T05:52:17.185083+0000","last_peered":"2023-10-09T05:52:17.185083+0000","last_clean":"2023-10-09T05:52:17.185083+0000","last_became_active":"2023-10-09T05:51:31.682566+0000","last_became_peered":"2023-10-09T05:51:31.682566+0000","last_unstale":"2023-10-09T05:52:17.185083+0000","last_undegraded":"2023-10-09T05:52:17.185083+0000","last_fullsized":"2023-10-09T05:52:17.185083+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:42.916316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":348,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181316+0000","last_change":"2023-10-09T05:48:54.489115+0000","last_active":"2023-10-09T05:52:17.181316+0000","last_peered":"2023-10-09T05:52:17.181316+0000","last_clean":"2023-10-09T05:52:17.181316+0000","last_became_active":"2023-10-09T05:48:54.488783+0000","last_became_peered":"2023-10-09T05:48:54.488783+0000","last_unstale":"2023-10-09T05:52:17.181316+0000","last_undegraded":"2023-10-09T05:52:17.181316+0000","last_fullsized":"2023-10-09T05:52:17.181316+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:13.193778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"358'9547","reported_seq":16621,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:42.300744+0000","last_change":"2023-10-09T05:51:13.190036+0000","last_active":"2023-10-09T05:52:42.300744+0000","last_peered":"2023-10-09T05:52:42.300744+0000","last_clean":"2023-10-09T05:48:53.467734+0000","last_became_active":"2023-10-09T05:51:13.176990+0000","last_became_peered":"2023-10-09T05:51:13.176990+0000","last_unstale":"2023-10-09T05:52:42.300744+0000","last_undegraded":"2023-10-09T05:51:13.120625+0000","last_fullsized":"2023-10-09T05:51:13.052350+0000","mapping_epoch":339,"log_start":"290'2222","ondisk_log_start":"290'2222","created":285,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":7325,"log_dups_size":0,"ondisk_log_size":7325,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3768320,"num_objects":57,"num_object_clones":0,"num_object_copies":171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1703,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6263,"num_write_kb":202272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2683,"num_bytes_recovered":175767552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[3,1],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"1,3","objects":-294},{"shards":"1,3,4","objects":58}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":541,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.182119+0000","last_change":"2023-10-09T05:44:51.489799+0000","last_active":"2023-10-09T05:52:17.182119+0000","last_peered":"2023-10-09T05:52:17.182119+0000","last_clean":"2023-10-09T05:52:17.182119+0000","last_became_active":"2023-10-09T05:44:51.489301+0000","last_became_peered":"2023-10-09T05:44:51.489301+0000","last_unstale":"2023-10-09T05:52:17.182119+0000","last_undegraded":"2023-10-09T05:52:17.182119+0000","last_fullsized":"2023-10-09T05:52:17.182119+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":157,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:27.404219+0000","last_change":"2023-10-09T05:51:43.127835+0000","last_active":"2023-10-09T05:52:27.404219+0000","last_peered":"2023-10-09T05:52:27.404219+0000","last_clean":"2023-10-09T05:52:27.404219+0000","last_became_active":"2023-10-09T05:51:43.127489+0000","last_became_peered":"2023-10-09T05:51:43.127489+0000","last_unstale":"2023-10-09T05:52:27.404219+0000","last_undegraded":"2023-10-09T05:52:27.404219+0000","last_fullsized":"2023-10-09T05:52:27.404219+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:20.191774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":333,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180192+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:52:17.180192+0000","last_peered":"2023-10-09T05:52:17.180192+0000","last_clean":"2023-10-09T05:52:17.180192+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:52:17.180192+0000","last_undegraded":"2023-10-09T05:52:17.180192+0000","last_fullsized":"2023-10-09T05:52:17.180192+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"358'8554","reported_seq":8661,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:39.538563+0000","last_change":"2023-10-09T05:52:37.860173+0000","last_active":"2023-10-09T05:52:39.538563+0000","last_peered":"2023-10-09T05:52:39.538563+0000","last_clean":"2023-10-09T05:52:39.538563+0000","last_became_active":"2023-10-09T05:50:08.040134+0000","last_became_peered":"2023-10-09T05:50:08.040134+0000","last_unstale":"2023-10-09T05:52:39.538563+0000","last_undegraded":"2023-10-09T05:52:39.538563+0000","last_fullsized":"2023-10-09T05:52:39.538563+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8554,"log_dups_size":0,"ondisk_log_size":8554,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3047424,"num_objects":46,"num_object_clones":0,"num_object_copies":138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3244,"num_write_kb":105312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":305,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.178064+0000","last_change":"2023-10-09T05:48:25.066163+0000","last_active":"2023-10-09T05:52:17.178064+0000","last_peered":"2023-10-09T05:52:17.178064+0000","last_clean":"2023-10-09T05:52:17.178064+0000","last_became_active":"2023-10-09T05:48:25.065145+0000","last_became_peered":"2023-10-09T05:48:25.065145+0000","last_unstale":"2023-10-09T05:52:17.178064+0000","last_undegraded":"2023-10-09T05:52:17.178064+0000","last_fullsized":"2023-10-09T05:52:17.178064+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":588,"reported_epoch":357,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:52:17.181397+0000","last_change":"2023-10-09T05:50:19.055196+0000","last_active":"2023-10-09T05:52:17.181397+0000","last_peered":"2023-10-09T05:52:17.181397+0000","last_clean":"2023-10-09T05:48:52.524136+0000","last_became_active":"2023-10-09T05:50:19.054850+0000","last_became_peered":"2023-10-09T05:50:19.054850+0000","last_unstale":"2023-10-09T05:52:17.181397+0000","last_undegraded":"2023-10-09T05:50:19.031371+0000","last_fullsized":"2023-10-09T05:52:17.181397+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":["4","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":298,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.176657+0000","last_change":"2023-10-09T05:48:25.067326+0000","last_active":"2023-10-09T05:52:17.176657+0000","last_peered":"2023-10-09T05:52:17.176657+0000","last_clean":"2023-10-09T05:52:17.176657+0000","last_became_active":"2023-10-09T05:48:25.066977+0000","last_became_peered":"2023-10-09T05:48:25.066977+0000","last_unstale":"2023-10-09T05:52:17.176657+0000","last_undegraded":"2023-10-09T05:52:17.176657+0000","last_fullsized":"2023-10-09T05:52:17.176657+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"358'8217","reported_seq":8322,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.338877+0000","last_change":"2023-10-09T05:50:17.989037+0000","last_active":"2023-10-09T05:52:40.338877+0000","last_peered":"2023-10-09T05:52:40.338877+0000","last_clean":"2023-10-09T05:52:40.338877+0000","last_became_active":"2023-10-09T05:50:08.041685+0000","last_became_peered":"2023-10-09T05:50:08.041685+0000","last_unstale":"2023-10-09T05:52:40.338877+0000","last_undegraded":"2023-10-09T05:52:40.338877+0000","last_fullsized":"2023-10-09T05:52:40.338877+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8217,"log_dups_size":0,"ondisk_log_size":8217,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6275078,"num_objects":96,"num_object_clones":0,"num_object_copies":288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":96,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3085,"num_write_kb":101776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":855,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.177991+0000","last_change":"2023-10-09T05:44:50.482211+0000","last_active":"2023-10-09T05:52:17.177991+0000","last_peered":"2023-10-09T05:52:17.177991+0000","last_clean":"2023-10-09T05:52:17.177991+0000","last_became_active":"2023-10-09T05:44:50.481749+0000","last_became_peered":"2023-10-09T05:44:50.481749+0000","last_unstale":"2023-10-09T05:52:17.177991+0000","last_undegraded":"2023-10-09T05:52:17.177991+0000","last_fullsized":"2023-10-09T05:52:17.177991+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":344,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181466+0000","last_change":"2023-10-09T05:48:54.585691+0000","last_active":"2023-10-09T05:52:17.181466+0000","last_peered":"2023-10-09T05:52:17.181466+0000","last_clean":"2023-10-09T05:52:17.181466+0000","last_became_active":"2023-10-09T05:48:54.498259+0000","last_became_peered":"2023-10-09T05:48:54.498259+0000","last_unstale":"2023-10-09T05:52:17.181466+0000","last_undegraded":"2023-10-09T05:52:17.181466+0000","last_fullsized":"2023-10-09T05:52:17.181466+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.684612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":308,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180841+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:52:17.180841+0000","last_peered":"2023-10-09T05:52:17.180841+0000","last_clean":"2023-10-09T05:52:17.180841+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:52:17.180841+0000","last_undegraded":"2023-10-09T05:52:17.180841+0000","last_fullsized":"2023-10-09T05:52:17.180841+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"358'9148","reported_seq":17954,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.199676+0000","last_change":"2023-10-09T05:52:13.371424+0000","last_active":"2023-10-09T05:52:40.199676+0000","last_peered":"2023-10-09T05:52:40.199676+0000","last_clean":"2023-10-09T05:52:40.199676+0000","last_became_active":"2023-10-09T05:49:35.024339+0000","last_became_peered":"2023-10-09T05:49:35.024339+0000","last_unstale":"2023-10-09T05:52:40.199676+0000","last_undegraded":"2023-10-09T05:52:40.199676+0000","last_fullsized":"2023-10-09T05:52:40.199676+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":9148,"log_dups_size":0,"ondisk_log_size":9148,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6225920,"num_objects":95,"num_object_clones":0,"num_object_copies":285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":95,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6411,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2892,"num_bytes_recovered":189530112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":335,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:29.110441+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:52:29.110441+0000","last_peered":"2023-10-09T05:52:29.110441+0000","last_clean":"2023-10-09T05:52:29.110441+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:52:29.110441+0000","last_undegraded":"2023-10-09T05:52:29.110441+0000","last_fullsized":"2023-10-09T05:52:29.110441+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":29,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.170685+0000","last_change":"2023-10-09T05:51:13.060067+0000","last_active":"2023-10-09T05:52:17.170685+0000","last_peered":"2023-10-09T05:52:17.170685+0000","last_clean":"2023-10-09T05:52:17.170685+0000","last_became_active":"2023-10-09T05:51:13.059725+0000","last_became_peered":"2023-10-09T05:51:13.059725+0000","last_unstale":"2023-10-09T05:52:17.170685+0000","last_undegraded":"2023-10-09T05:52:17.170685+0000","last_fullsized":"2023-10-09T05:52:17.170685+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:43.328688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"358'9305","reported_seq":9867,"reported_epoch":358,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:42.363434+0000","last_change":"2023-10-09T05:52:37.578223+0000","last_active":"2023-10-09T05:52:42.363434+0000","last_peered":"2023-10-09T05:52:42.363434+0000","last_clean":"2023-10-09T05:48:53.462142+0000","last_became_active":"2023-10-09T05:48:56.004223+0000","last_became_peered":"2023-10-09T05:48:56.004223+0000","last_unstale":"2023-10-09T05:52:42.363434+0000","last_undegraded":"2023-10-09T05:48:55.514193+0000","last_fullsized":"2023-10-09T05:48:55.479940+0000","mapping_epoch":306,"log_start":"291'2677","ondisk_log_start":"291'2677","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6628,"log_dups_size":0,"ondisk_log_size":6628,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3342336,"num_objects":51,"num_object_clones":0,"num_object_copies":153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2757,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6521,"num_write_kb":210304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":229,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,6","objects":-2706}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":353,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.175263+0000","last_change":"2023-10-09T05:48:54.583404+0000","last_active":"2023-10-09T05:52:17.175263+0000","last_peered":"2023-10-09T05:52:17.175263+0000","last_clean":"2023-10-09T05:52:17.175263+0000","last_became_active":"2023-10-09T05:48:54.492565+0000","last_became_peered":"2023-10-09T05:48:54.492565+0000","last_unstale":"2023-10-09T05:52:17.175263+0000","last_undegraded":"2023-10-09T05:52:17.175263+0000","last_fullsized":"2023-10-09T05:52:17.175263+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":356,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181778+0000","last_change":"2023-10-09T05:48:54.493079+0000","last_active":"2023-10-09T05:52:17.181778+0000","last_peered":"2023-10-09T05:52:17.181778+0000","last_clean":"2023-10-09T05:52:17.181778+0000","last_became_active":"2023-10-09T05:48:54.492013+0000","last_became_peered":"2023-10-09T05:48:54.492013+0000","last_unstale":"2023-10-09T05:52:17.181778+0000","last_undegraded":"2023-10-09T05:52:17.181778+0000","last_fullsized":"2023-10-09T05:52:17.181778+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"358'8957","reported_seq":9050,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:39.601287+0000","last_change":"2023-10-09T05:48:46.260747+0000","last_active":"2023-10-09T05:52:39.601287+0000","last_peered":"2023-10-09T05:52:39.601287+0000","last_clean":"2023-10-09T05:52:39.601287+0000","last_became_active":"2023-10-09T05:48:46.258900+0000","last_became_peered":"2023-10-09T05:48:46.258900+0000","last_unstale":"2023-10-09T05:52:39.601287+0000","last_undegraded":"2023-10-09T05:52:39.601287+0000","last_fullsized":"2023-10-09T05:52:39.601287+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8957,"log_dups_size":0,"ondisk_log_size":8957,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9273344,"num_objects":142,"num_object_clones":0,"num_object_copies":426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6270,"num_write_kb":205152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":312,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.176598+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:52:17.176598+0000","last_peered":"2023-10-09T05:52:17.176598+0000","last_clean":"2023-10-09T05:52:17.176598+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:52:17.176598+0000","last_undegraded":"2023-10-09T05:52:17.176598+0000","last_fullsized":"2023-10-09T05:52:17.176598+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":21,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180658+0000","last_change":"2023-10-09T05:51:31.668739+0000","last_active":"2023-10-09T05:52:17.180658+0000","last_peered":"2023-10-09T05:52:17.180658+0000","last_clean":"2023-10-09T05:52:17.180658+0000","last_became_active":"2023-10-09T05:51:31.668481+0000","last_became_peered":"2023-10-09T05:51:31.668481+0000","last_unstale":"2023-10-09T05:52:17.180658+0000","last_undegraded":"2023-10-09T05:52:17.180658+0000","last_fullsized":"2023-10-09T05:52:17.180658+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"358'8921","reported_seq":9013,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.401945+0000","last_change":"2023-10-09T05:48:46.308705+0000","last_active":"2023-10-09T05:52:40.401945+0000","last_peered":"2023-10-09T05:52:40.401945+0000","last_clean":"2023-10-09T05:52:40.401945+0000","last_became_active":"2023-10-09T05:48:46.258676+0000","last_became_peered":"2023-10-09T05:48:46.258676+0000","last_unstale":"2023-10-09T05:52:40.401945+0000","last_undegraded":"2023-10-09T05:52:40.401945+0000","last_fullsized":"2023-10-09T05:52:40.401945+0000","mapping_epoch":299,"log_start":"291'2281","ondisk_log_start":"291'2281","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":6640,"log_dups_size":0,"ondisk_log_size":6640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6389760,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6232,"num_write_kb":202528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":77,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:27.404229+0000","last_change":"2023-10-09T05:48:25.069406+0000","last_active":"2023-10-09T05:52:27.404229+0000","last_peered":"2023-10-09T05:52:27.404229+0000","last_clean":"2023-10-09T05:52:27.404229+0000","last_became_active":"2023-10-09T05:48:25.069064+0000","last_became_peered":"2023-10-09T05:48:25.069064+0000","last_unstale":"2023-10-09T05:52:27.404229+0000","last_undegraded":"2023-10-09T05:52:27.404229+0000","last_fullsized":"2023-10-09T05:52:27.404229+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.567858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":159,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.171013+0000","last_change":"2023-10-09T05:48:25.063854+0000","last_active":"2023-10-09T05:52:17.171013+0000","last_peered":"2023-10-09T05:52:17.171013+0000","last_clean":"2023-10-09T05:52:17.171013+0000","last_became_active":"2023-10-09T05:48:25.063579+0000","last_became_peered":"2023-10-09T05:48:25.063579+0000","last_unstale":"2023-10-09T05:52:17.171013+0000","last_undegraded":"2023-10-09T05:52:17.171013+0000","last_fullsized":"2023-10-09T05:52:17.171013+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:02.389789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.14","version":"358'9135","reported_seq":9227,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T05:52:40.345988+0000","last_change":"2023-10-09T05:48:46.325697+0000","last_active":"2023-10-09T05:52:40.345988+0000","last_peered":"2023-10-09T05:52:40.345988+0000","last_clean":"2023-10-09T05:52:40.345988+0000","last_became_active":"2023-10-09T05:48:46.268207+0000","last_became_peered":"2023-10-09T05:48:46.268207+0000","last_unstale":"2023-10-09T05:52:40.345988+0000","last_undegraded":"2023-10-09T05:52:40.345988+0000","last_fullsized":"2023-10-09T05:52:40.345988+0000","mapping_epoch":299,"log_start":"290'618","ondisk_log_start":"290'618","created":299,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:47:32.903924+0000","last_clean_scrub_stamp":"2023-10-09T05:47:32.903924+0000","objects_scrubbed":0,"log_size":8517,"log_dups_size":0,"ondisk_log_size":8517,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4816896,"num_objects":74,"num_object_clones":0,"num_object_copies":222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6408,"num_write_kb":207392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":348,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.180544+0000","last_change":"2023-10-09T05:51:13.071891+0000","last_active":"2023-10-09T05:52:17.180544+0000","last_peered":"2023-10-09T05:52:17.180544+0000","last_clean":"2023-10-09T05:52:17.180544+0000","last_became_active":"2023-10-09T05:51:13.058378+0000","last_became_peered":"2023-10-09T05:51:13.058378+0000","last_unstale":"2023-10-09T05:52:17.180544+0000","last_undegraded":"2023-10-09T05:52:17.180544+0000","last_fullsized":"2023-10-09T05:52:17.180544+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:41.278366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":341,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T05:52:17.181310+0000","last_change":"2023-10-09T05:48:54.492580+0000","last_active":"2023-10-09T05:52:17.181310+0000","last_peered":"2023-10-09T05:52:17.181310+0000","last_clean":"2023-10-09T05:52:17.181310+0000","last_became_active":"2023-10-09T05:48:54.492259+0000","last_became_peered":"2023-10-09T05:48:54.492259+0000","last_unstale":"2023-10-09T05:52:17.181310+0000","last_undegraded":"2023-10-09T05:52:17.181310+0000","last_fullsized":"2023-10-09T05:52:17.181310+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":10932,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":46,"stat_sum":{"num_bytes":222429208,"num_objects":3395,"num_object_clones":0,"num_object_copies":10185,"num_objects_missing_on_primary":451,"num_objects_missing":451,"num_objects_degraded":21432,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3395,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202497,"num_write_kb":6588097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35184,"num_bytes_recovered":2278293504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3912577024,"data_stored":3912564808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":371341,"ondisk_log_size":371341,"up":138,"acting":124,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":132,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846809,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1006432,"kb_used_data":88408,"kb_used_omap":14,"kb_used_meta":918001,"kb_avail":92738720,"statfs":{"total":95995035648,"available":94964449280,"internally_reserved":0,"allocated":90529792,"data_stored":89846341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":940033882},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":12.824999999999999,"5min":4.1820000000000004,"15min":4.3869999999999996},"min":{"1min":0.22800000000000001,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":105.39100000000001,"5min":105.39100000000001,"15min":220.77500000000001},"last":0.94899999999999995},{"interface":"front","average":{"1min":12.695,"5min":4.1100000000000003,"15min":4.359},"min":{"1min":0.29799999999999999,"5min":0.25900000000000001,"15min":0.22700000000000001},"max":{"1min":105.417,"5min":105.417,"15min":220.78700000000001},"last":0.97899999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":7.9139999999999997,"5min":2.363,"15min":3.1760000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":114.422,"5min":114.422,"15min":190.68100000000001},"last":1.99},{"interface":"front","average":{"1min":7.8090000000000002,"5min":2.5760000000000001,"15min":3.234},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":111.708,"5min":111.708,"15min":191.09200000000001},"last":2.8559999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":7.2279999999999998,"5min":2.6629999999999998,"15min":2.0590000000000002},"min":{"1min":0.374,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":100.684,"5min":100.684,"15min":100.684},"last":2.6400000000000001},{"interface":"front","average":{"1min":7.4290000000000003,"5min":2.4990000000000001,"15min":2.0129999999999999},"min":{"1min":0.32000000000000001,"5min":0.30499999999999999,"15min":0.28100000000000003},"max":{"1min":100.736,"5min":100.736,"15min":100.736},"last":6.0629999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":8.4090000000000007,"5min":4.3399999999999999,"15min":4.569},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":103.29300000000001,"5min":103.29300000000001,"15min":185.17099999999999},"last":1.909},{"interface":"front","average":{"1min":11.114000000000001,"5min":4.8150000000000004,"15min":4.6909999999999998},"min":{"1min":0.39700000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":103.202,"5min":103.202,"15min":183.35499999999999},"last":1.853}]},{"osd":4,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":9.8770000000000007,"5min":3.4350000000000001,"15min":1.5529999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.25700000000000001},"max":{"1min":159.50299999999999,"5min":159.50299999999999,"15min":159.50299999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":10.189,"5min":3.5230000000000001,"15min":1.611},"min":{"1min":0.32600000000000001,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":159.483,"5min":159.483,"15min":159.483},"last":0.443}]},{"osd":5,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.476,"15min":1.0329999999999999},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.26900000000000002},"max":{"1min":4.1319999999999997,"5min":49.210000000000001,"15min":49.210000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":2.246,"5min":1.7749999999999999,"15min":1.101},"min":{"1min":0.34999999999999998,"5min":0.308,"15min":0.28299999999999997},"max":{"1min":9.6739999999999995,"5min":49.274999999999999,"15min":49.274999999999999},"last":0.84499999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":0.90200000000000002,"15min":0.75600000000000001},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.1720000000000006,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.095,"15min":0.85499999999999998},"min":{"1min":0.313,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.8200000000000003,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.88600000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626835,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1537040,"kb_used_data":628808,"kb_used_omap":0,"kb_used_meta":908224,"kb_avail":92208112,"statfs":{"total":95995035648,"available":94421106688,"internally_reserved":0,"allocated":643899392,"data_stored":643231909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":930021376},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":2.2250000000000001,"15min":4.1349999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.20999999999999999},"max":{"1min":2.5379999999999998,"5min":42.869,"15min":162.08799999999999},"last":4.8300000000000001},{"interface":"front","average":{"1min":1.071,"5min":2.2639999999999998,"15min":4.0410000000000004},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.2169999999999996,"5min":42.892000000000003,"15min":161.90899999999999},"last":2.0600000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:52:10 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.9359999999999999,"15min":3.867},"min":{"1min":0.27300000000000002,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":9.3469999999999995,"5min":48.729999999999997,"15min":228.24299999999999},"last":4.1769999999999996},{"interface":"front","average":{"1min":2.274,"5min":2.1699999999999999,"15min":3.8340000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":13.843,"5min":48.741999999999997,"15min":228.22499999999999},"last":4.8090000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":1.014,"15min":3.3849999999999998},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":2.4369999999999998,"5min":6.1680000000000001,"15min":115.40600000000001},"last":4.8540000000000001},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.0960000000000001,"15min":3.5369999999999999},"min":{"1min":0.33600000000000002,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":3.794,"5min":6.1440000000000001,"15min":115.297},"last":4.2060000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:52:10 2023","interfaces":[{"interface":"back","average":{"1min":10.012,"5min":3.4159999999999999,"15min":4.4009999999999998},"min":{"1min":0.35999999999999999,"5min":0.32800000000000001,"15min":0.26800000000000002},"max":{"1min":145.124,"5min":145.124,"15min":145.124},"last":4.2560000000000002},{"interface":"front","average":{"1min":10.629,"5min":3.681,"15min":4.4610000000000003},"min":{"1min":0.37,"5min":0.33800000000000002,"15min":0.30299999999999999},"max":{"1min":145.26599999999999,"5min":145.26599999999999,"15min":145.26599999999999},"last":4.8739999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:52:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":3.9100000000000001,"15min":1.9430000000000001},"min":{"1min":0.28599999999999998,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":9.4979999999999993,"5min":257.375,"15min":257.375},"last":4.742},{"interface":"front","average":{"1min":1.4590000000000001,"5min":4.0060000000000002,"15min":2.1760000000000002},"min":{"1min":0.309,"5min":0.26100000000000001,"15min":0.246},"max":{"1min":5.5659999999999998,"5min":257.61200000000002,"15min":257.61200000000002},"last":2.0299999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:52:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":3.9100000000000001,"15min":2.2799999999999998},"min":{"1min":0.33100000000000002,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":6.7839999999999998,"5min":256.05700000000002,"15min":256.05700000000002},"last":4.7809999999999997},{"interface":"front","average":{"1min":1.7230000000000001,"5min":4.1059999999999999,"15min":2.3879999999999999},"min":{"1min":0.35799999999999998,"5min":0.29899999999999999,"15min":0.27500000000000002},"max":{"1min":6.6289999999999996,"5min":256.08499999999998,"15min":256.08499999999998},"last":2.0819999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:52:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4850000000000001,"5min":0.88100000000000001,"15min":1.091},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.308},"max":{"1min":7.3120000000000003,"5min":7.3120000000000003,"15min":115.514},"last":5.0010000000000003},{"interface":"front","average":{"1min":1.9430000000000001,"5min":1.202,"15min":1.333},"min":{"1min":0.38300000000000001,"5min":0.308,"15min":0.30599999999999999},"max":{"1min":6.601,"5min":6.601,"15min":116.53100000000001},"last":2.8159999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958796,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":778376,"kb_used_data":80896,"kb_used_omap":7,"kb_used_meta":697464,"kb_avail":92966776,"statfs":{"total":95995035648,"available":95197978624,"internally_reserved":0,"allocated":82837504,"data_stored":81648251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":714203662},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":4.3159999999999998,"5min":3.2010000000000001,"15min":4.6799999999999997},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":34.058,"5min":93.176000000000002,"15min":207.858},"last":2.8340000000000001},{"interface":"front","average":{"1min":4.1699999999999999,"5min":3.2130000000000001,"15min":5.1470000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.182},"max":{"1min":34.634,"5min":92.593999999999994,"15min":206.97200000000001},"last":5.641}]},{"osd":2,"last update":"Mon Oct 9 05:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3940000000000001,"5min":2.1469999999999998,"15min":3.6890000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.19800000000000001},"max":{"1min":7.2510000000000003,"5min":11.914,"15min":101.367},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.373,"5min":1.7310000000000001,"15min":3.452},"min":{"1min":0.27800000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":3.6259999999999999,"5min":16.702999999999999,"15min":99.861000000000004},"last":0.40200000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1230000000000002,"5min":3.2530000000000001,"15min":5.2149999999999999},"min":{"1min":0.25900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.053000000000001,"5min":92.353999999999999,"15min":162.33699999999999},"last":1.044},{"interface":"front","average":{"1min":3.863,"5min":3.1389999999999998,"15min":5.4470000000000001},"min":{"1min":0.36399999999999999,"5min":0.28599999999999998,"15min":0.26000000000000001},"max":{"1min":16.617999999999999,"5min":93.117999999999995,"15min":162.376},"last":0.51500000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":1.484,"15min":2.3839999999999999},"min":{"1min":0.40699999999999997,"5min":0.312,"15min":0.29799999999999999},"max":{"1min":5.2629999999999999,"5min":29.605,"15min":141.32900000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.153,"5min":1.5840000000000001,"15min":2.0369999999999999},"min":{"1min":0.34000000000000002,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":12.044,"5min":29.515999999999998,"15min":141.15899999999999},"last":0.437}]},{"osd":5,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5710000000000002,"5min":2.1469999999999998,"15min":2.5880000000000001},"min":{"1min":0.29399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.1749999999999998,"5min":32.774000000000001,"15min":235.18700000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.734,"5min":1.5589999999999999,"15min":2.742},"min":{"1min":0.41399999999999998,"5min":0.34499999999999997,"15min":0.32200000000000001},"max":{"1min":5.2910000000000004,"5min":36.976999999999997,"15min":235.79300000000001},"last":0.45000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5910000000000002,"5min":1.8340000000000001,"15min":2.613},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.316},"max":{"1min":7.2169999999999996,"5min":11.837999999999999,"15min":103.60899999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.7010000000000001,"5min":1.208,"15min":2.601},"min":{"1min":0.37,"5min":0.37,"15min":0.33400000000000002},"max":{"1min":5.1900000000000004,"5min":9.641,"15min":103.679},"last":0.56799999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1520000000000001,"5min":1.7330000000000001,"15min":2.1059999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.35999999999999999},"max":{"1min":12.093999999999999,"5min":35.021000000000001,"15min":126.68300000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.8879999999999999,"15min":2.1960000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.35499999999999998},"max":{"1min":12.196,"5min":35.493000000000002,"15min":126.738},"last":2.161}]}]},{"osd":0,"up_from":88,"seq":377957122322,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":763992,"kb_used_data":122192,"kb_used_omap":8,"kb_used_meta":641783,"kb_avail":92981160,"statfs":{"total":95995035648,"available":95212707840,"internally_reserved":0,"allocated":125124608,"data_stored":123861088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":657186568},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:52:08 2023","interfaces":[{"interface":"back","average":{"1min":3.133,"5min":2.4319999999999999,"15min":1.992},"min":{"1min":0.186,"5min":0.186,"15min":0.17199999999999999},"max":{"1min":18.164999999999999,"5min":63.534999999999997,"15min":63.534999999999997},"last":0.54000000000000004},{"interface":"front","average":{"1min":3.5619999999999998,"5min":2.4129999999999998,"15min":2.4489999999999998},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":16.484000000000002,"5min":63.677999999999997,"15min":63.677999999999997},"last":3.347}]},{"osd":2,"last update":"Mon Oct 9 05:51:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":2.7010000000000001,"15min":3.0369999999999999},"min":{"1min":0.33300000000000002,"5min":0.219,"15min":0.20000000000000001},"max":{"1min":9.4390000000000001,"5min":62.252000000000002,"15min":92.230000000000004},"last":2.3809999999999998},{"interface":"front","average":{"1min":2.4180000000000001,"5min":2.1720000000000002,"15min":2.8100000000000001},"min":{"1min":0.255,"5min":0.221,"15min":0.221},"max":{"1min":16.927,"5min":61.470999999999997,"15min":92.247},"last":0.66400000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:52:08 2023","interfaces":[{"interface":"back","average":{"1min":4.8259999999999996,"5min":2.1219999999999999,"15min":3.9870000000000001},"min":{"1min":0.307,"5min":0.253,"15min":0.218},"max":{"1min":59.838999999999999,"5min":59.838999999999999,"15min":123.024},"last":3.4289999999999998},{"interface":"front","average":{"1min":4.6399999999999997,"5min":2.8279999999999998,"15min":4.0220000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":59.859000000000002,"5min":72.632999999999996,"15min":99.162000000000006},"last":2.863}]},{"osd":4,"last update":"Mon Oct 9 05:51:38 2023","interfaces":[{"interface":"back","average":{"1min":9.0350000000000001,"5min":3.3420000000000001,"15min":2.109},"min":{"1min":0.439,"5min":0.29699999999999999,"15min":0.25700000000000001},"max":{"1min":149.63999999999999,"5min":149.63999999999999,"15min":149.63999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":8.3290000000000006,"5min":3.3119999999999998,"15min":2.4649999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":149.65899999999999,"5min":149.65899999999999,"15min":149.65899999999999},"last":3.3109999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:51:38 2023","interfaces":[{"interface":"back","average":{"1min":10.771000000000001,"5min":3.8370000000000002,"15min":3.9399999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.317},"max":{"1min":152.20500000000001,"5min":152.20500000000001,"15min":281.733},"last":3.27},{"interface":"front","average":{"1min":10.616,"5min":4.5860000000000003,"15min":4.2279999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.30199999999999999},"max":{"1min":154.30600000000001,"5min":154.30600000000001,"15min":282.95400000000001},"last":2.3029999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:51:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.472,"15min":1.76},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":6.6539999999999999,"5min":19.579000000000001,"15min":19.579000000000001},"last":3.375},{"interface":"front","average":{"1min":2.0750000000000002,"5min":1.847,"15min":1.9810000000000001},"min":{"1min":0.42499999999999999,"5min":0.36699999999999999,"15min":0.32500000000000001},"max":{"1min":10.441000000000001,"5min":19.710999999999999,"15min":19.710999999999999},"last":2.2669999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:52:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.504,"15min":1.4770000000000001},"min":{"1min":0.38500000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":7.6040000000000001,"5min":15.933999999999999,"15min":25.378},"last":0.627},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.486,"15min":1.861},"min":{"1min":0.39100000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":12.41,"5min":15.641,"15min":25.436},"last":3.4039999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795270,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1563524,"kb_used_data":803708,"kb_used_omap":11,"kb_used_meta":759796,"kb_avail":92181628,"statfs":{"total":95995035648,"available":94393987072,"internally_reserved":0,"allocated":822996992,"data_stored":821573132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":778031701},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,0,0,1,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":2.161,"15min":3.5129999999999999},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.17799999999999999},"max":{"1min":7.577,"5min":42.914999999999999,"15min":162.285},"last":0.75900000000000001},{"interface":"front","average":{"1min":3.2309999999999999,"5min":2.1120000000000001,"15min":3.8679999999999999},"min":{"1min":0.314,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":10.781000000000001,"5min":42.972999999999999,"15min":162.643},"last":0.38200000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:51:45 2023","interfaces":[{"interface":"back","average":{"1min":3.1909999999999998,"5min":3.8500000000000001,"15min":3.7389999999999999},"min":{"1min":0.23699999999999999,"5min":0.193,"15min":0.193},"max":{"1min":15.329000000000001,"5min":125.77500000000001,"15min":125.77500000000001},"last":0.504},{"interface":"front","average":{"1min":2.625,"5min":3.492,"15min":3.085},"min":{"1min":0.26900000000000002,"5min":0.24199999999999999,"15min":0.20999999999999999},"max":{"1min":11.074999999999999,"5min":125.857,"15min":125.857},"last":5.5510000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.855,"15min":4.9240000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.21099999999999999},"max":{"1min":9.5129999999999999,"5min":9.5129999999999999,"15min":159.35499999999999},"last":5.492},{"interface":"front","average":{"1min":2.7160000000000002,"5min":1.7170000000000001,"15min":4.7110000000000003},"min":{"1min":0.29399999999999998,"5min":0.217,"15min":0.217},"max":{"1min":11.303000000000001,"5min":11.303000000000001,"15min":159.196},"last":6.3769999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:51:45 2023","interfaces":[{"interface":"back","average":{"1min":4.915,"5min":1.901,"15min":2.1030000000000002},"min":{"1min":0.32600000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":68.052000000000007,"5min":68.052000000000007,"15min":115.84999999999999},"last":5.4450000000000003},{"interface":"front","average":{"1min":4.7869999999999999,"5min":2.0529999999999999,"15min":2.2210000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28699999999999998},"max":{"1min":68.727000000000004,"5min":68.727000000000004,"15min":116.149},"last":0.86699999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:51:45 2023","interfaces":[{"interface":"back","average":{"1min":4.7599999999999998,"5min":2.0179999999999998,"15min":2.1429999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":64.825000000000003,"5min":64.825000000000003,"15min":64.825000000000003},"last":0.47699999999999998},{"interface":"front","average":{"1min":5.0629999999999997,"5min":2.0019999999999998,"15min":1.843},"min":{"1min":0.39300000000000002,"5min":0.30499999999999999,"15min":0.26000000000000001},"max":{"1min":64.582999999999998,"5min":64.582999999999998,"15min":64.582999999999998},"last":5.5229999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:51:45 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.5069999999999999,"15min":1.7989999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.32000000000000001},"max":{"1min":9.5069999999999997,"5min":25.818999999999999,"15min":25.818999999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":1.8859999999999999,"5min":1.653,"15min":2.0099999999999998},"min":{"1min":0.34399999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.821,"5min":24.481000000000002,"15min":24.481000000000002},"last":0.42699999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":1.629,"15min":2.0190000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":12.071999999999999,"5min":12.071999999999999,"15min":51.701999999999998},"last":0.93500000000000005},{"interface":"front","average":{"1min":2.2170000000000001,"5min":1.74,"15min":2.2189999999999999},"min":{"1min":0.43099999999999999,"5min":0.378,"15min":0.373},"max":{"1min":6.2679999999999998,"5min":8.7430000000000003,"15min":51.716000000000001},"last":0.53000000000000003}]}]},{"osd":3,"up_from":103,"seq":442381631744,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1344788,"kb_used_data":594956,"kb_used_omap":4,"kb_used_meta":749819,"kb_avail":92400364,"statfs":{"total":95995035648,"available":94617972736,"internally_reserved":0,"allocated":609234944,"data_stored":607973113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":767815262},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":2.7879999999999998,"5min":3.2040000000000002,"15min":4.0789999999999997},"min":{"1min":0.218,"5min":0.188,"15min":0.184},"max":{"1min":8.5039999999999996,"5min":47.173000000000002,"15min":167.82400000000001},"last":10.35},{"interface":"front","average":{"1min":2.2000000000000002,"5min":3.0329999999999999,"15min":4.4950000000000001},"min":{"1min":0.22900000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":8.4879999999999995,"5min":47.908000000000001,"15min":167.89599999999999},"last":2.282}]},{"osd":1,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":3.7799999999999998,"15min":3.5680000000000001},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":5.9859999999999998,"5min":150.29599999999999,"15min":150.29599999999999},"last":10.234999999999999},{"interface":"front","average":{"1min":1.893,"5min":4.0460000000000003,"15min":3.746},"min":{"1min":0.216,"5min":0.216,"15min":0.19},"max":{"1min":13.018000000000001,"5min":151.25700000000001,"15min":151.25700000000001},"last":3.2709999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2730000000000001,"5min":1.6120000000000001,"15min":3.669},"min":{"1min":0.32700000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.577,"5min":12.577,"15min":190.43299999999999},"last":10.317},{"interface":"front","average":{"1min":2.3969999999999998,"5min":1.7250000000000001,"15min":4.0170000000000003},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.7390000000000008,"5min":9.7390000000000008,"15min":190.267},"last":2.774}]},{"osd":4,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":2.681,"15min":2.8180000000000001},"min":{"1min":0.31,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":8.0120000000000005,"5min":126.566,"15min":149.84700000000001},"last":2.2170000000000001},{"interface":"front","average":{"1min":1.4390000000000001,"5min":3.7949999999999999,"15min":2.8210000000000002},"min":{"1min":0.35999999999999999,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":4.8920000000000003,"5min":126.651,"15min":149.74700000000001},"last":10.289999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.7529999999999999,"15min":2.3370000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.28199999999999997},"max":{"1min":12.509,"5min":12.509,"15min":149.43000000000001},"last":3.2130000000000001},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.518,"15min":2.1619999999999999},"min":{"1min":0.39900000000000002,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":4.8140000000000001,"5min":7.8689999999999998,"15min":149.69300000000001},"last":10.186}]},{"osd":6,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":1.373,"15min":2.0680000000000001},"min":{"1min":0.35899999999999999,"5min":0.34499999999999997,"15min":0.33500000000000002},"max":{"1min":4.3360000000000003,"5min":7.9009999999999998,"15min":149.72399999999999},"last":10.263999999999999},{"interface":"front","average":{"1min":1.5389999999999999,"5min":1.6759999999999999,"15min":2.379},"min":{"1min":0.35899999999999999,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":12.558999999999999,"5min":12.558999999999999,"15min":149.589},"last":3.2440000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":4.8719999999999999,"5min":2.0870000000000002,"15min":2.4769999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":69.277000000000001,"5min":69.277000000000001,"15min":149.77199999999999},"last":2.1800000000000002},{"interface":"front","average":{"1min":4.4809999999999999,"5min":2.0329999999999999,"15min":2.5870000000000002},"min":{"1min":0.48299999999999998,"5min":0.374,"15min":0.373},"max":{"1min":69.299000000000007,"5min":69.299000000000007,"15min":149.80799999999999},"last":2.246}]}]},{"osd":4,"up_from":30,"seq":128849019291,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2052564,"kb_used_data":1159308,"kb_used_omap":0,"kb_used_meta":893248,"kb_avail":91692588,"statfs":{"total":95995035648,"available":93893210112,"internally_reserved":0,"allocated":1187131392,"data_stored":1186459591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914685952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:52 2023","interfaces":[{"interface":"back","average":{"1min":6.6379999999999999,"5min":4.6890000000000001,"15min":3.0910000000000002},"min":{"1min":0.26900000000000002,"5min":0.219,"15min":0.219},"max":{"1min":95.795000000000002,"5min":132.35599999999999,"15min":132.35599999999999},"last":4.3739999999999997},{"interface":"front","average":{"1min":7.7619999999999996,"5min":5.1719999999999997,"15min":3.4700000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23200000000000001},"max":{"1min":95.784999999999997,"5min":132.501,"15min":132.501},"last":8.3960000000000008}]},{"osd":1,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":6.4989999999999997,"5min":3.2890000000000001,"15min":3.0099999999999998},"min":{"1min":0.34999999999999998,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":59.978999999999999,"5min":68.367000000000004,"15min":187.369},"last":2.1030000000000002},{"interface":"front","average":{"1min":6.1360000000000001,"5min":3.2429999999999999,"15min":3.0819999999999999},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.27700000000000002},"max":{"1min":59.991,"5min":68.376000000000005,"15min":187.35300000000001},"last":8.4320000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:52:34 2023","interfaces":[{"interface":"back","average":{"1min":5.8710000000000004,"5min":2.5990000000000002,"15min":3.407},"min":{"1min":0.34899999999999998,"5min":0.308,"15min":0.26800000000000002},"max":{"1min":51.768000000000001,"5min":51.768000000000001,"15min":196.93700000000001},"last":2.1520000000000001},{"interface":"front","average":{"1min":5.5659999999999998,"5min":2.3199999999999998,"15min":3.3260000000000001},"min":{"1min":0.35099999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":51.753999999999998,"5min":51.753999999999998,"15min":197.08099999999999},"last":4.4260000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:52:02 2023","interfaces":[{"interface":"back","average":{"1min":2.343,"5min":1.929,"15min":4.5419999999999998},"min":{"1min":0.32900000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.347,"5min":30.111999999999998,"15min":211.48699999999999},"last":8.4559999999999995},{"interface":"front","average":{"1min":1.591,"5min":1.7370000000000001,"15min":4.4050000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.312},"max":{"1min":5.601,"5min":30.709,"15min":211.31100000000001},"last":4.4029999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:52:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.099,"15min":1.1919999999999999},"min":{"1min":0.27300000000000002,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":3.1190000000000002,"5min":12.34,"15min":94.646000000000001},"last":8.5190000000000001},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1,"15min":1.107},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.8490000000000002,"5min":10.191000000000001,"15min":94.692999999999998},"last":4.4589999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.93799999999999994,"15min":0.70499999999999996},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.4939999999999998,"5min":9.8889999999999993,"15min":9.8889999999999993},"last":4.4809999999999999},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.92700000000000005,"15min":0.875},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":2.387,"5min":6.0179999999999998,"15min":8.5269999999999992},"last":8.5429999999999993}]},{"osd":7,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":0.88600000000000001,"15min":1.034},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.5810000000000004,"5min":5.8650000000000002,"15min":96.131},"last":5.3789999999999996},{"interface":"front","average":{"1min":1.1459999999999999,"5min":0.995,"15min":1.1399999999999999},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.3159999999999998,"5min":7.6580000000000004,"15min":96.465000000000003},"last":2.1320000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823063,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1351880,"kb_used_data":461376,"kb_used_omap":0,"kb_used_meta":890496,"kb_avail":92393272,"statfs":{"total":95995035648,"available":94610710528,"internally_reserved":0,"allocated":472449024,"data_stored":471784983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911867904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:50 2023","interfaces":[{"interface":"back","average":{"1min":17.081,"5min":6.6459999999999999,"15min":3.819},"min":{"1min":0.33000000000000002,"5min":0.224,"15min":0.217},"max":{"1min":210.392,"5min":210.392,"15min":210.392},"last":0.245},{"interface":"front","average":{"1min":16.922999999999998,"5min":6.8470000000000004,"15min":3.9140000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":208.43600000000001,"5min":208.43600000000001,"15min":208.43600000000001},"last":0.371}]},{"osd":1,"last update":"Mon Oct 9 05:52:11 2023","interfaces":[{"interface":"back","average":{"1min":12.114000000000001,"5min":4.3680000000000003,"15min":3.4969999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.23999999999999999},"max":{"1min":198.35300000000001,"5min":198.35300000000001,"15min":198.35300000000001},"last":1.675},{"interface":"front","average":{"1min":11.247999999999999,"5min":4.4119999999999999,"15min":3.637},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":198.32900000000001,"5min":198.32900000000001,"15min":198.32900000000001},"last":0.96099999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:51:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":1.3200000000000001,"15min":2.1819999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":12.551,"5min":12.551,"15min":73.311999999999998},"last":0.88100000000000001},{"interface":"front","average":{"1min":2.6429999999999998,"5min":1.5529999999999999,"15min":2.2080000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":12.573,"5min":12.573,"15min":73.280000000000001},"last":1.696}]},{"osd":3,"last update":"Mon Oct 9 05:52:11 2023","interfaces":[{"interface":"back","average":{"1min":6.0309999999999997,"5min":3.379,"15min":4.2030000000000003},"min":{"1min":0.47399999999999998,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":63.069000000000003,"5min":85.069000000000003,"15min":95.236999999999995},"last":1.8779999999999999},{"interface":"front","average":{"1min":5.1680000000000001,"5min":3.3079999999999998,"15min":4.024},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":63.128,"5min":84.814999999999998,"15min":93.471000000000004},"last":1.3939999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:52:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":2.3330000000000002,"15min":1.2030000000000001},"min":{"1min":0.28000000000000003,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.8460000000000001,"5min":130.81,"15min":130.81},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.4890000000000001,"5min":2.5539999999999998,"15min":1.3},"min":{"1min":0.32300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":6.8330000000000002,"5min":133.803,"15min":133.803},"last":0.307}]},{"osd":6,"last update":"Mon Oct 9 05:52:26 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.8220000000000001,"15min":1.1899999999999999},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.8639999999999999,"5min":71.426000000000002,"15min":71.426000000000002},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.0550000000000002,"15min":1.2809999999999999},"min":{"1min":0.32300000000000001,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":4.109,"5min":71.498999999999995,"15min":71.498999999999995},"last":1.6459999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:52:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.383,"15min":1.0489999999999999},"min":{"1min":0.36799999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.085,"5min":31.533000000000001,"15min":31.533000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.6060000000000001,"5min":1.3759999999999999,"15min":1.0129999999999999},"min":{"1min":0.34300000000000003,"5min":0.316,"15min":0.308},"max":{"1min":8.5809999999999995,"5min":31.855,"15min":31.855},"last":0.40100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":107413504,"data_stored":107413504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57409536,"data_stored":57409536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":804786176,"data_stored":804782104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":600702976,"data_stored":600702976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1180106752,"data_stored":1180106752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":455479296,"data_stored":455475224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":624295936,"data_stored":624295936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":82382848,"data_stored":82378776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:52:48.804 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:52:48.824 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:48.850 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:48.891 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:48.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:48.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:48.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:49.022 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:49.063 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:49.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:49.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:49.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:49.196 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:49.237 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:49.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:49.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:49.345 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:49.371 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:49.412 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:49.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:49.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:49.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:49.547 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:49.588 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:49.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:49.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:49.657 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:52:49.657 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:52:49.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:52:49.657 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:52:49.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.657 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102141 7.8 7:1ebffa85:::benchmark_data_smithi008_354649_object7645:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.894031+0000", 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 570.75997756499999, 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3312663869999999, 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102141 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894031+0000", 2023-10-09T05:52:49.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894031+0000", 2023-10-09T05:52:49.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894030+0000", 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894035+0000", 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8479999999999997e-06 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894046+0000", 2023-10-09T05:52:49.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0501e-05 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894055+0000", 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0860000000000008e-06 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894107+0000", 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2182000000000003e-05 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.894119+0000", 2023-10-09T05:52:49.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1722000000000001e-05 2023-10-09T05:52:49.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870208+0000", 2023-10-09T05:52:49.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97608879900000001 2023-10-09T05:52:49.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870222+0000", 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3732e-05 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.209616+0000", 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3393948500000001 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.209842+0000", 2023-10-09T05:52:49.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00022551799999999999 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.210051+0000", 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020889199999999999 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.221116+0000", 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011064981 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.221206+0000", 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0245000000000001e-05 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.221293+0000", 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7059999999999997e-05 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.221361+0000", 2023-10-09T05:52:49.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7506e-05 2023-10-09T05:52:49.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.225298+0000", 2023-10-09T05:52:49.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039369940000000001 2023-10-09T05:52:49.666 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.666 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.666 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.666 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102155 7.8 7:1c0eb140:::benchmark_data_smithi008_354649_object7659:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.907631+0000", 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 570.74637808499995, 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.330968441, 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102155 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907631+0000", 2023-10-09T05:52:49.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907631+0000", 2023-10-09T05:52:49.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907630+0000", 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907635+0000", 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.152e-06 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907646+0000", 2023-10-09T05:52:49.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1117e-05 2023-10-09T05:52:49.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907651+0000", 2023-10-09T05:52:49.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5770000000000001e-06 2023-10-09T05:52:49.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907702+0000", 2023-10-09T05:52:49.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0269999999999998e-05 2023-10-09T05:52:49.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.907715+0000", 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2850000000000001e-05 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870248+0000", 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96253387899999998 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870257+0000", 2023-10-09T05:52:49.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3550000000000005e-06 2023-10-09T05:52:49.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.217936+0000", 2023-10-09T05:52:49.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3476795529999999 2023-10-09T05:52:49.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.218069+0000", 2023-10-09T05:52:49.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013219300000000001 2023-10-09T05:52:49.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:52:49.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.218202+0000", 2023-10-09T05:52:49.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013337599999999999 2023-10-09T05:52:49.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.229249+0000", 2023-10-09T05:52:49.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011046554 2023-10-09T05:52:49.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.233501+0000", 2023-10-09T05:52:49.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042527809999999997 2023-10-09T05:52:49.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.234614+0000", 2023-10-09T05:52:49.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001112273 2023-10-09T05:52:49.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.234665+0000", 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1879999999999998e-05 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.238599+0000", 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039336279999999998 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.675 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102163 7.8 7:18e716b9:::benchmark_data_smithi008_354649_object7667:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.920286+0000", 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 570.73372241100003, 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.322594327, 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102163 2023-10-09T05:52:49.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920286+0000", 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920286+0000", 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920285+0000", 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920291+0000", 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7899999999999996e-06 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920302+0000", 2023-10-09T05:52:49.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1046000000000001e-05 2023-10-09T05:52:49.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920310+0000", 2023-10-09T05:52:49.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6930000000000007e-06 2023-10-09T05:52:49.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920371+0000", 2023-10-09T05:52:49.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1540999999999999e-05 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.920382+0000", 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0910000000000001e-05 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870283+0000", 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94990104200000003 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870291+0000", 2023-10-09T05:52:49.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1130000000000001e-06 2023-10-09T05:52:49.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.225356+0000", 2023-10-09T05:52:49.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.355064152 2023-10-09T05:52:49.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.225545+0000", 2023-10-09T05:52:49.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000189061 2023-10-09T05:52:49.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:52:49.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.225732+0000", 2023-10-09T05:52:49.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018689299999999999 2023-10-09T05:52:49.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.233511+0000", 2023-10-09T05:52:49.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0077791470000000001 2023-10-09T05:52:49.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.238664+0000", 2023-10-09T05:52:49.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051530389999999999 2023-10-09T05:52:49.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.238929+0000", 2023-10-09T05:52:49.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00026558199999999999 2023-10-09T05:52:49.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.238994+0000", 2023-10-09T05:52:49.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4844999999999993e-05 2023-10-09T05:52:49.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.242881+0000", 2023-10-09T05:52:49.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038864920000000001 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:102168 7.8 7:17f42d0f:::benchmark_data_smithi008_354649_object7672:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:43:18.928352+0000", 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 570.72565694100001, 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3191062200000001, 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.685 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.686 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:52:49.686 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:52:49.686 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102168 2023-10-09T05:52:49.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.686 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928352+0000", 2023-10-09T05:52:49.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928352+0000", 2023-10-09T05:52:49.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928350+0000", 2023-10-09T05:52:49.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:49.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928359+0000", 2023-10-09T05:52:49.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3429999999999996e-06 2023-10-09T05:52:49.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928376+0000", 2023-10-09T05:52:49.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7161999999999999e-05 2023-10-09T05:52:49.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928386+0000", 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0054000000000001e-05 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928449+0000", 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2954999999999996e-05 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:18.928461+0000", 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2351000000000001e-05 2023-10-09T05:52:49.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870577+0000", 2023-10-09T05:52:49.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94211601300000003 2023-10-09T05:52:49.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:19.870588+0000", 2023-10-09T05:52:49.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0668e-05 2023-10-09T05:52:49.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230146+0000", 2023-10-09T05:52:49.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3595585649999999 2023-10-09T05:52:49.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230346+0000", 2023-10-09T05:52:49.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000200001 2023-10-09T05:52:49.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.230526+0000", 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017919 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.233655+0000", 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031297579999999998 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.238696+0000", 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050402520000000003 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.242968+0000", 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042724520000000004 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.243043+0000", 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5434999999999999e-05 2023-10-09T05:52:49.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:43:21.247458+0000", 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0044145460000000001 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125676 7.8 7:15209edd:::benchmark_data_smithi008_354649_object31180:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.953092+0000", 2023-10-09T05:52:49.694 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 526.700916269, 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33119336100000002, 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125676 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953092+0000", 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953092+0000", 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953091+0000", 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:49.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953100+0000", 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9600000000000006e-06 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953119+0000", 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9273999999999999e-05 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953130+0000", 2023-10-09T05:52:49.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0744000000000001e-05 2023-10-09T05:52:49.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953218+0000", 2023-10-09T05:52:49.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8712e-05 2023-10-09T05:52:49.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:52:49.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.953359+0000", 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014026800000000001 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.284269+0000", 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33090979500000001 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.284286+0000", 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7306999999999999e-05 2023-10-09T05:52:49.699 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45365.0:125684 7.3 e238/213 7:c9988d38:::benchmark_data_smithi008_354649_object31188:head v 238'7873, mlcod=238'7871)", 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:02.980604+0000", 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 526.67340517000002, 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.34659507299999998, 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980604+0000", 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980604+0000", 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980602+0000", 2023-10-09T05:52:49.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:49.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980617+0000", 2023-10-09T05:52:49.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4684999999999999e-05 2023-10-09T05:52:49.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980632+0000", 2023-10-09T05:52:49.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4992e-05 2023-10-09T05:52:49.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980644+0000", 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1753000000000001e-05 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980861+0000", 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00021713400000000001 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:02.980886+0000", 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5148999999999999e-05 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.327199+0000", 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.34631276100000002 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.704 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125676 7.8 7:15209edd:::benchmark_data_smithi008_354649_object31180:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:03.006382+0000", 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 526.647626844, 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0270797059999999, 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125676 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006382+0000", 2023-10-09T05:52:49.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006382+0000", 2023-10-09T05:52:49.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006379+0000", 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006392+0000", 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2278e-05 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006411+0000", 2023-10-09T05:52:49.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8771e-05 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.006423+0000", 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2140000000000001e-05 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.298138+0000", 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29171578399999998 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.298155+0000", 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6248000000000001e-05 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.958770+0000", 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.66061571799999996 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.958788+0000", 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7986e-05 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.990377+0000", 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0315884870000001 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.990742+0000", 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00036464899999999998 2023-10-09T05:52:49.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.025159+0000", 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.034417784999999999 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.025291+0000", 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013127599999999999 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.025442+0000", 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015114300000000001 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.028098+0000", 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026560030000000001 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.028113+0000", 2023-10-09T05:52:49.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4803999999999999e-05 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.031883+0000", 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037704729999999999 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.031952+0000", 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8750000000000004e-05 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.033462+0000", 2023-10-09T05:52:49.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001509809 2023-10-09T05:52:49.714 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.714 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.714 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.714 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125690 7.18 7:1f975c2f:::benchmark_data_smithi008_354649_object31194:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T05:52:49.714 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:03.043212+0000", 2023-10-09T05:52:49.714 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 526.61079718200006, 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1173635340000001, 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125690 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043212+0000", 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043212+0000", 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043209+0000", 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:52:49.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043218+0000", 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.081e-06 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043237+0000", 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8473000000000001e-05 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.043249+0000", 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2850999999999999e-05 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.298392+0000", 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.25514280299999997 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:03.298400+0000", 2023-10-09T05:52:49.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7619999999999995e-06 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.117212+0000", 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.81881223800000003 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.118138+0000", 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000925796 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.151854+0000", 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.033715429999999998 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.151931+0000", 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7076999999999994e-05 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:52:49.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.152013+0000", 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2369999999999999e-05 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.154638+0000", 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026251180000000001 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.154925+0000", 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00028725099999999998 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.158589+0000", 2023-10-09T05:52:49.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003663805 2023-10-09T05:52:49.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.158660+0000", 2023-10-09T05:52:49.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0753999999999994e-05 2023-10-09T05:52:49.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.160575+0000", 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00191519 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125699 7.8 7:12a3ca9b:::benchmark_data_smithi008_354649_object31203:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:04.049537+0000", 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 525.60447132800005, 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99354653900000001, 2023-10-09T05:52:49.723 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125699 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049537+0000", 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049537+0000", 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049536+0000", 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049542+0000", 2023-10-09T05:52:49.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9699999999999996e-06 2023-10-09T05:52:49.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049552+0000", 2023-10-09T05:52:49.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0336999999999999e-05 2023-10-09T05:52:49.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.049561+0000", 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5199999999999997e-06 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.080267+0000", 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030705823 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.080279+0000", 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2414e-05 2023-10-09T05:52:49.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.027631+0000", 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94735189399999997 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.027981+0000", 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00034967100000000002 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.728 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.035589+0000", 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007608498 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.035698+0000", 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000108607 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.035835+0000", 2023-10-09T05:52:49.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013765199999999999 2023-10-09T05:52:49.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.041951+0000", 2023-10-09T05:52:49.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061161460000000003 2023-10-09T05:52:49.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.041966+0000", 2023-10-09T05:52:49.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4367e-05 2023-10-09T05:52:49.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.042023+0000", 2023-10-09T05:52:49.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6901999999999998e-05 2023-10-09T05:52:49.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.042087+0000", 2023-10-09T05:52:49.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4338000000000006e-05 2023-10-09T05:52:49.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.043084+0000", 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00099690400000000006 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:125747 7.8 7:1254c147:::benchmark_data_smithi008_354649_object31251:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:44:04.185506+0000", 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 525.46850298100003, 2023-10-09T05:52:49.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85855084100000001, 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 125747 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185506+0000", 2023-10-09T05:52:49.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185506+0000", 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185504+0000", 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.735 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185511+0000", 2023-10-09T05:52:49.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5050000000000004e-06 2023-10-09T05:52:49.735 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.735 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185527+0000", 2023-10-09T05:52:49.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6192e-05 2023-10-09T05:52:49.735 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.185536+0000", 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8340000000000005e-06 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.186726+0000", 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011901629999999999 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:04.186737+0000", 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0543999999999999e-05 2023-10-09T05:52:49.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.028847+0000", 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.84211075000000002 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.029239+0000", 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00039186899999999998 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.039583+0000", 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010344183999999999 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.039701+0000", 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011815300000000001 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.039851+0000", 2023-10-09T05:52:49.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000149414 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.043440+0000", 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035890679999999999 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.043469+0000", 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8659000000000001e-05 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.043474+0000", 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2780000000000001e-06 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.043504+0000", 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9754e-05 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.740 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:44:05.044057+0000", 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00055292199999999996 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180268 7.1e 7:795f278a:::benchmark_data_smithi008_354649_object85772:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e269)", 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.024498+0000", 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 368.62951100800001, 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99404307599999997, 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.741 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.742 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.742 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:52:49.742 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:52:49.742 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180268 2023-10-09T05:52:49.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.742 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.742 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.742 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.742 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024498+0000", 2023-10-09T05:52:49.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024498+0000", 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024496+0000", 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024505+0000", 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2900000000000008e-06 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024528+0000", 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3096e-05 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024540+0000", 2023-10-09T05:52:49.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1959e-05 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.285107+0000", 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26056715899999999 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.285116+0000", 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5250000000000005e-06 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.016113+0000", 2023-10-09T05:52:49.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73099665800000002 2023-10-09T05:52:49.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.016255+0000", 2023-10-09T05:52:49.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014244400000000001 2023-10-09T05:52:49.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.016443+0000", 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018830400000000001 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.017705+0000", 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0012614029999999999 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018393+0000", 2023-10-09T05:52:49.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00068864900000000005 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018481+0000", 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7276999999999998e-05 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018513+0000", 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1922999999999999e-05 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018541+0000", 2023-10-09T05:52:49.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8174000000000002e-05 2023-10-09T05:52:49.749 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.749 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.749 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.749 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:75147 8.a 8:5b312b1e:::benchmark_data_smithi008_432048_object75146:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-09T05:52:49.749 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:31.714915+0000", 2023-10-09T05:52:49.749 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 257.93909366499997, 2023-10-09T05:52:49.750 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30074143399999997, 2023-10-09T05:52:49.750 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.750 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.750 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.750 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:49.750 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:49.750 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 75147 2023-10-09T05:52:49.750 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.750 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.750 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.750 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714915+0000", 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714915+0000", 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714914+0000", 2023-10-09T05:52:49.751 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714936+0000", 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2353000000000001e-05 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714945+0000", 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1670000000000005e-06 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.752 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.753 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714951+0000", 2023-10-09T05:52:49.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5999999999999997e-06 2023-10-09T05:52:49.753 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.753 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.753 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716442+0000", 2023-10-09T05:52:49.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001491514 2023-10-09T05:52:49.753 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.753 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716558+0000", 2023-10-09T05:52:49.754 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011547799999999999 2023-10-09T05:52:49.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.754 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:49.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716652+0000", 2023-10-09T05:52:49.754 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4291999999999998e-05 2023-10-09T05:52:49.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.754 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.014083+0000", 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.297430892 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015543+0000", 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014593900000000001 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015556+0000", 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3715e-05 2023-10-09T05:52:49.755 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015613+0000", 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6848000000000002e-05 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015657+0000", 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3412999999999999e-05 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.756 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.757 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:75150 8.3 e296/285 8:c720d3fb:::benchmark_data_smithi008_432048_object75149:head v 296'4679, mlcod=296'4677)", 2023-10-09T05:52:49.757 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:31.716949+0000", 2023-10-09T05:52:49.757 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 257.93706010099999, 2023-10-09T05:52:49.757 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30161086100000001, 2023-10-09T05:52:49.757 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.757 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.757 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.757 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.757 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716949+0000", 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716949+0000", 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716948+0000", 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:49.758 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716999+0000", 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0831999999999997e-05 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011201+0000", 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29420217900000001 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.759 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011219+0000", 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7883000000000001e-05 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011269+0000", 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9586999999999998e-05 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011301+0000", 2023-10-09T05:52:49.760 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1902000000000001e-05 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.018513+0000", 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0072121429999999999 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.018559+0000", 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6836999999999999e-05 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.761 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.762 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.762 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.762 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45782.0:75147 8.a e296/285 ondisk, result = 0)", 2023-10-09T05:52:49.762 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:31.718538+0000", 2023-10-09T05:52:49.762 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 257.935470716, 2023-10-09T05:52:49.762 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29700873900000002, 2023-10-09T05:52:49.762 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.762 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:52:49.762 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718538+0000", 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718538+0000", 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718537+0000", 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718541+0000", 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7900000000000001e-06 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.012365+0000", 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29382405900000003 2023-10-09T05:52:49.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.012376+0000", 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0210000000000001e-05 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015527+0000", 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031513660000000001 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.765 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015536+0000", 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1689999999999998e-06 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015547+0000", 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0713000000000001e-05 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.766 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:90579 8.a 8:55cc0eec:::benchmark_data_smithi008_432048_object90578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:49.809093+0000", 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 239.84491604900001, 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29641294499999998, 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 90579 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.767 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809093+0000", 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809093+0000", 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.768 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809091+0000", 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809117+0000", 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6276999999999999e-05 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809142+0000", 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4307e-05 2023-10-09T05:52:49.769 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809153+0000", 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1561e-05 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809289+0000", 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000136398 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.770 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809442+0000", 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000152211 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809589+0000", 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014731999999999999 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.819043+0000", 2023-10-09T05:52:49.771 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0094543100000000005 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.828230+0000", 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0091865950000000005 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105413+0000", 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27718348900000001 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.772 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105464+0000", 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0207000000000002e-05 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105506+0000", 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2001e-05 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.773 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.774 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.774 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:90580 8.a 8:5146ec73:::benchmark_data_smithi008_432048_object90579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-09T05:52:49.774 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:49.809186+0000", 2023-10-09T05:52:49.774 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 239.84482320999999, 2023-10-09T05:52:49.774 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29646979400000001, 2023-10-09T05:52:49.774 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.774 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.774 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.774 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:49.774 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 90580 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809186+0000", 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809186+0000", 2023-10-09T05:52:49.775 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.776 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.776 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.776 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.776 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809185+0000", 2023-10-09T05:52:49.776 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:49.776 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.776 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.776 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.776 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809210+0000", 2023-10-09T05:52:49.776 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5360000000000001e-05 2023-10-09T05:52:49.776 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809224+0000", 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3818000000000001e-05 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809232+0000", 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7230000000000004e-06 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.777 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.812756+0000", 2023-10-09T05:52:49.778 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035239479999999998 2023-10-09T05:52:49.778 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.778 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.778 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.778 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.812903+0000", 2023-10-09T05:52:49.778 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000147274 2023-10-09T05:52:49.778 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.778 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.778 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:52:49.778 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.813046+0000", 2023-10-09T05:52:49.778 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000142492 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.819054+0000", 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0060084450000000003 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.828260+0000", 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0092060430000000006 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.779 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105560+0000", 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27729968900000002 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105616+0000", 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6239999999999997e-05 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105655+0000", 2023-10-09T05:52:49.780 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9382999999999999e-05 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45782.0:168592 8.1a e352/349 ondisk, result = 0)", 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:53.754226+0000", 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 55.899782565000002, 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30718891300000001, 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.781 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.782 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.782 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754226+0000", 2023-10-09T05:52:49.782 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.782 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.782 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.782 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.782 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754226+0000", 2023-10-09T05:52:49.782 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.782 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.782 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.782 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754225+0000", 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754233+0000", 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3259999999999996e-06 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754249+0000", 2023-10-09T05:52:49.783 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5512e-05 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754258+0000", 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3989999999999993e-06 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754302+0000", 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4023999999999997e-05 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.784 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754311+0000", 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5580000000000001e-06 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061415+0000", 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.307104341 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.785 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:168594 8.f e352/339 8:f6b60d9c:::benchmark_data_smithi008_432048_object65652:head v 352'8443, mlcod=293'4003)", 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:53.755882+0000", 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 55.898126658000002, 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.31213674699999999, 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755882+0000", 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.786 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755882+0000", 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755881+0000", 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.787 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755888+0000", 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2779999999999999e-06 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061422+0000", 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30553472300000001 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061434+0000", 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2132e-05 2023-10-09T05:52:49.788 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.789 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.789 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.789 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061482+0000", 2023-10-09T05:52:49.789 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7129999999999998e-05 2023-10-09T05:52:49.789 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.789 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.789 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.789 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061517+0000", 2023-10-09T05:52:49.789 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5268999999999997e-05 2023-10-09T05:52:49.789 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.789 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.790 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.790 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.067965+0000", 2023-10-09T05:52:49.790 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064480149999999996 2023-10-09T05:52:49.790 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.790 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.790 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.790 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.068019+0000", 2023-10-09T05:52:49.790 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3918000000000001e-05 2023-10-09T05:52:49.790 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.790 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.790 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:168601 8.14 e352/299 8:2b44b62c:::benchmark_data_smithi008_432048_object65659:head v 352'7985, mlcod=352'7983)", 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:53.758418+0000", 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 55.895590452999997, 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.305713397, 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.791 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758418+0000", 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758418+0000", 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758417+0000", 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:52:49.792 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758427+0000", 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6940000000000007e-06 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061398+0000", 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30297090500000001 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.793 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061426+0000", 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7858999999999999e-05 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061482+0000", 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6051000000000003e-05 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061510+0000", 2023-10-09T05:52:49.794 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8085999999999998e-05 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.064085+0000", 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025752219999999998 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.064132+0000", 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6628999999999997e-05 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.795 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:189481 8.1a 8:5d874d6e:::benchmark_data_smithi008_432048_object86539:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:52:23.908117+0000", 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 25.745891864000001, 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35551341400000003, 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:52:49.796 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 189481 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908117+0000", 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908117+0000", 2023-10-09T05:52:49.797 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908116+0000", 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908121+0000", 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4879999999999998e-06 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908131+0000", 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7969999999999995e-06 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908136+0000", 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5509999999999999e-06 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908196+0000", 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9165e-05 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908548+0000", 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000352345 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.247746+0000", 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33919810900000003 2023-10-09T05:52:49.800 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.247846+0000", 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010041 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.248364+0000", 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00051785399999999999 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.801 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.252480+0000", 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041154950000000003 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.259547+0000", 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070675399999999998 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.263254+0000", 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037071130000000002 2023-10-09T05:52:49.802 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.803 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.803 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:52:49.803 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.263306+0000", 2023-10-09T05:52:49.803 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1272e-05 2023-10-09T05:52:49.803 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:52:49.803 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:52:49.803 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:52:49.803 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.263630+0000", 2023-10-09T05:52:49.803 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00032452000000000002 2023-10-09T05:52:49.804 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.804 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.804 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.804 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:52:49.804 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:52:49.804 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:52:49.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:49.830 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:49.872 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:49.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:49.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:49.980 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:50.155 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:50 smithi008 ceph-mon[184418]: pgmap v1090: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 20769/1635 objects degraded (1270.275%); 0 B/s, 94 objects/s recovering 2023-10-09T05:52:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:50 smithi008 ceph-mon[191227]: pgmap v1090: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 20769/1635 objects degraded (1270.275%); 0 B/s, 94 objects/s recovering 2023-10-09T05:52:50.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:50 smithi191 ceph-mon[161669]: pgmap v1090: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 20769/1635 objects degraded (1270.275%); 0 B/s, 94 objects/s recovering 2023-10-09T05:52:50.910 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:50.911 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:52:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:52:50] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.33.4" 2023-10-09T05:52:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:50.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:50.998 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:52:51.013 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:51.070 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:51.124 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:51.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:51.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:51.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:51.273 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:51.320 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:51.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:51.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:51.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:51.455 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:51.500 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:51.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:51.605 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:51.631 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:51.673 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:51.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:51.779 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:51.808 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:52.198 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:52:52.210 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:52.213 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:52.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:52.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:52.267 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:52.319 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:52.351 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:52.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:52 smithi008 ceph-mon[184418]: pgmap v1091: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 20769/1635 objects degraded (1270.275%); 0 B/s, 94 objects/s recovering 2023-10-09T05:52:52.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:52 smithi008 ceph-mon[191227]: pgmap v1091: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 20769/1635 objects degraded (1270.275%); 0 B/s, 94 objects/s recovering 2023-10-09T05:52:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:52 smithi191 ceph-mon[161669]: pgmap v1091: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 20769/1635 objects degraded (1270.275%); 0 B/s, 94 objects/s recovering 2023-10-09T05:52:52.550 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:52.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:52.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:52.636 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:52.655 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:52.687 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:52.736 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:52.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:52.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:52.843 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:52.871 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:52.915 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:52:52.915 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:52.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:52.937 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:52:52.961 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:52:52.981 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:52:53.002 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:52:53.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:53.033 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:52:53.047 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:53.055 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:52:53.077 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:52:53.079 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:52:53.079 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:52:53.079 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:52:53.079 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:52:53.079 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:52:53.079 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:52:53.079 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:52:53.079 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:52:53.089 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:53.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:53.195 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:53.220 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:53.262 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:53.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:53.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:53.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3210238360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20769/1635 objects degraded (1270.275%), 21 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:53.368 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:53.393 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:53.435 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:53.436 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3210238360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:53.436 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20769/1635 objects degraded (1270.275%), 21 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:53.437 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3210238360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:53.437 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20769/1635 objects degraded (1270.275%), 21 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:53.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:53.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:53.540 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:53.574 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:54.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:54 smithi008 ceph-mon[184418]: pgmap v1092: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 19656/1635 objects degraded (1202.202%); 0 B/s, 178 objects/s recovering 2023-10-09T05:52:54.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:54 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:54 smithi008 ceph-mon[191227]: pgmap v1092: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 19656/1635 objects degraded (1202.202%); 0 B/s, 178 objects/s recovering 2023-10-09T05:52:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:54 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:54 smithi191 ceph-mon[161669]: pgmap v1092: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 19656/1635 objects degraded (1202.202%); 0 B/s, 178 objects/s recovering 2023-10-09T05:52:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:54 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:55.381 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:55.382 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:55 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:55.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:55.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:55.384 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:55 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:55.485 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:55.508 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:55 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:55.511 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:55.552 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:55.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:55.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:55.660 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:55.685 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:55.727 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:55.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:55.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:55.836 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:55.866 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:56.084 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:56.318 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:56.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:56.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:56.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:56.445 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:56 smithi191 ceph-mon[161669]: pgmap v1093: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 19656/1635 objects degraded (1202.202%); 0 B/s, 178 objects/s recovering 2023-10-09T05:52:56.445 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:56 smithi191 ceph-mon[161669]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.445 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:56 smithi191 ceph-mon[161669]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.445 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:56 smithi191 ceph-mon[161669]: osdmap e360: 8 total, 8 up, 8 in 2023-10-09T05:52:56.445 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:56 smithi191 ceph-mon[161669]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.445 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:56 smithi191 ceph-mon[161669]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.447 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:56.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[191227]: pgmap v1093: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 19656/1635 objects degraded (1202.202%); 0 B/s, 178 objects/s recovering 2023-10-09T05:52:56.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[191227]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[191227]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[191227]: osdmap e360: 8 total, 8 up, 8 in 2023-10-09T05:52:56.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[191227]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[191227]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[184418]: pgmap v1093: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 19656/1635 objects degraded (1202.202%); 0 B/s, 178 objects/s recovering 2023-10-09T05:52:56.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[184418]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[184418]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[184418]: osdmap e360: 8 total, 8 up, 8 in 2023-10-09T05:52:56.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[184418]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:56 smithi008 ceph-mon[184418]: failed to encode map e360 with expected crc 2023-10-09T05:52:56.489 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:56.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:56.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:56.594 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:56.619 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:56.664 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:56.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:56.771 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:56.800 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:57.142 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[191227]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.142 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[191227]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.142 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[191227]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T05:52:57.142 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[191227]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.142 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[191227]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.142 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:57.198 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:57.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:57.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:57.304 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:57.339 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:57.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[184418]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[184418]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[184418]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T05:52:57.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[184418]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[184418]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:57 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:57.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:57 smithi191 ceph-mon[161669]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:57 smithi191 ceph-mon[161669]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:57 smithi191 ceph-mon[161669]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T05:52:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:57 smithi191 ceph-mon[161669]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:57 smithi191 ceph-mon[161669]: failed to encode map e361 with expected crc 2023-10-09T05:52:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:57 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:58.125 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_3' removed 2023-10-09T05:52:58.141 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:58.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:58.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:58.263 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:58.387 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:58.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[184418]: pgmap v1096: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 19595/1635 objects degraded (1198.471%); 0 B/s, 140 objects/s recovering 2023-10-09T05:52:58.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[184418]: 8.10 scrub starts 2023-10-09T05:52:58.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2470170250' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:52:58.457 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19595/1635 objects degraded (1198.471%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:58.457 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2470170250' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:52:58.457 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[184418]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T05:52:58.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[191227]: pgmap v1096: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 19595/1635 objects degraded (1198.471%); 0 B/s, 140 objects/s recovering 2023-10-09T05:52:58.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[191227]: 8.10 scrub starts 2023-10-09T05:52:58.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2470170250' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:52:58.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19595/1635 objects degraded (1198.471%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:58.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2470170250' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:52:58.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:58 smithi008 ceph-mon[191227]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T05:52:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:58 smithi191 ceph-mon[161669]: pgmap v1096: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 98 active+clean; 37 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 19595/1635 objects degraded (1198.471%); 0 B/s, 140 objects/s recovering 2023-10-09T05:52:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:58 smithi191 ceph-mon[161669]: 8.10 scrub starts 2023-10-09T05:52:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2470170250' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:52:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19595/1635 objects degraded (1198.471%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:52:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2470170250' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:52:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:58 smithi191 ceph-mon[161669]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T05:52:59.243 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:59.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:59.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:59.335 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:52:59.348 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:59.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:59 smithi008 ceph-mon[184418]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:59 smithi008 ceph-mon[184418]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:59 smithi008 ceph-mon[184418]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:59 smithi008 ceph-mon[184418]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:52:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:59.367 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:59 smithi008 ceph-mon[191227]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:59 smithi008 ceph-mon[191227]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:59 smithi008 ceph-mon[191227]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:59 smithi008 ceph-mon[191227]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:52:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:59.406 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:59.422 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:52:59.423 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:52:59.423 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T05:52:59.424 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T05:52:59.498 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:59.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:59.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:59 smithi191 ceph-mon[161669]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:59 smithi191 ceph-mon[161669]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:59 smithi191 ceph-mon[161669]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:59 smithi191 ceph-mon[161669]: failed to encode map e362 with expected crc 2023-10-09T05:52:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:52:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:59.594 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:52:59.594 INFO:teuthology.orchestra.run.smithi008.stdout:{"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded","count":1},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1473,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":362,"num_osds":8,"num_up_osds":8,"osd_up_since":1696829680,"num_in_osds":8,"osd_in_since":1696830533,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":72},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":73,"num_pools":4,"num_objects":28,"data_bytes":4790195,"bytes_used":10031980544,"bytes_avail":757928304640,"bytes_total":767960285184,"degraded_objects":10,"degraded_total":84,"degraded_ratio":0.11904761904761903},"fsmap":{"epoch":6,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi008.zfejpl","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.8:8443/","prometheus":"http://172.21.15.8:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T05:41:46.162076+0000","services":{}},"progress_events":{"3ed3ab59-29d3-494c-9b75-b5419b290738":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 48s)","progress":0.82352942228317261,"add_to_ceph_s":true},"e7a35707-3c6c-469d-ad94-cd0d5825f733":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T05:52:59.603 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:59.633 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:59.678 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:59.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:59.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:59.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:59.810 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:59.826 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:59.826 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:59.852 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:59.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:59.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:59.957 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:59.982 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:00.025 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:00.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:00.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:00.131 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:00.161 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:00.181 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:53:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:00 smithi191 ceph-mon[161669]: pgmap v1098: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T05:53:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2080001060' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:00 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/67631732' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:53:00.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:00 smithi008 ceph-mon[184418]: pgmap v1098: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T05:53:00.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2080001060' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:00.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:00 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:00.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/67631732' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:53:00.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:00 smithi008 ceph-mon[191227]: pgmap v1098: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 4.6 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T05:53:00.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2080001060' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:00.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:00 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:00.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/67631732' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:53:00.766 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:00.768 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:53:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:53:00] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.33.4" 2023-10-09T05:53:00.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:00.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:00.866 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T05:53:00.867 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:53:00.870 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:00.900 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:00.947 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:00.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:00.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:01.055 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:01.085 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:01.329 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:01.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:01.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:01.335 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:01 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:53:01.335 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:01 smithi008 ceph-mon[184418]: failed to encode map e363 with expected crc 2023-10-09T05:53:01.335 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:01 smithi008 ceph-mon[184418]: failed to encode map e363 with expected crc 2023-10-09T05:53:01.335 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:01 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:01.335 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:01 smithi008 ceph-mon[184418]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T05:53:01.336 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:01 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:53:01.336 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:01 smithi008 ceph-mon[191227]: failed to encode map e363 with expected crc 2023-10-09T05:53:01.336 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:01 smithi008 ceph-mon[191227]: failed to encode map e363 with expected crc 2023-10-09T05:53:01.336 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:01 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:01.336 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:01 smithi008 ceph-mon[191227]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T05:53:01.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:01.462 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:01 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:53:01.463 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:01 smithi191 ceph-mon[161669]: failed to encode map e363 with expected crc 2023-10-09T05:53:01.463 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:01 smithi191 ceph-mon[161669]: failed to encode map e363 with expected crc 2023-10-09T05:53:01.463 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:01 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:01.463 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:01 smithi191 ceph-mon[161669]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T05:53:01.465 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:01.513 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:01.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:01.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:01.547 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:53:01.582 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:01.618 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:01.645 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:01.646 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:01.692 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:01.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:01.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:01.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:01.826 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:01.858 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:01.867 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:01.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:01.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:01.974 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:02.004 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:02.431 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:02 smithi008 ceph-mon[184418]: pgmap v1100: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 4.6 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T05:53:02.431 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:02 smithi008 ceph-mon[184418]: failed to encode map e363 with expected crc 2023-10-09T05:53:02.431 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:02 smithi008 ceph-mon[184418]: failed to encode map e363 with expected crc 2023-10-09T05:53:02.431 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:02 smithi008 ceph-mon[191227]: pgmap v1100: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 4.6 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T05:53:02.431 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:02 smithi008 ceph-mon[191227]: failed to encode map e363 with expected crc 2023-10-09T05:53:02.432 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:02 smithi008 ceph-mon[191227]: failed to encode map e363 with expected crc 2023-10-09T05:53:02.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:02 smithi191 ceph-mon[161669]: pgmap v1100: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 4.6 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T05:53:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:02 smithi191 ceph-mon[161669]: failed to encode map e363 with expected crc 2023-10-09T05:53:02.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:02 smithi191 ceph-mon[161669]: failed to encode map e363 with expected crc 2023-10-09T05:53:03.058 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:03.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:03.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:03.162 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:03.194 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:03.240 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:03.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:03.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:03.338 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:53:03.347 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:03.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:53:03.365 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:53:03.374 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:03.417 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:03.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:03.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:03.522 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:03.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:53:03.557 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:03.610 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:03.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:03.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:03.714 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:03.740 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:03.782 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:03.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:03.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:03.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:03.916 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:03.957 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:03.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:03.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:04.062 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:04.087 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:04.129 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:04.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:04.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:04.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:04 smithi008 ceph-mon[184418]: pgmap v1101: 73 pgs: 73 active+clean; 4.6 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:53:04.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:04 smithi008 ceph-mon[184418]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded) 2023-10-09T05:53:04.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:04 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:53:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:04 smithi008 ceph-mon[191227]: pgmap v1101: 73 pgs: 73 active+clean; 4.6 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:53:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:04 smithi008 ceph-mon[191227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded) 2023-10-09T05:53:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:04 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:53:04.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:04.263 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:04.305 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:04.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:04.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:04.412 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:04.435 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:04 smithi191 ceph-mon[161669]: pgmap v1101: 73 pgs: 73 active+clean; 4.6 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:53:04.435 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:04 smithi191 ceph-mon[161669]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded) 2023-10-09T05:53:04.435 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:04 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:53:04.437 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:04.479 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:04.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:04.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:04.530 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:04.585 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:04.619 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:04.925 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:53:04.944 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:53:04.964 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:53:04.983 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:53:05.002 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:05.003 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:53:05.026 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:53:05.048 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:53:05.067 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:53:05.070 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:53:05.070 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:53:05.070 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:53:05.070 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:53:05.070 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:53:05.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:53:05.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:53:05.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:53:05.076 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:05.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:05.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:05.183 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:05.219 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:05.323 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:53:05.325 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:53:05.345 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:05 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1002489388' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:05.346 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:05 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1002489388' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:05.473 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:05.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:05.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:05 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1002489388' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:05.577 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:05.609 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:05.656 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:05.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:05.762 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:05.790 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:05.834 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:05.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:05.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:05.940 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:05.974 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:06.241 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:06 smithi008 ceph-mon[184418]: pgmap v1102: 73 pgs: 73 active+clean; 4.6 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:53:06.241 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:06 smithi008 ceph-mon[191227]: pgmap v1102: 73 pgs: 73 active+clean; 4.6 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:53:06.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:06 smithi191 ceph-mon[161669]: pgmap v1102: 73 pgs: 73 active+clean; 4.6 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:53:06.724 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:06.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:06.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:06.830 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:06.858 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:07.206 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:07 smithi008 ceph-mon[184418]: from='client.46145 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:53:07.207 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:07 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/621374547' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T05:53:07.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:07 smithi008 ceph-mon[191227]: from='client.46145 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:53:07.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:07 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/621374547' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T05:53:07.213 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_4' created 2023-10-09T05:53:07.495 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1102,"stamp":"2023-10-09T05:53:04.523084+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":182,"num_bytes_recovered":5386010,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9658964,"kb_used_data":3155732,"kb_used_omap":45,"kb_used_meta":6503122,"kb_avail":740302252,"statfs":{"total":767960285184,"available":758069506048,"internally_reserved":0,"allocated":3231469568,"data_stored":3223529222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":6659197219},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":94,"apply_latency_ms":94,"commit_latency_ns":94000000,"apply_latency_ns":94000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-33914886,"num_objects":-517,"num_object_clones":0,"num_object_copies":-1551,"num_objects_missing_on_primary":-451,"num_objects_missing":-451,"num_objects_degraded":-19595,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-517,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-205880,"num_write_kb":-6588097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-36969,"num_bytes_recovered":-2278293504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002325"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":362,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.143172+0000","last_change":"2023-10-09T05:48:54.494513+0000","last_active":"2023-10-09T05:52:56.143172+0000","last_peered":"2023-10-09T05:52:56.143172+0000","last_clean":"2023-10-09T05:52:56.143172+0000","last_became_active":"2023-10-09T05:48:54.493510+0000","last_became_peered":"2023-10-09T05:48:54.493510+0000","last_unstale":"2023-10-09T05:52:56.143172+0000","last_undegraded":"2023-10-09T05:52:56.143172+0000","last_fullsized":"2023-10-09T05:52:56.143172+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":292,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.144733+0000","last_change":"2023-10-09T05:50:18.047163+0000","last_active":"2023-10-09T05:52:56.144733+0000","last_peered":"2023-10-09T05:52:56.144733+0000","last_clean":"2023-10-09T05:52:56.144733+0000","last_became_active":"2023-10-09T05:50:18.046872+0000","last_became_peered":"2023-10-09T05:50:18.046872+0000","last_unstale":"2023-10-09T05:52:56.144733+0000","last_undegraded":"2023-10-09T05:52:56.144733+0000","last_fullsized":"2023-10-09T05:52:56.144733+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":106,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.142677+0000","last_change":"2023-10-09T05:50:18.055745+0000","last_active":"2023-10-09T05:52:56.142677+0000","last_peered":"2023-10-09T05:52:56.142677+0000","last_clean":"2023-10-09T05:52:56.142677+0000","last_became_active":"2023-10-09T05:50:18.055524+0000","last_became_peered":"2023-10-09T05:50:18.055524+0000","last_unstale":"2023-10-09T05:52:56.142677+0000","last_undegraded":"2023-10-09T05:52:56.142677+0000","last_fullsized":"2023-10-09T05:52:56.142677+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":380,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.517200+0000","last_change":"2023-10-09T05:51:22.261181+0000","last_active":"2023-10-09T05:52:56.517200+0000","last_peered":"2023-10-09T05:52:56.517200+0000","last_clean":"2023-10-09T05:52:56.517200+0000","last_became_active":"2023-10-09T05:51:22.260808+0000","last_became_peered":"2023-10-09T05:51:22.260808+0000","last_unstale":"2023-10-09T05:52:56.517200+0000","last_undegraded":"2023-10-09T05:52:56.517200+0000","last_fullsized":"2023-10-09T05:52:56.517200+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:17.362302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":331,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.486047+0000","last_change":"2023-10-09T05:48:54.492664+0000","last_active":"2023-10-09T05:52:56.486047+0000","last_peered":"2023-10-09T05:52:56.486047+0000","last_clean":"2023-10-09T05:52:56.486047+0000","last_became_active":"2023-10-09T05:48:54.492299+0000","last_became_peered":"2023-10-09T05:48:54.492299+0000","last_unstale":"2023-10-09T05:52:56.486047+0000","last_undegraded":"2023-10-09T05:52:56.486047+0000","last_fullsized":"2023-10-09T05:52:56.486047+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:43.198424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":196,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.115073+0000","last_change":"2023-10-09T05:48:54.492447+0000","last_active":"2023-10-09T05:52:58.115073+0000","last_peered":"2023-10-09T05:52:58.115073+0000","last_clean":"2023-10-09T05:52:58.115073+0000","last_became_active":"2023-10-09T05:48:54.492089+0000","last_became_peered":"2023-10-09T05:48:54.492089+0000","last_unstale":"2023-10-09T05:52:58.115073+0000","last_undegraded":"2023-10-09T05:52:58.115073+0000","last_fullsized":"2023-10-09T05:52:58.115073+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":318,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.486500+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:52:56.486500+0000","last_peered":"2023-10-09T05:52:56.486500+0000","last_clean":"2023-10-09T05:52:56.486500+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:52:56.486500+0000","last_undegraded":"2023-10-09T05:52:56.486500+0000","last_fullsized":"2023-10-09T05:52:56.486500+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":324,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.154125+0000","last_change":"2023-10-09T05:50:18.054081+0000","last_active":"2023-10-09T05:52:58.154125+0000","last_peered":"2023-10-09T05:52:58.154125+0000","last_clean":"2023-10-09T05:52:58.154125+0000","last_became_active":"2023-10-09T05:50:18.053746+0000","last_became_peered":"2023-10-09T05:50:18.053746+0000","last_unstale":"2023-10-09T05:52:58.154125+0000","last_undegraded":"2023-10-09T05:52:58.154125+0000","last_fullsized":"2023-10-09T05:52:58.154125+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:55.758681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":23,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.117722+0000","last_change":"2023-10-09T05:51:43.123678+0000","last_active":"2023-10-09T05:52:58.117722+0000","last_peered":"2023-10-09T05:52:58.117722+0000","last_clean":"2023-10-09T05:52:58.117722+0000","last_became_active":"2023-10-09T05:51:43.121947+0000","last_became_peered":"2023-10-09T05:51:43.121947+0000","last_unstale":"2023-10-09T05:52:58.117722+0000","last_undegraded":"2023-10-09T05:52:58.117722+0000","last_fullsized":"2023-10-09T05:52:58.117722+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":298,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.094626+0000","last_change":"2023-10-09T05:48:25.068328+0000","last_active":"2023-10-09T05:52:58.094626+0000","last_peered":"2023-10-09T05:52:58.094626+0000","last_clean":"2023-10-09T05:52:58.094626+0000","last_became_active":"2023-10-09T05:48:25.068094+0000","last_became_peered":"2023-10-09T05:48:25.068094+0000","last_unstale":"2023-10-09T05:52:58.094626+0000","last_undegraded":"2023-10-09T05:52:58.094626+0000","last_fullsized":"2023-10-09T05:52:58.094626+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":148,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.096156+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:52:58.096156+0000","last_peered":"2023-10-09T05:52:58.096156+0000","last_clean":"2023-10-09T05:52:58.096156+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:52:58.096156+0000","last_undegraded":"2023-10-09T05:52:58.096156+0000","last_fullsized":"2023-10-09T05:52:58.096156+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":144,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.096426+0000","last_change":"2023-10-09T05:50:18.045973+0000","last_active":"2023-10-09T05:52:58.096426+0000","last_peered":"2023-10-09T05:52:58.096426+0000","last_clean":"2023-10-09T05:52:58.096426+0000","last_became_active":"2023-10-09T05:50:18.045688+0000","last_became_peered":"2023-10-09T05:50:18.045688+0000","last_unstale":"2023-10-09T05:52:58.096426+0000","last_undegraded":"2023-10-09T05:52:58.096426+0000","last_fullsized":"2023-10-09T05:52:58.096426+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:51.105027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":388,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.143688+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:52:56.143688+0000","last_peered":"2023-10-09T05:52:56.143688+0000","last_clean":"2023-10-09T05:52:56.143688+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:52:56.143688+0000","last_undegraded":"2023-10-09T05:52:56.143688+0000","last_fullsized":"2023-10-09T05:52:56.143688+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":345,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.143079+0000","last_change":"2023-10-09T05:48:54.492082+0000","last_active":"2023-10-09T05:52:56.143079+0000","last_peered":"2023-10-09T05:52:56.143079+0000","last_clean":"2023-10-09T05:52:56.143079+0000","last_became_active":"2023-10-09T05:48:54.491753+0000","last_became_peered":"2023-10-09T05:48:54.491753+0000","last_unstale":"2023-10-09T05:52:56.143079+0000","last_undegraded":"2023-10-09T05:52:56.143079+0000","last_fullsized":"2023-10-09T05:52:56.143079+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":148,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.094850+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:52:58.094850+0000","last_peered":"2023-10-09T05:52:58.094850+0000","last_clean":"2023-10-09T05:52:58.094850+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:52:58.094850+0000","last_undegraded":"2023-10-09T05:52:58.094850+0000","last_fullsized":"2023-10-09T05:52:58.094850+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":326,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.095831+0000","last_change":"2023-10-09T05:50:18.039555+0000","last_active":"2023-10-09T05:52:58.095831+0000","last_peered":"2023-10-09T05:52:58.095831+0000","last_clean":"2023-10-09T05:52:58.095831+0000","last_became_active":"2023-10-09T05:50:18.039222+0000","last_became_peered":"2023-10-09T05:50:18.039222+0000","last_unstale":"2023-10-09T05:52:58.095831+0000","last_undegraded":"2023-10-09T05:52:58.095831+0000","last_fullsized":"2023-10-09T05:52:58.095831+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":351,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.485708+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:52:56.485708+0000","last_peered":"2023-10-09T05:52:56.485708+0000","last_clean":"2023-10-09T05:52:56.485708+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:52:56.485708+0000","last_undegraded":"2023-10-09T05:52:56.485708+0000","last_fullsized":"2023-10-09T05:52:56.485708+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":23,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.106335+0000","last_change":"2023-10-09T05:51:43.075341+0000","last_active":"2023-10-09T05:52:58.106335+0000","last_peered":"2023-10-09T05:52:58.106335+0000","last_clean":"2023-10-09T05:52:58.106335+0000","last_became_active":"2023-10-09T05:51:43.075024+0000","last_became_peered":"2023-10-09T05:51:43.075024+0000","last_unstale":"2023-10-09T05:52:58.106335+0000","last_undegraded":"2023-10-09T05:52:58.106335+0000","last_fullsized":"2023-10-09T05:52:58.106335+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":142,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.094730+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:52:58.094730+0000","last_peered":"2023-10-09T05:52:58.094730+0000","last_clean":"2023-10-09T05:52:58.094730+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:52:58.094730+0000","last_undegraded":"2023-10-09T05:52:58.094730+0000","last_fullsized":"2023-10-09T05:52:58.094730+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":351,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.516790+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:52:56.516790+0000","last_peered":"2023-10-09T05:52:56.516790+0000","last_clean":"2023-10-09T05:52:56.516790+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:52:56.516790+0000","last_undegraded":"2023-10-09T05:52:56.516790+0000","last_fullsized":"2023-10-09T05:52:56.516790+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":177,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.485918+0000","last_change":"2023-10-09T05:51:43.221482+0000","last_active":"2023-10-09T05:52:56.485918+0000","last_peered":"2023-10-09T05:52:56.485918+0000","last_clean":"2023-10-09T05:52:56.485918+0000","last_became_active":"2023-10-09T05:51:43.072606+0000","last_became_peered":"2023-10-09T05:51:43.072606+0000","last_unstale":"2023-10-09T05:52:56.485918+0000","last_undegraded":"2023-10-09T05:52:56.485918+0000","last_fullsized":"2023-10-09T05:52:56.485918+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:22.097091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":368,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.094987+0000","last_change":"2023-10-09T05:48:54.492012+0000","last_active":"2023-10-09T05:52:58.094987+0000","last_peered":"2023-10-09T05:52:58.094987+0000","last_clean":"2023-10-09T05:52:58.094987+0000","last_became_active":"2023-10-09T05:48:54.491644+0000","last_became_peered":"2023-10-09T05:48:54.491644+0000","last_unstale":"2023-10-09T05:52:58.094987+0000","last_undegraded":"2023-10-09T05:52:58.094987+0000","last_fullsized":"2023-10-09T05:52:58.094987+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":146,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.094899+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:52:58.094899+0000","last_peered":"2023-10-09T05:52:58.094899+0000","last_clean":"2023-10-09T05:52:58.094899+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:52:58.094899+0000","last_undegraded":"2023-10-09T05:52:58.094899+0000","last_fullsized":"2023-10-09T05:52:58.094899+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":23,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.105394+0000","last_change":"2023-10-09T05:51:43.075609+0000","last_active":"2023-10-09T05:52:58.105394+0000","last_peered":"2023-10-09T05:52:58.105394+0000","last_clean":"2023-10-09T05:52:58.105394+0000","last_became_active":"2023-10-09T05:51:43.075280+0000","last_became_peered":"2023-10-09T05:51:43.075280+0000","last_unstale":"2023-10-09T05:52:58.105394+0000","last_undegraded":"2023-10-09T05:52:58.105394+0000","last_fullsized":"2023-10-09T05:52:58.105394+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":172,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.095804+0000","last_change":"2023-10-09T05:48:54.539425+0000","last_active":"2023-10-09T05:52:58.095804+0000","last_peered":"2023-10-09T05:52:58.095804+0000","last_clean":"2023-10-09T05:52:58.095804+0000","last_became_active":"2023-10-09T05:48:54.486401+0000","last_became_peered":"2023-10-09T05:48:54.486401+0000","last_unstale":"2023-10-09T05:52:58.095804+0000","last_undegraded":"2023-10-09T05:52:58.095804+0000","last_fullsized":"2023-10-09T05:52:58.095804+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":378,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.142764+0000","last_change":"2023-10-09T05:48:54.492534+0000","last_active":"2023-10-09T05:52:56.142764+0000","last_peered":"2023-10-09T05:52:56.142764+0000","last_clean":"2023-10-09T05:52:56.142764+0000","last_became_active":"2023-10-09T05:48:54.492172+0000","last_became_peered":"2023-10-09T05:48:54.492172+0000","last_unstale":"2023-10-09T05:52:56.142764+0000","last_undegraded":"2023-10-09T05:52:56.142764+0000","last_fullsized":"2023-10-09T05:52:56.142764+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":172,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.096051+0000","last_change":"2023-10-09T05:51:43.265312+0000","last_active":"2023-10-09T05:52:58.096051+0000","last_peered":"2023-10-09T05:52:58.096051+0000","last_clean":"2023-10-09T05:52:58.096051+0000","last_became_active":"2023-10-09T05:51:43.076237+0000","last_became_peered":"2023-10-09T05:51:43.076237+0000","last_unstale":"2023-10-09T05:52:58.096051+0000","last_undegraded":"2023-10-09T05:52:58.096051+0000","last_fullsized":"2023-10-09T05:52:58.096051+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":340,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.094742+0000","last_change":"2023-10-09T05:50:18.039975+0000","last_active":"2023-10-09T05:52:58.094742+0000","last_peered":"2023-10-09T05:52:58.094742+0000","last_clean":"2023-10-09T05:52:58.094742+0000","last_became_active":"2023-10-09T05:50:18.039332+0000","last_became_peered":"2023-10-09T05:50:18.039332+0000","last_unstale":"2023-10-09T05:52:58.094742+0000","last_undegraded":"2023-10-09T05:52:58.094742+0000","last_fullsized":"2023-10-09T05:52:58.094742+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":330,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.113021+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:52:58.113021+0000","last_peered":"2023-10-09T05:52:58.113021+0000","last_clean":"2023-10-09T05:52:58.113021+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:52:58.113021+0000","last_undegraded":"2023-10-09T05:52:58.113021+0000","last_fullsized":"2023-10-09T05:52:58.113021+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":337,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.485972+0000","last_change":"2023-10-09T05:48:54.495800+0000","last_active":"2023-10-09T05:52:56.485972+0000","last_peered":"2023-10-09T05:52:56.485972+0000","last_clean":"2023-10-09T05:52:56.485972+0000","last_became_active":"2023-10-09T05:48:54.495540+0000","last_became_peered":"2023-10-09T05:48:54.495540+0000","last_unstale":"2023-10-09T05:52:56.485972+0000","last_undegraded":"2023-10-09T05:52:56.485972+0000","last_fullsized":"2023-10-09T05:52:56.485972+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:26.903669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":100,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.104232+0000","last_change":"2023-10-09T05:48:25.067945+0000","last_active":"2023-10-09T05:52:58.104232+0000","last_peered":"2023-10-09T05:52:58.104232+0000","last_clean":"2023-10-09T05:52:58.104232+0000","last_became_active":"2023-10-09T05:48:25.067721+0000","last_became_peered":"2023-10-09T05:48:25.067721+0000","last_unstale":"2023-10-09T05:52:58.104232+0000","last_undegraded":"2023-10-09T05:52:58.104232+0000","last_fullsized":"2023-10-09T05:52:58.104232+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:58.655544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":327,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.143539+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:52:56.143539+0000","last_peered":"2023-10-09T05:52:56.143539+0000","last_clean":"2023-10-09T05:52:56.143539+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:52:56.143539+0000","last_undegraded":"2023-10-09T05:52:56.143539+0000","last_fullsized":"2023-10-09T05:52:56.143539+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":196,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.143088+0000","last_change":"2023-10-09T05:48:54.494715+0000","last_active":"2023-10-09T05:52:56.143088+0000","last_peered":"2023-10-09T05:52:56.143088+0000","last_clean":"2023-10-09T05:52:56.143088+0000","last_became_active":"2023-10-09T05:48:54.493988+0000","last_became_peered":"2023-10-09T05:48:54.493988+0000","last_unstale":"2023-10-09T05:52:56.143088+0000","last_undegraded":"2023-10-09T05:52:56.143088+0000","last_fullsized":"2023-10-09T05:52:56.143088+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":360,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.144385+0000","last_change":"2023-10-09T05:48:54.493193+0000","last_active":"2023-10-09T05:52:56.144385+0000","last_peered":"2023-10-09T05:52:56.144385+0000","last_clean":"2023-10-09T05:52:56.144385+0000","last_became_active":"2023-10-09T05:48:54.492886+0000","last_became_peered":"2023-10-09T05:48:54.492886+0000","last_unstale":"2023-10-09T05:52:56.144385+0000","last_undegraded":"2023-10-09T05:52:56.144385+0000","last_fullsized":"2023-10-09T05:52:56.144385+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":361,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.142561+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:52:56.142561+0000","last_peered":"2023-10-09T05:52:56.142561+0000","last_clean":"2023-10-09T05:52:56.142561+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:52:56.142561+0000","last_undegraded":"2023-10-09T05:52:56.142561+0000","last_fullsized":"2023-10-09T05:52:56.142561+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":354,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.144075+0000","last_change":"2023-10-09T05:48:54.492708+0000","last_active":"2023-10-09T05:52:56.144075+0000","last_peered":"2023-10-09T05:52:56.144075+0000","last_clean":"2023-10-09T05:52:56.144075+0000","last_became_active":"2023-10-09T05:48:54.492382+0000","last_became_peered":"2023-10-09T05:48:54.492382+0000","last_unstale":"2023-10-09T05:52:56.144075+0000","last_undegraded":"2023-10-09T05:52:56.144075+0000","last_fullsized":"2023-10-09T05:52:56.144075+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":328,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.096791+0000","last_change":"2023-10-09T05:50:18.048706+0000","last_active":"2023-10-09T05:52:58.096791+0000","last_peered":"2023-10-09T05:52:58.096791+0000","last_clean":"2023-10-09T05:52:58.096791+0000","last_became_active":"2023-10-09T05:50:18.048410+0000","last_became_peered":"2023-10-09T05:50:18.048410+0000","last_unstale":"2023-10-09T05:52:58.096791+0000","last_undegraded":"2023-10-09T05:52:58.096791+0000","last_fullsized":"2023-10-09T05:52:58.096791+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.144072+0000","last_change":"2023-10-09T05:50:18.072866+0000","last_active":"2023-10-09T05:52:56.144072+0000","last_peered":"2023-10-09T05:52:56.144072+0000","last_clean":"2023-10-09T05:52:56.144072+0000","last_became_active":"2023-10-09T05:50:18.072550+0000","last_became_peered":"2023-10-09T05:50:18.072550+0000","last_unstale":"2023-10-09T05:52:56.144072+0000","last_undegraded":"2023-10-09T05:52:56.144072+0000","last_fullsized":"2023-10-09T05:52:56.144072+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":752,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.516547+0000","last_change":"2023-10-09T05:50:18.055235+0000","last_active":"2023-10-09T05:52:56.516547+0000","last_peered":"2023-10-09T05:52:56.516547+0000","last_clean":"2023-10-09T05:52:56.516547+0000","last_became_active":"2023-10-09T05:50:18.054656+0000","last_became_peered":"2023-10-09T05:50:18.054656+0000","last_unstale":"2023-10-09T05:52:56.516547+0000","last_undegraded":"2023-10-09T05:52:56.516547+0000","last_fullsized":"2023-10-09T05:52:56.516547+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:33.760262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":345,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.114236+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:52:58.114236+0000","last_peered":"2023-10-09T05:52:58.114236+0000","last_clean":"2023-10-09T05:52:58.114236+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:52:58.114236+0000","last_undegraded":"2023-10-09T05:52:58.114236+0000","last_fullsized":"2023-10-09T05:52:58.114236+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":23,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.106128+0000","last_change":"2023-10-09T05:51:43.248409+0000","last_active":"2023-10-09T05:52:58.106128+0000","last_peered":"2023-10-09T05:52:58.106128+0000","last_clean":"2023-10-09T05:52:58.106128+0000","last_became_active":"2023-10-09T05:51:43.154898+0000","last_became_peered":"2023-10-09T05:51:43.154898+0000","last_unstale":"2023-10-09T05:52:58.106128+0000","last_undegraded":"2023-10-09T05:52:58.106128+0000","last_fullsized":"2023-10-09T05:52:58.106128+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":853,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.109080+0000","last_change":"2023-10-09T05:48:54.592642+0000","last_active":"2023-10-09T05:52:58.109080+0000","last_peered":"2023-10-09T05:52:58.109080+0000","last_clean":"2023-10-09T05:52:58.109080+0000","last_became_active":"2023-10-09T05:48:54.491917+0000","last_became_peered":"2023-10-09T05:48:54.491917+0000","last_unstale":"2023-10-09T05:52:58.109080+0000","last_undegraded":"2023-10-09T05:52:58.109080+0000","last_fullsized":"2023-10-09T05:52:58.109080+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":328,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.143291+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:52:56.143291+0000","last_peered":"2023-10-09T05:52:56.143291+0000","last_clean":"2023-10-09T05:52:56.143291+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:52:56.143291+0000","last_undegraded":"2023-10-09T05:52:56.143291+0000","last_fullsized":"2023-10-09T05:52:56.143291+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":185,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.090006+0000","last_change":"2023-10-09T05:50:18.043775+0000","last_active":"2023-10-09T05:52:58.090006+0000","last_peered":"2023-10-09T05:52:58.090006+0000","last_clean":"2023-10-09T05:52:58.090006+0000","last_became_active":"2023-10-09T05:50:18.043047+0000","last_became_peered":"2023-10-09T05:50:18.043047+0000","last_unstale":"2023-10-09T05:52:58.090006+0000","last_undegraded":"2023-10-09T05:52:58.090006+0000","last_fullsized":"2023-10-09T05:52:58.090006+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:28.456979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":683,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.089347+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:52:58.089347+0000","last_peered":"2023-10-09T05:52:58.089347+0000","last_clean":"2023-10-09T05:52:58.089347+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:52:58.089347+0000","last_undegraded":"2023-10-09T05:52:58.089347+0000","last_fullsized":"2023-10-09T05:52:58.089347+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":144,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.087656+0000","last_change":"2023-10-09T05:50:18.063466+0000","last_active":"2023-10-09T05:52:58.087656+0000","last_peered":"2023-10-09T05:52:58.087656+0000","last_clean":"2023-10-09T05:52:58.087656+0000","last_became_active":"2023-10-09T05:50:18.063120+0000","last_became_peered":"2023-10-09T05:50:18.063120+0000","last_unstale":"2023-10-09T05:52:58.087656+0000","last_undegraded":"2023-10-09T05:52:58.087656+0000","last_fullsized":"2023-10-09T05:52:58.087656+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:13.683314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":167,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.114999+0000","last_change":"2023-10-09T05:50:18.069402+0000","last_active":"2023-10-09T05:52:58.114999+0000","last_peered":"2023-10-09T05:52:58.114999+0000","last_clean":"2023-10-09T05:52:58.114999+0000","last_became_active":"2023-10-09T05:50:18.069196+0000","last_became_peered":"2023-10-09T05:50:18.069196+0000","last_unstale":"2023-10-09T05:52:58.114999+0000","last_undegraded":"2023-10-09T05:52:58.114999+0000","last_fullsized":"2023-10-09T05:52:58.114999+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":1009,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.109398+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:52:58.109398+0000","last_peered":"2023-10-09T05:52:58.109398+0000","last_clean":"2023-10-09T05:52:58.109398+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:52:58.109398+0000","last_undegraded":"2023-10-09T05:52:58.109398+0000","last_fullsized":"2023-10-09T05:52:58.109398+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":513,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.096649+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:52:58.096649+0000","last_peered":"2023-10-09T05:52:58.096649+0000","last_clean":"2023-10-09T05:52:58.096649+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:52:58.096649+0000","last_undegraded":"2023-10-09T05:52:58.096649+0000","last_fullsized":"2023-10-09T05:52:58.096649+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":135,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.106865+0000","last_change":"2023-10-09T05:48:25.068759+0000","last_active":"2023-10-09T05:52:58.106865+0000","last_peered":"2023-10-09T05:52:58.106865+0000","last_clean":"2023-10-09T05:52:58.106865+0000","last_became_active":"2023-10-09T05:48:25.068358+0000","last_became_peered":"2023-10-09T05:48:25.068358+0000","last_unstale":"2023-10-09T05:52:58.106865+0000","last_undegraded":"2023-10-09T05:52:58.106865+0000","last_fullsized":"2023-10-09T05:52:58.106865+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":329,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.144664+0000","last_change":"2023-10-09T05:44:50.495966+0000","last_active":"2023-10-09T05:52:56.144664+0000","last_peered":"2023-10-09T05:52:56.144664+0000","last_clean":"2023-10-09T05:52:56.144664+0000","last_became_active":"2023-10-09T05:44:50.495628+0000","last_became_peered":"2023-10-09T05:44:50.495628+0000","last_unstale":"2023-10-09T05:52:56.144664+0000","last_undegraded":"2023-10-09T05:52:56.144664+0000","last_fullsized":"2023-10-09T05:52:56.144664+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":885,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.096734+0000","last_change":"2023-10-09T05:49:36.046279+0000","last_active":"2023-10-09T05:52:58.096734+0000","last_peered":"2023-10-09T05:52:58.096734+0000","last_clean":"2023-10-09T05:52:58.096734+0000","last_became_active":"2023-10-09T05:49:36.045828+0000","last_became_peered":"2023-10-09T05:49:36.045828+0000","last_unstale":"2023-10-09T05:52:58.096734+0000","last_undegraded":"2023-10-09T05:52:58.096734+0000","last_fullsized":"2023-10-09T05:52:58.096734+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":356,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.517445+0000","last_change":"2023-10-09T05:51:31.682830+0000","last_active":"2023-10-09T05:52:56.517445+0000","last_peered":"2023-10-09T05:52:56.517445+0000","last_clean":"2023-10-09T05:52:56.517445+0000","last_became_active":"2023-10-09T05:51:31.682566+0000","last_became_peered":"2023-10-09T05:51:31.682566+0000","last_unstale":"2023-10-09T05:52:56.517445+0000","last_undegraded":"2023-10-09T05:52:56.517445+0000","last_fullsized":"2023-10-09T05:52:56.517445+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:42.916316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":352,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.517108+0000","last_change":"2023-10-09T05:48:54.489115+0000","last_active":"2023-10-09T05:52:56.517108+0000","last_peered":"2023-10-09T05:52:56.517108+0000","last_clean":"2023-10-09T05:52:56.517108+0000","last_became_active":"2023-10-09T05:48:54.488783+0000","last_became_peered":"2023-10-09T05:48:54.488783+0000","last_unstale":"2023-10-09T05:52:56.517108+0000","last_undegraded":"2023-10-09T05:52:56.517108+0000","last_fullsized":"2023-10-09T05:52:56.517108+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:13.193778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":546,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.096205+0000","last_change":"2023-10-09T05:44:51.489799+0000","last_active":"2023-10-09T05:52:58.096205+0000","last_peered":"2023-10-09T05:52:58.096205+0000","last_clean":"2023-10-09T05:52:58.096205+0000","last_became_active":"2023-10-09T05:44:51.489301+0000","last_became_peered":"2023-10-09T05:44:51.489301+0000","last_unstale":"2023-10-09T05:52:58.096205+0000","last_undegraded":"2023-10-09T05:52:58.096205+0000","last_fullsized":"2023-10-09T05:52:58.096205+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":162,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.090109+0000","last_change":"2023-10-09T05:51:43.127835+0000","last_active":"2023-10-09T05:52:58.090109+0000","last_peered":"2023-10-09T05:52:58.090109+0000","last_clean":"2023-10-09T05:52:58.090109+0000","last_became_active":"2023-10-09T05:51:43.127489+0000","last_became_peered":"2023-10-09T05:51:43.127489+0000","last_unstale":"2023-10-09T05:52:58.090109+0000","last_undegraded":"2023-10-09T05:52:58.090109+0000","last_fullsized":"2023-10-09T05:52:58.090109+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:20.191774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":338,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.485724+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:52:56.485724+0000","last_peered":"2023-10-09T05:52:56.485724+0000","last_clean":"2023-10-09T05:52:56.485724+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:52:56.485724+0000","last_undegraded":"2023-10-09T05:52:56.485724+0000","last_fullsized":"2023-10-09T05:52:56.485724+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":310,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.095166+0000","last_change":"2023-10-09T05:48:25.066163+0000","last_active":"2023-10-09T05:52:58.095166+0000","last_peered":"2023-10-09T05:52:58.095166+0000","last_clean":"2023-10-09T05:52:58.095166+0000","last_became_active":"2023-10-09T05:48:25.065145+0000","last_became_peered":"2023-10-09T05:48:25.065145+0000","last_unstale":"2023-10-09T05:52:58.095166+0000","last_undegraded":"2023-10-09T05:52:58.095166+0000","last_fullsized":"2023-10-09T05:52:58.095166+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":616,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.687202+0000","last_change":"2023-10-09T05:52:58.687202+0000","last_active":"2023-10-09T05:52:58.687202+0000","last_peered":"2023-10-09T05:52:58.687202+0000","last_clean":"2023-10-09T05:52:58.687202+0000","last_became_active":"2023-10-09T05:50:19.054850+0000","last_became_peered":"2023-10-09T05:50:19.054850+0000","last_unstale":"2023-10-09T05:52:58.687202+0000","last_undegraded":"2023-10-09T05:52:58.687202+0000","last_fullsized":"2023-10-09T05:52:58.687202+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":303,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.096812+0000","last_change":"2023-10-09T05:48:25.067326+0000","last_active":"2023-10-09T05:52:58.096812+0000","last_peered":"2023-10-09T05:52:58.096812+0000","last_clean":"2023-10-09T05:52:58.096812+0000","last_became_active":"2023-10-09T05:48:25.066977+0000","last_became_peered":"2023-10-09T05:48:25.066977+0000","last_unstale":"2023-10-09T05:52:58.096812+0000","last_undegraded":"2023-10-09T05:52:58.096812+0000","last_fullsized":"2023-10-09T05:52:58.096812+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":860,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.095076+0000","last_change":"2023-10-09T05:44:50.482211+0000","last_active":"2023-10-09T05:52:58.095076+0000","last_peered":"2023-10-09T05:52:58.095076+0000","last_clean":"2023-10-09T05:52:58.095076+0000","last_became_active":"2023-10-09T05:44:50.481749+0000","last_became_peered":"2023-10-09T05:44:50.481749+0000","last_unstale":"2023-10-09T05:52:58.095076+0000","last_undegraded":"2023-10-09T05:52:58.095076+0000","last_fullsized":"2023-10-09T05:52:58.095076+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":348,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.517141+0000","last_change":"2023-10-09T05:48:54.585691+0000","last_active":"2023-10-09T05:52:56.517141+0000","last_peered":"2023-10-09T05:52:56.517141+0000","last_clean":"2023-10-09T05:52:56.517141+0000","last_became_active":"2023-10-09T05:48:54.498259+0000","last_became_peered":"2023-10-09T05:48:54.498259+0000","last_unstale":"2023-10-09T05:52:56.517141+0000","last_undegraded":"2023-10-09T05:52:56.517141+0000","last_fullsized":"2023-10-09T05:52:56.517141+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.684612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":312,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.142679+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:52:56.142679+0000","last_peered":"2023-10-09T05:52:56.142679+0000","last_clean":"2023-10-09T05:52:56.142679+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:52:56.142679+0000","last_undegraded":"2023-10-09T05:52:56.142679+0000","last_fullsized":"2023-10-09T05:52:56.142679+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":339,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.485865+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:52:56.485865+0000","last_peered":"2023-10-09T05:52:56.485865+0000","last_clean":"2023-10-09T05:52:56.485865+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:52:56.485865+0000","last_undegraded":"2023-10-09T05:52:56.485865+0000","last_fullsized":"2023-10-09T05:52:56.485865+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":34,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.090061+0000","last_change":"2023-10-09T05:51:13.060067+0000","last_active":"2023-10-09T05:52:58.090061+0000","last_peered":"2023-10-09T05:52:58.090061+0000","last_clean":"2023-10-09T05:52:58.090061+0000","last_became_active":"2023-10-09T05:51:13.059725+0000","last_became_peered":"2023-10-09T05:51:13.059725+0000","last_unstale":"2023-10-09T05:52:58.090061+0000","last_undegraded":"2023-10-09T05:52:58.090061+0000","last_fullsized":"2023-10-09T05:52:58.090061+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:43.328688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":358,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.112949+0000","last_change":"2023-10-09T05:48:54.583404+0000","last_active":"2023-10-09T05:52:58.112949+0000","last_peered":"2023-10-09T05:52:58.112949+0000","last_clean":"2023-10-09T05:52:58.112949+0000","last_became_active":"2023-10-09T05:48:54.492565+0000","last_became_peered":"2023-10-09T05:48:54.492565+0000","last_unstale":"2023-10-09T05:52:58.112949+0000","last_undegraded":"2023-10-09T05:52:58.112949+0000","last_fullsized":"2023-10-09T05:52:58.112949+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":360,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.143335+0000","last_change":"2023-10-09T05:48:54.493079+0000","last_active":"2023-10-09T05:52:56.143335+0000","last_peered":"2023-10-09T05:52:56.143335+0000","last_clean":"2023-10-09T05:52:56.143335+0000","last_became_active":"2023-10-09T05:48:54.492013+0000","last_became_peered":"2023-10-09T05:48:54.492013+0000","last_unstale":"2023-10-09T05:52:56.143335+0000","last_undegraded":"2023-10-09T05:52:56.143335+0000","last_fullsized":"2023-10-09T05:52:56.143335+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":317,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.114165+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:52:58.114165+0000","last_peered":"2023-10-09T05:52:58.114165+0000","last_clean":"2023-10-09T05:52:58.114165+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:52:58.114165+0000","last_undegraded":"2023-10-09T05:52:58.114165+0000","last_fullsized":"2023-10-09T05:52:58.114165+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":25,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.143003+0000","last_change":"2023-10-09T05:51:31.668739+0000","last_active":"2023-10-09T05:52:56.143003+0000","last_peered":"2023-10-09T05:52:56.143003+0000","last_clean":"2023-10-09T05:52:56.143003+0000","last_became_active":"2023-10-09T05:51:31.668481+0000","last_became_peered":"2023-10-09T05:51:31.668481+0000","last_unstale":"2023-10-09T05:52:56.143003+0000","last_undegraded":"2023-10-09T05:52:56.143003+0000","last_fullsized":"2023-10-09T05:52:56.143003+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":82,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.094527+0000","last_change":"2023-10-09T05:48:25.069406+0000","last_active":"2023-10-09T05:52:58.094527+0000","last_peered":"2023-10-09T05:52:58.094527+0000","last_clean":"2023-10-09T05:52:58.094527+0000","last_became_active":"2023-10-09T05:48:25.069064+0000","last_became_peered":"2023-10-09T05:48:25.069064+0000","last_unstale":"2023-10-09T05:52:58.094527+0000","last_undegraded":"2023-10-09T05:52:58.094527+0000","last_fullsized":"2023-10-09T05:52:58.094527+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.567858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":164,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T05:52:58.089260+0000","last_change":"2023-10-09T05:48:25.063854+0000","last_active":"2023-10-09T05:52:58.089260+0000","last_peered":"2023-10-09T05:52:58.089260+0000","last_clean":"2023-10-09T05:52:58.089260+0000","last_became_active":"2023-10-09T05:48:25.063579+0000","last_became_peered":"2023-10-09T05:48:25.063579+0000","last_unstale":"2023-10-09T05:52:58.089260+0000","last_undegraded":"2023-10-09T05:52:58.089260+0000","last_fullsized":"2023-10-09T05:52:58.089260+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:02.389789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":353,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.516677+0000","last_change":"2023-10-09T05:51:13.071891+0000","last_active":"2023-10-09T05:52:56.516677+0000","last_peered":"2023-10-09T05:52:56.516677+0000","last_clean":"2023-10-09T05:52:56.516677+0000","last_became_active":"2023-10-09T05:51:13.058378+0000","last_became_peered":"2023-10-09T05:51:13.058378+0000","last_unstale":"2023-10-09T05:52:56.516677+0000","last_undegraded":"2023-10-09T05:52:56.516677+0000","last_fullsized":"2023-10-09T05:52:56.516677+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:41.278366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":345,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T05:52:56.142553+0000","last_change":"2023-10-09T05:48:54.492580+0000","last_active":"2023-10-09T05:52:56.142553+0000","last_peered":"2023-10-09T05:52:56.142553+0000","last_clean":"2023-10-09T05:52:56.142553+0000","last_became_active":"2023-10-09T05:48:54.492259+0000","last_became_peered":"2023-10-09T05:48:54.492259+0000","last_unstale":"2023-10-09T05:52:56.142553+0000","last_undegraded":"2023-10-09T05:52:56.142553+0000","last_fullsized":"2023-10-09T05:52:56.142553+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":10932,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":142,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846813,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":913676,"kb_used_data":7940,"kb_used_omap":14,"kb_used_meta":905713,"kb_avail":92831476,"statfs":{"total":95995035648,"available":95059431424,"internally_reserved":0,"allocated":8130560,"data_stored":7441874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":927450970},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":12.824999999999999,"5min":4.1820000000000004,"15min":4.3869999999999996},"min":{"1min":0.22800000000000001,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":105.39100000000001,"5min":105.39100000000001,"15min":220.77500000000001},"last":0.436},{"interface":"front","average":{"1min":12.695,"5min":4.1100000000000003,"15min":4.359},"min":{"1min":0.29799999999999999,"5min":0.25900000000000001,"15min":0.22700000000000001},"max":{"1min":105.417,"5min":105.417,"15min":220.78700000000001},"last":0.38600000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":7.9139999999999997,"5min":2.363,"15min":3.1760000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":114.422,"5min":114.422,"15min":190.68100000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":7.8090000000000002,"5min":2.5760000000000001,"15min":3.234},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":111.708,"5min":111.708,"15min":191.09200000000001},"last":0.53000000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":7.2279999999999998,"5min":2.6629999999999998,"15min":2.0590000000000002},"min":{"1min":0.374,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":100.684,"5min":100.684,"15min":100.684},"last":0.36899999999999999},{"interface":"front","average":{"1min":7.4290000000000003,"5min":2.4990000000000001,"15min":2.0129999999999999},"min":{"1min":0.32000000000000001,"5min":0.30499999999999999,"15min":0.28100000000000003},"max":{"1min":100.736,"5min":100.736,"15min":100.736},"last":0.40699999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":8.4090000000000007,"5min":4.3399999999999999,"15min":4.569},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":103.29300000000001,"5min":103.29300000000001,"15min":185.17099999999999},"last":0.498},{"interface":"front","average":{"1min":11.114000000000001,"5min":4.8150000000000004,"15min":4.6909999999999998},"min":{"1min":0.39700000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":103.202,"5min":103.202,"15min":183.35499999999999},"last":0.41599999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":9.8770000000000007,"5min":3.4350000000000001,"15min":1.5529999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.25700000000000001},"max":{"1min":159.50299999999999,"5min":159.50299999999999,"15min":159.50299999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":10.189,"5min":3.5230000000000001,"15min":1.611},"min":{"1min":0.32600000000000001,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":159.483,"5min":159.483,"15min":159.483},"last":0.505}]},{"osd":5,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.476,"15min":1.0329999999999999},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.26900000000000002},"max":{"1min":4.1319999999999997,"5min":49.210000000000001,"15min":49.210000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.246,"5min":1.7749999999999999,"15min":1.101},"min":{"1min":0.34999999999999998,"5min":0.308,"15min":0.28299999999999997},"max":{"1min":9.6739999999999995,"5min":49.274999999999999,"15min":49.274999999999999},"last":0.48199999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":0.90200000000000002,"15min":0.75600000000000001},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.1720000000000006,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.095,"15min":0.85499999999999998},"min":{"1min":0.313,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.8200000000000003,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.52000000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626839,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1426936,"kb_used_data":530992,"kb_used_omap":0,"kb_used_meta":895936,"kb_avail":92318216,"statfs":{"total":95995035648,"available":94533853184,"internally_reserved":0,"allocated":543735808,"data_stored":543059765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917438464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:45 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":2.46,"15min":3.9430000000000001},"min":{"1min":0.255,"5min":0.248,"15min":0.20999999999999999},"max":{"1min":7.3449999999999998,"5min":42.869,"15min":162.08799999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.2240000000000002,"5min":2.5760000000000001,"15min":3.8809999999999998},"min":{"1min":0.26200000000000001,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":9.0749999999999993,"5min":42.892000000000003,"15min":161.90899999999999},"last":0.54700000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:52:10 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.9359999999999999,"15min":3.867},"min":{"1min":0.27300000000000002,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":9.3469999999999995,"5min":48.729999999999997,"15min":228.24299999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.274,"5min":2.1699999999999999,"15min":3.8340000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":13.843,"5min":48.741999999999997,"15min":228.22499999999999},"last":0.58599999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:52:45 2023","interfaces":[{"interface":"back","average":{"1min":6.8739999999999997,"5min":2.2530000000000001,"15min":3.5059999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27200000000000002},"max":{"1min":91.049999999999997,"5min":91.049999999999997,"15min":115.40600000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":7.3049999999999997,"5min":2.4420000000000002,"15min":3.6789999999999998},"min":{"1min":0.34000000000000002,"5min":0.33500000000000002,"15min":0.28699999999999998},"max":{"1min":91.105000000000004,"5min":91.105000000000004,"15min":115.297},"last":0.41299999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:52:10 2023","interfaces":[{"interface":"back","average":{"1min":10.012,"5min":3.4159999999999999,"15min":4.4009999999999998},"min":{"1min":0.35999999999999999,"5min":0.32800000000000001,"15min":0.26800000000000002},"max":{"1min":145.124,"5min":145.124,"15min":145.124},"last":0.45200000000000001},{"interface":"front","average":{"1min":10.629,"5min":3.681,"15min":4.4610000000000003},"min":{"1min":0.37,"5min":0.33800000000000002,"15min":0.30299999999999999},"max":{"1min":145.26599999999999,"5min":145.26599999999999,"15min":145.26599999999999},"last":0.55900000000000005}]},{"osd":4,"last update":"Mon Oct 9 05:52:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":3.9100000000000001,"15min":1.9430000000000001},"min":{"1min":0.28599999999999998,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":9.4979999999999993,"5min":257.375,"15min":257.375},"last":0.504},{"interface":"front","average":{"1min":1.4590000000000001,"5min":4.0060000000000002,"15min":2.1760000000000002},"min":{"1min":0.309,"5min":0.26100000000000001,"15min":0.246},"max":{"1min":5.5659999999999998,"5min":257.61200000000002,"15min":257.61200000000002},"last":0.48999999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:52:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":3.9100000000000001,"15min":2.2799999999999998},"min":{"1min":0.33100000000000002,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":6.7839999999999998,"5min":256.05700000000002,"15min":256.05700000000002},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.7230000000000001,"5min":4.1059999999999999,"15min":2.3879999999999999},"min":{"1min":0.35799999999999998,"5min":0.29899999999999999,"15min":0.27500000000000002},"max":{"1min":6.6289999999999996,"5min":256.08499999999998,"15min":256.08499999999998},"last":0.52500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:52:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4850000000000001,"5min":0.88100000000000001,"15min":1.091},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.308},"max":{"1min":7.3120000000000003,"5min":7.3120000000000003,"15min":115.514},"last":0.745},{"interface":"front","average":{"1min":1.9430000000000001,"5min":1.202,"15min":1.333},"min":{"1min":0.38300000000000001,"5min":0.308,"15min":0.30599999999999999},"max":{"1min":6.601,"5min":6.601,"15min":116.53100000000001},"last":0.57499999999999996}]}]},{"osd":1,"up_from":93,"seq":399431958800,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":719372,"kb_used_data":3844,"kb_used_omap":7,"kb_used_meta":715512,"kb_avail":93025780,"statfs":{"total":95995035648,"available":95258398720,"internally_reserved":0,"allocated":3936256,"data_stored":2718907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":732684814},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":4.3159999999999998,"5min":3.2010000000000001,"15min":4.6799999999999997},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":34.058,"5min":93.176000000000002,"15min":207.858},"last":0.34499999999999997},{"interface":"front","average":{"1min":4.1699999999999999,"5min":3.2130000000000001,"15min":5.1470000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.182},"max":{"1min":34.634,"5min":92.593999999999994,"15min":206.97200000000001},"last":0.33400000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3940000000000001,"5min":2.1469999999999998,"15min":3.6890000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.19800000000000001},"max":{"1min":7.2510000000000003,"5min":11.914,"15min":101.367},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.373,"5min":1.7310000000000001,"15min":3.452},"min":{"1min":0.27800000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":3.6259999999999999,"5min":16.702999999999999,"15min":99.861000000000004},"last":0.311}]},{"osd":3,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1230000000000002,"5min":3.2530000000000001,"15min":5.2149999999999999},"min":{"1min":0.25900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.053000000000001,"5min":92.353999999999999,"15min":162.33699999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":3.863,"5min":3.1389999999999998,"15min":5.4470000000000001},"min":{"1min":0.36399999999999999,"5min":0.28599999999999998,"15min":0.26000000000000001},"max":{"1min":16.617999999999999,"5min":93.117999999999995,"15min":162.376},"last":0.44800000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":1.484,"15min":2.3839999999999999},"min":{"1min":0.40699999999999997,"5min":0.312,"15min":0.29799999999999999},"max":{"1min":5.2629999999999999,"5min":29.605,"15min":141.32900000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":2.153,"5min":1.5840000000000001,"15min":2.0369999999999999},"min":{"1min":0.34000000000000002,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":12.044,"5min":29.515999999999998,"15min":141.15899999999999},"last":0.498}]},{"osd":5,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5710000000000002,"5min":2.1469999999999998,"15min":2.5880000000000001},"min":{"1min":0.29399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.1749999999999998,"5min":32.774000000000001,"15min":235.18700000000001},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.734,"5min":1.5589999999999999,"15min":2.742},"min":{"1min":0.41399999999999998,"5min":0.34499999999999997,"15min":0.32200000000000001},"max":{"1min":5.2910000000000004,"5min":36.976999999999997,"15min":235.79300000000001},"last":0.73099999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5910000000000002,"5min":1.8340000000000001,"15min":2.613},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.316},"max":{"1min":7.2169999999999996,"5min":11.837999999999999,"15min":103.60899999999999},"last":0.78300000000000003},{"interface":"front","average":{"1min":1.7010000000000001,"5min":1.208,"15min":2.601},"min":{"1min":0.37,"5min":0.37,"15min":0.33400000000000002},"max":{"1min":5.1900000000000004,"5min":9.641,"15min":103.679},"last":0.71499999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:52:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1520000000000001,"5min":1.7330000000000001,"15min":2.1059999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.35999999999999999},"max":{"1min":12.093999999999999,"5min":35.021000000000001,"15min":126.68300000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.8879999999999999,"15min":2.1960000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.35499999999999998},"max":{"1min":12.196,"5min":35.493000000000002,"15min":126.738},"last":0.82899999999999996}]}]},{"osd":0,"up_from":88,"seq":377957122326,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":672524,"kb_used_data":4484,"kb_used_omap":8,"kb_used_meta":668023,"kb_avail":93072628,"statfs":{"total":95995035648,"available":95306371072,"internally_reserved":0,"allocated":4591616,"data_stored":3308385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":684056328},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":22,"apply_latency_ms":22,"commit_latency_ns":22000000,"apply_latency_ns":22000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:52:08 2023","interfaces":[{"interface":"back","average":{"1min":3.133,"5min":2.4319999999999999,"15min":1.992},"min":{"1min":0.186,"5min":0.186,"15min":0.17199999999999999},"max":{"1min":18.164999999999999,"5min":63.534999999999997,"15min":63.534999999999997},"last":0.313},{"interface":"front","average":{"1min":3.5619999999999998,"5min":2.4129999999999998,"15min":2.4489999999999998},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":16.484000000000002,"5min":63.677999999999997,"15min":63.677999999999997},"last":0.33900000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:52:39 2023","interfaces":[{"interface":"back","average":{"1min":8.0429999999999993,"5min":3.9140000000000001,"15min":3.4500000000000002},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.20000000000000001},"max":{"1min":70.188999999999993,"5min":70.188999999999993,"15min":92.230000000000004},"last":0.28399999999999997},{"interface":"front","average":{"1min":7.0819999999999999,"5min":3.3399999999999999,"15min":3.161},"min":{"1min":0.25,"5min":0.221,"15min":0.221},"max":{"1min":71.144000000000005,"5min":71.144000000000005,"15min":92.247},"last":0.35399999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:52:08 2023","interfaces":[{"interface":"back","average":{"1min":4.8259999999999996,"5min":2.1219999999999999,"15min":3.9870000000000001},"min":{"1min":0.307,"5min":0.253,"15min":0.218},"max":{"1min":59.838999999999999,"5min":59.838999999999999,"15min":123.024},"last":0.41399999999999998},{"interface":"front","average":{"1min":4.6399999999999997,"5min":2.8279999999999998,"15min":4.0220000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":59.859000000000002,"5min":72.632999999999996,"15min":99.162000000000006},"last":0.39300000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":3.2330000000000001,"15min":2.0750000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.25700000000000001},"max":{"1min":3.3660000000000001,"5min":149.63999999999999,"15min":149.63999999999999},"last":0.434},{"interface":"front","average":{"1min":1.9850000000000001,"5min":3.3300000000000001,"15min":2.5099999999999998},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":12.340999999999999,"5min":149.65899999999999,"15min":149.65899999999999},"last":0.53800000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:52:39 2023","interfaces":[{"interface":"back","average":{"1min":1.855,"5min":3.847,"15min":3.9660000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.317},"max":{"1min":12.282999999999999,"5min":152.20500000000001,"15min":281.733},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.387,"5min":4.468,"15min":4.2169999999999996},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30199999999999999},"max":{"1min":5.4989999999999997,"5min":154.30600000000001,"15min":282.95400000000001},"last":0.51600000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:52:39 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":1.5189999999999999,"15min":1.8049999999999999},"min":{"1min":0.36399999999999999,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":12.377000000000001,"5min":15.789,"15min":19.579000000000001},"last":0.5},{"interface":"front","average":{"1min":1.27,"5min":1.7,"15min":1.9890000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":5.5700000000000003,"5min":19.710999999999999,"15min":19.710999999999999},"last":0.55500000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:52:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.504,"15min":1.4770000000000001},"min":{"1min":0.38500000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":7.6040000000000001,"5min":15.933999999999999,"15min":25.378},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.486,"15min":1.861},"min":{"1min":0.39100000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":12.41,"5min":15.641,"15min":25.436},"last":0.57999999999999996}]}]},{"osd":2,"up_from":98,"seq":420906795274,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1468476,"kb_used_data":704564,"kb_used_omap":11,"kb_used_meta":763892,"kb_avail":92276676,"statfs":{"total":95995035648,"available":94491316224,"internally_reserved":0,"allocated":721473536,"data_stored":720033290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":782226005},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:46 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.99,"15min":3.4430000000000001},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.17799999999999999},"max":{"1min":12.129,"5min":12.829000000000001,"15min":162.285},"last":0.377},{"interface":"front","average":{"1min":1.976,"5min":1.9670000000000001,"15min":3.891},"min":{"1min":0.26500000000000001,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":8.2379999999999995,"5min":10.781000000000001,"15min":162.643},"last":0.30399999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:52:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9169999999999998,"5min":4.3170000000000002,"15min":3.7959999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.193},"max":{"1min":11.69,"5min":125.77500000000001,"15min":125.77500000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.1589999999999998,"5min":3.7850000000000001,"15min":3.1040000000000001},"min":{"1min":0.28599999999999998,"5min":0.24199999999999999,"15min":0.20999999999999999},"max":{"1min":6.6299999999999999,"5min":125.857,"15min":125.857},"last":0.63900000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.855,"15min":4.9240000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.21099999999999999},"max":{"1min":9.5129999999999999,"5min":9.5129999999999999,"15min":159.35499999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":2.7160000000000002,"5min":1.7170000000000001,"15min":4.7110000000000003},"min":{"1min":0.29399999999999998,"5min":0.217,"15min":0.217},"max":{"1min":11.303000000000001,"5min":11.303000000000001,"15min":159.196},"last":0.55600000000000005}]},{"osd":4,"last update":"Mon Oct 9 05:52:46 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":2.0920000000000001,"15min":2.0720000000000001},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.28699999999999998},"max":{"1min":5.4450000000000003,"5min":68.052000000000007,"15min":115.84999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.0009999999999999,"5min":2.323,"15min":2.1419999999999999},"min":{"1min":0.36599999999999999,"5min":0.29199999999999998,"15min":0.28699999999999998},"max":{"1min":8.6349999999999998,"5min":68.727000000000004,"15min":116.149},"last":0.60699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:52:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":2.206,"15min":2.1190000000000002},"min":{"1min":0.36099999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.8859999999999992,"5min":64.825000000000003,"15min":64.825000000000003},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.2010000000000001,"15min":1.845},"min":{"1min":0.371,"5min":0.371,"15min":0.26000000000000001},"max":{"1min":5.5229999999999997,"5min":64.582999999999998,"15min":64.582999999999998},"last":0.58999999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:52:46 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.643,"15min":1.6839999999999999},"min":{"1min":0.41199999999999998,"5min":0.33200000000000002,"15min":0.32000000000000001},"max":{"1min":8.6790000000000003,"5min":25.818999999999999,"15min":25.818999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":2.0539999999999998,"5min":1.9159999999999999,"15min":2.0110000000000001},"min":{"1min":0.40100000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":8.7599999999999998,"5min":24.481000000000002,"15min":24.481000000000002},"last":0.57399999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":1.629,"15min":2.0190000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":12.071999999999999,"5min":12.071999999999999,"15min":51.701999999999998},"last":0.71899999999999997},{"interface":"front","average":{"1min":2.2170000000000001,"5min":1.74,"15min":2.2189999999999999},"min":{"1min":0.43099999999999999,"5min":0.378,"15min":0.373},"max":{"1min":6.2679999999999998,"5min":8.7430000000000003,"15min":51.716000000000001},"last":0.622}]}]},{"osd":3,"up_from":103,"seq":442381631748,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1252820,"kb_used_data":498892,"kb_used_omap":4,"kb_used_meta":753915,"kb_avail":92492332,"statfs":{"total":95995035648,"available":94712147968,"internally_reserved":0,"allocated":510865408,"data_stored":509579507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":772009566},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":2.7879999999999998,"5min":3.2040000000000002,"15min":4.0789999999999997},"min":{"1min":0.218,"5min":0.188,"15min":0.184},"max":{"1min":8.5039999999999996,"5min":47.173000000000002,"15min":167.82400000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.2000000000000002,"5min":3.0329999999999999,"15min":4.4950000000000001},"min":{"1min":0.22900000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":8.4879999999999995,"5min":47.908000000000001,"15min":167.89599999999999},"last":0.29799999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":3.7799999999999998,"15min":3.5680000000000001},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":5.9859999999999998,"5min":150.29599999999999,"15min":150.29599999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.893,"5min":4.0460000000000003,"15min":3.746},"min":{"1min":0.216,"5min":0.216,"15min":0.19},"max":{"1min":13.018000000000001,"5min":151.25700000000001,"15min":151.25700000000001},"last":0.32300000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2730000000000001,"5min":1.6120000000000001,"15min":3.669},"min":{"1min":0.32700000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.577,"5min":12.577,"15min":190.43299999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":2.3969999999999998,"5min":1.7250000000000001,"15min":4.0170000000000003},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.7390000000000008,"5min":9.7390000000000008,"15min":190.267},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":2.681,"15min":2.8180000000000001},"min":{"1min":0.31,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":8.0120000000000005,"5min":126.566,"15min":149.84700000000001},"last":0.502},{"interface":"front","average":{"1min":1.4390000000000001,"5min":3.7949999999999999,"15min":2.8210000000000002},"min":{"1min":0.35999999999999999,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":4.8920000000000003,"5min":126.651,"15min":149.74700000000001},"last":0.69999999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.7529999999999999,"15min":2.3370000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.28199999999999997},"max":{"1min":12.509,"5min":12.509,"15min":149.43000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.518,"15min":2.1619999999999999},"min":{"1min":0.39900000000000002,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":4.8140000000000001,"5min":7.8689999999999998,"15min":149.69300000000001},"last":0.66600000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":1.373,"15min":2.0680000000000001},"min":{"1min":0.35899999999999999,"5min":0.34499999999999997,"15min":0.33500000000000002},"max":{"1min":4.3360000000000003,"5min":7.9009999999999998,"15min":149.72399999999999},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.5389999999999999,"5min":1.6759999999999999,"15min":2.379},"min":{"1min":0.35899999999999999,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":12.558999999999999,"5min":12.558999999999999,"15min":149.589},"last":0.60399999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":4.8719999999999999,"5min":2.0870000000000002,"15min":2.4769999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":69.277000000000001,"5min":69.277000000000001,"15min":149.77199999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":4.4809999999999999,"5min":2.0329999999999999,"15min":2.5870000000000002},"min":{"1min":0.48299999999999998,"5min":0.374,"15min":0.373},"max":{"1min":69.299000000000007,"5min":69.299000000000007,"15min":149.80799999999999},"last":0.58899999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019295,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1945596,"kb_used_data":1044148,"kb_used_omap":0,"kb_used_meta":901440,"kb_avail":91799556,"statfs":{"total":95995035648,"available":94002745344,"internally_reserved":0,"allocated":1069207552,"data_stored":1068527191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923074560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7969999999999999,"5min":4.8650000000000002,"15min":3.1499999999999999},"min":{"1min":0.246,"5min":0.219,"15min":0.219},"max":{"1min":7.024,"5min":132.35599999999999,"15min":132.35599999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.3079999999999998,"5min":5.4429999999999996,"15min":3.5510000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":8.3960000000000008,"5min":132.501,"15min":132.501},"last":0.36599999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":6.4989999999999997,"5min":3.2890000000000001,"15min":3.0099999999999998},"min":{"1min":0.34999999999999998,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":59.978999999999999,"5min":68.367000000000004,"15min":187.369},"last":0.40699999999999997},{"interface":"front","average":{"1min":6.1360000000000001,"5min":3.2429999999999999,"15min":3.0819999999999999},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.27700000000000002},"max":{"1min":59.991,"5min":68.376000000000005,"15min":187.35300000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:52:34 2023","interfaces":[{"interface":"back","average":{"1min":5.8710000000000004,"5min":2.5990000000000002,"15min":3.407},"min":{"1min":0.34899999999999998,"5min":0.308,"15min":0.26800000000000002},"max":{"1min":51.768000000000001,"5min":51.768000000000001,"15min":196.93700000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":5.5659999999999998,"5min":2.3199999999999998,"15min":3.3260000000000001},"min":{"1min":0.35099999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":51.753999999999998,"5min":51.753999999999998,"15min":197.08099999999999},"last":0.49099999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:52:02 2023","interfaces":[{"interface":"back","average":{"1min":2.343,"5min":1.929,"15min":4.5419999999999998},"min":{"1min":0.32900000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.347,"5min":30.111999999999998,"15min":211.48699999999999},"last":0.505},{"interface":"front","average":{"1min":1.591,"5min":1.7370000000000001,"15min":4.4050000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.312},"max":{"1min":5.601,"5min":30.709,"15min":211.31100000000001},"last":0.57399999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:52:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.099,"15min":1.1919999999999999},"min":{"1min":0.27300000000000002,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":3.1190000000000002,"5min":12.34,"15min":94.646000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1,"15min":1.107},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.8490000000000002,"5min":10.191000000000001,"15min":94.692999999999998},"last":0.59099999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.93799999999999994,"15min":0.70499999999999996},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.4939999999999998,"5min":9.8889999999999993,"15min":9.8889999999999993},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.92700000000000005,"15min":0.875},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":2.387,"5min":6.0179999999999998,"15min":8.5269999999999992},"last":0.48499999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":0.88600000000000001,"15min":1.034},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.5810000000000004,"5min":5.8650000000000002,"15min":96.131},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.1459999999999999,"5min":0.995,"15min":1.1399999999999999},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.3159999999999998,"5min":7.6580000000000004,"15min":96.465000000000003},"last":0.54100000000000004}]}]},{"osd":5,"up_from":36,"seq":154618823067,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1259564,"kb_used_data":360868,"kb_used_omap":0,"kb_used_meta":898688,"kb_avail":92485588,"statfs":{"total":95995035648,"available":94705242112,"internally_reserved":0,"allocated":369528832,"data_stored":368860303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920256512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:52 2023","interfaces":[{"interface":"back","average":{"1min":5.0279999999999996,"5min":7.5510000000000002,"15min":4.0049999999999999},"min":{"1min":0.245,"5min":0.224,"15min":0.217},"max":{"1min":77.456999999999994,"5min":210.392,"15min":210.392},"last":0.34000000000000002},{"interface":"front","average":{"1min":5.8179999999999996,"5min":7.8650000000000002,"15min":4.2030000000000003},"min":{"1min":0.28899999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":77.828999999999994,"5min":208.43600000000001,"15min":208.43600000000001},"last":0.38900000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:52:11 2023","interfaces":[{"interface":"back","average":{"1min":12.114000000000001,"5min":4.3680000000000003,"15min":3.4969999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.23999999999999999},"max":{"1min":198.35300000000001,"5min":198.35300000000001,"15min":198.35300000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":11.247999999999999,"5min":4.4119999999999999,"15min":3.637},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":198.32900000000001,"5min":198.32900000000001,"15min":198.32900000000001},"last":0.35599999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:52:52 2023","interfaces":[{"interface":"back","average":{"1min":14.33,"5min":4.0650000000000004,"15min":3.0640000000000001},"min":{"1min":0.39800000000000002,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":195.92099999999999,"5min":195.92099999999999,"15min":195.92099999999999},"last":3.2850000000000001},{"interface":"front","average":{"1min":13.996,"5min":4.2279999999999998,"15min":3.0579999999999998},"min":{"1min":0.38600000000000001,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":195.905,"5min":195.905,"15min":195.905},"last":2.6739999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:52:11 2023","interfaces":[{"interface":"back","average":{"1min":6.0309999999999997,"5min":3.379,"15min":4.2030000000000003},"min":{"1min":0.47399999999999998,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":63.069000000000003,"5min":85.069000000000003,"15min":95.236999999999995},"last":1.0169999999999999},{"interface":"front","average":{"1min":5.1680000000000001,"5min":3.3079999999999998,"15min":4.024},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":63.128,"5min":84.814999999999998,"15min":93.471000000000004},"last":2.8109999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:52:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":2.3330000000000002,"15min":1.2030000000000001},"min":{"1min":0.28000000000000003,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.8460000000000001,"5min":130.81,"15min":130.81},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.4890000000000001,"5min":2.5539999999999998,"15min":1.3},"min":{"1min":0.32300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":6.8330000000000002,"5min":133.803,"15min":133.803},"last":0.44700000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:52:26 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.8220000000000001,"15min":1.1899999999999999},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.8639999999999999,"5min":71.426000000000002,"15min":71.426000000000002},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.0550000000000002,"15min":1.2809999999999999},"min":{"1min":0.32300000000000001,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":4.109,"5min":71.498999999999995,"15min":71.498999999999995},"last":0.36699999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:52:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.383,"15min":1.0489999999999999},"min":{"1min":0.36799999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.085,"5min":31.533000000000001,"15min":31.533000000000001},"last":0.437},{"interface":"front","average":{"1min":1.6060000000000001,"5min":1.3759999999999999,"15min":1.0129999999999999},"min":{"1min":0.34300000000000003,"5min":0.316,"15min":0.308},"max":{"1min":8.5809999999999995,"5min":31.855,"15min":31.855},"last":0.46700000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:53:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:07 smithi191 ceph-mon[161669]: from='client.46145 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:53:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:07 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/621374547' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T05:53:07.915 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:07.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:07.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:08.018 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:08.050 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:08.100 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:08.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:08.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:08.196 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:53:08.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[191227]: pgmap v1103: 73 pgs: 73 active+clean; 4.6 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:53:08.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[191227]: failed to encode map e364 with expected crc 2023-10-09T05:53:08.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/621374547' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T05:53:08.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[191227]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T05:53:08.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[191227]: failed to encode map e364 with expected crc 2023-10-09T05:53:08.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[191227]: failed to encode map e364 with expected crc 2023-10-09T05:53:08.208 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:08.239 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:08.285 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:08.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:08.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:08 smithi191 ceph-mon[161669]: pgmap v1103: 73 pgs: 73 active+clean; 4.6 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:53:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:08 smithi191 ceph-mon[161669]: failed to encode map e364 with expected crc 2023-10-09T05:53:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/621374547' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T05:53:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:08 smithi191 ceph-mon[161669]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T05:53:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:08 smithi191 ceph-mon[161669]: failed to encode map e364 with expected crc 2023-10-09T05:53:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:08 smithi191 ceph-mon[161669]: failed to encode map e364 with expected crc 2023-10-09T05:53:08.392 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:08.397 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T05:53:08.419 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:08.463 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[184418]: pgmap v1103: 73 pgs: 73 active+clean; 4.6 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:53:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[184418]: failed to encode map e364 with expected crc 2023-10-09T05:53:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/621374547' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T05:53:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[184418]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T05:53:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[184418]: failed to encode map e364 with expected crc 2023-10-09T05:53:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:08 smithi008 ceph-mon[184418]: failed to encode map e364 with expected crc 2023-10-09T05:53:08.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:08.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:08.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:08.596 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:08.637 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:08.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:08.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:08.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:08.768 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:08.810 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:08.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:08.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:08.917 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:08.954 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:08.963 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:08.997 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:53:08.997 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:53:08.997 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:53:08.997 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:53:08.998 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:53:09.019 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:09.222 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:09.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:09.226 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:53:09.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:09.354 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:09.396 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:09.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:09.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:09.502 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:09.529 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[184418]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.529 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[184418]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.529 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[184418]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T05:53:09.529 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[184418]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.530 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[184418]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.530 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[184418]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.530 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[184418]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.530 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3932374870' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:09.530 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:09.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[191227]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[191227]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[191227]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T05:53:09.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[191227]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.531 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[191227]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.531 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[191227]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.531 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[191227]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.531 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3932374870' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:09.531 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:09 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:09.533 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:09.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:09 smithi191 ceph-mon[161669]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:09 smithi191 ceph-mon[161669]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:09 smithi191 ceph-mon[161669]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T05:53:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:09 smithi191 ceph-mon[161669]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:09 smithi191 ceph-mon[161669]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:09 smithi191 ceph-mon[161669]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:09 smithi191 ceph-mon[161669]: failed to encode map e365 with expected crc 2023-10-09T05:53:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:09 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3932374870' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:09.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:09 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:10.142 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:10.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:10.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:10.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:10.279 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:10.329 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:10.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:10.437 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:10.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[184418]: pgmap v1106: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 4.6 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:53:10.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:10.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[184418]: failed to encode map e366 with expected crc 2023-10-09T05:53:10.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:10.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[184418]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T05:53:10.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:10.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[184418]: failed to encode map e366 with expected crc 2023-10-09T05:53:10.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[191227]: pgmap v1106: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 4.6 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:53:10.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:10.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[191227]: failed to encode map e366 with expected crc 2023-10-09T05:53:10.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:10.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[191227]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T05:53:10.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:10.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:10 smithi008 ceph-mon[191227]: failed to encode map e366 with expected crc 2023-10-09T05:53:10.469 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:10.501 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:53:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:10 smithi191 ceph-mon[161669]: pgmap v1106: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 4.6 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T05:53:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:10 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:10 smithi191 ceph-mon[161669]: failed to encode map e366 with expected crc 2023-10-09T05:53:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:10 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:10 smithi191 ceph-mon[161669]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T05:53:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:10 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:10 smithi191 ceph-mon[161669]: failed to encode map e366 with expected crc 2023-10-09T05:53:10.570 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:53:10.618 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:10.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:10.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:10.727 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:10.752 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:53:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:53:10] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.33.4" 2023-10-09T05:53:10.759 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:10.807 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:10.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:10.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:10.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:10.941 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:10.986 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:10.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:10.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:11.091 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:11.144 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:11.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:11 smithi008 ceph-mon[184418]: failed to encode map e366 with expected crc 2023-10-09T05:53:11.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:11 smithi008 ceph-mon[184418]: failed to encode map e366 with expected crc 2023-10-09T05:53:11.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:11 smithi008 ceph-mon[191227]: failed to encode map e366 with expected crc 2023-10-09T05:53:11.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:11 smithi008 ceph-mon[191227]: failed to encode map e366 with expected crc 2023-10-09T05:53:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:11 smithi191 ceph-mon[161669]: failed to encode map e366 with expected crc 2023-10-09T05:53:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:11 smithi191 ceph-mon[161669]: failed to encode map e366 with expected crc 2023-10-09T05:53:11.845 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:11.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:11.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:11.948 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:11.979 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:12.027 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:12.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:12.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:12.133 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:12.176 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:12.219 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:12.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:12 smithi191 ceph-mon[161669]: pgmap v1108: 89 pgs: 2 creating+peering, 6 unknown, 81 active+clean; 4.6 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:53:12.342 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:12.347 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:12.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:12.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:12.390 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:12 smithi008 ceph-mon[184418]: pgmap v1108: 89 pgs: 2 creating+peering, 6 unknown, 81 active+clean; 4.6 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:53:12.390 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:12 smithi008 ceph-mon[191227]: pgmap v1108: 89 pgs: 2 creating+peering, 6 unknown, 81 active+clean; 4.6 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:53:12.452 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:12.484 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:12.533 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:12.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:12.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:12.640 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:12.670 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:12.697 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:53:13.112 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:53:13.131 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:53:13.163 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:53:13.183 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:53:13.202 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:53:13.221 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:53:13.236 INFO:teuthology.orchestra.run.smithi008.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T05:53:13.253 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:53:13.275 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:53:13.277 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:53:13.277 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:53:13.278 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:53:13.278 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:53:13.278 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:53:13.278 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:53:13.278 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:53:13.278 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:53:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3862149963' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:53:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:13 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:53:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2787089465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:13.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3862149963' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:53:13.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:13 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:53:13.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2787089465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3862149963' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:53:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:13 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:53:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2787089465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:14 smithi191 ceph-mon[161669]: pgmap v1109: 89 pgs: 89 active+clean; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T05:53:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:14 smithi191 ceph-mon[161669]: failed to encode map e367 with expected crc 2023-10-09T05:53:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:14 smithi191 ceph-mon[161669]: failed to encode map e367 with expected crc 2023-10-09T05:53:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:14 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:53:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:14 smithi191 ceph-mon[161669]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T05:53:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:14.610 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[184418]: pgmap v1109: 89 pgs: 89 active+clean; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T05:53:14.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[184418]: failed to encode map e367 with expected crc 2023-10-09T05:53:14.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[184418]: failed to encode map e367 with expected crc 2023-10-09T05:53:14.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:53:14.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[184418]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T05:53:14.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:14.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:14.611 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[191227]: pgmap v1109: 89 pgs: 89 active+clean; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T05:53:14.611 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[191227]: failed to encode map e367 with expected crc 2023-10-09T05:53:14.611 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[191227]: failed to encode map e367 with expected crc 2023-10-09T05:53:14.612 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:53:14.612 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[191227]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T05:53:14.612 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:14.612 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:15.184 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:15.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:15.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:15.288 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:15.320 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:15.369 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:15.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:15.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:15.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:15 smithi008 ceph-mon[184418]: pgmap v1111: 89 pgs: 89 active+clean; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T05:53:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:15 smithi008 ceph-mon[191227]: pgmap v1111: 89 pgs: 89 active+clean; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T05:53:15.476 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:15.508 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:15 smithi191 ceph-mon[161669]: pgmap v1111: 89 pgs: 89 active+clean; 4.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T05:53:15.654 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:15.654 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:15.862 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:15.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:15.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:15.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:15.998 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:16.041 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:16.043 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:16.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:16.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:16.148 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:16.177 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:16.228 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:16.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:16.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:16.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:16.359 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:16.403 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:16.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:16.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:16.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:16.535 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:16.576 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:16.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:16.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:16.681 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:16.712 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:16.776 DEBUG:teuthology.orchestra.run.smithi008:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T05:53:16.779 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:53:16.779 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:53:16.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:hints = 1 2023-10-09T05:53:16.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:53:16.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object prefix: benchmark_data_smithi008_502820 2023-10-09T05:53:16.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:53:16.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:53:17.007 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:53:17.016 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:53:17.080 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:17.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:17.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:17.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:53:17.187 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:17.247 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:17.616 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:17.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 1 16 2291 2275 142.18 142.188 0.00516184 0.0069997 2023-10-09T05:53:17.842 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:17.847 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:17 smithi008 ceph-mon[191227]: pgmap v1112: 89 pgs: 89 active+clean; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T05:53:17.847 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:17 smithi008 ceph-mon[191227]: from='client.46175 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:53:17.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:17.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:17.848 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:17 smithi008 ceph-mon[184418]: pgmap v1112: 89 pgs: 89 active+clean; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T05:53:17.848 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:17 smithi008 ceph-mon[184418]: from='client.46175 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:53:17.849 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:17 smithi191 ceph-mon[161669]: pgmap v1112: 89 pgs: 89 active+clean; 4.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T05:53:17.849 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:17 smithi191 ceph-mon[161669]: from='client.46175 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:53:17.880 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:53:17.950 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:18.017 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:18.310 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1112,"stamp":"2023-10-09T05:53:16.525757+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":182,"num_bytes_recovered":5386010,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8920080,"kb_used_data":2416848,"kb_used_omap":45,"kb_used_meta":6503122,"kb_avail":741041136,"statfs":{"total":767960285184,"available":758826123264,"internally_reserved":0,"allocated":2474852352,"data_stored":2466850965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":6659197219},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001867"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":367,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706766+0000","last_change":"2023-10-09T05:48:54.494513+0000","last_active":"2023-10-09T05:53:09.706766+0000","last_peered":"2023-10-09T05:53:09.706766+0000","last_clean":"2023-10-09T05:53:09.706766+0000","last_became_active":"2023-10-09T05:48:54.493510+0000","last_became_peered":"2023-10-09T05:48:54.493510+0000","last_unstale":"2023-10-09T05:53:09.706766+0000","last_undegraded":"2023-10-09T05:53:09.706766+0000","last_fullsized":"2023-10-09T05:53:09.706766+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":297,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.707067+0000","last_change":"2023-10-09T05:50:18.047163+0000","last_active":"2023-10-09T05:53:09.707067+0000","last_peered":"2023-10-09T05:53:09.707067+0000","last_clean":"2023-10-09T05:53:09.707067+0000","last_became_active":"2023-10-09T05:50:18.046872+0000","last_became_peered":"2023-10-09T05:50:18.046872+0000","last_unstale":"2023-10-09T05:53:09.707067+0000","last_undegraded":"2023-10-09T05:53:09.707067+0000","last_fullsized":"2023-10-09T05:53:09.707067+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":111,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706617+0000","last_change":"2023-10-09T05:50:18.055745+0000","last_active":"2023-10-09T05:53:09.706617+0000","last_peered":"2023-10-09T05:53:09.706617+0000","last_clean":"2023-10-09T05:53:09.706617+0000","last_became_active":"2023-10-09T05:50:18.055524+0000","last_became_peered":"2023-10-09T05:50:18.055524+0000","last_unstale":"2023-10-09T05:53:09.706617+0000","last_undegraded":"2023-10-09T05:53:09.706617+0000","last_fullsized":"2023-10-09T05:53:09.706617+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":385,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703606+0000","last_change":"2023-10-09T05:51:22.261181+0000","last_active":"2023-10-09T05:53:09.703606+0000","last_peered":"2023-10-09T05:53:09.703606+0000","last_clean":"2023-10-09T05:53:09.703606+0000","last_became_active":"2023-10-09T05:51:22.260808+0000","last_became_peered":"2023-10-09T05:51:22.260808+0000","last_unstale":"2023-10-09T05:53:09.703606+0000","last_undegraded":"2023-10-09T05:53:09.703606+0000","last_fullsized":"2023-10-09T05:53:09.703606+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:17.362302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":336,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703080+0000","last_change":"2023-10-09T05:48:54.492664+0000","last_active":"2023-10-09T05:53:09.703080+0000","last_peered":"2023-10-09T05:53:09.703080+0000","last_clean":"2023-10-09T05:53:09.703080+0000","last_became_active":"2023-10-09T05:48:54.492299+0000","last_became_peered":"2023-10-09T05:48:54.492299+0000","last_unstale":"2023-10-09T05:53:09.703080+0000","last_undegraded":"2023-10-09T05:53:09.703080+0000","last_fullsized":"2023-10-09T05:53:09.703080+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:43.198424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":200,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.705267+0000","last_change":"2023-10-09T05:48:54.492447+0000","last_active":"2023-10-09T05:53:09.705267+0000","last_peered":"2023-10-09T05:53:09.705267+0000","last_clean":"2023-10-09T05:53:09.705267+0000","last_became_active":"2023-10-09T05:48:54.492089+0000","last_became_peered":"2023-10-09T05:48:54.492089+0000","last_unstale":"2023-10-09T05:53:09.705267+0000","last_undegraded":"2023-10-09T05:53:09.705267+0000","last_fullsized":"2023-10-09T05:53:09.705267+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":323,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703175+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:53:09.703175+0000","last_peered":"2023-10-09T05:53:09.703175+0000","last_clean":"2023-10-09T05:53:09.703175+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:53:09.703175+0000","last_undegraded":"2023-10-09T05:53:09.703175+0000","last_fullsized":"2023-10-09T05:53:09.703175+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":327,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.431384+0000","last_change":"2023-10-09T05:50:18.054081+0000","last_active":"2023-10-09T05:53:08.431384+0000","last_peered":"2023-10-09T05:53:08.431384+0000","last_clean":"2023-10-09T05:53:08.431384+0000","last_became_active":"2023-10-09T05:50:18.053746+0000","last_became_peered":"2023-10-09T05:50:18.053746+0000","last_unstale":"2023-10-09T05:53:08.431384+0000","last_undegraded":"2023-10-09T05:53:08.431384+0000","last_fullsized":"2023-10-09T05:53:08.431384+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:55.758681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":27,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706300+0000","last_change":"2023-10-09T05:51:43.123678+0000","last_active":"2023-10-09T05:53:09.706300+0000","last_peered":"2023-10-09T05:53:09.706300+0000","last_clean":"2023-10-09T05:53:09.706300+0000","last_became_active":"2023-10-09T05:51:43.121947+0000","last_became_peered":"2023-10-09T05:51:43.121947+0000","last_unstale":"2023-10-09T05:53:09.706300+0000","last_undegraded":"2023-10-09T05:53:09.706300+0000","last_fullsized":"2023-10-09T05:53:09.706300+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":302,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221562+0000","last_change":"2023-10-09T05:48:25.068328+0000","last_active":"2023-10-09T05:53:09.221562+0000","last_peered":"2023-10-09T05:53:09.221562+0000","last_clean":"2023-10-09T05:53:09.221562+0000","last_became_active":"2023-10-09T05:48:25.068094+0000","last_became_peered":"2023-10-09T05:48:25.068094+0000","last_unstale":"2023-10-09T05:53:09.221562+0000","last_undegraded":"2023-10-09T05:53:09.221562+0000","last_fullsized":"2023-10-09T05:53:09.221562+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":152,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221991+0000","last_change":"2023-10-09T05:43:20.365211+0000","last_active":"2023-10-09T05:53:09.221991+0000","last_peered":"2023-10-09T05:53:09.221991+0000","last_clean":"2023-10-09T05:53:09.221991+0000","last_became_active":"2023-10-09T05:43:19.885169+0000","last_became_peered":"2023-10-09T05:43:19.885169+0000","last_unstale":"2023-10-09T05:53:09.221991+0000","last_undegraded":"2023-10-09T05:53:09.221991+0000","last_fullsized":"2023-10-09T05:53:09.221991+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":147,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.213103+0000","last_change":"2023-10-09T05:50:18.045973+0000","last_active":"2023-10-09T05:53:08.213103+0000","last_peered":"2023-10-09T05:53:08.213103+0000","last_clean":"2023-10-09T05:53:08.213103+0000","last_became_active":"2023-10-09T05:50:18.045688+0000","last_became_peered":"2023-10-09T05:50:18.045688+0000","last_unstale":"2023-10-09T05:53:08.213103+0000","last_undegraded":"2023-10-09T05:53:08.213103+0000","last_fullsized":"2023-10-09T05:53:08.213103+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:51.105027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":393,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.707149+0000","last_change":"2023-10-09T05:43:20.061432+0000","last_active":"2023-10-09T05:53:09.707149+0000","last_peered":"2023-10-09T05:53:09.707149+0000","last_clean":"2023-10-09T05:53:09.707149+0000","last_became_active":"2023-10-09T05:43:19.885865+0000","last_became_peered":"2023-10-09T05:43:19.885865+0000","last_unstale":"2023-10-09T05:53:09.707149+0000","last_undegraded":"2023-10-09T05:53:09.707149+0000","last_fullsized":"2023-10-09T05:53:09.707149+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":350,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706754+0000","last_change":"2023-10-09T05:48:54.492082+0000","last_active":"2023-10-09T05:53:09.706754+0000","last_peered":"2023-10-09T05:53:09.706754+0000","last_clean":"2023-10-09T05:53:09.706754+0000","last_became_active":"2023-10-09T05:48:54.491753+0000","last_became_peered":"2023-10-09T05:48:54.491753+0000","last_unstale":"2023-10-09T05:53:09.706754+0000","last_undegraded":"2023-10-09T05:53:09.706754+0000","last_fullsized":"2023-10-09T05:53:09.706754+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":152,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221803+0000","last_change":"2023-10-09T05:43:20.062113+0000","last_active":"2023-10-09T05:53:09.221803+0000","last_peered":"2023-10-09T05:53:09.221803+0000","last_clean":"2023-10-09T05:53:09.221803+0000","last_became_active":"2023-10-09T05:43:19.883417+0000","last_became_peered":"2023-10-09T05:43:19.883417+0000","last_unstale":"2023-10-09T05:53:09.221803+0000","last_undegraded":"2023-10-09T05:53:09.221803+0000","last_fullsized":"2023-10-09T05:53:09.221803+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":330,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221742+0000","last_change":"2023-10-09T05:50:18.039555+0000","last_active":"2023-10-09T05:53:09.221742+0000","last_peered":"2023-10-09T05:53:09.221742+0000","last_clean":"2023-10-09T05:53:09.221742+0000","last_became_active":"2023-10-09T05:50:18.039222+0000","last_became_peered":"2023-10-09T05:50:18.039222+0000","last_unstale":"2023-10-09T05:53:09.221742+0000","last_undegraded":"2023-10-09T05:53:09.221742+0000","last_fullsized":"2023-10-09T05:53:09.221742+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":356,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703550+0000","last_change":"2023-10-09T05:43:19.886004+0000","last_active":"2023-10-09T05:53:09.703550+0000","last_peered":"2023-10-09T05:53:09.703550+0000","last_clean":"2023-10-09T05:53:09.703550+0000","last_became_active":"2023-10-09T05:43:19.885664+0000","last_became_peered":"2023-10-09T05:43:19.885664+0000","last_unstale":"2023-10-09T05:53:09.703550+0000","last_undegraded":"2023-10-09T05:53:09.703550+0000","last_fullsized":"2023-10-09T05:53:09.703550+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:22.772901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":27,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.705095+0000","last_change":"2023-10-09T05:51:43.075341+0000","last_active":"2023-10-09T05:53:09.705095+0000","last_peered":"2023-10-09T05:53:09.705095+0000","last_clean":"2023-10-09T05:53:09.705095+0000","last_became_active":"2023-10-09T05:51:43.075024+0000","last_became_peered":"2023-10-09T05:51:43.075024+0000","last_unstale":"2023-10-09T05:53:09.705095+0000","last_undegraded":"2023-10-09T05:53:09.705095+0000","last_fullsized":"2023-10-09T05:53:09.705095+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":146,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221653+0000","last_change":"2023-10-09T05:43:19.885599+0000","last_active":"2023-10-09T05:53:09.221653+0000","last_peered":"2023-10-09T05:53:09.221653+0000","last_clean":"2023-10-09T05:53:09.221653+0000","last_became_active":"2023-10-09T05:43:19.885226+0000","last_became_peered":"2023-10-09T05:43:19.885226+0000","last_unstale":"2023-10-09T05:53:09.221653+0000","last_undegraded":"2023-10-09T05:53:09.221653+0000","last_fullsized":"2023-10-09T05:53:09.221653+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":356,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703702+0000","last_change":"2023-10-09T05:43:19.889076+0000","last_active":"2023-10-09T05:53:09.703702+0000","last_peered":"2023-10-09T05:53:09.703702+0000","last_clean":"2023-10-09T05:53:09.703702+0000","last_became_active":"2023-10-09T05:43:19.888617+0000","last_became_peered":"2023-10-09T05:43:19.888617+0000","last_unstale":"2023-10-09T05:53:09.703702+0000","last_undegraded":"2023-10-09T05:53:09.703702+0000","last_fullsized":"2023-10-09T05:53:09.703702+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:42.881239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":182,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.702932+0000","last_change":"2023-10-09T05:51:43.221482+0000","last_active":"2023-10-09T05:53:09.702932+0000","last_peered":"2023-10-09T05:53:09.702932+0000","last_clean":"2023-10-09T05:53:09.702932+0000","last_became_active":"2023-10-09T05:51:43.072606+0000","last_became_peered":"2023-10-09T05:51:43.072606+0000","last_unstale":"2023-10-09T05:53:09.702932+0000","last_undegraded":"2023-10-09T05:53:09.702932+0000","last_fullsized":"2023-10-09T05:53:09.702932+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:22.097091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":372,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.222024+0000","last_change":"2023-10-09T05:48:54.492012+0000","last_active":"2023-10-09T05:53:09.222024+0000","last_peered":"2023-10-09T05:53:09.222024+0000","last_clean":"2023-10-09T05:53:09.222024+0000","last_became_active":"2023-10-09T05:48:54.491644+0000","last_became_peered":"2023-10-09T05:48:54.491644+0000","last_unstale":"2023-10-09T05:53:09.222024+0000","last_undegraded":"2023-10-09T05:53:09.222024+0000","last_fullsized":"2023-10-09T05:53:09.222024+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":12,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703607+0000","last_change":"2023-10-09T05:53:08.222771+0000","last_active":"2023-10-09T05:53:09.703607+0000","last_peered":"2023-10-09T05:53:09.703607+0000","last_clean":"2023-10-09T05:53:09.703607+0000","last_became_active":"2023-10-09T05:53:08.222417+0000","last_became_peered":"2023-10-09T05:53:08.222417+0000","last_unstale":"2023-10-09T05:53:09.703607+0000","last_undegraded":"2023-10-09T05:53:09.703607+0000","last_fullsized":"2023-10-09T05:53:09.703607+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:49.364969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":150,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221957+0000","last_change":"2023-10-09T05:43:20.452140+0000","last_active":"2023-10-09T05:53:09.221957+0000","last_peered":"2023-10-09T05:53:09.221957+0000","last_clean":"2023-10-09T05:53:09.221957+0000","last_became_active":"2023-10-09T05:43:19.888075+0000","last_became_peered":"2023-10-09T05:43:19.888075+0000","last_unstale":"2023-10-09T05:53:09.221957+0000","last_undegraded":"2023-10-09T05:53:09.221957+0000","last_fullsized":"2023-10-09T05:53:09.221957+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":27,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.704624+0000","last_change":"2023-10-09T05:51:43.075609+0000","last_active":"2023-10-09T05:53:09.704624+0000","last_peered":"2023-10-09T05:53:09.704624+0000","last_clean":"2023-10-09T05:53:09.704624+0000","last_became_active":"2023-10-09T05:51:43.075280+0000","last_became_peered":"2023-10-09T05:51:43.075280+0000","last_unstale":"2023-10-09T05:53:09.704624+0000","last_undegraded":"2023-10-09T05:53:09.704624+0000","last_fullsized":"2023-10-09T05:53:09.704624+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":11,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706987+0000","last_change":"2023-10-09T05:53:08.434228+0000","last_active":"2023-10-09T05:53:09.706987+0000","last_peered":"2023-10-09T05:53:09.706987+0000","last_clean":"2023-10-09T05:53:09.706987+0000","last_became_active":"2023-10-09T05:53:08.433983+0000","last_became_peered":"2023-10-09T05:53:08.433983+0000","last_unstale":"2023-10-09T05:53:09.706987+0000","last_undegraded":"2023-10-09T05:53:09.706987+0000","last_fullsized":"2023-10-09T05:53:09.706987+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":176,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.222040+0000","last_change":"2023-10-09T05:48:54.539425+0000","last_active":"2023-10-09T05:53:09.222040+0000","last_peered":"2023-10-09T05:53:09.222040+0000","last_clean":"2023-10-09T05:53:09.222040+0000","last_became_active":"2023-10-09T05:48:54.486401+0000","last_became_peered":"2023-10-09T05:48:54.486401+0000","last_unstale":"2023-10-09T05:53:09.222040+0000","last_undegraded":"2023-10-09T05:53:09.222040+0000","last_fullsized":"2023-10-09T05:53:09.222040+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":383,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706732+0000","last_change":"2023-10-09T05:48:54.492534+0000","last_active":"2023-10-09T05:53:09.706732+0000","last_peered":"2023-10-09T05:53:09.706732+0000","last_clean":"2023-10-09T05:53:09.706732+0000","last_became_active":"2023-10-09T05:48:54.492172+0000","last_became_peered":"2023-10-09T05:48:54.492172+0000","last_unstale":"2023-10-09T05:53:09.706732+0000","last_undegraded":"2023-10-09T05:53:09.706732+0000","last_fullsized":"2023-10-09T05:53:09.706732+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":11,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.705120+0000","last_change":"2023-10-09T05:53:08.432184+0000","last_active":"2023-10-09T05:53:09.705120+0000","last_peered":"2023-10-09T05:53:09.705120+0000","last_clean":"2023-10-09T05:53:09.705120+0000","last_became_active":"2023-10-09T05:53:08.431880+0000","last_became_peered":"2023-10-09T05:53:08.431880+0000","last_unstale":"2023-10-09T05:53:09.705120+0000","last_undegraded":"2023-10-09T05:53:09.705120+0000","last_fullsized":"2023-10-09T05:53:09.705120+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":176,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221927+0000","last_change":"2023-10-09T05:51:43.265312+0000","last_active":"2023-10-09T05:53:09.221927+0000","last_peered":"2023-10-09T05:53:09.221927+0000","last_clean":"2023-10-09T05:53:09.221927+0000","last_became_active":"2023-10-09T05:51:43.076237+0000","last_became_peered":"2023-10-09T05:51:43.076237+0000","last_unstale":"2023-10-09T05:53:09.221927+0000","last_undegraded":"2023-10-09T05:53:09.221927+0000","last_fullsized":"2023-10-09T05:53:09.221927+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":344,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221655+0000","last_change":"2023-10-09T05:50:18.039975+0000","last_active":"2023-10-09T05:53:09.221655+0000","last_peered":"2023-10-09T05:53:09.221655+0000","last_clean":"2023-10-09T05:53:09.221655+0000","last_became_active":"2023-10-09T05:50:18.039332+0000","last_became_peered":"2023-10-09T05:50:18.039332+0000","last_unstale":"2023-10-09T05:53:09.221655+0000","last_undegraded":"2023-10-09T05:53:09.221655+0000","last_fullsized":"2023-10-09T05:53:09.221655+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":11,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706245+0000","last_change":"2023-10-09T05:53:08.373761+0000","last_active":"2023-10-09T05:53:09.706245+0000","last_peered":"2023-10-09T05:53:09.706245+0000","last_clean":"2023-10-09T05:53:09.706245+0000","last_became_active":"2023-10-09T05:53:08.373499+0000","last_became_peered":"2023-10-09T05:53:08.373499+0000","last_unstale":"2023-10-09T05:53:09.706245+0000","last_undegraded":"2023-10-09T05:53:09.706245+0000","last_fullsized":"2023-10-09T05:53:09.706245+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":334,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.704971+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:53:09.704971+0000","last_peered":"2023-10-09T05:53:09.704971+0000","last_clean":"2023-10-09T05:53:09.704971+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:53:09.704971+0000","last_undegraded":"2023-10-09T05:53:09.704971+0000","last_fullsized":"2023-10-09T05:53:09.704971+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":342,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703074+0000","last_change":"2023-10-09T05:48:54.495800+0000","last_active":"2023-10-09T05:53:09.703074+0000","last_peered":"2023-10-09T05:53:09.703074+0000","last_clean":"2023-10-09T05:53:09.703074+0000","last_became_active":"2023-10-09T05:48:54.495540+0000","last_became_peered":"2023-10-09T05:48:54.495540+0000","last_unstale":"2023-10-09T05:53:09.703074+0000","last_undegraded":"2023-10-09T05:53:09.703074+0000","last_fullsized":"2023-10-09T05:53:09.703074+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:26.903669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":11,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221454+0000","last_change":"2023-10-09T05:53:08.433005+0000","last_active":"2023-10-09T05:53:09.221454+0000","last_peered":"2023-10-09T05:53:09.221454+0000","last_clean":"2023-10-09T05:53:09.221454+0000","last_became_active":"2023-10-09T05:53:08.432721+0000","last_became_peered":"2023-10-09T05:53:08.432721+0000","last_unstale":"2023-10-09T05:53:09.221454+0000","last_undegraded":"2023-10-09T05:53:09.221454+0000","last_fullsized":"2023-10-09T05:53:09.221454+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":105,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703178+0000","last_change":"2023-10-09T05:48:25.067945+0000","last_active":"2023-10-09T05:53:09.703178+0000","last_peered":"2023-10-09T05:53:09.703178+0000","last_clean":"2023-10-09T05:53:09.703178+0000","last_became_active":"2023-10-09T05:48:25.067721+0000","last_became_peered":"2023-10-09T05:48:25.067721+0000","last_unstale":"2023-10-09T05:53:09.703178+0000","last_undegraded":"2023-10-09T05:53:09.703178+0000","last_fullsized":"2023-10-09T05:53:09.703178+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:58.655544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":332,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.707116+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:53:09.707116+0000","last_peered":"2023-10-09T05:53:09.707116+0000","last_clean":"2023-10-09T05:53:09.707116+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:53:09.707116+0000","last_undegraded":"2023-10-09T05:53:09.707116+0000","last_fullsized":"2023-10-09T05:53:09.707116+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":11,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.375910+0000","last_change":"2023-10-09T05:53:08.375690+0000","last_active":"2023-10-09T05:53:08.375910+0000","last_peered":"2023-10-09T05:53:08.375910+0000","last_clean":"2023-10-09T05:53:08.375910+0000","last_became_active":"2023-10-09T05:53:08.375441+0000","last_became_peered":"2023-10-09T05:53:08.375441+0000","last_unstale":"2023-10-09T05:53:08.375910+0000","last_undegraded":"2023-10-09T05:53:08.375910+0000","last_fullsized":"2023-10-09T05:53:08.375910+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:39:15.702086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":201,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706771+0000","last_change":"2023-10-09T05:48:54.494715+0000","last_active":"2023-10-09T05:53:09.706771+0000","last_peered":"2023-10-09T05:53:09.706771+0000","last_clean":"2023-10-09T05:53:09.706771+0000","last_became_active":"2023-10-09T05:48:54.493988+0000","last_became_peered":"2023-10-09T05:48:54.493988+0000","last_unstale":"2023-10-09T05:53:09.706771+0000","last_undegraded":"2023-10-09T05:53:09.706771+0000","last_fullsized":"2023-10-09T05:53:09.706771+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":365,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706874+0000","last_change":"2023-10-09T05:48:54.493193+0000","last_active":"2023-10-09T05:53:09.706874+0000","last_peered":"2023-10-09T05:53:09.706874+0000","last_clean":"2023-10-09T05:53:09.706874+0000","last_became_active":"2023-10-09T05:48:54.492886+0000","last_became_peered":"2023-10-09T05:48:54.492886+0000","last_unstale":"2023-10-09T05:53:09.706874+0000","last_undegraded":"2023-10-09T05:53:09.706874+0000","last_fullsized":"2023-10-09T05:53:09.706874+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":11,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221741+0000","last_change":"2023-10-09T05:53:08.431947+0000","last_active":"2023-10-09T05:53:09.221741+0000","last_peered":"2023-10-09T05:53:09.221741+0000","last_clean":"2023-10-09T05:53:09.221741+0000","last_became_active":"2023-10-09T05:53:08.431601+0000","last_became_peered":"2023-10-09T05:53:08.431601+0000","last_unstale":"2023-10-09T05:53:09.221741+0000","last_undegraded":"2023-10-09T05:53:09.221741+0000","last_fullsized":"2023-10-09T05:53:09.221741+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":366,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706533+0000","last_change":"2023-10-09T05:43:19.886708+0000","last_active":"2023-10-09T05:53:09.706533+0000","last_peered":"2023-10-09T05:53:09.706533+0000","last_clean":"2023-10-09T05:53:09.706533+0000","last_became_active":"2023-10-09T05:43:19.886187+0000","last_became_peered":"2023-10-09T05:43:19.886187+0000","last_unstale":"2023-10-09T05:53:09.706533+0000","last_undegraded":"2023-10-09T05:53:09.706533+0000","last_fullsized":"2023-10-09T05:53:09.706533+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":359,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706928+0000","last_change":"2023-10-09T05:48:54.492708+0000","last_active":"2023-10-09T05:53:09.706928+0000","last_peered":"2023-10-09T05:53:09.706928+0000","last_clean":"2023-10-09T05:53:09.706928+0000","last_became_active":"2023-10-09T05:48:54.492382+0000","last_became_peered":"2023-10-09T05:48:54.492382+0000","last_unstale":"2023-10-09T05:53:09.706928+0000","last_undegraded":"2023-10-09T05:53:09.706928+0000","last_fullsized":"2023-10-09T05:53:09.706928+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":11,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.433248+0000","last_change":"2023-10-09T05:53:08.432676+0000","last_active":"2023-10-09T05:53:08.433248+0000","last_peered":"2023-10-09T05:53:08.433248+0000","last_clean":"2023-10-09T05:53:08.433248+0000","last_became_active":"2023-10-09T05:53:08.432370+0000","last_became_peered":"2023-10-09T05:53:08.432370+0000","last_unstale":"2023-10-09T05:53:08.433248+0000","last_undegraded":"2023-10-09T05:53:08.433248+0000","last_fullsized":"2023-10-09T05:53:08.433248+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:08.122834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":332,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221486+0000","last_change":"2023-10-09T05:50:18.048706+0000","last_active":"2023-10-09T05:53:09.221486+0000","last_peered":"2023-10-09T05:53:09.221486+0000","last_clean":"2023-10-09T05:53:09.221486+0000","last_became_active":"2023-10-09T05:50:18.048410+0000","last_became_peered":"2023-10-09T05:50:18.048410+0000","last_unstale":"2023-10-09T05:53:09.221486+0000","last_undegraded":"2023-10-09T05:53:09.221486+0000","last_fullsized":"2023-10-09T05:53:09.221486+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":111,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.707120+0000","last_change":"2023-10-09T05:50:18.072866+0000","last_active":"2023-10-09T05:53:09.707120+0000","last_peered":"2023-10-09T05:53:09.707120+0000","last_clean":"2023-10-09T05:53:09.707120+0000","last_became_active":"2023-10-09T05:50:18.072550+0000","last_became_peered":"2023-10-09T05:50:18.072550+0000","last_unstale":"2023-10-09T05:53:09.707120+0000","last_undegraded":"2023-10-09T05:53:09.707120+0000","last_fullsized":"2023-10-09T05:53:09.707120+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":11,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.705039+0000","last_change":"2023-10-09T05:53:08.432523+0000","last_active":"2023-10-09T05:53:09.705039+0000","last_peered":"2023-10-09T05:53:09.705039+0000","last_clean":"2023-10-09T05:53:09.705039+0000","last_became_active":"2023-10-09T05:53:08.431983+0000","last_became_peered":"2023-10-09T05:53:08.431983+0000","last_unstale":"2023-10-09T05:53:09.705039+0000","last_undegraded":"2023-10-09T05:53:09.705039+0000","last_fullsized":"2023-10-09T05:53:09.705039+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":757,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703901+0000","last_change":"2023-10-09T05:50:18.055235+0000","last_active":"2023-10-09T05:53:09.703901+0000","last_peered":"2023-10-09T05:53:09.703901+0000","last_clean":"2023-10-09T05:53:09.703901+0000","last_became_active":"2023-10-09T05:50:18.054656+0000","last_became_peered":"2023-10-09T05:50:18.054656+0000","last_unstale":"2023-10-09T05:53:09.703901+0000","last_undegraded":"2023-10-09T05:53:09.703901+0000","last_fullsized":"2023-10-09T05:53:09.703901+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:33.760262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":349,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.705176+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:53:09.705176+0000","last_peered":"2023-10-09T05:53:09.705176+0000","last_clean":"2023-10-09T05:53:09.705176+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:53:09.705176+0000","last_undegraded":"2023-10-09T05:53:09.705176+0000","last_fullsized":"2023-10-09T05:53:09.705176+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":27,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.704741+0000","last_change":"2023-10-09T05:51:43.248409+0000","last_active":"2023-10-09T05:53:09.704741+0000","last_peered":"2023-10-09T05:53:09.704741+0000","last_clean":"2023-10-09T05:53:09.704741+0000","last_became_active":"2023-10-09T05:51:43.154898+0000","last_became_peered":"2023-10-09T05:51:43.154898+0000","last_unstale":"2023-10-09T05:53:09.704741+0000","last_undegraded":"2023-10-09T05:53:09.704741+0000","last_fullsized":"2023-10-09T05:53:09.704741+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":12,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.702872+0000","last_change":"2023-10-09T05:53:08.222664+0000","last_active":"2023-10-09T05:53:09.702872+0000","last_peered":"2023-10-09T05:53:09.702872+0000","last_clean":"2023-10-09T05:53:09.702872+0000","last_became_active":"2023-10-09T05:53:08.222324+0000","last_became_peered":"2023-10-09T05:53:08.222324+0000","last_unstale":"2023-10-09T05:53:09.702872+0000","last_undegraded":"2023-10-09T05:53:09.702872+0000","last_fullsized":"2023-10-09T05:53:09.702872+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:06.051121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":857,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.704834+0000","last_change":"2023-10-09T05:48:54.592642+0000","last_active":"2023-10-09T05:53:09.704834+0000","last_peered":"2023-10-09T05:53:09.704834+0000","last_clean":"2023-10-09T05:53:09.704834+0000","last_became_active":"2023-10-09T05:48:54.491917+0000","last_became_peered":"2023-10-09T05:48:54.491917+0000","last_unstale":"2023-10-09T05:53:09.704834+0000","last_undegraded":"2023-10-09T05:53:09.704834+0000","last_fullsized":"2023-10-09T05:53:09.704834+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":333,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706875+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:53:09.706875+0000","last_peered":"2023-10-09T05:53:09.706875+0000","last_clean":"2023-10-09T05:53:09.706875+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:53:09.706875+0000","last_undegraded":"2023-10-09T05:53:09.706875+0000","last_fullsized":"2023-10-09T05:53:09.706875+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":188,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.213937+0000","last_change":"2023-10-09T05:50:18.043775+0000","last_active":"2023-10-09T05:53:08.213937+0000","last_peered":"2023-10-09T05:53:08.213937+0000","last_clean":"2023-10-09T05:53:08.213937+0000","last_became_active":"2023-10-09T05:50:18.043047+0000","last_became_peered":"2023-10-09T05:50:18.043047+0000","last_unstale":"2023-10-09T05:53:08.213937+0000","last_undegraded":"2023-10-09T05:53:08.213937+0000","last_fullsized":"2023-10-09T05:53:08.213937+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:28.456979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":11,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.704719+0000","last_change":"2023-10-09T05:53:08.371616+0000","last_active":"2023-10-09T05:53:09.704719+0000","last_peered":"2023-10-09T05:53:09.704719+0000","last_clean":"2023-10-09T05:53:09.704719+0000","last_became_active":"2023-10-09T05:53:08.371287+0000","last_became_peered":"2023-10-09T05:53:08.371287+0000","last_unstale":"2023-10-09T05:53:09.704719+0000","last_undegraded":"2023-10-09T05:53:09.704719+0000","last_fullsized":"2023-10-09T05:53:09.704719+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":686,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.212644+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:53:08.212644+0000","last_peered":"2023-10-09T05:53:08.212644+0000","last_clean":"2023-10-09T05:53:08.212644+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:53:08.212644+0000","last_undegraded":"2023-10-09T05:53:08.212644+0000","last_fullsized":"2023-10-09T05:53:08.212644+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":147,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.212524+0000","last_change":"2023-10-09T05:50:18.063466+0000","last_active":"2023-10-09T05:53:08.212524+0000","last_peered":"2023-10-09T05:53:08.212524+0000","last_clean":"2023-10-09T05:53:08.212524+0000","last_became_active":"2023-10-09T05:50:18.063120+0000","last_became_peered":"2023-10-09T05:50:18.063120+0000","last_unstale":"2023-10-09T05:53:08.212524+0000","last_undegraded":"2023-10-09T05:53:08.212524+0000","last_fullsized":"2023-10-09T05:53:08.212524+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:13.683314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":171,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.705260+0000","last_change":"2023-10-09T05:50:18.069402+0000","last_active":"2023-10-09T05:53:09.705260+0000","last_peered":"2023-10-09T05:53:09.705260+0000","last_clean":"2023-10-09T05:53:09.705260+0000","last_became_active":"2023-10-09T05:50:18.069196+0000","last_became_peered":"2023-10-09T05:50:18.069196+0000","last_unstale":"2023-10-09T05:53:09.705260+0000","last_undegraded":"2023-10-09T05:53:09.705260+0000","last_fullsized":"2023-10-09T05:53:09.705260+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":12,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.702822+0000","last_change":"2023-10-09T05:53:08.432957+0000","last_active":"2023-10-09T05:53:09.702822+0000","last_peered":"2023-10-09T05:53:09.702822+0000","last_clean":"2023-10-09T05:53:09.702822+0000","last_became_active":"2023-10-09T05:53:08.432681+0000","last_became_peered":"2023-10-09T05:53:08.432681+0000","last_unstale":"2023-10-09T05:53:09.702822+0000","last_undegraded":"2023-10-09T05:53:09.702822+0000","last_fullsized":"2023-10-09T05:53:09.702822+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.595860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":1013,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.704928+0000","last_change":"2023-10-09T05:43:22.950370+0000","last_active":"2023-10-09T05:53:09.704928+0000","last_peered":"2023-10-09T05:53:09.704928+0000","last_clean":"2023-10-09T05:53:09.704928+0000","last_became_active":"2023-10-09T05:43:22.949790+0000","last_became_peered":"2023-10-09T05:43:22.949790+0000","last_unstale":"2023-10-09T05:53:09.704928+0000","last_undegraded":"2023-10-09T05:53:09.704928+0000","last_fullsized":"2023-10-09T05:53:09.704928+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":517,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221380+0000","last_change":"2023-10-09T05:44:04.972868+0000","last_active":"2023-10-09T05:53:09.221380+0000","last_peered":"2023-10-09T05:53:09.221380+0000","last_clean":"2023-10-09T05:53:09.221380+0000","last_became_active":"2023-10-09T05:44:04.972462+0000","last_became_peered":"2023-10-09T05:44:04.972462+0000","last_unstale":"2023-10-09T05:53:09.221380+0000","last_undegraded":"2023-10-09T05:53:09.221380+0000","last_fullsized":"2023-10-09T05:53:09.221380+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":139,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.704808+0000","last_change":"2023-10-09T05:48:25.068759+0000","last_active":"2023-10-09T05:53:09.704808+0000","last_peered":"2023-10-09T05:53:09.704808+0000","last_clean":"2023-10-09T05:53:09.704808+0000","last_became_active":"2023-10-09T05:48:25.068358+0000","last_became_peered":"2023-10-09T05:48:25.068358+0000","last_unstale":"2023-10-09T05:53:09.704808+0000","last_undegraded":"2023-10-09T05:53:09.704808+0000","last_fullsized":"2023-10-09T05:53:09.704808+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":334,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.707019+0000","last_change":"2023-10-09T05:44:50.495966+0000","last_active":"2023-10-09T05:53:09.707019+0000","last_peered":"2023-10-09T05:53:09.707019+0000","last_clean":"2023-10-09T05:53:09.707019+0000","last_became_active":"2023-10-09T05:44:50.495628+0000","last_became_peered":"2023-10-09T05:44:50.495628+0000","last_unstale":"2023-10-09T05:53:09.707019+0000","last_undegraded":"2023-10-09T05:53:09.707019+0000","last_fullsized":"2023-10-09T05:53:09.707019+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":12,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.702991+0000","last_change":"2023-10-09T05:53:08.360961+0000","last_active":"2023-10-09T05:53:09.702991+0000","last_peered":"2023-10-09T05:53:09.702991+0000","last_clean":"2023-10-09T05:53:09.702991+0000","last_became_active":"2023-10-09T05:53:08.360644+0000","last_became_peered":"2023-10-09T05:53:08.360644+0000","last_unstale":"2023-10-09T05:53:09.702991+0000","last_undegraded":"2023-10-09T05:53:09.702991+0000","last_fullsized":"2023-10-09T05:53:09.702991+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:29.685493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":889,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221387+0000","last_change":"2023-10-09T05:49:36.046279+0000","last_active":"2023-10-09T05:53:09.221387+0000","last_peered":"2023-10-09T05:53:09.221387+0000","last_clean":"2023-10-09T05:53:09.221387+0000","last_became_active":"2023-10-09T05:49:36.045828+0000","last_became_peered":"2023-10-09T05:49:36.045828+0000","last_unstale":"2023-10-09T05:53:09.221387+0000","last_undegraded":"2023-10-09T05:53:09.221387+0000","last_fullsized":"2023-10-09T05:53:09.221387+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":361,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.702957+0000","last_change":"2023-10-09T05:51:31.682830+0000","last_active":"2023-10-09T05:53:09.702957+0000","last_peered":"2023-10-09T05:53:09.702957+0000","last_clean":"2023-10-09T05:53:09.702957+0000","last_became_active":"2023-10-09T05:51:31.682566+0000","last_became_peered":"2023-10-09T05:51:31.682566+0000","last_unstale":"2023-10-09T05:53:09.702957+0000","last_undegraded":"2023-10-09T05:53:09.702957+0000","last_fullsized":"2023-10-09T05:53:09.702957+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:42.916316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":357,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703430+0000","last_change":"2023-10-09T05:48:54.489115+0000","last_active":"2023-10-09T05:53:09.703430+0000","last_peered":"2023-10-09T05:53:09.703430+0000","last_clean":"2023-10-09T05:53:09.703430+0000","last_became_active":"2023-10-09T05:48:54.488783+0000","last_became_peered":"2023-10-09T05:48:54.488783+0000","last_unstale":"2023-10-09T05:53:09.703430+0000","last_undegraded":"2023-10-09T05:53:09.703430+0000","last_fullsized":"2023-10-09T05:53:09.703430+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:13.193778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":11,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221862+0000","last_change":"2023-10-09T05:53:08.373960+0000","last_active":"2023-10-09T05:53:09.221862+0000","last_peered":"2023-10-09T05:53:09.221862+0000","last_clean":"2023-10-09T05:53:09.221862+0000","last_became_active":"2023-10-09T05:53:08.373647+0000","last_became_peered":"2023-10-09T05:53:08.373647+0000","last_unstale":"2023-10-09T05:53:09.221862+0000","last_undegraded":"2023-10-09T05:53:09.221862+0000","last_fullsized":"2023-10-09T05:53:09.221862+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":550,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.222003+0000","last_change":"2023-10-09T05:44:51.489799+0000","last_active":"2023-10-09T05:53:09.222003+0000","last_peered":"2023-10-09T05:53:09.222003+0000","last_clean":"2023-10-09T05:53:09.222003+0000","last_became_active":"2023-10-09T05:44:51.489301+0000","last_became_peered":"2023-10-09T05:44:51.489301+0000","last_unstale":"2023-10-09T05:53:09.222003+0000","last_undegraded":"2023-10-09T05:53:09.222003+0000","last_fullsized":"2023-10-09T05:53:09.222003+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":165,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.214020+0000","last_change":"2023-10-09T05:51:43.127835+0000","last_active":"2023-10-09T05:53:08.214020+0000","last_peered":"2023-10-09T05:53:08.214020+0000","last_clean":"2023-10-09T05:53:08.214020+0000","last_became_active":"2023-10-09T05:51:43.127489+0000","last_became_peered":"2023-10-09T05:51:43.127489+0000","last_unstale":"2023-10-09T05:53:08.214020+0000","last_undegraded":"2023-10-09T05:53:08.214020+0000","last_fullsized":"2023-10-09T05:53:08.214020+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:20.191774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":343,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703352+0000","last_change":"2023-10-09T05:43:19.887078+0000","last_active":"2023-10-09T05:53:09.703352+0000","last_peered":"2023-10-09T05:53:09.703352+0000","last_clean":"2023-10-09T05:53:09.703352+0000","last_became_active":"2023-10-09T05:43:19.885975+0000","last_became_peered":"2023-10-09T05:43:19.885975+0000","last_unstale":"2023-10-09T05:53:09.703352+0000","last_undegraded":"2023-10-09T05:53:09.703352+0000","last_fullsized":"2023-10-09T05:53:09.703352+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:53.255114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":11,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221804+0000","last_change":"2023-10-09T05:53:08.371418+0000","last_active":"2023-10-09T05:53:09.221804+0000","last_peered":"2023-10-09T05:53:09.221804+0000","last_clean":"2023-10-09T05:53:09.221804+0000","last_became_active":"2023-10-09T05:53:08.371127+0000","last_became_peered":"2023-10-09T05:53:08.371127+0000","last_unstale":"2023-10-09T05:53:09.221804+0000","last_undegraded":"2023-10-09T05:53:09.221804+0000","last_fullsized":"2023-10-09T05:53:09.221804+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":314,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.222083+0000","last_change":"2023-10-09T05:48:25.066163+0000","last_active":"2023-10-09T05:53:09.222083+0000","last_peered":"2023-10-09T05:53:09.222083+0000","last_clean":"2023-10-09T05:53:09.222083+0000","last_became_active":"2023-10-09T05:48:25.065145+0000","last_became_peered":"2023-10-09T05:48:25.065145+0000","last_unstale":"2023-10-09T05:53:09.222083+0000","last_undegraded":"2023-10-09T05:53:09.222083+0000","last_fullsized":"2023-10-09T05:53:09.222083+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":621,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.707104+0000","last_change":"2023-10-09T05:52:58.687202+0000","last_active":"2023-10-09T05:53:09.707104+0000","last_peered":"2023-10-09T05:53:09.707104+0000","last_clean":"2023-10-09T05:53:09.707104+0000","last_became_active":"2023-10-09T05:50:19.054850+0000","last_became_peered":"2023-10-09T05:50:19.054850+0000","last_unstale":"2023-10-09T05:53:09.707104+0000","last_undegraded":"2023-10-09T05:53:09.707104+0000","last_fullsized":"2023-10-09T05:53:09.707104+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":307,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.221485+0000","last_change":"2023-10-09T05:48:25.067326+0000","last_active":"2023-10-09T05:53:09.221485+0000","last_peered":"2023-10-09T05:53:09.221485+0000","last_clean":"2023-10-09T05:53:09.221485+0000","last_became_active":"2023-10-09T05:48:25.066977+0000","last_became_peered":"2023-10-09T05:48:25.066977+0000","last_unstale":"2023-10-09T05:53:09.221485+0000","last_undegraded":"2023-10-09T05:53:09.221485+0000","last_fullsized":"2023-10-09T05:53:09.221485+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":11,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706895+0000","last_change":"2023-10-09T05:53:08.434139+0000","last_active":"2023-10-09T05:53:09.706895+0000","last_peered":"2023-10-09T05:53:09.706895+0000","last_clean":"2023-10-09T05:53:09.706895+0000","last_became_active":"2023-10-09T05:53:08.433910+0000","last_became_peered":"2023-10-09T05:53:08.433910+0000","last_unstale":"2023-10-09T05:53:09.706895+0000","last_undegraded":"2023-10-09T05:53:09.706895+0000","last_fullsized":"2023-10-09T05:53:09.706895+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":864,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.222020+0000","last_change":"2023-10-09T05:44:50.482211+0000","last_active":"2023-10-09T05:53:09.222020+0000","last_peered":"2023-10-09T05:53:09.222020+0000","last_clean":"2023-10-09T05:53:09.222020+0000","last_became_active":"2023-10-09T05:44:50.481749+0000","last_became_peered":"2023-10-09T05:44:50.481749+0000","last_unstale":"2023-10-09T05:53:09.222020+0000","last_undegraded":"2023-10-09T05:53:09.222020+0000","last_fullsized":"2023-10-09T05:53:09.222020+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":353,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703521+0000","last_change":"2023-10-09T05:48:54.585691+0000","last_active":"2023-10-09T05:53:09.703521+0000","last_peered":"2023-10-09T05:53:09.703521+0000","last_clean":"2023-10-09T05:53:09.703521+0000","last_became_active":"2023-10-09T05:48:54.498259+0000","last_became_peered":"2023-10-09T05:48:54.498259+0000","last_unstale":"2023-10-09T05:53:09.703521+0000","last_undegraded":"2023-10-09T05:53:09.703521+0000","last_fullsized":"2023-10-09T05:53:09.703521+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.684612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":317,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706614+0000","last_change":"2023-10-09T05:21:16.803732+0000","last_active":"2023-10-09T05:53:09.706614+0000","last_peered":"2023-10-09T05:53:09.706614+0000","last_clean":"2023-10-09T05:53:09.706614+0000","last_became_active":"2023-10-09T05:21:16.803443+0000","last_became_peered":"2023-10-09T05:21:16.803443+0000","last_unstale":"2023-10-09T05:53:09.706614+0000","last_undegraded":"2023-10-09T05:53:09.706614+0000","last_fullsized":"2023-10-09T05:53:09.706614+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":344,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703273+0000","last_change":"2023-10-09T05:43:19.886769+0000","last_active":"2023-10-09T05:53:09.703273+0000","last_peered":"2023-10-09T05:53:09.703273+0000","last_clean":"2023-10-09T05:53:09.703273+0000","last_became_active":"2023-10-09T05:43:19.886314+0000","last_became_peered":"2023-10-09T05:43:19.886314+0000","last_unstale":"2023-10-09T05:53:09.703273+0000","last_undegraded":"2023-10-09T05:53:09.703273+0000","last_fullsized":"2023-10-09T05:53:09.703273+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:02.594919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":37,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.213966+0000","last_change":"2023-10-09T05:51:13.060067+0000","last_active":"2023-10-09T05:53:08.213966+0000","last_peered":"2023-10-09T05:53:08.213966+0000","last_clean":"2023-10-09T05:53:08.213966+0000","last_became_active":"2023-10-09T05:51:13.059725+0000","last_became_peered":"2023-10-09T05:51:13.059725+0000","last_unstale":"2023-10-09T05:53:08.213966+0000","last_undegraded":"2023-10-09T05:53:08.213966+0000","last_fullsized":"2023-10-09T05:53:08.213966+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:43.328688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":362,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.704970+0000","last_change":"2023-10-09T05:48:54.583404+0000","last_active":"2023-10-09T05:53:09.704970+0000","last_peered":"2023-10-09T05:53:09.704970+0000","last_clean":"2023-10-09T05:53:09.704970+0000","last_became_active":"2023-10-09T05:48:54.492565+0000","last_became_peered":"2023-10-09T05:48:54.492565+0000","last_unstale":"2023-10-09T05:53:09.704970+0000","last_undegraded":"2023-10-09T05:53:09.704970+0000","last_fullsized":"2023-10-09T05:53:09.704970+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":365,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706938+0000","last_change":"2023-10-09T05:48:54.493079+0000","last_active":"2023-10-09T05:53:09.706938+0000","last_peered":"2023-10-09T05:53:09.706938+0000","last_clean":"2023-10-09T05:53:09.706938+0000","last_became_active":"2023-10-09T05:48:54.492013+0000","last_became_peered":"2023-10-09T05:48:54.492013+0000","last_unstale":"2023-10-09T05:53:09.706938+0000","last_undegraded":"2023-10-09T05:53:09.706938+0000","last_fullsized":"2023-10-09T05:53:09.706938+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":321,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.705126+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:53:09.705126+0000","last_peered":"2023-10-09T05:53:09.705126+0000","last_clean":"2023-10-09T05:53:09.705126+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:53:09.705126+0000","last_undegraded":"2023-10-09T05:53:09.705126+0000","last_fullsized":"2023-10-09T05:53:09.705126+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":30,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706592+0000","last_change":"2023-10-09T05:51:31.668739+0000","last_active":"2023-10-09T05:53:09.706592+0000","last_peered":"2023-10-09T05:53:09.706592+0000","last_clean":"2023-10-09T05:53:09.706592+0000","last_became_active":"2023-10-09T05:51:31.668481+0000","last_became_peered":"2023-10-09T05:51:31.668481+0000","last_unstale":"2023-10-09T05:53:09.706592+0000","last_undegraded":"2023-10-09T05:53:09.706592+0000","last_fullsized":"2023-10-09T05:53:09.706592+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":85,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.214097+0000","last_change":"2023-10-09T05:48:25.069406+0000","last_active":"2023-10-09T05:53:08.214097+0000","last_peered":"2023-10-09T05:53:08.214097+0000","last_clean":"2023-10-09T05:53:08.214097+0000","last_became_active":"2023-10-09T05:48:25.069064+0000","last_became_peered":"2023-10-09T05:48:25.069064+0000","last_unstale":"2023-10-09T05:53:08.214097+0000","last_undegraded":"2023-10-09T05:53:08.214097+0000","last_fullsized":"2023-10-09T05:53:08.214097+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.567858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":167,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.212511+0000","last_change":"2023-10-09T05:48:25.063854+0000","last_active":"2023-10-09T05:53:08.212511+0000","last_peered":"2023-10-09T05:53:08.212511+0000","last_clean":"2023-10-09T05:53:08.212511+0000","last_became_active":"2023-10-09T05:48:25.063579+0000","last_became_peered":"2023-10-09T05:48:25.063579+0000","last_unstale":"2023-10-09T05:53:08.212511+0000","last_undegraded":"2023-10-09T05:53:08.212511+0000","last_fullsized":"2023-10-09T05:53:08.212511+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:02.389789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":358,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.703271+0000","last_change":"2023-10-09T05:51:13.071891+0000","last_active":"2023-10-09T05:53:09.703271+0000","last_peered":"2023-10-09T05:53:09.703271+0000","last_clean":"2023-10-09T05:53:09.703271+0000","last_became_active":"2023-10-09T05:51:13.058378+0000","last_became_peered":"2023-10-09T05:51:13.058378+0000","last_unstale":"2023-10-09T05:53:09.703271+0000","last_undegraded":"2023-10-09T05:53:09.703271+0000","last_fullsized":"2023-10-09T05:53:09.703271+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:41.278366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":350,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.706489+0000","last_change":"2023-10-09T05:48:54.492580+0000","last_active":"2023-10-09T05:53:09.706489+0000","last_peered":"2023-10-09T05:53:09.706489+0000","last_clean":"2023-10-09T05:53:09.706489+0000","last_became_active":"2023-10-09T05:48:54.492259+0000","last_became_peered":"2023-10-09T05:48:54.492259+0000","last_unstale":"2023-10-09T05:53:09.706489+0000","last_undegraded":"2023-10-09T05:53:09.706489+0000","last_fullsized":"2023-10-09T05:53:09.706489+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":10932,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":142,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846816,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":913708,"kb_used_data":7972,"kb_used_omap":14,"kb_used_meta":905713,"kb_avail":92831444,"statfs":{"total":95995035648,"available":95059398656,"internally_reserved":0,"allocated":8163328,"data_stored":7473095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":927450970},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":12.824999999999999,"5min":4.1820000000000004,"15min":4.3869999999999996},"min":{"1min":0.22800000000000001,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":105.39100000000001,"5min":105.39100000000001,"15min":220.77500000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":12.695,"5min":4.1100000000000003,"15min":4.359},"min":{"1min":0.29799999999999999,"5min":0.25900000000000001,"15min":0.22700000000000001},"max":{"1min":105.417,"5min":105.417,"15min":220.78700000000001},"last":0.61099999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":7.9139999999999997,"5min":2.363,"15min":3.1760000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":114.422,"5min":114.422,"15min":190.68100000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":7.8090000000000002,"5min":2.5760000000000001,"15min":3.234},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":111.708,"5min":111.708,"15min":191.09200000000001},"last":0.44}]},{"osd":2,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":7.2279999999999998,"5min":2.6629999999999998,"15min":2.0590000000000002},"min":{"1min":0.374,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":100.684,"5min":100.684,"15min":100.684},"last":0.59799999999999998},{"interface":"front","average":{"1min":7.4290000000000003,"5min":2.4990000000000001,"15min":2.0129999999999999},"min":{"1min":0.32000000000000001,"5min":0.30499999999999999,"15min":0.28100000000000003},"max":{"1min":100.736,"5min":100.736,"15min":100.736},"last":0.47999999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":8.4090000000000007,"5min":4.3399999999999999,"15min":4.569},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":103.29300000000001,"5min":103.29300000000001,"15min":185.17099999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":11.114000000000001,"5min":4.8150000000000004,"15min":4.6909999999999998},"min":{"1min":0.39700000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":103.202,"5min":103.202,"15min":183.35499999999999},"last":0.51100000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":9.8770000000000007,"5min":3.4350000000000001,"15min":1.5529999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.25700000000000001},"max":{"1min":159.50299999999999,"5min":159.50299999999999,"15min":159.50299999999999},"last":0.623},{"interface":"front","average":{"1min":10.189,"5min":3.5230000000000001,"15min":1.611},"min":{"1min":0.32600000000000001,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":159.483,"5min":159.483,"15min":159.483},"last":0.58799999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.476,"15min":1.0329999999999999},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.26900000000000002},"max":{"1min":4.1319999999999997,"5min":49.210000000000001,"15min":49.210000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.246,"5min":1.7749999999999999,"15min":1.101},"min":{"1min":0.34999999999999998,"5min":0.308,"15min":0.28299999999999997},"max":{"1min":9.6739999999999995,"5min":49.274999999999999,"15min":49.274999999999999},"last":0.64700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:52:13 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":0.90200000000000002,"15min":0.75600000000000001},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.1720000000000006,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.095,"15min":0.85499999999999998},"min":{"1min":0.313,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.8200000000000003,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.54900000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626842,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1311776,"kb_used_data":415832,"kb_used_omap":0,"kb_used_meta":895936,"kb_avail":92433376,"statfs":{"total":95995035648,"available":94651777024,"internally_reserved":0,"allocated":425811968,"data_stored":425133631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917438464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:45 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":2.46,"15min":3.9430000000000001},"min":{"1min":0.255,"5min":0.248,"15min":0.20999999999999999},"max":{"1min":7.3449999999999998,"5min":42.869,"15min":162.08799999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.2240000000000002,"5min":2.5760000000000001,"15min":3.8809999999999998},"min":{"1min":0.26200000000000001,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":9.0749999999999993,"5min":42.892000000000003,"15min":161.90899999999999},"last":0.55200000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:53:11 2023","interfaces":[{"interface":"back","average":{"1min":6.5970000000000004,"5min":3.1560000000000001,"15min":4.0110000000000001},"min":{"1min":0.39400000000000002,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":101.374,"5min":101.374,"15min":228.24299999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":6.3070000000000004,"5min":3.3239999999999998,"15min":3.9620000000000002},"min":{"1min":0.374,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":101.35899999999999,"5min":101.35899999999999,"15min":228.22499999999999},"last":0.68000000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:52:45 2023","interfaces":[{"interface":"back","average":{"1min":6.8739999999999997,"5min":2.2530000000000001,"15min":3.5059999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27200000000000002},"max":{"1min":91.049999999999997,"5min":91.049999999999997,"15min":115.40600000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":7.3049999999999997,"5min":2.4420000000000002,"15min":3.6789999999999998},"min":{"1min":0.34000000000000002,"5min":0.33500000000000002,"15min":0.28699999999999998},"max":{"1min":91.105000000000004,"5min":91.105000000000004,"15min":115.297},"last":0.58299999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:53:11 2023","interfaces":[{"interface":"back","average":{"1min":7.0090000000000003,"5min":4.6769999999999996,"15min":4.5709999999999997},"min":{"1min":0.36399999999999999,"5min":0.33300000000000002,"15min":0.307},"max":{"1min":93.366,"5min":145.124,"15min":145.124},"last":0.46000000000000002},{"interface":"front","average":{"1min":5.8899999999999997,"5min":4.7229999999999999,"15min":4.5679999999999996},"min":{"1min":0.313,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":93.311000000000007,"5min":145.26599999999999,"15min":145.26599999999999},"last":0.61499999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:52:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":3.9100000000000001,"15min":1.9430000000000001},"min":{"1min":0.28599999999999998,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":9.4979999999999993,"5min":257.375,"15min":257.375},"last":0.439},{"interface":"front","average":{"1min":1.4590000000000001,"5min":4.0060000000000002,"15min":2.1760000000000002},"min":{"1min":0.309,"5min":0.26100000000000001,"15min":0.246},"max":{"1min":5.5659999999999998,"5min":257.61200000000002,"15min":257.61200000000002},"last":0.40500000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:52:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":3.9100000000000001,"15min":2.2799999999999998},"min":{"1min":0.33100000000000002,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":6.7839999999999998,"5min":256.05700000000002,"15min":256.05700000000002},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.7230000000000001,"5min":4.1059999999999999,"15min":2.3879999999999999},"min":{"1min":0.35799999999999998,"5min":0.29899999999999999,"15min":0.27500000000000002},"max":{"1min":6.6289999999999996,"5min":256.08499999999998,"15min":256.08499999999998},"last":0.48599999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:53:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.046,"15min":1.131},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.308},"max":{"1min":5.0010000000000003,"5min":7.3120000000000003,"15min":115.514},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.7030000000000001,"5min":1.431,"15min":1.3999999999999999},"min":{"1min":0.38,"5min":0.308,"15min":0.30599999999999999},"max":{"1min":6.1950000000000003,"5min":6.601,"15min":116.53100000000001},"last":0.59799999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958802,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":719408,"kb_used_data":3880,"kb_used_omap":7,"kb_used_meta":715512,"kb_avail":93025744,"statfs":{"total":95995035648,"available":95258361856,"internally_reserved":0,"allocated":3973120,"data_stored":2743856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":732684814},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":3.3370000000000002,"15min":3.8559999999999999},"min":{"1min":0.217,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":4.3330000000000002,"5min":93.176000000000002,"15min":93.176000000000002},"last":0.31900000000000001},{"interface":"front","average":{"1min":1.677,"5min":3.4199999999999999,"15min":4.3490000000000002},"min":{"1min":0.24399999999999999,"5min":0.223,"15min":0.182},"max":{"1min":5.641,"5min":92.593999999999994,"15min":92.593999999999994},"last":0.377}]},{"osd":2,"last update":"Mon Oct 9 05:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3940000000000001,"5min":2.1469999999999998,"15min":3.6890000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.19800000000000001},"max":{"1min":7.2510000000000003,"5min":11.914,"15min":101.367},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.373,"5min":1.7310000000000001,"15min":3.452},"min":{"1min":0.27800000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":3.6259999999999999,"5min":16.702999999999999,"15min":99.861000000000004},"last":0.42299999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:53:08 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":3.1419999999999999,"15min":4.9260000000000002},"min":{"1min":0.25700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.7509999999999999,"5min":92.353999999999999,"15min":162.33699999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.1779999999999999,"5min":3.089,"15min":5.1660000000000004},"min":{"1min":0.372,"5min":0.28599999999999998,"15min":0.26000000000000001},"max":{"1min":4.4770000000000003,"5min":93.117999999999995,"15min":162.376},"last":0.54700000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:53:08 2023","interfaces":[{"interface":"back","average":{"1min":1.704,"5min":1.681,"15min":2.0099999999999998},"min":{"1min":0.436,"5min":0.312,"15min":0.29799999999999999},"max":{"1min":5.3090000000000002,"5min":29.605,"15min":63.116},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.696,"15min":1.641},"min":{"1min":0.38800000000000001,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":4.391,"5min":29.515999999999998,"15min":40.475999999999999},"last":0.626}]},{"osd":5,"last update":"Mon Oct 9 05:53:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":2.0419999999999998,"15min":1.8919999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.222,"5min":32.774000000000001,"15min":40.633000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.5,"5min":1.73,"15min":2.077},"min":{"1min":0.40999999999999998,"5min":0.34499999999999997,"15min":0.32200000000000001},"max":{"1min":5.0800000000000001,"5min":36.976999999999997,"15min":63.162999999999997},"last":0.64800000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:53:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.798,"15min":2.613},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.316},"max":{"1min":8.3209999999999997,"5min":10.238,"15min":103.60899999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.897,"5min":1.4550000000000001,"15min":2.6429999999999998},"min":{"1min":0.48099999999999998,"5min":0.37,"15min":0.37},"max":{"1min":4.8819999999999997,"5min":9.641,"15min":103.679},"last":0.58099999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:53:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":1.8400000000000001,"15min":1.7490000000000001},"min":{"1min":0.45000000000000001,"5min":0.375,"15min":0.35999999999999999},"max":{"1min":5.8170000000000002,"5min":35.021000000000001,"15min":42.204000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.4570000000000001,"5min":2.016,"15min":1.853},"min":{"1min":0.48899999999999999,"5min":0.39200000000000002,"15min":0.35499999999999998},"max":{"1min":5.4180000000000001,"5min":35.493000000000002,"15min":42.228999999999999},"last":0.73499999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122329,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":672580,"kb_used_data":4540,"kb_used_omap":8,"kb_used_meta":668023,"kb_avail":93072572,"statfs":{"total":95995035648,"available":95306313728,"internally_reserved":0,"allocated":4648960,"data_stored":3348862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":684056328},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:53:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":2.4049999999999998,"15min":1.98},"min":{"1min":0.219,"5min":0.186,"15min":0.17999999999999999},"max":{"1min":0.54000000000000004,"5min":63.534999999999997,"15min":63.534999999999997},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.85899999999999999,"5min":2.4809999999999999,"15min":2.4580000000000002},"min":{"1min":0.26500000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.5990000000000002,"5min":63.677999999999997,"15min":63.677999999999997},"last":0.34899999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:52:39 2023","interfaces":[{"interface":"back","average":{"1min":8.0429999999999993,"5min":3.9140000000000001,"15min":3.4500000000000002},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.20000000000000001},"max":{"1min":70.188999999999993,"5min":70.188999999999993,"15min":92.230000000000004},"last":0.312},{"interface":"front","average":{"1min":7.0819999999999999,"5min":3.3399999999999999,"15min":3.161},"min":{"1min":0.25,"5min":0.221,"15min":0.221},"max":{"1min":71.144000000000005,"5min":71.144000000000005,"15min":92.247},"last":0.377}]},{"osd":3,"last update":"Mon Oct 9 05:53:09 2023","interfaces":[{"interface":"back","average":{"1min":5.375,"5min":3.0529999999999999,"15min":4.2859999999999996},"min":{"1min":0.249,"5min":0.249,"15min":0.218},"max":{"1min":62.698999999999998,"5min":62.698999999999998,"15min":123.024},"last":0.39200000000000002},{"interface":"front","average":{"1min":5.2939999999999996,"5min":3.754,"15min":4.3230000000000004},"min":{"1min":0.28599999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":60.383000000000003,"5min":72.632999999999996,"15min":99.162000000000006},"last":0.36299999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":3.2330000000000001,"15min":2.0750000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.25700000000000001},"max":{"1min":3.3660000000000001,"5min":149.63999999999999,"15min":149.63999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.9850000000000001,"5min":3.3300000000000001,"15min":2.5099999999999998},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":12.340999999999999,"5min":149.65899999999999,"15min":149.65899999999999},"last":0.52100000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:52:39 2023","interfaces":[{"interface":"back","average":{"1min":1.855,"5min":3.847,"15min":3.9660000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.317},"max":{"1min":12.282999999999999,"5min":152.20500000000001,"15min":281.733},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.387,"5min":4.468,"15min":4.2169999999999996},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30199999999999999},"max":{"1min":5.4989999999999997,"5min":154.30600000000001,"15min":282.95400000000001},"last":0.54200000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:52:39 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":1.5189999999999999,"15min":1.8049999999999999},"min":{"1min":0.36399999999999999,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":12.377000000000001,"5min":15.789,"15min":19.579000000000001},"last":0.497},{"interface":"front","average":{"1min":1.27,"5min":1.7,"15min":1.9890000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":5.5700000000000003,"5min":19.710999999999999,"15min":19.710999999999999},"last":0.58799999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:53:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.577,"15min":1.5029999999999999},"min":{"1min":0.35199999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.6420000000000003,"5min":15.933999999999999,"15min":25.378},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.613,"15min":1.8879999999999999},"min":{"1min":0.44600000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.7869999999999999,"5min":15.641,"15min":25.436},"last":0.57699999999999996}]}]},{"osd":2,"up_from":98,"seq":420906795276,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1353312,"kb_used_data":589400,"kb_used_omap":11,"kb_used_meta":763892,"kb_avail":92391840,"statfs":{"total":95995035648,"available":94609244160,"internally_reserved":0,"allocated":603545600,"data_stored":602093439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":782226005},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:46 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.99,"15min":3.4430000000000001},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.17799999999999999},"max":{"1min":12.129,"5min":12.829000000000001,"15min":162.285},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.976,"5min":1.9670000000000001,"15min":3.891},"min":{"1min":0.26500000000000001,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":8.2379999999999995,"5min":10.781000000000001,"15min":162.643},"last":0.47899999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:52:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9169999999999998,"5min":4.3170000000000002,"15min":3.7959999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.193},"max":{"1min":11.69,"5min":125.77500000000001,"15min":125.77500000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.1589999999999998,"5min":3.7850000000000001,"15min":3.1040000000000001},"min":{"1min":0.28599999999999998,"5min":0.24199999999999999,"15min":0.20999999999999999},"max":{"1min":6.6299999999999999,"5min":125.857,"15min":125.857},"last":0.33500000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.855,"15min":4.9240000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.21099999999999999},"max":{"1min":9.5129999999999999,"5min":9.5129999999999999,"15min":159.35499999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.7160000000000002,"5min":1.7170000000000001,"15min":4.7110000000000003},"min":{"1min":0.29399999999999998,"5min":0.217,"15min":0.217},"max":{"1min":11.303000000000001,"5min":11.303000000000001,"15min":159.196},"last":0.438}]},{"osd":4,"last update":"Mon Oct 9 05:52:46 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":2.0920000000000001,"15min":2.0720000000000001},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.28699999999999998},"max":{"1min":5.4450000000000003,"5min":68.052000000000007,"15min":115.84999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.0009999999999999,"5min":2.323,"15min":2.1419999999999999},"min":{"1min":0.36599999999999999,"5min":0.29199999999999998,"15min":0.28699999999999998},"max":{"1min":8.6349999999999998,"5min":68.727000000000004,"15min":116.149},"last":0.55500000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:52:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":2.206,"15min":2.1190000000000002},"min":{"1min":0.36099999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.8859999999999992,"5min":64.825000000000003,"15min":64.825000000000003},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.2010000000000001,"15min":1.845},"min":{"1min":0.371,"5min":0.371,"15min":0.26000000000000001},"max":{"1min":5.5229999999999997,"5min":64.582999999999998,"15min":64.582999999999998},"last":0.58899999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:52:46 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.643,"15min":1.6839999999999999},"min":{"1min":0.41199999999999998,"5min":0.33200000000000002,"15min":0.32000000000000001},"max":{"1min":8.6790000000000003,"5min":25.818999999999999,"15min":25.818999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":2.0539999999999998,"5min":1.9159999999999999,"15min":2.0110000000000001},"min":{"1min":0.40100000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":8.7599999999999998,"5min":24.481000000000002,"15min":24.481000000000002},"last":0.61799999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":1.629,"15min":2.0190000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":12.071999999999999,"5min":12.071999999999999,"15min":51.701999999999998},"last":0.495},{"interface":"front","average":{"1min":2.2170000000000001,"5min":1.74,"15min":2.2189999999999999},"min":{"1min":0.43099999999999999,"5min":0.378,"15min":0.373},"max":{"1min":6.2679999999999998,"5min":8.7430000000000003,"15min":51.716000000000001},"last":0.57199999999999995}]}]},{"osd":3,"up_from":103,"seq":442381631750,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1118456,"kb_used_data":364528,"kb_used_omap":4,"kb_used_meta":753915,"kb_avail":92626696,"statfs":{"total":95995035648,"available":94849736704,"internally_reserved":0,"allocated":373276672,"data_stored":371978856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":772009566},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:10 2023","interfaces":[{"interface":"back","average":{"1min":23.097999999999999,"5min":7.2030000000000003,"15min":5.5259999999999998},"min":{"1min":0.28499999999999998,"5min":0.19700000000000001,"15min":0.184},"max":{"1min":312.56599999999997,"5min":312.56599999999997,"15min":312.56599999999997},"last":0.499},{"interface":"front","average":{"1min":21.969000000000001,"5min":6.8330000000000002,"15min":5.8929999999999998},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":311.02300000000002,"5min":311.02300000000002,"15min":311.02300000000002},"last":0.54100000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:53:10 2023","interfaces":[{"interface":"back","average":{"1min":7.8330000000000002,"5min":5.2110000000000003,"15min":3.657},"min":{"1min":0.22,"5min":0.214,"15min":0.214},"max":{"1min":71.179000000000002,"5min":150.29599999999999,"15min":150.29599999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":8.0879999999999992,"5min":5.5270000000000001,"15min":3.9049999999999998},"min":{"1min":0.224,"5min":0.216,"15min":0.19},"max":{"1min":71.284999999999997,"5min":151.25700000000001,"15min":151.25700000000001},"last":0.434}]},{"osd":2,"last update":"Mon Oct 9 05:53:10 2023","interfaces":[{"interface":"back","average":{"1min":8.1150000000000002,"5min":3.0899999999999999,"15min":4.0910000000000002},"min":{"1min":0.248,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":72.602000000000004,"5min":72.602000000000004,"15min":190.43299999999999},"last":0.78100000000000003},{"interface":"front","average":{"1min":7.2949999999999999,"5min":3.0139999999999998,"15min":4.4260000000000002},"min":{"1min":0.317,"5min":0.221,"15min":0.20499999999999999},"max":{"1min":69.254000000000005,"5min":69.254000000000005,"15min":190.267},"last":0.623}]},{"osd":4,"last update":"Mon Oct 9 05:53:10 2023","interfaces":[{"interface":"back","average":{"1min":4.9989999999999997,"5min":3.532,"15min":3.069},"min":{"1min":0.371,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":69.224999999999994,"5min":126.566,"15min":149.84700000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":5.6820000000000004,"5min":4.766,"15min":3.0939999999999999},"min":{"1min":0.34399999999999997,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":70.439999999999998,"5min":126.651,"15min":149.74700000000001},"last":0.70199999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:53:10 2023","interfaces":[{"interface":"back","average":{"1min":6.0410000000000004,"5min":2.7890000000000001,"15min":2.6669999999999998},"min":{"1min":0.40000000000000002,"5min":0.307,"15min":0.28199999999999997},"max":{"1min":71.203999999999994,"5min":71.203999999999994,"15min":149.43000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":5.9550000000000001,"5min":2.528,"15min":2.4700000000000002},"min":{"1min":0.46600000000000003,"5min":0.32600000000000001,"15min":0.28599999999999998},"max":{"1min":70.397000000000006,"5min":70.397000000000006,"15min":149.69300000000001},"last":0.76300000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:53:10 2023","interfaces":[{"interface":"back","average":{"1min":5.9189999999999996,"5min":2.3740000000000001,"15min":2.3730000000000002},"min":{"1min":0.49099999999999999,"5min":0.35899999999999999,"15min":0.33500000000000002},"max":{"1min":70.474000000000004,"5min":70.474000000000004,"15min":149.72399999999999},"last":0.73999999999999999},{"interface":"front","average":{"1min":5.5030000000000001,"5min":2.573,"15min":2.6720000000000002},"min":{"1min":0.41199999999999998,"5min":0.35399999999999998,"15min":0.307},"max":{"1min":71.251999999999995,"5min":71.251999999999995,"15min":149.589},"last":0.51800000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":4.8719999999999999,"5min":2.0870000000000002,"15min":2.4769999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":69.277000000000001,"5min":69.277000000000001,"15min":149.77199999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":4.4809999999999999,"5min":2.0329999999999999,"15min":2.5870000000000002},"min":{"1min":0.48299999999999998,"5min":0.374,"15min":0.373},"max":{"1min":69.299000000000007,"5min":69.299000000000007,"15min":149.80799999999999},"last":0.71799999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019298,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1657636,"kb_used_data":756188,"kb_used_omap":0,"kb_used_meta":901440,"kb_avail":92087516,"statfs":{"total":95995035648,"available":94297616384,"internally_reserved":0,"allocated":774336512,"data_stored":773653857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923074560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7969999999999999,"5min":4.8650000000000002,"15min":3.1499999999999999},"min":{"1min":0.246,"5min":0.219,"15min":0.219},"max":{"1min":7.024,"5min":132.35599999999999,"15min":132.35599999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":2.3079999999999998,"5min":5.4429999999999996,"15min":3.5510000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":8.3960000000000008,"5min":132.501,"15min":132.501},"last":0.68799999999999994}]},{"osd":1,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":6.4989999999999997,"5min":3.2890000000000001,"15min":3.0099999999999998},"min":{"1min":0.34999999999999998,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":59.978999999999999,"5min":68.367000000000004,"15min":187.369},"last":0.82799999999999996},{"interface":"front","average":{"1min":6.1360000000000001,"5min":3.2429999999999999,"15min":3.0819999999999999},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.27700000000000002},"max":{"1min":59.991,"5min":68.376000000000005,"15min":187.35300000000001},"last":0.79700000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:52:34 2023","interfaces":[{"interface":"back","average":{"1min":5.8710000000000004,"5min":2.5990000000000002,"15min":3.407},"min":{"1min":0.34899999999999998,"5min":0.308,"15min":0.26800000000000002},"max":{"1min":51.768000000000001,"5min":51.768000000000001,"15min":196.93700000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":5.5659999999999998,"5min":2.3199999999999998,"15min":3.3260000000000001},"min":{"1min":0.35099999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":51.753999999999998,"5min":51.753999999999998,"15min":197.08099999999999},"last":0.75700000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:53:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":2.093,"15min":4.5789999999999997},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.4559999999999995,"5min":30.111999999999998,"15min":211.48699999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.573,"5min":1.921,"15min":4.4299999999999997},"min":{"1min":0.378,"5min":0.33300000000000002,"15min":0.312},"max":{"1min":5.7539999999999996,"5min":30.709,"15min":211.31100000000001},"last":0.80900000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:53:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.23,"15min":0.84499999999999997},"min":{"1min":0.27900000000000003,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":8.5190000000000001,"5min":12.34,"15min":12.34},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.1599999999999999,"15min":0.78700000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.6900000000000004,"5min":10.191000000000001,"15min":10.191000000000001},"last":0.66600000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.93799999999999994,"15min":0.70499999999999996},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.4939999999999998,"5min":9.8889999999999993,"15min":9.8889999999999993},"last":0.624},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.92700000000000005,"15min":0.875},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":2.387,"5min":6.0179999999999998,"15min":8.5269999999999992},"last":0.64500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":0.88600000000000001,"15min":1.034},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.5810000000000004,"5min":5.8650000000000002,"15min":96.131},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.1459999999999999,"5min":0.995,"15min":1.1399999999999999},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.3159999999999998,"5min":7.6580000000000004,"15min":96.465000000000003},"last":0.73899999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823070,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1173204,"kb_used_data":274508,"kb_used_omap":0,"kb_used_meta":898688,"kb_avail":92571948,"statfs":{"total":95995035648,"available":94793674752,"internally_reserved":0,"allocated":281096192,"data_stored":280425369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920256512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:52 2023","interfaces":[{"interface":"back","average":{"1min":5.0279999999999996,"5min":7.5510000000000002,"15min":4.0049999999999999},"min":{"1min":0.245,"5min":0.224,"15min":0.217},"max":{"1min":77.456999999999994,"5min":210.392,"15min":210.392},"last":0.45900000000000002},{"interface":"front","average":{"1min":5.8179999999999996,"5min":7.8650000000000002,"15min":4.2030000000000003},"min":{"1min":0.28899999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":77.828999999999994,"5min":208.43600000000001,"15min":208.43600000000001},"last":0.72999999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:53:11 2023","interfaces":[{"interface":"back","average":{"1min":4.9900000000000002,"5min":5.2370000000000001,"15min":3.7869999999999999},"min":{"1min":0.34200000000000003,"5min":0.247,"15min":0.23999999999999999},"max":{"1min":51.393000000000001,"5min":198.35300000000001,"15min":198.35300000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":4.3330000000000002,"5min":5.1390000000000002,"15min":3.871},"min":{"1min":0.35599999999999998,"5min":0.248,"15min":0.248},"max":{"1min":51.472000000000001,"5min":198.32900000000001,"15min":198.32900000000001},"last":0.66500000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:52:52 2023","interfaces":[{"interface":"back","average":{"1min":14.33,"5min":4.0650000000000004,"15min":3.0640000000000001},"min":{"1min":0.39800000000000002,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":195.92099999999999,"5min":195.92099999999999,"15min":195.92099999999999},"last":0.74299999999999999},{"interface":"front","average":{"1min":13.996,"5min":4.2279999999999998,"15min":3.0579999999999998},"min":{"1min":0.38600000000000001,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":195.905,"5min":195.905,"15min":195.905},"last":0.69399999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:53:11 2023","interfaces":[{"interface":"back","average":{"1min":4.2149999999999999,"5min":4.0750000000000002,"15min":4.4400000000000004},"min":{"1min":0.40600000000000003,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":47.488999999999997,"5min":85.069000000000003,"15min":95.236999999999995},"last":0.76500000000000001},{"interface":"front","average":{"1min":4.5810000000000004,"5min":4.0209999999999999,"15min":4.2460000000000004},"min":{"1min":0.39900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":47.549999999999997,"5min":84.814999999999998,"15min":93.471000000000004},"last":0.89200000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:53:11 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":2.359,"15min":1.2190000000000001},"min":{"1min":0.29099999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.1139999999999999,"5min":130.81,"15min":130.81},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.84199999999999997,"5min":2.5840000000000001,"15min":1.3169999999999999},"min":{"1min":0.27000000000000002,"5min":0.253,"15min":0.253},"max":{"1min":3.7850000000000001,"5min":133.803,"15min":133.803},"last":0.82899999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:52:26 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.8220000000000001,"15min":1.1899999999999999},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.8639999999999999,"5min":71.426000000000002,"15min":71.426000000000002},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.0550000000000002,"15min":1.2809999999999999},"min":{"1min":0.32300000000000001,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":4.109,"5min":71.498999999999995,"15min":71.498999999999995},"last":0.84499999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:52:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.383,"15min":1.0489999999999999},"min":{"1min":0.36799999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.085,"5min":31.533000000000001,"15min":31.533000000000001},"last":0.874},{"interface":"front","average":{"1min":1.6060000000000001,"5min":1.3759999999999999,"15min":1.0129999999999999},"min":{"1min":0.34300000000000003,"5min":0.316,"15min":0.308},"max":{"1min":8.5809999999999995,"5min":31.855,"15min":31.855},"last":1.03}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:53:18.320 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T05:53:18.320 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:53:18.320 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T05:53:18.320 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:53:18.320 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:53:18.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T05:53:18.350 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:18.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:18.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:18.457 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:18.484 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:18.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 2 16 4583 4567 142.709 143.25 0.00189228 0.00692962 2023-10-09T05:53:19.083 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:19.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:19.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:19.188 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:19.221 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:19.261 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:19.451 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:19.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:19.559 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:19.594 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:19.739 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:19.786 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:19.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:19.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 3 16 6197 6181 128.761 100.875 0.00644976 0.00774931 2023-10-09T05:53:19.893 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:19.951 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:20.256 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:20 smithi191 ceph-mon[161669]: pgmap v1113: 89 pgs: 89 active+clean; 19 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 1.5 MiB/s wr, 24 op/s 2023-10-09T05:53:20.259 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:20.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:20.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:20.279 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:20.279 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:20.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:20.401 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:20.443 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:20.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:20.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:20.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:20 smithi008 ceph-mon[184418]: pgmap v1113: 89 pgs: 89 active+clean; 19 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 1.5 MiB/s wr, 24 op/s 2023-10-09T05:53:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:20 smithi008 ceph-mon[191227]: pgmap v1113: 89 pgs: 89 active+clean; 19 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 1.5 MiB/s wr, 24 op/s 2023-10-09T05:53:20.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:20.573 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:20.715 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:20.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:20.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:20.819 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:20.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 4 16 7686 7670 119.834 93.0625 0.00235235 0.00784865 2023-10-09T05:53:20.864 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:53:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:53:20] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.33.4" 2023-10-09T05:53:20.871 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:21.116 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:53:21.146 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:21 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1529876041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:21.146 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:21 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:21.146 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:21 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1529876041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:21.146 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:21 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:21.148 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:21.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:21.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:21.252 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:21.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:21 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1529876041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:21 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:21.312 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:21.404 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:21.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:21.508 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:21.535 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:21.579 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:21.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:21.682 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:21.718 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:21.768 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:21.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:21.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:21.827 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 5 16 9062 9046 113.065 86 0.00170995 0.00866583 2023-10-09T05:53:21.871 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:21.902 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:22.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[184418]: pgmap v1114: 89 pgs: 89 active+clean; 222 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 22 MiB/s wr, 348 op/s 2023-10-09T05:53:22.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:22.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[184418]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[184418]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:22.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[184418]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T05:53:22.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[184418]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.113 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[184418]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.194 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:22.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:22.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:22.241 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:53:22.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:22 smithi191 ceph-mon[161669]: pgmap v1114: 89 pgs: 89 active+clean; 222 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 22 MiB/s wr, 348 op/s 2023-10-09T05:53:22.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:22 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:22.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:22 smithi191 ceph-mon[161669]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:22 smithi191 ceph-mon[161669]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:22 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:22.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:22 smithi191 ceph-mon[161669]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T05:53:22.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:22 smithi191 ceph-mon[161669]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:22 smithi191 ceph-mon[161669]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:22.353 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:22.378 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[191227]: pgmap v1114: 89 pgs: 89 active+clean; 222 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 22 MiB/s wr, 348 op/s 2023-10-09T05:53:22.378 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:22.378 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[191227]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.378 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[191227]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.378 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:22.378 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[191227]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T05:53:22.378 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[191227]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.379 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:22 smithi008 ceph-mon[191227]: failed to encode map e368 with expected crc 2023-10-09T05:53:22.489 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:22.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:22.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:22.600 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:22.634 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:53:22.658 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:22.672 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:53:22.679 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:53:22.679 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "" 2023-10-09T05:53:22.679 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:53:22.709 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:53:22.745 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:53:22.754 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:22.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:22.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:22.782 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:53:22.823 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:53:22.828 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 6 16 11439 11423 118.979 148.562 0.00371949 0.00839632 2023-10-09T05:53:22.857 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:53:22.860 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:22.893 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:53:22.898 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:53:22.898 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:53:22.898 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:53:22.899 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:53:22.899 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:53:22.899 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:53:22.899 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:53:22.899 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:53:22.915 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:23.008 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:23.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:23.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:23.114 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:23.168 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:23.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2949545870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:23.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2949545870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:23.268 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:23.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:23.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2949545870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:23.374 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:23.432 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:23.503 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:23.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:23.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:23.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:23.633 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:23.675 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:23.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:23.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:23.779 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:23.805 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:23.828 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 7 16 13264 13248 118.275 114.062 0.00166194 0.00802711 2023-10-09T05:53:23.846 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:23.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:23.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:23.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:23.999 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:24.159 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:24.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:24.249 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:53:24.266 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:24.268 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:24.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:24 smithi191 ceph-mon[161669]: pgmap v1116: 89 pgs: 89 active+clean; 471 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2023-10-09T05:53:24.358 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:24 smithi008 ceph-mon[184418]: pgmap v1116: 89 pgs: 89 active+clean; 471 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2023-10-09T05:53:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:24 smithi008 ceph-mon[191227]: pgmap v1116: 89 pgs: 89 active+clean; 471 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2023-10-09T05:53:24.466 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:24.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:24.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:24.572 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:24.626 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:24.746 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:24.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:24.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:24.828 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 8 16 15699 15683 122.512 152.188 0.00398402 0.00815505 2023-10-09T05:53:24.855 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:24.913 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:24.964 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:25.015 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:25.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:25.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:25.122 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:25.181 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:25.278 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:25.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:25.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:25.383 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:25.411 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:25.556 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:25 smithi008 ceph-mon[184418]: pgmap v1117: 89 pgs: 89 active+clean; 471 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 47 MiB/s wr, 746 op/s 2023-10-09T05:53:25.556 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:25 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:25.580 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:25.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:25.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:25.687 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:25.747 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:25.828 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 9 16 17567 17551 121.87 116.75 0.00315702 0.00820158 2023-10-09T05:53:25.885 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:25.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:25.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:25.891 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:25 smithi191 ceph-mon[161669]: pgmap v1117: 89 pgs: 89 active+clean; 471 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 47 MiB/s wr, 746 op/s 2023-10-09T05:53:25.891 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:25 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:25.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:25 smithi008 ceph-mon[191227]: pgmap v1117: 89 pgs: 89 active+clean; 471 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 47 MiB/s wr, 746 op/s 2023-10-09T05:53:25.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:25 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:25.991 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:26.049 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:26.153 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:26.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:26.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:26.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:26.316 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:26.317 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:53:26.413 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:26.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:26.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:26.518 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:26.526 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:53:26.527 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:53:26.527 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:53:26.527 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:53:26.527 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:53:26.585 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:26.670 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:26.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:26.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:26.773 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:26.800 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:26.828 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 10 16 19502 19486 121.776 120.938 0.0017972 0.00812619 2023-10-09T05:53:26.843 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:26.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:26.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:26.947 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:27.000 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:27.063 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:27.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:27.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:27.166 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:27.195 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:27.244 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:27.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:27.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:27.348 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:27.397 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:27.424 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:27.665 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:27 smithi191 ceph-mon[161669]: pgmap v1118: 89 pgs: 89 active+clean; 981 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T05:53:27.692 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:27.694 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:53:27.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:27.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:27.800 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:27.828 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 11 16 21159 21143 120.119 103.562 0.012859 0.008315 2023-10-09T05:53:27.855 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:27 smithi008 ceph-mon[184418]: pgmap v1118: 89 pgs: 89 active+clean; 981 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T05:53:27.855 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:27 smithi008 ceph-mon[191227]: pgmap v1118: 89 pgs: 89 active+clean; 981 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T05:53:27.861 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:27.966 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:27.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:27.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:28.076 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:28.135 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:28.246 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:28.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:28.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:28.303 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:28.353 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:28.404 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:28.522 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:28.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:28.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:28.630 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:28.690 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:28.828 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 12 16 23123 23107 120.337 122.75 0.0056076 0.00828073 2023-10-09T05:53:28.925 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:28.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:28.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:29.030 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:29.058 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:29.118 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:53:29.142 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:29.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:29.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:29.187 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:53:29.187 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:53:29.187 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T05:53:29.187 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:53:29.187 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:53:29.188 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T05:53:29.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:29.287 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/493768083' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:29.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:29 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:29.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:29.296 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:29.389 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:29.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:29.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:29.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/493768083' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:29.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:29 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:29.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:29.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/493768083' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:29.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:29 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:29.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:29.496 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:29.545 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:29.638 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:29.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:29.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:29.741 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:29.784 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:29.829 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 13 16 24522 24506 117.805 87.4375 0.0036097 0.00848178 2023-10-09T05:53:30.035 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:30.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:30.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:30.139 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:30.194 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:30 smithi191 ceph-mon[161669]: pgmap v1119: 89 pgs: 89 active+clean; 1.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T05:53:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:30 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:30 smithi191 ceph-mon[161669]: failed to encode map e369 with expected crc 2023-10-09T05:53:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:30 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:53:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:30 smithi191 ceph-mon[161669]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T05:53:30.318 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:30.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:30.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:30.424 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:30.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:30 smithi008 ceph-mon[184418]: pgmap v1119: 89 pgs: 89 active+clean; 1.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T05:53:30.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:30 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:30.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:30 smithi008 ceph-mon[184418]: failed to encode map e369 with expected crc 2023-10-09T05:53:30.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:30 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:53:30.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:30 smithi008 ceph-mon[184418]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T05:53:30.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:30 smithi008 ceph-mon[191227]: pgmap v1119: 89 pgs: 89 active+clean; 1.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T05:53:30.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:30 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:30.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:30 smithi008 ceph-mon[191227]: failed to encode map e369 with expected crc 2023-10-09T05:53:30.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:30 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:53:30.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:30 smithi008 ceph-mon[191227]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T05:53:30.480 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:30.616 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:30.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:30.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:30.724 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:53:30.724 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:53:30.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:53:30.724 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:53:30.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.724 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125677 7.11 7:8932b233:::benchmark_data_smithi008_354649_object31181:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:53:30.724 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:53:30.724 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 567.72183079900003, 2023-10-09T05:53:30.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0454934010000001, 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125677 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:53:30.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997453+0000", 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999967 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997449+0000", 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3732e-05 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997463+0000", 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9613999999999999e-05 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997483+0000", 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0620000000000007e-06 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997492+0000", 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.234801074 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232293+0000", 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2490000000000001e-06 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232301+0000", 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81054634999999997 2023-10-09T05:53:30.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.729 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.729 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.729 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042848+0000", 2023-10-09T05:53:30.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8449999999999994e-05 2023-10-09T05:53:30.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.729 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.729 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.729 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042946+0000", 2023-10-09T05:53:30.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0454934010000001 2023-10-09T05:53:30.729 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.729 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125688 7.11 7:8f585b70:::benchmark_data_smithi008_354649_object31192:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e239)", 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 567.72178323000003, 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0556018709999999, 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:30.730 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125688 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:53:30.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997500+0000", 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2820000000000001e-06 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997503+0000", 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0820000000000004e-06 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997509+0000", 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0659999999999997e-06 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:02.997513+0000", 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.234809199 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232322+0000", 2023-10-09T05:53:30.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6849999999999998e-06 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.232328+0000", 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81122250699999998 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043550+0000", 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114354 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043665+0000", 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00033335100000000001 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.043998+0000", 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001666088 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.045664+0000", 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041383430000000001 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.049803+0000", 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032002139999999998 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053003+0000", 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7229999999999999e-05 2023-10-09T05:53:30.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053060+0000", 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1964e-05 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.053102+0000", 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0556018709999999 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180297 7.21 7:846cd9a9:::benchmark_data_smithi008_354649_object85801:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:41.587040+0000", 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 409.13224372899998, 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48169166200000002, 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180297 2023-10-09T05:53:30.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587040+0000", 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587040+0000", 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587038+0000", 2023-10-09T05:53:30.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6429999999999995e-06 2023-10-09T05:53:30.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587046+0000", 2023-10-09T05:53:30.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4168999999999999e-05 2023-10-09T05:53:30.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587060+0000", 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9569999999999999e-06 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587066+0000", 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7617000000000003e-05 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587104+0000", 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1820000000000003e-06 2023-10-09T05:53:30.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.587110+0000", 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.47181123800000002 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.058921+0000", 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117278 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.059038+0000", 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000192509 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.059231+0000", 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001372269 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060603+0000", 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2033000000000003e-05 2023-10-09T05:53:30.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060635+0000", 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079930550000000006 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.068628+0000", 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9888e-05 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.068688+0000", 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3340999999999997e-05 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.068731+0000", 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.48169166200000002 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180300 7.1 7:807d12ef:::benchmark_data_smithi008_354649_object85804:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:41.743813+0000", 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 408.97546995800002, 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.316588969, 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180300 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.746 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743813+0000", 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743813+0000", 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743812+0000", 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2349999999999999e-06 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743820+0000", 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5903999999999999e-05 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743836+0000", 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0990000000000004e-06 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743842+0000", 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5695000000000001e-05 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743897+0000", 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4329999999999993e-06 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.743907+0000", 2023-10-09T05:53:30.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29237399200000003 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.036281+0000", 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9270000000000004e-05 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.036370+0000", 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028330299999999999 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.036653+0000", 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020744069999999999 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.038728+0000", 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024384929999999999 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.041166+0000", 2023-10-09T05:53:30.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019156355999999999 2023-10-09T05:53:30.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060323+0000", 2023-10-09T05:53:30.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2685999999999997e-05 2023-10-09T05:53:30.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060375+0000", 2023-10-09T05:53:30.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6947000000000001e-05 2023-10-09T05:53:30.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.060402+0000", 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.316588969 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86697 8.5 8:a67798ab:::benchmark_data_smithi008_432048_object86696:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088415+0000", 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 285.63086852700002, 2023-10-09T05:53:30.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.168776316, 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86697 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088415+0000", 2023-10-09T05:53:30.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088415+0000", 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088413+0000", 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6736999999999995e-05 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088479+0000", 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0219000000000001e-05 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088500+0000", 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1593000000000001e-05 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088511+0000", 2023-10-09T05:53:30.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017065599999999999 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088682+0000", 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1679999999999993e-06 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088691+0000", 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18748368300000001 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276175+0000", 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1679999999999993e-06 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.276184+0000", 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98094864800000003 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.257133+0000", 2023-10-09T05:53:30.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8631000000000002e-05 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.257191+0000", 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.168776316 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86704 8.15 8:af611326:::benchmark_data_smithi008_432048_object86703:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:53:30.759 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088533+0000", 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 285.63075061500001, 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1725390840000001, 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86704 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088533+0000", 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088533+0000", 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088532+0000", 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3013000000000002e-05 2023-10-09T05:53:30.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088585+0000", 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5350000000000004e-06 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088594+0000", 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3850000000000001e-06 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088599+0000", 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19036776799999999 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.278967+0000", 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1251e-05 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.278978+0000", 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98199783900000004 2023-10-09T05:53:30.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.260976+0000", 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5806999999999999e-05 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261072+0000", 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1725390840000001 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.765 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86708 8.15 8:ad569a26:::benchmark_data_smithi008_432048_object86707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:53:30.765 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:53:30.765 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 285.63067800200002, 2023-10-09T05:53:30.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1769074829999999, 2023-10-09T05:53:30.765 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.765 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:53:30.765 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.765 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.765 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.765 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86708 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:30.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088605+0000", 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6990000000000002e-05 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088652+0000", 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5610000000000004e-06 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088658+0000", 2023-10-09T05:53:30.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7790000000000003e-06 2023-10-09T05:53:30.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088663+0000", 2023-10-09T05:53:30.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19033387900000001 2023-10-09T05:53:30.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.278997+0000", 2023-10-09T05:53:30.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0989999999999998e-06 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.279002+0000", 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98210046799999995 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261102+0000", 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094320799999999996 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:53:30.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.262046+0000", 2023-10-09T05:53:30.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003411837 2023-10-09T05:53:30.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265458+0000", 2023-10-09T05:53:30.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5321000000000001e-05 2023-10-09T05:53:30.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265513+0000", 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1769074829999999 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86714 8.15 8:a990e10e:::benchmark_data_smithi008_432048_object86713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.088669+0000", 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 285.63061442999998, 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.177276395, 2023-10-09T05:53:30.771 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86714 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088669+0000", 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.772 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088669+0000", 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088668+0000", 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1603999999999997e-05 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088710+0000", 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8530000000000003e-06 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088717+0000", 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7299999999999996e-06 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.088722+0000", 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.190296668 2023-10-09T05:53:30.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.279018+0000", 2023-10-09T05:53:30.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.271e-06 2023-10-09T05:53:30.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:30.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.279024+0000", 2023-10-09T05:53:30.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98307463399999995 2023-10-09T05:53:30.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.262098+0000", 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00063494799999999998 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.262733+0000", 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031703159999999998 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265903+0000", 2023-10-09T05:53:30.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1888999999999998e-05 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.265945+0000", 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.177276395 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86722 8.12 8:4898e42c:::benchmark_data_smithi008_432048_object86721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-10-09T05:53:30.777 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.349501+0000", 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 285.36978262700001, 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91716482799999999, 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86722 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.778 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349501+0000", 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349501+0000", 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999971 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349498+0000", 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0645999999999997e-05 2023-10-09T05:53:30.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349539+0000", 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.594e-05 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349554+0000", 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2979999999999997e-06 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.349564+0000", 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001176355 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.350740+0000", 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0907000000000001e-05 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.350751+0000", 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91074052900000002 2023-10-09T05:53:30.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261492+0000", 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000151316 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261643+0000", 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021199000000000001 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261855+0000", 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046603080000000002 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266515+0000", 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.63e-06 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266522+0000", 2023-10-09T05:53:30.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0194e-05 2023-10-09T05:53:30.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.784 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.784 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266582+0000", 2023-10-09T05:53:30.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8968999999999998e-05 2023-10-09T05:53:30.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.784 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.784 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266631+0000", 2023-10-09T05:53:30.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4625000000000001e-05 2023-10-09T05:53:30.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266666+0000", 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91716482799999999 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88187 8.b 8:d5bc5a72:::benchmark_data_smithi008_432048_object88186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.469959+0000", 2023-10-09T05:53:30.785 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 283.24932441300001, 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30218297900000002, 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88187 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.469959+0000", 2023-10-09T05:53:30.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.469959+0000", 2023-10-09T05:53:30.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:53:30.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.469957+0000", 2023-10-09T05:53:30.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9060000000000002e-05 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.469997+0000", 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2267e-05 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470009+0000", 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6200000000000004e-06 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470014+0000", 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1153000000000003e-05 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470066+0000", 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000135251 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470201+0000", 2023-10-09T05:53:30.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017363400000000001 2023-10-09T05:53:30.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:53:30.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470374+0000", 2023-10-09T05:53:30.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30140982500000002 2023-10-09T05:53:30.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.771784+0000", 2023-10-09T05:53:30.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9709e-05 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.771844+0000", 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020167500000000001 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772046+0000", 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8795999999999997e-05 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772104+0000", 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.748e-05 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772142+0000", 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30218297900000002 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88189 8.f 8:f2efcde2:::benchmark_data_smithi008_432048_object88188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.470029+0000", 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 283.24925388299999, 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30080939000000001, 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88189 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.793 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470029+0000", 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470029+0000", 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470029+0000", 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0537000000000002e-05 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470069+0000", 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.289e-06 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470077+0000", 2023-10-09T05:53:30.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0610000000000003e-06 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470082+0000", 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0009999999999997e-05 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470132+0000", 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012962999999999999 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470261+0000", 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000156281 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470418+0000", 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003489214 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473907+0000", 2023-10-09T05:53:30.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29485467900000001 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768762+0000", 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019996879999999999 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770761+0000", 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3291000000000003e-05 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770805+0000", 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4267999999999998e-05 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770839+0000", 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30080939000000001 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88193 8.0 8:01ab35b2:::benchmark_data_smithi008_432048_object88192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.470097+0000", 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 283.249186157, 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.300629545, 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88193 2023-10-09T05:53:30.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470097+0000", 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470097+0000", 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470097+0000", 2023-10-09T05:53:30.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0299000000000002e-05 2023-10-09T05:53:30.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470137+0000", 2023-10-09T05:53:30.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3930000000000002e-06 2023-10-09T05:53:30.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470144+0000", 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5410000000000002e-06 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470149+0000", 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6909000000000001e-05 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470196+0000", 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112024 2023-10-09T05:53:30.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470308+0000", 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011089799999999999 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470419+0000", 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015793319999999999 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.471998+0000", 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29678586000000001 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768784+0000", 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001849271 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770633+0000", 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1421000000000001e-05 2023-10-09T05:53:30.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.806 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.806 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770685+0000", 2023-10-09T05:53:30.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2166000000000003e-05 2023-10-09T05:53:30.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.806 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.806 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770727+0000", 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.300629545 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88195 8.b 8:d594efea:::benchmark_data_smithi008_432048_object88194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.470165+0000", 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 283.249117904, 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.302059788, 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.807 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88195 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470165+0000", 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.808 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470165+0000", 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470165+0000", 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6063999999999999e-05 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470201+0000", 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9789999999999996e-06 2023-10-09T05:53:30.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470208+0000", 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7940000000000002e-06 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470213+0000", 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030594849999999998 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473272+0000", 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000137114 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473409+0000", 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018650599999999999 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473596+0000", 2023-10-09T05:53:30.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29822002399999997 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.771816+0000", 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014296999999999999 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.771959+0000", 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019265600000000001 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772151+0000", 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9477999999999997e-05 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772191+0000", 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4397000000000003e-05 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772225+0000", 2023-10-09T05:53:30.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.302059788 2023-10-09T05:53:30.814 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.814 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.814 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.814 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88196 8.15 8:ae709537:::benchmark_data_smithi008_432048_object88195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:53:30.814 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.470222+0000", 2023-10-09T05:53:30.814 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 283.24906183399997, 2023-10-09T05:53:30.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30074758899999998, 2023-10-09T05:53:30.814 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.814 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88196 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470222+0000", 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470222+0000", 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470221+0000", 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4220000000000001e-05 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470255+0000", 2023-10-09T05:53:30.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.585e-06 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470264+0000", 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8439999999999997e-06 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470269+0000", 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2599000000000002e-05 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470321+0000", 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011573700000000001 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470437+0000", 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119529 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:53:30.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.470556+0000", 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001557498 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.472114+0000", 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29872052100000002 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770834+0000", 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7939000000000002e-05 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770892+0000", 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.159e-05 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770934+0000", 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5179000000000003e-05 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.820 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770969+0000", 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30074758899999998 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88197 8.15 8:adcf56d2:::benchmark_data_smithi008_432048_object88196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.473499+0000", 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 283.24578389499999, 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29885566000000002, 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.821 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88197 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473499+0000", 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.822 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473499+0000", 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473498+0000", 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1958e-05 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.823 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473540+0000", 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1843e-05 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473552+0000", 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3709999999999996e-06 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473558+0000", 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00027575100000000002 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473834+0000", 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105849 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473940+0000", 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014184199999999999 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:53:30.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.474081+0000", 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29451698100000001 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768598+0000", 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022665799999999998 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.770865+0000", 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014092810000000001 2023-10-09T05:53:30.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772274+0000", 2023-10-09T05:53:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5215e-05 2023-10-09T05:53:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772320+0000", 2023-10-09T05:53:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5620000000000001e-05 2023-10-09T05:53:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.828 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.828 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772355+0000", 2023-10-09T05:53:30.828 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29885566000000002 2023-10-09T05:53:30.828 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.828 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.828 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.828 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.828 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.828 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:88199 8.b 8:d46d9522:::benchmark_data_smithi008_432048_object88198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-10-09T05:53:30.828 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:47.473580+0000", 2023-10-09T05:53:30.828 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 283.24570366699999, 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30265360200000002, 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 88199 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.829 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473580+0000", 2023-10-09T05:53:30.830 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.830 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.830 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.830 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.830 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473580+0000", 2023-10-09T05:53:30.830 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:30.830 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.830 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.830 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.830 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473579+0000", 2023-10-09T05:53:30.830 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7914999999999999e-05 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473617+0000", 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3939999999999998e-06 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473624+0000", 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3409999999999996e-06 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.831 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.473629+0000", 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030601980000000001 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.476689+0000", 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019625999999999999 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.476885+0000", 2023-10-09T05:53:30.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29183931099999999 2023-10-09T05:53:30.833 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.833 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.833 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:53:30.833 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.768725+0000", 2023-10-09T05:53:30.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038368299999999998 2023-10-09T05:53:30.833 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.833 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.833 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.833 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.772561+0000", 2023-10-09T05:53:30.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024502909999999998 2023-10-09T05:53:30.833 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.775012+0000", 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001103314 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.776115+0000", 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0850000000000002e-05 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.776176+0000", 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7447000000000001e-05 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:47.776233+0000", 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30265360200000002 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.835 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:144849 8.1b 8:df42c21d:::benchmark_data_smithi008_432048_object41907:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:51:15.171252+0000", 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 135.54803179699999, 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.34235762199999997, 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 144849 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.836 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.171252+0000", 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.171252+0000", 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.837 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.171250+0000", 2023-10-09T05:53:30.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1919999999999994e-06 2023-10-09T05:53:30.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.838 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.171259+0000", 2023-10-09T05:53:30.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8190999999999999e-05 2023-10-09T05:53:30.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.838 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.171277+0000", 2023-10-09T05:53:30.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9559999999999997e-06 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.171285+0000", 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3584000000000001e-05 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.171329+0000", 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014543299999999999 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.171474+0000", 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00048189199999999999 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.171956+0000", 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024546479999999998 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.174411+0000", 2023-10-09T05:53:30.840 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005342584 2023-10-09T05:53:30.841 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.841 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.841 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.841 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.179753+0000", 2023-10-09T05:53:30.841 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.33320208099999998 2023-10-09T05:53:30.841 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.841 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.841 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.841 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.512955+0000", 2023-10-09T05:53:30.841 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7524000000000001e-05 2023-10-09T05:53:30.841 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.513003+0000", 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060642699999999999 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.513609+0000", 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.34235762199999997 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.842 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.843 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.843 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.843 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:144863 8.2b 8:d44686fc:::benchmark_data_smithi008_432048_object41921:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-09T05:53:30.843 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:51:15.188223+0000", 2023-10-09T05:53:30.843 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 135.53106068400001, 2023-10-09T05:53:30.843 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.32401575300000002, 2023-10-09T05:53:30.843 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.843 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.843 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.843 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.843 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 144863 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.188223+0000", 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.188223+0000", 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:30.844 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.188221+0000", 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2809999999999997e-06 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.188230+0000", 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8020999999999999e-05 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.845 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.188248+0000", 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1799999999999999e-06 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.188255+0000", 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5791000000000002e-05 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.188311+0000", 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033730140000000001 2023-10-09T05:53:30.846 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.191684+0000", 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032481630000000001 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.194932+0000", 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.925e-05 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.195031+0000", 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028294600000000001 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.195314+0000", 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001070753 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.196385+0000", 2023-10-09T05:53:30.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016438550000000001 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.198029+0000", 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.313591809 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.511621+0000", 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2552000000000003e-05 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.850 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.850 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.511653+0000", 2023-10-09T05:53:30.850 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00058519099999999999 2023-10-09T05:53:30.850 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.850 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.850 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.850 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:15.512238+0000", 2023-10-09T05:53:30.850 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.32401575300000002 2023-10-09T05:53:30.850 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.850 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.850 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:168586 8.b 8:d162300c:::benchmark_data_smithi008_432048_object65644:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:51:53.735185+0000", 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 96.984098759999995, 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.34259732199999998, 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 168586 2023-10-09T05:53:30.851 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.735185+0000", 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.735185+0000", 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.735183+0000", 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6400000000000003e-06 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.735192+0000", 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3397999999999999e-05 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.735205+0000", 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1300000000000003e-06 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.735212+0000", 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6029999999999997e-05 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.735268+0000", 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00073543700000000003 2023-10-09T05:53:30.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.736004+0000", 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016080437 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.752084+0000", 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2169999999999993e-05 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.855 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.752176+0000", 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00035090499999999999 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.752527+0000", 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001328954 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.753856+0000", 2023-10-09T05:53:30.856 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018728449999999999 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:53.755729+0000", 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.32142850699999997 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:54.077158+0000", 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2362999999999999e-05 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.857 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.858 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:54.077220+0000", 2023-10-09T05:53:30.858 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00056188699999999996 2023-10-09T05:53:30.858 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.858 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.858 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.858 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:51:54.077782+0000", 2023-10-09T05:53:30.858 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.34259732199999998 2023-10-09T05:53:30.858 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.858 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.858 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.859 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.859 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.859 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:13235 9.4 9:2922d6a6:::benchmark_data_smithi008_502820_object13234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T05:53:30.859 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:53:23.472781+0000", 2023-10-09T05:53:30.859 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 7.2465026029999997, 2023-10-09T05:53:30.859 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.37626717199999998, 2023-10-09T05:53:30.859 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:30.859 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:30.859 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:30.859 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:53:30.860 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:53:30.860 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 13235 2023-10-09T05:53:30.860 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.860 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:30.860 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.860 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:30.860 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.472781+0000", 2023-10-09T05:53:30.860 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:30.860 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.860 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.860 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.472781+0000", 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.472780+0000", 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9483000000000001e-05 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.472820+0000", 2023-10-09T05:53:30.861 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8140000000000007e-06 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.472828+0000", 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1370000000000001e-06 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.472833+0000", 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010342050000000001 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.862 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:30.863 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.473868+0000", 2023-10-09T05:53:30.863 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011038 2023-10-09T05:53:30.863 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.863 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.863 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:30.863 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.473978+0000", 2023-10-09T05:53:30.863 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013944399999999999 2023-10-09T05:53:30.863 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.863 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.863 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:53:30.863 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.474118+0000", 2023-10-09T05:53:30.864 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00079111000000000003 2023-10-09T05:53:30.864 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.864 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.864 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:30.864 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.474909+0000", 2023-10-09T05:53:30.864 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012677280000000001 2023-10-09T05:53:30.864 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.864 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.864 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.864 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.476176+0000", 2023-10-09T05:53:30.864 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.37280109700000003 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.848977+0000", 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0655999999999998e-05 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.849018+0000", 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9824000000000001e-05 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:30.865 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:30.866 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:30.866 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:23.849048+0000", 2023-10-09T05:53:30.866 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.37626717199999998 2023-10-09T05:53:30.866 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.866 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.866 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.866 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:30.866 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:30.866 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:53:30.867 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:30.867 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 14 16 26138 26122 116.604 101 0.00426464 0.00857035 2023-10-09T05:53:30.905 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:30.922 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:53:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:53:30] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.33.4" 2023-10-09T05:53:30.934 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:31.044 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:31.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:31.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:31.104 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:53:31.142 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:53:31.149 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:31.172 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:53:31.191 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:31 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3608816669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:31.194 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:31.214 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:53:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:31 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3608816669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:31.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:31 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3608816669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:31.249 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:53:31.272 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:31.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:31.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:31.298 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:53:31.323 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:53:31.341 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:53:31.342 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:53:31.342 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:53:31.343 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:53:31.343 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:53:31.343 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:53:31.343 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:53:31.343 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:53:31.343 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:53:31.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:31.405 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:31.447 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:31.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:31.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:31.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:31.575 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:31.617 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:31.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:31.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:31.721 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:31.775 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:31.829 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 15 16 27417 27401 114.16 79.9375 0.00595299 0.00875405 2023-10-09T05:53:31.999 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:32.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:53:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:32.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:32.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:32.175 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:32.227 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:32.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:32.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:32.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:32 smithi191 ceph-mon[161669]: pgmap v1121: 89 pgs: 89 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T05:53:32.334 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:32.362 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:32.408 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:32.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:32.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:32.440 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:32.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:32 smithi008 ceph-mon[184418]: pgmap v1121: 89 pgs: 89 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T05:53:32.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:32 smithi008 ceph-mon[191227]: pgmap v1121: 89 pgs: 89 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T05:53:32.514 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:32.554 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:53:32.572 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:32.694 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:32.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:32.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:32.798 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:32.825 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:32.829 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 16 16 28565 28549 111.509 71.75 0.00181156 0.00893926 2023-10-09T05:53:32.869 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:32.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:32.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:32.972 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:33.033 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:33.077 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:33.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:33.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:33.180 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:33.241 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:33.462 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:33.545 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:33.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:33.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:33.650 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:33.702 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:33.829 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 17 16 29741 29725 109.27 73.5 0.0115114 0.00914743 2023-10-09T05:53:33.885 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:33.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:33.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:33.991 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:34.053 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:34.084 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:53:34.085 INFO:teuthology.orchestra.run.smithi008.stdout: "success": "" 2023-10-09T05:53:34.085 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:53:34.213 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:34.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:34.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:34.322 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:34.371 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:34 smithi008 ceph-mon[191227]: pgmap v1122: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T05:53:34.371 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:34 smithi008 ceph-mon[184418]: pgmap v1122: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T05:53:34.380 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:34.435 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:34.436 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:34.469 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:34 smithi191 ceph-mon[161669]: pgmap v1122: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T05:53:34.830 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 18 16 31243 31227 108.409 93.875 0.00924913 0.00921936 2023-10-09T05:53:35.256 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:35.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:35.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:35.355 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:53:35.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:35.392 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:35.459 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:35.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:35.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:35.465 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:53:35.566 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:35.628 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:35.830 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 19 16 32413 32397 106.552 73.125 0.00537225 0.00938076 2023-10-09T05:53:35.940 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:35.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:35.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:36.050 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:36.110 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:36.120 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:36.197 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:36 smithi008 ceph-mon[184418]: pgmap v1123: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T05:53:36.197 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:36 smithi008 ceph-mon[191227]: pgmap v1123: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T05:53:36.208 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:36.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:36.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:36.320 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:36.369 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:36.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:36 smithi191 ceph-mon[161669]: pgmap v1123: 89 pgs: 89 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T05:53:36.401 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:36.485 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:36.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:36.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:36.592 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:36.653 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:36.775 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:36.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:36.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:36.830 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:53:36.831246+0000 min lat: 0.00115306 max lat: 0.468193 avg lat: 0.0094028 2023-10-09T05:53:36.831 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:53:36.831 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 20 16 34032 34016 106.283 101.188 0.00883648 0.0094028 2023-10-09T05:53:36.880 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:36.943 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:37.140 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:53:37.215 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:37.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:37.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:37.319 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:37.348 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:37.416 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:37.420 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:37 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2235629680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:37.420 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:37 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2235629680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:37.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:37.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:37.522 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:37.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:37 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2235629680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:37.575 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:37.831 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 21 16 35187 35171 104.659 72.1875 0.00542512 0.009544 2023-10-09T05:53:38.274 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:38.274 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:38.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:38 smithi191 ceph-mon[161669]: pgmap v1124: 89 pgs: 89 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T05:53:38.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:38 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:53:38.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:38 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T05:53:38.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:38 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2235629680' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:38.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:38 smithi191 ceph-mon[161669]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T05:53:38.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:38 smithi191 ceph-mon[161669]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:38 smithi191 ceph-mon[161669]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:38 smithi191 ceph-mon[161669]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:38 smithi191 ceph-mon[161669]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:38 smithi191 ceph-mon[161669]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[184418]: pgmap v1124: 89 pgs: 89 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T05:53:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:53:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T05:53:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2235629680' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[184418]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T05:53:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[184418]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[184418]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[184418]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[184418]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[184418]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[191227]: pgmap v1124: 89 pgs: 89 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T05:53:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:53:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T05:53:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2235629680' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[191227]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T05:53:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[191227]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[191227]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[191227]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[191227]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:38 smithi008 ceph-mon[191227]: failed to encode map e370 with expected crc 2023-10-09T05:53:38.831 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 22 16 36577 36561 103.85 86.875 0.00247136 0.00959533 2023-10-09T05:53:39.021 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:39.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:39.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:39.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:39.186 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:39.350 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:39.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:39.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:39.359 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:53:39.457 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:39.513 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:39.671 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:39.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:39.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:39.783 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:39.831 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 23 16 38023 38007 103.264 90.375 0.0105801 0.00967796 2023-10-09T05:53:39.844 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:53:39.853 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:39.887 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:53:39.926 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:53:39.959 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:39.962 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:53:39.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:39.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:39.993 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:53:40.004 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:40.029 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:53:40.060 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:53:40.064 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:40.103 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:53:40.105 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:53:40.105 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:53:40.105 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:53:40.105 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:53:40.105 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:53:40.105 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:53:40.105 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:53:40.105 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:53:40.133 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:40.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:40 smithi008 ceph-mon[184418]: pgmap v1126: 89 pgs: 89 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T05:53:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:40 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3482015880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:40 smithi008 ceph-mon[191227]: pgmap v1126: 89 pgs: 89 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T05:53:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:40 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3482015880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:40.267 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:40.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:40.376 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:40.425 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:40 smithi191 ceph-mon[161669]: pgmap v1126: 89 pgs: 89 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T05:53:40.426 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:40 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3482015880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:40.431 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:40.516 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:40.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:40.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:40.671 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:40.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:53:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:53:40] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.33.4" 2023-10-09T05:53:40.779 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:40.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:40.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:40.831 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 24 16 39375 39359 102.482 84.5 0.00219675 0.00968033 2023-10-09T05:53:40.882 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:40.921 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:40.969 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:53:40.969 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:53:40.970 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T05:53:40.970 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T05:53:41.139 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:53:41.227 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:41.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:41.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:41.335 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:41.377 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:41.706 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:41.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:41.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:41.723 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:41.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:41.832 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 25 16 40776 40760 101.879 87.5625 0.00847096 0.00980917 2023-10-09T05:53:41.861 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:41.914 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:42.005 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:42.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:42.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:42.118 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:42.186 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:42.296 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:42.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:42 smithi191 ceph-mon[161669]: pgmap v1127: 89 pgs: 89 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:53:42.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:42 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:42.302 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:42 smithi008 ceph-mon[184418]: pgmap v1127: 89 pgs: 89 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:53:42.302 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:42 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:42.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:42.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:42 smithi008 ceph-mon[191227]: pgmap v1127: 89 pgs: 89 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T05:53:42.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:42 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:42.335 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:53:42.336 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:53:42.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:53:42.336 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:53:42.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.336 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:125694 7.10 7:0c5ab723:::benchmark_data_smithi008_354649_object31198:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-09T05:53:42.336 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:44:03.091387+0000", 2023-10-09T05:53:42.336 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 579.24073590099999, 2023-10-09T05:53:42.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95149025600000003, 2023-10-09T05:53:42.336 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.337 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.337 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.337 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:42.337 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:42.337 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 125694 2023-10-09T05:53:42.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.337 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091387+0000", 2023-10-09T05:53:42.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.338 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.338 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.338 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.338 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091387+0000", 2023-10-09T05:53:42.338 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999967 2023-10-09T05:53:42.338 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.338 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.338 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.338 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091384+0000", 2023-10-09T05:53:42.338 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1436e-05 2023-10-09T05:53:42.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.339 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091396+0000", 2023-10-09T05:53:42.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3645999999999999e-05 2023-10-09T05:53:42.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.339 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091409+0000", 2023-10-09T05:53:42.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4099999999999997e-06 2023-10-09T05:53:42.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.340 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.340 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091419+0000", 2023-10-09T05:53:42.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010047699999999999 2023-10-09T05:53:42.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.340 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.340 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091519+0000", 2023-10-09T05:53:42.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6649999999999998e-06 2023-10-09T05:53:42.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:03.091529+0000", 2023-10-09T05:53:42.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93579876100000003 2023-10-09T05:53:42.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.027328+0000", 2023-10-09T05:53:42.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013571199999999999 2023-10-09T05:53:42.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.027463+0000", 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015890500000000001 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.027622+0000", 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027023300000000002 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.030325+0000", 2023-10-09T05:53:42.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040089269999999998 2023-10-09T05:53:42.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.343 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.343 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.343 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.034334+0000", 2023-10-09T05:53:42.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.008445625 2023-10-09T05:53:42.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.343 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.343 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.343 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042779+0000", 2023-10-09T05:53:42.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8933e-05 2023-10-09T05:53:42.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.344 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042838+0000", 2023-10-09T05:53:42.344 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9582999999999997e-05 2023-10-09T05:53:42.344 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.344 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:44:04.042878+0000", 2023-10-09T05:53:42.344 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95149025600000003 2023-10-09T05:53:42.344 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.344 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150727 7.b 7:ddb4ebd7:::benchmark_data_smithi008_354649_object56231:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e252)", 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:16.972532+0000", 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 505.35959136500003, 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.216839464, 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:42.345 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:42.346 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150727 2023-10-09T05:53:42.346 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.346 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.346 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.346 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.346 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972532+0000", 2023-10-09T05:53:42.346 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.346 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.346 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.346 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.346 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972532+0000", 2023-10-09T05:53:42.347 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:42.347 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.347 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.347 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.347 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972530+0000", 2023-10-09T05:53:42.347 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8669999999999994e-06 2023-10-09T05:53:42.347 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.347 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.347 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.347 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972538+0000", 2023-10-09T05:53:42.347 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3001e-05 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972551+0000", 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5217e-05 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.972566+0000", 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002708668 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.975275+0000", 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9929999999999998e-06 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:16.975282+0000", 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93068869399999998 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.905971+0000", 2023-10-09T05:53:42.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0430000000000008e-06 2023-10-09T05:53:42.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.905979+0000", 2023-10-09T05:53:42.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.232000741 2023-10-09T05:53:42.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.137980+0000", 2023-10-09T05:53:42.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021446479999999999 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.140124+0000", 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6557000000000004e-05 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.140221+0000", 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011995643 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.152216+0000", 2023-10-09T05:53:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00300789 2023-10-09T05:53:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.155224+0000", 2023-10-09T05:53:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015419295 2023-10-09T05:53:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.170644+0000", 2023-10-09T05:53:42.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018450558999999998 2023-10-09T05:53:42.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189094+0000", 2023-10-09T05:53:42.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119319 2023-10-09T05:53:42.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189213+0000", 2023-10-09T05:53:42.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000158014 2023-10-09T05:53:42.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189371+0000", 2023-10-09T05:53:42.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.216839464 2023-10-09T05:53:42.354 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.354 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.354 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150733 7.0 7:0389df5e:::benchmark_data_smithi008_354649_object56237:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:17.005416+0000", 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 505.326706973, 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2781442140000001, 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150733 2023-10-09T05:53:42.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.356 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005416+0000", 2023-10-09T05:53:42.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005416+0000", 2023-10-09T05:53:42.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:53:42.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005415+0000", 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0110000000000003e-06 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005422+0000", 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3579e-05 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005435+0000", 2023-10-09T05:53:42.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7229999999999998e-06 2023-10-09T05:53:42.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005442+0000", 2023-10-09T05:53:42.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8405000000000001e-05 2023-10-09T05:53:42.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005501+0000", 2023-10-09T05:53:42.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4299999999999995e-06 2023-10-09T05:53:42.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.005510+0000", 2023-10-09T05:53:42.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.902713443 2023-10-09T05:53:42.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.908223+0000", 2023-10-09T05:53:42.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7690000000000008e-06 2023-10-09T05:53:42.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.908233+0000", 2023-10-09T05:53:42.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3617464159999999 2023-10-09T05:53:42.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.269980+0000", 2023-10-09T05:53:42.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119249 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.270099+0000", 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112638 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.270212+0000", 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059159390000000003 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.276127+0000", 2023-10-09T05:53:42.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033336210000000002 2023-10-09T05:53:42.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.279461+0000", 2023-10-09T05:53:42.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002913522 2023-10-09T05:53:42.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.282375+0000", 2023-10-09T05:53:42.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012422099999999999 2023-10-09T05:53:42.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.282499+0000", 2023-10-09T05:53:42.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00106174 2023-10-09T05:53:42.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.364 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.364 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.283561+0000", 2023-10-09T05:53:42.364 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2781442140000001 2023-10-09T05:53:42.364 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.364 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.364 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.364 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.364 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.364 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150737 7.b 7:d87fa168:::benchmark_data_smithi008_354649_object56241:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:53:42.364 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:17.110860+0000", 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 505.22126363500001, 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0787747959999998, 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150737 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.365 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110860+0000", 2023-10-09T05:53:42.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110860+0000", 2023-10-09T05:53:42.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:42.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110858+0000", 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9359999999999994e-06 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110868+0000", 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9113000000000001e-05 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110887+0000", 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4679999999999992e-06 2023-10-09T05:53:42.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110895+0000", 2023-10-09T05:53:42.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3297e-05 2023-10-09T05:53:42.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110969+0000", 2023-10-09T05:53:42.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3549e-05 2023-10-09T05:53:42.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.110982+0000", 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.79500997399999995 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.905992+0000", 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1529999999999999e-06 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.905996+0000", 2023-10-09T05:53:42.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.237875888 2023-10-09T05:53:42.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.143872+0000", 2023-10-09T05:53:42.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0003837 2023-10-09T05:53:42.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.144256+0000", 2023-10-09T05:53:42.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015668100000000001 2023-10-09T05:53:42.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:53:42.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.144412+0000", 2023-10-09T05:53:42.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0092940639999999995 2023-10-09T05:53:42.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.153707+0000", 2023-10-09T05:53:42.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001565924 2023-10-09T05:53:42.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.155272+0000", 2023-10-09T05:53:42.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.027123054000000001 2023-10-09T05:53:42.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.182396+0000", 2023-10-09T05:53:42.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070547509999999997 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189450+0000", 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3298000000000003e-05 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189514+0000", 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120955 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189635+0000", 2023-10-09T05:53:42.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0787747959999998 2023-10-09T05:53:42.374 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.374 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.374 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.374 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150750 7.b 7:d9e0951c:::benchmark_data_smithi008_354649_object56254:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:53:42.374 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:17.325970+0000", 2023-10-09T05:53:42.375 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 505.00615286999999, 2023-10-09T05:53:42.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8639324529999999, 2023-10-09T05:53:42.375 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.375 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.375 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.375 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:42.375 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:42.375 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150750 2023-10-09T05:53:42.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.375 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.325970+0000", 2023-10-09T05:53:42.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.325970+0000", 2023-10-09T05:53:42.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:42.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.325969+0000", 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8060000000000001e-06 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.325978+0000", 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8423e-05 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.325996+0000", 2023-10-09T05:53:42.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7100000000000001e-06 2023-10-09T05:53:42.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.378 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.326003+0000", 2023-10-09T05:53:42.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1128000000000002e-05 2023-10-09T05:53:42.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.378 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.326064+0000", 2023-10-09T05:53:42.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0229e-05 2023-10-09T05:53:42.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.326074+0000", 2023-10-09T05:53:42.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.57993254100000002 2023-10-09T05:53:42.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906007+0000", 2023-10-09T05:53:42.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8789999999999996e-06 2023-10-09T05:53:42.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906010+0000", 2023-10-09T05:53:42.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.24214676 2023-10-09T05:53:42.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.148157+0000", 2023-10-09T05:53:42.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012305199999999999 2023-10-09T05:53:42.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.148280+0000", 2023-10-09T05:53:42.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000149951 2023-10-09T05:53:42.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:53:42.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.148430+0000", 2023-10-09T05:53:42.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006522707 2023-10-09T05:53:42.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.154953+0000", 2023-10-09T05:53:42.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00036319500000000002 2023-10-09T05:53:42.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.155316+0000", 2023-10-09T05:53:42.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.027118198999999999 2023-10-09T05:53:42.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.182434+0000", 2023-10-09T05:53:42.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073679979999999997 2023-10-09T05:53:42.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189802+0000", 2023-10-09T05:53:42.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1600999999999998e-05 2023-10-09T05:53:42.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189834+0000", 2023-10-09T05:53:42.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8996000000000001e-05 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.189903+0000", 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8639324529999999 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150752 7.10 7:089e0e87:::benchmark_data_smithi008_354649_object56256:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:53:42.384 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:17.410069+0000", 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 504.92205410999998, 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.881545647, 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150752 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410069+0000", 2023-10-09T05:53:42.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410069+0000", 2023-10-09T05:53:42.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:42.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410068+0000", 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5140000000000001e-06 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410076+0000", 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7410000000000001e-05 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410093+0000", 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0040000000000002e-06 2023-10-09T05:53:42.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410100+0000", 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3235000000000001e-05 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410154+0000", 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7259999999999997e-06 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.410163+0000", 2023-10-09T05:53:42.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.49602615999999999 2023-10-09T05:53:42.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906190+0000", 2023-10-09T05:53:42.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.164e-06 2023-10-09T05:53:42.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906195+0000", 2023-10-09T05:53:42.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.36729938 2023-10-09T05:53:42.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.273494+0000", 2023-10-09T05:53:42.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016964899999999999 2023-10-09T05:53:42.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.273664+0000", 2023-10-09T05:53:42.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015241199999999999 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.273816+0000", 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0056057640000000001 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.279422+0000", 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079792330000000005 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.287401+0000", 2023-10-09T05:53:42.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031066990000000001 2023-10-09T05:53:42.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.290508+0000", 2023-10-09T05:53:42.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9672999999999997e-05 2023-10-09T05:53:42.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.290558+0000", 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00105737 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.291615+0000", 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.881545647 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.394 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:150753 7.10 7:0ce0167e:::benchmark_data_smithi008_354649_object56257:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:53:42.394 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:45:17.452910+0000", 2023-10-09T05:53:42.394 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 504.87921320599997, 2023-10-09T05:53:42.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8600227089999999, 2023-10-09T05:53:42.394 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.394 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.394 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.394 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:42.394 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:42.394 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 150753 2023-10-09T05:53:42.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452910+0000", 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452910+0000", 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452909+0000", 2023-10-09T05:53:42.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4999999999999999e-06 2023-10-09T05:53:42.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452917+0000", 2023-10-09T05:53:42.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6939000000000002e-05 2023-10-09T05:53:42.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452934+0000", 2023-10-09T05:53:42.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2430000000000002e-06 2023-10-09T05:53:42.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452940+0000", 2023-10-09T05:53:42.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6143000000000001e-05 2023-10-09T05:53:42.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.452996+0000", 2023-10-09T05:53:42.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7030000000000001e-06 2023-10-09T05:53:42.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.453006+0000", 2023-10-09T05:53:42.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.45320798200000001 2023-10-09T05:53:42.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906214+0000", 2023-10-09T05:53:42.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0080000000000002e-06 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:17.906219+0000", 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3699190080000001 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.276138+0000", 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000113307 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.276251+0000", 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014198699999999999 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.276393+0000", 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071874319999999997 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.283581+0000", 2023-10-09T05:53:42.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038658339999999999 2023-10-09T05:53:42.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.287447+0000", 2023-10-09T05:53:42.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024229746 2023-10-09T05:53:42.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.311676+0000", 2023-10-09T05:53:42.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6548e-05 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.311753+0000", 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001179905 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:45:19.312933+0000", 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8600227089999999 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.402 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180287 7.0 7:0049df52:::benchmark_data_smithi008_354649_object85791:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:53:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:41.442036+0000", 2023-10-09T05:53:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 420.89008707300002, 2023-10-09T05:53:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.551793397, 2023-10-09T05:53:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180287 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442036+0000", 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442036+0000", 2023-10-09T05:53:42.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:42.405 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.405 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.405 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.405 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442034+0000", 2023-10-09T05:53:42.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9820000000000003e-06 2023-10-09T05:53:42.405 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.405 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.405 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.405 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442044+0000", 2023-10-09T05:53:42.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9191999999999999e-05 2023-10-09T05:53:42.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.406 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.406 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442063+0000", 2023-10-09T05:53:42.406 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6090000000000001e-06 2023-10-09T05:53:42.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.406 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.406 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442072+0000", 2023-10-09T05:53:42.406 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4536999999999996e-05 2023-10-09T05:53:42.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442137+0000", 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1471000000000001e-05 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442148+0000", 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.53468159800000004 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.976830+0000", 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.429e-05 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.976844+0000", 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99917367300000004 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.976018+0000", 2023-10-09T05:53:42.408 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00050756399999999995 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.976525+0000", 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0066100669999999999 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.983135+0000", 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8519999999999999e-05 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.409 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.410 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.410 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.983234+0000", 2023-10-09T05:53:42.410 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00039802300000000001 2023-10-09T05:53:42.410 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.410 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.410 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:53:42.410 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.983632+0000", 2023-10-09T05:53:42.410 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045434739999999996 2023-10-09T05:53:42.410 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.410 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.410 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.988175+0000", 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002886684 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.991062+0000", 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025684789999999998 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.993630+0000", 2023-10-09T05:53:42.411 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2209000000000002e-05 2023-10-09T05:53:42.412 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.412 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.412 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.412 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.993673+0000", 2023-10-09T05:53:42.412 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000157071 2023-10-09T05:53:42.412 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.412 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.412 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.412 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.993830+0000", 2023-10-09T05:53:42.412 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.551793397 2023-10-09T05:53:42.412 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180312 7.1b 7:dfc0fe71:::benchmark_data_smithi008_354649_object85816:head [delete] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:42.080432+0000", 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 420.25169131000001, 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9004082920000001, 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.413 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180312 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080432+0000", 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.414 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080432+0000", 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080430+0000", 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4959999999999999e-06 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080438+0000", 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4079e-05 2023-10-09T05:53:42.415 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.416 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.416 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.416 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080452+0000", 2023-10-09T05:53:42.416 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9e-06 2023-10-09T05:53:42.416 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.416 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.416 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.416 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080459+0000", 2023-10-09T05:53:42.416 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7777e-05 2023-10-09T05:53:42.416 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080506+0000", 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8789999999999998e-06 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080516+0000", 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86410833499999995 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.417 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.418 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.944625+0000", 2023-10-09T05:53:42.418 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0523000000000001e-05 2023-10-09T05:53:42.418 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.418 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.418 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.944635+0000", 2023-10-09T05:53:42.418 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.029066319 2023-10-09T05:53:42.418 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.418 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.418 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.973701+0000", 2023-10-09T05:53:42.419 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000104314 2023-10-09T05:53:42.419 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.419 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.419 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.419 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.973806+0000", 2023-10-09T05:53:42.419 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105163 2023-10-09T05:53:42.419 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.419 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.419 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:53:42.419 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.973911+0000", 2023-10-09T05:53:42.420 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048823 2023-10-09T05:53:42.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.420 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.420 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.420 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.978793+0000", 2023-10-09T05:53:42.420 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001543518 2023-10-09T05:53:42.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.420 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.420 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.420 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980337+0000", 2023-10-09T05:53:42.421 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018337200000000001 2023-10-09T05:53:42.421 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.421 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.421 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.421 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980520+0000", 2023-10-09T05:53:42.421 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000125826 2023-10-09T05:53:42.421 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.421 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.421 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.421 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980646+0000", 2023-10-09T05:53:42.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9175000000000001e-05 2023-10-09T05:53:42.422 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.422 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.422 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.422 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980715+0000", 2023-10-09T05:53:42.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012521 2023-10-09T05:53:42.422 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.422 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.422 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.422 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980840+0000", 2023-10-09T05:53:42.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9004082920000001 2023-10-09T05:53:42.423 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.423 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.423 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.423 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.423 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.423 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180373 7.1b 7:d95ec086:::benchmark_data_smithi008_354649_object85877:head [delete] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-10-09T05:53:42.423 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:42.192520+0000", 2023-10-09T05:53:42.423 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 420.139603776, 2023-10-09T05:53:42.423 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7885932069999999, 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180373 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192520+0000", 2023-10-09T05:53:42.424 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192520+0000", 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192518+0000", 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.024e-06 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.426 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192525+0000", 2023-10-09T05:53:42.426 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1698999999999999e-05 2023-10-09T05:53:42.426 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.426 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.426 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192537+0000", 2023-10-09T05:53:42.426 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9819999999999999e-06 2023-10-09T05:53:42.426 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.426 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.427 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192542+0000", 2023-10-09T05:53:42.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9879000000000001e-05 2023-10-09T05:53:42.427 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.427 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.427 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.427 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192592+0000", 2023-10-09T05:53:42.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.173e-06 2023-10-09T05:53:42.427 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.427 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.427 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.427 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192598+0000", 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.75205944400000002 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.944657+0000", 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2369999999999997e-06 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.944664+0000", 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.030814664 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.975478+0000", 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014284399999999999 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.975621+0000", 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015404700000000001 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.429 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:53:42.430 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.975775+0000", 2023-10-09T05:53:42.430 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030813910000000002 2023-10-09T05:53:42.430 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.430 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.430 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.430 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.978856+0000", 2023-10-09T05:53:42.430 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001523546 2023-10-09T05:53:42.430 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.430 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.430 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.430 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980380+0000", 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000185644 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980566+0000", 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000327907 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980894+0000", 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1761000000000006e-05 2023-10-09T05:53:42.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980965+0000", 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000147479 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.981113+0000", 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7885932069999999 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.432 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.433 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.433 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180398 7.1b 7:ddd0bb0e:::benchmark_data_smithi008_354649_object85902:head [delete] snapc 0=[] ondisk+write+known_if_redirected e271)", 2023-10-09T05:53:42.433 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:42.996383+0000", 2023-10-09T05:53:42.433 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 419.33573986200003, 2023-10-09T05:53:42.433 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98488713800000005, 2023-10-09T05:53:42.433 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.433 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.433 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.433 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:53:42.433 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:53:42.434 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180398 2023-10-09T05:53:42.434 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.434 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.434 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.434 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.434 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.996383+0000", 2023-10-09T05:53:42.434 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.434 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.434 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.434 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.434 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.996383+0000", 2023-10-09T05:53:42.435 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:53:42.435 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.435 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.435 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.435 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.996383+0000", 2023-10-09T05:53:42.435 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4750000000000004e-06 2023-10-09T05:53:42.435 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.435 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.435 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.435 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.996387+0000", 2023-10-09T05:53:42.435 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1370000000000002e-06 2023-10-09T05:53:42.436 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.436 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.436 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.436 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.996395+0000", 2023-10-09T05:53:42.436 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2610000000000004e-06 2023-10-09T05:53:42.436 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.436 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.436 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.436 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.996400+0000", 2023-10-09T05:53:42.436 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3977999999999998e-05 2023-10-09T05:53:42.436 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.437 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.437 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.437 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.996444+0000", 2023-10-09T05:53:42.437 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3369999999999996e-06 2023-10-09T05:53:42.437 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.437 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.437 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.437 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.996450+0000", 2023-10-09T05:53:42.437 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98053794000000005 2023-10-09T05:53:42.437 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.437 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.438 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.438 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.976988+0000", 2023-10-09T05:53:42.438 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118853 2023-10-09T05:53:42.438 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.438 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.438 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.438 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.977107+0000", 2023-10-09T05:53:42.438 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012536700000000001 2023-10-09T05:53:42.438 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.438 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.438 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:53:42.439 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.977232+0000", 2023-10-09T05:53:42.439 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001669184 2023-10-09T05:53:42.439 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.439 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.439 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.439 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.978901+0000", 2023-10-09T05:53:42.439 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015792149999999999 2023-10-09T05:53:42.439 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.439 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.439 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.439 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980480+0000", 2023-10-09T05:53:42.440 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012991500000000001 2023-10-09T05:53:42.440 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.440 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.440 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.440 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980610+0000", 2023-10-09T05:53:42.440 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00054580799999999995 2023-10-09T05:53:42.440 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.440 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.440 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.440 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.981156+0000", 2023-10-09T05:53:42.440 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4378000000000002e-05 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.981191+0000", 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0045999999999999e-05 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.981271+0000", 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98488713800000005 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.441 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.442 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.442 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.442 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86696 8.14 8:2cb5a248:::benchmark_data_smithi008_432048_object86695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:53:42.442 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078540+0000", 2023-10-09T05:53:42.442 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 297.25358319100002, 2023-10-09T05:53:42.442 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.192233133, 2023-10-09T05:53:42.442 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.442 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:53:42.442 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.442 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:42.443 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:42.443 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86696 2023-10-09T05:53:42.443 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.443 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.443 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.443 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.443 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078540+0000", 2023-10-09T05:53:42.443 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.443 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.443 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.443 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078540+0000", 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078538+0000", 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5841000000000003e-05 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078594+0000", 2023-10-09T05:53:42.444 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4001e-05 2023-10-09T05:53:42.445 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.445 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.445 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.445 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078608+0000", 2023-10-09T05:53:42.445 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3899999999999998e-06 2023-10-09T05:53:42.445 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.445 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.445 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.445 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078614+0000", 2023-10-09T05:53:42.445 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21596270400000001 2023-10-09T05:53:42.445 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.446 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.446 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.446 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294577+0000", 2023-10-09T05:53:42.446 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7299999999999994e-06 2023-10-09T05:53:42.446 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.446 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.446 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.446 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294586+0000", 2023-10-09T05:53:42.446 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97611344200000005 2023-10-09T05:53:42.446 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.446 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.270699+0000", 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4124999999999999e-05 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.270773+0000", 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.192233133 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.447 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.448 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.448 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86699 8.3 8:c3d6f3c4:::benchmark_data_smithi008_432048_object86698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:53:42.448 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:53:42.448 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 297.25349810900002, 2023-10-09T05:53:42.448 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650868249999999, 2023-10-09T05:53:42.448 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.448 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:53:42.448 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.448 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:42.448 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:42.448 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86699 2023-10-09T05:53:42.449 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.449 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.449 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.449 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.449 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:53:42.449 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.449 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.449 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.449 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.449 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:53:42.449 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:42.450 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.450 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.450 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.450 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:53:42.450 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0989999999999999e-05 2023-10-09T05:53:42.450 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.450 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.450 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.450 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078666+0000", 2023-10-09T05:53:42.450 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7989999999999996e-06 2023-10-09T05:53:42.450 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.451 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.451 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.451 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078672+0000", 2023-10-09T05:53:42.451 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0949999999999998e-06 2023-10-09T05:53:42.451 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.451 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.451 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.451 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078676+0000", 2023-10-09T05:53:42.451 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15420414599999999 2023-10-09T05:53:42.451 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.452 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.452 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.452 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.232881+0000", 2023-10-09T05:53:42.452 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6739999999999992e-06 2023-10-09T05:53:42.452 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.452 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.452 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.452 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.232890+0000", 2023-10-09T05:53:42.452 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0107454360000001 2023-10-09T05:53:42.452 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.452 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.243636+0000", 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6348999999999997e-05 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.243712+0000", 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650868249999999 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.453 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.454 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.454 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86700 8.18 8:1e3af286:::benchmark_data_smithi008_432048_object86699:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:53:42.454 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078684+0000", 2023-10-09T05:53:42.454 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 297.25343950600001, 2023-10-09T05:53:42.454 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1857459349999999, 2023-10-09T05:53:42.454 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.454 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:53:42.454 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.454 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:42.454 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:42.454 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86700 2023-10-09T05:53:42.455 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.457 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.458 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.458 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.458 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078684+0000", 2023-10-09T05:53:42.458 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.458 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.458 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.458 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.458 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078684+0000", 2023-10-09T05:53:42.458 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:42.458 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.459 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.459 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078683+0000", 2023-10-09T05:53:42.459 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9190999999999998e-05 2023-10-09T05:53:42.459 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.459 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.459 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078723+0000", 2023-10-09T05:53:42.459 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0850000000000002e-06 2023-10-09T05:53:42.459 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.460 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.460 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078729+0000", 2023-10-09T05:53:42.460 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7340000000000002e-06 2023-10-09T05:53:42.460 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.460 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.460 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.460 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078732+0000", 2023-10-09T05:53:42.460 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.179763071 2023-10-09T05:53:42.460 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.460 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.460 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.461 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258495+0000", 2023-10-09T05:53:42.461 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8499999999999999e-06 2023-10-09T05:53:42.461 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.461 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.461 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.461 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258501+0000", 2023-10-09T05:53:42.461 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0058671960000001 2023-10-09T05:53:42.461 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.461 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.461 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.461 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.264369+0000", 2023-10-09T05:53:42.462 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1284999999999993e-05 2023-10-09T05:53:42.462 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.462 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.462 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.462 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.264430+0000", 2023-10-09T05:53:42.462 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1857459349999999 2023-10-09T05:53:42.462 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.462 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.462 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.462 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.462 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86707 8.18 8:1d2a32f3:::benchmark_data_smithi008_432048_object86706:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078737+0000", 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 297.25338660699998, 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.195960744, 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86707 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.463 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.464 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.464 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078737+0000", 2023-10-09T05:53:42.464 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.464 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.464 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.464 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078737+0000", 2023-10-09T05:53:42.464 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:42.464 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078736+0000", 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9480999999999997e-05 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078776+0000", 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9616000000000002e-05 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.465 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078815+0000", 2023-10-09T05:53:42.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2260000000000005e-06 2023-10-09T05:53:42.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.466 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078824+0000", 2023-10-09T05:53:42.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.179694931 2023-10-09T05:53:42.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.466 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258519+0000", 2023-10-09T05:53:42.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3319999999999999e-06 2023-10-09T05:53:42.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.467 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.467 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258524+0000", 2023-10-09T05:53:42.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0062883300000001 2023-10-09T05:53:42.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.467 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.467 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.264812+0000", 2023-10-09T05:53:42.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002849922 2023-10-09T05:53:42.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.468 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.468 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:53:42.468 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267662+0000", 2023-10-09T05:53:42.468 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0069641929999999996 2023-10-09T05:53:42.468 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.468 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.468 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.468 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274626+0000", 2023-10-09T05:53:42.468 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1110000000000002e-05 2023-10-09T05:53:42.468 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.468 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274697+0000", 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.195960744 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86710 8.14 8:28e4dd35:::benchmark_data_smithi008_432048_object86709:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 297.25329499700001, 2023-10-09T05:53:42.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2037845970000001, 2023-10-09T05:53:42.470 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.470 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:53:42.470 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.470 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:42.470 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:42.470 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86710 2023-10-09T05:53:42.470 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.470 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.470 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.470 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.470 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5683000000000001e-05 2023-10-09T05:53:42.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.472 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.472 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.472 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078894+0000", 2023-10-09T05:53:42.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5679999999999996e-06 2023-10-09T05:53:42.472 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.472 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.472 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.472 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078900+0000", 2023-10-09T05:53:42.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0676000000000001e-05 2023-10-09T05:53:42.472 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.472 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078911+0000", 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.215693774 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294605+0000", 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2909999999999998e-06 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.473 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294610+0000", 2023-10-09T05:53:42.474 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97619070799999996 2023-10-09T05:53:42.474 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.474 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.474 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.270801+0000", 2023-10-09T05:53:42.474 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037587950000000001 2023-10-09T05:53:42.474 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.474 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:53:42.474 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274559+0000", 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079496329999999994 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282509+0000", 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103909 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282613+0000", 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2037845970000001 2023-10-09T05:53:42.475 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.476 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.476 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.476 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.476 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.476 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86715 8.14 8:2d9c7f0c:::benchmark_data_smithi008_432048_object86714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:53:42.476 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:53:42.476 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 297.25320728700001, 2023-10-09T05:53:42.476 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.203914588, 2023-10-09T05:53:42.476 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.476 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.476 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.477 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:42.477 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:42.477 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86715 2023-10-09T05:53:42.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.477 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.477 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.477 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:53:42.477 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.478 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.478 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.478 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:53:42.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:42.478 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.478 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.478 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.478 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:53:42.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2407999999999997e-05 2023-10-09T05:53:42.478 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.478 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078968+0000", 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4019999999999999e-06 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078974+0000", 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1539000000000001e-05 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.479 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078986+0000", 2023-10-09T05:53:42.480 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.215643644 2023-10-09T05:53:42.480 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.480 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.480 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.480 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294630+0000", 2023-10-09T05:53:42.480 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5470000000000004e-06 2023-10-09T05:53:42.480 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.480 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.480 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.480 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294635+0000", 2023-10-09T05:53:42.480 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.979978393 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274614+0000", 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014625400000000001 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274760+0000", 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017972999999999999 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.481 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274940+0000", 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017525329999999999 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.276692+0000", 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003165429 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.482 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.279857+0000", 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028214580000000002 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282679+0000", 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9833000000000002e-05 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282729+0000", 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010188600000000001 2023-10-09T05:53:42.483 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282831+0000", 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.203914588 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86716 8.8 8:1416e0db:::benchmark_data_smithi008_432048_object86715:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:53:42.484 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 297.25313296100001, 2023-10-09T05:53:42.485 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1828720580000001, 2023-10-09T05:53:42.485 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.485 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.485 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.485 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:42.485 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:42.485 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86716 2023-10-09T05:53:42.485 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.485 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.485 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.485 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:53:42.486 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0300000000000002e-05 2023-10-09T05:53:42.487 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.487 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.487 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.487 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.079050+0000", 2023-10-09T05:53:42.487 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0120000000000002e-06 2023-10-09T05:53:42.487 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.487 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.487 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.487 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.079056+0000", 2023-10-09T05:53:42.487 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.552e-06 2023-10-09T05:53:42.487 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.079060+0000", 2023-10-09T05:53:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17917138099999999 2023-10-09T05:53:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258231+0000", 2023-10-09T05:53:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0326000000000001e-05 2023-10-09T05:53:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258242+0000", 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99156280299999999 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.249804+0000", 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000172201 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.489 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.249977+0000", 2023-10-09T05:53:42.490 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000205291 2023-10-09T05:53:42.490 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.490 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.490 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:53:42.490 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.250182+0000", 2023-10-09T05:53:42.490 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0089602589999999999 2023-10-09T05:53:42.490 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.490 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.490 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.490 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.259142+0000", 2023-10-09T05:53:42.490 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025977299999999999 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261740+0000", 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1816999999999997e-05 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261772+0000", 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1427000000000001e-05 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.491 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261823+0000", 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9397999999999998e-05 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261862+0000", 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1828720580000001 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.492 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.493 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.493 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86719 8.8 8:14442d3e:::benchmark_data_smithi008_432048_object86718:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-10-09T05:53:42.493 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.102694+0000", 2023-10-09T05:53:42.493 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 297.22942934700001, 2023-10-09T05:53:42.493 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650766299999999, 2023-10-09T05:53:42.493 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.493 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.493 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.493 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:42.493 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86719 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102694+0000", 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102694+0000", 2023-10-09T05:53:42.494 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999962 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102690+0000", 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0977000000000002e-05 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102751+0000", 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2374e-05 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.495 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102774+0000", 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2845e-05 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102786+0000", 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15555860099999999 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258345+0000", 2023-10-09T05:53:42.496 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0730000000000005e-06 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258353+0000", 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0014172180000001 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.259770+0000", 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000128658 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.497 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.259899+0000", 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016774100000000001 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.260067+0000", 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039100710000000002 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.263977+0000", 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029711770000000002 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266948+0000", 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00073392599999999998 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267682+0000", 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3998e-05 2023-10-09T05:53:42.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267736+0000", 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4730000000000001e-05 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267771+0000", 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650766299999999 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.500 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.501 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.501 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.501 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:119626 8.13 8:cd715123:::benchmark_data_smithi008_432048_object16684:head [delete] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-10-09T05:53:42.501 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:50:18.137391+0000", 2023-10-09T05:53:42.501 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.19473214600001, 2023-10-09T05:53:42.501 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97353319299999996, 2023-10-09T05:53:42.501 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:53:42.501 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:53:42.501 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:53:42.501 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:53:42.501 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119626 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:18.137391+0000", 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:18.137391+0000", 2023-10-09T05:53:42.502 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999971 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:18.137389+0000", 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1839e-05 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:18.137400+0000", 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2561e-05 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.503 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:18.137423+0000", 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2909999999999996e-06 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:18.137432+0000", 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1499000000000001e-05 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.504 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:18.137504+0000", 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0787e-05 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:18.137515+0000", 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.959876536 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:19.097391+0000", 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00067621499999999998 2023-10-09T05:53:42.505 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.506 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.506 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:53:42.506 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:19.098067+0000", 2023-10-09T05:53:42.506 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0064741529999999999 2023-10-09T05:53:42.506 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.506 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.506 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:53:42.506 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:19.104541+0000", 2023-10-09T05:53:42.506 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6386999999999997e-05 2023-10-09T05:53:42.506 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.506 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:19.104638+0000", 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013967 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:19.104778+0000", 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021606469999999999 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:19.106938+0000", 2023-10-09T05:53:42.507 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.532e-05 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:19.106983+0000", 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002967552 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:19.109951+0000", 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6063999999999999e-05 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:19.109987+0000", 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00093729600000000001 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:50:19.110924+0000", 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97353319299999996 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.509 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:53:42.510 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:53:42.510 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:53:42.510 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:42.582 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:42.832 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 26 16 42141 42125 101.241 85.3125 0.00350391 0.00987267 2023-10-09T05:53:43.381 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:43.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:43.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:43.421 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:53:43.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:53:43.487 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:43.554 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:43.660 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:43.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:43.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:43.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:43.833 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 27 16 43123 43107 99.7638 61.375 0.00440214 0.0100178 2023-10-09T05:53:43.838 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:44.002 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:44.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:44.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:44.110 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:44.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[184418]: pgmap v1128: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:53:44.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:44.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:44.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:53:44.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]: dispatch 2023-10-09T05:53:44.162 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T05:53:44.186 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:44.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:44 smithi191 ceph-mon[161669]: pgmap v1128: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:53:44.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:44.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:44.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:53:44.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]: dispatch 2023-10-09T05:53:44.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T05:53:44.372 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:44.451 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:44.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:44.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:44.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[191227]: pgmap v1128: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:53:44.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:44.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:44.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:53:44.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]: dispatch 2023-10-09T05:53:44.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T05:53:44.564 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:44.637 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:44.834 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 28 16 44497 44481 99.2638 85.875 0.00786484 0.0100682 2023-10-09T05:53:44.900 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:44.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:44.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:45.007 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:45.046 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:45.069 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:45.250 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:45.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:45.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:45.361 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:45.430 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:45.525 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:45.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:45.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:45.637 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:45.665 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:45.708 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:45.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:45.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:45.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:45.834 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 29 16 45655 45639 98.3366 72.375 0.00881799 0.0101647 2023-10-09T05:53:45.859 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:45.939 INFO:teuthology.orchestra.run.smithi008.stderr:marked out osd.7. 2023-10-09T05:53:45.941 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:53:46.195 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:45 smithi191 ceph-mon[161669]: pgmap v1129: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:53:46.195 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:45 smithi191 ceph-mon[161669]: failed to encode map e371 with expected crc 2023-10-09T05:53:46.195 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:45 smithi191 ceph-mon[161669]: failed to encode map e371 with expected crc 2023-10-09T05:53:46.195 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:45 smithi191 ceph-mon[161669]: failed to encode map e371 with expected crc 2023-10-09T05:53:46.195 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:53:46.196 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]': finished 2023-10-09T05:53:46.196 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T05:53:46.196 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:45 smithi191 ceph-mon[161669]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T05:53:46.196 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3478119297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:46.196 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1204988026' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:53:46.196 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:45 smithi191 ceph-mon[161669]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:53:46.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 conmon[184394]: 2023-10-09T05:53:45.884+0000 7f8a66521700 -1 mon.a@0(leader).osd e371 definitely_dead 0 2023-10-09T05:53:46.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[184418]: pgmap v1129: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:53:46.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[184418]: failed to encode map e371 with expected crc 2023-10-09T05:53:46.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[184418]: failed to encode map e371 with expected crc 2023-10-09T05:53:46.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[184418]: failed to encode map e371 with expected crc 2023-10-09T05:53:46.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:53:46.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]': finished 2023-10-09T05:53:46.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T05:53:46.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[184418]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T05:53:46.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3478119297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:46.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1204988026' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:53:46.202 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[184418]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:53:46.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[191227]: pgmap v1129: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:53:46.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[191227]: failed to encode map e371 with expected crc 2023-10-09T05:53:46.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[191227]: failed to encode map e371 with expected crc 2023-10-09T05:53:46.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[191227]: failed to encode map e371 with expected crc 2023-10-09T05:53:46.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:53:46.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]': finished 2023-10-09T05:53:46.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T05:53:46.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[191227]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T05:53:46.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3478119297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:46.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1204988026' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:53:46.203 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:45 smithi008 ceph-mon[191227]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:53:46.258 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:46.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:46.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:46.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:46.395 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:46.492 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:46.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:46.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:46.595 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:46.624 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:46.672 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:46.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:46.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:46.710 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:53:46.710 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:53:46.711 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T05:53:46.711 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:53:46.776 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:46.803 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:46.834 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 30 16 45782 45766 95.3235 7.9375 0.00218989 0.0101892 2023-10-09T05:53:46.897 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:46.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:46.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:46.951 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:46.951 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: failed to encode map e372 with expected crc 2023-10-09T05:53:46.951 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: failed to encode map e372 with expected crc 2023-10-09T05:53:46.951 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: failed to encode map e372 with expected crc 2023-10-09T05:53:46.952 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: failed to encode map e372 with expected crc 2023-10-09T05:53:46.952 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: failed to encode map e372 with expected crc 2023-10-09T05:53:46.952 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3478119297' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:46.952 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.001 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:47.034 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:47.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1204988026' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:53:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: osdmap e372: 8 total, 8 up, 7 in 2023-10-09T05:53:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[184418]: osdmap e373: 8 total, 8 up, 7 in 2023-10-09T05:53:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:47.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3478119297' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:47.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1204988026' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:53:47.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: osdmap e372: 8 total, 8 up, 7 in 2023-10-09T05:53:47.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:46 smithi008 ceph-mon[191227]: osdmap e373: 8 total, 8 up, 7 in 2023-10-09T05:53:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3478119297' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:47.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1204988026' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:53:47.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: osdmap e372: 8 total, 8 up, 7 in 2023-10-09T05:53:47.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: failed to encode map e372 with expected crc 2023-10-09T05:53:47.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:46 smithi191 ceph-mon[161669]: osdmap e373: 8 total, 8 up, 7 in 2023-10-09T05:53:47.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:53:47.834 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 31 16 45782 45766 92.2488 0 - 0.0101892 2023-10-09T05:53:47.916 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:47.937 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:47.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:47.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:48.045 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:48.160 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:48.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[184418]: pgmap v1132: 89 pgs: 2 peering, 87 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:53:48.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[184418]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[184418]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[184418]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[184418]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[184418]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[184418]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[184418]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[184418]: osdmap e374: 8 total, 8 up, 7 in 2023-10-09T05:53:48.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[191227]: pgmap v1132: 89 pgs: 2 peering, 87 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:53:48.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[191227]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[191227]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[191227]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[191227]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[191227]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[191227]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[191227]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:47 smithi008 ceph-mon[191227]: osdmap e374: 8 total, 8 up, 7 in 2023-10-09T05:53:48.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:47 smithi191 ceph-mon[161669]: pgmap v1132: 89 pgs: 2 peering, 87 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:53:48.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:47 smithi191 ceph-mon[161669]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:47 smithi191 ceph-mon[161669]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:47 smithi191 ceph-mon[161669]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:47 smithi191 ceph-mon[161669]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:47 smithi191 ceph-mon[161669]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:47 smithi191 ceph-mon[161669]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:47 smithi191 ceph-mon[161669]: failed to encode map e373 with expected crc 2023-10-09T05:53:48.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:47 smithi191 ceph-mon[161669]: osdmap e374: 8 total, 8 up, 7 in 2023-10-09T05:53:48.699 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:48.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:48.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:48.809 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:48.834 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 32 16 46309 46293 90.3955 16.4688 0.00861681 0.0110548 2023-10-09T05:53:48.884 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:49.086 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:49.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:49.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:49.206 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:49.249 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:49.331 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[184418]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.331 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[184418]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.331 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[184418]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.331 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[184418]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.331 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[184418]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.332 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[184418]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.332 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[191227]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.332 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[191227]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.332 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[191227]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.332 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[191227]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.332 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[191227]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.332 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:49 smithi008 ceph-mon[191227]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.388 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:49.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:49.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:49.459 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:49 smithi191 ceph-mon[161669]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.459 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:49 smithi191 ceph-mon[161669]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.459 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:49 smithi191 ceph-mon[161669]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.459 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:49 smithi191 ceph-mon[161669]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.459 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:49 smithi191 ceph-mon[161669]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.459 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:49 smithi191 ceph-mon[161669]: failed to encode map e374 with expected crc 2023-10-09T05:53:49.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:49.542 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:49.786 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:53:49.828 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:53:49.834 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 33 16 46923 46907 88.8192 38.375 0.0175023 0.0112508 2023-10-09T05:53:49.862 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:53:49.907 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:53:49.913 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:49.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:49.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:49.950 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:53:49.996 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:53:50.023 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:50.033 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:53:50.062 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:50.068 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:53:50.073 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:53:50.073 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:53:50.074 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:53:50.074 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:53:50.074 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:53:50.074 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:53:50.074 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:53:50.074 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:53:50.215 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:50.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:50.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:50.319 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:50.347 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:50.347 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:50.379 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:50 smithi191 ceph-mon[161669]: pgmap v1135: 89 pgs: 3 remapped+peering, 2 peering, 84 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T05:53:50.379 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:50 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:53:50.379 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:50 smithi191 ceph-mon[161669]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.379 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:50 smithi191 ceph-mon[161669]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.379 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:50 smithi191 ceph-mon[161669]: osdmap e375: 8 total, 8 up, 7 in 2023-10-09T05:53:50.379 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:50 smithi191 ceph-mon[161669]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.379 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:50 smithi191 ceph-mon[161669]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.380 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2668440986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:50.384 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:50.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[191227]: pgmap v1135: 89 pgs: 3 remapped+peering, 2 peering, 84 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T05:53:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:53:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[191227]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[191227]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[191227]: osdmap e375: 8 total, 8 up, 7 in 2023-10-09T05:53:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[191227]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[191227]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2668440986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:50.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[184418]: pgmap v1135: 89 pgs: 3 remapped+peering, 2 peering, 84 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T05:53:50.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:53:50.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[184418]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[184418]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[184418]: osdmap e375: 8 total, 8 up, 7 in 2023-10-09T05:53:50.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[184418]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[184418]: failed to encode map e375 with expected crc 2023-10-09T05:53:50.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2668440986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:50.526 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:50.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:50.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:50.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:50.670 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:50.834 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 34 16 47595 47579 87.4423 42 0.00569108 0.0113767 2023-10-09T05:53:50.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:53:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:53:50] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.33.4" 2023-10-09T05:53:51.048 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:53:51.113 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:51.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:51.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:51.217 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:51.229 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:53:51.259 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:51.307 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:51.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:51.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:51.412 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:51.440 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[184418]: failed to encode map e376 with expected crc 2023-10-09T05:53:51.441 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[184418]: failed to encode map e376 with expected crc 2023-10-09T05:53:51.441 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[184418]: osdmap e376: 8 total, 8 up, 7 in 2023-10-09T05:53:51.441 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:51.441 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[184418]: failed to encode map e376 with expected crc 2023-10-09T05:53:51.441 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:51.441 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[191227]: failed to encode map e376 with expected crc 2023-10-09T05:53:51.442 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[191227]: failed to encode map e376 with expected crc 2023-10-09T05:53:51.442 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[191227]: osdmap e376: 8 total, 8 up, 7 in 2023-10-09T05:53:51.442 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:51.442 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[191227]: failed to encode map e376 with expected crc 2023-10-09T05:53:51.442 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:51 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:51.444 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:51 smithi191 ceph-mon[161669]: failed to encode map e376 with expected crc 2023-10-09T05:53:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:51 smithi191 ceph-mon[161669]: failed to encode map e376 with expected crc 2023-10-09T05:53:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:51 smithi191 ceph-mon[161669]: osdmap e376: 8 total, 8 up, 7 in 2023-10-09T05:53:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:51 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:51 smithi191 ceph-mon[161669]: failed to encode map e376 with expected crc 2023-10-09T05:53:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:51 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:51.641 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:51.844 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 35 16 48098 48082 85.8322 31.4375 0.0177561 0.0116436 2023-10-09T05:53:52.050 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:52.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:52.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:52.171 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:52.247 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:52.305 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:53:52.305 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:53:52.306 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:53:52.306 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.784232 2023-10-09T05:53:52.306 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0.7842315042078791 2023-10-09T05:53:52.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:52 smithi008 ceph-mon[191227]: pgmap v1138: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 3 peering, 78 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 19 MiB/s wr, 303 op/s; 5736/137592 objects degraded (4.169%) 2023-10-09T05:53:52.377 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:52 smithi008 ceph-mon[191227]: Health check failed: Degraded data redundancy: 5736/137592 objects degraded (4.169%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:53:52.377 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:52 smithi008 ceph-mon[184418]: pgmap v1138: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 3 peering, 78 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 19 MiB/s wr, 303 op/s; 5736/137592 objects degraded (4.169%) 2023-10-09T05:53:52.377 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:52 smithi008 ceph-mon[184418]: Health check failed: Degraded data redundancy: 5736/137592 objects degraded (4.169%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:53:52.433 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:52.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:52.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:52 smithi191 ceph-mon[161669]: pgmap v1138: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 3 peering, 78 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 19 MiB/s wr, 303 op/s; 5736/137592 objects degraded (4.169%) 2023-10-09T05:53:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:52 smithi191 ceph-mon[161669]: Health check failed: Degraded data redundancy: 5736/137592 objects degraded (4.169%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:53:52.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:52.586 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:52.706 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:52.790 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:52.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:52.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:52.844 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 36 16 48855 48839 84.7496 47.3125 0.00735379 0.0117933 2023-10-09T05:53:52.898 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:53.003 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:53.268 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:53.268 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:53.396 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:53.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:53.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:53.502 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:53.542 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:53.600 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:53.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:53.704 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:53.792 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:53.844 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 37 16 49627 49611 83.7632 48.25 0.014621 0.0119304 2023-10-09T05:53:54.032 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:53:54.116 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:54 smithi008 ceph-mon[184418]: pgmap v1139: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 peering, 76 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 33 MiB/s wr, 524 op/s; 20089/142788 objects degraded (14.069%); 5340/142788 objects misplaced (3.740%); 4.0 MiB/s, 1 keys/s, 64 objects/s recovering 2023-10-09T05:53:54.116 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:54 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:53:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:54 smithi191 ceph-mon[161669]: pgmap v1139: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 peering, 76 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 33 MiB/s wr, 524 op/s; 20089/142788 objects degraded (14.069%); 5340/142788 objects misplaced (3.740%); 4.0 MiB/s, 1 keys/s, 64 objects/s recovering 2023-10-09T05:53:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:54 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:53:54.415 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:53:54.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:54 smithi008 ceph-mon[191227]: pgmap v1139: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 peering, 76 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 33 MiB/s wr, 524 op/s; 20089/142788 objects degraded (14.069%); 5340/142788 objects misplaced (3.740%); 4.0 MiB/s, 1 keys/s, 64 objects/s recovering 2023-10-09T05:53:54.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:54 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:53:54.535 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:54.844 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 38 16 50319 50303 82.6974 43.25 0.0303443 0.0120869 2023-10-09T05:53:55.562 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:55.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:55.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:55.673 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:55.678 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:55.783 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:55.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 39 16 50991 50975 81.6541 42 0.0192058 0.0122384 2023-10-09T05:53:56.068 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:56.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:56.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:56.174 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:56.179 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:56 smithi191 ceph-mon[161669]: pgmap v1140: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 peering, 76 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 23 MiB/s wr, 360 op/s; 20089/142788 objects degraded (14.069%); 5340/142788 objects misplaced (3.740%); 4.0 MiB/s, 1 keys/s, 56 objects/s recovering 2023-10-09T05:53:56.220 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:56.289 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:56.389 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:56.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:56.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:56.396 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:56 smithi008 ceph-mon[191227]: pgmap v1140: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 peering, 76 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 23 MiB/s wr, 360 op/s; 20089/142788 objects degraded (14.069%); 5340/142788 objects misplaced (3.740%); 4.0 MiB/s, 1 keys/s, 56 objects/s recovering 2023-10-09T05:53:56.400 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:56 smithi008 ceph-mon[184418]: pgmap v1140: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 peering, 76 active+clean; 2.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 23 MiB/s wr, 360 op/s; 20089/142788 objects degraded (14.069%); 5340/142788 objects misplaced (3.740%); 4.0 MiB/s, 1 keys/s, 56 objects/s recovering 2023-10-09T05:53:56.468 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:56.468 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:56.494 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:56.523 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:56.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:53:56.845813+0000 min lat: 0.00115306 max lat: 2.32752 avg lat: 0.0123803 2023-10-09T05:53:56.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:53:56.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 40 16 51704 51688 80.7266 44.5625 0.00909049 0.0123803 2023-10-09T05:53:56.898 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:56.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:56.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:57.002 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:57.128 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:57.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:53:57.732 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:57.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 41 16 52395 52379 79.8112 43.1875 0.0254602 0.0125217 2023-10-09T05:53:58.033 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:58.042 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:57 smithi008 ceph-mon[184418]: pgmap v1141: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 614 op/s; 20089/150402 objects degraded (13.357%); 5340/150402 objects misplaced (3.550%); 3.3 MiB/s, 1 keys/s, 46 objects/s recovering 2023-10-09T05:53:58.042 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:57 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:58.043 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:57 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:58.043 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:57 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20089/150402 objects degraded (13.357%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:53:58.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:57 smithi008 ceph-mon[191227]: pgmap v1141: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 614 op/s; 20089/150402 objects degraded (13.357%); 5340/150402 objects misplaced (3.550%); 3.3 MiB/s, 1 keys/s, 46 objects/s recovering 2023-10-09T05:53:58.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:57 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:58.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:57 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:58.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:57 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20089/150402 objects degraded (13.357%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:53:58.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:58.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:57 smithi191 ceph-mon[161669]: pgmap v1141: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 614 op/s; 20089/150402 objects degraded (13.357%); 5340/150402 objects misplaced (3.550%); 3.3 MiB/s, 1 keys/s, 46 objects/s recovering 2023-10-09T05:53:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:57 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:57 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:53:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:57 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20089/150402 objects degraded (13.357%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:53:58.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:58.186 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:58.365 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:58.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:58.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:58.470 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:58.578 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:58.796 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:53:58.828 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1821567819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:58.828 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:58 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:58.828 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1821567819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:58.828 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:58 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:58.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 42 16 53115 53099 78.9826 45 0.00800301 0.0126206 2023-10-09T05:53:59.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1821567819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:59.053 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:58 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:59.566 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:59.566 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:59.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 43 16 53827 53811 78.1809 44.5 0.0105091 0.012783 2023-10-09T05:54:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:59 smithi191 ceph-mon[161669]: pgmap v1142: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 615 op/s; 20089/153012 objects degraded (13.129%); 4841/153012 objects misplaced (3.164%); 6.1 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T05:54:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:59 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:59 smithi191 ceph-mon[161669]: failed to encode map e377 with expected crc 2023-10-09T05:54:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:59 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:59 smithi191 ceph-mon[161669]: osdmap e377: 8 total, 8 up, 7 in 2023-10-09T05:54:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:53:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:00.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[191227]: pgmap v1142: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 615 op/s; 20089/153012 objects degraded (13.129%); 4841/153012 objects misplaced (3.164%); 6.1 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T05:54:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[191227]: failed to encode map e377 with expected crc 2023-10-09T05:54:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[191227]: osdmap e377: 8 total, 8 up, 7 in 2023-10-09T05:54:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[184418]: pgmap v1142: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 615 op/s; 20089/153012 objects degraded (13.129%); 4841/153012 objects misplaced (3.164%); 6.1 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T05:54:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:00.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[184418]: failed to encode map e377 with expected crc 2023-10-09T05:54:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[184418]: osdmap e377: 8 total, 8 up, 7 in 2023-10-09T05:54:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:00.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:53:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:00.334 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:54:00.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:00 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:00.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:00 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:00.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:00 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:00.804 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:54:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:54:00] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.33.4" 2023-10-09T05:54:00.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 44 16 54634 54618 77.5504 50.4375 0.01345 0.012889 2023-10-09T05:54:00.853 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:00.869 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:00.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:00.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:00.983 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:01.018 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:01.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:00 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:01.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:00 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:01.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:00 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:01.149 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:01.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:01.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:01.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:00 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:00 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:00 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:01.254 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:01.282 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:01.816 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:01.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:01.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:01.845 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 45 16 55233 55217 76.6592 37.4375 0.0114446 0.0130373 2023-10-09T05:54:01.922 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:02.040 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:02.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:02 smithi008 ceph-mon[184418]: pgmap v1144: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 603 op/s; 20089/155709 objects degraded (12.902%); 4841/155709 objects misplaced (3.109%); 5.7 MiB/s, 0 keys/s, 87 objects/s recovering 2023-10-09T05:54:02.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:02 smithi008 ceph-mon[191227]: pgmap v1144: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 603 op/s; 20089/155709 objects degraded (12.902%); 4841/155709 objects misplaced (3.109%); 5.7 MiB/s, 0 keys/s, 87 objects/s recovering 2023-10-09T05:54:02.249 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:02.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:02.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:02.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:02 smithi191 ceph-mon[161669]: pgmap v1144: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 38 MiB/s wr, 603 op/s; 20089/155709 objects degraded (12.902%); 4841/155709 objects misplaced (3.109%); 5.7 MiB/s, 0 keys/s, 87 objects/s recovering 2023-10-09T05:54:02.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:02.409 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:02.589 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:02.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:02.655 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:54:02.655 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:54:02.694 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:02.767 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:02.851 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 46 16 55959 55943 75.9696 45.375 0.016765 0.013152 2023-10-09T05:54:02.938 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:54:02.940 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:02.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:02.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:03.002 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:54:03.025 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:03 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1667016642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:03.049 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:03.063 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:54:03.088 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:03.126 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:54:03.151 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:03.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:03.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:03.187 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:54:03.232 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:54:03.254 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:03.282 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:54:03.284 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:03 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1667016642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:03.290 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:03.310 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:54:03.324 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:54:03.325 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:54:03.325 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:54:03.325 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:54:03.325 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:54:03.325 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:54:03.325 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:54:03.325 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:54:03.364 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:03 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1667016642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:03.435 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:03.435 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:54:03.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:03.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:03.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:03.573 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:03.629 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:03.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:03.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:03.736 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:03.855 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:03.856 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 47 16 56692 56676 75.3267 45.8125 0.0338277 0.0132674 2023-10-09T05:54:03.997 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:54:04.196 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.4 primary-affinity to 0.784232 (8513952) 2023-10-09T05:54:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:04 smithi008 ceph-mon[191227]: pgmap v1145: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 44 MiB/s wr, 698 op/s; 20089/163740 objects degraded (12.269%); 4301/163740 objects misplaced (2.627%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:54:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:04 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20089/163740 objects degraded (12.269%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3608665954' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7842315042078791}]: dispatch 2023-10-09T05:54:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:04 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7842315042078791}]: dispatch 2023-10-09T05:54:04.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:04 smithi008 ceph-mon[184418]: pgmap v1145: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 44 MiB/s wr, 698 op/s; 20089/163740 objects degraded (12.269%); 4301/163740 objects misplaced (2.627%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:54:04.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:04 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20089/163740 objects degraded (12.269%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:04.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3608665954' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7842315042078791}]: dispatch 2023-10-09T05:54:04.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:04 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7842315042078791}]: dispatch 2023-10-09T05:54:04.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:04 smithi191 ceph-mon[161669]: pgmap v1145: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 44 MiB/s wr, 698 op/s; 20089/163740 objects degraded (12.269%); 4301/163740 objects misplaced (2.627%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:54:04.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:04 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20089/163740 objects degraded (12.269%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:04.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3608665954' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7842315042078791}]: dispatch 2023-10-09T05:54:04.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:04 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7842315042078791}]: dispatch 2023-10-09T05:54:04.415 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:04.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:04.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:04.520 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:04.568 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:04.852 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 48 16 57140 57124 74.3411 28 0.00515572 0.0133055 2023-10-09T05:54:05.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:05 smithi008 ceph-mon[184418]: failed to encode map e378 with expected crc 2023-10-09T05:54:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:05 smithi008 ceph-mon[184418]: failed to encode map e378 with expected crc 2023-10-09T05:54:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:05 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7842315042078791}]': finished 2023-10-09T05:54:05.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:05 smithi008 ceph-mon[184418]: osdmap e378: 8 total, 8 up, 7 in 2023-10-09T05:54:05.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:05 smithi008 ceph-mon[191227]: failed to encode map e378 with expected crc 2023-10-09T05:54:05.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:05 smithi008 ceph-mon[191227]: failed to encode map e378 with expected crc 2023-10-09T05:54:05.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:05 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7842315042078791}]': finished 2023-10-09T05:54:05.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:05 smithi008 ceph-mon[191227]: osdmap e378: 8 total, 8 up, 7 in 2023-10-09T05:54:05.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:05 smithi191 ceph-mon[161669]: failed to encode map e378 with expected crc 2023-10-09T05:54:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:05 smithi191 ceph-mon[161669]: failed to encode map e378 with expected crc 2023-10-09T05:54:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:05 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7842315042078791}]': finished 2023-10-09T05:54:05.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:05 smithi191 ceph-mon[161669]: osdmap e378: 8 total, 8 up, 7 in 2023-10-09T05:54:05.855 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 49 16 57711 57695 73.5525 35.6875 0.0084075 0.0135891 2023-10-09T05:54:06.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[191227]: pgmap v1147: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 555 op/s; 20089/163740 objects degraded (12.269%); 4301/163740 objects misplaced (2.627%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T05:54:06.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[191227]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[191227]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[191227]: osdmap e379: 8 total, 8 up, 7 in 2023-10-09T05:54:06.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[191227]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[191227]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[191227]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[191227]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[184418]: pgmap v1147: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 555 op/s; 20089/163740 objects degraded (12.269%); 4301/163740 objects misplaced (2.627%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T05:54:06.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[184418]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[184418]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[184418]: osdmap e379: 8 total, 8 up, 7 in 2023-10-09T05:54:06.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[184418]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[184418]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[184418]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:06 smithi008 ceph-mon[184418]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:06 smithi191 ceph-mon[161669]: pgmap v1147: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 555 op/s; 20089/163740 objects degraded (12.269%); 4301/163740 objects misplaced (2.627%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T05:54:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:06 smithi191 ceph-mon[161669]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:06 smithi191 ceph-mon[161669]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:06 smithi191 ceph-mon[161669]: osdmap e379: 8 total, 8 up, 7 in 2023-10-09T05:54:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:06 smithi191 ceph-mon[161669]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:06 smithi191 ceph-mon[161669]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:06 smithi191 ceph-mon[161669]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:06 smithi191 ceph-mon[161669]: failed to encode map e379 with expected crc 2023-10-09T05:54:06.664 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:06.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:06.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:06.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:06.800 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:06.847 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:06.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:06.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:06.854 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 50 16 58345 58329 72.8717 39.625 0.0151456 0.0137167 2023-10-09T05:54:06.953 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:06.986 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:07.054 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:07.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:07.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:07.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:07.195 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:07.254 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:07.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:07.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:07.361 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:07.391 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:07.426 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:07 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:07.429 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:07 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:07.435 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:07.456 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:07.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:07 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:07.593 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:07.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:07.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:07.705 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:07.787 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:07.855 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 51 16 59105 59089 72.3737 47.5 0.0170589 0.013812 2023-10-09T05:54:07.934 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:07.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:07.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:08.039 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:08.082 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:08.138 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:08.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:08.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:08.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:08.280 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:08.351 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:08.353 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:08 smithi008 ceph-mon[184418]: pgmap v1149: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 46 MiB/s wr, 740 op/s; 20089/170253 objects degraded (11.799%); 4301/170253 objects misplaced (2.526%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T05:54:08.353 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20089/170253 objects degraded (11.799%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:08.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:08 smithi008 ceph-mon[191227]: pgmap v1149: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 46 MiB/s wr, 740 op/s; 20089/170253 objects degraded (11.799%); 4301/170253 objects misplaced (2.526%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T05:54:08.353 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20089/170253 objects degraded (11.799%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:08.356 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:08 smithi191 ceph-mon[161669]: pgmap v1149: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 46 MiB/s wr, 740 op/s; 20089/170253 objects degraded (11.799%); 4301/170253 objects misplaced (2.526%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T05:54:08.356 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20089/170253 objects degraded (11.799%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:08.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:08.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:08.458 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:08.553 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:08.855 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 52 16 59650 59634 71.6372 34.0625 0.00988654 0.0139527 2023-10-09T05:54:09.113 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:54:09.620 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:09.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:09.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:09.730 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:09.810 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:09.855 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 53 16 60440 60424 71.2173 49.375 0.0180364 0.0140347 2023-10-09T05:54:10.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:10 smithi191 ceph-mon[161669]: pgmap v1150: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 46 MiB/s wr, 736 op/s; 20089/173382 objects degraded (11.587%); 3577/173382 objects misplaced (2.063%); 9.9 MiB/s, 157 objects/s recovering 2023-10-09T05:54:10.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:10 smithi008 ceph-mon[191227]: pgmap v1150: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 46 MiB/s wr, 736 op/s; 20089/173382 objects degraded (11.587%); 3577/173382 objects misplaced (2.063%); 9.9 MiB/s, 157 objects/s recovering 2023-10-09T05:54:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:10 smithi008 ceph-mon[184418]: pgmap v1150: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 46 MiB/s wr, 736 op/s; 20089/173382 objects degraded (11.587%); 3577/173382 objects misplaced (2.063%); 9.9 MiB/s, 157 objects/s recovering 2023-10-09T05:54:10.716 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:54:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:54:10] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.33.4" 2023-10-09T05:54:10.855 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 54 16 61185 61169 70.7608 46.5625 0.0160862 0.0141256 2023-10-09T05:54:10.917 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:10.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:10.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:11.035 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:11.061 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:11.112 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:11.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:11.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:11.221 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:11.254 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:11.304 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:11.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:11.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:11.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:11.444 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:11.501 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:11.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:11.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:11.608 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:11.642 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:11 smithi008 ceph-mon[184418]: pgmap v1151: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 32 MiB/s wr, 504 op/s; 20089/175857 objects degraded (11.423%); 3577/175857 objects misplaced (2.034%); 5.7 MiB/s, 89 objects/s recovering 2023-10-09T05:54:11.642 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:11 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:54:11.646 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:11 smithi008 ceph-mon[191227]: pgmap v1151: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 32 MiB/s wr, 504 op/s; 20089/175857 objects degraded (11.423%); 3577/175857 objects misplaced (2.034%); 5.7 MiB/s, 89 objects/s recovering 2023-10-09T05:54:11.646 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:11 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:11 smithi191 ceph-mon[161669]: pgmap v1151: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 32 MiB/s wr, 504 op/s; 20089/175857 objects degraded (11.423%); 3577/175857 objects misplaced (2.034%); 5.7 MiB/s, 89 objects/s recovering 2023-10-09T05:54:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:11 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:54:11.862 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 55 16 61863 61847 70.2427 42.375 0.0176409 0.01423 2023-10-09T05:54:12.003 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:12.063 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:12.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:12.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:12.162 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:54:12.162 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:54:12.162 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-10-09T05:54:12.162 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:54:12.185 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:54:12.185 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:54:12.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.185 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:62034 9.9 9:93a9f225:::benchmark_data_smithi008_502820_object62033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e379)", 2023-10-09T05:54:12.185 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:12.139347+0000", 2023-10-09T05:54:12.185 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.026589727000000001, 2023-10-09T05:54:12.185 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.02663014, 2023-10-09T05:54:12.185 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 62034 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.139347+0000", 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.186 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.139347+0000", 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.139345+0000", 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:12.187 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.139372+0000", 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6472e-05 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.139384+0000", 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1967000000000001e-05 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.139392+0000", 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9359999999999999e-06 2023-10-09T05:54:12.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.139453+0000", 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1484999999999998e-05 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.139650+0000", 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00019668499999999999 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:54:12.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.139845+0000", 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00019455199999999999 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.142523+0000", 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002678555 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.147342+0000", 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048189089999999997 2023-10-09T05:54:12.190 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.36258.0:62043 9.e e379/373 9:778c5b7b:::benchmark_data_smithi008_502820_object62042:head v 379'3844, mlcod=0'0)", 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:12.152714+0000", 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.013222065999999999, 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013361143000000001, 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.152714+0000", 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.152714+0000", 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.152714+0000", 2023-10-09T05:54:12.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.152818+0000", 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000104509 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.152836+0000", 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7436e-05 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.193 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.152843+0000", 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.588e-06 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.153230+0000", 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00038634800000000001 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.153260+0000", 2023-10-09T05:54:12.194 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0213999999999999e-05 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.36258.0:62041 9.c e379/373 9:3bb62845:::benchmark_data_smithi008_502820_object62040:head v 379'3886, mlcod=0'0)", 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:12.159306+0000", 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.006630726, 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068340240000000002, 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:54:12.195 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159306+0000", 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159306+0000", 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:12.196 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159305+0000", 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159354+0000", 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9814000000000001e-05 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159368+0000", 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3438000000000001e-05 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.197 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159377+0000", 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8990000000000007e-06 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159423+0000", 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6208000000000002e-05 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:12.198 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159446+0000", 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3033000000000001e-05 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:62044 9.9 9:9c1659b4:::benchmark_data_smithi008_502820_object62043:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e379)", 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:12.159447+0000", 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0064890449999999997, 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0067685430000000001, 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:12.199 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 62044 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159447+0000", 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.200 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159447+0000", 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159446+0000", 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159469+0000", 2023-10-09T05:54:12.201 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2467999999999999e-05 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159479+0000", 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.012e-05 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159486+0000", 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6850000000000004e-06 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.202 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159530+0000", 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4814999999999997e-05 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159706+0000", 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017599900000000001 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159877+0000", 2023-10-09T05:54:12.203 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017008099999999999 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:62045 9.5 9:a75c07ba:::benchmark_data_smithi008_502820_object62044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e379)", 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:12.159499+0000", 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0064373770000000002, 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006798264, 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:54:12.204 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 62045 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159499+0000", 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:12.205 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159499+0000", 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159499+0000", 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159519+0000", 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0149e-05 2023-10-09T05:54:12.206 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159525+0000", 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.421e-06 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159530+0000", 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1900000000000003e-06 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.207 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159597+0000", 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6837999999999999e-05 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159755+0000", 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000157739 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.159936+0000", 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018088699999999999 2023-10-09T05:54:12.208 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.36258.0:62046 9.e e379/373)", 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:12.162347+0000", 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0035892400000000001, 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040208479999999996, 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "queued for pg", 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:12.209 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:12.210 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.162347+0000", 2023-10-09T05:54:12.210 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.210 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:12.210 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.162347+0000", 2023-10-09T05:54:12.210 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:12.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.210 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:12.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.162347+0000", 2023-10-09T05:54:12.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:54:12.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:12.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.162388+0000", 2023-10-09T05:54:12.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1940000000000002e-05 2023-10-09T05:54:12.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.162401+0000", 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2689e-05 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:12.162409+0000", 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9140000000000008e-06 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:12.214 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.215 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:12.215 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:54:12.215 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 6 2023-10-09T05:54:12.215 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:54:12.215 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:12.325 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:12.611 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:12.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:12.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:12.727 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:12.820 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:12.857 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 56 16 62480 62464 69.6772 38.5625 0.00655939 0.0143449 2023-10-09T05:54:13.700 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:54:13.857 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 57 16 63089 63073 69.1227 38.0625 0.0230549 0.0144587 2023-10-09T05:54:13.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:13 smithi008 ceph-mon[184418]: pgmap v1152: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 49 MiB/s wr, 787 op/s; 20089/183606 objects degraded (10.941%); 2911/183606 objects misplaced (1.585%); 10 MiB/s, 164 objects/s recovering 2023-10-09T05:54:13.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1098019330' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:13.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:13 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:13.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20089/183606 objects degraded (10.941%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:13.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:13 smithi008 ceph-mon[191227]: pgmap v1152: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 49 MiB/s wr, 787 op/s; 20089/183606 objects degraded (10.941%); 2911/183606 objects misplaced (1.585%); 10 MiB/s, 164 objects/s recovering 2023-10-09T05:54:13.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1098019330' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:13.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:13 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:13.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20089/183606 objects degraded (10.941%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:14.041 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:14.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:14.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:13 smithi191 ceph-mon[161669]: pgmap v1152: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 49 MiB/s wr, 787 op/s; 20089/183606 objects degraded (10.941%); 2911/183606 objects misplaced (1.585%); 10 MiB/s, 164 objects/s recovering 2023-10-09T05:54:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1098019330' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:13 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:14.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20089/183606 objects degraded (10.941%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:14.150 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:14.233 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:54:14.265 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:14.650 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:14.650 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[191227]: failed to encode map e380 with expected crc 2023-10-09T05:54:14.651 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:54:14.651 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[191227]: osdmap e380: 8 total, 8 up, 7 in 2023-10-09T05:54:14.651 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[191227]: failed to encode map e380 with expected crc 2023-10-09T05:54:14.651 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:14.741 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:14.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:14.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:14.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:14.857 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 58 16 63749 63733 68.6422 41.25 0.0318883 0.0145617 2023-10-09T05:54:14.879 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:14.942 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:14.946 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:14 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:14.946 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:14 smithi191 ceph-mon[161669]: failed to encode map e380 with expected crc 2023-10-09T05:54:14.946 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:14 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:54:14.946 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:14 smithi191 ceph-mon[161669]: osdmap e380: 8 total, 8 up, 7 in 2023-10-09T05:54:14.946 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:14 smithi191 ceph-mon[161669]: failed to encode map e380 with expected crc 2023-10-09T05:54:14.946 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:14.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:14.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:14.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:14.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[184418]: failed to encode map e380 with expected crc 2023-10-09T05:54:14.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:54:14.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[184418]: osdmap e380: 8 total, 8 up, 7 in 2023-10-09T05:54:14.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[184418]: failed to encode map e380 with expected crc 2023-10-09T05:54:14.969 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:15.048 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:15.121 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:15.721 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:15.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:15.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:15.832 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:15.857 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 59 16 64547 64531 68.3242 49.875 0.0171238 0.0146302 2023-10-09T05:54:15.867 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:15.926 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:15.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:15.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:15.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:15 smithi008 ceph-mon[184418]: pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 617 op/s; 20089/183606 objects degraded (10.941%); 2911/183606 objects misplaced (1.585%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T05:54:15.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:15 smithi008 ceph-mon[191227]: pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 617 op/s; 20089/183606 objects degraded (10.941%); 2911/183606 objects misplaced (1.585%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T05:54:16.031 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:16.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:15 smithi191 ceph-mon[161669]: pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 617 op/s; 20089/183606 objects degraded (10.941%); 2911/183606 objects misplaced (1.585%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T05:54:16.058 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:16.103 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:16.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:16.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:16.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:16.248 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:16.316 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:16.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:16.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:16.424 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:16.508 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:16.673 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:16.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:16.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:16.782 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:16.792 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:16.812 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:16.863 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:16.864 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:54:16.864717+0000 min lat: 0.00115306 max lat: 2.32752 avg lat: 0.0147385 2023-10-09T05:54:16.864 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:54:16.864 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 60 16 65156 65140 67.8127 38.0625 0.0201826 0.0147385 2023-10-09T05:54:16.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:16.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:16.885 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:16.974 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:17.010 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:17.074 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:17.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:17.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:17.179 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:17.288 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:17.778 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:54:17.779 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:17 smithi008 ceph-mon[184418]: pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 672 op/s; 20089/190467 objects degraded (10.547%); 2911/190467 objects misplaced (1.528%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T05:54:17.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:17.782 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:17.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:17.864 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 61 16 65871 65855 67.4337 44.6875 0.0205085 0.0148224 2023-10-09T05:54:18.270 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:18.271 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:17 smithi191 ceph-mon[161669]: pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 672 op/s; 20089/190467 objects degraded (10.547%); 2911/190467 objects misplaced (1.528%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T05:54:18.271 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:54:18.271 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:54:18.272 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:17 smithi008 ceph-mon[191227]: pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 672 op/s; 20089/190467 objects degraded (10.547%); 2911/190467 objects misplaced (1.528%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T05:54:18.273 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:54:18.273 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:54:18.273 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:54:18.273 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:54:18.273 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:54:18.273 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:54:18.273 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:54:18.273 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:54:18.274 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:54:18.274 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:54:18.274 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:54:18.274 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:54:18.274 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:54:18.274 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:54:18.330 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:18.695 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3709864151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:18.869 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20089/190467 objects degraded (10.547%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:18.871 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 62 16 66578 66562 67.0589 44.1875 0.0207675 0.0149069 2023-10-09T05:54:18.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3709864151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:18.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20089/190467 objects degraded (10.547%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3709864151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20089/190467 objects degraded (10.547%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:19.864 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 63 16 67191 67175 66.6028 38.3125 0.0137537 0.015008 2023-10-09T05:54:20.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:20 smithi008 ceph-mon[184418]: pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 677 op/s; 20089/193755 objects degraded (10.368%); 2207/193755 objects misplaced (1.139%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:54:20.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:20 smithi008 ceph-mon[191227]: pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 677 op/s; 20089/193755 objects degraded (10.368%); 2207/193755 objects misplaced (1.139%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:54:20.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:20 smithi191 ceph-mon[161669]: pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 677 op/s; 20089/193755 objects degraded (10.368%); 2207/193755 objects misplaced (1.139%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:54:20.712 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:20.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:20.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:20.827 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:20.853 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:54:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:54:20] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.33.4" 2023-10-09T05:54:20.857 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:20.864 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 64 16 67908 67892 66.2624 44.8125 0.0081314 0.0150676 2023-10-09T05:54:21.323 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:21.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:21.432 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:21.520 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:21.644 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:21.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:21.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:21.710 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:21.760 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:21.792 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:21.852 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:21.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:21.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:21.864 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 65 16 68604 68588 65.9123 43.5 0.0197147 0.0151649 2023-10-09T05:54:21.957 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:22.049 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:22.470 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:22 smithi008 ceph-mon[184418]: pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 681 op/s; 20089/196326 objects degraded (10.232%); 2207/196326 objects misplaced (1.124%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:54:22.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:22 smithi008 ceph-mon[191227]: pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 681 op/s; 20089/196326 objects degraded (10.232%); 2207/196326 objects misplaced (1.124%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:54:22.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:22 smithi191 ceph-mon[161669]: pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 681 op/s; 20089/196326 objects degraded (10.232%); 2207/196326 objects misplaced (1.124%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T05:54:22.569 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:22.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:22.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:22.624 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:54:22.624 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:54:22.677 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:22.711 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:22.763 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:22.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:22.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:22.865 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 66 16 69313 69297 65.5849 44.3125 0.0351405 0.0152407 2023-10-09T05:54:22.871 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:22.906 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:22.962 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:22.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:22.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:23.015 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:54:23.040 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:54:23.069 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:23.147 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20089/196326 objects degraded (10.232%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:23.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20089/196326 objects degraded (10.232%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:23.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20089/196326 objects degraded (10.232%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:23.492 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:54:23.865 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 67 16 70072 70056 65.3141 47.4375 0.00679573 0.0153032 2023-10-09T05:54:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:24 smithi008 ceph-mon[184418]: pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s; 20089/204165 objects degraded (9.840%); 1361/204165 objects misplaced (0.667%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T05:54:24.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:24 smithi008 ceph-mon[184418]: from='client.56146 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:54:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:24 smithi008 ceph-mon[191227]: pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s; 20089/204165 objects degraded (9.840%); 1361/204165 objects misplaced (0.667%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T05:54:24.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:24 smithi008 ceph-mon[191227]: from='client.56146 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:54:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:24 smithi191 ceph-mon[161669]: pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s; 20089/204165 objects degraded (9.840%); 1361/204165 objects misplaced (0.667%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T05:54:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:24 smithi191 ceph-mon[161669]: from='client.56146 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:54:24.867 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 68 16 70593 70577 64.8304 32.5625 0.0224097 0.015419 2023-10-09T05:54:25.486 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:25.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:25.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:25.604 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:25.641 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:25.709 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:25.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:25.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:25.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:25.844 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:25.869 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 69 16 71295 71279 64.5253 43.875 0.150299 0.0154806 2023-10-09T05:54:25.910 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:25.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:25.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:26.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:26.058 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:26.132 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:26.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:26.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:26.235 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:26 smithi191 ceph-mon[161669]: pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 628 op/s; 20089/204165 objects degraded (9.840%); 1361/204165 objects misplaced (0.667%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T05:54:26.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:26.270 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:26.307 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:54:26.325 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:26.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:26.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:26.431 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:26.454 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1158,"stamp":"2023-10-09T05:54:22.565920+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4463007667,"num_objects":68055,"num_object_clones":0,"num_object_copies":204165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20089,"num_objects_misplaced":1361,"num_objects_unfound":0,"num_objects_dirty":68055,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":69133,"num_write_kb":4368869,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4521,"num_bytes_recovered":289620364,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":69197,"ondisk_log_size":69197,"up":267,"acting":255,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":275,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17675796,"kb_used_data":11874652,"kb_used_omap":31,"kb_used_meta":5801056,"kb_avail":638540268,"statfs":{"total":671965249536,"available":653865234432,"internally_reserved":0,"allocated":12159643648,"data_stored":12151951413,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32311,"internal_metadata":5940281801},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":449118208,"num_objects":6853,"num_object_clones":0,"num_object_copies":20559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1550,"num_objects_unfound":0,"num_objects_dirty":6853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6853,"num_write_kb":438592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1550,"num_bytes_recovered":101580800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.011511"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":380,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.162481+0000","last_change":"2023-10-09T05:48:54.494513+0000","last_active":"2023-10-09T05:54:05.162481+0000","last_peered":"2023-10-09T05:54:05.162481+0000","last_clean":"2023-10-09T05:54:05.162481+0000","last_became_active":"2023-10-09T05:48:54.493510+0000","last_became_peered":"2023-10-09T05:48:54.493510+0000","last_unstale":"2023-10-09T05:54:05.162481+0000","last_undegraded":"2023-10-09T05:54:05.162481+0000","last_fullsized":"2023-10-09T05:54:05.162481+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":310,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160925+0000","last_change":"2023-10-09T05:50:18.047163+0000","last_active":"2023-10-09T05:54:05.160925+0000","last_peered":"2023-10-09T05:54:05.160925+0000","last_clean":"2023-10-09T05:54:05.160925+0000","last_became_active":"2023-10-09T05:50:18.046872+0000","last_became_peered":"2023-10-09T05:50:18.046872+0000","last_unstale":"2023-10-09T05:54:05.160925+0000","last_undegraded":"2023-10-09T05:54:05.160925+0000","last_fullsized":"2023-10-09T05:54:05.160925+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":124,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160978+0000","last_change":"2023-10-09T05:50:18.055745+0000","last_active":"2023-10-09T05:54:05.160978+0000","last_peered":"2023-10-09T05:54:05.160978+0000","last_clean":"2023-10-09T05:54:05.160978+0000","last_became_active":"2023-10-09T05:50:18.055524+0000","last_became_peered":"2023-10-09T05:50:18.055524+0000","last_unstale":"2023-10-09T05:54:05.160978+0000","last_undegraded":"2023-10-09T05:54:05.160978+0000","last_fullsized":"2023-10-09T05:54:05.160978+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":408,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.156532+0000","last_change":"2023-10-09T05:53:46.952306+0000","last_active":"2023-10-09T05:54:05.156532+0000","last_peered":"2023-10-09T05:54:05.156532+0000","last_clean":"2023-10-09T05:54:05.156532+0000","last_became_active":"2023-10-09T05:53:46.950661+0000","last_became_peered":"2023-10-09T05:53:46.950661+0000","last_unstale":"2023-10-09T05:54:05.156532+0000","last_undegraded":"2023-10-09T05:54:05.156532+0000","last_fullsized":"2023-10-09T05:54:05.156532+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:40.279962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":350,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.154432+0000","last_change":"2023-10-09T05:48:54.492664+0000","last_active":"2023-10-09T05:54:05.154432+0000","last_peered":"2023-10-09T05:54:05.154432+0000","last_clean":"2023-10-09T05:54:05.154432+0000","last_became_active":"2023-10-09T05:48:54.492299+0000","last_became_peered":"2023-10-09T05:48:54.492299+0000","last_unstale":"2023-10-09T05:54:05.154432+0000","last_undegraded":"2023-10-09T05:54:05.154432+0000","last_fullsized":"2023-10-09T05:54:05.154432+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:43.198424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":213,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.156750+0000","last_change":"2023-10-09T05:48:54.492447+0000","last_active":"2023-10-09T05:54:05.156750+0000","last_peered":"2023-10-09T05:54:05.156750+0000","last_clean":"2023-10-09T05:54:05.156750+0000","last_became_active":"2023-10-09T05:48:54.492089+0000","last_became_peered":"2023-10-09T05:48:54.492089+0000","last_unstale":"2023-10-09T05:54:05.156750+0000","last_undegraded":"2023-10-09T05:54:05.156750+0000","last_fullsized":"2023-10-09T05:54:05.156750+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":337,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:09.823221+0000","last_change":"2023-10-09T05:31:32.864410+0000","last_active":"2023-10-09T05:54:09.823221+0000","last_peered":"2023-10-09T05:54:09.823221+0000","last_clean":"2023-10-09T05:54:09.823221+0000","last_became_active":"2023-10-09T05:31:32.864071+0000","last_became_peered":"2023-10-09T05:31:32.864071+0000","last_unstale":"2023-10-09T05:54:09.823221+0000","last_undegraded":"2023-10-09T05:54:09.823221+0000","last_fullsized":"2023-10-09T05:54:09.823221+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:09.664984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":341,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.159803+0000","last_change":"2023-10-09T05:50:18.054081+0000","last_active":"2023-10-09T05:54:05.159803+0000","last_peered":"2023-10-09T05:54:05.159803+0000","last_clean":"2023-10-09T05:54:05.159803+0000","last_became_active":"2023-10-09T05:50:18.053746+0000","last_became_peered":"2023-10-09T05:50:18.053746+0000","last_unstale":"2023-10-09T05:54:05.159803+0000","last_undegraded":"2023-10-09T05:54:05.159803+0000","last_fullsized":"2023-10-09T05:54:05.159803+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:55.758681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":49,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155902+0000","last_change":"2023-10-09T05:53:46.959609+0000","last_active":"2023-10-09T05:54:05.155902+0000","last_peered":"2023-10-09T05:54:05.155902+0000","last_clean":"2023-10-09T05:54:05.155902+0000","last_became_active":"2023-10-09T05:53:46.959296+0000","last_became_peered":"2023-10-09T05:53:46.959296+0000","last_unstale":"2023-10-09T05:54:05.155902+0000","last_undegraded":"2023-10-09T05:54:05.155902+0000","last_fullsized":"2023-10-09T05:54:05.155902+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":17,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155043+0000","last_change":"2023-10-09T05:53:46.944656+0000","last_active":"2023-10-09T05:54:05.155043+0000","last_peered":"2023-10-09T05:54:05.155043+0000","last_clean":"2023-10-09T05:54:05.155043+0000","last_became_active":"2023-10-09T05:53:46.944247+0000","last_became_peered":"2023-10-09T05:53:46.944247+0000","last_unstale":"2023-10-09T05:54:05.155043+0000","last_undegraded":"2023-10-09T05:54:05.155043+0000","last_fullsized":"2023-10-09T05:54:05.155043+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":35,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160760+0000","last_change":"2023-10-09T05:53:47.403098+0000","last_active":"2023-10-09T05:54:05.160760+0000","last_peered":"2023-10-09T05:54:05.160760+0000","last_clean":"2023-10-09T05:54:05.160760+0000","last_became_active":"2023-10-09T05:53:46.962314+0000","last_became_peered":"2023-10-09T05:53:46.962314+0000","last_unstale":"2023-10-09T05:54:05.160760+0000","last_undegraded":"2023-10-09T05:54:05.160760+0000","last_fullsized":"2023-10-09T05:54:05.160760+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":19,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155311+0000","last_change":"2023-10-09T05:53:46.063792+0000","last_active":"2023-10-09T05:54:05.155311+0000","last_peered":"2023-10-09T05:54:05.155311+0000","last_clean":"2023-10-09T05:54:05.155311+0000","last_became_active":"2023-10-09T05:53:46.061913+0000","last_became_peered":"2023-10-09T05:53:46.061913+0000","last_unstale":"2023-10-09T05:54:05.155311+0000","last_undegraded":"2023-10-09T05:54:05.155311+0000","last_fullsized":"2023-10-09T05:54:05.155311+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.248908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":425,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.162776+0000","last_change":"2023-10-09T05:53:47.133385+0000","last_active":"2023-10-09T05:54:05.162776+0000","last_peered":"2023-10-09T05:54:05.162776+0000","last_clean":"2023-10-09T05:54:05.162776+0000","last_became_active":"2023-10-09T05:53:46.945068+0000","last_became_peered":"2023-10-09T05:53:46.945068+0000","last_unstale":"2023-10-09T05:54:05.162776+0000","last_undegraded":"2023-10-09T05:54:05.162776+0000","last_fullsized":"2023-10-09T05:54:05.162776+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":372,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160781+0000","last_change":"2023-10-09T05:53:45.952781+0000","last_active":"2023-10-09T05:54:05.160781+0000","last_peered":"2023-10-09T05:54:05.160781+0000","last_clean":"2023-10-09T05:54:05.160781+0000","last_became_active":"2023-10-09T05:53:45.952446+0000","last_became_peered":"2023-10-09T05:53:45.952446+0000","last_unstale":"2023-10-09T05:54:05.160781+0000","last_undegraded":"2023-10-09T05:54:05.160781+0000","last_fullsized":"2023-10-09T05:54:05.160781+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":23,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:07.478965+0000","last_change":"2023-10-09T05:54:05.165476+0000","last_active":"2023-10-09T05:54:07.478965+0000","last_peered":"2023-10-09T05:54:07.478965+0000","last_clean":"2023-10-09T05:54:07.478965+0000","last_became_active":"2023-10-09T05:54:05.165139+0000","last_became_peered":"2023-10-09T05:54:05.165139+0000","last_unstale":"2023-10-09T05:54:07.478965+0000","last_undegraded":"2023-10-09T05:54:07.478965+0000","last_fullsized":"2023-10-09T05:54:07.478965+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:36.486411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":18,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.153245+0000","last_change":"2023-10-09T05:53:46.949286+0000","last_active":"2023-10-09T05:54:05.153245+0000","last_peered":"2023-10-09T05:54:05.153245+0000","last_clean":"2023-10-09T05:54:05.153245+0000","last_became_active":"2023-10-09T05:53:46.948933+0000","last_became_peered":"2023-10-09T05:53:46.948933+0000","last_unstale":"2023-10-09T05:54:05.153245+0000","last_undegraded":"2023-10-09T05:54:05.153245+0000","last_fullsized":"2023-10-09T05:54:05.153245+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:47.161513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":379,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155779+0000","last_change":"2023-10-09T05:53:46.946605+0000","last_active":"2023-10-09T05:54:05.155779+0000","last_peered":"2023-10-09T05:54:05.155779+0000","last_clean":"2023-10-09T05:54:05.155779+0000","last_became_active":"2023-10-09T05:53:46.946274+0000","last_became_peered":"2023-10-09T05:53:46.946274+0000","last_unstale":"2023-10-09T05:54:05.155779+0000","last_undegraded":"2023-10-09T05:54:05.155779+0000","last_fullsized":"2023-10-09T05:54:05.155779+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:36.695351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":49,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.156025+0000","last_change":"2023-10-09T05:53:46.947555+0000","last_active":"2023-10-09T05:54:05.156025+0000","last_peered":"2023-10-09T05:54:05.156025+0000","last_clean":"2023-10-09T05:54:05.156025+0000","last_became_active":"2023-10-09T05:53:46.947206+0000","last_became_peered":"2023-10-09T05:53:46.947206+0000","last_unstale":"2023-10-09T05:54:05.156025+0000","last_undegraded":"2023-10-09T05:54:05.156025+0000","last_fullsized":"2023-10-09T05:54:05.156025+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":143,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.161063+0000","last_change":"2023-10-09T05:53:46.951180+0000","last_active":"2023-10-09T05:54:05.161063+0000","last_peered":"2023-10-09T05:54:05.161063+0000","last_clean":"2023-10-09T05:54:05.161063+0000","last_became_active":"2023-10-09T05:53:46.950816+0000","last_became_peered":"2023-10-09T05:53:46.950816+0000","last_unstale":"2023-10-09T05:54:05.161063+0000","last_undegraded":"2023-10-09T05:54:05.161063+0000","last_fullsized":"2023-10-09T05:54:05.161063+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":379,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:09.823194+0000","last_change":"2023-10-09T05:53:46.945942+0000","last_active":"2023-10-09T05:54:09.823194+0000","last_peered":"2023-10-09T05:54:09.823194+0000","last_clean":"2023-10-09T05:54:09.823194+0000","last_became_active":"2023-10-09T05:53:46.945587+0000","last_became_peered":"2023-10-09T05:53:46.945587+0000","last_unstale":"2023-10-09T05:54:09.823194+0000","last_undegraded":"2023-10-09T05:54:09.823194+0000","last_fullsized":"2023-10-09T05:54:09.823194+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:12.067695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":196,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-10-09T05:54:13.675039+0000","last_change":"2023-10-09T05:51:43.221482+0000","last_active":"2023-10-09T05:54:13.675039+0000","last_peered":"2023-10-09T05:54:13.675039+0000","last_clean":"2023-10-09T05:54:13.675039+0000","last_became_active":"2023-10-09T05:51:43.072606+0000","last_became_peered":"2023-10-09T05:51:43.072606+0000","last_unstale":"2023-10-09T05:54:13.675039+0000","last_undegraded":"2023-10-09T05:54:13.675039+0000","last_fullsized":"2023-10-09T05:54:13.675039+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:22.097091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":18,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.154010+0000","last_change":"2023-10-09T05:53:46.946094+0000","last_active":"2023-10-09T05:54:05.154010+0000","last_peered":"2023-10-09T05:54:05.154010+0000","last_clean":"2023-10-09T05:54:05.154010+0000","last_became_active":"2023-10-09T05:53:46.945735+0000","last_became_peered":"2023-10-09T05:53:46.945735+0000","last_unstale":"2023-10-09T05:54:05.154010+0000","last_undegraded":"2023-10-09T05:54:05.154010+0000","last_fullsized":"2023-10-09T05:54:05.154010+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:27.320170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"380'4213","reported_seq":4251,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:19.488000+0000","last_change":"2023-10-09T05:53:48.156183+0000","last_active":"2023-10-09T05:54:19.488000+0000","last_peered":"2023-10-09T05:54:19.488000+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T05:53:48.114132+0000","last_became_peered":"2023-10-09T05:53:48.114132+0000","last_unstale":"2023-10-09T05:54:19.488000+0000","last_undegraded":"2023-10-09T05:53:48.103239+0000","last_fullsized":"2023-10-09T05:53:47.948316+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4213,"log_dups_size":0,"ondisk_log_size":4213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:55.538793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276103168,"num_objects":4213,"num_object_clones":0,"num_object_copies":12639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2885,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4213,"num_write_kb":269632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":1328},{"shards":"0,5,7","objects":2885}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":127,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160861+0000","last_change":"2023-10-09T05:53:47.265625+0000","last_active":"2023-10-09T05:54:05.160861+0000","last_peered":"2023-10-09T05:54:05.160861+0000","last_clean":"2023-10-09T05:54:05.160861+0000","last_became_active":"2023-10-09T05:53:46.960232+0000","last_became_peered":"2023-10-09T05:53:46.960232+0000","last_unstale":"2023-10-09T05:54:05.160861+0000","last_undegraded":"2023-10-09T05:54:05.160861+0000","last_fullsized":"2023-10-09T05:54:05.160861+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":49,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155055+0000","last_change":"2023-10-09T05:53:46.945666+0000","last_active":"2023-10-09T05:54:05.155055+0000","last_peered":"2023-10-09T05:54:05.155055+0000","last_clean":"2023-10-09T05:54:05.155055+0000","last_became_active":"2023-10-09T05:53:46.945309+0000","last_became_peered":"2023-10-09T05:53:46.945309+0000","last_unstale":"2023-10-09T05:54:05.155055+0000","last_undegraded":"2023-10-09T05:54:05.155055+0000","last_fullsized":"2023-10-09T05:54:05.155055+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"380'4286","reported_seq":4315,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-10-09T05:54:22.036312+0000","last_change":"2023-10-09T05:54:05.169383+0000","last_active":"2023-10-09T05:54:22.036312+0000","last_peered":"2023-10-09T05:54:22.036312+0000","last_clean":"2023-10-09T05:54:22.036312+0000","last_became_active":"2023-10-09T05:54:05.163196+0000","last_became_peered":"2023-10-09T05:54:05.163196+0000","last_unstale":"2023-10-09T05:54:22.036312+0000","last_undegraded":"2023-10-09T05:54:22.036312+0000","last_fullsized":"2023-10-09T05:54:22.036312+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4286,"log_dups_size":0,"ondisk_log_size":4286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:22.536943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280887296,"num_objects":4286,"num_object_clones":0,"num_object_copies":12858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4286,"num_write_kb":274304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":127,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160701+0000","last_change":"2023-10-09T05:53:47.311655+0000","last_active":"2023-10-09T05:54:05.160701+0000","last_peered":"2023-10-09T05:54:05.160701+0000","last_clean":"2023-10-09T05:54:05.160701+0000","last_became_active":"2023-10-09T05:53:46.960645+0000","last_became_peered":"2023-10-09T05:53:46.960645+0000","last_unstale":"2023-10-09T05:54:05.160701+0000","last_undegraded":"2023-10-09T05:54:05.160701+0000","last_fullsized":"2023-10-09T05:54:05.160701+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":326,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.164113+0000","last_change":"2023-10-09T05:54:05.163758+0000","last_active":"2023-10-09T05:54:05.164113+0000","last_peered":"2023-10-09T05:54:05.164113+0000","last_clean":"2023-10-09T05:54:05.164113+0000","last_became_active":"2023-10-09T05:54:05.163374+0000","last_became_peered":"2023-10-09T05:54:05.163374+0000","last_unstale":"2023-10-09T05:54:05.164113+0000","last_undegraded":"2023-10-09T05:54:05.164113+0000","last_fullsized":"2023-10-09T05:54:05.164113+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.7","version":"380'4256","reported_seq":4288,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:20.594449+0000","last_change":"2023-10-09T05:53:48.093566+0000","last_active":"2023-10-09T05:54:20.594449+0000","last_peered":"2023-10-09T05:54:20.594449+0000","last_clean":"2023-10-09T05:53:45.891460+0000","last_became_active":"2023-10-09T05:53:48.087661+0000","last_became_peered":"2023-10-09T05:53:48.087661+0000","last_unstale":"2023-10-09T05:54:20.594449+0000","last_undegraded":"2023-10-09T05:53:47.967757+0000","last_fullsized":"2023-10-09T05:53:47.934982+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4256,"log_dups_size":0,"ondisk_log_size":4256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278921216,"num_objects":4256,"num_object_clones":0,"num_object_copies":12768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2928,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4256,"num_write_kb":272384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5","objects":1328},{"shards":"3,5,7","objects":2928}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":169,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155960+0000","last_change":"2023-10-09T05:53:47.177039+0000","last_active":"2023-10-09T05:54:05.155960+0000","last_peered":"2023-10-09T05:54:05.155960+0000","last_clean":"2023-10-09T05:54:05.155960+0000","last_became_active":"2023-10-09T05:53:46.947204+0000","last_became_peered":"2023-10-09T05:53:46.947204+0000","last_unstale":"2023-10-09T05:54:05.155960+0000","last_undegraded":"2023-10-09T05:54:05.155960+0000","last_fullsized":"2023-10-09T05:54:05.155960+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":17,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155079+0000","last_change":"2023-10-09T05:53:46.962297+0000","last_active":"2023-10-09T05:54:05.155079+0000","last_peered":"2023-10-09T05:54:05.155079+0000","last_clean":"2023-10-09T05:54:05.155079+0000","last_became_active":"2023-10-09T05:53:46.961944+0000","last_became_peered":"2023-10-09T05:53:46.961944+0000","last_unstale":"2023-10-09T05:54:05.155079+0000","last_undegraded":"2023-10-09T05:54:05.155079+0000","last_fullsized":"2023-10-09T05:54:05.155079+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.6","version":"380'4185","reported_seq":4208,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-10-09T05:54:20.569616+0000","last_change":"2023-10-09T05:53:08.373761+0000","last_active":"2023-10-09T05:54:20.569616+0000","last_peered":"2023-10-09T05:54:20.569616+0000","last_clean":"2023-10-09T05:54:20.569616+0000","last_became_active":"2023-10-09T05:53:08.373499+0000","last_became_peered":"2023-10-09T05:53:08.373499+0000","last_unstale":"2023-10-09T05:54:20.569616+0000","last_undegraded":"2023-10-09T05:54:20.569616+0000","last_fullsized":"2023-10-09T05:54:20.569616+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4185,"log_dups_size":0,"ondisk_log_size":4185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274268160,"num_objects":4185,"num_object_clones":0,"num_object_copies":12555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4185,"num_write_kb":267840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":347,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155399+0000","last_change":"2023-10-09T05:31:32.866394+0000","last_active":"2023-10-09T05:54:05.155399+0000","last_peered":"2023-10-09T05:54:05.155399+0000","last_clean":"2023-10-09T05:54:05.155399+0000","last_became_active":"2023-10-09T05:31:32.866047+0000","last_became_peered":"2023-10-09T05:31:32.866047+0000","last_unstale":"2023-10-09T05:54:05.155399+0000","last_undegraded":"2023-10-09T05:54:05.155399+0000","last_fullsized":"2023-10-09T05:54:05.155399+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":355,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.154431+0000","last_change":"2023-10-09T05:48:54.495800+0000","last_active":"2023-10-09T05:54:05.154431+0000","last_peered":"2023-10-09T05:54:05.154431+0000","last_clean":"2023-10-09T05:54:05.154431+0000","last_became_active":"2023-10-09T05:48:54.495540+0000","last_became_peered":"2023-10-09T05:48:54.495540+0000","last_unstale":"2023-10-09T05:54:05.154431+0000","last_undegraded":"2023-10-09T05:54:05.154431+0000","last_fullsized":"2023-10-09T05:54:05.154431+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:26.903669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"380'4358","reported_seq":8460,"reported_epoch":380,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:54:22.519477+0000","last_change":"2023-10-09T05:53:48.210689+0000","last_active":"2023-10-09T05:54:22.519477+0000","last_peered":"2023-10-09T05:54:22.519477+0000","last_clean":"2023-10-09T05:53:45.918109+0000","last_became_active":"2023-10-09T05:53:48.094653+0000","last_became_peered":"2023-10-09T05:53:48.094653+0000","last_unstale":"2023-10-09T05:54:22.519477+0000","last_undegraded":"2023-10-09T05:54:22.519477+0000","last_fullsized":"2023-10-09T05:53:47.933493+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4358,"log_dups_size":0,"ondisk_log_size":4358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:16.164567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285605888,"num_objects":4358,"num_object_clones":0,"num_object_copies":13074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":815,"num_objects_unfound":0,"num_objects_dirty":4358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4358,"num_write_kb":278912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2036,"num_bytes_recovered":133431296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6","objects":3543},{"shards":"3,6,7","objects":815}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":118,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.154543+0000","last_change":"2023-10-09T05:48:25.067945+0000","last_active":"2023-10-09T05:54:05.154543+0000","last_peered":"2023-10-09T05:54:05.154543+0000","last_clean":"2023-10-09T05:54:05.154543+0000","last_became_active":"2023-10-09T05:48:25.067721+0000","last_became_peered":"2023-10-09T05:48:25.067721+0000","last_unstale":"2023-10-09T05:54:05.154543+0000","last_undegraded":"2023-10-09T05:54:05.154543+0000","last_fullsized":"2023-10-09T05:54:05.154543+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:58.655544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":345,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.162684+0000","last_change":"2023-10-09T05:31:32.864605+0000","last_active":"2023-10-09T05:54:05.162684+0000","last_peered":"2023-10-09T05:54:05.162684+0000","last_clean":"2023-10-09T05:54:05.162684+0000","last_became_active":"2023-10-09T05:31:32.864202+0000","last_became_peered":"2023-10-09T05:31:32.864202+0000","last_unstale":"2023-10-09T05:54:05.162684+0000","last_undegraded":"2023-10-09T05:54:05.162684+0000","last_fullsized":"2023-10-09T05:54:05.162684+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"380'4261","reported_seq":4286,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-10-09T05:54:22.416232+0000","last_change":"2023-10-09T05:53:08.375690+0000","last_active":"2023-10-09T05:54:22.416232+0000","last_peered":"2023-10-09T05:54:22.416232+0000","last_clean":"2023-10-09T05:54:22.416232+0000","last_became_active":"2023-10-09T05:53:08.375441+0000","last_became_peered":"2023-10-09T05:53:08.375441+0000","last_unstale":"2023-10-09T05:54:22.416232+0000","last_undegraded":"2023-10-09T05:54:22.416232+0000","last_fullsized":"2023-10-09T05:54:22.416232+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4261,"log_dups_size":0,"ondisk_log_size":4261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:39:15.702086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279248896,"num_objects":4261,"num_object_clones":0,"num_object_copies":12783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4261,"num_write_kb":272704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.162440+0000","last_change":"2023-10-09T05:48:54.494715+0000","last_active":"2023-10-09T05:54:05.162440+0000","last_peered":"2023-10-09T05:54:05.162440+0000","last_clean":"2023-10-09T05:54:05.162440+0000","last_became_active":"2023-10-09T05:48:54.493988+0000","last_became_peered":"2023-10-09T05:48:54.493988+0000","last_unstale":"2023-10-09T05:54:05.162440+0000","last_undegraded":"2023-10-09T05:54:05.162440+0000","last_fullsized":"2023-10-09T05:54:05.162440+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":308,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:07.479018+0000","last_change":"2023-10-09T05:54:05.161959+0000","last_active":"2023-10-09T05:54:07.479018+0000","last_peered":"2023-10-09T05:54:07.479018+0000","last_clean":"2023-10-09T05:54:07.479018+0000","last_became_active":"2023-10-09T05:54:05.161671+0000","last_became_peered":"2023-10-09T05:54:05.161671+0000","last_unstale":"2023-10-09T05:54:07.479018+0000","last_undegraded":"2023-10-09T05:54:07.479018+0000","last_fullsized":"2023-10-09T05:54:07.479018+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:47.282176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"380'4260","reported_seq":8886,"reported_epoch":380,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:54:22.041330+0000","last_change":"2023-10-09T05:53:48.516712+0000","last_active":"2023-10-09T05:54:22.041330+0000","last_peered":"2023-10-09T05:54:22.041330+0000","last_clean":"2023-10-09T05:53:45.901038+0000","last_became_active":"2023-10-09T05:53:48.075439+0000","last_became_peered":"2023-10-09T05:53:48.075439+0000","last_unstale":"2023-10-09T05:54:22.041330+0000","last_undegraded":"2023-10-09T05:54:22.041330+0000","last_fullsized":"2023-10-09T05:53:47.934505+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4260,"log_dups_size":0,"ondisk_log_size":4260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:39.511312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279183360,"num_objects":4260,"num_object_clones":0,"num_object_copies":12780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":546,"num_objects_unfound":0,"num_objects_dirty":4260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4260,"num_write_kb":272640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2298,"num_bytes_recovered":150601728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[1,3],"avail_no_missing":["1","3","7"],"object_location_counts":[{"shards":"1,3","objects":3714},{"shards":"1,3,7","objects":546}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":388,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160887+0000","last_change":"2023-10-09T05:53:46.963923+0000","last_active":"2023-10-09T05:54:05.160887+0000","last_peered":"2023-10-09T05:54:05.160887+0000","last_clean":"2023-10-09T05:54:05.160887+0000","last_became_active":"2023-10-09T05:53:46.963593+0000","last_became_peered":"2023-10-09T05:53:46.963593+0000","last_unstale":"2023-10-09T05:54:05.160887+0000","last_undegraded":"2023-10-09T05:54:05.160887+0000","last_fullsized":"2023-10-09T05:54:05.160887+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":372,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160608+0000","last_change":"2023-10-09T05:48:54.492708+0000","last_active":"2023-10-09T05:54:05.160608+0000","last_peered":"2023-10-09T05:54:05.160608+0000","last_clean":"2023-10-09T05:54:05.160608+0000","last_became_active":"2023-10-09T05:48:54.492382+0000","last_became_peered":"2023-10-09T05:48:54.492382+0000","last_unstale":"2023-10-09T05:54:05.160608+0000","last_undegraded":"2023-10-09T05:54:05.160608+0000","last_fullsized":"2023-10-09T05:54:05.160608+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"380'4344","reported_seq":4368,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-10-09T05:54:22.450800+0000","last_change":"2023-10-09T05:53:08.432676+0000","last_active":"2023-10-09T05:54:22.450800+0000","last_peered":"2023-10-09T05:54:22.450800+0000","last_clean":"2023-10-09T05:54:22.450800+0000","last_became_active":"2023-10-09T05:53:08.432370+0000","last_became_peered":"2023-10-09T05:53:08.432370+0000","last_unstale":"2023-10-09T05:54:22.450800+0000","last_undegraded":"2023-10-09T05:54:22.450800+0000","last_fullsized":"2023-10-09T05:54:22.450800+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4344,"log_dups_size":0,"ondisk_log_size":4344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:08.122834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284688384,"num_objects":4344,"num_object_clones":0,"num_object_copies":13032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4344,"num_write_kb":278016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":323,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:07.512962+0000","last_change":"2023-10-09T05:54:05.162479+0000","last_active":"2023-10-09T05:54:07.512962+0000","last_peered":"2023-10-09T05:54:07.512962+0000","last_clean":"2023-10-09T05:54:07.512962+0000","last_became_active":"2023-10-09T05:54:05.162120+0000","last_became_peered":"2023-10-09T05:54:05.162120+0000","last_unstale":"2023-10-09T05:54:07.512962+0000","last_undegraded":"2023-10-09T05:54:07.512962+0000","last_fullsized":"2023-10-09T05:54:07.512962+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:54.264293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":124,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160612+0000","last_change":"2023-10-09T05:50:18.072866+0000","last_active":"2023-10-09T05:54:05.160612+0000","last_peered":"2023-10-09T05:54:05.160612+0000","last_clean":"2023-10-09T05:54:05.160612+0000","last_became_active":"2023-10-09T05:50:18.072550+0000","last_became_peered":"2023-10-09T05:50:18.072550+0000","last_unstale":"2023-10-09T05:54:05.160612+0000","last_undegraded":"2023-10-09T05:54:05.160612+0000","last_fullsized":"2023-10-09T05:54:05.160612+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.f","version":"380'4306","reported_seq":4330,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-10-09T05:54:20.598639+0000","last_change":"2023-10-09T05:53:08.432523+0000","last_active":"2023-10-09T05:54:20.598639+0000","last_peered":"2023-10-09T05:54:20.598639+0000","last_clean":"2023-10-09T05:54:20.598639+0000","last_became_active":"2023-10-09T05:53:08.431983+0000","last_became_peered":"2023-10-09T05:53:08.431983+0000","last_unstale":"2023-10-09T05:54:20.598639+0000","last_undegraded":"2023-10-09T05:54:20.598639+0000","last_fullsized":"2023-10-09T05:54:20.598639+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4306,"log_dups_size":0,"ondisk_log_size":4306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282198016,"num_objects":4306,"num_object_clones":0,"num_object_copies":12918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4306,"num_write_kb":275584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":814,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155987+0000","last_change":"2023-10-09T05:53:50.174532+0000","last_active":"2023-10-09T05:54:05.155987+0000","last_peered":"2023-10-09T05:54:05.155987+0000","last_clean":"2023-10-09T05:54:05.155987+0000","last_became_active":"2023-10-09T05:53:50.173844+0000","last_became_peered":"2023-10-09T05:53:50.173844+0000","last_unstale":"2023-10-09T05:54:05.155987+0000","last_undegraded":"2023-10-09T05:54:05.155987+0000","last_fullsized":"2023-10-09T05:54:05.155987+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.790714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":362,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.156159+0000","last_change":"2023-10-09T05:31:32.866456+0000","last_active":"2023-10-09T05:54:05.156159+0000","last_peered":"2023-10-09T05:54:05.156159+0000","last_clean":"2023-10-09T05:54:05.156159+0000","last_became_active":"2023-10-09T05:31:32.866073+0000","last_became_peered":"2023-10-09T05:31:32.866073+0000","last_unstale":"2023-10-09T05:54:05.156159+0000","last_undegraded":"2023-10-09T05:54:05.156159+0000","last_fullsized":"2023-10-09T05:54:05.156159+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":40,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155633+0000","last_change":"2023-10-09T05:51:43.248409+0000","last_active":"2023-10-09T05:54:05.155633+0000","last_peered":"2023-10-09T05:54:05.155633+0000","last_clean":"2023-10-09T05:54:05.155633+0000","last_became_active":"2023-10-09T05:51:43.154898+0000","last_became_peered":"2023-10-09T05:51:43.154898+0000","last_unstale":"2023-10-09T05:54:05.155633+0000","last_undegraded":"2023-10-09T05:54:05.155633+0000","last_fullsized":"2023-10-09T05:54:05.155633+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"380'4150","reported_seq":4185,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:19.849656+0000","last_change":"2023-10-09T05:53:48.182111+0000","last_active":"2023-10-09T05:54:19.849656+0000","last_peered":"2023-10-09T05:54:19.849656+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T05:53:48.127859+0000","last_became_peered":"2023-10-09T05:53:48.127859+0000","last_unstale":"2023-10-09T05:54:19.849656+0000","last_undegraded":"2023-10-09T05:53:48.119724+0000","last_fullsized":"2023-10-09T05:53:47.944500+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4150,"log_dups_size":0,"ondisk_log_size":4150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:12.814441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271974400,"num_objects":4150,"num_object_clones":0,"num_object_copies":12450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2825,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4150,"num_write_kb":265600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":1325},{"shards":"0,5,7","objects":2824}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":870,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155804+0000","last_change":"2023-10-09T05:48:54.592642+0000","last_active":"2023-10-09T05:54:05.155804+0000","last_peered":"2023-10-09T05:54:05.155804+0000","last_clean":"2023-10-09T05:54:05.155804+0000","last_became_active":"2023-10-09T05:48:54.491917+0000","last_became_peered":"2023-10-09T05:48:54.491917+0000","last_unstale":"2023-10-09T05:54:05.155804+0000","last_undegraded":"2023-10-09T05:54:05.155804+0000","last_fullsized":"2023-10-09T05:54:05.155804+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":346,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.161140+0000","last_change":"2023-10-09T05:29:59.475194+0000","last_active":"2023-10-09T05:54:05.161140+0000","last_peered":"2023-10-09T05:54:05.161140+0000","last_clean":"2023-10-09T05:54:05.161140+0000","last_became_active":"2023-10-09T05:29:59.474745+0000","last_became_peered":"2023-10-09T05:29:59.474745+0000","last_unstale":"2023-10-09T05:54:05.161140+0000","last_undegraded":"2023-10-09T05:54:05.161140+0000","last_fullsized":"2023-10-09T05:54:05.161140+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":18,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155151+0000","last_change":"2023-10-09T05:53:45.946199+0000","last_active":"2023-10-09T05:54:05.155151+0000","last_peered":"2023-10-09T05:54:05.155151+0000","last_clean":"2023-10-09T05:54:05.155151+0000","last_became_active":"2023-10-09T05:53:45.945870+0000","last_became_peered":"2023-10-09T05:53:45.945870+0000","last_unstale":"2023-10-09T05:54:05.155151+0000","last_undegraded":"2023-10-09T05:54:05.155151+0000","last_fullsized":"2023-10-09T05:54:05.155151+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"380'4281","reported_seq":4313,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:20.581832+0000","last_change":"2023-10-09T05:53:48.079726+0000","last_active":"2023-10-09T05:54:20.581832+0000","last_peered":"2023-10-09T05:54:20.581832+0000","last_clean":"2023-10-09T05:53:45.927481+0000","last_became_active":"2023-10-09T05:53:48.072168+0000","last_became_peered":"2023-10-09T05:53:48.072168+0000","last_unstale":"2023-10-09T05:54:20.581832+0000","last_undegraded":"2023-10-09T05:53:47.967226+0000","last_fullsized":"2023-10-09T05:53:47.933929+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4281,"log_dups_size":0,"ondisk_log_size":4281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280559616,"num_objects":4281,"num_object_clones":0,"num_object_copies":12843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2829,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4281,"num_write_kb":273984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5","objects":1452},{"shards":"1,5,7","objects":2829}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":701,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.151150+0000","last_change":"2023-10-09T05:31:32.858194+0000","last_active":"2023-10-09T05:54:05.151150+0000","last_peered":"2023-10-09T05:54:05.151150+0000","last_clean":"2023-10-09T05:54:05.151150+0000","last_became_active":"2023-10-09T05:31:32.857647+0000","last_became_peered":"2023-10-09T05:31:32.857647+0000","last_unstale":"2023-10-09T05:54:05.151150+0000","last_undegraded":"2023-10-09T05:54:05.151150+0000","last_fullsized":"2023-10-09T05:54:05.151150+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:50.594451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":162,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-10-09T05:54:13.670893+0000","last_change":"2023-10-09T05:50:18.063466+0000","last_active":"2023-10-09T05:54:13.670893+0000","last_peered":"2023-10-09T05:54:13.670893+0000","last_clean":"2023-10-09T05:54:13.670893+0000","last_became_active":"2023-10-09T05:50:18.063120+0000","last_became_peered":"2023-10-09T05:50:18.063120+0000","last_unstale":"2023-10-09T05:54:13.670893+0000","last_undegraded":"2023-10-09T05:54:13.670893+0000","last_fullsized":"2023-10-09T05:54:13.670893+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:13.683314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":184,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.156671+0000","last_change":"2023-10-09T05:50:18.069402+0000","last_active":"2023-10-09T05:54:05.156671+0000","last_peered":"2023-10-09T05:54:05.156671+0000","last_clean":"2023-10-09T05:54:05.156671+0000","last_became_active":"2023-10-09T05:50:18.069196+0000","last_became_peered":"2023-10-09T05:50:18.069196+0000","last_unstale":"2023-10-09T05:54:05.156671+0000","last_undegraded":"2023-10-09T05:54:05.156671+0000","last_fullsized":"2023-10-09T05:54:05.156671+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"380'4201","reported_seq":4226,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.824424+0000","last_change":"2023-10-09T05:53:08.432957+0000","last_active":"2023-10-09T05:54:19.824424+0000","last_peered":"2023-10-09T05:54:19.824424+0000","last_clean":"2023-10-09T05:54:19.824424+0000","last_became_active":"2023-10-09T05:53:08.432681+0000","last_became_peered":"2023-10-09T05:53:08.432681+0000","last_unstale":"2023-10-09T05:54:19.824424+0000","last_undegraded":"2023-10-09T05:54:19.824424+0000","last_fullsized":"2023-10-09T05:54:19.824424+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4201,"log_dups_size":0,"ondisk_log_size":4201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:20.595860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275316736,"num_objects":4201,"num_object_clones":0,"num_object_copies":12603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4201,"num_write_kb":268864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":1061,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155557+0000","last_change":"2023-10-09T05:53:50.172162+0000","last_active":"2023-10-09T05:54:05.155557+0000","last_peered":"2023-10-09T05:54:05.155557+0000","last_clean":"2023-10-09T05:54:05.155557+0000","last_became_active":"2023-10-09T05:53:50.171632+0000","last_became_peered":"2023-10-09T05:53:50.171632+0000","last_unstale":"2023-10-09T05:54:05.155557+0000","last_undegraded":"2023-10-09T05:54:05.155557+0000","last_fullsized":"2023-10-09T05:54:05.155557+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":416,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:05.154202+0000","last_change":"2023-10-09T05:53:48.006744+0000","last_active":"2023-10-09T05:54:05.154202+0000","last_peered":"2023-10-09T05:54:05.154202+0000","last_clean":"2023-10-09T05:43:08.904107+0000","last_became_active":"2023-10-09T05:53:48.006406+0000","last_became_peered":"2023-10-09T05:53:48.006406+0000","last_unstale":"2023-10-09T05:54:05.154202+0000","last_undegraded":"2023-10-09T05:53:47.957008+0000","last_fullsized":"2023-10-09T05:53:47.956494+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:50.327405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":152,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155722+0000","last_change":"2023-10-09T05:48:25.068759+0000","last_active":"2023-10-09T05:54:05.155722+0000","last_peered":"2023-10-09T05:54:05.155722+0000","last_clean":"2023-10-09T05:54:05.155722+0000","last_became_active":"2023-10-09T05:48:25.068358+0000","last_became_peered":"2023-10-09T05:48:25.068358+0000","last_unstale":"2023-10-09T05:54:05.155722+0000","last_undegraded":"2023-10-09T05:54:05.155722+0000","last_fullsized":"2023-10-09T05:54:05.155722+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":356,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160847+0000","last_change":"2023-10-09T05:53:46.951548+0000","last_active":"2023-10-09T05:54:05.160847+0000","last_peered":"2023-10-09T05:54:05.160847+0000","last_clean":"2023-10-09T05:54:05.160847+0000","last_became_active":"2023-10-09T05:53:46.951247+0000","last_became_peered":"2023-10-09T05:53:46.951247+0000","last_unstale":"2023-10-09T05:54:05.160847+0000","last_undegraded":"2023-10-09T05:54:05.160847+0000","last_fullsized":"2023-10-09T05:54:05.160847+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"380'4270","reported_seq":4295,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.822946+0000","last_change":"2023-10-09T05:53:08.360961+0000","last_active":"2023-10-09T05:54:19.822946+0000","last_peered":"2023-10-09T05:54:19.822946+0000","last_clean":"2023-10-09T05:54:19.822946+0000","last_became_active":"2023-10-09T05:53:08.360644+0000","last_became_peered":"2023-10-09T05:53:08.360644+0000","last_unstale":"2023-10-09T05:54:19.822946+0000","last_undegraded":"2023-10-09T05:54:19.822946+0000","last_fullsized":"2023-10-09T05:54:19.822946+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4270,"log_dups_size":0,"ondisk_log_size":4270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:29.685493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279838720,"num_objects":4270,"num_object_clones":0,"num_object_copies":12810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4270,"num_write_kb":273280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":711,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:05.154387+0000","last_change":"2023-10-09T05:53:47.960204+0000","last_active":"2023-10-09T05:54:05.154387+0000","last_peered":"2023-10-09T05:54:05.154387+0000","last_clean":"2023-10-09T05:22:39.601058+0000","last_became_active":"2023-10-09T05:53:47.959667+0000","last_became_peered":"2023-10-09T05:53:47.959667+0000","last_unstale":"2023-10-09T05:54:05.154387+0000","last_undegraded":"2023-10-09T05:53:47.945301+0000","last_fullsized":"2023-10-09T05:53:47.944633+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:36.937775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1],"avail_no_missing":["0","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":374,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.154480+0000","last_change":"2023-10-09T05:51:31.682830+0000","last_active":"2023-10-09T05:54:05.154480+0000","last_peered":"2023-10-09T05:54:05.154480+0000","last_clean":"2023-10-09T05:54:05.154480+0000","last_became_active":"2023-10-09T05:51:31.682566+0000","last_became_peered":"2023-10-09T05:51:31.682566+0000","last_unstale":"2023-10-09T05:54:05.154480+0000","last_undegraded":"2023-10-09T05:54:05.154480+0000","last_fullsized":"2023-10-09T05:54:05.154480+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:42.916316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":371,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.156446+0000","last_change":"2023-10-09T05:48:54.489115+0000","last_active":"2023-10-09T05:54:05.156446+0000","last_peered":"2023-10-09T05:54:05.156446+0000","last_clean":"2023-10-09T05:54:05.156446+0000","last_became_active":"2023-10-09T05:48:54.488783+0000","last_became_peered":"2023-10-09T05:48:54.488783+0000","last_unstale":"2023-10-09T05:54:05.156446+0000","last_undegraded":"2023-10-09T05:54:05.156446+0000","last_fullsized":"2023-10-09T05:54:05.156446+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:13.193778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"380'4181","reported_seq":4212,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:20.555066+0000","last_change":"2023-10-09T05:53:48.165730+0000","last_active":"2023-10-09T05:54:20.555066+0000","last_peered":"2023-10-09T05:54:20.555066+0000","last_clean":"2023-10-09T05:53:45.896176+0000","last_became_active":"2023-10-09T05:53:48.159223+0000","last_became_peered":"2023-10-09T05:53:48.159223+0000","last_unstale":"2023-10-09T05:54:20.555066+0000","last_undegraded":"2023-10-09T05:53:47.957749+0000","last_fullsized":"2023-10-09T05:53:47.935969+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4181,"log_dups_size":0,"ondisk_log_size":4181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274006016,"num_objects":4181,"num_object_clones":0,"num_object_copies":12543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2868,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4181,"num_write_kb":267584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4","objects":1313},{"shards":"1,4,7","objects":2868}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":489,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:05.155004+0000","last_change":"2023-10-09T05:53:48.108937+0000","last_active":"2023-10-09T05:54:05.155004+0000","last_peered":"2023-10-09T05:54:05.155004+0000","last_clean":"2023-10-09T05:43:08.905133+0000","last_became_active":"2023-10-09T05:53:48.108379+0000","last_became_peered":"2023-10-09T05:53:48.108379+0000","last_unstale":"2023-10-09T05:54:05.155004+0000","last_undegraded":"2023-10-09T05:53:47.948868+0000","last_fullsized":"2023-10-09T05:53:47.948319+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:06.245327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":179,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.153426+0000","last_change":"2023-10-09T05:51:43.127835+0000","last_active":"2023-10-09T05:54:05.153426+0000","last_peered":"2023-10-09T05:54:05.153426+0000","last_clean":"2023-10-09T05:54:05.153426+0000","last_became_active":"2023-10-09T05:51:43.127489+0000","last_became_peered":"2023-10-09T05:51:43.127489+0000","last_unstale":"2023-10-09T05:54:05.153426+0000","last_undegraded":"2023-10-09T05:54:05.153426+0000","last_fullsized":"2023-10-09T05:54:05.153426+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:20.191774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":365,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.154718+0000","last_change":"2023-10-09T05:53:46.951615+0000","last_active":"2023-10-09T05:54:05.154718+0000","last_peered":"2023-10-09T05:54:05.154718+0000","last_clean":"2023-10-09T05:54:05.154718+0000","last_became_active":"2023-10-09T05:53:46.951099+0000","last_became_peered":"2023-10-09T05:53:46.951099+0000","last_unstale":"2023-10-09T05:54:05.154718+0000","last_undegraded":"2023-10-09T05:54:05.154718+0000","last_fullsized":"2023-10-09T05:54:05.154718+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:11.581702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"380'4236","reported_seq":4267,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:20.585536+0000","last_change":"2023-10-09T05:53:48.102451+0000","last_active":"2023-10-09T05:54:20.585536+0000","last_peered":"2023-10-09T05:54:20.585536+0000","last_clean":"2023-10-09T05:53:45.883404+0000","last_became_active":"2023-10-09T05:53:48.098870+0000","last_became_peered":"2023-10-09T05:53:48.098870+0000","last_unstale":"2023-10-09T05:54:20.585536+0000","last_undegraded":"2023-10-09T05:53:47.964804+0000","last_fullsized":"2023-10-09T05:53:47.934173+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4236,"log_dups_size":0,"ondisk_log_size":4236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277610496,"num_objects":4236,"num_object_clones":0,"num_object_copies":12708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2855,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4236,"num_write_kb":271104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5","objects":1381},{"shards":"1,5,7","objects":2855}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":33,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.156309+0000","last_change":"2023-10-09T05:53:47.179044+0000","last_active":"2023-10-09T05:54:05.156309+0000","last_peered":"2023-10-09T05:54:05.156309+0000","last_clean":"2023-10-09T05:54:05.156309+0000","last_became_active":"2023-10-09T05:53:46.950194+0000","last_became_peered":"2023-10-09T05:53:46.950194+0000","last_unstale":"2023-10-09T05:54:05.156309+0000","last_undegraded":"2023-10-09T05:54:05.156309+0000","last_fullsized":"2023-10-09T05:54:05.156309+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:26.142474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":634,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160520+0000","last_change":"2023-10-09T05:52:58.687202+0000","last_active":"2023-10-09T05:54:05.160520+0000","last_peered":"2023-10-09T05:54:05.160520+0000","last_clean":"2023-10-09T05:54:05.160520+0000","last_became_active":"2023-10-09T05:50:19.054850+0000","last_became_peered":"2023-10-09T05:50:19.054850+0000","last_unstale":"2023-10-09T05:54:05.160520+0000","last_undegraded":"2023-10-09T05:54:05.160520+0000","last_fullsized":"2023-10-09T05:54:05.160520+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":245,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.154083+0000","last_change":"2023-10-09T05:53:46.946475+0000","last_active":"2023-10-09T05:54:05.154083+0000","last_peered":"2023-10-09T05:54:05.154083+0000","last_clean":"2023-10-09T05:54:05.154083+0000","last_became_active":"2023-10-09T05:53:46.945809+0000","last_became_peered":"2023-10-09T05:53:46.945809+0000","last_unstale":"2023-10-09T05:54:05.154083+0000","last_undegraded":"2023-10-09T05:54:05.154083+0000","last_fullsized":"2023-10-09T05:54:05.154083+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:43.983847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"380'4239","reported_seq":4271,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:20.511525+0000","last_change":"2023-10-09T05:53:48.156042+0000","last_active":"2023-10-09T05:54:20.511525+0000","last_peered":"2023-10-09T05:54:20.511525+0000","last_clean":"2023-10-09T05:53:45.907519+0000","last_became_active":"2023-10-09T05:53:48.152299+0000","last_became_peered":"2023-10-09T05:53:48.152299+0000","last_unstale":"2023-10-09T05:54:20.511525+0000","last_undegraded":"2023-10-09T05:53:47.960433+0000","last_fullsized":"2023-10-09T05:53:47.935660+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4239,"log_dups_size":0,"ondisk_log_size":4239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277807104,"num_objects":4239,"num_object_clones":0,"num_object_copies":12717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4239,"num_write_kb":271296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4","objects":1366},{"shards":"3,4,7","objects":2873}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":550,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.154094+0000","last_change":"2023-10-09T05:53:50.166917+0000","last_active":"2023-10-09T05:54:05.154094+0000","last_peered":"2023-10-09T05:54:05.154094+0000","last_clean":"2023-10-09T05:54:05.154094+0000","last_became_active":"2023-10-09T05:53:50.166410+0000","last_became_peered":"2023-10-09T05:53:50.166410+0000","last_unstale":"2023-10-09T05:54:05.154094+0000","last_undegraded":"2023-10-09T05:54:05.154094+0000","last_fullsized":"2023-10-09T05:54:05.154094+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:12.470525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":367,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.156491+0000","last_change":"2023-10-09T05:48:54.585691+0000","last_active":"2023-10-09T05:54:05.156491+0000","last_peered":"2023-10-09T05:54:05.156491+0000","last_clean":"2023-10-09T05:54:05.156491+0000","last_became_active":"2023-10-09T05:48:54.498259+0000","last_became_peered":"2023-10-09T05:48:54.498259+0000","last_unstale":"2023-10-09T05:54:05.156491+0000","last_undegraded":"2023-10-09T05:54:05.156491+0000","last_fullsized":"2023-10-09T05:54:05.156491+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:20.684612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.164236+0000","last_change":"2023-10-09T05:54:05.163953+0000","last_active":"2023-10-09T05:54:05.164236+0000","last_peered":"2023-10-09T05:54:05.164236+0000","last_clean":"2023-10-09T05:54:05.164236+0000","last_became_active":"2023-10-09T05:54:05.163410+0000","last_became_peered":"2023-10-09T05:54:05.163410+0000","last_unstale":"2023-10-09T05:54:05.164236+0000","last_undegraded":"2023-10-09T05:54:05.164236+0000","last_fullsized":"2023-10-09T05:54:05.164236+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":366,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.154632+0000","last_change":"2023-10-09T05:53:46.950206+0000","last_active":"2023-10-09T05:54:05.154632+0000","last_peered":"2023-10-09T05:54:05.154632+0000","last_clean":"2023-10-09T05:54:05.154632+0000","last_became_active":"2023-10-09T05:53:46.949862+0000","last_became_peered":"2023-10-09T05:53:46.949862+0000","last_unstale":"2023-10-09T05:54:05.154632+0000","last_undegraded":"2023-10-09T05:54:05.154632+0000","last_fullsized":"2023-10-09T05:54:05.154632+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:32.602958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":52,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.153350+0000","last_change":"2023-10-09T05:51:13.060067+0000","last_active":"2023-10-09T05:54:05.153350+0000","last_peered":"2023-10-09T05:54:05.153350+0000","last_clean":"2023-10-09T05:54:05.153350+0000","last_became_active":"2023-10-09T05:51:13.059725+0000","last_became_peered":"2023-10-09T05:51:13.059725+0000","last_unstale":"2023-10-09T05:54:05.153350+0000","last_undegraded":"2023-10-09T05:54:05.153350+0000","last_fullsized":"2023-10-09T05:54:05.153350+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:43.328688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":375,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.155328+0000","last_change":"2023-10-09T05:48:54.583404+0000","last_active":"2023-10-09T05:54:05.155328+0000","last_peered":"2023-10-09T05:54:05.155328+0000","last_clean":"2023-10-09T05:54:05.155328+0000","last_became_active":"2023-10-09T05:48:54.492565+0000","last_became_peered":"2023-10-09T05:48:54.492565+0000","last_unstale":"2023-10-09T05:54:05.155328+0000","last_undegraded":"2023-10-09T05:54:05.155328+0000","last_fullsized":"2023-10-09T05:54:05.155328+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":378,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.161153+0000","last_change":"2023-10-09T05:48:54.493079+0000","last_active":"2023-10-09T05:54:05.161153+0000","last_peered":"2023-10-09T05:54:05.161153+0000","last_clean":"2023-10-09T05:54:05.161153+0000","last_became_active":"2023-10-09T05:48:54.492013+0000","last_became_peered":"2023-10-09T05:48:54.492013+0000","last_unstale":"2023-10-09T05:54:05.161153+0000","last_undegraded":"2023-10-09T05:54:05.161153+0000","last_fullsized":"2023-10-09T05:54:05.161153+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":334,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.156093+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:54:05.156093+0000","last_peered":"2023-10-09T05:54:05.156093+0000","last_clean":"2023-10-09T05:54:05.156093+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:54:05.156093+0000","last_undegraded":"2023-10-09T05:54:05.156093+0000","last_fullsized":"2023-10-09T05:54:05.156093+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":153,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.166291+0000","last_change":"2023-10-09T05:54:05.165471+0000","last_active":"2023-10-09T05:54:05.166291+0000","last_peered":"2023-10-09T05:54:05.166291+0000","last_clean":"2023-10-09T05:54:05.166291+0000","last_became_active":"2023-10-09T05:54:05.165145+0000","last_became_peered":"2023-10-09T05:54:05.165145+0000","last_unstale":"2023-10-09T05:54:05.166291+0000","last_undegraded":"2023-10-09T05:54:05.166291+0000","last_fullsized":"2023-10-09T05:54:05.166291+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:03.825221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":99,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.153706+0000","last_change":"2023-10-09T05:48:25.069406+0000","last_active":"2023-10-09T05:54:05.153706+0000","last_peered":"2023-10-09T05:54:05.153706+0000","last_clean":"2023-10-09T05:54:05.153706+0000","last_became_active":"2023-10-09T05:48:25.069064+0000","last_became_peered":"2023-10-09T05:48:25.069064+0000","last_unstale":"2023-10-09T05:54:05.153706+0000","last_undegraded":"2023-10-09T05:54:05.153706+0000","last_fullsized":"2023-10-09T05:54:05.153706+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:01.567858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":182,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.151064+0000","last_change":"2023-10-09T05:48:25.063854+0000","last_active":"2023-10-09T05:54:05.151064+0000","last_peered":"2023-10-09T05:54:05.151064+0000","last_clean":"2023-10-09T05:54:05.151064+0000","last_became_active":"2023-10-09T05:48:25.063579+0000","last_became_peered":"2023-10-09T05:48:25.063579+0000","last_unstale":"2023-10-09T05:54:05.151064+0000","last_undegraded":"2023-10-09T05:54:05.151064+0000","last_fullsized":"2023-10-09T05:54:05.151064+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:02.389789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":371,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.156256+0000","last_change":"2023-10-09T05:51:13.071891+0000","last_active":"2023-10-09T05:54:05.156256+0000","last_peered":"2023-10-09T05:54:05.156256+0000","last_clean":"2023-10-09T05:54:05.156256+0000","last_became_active":"2023-10-09T05:51:13.058378+0000","last_became_peered":"2023-10-09T05:51:13.058378+0000","last_unstale":"2023-10-09T05:54:05.156256+0000","last_undegraded":"2023-10-09T05:54:05.156256+0000","last_fullsized":"2023-10-09T05:54:05.156256+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:41.278366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":363,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-10-09T05:54:05.160419+0000","last_change":"2023-10-09T05:48:54.492580+0000","last_active":"2023-10-09T05:54:05.160419+0000","last_peered":"2023-10-09T05:54:05.160419+0000","last_clean":"2023-10-09T05:54:05.160419+0000","last_became_active":"2023-10-09T05:48:54.492259+0000","last_became_peered":"2023-10-09T05:48:54.492259+0000","last_unstale":"2023-10-09T05:54:05.160419+0000","last_undegraded":"2023-10-09T05:54:05.160419+0000","last_fullsized":"2023-10-09T05:54:05.160419+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":15654,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4458217472,"num_objects":68027,"num_object_clones":0,"num_object_copies":204081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20063,"num_objects_misplaced":1361,"num_objects_unfound":0,"num_objects_dirty":68027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":68027,"num_write_kb":4353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4337,"num_bytes_recovered":284229632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13693747200,"data_stored":13693747200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":68027,"ondisk_log_size":68027,"up":48,"acting":39,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":142,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":545460846829,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626855,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1762860,"kb_used_data":866916,"kb_used_omap":0,"kb_used_meta":895936,"kb_avail":91982292,"statfs":{"total":95995035648,"available":94189867008,"internally_reserved":0,"allocated":887721984,"data_stored":887005679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917438464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":2.3290000000000002,"15min":3.9220000000000002},"min":{"1min":0.29099999999999998,"5min":0.248,"15min":0.20999999999999999},"max":{"1min":1.9450000000000001,"5min":42.869,"15min":162.08799999999999},"last":11.193},{"interface":"front","average":{"1min":0.59599999999999997,"5min":2.4159999999999999,"15min":3.8650000000000002},"min":{"1min":0.317,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":2.4089999999999998,"5min":42.892000000000003,"15min":161.90899999999999},"last":0.316}]},{"osd":1,"last update":"Mon Oct 9 05:54:13 2023","interfaces":[{"interface":"back","average":{"1min":10.323,"5min":4.9249999999999998,"15min":4.5519999999999996},"min":{"1min":0.28199999999999997,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":165.66399999999999,"5min":165.66399999999999,"15min":228.24299999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":10.321999999999999,"5min":5.101,"15min":4.5190000000000001},"min":{"1min":0.36799999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":165.649,"5min":165.649,"15min":228.22499999999999},"last":0.441}]},{"osd":2,"last update":"Mon Oct 9 05:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":2.1920000000000002,"15min":3.4319999999999999},"min":{"1min":0.36799999999999999,"5min":0.29599999999999999,"15min":0.27200000000000002},"max":{"1min":1.9610000000000001,"5min":91.049999999999997,"15min":115.40600000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.76000000000000001,"5min":2.319,"15min":3.5990000000000002},"min":{"1min":0.35699999999999998,"5min":0.33600000000000002,"15min":0.29499999999999998},"max":{"1min":2.778,"5min":91.105000000000004,"15min":115.297},"last":0.46800000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:54:13 2023","interfaces":[{"interface":"back","average":{"1min":9.798,"5min":6.1559999999999997,"15min":5.0220000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.307},"max":{"1min":152.74700000000001,"5min":152.74700000000001,"15min":152.74700000000001},"last":1.1950000000000001},{"interface":"front","average":{"1min":9.4079999999999995,"5min":6.0869999999999997,"15min":4.9939999999999998},"min":{"1min":0.375,"5min":0.313,"15min":0.313},"max":{"1min":152.78100000000001,"5min":152.78100000000001,"15min":152.78100000000001},"last":2.0579999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:53:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":3.8799999999999999,"15min":1.982},"min":{"1min":0.29399999999999998,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":4.742,"5min":257.375,"15min":257.375},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.1970000000000001,"5min":3.9849999999999999,"15min":2.218},"min":{"1min":0.26900000000000002,"5min":0.26100000000000001,"15min":0.246},"max":{"1min":4.5389999999999997,"5min":257.61200000000002,"15min":257.61200000000002},"last":0.36399999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":4.3300000000000001,"15min":2.4260000000000002},"min":{"1min":0.33800000000000002,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":37.679000000000002,"5min":256.05700000000002,"15min":256.05700000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.8809999999999998,"5min":4.4400000000000004,"15min":2.5459999999999998},"min":{"1min":0.311,"5min":0.29899999999999999,"15min":0.27500000000000002},"max":{"1min":38.081000000000003,"5min":256.08499999999998,"15min":256.08499999999998},"last":0.45400000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:54:13 2023","interfaces":[{"interface":"back","average":{"1min":12.811,"5min":3.3690000000000002,"15min":1.952},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":204.15299999999999,"5min":204.15299999999999,"15min":204.15299999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":12.731,"5min":3.5670000000000002,"15min":2.2109999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":203.85499999999999,"5min":203.85499999999999,"15min":203.85499999999999},"last":0.49399999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958816,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3327576,"kb_used_data":2539856,"kb_used_omap":7,"kb_used_meta":787704,"kb_avail":90417576,"statfs":{"total":95995035648,"available":92587597824,"internally_reserved":0,"allocated":2600812544,"data_stored":2599527227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":806609422},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:54:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":3.5840000000000001,"15min":3.8340000000000001},"min":{"1min":0.25800000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":9.3480000000000008,"5min":93.176000000000002,"15min":93.176000000000002},"last":1.663},{"interface":"front","average":{"1min":3.2229999999999999,"5min":3.9220000000000002,"15min":4.1280000000000001},"min":{"1min":0.29199999999999998,"5min":0.223,"15min":0.182},"max":{"1min":16.469999999999999,"5min":92.593999999999994,"15min":92.593999999999994},"last":3.7719999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:53:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.9670000000000001,"15min":3.0649999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19800000000000001},"max":{"1min":6.1689999999999996,"5min":10.210000000000001,"15min":99.381},"last":1.0920000000000001},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.6990000000000001,"15min":2.8839999999999999},"min":{"1min":0.31,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":3.8959999999999999,"5min":16.702999999999999,"15min":99.861000000000004},"last":0.41699999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:54:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":3.2799999999999998,"15min":4.7789999999999999},"min":{"1min":0.27800000000000002,"5min":0.25700000000000001,"15min":0.20200000000000001},"max":{"1min":7.3410000000000002,"5min":92.353999999999999,"15min":162.33699999999999},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.4279999999999999,"5min":3.1499999999999999,"15min":4.6890000000000001},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.26000000000000001},"max":{"1min":5.218,"5min":93.117999999999995,"15min":162.376},"last":0.61199999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:54:08 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":1.544,"15min":1.7410000000000001},"min":{"1min":0.45900000000000002,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":8.7080000000000002,"5min":9.6899999999999995,"15min":42.106999999999999},"last":3.8759999999999999},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.423,"15min":1.611},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32100000000000001},"max":{"1min":4.9710000000000001,"5min":12.044,"15min":40.475999999999999},"last":0.32700000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:54:08 2023","interfaces":[{"interface":"back","average":{"1min":7.4100000000000001,"5min":2.8740000000000001,"15min":2.266},"min":{"1min":0.39800000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":95.060000000000002,"5min":95.060000000000002,"15min":95.060000000000002},"last":1.1679999999999999},{"interface":"front","average":{"1min":7.9349999999999996,"5min":2.7130000000000001,"15min":2.2050000000000001},"min":{"1min":0.498,"5min":0.34499999999999997,"15min":0.32200000000000001},"max":{"1min":95.593000000000004,"5min":95.593000000000004,"15min":95.593000000000004},"last":3.8119999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:54:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.7569999999999999,"15min":2.5750000000000002},"min":{"1min":0.48099999999999998,"5min":0.33300000000000002,"15min":0.316},"max":{"1min":7.2750000000000004,"5min":10.238,"15min":103.60899999999999},"last":2.0800000000000001},{"interface":"front","average":{"1min":1.974,"5min":1.605,"15min":2.3599999999999999},"min":{"1min":0.42799999999999999,"5min":0.37,"15min":0.37},"max":{"1min":8.4760000000000009,"5min":9.641,"15min":103.679},"last":3.8450000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:54:08 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.411,"15min":1.7050000000000001},"min":{"1min":0.39800000000000002,"5min":0.375,"15min":0.35999999999999999},"max":{"1min":5,"5min":12.093999999999999,"15min":42.204000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.28,"5min":1.5489999999999999,"15min":1.829},"min":{"1min":0.42999999999999999,"5min":0.39200000000000002,"15min":0.35499999999999998},"max":{"1min":5.0279999999999996,"5min":12.196,"15min":42.228999999999999},"last":2.165}]}]},{"osd":0,"up_from":88,"seq":377957122342,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1939888,"kb_used_data":1239848,"kb_used_omap":8,"kb_used_meta":700023,"kb_avail":91805264,"statfs":{"total":95995035648,"available":94008590336,"internally_reserved":0,"allocated":1269604352,"data_stored":1268236835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":716824328},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":2.532,"15min":1.895},"min":{"1min":0.246,"5min":0.186,"15min":0.17999999999999999},"max":{"1min":8.8529999999999998,"5min":63.534999999999997,"15min":63.534999999999997},"last":5.8380000000000001},{"interface":"front","average":{"1min":2.4649999999999999,"5min":2.754,"15min":2.3639999999999999},"min":{"1min":0.28899999999999998,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.2569999999999997,"5min":63.677999999999997,"15min":63.677999999999997},"last":5.9039999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:53:40 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":3.7629999999999999,"15min":3.2919999999999998},"min":{"1min":0.24399999999999999,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":2.7389999999999999,"5min":70.188999999999993,"15min":92.230000000000004},"last":8.4120000000000008},{"interface":"front","average":{"1min":0.51400000000000001,"5min":3.2229999999999999,"15min":3.0779999999999998},"min":{"1min":0.26400000000000001,"5min":0.23400000000000001,"15min":0.221},"max":{"1min":1.929,"5min":71.144000000000005,"15min":92.247},"last":8.6460000000000008}]},{"osd":3,"last update":"Mon Oct 9 05:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":3.246,"15min":3.98},"min":{"1min":0.29299999999999998,"5min":0.249,"15min":0.218},"max":{"1min":8.5619999999999994,"5min":62.698999999999998,"15min":123.024},"last":1.1200000000000001},{"interface":"front","average":{"1min":2.4430000000000001,"5min":3.9569999999999999,"15min":4.056},"min":{"1min":0.30599999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":14.122,"5min":72.632999999999996,"15min":99.162000000000006},"last":8.3770000000000007}]},{"osd":4,"last update":"Mon Oct 9 05:53:40 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":3.1989999999999998,"15min":1.8360000000000001},"min":{"1min":0.34899999999999998,"5min":0.28699999999999998,"15min":0.25700000000000001},"max":{"1min":4.0220000000000002,"5min":149.63999999999999,"15min":149.63999999999999},"last":1.1519999999999999},{"interface":"front","average":{"1min":1.363,"5min":3.4020000000000001,"15min":2.2799999999999998},"min":{"1min":0.41499999999999998,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":3.9929999999999999,"5min":149.65899999999999,"15min":149.65899999999999},"last":1.0649999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:53:40 2023","interfaces":[{"interface":"back","average":{"1min":14.904999999999999,"5min":6.617,"15min":4.875},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.317},"max":{"1min":246.761,"5min":246.761,"15min":281.733},"last":1.0089999999999999},{"interface":"front","average":{"1min":14.659000000000001,"5min":7.1550000000000002,"15min":5.0970000000000004},"min":{"1min":0.32800000000000001,"5min":0.30399999999999999,"15min":0.30199999999999999},"max":{"1min":247.48400000000001,"5min":247.48400000000001,"15min":282.95400000000001},"last":8.3059999999999992}]},{"osd":6,"last update":"Mon Oct 9 05:53:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":1.579,"15min":1.8120000000000001},"min":{"1min":0.42499999999999999,"5min":0.34000000000000002,"15min":0.30599999999999999},"max":{"1min":3.8170000000000002,"5min":15.789,"15min":19.579000000000001},"last":1.04},{"interface":"front","average":{"1min":0.81000000000000005,"5min":1.615,"15min":1.9139999999999999},"min":{"1min":0.44700000000000001,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":2.7829999999999999,"5min":19.710999999999999,"15min":19.710999999999999},"last":8.3469999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.5129999999999999,"15min":1.4730000000000001},"min":{"1min":0.436,"5min":0.35199999999999998,"15min":0.33700000000000002},"max":{"1min":4.2400000000000002,"5min":15.933999999999999,"15min":25.378},"last":1.181},{"interface":"front","average":{"1min":1.819,"5min":1.6659999999999999,"15min":1.833},"min":{"1min":0.48299999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.1429999999999998,"5min":15.641,"15min":25.436},"last":1.089}]}]},{"osd":2,"up_from":98,"seq":420906795290,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1763556,"kb_used_data":963548,"kb_used_omap":11,"kb_used_meta":799988,"kb_avail":91981596,"statfs":{"total":95995035648,"available":94189154304,"internally_reserved":0,"allocated":986673152,"data_stored":985143095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":819188309},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:48 2023","interfaces":[{"interface":"back","average":{"1min":8.1790000000000003,"5min":3.427,"15min":3.8319999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.17799999999999999},"max":{"1min":140.71100000000001,"5min":140.71100000000001,"15min":162.285},"last":0.71699999999999997},{"interface":"front","average":{"1min":8.702,"5min":3.5249999999999999,"15min":4.29},"min":{"1min":0.28999999999999998,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":142.542,"5min":142.542,"15min":162.643},"last":3.411}]},{"osd":1,"last update":"Mon Oct 9 05:53:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":4.7229999999999999,"15min":3.8239999999999998},"min":{"1min":0.25,"5min":0.23599999999999999,"15min":0.193},"max":{"1min":38.677999999999997,"5min":125.77500000000001,"15min":125.77500000000001},"last":5.4889999999999999},{"interface":"front","average":{"1min":5.9269999999999996,"5min":4.75,"15min":3.4049999999999998},"min":{"1min":0.26700000000000002,"5min":0.255,"15min":0.20999999999999999},"max":{"1min":57.715000000000003,"5min":125.857,"15min":125.857},"last":5.8890000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:53:23 2023","interfaces":[{"interface":"back","average":{"1min":3.9820000000000002,"5min":2.4870000000000001,"15min":4.9669999999999996},"min":{"1min":0.25,"5min":0.25,"15min":0.21099999999999999},"max":{"1min":29.606999999999999,"5min":29.606999999999999,"15min":159.35499999999999},"last":3.0739999999999998},{"interface":"front","average":{"1min":3.7709999999999999,"5min":2.2749999999999999,"15min":4.649},"min":{"1min":0.28499999999999998,"5min":0.217,"15min":0.217},"max":{"1min":29.920999999999999,"5min":29.920999999999999,"15min":159.196},"last":3.9399999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:53:48 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":2.0449999999999999,"15min":1.7030000000000001},"min":{"1min":0.34399999999999997,"5min":0.32200000000000001,"15min":0.28699999999999998},"max":{"1min":3.8029999999999999,"5min":68.052000000000007,"15min":68.052000000000007},"last":3.1619999999999999},{"interface":"front","average":{"1min":1.026,"5min":2.2040000000000002,"15min":1.7709999999999999},"min":{"1min":0.371,"5min":0.29199999999999998,"15min":0.28699999999999998},"max":{"1min":5.5679999999999996,"5min":68.727000000000004,"15min":68.727000000000004},"last":0.749}]},{"osd":5,"last update":"Mon Oct 9 05:53:48 2023","interfaces":[{"interface":"back","average":{"1min":19.084,"5min":5.726,"15min":3.1659999999999999},"min":{"1min":0.38400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":207.80500000000001,"5min":207.80500000000001,"15min":207.80500000000001},"last":3.2850000000000001},{"interface":"front","average":{"1min":19.094000000000001,"5min":5.7640000000000002,"15min":2.9630000000000001},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.26000000000000001},"max":{"1min":207.54900000000001,"5min":207.54900000000001,"15min":207.54900000000001},"last":3.1280000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:53:48 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.3480000000000001,"15min":1.6499999999999999},"min":{"1min":0.371,"5min":0.33200000000000002,"15min":0.32000000000000001},"max":{"1min":5.5410000000000004,"5min":9.5069999999999997,"15min":25.818999999999999},"last":0.79600000000000004},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.7010000000000001,"15min":1.968},"min":{"1min":0.44400000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.0090000000000003,"5min":10.821,"15min":24.481000000000002},"last":3.3359999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:53:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.7030000000000001,"15min":1.8879999999999999},"min":{"1min":0.44800000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.9500000000000002,"5min":12.071999999999999,"15min":51.701999999999998},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.411,"5min":1.8260000000000001,"15min":2.1520000000000001},"min":{"1min":0.44600000000000001,"5min":0.378,"15min":0.373},"max":{"1min":8.3010000000000002,"5min":8.7430000000000003,"15min":51.716000000000001},"last":3.3730000000000002}]}]},{"osd":3,"up_from":103,"seq":442381631764,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3254580,"kb_used_data":2437292,"kb_used_omap":4,"kb_used_meta":817275,"kb_avail":90490572,"statfs":{"total":95995035648,"available":92662345728,"internally_reserved":0,"allocated":2495787008,"data_stored":2494415259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":836890206},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":6.4950000000000001,"15min":5.3959999999999999},"min":{"1min":0.248,"5min":0.19700000000000001,"15min":0.184},"max":{"1min":3.4540000000000002,"5min":312.56599999999997,"15min":312.56599999999997},"last":4.8479999999999999},{"interface":"front","average":{"1min":2.0630000000000002,"5min":6.1849999999999996,"15min":5.7789999999999999},"min":{"1min":0.23300000000000001,"5min":0.219,"15min":0.20499999999999999},"max":{"1min":11.282,"5min":311.02300000000002,"15min":311.02300000000002},"last":1.105}]},{"osd":1,"last update":"Mon Oct 9 05:54:11 2023","interfaces":[{"interface":"back","average":{"1min":12.135999999999999,"5min":7.1769999999999996,"15min":4.2649999999999997},"min":{"1min":0.25700000000000001,"5min":0.214,"15min":0.214},"max":{"1min":136.52799999999999,"5min":150.29599999999999,"15min":150.29599999999999},"last":3.7040000000000002},{"interface":"front","average":{"1min":11.856999999999999,"5min":7.3550000000000004,"15min":4.5289999999999999},"min":{"1min":0.27200000000000002,"5min":0.216,"15min":0.19},"max":{"1min":136.47999999999999,"5min":151.25700000000001,"15min":151.25700000000001},"last":8.0280000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5750000000000002,"5min":3.2040000000000002,"15min":3.8450000000000002},"min":{"1min":0.28999999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":9.5039999999999996,"5min":72.602000000000004,"15min":190.43299999999999},"last":0.433},{"interface":"front","average":{"1min":1.9419999999999999,"5min":3.0830000000000002,"15min":4.0839999999999996},"min":{"1min":0.30399999999999999,"5min":0.251,"15min":0.20499999999999999},"max":{"1min":12.807,"5min":69.254000000000005,"15min":190.267},"last":1.133}]},{"osd":4,"last update":"Mon Oct 9 05:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0569999999999999,"5min":3.605,"15min":3.048},"min":{"1min":0.35899999999999999,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":16.713000000000001,"5min":126.566,"15min":149.84700000000001},"last":1.212},{"interface":"front","average":{"1min":1.7589999999999999,"5min":4.7069999999999999,"15min":3.0830000000000002},"min":{"1min":0.34599999999999997,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":16.349,"5min":126.651,"15min":149.74700000000001},"last":0.47699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9529999999999998,"5min":2.7559999999999998,"15min":2.746},"min":{"1min":0.309,"5min":0.307,"15min":0.28199999999999997},"max":{"1min":35.648000000000003,"5min":71.203999999999994,"15min":149.43000000000001},"last":4.1509999999999998},{"interface":"front","average":{"1min":2.7530000000000001,"5min":2.5369999999999999,"15min":2.5249999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":35.686999999999998,"5min":70.397000000000006,"15min":149.69300000000001},"last":0.501}]},{"osd":6,"last update":"Mon Oct 9 05:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":2.2200000000000002,"15min":2.3149999999999999},"min":{"1min":0.39600000000000002,"5min":0.35899999999999999,"15min":0.33500000000000002},"max":{"1min":3.1259999999999999,"5min":70.474000000000004,"15min":149.72399999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.377,"5min":2.3450000000000002,"15min":2.6339999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.307},"max":{"1min":5.2160000000000002,"5min":71.251999999999995,"15min":149.589},"last":4.1950000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:53:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":2.2559999999999998,"15min":2.5139999999999998},"min":{"1min":0.46400000000000002,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":10.775,"5min":69.277000000000001,"15min":149.77199999999999},"last":1.246},{"interface":"front","average":{"1min":1.4570000000000001,"5min":2.1440000000000001,"15min":2.5779999999999998},"min":{"1min":0.41399999999999998,"5min":0.374,"15min":0.373},"max":{"1min":10.913,"5min":69.299000000000007,"15min":149.80799999999999},"last":1.3140000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019311,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2052524,"kb_used_data":1155172,"kb_used_omap":0,"kb_used_meta":897344,"kb_avail":91692628,"statfs":{"total":95995035648,"available":93893251072,"internally_reserved":0,"allocated":1182896128,"data_stored":1182179567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918880256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:58 2023","interfaces":[{"interface":"back","average":{"1min":3.298,"5min":4.8440000000000003,"15min":3.3109999999999999},"min":{"1min":0.316,"5min":0.219,"15min":0.219},"max":{"1min":50.305999999999997,"5min":132.35599999999999,"15min":132.35599999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":3.4390000000000001,"5min":5.5030000000000001,"15min":3.7090000000000001},"min":{"1min":0.36599999999999999,"5min":0.23699999999999999,"15min":0.23300000000000001},"max":{"1min":51.537999999999997,"5min":132.501,"15min":132.501},"last":2.4910000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:53:20 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":3.3319999999999999,"15min":3.0179999999999998},"min":{"1min":0.29799999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":5.9809999999999999,"5min":68.367000000000004,"15min":187.369},"last":32.767000000000003},{"interface":"front","average":{"1min":1.599,"5min":3.3959999999999999,"15min":3.1099999999999999},"min":{"1min":0.312,"5min":0.28599999999999998,"15min":0.27700000000000002},"max":{"1min":8.4320000000000004,"5min":68.376000000000005,"15min":187.35300000000001},"last":2.536}]},{"osd":2,"last update":"Mon Oct 9 05:53:34 2023","interfaces":[{"interface":"back","average":{"1min":3.113,"5min":2.6739999999999999,"15min":2.8370000000000002},"min":{"1min":0.33700000000000002,"5min":0.308,"15min":0.26800000000000002},"max":{"1min":49.405000000000001,"5min":51.768000000000001,"15min":100.148},"last":2.8100000000000001},{"interface":"front","average":{"1min":3.2810000000000001,"5min":2.5499999999999998,"15min":2.77},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":49.292999999999999,"5min":51.753999999999998,"15min":100.086},"last":0.51900000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:54:09 2023","interfaces":[{"interface":"back","average":{"1min":6.0670000000000002,"5min":2.7970000000000002,"15min":3.3860000000000001},"min":{"1min":0.312,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":89.584999999999994,"5min":89.584999999999994,"15min":123.59},"last":3.52},{"interface":"front","average":{"1min":5.46,"5min":2.4740000000000002,"15min":3.2240000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":89.480000000000004,"5min":89.480000000000004,"15min":123.566},"last":7.9770000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:54:09 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.76500000000000001,"15min":0.84799999999999998},"min":{"1min":0.29099999999999998,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":2.8900000000000001,"5min":8.5190000000000001,"15min":12.34},"last":2.7290000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.78000000000000003,"15min":0.80200000000000005},"min":{"1min":0.35899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.0539999999999998,"5min":5.6900000000000004,"15min":10.191000000000001},"last":0.67200000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.002,"15min":0.71399999999999997},"min":{"1min":0.28699999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.181,"5min":9.8889999999999993,"15min":9.8889999999999993},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.98099999999999998,"15min":0.878},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28399999999999997},"max":{"1min":8.5429999999999993,"5min":8.5429999999999993,"15min":8.5429999999999993},"last":2.9540000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:53:20 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":0.94999999999999996,"15min":1.0669999999999999},"min":{"1min":0.33700000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.7590000000000003,"5min":7.7590000000000003,"15min":96.131},"last":2.8540000000000001},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.98599999999999999,"15min":1.159},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.0330000000000004,"5min":7.6580000000000004,"15min":96.465000000000003},"last":2.7629999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823083,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3574812,"kb_used_data":2672020,"kb_used_omap":0,"kb_used_meta":902784,"kb_avail":90170340,"statfs":{"total":95995035648,"available":92334428160,"internally_reserved":0,"allocated":2736148480,"data_stored":2735443751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924450816},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:53 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":7.593,"15min":3.9950000000000001},"min":{"1min":0.249,"5min":0.224,"15min":0.217},"max":{"1min":6.298,"5min":210.392,"15min":210.392},"last":120.405},{"interface":"front","average":{"1min":1.2549999999999999,"5min":7.9489999999999998,"15min":4.133},"min":{"1min":0.34699999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.2240000000000002,"5min":208.43600000000001,"15min":208.43600000000001},"last":120.441}]},{"osd":1,"last update":"Mon Oct 9 05:54:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":5.069,"15min":3.3599999999999999},"min":{"1min":0.25600000000000001,"5min":0.247,"15min":0.23999999999999999},"max":{"1min":3.456,"5min":198.35300000000001,"15min":198.35300000000001},"last":284.13900000000001},{"interface":"front","average":{"1min":1.3049999999999999,"5min":5.1779999999999999,"15min":3.496},"min":{"1min":0.30199999999999999,"5min":0.248,"15min":0.248},"max":{"1min":6.2430000000000003,"5min":198.32900000000001,"15min":198.32900000000001},"last":284.07900000000001}]},{"osd":2,"last update":"Mon Oct 9 05:53:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":4.1340000000000003,"15min":3.0750000000000002},"min":{"1min":0.308,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":11.965,"5min":195.92099999999999,"15min":195.92099999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.29,"5min":4.1699999999999999,"15min":3.004},"min":{"1min":0.38200000000000001,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":3.4319999999999999,"5min":195.905,"15min":195.905},"last":0.29999999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:54:15 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":4.0259999999999998,"15min":4.1849999999999996},"min":{"1min":0.33600000000000002,"5min":0.32500000000000001,"15min":0.30399999999999999},"max":{"1min":9.0269999999999992,"5min":85.069000000000003,"15min":95.236999999999995},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.986,"5min":4.0209999999999999,"15min":4.0549999999999997},"min":{"1min":0.35499999999999998,"5min":0.308,"15min":0.308},"max":{"1min":6.335,"5min":84.814999999999998,"15min":93.471000000000004},"last":0.36599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:54:15 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.72899999999999998,"15min":1.2290000000000001},"min":{"1min":0.31,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.343,"5min":5.8460000000000001,"15min":130.81},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.472,"5min":0.91600000000000004,"15min":1.367},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":9.3170000000000002,"5min":9.3170000000000002,"15min":133.803},"last":0.39700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:53:27 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.8120000000000001,"15min":1.1919999999999999},"min":{"1min":0.30599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.8399999999999999,"5min":71.426000000000002,"15min":71.426000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.67300000000000004,"5min":2.0510000000000002,"15min":1.2829999999999999},"min":{"1min":0.33700000000000002,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":1.6459999999999999,"5min":71.498999999999995,"15min":71.498999999999995},"last":0.378}]},{"osd":7,"last update":"Mon Oct 9 05:53:27 2023","interfaces":[{"interface":"back","average":{"1min":5.6230000000000002,"5min":2.3399999999999999,"15min":1.3740000000000001},"min":{"1min":0.35099999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":86.506,"5min":86.506,"15min":86.506},"last":0.44500000000000001},{"interface":"front","average":{"1min":5.9630000000000001,"5min":2.3769999999999998,"15min":1.3440000000000001},"min":{"1min":0.35199999999999998,"5min":0.316,"15min":0.308},"max":{"1min":86.924000000000007,"5min":86.924000000000007,"15min":86.924000000000007},"last":0.53200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1274019840,"data_stored":1274019840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2621440000,"data_stored":2621440000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":934805504,"data_stored":934805504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2466119680,"data_stored":2466119680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1136197632,"data_stored":1136197632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2734751744,"data_stored":2734751744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":838402048,"data_stored":838402048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1688010752,"data_stored":1688010752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:54:26.462 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:54:26.462 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 26 2023-10-09T05:54:26.571 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:26 smithi008 ceph-mon[184418]: pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 628 op/s; 20089/204165 objects degraded (9.840%); 1361/204165 objects misplaced (0.667%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T05:54:26.571 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:26 smithi008 ceph-mon[191227]: pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 628 op/s; 20089/204165 objects degraded (9.840%); 1361/204165 objects misplaced (0.667%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T05:54:26.573 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:26.709 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:26.861 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:26.869 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 70 16 71967 71951 64.2036 42 0.0309279 0.0155676 2023-10-09T05:54:26.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:26.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:26.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:27.007 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:27.070 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:27.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:27.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:27.178 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:27.210 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:27.275 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:27.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:27.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:27.385 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:27.452 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:27.607 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:27.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:27.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:27.711 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:27.723 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:27.793 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:27.869 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 71 16 72599 72583 63.8558 39.5 0.0392491 0.015653 2023-10-09T05:54:28.157 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:54:28.442 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:28.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[184418]: pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 20089/211119 objects degraded (9.515%); 1361/211119 objects misplaced (0.645%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T05:54:28.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1539674652' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20089/211119 objects degraded (9.515%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:54:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[184418]: osdmap e381: 8 total, 8 up, 7 in 2023-10-09T05:54:28.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[191227]: pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 20089/211119 objects degraded (9.515%); 1361/211119 objects misplaced (0.645%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T05:54:28.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1539674652' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:28.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:28.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20089/211119 objects degraded (9.515%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:28.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:54:28.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:28.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:28 smithi008 ceph-mon[191227]: osdmap e381: 8 total, 8 up, 7 in 2023-10-09T05:54:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:28 smithi191 ceph-mon[161669]: pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 20089/211119 objects degraded (9.515%); 1361/211119 objects misplaced (0.645%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T05:54:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1539674652' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:28 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20089/211119 objects degraded (9.515%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:28 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:54:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:28 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:28 smithi191 ceph-mon[161669]: osdmap e381: 8 total, 8 up, 7 in 2023-10-09T05:54:28.869 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 72 16 73020 73004 63.3346 26.3125 0.00481282 0.0156849 2023-10-09T05:54:28.970 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:28.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:28.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:29.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:29.100 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:29.150 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:29.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:29.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:29.256 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:29.273 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:29 smithi008 ceph-mon[191227]: failed to encode map e381 with expected crc 2023-10-09T05:54:29.273 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:29 smithi008 ceph-mon[191227]: failed to encode map e381 with expected crc 2023-10-09T05:54:29.273 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:29.273 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:29.273 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:29 smithi008 ceph-mon[191227]: osdmap e382: 8 total, 8 up, 7 in 2023-10-09T05:54:29.277 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:29 smithi008 ceph-mon[184418]: failed to encode map e381 with expected crc 2023-10-09T05:54:29.277 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:29 smithi008 ceph-mon[184418]: failed to encode map e381 with expected crc 2023-10-09T05:54:29.277 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:29.277 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:29.277 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:29 smithi008 ceph-mon[184418]: osdmap e382: 8 total, 8 up, 7 in 2023-10-09T05:54:29.321 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:29.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:29 smithi191 ceph-mon[161669]: failed to encode map e381 with expected crc 2023-10-09T05:54:29.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:29 smithi191 ceph-mon[161669]: failed to encode map e381 with expected crc 2023-10-09T05:54:29.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:29.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:29.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:29 smithi191 ceph-mon[161669]: osdmap e382: 8 total, 8 up, 7 in 2023-10-09T05:54:29.569 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:29.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:29.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:29.705 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:29.747 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:29.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:29.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:29.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:29.869 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 73 16 73271 73255 62.6822 15.6875 0.0145096 0.0158449 2023-10-09T05:54:29.878 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:29.920 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:29.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:30.025 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:30.058 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:30.225 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:30.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:30.344 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:30.454 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:30.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:30 smithi008 ceph-mon[184418]: pgmap v1162: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 681 op/s; 20089/214212 objects degraded (9.378%); 457/214212 objects misplaced (0.213%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:54:30.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:30 smithi008 ceph-mon[184418]: failed to encode map e382 with expected crc 2023-10-09T05:54:30.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:30 smithi008 ceph-mon[184418]: failed to encode map e382 with expected crc 2023-10-09T05:54:30.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:30 smithi008 ceph-mon[184418]: failed to encode map e382 with expected crc 2023-10-09T05:54:30.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:30 smithi008 ceph-mon[184418]: osdmap e383: 8 total, 8 up, 7 in 2023-10-09T05:54:30.517 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:54:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:54:30] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.33.4" 2023-10-09T05:54:30.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:30 smithi008 ceph-mon[191227]: pgmap v1162: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 681 op/s; 20089/214212 objects degraded (9.378%); 457/214212 objects misplaced (0.213%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:54:30.518 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:30 smithi008 ceph-mon[191227]: failed to encode map e382 with expected crc 2023-10-09T05:54:30.518 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:30 smithi008 ceph-mon[191227]: failed to encode map e382 with expected crc 2023-10-09T05:54:30.518 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:30 smithi008 ceph-mon[191227]: failed to encode map e382 with expected crc 2023-10-09T05:54:30.518 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:30 smithi008 ceph-mon[191227]: osdmap e383: 8 total, 8 up, 7 in 2023-10-09T05:54:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:30 smithi191 ceph-mon[161669]: pgmap v1162: 89 pgs: 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 681 op/s; 20089/214212 objects degraded (9.378%); 457/214212 objects misplaced (0.213%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:54:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:30 smithi191 ceph-mon[161669]: failed to encode map e382 with expected crc 2023-10-09T05:54:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:30 smithi191 ceph-mon[161669]: failed to encode map e382 with expected crc 2023-10-09T05:54:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:30 smithi191 ceph-mon[161669]: failed to encode map e382 with expected crc 2023-10-09T05:54:30.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:30 smithi191 ceph-mon[161669]: osdmap e383: 8 total, 8 up, 7 in 2023-10-09T05:54:30.871 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 74 16 73808 73792 62.288 33.5625 0.0139327 0.0160473 2023-10-09T05:54:31.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:31 smithi008 ceph-mon[184418]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:31 smithi008 ceph-mon[184418]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:31 smithi008 ceph-mon[184418]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:31 smithi008 ceph-mon[184418]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:31 smithi008 ceph-mon[191227]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:31 smithi008 ceph-mon[191227]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:31 smithi008 ceph-mon[191227]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:31 smithi008 ceph-mon[191227]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.721 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:31.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:31.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:31 smithi191 ceph-mon[161669]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:31 smithi191 ceph-mon[161669]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:31 smithi191 ceph-mon[161669]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:31 smithi191 ceph-mon[161669]: failed to encode map e383 with expected crc 2023-10-09T05:54:31.833 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:31.870 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 75 16 74401 74385 61.9518 37.0625 0.0232524 0.0161358 2023-10-09T05:54:31.928 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:32.163 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:32.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:32.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:32.276 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:32.311 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:32.387 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:32.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:32.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:32.496 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:32.559 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:32.640 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:32 smithi008 ceph-mon[191227]: pgmap v1165: 89 pgs: 3 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 14331/217365 objects degraded (6.593%); 5666/217365 objects misplaced (2.607%); 15 MiB/s, 242 objects/s recovering 2023-10-09T05:54:32.642 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:32 smithi008 ceph-mon[184418]: pgmap v1165: 89 pgs: 3 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 14331/217365 objects degraded (6.593%); 5666/217365 objects misplaced (2.607%); 15 MiB/s, 242 objects/s recovering 2023-10-09T05:54:32.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:32 smithi191 ceph-mon[161669]: pgmap v1165: 89 pgs: 3 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 14331/217365 objects degraded (6.593%); 5666/217365 objects misplaced (2.607%); 15 MiB/s, 242 objects/s recovering 2023-10-09T05:54:32.875 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 76 16 75162 75146 61.7625 47.5625 0.01416 0.0161845 2023-10-09T05:54:33.179 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:33.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:33.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:33.239 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:54:33.276 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:54:33.297 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:33.346 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:54:33.373 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:33 smithi008 ceph-mon[184418]: pgmap v1166: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 564 op/s; 14331/221289 objects degraded (6.476%); 5576/221289 objects misplaced (2.520%); 16 MiB/s, 257 objects/s recovering 2023-10-09T05:54:33.373 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14331/217365 objects degraded (6.593%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:33.373 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:33 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/620264097' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:33.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:33 smithi008 ceph-mon[191227]: pgmap v1166: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 564 op/s; 14331/221289 objects degraded (6.476%); 5576/221289 objects misplaced (2.520%); 16 MiB/s, 257 objects/s recovering 2023-10-09T05:54:33.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14331/217365 objects degraded (6.593%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:33.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:33 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/620264097' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:33.388 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:54:33.389 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:33.420 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:54:33.493 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:54:33.537 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:54:33.570 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:54:33.574 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:54:33.574 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:54:33.574 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:54:33.574 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:54:33.574 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:54:33.574 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:54:33.575 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:54:33.575 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:54:33.658 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:54:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:54:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:54:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:54:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150726 7.4 7:2606d17b:::benchmark_data_smithi008_354649_object56230:head [delete] snapc 0=[] ondisk+write+known_if_redirected e251)", 2023-10-09T05:54:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:16.967446+0000", 2023-10-09T05:54:33.658 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 556.66878630500003, 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.266506991, 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150726 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967446+0000", 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967446+0000", 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967445+0000", 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:33.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967448+0000", 2023-10-09T05:54:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.942e-06 2023-10-09T05:54:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967454+0000", 2023-10-09T05:54:33.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8379999999999998e-06 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.967460+0000", 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7620000000000001e-06 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.984917+0000", 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017457026 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.984930+0000", 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3217000000000001e-05 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.923010+0000", 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93808012799999996 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.923019+0000", 2023-10-09T05:54:33.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6230000000000003e-06 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.216263+0000", 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2932439330000001 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.216410+0000", 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000146861 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.216920+0000", 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00051030300000000004 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.224475+0000", 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0075551230000000004 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.224528+0000", 2023-10-09T05:54:33.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.295e-05 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.233016+0000", 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0084877560000000008 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.233080+0000", 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4058000000000002e-05 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.233953+0000", 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00087297799999999995 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150729 7.9 7:91e0d910:::benchmark_data_smithi008_354649_object56233:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:16.994744+0000", 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 556.64148841199994, 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.138625711, 2023-10-09T05:54:33.667 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150729 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994744+0000", 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994744+0000", 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994742+0000", 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994749+0000", 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.421e-06 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994762+0000", 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3101999999999999e-05 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994771+0000", 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0070000000000003e-06 2023-10-09T05:54:33.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994839+0000", 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7821e-05 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:16.994853+0000", 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4782e-05 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917318+0000", 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92246414300000001 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917330+0000", 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2471e-05 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.116457+0000", 2023-10-09T05:54:33.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.199127415 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.116616+0000", 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015812399999999999 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.117033+0000", 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00041699599999999999 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.123489+0000", 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064567210000000003 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.125273+0000", 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001783722 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.132228+0000", 2023-10-09T05:54:33.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0069553769999999996 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.132292+0000", 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3936999999999994e-05 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.133369+0000", 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001077103 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.675 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150734 7.14 7:2e650744:::benchmark_data_smithi008_354649_object56238:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.025678+0000", 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 556.610553998, 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.200712695, 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:54:33.676 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150734 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025678+0000", 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025678+0000", 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025676+0000", 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025686+0000", 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2490000000000007e-06 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025700+0000", 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4002000000000001e-05 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025708+0000", 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4409999999999993e-06 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025787+0000", 2023-10-09T05:54:33.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8909e-05 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.025804+0000", 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6718000000000002e-05 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922949+0000", 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89714522100000005 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.922958+0000", 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3349999999999996e-06 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.210519+0000", 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2875604629999999 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.210670+0000", 2023-10-09T05:54:33.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015123 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.210978+0000", 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00030860600000000001 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.217348+0000", 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0063698720000000004 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.217358+0000", 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3179999999999996e-06 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.225201+0000", 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078434549999999992 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.225268+0000", 2023-10-09T05:54:33.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7101999999999996e-05 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.226391+0000", 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001122556 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:150735 7.9 7:92c6da78:::benchmark_data_smithi008_354649_object56239:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-09T05:54:33.684 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:45:17.026350+0000", 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 556.60988218399996, 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1077192149999999, 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 150735 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026350+0000", 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026350+0000", 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026349+0000", 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:54:33.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026354+0000", 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4159999999999997e-06 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026363+0000", 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4760000000000004e-06 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026368+0000", 2023-10-09T05:54:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8380000000000001e-06 2023-10-09T05:54:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026430+0000", 2023-10-09T05:54:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2142000000000002e-05 2023-10-09T05:54:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.026444+0000", 2023-10-09T05:54:33.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4007e-05 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917360+0000", 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.89091560000000003 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:17.917368+0000", 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4489999999999999e-06 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.121241+0000", 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.203872729 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.121389+0000", 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014778299999999999 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:54:33.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.121817+0000", 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00042842200000000003 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.125281+0000", 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034634980000000002 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.126920+0000", 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016398389999999999 2023-10-09T05:54:33.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.133435+0000", 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0065145170000000001 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.133481+0000", 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6368000000000001e-05 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:45:19.134069+0000", 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00058786500000000002 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180271 7.1d 7:baeacd59:::benchmark_data_smithi008_354649_object85775:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 472.50671977500002, 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.855218754, 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.693 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180271 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129509+0000", 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129525+0000", 2023-10-09T05:54:33.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5430999999999998e-05 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129553+0000", 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8175e-05 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129568+0000", 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5267000000000001e-05 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.333120+0000", 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20355221500000001 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.333134+0000", 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4073000000000001e-05 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929537+0000", 2023-10-09T05:54:33.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5964021320000001 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929548+0000", 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1759999999999999e-05 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.974829+0000", 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.045281066 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.974986+0000", 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000156174 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.975339+0000", 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00035308699999999998 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977566+0000", 2023-10-09T05:54:33.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022271399999999998 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984241+0000", 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066755160000000003 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984422+0000", 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000180633 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984530+0000", 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010849999999999999 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984609+0000", 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8506999999999999e-05 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984731+0000", 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012211599999999999 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180280 7.1d 7:b9a77225:::benchmark_data_smithi008_354649_object85784:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 472.27437206500002, 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6230030800000002, 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.702 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180280 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361859+0000", 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361865+0000", 2023-10-09T05:54:33.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9800000000000003e-06 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361876+0000", 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1178999999999999e-05 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361882+0000", 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0070000000000003e-06 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361924+0000", 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2243999999999997e-05 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361931+0000", 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.097e-06 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929573+0000", 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.567641397 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929580+0000", 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4569999999999999e-06 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977745+0000", 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0481653790000001 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977840+0000", 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4581999999999997e-05 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.978096+0000", 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00025544499999999998 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984291+0000", 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061956720000000002 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984301+0000", 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7710000000000001e-06 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984457+0000", 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000155885 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984780+0000", 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00032328700000000001 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984806+0000", 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6290000000000001e-05 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984863+0000", 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6693000000000001e-05 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180292 7.1f 7:fc8bce44:::benchmark_data_smithi008_354649_object85796:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 472.10874611000003, 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4946860599999998, 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.711 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:54:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:54:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180292 2023-10-09T05:54:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527485+0000", 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527491+0000", 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.139e-06 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527504+0000", 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2964e-05 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527512+0000", 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2230000000000007e-06 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527553+0000", 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1266000000000001e-05 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527560+0000", 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6599999999999998e-06 2023-10-09T05:54:33.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929860+0000", 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4023002959999999 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929869+0000", 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9080000000000001e-06 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986248+0000", 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0563782779999999 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986407+0000", 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000159588 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986738+0000", 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00033078600000000001 2023-10-09T05:54:33.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.015271+0000", 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028533060999999998 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.015291+0000", 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0077000000000001e-05 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.021759+0000", 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064680570000000001 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.021808+0000", 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8615000000000001e-05 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.022172+0000", 2023-10-09T05:54:33.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00036433100000000002 2023-10-09T05:54:33.720 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.720 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.720 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.720 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45725 9.9 9:9b3d785e:::benchmark_data_smithi008_502820_object45724:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T05:54:33.720 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:45.884511+0000", 2023-10-09T05:54:33.720 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.751720673000001, 2023-10-09T05:54:33.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2629352659999999, 2023-10-09T05:54:33.720 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.720 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45725 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884511+0000", 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884511+0000", 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884510+0000", 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884551+0000", 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0584000000000003e-05 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884562+0000", 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1015999999999999e-05 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884569+0000", 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7160000000000003e-06 2023-10-09T05:54:33.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884627+0000", 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7799e-05 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884762+0000", 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000134609 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884865+0000", 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000103564 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.062004+0000", 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.177138507 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.062019+0000", 2023-10-09T05:54:33.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5153e-05 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933035+0000", 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.87101594900000001 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933052+0000", 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7210999999999998e-05 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.147320+0000", 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2142682119999999 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.147447+0000", 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000126291 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45727 9.5 9:af6b034d:::benchmark_data_smithi008_502820_object45726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:45.884586+0000", 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.751646082999997, 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.280584508, 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45727 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884586+0000", 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884586+0000", 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884585+0000", 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884616+0000", 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0423e-05 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884625+0000", 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0250000000000008e-06 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884631+0000", 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3840000000000002e-06 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884669+0000", 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7698999999999999e-05 2023-10-09T05:54:33.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884795+0000", 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012634500000000001 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884920+0000", 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012516900000000001 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.061847+0000", 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17692612699999999 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.061860+0000", 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3361e-05 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933452+0000", 2023-10-09T05:54:33.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.87159255700000005 2023-10-09T05:54:33.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933460+0000", 2023-10-09T05:54:33.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5549999999999997e-06 2023-10-09T05:54:33.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.120071+0000", 2023-10-09T05:54:33.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1866107539999999 2023-10-09T05:54:33.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.120662+0000", 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00059118300000000001 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165062+0000", 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.044399986000000002 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165171+0000", 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010856700000000001 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45730 9.5 9:a76420fe:::benchmark_data_smithi008_502820_object45729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:45.884645+0000", 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.751586644, 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2944595059999999, 2023-10-09T05:54:33.736 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.737 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:54:33.737 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.737 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.737 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.737 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45730 2023-10-09T05:54:33.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.737 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884645+0000", 2023-10-09T05:54:33.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884645+0000", 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884645+0000", 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884669+0000", 2023-10-09T05:54:33.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4326000000000001e-05 2023-10-09T05:54:33.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.741 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.741 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884678+0000", 2023-10-09T05:54:33.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8829999999999995e-06 2023-10-09T05:54:33.741 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.742 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.742 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884684+0000", 2023-10-09T05:54:33.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.806e-06 2023-10-09T05:54:33.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.742 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.742 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.742 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055897+0000", 2023-10-09T05:54:33.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17121303700000001 2023-10-09T05:54:33.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.742 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.742 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.056018+0000", 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000120831 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.056156+0000", 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013870000000000001 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066733+0000", 2023-10-09T05:54:33.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010576189999999999 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066739+0000", 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4500000000000001e-06 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933477+0000", 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86673776000000002 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933485+0000", 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2779999999999993e-06 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165241+0000", 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.231755605 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:54:33.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165788+0000", 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00054761800000000004 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.178974+0000", 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013186164 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.179105+0000", 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000130462 2023-10-09T05:54:33.746 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45731 9.9 9:9d7980f6:::benchmark_data_smithi008_502820_object45730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:45.884828+0000", 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.751404543, 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.325062725, 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.747 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45731 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884828+0000", 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884828+0000", 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884827+0000", 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884851+0000", 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4647999999999998e-05 2023-10-09T05:54:33.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.750 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.750 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.750 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884863+0000", 2023-10-09T05:54:33.750 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1389000000000001e-05 2023-10-09T05:54:33.750 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.750 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.750 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.750 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884870+0000", 2023-10-09T05:54:33.750 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1250000000000004e-06 2023-10-09T05:54:33.750 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.750 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055752+0000", 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17088223499999999 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055870+0000", 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011758500000000001 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:54:33.751 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.056006+0000", 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000136243 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064121+0000", 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081152840000000004 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064134+0000", 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2826999999999999e-05 2023-10-09T05:54:33.752 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933097+0000", 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86896320299999996 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933109+0000", 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1987999999999999e-05 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.184026+0000", 2023-10-09T05:54:33.754 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2509170430000001 2023-10-09T05:54:33.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.754 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:54:33.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.184835+0000", 2023-10-09T05:54:33.754 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00080872799999999999 2023-10-09T05:54:33.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.754 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.209763+0000", 2023-10-09T05:54:33.755 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024928049000000001 2023-10-09T05:54:33.755 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.755 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.755 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.755 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.209890+0000", 2023-10-09T05:54:33.755 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000127267 2023-10-09T05:54:33.755 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.755 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.755 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.755 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.755 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45736 9.9 9:9ccca68b:::benchmark_data_smithi008_502820_object45735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:46.054780+0000", 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.581451811000001, 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1574720140000001, 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45736 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.756 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054780+0000", 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054780+0000", 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.757 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054780+0000", 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054803+0000", 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3753000000000001e-05 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054827+0000", 2023-10-09T05:54:33.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3315e-05 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054834+0000", 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7190000000000004e-06 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064178+0000", 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0093441159999999995 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.759 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064190+0000", 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2302e-05 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933141+0000", 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86895131199999998 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.760 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933151+0000", 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4089999999999992e-06 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.184895+0000", 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2517445010000001 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.185025+0000", 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012967100000000001 2023-10-09T05:54:33.761 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.762 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:54:33.762 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.185202+0000", 2023-10-09T05:54:33.762 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017737699999999999 2023-10-09T05:54:33.762 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.762 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.762 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.195793+0000", 2023-10-09T05:54:33.762 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010591088 2023-10-09T05:54:33.762 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.204622+0000", 2023-10-09T05:54:33.763 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0088287469999999996 2023-10-09T05:54:33.763 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.763 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.209933+0000", 2023-10-09T05:54:33.763 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0053108310000000002 2023-10-09T05:54:33.763 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.763 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.209987+0000", 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3817000000000003e-05 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.212252+0000", 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022656040000000001 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.764 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45740 9.9 9:9b88b00f:::benchmark_data_smithi008_502820_object45739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:46.054841+0000", 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.581391166000003, 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1674305079999998, 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45740 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.765 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054841+0000", 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054841+0000", 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.766 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.767 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054840+0000", 2023-10-09T05:54:33.767 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:33.767 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.767 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.767 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.767 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054886+0000", 2023-10-09T05:54:33.767 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5235999999999999e-05 2023-10-09T05:54:33.767 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.767 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.767 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.767 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054895+0000", 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8400000000000007e-06 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.054902+0000", 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2310000000000001e-06 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064227+0000", 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0093249939999999996 2023-10-09T05:54:33.768 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064236+0000", 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2520000000000005e-06 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933188+0000", 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.868951735 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.769 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933197+0000", 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4749999999999999e-06 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.186877+0000", 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2536796479999999 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.187045+0000", 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016778600000000001 2023-10-09T05:54:33.770 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.187242+0000", 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00019769500000000001 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.195816+0000", 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0085738629999999993 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.771 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.204632+0000", 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0088162469999999993 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.212313+0000", 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0076801780000000002 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.212372+0000", 2023-10-09T05:54:33.772 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9122000000000001e-05 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.222271+0000", 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0098997800000000004 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45760 9.5 9:a9d70d63:::benchmark_data_smithi008_502820_object45759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:46.055094+0000", 2023-10-09T05:54:33.773 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.581138033000002, 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.125521241, 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45760 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.774 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055094+0000", 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055094+0000", 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055094+0000", 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:33.775 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055118+0000", 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4046e-05 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055156+0000", 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8473999999999998e-05 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.776 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055174+0000", 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7595e-05 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066754+0000", 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011579924 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066757+0000", 2023-10-09T05:54:33.777 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8419999999999998e-06 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933506+0000", 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86674832700000004 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933513+0000", 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4699999999999996e-06 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.778 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.169217+0000", 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2357041529999999 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.169409+0000", 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00019163399999999999 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:54:33.779 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.169612+0000", 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020308899999999999 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.176407+0000", 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0067946589999999998 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.179175+0000", 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027682760000000001 2023-10-09T05:54:33.780 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.179245+0000", 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0186999999999997e-05 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.179303+0000", 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8066000000000003e-05 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.781 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.782 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.180615+0000", 2023-10-09T05:54:33.782 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013119710000000001 2023-10-09T05:54:33.782 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.782 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.782 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.782 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.782 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.782 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45769 9.5 9:a3876d49:::benchmark_data_smithi008_502820_object45768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e372)", 2023-10-09T05:54:33.782 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:46.055181+0000", 2023-10-09T05:54:33.782 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.581050744999999, 2023-10-09T05:54:33.782 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1268914400000001, 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45769 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055181+0000", 2023-10-09T05:54:33.783 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055181+0000", 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055181+0000", 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.784 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055208+0000", 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7634e-05 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055215+0000", 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9929999999999998e-06 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.785 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055221+0000", 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6300000000000003e-06 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066774+0000", 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011552461999999999 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066781+0000", 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5290000000000003e-06 2023-10-09T05:54:33.786 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933533+0000", 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86675161499999998 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933541+0000", 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2660000000000001e-06 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.787 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.171660+0000", 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2381189240000001 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.171834+0000", 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000174606 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.172016+0000", 2023-10-09T05:54:33.788 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018137900000000001 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.178910+0000", 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068946220000000004 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.180671+0000", 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017604439999999999 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.789 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.180736+0000", 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4945999999999998e-05 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.180801+0000", 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4859999999999999e-05 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.790 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.182073+0000", 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001272079 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45771 9.9 9:9b423c57:::benchmark_data_smithi008_502820_object45770:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e372)", 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:46.059297+0000", 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.576935220000003, 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1653252859999998, 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.791 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45771 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.059297+0000", 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.792 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.059297+0000", 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.059296+0000", 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.793 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.059319+0000", 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3708000000000001e-05 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.059328+0000", 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5299999999999996e-06 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.059335+0000", 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6309999999999997e-06 2023-10-09T05:54:33.794 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.795 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.795 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.795 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064476+0000", 2023-10-09T05:54:33.795 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051417600000000004 2023-10-09T05:54:33.795 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.795 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.795 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.795 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064488+0000", 2023-10-09T05:54:33.795 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1712000000000001e-05 2023-10-09T05:54:33.795 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.795 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933244+0000", 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86875577199999998 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933254+0000", 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0455000000000001e-05 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.796 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.188944+0000", 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2556891889999999 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.189109+0000", 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016521999999999999 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.189307+0000", 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00019821199999999999 2023-10-09T05:54:33.797 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.195848+0000", 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0065405960000000001 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.204641+0000", 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0087938840000000001 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.798 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.222334+0000", 2023-10-09T05:54:33.799 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017692304999999998 2023-10-09T05:54:33.799 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.799 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.222366+0000", 2023-10-09T05:54:33.799 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2036000000000003e-05 2023-10-09T05:54:33.799 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.799 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.799 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.799 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.224622+0000", 2023-10-09T05:54:33.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002256395 2023-10-09T05:54:33.800 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.800 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.800 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.800 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.800 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.800 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45772 9.5 9:a07c9773:::benchmark_data_smithi008_502820_object45771:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e372)", 2023-10-09T05:54:33.800 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:46.065806+0000", 2023-10-09T05:54:33.800 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.570426165999997, 2023-10-09T05:54:33.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1281058470000001, 2023-10-09T05:54:33.800 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45772 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.065806+0000", 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.801 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.802 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.065806+0000", 2023-10-09T05:54:33.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.802 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.802 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.065805+0000", 2023-10-09T05:54:33.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:54:33.802 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.802 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.065829+0000", 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4182e-05 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.065838+0000", 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1959999999999997e-06 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.803 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.065846+0000", 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9589999999999995e-06 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066803+0000", 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095733999999999999 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066811+0000", 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8560000000000007e-06 2023-10-09T05:54:33.804 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933562+0000", 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86675103499999995 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933571+0000", 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4440000000000008e-06 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.805 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.806 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.182116+0000", 2023-10-09T05:54:33.806 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.24854548 2023-10-09T05:54:33.806 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.806 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.806 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.806 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.182299+0000", 2023-10-09T05:54:33.806 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018272899999999999 2023-10-09T05:54:33.806 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.806 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.806 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:54:33.806 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.182470+0000", 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017088199999999999 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.187991+0000", 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0055209689999999997 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.189844+0000", 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018528139999999999 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.807 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.808 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.808 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.191472+0000", 2023-10-09T05:54:33.808 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001628055 2023-10-09T05:54:33.808 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.808 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.808 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.808 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.191527+0000", 2023-10-09T05:54:33.808 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5310999999999999e-05 2023-10-09T05:54:33.808 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.808 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.808 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.193912+0000", 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023847679999999998 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45773 9.5 9:aa925fb8:::benchmark_data_smithi008_502820_object45772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e372)", 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:46.066379+0000", 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.569853514999998, 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.129831019, 2023-10-09T05:54:33.809 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45773 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066379+0000", 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.810 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.811 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.811 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.811 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066379+0000", 2023-10-09T05:54:33.811 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.811 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.811 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.811 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.811 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066378+0000", 2023-10-09T05:54:33.811 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:54:33.811 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.811 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066400+0000", 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2605000000000001e-05 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066412+0000", 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1925e-05 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.812 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066420+0000", 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1189999999999997e-06 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066834+0000", 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00041378500000000001 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066842+0000", 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4139999999999997e-06 2023-10-09T05:54:33.813 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933592+0000", 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.866749993 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933599+0000", 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8709999999999997e-06 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.814 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.184508+0000", 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2509081559999999 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.184680+0000", 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017269299999999999 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.184897+0000", 2023-10-09T05:54:33.815 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00021652499999999999 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.187998+0000", 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003101155 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.190234+0000", 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022358830000000001 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.816 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.193993+0000", 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037591669999999999 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.194064+0000", 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0842999999999999e-05 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.817 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.196210+0000", 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021456859999999999 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45774 9.9 9:9735d06c:::benchmark_data_smithi008_502820_object45773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e372)", 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:46.066428+0000", 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.569804185999999, 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.160895724, 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.818 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.819 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.819 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.819 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.819 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45774 2023-10-09T05:54:33.819 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.819 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.819 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.819 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.819 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066428+0000", 2023-10-09T05:54:33.819 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.819 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066428+0000", 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066427+0000", 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.820 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066450+0000", 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2334999999999999e-05 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066457+0000", 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1890000000000001e-06 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066463+0000", 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8459999999999996e-06 2023-10-09T05:54:33.821 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.822 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.822 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.822 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066515+0000", 2023-10-09T05:54:33.822 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1854000000000002e-05 2023-10-09T05:54:33.822 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.822 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.822 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.822 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066525+0000", 2023-10-09T05:54:33.822 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0837e-05 2023-10-09T05:54:33.822 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.822 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933291+0000", 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86676584300000004 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933301+0000", 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4369999999999996e-06 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.823 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.191093+0000", 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.257792051 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.191309+0000", 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00021658800000000001 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.191542+0000", 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00023214000000000001 2023-10-09T05:54:33.824 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.195870+0000", 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0043288069999999996 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.204764+0000", 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0088936039999999994 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.825 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.225534+0000", 2023-10-09T05:54:33.826 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020769777999999999 2023-10-09T05:54:33.826 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.826 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.826 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.826 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.225575+0000", 2023-10-09T05:54:33.826 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1115999999999997e-05 2023-10-09T05:54:33.826 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.826 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.826 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.826 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.227324+0000", 2023-10-09T05:54:33.826 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017488060000000001 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45777 9.9 9:9f15e114:::benchmark_data_smithi008_502820_object45776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e372)", 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:46.070034+0000", 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 47.566197635999998, 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.158829356, 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45777 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.070034+0000", 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:54:33.828 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.070034+0000", 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.070033+0000", 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.070058+0000", 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5089999999999999e-05 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.829 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.070074+0000", 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5880999999999999e-05 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.070081+0000", 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3860000000000004e-06 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.830 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.070139+0000", 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8369000000000003e-05 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.070147+0000", 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8339999999999999e-06 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933331+0000", 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.863184587 2023-10-09T05:54:33.831 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933341+0000", 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0127e-05 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.195885+0000", 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2625438410000001 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.832 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.196019+0000", 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000133869 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.196178+0000", 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015871599999999999 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.204706+0000", 2023-10-09T05:54:33.833 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0085282190000000001 2023-10-09T05:54:33.834 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.834 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.834 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:54:33.834 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.204770+0000", 2023-10-09T05:54:33.834 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3885000000000002e-05 2023-10-09T05:54:33.834 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.834 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.834 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:33.834 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.227371+0000", 2023-10-09T05:54:33.834 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022601317999999999 2023-10-09T05:54:33.834 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.227406+0000", 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.519e-05 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.228864+0000", 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001457348 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.835 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.836 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:54:33.836 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:54:33.836 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:54:33.836 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:33 smithi191 ceph-mon[161669]: pgmap v1166: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 564 op/s; 14331/221289 objects degraded (6.476%); 5576/221289 objects misplaced (2.520%); 16 MiB/s, 257 objects/s recovering 2023-10-09T05:54:33.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14331/217365 objects degraded (6.593%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:33.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:33 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/620264097' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:33.837 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:33.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:33.877 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 77 16 75831 75815 61.4981 41.8125 0.0243216 0.0162543 2023-10-09T05:54:33.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:33.970 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:33.986 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:54:34.041 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:34.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:34.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:34.152 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:34.253 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:34.574 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:34.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:34.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:34.680 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:34.771 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:34.877 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 78 16 76568 76552 61.3002 46.0625 0.0163294 0.0163071 2023-10-09T05:54:35.610 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:35.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:35.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:35.692 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 9 pg_num to 26 2023-10-09T05:54:35.721 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:35.756 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:35.814 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:35.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:35.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:35.826 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:54:35.879 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 79 16 77181 77165 61.0083 38.3125 0.0343527 0.016383 2023-10-09T05:54:35.916 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:35 smithi008 ceph-mon[184418]: pgmap v1167: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 23 MiB/s wr, 366 op/s; 14331/221289 objects degraded (6.476%); 5576/221289 objects misplaced (2.520%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:54:35.916 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1568687406' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:54:35.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:35 smithi008 ceph-mon[191227]: pgmap v1167: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 23 MiB/s wr, 366 op/s; 14331/221289 objects degraded (6.476%); 5576/221289 objects misplaced (2.520%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:54:35.917 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1568687406' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:54:35.921 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:35.960 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:35 smithi191 ceph-mon[161669]: pgmap v1167: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 23 MiB/s wr, 366 op/s; 14331/221289 objects degraded (6.476%); 5576/221289 objects misplaced (2.520%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:54:35.960 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1568687406' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:54:35.962 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:36.021 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:36.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:36.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:36.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:36.159 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:36.197 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:36.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:36.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:36.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:36.330 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:36.382 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:36.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:36.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:36.491 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:36.530 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:36.589 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:36.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:36.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:36.699 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:36.776 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:36.880 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T05:54:36.880911+0000 min lat: 0.00115306 max lat: 2.32752 avg lat: 0.0164626 2023-10-09T05:54:36.880 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:54:36.880 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 80 16 77622 77606 60.5897 27.5625 0.20088 0.0164626 2023-10-09T05:54:36.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[184418]: failed to encode map e384 with expected crc 2023-10-09T05:54:36.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1568687406' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:54:36.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[184418]: osdmap e384: 8 total, 8 up, 7 in 2023-10-09T05:54:36.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[184418]: failed to encode map e384 with expected crc 2023-10-09T05:54:36.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[184418]: failed to encode map e384 with expected crc 2023-10-09T05:54:36.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[184418]: failed to encode map e384 with expected crc 2023-10-09T05:54:36.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[184418]: failed to encode map e384 with expected crc 2023-10-09T05:54:36.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:54:36.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[191227]: failed to encode map e384 with expected crc 2023-10-09T05:54:36.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1568687406' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:54:36.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[191227]: osdmap e384: 8 total, 8 up, 7 in 2023-10-09T05:54:36.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[191227]: failed to encode map e384 with expected crc 2023-10-09T05:54:36.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[191227]: failed to encode map e384 with expected crc 2023-10-09T05:54:36.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[191227]: failed to encode map e384 with expected crc 2023-10-09T05:54:36.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[191227]: failed to encode map e384 with expected crc 2023-10-09T05:54:36.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:36 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:54:37.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:36 smithi191 ceph-mon[161669]: failed to encode map e384 with expected crc 2023-10-09T05:54:37.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1568687406' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:54:37.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:36 smithi191 ceph-mon[161669]: osdmap e384: 8 total, 8 up, 7 in 2023-10-09T05:54:37.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:36 smithi191 ceph-mon[161669]: failed to encode map e384 with expected crc 2023-10-09T05:54:37.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:36 smithi191 ceph-mon[161669]: failed to encode map e384 with expected crc 2023-10-09T05:54:37.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:36 smithi191 ceph-mon[161669]: failed to encode map e384 with expected crc 2023-10-09T05:54:37.027 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:36 smithi191 ceph-mon[161669]: failed to encode map e384 with expected crc 2023-10-09T05:54:37.027 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:36 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:54:37.027 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:37.028 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:37.113 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:37.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:37.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:37.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:37.245 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:37.288 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:37.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:37.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:37.394 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:37.426 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:37.751 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:54:37.802 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[191227]: pgmap v1169: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 685 op/s; 14331/229476 objects degraded (6.245%); 5325/229476 objects misplaced (2.321%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T05:54:37.803 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[191227]: failed to encode map e385 with expected crc 2023-10-09T05:54:37.803 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:54:37.803 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[191227]: osdmap e385: 8 total, 8 up, 7 in 2023-10-09T05:54:37.803 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[191227]: failed to encode map e385 with expected crc 2023-10-09T05:54:37.803 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[191227]: failed to encode map e385 with expected crc 2023-10-09T05:54:37.803 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[184418]: pgmap v1169: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 685 op/s; 14331/229476 objects degraded (6.245%); 5325/229476 objects misplaced (2.321%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T05:54:37.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[184418]: failed to encode map e385 with expected crc 2023-10-09T05:54:37.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:54:37.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[184418]: osdmap e385: 8 total, 8 up, 7 in 2023-10-09T05:54:37.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[184418]: failed to encode map e385 with expected crc 2023-10-09T05:54:37.804 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:37 smithi008 ceph-mon[184418]: failed to encode map e385 with expected crc 2023-10-09T05:54:37.825 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:37.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:37.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:37.880 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 81 16 77731 77715 59.9261 6.8125 0.00786854 0.0166005 2023-10-09T05:54:37.946 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:37.982 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:38.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:37 smithi191 ceph-mon[161669]: pgmap v1169: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 685 op/s; 14331/229476 objects degraded (6.245%); 5325/229476 objects misplaced (2.321%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T05:54:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:37 smithi191 ceph-mon[161669]: failed to encode map e385 with expected crc 2023-10-09T05:54:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:37 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:54:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:37 smithi191 ceph-mon[161669]: osdmap e385: 8 total, 8 up, 7 in 2023-10-09T05:54:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:37 smithi191 ceph-mon[161669]: failed to encode map e385 with expected crc 2023-10-09T05:54:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:37 smithi191 ceph-mon[161669]: failed to encode map e385 with expected crc 2023-10-09T05:54:38.880 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 82 16 78240 78224 59.5834 31.8125 0.00240727 0.0166756 2023-10-09T05:54:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: osdmap e386: 8 total, 8 up, 7 in 2023-10-09T05:54:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14331/229476 objects degraded (6.245%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: failed to encode map e387 with expected crc 2023-10-09T05:54:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: osdmap e387: 8 total, 8 up, 7 in 2023-10-09T05:54:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: failed to encode map e387 with expected crc 2023-10-09T05:54:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: failed to encode map e387 with expected crc 2023-10-09T05:54:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: failed to encode map e387 with expected crc 2023-10-09T05:54:38.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[184418]: failed to encode map e387 with expected crc 2023-10-09T05:54:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: osdmap e386: 8 total, 8 up, 7 in 2023-10-09T05:54:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: failed to encode map e386 with expected crc 2023-10-09T05:54:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14331/229476 objects degraded (6.245%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:38.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: failed to encode map e387 with expected crc 2023-10-09T05:54:38.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: osdmap e387: 8 total, 8 up, 7 in 2023-10-09T05:54:38.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: failed to encode map e387 with expected crc 2023-10-09T05:54:38.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: failed to encode map e387 with expected crc 2023-10-09T05:54:38.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: failed to encode map e387 with expected crc 2023-10-09T05:54:38.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:38 smithi008 ceph-mon[191227]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: failed to encode map e386 with expected crc 2023-10-09T05:54:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: failed to encode map e386 with expected crc 2023-10-09T05:54:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: failed to encode map e386 with expected crc 2023-10-09T05:54:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: osdmap e386: 8 total, 8 up, 7 in 2023-10-09T05:54:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: failed to encode map e386 with expected crc 2023-10-09T05:54:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: failed to encode map e386 with expected crc 2023-10-09T05:54:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: failed to encode map e386 with expected crc 2023-10-09T05:54:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14331/229476 objects degraded (6.245%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: osdmap e387: 8 total, 8 up, 7 in 2023-10-09T05:54:39.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:38 smithi191 ceph-mon[161669]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.358 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:39.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:39.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:39.469 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:39.503 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:39.559 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:39.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:39.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:39.671 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:39.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[184418]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[184418]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[184418]: 9.0 scrub starts 2023-10-09T05:54:39.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[184418]: pgmap v1173: 99 pgs: 3 peering, 10 unknown, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 14331/231090 objects degraded (6.201%); 5325/231090 objects misplaced (2.304%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:54:39.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[184418]: failed to encode map e388 with expected crc 2023-10-09T05:54:39.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[184418]: osdmap e388: 8 total, 8 up, 7 in 2023-10-09T05:54:39.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[184418]: failed to encode map e388 with expected crc 2023-10-09T05:54:39.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[184418]: failed to encode map e388 with expected crc 2023-10-09T05:54:39.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[191227]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[191227]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[191227]: 9.0 scrub starts 2023-10-09T05:54:39.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[191227]: pgmap v1173: 99 pgs: 3 peering, 10 unknown, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 14331/231090 objects degraded (6.201%); 5325/231090 objects misplaced (2.304%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:54:39.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[191227]: failed to encode map e388 with expected crc 2023-10-09T05:54:39.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[191227]: osdmap e388: 8 total, 8 up, 7 in 2023-10-09T05:54:39.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[191227]: failed to encode map e388 with expected crc 2023-10-09T05:54:39.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:39 smithi008 ceph-mon[191227]: failed to encode map e388 with expected crc 2023-10-09T05:54:39.718 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:39.782 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:39.793 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:39 smithi191 ceph-mon[161669]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.793 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:39 smithi191 ceph-mon[161669]: failed to encode map e387 with expected crc 2023-10-09T05:54:39.794 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:39 smithi191 ceph-mon[161669]: 9.0 scrub starts 2023-10-09T05:54:39.794 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:39 smithi191 ceph-mon[161669]: pgmap v1173: 99 pgs: 3 peering, 10 unknown, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 14331/231090 objects degraded (6.201%); 5325/231090 objects misplaced (2.304%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:54:39.794 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:39 smithi191 ceph-mon[161669]: failed to encode map e388 with expected crc 2023-10-09T05:54:39.794 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:39 smithi191 ceph-mon[161669]: osdmap e388: 8 total, 8 up, 7 in 2023-10-09T05:54:39.794 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:39 smithi191 ceph-mon[161669]: failed to encode map e388 with expected crc 2023-10-09T05:54:39.795 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:39 smithi191 ceph-mon[161669]: failed to encode map e388 with expected crc 2023-10-09T05:54:39.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:39.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:39.880 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 83 16 78904 78888 59.3656 41.5 0.00658051 0.0168386 2023-10-09T05:54:39.897 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:39.942 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:39.965 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:40.003 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:40.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:40.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:40.109 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:40.142 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:40.195 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:40.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:40.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:40.230 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:54:40.231 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:54:40.231 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T05:54:40.231 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:54:40.299 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:40.331 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:40.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:40 smithi008 ceph-mon[191227]: failed to encode map e388 with expected crc 2023-10-09T05:54:40.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:40 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2989345380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:40.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:40 smithi008 ceph-mon[184418]: failed to encode map e388 with expected crc 2023-10-09T05:54:40.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:40 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2989345380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:40.707 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:54:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:54:40] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.33.4" 2023-10-09T05:54:40.724 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:54:40.880 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 84 16 79422 79406 59.0444 32.375 0.0158024 0.0169307 2023-10-09T05:54:41.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:40 smithi191 ceph-mon[161669]: failed to encode map e388 with expected crc 2023-10-09T05:54:41.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:40 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2989345380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:41.352 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:41.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:41.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:41.480 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:54:41.491 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:41.553 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:41.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:41.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:41.659 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:41.711 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:41.773 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:41.778 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:41 smithi191 ceph-mon[161669]: 9.2 scrub starts 2023-10-09T05:54:41.778 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:41 smithi191 ceph-mon[161669]: pgmap v1175: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 peering, 6 unknown, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 24 MiB/s wr, 386 op/s; 16857/234114 objects degraded (7.200%); 2662/234114 objects misplaced (1.137%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:54:41.779 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:41 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:41.779 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:41 smithi191 ceph-mon[161669]: failed to encode map e389 with expected crc 2023-10-09T05:54:41.779 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:41 smithi191 ceph-mon[161669]: failed to encode map e389 with expected crc 2023-10-09T05:54:41.779 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:41 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2989345380' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:54:41.779 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:41 smithi191 ceph-mon[161669]: osdmap e389: 8 total, 8 up, 7 in 2023-10-09T05:54:41.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:41.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:41.835 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:54:41.836 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:54:41.836 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T05:54:41.836 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:54:41.836 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:54:41.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T05:54:41.881 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 85 16 80426 80410 59.0879 62.75 0.00406345 0.0169162 2023-10-09T05:54:41.881 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:41.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[184418]: 9.2 scrub starts 2023-10-09T05:54:41.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[184418]: pgmap v1175: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 peering, 6 unknown, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 24 MiB/s wr, 386 op/s; 16857/234114 objects degraded (7.200%); 2662/234114 objects misplaced (1.137%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:54:41.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:41.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[184418]: failed to encode map e389 with expected crc 2023-10-09T05:54:41.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[184418]: failed to encode map e389 with expected crc 2023-10-09T05:54:41.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2989345380' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:54:41.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[184418]: osdmap e389: 8 total, 8 up, 7 in 2023-10-09T05:54:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[191227]: 9.2 scrub starts 2023-10-09T05:54:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[191227]: pgmap v1175: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 peering, 6 unknown, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 24 MiB/s wr, 386 op/s; 16857/234114 objects degraded (7.200%); 2662/234114 objects misplaced (1.137%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:54:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[191227]: failed to encode map e389 with expected crc 2023-10-09T05:54:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[191227]: failed to encode map e389 with expected crc 2023-10-09T05:54:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2989345380' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:54:41.907 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:41 smithi008 ceph-mon[191227]: osdmap e389: 8 total, 8 up, 7 in 2023-10-09T05:54:41.908 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:42.363 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:42.881 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 86 16 80983 80967 58.8058 34.8125 0.0158224 0.0169999 2023-10-09T05:54:42.935 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:42.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:42.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:43.047 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:43.123 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:43.258 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:43.380 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:43.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:43.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:43.488 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:43.526 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:43.599 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:43.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:43.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:43.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:43.749 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:43.786 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:54:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:43 smithi191 ceph-mon[161669]: pgmap v1177: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 30 MiB/s wr, 475 op/s; 16857/238080 objects degraded (7.080%); 2662/238080 objects misplaced (1.118%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:54:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16857/238080 objects degraded (7.080%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:43.826 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:54:43.863 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:54:43.881 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 87 16 81874 81858 58.7699 55.6875 0.0175994 0.0170098 2023-10-09T05:54:43.911 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:54:43.955 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:54:44.000 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:54:44.045 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:54:44.076 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:44.081 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:54:44.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:44.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:44.096 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:54:44.096 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:54:44.096 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:54:44.096 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:54:44.097 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:54:44.097 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:54:44.097 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:54:44.097 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:54:44.126 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:43 smithi008 ceph-mon[184418]: pgmap v1177: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 30 MiB/s wr, 475 op/s; 16857/238080 objects degraded (7.080%); 2662/238080 objects misplaced (1.118%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:54:44.126 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16857/238080 objects degraded (7.080%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:44.127 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:43 smithi008 ceph-mon[191227]: pgmap v1177: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 30 MiB/s wr, 475 op/s; 16857/238080 objects degraded (7.080%); 2662/238080 objects misplaced (1.118%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:54:44.127 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16857/238080 objects degraded (7.080%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:44.184 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:44.238 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:44.389 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:44.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:44.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:44.494 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:44.587 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:44.632 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:54:44.632 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "" 2023-10-09T05:54:44.632 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:54:44.881 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 88 16 82706 82690 58.6929 52 0.00243554 0.0170297 2023-10-09T05:54:44.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/129139582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:44.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/129139582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:44.980 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:54:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/129139582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:54:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:45.761 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:45.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:45.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:45.867 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:45.881 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 89 16 83497 83481 58.5889 49.4375 0.0149735 0.0170625 2023-10-09T05:54:45.962 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:46.155 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:46.167 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:45 smithi008 ceph-mon[184418]: pgmap v1178: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 23 MiB/s wr, 363 op/s; 16857/238080 objects degraded (7.080%); 2662/238080 objects misplaced (1.118%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T05:54:46.167 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:45 smithi008 ceph-mon[191227]: pgmap v1178: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 23 MiB/s wr, 363 op/s; 16857/238080 objects degraded (7.080%); 2662/238080 objects misplaced (1.118%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T05:54:46.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:46.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:46.232 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:54:46.232 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:54:46.269 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:46.273 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:46.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:45 smithi191 ceph-mon[161669]: pgmap v1178: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 23 MiB/s wr, 363 op/s; 16857/238080 objects degraded (7.080%); 2662/238080 objects misplaced (1.118%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T05:54:46.364 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:46.545 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:46.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:46.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:46.650 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:46.676 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:46.881 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total time run: 90.0154 2023-10-09T05:54:46.881 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total writes made: 84170 2023-10-09T05:54:46.882 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Write size: 65536 2023-10-09T05:54:46.882 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object size: 65536 2023-10-09T05:54:46.882 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Bandwidth (MB/sec): 58.4414 2023-10-09T05:54:46.882 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Bandwidth: 32.5953 2023-10-09T05:54:46.882 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max bandwidth (MB/sec): 152.188 2023-10-09T05:54:46.882 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min bandwidth (MB/sec): 0 2023-10-09T05:54:46.883 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average IOPS: 935 2023-10-09T05:54:46.883 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev IOPS: 521.533 2023-10-09T05:54:46.883 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max IOPS: 2435 2023-10-09T05:54:46.883 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min IOPS: 0 2023-10-09T05:54:46.883 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average Latency(s): 0.0171067 2023-10-09T05:54:46.883 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Latency(s): 0.049183 2023-10-09T05:54:46.883 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max latency(s): 2.53611 2023-10-09T05:54:46.883 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min latency(s): 0.00115306 2023-10-09T05:54:46.890 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:54:47.179 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:47.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:47.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:47.286 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:47.341 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:54:47.361 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:47.499 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:47.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:47.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:47.611 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:47.660 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:47.740 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:47.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:47.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:47.770 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:54:47.846 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:47.953 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:48.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:47 smithi008 ceph-mon[184418]: pgmap v1179: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s; 16857/247872 objects degraded (6.801%); 2592/247872 objects misplaced (1.046%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:54:48.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:47 smithi008 ceph-mon[191227]: pgmap v1179: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s; 16857/247872 objects degraded (6.801%); 2592/247872 objects misplaced (1.046%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:54:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:47 smithi191 ceph-mon[161669]: pgmap v1179: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s; 16857/247872 objects degraded (6.801%); 2592/247872 objects misplaced (1.046%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:54:48.588 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:48.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:48.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:48.696 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:48.734 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:48.796 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:48.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:48.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:48.846 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:48.904 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:48.912 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:48.945 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16857/247872 objects degraded (6.801%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:48.979 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:49.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16857/247872 objects degraded (6.801%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:49.204 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16857/247872 objects degraded (6.801%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:49.294 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:49.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:49.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:49.401 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:49.512 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:49.616 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:49.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:49.727 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:49.769 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:49.839 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:49.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:49.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:49.945 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:50.020 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:50.229 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:54:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:50 smithi008 ceph-mon[184418]: pgmap v1180: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 677 op/s; 16857/249633 objects degraded (6.753%); 2592/249633 objects misplaced (1.038%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:54:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3849374877' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:50 smithi008 ceph-mon[191227]: pgmap v1180: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 677 op/s; 16857/249633 objects degraded (6.753%); 2592/249633 objects misplaced (1.038%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:54:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3849374877' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:50 smithi191 ceph-mon[161669]: pgmap v1180: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 677 op/s; 16857/249633 objects degraded (6.753%); 2592/249633 objects misplaced (1.038%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:54:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3849374877' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:50.578 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:54:50.578 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:54:50.578 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:54:50.578 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2023-10-09T05:54:50.578 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0 2023-10-09T05:54:50.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:54:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:54:50] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.33.4" 2023-10-09T05:54:51.244 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:51.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:51.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:51.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:51.378 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:51.423 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:51.434 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:51 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:51.434 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:51 smithi191 ceph-mon[161669]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.435 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:51 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3849374877' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:51.435 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:51 smithi191 ceph-mon[161669]: osdmap e390: 8 total, 8 up, 7 in 2023-10-09T05:54:51.435 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:51 smithi191 ceph-mon[161669]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.435 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:51 smithi191 ceph-mon[161669]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.435 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:51 smithi191 ceph-mon[161669]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.435 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:51 smithi191 ceph-mon[161669]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:51.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:51.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:51.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[191227]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3849374877' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:51.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[191227]: osdmap e390: 8 total, 8 up, 7 in 2023-10-09T05:54:51.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[191227]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[191227]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[191227]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[191227]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:51.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[184418]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3849374877' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:51.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[184418]: osdmap e390: 8 total, 8 up, 7 in 2023-10-09T05:54:51.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[184418]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[184418]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[184418]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:51 smithi008 ceph-mon[184418]: failed to encode map e390 with expected crc 2023-10-09T05:54:51.537 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:51.598 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:52.111 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:52.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:52.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:52.218 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:52.310 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:52 smithi008 ceph-mon[191227]: pgmap v1182: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 31 MiB/s wr, 565 op/s; 16701/248964 objects degraded (6.708%); 2592/248964 objects misplaced (1.041%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:54:52.314 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:52 smithi008 ceph-mon[184418]: pgmap v1182: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 31 MiB/s wr, 565 op/s; 16701/248964 objects degraded (6.708%); 2592/248964 objects misplaced (1.041%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:54:52.321 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:52.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:52 smithi191 ceph-mon[161669]: pgmap v1182: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 31 MiB/s wr, 565 op/s; 16701/248964 objects degraded (6.708%); 2592/248964 objects misplaced (1.041%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:54:52.743 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:52.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:52.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:52.853 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:52.918 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:53.284 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:53.301 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:54:53.301 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:54:53.329 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16701/248964 objects degraded (6.708%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:53.329 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16701/248964 objects degraded (6.708%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:53.347 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:53.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:53.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:53.433 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:54:53.453 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:53.476 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:54:53.485 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16701/248964 objects degraded (6.708%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:53.488 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:53.524 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:54:53.543 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:53.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:53.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:53.563 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:54:53.608 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:54:53.644 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:54:53.657 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:53.680 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:54:53.719 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:54:53.724 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:53.744 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:54:53.744 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:54:53.745 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:54:53.745 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:54:53.745 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:54:53.745 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:54:53.745 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:54:53.745 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:54:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:54 smithi191 ceph-mon[161669]: pgmap v1183: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 798 op/s; 16504/243144 objects degraded (6.788%); 2520/243144 objects misplaced (1.036%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:54:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:54 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/469039883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:54.613 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:54.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:54.619 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:54 smithi008 ceph-mon[184418]: pgmap v1183: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 798 op/s; 16504/243144 objects degraded (6.788%); 2520/243144 objects misplaced (1.036%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:54:54.619 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:54 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/469039883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:54.619 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:54 smithi008 ceph-mon[191227]: pgmap v1183: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 798 op/s; 16504/243144 objects degraded (6.788%); 2520/243144 objects misplaced (1.036%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:54:54.619 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:54 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/469039883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:54.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:54.721 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:54.780 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:54.831 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:54:55.147 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:55.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:55.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:55.259 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:55.309 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:55.401 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:55.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:55.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:55.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:55.555 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:54:55.572 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:55.630 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:55.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:55.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:55.738 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:55.855 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:55.987 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:55.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:55.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:56.097 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:56.161 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:56.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:56 smithi008 ceph-mon[191227]: pgmap v1184: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 798 op/s; 16504/243144 objects degraded (6.788%); 2520/243144 objects misplaced (1.036%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:54:56.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:56 smithi008 ceph-mon[184418]: pgmap v1184: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 798 op/s; 16504/243144 objects degraded (6.788%); 2520/243144 objects misplaced (1.036%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:54:56.497 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:56.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:56.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:56.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:56 smithi191 ceph-mon[161669]: pgmap v1184: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 798 op/s; 16504/243144 objects degraded (6.788%); 2520/243144 objects misplaced (1.036%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:54:56.606 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:56.643 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:56.699 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:56.959 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:56.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:56.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:57.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:57.135 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:57.209 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:57.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:57.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:57.239 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:57.317 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:57.354 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:57.410 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:57.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:57.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:57.470 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:54:57.516 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:57.560 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:57.624 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:57.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:57.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:57.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:57.767 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:57.830 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:57.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:57.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:57.936 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:57.976 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:58.043 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:58.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:58.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:58.152 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:58.181 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:58.244 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.4 primary-affinity to 0 (802) 2023-10-09T05:54:58.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:58 smithi008 ceph-mon[191227]: pgmap v1185: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 9.8 MiB/s wr, 809 op/s; 16040/233022 objects degraded (6.883%); 2419/233022 objects misplaced (1.038%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T05:54:58.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1144340227' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T05:54:58.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:58 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T05:54:58.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16040/233022 objects degraded (6.883%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:58 smithi008 ceph-mon[184418]: pgmap v1185: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 9.8 MiB/s wr, 809 op/s; 16040/233022 objects degraded (6.883%); 2419/233022 objects misplaced (1.038%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T05:54:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1144340227' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T05:54:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:58 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T05:54:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16040/233022 objects degraded (6.883%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:58.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:58 smithi191 ceph-mon[161669]: pgmap v1185: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 9.8 MiB/s wr, 809 op/s; 16040/233022 objects degraded (6.883%); 2419/233022 objects misplaced (1.038%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T05:54:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1144340227' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T05:54:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:58 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T05:54:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16040/233022 objects degraded (6.883%), 13 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:59.270 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:59.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:59.381 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:59.431 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:59.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[184418]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[184418]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[184418]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T05:54:59.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[184418]: osdmap e391: 8 total, 8 up, 7 in 2023-10-09T05:54:59.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[184418]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:59.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[184418]: osdmap e392: 8 total, 8 up, 7 in 2023-10-09T05:54:59.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[191227]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[191227]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[191227]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T05:54:59.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[191227]: osdmap e391: 8 total, 8 up, 7 in 2023-10-09T05:54:59.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[191227]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.518 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:59.518 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:54:59 smithi008 ceph-mon[191227]: osdmap e392: 8 total, 8 up, 7 in 2023-10-09T05:54:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:59 smithi191 ceph-mon[161669]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:59 smithi191 ceph-mon[161669]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:59 smithi191 ceph-mon[161669]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:59 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T05:54:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:59 smithi191 ceph-mon[161669]: osdmap e391: 8 total, 8 up, 7 in 2023-10-09T05:54:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:59 smithi191 ceph-mon[161669]: failed to encode map e391 with expected crc 2023-10-09T05:54:59.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:59.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:54:59 smithi191 ceph-mon[161669]: osdmap e392: 8 total, 8 up, 7 in 2023-10-09T05:54:59.643 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:59.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:59.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:59.747 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:59.777 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:00.327 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:00.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:00.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:00.424 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:00.432 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:00.459 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:00.512 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:00.515 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: pgmap v1187: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 121 B/s rd, 3.9 MiB/s wr, 851 op/s; 16040/230637 objects degraded (6.955%); 2419/230637 objects misplaced (1.049%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:55:00.515 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.515 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.516 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.516 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.516 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.516 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.516 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.516 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.517 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.517 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:00.517 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:00 smithi191 ceph-mon[161669]: osdmap e393: 8 total, 8 up, 7 in 2023-10-09T05:55:00.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:00.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:00.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: pgmap v1187: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 121 B/s rd, 3.9 MiB/s wr, 851 op/s; 16040/230637 objects degraded (6.955%); 2419/230637 objects misplaced (1.049%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:55:00.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:00.530 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[191227]: osdmap e393: 8 total, 8 up, 7 in 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: pgmap v1187: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 121 B/s rd, 3.9 MiB/s wr, 851 op/s; 16040/230637 objects degraded (6.955%); 2419/230637 objects misplaced (1.049%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: failed to encode map e392 with expected crc 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:00.531 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:00 smithi008 ceph-mon[184418]: osdmap e393: 8 total, 8 up, 7 in 2023-10-09T05:55:00.532 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:55:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:55:00] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-10-09T05:55:00.619 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:00.659 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:00.822 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:00.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:00.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:00.929 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:00.959 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:01.008 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:01.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:01.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:01.111 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:01.145 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:01.246 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:01.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:01.273 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:55:01.349 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:01.383 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:01.535 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:01.543 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:01 smithi191 ceph-mon[161669]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.543 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:01 smithi191 ceph-mon[161669]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.543 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:01 smithi191 ceph-mon[161669]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.543 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:01 smithi191 ceph-mon[161669]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.543 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:01 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/619177873' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:01.543 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:01 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:01.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:01.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:01.645 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:01.672 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:01.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[191227]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[191227]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[191227]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[191227]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/619177873' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[184418]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[184418]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[184418]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[184418]: failed to encode map e393 with expected crc 2023-10-09T05:55:01.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/619177873' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:01.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:01 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:01.768 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:01.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:01.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:01.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:01.916 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:01.972 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:55:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:55:01.973 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T05:55:01.973 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:55:01.978 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:01.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:01.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:02.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:02.115 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:02.174 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:02.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:02.279 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:02.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[184418]: pgmap v1190: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 12980/227955 objects degraded (5.694%); 2419/227955 objects misplaced (1.061%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T05:55:02.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:02.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[184418]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:02.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[184418]: osdmap e394: 8 total, 8 up, 7 in 2023-10-09T05:55:02.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[184418]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[184418]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.305 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[184418]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[191227]: pgmap v1190: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 12980/227955 objects degraded (5.694%); 2419/227955 objects misplaced (1.061%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T05:55:02.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:02.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[191227]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:02.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[191227]: osdmap e394: 8 total, 8 up, 7 in 2023-10-09T05:55:02.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[191227]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[191227]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.305 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:02 smithi008 ceph-mon[191227]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.320 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:02.524 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:02.528 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:02 smithi191 ceph-mon[161669]: pgmap v1190: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 843 op/s; 12980/227955 objects degraded (5.694%); 2419/227955 objects misplaced (1.061%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T05:55:02.528 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:02 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:02.528 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:02 smithi191 ceph-mon[161669]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.528 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:02 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:02.528 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:02 smithi191 ceph-mon[161669]: osdmap e394: 8 total, 8 up, 7 in 2023-10-09T05:55:02.528 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:02 smithi191 ceph-mon[161669]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.528 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:02 smithi191 ceph-mon[161669]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.528 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:02 smithi191 ceph-mon[161669]: failed to encode map e394 with expected crc 2023-10-09T05:55:02.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:02.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:02.630 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:02.662 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:02.811 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:02.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:02.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:02.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:02.941 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:03.045 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:03.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:03.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:03.150 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:03.182 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:03.270 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:55:03.295 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:03.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:03.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:03.399 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:03.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 12980/227955 objects degraded (5.694%), 12 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:03.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 12980/227955 objects degraded (5.694%), 12 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:03.468 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:55:03.472 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:03.500 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:55:03.530 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:55:03.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 12980/227955 objects degraded (5.694%), 12 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:03.556 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:55:03.583 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:03.584 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:55:03.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:03.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:03.609 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:55:03.641 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:55:03.666 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:55:03.675 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:55:03.675 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:55:03.675 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:55:03.675 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:55:03.675 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:55:03.676 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:55:03.676 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:55:03.676 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:55:03.692 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:03.721 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:03.773 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:03.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:03.879 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:03.919 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:04.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:04 smithi191 ceph-mon[161669]: pgmap v1192: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 12957/223821 objects degraded (5.789%); 1975/223821 objects misplaced (0.882%); 9.4 MiB/s, 144 objects/s recovering 2023-10-09T05:55:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1983230924' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:04 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:04 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:04.582 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:55:04.582 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:55:04.582 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:55:04.582 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2023-10-09T05:55:04.582 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 0 2023-10-09T05:55:04.612 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:04 smithi008 ceph-mon[184418]: pgmap v1192: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 12957/223821 objects degraded (5.789%); 1975/223821 objects misplaced (0.882%); 9.4 MiB/s, 144 objects/s recovering 2023-10-09T05:55:04.612 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1983230924' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:04.612 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:04 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:04.612 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:04 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:04.613 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:04 smithi008 ceph-mon[191227]: pgmap v1192: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 12957/223821 objects degraded (5.789%); 1975/223821 objects misplaced (0.882%); 9.4 MiB/s, 144 objects/s recovering 2023-10-09T05:55:04.613 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1983230924' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:04.613 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:04 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:04.613 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:04 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:04.882 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:04.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:04.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:04.992 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:04.999 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:05.049 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:05.194 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:05.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:05.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:05.298 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:05.329 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:05.365 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:05.378 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:05.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:05.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:05.486 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:05.532 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:05.574 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:55:05.764 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:05.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:05.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:05.869 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:05.896 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:05.942 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:05.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:05.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:06.048 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:06.075 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:06.270 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:06 smithi191 ceph-mon[161669]: pgmap v1193: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 378 op/s; 12957/223821 objects degraded (5.789%); 1975/223821 objects misplaced (0.882%); 6.8 MiB/s, 104 objects/s recovering 2023-10-09T05:55:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:06 smithi008 ceph-mon[184418]: pgmap v1193: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 378 op/s; 12957/223821 objects degraded (5.789%); 1975/223821 objects misplaced (0.882%); 6.8 MiB/s, 104 objects/s recovering 2023-10-09T05:55:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:06 smithi008 ceph-mon[191227]: pgmap v1193: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 378 op/s; 12957/223821 objects degraded (5.789%); 1975/223821 objects misplaced (0.882%); 6.8 MiB/s, 104 objects/s recovering 2023-10-09T05:55:06.592 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:06.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:06.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:06.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:06.724 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:06.737 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:55:06.782 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:06.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:06.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:06.887 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:06.934 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:07.377 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:07.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:07.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:07.484 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:07.517 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:07 smithi191 ceph-mon[161669]: 9.0 scrub starts 2023-10-09T05:55:07.664 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:07 smithi008 ceph-mon[184418]: 9.0 scrub starts 2023-10-09T05:55:07.664 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:07 smithi008 ceph-mon[191227]: 9.0 scrub starts 2023-10-09T05:55:07.836 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:07.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:07.941 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:55:07.941 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:55:07.951 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:08.008 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:08.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:08 smithi191 ceph-mon[161669]: pgmap v1194: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 15667/221946 objects degraded (7.059%); 1494/221946 objects misplaced (0.673%); 11 MiB/s, 170 objects/s recovering 2023-10-09T05:55:08.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15667/221946 objects degraded (7.059%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:08.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:08 smithi008 ceph-mon[184418]: pgmap v1194: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 15667/221946 objects degraded (7.059%); 1494/221946 objects misplaced (0.673%); 11 MiB/s, 170 objects/s recovering 2023-10-09T05:55:08.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15667/221946 objects degraded (7.059%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:08 smithi008 ceph-mon[191227]: pgmap v1194: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 15667/221946 objects degraded (7.059%); 1494/221946 objects misplaced (0.673%); 11 MiB/s, 170 objects/s recovering 2023-10-09T05:55:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15667/221946 objects degraded (7.059%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:09.084 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:09.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:09.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:09.115 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:55:09.189 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:09.193 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:09.258 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:09.736 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:09.783 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:09.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:09.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:09.890 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:09.938 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:10.103 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:10.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:10.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:10.210 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:10.256 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:10.312 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.5 primary-affinity to 0 (802) 2023-10-09T05:55:10.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:10 smithi008 ceph-mon[191227]: pgmap v1195: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 274 op/s; 15667/221085 objects degraded (7.086%); 1494/221085 objects misplaced (0.676%); 8.2 MiB/s, 128 objects/s recovering 2023-10-09T05:55:10.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:10 smithi008 ceph-mon[191227]: 9.2 scrub starts 2023-10-09T05:55:10.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:10 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2169194900' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T05:55:10.339 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:10 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T05:55:10.341 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:10 smithi008 ceph-mon[184418]: pgmap v1195: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 274 op/s; 15667/221085 objects degraded (7.086%); 1494/221085 objects misplaced (0.676%); 8.2 MiB/s, 128 objects/s recovering 2023-10-09T05:55:10.341 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:10 smithi008 ceph-mon[184418]: 9.2 scrub starts 2023-10-09T05:55:10.341 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:10 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2169194900' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T05:55:10.341 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:10 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T05:55:10.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:55:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:55:10] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-10-09T05:55:10.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:10 smithi191 ceph-mon[161669]: pgmap v1195: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 274 op/s; 15667/221085 objects degraded (7.086%); 1494/221085 objects misplaced (0.676%); 8.2 MiB/s, 128 objects/s recovering 2023-10-09T05:55:10.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:10 smithi191 ceph-mon[161669]: 9.2 scrub starts 2023-10-09T05:55:10.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:10 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2169194900' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T05:55:10.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:10 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T05:55:11.511 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:11.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T05:55:11.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: osdmap e395: 8 total, 8 up, 7 in 2023-10-09T05:55:11.519 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.520 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[184418]: osdmap e396: 8 total, 8 up, 7 in 2023-10-09T05:55:11.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T05:55:11.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: osdmap e395: 8 total, 8 up, 7 in 2023-10-09T05:55:11.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.521 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:11 smithi008 ceph-mon[191227]: osdmap e396: 8 total, 8 up, 7 in 2023-10-09T05:55:11.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:11.624 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:11.678 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:11.796 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T05:55:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: osdmap e395: 8 total, 8 up, 7 in 2023-10-09T05:55:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: failed to encode map e395 with expected crc 2023-10-09T05:55:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:11 smithi191 ceph-mon[161669]: osdmap e396: 8 total, 8 up, 7 in 2023-10-09T05:55:11.810 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:11.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:11.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:11.916 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:11.950 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:12.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[184418]: pgmap v1197: 99 pgs: 2 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 180 op/s; 15525/219876 objects degraded (7.061%); 1494/219876 objects misplaced (0.679%); 5.1 MiB/s, 79 objects/s recovering 2023-10-09T05:55:12.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[184418]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[184418]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[184418]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[184418]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:55:12.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:12.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:12.698 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[184418]: osdmap e397: 8 total, 8 up, 7 in 2023-10-09T05:55:12.699 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[191227]: pgmap v1197: 99 pgs: 2 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 180 op/s; 15525/219876 objects degraded (7.061%); 1494/219876 objects misplaced (0.679%); 5.1 MiB/s, 79 objects/s recovering 2023-10-09T05:55:12.699 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[191227]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.699 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[191227]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.699 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[191227]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.699 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[191227]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.699 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:55:12.699 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:12.699 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:12.699 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:12 smithi008 ceph-mon[191227]: osdmap e397: 8 total, 8 up, 7 in 2023-10-09T05:55:12.732 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:12.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:12.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:12 smithi191 ceph-mon[161669]: pgmap v1197: 99 pgs: 2 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 180 op/s; 15525/219876 objects degraded (7.061%); 1494/219876 objects misplaced (0.679%); 5.1 MiB/s, 79 objects/s recovering 2023-10-09T05:55:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:12 smithi191 ceph-mon[161669]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:12 smithi191 ceph-mon[161669]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:12 smithi191 ceph-mon[161669]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:12 smithi191 ceph-mon[161669]: failed to encode map e396 with expected crc 2023-10-09T05:55:12.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:12 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:55:12.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:12 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:12.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:12 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:12.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:12 smithi191 ceph-mon[161669]: osdmap e397: 8 total, 8 up, 7 in 2023-10-09T05:55:12.842 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:12.914 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:13.152 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:55:13.191 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:55:13.227 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:55:13.272 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:55:13.315 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:55:13.351 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:13.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:13.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:13.360 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:55:13.395 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:55:13.424 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:55:13.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:55:13.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:55:13.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:55:13.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:55:13.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:55:13.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:55:13.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:55:13.428 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:55:13.461 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:13.527 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:13.680 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: pgmap v1200: 99 pgs: 5 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 12594/217761 objects degraded (5.783%); 908/217761 objects misplaced (0.417%); 8.4 MiB/s, 129 objects/s recovering 2023-10-09T05:55:13.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2260354538' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:13.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15525/219876 objects degraded (7.061%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:13.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:13.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:13.682 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:13.682 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.682 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.682 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.682 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.682 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.682 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.682 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: pgmap v1200: 99 pgs: 5 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 12594/217761 objects degraded (5.783%); 908/217761 objects misplaced (0.417%); 8.4 MiB/s, 129 objects/s recovering 2023-10-09T05:55:13.682 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2260354538' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:13.682 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15525/219876 objects degraded (7.061%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:13.683 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:13.683 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:13.683 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: failed to encode map e397 with expected crc 2023-10-09T05:55:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: pgmap v1200: 99 pgs: 5 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 12594/217761 objects degraded (5.783%); 908/217761 objects misplaced (0.417%); 8.4 MiB/s, 129 objects/s recovering 2023-10-09T05:55:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2260354538' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:13.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15525/219876 objects degraded (7.061%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:13.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:13.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:13.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:14.386 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:55:14.502 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:14.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:14.606 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:14.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:14 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2586978049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:14.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:14 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:14.631 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:14.632 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:14.634 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:14.680 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:14.682 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:14 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2586978049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:14.682 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:14 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:14.682 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:14.682 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:14.683 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:14 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2586978049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:14.683 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:14 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:14.683 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:14.683 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:14.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:14.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:14.786 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:14.812 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:14.856 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:14.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:14.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:14.961 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:14.988 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:15.037 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:15.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:15.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:15.145 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:15.179 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:15.584 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:15.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:15.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:15.657 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:15.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[191227]: failed to encode map e398 with expected crc 2023-10-09T05:55:15.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[191227]: failed to encode map e398 with expected crc 2023-10-09T05:55:15.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:15.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[191227]: osdmap e398: 8 total, 8 up, 7 in 2023-10-09T05:55:15.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[191227]: pgmap v1202: 99 pgs: 5 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 12594/217761 objects degraded (5.783%); 908/217761 objects misplaced (0.417%); 9.6 MiB/s, 148 objects/s recovering 2023-10-09T05:55:15.658 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:15.658 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[184418]: failed to encode map e398 with expected crc 2023-10-09T05:55:15.658 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[184418]: failed to encode map e398 with expected crc 2023-10-09T05:55:15.659 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:15.659 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[184418]: osdmap e398: 8 total, 8 up, 7 in 2023-10-09T05:55:15.659 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:15 smithi008 ceph-mon[184418]: pgmap v1202: 99 pgs: 5 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 12594/217761 objects degraded (5.783%); 908/217761 objects misplaced (0.417%); 9.6 MiB/s, 148 objects/s recovering 2023-10-09T05:55:15.688 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:15.745 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:15 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:15 smithi191 ceph-mon[161669]: failed to encode map e398 with expected crc 2023-10-09T05:55:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:15 smithi191 ceph-mon[161669]: failed to encode map e398 with expected crc 2023-10-09T05:55:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:15 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:15 smithi191 ceph-mon[161669]: osdmap e398: 8 total, 8 up, 7 in 2023-10-09T05:55:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:15 smithi191 ceph-mon[161669]: pgmap v1202: 99 pgs: 5 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 12594/217761 objects degraded (5.783%); 908/217761 objects misplaced (0.417%); 9.6 MiB/s, 148 objects/s recovering 2023-10-09T05:55:15.851 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:15.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:15.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:15.956 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:16.001 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:16.190 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:16.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:16.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:16.244 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:55:16.244 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:55:16.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:55:16.245 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:55:16.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.245 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180268 7.1e 7:795f278a:::benchmark_data_smithi008_354649_object85772:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e269)", 2023-10-09T05:55:16.245 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.024498+0000", 2023-10-09T05:55:16.245 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 515.21694080500004, 2023-10-09T05:55:16.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99404307599999997, 2023-10-09T05:55:16.245 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.245 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.245 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.245 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:55:16.246 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:55:16.246 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180268 2023-10-09T05:55:16.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.246 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024498+0000", 2023-10-09T05:55:16.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024498+0000", 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024496+0000", 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024505+0000", 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2900000000000008e-06 2023-10-09T05:55:16.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024528+0000", 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3096e-05 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.024540+0000", 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1959e-05 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.285107+0000", 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26056715899999999 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.285116+0000", 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5250000000000005e-06 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.016113+0000", 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73099665800000002 2023-10-09T05:55:16.249 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.016255+0000", 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014244400000000001 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,5", 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.016443+0000", 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018830400000000001 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.250 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.017705+0000", 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0012614029999999999 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018393+0000", 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00068864900000000005 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018481+0000", 2023-10-09T05:55:16.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7276999999999998e-05 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018513+0000", 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1922999999999999e-05 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.018541+0000", 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8174000000000002e-05 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.252 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:75147 8.a 8:5b312b1e:::benchmark_data_smithi008_432048_object75146:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:31.714915+0000", 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 404.526523462, 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30074143399999997, 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:16.253 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 75147 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714915+0000", 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714915+0000", 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714914+0000", 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714936+0000", 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2353000000000001e-05 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714945+0000", 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1670000000000005e-06 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714951+0000", 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5999999999999997e-06 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716442+0000", 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001491514 2023-10-09T05:55:16.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716558+0000", 2023-10-09T05:55:16.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011547799999999999 2023-10-09T05:55:16.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:55:16.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716652+0000", 2023-10-09T05:55:16.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4291999999999998e-05 2023-10-09T05:55:16.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.014083+0000", 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.297430892 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015543+0000", 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014593900000000001 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015556+0000", 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3715e-05 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015613+0000", 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6848000000000002e-05 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015657+0000", 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3412999999999999e-05 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.259 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.260 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.260 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:75150 8.3 e296/285 8:c720d3fb:::benchmark_data_smithi008_432048_object75149:head v 296'4679, mlcod=296'4677)", 2023-10-09T05:55:16.260 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:31.716949+0000", 2023-10-09T05:55:16.260 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 404.52448989800001, 2023-10-09T05:55:16.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30161086100000001, 2023-10-09T05:55:16.260 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.260 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.260 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716949+0000", 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716949+0000", 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716948+0000", 2023-10-09T05:55:16.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:16.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.262 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.262 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716999+0000", 2023-10-09T05:55:16.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0831999999999997e-05 2023-10-09T05:55:16.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.262 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.262 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011201+0000", 2023-10-09T05:55:16.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29420217900000001 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011219+0000", 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7883000000000001e-05 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011269+0000", 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9586999999999998e-05 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011301+0000", 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1902000000000001e-05 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.018513+0000", 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0072121429999999999 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.018559+0000", 2023-10-09T05:55:16.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6836999999999999e-05 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45782.0:75147 8.a e296/285 ondisk, result = 0)", 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:31.718538+0000", 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 404.52290051300002, 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29700873900000002, 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:55:16.265 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718538+0000", 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718538+0000", 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718537+0000", 2023-10-09T05:55:16.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:16.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718541+0000", 2023-10-09T05:55:16.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7900000000000001e-06 2023-10-09T05:55:16.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.012365+0000", 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29382405900000003 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.012376+0000", 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0210000000000001e-05 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015527+0000", 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031513660000000001 2023-10-09T05:55:16.268 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.269 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015536+0000", 2023-10-09T05:55:16.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1689999999999998e-06 2023-10-09T05:55:16.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.269 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015547+0000", 2023-10-09T05:55:16.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0713000000000001e-05 2023-10-09T05:55:16.269 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.269 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:90579 8.a 8:55cc0eec:::benchmark_data_smithi008_432048_object90578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:49.809093+0000", 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 386.43234584599998, 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29641294499999998, 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:55:16.270 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:16.271 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 90579 2023-10-09T05:55:16.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.271 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809093+0000", 2023-10-09T05:55:16.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809093+0000", 2023-10-09T05:55:16.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.272 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.272 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809091+0000", 2023-10-09T05:55:16.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:16.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.272 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.272 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809117+0000", 2023-10-09T05:55:16.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6276999999999999e-05 2023-10-09T05:55:16.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809142+0000", 2023-10-09T05:55:16.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4307e-05 2023-10-09T05:55:16.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809153+0000", 2023-10-09T05:55:16.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1561e-05 2023-10-09T05:55:16.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.274 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.274 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809289+0000", 2023-10-09T05:55:16.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000136398 2023-10-09T05:55:16.274 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.274 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.274 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809442+0000", 2023-10-09T05:55:16.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000152211 2023-10-09T05:55:16.274 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:55:16.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809589+0000", 2023-10-09T05:55:16.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014731999999999999 2023-10-09T05:55:16.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.819043+0000", 2023-10-09T05:55:16.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0094543100000000005 2023-10-09T05:55:16.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.828230+0000", 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0091865950000000005 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105413+0000", 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27718348900000001 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105464+0000", 2023-10-09T05:55:16.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0207000000000002e-05 2023-10-09T05:55:16.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105506+0000", 2023-10-09T05:55:16.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2001e-05 2023-10-09T05:55:16.277 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.277 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.277 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:90580 8.a 8:5146ec73:::benchmark_data_smithi008_432048_object90579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:49.809186+0000", 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 386.43225300699999, 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29646979400000001, 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 90580 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.278 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809186+0000", 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809186+0000", 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809185+0000", 2023-10-09T05:55:16.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:16.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809210+0000", 2023-10-09T05:55:16.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5360000000000001e-05 2023-10-09T05:55:16.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809224+0000", 2023-10-09T05:55:16.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3818000000000001e-05 2023-10-09T05:55:16.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809232+0000", 2023-10-09T05:55:16.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7230000000000004e-06 2023-10-09T05:55:16.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.812756+0000", 2023-10-09T05:55:16.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035239479999999998 2023-10-09T05:55:16.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.282 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.282 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.812903+0000", 2023-10-09T05:55:16.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000147274 2023-10-09T05:55:16.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.282 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:55:16.282 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.813046+0000", 2023-10-09T05:55:16.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000142492 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.819054+0000", 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0060084450000000003 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.828260+0000", 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0092060430000000006 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105560+0000", 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27729968900000002 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105616+0000", 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6239999999999997e-05 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105655+0000", 2023-10-09T05:55:16.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9382999999999999e-05 2023-10-09T05:55:16.285 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.285 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.285 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.285 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45782.0:168592 8.1a e352/349 ondisk, result = 0)", 2023-10-09T05:55:16.285 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:53.754226+0000", 2023-10-09T05:55:16.285 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 202.48721236200001, 2023-10-09T05:55:16.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30718891300000001, 2023-10-09T05:55:16.285 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.286 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:55:16.286 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754226+0000", 2023-10-09T05:55:16.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754226+0000", 2023-10-09T05:55:16.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754225+0000", 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754233+0000", 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3259999999999996e-06 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754249+0000", 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5512e-05 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754258+0000", 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3989999999999993e-06 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754302+0000", 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4023999999999997e-05 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754311+0000", 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5580000000000001e-06 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061415+0000", 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.307104341 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.289 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.290 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.290 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:168594 8.f e352/339 8:f6b60d9c:::benchmark_data_smithi008_432048_object65652:head v 352'8443, mlcod=293'4003)", 2023-10-09T05:55:16.290 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:53.755882+0000", 2023-10-09T05:55:16.290 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 202.48555645499999, 2023-10-09T05:55:16.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.31213674699999999, 2023-10-09T05:55:16.290 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.290 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.290 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755882+0000", 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755882+0000", 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755881+0000", 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755888+0000", 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2779999999999999e-06 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061422+0000", 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30553472300000001 2023-10-09T05:55:16.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061434+0000", 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2132e-05 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061482+0000", 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7129999999999998e-05 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061517+0000", 2023-10-09T05:55:16.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5268999999999997e-05 2023-10-09T05:55:16.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.067965+0000", 2023-10-09T05:55:16.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064480149999999996 2023-10-09T05:55:16.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.068019+0000", 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3918000000000001e-05 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:168601 8.14 e352/299 8:2b44b62c:::benchmark_data_smithi008_432048_object65659:head v 352'7985, mlcod=352'7983)", 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:53.758418+0000", 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 202.48302025000001, 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.305713397, 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.295 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.296 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758418+0000", 2023-10-09T05:55:16.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758418+0000", 2023-10-09T05:55:16.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758417+0000", 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758427+0000", 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6940000000000007e-06 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061398+0000", 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30297090500000001 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061426+0000", 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7858999999999999e-05 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061482+0000", 2023-10-09T05:55:16.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6051000000000003e-05 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061510+0000", 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8085999999999998e-05 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.064085+0000", 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025752219999999998 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.064132+0000", 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6628999999999997e-05 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:189481 8.1a 8:5d874d6e:::benchmark_data_smithi008_432048_object86539:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:52:23.908117+0000", 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 172.33332166100001, 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35551341400000003, 2023-10-09T05:55:16.300 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 189481 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908117+0000", 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908117+0000", 2023-10-09T05:55:16.302 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908116+0000", 2023-10-09T05:55:16.302 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:16.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908121+0000", 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4879999999999998e-06 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908131+0000", 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7969999999999995e-06 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908136+0000", 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5509999999999999e-06 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908196+0000", 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9165e-05 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908548+0000", 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000352345 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.247746+0000", 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33919810900000003 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.247846+0000", 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010041 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:55:16.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.248364+0000", 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00051785399999999999 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.252480+0000", 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041154950000000003 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.259547+0000", 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070675399999999998 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.263254+0000", 2023-10-09T05:55:16.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037071130000000002 2023-10-09T05:55:16.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.263306+0000", 2023-10-09T05:55:16.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1272e-05 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.263630+0000", 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00032452000000000002 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:13236 9.2 9:4c9db24c:::benchmark_data_smithi008_502820_object13235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T05:55:16.308 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:23.473276+0000", 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 112.768162101, 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37738287399999998, 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 13236 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.309 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473276+0000", 2023-10-09T05:55:16.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.310 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473276+0000", 2023-10-09T05:55:16.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.310 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473275+0000", 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473299+0000", 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3838000000000001e-05 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473311+0000", 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1341e-05 2023-10-09T05:55:16.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473318+0000", 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2239999999999998e-06 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.474378+0000", 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010603629999999999 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.474523+0000", 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014455100000000001 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.474663+0000", 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014071900000000001 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475992+0000", 2023-10-09T05:55:16.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001328678 2023-10-09T05:55:16.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475999+0000", 2023-10-09T05:55:16.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.007e-06 2023-10-09T05:55:16.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850549+0000", 2023-10-09T05:55:16.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37454978100000003 2023-10-09T05:55:16.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850610+0000", 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0807999999999998e-05 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850659+0000", 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9536999999999997e-05 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.315 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:13240 9.2 9:4ac49363:::benchmark_data_smithi008_502820_object13239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:23.475894+0000", 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 112.765544041, 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37490599200000002, 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 13240 2023-10-09T05:55:16.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475894+0000", 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475894+0000", 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475893+0000", 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475917+0000", 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4465999999999999e-05 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475929+0000", 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1144e-05 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475937+0000", 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9340000000000006e-06 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476020+0000", 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3170999999999994e-05 2023-10-09T05:55:16.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476141+0000", 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012152 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476295+0000", 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000153709 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.811119+0000", 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33482404700000001 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.834120+0000", 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.023000782000000001 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850714+0000", 2023-10-09T05:55:16.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016593891 2023-10-09T05:55:16.322 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.322 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.322 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.322 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850764+0000", 2023-10-09T05:55:16.322 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0479999999999998e-05 2023-10-09T05:55:16.322 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.322 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.322 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.322 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850800+0000", 2023-10-09T05:55:16.322 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6355e-05 2023-10-09T05:55:16.322 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:13243 9.0 9:0610f3a0:::benchmark_data_smithi008_502820_object13242:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:23.475944+0000", 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 112.765494487, 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37392950800000002, 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.323 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:55:16.324 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:16.324 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 13243 2023-10-09T05:55:16.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.324 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475944+0000", 2023-10-09T05:55:16.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475944+0000", 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475943+0000", 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475972+0000", 2023-10-09T05:55:16.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8776999999999999e-05 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475980+0000", 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3510000000000002e-06 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475990+0000", 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8600000000000005e-06 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.327 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476039+0000", 2023-10-09T05:55:16.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9144000000000002e-05 2023-10-09T05:55:16.327 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.327 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.327 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476196+0000", 2023-10-09T05:55:16.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015608700000000001 2023-10-09T05:55:16.327 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.327 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476325+0000", 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000129407 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.811118+0000", 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33479272500000001 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.811147+0000", 2023-10-09T05:55:16.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8900000000000001e-05 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.849758+0000", 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.038611131 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.849825+0000", 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7037000000000002e-05 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.849874+0000", 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8823000000000003e-05 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:13247 9.2 9:4cd4543f:::benchmark_data_smithi008_502820_object13246:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:23.477800+0000", 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 112.763638628, 2023-10-09T05:55:16.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37350776499999999, 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 13247 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.477800+0000", 2023-10-09T05:55:16.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.477800+0000", 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.477799+0000", 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.477826+0000", 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7291e-05 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.809475+0000", 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33164901299999999 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.333 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.809508+0000", 2023-10-09T05:55:16.334 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2771000000000002e-05 2023-10-09T05:55:16.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.334 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.810358+0000", 2023-10-09T05:55:16.334 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00085006899999999995 2023-10-09T05:55:16.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.334 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.810484+0000", 2023-10-09T05:55:16.334 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000125742 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.810587+0000", 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010355 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.811302+0000", 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00071510300000000003 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.834210+0000", 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022907375000000001 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.851187+0000", 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016977590000000001 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.851262+0000", 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5185000000000006e-05 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.851308+0000", 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5237000000000001e-05 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.337 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:62837 9.2 9:4c5dbb04:::benchmark_data_smithi008_502820_object62836:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e379)", 2023-10-09T05:55:16.338 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:13.285461+0000", 2023-10-09T05:55:16.338 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 62.955977631000003, 2023-10-09T05:55:16.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27731913600000002, 2023-10-09T05:55:16.338 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.338 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.338 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.338 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:55:16.338 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:16.338 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 62837 2023-10-09T05:55:16.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.338 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285461+0000", 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285461+0000", 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285460+0000", 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285481+0000", 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0845000000000001e-05 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285490+0000", 2023-10-09T05:55:16.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8800000000000003e-06 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285496+0000", 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7010000000000001e-06 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285541+0000", 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5210000000000003e-05 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285688+0000", 2023-10-09T05:55:16.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000146093 2023-10-09T05:55:16.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:55:16.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285847+0000", 2023-10-09T05:55:16.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015936999999999999 2023-10-09T05:55:16.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.288768+0000", 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029215479999999999 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.297607+0000", 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.008838472 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.562687+0000", 2023-10-09T05:55:16.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26507981600000002 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.562740+0000", 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3507000000000001e-05 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.562780+0000", 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9818999999999999e-05 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.344 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:64640 9.2 9:4a222491:::benchmark_data_smithi008_502820_object64639:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e380)", 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:15.970717+0000", 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 60.270721422000001, 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.227555014, 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:55:16.345 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:16.346 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 64640 2023-10-09T05:55:16.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.346 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.346 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.346 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:15.970717+0000", 2023-10-09T05:55:16.346 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.346 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.346 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:15.970717+0000", 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:15.970715+0000", 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:15.970742+0000", 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6665e-05 2023-10-09T05:55:16.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:15.970755+0000", 2023-10-09T05:55:16.348 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3349999999999999e-05 2023-10-09T05:55:16.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:15.970764+0000", 2023-10-09T05:55:16.348 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5040000000000002e-06 2023-10-09T05:55:16.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:15.970816+0000", 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2271999999999997e-05 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:15.970959+0000", 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014279200000000001 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:55:16.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:15.971092+0000", 2023-10-09T05:55:16.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013353400000000001 2023-10-09T05:55:16.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:15.974119+0000", 2023-10-09T05:55:16.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030260980000000001 2023-10-09T05:55:16.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:16.197479+0000", 2023-10-09T05:55:16.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.22336067200000001 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:16.198179+0000", 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00069957199999999998 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:16.198231+0000", 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2609999999999999e-05 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.352 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:16.198272+0000", 2023-10-09T05:55:16.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0729999999999998e-05 2023-10-09T05:55:16.352 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.352 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.352 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.352 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.352 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77599 9.0 9:0003d9cd:::benchmark_data_smithi008_502820_object77598:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T05:55:16.352 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:36.817794+0000", 2023-10-09T05:55:16.352 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 39.423644285999998, 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.88810962599999999, 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77599 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.353 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817794+0000", 2023-10-09T05:55:16.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817794+0000", 2023-10-09T05:55:16.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817793+0000", 2023-10-09T05:55:16.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817819+0000", 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6563999999999998e-05 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817840+0000", 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1183999999999999e-05 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817850+0000", 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5040000000000008e-06 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817907+0000", 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6700000000000003e-05 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:55:16.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817919+0000", 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2755000000000001e-05 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.705815+0000", 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.88789530699999997 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.705904+0000", 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9301000000000004e-05 2023-10-09T05:55:16.357 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.358 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.358 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.358 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.358 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.358 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77622 9.2 9:43e6566f:::benchmark_data_smithi008_502820_object77621:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e385)", 2023-10-09T05:55:16.358 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:36.872539+0000", 2023-10-09T05:55:16.358 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 39.368899269000003, 2023-10-09T05:55:16.358 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86671261899999996, 2023-10-09T05:55:16.358 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.358 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.358 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77622 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872539+0000", 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.359 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872539+0000", 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872537+0000", 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872566+0000", 2023-10-09T05:55:16.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8402999999999999e-05 2023-10-09T05:55:16.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872580+0000", 2023-10-09T05:55:16.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4102e-05 2023-10-09T05:55:16.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872590+0000", 2023-10-09T05:55:16.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8509999999999994e-06 2023-10-09T05:55:16.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872642+0000", 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2284999999999998e-05 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872653+0000", 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0428000000000001e-05 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.363 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.703374+0000", 2023-10-09T05:55:16.363 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83072101700000001 2023-10-09T05:55:16.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.363 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.363 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.703547+0000", 2023-10-09T05:55:16.363 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000173058 2023-10-09T05:55:16.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.363 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:55:16.363 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.703705+0000", 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000158813 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.714288+0000", 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010582225000000001 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.725463+0000", 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011175243 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.739134+0000", 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013670748 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.739206+0000", 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2875999999999998e-05 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.739252+0000", 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5386000000000002e-05 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77624 9.10 9:0cfb0a88:::benchmark_data_smithi008_502820_object77623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e385)", 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:37.004849+0000", 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 39.236589455999997, 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.76289203999999999, 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.366 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77624 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004849+0000", 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004849+0000", 2023-10-09T05:55:16.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004847+0000", 2023-10-09T05:55:16.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:16.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004881+0000", 2023-10-09T05:55:16.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3456e-05 2023-10-09T05:55:16.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004897+0000", 2023-10-09T05:55:16.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.594e-05 2023-10-09T05:55:16.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004908+0000", 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1667e-05 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004998+0000", 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9231e-05 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:55:16.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.005010+0000", 2023-10-09T05:55:16.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.225e-05 2023-10-09T05:55:16.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.747777+0000", 2023-10-09T05:55:16.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.742767222 2023-10-09T05:55:16.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.747948+0000", 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017099300000000001 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.752637+0000", 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0046887279999999996 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.763870+0000", 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011232956000000001 2023-10-09T05:55:16.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.374 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.764548+0000", 2023-10-09T05:55:16.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00067862 2023-10-09T05:55:16.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.374 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.767630+0000", 2023-10-09T05:55:16.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030816730000000001 2023-10-09T05:55:16.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.767696+0000", 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5646000000000001e-05 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.767741+0000", 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5454000000000002e-05 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.376 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93493 9.2 9:40721287:::benchmark_data_smithi008_502820_object9320:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T05:55:16.376 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:58.119061+0000", 2023-10-09T05:55:16.376 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 18.122377350000001, 2023-10-09T05:55:16.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24733692800000001, 2023-10-09T05:55:16.376 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:55:16.376 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:16.376 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:55:16.376 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:55:16.376 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:16.376 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93493 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119061+0000", 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119061+0000", 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:55:16.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119060+0000", 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119065+0000", 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8840000000000002e-06 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119077+0000", 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2122e-05 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119083+0000", 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5169999999999999e-06 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119670+0000", 2023-10-09T05:55:16.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00058677600000000001 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119757+0000", 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7323000000000004e-05 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119823+0000", 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6447999999999998e-05 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.121014+0000", 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001191089 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.121251+0000", 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00023627199999999999 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.366302+0000", 2023-10-09T05:55:16.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.245051305 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.366355+0000", 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3211999999999998e-05 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.366398+0000", 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2898000000000002e-05 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.382 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.383 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.383 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:55:16.383 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:55:16.383 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:55:16.383 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:16.431 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:16.510 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:16.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:16.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:16.617 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:16.643 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:16.684 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:55:16.688 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:16.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:16.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:16.700 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:16 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T05:55:16.792 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:16.831 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:16.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:16 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T05:55:17.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:16 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T05:55:17.159 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:17.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:17.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:17.180 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:55:17.270 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:17.340 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:17.340 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:17.444 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:17.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:17.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:17.555 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:17.615 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:17.700 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:17.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:17.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:17 smithi008 ceph-mon[184418]: pgmap v1203: 99 pgs: 1 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 12463/216081 objects degraded (5.768%); 267/216081 objects misplaced (0.124%); 15 MiB/s, 234 objects/s recovering 2023-10-09T05:55:17.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:17.753 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:55:17.800 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:55:17.800 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:55:17.801 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T05:55:17.801 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T05:55:17.809 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:17.929 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:17.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:17 smithi008 ceph-mon[191227]: pgmap v1203: 99 pgs: 1 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 12463/216081 objects degraded (5.768%); 267/216081 objects misplaced (0.124%); 15 MiB/s, 234 objects/s recovering 2023-10-09T05:55:18.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:17 smithi191 ceph-mon[161669]: pgmap v1203: 99 pgs: 1 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 12463/216081 objects degraded (5.768%); 267/216081 objects misplaced (0.124%); 15 MiB/s, 234 objects/s recovering 2023-10-09T05:55:18.225 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:18.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:18.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:18.334 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:18.361 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:18.404 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:18.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:18.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:18.510 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:18.537 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:18.586 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:18.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:18.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:18.693 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:18.708 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 12463/216081 objects degraded (5.768%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:18.745 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:18.873 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:18.886 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:18.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:18.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:18.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 12463/216081 objects degraded (5.768%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:18.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 12463/216081 objects degraded (5.768%), 12 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:19.003 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:19.030 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:19.089 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:19.341 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:19.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:19.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:19.447 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:19.473 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:19.521 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:19.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:19.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:19.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:19.653 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:19.698 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:19.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:19.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:19.749 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:19.805 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:19.832 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:19.879 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:19.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:19.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:19.984 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:20.009 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:20.052 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:20.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:20.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:20.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:20.188 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:20.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:20 smithi008 ceph-mon[184418]: pgmap v1204: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 15142/214164 objects degraded (7.070%); 267/214164 objects misplaced (0.125%); 8.7 MiB/s, 136 objects/s recovering 2023-10-09T05:55:20.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:20 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1289039186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:20 smithi008 ceph-mon[191227]: pgmap v1204: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 15142/214164 objects degraded (7.070%); 267/214164 objects misplaced (0.125%); 8.7 MiB/s, 136 objects/s recovering 2023-10-09T05:55:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:20 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1289039186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:20.237 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:20.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:20.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:20.345 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:20.372 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:20.423 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:20.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:20.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:20.506 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:55:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:55:20] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.33.4" 2023-10-09T05:55:20.528 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:20.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:20 smithi191 ceph-mon[161669]: pgmap v1204: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 15142/214164 objects degraded (7.070%); 267/214164 objects misplaced (0.125%); 8.7 MiB/s, 136 objects/s recovering 2023-10-09T05:55:20.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:20 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1289039186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:20.559 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:55:20.559 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:55:20.559 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:55:20.560 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:55:20.560 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:55:20.560 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:55:20.560 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:55:20.560 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:55:20.560 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:55:20.560 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:55:20.560 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:55:20.560 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:55:20.561 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:55:20.561 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:55:20.561 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:55:20.561 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:55:20.563 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:21.346 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:21.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:21.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:21.450 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:21.494 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:22.317 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:22.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:22.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:22.423 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:22.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:22 smithi191 ceph-mon[161669]: pgmap v1205: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 14998/212736 objects degraded (7.050%); 267/212736 objects misplaced (0.126%); 8.8 MiB/s, 138 objects/s recovering 2023-10-09T05:55:22.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:22 smithi191 ceph-mon[161669]: failed to encode map e399 with expected crc 2023-10-09T05:55:22.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:22 smithi191 ceph-mon[161669]: osdmap e399: 8 total, 8 up, 7 in 2023-10-09T05:55:22.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:22 smithi191 ceph-mon[161669]: failed to encode map e399 with expected crc 2023-10-09T05:55:22.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:22 smithi191 ceph-mon[161669]: failed to encode map e399 with expected crc 2023-10-09T05:55:22.450 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:22.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:22 smithi008 ceph-mon[184418]: pgmap v1205: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 14998/212736 objects degraded (7.050%); 267/212736 objects misplaced (0.126%); 8.8 MiB/s, 138 objects/s recovering 2023-10-09T05:55:22.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:22 smithi008 ceph-mon[184418]: failed to encode map e399 with expected crc 2023-10-09T05:55:22.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:22 smithi008 ceph-mon[184418]: osdmap e399: 8 total, 8 up, 7 in 2023-10-09T05:55:22.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:22 smithi008 ceph-mon[184418]: failed to encode map e399 with expected crc 2023-10-09T05:55:22.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:22 smithi008 ceph-mon[184418]: failed to encode map e399 with expected crc 2023-10-09T05:55:22.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:22 smithi008 ceph-mon[191227]: pgmap v1205: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 14998/212736 objects degraded (7.050%); 267/212736 objects misplaced (0.126%); 8.8 MiB/s, 138 objects/s recovering 2023-10-09T05:55:22.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:22 smithi008 ceph-mon[191227]: failed to encode map e399 with expected crc 2023-10-09T05:55:22.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:22 smithi008 ceph-mon[191227]: osdmap e399: 8 total, 8 up, 7 in 2023-10-09T05:55:22.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:22 smithi008 ceph-mon[191227]: failed to encode map e399 with expected crc 2023-10-09T05:55:22.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:22 smithi008 ceph-mon[191227]: failed to encode map e399 with expected crc 2023-10-09T05:55:22.496 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:22.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:22.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:22.601 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:22.627 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:22.677 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:22.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:22.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:22.786 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:22.811 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:22.854 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:22.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:22.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:22.962 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:22.990 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:23.037 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:23.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:23.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:23.144 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:23.170 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:23.220 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:23.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:23.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:23.327 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:23.394 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:23.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[184418]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[184418]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[184418]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[184418]: osdmap e400: 8 total, 8 up, 7 in 2023-10-09T05:55:23.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[184418]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.469 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14998/212736 objects degraded (7.050%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:23.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[191227]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[191227]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[191227]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[191227]: osdmap e400: 8 total, 8 up, 7 in 2023-10-09T05:55:23.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[191227]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.470 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14998/212736 objects degraded (7.050%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:23 smithi191 ceph-mon[161669]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:23 smithi191 ceph-mon[161669]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:23 smithi191 ceph-mon[161669]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:23 smithi191 ceph-mon[161669]: osdmap e400: 8 total, 8 up, 7 in 2023-10-09T05:55:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:23 smithi191 ceph-mon[161669]: failed to encode map e400 with expected crc 2023-10-09T05:55:23.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14998/212736 objects degraded (7.050%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:23.689 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:23.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:23.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:23.798 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:23.832 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:23.889 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:23.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:23.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:23.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:24.021 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:24.069 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:24.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:24.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:24.175 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:24.201 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:24.245 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:24.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:24.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:24.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:24 smithi191 ceph-mon[161669]: pgmap v1208: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 11701/210273 objects degraded (5.565%); 1832/210273 objects misplaced (0.871%); 13 MiB/s, 262 objects/s recovering 2023-10-09T05:55:24.353 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:24.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:24 smithi008 ceph-mon[184418]: pgmap v1208: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 11701/210273 objects degraded (5.565%); 1832/210273 objects misplaced (0.871%); 13 MiB/s, 262 objects/s recovering 2023-10-09T05:55:24.415 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:24 smithi008 ceph-mon[191227]: pgmap v1208: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 11701/210273 objects degraded (5.565%); 1832/210273 objects misplaced (0.871%); 13 MiB/s, 262 objects/s recovering 2023-10-09T05:55:24.417 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:24.492 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:55:24.557 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:24.571 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:55:24.571 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:55:24.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:24.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:24.675 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:24.702 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:24.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:24 smithi008 conmon[184394]: 2023-10-09T05:55:24.462+0000 7f8a66521700 -1 mon.a@0(leader).osd e400 definitely_dead 0 2023-10-09T05:55:24.749 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:24.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:24.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:24.859 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:24.923 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:25.246 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:55:25.255 INFO:teuthology.orchestra.run.smithi008.stderr:marked in osd.7. 2023-10-09T05:55:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:25 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3186012042' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:55:25.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:25 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2228127980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:25.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:25 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3186012042' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:55:25.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:25 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2228127980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:25 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3186012042' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:55:25.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:25 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2228127980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:26.459 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[184418]: pgmap v1209: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 11701/210273 objects degraded (5.565%); 1832/210273 objects misplaced (0.871%); 6.9 MiB/s, 166 objects/s recovering 2023-10-09T05:55:26.459 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:26.459 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[184418]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.459 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[184418]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.460 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[184418]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.460 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3186012042' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:55:26.460 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2228127980' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:55:26.460 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[184418]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T05:55:26.460 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[184418]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.460 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[184418]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.460 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[191227]: pgmap v1209: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 11701/210273 objects degraded (5.565%); 1832/210273 objects misplaced (0.871%); 6.9 MiB/s, 166 objects/s recovering 2023-10-09T05:55:26.460 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:26.460 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[191227]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.460 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[191227]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.461 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[191227]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.461 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3186012042' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:55:26.461 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2228127980' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:55:26.461 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[191227]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T05:55:26.461 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[191227]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.461 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:26 smithi008 ceph-mon[191227]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.471 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:26.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:26.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:26.483 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:55:26.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:26 smithi191 ceph-mon[161669]: pgmap v1209: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 11701/210273 objects degraded (5.565%); 1832/210273 objects misplaced (0.871%); 6.9 MiB/s, 166 objects/s recovering 2023-10-09T05:55:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:26 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:26 smithi191 ceph-mon[161669]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:26 smithi191 ceph-mon[161669]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:26 smithi191 ceph-mon[161669]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3186012042' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:55:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2228127980' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:55:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:26 smithi191 ceph-mon[161669]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T05:55:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:26 smithi191 ceph-mon[161669]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:26 smithi191 ceph-mon[161669]: failed to encode map e401 with expected crc 2023-10-09T05:55:26.576 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:26.606 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:26.960 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:26.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:26.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:27.064 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:27.092 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:27.361 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[184418]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.361 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[184418]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.361 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[184418]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.361 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[184418]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T05:55:27.361 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[184418]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.361 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[184418]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.361 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[184418]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[184418]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[184418]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[184418]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[184418]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T05:55:27.362 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[191227]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.362 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[191227]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.362 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[191227]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.362 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[191227]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T05:55:27.362 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[191227]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[191227]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[191227]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[191227]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[191227]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[191227]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:27 smithi008 ceph-mon[191227]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T05:55:27.546 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:27.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:27.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:27.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:27 smithi191 ceph-mon[161669]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:27 smithi191 ceph-mon[161669]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:27 smithi191 ceph-mon[161669]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:27 smithi191 ceph-mon[161669]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T05:55:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:27 smithi191 ceph-mon[161669]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:27 smithi191 ceph-mon[161669]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:27 smithi191 ceph-mon[161669]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:27 smithi191 ceph-mon[161669]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:27 smithi191 ceph-mon[161669]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:27 smithi191 ceph-mon[161669]: failed to encode map e402 with expected crc 2023-10-09T05:55:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:27 smithi191 ceph-mon[161669]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T05:55:27.650 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:27.668 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T05:55:27.682 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:27.726 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:27.748 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:27.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:27.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:27.780 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:27.854 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:27.882 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:27.944 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:27.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:27.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:28.048 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:28.077 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:28.230 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:28.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:28.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:28.338 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:28.371 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[184418]: pgmap v1212: 99 pgs: 1 remapped+peering, 25 peering, 2 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 3.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 4464/186255 objects degraded (2.397%) 2023-10-09T05:55:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[184418]: failed to encode map e403 with expected crc 2023-10-09T05:55:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[184418]: 9.13 starting backfill to osd.7 from (0'0,371'2844] MIN to 400'5504 2023-10-09T05:55:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[184418]: failed to encode map e403 with expected crc 2023-10-09T05:55:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 7 pgs peering (PG_AVAILABILITY) 2023-10-09T05:55:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[184418]: failed to encode map e403 with expected crc 2023-10-09T05:55:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[184418]: 9.a starting backfill to osd.7 from (0'0,371'2829] MIN to 400'6252 2023-10-09T05:55:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[184418]: 9.11 starting backfill to osd.7 from (0'0,371'2851] MIN to 400'5579 2023-10-09T05:55:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 4464/186255 objects degraded (2.397%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[191227]: pgmap v1212: 99 pgs: 1 remapped+peering, 25 peering, 2 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 3.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 4464/186255 objects degraded (2.397%) 2023-10-09T05:55:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[191227]: failed to encode map e403 with expected crc 2023-10-09T05:55:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[191227]: 9.13 starting backfill to osd.7 from (0'0,371'2844] MIN to 400'5504 2023-10-09T05:55:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[191227]: failed to encode map e403 with expected crc 2023-10-09T05:55:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 7 pgs peering (PG_AVAILABILITY) 2023-10-09T05:55:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[191227]: failed to encode map e403 with expected crc 2023-10-09T05:55:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[191227]: 9.a starting backfill to osd.7 from (0'0,371'2829] MIN to 400'6252 2023-10-09T05:55:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[191227]: 9.11 starting backfill to osd.7 from (0'0,371'2851] MIN to 400'5579 2023-10-09T05:55:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 4464/186255 objects degraded (2.397%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:28 smithi191 ceph-mon[161669]: pgmap v1212: 99 pgs: 1 remapped+peering, 25 peering, 2 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 3.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 4464/186255 objects degraded (2.397%) 2023-10-09T05:55:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:28 smithi191 ceph-mon[161669]: failed to encode map e403 with expected crc 2023-10-09T05:55:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:28 smithi191 ceph-mon[161669]: 9.13 starting backfill to osd.7 from (0'0,371'2844] MIN to 400'5504 2023-10-09T05:55:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:28 smithi191 ceph-mon[161669]: failed to encode map e403 with expected crc 2023-10-09T05:55:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:28 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 7 pgs peering (PG_AVAILABILITY) 2023-10-09T05:55:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:28 smithi191 ceph-mon[161669]: failed to encode map e403 with expected crc 2023-10-09T05:55:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:28 smithi191 ceph-mon[161669]: 9.a starting backfill to osd.7 from (0'0,371'2829] MIN to 400'6252 2023-10-09T05:55:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:28 smithi191 ceph-mon[161669]: 9.11 starting backfill to osd.7 from (0'0,371'2851] MIN to 400'5579 2023-10-09T05:55:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 4464/186255 objects degraded (2.397%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:28.975 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:28.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:28.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:29.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:29.105 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:29.148 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:29.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:29.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:29.253 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:29.281 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:29.365 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:55:29.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:29.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:29.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:29.840 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:29.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:29.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:29.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:29.969 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:30.017 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:30.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:30.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:30.030 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:30.121 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:30.157 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:30.370 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:55:30.371 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:55:30.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:30 smithi008 ceph-mon[184418]: pgmap v1214: 99 pgs: 1 activating+remapped, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 19 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16334/203163 objects degraded (8.040%); 8711/203163 objects misplaced (4.288%); 2.0 MiB/s, 52 objects/s recovering 2023-10-09T05:55:30.397 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:30 smithi008 ceph-mon[191227]: pgmap v1214: 99 pgs: 1 activating+remapped, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 19 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16334/203163 objects degraded (8.040%); 8711/203163 objects misplaced (4.288%); 2.0 MiB/s, 52 objects/s recovering 2023-10-09T05:55:30.463 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:30.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:30.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:30.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:30 smithi191 ceph-mon[161669]: pgmap v1214: 99 pgs: 1 activating+remapped, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 19 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16334/203163 objects degraded (8.040%); 8711/203163 objects misplaced (4.288%); 2.0 MiB/s, 52 objects/s recovering 2023-10-09T05:55:30.569 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:30.596 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:30.638 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:30.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:30.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:30.654 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:55:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:55:30] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-10-09T05:55:30.745 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:30.778 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:31.349 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:55:31.379 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:31.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:31.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:31.484 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:31.515 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:31 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2524069668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:31.515 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:31 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2524069668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:31.519 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:31.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:31 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2524069668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:31.673 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:31.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:31.777 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:31.809 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:31.855 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:31.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:31.959 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:31.988 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:32.267 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:32.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:32.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:32.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:32.376 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:32.397 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:32.446 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:32.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:32.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:32.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:32 smithi191 ceph-mon[161669]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 2 activating+undersized+degraded+remapped, 1 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16308/203199 objects degraded (8.026%); 11051/203199 objects misplaced (5.439%); 2.9 MiB/s, 1 keys/s, 69 objects/s recovering 2023-10-09T05:55:32.551 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:32.579 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:32 smithi008 ceph-mon[184418]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 2 activating+undersized+degraded+remapped, 1 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16308/203199 objects degraded (8.026%); 11051/203199 objects misplaced (5.439%); 2.9 MiB/s, 1 keys/s, 69 objects/s recovering 2023-10-09T05:55:32.580 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:32 smithi008 ceph-mon[191227]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 2 activating+undersized+degraded+remapped, 1 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16308/203199 objects degraded (8.026%); 11051/203199 objects misplaced (5.439%); 2.9 MiB/s, 1 keys/s, 69 objects/s recovering 2023-10-09T05:55:32.582 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:32.668 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:55:32.668 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:55:32.669 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:55:32.669 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.675377 2023-10-09T05:55:32.669 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 0.6753772496795585 2023-10-09T05:55:32.833 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:32.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:32.938 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:32.969 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:33.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:33 smithi191 ceph-mon[161669]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:55:33.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16308/203199 objects degraded (8.026%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:33.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:33 smithi008 ceph-mon[184418]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:55:33.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16308/203199 objects degraded (8.026%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:33.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:33 smithi008 ceph-mon[191227]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:55:33.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16308/203199 objects degraded (8.026%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:33.763 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:33.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:33.867 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:33.902 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:34.016 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:34.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:34.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:34.121 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:34.146 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:34.190 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:34.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:34.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:34.276 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:55:34.277 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:34.295 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:34.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:34 smithi008 ceph-mon[184418]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16308/203199 objects degraded (8.026%); 11051/203199 objects misplaced (5.439%); 1.1 MiB/s, 1 keys/s, 36 objects/s recovering 2023-10-09T05:55:34.304 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3799727149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:34.304 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:34 smithi008 ceph-mon[191227]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16308/203199 objects degraded (8.026%); 11051/203199 objects misplaced (5.439%); 1.1 MiB/s, 1 keys/s, 36 objects/s recovering 2023-10-09T05:55:34.304 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3799727149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:34.334 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:34.388 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:55:34.388 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:55:34.443 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:34.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:34.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:34.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:34 smithi191 ceph-mon[161669]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16308/203199 objects degraded (8.026%); 11051/203199 objects misplaced (5.439%); 1.1 MiB/s, 1 keys/s, 36 objects/s recovering 2023-10-09T05:55:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3799727149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:34.572 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:34.614 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:34.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:34.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:34.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:34.748 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:34.791 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:34.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:34.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:34.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:34.923 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:34.966 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:34.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:34.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:35.072 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:35.098 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:35.140 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:35.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:35.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:35.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:35.274 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:35.321 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:35.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:35.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:35.427 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:35.461 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:35.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:35 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3799727149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:35 smithi191 ceph-mon[161669]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T05:55:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:35 smithi191 ceph-mon[161669]: failed to encode map e404 with expected crc 2023-10-09T05:55:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:35 smithi191 ceph-mon[161669]: failed to encode map e404 with expected crc 2023-10-09T05:55:35.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:35 smithi191 ceph-mon[161669]: failed to encode map e404 with expected crc 2023-10-09T05:55:35.696 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:35.696 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3799727149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:35.696 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[184418]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T05:55:35.696 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[184418]: failed to encode map e404 with expected crc 2023-10-09T05:55:35.696 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[184418]: failed to encode map e404 with expected crc 2023-10-09T05:55:35.697 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[184418]: failed to encode map e404 with expected crc 2023-10-09T05:55:35.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:35.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3799727149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:35.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[191227]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T05:55:35.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[191227]: failed to encode map e404 with expected crc 2023-10-09T05:55:35.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[191227]: failed to encode map e404 with expected crc 2023-10-09T05:55:35.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:35 smithi008 ceph-mon[191227]: failed to encode map e404 with expected crc 2023-10-09T05:55:35.716 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:55:35.750 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:35.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:35.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:35.880 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:35.932 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:35.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:36.037 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:36.065 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:36.111 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:36.111 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:36.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:36.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:36.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:36.244 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:36.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:36 smithi008 ceph-mon[184418]: pgmap v1218: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 75 MiB/s wr, 1.70k op/s; 16308/203199 objects degraded (8.026%); 11051/203199 objects misplaced (5.439%); 52 MiB/s, 1 keys/s, 910 objects/s recovering 2023-10-09T05:55:36.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:36 smithi008 ceph-mon[191227]: pgmap v1218: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 75 MiB/s wr, 1.70k op/s; 16308/203199 objects degraded (8.026%); 11051/203199 objects misplaced (5.439%); 52 MiB/s, 1 keys/s, 910 objects/s recovering 2023-10-09T05:55:36.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:36 smithi191 ceph-mon[161669]: pgmap v1218: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 75 MiB/s wr, 1.70k op/s; 16308/203199 objects degraded (8.026%); 11051/203199 objects misplaced (5.439%); 52 MiB/s, 1 keys/s, 910 objects/s recovering 2023-10-09T05:55:36.811 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:36.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:36.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:36.914 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:36.948 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:37.001 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:37.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:37.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:37.104 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:37.136 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:37.187 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:37.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:37.291 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:37.319 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:37.365 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:37.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:37.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:37.463 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:55:37.469 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:37.535 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:37.584 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:37.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:37.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:37.688 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:37.719 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:37.773 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:37.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:37.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:37.862 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:37.877 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:37.905 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:37.950 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:37.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:37.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:38.054 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:38.083 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:38.140 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:38.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:38.244 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:38.274 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:38.290 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.5 primary-affinity to 0.675377 (8442612) 2023-10-09T05:55:38.374 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:38.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:38.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:38.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:38 smithi008 ceph-mon[184418]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 109 B/s rd, 64 MiB/s wr, 1.45k op/s; 16308/203199 objects degraded (8.026%); 10935/203199 objects misplaced (5.381%); 45 MiB/s, 1 keys/s, 789 objects/s recovering 2023-10-09T05:55:38.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:38 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2814556580' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.6753772496795585}]: dispatch 2023-10-09T05:55:38.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:38 smithi008 ceph-mon[191227]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 109 B/s rd, 64 MiB/s wr, 1.45k op/s; 16308/203199 objects degraded (8.026%); 10935/203199 objects misplaced (5.381%); 45 MiB/s, 1 keys/s, 789 objects/s recovering 2023-10-09T05:55:38.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:38 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2814556580' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.6753772496795585}]: dispatch 2023-10-09T05:55:38.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:38.502 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:38 smithi191 ceph-mon[161669]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 109 B/s rd, 64 MiB/s wr, 1.45k op/s; 16308/203199 objects degraded (8.026%); 10935/203199 objects misplaced (5.381%); 45 MiB/s, 1 keys/s, 789 objects/s recovering 2023-10-09T05:55:38.502 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:38 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2814556580' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.6753772496795585}]: dispatch 2023-10-09T05:55:38.505 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:38.551 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:38.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:38.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:38.657 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:38.689 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:38.699 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:55:38.725 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:55:38.744 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:55:38.763 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:55:38.781 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:55:38.799 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:55:38.817 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:55:38.834 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:55:38.836 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:55:38.836 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:55:38.836 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:55:38.836 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:55:38.836 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:55:38.836 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:55:38.837 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:55:38.837 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:55:39.322 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:39.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:39.426 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:39.453 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:39.501 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:39.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:39.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:39.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:39 smithi191 ceph-mon[161669]: failed to encode map e405 with expected crc 2023-10-09T05:55:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2814556580' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.6753772496795585}]': finished 2023-10-09T05:55:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:39 smithi191 ceph-mon[161669]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T05:55:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:39 smithi191 ceph-mon[161669]: failed to encode map e405 with expected crc 2023-10-09T05:55:39.604 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:39.633 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:39 smithi008 ceph-mon[184418]: failed to encode map e405 with expected crc 2023-10-09T05:55:39.633 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2814556580' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.6753772496795585}]': finished 2023-10-09T05:55:39.633 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:39 smithi008 ceph-mon[184418]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T05:55:39.633 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:39 smithi008 ceph-mon[184418]: failed to encode map e405 with expected crc 2023-10-09T05:55:39.634 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:39 smithi008 ceph-mon[191227]: failed to encode map e405 with expected crc 2023-10-09T05:55:39.634 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2814556580' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.6753772496795585}]': finished 2023-10-09T05:55:39.634 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:39 smithi008 ceph-mon[191227]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T05:55:39.634 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:39 smithi008 ceph-mon[191227]: failed to encode map e405 with expected crc 2023-10-09T05:55:39.636 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:40.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:40 smithi191 ceph-mon[161669]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16308/203199 objects degraded (8.026%); 10935/203199 objects misplaced (5.381%); 928 KiB/s, 14 objects/s recovering 2023-10-09T05:55:40.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:40 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16308/203199 objects degraded (8.026%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:55:40.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:40 smithi191 ceph-mon[161669]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:40 smithi191 ceph-mon[161669]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:40 smithi191 ceph-mon[161669]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:40 smithi191 ceph-mon[161669]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T05:55:40.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:40 smithi191 ceph-mon[161669]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.294 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:40.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:40.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:40.400 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:40.447 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:40.519 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:40.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:40.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:40.562 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:55:40.562 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:55:40.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.562 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:99476 9.c 9:32a7e87c:::benchmark_data_smithi008_502820_object15303:head [delete] snapc 0=[] ondisk+write+known_if_redirected e407)", 2023-10-09T05:55:40.562 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:40.554991+0000", 2023-10-09T05:55:40.563 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0068320580000000002, 2023-10-09T05:55:40.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068533639999999998, 2023-10-09T05:55:40.563 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:40.563 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:55:40.563 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:40.563 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:55:40.563 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:40.563 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99476 2023-10-09T05:55:40.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.564 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:40.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:40.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.554991+0000", 2023-10-09T05:55:40.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:40.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:40.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.554991+0000", 2023-10-09T05:55:40.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:40.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:40.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.554990+0000", 2023-10-09T05:55:40.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7619999999999998e-06 2023-10-09T05:55:40.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:40.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.554996+0000", 2023-10-09T05:55:40.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3597999999999999e-05 2023-10-09T05:55:40.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:40.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.555010+0000", 2023-10-09T05:55:40.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7649999999999999e-06 2023-10-09T05:55:40.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:40.568 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.555016+0000", 2023-10-09T05:55:40.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4258999999999998e-05 2023-10-09T05:55:40.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:40.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.555060+0000", 2023-10-09T05:55:40.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013414500000000001 2023-10-09T05:55:40.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:40.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.555194+0000", 2023-10-09T05:55:40.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011669 2023-10-09T05:55:40.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.571 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:40.571 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.555311+0000", 2023-10-09T05:55:40.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031315900000000001 2023-10-09T05:55:40.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.571 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:40.571 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.558442+0000", 2023-10-09T05:55:40.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00031598500000000002 2023-10-09T05:55:40.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:40.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.558758+0000", 2023-10-09T05:55:40.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003767261 2023-10-09T05:55:40.572 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:40.572 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:40.573 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:40.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.573 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "MOSDPGPushReply(9.c 407/401 [PushReplyOp(9:37e1522a:::benchmark_data_smithi008_502820_object50403:head)])", 2023-10-09T05:55:40.573 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:40.561282+0000", 2023-10-09T05:55:40.573 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.00054108100000000001, 2023-10-09T05:55:40.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00063315599999999995, 2023-10-09T05:55:40.573 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:40.574 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:55:40.574 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:40.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:40.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561282+0000", 2023-10-09T05:55:40.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:40.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:40.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561282+0000", 2023-10-09T05:55:40.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:55:40.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:40.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561281+0000", 2023-10-09T05:55:40.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0809999999999996e-06 2023-10-09T05:55:40.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:40.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561288+0000", 2023-10-09T05:55:40.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2965999999999999e-05 2023-10-09T05:55:40.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:40.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561301+0000", 2023-10-09T05:55:40.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6509999999999998e-06 2023-10-09T05:55:40.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561307+0000", 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5074000000000002e-05 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561362+0000", 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9898e-05 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:40.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.579 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.36258.0:99476 9.c e407/401)", 2023-10-09T05:55:40.579 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:40.561321+0000", 2023-10-09T05:55:40.579 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.00050262400000000002, 2023-10-09T05:55:40.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00062436299999999996, 2023-10-09T05:55:40.579 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:40.579 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2023-10-09T05:55:40.579 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:40.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:40.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561321+0000", 2023-10-09T05:55:40.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:40.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:40.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561321+0000", 2023-10-09T05:55:40.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967296 2023-10-09T05:55:40.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:40.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561320+0000", 2023-10-09T05:55:40.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6879999999999999e-06 2023-10-09T05:55:40.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:40.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561323+0000", 2023-10-09T05:55:40.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0484e-05 2023-10-09T05:55:40.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:40.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561334+0000", 2023-10-09T05:55:40.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9140000000000001e-06 2023-10-09T05:55:40.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:40.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:40.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:40.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.561337+0000", 2023-10-09T05:55:40.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6875e-05 2023-10-09T05:55:40.584 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:40.584 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:40.584 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:40.584 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:40.584 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:55:40.585 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 3 2023-10-09T05:55:40.585 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:55:40.590 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:55:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:55:40] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-10-09T05:55:40.590 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[184418]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16308/203199 objects degraded (8.026%); 10935/203199 objects misplaced (5.381%); 928 KiB/s, 14 objects/s recovering 2023-10-09T05:55:40.590 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16308/203199 objects degraded (8.026%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:55:40.590 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[184418]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.590 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[184418]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.591 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[184418]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.591 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[184418]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T05:55:40.591 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[184418]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.591 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[191227]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 16308/203199 objects degraded (8.026%); 10935/203199 objects misplaced (5.381%); 928 KiB/s, 14 objects/s recovering 2023-10-09T05:55:40.591 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16308/203199 objects degraded (8.026%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:55:40.591 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[191227]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.591 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[191227]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.591 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[191227]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.591 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[191227]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T05:55:40.591 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:40 smithi008 ceph-mon[191227]: failed to encode map e406 with expected crc 2023-10-09T05:55:40.625 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:40.656 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:40.751 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:40.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:40.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:40.854 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:40.887 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:40.948 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:40.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:40.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:41.052 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:41.079 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:41.297 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:55:41.391 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:41 smithi191 ceph-mon[161669]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.391 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:41 smithi191 ceph-mon[161669]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T05:55:41.391 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:41 smithi191 ceph-mon[161669]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.391 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:41 smithi191 ceph-mon[161669]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.391 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:41 smithi191 ceph-mon[161669]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.391 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:41 smithi191 ceph-mon[161669]: 9.a continuing backfill to osd.7 from (0'0,400'6252] MIN to 400'6252 2023-10-09T05:55:41.391 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:41 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3255636567' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:41.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:55:41.421 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:41.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:41.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:41.528 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:41.559 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[184418]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.559 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[184418]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T05:55:41.559 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[184418]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.560 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[184418]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.560 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[184418]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.560 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[184418]: 9.a continuing backfill to osd.7 from (0'0,400'6252] MIN to 400'6252 2023-10-09T05:55:41.560 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3255636567' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:41.560 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[191227]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.560 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[191227]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T05:55:41.560 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[191227]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.560 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[191227]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.560 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[191227]: failed to encode map e407 with expected crc 2023-10-09T05:55:41.561 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[191227]: 9.a continuing backfill to osd.7 from (0'0,400'6252] MIN to 400'6252 2023-10-09T05:55:41.561 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:41 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3255636567' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:41.563 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:41.619 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:41.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:41.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:41.726 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:41.754 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:41.801 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:41.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:41.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:41.808 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:41.905 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:41.931 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:42.032 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:42.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:42.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:42.139 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:42.170 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:42.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:42.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:42.321 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:42.355 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:42.376 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:55:42.412 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:42.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:42.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:42.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[184418]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 25 MiB/s wr, 553 op/s; 20883/207363 objects degraded (10.071%); 11615/207363 objects misplaced (5.601%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T05:55:42.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:55:42.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:42.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[184418]: failed to encode map e408 with expected crc 2023-10-09T05:55:42.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[184418]: failed to encode map e408 with expected crc 2023-10-09T05:55:42.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3255636567' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:42.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[184418]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T05:55:42.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[184418]: failed to encode map e408 with expected crc 2023-10-09T05:55:42.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[191227]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 25 MiB/s wr, 553 op/s; 20883/207363 objects degraded (10.071%); 11615/207363 objects misplaced (5.601%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T05:55:42.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:55:42.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:42.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[191227]: failed to encode map e408 with expected crc 2023-10-09T05:55:42.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[191227]: failed to encode map e408 with expected crc 2023-10-09T05:55:42.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3255636567' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:42.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[191227]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T05:55:42.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:42 smithi008 ceph-mon[191227]: failed to encode map e408 with expected crc 2023-10-09T05:55:42.517 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:42 smithi191 ceph-mon[161669]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 25 MiB/s wr, 553 op/s; 20883/207363 objects degraded (10.071%); 11615/207363 objects misplaced (5.601%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T05:55:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:42 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:55:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:42 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:42 smithi191 ceph-mon[161669]: failed to encode map e408 with expected crc 2023-10-09T05:55:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:42 smithi191 ceph-mon[161669]: failed to encode map e408 with expected crc 2023-10-09T05:55:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3255636567' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:42 smithi191 ceph-mon[161669]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T05:55:42.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:42 smithi191 ceph-mon[161669]: failed to encode map e408 with expected crc 2023-10-09T05:55:42.551 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:42.605 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:42.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:42.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:42.713 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:42.742 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:42.807 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:42.848 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:42.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:42.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:42.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:42.984 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:43.035 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:43.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:43.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:43.140 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:43.171 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:43.221 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:43.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:43.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:43.326 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:43.358 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:43.430 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:55:43.430 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:55:43.430 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:55:43.430 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:55:43.430 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:55:43.513 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:43.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:43.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:43.616 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:43.645 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:43.688 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:43.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:43.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:43.792 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:43.821 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:44.068 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:44.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:44.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:44.174 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:44.203 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:55:44.205 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:55:44.212 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:44.261 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:44.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:44.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:44.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:44 smithi008 ceph-mon[184418]: pgmap v1226: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 236 B/s rd, 34 MiB/s wr, 826 op/s; 20883/206583 objects degraded (10.109%); 11573/206583 objects misplaced (5.602%); 9.4 MiB/s, 156 objects/s recovering 2023-10-09T05:55:44.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:44.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:44.366 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:44 smithi008 ceph-mon[191227]: pgmap v1226: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 236 B/s rd, 34 MiB/s wr, 826 op/s; 20883/206583 objects degraded (10.109%); 11573/206583 objects misplaced (5.602%); 9.4 MiB/s, 156 objects/s recovering 2023-10-09T05:55:44.366 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:44.366 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:44.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:44.394 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:44.448 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:44.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:44 smithi191 ceph-mon[161669]: pgmap v1226: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 236 B/s rd, 34 MiB/s wr, 826 op/s; 20883/206583 objects degraded (10.109%); 11573/206583 objects misplaced (5.602%); 9.4 MiB/s, 156 objects/s recovering 2023-10-09T05:55:44.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:55:44.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:44.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:44.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:44.553 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:44.574 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:55:44.583 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:44.606 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:55:44.625 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:44.636 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:55:44.642 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:44.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:44.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:44.661 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:44.661 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:55:44.694 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:55:44.719 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:55:44.720 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:55:44.720 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:55:44.720 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 2, 3, 4, 5, 6, 1, 7] 2023-10-09T05:55:44.720 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd out 3 2023-10-09T05:55:44.748 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:44.751 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:55:44.777 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:44.786 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:55:44.787 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:55:44.788 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:55:44.788 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:55:44.788 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:55:44.788 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:55:44.788 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:55:44.788 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:55:44.788 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:55:44.882 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:44.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:44.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:44.986 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:45.013 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:45.212 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:45.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:45.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:45.317 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:45.339 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1160712305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:45.340 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1160712305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:45.347 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:45.392 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:45.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:45.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:45.497 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:45.528 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:45.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1160712305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:45.740 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:45.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:45.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:45.843 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:45.872 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:46.258 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:55:46.258 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:55:46.318 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:46.356 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:46.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:46.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:46.463 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:46.498 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:46.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:46 smithi191 ceph-mon[161669]: pgmap v1227: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 170 B/s rd, 25 MiB/s wr, 631 op/s; 20836/205950 objects degraded (10.117%); 11617/205950 objects misplaced (5.641%); 7.1 MiB/s, 117 objects/s recovering 2023-10-09T05:55:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:46 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20836/205950 objects degraded (10.117%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:46.579 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:46 smithi008 ceph-mon[184418]: pgmap v1227: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 170 B/s rd, 25 MiB/s wr, 631 op/s; 20836/205950 objects degraded (10.117%); 11617/205950 objects misplaced (5.641%); 7.1 MiB/s, 117 objects/s recovering 2023-10-09T05:55:46.579 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:46 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20836/205950 objects degraded (10.117%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:46.580 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:46 smithi008 ceph-mon[191227]: pgmap v1227: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 170 B/s rd, 25 MiB/s wr, 631 op/s; 20836/205950 objects degraded (10.117%); 11617/205950 objects misplaced (5.641%); 7.1 MiB/s, 117 objects/s recovering 2023-10-09T05:55:46.580 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:46 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20836/205950 objects degraded (10.117%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:46.647 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:46.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:46.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:46.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:46.780 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:47.064 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:55:47.080 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:47.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:47.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:47.185 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:47.215 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:47.274 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:47.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:47.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:47.381 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:47.412 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:47.463 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:47.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:47.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:47.485 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:47.567 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:47.596 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:47.842 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:47.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:47.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:47.949 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:48.000 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:48.103 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:48.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:48.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:48.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:48.236 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:48.281 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:48.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:48.287 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:55:48.318 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:55:48.384 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:48.429 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:48 smithi008 ceph-mon[184418]: pgmap v1228: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 139 B/s rd, 0 B/s wr, 78 op/s; 20836/205683 objects degraded (10.130%); 11492/205683 objects misplaced (5.587%); 2.1 MiB/s, 36 objects/s recovering 2023-10-09T05:55:48.429 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:48 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4088235569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:48.429 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:48 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:48.429 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:48 smithi008 ceph-mon[191227]: pgmap v1228: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 139 B/s rd, 0 B/s wr, 78 op/s; 20836/205683 objects degraded (10.130%); 11492/205683 objects misplaced (5.587%); 2.1 MiB/s, 36 objects/s recovering 2023-10-09T05:55:48.429 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:48 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4088235569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:48.430 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:48 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:48.432 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:48.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:48 smithi191 ceph-mon[161669]: pgmap v1228: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 139 B/s rd, 0 B/s wr, 78 op/s; 20836/205683 objects degraded (10.130%); 11492/205683 objects misplaced (5.587%); 2.1 MiB/s, 36 objects/s recovering 2023-10-09T05:55:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:48 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4088235569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:48 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:48.589 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:48.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:48.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:48.693 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:48.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:48 smithi008 conmon[184394]: 2023-10-09T05:55:48.614+0000 7f8a66521700 -1 mon.a@0(leader).osd e409 definitely_dead 0 2023-10-09T05:55:48.724 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:48.805 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:48.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:48.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:48.908 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:48.939 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:49.139 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:55:49.139 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:55:49.146 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:49.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:49.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:49.178 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:49.252 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:49.280 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:49.351 INFO:teuthology.orchestra.run.smithi008.stderr:marked out osd.3. 2023-10-09T05:55:49.509 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:49.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:49.613 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:49.640 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:49 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:49.640 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:49 smithi191 ceph-mon[161669]: failed to encode map e409 with expected crc 2023-10-09T05:55:49.640 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:49 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:49.640 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:49 smithi191 ceph-mon[161669]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T05:55:49.640 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:49 smithi191 ceph-mon[161669]: failed to encode map e409 with expected crc 2023-10-09T05:55:49.641 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4252090246' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T05:55:49.641 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:49 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T05:55:49.641 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:49 smithi191 ceph-mon[161669]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T05:55:49.643 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:49.649 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:49.649 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[184418]: failed to encode map e409 with expected crc 2023-10-09T05:55:49.649 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:49.650 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[184418]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T05:55:49.650 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[184418]: failed to encode map e409 with expected crc 2023-10-09T05:55:49.650 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4252090246' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T05:55:49.650 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T05:55:49.650 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[184418]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T05:55:49.650 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:49.650 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[191227]: failed to encode map e409 with expected crc 2023-10-09T05:55:49.650 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:49.650 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[191227]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T05:55:49.651 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[191227]: failed to encode map e409 with expected crc 2023-10-09T05:55:49.651 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4252090246' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T05:55:49.651 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T05:55:49.651 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:49 smithi008 ceph-mon[191227]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T05:55:49.688 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:49.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:49.791 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:49.818 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:49.934 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:49.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:49.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:49.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:55:50.037 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:50.126 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:50.172 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:50.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:50.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:50.278 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:50.306 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:50.374 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:50.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:50.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:50.438 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:50.480 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:50.512 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:50.568 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:50.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:50.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:50.634 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:50 smithi191 ceph-mon[161669]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 0 B/s wr, 111 op/s; 20836/204696 objects degraded (10.179%); 11444/204696 objects misplaced (5.591%); 1024 KiB/s, 19 objects/s recovering 2023-10-09T05:55:50.634 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:50 smithi191 ceph-mon[161669]: failed to encode map e410 with expected crc 2023-10-09T05:55:50.634 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:50 smithi191 ceph-mon[161669]: failed to encode map e410 with expected crc 2023-10-09T05:55:50.634 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:50 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T05:55:50.634 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:50 smithi191 ceph-mon[161669]: osdmap e410: 8 total, 8 up, 7 in 2023-10-09T05:55:50.672 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:50.699 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:50.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:50 smithi008 ceph-mon[184418]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 0 B/s wr, 111 op/s; 20836/204696 objects degraded (10.179%); 11444/204696 objects misplaced (5.591%); 1024 KiB/s, 19 objects/s recovering 2023-10-09T05:55:50.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:50 smithi008 ceph-mon[184418]: failed to encode map e410 with expected crc 2023-10-09T05:55:50.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:50 smithi008 ceph-mon[184418]: failed to encode map e410 with expected crc 2023-10-09T05:55:50.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:50 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T05:55:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:50 smithi008 ceph-mon[184418]: osdmap e410: 8 total, 8 up, 7 in 2023-10-09T05:55:50.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:50 smithi008 ceph-mon[191227]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 0 B/s wr, 111 op/s; 20836/204696 objects degraded (10.179%); 11444/204696 objects misplaced (5.591%); 1024 KiB/s, 19 objects/s recovering 2023-10-09T05:55:50.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:50 smithi008 ceph-mon[191227]: failed to encode map e410 with expected crc 2023-10-09T05:55:50.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:50 smithi008 ceph-mon[191227]: failed to encode map e410 with expected crc 2023-10-09T05:55:50.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:50 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T05:55:50.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:50 smithi008 ceph-mon[191227]: osdmap e410: 8 total, 8 up, 7 in 2023-10-09T05:55:50.716 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:55:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:55:50] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.33.4" 2023-10-09T05:55:50.741 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:50.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:50.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:50.844 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:50.874 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:51.101 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:55:51.126 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:51.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:51.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:51.230 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:51.263 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:51.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[184418]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[184418]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[184418]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[184418]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[184418]: osdmap e411: 8 total, 8 up, 7 in 2023-10-09T05:55:51.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[184418]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[184418]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[184418]: 9.11 continuing backfill to osd.7 from (0'0,409'5627] MIN to 409'5627 2023-10-09T05:55:51.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[191227]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[191227]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[191227]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[191227]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[191227]: osdmap e411: 8 total, 8 up, 7 in 2023-10-09T05:55:51.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[191227]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[191227]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:51 smithi008 ceph-mon[191227]: 9.11 continuing backfill to osd.7 from (0'0,409'5627] MIN to 409'5627 2023-10-09T05:55:51.555 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:51.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:51.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:51.662 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:51.697 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:51.752 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:51.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:51.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:51.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:51 smithi191 ceph-mon[161669]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:51 smithi191 ceph-mon[161669]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:51 smithi191 ceph-mon[161669]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:51 smithi191 ceph-mon[161669]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:51 smithi191 ceph-mon[161669]: osdmap e411: 8 total, 8 up, 7 in 2023-10-09T05:55:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:51 smithi191 ceph-mon[161669]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:51 smithi191 ceph-mon[161669]: failed to encode map e411 with expected crc 2023-10-09T05:55:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:51 smithi191 ceph-mon[161669]: 9.11 continuing backfill to osd.7 from (0'0,409'5627] MIN to 409'5627 2023-10-09T05:55:51.821 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:55:51.828 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:51.843 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:55:51.857 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:51.865 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:55:51.891 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:55:51.891 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:51.918 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:55:51.944 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:55:51.971 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:55:51.975 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:51.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:51.993 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:55:51.994 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:55:51.994 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:55:51.994 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:55:51.995 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:55:51.995 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:55:51.995 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:55:51.995 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:55:51.995 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:55:52.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:52.108 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:55:52.108 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:55:52.108 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:55:52.108 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:55:52.108 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:55:52.126 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:52.294 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:52.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:52.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:52.399 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:52.429 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: pgmap v1233: 99 pgs: 1 remapped+peering, 15 peering, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 20791/203676 objects degraded (10.208%); 9434/203676 objects misplaced (4.632%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:52.429 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20791/203676 objects degraded (10.208%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:52.429 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.430 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.430 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.430 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: osdmap e412: 8 total, 8 up, 7 in 2023-10-09T05:55:52.430 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.430 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.430 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: 9.13 continuing backfill to osd.7 from (0'0,409'5547] MIN to 409'5547 2023-10-09T05:55:52.430 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.430 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.430 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3425789455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:52.433 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: pgmap v1233: 99 pgs: 1 remapped+peering, 15 peering, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 20791/203676 objects degraded (10.208%); 9434/203676 objects misplaced (4.632%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20791/203676 objects degraded (10.208%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: osdmap e412: 8 total, 8 up, 7 in 2023-10-09T05:55:52.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: 9.13 continuing backfill to osd.7 from (0'0,409'5547] MIN to 409'5547 2023-10-09T05:55:52.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3425789455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:52.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: pgmap v1233: 99 pgs: 1 remapped+peering, 15 peering, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 20791/203676 objects degraded (10.208%); 9434/203676 objects misplaced (4.632%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:52.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20791/203676 objects degraded (10.208%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: osdmap e412: 8 total, 8 up, 7 in 2023-10-09T05:55:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: 9.13 continuing backfill to osd.7 from (0'0,409'5547] MIN to 409'5547 2023-10-09T05:55:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: failed to encode map e412 with expected crc 2023-10-09T05:55:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3425789455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:52.484 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:52.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:52.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:52.589 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:52.623 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:52.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:55:53.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[184418]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[184418]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[184418]: osdmap e413: 8 total, 8 up, 7 in 2023-10-09T05:55:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[184418]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[184418]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[184418]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[184418]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[184418]: pgmap v1236: 99 pgs: 1 activating+remapped, 1 remapped+peering, 15 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 31351/203385 objects degraded (15.415%); 9420/203385 objects misplaced (4.632%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:55:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[191227]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[191227]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[191227]: osdmap e413: 8 total, 8 up, 7 in 2023-10-09T05:55:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[191227]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[191227]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[191227]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[191227]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:53 smithi008 ceph-mon[191227]: pgmap v1236: 99 pgs: 1 activating+remapped, 1 remapped+peering, 15 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 31351/203385 objects degraded (15.415%); 9420/203385 objects misplaced (4.632%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:55:53.472 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:53.714 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:53 smithi191 ceph-mon[161669]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.714 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:53 smithi191 ceph-mon[161669]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.714 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:53 smithi191 ceph-mon[161669]: osdmap e413: 8 total, 8 up, 7 in 2023-10-09T05:55:53.714 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:53 smithi191 ceph-mon[161669]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.714 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:53 smithi191 ceph-mon[161669]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.714 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:53 smithi191 ceph-mon[161669]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.714 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:53 smithi191 ceph-mon[161669]: failed to encode map e413 with expected crc 2023-10-09T05:55:53.714 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:53 smithi191 ceph-mon[161669]: pgmap v1236: 99 pgs: 1 activating+remapped, 1 remapped+peering, 15 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 31351/203385 objects degraded (15.415%); 9420/203385 objects misplaced (4.632%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:55:53.727 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:53.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:53.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:53.832 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:53.865 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:53.967 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:53.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:53.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:54.071 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:54.104 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:54.158 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:54.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:54.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:54.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:54.288 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:54.340 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:54.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:54.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:54.409 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:55:54.454 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:54.489 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:54.593 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:54.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:54.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:54.682 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:54 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2570764468' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:54.683 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:54 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:54.697 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:54.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:54 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2570764468' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:54.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:54 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:54 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2570764468' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:54.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:54 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:54.728 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:54.777 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:54.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:54.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:54.881 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:54.909 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:54.957 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:54.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:54.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:55.061 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:55.090 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:55.142 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:55:55.142 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:55:55.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:55:55.142 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:55:55.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.143 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180287 7.0 7:0049df52:::benchmark_data_smithi008_354649_object85791:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:55:55.143 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:41.442036+0000", 2023-10-09T05:55:55.143 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 553.69687696100004, 2023-10-09T05:55:55.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.551793397, 2023-10-09T05:55:55.144 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.144 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.144 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.144 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:55:55.145 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:55:55.145 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180287 2023-10-09T05:55:55.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.145 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442036+0000", 2023-10-09T05:55:55.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442036+0000", 2023-10-09T05:55:55.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:55.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442034+0000", 2023-10-09T05:55:55.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9820000000000003e-06 2023-10-09T05:55:55.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442044+0000", 2023-10-09T05:55:55.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9191999999999999e-05 2023-10-09T05:55:55.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442063+0000", 2023-10-09T05:55:55.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6090000000000001e-06 2023-10-09T05:55:55.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442072+0000", 2023-10-09T05:55:55.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4536999999999996e-05 2023-10-09T05:55:55.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442137+0000", 2023-10-09T05:55:55.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1471000000000001e-05 2023-10-09T05:55:55.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.442148+0000", 2023-10-09T05:55:55.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.53468159800000004 2023-10-09T05:55:55.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.976830+0000", 2023-10-09T05:55:55.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.429e-05 2023-10-09T05:55:55.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:41.976844+0000", 2023-10-09T05:55:55.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99917367300000004 2023-10-09T05:55:55.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.976018+0000", 2023-10-09T05:55:55.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00050756399999999995 2023-10-09T05:55:55.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:55.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.976525+0000", 2023-10-09T05:55:55.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0066100669999999999 2023-10-09T05:55:55.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.983135+0000", 2023-10-09T05:55:55.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8519999999999999e-05 2023-10-09T05:55:55.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.983234+0000", 2023-10-09T05:55:55.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00039802300000000001 2023-10-09T05:55:55.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:55:55.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.983632+0000", 2023-10-09T05:55:55.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045434739999999996 2023-10-09T05:55:55.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.988175+0000", 2023-10-09T05:55:55.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002886684 2023-10-09T05:55:55.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.991062+0000", 2023-10-09T05:55:55.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025684789999999998 2023-10-09T05:55:55.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.993630+0000", 2023-10-09T05:55:55.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2209000000000002e-05 2023-10-09T05:55:55.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.993673+0000", 2023-10-09T05:55:55.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000157071 2023-10-09T05:55:55.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.993830+0000", 2023-10-09T05:55:55.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.551793397 2023-10-09T05:55:55.165 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.165 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.165 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.165 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180312 7.1b 7:dfc0fe71:::benchmark_data_smithi008_354649_object85816:head [delete] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-10-09T05:55:55.165 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:42.080432+0000", 2023-10-09T05:55:55.165 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 553.05848119799998, 2023-10-09T05:55:55.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9004082920000001, 2023-10-09T05:55:55.166 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.166 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.166 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.166 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:55:55.166 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:55:55.166 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180312 2023-10-09T05:55:55.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.166 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080432+0000", 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080432+0000", 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080430+0000", 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4959999999999999e-06 2023-10-09T05:55:55.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080438+0000", 2023-10-09T05:55:55.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4079e-05 2023-10-09T05:55:55.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080452+0000", 2023-10-09T05:55:55.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9e-06 2023-10-09T05:55:55.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080459+0000", 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7777e-05 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080506+0000", 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8789999999999998e-06 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.080516+0000", 2023-10-09T05:55:55.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86410833499999995 2023-10-09T05:55:55.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.944625+0000", 2023-10-09T05:55:55.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0523000000000001e-05 2023-10-09T05:55:55.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.944635+0000", 2023-10-09T05:55:55.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.029066319 2023-10-09T05:55:55.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.973701+0000", 2023-10-09T05:55:55.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000104314 2023-10-09T05:55:55.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.973806+0000", 2023-10-09T05:55:55.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105163 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.973911+0000", 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048823 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.978793+0000", 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001543518 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980337+0000", 2023-10-09T05:55:55.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018337200000000001 2023-10-09T05:55:55.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980520+0000", 2023-10-09T05:55:55.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000125826 2023-10-09T05:55:55.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980646+0000", 2023-10-09T05:55:55.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9175000000000001e-05 2023-10-09T05:55:55.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980715+0000", 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012521 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980840+0000", 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9004082920000001 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.175 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45365.0:180373 7.1b 7:d95ec086:::benchmark_data_smithi008_354649_object85877:head [delete] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-10-09T05:55:55.176 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:46:42.192520+0000", 2023-10-09T05:55:55.176 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 552.94639366399997, 2023-10-09T05:55:55.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7885932069999999, 2023-10-09T05:55:55.176 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.176 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.176 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45365", 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 180373 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192520+0000", 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192520+0000", 2023-10-09T05:55:55.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:55.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192518+0000", 2023-10-09T05:55:55.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.024e-06 2023-10-09T05:55:55.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192525+0000", 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1698999999999999e-05 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192537+0000", 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9819999999999999e-06 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192542+0000", 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9879000000000001e-05 2023-10-09T05:55:55.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192592+0000", 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.173e-06 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.192598+0000", 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.75205944400000002 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.944657+0000", 2023-10-09T05:55:55.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2369999999999997e-06 2023-10-09T05:55:55.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:42.944664+0000", 2023-10-09T05:55:55.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.030814664 2023-10-09T05:55:55.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.975478+0000", 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014284399999999999 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.975621+0000", 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015404700000000001 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.975775+0000", 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030813910000000002 2023-10-09T05:55:55.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.978856+0000", 2023-10-09T05:55:55.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001523546 2023-10-09T05:55:55.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980380+0000", 2023-10-09T05:55:55.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000185644 2023-10-09T05:55:55.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980566+0000", 2023-10-09T05:55:55.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000327907 2023-10-09T05:55:55.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980894+0000", 2023-10-09T05:55:55.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1761000000000006e-05 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.980965+0000", 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000147479 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:46:43.981113+0000", 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7885932069999999 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.185 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86696 8.14 8:2cb5a248:::benchmark_data_smithi008_432048_object86695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078540+0000", 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.06037307899999, 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.192233133, 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:55:55.186 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86696 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078540+0000", 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078540+0000", 2023-10-09T05:55:55.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:55.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078538+0000", 2023-10-09T05:55:55.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5841000000000003e-05 2023-10-09T05:55:55.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078594+0000", 2023-10-09T05:55:55.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4001e-05 2023-10-09T05:55:55.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078608+0000", 2023-10-09T05:55:55.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3899999999999998e-06 2023-10-09T05:55:55.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078614+0000", 2023-10-09T05:55:55.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21596270400000001 2023-10-09T05:55:55.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294577+0000", 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7299999999999994e-06 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294586+0000", 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97611344200000005 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.270699+0000", 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4124999999999999e-05 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.270773+0000", 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.192233133 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.191 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86699 8.3 8:c3d6f3c4:::benchmark_data_smithi008_432048_object86698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.06028799699999, 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650868249999999, 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86699 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0989999999999999e-05 2023-10-09T05:55:55.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078666+0000", 2023-10-09T05:55:55.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7989999999999996e-06 2023-10-09T05:55:55.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078672+0000", 2023-10-09T05:55:55.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0949999999999998e-06 2023-10-09T05:55:55.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078676+0000", 2023-10-09T05:55:55.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15420414599999999 2023-10-09T05:55:55.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.232881+0000", 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6739999999999992e-06 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.232890+0000", 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0107454360000001 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.243636+0000", 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6348999999999997e-05 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.243712+0000", 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650868249999999 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86700 8.18 8:1e3af286:::benchmark_data_smithi008_432048_object86699:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:55:55.197 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078684+0000", 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.06022939399998, 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1857459349999999, 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86700 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078684+0000", 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078684+0000", 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078683+0000", 2023-10-09T05:55:55.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9190999999999998e-05 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078723+0000", 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0850000000000002e-06 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078729+0000", 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7340000000000002e-06 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078732+0000", 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.179763071 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258495+0000", 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8499999999999999e-06 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258501+0000", 2023-10-09T05:55:55.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0058671960000001 2023-10-09T05:55:55.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.264369+0000", 2023-10-09T05:55:55.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1284999999999993e-05 2023-10-09T05:55:55.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.264430+0000", 2023-10-09T05:55:55.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1857459349999999 2023-10-09T05:55:55.202 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.203 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.203 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.203 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86707 8.18 8:1d2a32f3:::benchmark_data_smithi008_432048_object86706:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:55:55.203 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078737+0000", 2023-10-09T05:55:55.203 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.06017649500001, 2023-10-09T05:55:55.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.195960744, 2023-10-09T05:55:55.203 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.203 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:55:55.204 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.204 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:55:55.204 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:55.204 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86707 2023-10-09T05:55:55.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.204 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078737+0000", 2023-10-09T05:55:55.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078737+0000", 2023-10-09T05:55:55.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:55.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078736+0000", 2023-10-09T05:55:55.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9480999999999997e-05 2023-10-09T05:55:55.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078776+0000", 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9616000000000002e-05 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078815+0000", 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2260000000000005e-06 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078824+0000", 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.179694931 2023-10-09T05:55:55.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258519+0000", 2023-10-09T05:55:55.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3319999999999999e-06 2023-10-09T05:55:55.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258524+0000", 2023-10-09T05:55:55.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0062883300000001 2023-10-09T05:55:55.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.264812+0000", 2023-10-09T05:55:55.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002849922 2023-10-09T05:55:55.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:55.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267662+0000", 2023-10-09T05:55:55.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0069641929999999996 2023-10-09T05:55:55.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274626+0000", 2023-10-09T05:55:55.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1110000000000002e-05 2023-10-09T05:55:55.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274697+0000", 2023-10-09T05:55:55.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.195960744 2023-10-09T05:55:55.210 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.211 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.211 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.211 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86710 8.14 8:28e4dd35:::benchmark_data_smithi008_432048_object86709:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:55:55.211 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:55:55.212 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.06008488499998, 2023-10-09T05:55:55.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2037845970000001, 2023-10-09T05:55:55.212 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.212 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:55:55.212 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.212 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:55:55.212 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:55.212 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86710 2023-10-09T05:55:55.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.213 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:55:55.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:55:55.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:55.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:55:55.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5683000000000001e-05 2023-10-09T05:55:55.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078894+0000", 2023-10-09T05:55:55.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5679999999999996e-06 2023-10-09T05:55:55.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078900+0000", 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0676000000000001e-05 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078911+0000", 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.215693774 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294605+0000", 2023-10-09T05:55:55.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2909999999999998e-06 2023-10-09T05:55:55.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294610+0000", 2023-10-09T05:55:55.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97619070799999996 2023-10-09T05:55:55.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.270801+0000", 2023-10-09T05:55:55.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037587950000000001 2023-10-09T05:55:55.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:55.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274559+0000", 2023-10-09T05:55:55.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079496329999999994 2023-10-09T05:55:55.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282509+0000", 2023-10-09T05:55:55.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103909 2023-10-09T05:55:55.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282613+0000", 2023-10-09T05:55:55.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2037845970000001 2023-10-09T05:55:55.219 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.219 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.219 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.220 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86715 8.14 8:2d9c7f0c:::benchmark_data_smithi008_432048_object86714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:55:55.220 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:55:55.220 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.05999717499998, 2023-10-09T05:55:55.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.203914588, 2023-10-09T05:55:55.220 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.220 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.220 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.220 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:55:55.220 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86715 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:55:55.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:55.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:55:55.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2407999999999997e-05 2023-10-09T05:55:55.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078968+0000", 2023-10-09T05:55:55.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4019999999999999e-06 2023-10-09T05:55:55.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078974+0000", 2023-10-09T05:55:55.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1539000000000001e-05 2023-10-09T05:55:55.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078986+0000", 2023-10-09T05:55:55.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.215643644 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294630+0000", 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5470000000000004e-06 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294635+0000", 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.979978393 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274614+0000", 2023-10-09T05:55:55.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014625400000000001 2023-10-09T05:55:55.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274760+0000", 2023-10-09T05:55:55.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017972999999999999 2023-10-09T05:55:55.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:55:55.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274940+0000", 2023-10-09T05:55:55.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017525329999999999 2023-10-09T05:55:55.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.276692+0000", 2023-10-09T05:55:55.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003165429 2023-10-09T05:55:55.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.279857+0000", 2023-10-09T05:55:55.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028214580000000002 2023-10-09T05:55:55.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282679+0000", 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9833000000000002e-05 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282729+0000", 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010188600000000001 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282831+0000", 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.203914588 2023-10-09T05:55:55.228 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86716 8.8 8:1416e0db:::benchmark_data_smithi008_432048_object86715:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.05992284899997, 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1828720580000001, 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.229 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:55:55.230 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:55.230 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86716 2023-10-09T05:55:55.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.230 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:55:55.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0300000000000002e-05 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.079050+0000", 2023-10-09T05:55:55.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0120000000000002e-06 2023-10-09T05:55:55.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.079056+0000", 2023-10-09T05:55:55.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.552e-06 2023-10-09T05:55:55.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.079060+0000", 2023-10-09T05:55:55.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17917138099999999 2023-10-09T05:55:55.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258231+0000", 2023-10-09T05:55:55.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0326000000000001e-05 2023-10-09T05:55:55.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258242+0000", 2023-10-09T05:55:55.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99156280299999999 2023-10-09T05:55:55.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.249804+0000", 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000172201 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.249977+0000", 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000205291 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:55.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.250182+0000", 2023-10-09T05:55:55.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0089602589999999999 2023-10-09T05:55:55.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.259142+0000", 2023-10-09T05:55:55.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025977299999999999 2023-10-09T05:55:55.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.236 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261740+0000", 2023-10-09T05:55:55.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1816999999999997e-05 2023-10-09T05:55:55.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.236 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261772+0000", 2023-10-09T05:55:55.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1427000000000001e-05 2023-10-09T05:55:55.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.237 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.237 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261823+0000", 2023-10-09T05:55:55.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9397999999999998e-05 2023-10-09T05:55:55.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261862+0000", 2023-10-09T05:55:55.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1828720580000001 2023-10-09T05:55:55.238 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.238 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.238 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.238 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86719 8.8 8:14442d3e:::benchmark_data_smithi008_432048_object86718:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-10-09T05:55:55.239 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.102694+0000", 2023-10-09T05:55:55.239 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 430.03621923499998, 2023-10-09T05:55:55.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650766299999999, 2023-10-09T05:55:55.239 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.239 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:55:55.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:55:55.239 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86719 2023-10-09T05:55:55.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.239 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102694+0000", 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102694+0000", 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999962 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102690+0000", 2023-10-09T05:55:55.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0977000000000002e-05 2023-10-09T05:55:55.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102751+0000", 2023-10-09T05:55:55.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2374e-05 2023-10-09T05:55:55.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102774+0000", 2023-10-09T05:55:55.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2845e-05 2023-10-09T05:55:55.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102786+0000", 2023-10-09T05:55:55.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15555860099999999 2023-10-09T05:55:55.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258345+0000", 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0730000000000005e-06 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258353+0000", 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0014172180000001 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.259770+0000", 2023-10-09T05:55:55.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000128658 2023-10-09T05:55:55.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.259899+0000", 2023-10-09T05:55:55.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016774100000000001 2023-10-09T05:55:55.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:55.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.260067+0000", 2023-10-09T05:55:55.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039100710000000002 2023-10-09T05:55:55.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.263977+0000", 2023-10-09T05:55:55.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029711770000000002 2023-10-09T05:55:55.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266948+0000", 2023-10-09T05:55:55.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00073392599999999998 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267682+0000", 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3998e-05 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267736+0000", 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4730000000000001e-05 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267771+0000", 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650766299999999 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:45770 9.a 9:54ab40b6:::benchmark_data_smithi008_502820_object45769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e372)", 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:53:46.056257+0000", 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 129.08265626299999, 2023-10-09T05:55:55.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0314687650000001, 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 45770 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056257+0000", 2023-10-09T05:55:55.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056257+0000", 2023-10-09T05:55:55.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999952 2023-10-09T05:55:55.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056252+0000", 2023-10-09T05:55:55.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2493e-05 2023-10-09T05:55:55.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056305+0000", 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6889e-05 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056322+0000", 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3180000000000001e-06 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056329+0000", 2023-10-09T05:55:55.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6201e-05 2023-10-09T05:55:55.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056375+0000", 2023-10-09T05:55:55.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7790000000000007e-06 2023-10-09T05:55:55.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056385+0000", 2023-10-09T05:55:55.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88098339699999995 2023-10-09T05:55:55.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.937368+0000", 2023-10-09T05:55:55.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1769e-05 2023-10-09T05:55:55.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.937380+0000", 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1403479299999999 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.077728+0000", 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000195169 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.077923+0000", 2023-10-09T05:55:55.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015467500000000001 2023-10-09T05:55:55.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:55:55.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.078078+0000", 2023-10-09T05:55:55.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0036624750000000001 2023-10-09T05:55:55.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.081740+0000", 2023-10-09T05:55:55.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003406278 2023-10-09T05:55:55.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.085147+0000", 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00062187400000000002 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.085768+0000", 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7609999999999997e-05 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.085846+0000", 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018796590000000001 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.087726+0000", 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0314687650000001 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:77607 9.7 9:e59624ae:::benchmark_data_smithi008_502820_object77606:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.819106+0000", 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 78.319806904999993, 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585100272, 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 77607 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819106+0000", 2023-10-09T05:55:55.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819106+0000", 2023-10-09T05:55:55.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:55.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819106+0000", 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1301000000000002e-05 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819157+0000", 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6730000000000003e-06 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819164+0000", 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6500000000000004e-06 2023-10-09T05:55:55.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819168+0000", 2023-10-09T05:55:55.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.052763364 2023-10-09T05:55:55.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871932+0000", 2023-10-09T05:55:55.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.26e-05 2023-10-09T05:55:55.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871944+0000", 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81505207000000002 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.686996+0000", 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.464e-05 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.687011+0000", 2023-10-09T05:55:55.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.70573758499999995 2023-10-09T05:55:55.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.392749+0000", 2023-10-09T05:55:55.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00064714099999999995 2023-10-09T05:55:55.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:55.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393396+0000", 2023-10-09T05:55:55.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010737744 2023-10-09T05:55:55.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404133+0000", 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3168000000000001e-05 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404207+0000", 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585100272 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.263 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:77621 9.7 9:e7b7c537:::benchmark_data_smithi008_502820_object77620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.819295+0000", 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 78.319618583999997, 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585638342, 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:55.264 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 77621 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819295+0000", 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819295+0000", 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819294+0000", 2023-10-09T05:55:55.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1934000000000002e-05 2023-10-09T05:55:55.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819336+0000", 2023-10-09T05:55:55.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5550000000000002e-06 2023-10-09T05:55:55.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819342+0000", 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.766e-06 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819345+0000", 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.052626693000000002 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871972+0000", 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2029999999999998e-06 2023-10-09T05:55:55.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871981+0000", 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81507707100000004 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.687058+0000", 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7999999999999993e-06 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.687068+0000", 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.70636606700000004 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393434+0000", 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011293 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393547+0000", 2023-10-09T05:55:55.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000142881 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393690+0000", 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0044343000000000004 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.398124+0000", 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6621e-05 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.398171+0000", 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060740669999999998 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404245+0000", 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8288999999999996e-05 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404313+0000", 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00061965000000000002 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404933+0000", 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585638342 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.272 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:99256 9.a 9:55538dec:::benchmark_data_smithi008_502820_object15083:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e406)", 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:39.286057+0000", 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 15.852855996000001, 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0366795289999999, 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99256 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286057+0000", 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286057+0000", 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286056+0000", 2023-10-09T05:55:55.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2190000000000001e-06 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286062+0000", 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2563e-05 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286074+0000", 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9100000000000005e-06 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286082+0000", 2023-10-09T05:55:55.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6157999999999997e-05 2023-10-09T05:55:55.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286168+0000", 2023-10-09T05:55:55.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4859999999999997e-06 2023-10-09T05:55:55.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286177+0000", 2023-10-09T05:55:55.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0218160709999999 2023-10-09T05:55:55.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.307993+0000", 2023-10-09T05:55:55.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112348 2023-10-09T05:55:55.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.308105+0000", 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0001139 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.308219+0000", 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029746149999999999 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.311194+0000", 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011134378 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322328+0000", 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6309000000000004e-05 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322404+0000", 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3089000000000005e-05 2023-10-09T05:55:55.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322468+0000", 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1542e-05 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322539+0000", 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000197648 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322737+0000", 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0366795289999999 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:99261 9.a 9:56a2e757:::benchmark_data_smithi008_502820_object15088:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e406)", 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:39.286092+0000", 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 15.852821225, 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.036949087, 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.281 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99261 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286092+0000", 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286092+0000", 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286092+0000", 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.79e-06 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286094+0000", 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9939999999999997e-06 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286100+0000", 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.138e-06 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286105+0000", 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5616000000000001e-05 2023-10-09T05:55:55.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286200+0000", 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0020000000000003e-06 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286206+0000", 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0250316589999999 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.311238+0000", 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000140032 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.311378+0000", 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000164034 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.311542+0000", 2023-10-09T05:55:55.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011212313999999999 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322754+0000", 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3466e-05 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322798+0000", 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4289000000000001e-05 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322852+0000", 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9132999999999999e-05 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322901+0000", 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9549999999999999e-05 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322941+0000", 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100512 2023-10-09T05:55:55.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323041+0000", 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.036949087 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:99272 9.a 9:54aba215:::benchmark_data_smithi008_502820_object15099:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e406)", 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:39.286108+0000", 2023-10-09T05:55:55.289 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 15.852804775999999, 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0372746719999999, 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99272 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286108+0000", 2023-10-09T05:55:55.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286108+0000", 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286108+0000", 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2369999999999999e-06 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286110+0000", 2023-10-09T05:55:55.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.49e-06 2023-10-09T05:55:55.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286116+0000", 2023-10-09T05:55:55.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7799999999999998e-06 2023-10-09T05:55:55.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286120+0000", 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106143 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286226+0000", 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6489999999999996e-06 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286231+0000", 2023-10-09T05:55:55.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0284209600000001 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.314652+0000", 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016349299999999999 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.314816+0000", 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019973899999999999 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.315016+0000", 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0077500399999999997 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322766+0000", 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000319585 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323085+0000", 2023-10-09T05:55:55.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9708e-05 2023-10-09T05:55:55.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323125+0000", 2023-10-09T05:55:55.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2876e-05 2023-10-09T05:55:55.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323178+0000", 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2748000000000003e-05 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323241+0000", 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014258200000000001 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323383+0000", 2023-10-09T05:55:55.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0372746719999999 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:99275 9.a 9:5b23c827:::benchmark_data_smithi008_502820_object15102:head [delete] snapc 0=[] ondisk+write+known_if_redirected e406)", 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:39.377005+0000", 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 15.761908624, 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94672199099999998, 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.298 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99275 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.377005+0000", 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.377005+0000", 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.377003+0000", 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.562e-06 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.377010+0000", 2023-10-09T05:55:55.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4184e-05 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.377024+0000", 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6849999999999998e-06 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.377030+0000", 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0467999999999999e-05 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.377070+0000", 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4789999999999999e-06 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.377077+0000", 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94124131600000005 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.318319+0000", 2023-10-09T05:55:55.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016720299999999999 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.318486+0000", 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000202358 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.318688+0000", 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041750470000000003 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322863+0000", 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00057947900000000004 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323443+0000", 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6833000000000003e-05 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323500+0000", 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7960000000000002e-05 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323548+0000", 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1544999999999994e-05 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323609+0000", 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117382 2023-10-09T05:55:55.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323727+0000", 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94672199099999998 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:99289 9.a 9:571e1078:::benchmark_data_smithi008_502820_object15116:head [delete] snapc 0=[] ondisk+write+known_if_redirected e406)", 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:39.553879+0000", 2023-10-09T05:55:55.306 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 15.585034497000001, 2023-10-09T05:55:55.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78563279100000005, 2023-10-09T05:55:55.307 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.307 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.307 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.307 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:55:55.307 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:55.307 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99289 2023-10-09T05:55:55.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.307 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.553879+0000", 2023-10-09T05:55:55.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.553879+0000", 2023-10-09T05:55:55.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:55:55.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.553878+0000", 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6330000000000001e-06 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.553883+0000", 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5060000000000001e-06 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.553893+0000", 2023-10-09T05:55:55.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0289999999999996e-06 2023-10-09T05:55:55.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.310 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.553898+0000", 2023-10-09T05:55:55.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1085999999999999e-05 2023-10-09T05:55:55.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.310 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.553939+0000", 2023-10-09T05:55:55.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0489999999999998e-06 2023-10-09T05:55:55.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.553946+0000", 2023-10-09T05:55:55.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.76983127699999998 2023-10-09T05:55:55.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323777+0000", 2023-10-09T05:55:55.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013273399999999999 2023-10-09T05:55:55.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323910+0000", 2023-10-09T05:55:55.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015743500000000001 2023-10-09T05:55:55.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T05:55:55.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.324067+0000", 2023-10-09T05:55:55.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.007427805 2023-10-09T05:55:55.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.331495+0000", 2023-10-09T05:55:55.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2020000000000004e-06 2023-10-09T05:55:55.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.331503+0000", 2023-10-09T05:55:55.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0076999149999999999 2023-10-09T05:55:55.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.339203+0000", 2023-10-09T05:55:55.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8977e-05 2023-10-09T05:55:55.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.339272+0000", 2023-10-09T05:55:55.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1359999999999994e-05 2023-10-09T05:55:55.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.339344+0000", 2023-10-09T05:55:55.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000167866 2023-10-09T05:55:55.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.339512+0000", 2023-10-09T05:55:55.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78563279100000005 2023-10-09T05:55:55.316 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.316 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.316 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:99312 9.a 9:55ee9803:::benchmark_data_smithi008_502820_object15139:head [delete] snapc 0=[] ondisk+write+known_if_redirected e406)", 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:39.652752+0000", 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 15.486161021999999, 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.69394356599999996, 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99312 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.317 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.652752+0000", 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.652752+0000", 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.318 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.652750+0000", 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4589999999999997e-06 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.652756+0000", 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8910000000000007e-06 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.652766+0000", 2023-10-09T05:55:55.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1660000000000002e-06 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.652771+0000", 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7353e-05 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.652808+0000", 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2589999999999997e-06 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.652815+0000", 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.67479096400000005 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.327606+0000", 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016133 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:55.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.327767+0000", 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018335000000000001 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.327950+0000", 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011266701 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.339217+0000", 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037819170000000001 2023-10-09T05:55:55.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.342999+0000", 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015963069999999999 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.344595+0000", 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001859543 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.346455+0000", 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8088000000000003e-05 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.346523+0000", 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017303500000000001 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.346696+0000", 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.69394356599999996 2023-10-09T05:55:55.324 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.325 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.325 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.325 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:55.325 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:55.325 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:55:55.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:55.326 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:55.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:55.414 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 6, 1, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:55:55.414 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:55:55.414 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-10-09T05:55:55.414 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd in 3 2023-10-09T05:55:55.428 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:55.499 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:55.615 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:55.623 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:55 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:55.623 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:55 smithi191 ceph-mon[161669]: failed to encode map e414 with expected crc 2023-10-09T05:55:55.623 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:55 smithi191 ceph-mon[161669]: failed to encode map e414 with expected crc 2023-10-09T05:55:55.623 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:55 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:55:55.623 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:55 smithi191 ceph-mon[161669]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T05:55:55.624 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:55 smithi191 ceph-mon[161669]: failed to encode map e414 with expected crc 2023-10-09T05:55:55.624 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:55 smithi191 ceph-mon[161669]: pgmap v1238: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 remapped+peering, 15 peering, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 35 op/s; 34073/203214 objects degraded (16.767%); 10889/203214 objects misplaced (5.358%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:55:55.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:55.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:55.664 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:55:55.706 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:55.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[184418]: failed to encode map e414 with expected crc 2023-10-09T05:55:55.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[184418]: failed to encode map e414 with expected crc 2023-10-09T05:55:55.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:55:55.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[184418]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T05:55:55.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[184418]: failed to encode map e414 with expected crc 2023-10-09T05:55:55.707 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[184418]: pgmap v1238: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 remapped+peering, 15 peering, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 35 op/s; 34073/203214 objects degraded (16.767%); 10889/203214 objects misplaced (5.358%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:55:55.708 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:55.708 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[191227]: failed to encode map e414 with expected crc 2023-10-09T05:55:55.708 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[191227]: failed to encode map e414 with expected crc 2023-10-09T05:55:55.708 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:55:55.708 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[191227]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T05:55:55.708 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[191227]: failed to encode map e414 with expected crc 2023-10-09T05:55:55.708 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:55 smithi008 ceph-mon[191227]: pgmap v1238: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 remapped+peering, 15 peering, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 35 op/s; 34073/203214 objects degraded (16.767%); 10889/203214 objects misplaced (5.358%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T05:55:55.726 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:55.774 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:55.861 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:55.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:55.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:55.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:55:55.962 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:55.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:56.040 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:56.154 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:56.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:56.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:56.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:56.297 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:56.380 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:56.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:56.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:56.464 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:56.489 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:56.502 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:56.541 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:56.669 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:56.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:56.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:56.780 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:56.809 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:56 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 46663/203010 objects degraded (22.986%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:56.813 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:56.814 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:56 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 46663/203010 objects degraded (22.986%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:57.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:56 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 46663/203010 objects degraded (22.986%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:57.330 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:55:57.362 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:57.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:57.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:57.469 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:57.554 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:57.736 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:57.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:57.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:57.844 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:57.889 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:57.943 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:57.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:57.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:58.048 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:58.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:57 smithi191 ceph-mon[161669]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 5 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 46663/203010 objects degraded (22.986%); 10880/203010 objects misplaced (5.359%); 2.9 MiB/s, 36 objects/s recovering 2023-10-09T05:55:58.082 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:57 smithi008 ceph-mon[184418]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 5 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 46663/203010 objects degraded (22.986%); 10880/203010 objects misplaced (5.359%); 2.9 MiB/s, 36 objects/s recovering 2023-10-09T05:55:58.082 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:57 smithi008 ceph-mon[191227]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 5 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 46663/203010 objects degraded (22.986%); 10880/203010 objects misplaced (5.359%); 2.9 MiB/s, 36 objects/s recovering 2023-10-09T05:55:58.086 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:58.267 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:58.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:58.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:58.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:58.386 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:55:58.386 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:55:58.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:58.386 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:101751 9.1 e414/411 9:8072751f:::benchmark_data_smithi008_502820_object17578:head v 414'5649, mlcod=368'1112)", 2023-10-09T05:55:58.386 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:58.377666+0000", 2023-10-09T05:55:58.386 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0018300079999999999, 2023-10-09T05:55:58.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001866976, 2023-10-09T05:55:58.386 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:58.377666+0000", 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:58.377666+0000", 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:58.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:58.377665+0000", 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3845e-05 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:58.377678+0000", 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1239000000000001e-05 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:58.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:58.377700+0000", 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7589999999999992e-06 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:58.377709+0000", 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8093e-05 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:58.377778+0000", 2023-10-09T05:55:58.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7392000000000001e-05 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:58.377805+0000", 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013862799999999999 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 1 2023-10-09T05:55:58.390 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:55:58.434 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:58.596 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:58.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:58.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:58.704 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:58.721 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:58.747 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:58.814 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:58.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:58.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:58.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:58.950 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:59.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:58 smithi008 conmon[184394]: 2023-10-09T05:55:58.939+0000 7f8a66521700 -1 mon.a@0(leader).osd e414 definitely_dead 0 2023-10-09T05:55:59.097 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:55:59.118 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:55:59.124 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:59.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:55:59.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:59.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:59.144 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:55:59.168 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:55:59.189 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:55:59.206 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:59.206 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1052682370' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T05:55:59.207 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:59 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T05:55:59.207 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:55:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2812447645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:59.220 INFO:teuthology.orchestra.run.smithi008.stderr:marked in osd.3. 2023-10-09T05:55:59.221 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:55:59.245 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:59.262 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:55:59.279 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:59.283 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:55:59.284 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:55:59.285 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:55:59.285 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:55:59.285 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:55:59.285 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:55:59.285 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:55:59.285 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:55:59.285 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:55:59.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:59.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1052682370' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T05:55:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:59 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T05:55:59.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:55:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2812447645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:59.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:59.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1052682370' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T05:55:59.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:59 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T05:55:59.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:55:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2812447645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:59.567 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:00.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[184418]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 46663/202101 objects degraded (23.089%); 10828/202101 objects misplaced (5.358%); 2.4 MiB/s, 30 objects/s recovering 2023-10-09T05:56:00.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[184418]: failed to encode map e415 with expected crc 2023-10-09T05:56:00.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[184418]: failed to encode map e415 with expected crc 2023-10-09T05:56:00.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T05:56:00.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[184418]: osdmap e415: 8 total, 8 up, 8 in 2023-10-09T05:56:00.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[184418]: failed to encode map e415 with expected crc 2023-10-09T05:56:00.434 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[191227]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 46663/202101 objects degraded (23.089%); 10828/202101 objects misplaced (5.358%); 2.4 MiB/s, 30 objects/s recovering 2023-10-09T05:56:00.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[191227]: failed to encode map e415 with expected crc 2023-10-09T05:56:00.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[191227]: failed to encode map e415 with expected crc 2023-10-09T05:56:00.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T05:56:00.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[191227]: osdmap e415: 8 total, 8 up, 8 in 2023-10-09T05:56:00.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:00 smithi008 ceph-mon[191227]: failed to encode map e415 with expected crc 2023-10-09T05:56:00.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:00 smithi191 ceph-mon[161669]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 46663/202101 objects degraded (23.089%); 10828/202101 objects misplaced (5.358%); 2.4 MiB/s, 30 objects/s recovering 2023-10-09T05:56:00.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:00 smithi191 ceph-mon[161669]: failed to encode map e415 with expected crc 2023-10-09T05:56:00.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:00 smithi191 ceph-mon[161669]: failed to encode map e415 with expected crc 2023-10-09T05:56:00.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:00 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T05:56:00.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:00 smithi191 ceph-mon[161669]: osdmap e415: 8 total, 8 up, 8 in 2023-10-09T05:56:00.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:00 smithi191 ceph-mon[161669]: failed to encode map e415 with expected crc 2023-10-09T05:56:00.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:56:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:56:00] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-10-09T05:56:01.034 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:01.034 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:01.057 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-10-09T05:56:01.252 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:01.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:01.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:01.332 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[184418]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.332 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[184418]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.332 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[184418]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.332 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[184418]: osdmap e416: 8 total, 8 up, 8 in 2023-10-09T05:56:01.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[184418]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[184418]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[184418]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[184418]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.333 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[184418]: 9.11 continuing backfill to osd.7 from (0'0,414'5672] MIN to 414'5672 2023-10-09T05:56:01.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[191227]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[191227]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[191227]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.335 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[191227]: osdmap e416: 8 total, 8 up, 8 in 2023-10-09T05:56:01.335 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[191227]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.335 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[191227]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.335 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[191227]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.335 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[191227]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.335 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:01 smithi008 ceph-mon[191227]: 9.11 continuing backfill to osd.7 from (0'0,414'5672] MIN to 414'5672 2023-10-09T05:56:01.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:01.402 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:01.465 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:01.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:01.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:01.475 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:01 smithi191 ceph-mon[161669]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.475 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:01 smithi191 ceph-mon[161669]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.475 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:01 smithi191 ceph-mon[161669]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.475 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:01 smithi191 ceph-mon[161669]: osdmap e416: 8 total, 8 up, 8 in 2023-10-09T05:56:01.475 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:01 smithi191 ceph-mon[161669]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.475 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:01 smithi191 ceph-mon[161669]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.475 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:01 smithi191 ceph-mon[161669]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.475 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:01 smithi191 ceph-mon[161669]: failed to encode map e416 with expected crc 2023-10-09T05:56:01.475 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:01 smithi191 ceph-mon[161669]: 9.11 continuing backfill to osd.7 from (0'0,414'5672] MIN to 414'5672 2023-10-09T05:56:01.574 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:01.612 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:01.741 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:01.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:01.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:01.846 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:01.876 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:01.923 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:01.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:01.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:02.027 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:02.056 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:02.101 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:02.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:02.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:02.112 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:56:02.204 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:02.236 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:02.244 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:56:02.302 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:02.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:02.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:02.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:02.439 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:02 smithi191 ceph-mon[161669]: pgmap v1243: 99 pgs: 2 remapped+peering, 8 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 65 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 33416/200091 objects degraded (16.700%); 10733/200091 objects misplaced (5.364%); 1.9 MiB/s, 21 objects/s recovering 2023-10-09T05:56:02.439 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:02 smithi191 ceph-mon[161669]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:02 smithi191 ceph-mon[161669]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:02 smithi191 ceph-mon[161669]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:02 smithi191 ceph-mon[161669]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:02 smithi191 ceph-mon[161669]: osdmap e417: 8 total, 8 up, 8 in 2023-10-09T05:56:02.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:02 smithi191 ceph-mon[161669]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:02 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:56:02.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:02 smithi191 ceph-mon[161669]: 9.13 continuing backfill to osd.7 from (0'0,414'5583] MIN to 414'5583 2023-10-09T05:56:02.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:02 smithi191 ceph-mon[161669]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:02 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3264691466' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:02.441 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:02.502 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:02.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:02.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:02.553 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:02.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:02.639 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:02.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[184418]: pgmap v1243: 99 pgs: 2 remapped+peering, 8 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 65 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 33416/200091 objects degraded (16.700%); 10733/200091 objects misplaced (5.364%); 1.9 MiB/s, 21 objects/s recovering 2023-10-09T05:56:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[184418]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[184418]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[184418]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[184418]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[184418]: osdmap e417: 8 total, 8 up, 8 in 2023-10-09T05:56:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[184418]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:56:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[184418]: 9.13 continuing backfill to osd.7 from (0'0,414'5583] MIN to 414'5583 2023-10-09T05:56:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[184418]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3264691466' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:02.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[191227]: pgmap v1243: 99 pgs: 2 remapped+peering, 8 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 65 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 33416/200091 objects degraded (16.700%); 10733/200091 objects misplaced (5.364%); 1.9 MiB/s, 21 objects/s recovering 2023-10-09T05:56:02.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[191227]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[191227]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[191227]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[191227]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[191227]: osdmap e417: 8 total, 8 up, 8 in 2023-10-09T05:56:02.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[191227]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:56:02.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[191227]: 9.13 continuing backfill to osd.7 from (0'0,414'5583] MIN to 414'5583 2023-10-09T05:56:02.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[191227]: failed to encode map e417 with expected crc 2023-10-09T05:56:02.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:02 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3264691466' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:02.719 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:02.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:02.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:02.823 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:02.857 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:02.903 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:02.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:03.006 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:03.035 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:03.087 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:03.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:03.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:03.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:03.221 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:03.353 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:56:03.370 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:03.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:03.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:03.474 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:03.501 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:03.546 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:03.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:03 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:03.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:03 smithi191 ceph-mon[161669]: failed to encode map e418 with expected crc 2023-10-09T05:56:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:03 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3264691466' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:03 smithi191 ceph-mon[161669]: osdmap e418: 8 total, 8 up, 8 in 2023-10-09T05:56:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:03 smithi191 ceph-mon[161669]: failed to encode map e418 with expected crc 2023-10-09T05:56:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:03 smithi191 ceph-mon[161669]: failed to encode map e418 with expected crc 2023-10-09T05:56:03.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 33416/200091 objects degraded (16.700%), 19 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:03.550 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:03.550 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[184418]: failed to encode map e418 with expected crc 2023-10-09T05:56:03.551 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3264691466' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:03.551 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[184418]: osdmap e418: 8 total, 8 up, 8 in 2023-10-09T05:56:03.551 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[184418]: failed to encode map e418 with expected crc 2023-10-09T05:56:03.551 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[184418]: failed to encode map e418 with expected crc 2023-10-09T05:56:03.551 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 33416/200091 objects degraded (16.700%), 19 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:03.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:03.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:03.552 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:03.552 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[191227]: failed to encode map e418 with expected crc 2023-10-09T05:56:03.552 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3264691466' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:03.552 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[191227]: osdmap e418: 8 total, 8 up, 8 in 2023-10-09T05:56:03.552 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[191227]: failed to encode map e418 with expected crc 2023-10-09T05:56:03.553 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[191227]: failed to encode map e418 with expected crc 2023-10-09T05:56:03.553 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 33416/200091 objects degraded (16.700%), 19 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:03.652 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:03.681 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:03.731 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:03.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:03.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:03.781 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:03.835 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:03.862 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:03.967 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:03.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:03.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:04.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:04.098 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:04.304 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:04.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:04.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:04.408 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:04.432 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:04 smithi008 ceph-mon[184418]: pgmap v1246: 99 pgs: 2 remapped+peering, 8 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 22828/199419 objects degraded (11.447%); 10701/199419 objects misplaced (5.366%); 3.3 MiB/s, 72 objects/s recovering 2023-10-09T05:56:04.432 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:04 smithi008 ceph-mon[191227]: pgmap v1246: 99 pgs: 2 remapped+peering, 8 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 22828/199419 objects degraded (11.447%); 10701/199419 objects misplaced (5.366%); 3.3 MiB/s, 72 objects/s recovering 2023-10-09T05:56:04.438 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:04 smithi191 ceph-mon[161669]: pgmap v1246: 99 pgs: 2 remapped+peering, 8 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 22828/199419 objects degraded (11.447%); 10701/199419 objects misplaced (5.366%); 3.3 MiB/s, 72 objects/s recovering 2023-10-09T05:56:04.441 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:04.497 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:04.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:04.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:04.601 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:04.633 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:04.736 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:04.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:04.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:04.824 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:04.824 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:04.840 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:04.873 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:04.946 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:04.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:04.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:05.050 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:05.087 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:05.163 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:05.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:05.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:05.267 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:05.304 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:05.429 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:05.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:05.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:05.454 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:56:05.533 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:05.562 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:05.591 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:56:05.605 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:05.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:05.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:05.609 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:56:05.630 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:56:05.651 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:56:05.670 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:56:05.689 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:56:05.708 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:05.747 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:56:05.749 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:05.769 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:56:05.772 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:56:05.772 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:56:05.772 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:56:05.772 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:56:05.772 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:56:05.772 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:56:05.772 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:56:05.773 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:56:05.809 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:05.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:05.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:05.913 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:05.940 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:06.033 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:06.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:06.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:06.057 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 6, 1, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:56:06.058 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:56:06.058 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 2, 4, 5, 6, 1, 7, 3] 2023-10-09T05:56:06.058 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd out 0 2023-10-09T05:56:06.083 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:56:06.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:06.167 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:06.218 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:06.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:06.328 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:06.359 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:06.413 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:06.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:06.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:06 smithi008 ceph-mon[184418]: pgmap v1247: 99 pgs: 2 remapped+peering, 8 peering, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 19985/199143 objects degraded (10.036%); 9331/199143 objects misplaced (4.686%); 2.3 MiB/s, 51 objects/s recovering 2023-10-09T05:56:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1320334909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:06 smithi008 ceph-mon[191227]: pgmap v1247: 99 pgs: 2 remapped+peering, 8 peering, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 19985/199143 objects degraded (10.036%); 9331/199143 objects misplaced (4.686%); 2.3 MiB/s, 51 objects/s recovering 2023-10-09T05:56:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1320334909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:06.504 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:06.517 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:06.546 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:06 smithi191 ceph-mon[161669]: pgmap v1247: 99 pgs: 2 remapped+peering, 8 peering, 7 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 19985/199143 objects degraded (10.036%); 9331/199143 objects misplaced (4.686%); 2.3 MiB/s, 51 objects/s recovering 2023-10-09T05:56:06.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1320334909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:06.549 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:06.643 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:06.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:06.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:06.746 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:06.775 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:06.816 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:06.835 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:06.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:06.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:06.939 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:06.966 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:06.974 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:07.090 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:07.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:07.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:07.195 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:07.227 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:07.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:07 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:56:07.333 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:07.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:07.439 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:07.470 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:07.522 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:07.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:07.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:07.624 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:07 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:56:07.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:07.653 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:07 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:56:07.656 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:07.702 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:07.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:07.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:07.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:07.839 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:07.890 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:07.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:07.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:07.995 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:08.030 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:08.338 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:08.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:08.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:08.441 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:08.471 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:08.533 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:08.536 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:08 smithi191 ceph-mon[161669]: pgmap v1248: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 21114/198759 objects degraded (10.623%); 11012/198759 objects misplaced (5.540%); 5.5 MiB/s, 109 objects/s recovering 2023-10-09T05:56:08.536 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 21114/198759 objects degraded (10.623%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:08.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:08.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:08.573 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:08 smithi008 ceph-mon[184418]: pgmap v1248: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 21114/198759 objects degraded (10.623%); 11012/198759 objects misplaced (5.540%); 5.5 MiB/s, 109 objects/s recovering 2023-10-09T05:56:08.573 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 21114/198759 objects degraded (10.623%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:08.574 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:08 smithi008 ceph-mon[191227]: pgmap v1248: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 21114/198759 objects degraded (10.623%); 11012/198759 objects misplaced (5.540%); 5.5 MiB/s, 109 objects/s recovering 2023-10-09T05:56:08.574 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 21114/198759 objects degraded (10.623%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:08.638 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:08.669 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:08.742 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:56:08.742 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:56:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:102698 9.c 9:335b60c2:::benchmark_data_smithi008_502820_object18525:head [delete] snapc 0=[] ondisk+write+known_if_redirected e418)", 2023-10-09T05:56:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:56:08.703002+0000", 2023-10-09T05:56:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.039400743000000002, 2023-10-09T05:56:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.039431954999999998, 2023-10-09T05:56:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:56:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:56:08.743 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:56:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 102698 2023-10-09T05:56:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:56:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:56:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.703002+0000", 2023-10-09T05:56:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:56:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:56:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.703002+0000", 2023-10-09T05:56:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:56:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.703001+0000", 2023-10-09T05:56:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4669999999999998e-06 2023-10-09T05:56:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:56:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.703008+0000", 2023-10-09T05:56:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3597e-05 2023-10-09T05:56:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:56:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.703022+0000", 2023-10-09T05:56:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9389999999999996e-06 2023-10-09T05:56:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:56:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.703027+0000", 2023-10-09T05:56:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014189230000000001 2023-10-09T05:56:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:56:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.717216+0000", 2023-10-09T05:56:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003503085 2023-10-09T05:56:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:56:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.720719+0000", 2023-10-09T05:56:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0054115459999999997 2023-10-09T05:56:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.726130+0000", 2023-10-09T05:56:08.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073444319999999997 2023-10-09T05:56:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.733475+0000", 2023-10-09T05:56:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00088047500000000003 2023-10-09T05:56:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.734355+0000", 2023-10-09T05:56:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00042282000000000002 2023-10-09T05:56:08.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:56:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.734778+0000", 2023-10-09T05:56:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020465990000000001 2023-10-09T05:56:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:56:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:56:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:56:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:08.736825+0000", 2023-10-09T05:56:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033822790999999998 2023-10-09T05:56:08.750 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:56:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:56:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:56:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:56:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T05:56:08.751 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 1 2023-10-09T05:56:08.751 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:56:08.842 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:56:08.892 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:08.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:08.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:08.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:09.027 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:09.074 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:09.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:09.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:09.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:09.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:09.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:09.359 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:09.391 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:09.498 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:09.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:09.558 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:09.603 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:09.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:09 smithi008 conmon[184394]: 2023-10-09T05:56:09.556+0000 7f8a66521700 -1 mon.a@0(leader).osd e418 definitely_dead 0 2023-10-09T05:56:09.640 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:09.711 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:09.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:09.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:09.816 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:09.849 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:10.018 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:10.022 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:56:10.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:10.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:10.124 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:10.154 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:10.200 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:10.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:10.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:10.305 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:10.327 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:56:10.333 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:10 smithi008 ceph-mon[191227]: pgmap v1249: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 21114/198297 objects degraded (10.648%); 10991/198297 objects misplaced (5.543%); 4.4 MiB/s, 87 objects/s recovering 2023-10-09T05:56:10.333 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:10 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2035239748' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T05:56:10.333 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:10 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T05:56:10.333 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:10 smithi008 ceph-mon[191227]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T05:56:10.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:10 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1129262711' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:10.334 INFO:teuthology.orchestra.run.smithi008.stderr:marked out osd.0. 2023-10-09T05:56:10.358 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:10.403 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:10.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:10.483 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:10.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:10.539 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:10 smithi191 ceph-mon[161669]: pgmap v1249: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 21114/198297 objects degraded (10.648%); 10991/198297 objects misplaced (5.543%); 4.4 MiB/s, 87 objects/s recovering 2023-10-09T05:56:10.539 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:10 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2035239748' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T05:56:10.539 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:10 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T05:56:10.539 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:10 smithi191 ceph-mon[161669]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T05:56:10.539 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:10 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1129262711' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:10.544 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:10.618 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:10.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:10.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:10.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:56:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:56:10] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-10-09T05:56:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:10 smithi008 ceph-mon[184418]: pgmap v1249: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 21114/198297 objects degraded (10.648%); 10991/198297 objects misplaced (5.543%); 4.4 MiB/s, 87 objects/s recovering 2023-10-09T05:56:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:10 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2035239748' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T05:56:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:10 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T05:56:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:10 smithi008 ceph-mon[184418]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T05:56:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:10 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1129262711' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:10.721 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:10.752 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:11.209 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:11.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:11.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:11.313 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:11.372 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:11.426 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:11.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:11.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[184418]: failed to encode map e419 with expected crc 2023-10-09T05:56:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[184418]: failed to encode map e419 with expected crc 2023-10-09T05:56:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T05:56:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1129262711' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:56:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[184418]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T05:56:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[184418]: failed to encode map e419 with expected crc 2023-10-09T05:56:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[184418]: 9.0 scrub starts 2023-10-09T05:56:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[184418]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T05:56:11.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:11.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[191227]: failed to encode map e419 with expected crc 2023-10-09T05:56:11.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[191227]: failed to encode map e419 with expected crc 2023-10-09T05:56:11.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T05:56:11.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1129262711' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:56:11.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[191227]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T05:56:11.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[191227]: failed to encode map e419 with expected crc 2023-10-09T05:56:11.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[191227]: 9.0 scrub starts 2023-10-09T05:56:11.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:11 smithi008 ceph-mon[191227]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T05:56:11.529 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:11.558 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:11 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:11 smithi191 ceph-mon[161669]: failed to encode map e419 with expected crc 2023-10-09T05:56:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:11 smithi191 ceph-mon[161669]: failed to encode map e419 with expected crc 2023-10-09T05:56:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:11 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T05:56:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1129262711' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:56:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:11 smithi191 ceph-mon[161669]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T05:56:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:11 smithi191 ceph-mon[161669]: failed to encode map e419 with expected crc 2023-10-09T05:56:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:11 smithi191 ceph-mon[161669]: 9.0 scrub starts 2023-10-09T05:56:11.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:11 smithi191 ceph-mon[161669]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T05:56:12.065 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:12.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:12.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:12.198 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:12.251 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:12.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:12.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:12.357 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:12.390 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:12.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: pgmap v1251: 99 pgs: 12 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 110 op/s; 20985/197316 objects degraded (10.635%); 10183/197316 objects misplaced (5.161%); 11 MiB/s, 184 objects/s recovering 2023-10-09T05:56:12.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: 9.10 scrub starts 2023-10-09T05:56:12.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[184418]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T05:56:12.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: pgmap v1251: 99 pgs: 12 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 110 op/s; 20985/197316 objects degraded (10.635%); 10183/197316 objects misplaced (5.161%); 11 MiB/s, 184 objects/s recovering 2023-10-09T05:56:12.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: 9.10 scrub starts 2023-10-09T05:56:12.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:12 smithi008 ceph-mon[191227]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T05:56:12.502 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: pgmap v1251: 99 pgs: 12 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 110 op/s; 20985/197316 objects degraded (10.635%); 10183/197316 objects misplaced (5.161%); 11 MiB/s, 184 objects/s recovering 2023-10-09T05:56:12.502 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.502 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.502 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.502 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.502 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.502 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: failed to encode map e420 with expected crc 2023-10-09T05:56:12.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: 9.10 scrub starts 2023-10-09T05:56:12.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:12 smithi191 ceph-mon[161669]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T05:56:12.551 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:12.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:12.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:12.634 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:56:12.655 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:12.656 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:56:12.682 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:56:12.692 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:12.705 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:56:12.738 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:56:12.740 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:12.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:12.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:12.762 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:56:12.771 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:56:12.771 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:56:12.771 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:56:12.771 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:56:12.771 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:56:12.783 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:56:12.801 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:56:12.803 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:56:12.803 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:56:12.803 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:56:12.803 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:56:12.803 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:56:12.803 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:56:12.803 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:56:12.803 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:56:12.846 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:12.877 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:12.975 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:12.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:12.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:13.078 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:13.105 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:13.150 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:13.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:13.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:13.254 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:13.285 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:13.338 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:13.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:13.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:13.430 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:56:13.442 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:13.455 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[184418]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[184418]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[184418]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[184418]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[184418]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[184418]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2155146430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:13.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[184418]: pgmap v1254: 99 pgs: 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 20995/196563 objects degraded (10.681%); 10144/196563 objects misplaced (5.161%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:56:13.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[184418]: 9.6 scrub starts 2023-10-09T05:56:13.456 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 20985/197316 objects degraded (10.635%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:13.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[191227]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[191227]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[191227]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[191227]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[191227]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[191227]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2155146430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:13.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[191227]: pgmap v1254: 99 pgs: 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 20995/196563 objects degraded (10.681%); 10144/196563 objects misplaced (5.161%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:56:13.457 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[191227]: 9.6 scrub starts 2023-10-09T05:56:13.458 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 20985/197316 objects degraded (10.635%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:13.485 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:13 smithi191 ceph-mon[161669]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:13 smithi191 ceph-mon[161669]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:13 smithi191 ceph-mon[161669]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:13 smithi191 ceph-mon[161669]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:13 smithi191 ceph-mon[161669]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:13 smithi191 ceph-mon[161669]: failed to encode map e421 with expected crc 2023-10-09T05:56:13.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2155146430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:13.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:13 smithi191 ceph-mon[161669]: pgmap v1254: 99 pgs: 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 20995/196563 objects degraded (10.681%); 10144/196563 objects misplaced (5.161%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:56:13.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:13 smithi191 ceph-mon[161669]: 9.6 scrub starts 2023-10-09T05:56:13.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 20985/197316 objects degraded (10.635%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:13.610 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:13.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:13.713 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:13.743 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:13.926 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:13.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:13.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:13.969 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:13.987 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:56:14.031 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:14.058 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:14.104 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:14.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:14.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:14.208 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:14.236 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:14.342 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:14.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:14.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:14.446 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:14.476 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:14.536 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:14.674 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:14.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:14.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:14.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:14.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:14.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:14.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:14.778 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:14.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:14.822 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:14.891 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:14.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:14.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:14.995 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:15.023 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:15.322 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:15.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:15.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:15.425 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:15.454 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:15 smithi191 ceph-mon[161669]: pgmap v1255: 99 pgs: 12 peering, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 20977/196389 objects degraded (10.681%); 15468/196389 objects misplaced (7.876%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T05:56:15.457 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:15.624 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:15 smithi008 ceph-mon[184418]: pgmap v1255: 99 pgs: 12 peering, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 20977/196389 objects degraded (10.681%); 15468/196389 objects misplaced (7.876%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T05:56:15.624 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:15 smithi008 ceph-mon[191227]: pgmap v1255: 99 pgs: 12 peering, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 20977/196389 objects degraded (10.681%); 15468/196389 objects misplaced (7.876%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T05:56:15.669 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:56:15.713 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:15.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:15.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:15.860 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:16.041 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:16.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:16.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:16.152 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:16.195 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:16.311 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:16.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:16.321 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:16.371 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:56:16.416 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:16.472 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:16.530 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:16.530 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:16.564 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:16 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2833325143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:16.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:16 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:16.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:16 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2833325143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:16.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:16 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:16.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:16 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2833325143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:16.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:16 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:16.737 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:16.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:16.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:16.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:16.887 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:16.929 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:16.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:17.032 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:17.064 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:17.143 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:17.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:17.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:17.173 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 5, 6, 1, 7, 3] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:56:17.174 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:56:17.174 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T05:56:17.174 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:56:17.249 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:17.282 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:17.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:17 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:56:17.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:17 smithi191 ceph-mon[161669]: failed to encode map e422 with expected crc 2023-10-09T05:56:17.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:17 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:17.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:17 smithi191 ceph-mon[161669]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T05:56:17.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:17 smithi191 ceph-mon[161669]: failed to encode map e422 with expected crc 2023-10-09T05:56:17.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:17 smithi191 ceph-mon[161669]: failed to encode map e422 with expected crc 2023-10-09T05:56:17.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:17 smithi191 ceph-mon[161669]: pgmap v1257: 99 pgs: 1 activating+degraded, 2 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 30742/195852 objects degraded (15.697%); 14769/195852 objects misplaced (7.541%); 7.4 MiB/s, 116 objects/s recovering 2023-10-09T05:56:17.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:56:17.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[184418]: failed to encode map e422 with expected crc 2023-10-09T05:56:17.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:17.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[184418]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T05:56:17.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[184418]: failed to encode map e422 with expected crc 2023-10-09T05:56:17.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[184418]: failed to encode map e422 with expected crc 2023-10-09T05:56:17.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[184418]: pgmap v1257: 99 pgs: 1 activating+degraded, 2 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 30742/195852 objects degraded (15.697%); 14769/195852 objects misplaced (7.541%); 7.4 MiB/s, 116 objects/s recovering 2023-10-09T05:56:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:56:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[191227]: failed to encode map e422 with expected crc 2023-10-09T05:56:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[191227]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T05:56:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[191227]: failed to encode map e422 with expected crc 2023-10-09T05:56:17.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[191227]: failed to encode map e422 with expected crc 2023-10-09T05:56:17.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:17 smithi008 ceph-mon[191227]: pgmap v1257: 99 pgs: 1 activating+degraded, 2 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 30742/195852 objects degraded (15.697%); 14769/195852 objects misplaced (7.541%); 7.4 MiB/s, 116 objects/s recovering 2023-10-09T05:56:17.469 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:56:17.511 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:17.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:17.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:17.615 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:17.646 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:17.808 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:17.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:17.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:17.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:17.955 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:18.059 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:18.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:18.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:18.163 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:18.191 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:18.290 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:18.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:18.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:18.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 30742/195852 objects degraded (15.697%), 22 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:18.400 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:18.454 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:18.458 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 30742/195852 objects degraded (15.697%), 22 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:18.530 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:18.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:18.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:18.639 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:18.663 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 30742/195852 objects degraded (15.697%), 22 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:18.689 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:18.782 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:18.789 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:18.830 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:18.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:18.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:18.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:18.984 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:19.038 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:19.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:19.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:19.149 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:19.184 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:19.238 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:19.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:19.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:19.343 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:19.388 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:19.560 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:19.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:19.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:19.662 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:56:19.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:19.696 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:19.739 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:56:19.750 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:19.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:19.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:19.760 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:56:19.791 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:56:19.814 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:56:19.845 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:56:19.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:19.872 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:56:19.880 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:19.900 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:56:19.922 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:19.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:19.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:19.931 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:56:19.932 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:56:19.932 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:56:19.933 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:56:19.933 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:56:19.933 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:56:19.933 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:56:19.933 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:56:19.933 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:56:20.022 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[191227]: pgmap v1258: 99 pgs: 1 activating+degraded, 2 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 30742/195387 objects degraded (15.734%); 14744/195387 objects misplaced (7.546%); 6.1 MiB/s, 95 objects/s recovering 2023-10-09T05:56:20.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[191227]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[191227]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[191227]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T05:56:20.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[191227]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[191227]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/536949099' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:20.027 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:20.058 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:20 smithi191 ceph-mon[161669]: pgmap v1258: 99 pgs: 1 activating+degraded, 2 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 30742/195387 objects degraded (15.734%); 14744/195387 objects misplaced (7.546%); 6.1 MiB/s, 95 objects/s recovering 2023-10-09T05:56:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:20 smithi191 ceph-mon[161669]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:20 smithi191 ceph-mon[161669]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:20 smithi191 ceph-mon[161669]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T05:56:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:20 smithi191 ceph-mon[161669]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:20 smithi191 ceph-mon[161669]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:20 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/536949099' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:20.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[184418]: pgmap v1258: 99 pgs: 1 activating+degraded, 2 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 30742/195387 objects degraded (15.734%); 14744/195387 objects misplaced (7.546%); 6.1 MiB/s, 95 objects/s recovering 2023-10-09T05:56:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[184418]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[184418]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[184418]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T05:56:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[184418]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[184418]: failed to encode map e423 with expected crc 2023-10-09T05:56:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:20 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/536949099' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:20.781 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:56:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:56:20] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.33.4" 2023-10-09T05:56:21.115 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:21.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:21.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:21.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:21.245 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:21.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:21.308 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:21.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:21.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:21.434 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:21.495 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:21.504 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:21 smithi191 ceph-mon[161669]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.504 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:21 smithi191 ceph-mon[161669]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T05:56:21.505 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:21 smithi191 ceph-mon[161669]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.505 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:21 smithi191 ceph-mon[161669]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.505 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:21 smithi191 ceph-mon[161669]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.505 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:21 smithi191 ceph-mon[161669]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.505 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:21 smithi191 ceph-mon[161669]: pgmap v1261: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded, 2 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 29094/194733 objects degraded (14.940%); 15849/194733 objects misplaced (8.139%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T05:56:21.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:21.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:21.607 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:21.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[184418]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[184418]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T05:56:21.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[184418]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[184418]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.636 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[184418]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.637 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[184418]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.637 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[184418]: pgmap v1261: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded, 2 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 29094/194733 objects degraded (14.940%); 15849/194733 objects misplaced (8.139%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T05:56:21.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[191227]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[191227]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T05:56:21.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[191227]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[191227]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[191227]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.637 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[191227]: failed to encode map e424 with expected crc 2023-10-09T05:56:21.638 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:21 smithi008 ceph-mon[191227]: pgmap v1261: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded, 2 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 29094/194733 objects degraded (14.940%); 15849/194733 objects misplaced (8.139%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T05:56:21.639 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:21.751 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:21.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:21.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:21.855 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:21.888 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:21.951 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:21.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:21.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:22.011 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:22.066 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:22.100 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:22.160 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:22.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:22.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:22.267 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:22.302 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:22.375 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:22.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:22.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:22.387 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:22 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:22.387 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:22 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:22.387 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:22 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:22.390 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:22 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:22.390 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:22 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:22.390 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:22 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:22.486 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:22.527 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:22 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:22.527 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:22 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:22.527 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:22 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:56:22.529 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:22.605 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:22.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:22.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:22.660 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:56:22.662 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:56:22.714 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:22.745 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:22.800 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:56:22.800 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:56:22.800 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.800 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:104213 9.e 9:7c792cb7:::benchmark_data_smithi008_502820_object20040:head [delete] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.800 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:22.596711+0000", 2023-10-09T05:56:22.800 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.20179164499999999, 2023-10-09T05:56:22.800 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20183532600000001, 2023-10-09T05:56:22.800 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.801 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:56:22.801 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.801 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.801 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.801 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 104213 2023-10-09T05:56:22.801 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.801 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.801 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.801 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.596711+0000", 2023-10-09T05:56:22.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.802 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.802 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.596711+0000", 2023-10-09T05:56:22.802 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.802 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.802 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.802 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.802 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.596709+0000", 2023-10-09T05:56:22.803 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:22.803 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.803 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.803 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.803 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.596719+0000", 2023-10-09T05:56:22.803 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9850000000000001e-06 2023-10-09T05:56:22.803 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.803 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.803 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.803 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.596738+0000", 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8649000000000001e-05 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.596749+0000", 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1035e-05 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.734592+0000", 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13784357899999999 2023-10-09T05:56:22.804 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.805 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.805 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:22.805 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.742437+0000", 2023-10-09T05:56:22.805 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078443720000000005 2023-10-09T05:56:22.805 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.805 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.805 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.805 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.797053+0000", 2023-10-09T05:56:22.805 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.054616112000000001 2023-10-09T05:56:22.805 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.805 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.806 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.806 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.806 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.806 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:104215 9.e 9:7301c025:::benchmark_data_smithi008_502820_object20042:head [delete] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.806 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:22.602225+0000", 2023-10-09T05:56:22.806 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.19627763300000001, 2023-10-09T05:56:22.806 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.19642282, 2023-10-09T05:56:22.806 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.806 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:56:22.806 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.806 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.807 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.807 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 104215 2023-10-09T05:56:22.807 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.807 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.807 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.807 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.807 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.602225+0000", 2023-10-09T05:56:22.807 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.807 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.807 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.808 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.808 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.602225+0000", 2023-10-09T05:56:22.808 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.808 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.808 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.808 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.808 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.602223+0000", 2023-10-09T05:56:22.808 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:22.808 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.808 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.602233+0000", 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0220999999999999e-05 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.602249+0000", 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5358000000000001e-05 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.602263+0000", 2023-10-09T05:56:22.809 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4087000000000001e-05 2023-10-09T05:56:22.810 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.810 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.810 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.810 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.743873+0000", 2023-10-09T05:56:22.810 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14161063800000001 2023-10-09T05:56:22.810 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.810 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.810 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:22.810 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.757152+0000", 2023-10-09T05:56:22.810 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013278642 2023-10-09T05:56:22.810 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:103921 9.e 9:7b75543b:::benchmark_data_smithi008_502820_object19748:head [delete] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:21.471462+0000", 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 1.3270411100000001, 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3272441289999999, 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.811 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 103921 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.471462+0000", 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.812 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.471462+0000", 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.471460+0000", 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.471466+0000", 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8939999999999998e-06 2023-10-09T05:56:22.813 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.814 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.814 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.814 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.471479+0000", 2023-10-09T05:56:22.814 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2548000000000001e-05 2023-10-09T05:56:22.814 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.814 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.814 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.814 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.471485+0000", 2023-10-09T05:56:22.814 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5089999999999999e-06 2023-10-09T05:56:22.814 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.814 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.499751+0000", 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.02826534 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.510837+0000", 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011086528 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.815 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.595876+0000", 2023-10-09T05:56:22.816 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.085039025000000004 2023-10-09T05:56:22.816 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.816 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.816 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:22.816 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.607145+0000", 2023-10-09T05:56:22.816 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011269173 2023-10-09T05:56:22.816 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.816 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.816 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.816 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.817 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.817 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:104018 9.e 9:7df15858:::benchmark_data_smithi008_502820_object19845:head [delete] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.817 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:21.851627+0000", 2023-10-09T05:56:22.817 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.94687533599999996, 2023-10-09T05:56:22.817 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94713467200000001, 2023-10-09T05:56:22.817 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.817 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:56:22.817 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.817 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.817 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.818 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 104018 2023-10-09T05:56:22.818 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.818 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.818 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.818 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.818 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.851627+0000", 2023-10-09T05:56:22.818 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.818 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.818 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.819 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.819 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.851627+0000", 2023-10-09T05:56:22.819 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.819 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.819 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.819 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.819 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.851626+0000", 2023-10-09T05:56:22.819 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:22.819 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.819 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.820 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.820 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.851633+0000", 2023-10-09T05:56:22.820 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1860000000000003e-06 2023-10-09T05:56:22.820 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.820 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.820 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.820 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.851652+0000", 2023-10-09T05:56:22.820 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9089e-05 2023-10-09T05:56:22.820 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.821 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.821 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.821 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.851659+0000", 2023-10-09T05:56:22.821 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2740000000000002e-06 2023-10-09T05:56:22.821 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.821 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.821 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.821 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.935235+0000", 2023-10-09T05:56:22.821 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.083576205000000001 2023-10-09T05:56:22.821 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.822 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.822 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:22.822 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.943131+0000", 2023-10-09T05:56:22.822 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078959119999999997 2023-10-09T05:56:22.822 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.822 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.822 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.822 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.982819+0000", 2023-10-09T05:56:22.822 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.039687980999999997 2023-10-09T05:56:22.823 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.823 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.823 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:22.823 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.997537+0000", 2023-10-09T05:56:22.823 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014717747999999999 2023-10-09T05:56:22.823 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.823 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.823 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.824 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.824 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.824 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:104021 9.e 9:776edf78:::benchmark_data_smithi008_502820_object19848:head [delete] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.824 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:21.875245+0000", 2023-10-09T05:56:22.824 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.92325787100000001, 2023-10-09T05:56:22.824 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.923595728, 2023-10-09T05:56:22.824 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.824 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:56:22.824 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.825 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.825 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.825 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 104021 2023-10-09T05:56:22.825 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.825 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.825 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.825 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.825 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.875245+0000", 2023-10-09T05:56:22.826 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.826 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.826 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.826 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.826 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.875245+0000", 2023-10-09T05:56:22.826 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.826 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.826 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.827 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.827 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.875243+0000", 2023-10-09T05:56:22.827 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:22.827 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.827 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.827 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.827 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.875252+0000", 2023-10-09T05:56:22.827 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7620000000000001e-06 2023-10-09T05:56:22.828 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.828 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.828 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.828 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.875267+0000", 2023-10-09T05:56:22.828 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5207e-05 2023-10-09T05:56:22.828 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.828 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.828 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.828 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.875275+0000", 2023-10-09T05:56:22.829 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6279999999999996e-06 2023-10-09T05:56:22.829 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.829 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.829 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.829 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.943990+0000", 2023-10-09T05:56:22.829 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.068715464000000004 2023-10-09T05:56:22.829 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.829 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.830 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:22.830 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.950996+0000", 2023-10-09T05:56:22.830 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070062450000000004 2023-10-09T05:56:22.830 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.830 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.830 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.830 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.036131+0000", 2023-10-09T05:56:22.830 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.08513424 2023-10-09T05:56:22.830 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.043648+0000", 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0075170310000000004 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:104235 9.e 9:72b97467:::benchmark_data_smithi008_502820_object20062:head [delete] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:22.671716+0000", 2023-10-09T05:56:22.831 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.12678682199999999, 2023-10-09T05:56:22.832 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.12720313799999999, 2023-10-09T05:56:22.833 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.833 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:56:22.833 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.833 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.833 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.833 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 104235 2023-10-09T05:56:22.834 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.834 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.834 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.834 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.834 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.671716+0000", 2023-10-09T05:56:22.834 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.834 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.834 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.834 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.834 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.671716+0000", 2023-10-09T05:56:22.834 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.671714+0000", 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.671725+0000", 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0944000000000001e-05 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.835 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.836 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.671750+0000", 2023-10-09T05:56:22.836 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5091000000000001e-05 2023-10-09T05:56:22.836 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.836 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.836 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.836 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.671761+0000", 2023-10-09T05:56:22.836 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0969e-05 2023-10-09T05:56:22.836 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.836 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.836 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.836 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.758408+0000", 2023-10-09T05:56:22.837 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.086647482999999997 2023-10-09T05:56:22.837 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.837 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.837 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:22.837 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.770035+0000", 2023-10-09T05:56:22.837 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011626778000000001 2023-10-09T05:56:22.837 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.837 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.837 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.837 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:103894 9.e 9:7433fdde:::benchmark_data_smithi008_502820_object19721:head [delete] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:21.216110+0000", 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 1.58239233, 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.582868036, 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 103894 2023-10-09T05:56:22.838 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.839 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.839 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.839 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.839 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.216110+0000", 2023-10-09T05:56:22.839 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.839 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.839 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.839 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.839 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.216110+0000", 2023-10-09T05:56:22.839 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.839 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.216109+0000", 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.216117+0000", 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1769999999999998e-06 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.840 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.841 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.216131+0000", 2023-10-09T05:56:22.841 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4343e-05 2023-10-09T05:56:22.841 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.841 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.841 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.841 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.216142+0000", 2023-10-09T05:56:22.841 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0326000000000001e-05 2023-10-09T05:56:22.841 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.841 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.841 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.841 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.306676+0000", 2023-10-09T05:56:22.842 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.090534811000000007 2023-10-09T05:56:22.842 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.842 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.842 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:22.842 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.312565+0000", 2023-10-09T05:56:22.842 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058885989999999996 2023-10-09T05:56:22.842 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.842 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.842 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.842 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.357046+0000", 2023-10-09T05:56:22.842 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.044480612000000003 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.363230+0000", 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061848010000000002 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:104236 9.e 9.de469dce (undecoded) ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.843 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:22.680109+0000", 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.118393926, 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.118926006, 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "queued for pg", 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 104236 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.844 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.680109+0000", 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.680109+0000", 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.680107+0000", 2023-10-09T05:56:22.845 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.680117+0000", 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0312999999999999e-05 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.680136+0000", 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9168e-05 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.846 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.680147+0000", 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0740999999999999e-05 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:103942 9.e 9:759f06e5:::benchmark_data_smithi008_502820_object19769:head [delete] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:21.548502+0000", 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 1.250001063, 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2505665580000001, 2023-10-09T05:56:22.847 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.848 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:56:22.848 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.848 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.848 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.848 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 103942 2023-10-09T05:56:22.848 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.848 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.848 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.848 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.848 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.548502+0000", 2023-10-09T05:56:22.848 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.548502+0000", 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.548500+0000", 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.849 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.850 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.548509+0000", 2023-10-09T05:56:22.850 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1969999999999996e-06 2023-10-09T05:56:22.850 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.850 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.850 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.850 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.548522+0000", 2023-10-09T05:56:22.850 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3138e-05 2023-10-09T05:56:22.850 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.850 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.850 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.850 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.548531+0000", 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0350000000000007e-06 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.684304+0000", 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13577302899999999 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.690739+0000", 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064350939999999997 2023-10-09T05:56:22.851 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.761106+0000", 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.070366854000000006 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.767236+0000", 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061307499999999999 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.852 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.853 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.853 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.853 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:104177 9.e 9:74f645f8:::benchmark_data_smithi008_502820_object20004:head [delete] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.853 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:22.482420+0000", 2023-10-09T05:56:22.853 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.316082739, 2023-10-09T05:56:22.853 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.31669348400000003, 2023-10-09T05:56:22.853 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.853 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:56:22.853 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.853 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.853 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.854 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 104177 2023-10-09T05:56:22.854 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.854 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.854 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.854 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.854 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.482420+0000", 2023-10-09T05:56:22.854 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.854 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.855 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.855 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.855 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.482420+0000", 2023-10-09T05:56:22.855 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.855 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.855 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.855 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.855 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.482418+0000", 2023-10-09T05:56:22.855 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:22.855 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.855 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.856 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.856 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.482427+0000", 2023-10-09T05:56:22.856 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3689999999999996e-06 2023-10-09T05:56:22.856 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.856 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.856 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.856 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.482445+0000", 2023-10-09T05:56:22.856 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7795000000000002e-05 2023-10-09T05:56:22.856 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.856 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.856 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.857 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.482458+0000", 2023-10-09T05:56:22.857 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3016999999999999e-05 2023-10-09T05:56:22.857 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.857 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.857 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.857 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.506205+0000", 2023-10-09T05:56:22.857 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.023746730000000001 2023-10-09T05:56:22.857 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.858 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.858 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:22.858 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.515231+0000", 2023-10-09T05:56:22.858 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0090261430000000004 2023-10-09T05:56:22.858 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.858 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.858 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.858 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.685632+0000", 2023-10-09T05:56:22.858 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17040128099999999 2023-10-09T05:56:22.859 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.859 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.859 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:22.859 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.696344+0000", 2023-10-09T05:56:22.859 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010711083 2023-10-09T05:56:22.859 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.859 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.859 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.859 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.859 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.859 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "MOSDPGPush(9.e 424/402 [PushOp(9:7301c025:::benchmark_data_smithi008_502820_object20042:head, version: 368'1265, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:7301c025:::benchmark_data_smithi008_502820_object20042:head@368'1265, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:56:22.860 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:22.757622+0000", 2023-10-09T05:56:22.860 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.040880574000000003, 2023-10-09T05:56:22.860 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.041566556999999997, 2023-10-09T05:56:22.860 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.860 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "queued for pg", 2023-10-09T05:56:22.860 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.860 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.860 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.860 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.757622+0000", 2023-10-09T05:56:22.861 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.861 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.861 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.861 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.861 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.757622+0000", 2023-10-09T05:56:22.861 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.861 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.861 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.861 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.862 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.757621+0000", 2023-10-09T05:56:22.862 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:56:22.862 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.862 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.862 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.862 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.757647+0000", 2023-10-09T05:56:22.862 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6191e-05 2023-10-09T05:56:22.862 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.862 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.863 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.863 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.757664+0000", 2023-10-09T05:56:22.863 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6986e-05 2023-10-09T05:56:22.863 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.863 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.863 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.863 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.757682+0000", 2023-10-09T05:56:22.863 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7980000000000001e-05 2023-10-09T05:56:22.863 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.864 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.864 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.864 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.864 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.864 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:104041 9.e 9:709a24aa:::benchmark_data_smithi008_502820_object19868:head [delete] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.864 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:21.935806+0000", 2023-10-09T05:56:22.864 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.86269707399999995, 2023-10-09T05:56:22.864 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86341352599999999, 2023-10-09T05:56:22.864 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.865 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:56:22.865 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.865 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.865 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.865 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 104041 2023-10-09T05:56:22.865 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.865 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.865 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.865 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.866 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.935806+0000", 2023-10-09T05:56:22.866 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.866 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.866 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.866 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.866 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.935806+0000", 2023-10-09T05:56:22.866 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.866 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.866 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.867 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.867 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.935804+0000", 2023-10-09T05:56:22.867 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:22.867 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.867 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.867 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.867 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.935813+0000", 2023-10-09T05:56:22.868 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3530000000000001e-06 2023-10-09T05:56:22.868 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.868 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.868 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.868 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.935831+0000", 2023-10-09T05:56:22.868 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7486000000000002e-05 2023-10-09T05:56:22.868 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.869 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.869 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.869 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.935841+0000", 2023-10-09T05:56:22.869 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0421999999999999e-05 2023-10-09T05:56:22.869 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.869 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.869 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.869 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.951603+0000", 2023-10-09T05:56:22.869 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015761588999999999 2023-10-09T05:56:22.870 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.870 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.870 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:22.870 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:21.957617+0000", 2023-10-09T05:56:22.870 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006014742 2023-10-09T05:56:22.870 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.870 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.870 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.870 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.068473+0000", 2023-10-09T05:56:22.871 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.11085526499999999 2023-10-09T05:56:22.871 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.871 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.871 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:22.871 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.074885+0000", 2023-10-09T05:56:22.871 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006412488 2023-10-09T05:56:22.871 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.871 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.871 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.871 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.871 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:104144 9.e 9:75735379:::benchmark_data_smithi008_502820_object19971:head [delete] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:22.380506+0000", 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.41799628500000002, 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.41876374300000002, 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "delayed", 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 104144 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.872 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.380506+0000", 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.380506+0000", 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.873 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.380505+0000", 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.380514+0000", 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7809999999999994e-06 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.380532+0000", 2023-10-09T05:56:22.874 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8581e-05 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.380542+0000", 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0346999999999999e-05 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.497406+0000", 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.116863094 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.875 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.505401+0000", 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0079954610000000006 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.626774+0000", 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.121373197 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:22.876 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.634648+0000", 2023-10-09T05:56:22.877 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078741369999999998 2023-10-09T05:56:22.877 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.877 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.877 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.877 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.877 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.877 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "MOSDPGPush(9.e 424/402 [PushOp(9:72b97467:::benchmark_data_smithi008_502820_object20062:head, version: 368'1266, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:72b97467:::benchmark_data_smithi008_502820_object20062:head@368'1266, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:56:22.877 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:22.770791+0000", 2023-10-09T05:56:22.877 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.027711533, 2023-10-09T05:56:22.877 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028539035000000001, 2023-10-09T05:56:22.877 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.878 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "queued for pg", 2023-10-09T05:56:22.878 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.878 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.878 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.878 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.770791+0000", 2023-10-09T05:56:22.878 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.878 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.878 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.878 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.878 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.770791+0000", 2023-10-09T05:56:22.878 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.879 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.879 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.879 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.879 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.770790+0000", 2023-10-09T05:56:22.879 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:56:22.879 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.879 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.879 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.879 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.770847+0000", 2023-10-09T05:56:22.879 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6512999999999999e-05 2023-10-09T05:56:22.879 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.770864+0000", 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6977999999999999e-05 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.770883+0000", 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9578000000000001e-05 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.880 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:104253 9.e 9.b1bfaebe (undecoded) ondisk+write+known_if_redirected e424)", 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:22.780013+0000", 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.018489736999999999, 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019359646000000001, 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "queued for pg", 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 104253 2023-10-09T05:56:22.881 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.780013+0000", 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.780013+0000", 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.882 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.780012+0000", 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.780019+0000", 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3690000000000001e-06 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:22.883 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.780036+0000", 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6787999999999999e-05 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:22.780047+0000", 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0756e-05 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:56:22.884 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 15 2023-10-09T05:56:22.885 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:56:22.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:22.886 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:22.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:22.987 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:23.021 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:23.391 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:23 smithi008 ceph-mon[191227]: pgmap v1262: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 137 op/s; 29050/193266 objects degraded (15.031%); 15597/193266 objects misplaced (8.070%); 2.7 MiB/s, 125 objects/s recovering 2023-10-09T05:56:23.391 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:23 smithi008 ceph-mon[191227]: from='client.46424 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:56:23.391 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 29094/194733 objects degraded (14.940%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:23.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:23 smithi008 ceph-mon[184418]: pgmap v1262: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 137 op/s; 29050/193266 objects degraded (15.031%); 15597/193266 objects misplaced (8.070%); 2.7 MiB/s, 125 objects/s recovering 2023-10-09T05:56:23.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:23 smithi008 ceph-mon[184418]: from='client.46424 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:56:23.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 29094/194733 objects degraded (14.940%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:23.754 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:56:23.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:23 smithi191 ceph-mon[161669]: pgmap v1262: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 137 op/s; 29050/193266 objects degraded (15.031%); 15597/193266 objects misplaced (8.070%); 2.7 MiB/s, 125 objects/s recovering 2023-10-09T05:56:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:23 smithi191 ceph-mon[161669]: from='client.46424 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:56:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 29094/194733 objects degraded (14.940%), 21 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:24.383 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:24.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:24.488 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:24.521 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:24.582 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:24.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:24.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:24.597 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1262,"stamp":"2023-10-09T05:56:22.617980+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4224849867,"num_objects":64422,"num_object_clones":0,"num_object_copies":193266,"num_objects_missing_on_primary":15242,"num_objects_missing":15242,"num_objects_degraded":29050,"num_objects_misplaced":15597,"num_objects_unfound":0,"num_objects_dirty":64422,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":105054,"num_write_kb":5402022,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16502,"num_bytes_recovered":999113202,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":151486,"ondisk_log_size":151486,"up":297,"acting":285,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18031972,"kb_used_data":11862892,"kb_used_omap":37,"kb_used_meta":6168986,"kb_avail":638184092,"statfs":{"total":671965249536,"available":653500510208,"internally_reserved":0,"allocated":12147601408,"data_stored":12139598626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38373,"internal_metadata":6317042203},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,3,3],"upper_bound":16384},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-56492032,"num_objects":-862,"num_object_clones":0,"num_object_copies":-2586,"num_objects_missing_on_primary":-86,"num_objects_missing":-86,"num_objects_degraded":-1692,"num_objects_misplaced":828,"num_objects_unfound":0,"num_objects_dirty":-862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":862,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":787,"num_bytes_recovered":18022400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.256609"},"pg_stats":[{"pgid":"9.10","version":"424'5629","reported_seq":5708,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T05:56:22.552113+0000","last_change":"2023-10-09T05:56:12.319627+0000","last_active":"2023-10-09T05:56:22.552113+0000","last_peered":"2023-10-09T05:56:22.552113+0000","last_clean":"2023-10-09T05:56:22.552113+0000","last_became_active":"2023-10-09T05:54:37.745539+0000","last_became_peered":"2023-10-09T05:54:37.745539+0000","last_unstale":"2023-10-09T05:56:22.552113+0000","last_undegraded":"2023-10-09T05:56:22.552113+0000","last_fullsized":"2023-10-09T05:56:22.552113+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5629,"log_dups_size":0,"ondisk_log_size":5629,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127729664,"num_objects":1949,"num_object_clones":0,"num_object_copies":5847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3239,"num_write_kb":166016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":44,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.417517+0000","last_change":"2023-10-09T05:54:59.256087+0000","last_active":"2023-10-09T05:56:19.417517+0000","last_peered":"2023-10-09T05:56:19.417517+0000","last_clean":"2023-10-09T05:56:19.417517+0000","last_became_active":"2023-10-09T05:54:59.255773+0000","last_became_peered":"2023-10-09T05:54:59.255773+0000","last_unstale":"2023-10-09T05:56:19.417517+0000","last_undegraded":"2023-10-09T05:56:19.417517+0000","last_fullsized":"2023-10-09T05:56:19.417517+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:19.545754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":19,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.394449+0000","last_change":"2023-10-09T05:56:00.247579+0000","last_active":"2023-10-09T05:56:19.394449+0000","last_peered":"2023-10-09T05:56:19.394449+0000","last_clean":"2023-10-09T05:56:19.394449+0000","last_became_active":"2023-10-09T05:56:00.238151+0000","last_became_peered":"2023-10-09T05:56:00.238151+0000","last_unstale":"2023-10-09T05:56:19.394449+0000","last_undegraded":"2023-10-09T05:56:19.394449+0000","last_fullsized":"2023-10-09T05:56:19.394449+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:05.692190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"424'5655","reported_seq":11470,"reported_epoch":424,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:56:22.070016+0000","last_change":"2023-10-09T05:56:01.319470+0000","last_active":"2023-10-09T05:56:22.070016+0000","last_peered":"2023-10-09T05:56:22.070016+0000","last_clean":"2023-10-09T05:55:24.768851+0000","last_became_active":"2023-10-09T05:56:01.311996+0000","last_became_peered":"2023-10-09T05:56:01.311996+0000","last_unstale":"2023-10-09T05:56:22.070016+0000","last_undegraded":"2023-10-09T05:56:22.070016+0000","last_fullsized":"2023-10-09T05:56:22.070016+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5655,"log_dups_size":0,"ondisk_log_size":5655,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132382720,"num_objects":2020,"num_object_clones":0,"num_object_copies":6060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2020,"num_objects_unfound":0,"num_objects_dirty":2020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3242,"num_write_kb":168384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1422,"num_bytes_recovered":93192192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":2020}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":125,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.415488+0000","last_change":"2023-10-09T05:56:00.243822+0000","last_active":"2023-10-09T05:56:19.415488+0000","last_peered":"2023-10-09T05:56:19.415488+0000","last_clean":"2023-10-09T05:56:19.415488+0000","last_became_active":"2023-10-09T05:56:00.243483+0000","last_became_peered":"2023-10-09T05:56:00.243483+0000","last_unstale":"2023-10-09T05:56:19.415488+0000","last_undegraded":"2023-10-09T05:56:19.415488+0000","last_fullsized":"2023-10-09T05:56:19.415488+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:39.811952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":476,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.389106+0000","last_change":"2023-10-09T05:56:11.341136+0000","last_active":"2023-10-09T05:56:19.389106+0000","last_peered":"2023-10-09T05:56:19.389106+0000","last_clean":"2023-10-09T05:56:19.389106+0000","last_became_active":"2023-10-09T05:56:11.339778+0000","last_became_peered":"2023-10-09T05:56:11.339778+0000","last_unstale":"2023-10-09T05:56:19.389106+0000","last_undegraded":"2023-10-09T05:56:19.389106+0000","last_fullsized":"2023-10-09T05:56:19.389106+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.12","version":"424'5722","reported_seq":5888,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T05:56:22.476534+0000","last_change":"2023-10-09T05:56:01.366668+0000","last_active":"2023-10-09T05:56:22.476534+0000","last_peered":"2023-10-09T05:56:22.476534+0000","last_clean":"2023-10-09T05:56:22.476534+0000","last_became_active":"2023-10-09T05:56:01.226869+0000","last_became_peered":"2023-10-09T05:56:01.226869+0000","last_unstale":"2023-10-09T05:56:22.476534+0000","last_undegraded":"2023-10-09T05:56:22.476534+0000","last_fullsized":"2023-10-09T05:56:22.476534+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5722,"log_dups_size":0,"ondisk_log_size":5722,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131497984,"num_objects":2007,"num_object_clones":0,"num_object_copies":6021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3301,"num_write_kb":169824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.416695+0000","last_change":"2023-10-09T05:56:11.336427+0000","last_active":"2023-10-09T05:56:19.416695+0000","last_peered":"2023-10-09T05:56:19.416695+0000","last_clean":"2023-10-09T05:56:19.416695+0000","last_became_active":"2023-10-09T05:56:11.335732+0000","last_became_peered":"2023-10-09T05:56:11.335732+0000","last_unstale":"2023-10-09T05:56:19.416695+0000","last_undegraded":"2023-10-09T05:56:19.416695+0000","last_fullsized":"2023-10-09T05:56:19.416695+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:25.535021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":249,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.431522+0000","last_change":"2023-10-09T05:55:11.312896+0000","last_active":"2023-10-09T05:56:16.431522+0000","last_peered":"2023-10-09T05:56:16.431522+0000","last_clean":"2023-10-09T05:56:16.431522+0000","last_became_active":"2023-10-09T05:55:11.312597+0000","last_became_peered":"2023-10-09T05:55:11.312597+0000","last_unstale":"2023-10-09T05:56:16.431522+0000","last_undegraded":"2023-10-09T05:56:16.431522+0000","last_fullsized":"2023-10-09T05:56:16.431522+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":397,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.390412+0000","last_change":"2023-10-09T05:56:11.346423+0000","last_active":"2023-10-09T05:56:16.390412+0000","last_peered":"2023-10-09T05:56:16.390412+0000","last_clean":"2023-10-09T05:56:16.390412+0000","last_became_active":"2023-10-09T05:56:11.346098+0000","last_became_peered":"2023-10-09T05:56:11.346098+0000","last_unstale":"2023-10-09T05:56:16.390412+0000","last_undegraded":"2023-10-09T05:56:16.390412+0000","last_fullsized":"2023-10-09T05:56:16.390412+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":19,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.394345+0000","last_change":"2023-10-09T05:56:00.263498+0000","last_active":"2023-10-09T05:56:19.394345+0000","last_peered":"2023-10-09T05:56:19.394345+0000","last_clean":"2023-10-09T05:56:19.394345+0000","last_became_active":"2023-10-09T05:56:00.262929+0000","last_became_peered":"2023-10-09T05:56:00.262929+0000","last_unstale":"2023-10-09T05:56:19.394345+0000","last_undegraded":"2023-10-09T05:56:19.394345+0000","last_fullsized":"2023-10-09T05:56:19.394345+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:56.874781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":97,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.378387+0000","last_change":"2023-10-09T05:55:39.287521+0000","last_active":"2023-10-09T05:56:16.378387+0000","last_peered":"2023-10-09T05:56:16.378387+0000","last_clean":"2023-10-09T05:56:16.378387+0000","last_became_active":"2023-10-09T05:55:39.287263+0000","last_became_peered":"2023-10-09T05:55:39.287263+0000","last_unstale":"2023-10-09T05:56:16.378387+0000","last_undegraded":"2023-10-09T05:56:16.378387+0000","last_fullsized":"2023-10-09T05:56:16.378387+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":50,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.392943+0000","last_change":"2023-10-09T05:56:00.235776+0000","last_active":"2023-10-09T05:56:19.392943+0000","last_peered":"2023-10-09T05:56:19.392943+0000","last_clean":"2023-10-09T05:56:19.392943+0000","last_became_active":"2023-10-09T05:56:00.235383+0000","last_became_peered":"2023-10-09T05:56:00.235383+0000","last_unstale":"2023-10-09T05:56:19.392943+0000","last_undegraded":"2023-10-09T05:56:19.392943+0000","last_fullsized":"2023-10-09T05:56:19.392943+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":62,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.388230+0000","last_change":"2023-10-09T05:56:00.348771+0000","last_active":"2023-10-09T05:56:19.388230+0000","last_peered":"2023-10-09T05:56:19.388230+0000","last_clean":"2023-10-09T05:56:19.388230+0000","last_became_active":"2023-10-09T05:56:00.224347+0000","last_became_peered":"2023-10-09T05:56:00.224347+0000","last_unstale":"2023-10-09T05:56:19.388230+0000","last_undegraded":"2023-10-09T05:56:19.388230+0000","last_fullsized":"2023-10-09T05:56:19.388230+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":166,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.417519+0000","last_change":"2023-10-09T05:56:11.346737+0000","last_active":"2023-10-09T05:56:19.417519+0000","last_peered":"2023-10-09T05:56:19.417519+0000","last_clean":"2023-10-09T05:56:19.417519+0000","last_became_active":"2023-10-09T05:56:11.346391+0000","last_became_peered":"2023-10-09T05:56:11.346391+0000","last_unstale":"2023-10-09T05:56:19.417519+0000","last_undegraded":"2023-10-09T05:56:19.417519+0000","last_fullsized":"2023-10-09T05:56:19.417519+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:33.857549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":42,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.387942+0000","last_change":"2023-10-09T05:55:26.703302+0000","last_active":"2023-10-09T05:56:19.387942+0000","last_peered":"2023-10-09T05:56:19.387942+0000","last_clean":"2023-10-09T05:56:19.387942+0000","last_became_active":"2023-10-09T05:55:26.265755+0000","last_became_peered":"2023-10-09T05:55:26.265755+0000","last_unstale":"2023-10-09T05:56:19.387942+0000","last_undegraded":"2023-10-09T05:56:19.387942+0000","last_fullsized":"2023-10-09T05:56:19.387942+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":399,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.416863+0000","last_change":"2023-10-09T05:54:59.255627+0000","last_active":"2023-10-09T05:56:19.416863+0000","last_peered":"2023-10-09T05:56:19.416863+0000","last_clean":"2023-10-09T05:56:19.416863+0000","last_became_active":"2023-10-09T05:54:59.255251+0000","last_became_peered":"2023-10-09T05:54:59.255251+0000","last_unstale":"2023-10-09T05:56:19.416863+0000","last_undegraded":"2023-10-09T05:56:19.416863+0000","last_fullsized":"2023-10-09T05:56:19.416863+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:37:19.719962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.19","version":"424'5630","reported_seq":6672,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:20.657166+0000","last_change":"2023-10-09T05:56:15.816895+0000","last_active":"2023-10-09T05:56:20.657166+0000","last_peered":"2023-10-09T05:56:20.657166+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T05:56:15.742099+0000","last_became_peered":"2023-10-09T05:56:15.742099+0000","last_unstale":"2023-10-09T05:56:20.657166+0000","last_undegraded":"2023-10-09T05:56:15.624687+0000","last_fullsized":"2023-10-09T05:56:12.325744+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5630,"log_dups_size":0,"ondisk_log_size":5630,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132612096,"num_objects":2024,"num_object_clones":0,"num_object_copies":6072,"num_objects_missing_on_primary":1689,"num_objects_missing":1689,"num_objects_degraded":2629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3224,"num_write_kb":167904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":472,"num_bytes_recovered":30900224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1217},{"shards":"0,5","objects":1},{"shards":"0,5,7","objects":810},{"shards":"5,7","objects":-23}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":38,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.387832+0000","last_change":"2023-10-09T05:55:27.092744+0000","last_active":"2023-10-09T05:56:19.387832+0000","last_peered":"2023-10-09T05:56:19.387832+0000","last_clean":"2023-10-09T05:56:19.387832+0000","last_became_active":"2023-10-09T05:55:26.271096+0000","last_became_peered":"2023-10-09T05:55:26.271096+0000","last_unstale":"2023-10-09T05:56:19.387832+0000","last_undegraded":"2023-10-09T05:56:19.387832+0000","last_fullsized":"2023-10-09T05:56:19.387832+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":50,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.387855+0000","last_change":"2023-10-09T05:56:00.235825+0000","last_active":"2023-10-09T05:56:19.387855+0000","last_peered":"2023-10-09T05:56:19.387855+0000","last_clean":"2023-10-09T05:56:19.387855+0000","last_became_active":"2023-10-09T05:56:00.235486+0000","last_became_peered":"2023-10-09T05:56:00.235486+0000","last_unstale":"2023-10-09T05:56:19.387855+0000","last_undegraded":"2023-10-09T05:56:19.387855+0000","last_fullsized":"2023-10-09T05:56:19.387855+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"424'5645","reported_seq":5757,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:20.566207+0000","last_change":"2023-10-09T05:56:16.273354+0000","last_active":"2023-10-09T05:56:20.566207+0000","last_peered":"2023-10-09T05:56:20.566207+0000","last_clean":"2023-10-09T05:55:49.156862+0000","last_became_active":"2023-10-09T05:56:16.208076+0000","last_became_peered":"2023-10-09T05:56:16.208076+0000","last_unstale":"2023-10-09T05:56:20.566207+0000","last_undegraded":"2023-10-09T05:56:16.138265+0000","last_fullsized":"2023-10-09T05:56:20.566207+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5645,"log_dups_size":0,"ondisk_log_size":5645,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130547712,"num_objects":1992,"num_object_clones":0,"num_object_copies":5976,"num_objects_missing_on_primary":2612,"num_objects_missing":2612,"num_objects_degraded":2655,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3232,"num_write_kb":167168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":1900544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":2021},{"shards":"3,5,7","objects":-29}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":450,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.387546+0000","last_change":"2023-10-09T05:56:11.340136+0000","last_active":"2023-10-09T05:56:19.387546+0000","last_peered":"2023-10-09T05:56:19.387546+0000","last_clean":"2023-10-09T05:56:19.387546+0000","last_became_active":"2023-10-09T05:56:11.339802+0000","last_became_peered":"2023-10-09T05:56:11.339802+0000","last_unstale":"2023-10-09T05:56:19.387546+0000","last_undegraded":"2023-10-09T05:56:19.387546+0000","last_fullsized":"2023-10-09T05:56:19.387546+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":115,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.386906+0000","last_change":"2023-10-09T05:56:00.243380+0000","last_active":"2023-10-09T05:56:16.386906+0000","last_peered":"2023-10-09T05:56:16.386906+0000","last_clean":"2023-10-09T05:56:16.386906+0000","last_became_active":"2023-10-09T05:56:00.243005+0000","last_became_peered":"2023-10-09T05:56:00.243005+0000","last_unstale":"2023-10-09T05:56:16.386906+0000","last_undegraded":"2023-10-09T05:56:16.386906+0000","last_fullsized":"2023-10-09T05:56:16.386906+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":32,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.392870+0000","last_change":"2023-10-09T05:55:26.272219+0000","last_active":"2023-10-09T05:56:19.392870+0000","last_peered":"2023-10-09T05:56:19.392870+0000","last_clean":"2023-10-09T05:56:19.392870+0000","last_became_active":"2023-10-09T05:55:26.271861+0000","last_became_peered":"2023-10-09T05:55:26.271861+0000","last_unstale":"2023-10-09T05:56:19.392870+0000","last_undegraded":"2023-10-09T05:56:19.392870+0000","last_fullsized":"2023-10-09T05:56:19.392870+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":447,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.389181+0000","last_change":"2023-10-09T05:56:11.341328+0000","last_active":"2023-10-09T05:56:19.389181+0000","last_peered":"2023-10-09T05:56:19.389181+0000","last_clean":"2023-10-09T05:56:19.389181+0000","last_became_active":"2023-10-09T05:56:11.340361+0000","last_became_peered":"2023-10-09T05:56:11.340361+0000","last_unstale":"2023-10-09T05:56:19.389181+0000","last_undegraded":"2023-10-09T05:56:19.389181+0000","last_fullsized":"2023-10-09T05:56:19.389181+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":232,"reported_epoch":422,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:16.430188+0000","last_change":"2023-10-09T05:56:11.379504+0000","last_active":"2023-10-09T05:56:16.430188+0000","last_peered":"2023-10-09T05:56:16.430188+0000","last_clean":"2023-10-09T05:44:39.991208+0000","last_became_active":"2023-10-09T05:56:11.379097+0000","last_became_peered":"2023-10-09T05:56:11.379097+0000","last_unstale":"2023-10-09T05:56:16.430188+0000","last_undegraded":"2023-10-09T05:56:11.342416+0000","last_fullsized":"2023-10-09T05:56:16.430188+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":49,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.389582+0000","last_change":"2023-10-09T05:56:00.225236+0000","last_active":"2023-10-09T05:56:19.389582+0000","last_peered":"2023-10-09T05:56:19.389582+0000","last_clean":"2023-10-09T05:56:19.389582+0000","last_became_active":"2023-10-09T05:56:00.224653+0000","last_became_peered":"2023-10-09T05:56:00.224653+0000","last_unstale":"2023-10-09T05:56:19.389582+0000","last_undegraded":"2023-10-09T05:56:19.389582+0000","last_fullsized":"2023-10-09T05:56:19.389582+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"423'5678","reported_seq":6832,"reported_epoch":423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:19.519503+0000","last_change":"2023-10-09T05:56:12.532970+0000","last_active":"2023-10-09T05:56:19.519503+0000","last_peered":"2023-10-09T05:56:19.519503+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T05:56:12.505348+0000","last_became_peered":"2023-10-09T05:56:12.505348+0000","last_unstale":"2023-10-09T05:56:19.519503+0000","last_undegraded":"2023-10-09T05:56:12.423351+0000","last_fullsized":"2023-10-09T05:56:12.346143+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5678,"log_dups_size":0,"ondisk_log_size":5678,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133332992,"num_objects":2034,"num_object_clones":0,"num_object_copies":6102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2735,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3236,"num_write_kb":168672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":346,"num_bytes_recovered":22675456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,1,5","objects":1009},{"shards":"0,5,7","objects":844},{"shards":"5,7","objects":159}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":42,"reported_epoch":423,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:19.389689+0000","last_change":"2023-10-09T05:56:11.338260+0000","last_active":"2023-10-09T05:56:19.389689+0000","last_peered":"2023-10-09T05:56:19.389689+0000","last_clean":"2023-10-09T05:56:02.699477+0000","last_became_active":"2023-10-09T05:56:11.338062+0000","last_became_peered":"2023-10-09T05:56:11.338062+0000","last_unstale":"2023-10-09T05:56:19.389689+0000","last_undegraded":"2023-10-09T05:56:11.322630+0000","last_fullsized":"2023-10-09T05:56:19.389689+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":97,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.378161+0000","last_change":"2023-10-09T05:55:39.368653+0000","last_active":"2023-10-09T05:56:16.378161+0000","last_peered":"2023-10-09T05:56:16.378161+0000","last_clean":"2023-10-09T05:56:16.378161+0000","last_became_active":"2023-10-09T05:55:39.368361+0000","last_became_peered":"2023-10-09T05:55:39.368361+0000","last_unstale":"2023-10-09T05:56:16.378161+0000","last_undegraded":"2023-10-09T05:56:16.378161+0000","last_fullsized":"2023-10-09T05:56:16.378161+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"424'5683","reported_seq":5780,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:21.896455+0000","last_change":"2023-10-09T05:56:01.252121+0000","last_active":"2023-10-09T05:56:21.896455+0000","last_peered":"2023-10-09T05:56:21.896455+0000","last_clean":"2023-10-09T05:55:48.934818+0000","last_became_active":"2023-10-09T05:56:01.246051+0000","last_became_peered":"2023-10-09T05:56:01.246051+0000","last_unstale":"2023-10-09T05:56:21.896455+0000","last_undegraded":"2023-10-09T05:56:01.224717+0000","last_fullsized":"2023-10-09T05:56:21.896455+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5683,"log_dups_size":0,"ondisk_log_size":5683,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132972544,"num_objects":2029,"num_object_clones":0,"num_object_copies":6087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3253,"num_write_kb":169024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":2029}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":36,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.389283+0000","last_change":"2023-10-09T05:55:26.358519+0000","last_active":"2023-10-09T05:56:19.389283+0000","last_peered":"2023-10-09T05:56:19.389283+0000","last_clean":"2023-10-09T05:56:19.389283+0000","last_became_active":"2023-10-09T05:55:26.260138+0000","last_became_peered":"2023-10-09T05:55:26.260138+0000","last_unstale":"2023-10-09T05:56:19.389283+0000","last_undegraded":"2023-10-09T05:56:19.389283+0000","last_fullsized":"2023-10-09T05:56:19.389283+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":449,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.391655+0000","last_change":"2023-10-09T05:56:00.235727+0000","last_active":"2023-10-09T05:56:16.391655+0000","last_peered":"2023-10-09T05:56:16.391655+0000","last_clean":"2023-10-09T05:56:16.391655+0000","last_became_active":"2023-10-09T05:56:00.235388+0000","last_became_peered":"2023-10-09T05:56:00.235388+0000","last_unstale":"2023-10-09T05:56:16.391655+0000","last_undegraded":"2023-10-09T05:56:16.391655+0000","last_fullsized":"2023-10-09T05:56:16.391655+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"423'5641","reported_seq":6710,"reported_epoch":423,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:56:19.733916+0000","last_change":"2023-10-09T05:56:19.004096+0000","last_active":"2023-10-09T05:56:19.733916+0000","last_peered":"2023-10-09T05:56:19.733916+0000","last_clean":"2023-10-09T05:53:45.891460+0000","last_became_active":"2023-10-09T05:56:02.059564+0000","last_became_peered":"2023-10-09T05:56:02.059564+0000","last_unstale":"2023-10-09T05:56:19.733916+0000","last_undegraded":"2023-10-09T05:56:19.733916+0000","last_fullsized":"2023-10-09T05:56:01.204624+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5641,"log_dups_size":0,"ondisk_log_size":5641,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131661824,"num_objects":2009,"num_object_clones":0,"num_object_copies":6027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1116,"num_objects_unfound":0,"num_objects_dirty":2009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3217,"num_write_kb":167232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":480,"num_bytes_recovered":30343168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[5,3],"avail_no_missing":["5"],"object_location_counts":[{"shards":"2,3,5","objects":1116},{"shards":"3,5","objects":893}],"blocked_by":[],"up_primary":7,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":36,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.388360+0000","last_change":"2023-10-09T05:55:27.219053+0000","last_active":"2023-10-09T05:56:19.388360+0000","last_peered":"2023-10-09T05:56:19.388360+0000","last_clean":"2023-10-09T05:56:19.388360+0000","last_became_active":"2023-10-09T05:55:26.335357+0000","last_became_peered":"2023-10-09T05:55:26.335357+0000","last_unstale":"2023-10-09T05:56:19.388360+0000","last_undegraded":"2023-10-09T05:56:19.388360+0000","last_fullsized":"2023-10-09T05:56:19.388360+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":32,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.387449+0000","last_change":"2023-10-09T05:55:26.254446+0000","last_active":"2023-10-09T05:56:19.387449+0000","last_peered":"2023-10-09T05:56:19.387449+0000","last_clean":"2023-10-09T05:56:19.387449+0000","last_became_active":"2023-10-09T05:55:26.254088+0000","last_became_peered":"2023-10-09T05:55:26.254088+0000","last_unstale":"2023-10-09T05:56:19.387449+0000","last_undegraded":"2023-10-09T05:56:19.387449+0000","last_fullsized":"2023-10-09T05:56:19.387449+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"424'5584","reported_seq":5670,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T05:56:22.446914+0000","last_change":"2023-10-09T05:56:12.824335+0000","last_active":"2023-10-09T05:56:22.446914+0000","last_peered":"2023-10-09T05:56:22.446914+0000","last_clean":"2023-10-09T05:56:22.446914+0000","last_became_active":"2023-10-09T05:55:11.317275+0000","last_became_peered":"2023-10-09T05:55:11.317275+0000","last_unstale":"2023-10-09T05:56:22.446914+0000","last_undegraded":"2023-10-09T05:56:22.446914+0000","last_fullsized":"2023-10-09T05:56:22.446914+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5584,"log_dups_size":0,"ondisk_log_size":5584,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130744320,"num_objects":1995,"num_object_clones":0,"num_object_copies":5985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3185,"num_write_kb":165760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":408,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.377872+0000","last_change":"2023-10-09T05:56:00.245186+0000","last_active":"2023-10-09T05:56:16.377872+0000","last_peered":"2023-10-09T05:56:16.377872+0000","last_clean":"2023-10-09T05:56:16.377872+0000","last_became_active":"2023-10-09T05:56:00.244871+0000","last_became_peered":"2023-10-09T05:56:00.244871+0000","last_unstale":"2023-10-09T05:56:16.377872+0000","last_undegraded":"2023-10-09T05:56:16.377872+0000","last_fullsized":"2023-10-09T05:56:16.377872+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":286,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.416407+0000","last_change":"2023-10-09T05:56:11.340232+0000","last_active":"2023-10-09T05:56:19.416407+0000","last_peered":"2023-10-09T05:56:19.416407+0000","last_clean":"2023-10-09T05:56:19.416407+0000","last_became_active":"2023-10-09T05:56:11.339675+0000","last_became_peered":"2023-10-09T05:56:11.339675+0000","last_unstale":"2023-10-09T05:56:19.416407+0000","last_undegraded":"2023-10-09T05:56:19.416407+0000","last_fullsized":"2023-10-09T05:56:19.416407+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:50.837847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"424'5708","reported_seq":16477,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.430247+0000","last_change":"2023-10-09T05:56:20.425478+0000","last_active":"2023-10-09T05:56:20.430247+0000","last_peered":"2023-10-09T05:56:20.430247+0000","last_clean":"2023-10-09T05:56:20.430247+0000","last_became_active":"2023-10-09T05:56:20.416446+0000","last_became_peered":"2023-10-09T05:56:20.416446+0000","last_unstale":"2023-10-09T05:56:20.430247+0000","last_undegraded":"2023-10-09T05:56:20.430247+0000","last_fullsized":"2023-10-09T05:56:20.430247+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5708,"log_dups_size":0,"ondisk_log_size":5708,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132939776,"num_objects":2028,"num_object_clones":0,"num_object_copies":6084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3254,"num_write_kb":169056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4116,"num_bytes_recovered":234323968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":286,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.394278+0000","last_change":"2023-10-09T05:56:11.343723+0000","last_active":"2023-10-09T05:56:19.394278+0000","last_peered":"2023-10-09T05:56:19.394278+0000","last_clean":"2023-10-09T05:56:19.394278+0000","last_became_active":"2023-10-09T05:56:11.337217+0000","last_became_peered":"2023-10-09T05:56:11.337217+0000","last_unstale":"2023-10-09T05:56:19.394278+0000","last_undegraded":"2023-10-09T05:56:19.394278+0000","last_fullsized":"2023-10-09T05:56:19.394278+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:55.912242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":406,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.395967+0000","last_change":"2023-10-09T05:56:00.235557+0000","last_active":"2023-10-09T05:56:16.395967+0000","last_peered":"2023-10-09T05:56:16.395967+0000","last_clean":"2023-10-09T05:56:16.395967+0000","last_became_active":"2023-10-09T05:56:00.235199+0000","last_became_peered":"2023-10-09T05:56:00.235199+0000","last_unstale":"2023-10-09T05:56:16.395967+0000","last_undegraded":"2023-10-09T05:56:16.395967+0000","last_fullsized":"2023-10-09T05:56:16.395967+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"424'5595","reported_seq":5677,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T05:56:22.272117+0000","last_change":"2023-10-09T05:56:11.007809+0000","last_active":"2023-10-09T05:56:22.272117+0000","last_peered":"2023-10-09T05:56:22.272117+0000","last_clean":"2023-10-09T05:56:22.272117+0000","last_became_active":"2023-10-09T05:54:37.703304+0000","last_became_peered":"2023-10-09T05:54:37.703304+0000","last_unstale":"2023-10-09T05:56:22.272117+0000","last_undegraded":"2023-10-09T05:56:22.272117+0000","last_fullsized":"2023-10-09T05:56:22.272117+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5595,"log_dups_size":0,"ondisk_log_size":5595,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130744320,"num_objects":1995,"num_object_clones":0,"num_object_copies":5985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3205,"num_write_kb":166400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":254,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.430725+0000","last_change":"2023-10-09T05:54:59.250695+0000","last_active":"2023-10-09T05:56:16.430725+0000","last_peered":"2023-10-09T05:56:16.430725+0000","last_clean":"2023-10-09T05:56:16.430725+0000","last_became_active":"2023-10-09T05:54:59.250380+0000","last_became_peered":"2023-10-09T05:54:59.250380+0000","last_unstale":"2023-10-09T05:56:16.430725+0000","last_undegraded":"2023-10-09T05:56:16.430725+0000","last_fullsized":"2023-10-09T05:56:16.430725+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":353,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.382974+0000","last_change":"2023-10-09T05:54:05.161959+0000","last_active":"2023-10-09T05:56:20.382974+0000","last_peered":"2023-10-09T05:56:20.382974+0000","last_clean":"2023-10-09T05:56:20.382974+0000","last_became_active":"2023-10-09T05:54:05.161671+0000","last_became_peered":"2023-10-09T05:54:05.161671+0000","last_unstale":"2023-10-09T05:56:20.382974+0000","last_undegraded":"2023-10-09T05:56:20.382974+0000","last_fullsized":"2023-10-09T05:56:20.382974+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:47.282176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"424'5637","reported_seq":11537,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:22.067014+0000","last_change":"2023-10-09T05:56:01.447837+0000","last_active":"2023-10-09T05:56:22.067014+0000","last_peered":"2023-10-09T05:56:22.067014+0000","last_clean":"2023-10-09T05:55:25.103026+0000","last_became_active":"2023-10-09T05:56:01.440057+0000","last_became_peered":"2023-10-09T05:56:01.440057+0000","last_unstale":"2023-10-09T05:56:22.067014+0000","last_undegraded":"2023-10-09T05:56:01.309926+0000","last_fullsized":"2023-10-09T05:56:01.207038+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5637,"log_dups_size":0,"ondisk_log_size":5637,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130285568,"num_objects":1988,"num_object_clones":0,"num_object_copies":5964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3224,"num_write_kb":166784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1468,"num_bytes_recovered":96206848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1,3","objects":-87},{"shards":"1,3,5","objects":2017}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":425,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.416900+0000","last_change":"2023-10-09T05:55:26.256379+0000","last_active":"2023-10-09T05:56:19.416900+0000","last_peered":"2023-10-09T05:56:19.416900+0000","last_clean":"2023-10-09T05:56:19.416900+0000","last_became_active":"2023-10-09T05:55:26.256062+0000","last_became_peered":"2023-10-09T05:55:26.256062+0000","last_unstale":"2023-10-09T05:56:19.416900+0000","last_undegraded":"2023-10-09T05:56:19.416900+0000","last_fullsized":"2023-10-09T05:56:19.416900+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:00.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":415,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.392671+0000","last_change":"2023-10-09T05:48:54.492708+0000","last_active":"2023-10-09T05:56:16.392671+0000","last_peered":"2023-10-09T05:56:16.392671+0000","last_clean":"2023-10-09T05:56:16.392671+0000","last_became_active":"2023-10-09T05:48:54.492382+0000","last_became_peered":"2023-10-09T05:48:54.492382+0000","last_unstale":"2023-10-09T05:56:16.392671+0000","last_undegraded":"2023-10-09T05:56:16.392671+0000","last_fullsized":"2023-10-09T05:56:16.392671+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"424'5688","reported_seq":5877,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T05:56:22.192608+0000","last_change":"2023-10-09T05:56:01.421456+0000","last_active":"2023-10-09T05:56:22.192608+0000","last_peered":"2023-10-09T05:56:22.192608+0000","last_clean":"2023-10-09T05:56:22.192608+0000","last_became_active":"2023-10-09T05:56:01.227129+0000","last_became_peered":"2023-10-09T05:56:01.227129+0000","last_unstale":"2023-10-09T05:56:22.192608+0000","last_undegraded":"2023-10-09T05:56:22.192608+0000","last_fullsized":"2023-10-09T05:56:22.192608+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129138688,"num_objects":1970,"num_object_clones":0,"num_object_copies":5910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3266,"num_write_kb":167584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":49,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.413686+0000","last_change":"2023-10-09T05:56:00.235969+0000","last_active":"2023-10-09T05:56:19.413686+0000","last_peered":"2023-10-09T05:56:19.413686+0000","last_clean":"2023-10-09T05:56:19.413686+0000","last_became_active":"2023-10-09T05:56:00.235240+0000","last_became_peered":"2023-10-09T05:56:00.235240+0000","last_unstale":"2023-10-09T05:56:19.413686+0000","last_undegraded":"2023-10-09T05:56:19.413686+0000","last_fullsized":"2023-10-09T05:56:19.413686+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":110,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.415964+0000","last_change":"2023-10-09T05:56:00.225724+0000","last_active":"2023-10-09T05:56:19.415964+0000","last_peered":"2023-10-09T05:56:19.415964+0000","last_clean":"2023-10-09T05:56:19.415964+0000","last_became_active":"2023-10-09T05:56:00.225335+0000","last_became_peered":"2023-10-09T05:56:00.225335+0000","last_unstale":"2023-10-09T05:56:19.415964+0000","last_undegraded":"2023-10-09T05:56:19.415964+0000","last_fullsized":"2023-10-09T05:56:19.415964+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:41.405222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"423'6609","reported_seq":6693,"reported_epoch":423,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:19.550031+0000","last_change":"2023-10-09T05:56:01.503330+0000","last_active":"2023-10-09T05:56:19.550031+0000","last_peered":"2023-10-09T05:56:19.550031+0000","last_clean":"2023-10-09T05:55:49.154347+0000","last_became_active":"2023-10-09T05:56:01.480211+0000","last_became_peered":"2023-10-09T05:56:01.480211+0000","last_unstale":"2023-10-09T05:56:19.550031+0000","last_undegraded":"2023-10-09T05:56:01.465289+0000","last_fullsized":"2023-10-09T05:56:19.550031+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267190272,"num_objects":4077,"num_object_clones":0,"num_object_copies":12231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":93,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6609,"num_write_kb":341952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,3,5","objects":4077}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":852,"reported_epoch":423,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:19.387698+0000","last_change":"2023-10-09T05:56:12.339160+0000","last_active":"2023-10-09T05:56:19.387698+0000","last_peered":"2023-10-09T05:56:19.387698+0000","last_clean":"2023-10-09T05:55:22.199269+0000","last_became_active":"2023-10-09T05:56:12.338771+0000","last_became_peered":"2023-10-09T05:56:12.338771+0000","last_unstale":"2023-10-09T05:56:19.387698+0000","last_undegraded":"2023-10-09T05:56:12.324800+0000","last_fullsized":"2023-10-09T05:56:19.387698+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":422,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.387087+0000","last_change":"2023-10-09T05:56:00.220470+0000","last_active":"2023-10-09T05:56:16.387087+0000","last_peered":"2023-10-09T05:56:16.387087+0000","last_clean":"2023-10-09T05:56:16.387087+0000","last_became_active":"2023-10-09T05:56:00.220204+0000","last_became_peered":"2023-10-09T05:56:00.220204+0000","last_unstale":"2023-10-09T05:56:16.387087+0000","last_undegraded":"2023-10-09T05:56:16.387087+0000","last_fullsized":"2023-10-09T05:56:16.387087+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":90,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.379074+0000","last_change":"2023-10-09T05:55:39.308036+0000","last_active":"2023-10-09T05:56:16.379074+0000","last_peered":"2023-10-09T05:56:16.379074+0000","last_clean":"2023-10-09T05:56:16.379074+0000","last_became_active":"2023-10-09T05:55:39.307675+0000","last_became_peered":"2023-10-09T05:55:39.307675+0000","last_unstale":"2023-10-09T05:56:16.379074+0000","last_undegraded":"2023-10-09T05:56:16.379074+0000","last_fullsized":"2023-10-09T05:56:16.379074+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"424'5599","reported_seq":6601,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:20.631244+0000","last_change":"2023-10-09T05:56:15.816626+0000","last_active":"2023-10-09T05:56:20.631244+0000","last_peered":"2023-10-09T05:56:20.631244+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T05:56:15.741101+0000","last_became_peered":"2023-10-09T05:56:15.741101+0000","last_unstale":"2023-10-09T05:56:20.631244+0000","last_undegraded":"2023-10-09T05:56:15.691240+0000","last_fullsized":"2023-10-09T05:56:12.324475+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5599,"log_dups_size":0,"ondisk_log_size":5599,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132677632,"num_objects":2024,"num_object_clones":0,"num_object_copies":6072,"num_objects_missing_on_primary":1639,"num_objects_missing":1639,"num_objects_degraded":2604,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3192,"num_write_kb":166944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":450,"num_bytes_recovered":29523968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1190},{"shards":"0,5,7","objects":830},{"shards":"5,7","objects":-23}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":913,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.379250+0000","last_change":"2023-10-09T05:48:54.592642+0000","last_active":"2023-10-09T05:56:16.379250+0000","last_peered":"2023-10-09T05:56:16.379250+0000","last_clean":"2023-10-09T05:56:16.379250+0000","last_became_active":"2023-10-09T05:48:54.491917+0000","last_became_peered":"2023-10-09T05:48:54.491917+0000","last_unstale":"2023-10-09T05:56:16.379250+0000","last_undegraded":"2023-10-09T05:56:16.379250+0000","last_fullsized":"2023-10-09T05:56:16.379250+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":370,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.391474+0000","last_change":"2023-10-09T05:56:11.353151+0000","last_active":"2023-10-09T05:56:16.391474+0000","last_peered":"2023-10-09T05:56:16.391474+0000","last_clean":"2023-10-09T05:56:16.391474+0000","last_became_active":"2023-10-09T05:56:11.352842+0000","last_became_peered":"2023-10-09T05:56:11.352842+0000","last_unstale":"2023-10-09T05:56:16.391474+0000","last_undegraded":"2023-10-09T05:56:16.391474+0000","last_fullsized":"2023-10-09T05:56:16.391474+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":249,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.416965+0000","last_change":"2023-10-09T05:56:00.228904+0000","last_active":"2023-10-09T05:56:19.416965+0000","last_peered":"2023-10-09T05:56:19.416965+0000","last_clean":"2023-10-09T05:56:19.416965+0000","last_became_active":"2023-10-09T05:56:00.228568+0000","last_became_peered":"2023-10-09T05:56:00.228568+0000","last_unstale":"2023-10-09T05:56:19.416965+0000","last_undegraded":"2023-10-09T05:56:19.416965+0000","last_fullsized":"2023-10-09T05:56:19.416965+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:18.837848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"423'6560","reported_seq":12307,"reported_epoch":423,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:56:19.382713+0000","last_change":"2023-10-09T05:55:40.311001+0000","last_active":"2023-10-09T05:56:19.382713+0000","last_peered":"2023-10-09T05:56:19.382713+0000","last_clean":"2023-10-09T05:55:25.212985+0000","last_became_active":"2023-10-09T05:55:40.298416+0000","last_became_peered":"2023-10-09T05:55:40.298416+0000","last_unstale":"2023-10-09T05:56:19.382713+0000","last_undegraded":"2023-10-09T05:56:19.382713+0000","last_fullsized":"2023-10-09T05:56:19.382713+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6560,"log_dups_size":0,"ondisk_log_size":6560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265158656,"num_objects":4046,"num_object_clones":0,"num_object_copies":12138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4046,"num_objects_unfound":0,"num_objects_dirty":4046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6560,"num_write_kb":339392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2829,"num_bytes_recovered":147193856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":4046}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":770,"reported_epoch":423,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:19.416618+0000","last_change":"2023-10-09T05:56:11.337498+0000","last_active":"2023-10-09T05:56:19.416618+0000","last_peered":"2023-10-09T05:56:19.416618+0000","last_clean":"2023-10-09T05:56:02.545242+0000","last_became_active":"2023-10-09T05:56:11.336951+0000","last_became_peered":"2023-10-09T05:56:11.336951+0000","last_unstale":"2023-10-09T05:56:19.416618+0000","last_undegraded":"2023-10-09T05:56:11.323856+0000","last_fullsized":"2023-10-09T05:56:19.416618+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:54.695500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":["2","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":207,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.383316+0000","last_change":"2023-10-09T05:50:18.063466+0000","last_active":"2023-10-09T05:56:20.383316+0000","last_peered":"2023-10-09T05:56:20.383316+0000","last_clean":"2023-10-09T05:56:20.383316+0000","last_became_active":"2023-10-09T05:50:18.063120+0000","last_became_peered":"2023-10-09T05:50:18.063120+0000","last_unstale":"2023-10-09T05:56:20.383316+0000","last_undegraded":"2023-10-09T05:56:20.383316+0000","last_fullsized":"2023-10-09T05:56:20.383316+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:13.683314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":235,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.390491+0000","last_change":"2023-10-09T05:56:11.340005+0000","last_active":"2023-10-09T05:56:16.390491+0000","last_peered":"2023-10-09T05:56:16.390491+0000","last_clean":"2023-10-09T05:56:16.390491+0000","last_became_active":"2023-10-09T05:56:11.339643+0000","last_became_peered":"2023-10-09T05:56:11.339643+0000","last_unstale":"2023-10-09T05:56:16.390491+0000","last_undegraded":"2023-10-09T05:56:16.390491+0000","last_fullsized":"2023-10-09T05:56:16.390491+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"422'5615","reported_seq":5726,"reported_epoch":423,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:19.419387+0000","last_change":"2023-10-09T05:56:16.252401+0000","last_active":"2023-10-09T05:56:19.419387+0000","last_peered":"2023-10-09T05:56:19.419387+0000","last_clean":"2023-10-09T05:55:49.071451+0000","last_became_active":"2023-10-09T05:56:16.190148+0000","last_became_peered":"2023-10-09T05:56:16.190148+0000","last_unstale":"2023-10-09T05:56:19.419387+0000","last_undegraded":"2023-10-09T05:56:13.840760+0000","last_fullsized":"2023-10-09T05:56:19.419387+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5615,"log_dups_size":0,"ondisk_log_size":5615,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133169152,"num_objects":2032,"num_object_clones":0,"num_object_copies":6096,"num_objects_missing_on_primary":2566,"num_objects_missing":2566,"num_objects_degraded":2607,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3202,"num_write_kb":167488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":1376256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":2003},{"shards":"3,5,7","objects":29}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":1121,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.378780+0000","last_change":"2023-10-09T05:56:00.632765+0000","last_active":"2023-10-09T05:56:16.378780+0000","last_peered":"2023-10-09T05:56:16.378780+0000","last_clean":"2023-10-09T05:56:16.378780+0000","last_became_active":"2023-10-09T05:55:40.295413+0000","last_became_peered":"2023-10-09T05:55:40.295413+0000","last_unstale":"2023-10-09T05:56:16.378780+0000","last_undegraded":"2023-10-09T05:56:16.378780+0000","last_fullsized":"2023-10-09T05:56:16.378780+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":561,"reported_epoch":423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:19.425761+0000","last_change":"2023-10-09T05:56:16.297651+0000","last_active":"2023-10-09T05:56:19.425761+0000","last_peered":"2023-10-09T05:56:19.425761+0000","last_clean":"2023-10-09T05:56:02.257688+0000","last_became_active":"2023-10-09T05:56:16.262977+0000","last_became_peered":"2023-10-09T05:56:16.262977+0000","last_unstale":"2023-10-09T05:56:19.425761+0000","last_undegraded":"2023-10-09T05:56:16.181686+0000","last_fullsized":"2023-10-09T05:56:16.146296+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","0","6"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":308,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.379163+0000","last_change":"2023-10-09T05:56:00.243281+0000","last_active":"2023-10-09T05:56:16.379163+0000","last_peered":"2023-10-09T05:56:16.379163+0000","last_clean":"2023-10-09T05:56:16.379163+0000","last_became_active":"2023-10-09T05:56:00.242999+0000","last_became_peered":"2023-10-09T05:56:00.242999+0000","last_unstale":"2023-10-09T05:56:16.379163+0000","last_undegraded":"2023-10-09T05:56:16.379163+0000","last_fullsized":"2023-10-09T05:56:16.379163+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":32,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.388146+0000","last_change":"2023-10-09T05:55:26.336440+0000","last_active":"2023-10-09T05:56:19.388146+0000","last_peered":"2023-10-09T05:56:19.388146+0000","last_clean":"2023-10-09T05:56:19.388146+0000","last_became_active":"2023-10-09T05:55:26.335530+0000","last_became_peered":"2023-10-09T05:55:26.335530+0000","last_unstale":"2023-10-09T05:56:19.388146+0000","last_undegraded":"2023-10-09T05:56:19.388146+0000","last_fullsized":"2023-10-09T05:56:19.388146+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"422'6522","reported_seq":6596,"reported_epoch":422,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T05:56:18.844593+0000","last_change":"2023-10-09T05:56:12.533036+0000","last_active":"2023-10-09T05:56:18.844593+0000","last_peered":"2023-10-09T05:56:18.844593+0000","last_clean":"2023-10-09T05:56:10.073783+0000","last_became_active":"2023-10-09T05:56:12.516093+0000","last_became_peered":"2023-10-09T05:56:12.516093+0000","last_unstale":"2023-10-09T05:56:18.844593+0000","last_undegraded":"2023-10-09T05:56:18.844593+0000","last_fullsized":"2023-10-09T05:56:18.844593+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6522,"log_dups_size":0,"ondisk_log_size":6522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:46.643480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268042240,"num_objects":4090,"num_object_clones":0,"num_object_copies":12270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5306,"num_objects_unfound":0,"num_objects_dirty":4090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6522,"num_write_kb":339584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[0,6,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":-39}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":933,"reported_epoch":423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:19.426265+0000","last_change":"2023-10-09T05:56:16.297927+0000","last_active":"2023-10-09T05:56:19.426265+0000","last_peered":"2023-10-09T05:56:19.426265+0000","last_clean":"2023-10-09T05:56:02.257755+0000","last_became_active":"2023-10-09T05:56:16.263458+0000","last_became_peered":"2023-10-09T05:56:16.263458+0000","last_unstale":"2023-10-09T05:56:19.426265+0000","last_undegraded":"2023-10-09T05:56:16.199131+0000","last_fullsized":"2023-10-09T05:56:16.146936+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":435,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.391405+0000","last_change":"2023-10-09T05:56:11.334999+0000","last_active":"2023-10-09T05:56:16.391405+0000","last_peered":"2023-10-09T05:56:16.391405+0000","last_clean":"2023-10-09T05:56:16.391405+0000","last_became_active":"2023-10-09T05:56:11.334587+0000","last_became_peered":"2023-10-09T05:56:11.334587+0000","last_unstale":"2023-10-09T05:56:16.391405+0000","last_undegraded":"2023-10-09T05:56:16.391405+0000","last_fullsized":"2023-10-09T05:56:16.391405+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":301,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.417142+0000","last_change":"2023-10-09T05:56:11.340751+0000","last_active":"2023-10-09T05:56:19.417142+0000","last_peered":"2023-10-09T05:56:19.417142+0000","last_clean":"2023-10-09T05:56:19.417142+0000","last_became_active":"2023-10-09T05:56:11.340499+0000","last_became_peered":"2023-10-09T05:56:11.340499+0000","last_unstale":"2023-10-09T05:56:19.417142+0000","last_undegraded":"2023-10-09T05:56:19.417142+0000","last_fullsized":"2023-10-09T05:56:19.417142+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:36.120602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"422'6444","reported_seq":7834,"reported_epoch":422,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:18.993466+0000","last_change":"2023-10-09T05:56:00.305724+0000","last_active":"2023-10-09T05:56:18.993466+0000","last_peered":"2023-10-09T05:56:18.993466+0000","last_clean":"2023-10-09T05:53:45.896176+0000","last_became_active":"2023-10-09T05:55:38.711627+0000","last_became_peered":"2023-10-09T05:55:38.711627+0000","last_unstale":"2023-10-09T05:56:18.993466+0000","last_undegraded":"2023-10-09T05:56:00.290032+0000","last_fullsized":"2023-10-09T05:55:27.232363+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6444,"log_dups_size":0,"ondisk_log_size":6444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:26.171175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262144000,"num_objects":4000,"num_object_clones":0,"num_object_copies":12000,"num_objects_missing_on_primary":2205,"num_objects_missing":2205,"num_objects_degraded":2205,"num_objects_misplaced":1082,"num_objects_unfound":0,"num_objects_dirty":4000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6444,"num_write_kb":334208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":639,"num_bytes_recovered":41615360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[0,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2354},{"shards":"0,1,4,7","objects":12},{"shards":"1,4,7","objects":1634}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":595,"reported_epoch":423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:19.388674+0000","last_change":"2023-10-09T05:56:12.339234+0000","last_active":"2023-10-09T05:56:19.388674+0000","last_peered":"2023-10-09T05:56:19.388674+0000","last_clean":"2023-10-09T05:56:02.265149+0000","last_became_active":"2023-10-09T05:56:12.338651+0000","last_became_peered":"2023-10-09T05:56:12.338651+0000","last_unstale":"2023-10-09T05:56:19.388674+0000","last_undegraded":"2023-10-09T05:56:12.325644+0000","last_fullsized":"2023-10-09T05:56:12.324625+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":224,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.417185+0000","last_change":"2023-10-09T05:51:43.127835+0000","last_active":"2023-10-09T05:56:19.417185+0000","last_peered":"2023-10-09T05:56:19.417185+0000","last_clean":"2023-10-09T05:56:19.417185+0000","last_became_active":"2023-10-09T05:51:43.127489+0000","last_became_peered":"2023-10-09T05:51:43.127489+0000","last_unstale":"2023-10-09T05:56:19.417185+0000","last_undegraded":"2023-10-09T05:56:19.417185+0000","last_fullsized":"2023-10-09T05:56:19.417185+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:20.191774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":403,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.378693+0000","last_change":"2023-10-09T05:56:11.340202+0000","last_active":"2023-10-09T05:56:16.378693+0000","last_peered":"2023-10-09T05:56:16.378693+0000","last_clean":"2023-10-09T05:56:16.378693+0000","last_became_active":"2023-10-09T05:56:11.339869+0000","last_became_peered":"2023-10-09T05:56:11.339869+0000","last_unstale":"2023-10-09T05:56:16.378693+0000","last_undegraded":"2023-10-09T05:56:16.378693+0000","last_fullsized":"2023-10-09T05:56:16.378693+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"424'6469","reported_seq":7843,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:22.399165+0000","last_change":"2023-10-09T05:56:01.470487+0000","last_active":"2023-10-09T05:56:22.399165+0000","last_peered":"2023-10-09T05:56:22.399165+0000","last_clean":"2023-10-09T05:53:45.883404+0000","last_became_active":"2023-10-09T05:55:27.462550+0000","last_became_peered":"2023-10-09T05:55:27.462550+0000","last_unstale":"2023-10-09T05:56:22.399165+0000","last_undegraded":"2023-10-09T05:56:01.468832+0000","last_fullsized":"2023-10-09T05:55:27.236257+0000","mapping_epoch":402,"log_start":"369'1661","ondisk_log_start":"369'1661","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4808,"log_dups_size":0,"ondisk_log_size":4808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:54.072617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262144024,"num_objects":4001,"num_object_clones":0,"num_object_copies":12003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3238,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4001,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6469,"num_write_kb":334977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":641,"num_bytes_recovered":42008576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,4,5","objects":2308},{"shards":"1,5","objects":1693}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":68,"reported_epoch":423,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:19.389953+0000","last_change":"2023-10-09T05:56:11.349145+0000","last_active":"2023-10-09T05:56:19.389953+0000","last_peered":"2023-10-09T05:56:19.389953+0000","last_clean":"2023-10-09T05:56:02.699661+0000","last_became_active":"2023-10-09T05:56:11.340483+0000","last_became_peered":"2023-10-09T05:56:11.340483+0000","last_unstale":"2023-10-09T05:56:19.389953+0000","last_undegraded":"2023-10-09T05:56:11.321986+0000","last_fullsized":"2023-10-09T05:56:19.389953+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":["7","0","3"],"object_location_counts":[{"shards":"0,3,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":507,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.416588+0000","last_change":"2023-10-09T05:54:59.255889+0000","last_active":"2023-10-09T05:56:19.416588+0000","last_peered":"2023-10-09T05:56:19.416588+0000","last_clean":"2023-10-09T05:56:19.416588+0000","last_became_active":"2023-10-09T05:54:59.255366+0000","last_became_peered":"2023-10-09T05:54:59.255366+0000","last_unstale":"2023-10-09T05:56:19.416588+0000","last_undegraded":"2023-10-09T05:56:19.416588+0000","last_fullsized":"2023-10-09T05:56:19.416588+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:22.233818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.413771+0000","last_change":"2023-10-09T05:56:16.241536+0000","last_active":"2023-10-09T05:56:19.413771+0000","last_peered":"2023-10-09T05:56:19.413771+0000","last_clean":"2023-10-09T05:56:19.413771+0000","last_became_active":"2023-10-09T05:56:16.180128+0000","last_became_peered":"2023-10-09T05:56:16.180128+0000","last_unstale":"2023-10-09T05:56:19.413771+0000","last_undegraded":"2023-10-09T05:56:19.413771+0000","last_fullsized":"2023-10-09T05:56:19.413771+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"424'6529","reported_seq":8937,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:21.000537+0000","last_change":"2023-10-09T05:56:05.004837+0000","last_active":"2023-10-09T05:56:21.000537+0000","last_peered":"2023-10-09T05:56:21.000537+0000","last_clean":"2023-10-09T05:53:45.907519+0000","last_became_active":"2023-10-09T05:56:03.326563+0000","last_became_peered":"2023-10-09T05:56:03.326563+0000","last_unstale":"2023-10-09T05:56:21.000537+0000","last_undegraded":"2023-10-09T05:56:03.288425+0000","last_fullsized":"2023-10-09T05:56:21.000537+0000","mapping_epoch":416,"log_start":"367'355","ondisk_log_start":"367'355","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6174,"log_dups_size":0,"ondisk_log_size":6174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266141696,"num_objects":4061,"num_object_clones":0,"num_object_copies":12183,"num_objects_missing_on_primary":2853,"num_objects_missing":2853,"num_objects_degraded":2939,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6529,"num_write_kb":338880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1149,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["4"],"object_location_counts":[{"shards":"0,3,4","objects":1904},{"shards":"3,4,7","objects":2157}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":594,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.389848+0000","last_change":"2023-10-09T05:55:55.247195+0000","last_active":"2023-10-09T05:56:19.389848+0000","last_peered":"2023-10-09T05:56:19.389848+0000","last_clean":"2023-10-09T05:56:19.389848+0000","last_became_active":"2023-10-09T05:55:55.246499+0000","last_became_peered":"2023-10-09T05:55:55.246499+0000","last_unstale":"2023-10-09T05:56:19.389848+0000","last_undegraded":"2023-10-09T05:56:19.389848+0000","last_fullsized":"2023-10-09T05:56:19.389848+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":12,"reported_epoch":423,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:19.388951+0000","last_change":"2023-10-09T05:56:11.349888+0000","last_active":"2023-10-09T05:56:19.388951+0000","last_peered":"2023-10-09T05:56:19.388951+0000","last_clean":"2023-10-09T05:56:02.259693+0000","last_became_active":"2023-10-09T05:56:11.349646+0000","last_became_peered":"2023-10-09T05:56:11.349646+0000","last_unstale":"2023-10-09T05:56:19.388951+0000","last_undegraded":"2023-10-09T05:56:11.325475+0000","last_fullsized":"2023-10-09T05:56:19.388951+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":358,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.377554+0000","last_change":"2023-10-09T05:55:39.368525+0000","last_active":"2023-10-09T05:56:16.377554+0000","last_peered":"2023-10-09T05:56:16.377554+0000","last_clean":"2023-10-09T05:56:16.377554+0000","last_became_active":"2023-10-09T05:55:39.368203+0000","last_became_peered":"2023-10-09T05:55:39.368203+0000","last_unstale":"2023-10-09T05:56:16.377554+0000","last_undegraded":"2023-10-09T05:56:16.377554+0000","last_fullsized":"2023-10-09T05:56:16.377554+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"424'5738","reported_seq":11120,"reported_epoch":424,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:56:22.390569+0000","last_change":"2023-10-09T05:56:00.270069+0000","last_active":"2023-10-09T05:56:22.390569+0000","last_peered":"2023-10-09T05:56:22.390569+0000","last_clean":"2023-10-09T05:55:25.040502+0000","last_became_active":"2023-10-09T05:56:00.230913+0000","last_became_peered":"2023-10-09T05:56:00.230913+0000","last_unstale":"2023-10-09T05:56:22.390569+0000","last_undegraded":"2023-10-09T05:56:22.390569+0000","last_fullsized":"2023-10-09T05:56:22.390569+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5738,"log_dups_size":0,"ondisk_log_size":5738,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132808704,"num_objects":2027,"num_object_clones":0,"num_object_copies":6081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2027,"num_objects_unfound":0,"num_objects_dirty":2027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3285,"num_write_kb":169952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1426,"num_bytes_recovered":93421568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[3,6,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2027}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":405,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.387925+0000","last_change":"2023-10-09T05:56:11.340214+0000","last_active":"2023-10-09T05:56:19.387925+0000","last_peered":"2023-10-09T05:56:19.387925+0000","last_clean":"2023-10-09T05:56:19.387925+0000","last_became_active":"2023-10-09T05:56:11.339842+0000","last_became_peered":"2023-10-09T05:56:11.339842+0000","last_unstale":"2023-10-09T05:56:19.387925+0000","last_undegraded":"2023-10-09T05:56:19.387925+0000","last_fullsized":"2023-10-09T05:56:19.387925+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":115,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.417033+0000","last_change":"2023-10-09T05:56:00.226695+0000","last_active":"2023-10-09T05:56:19.417033+0000","last_peered":"2023-10-09T05:56:19.417033+0000","last_clean":"2023-10-09T05:56:19.417033+0000","last_became_active":"2023-10-09T05:56:00.226359+0000","last_became_peered":"2023-10-09T05:56:00.226359+0000","last_unstale":"2023-10-09T05:56:19.417033+0000","last_undegraded":"2023-10-09T05:56:19.417033+0000","last_fullsized":"2023-10-09T05:56:19.417033+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:20.323704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"424'5636","reported_seq":5720,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T05:56:22.367859+0000","last_change":"2023-10-09T05:55:11.328085+0000","last_active":"2023-10-09T05:56:22.367859+0000","last_peered":"2023-10-09T05:56:22.367859+0000","last_clean":"2023-10-09T05:56:22.367859+0000","last_became_active":"2023-10-09T05:55:11.322136+0000","last_became_peered":"2023-10-09T05:55:11.322136+0000","last_unstale":"2023-10-09T05:56:22.367859+0000","last_undegraded":"2023-10-09T05:56:22.367859+0000","last_fullsized":"2023-10-09T05:56:22.367859+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5636,"log_dups_size":0,"ondisk_log_size":5636,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129171456,"num_objects":1971,"num_object_clones":0,"num_object_copies":5913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3237,"num_write_kb":166656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":418,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.377791+0000","last_change":"2023-10-09T05:48:54.583404+0000","last_active":"2023-10-09T05:56:16.377791+0000","last_peered":"2023-10-09T05:56:16.377791+0000","last_clean":"2023-10-09T05:56:16.377791+0000","last_became_active":"2023-10-09T05:48:54.492565+0000","last_became_peered":"2023-10-09T05:48:54.492565+0000","last_unstale":"2023-10-09T05:56:16.377791+0000","last_undegraded":"2023-10-09T05:56:16.377791+0000","last_fullsized":"2023-10-09T05:56:16.377791+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":425,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.391553+0000","last_change":"2023-10-09T05:56:00.224626+0000","last_active":"2023-10-09T05:56:16.391553+0000","last_peered":"2023-10-09T05:56:16.391553+0000","last_clean":"2023-10-09T05:56:16.391553+0000","last_became_active":"2023-10-09T05:56:00.224322+0000","last_became_peered":"2023-10-09T05:56:00.224322+0000","last_unstale":"2023-10-09T05:56:16.391553+0000","last_undegraded":"2023-10-09T05:56:16.391553+0000","last_fullsized":"2023-10-09T05:56:16.391553+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.17","version":"424'5676","reported_seq":6724,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:20.871020+0000","last_change":"2023-10-09T05:56:02.661605+0000","last_active":"2023-10-09T05:56:20.871020+0000","last_peered":"2023-10-09T05:56:20.871020+0000","last_clean":"2023-10-09T05:53:45.891460+0000","last_became_active":"2023-10-09T05:56:02.061426+0000","last_became_peered":"2023-10-09T05:56:02.061426+0000","last_unstale":"2023-10-09T05:56:20.871020+0000","last_undegraded":"2023-10-09T05:56:02.046136+0000","last_fullsized":"2023-10-09T05:56:01.204705+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5676,"log_dups_size":0,"ondisk_log_size":5676,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131334144,"num_objects":2004,"num_object_clones":0,"num_object_copies":6012,"num_objects_missing_on_primary":1665,"num_objects_missing":1665,"num_objects_degraded":1717,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3252,"num_write_kb":168192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":469,"num_bytes_recovered":30670848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,3],"avail_no_missing":["5"],"object_location_counts":[{"shards":"2,3,5","objects":1197},{"shards":"3,7","objects":807}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":377,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.386999+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:56:16.386999+0000","last_peered":"2023-10-09T05:56:16.386999+0000","last_clean":"2023-10-09T05:56:16.386999+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:56:16.386999+0000","last_undegraded":"2023-10-09T05:56:16.386999+0000","last_fullsized":"2023-10-09T05:56:16.386999+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":15,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.417254+0000","last_change":"2023-10-09T05:56:11.337104+0000","last_active":"2023-10-09T05:56:19.417254+0000","last_peered":"2023-10-09T05:56:19.417254+0000","last_clean":"2023-10-09T05:56:19.417254+0000","last_became_active":"2023-10-09T05:56:11.336690+0000","last_became_peered":"2023-10-09T05:56:11.336690+0000","last_unstale":"2023-10-09T05:56:19.417254+0000","last_undegraded":"2023-10-09T05:56:19.417254+0000","last_fullsized":"2023-10-09T05:56:19.417254+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:56.369633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.14","version":"424'5700","reported_seq":5800,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:22.269230+0000","last_change":"2023-10-09T05:56:01.281023+0000","last_active":"2023-10-09T05:56:22.269230+0000","last_peered":"2023-10-09T05:56:22.269230+0000","last_clean":"2023-10-09T05:55:48.964110+0000","last_became_active":"2023-10-09T05:56:01.245699+0000","last_became_peered":"2023-10-09T05:56:01.245699+0000","last_unstale":"2023-10-09T05:56:22.269230+0000","last_undegraded":"2023-10-09T05:56:01.216362+0000","last_fullsized":"2023-10-09T05:56:22.269230+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5700,"log_dups_size":0,"ondisk_log_size":5700,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131334144,"num_objects":2004,"num_object_clones":0,"num_object_copies":6012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":40,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3270,"num_write_kb":168768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":2004}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":162,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.417331+0000","last_change":"2023-10-09T05:56:00.221032+0000","last_active":"2023-10-09T05:56:19.417331+0000","last_peered":"2023-10-09T05:56:19.417331+0000","last_clean":"2023-10-09T05:56:19.417331+0000","last_became_active":"2023-10-09T05:56:00.220693+0000","last_became_peered":"2023-10-09T05:56:00.220693+0000","last_unstale":"2023-10-09T05:56:19.417331+0000","last_undegraded":"2023-10-09T05:56:19.417331+0000","last_fullsized":"2023-10-09T05:56:19.417331+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:55.448253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":254,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.416788+0000","last_change":"2023-10-09T05:56:11.354072+0000","last_active":"2023-10-09T05:56:19.416788+0000","last_peered":"2023-10-09T05:56:19.416788+0000","last_clean":"2023-10-09T05:56:19.416788+0000","last_became_active":"2023-10-09T05:56:11.353547+0000","last_became_peered":"2023-10-09T05:56:11.353547+0000","last_unstale":"2023-10-09T05:56:19.416788+0000","last_undegraded":"2023-10-09T05:56:19.416788+0000","last_fullsized":"2023-10-09T05:56:19.416788+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:42.629830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"423'5740","reported_seq":6776,"reported_epoch":423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:19.383900+0000","last_change":"2023-10-09T05:56:12.557070+0000","last_active":"2023-10-09T05:56:19.383900+0000","last_peered":"2023-10-09T05:56:19.383900+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T05:56:12.536142+0000","last_became_peered":"2023-10-09T05:56:12.536142+0000","last_unstale":"2023-10-09T05:56:19.383900+0000","last_undegraded":"2023-10-09T05:56:12.474066+0000","last_fullsized":"2023-10-09T05:56:12.424748+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5740,"log_dups_size":0,"ondisk_log_size":5740,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132153344,"num_objects":2017,"num_object_clones":0,"num_object_copies":6051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2864,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3299,"num_write_kb":170080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":287,"num_bytes_recovered":18808832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,1,5","objects":1127},{"shards":"0,5,7","objects":799},{"shards":"5,7","objects":61}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":337,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T05:56:19.417687+0000","last_change":"2023-10-09T05:56:11.384692+0000","last_active":"2023-10-09T05:56:19.417687+0000","last_peered":"2023-10-09T05:56:19.417687+0000","last_clean":"2023-10-09T05:56:19.417687+0000","last_became_active":"2023-10-09T05:56:11.346444+0000","last_became_peered":"2023-10-09T05:56:11.346444+0000","last_unstale":"2023-10-09T05:56:19.417687+0000","last_undegraded":"2023-10-09T05:56:19.417687+0000","last_fullsized":"2023-10-09T05:56:19.417687+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:10.451624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":406,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T05:56:16.391163+0000","last_change":"2023-10-09T05:48:54.492580+0000","last_active":"2023-10-09T05:56:16.391163+0000","last_peered":"2023-10-09T05:56:16.391163+0000","last_clean":"2023-10-09T05:56:16.391163+0000","last_became_active":"2023-10-09T05:48:54.492259+0000","last_became_peered":"2023-10-09T05:48:54.492259+0000","last_unstale":"2023-10-09T05:56:16.391163+0000","last_undegraded":"2023-10-09T05:56:16.391163+0000","last_fullsized":"2023-10-09T05:56:16.391163+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":7,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":15756,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":4},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":4220059672,"num_objects":64394,"num_object_clones":0,"num_object_copies":193182,"num_objects_missing_on_primary":15229,"num_objects_missing":15229,"num_objects_degraded":28995,"num_objects_misplaced":15597,"num_objects_unfound":0,"num_objects_dirty":64394,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":103948,"num_write_kb":5386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16323,"num_bytes_recovered":989528064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13470871552,"data_stored":13470859336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":150316,"ondisk_log_size":150316,"up":78,"acting":69,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":46,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846853,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2276072,"kb_used_data":1362144,"kb_used_omap":14,"kb_used_meta":913905,"kb_avail":91469080,"statfs":{"total":95995035648,"available":93664337920,"internally_reserved":0,"allocated":1394835456,"data_stored":1394020024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":935839578},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,3,3],"upper_bound":16384},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:55:16 2023","interfaces":[{"interface":"back","average":{"1min":6.8460000000000001,"5min":7.7690000000000001,"15min":4.9109999999999996},"min":{"1min":0.24299999999999999,"5min":0.22800000000000001,"15min":0.20699999999999999},"max":{"1min":117.756,"5min":212.64400000000001,"15min":212.64400000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":6.8170000000000002,"5min":7.7530000000000001,"15min":4.8970000000000002},"min":{"1min":0.29899999999999999,"5min":0.29799999999999999,"15min":0.245},"max":{"1min":117.73,"5min":212.655,"15min":212.655},"last":0.32700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:55:20 2023","interfaces":[{"interface":"back","average":{"1min":4.9189999999999996,"5min":4.0270000000000001,"15min":2.8300000000000001},"min":{"1min":0.26900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":78.013000000000005,"5min":114.422,"15min":114.422},"last":0.77800000000000002},{"interface":"front","average":{"1min":5.0309999999999997,"5min":4.1230000000000002,"15min":2.923},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.255},"max":{"1min":78.174999999999997,"5min":111.708,"15min":111.708},"last":0.372}]},{"osd":2,"last update":"Mon Oct 9 05:55:20 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":5.0800000000000001,"15min":2.6949999999999998},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":12.047000000000001,"5min":205.863,"15min":205.863},"last":0.871},{"interface":"front","average":{"1min":1.1279999999999999,"5min":4.9109999999999996,"15min":2.6019999999999999},"min":{"1min":0.35199999999999998,"5min":0.313,"15min":0.28100000000000003},"max":{"1min":5.5010000000000003,"5min":205.88399999999999,"15min":205.88399999999999},"last":0.43099999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:55:16 2023","interfaces":[{"interface":"back","average":{"1min":7.5359999999999996,"5min":4.4610000000000003,"15min":4.8550000000000004},"min":{"1min":0.315,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":115.69799999999999,"5min":115.69799999999999,"15min":185.17099999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":7.7709999999999999,"5min":5.0720000000000001,"15min":5.032},"min":{"1min":0.39700000000000002,"5min":0.33000000000000002,"15min":0.29299999999999998},"max":{"1min":115.75700000000001,"5min":115.75700000000001,"15min":183.35499999999999},"last":0.45000000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":4.4790000000000001,"15min":1.923},"min":{"1min":0.26300000000000001,"5min":0.26000000000000001,"15min":0.25700000000000001},"max":{"1min":5.6130000000000004,"5min":159.50299999999999,"15min":159.50299999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.68500000000000005,"5min":4.3049999999999997,"15min":1.917},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":2.746,"5min":159.483,"15min":159.483},"last":0.64900000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:55:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2480000000000002,"5min":3.431,"15min":1.6859999999999999},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":94.093999999999994,"5min":94.093999999999994,"15min":94.093999999999994},"last":0.55500000000000005},{"interface":"front","average":{"1min":5.3810000000000002,"5min":3.7639999999999998,"15min":1.7909999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":95.579999999999998,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.81399999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.012,"15min":0.80600000000000005},"min":{"1min":0.32900000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.4700000000000002,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.022,"15min":0.90000000000000002},"min":{"1min":0.30599999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.5940000000000001,"5min":7.7359999999999998,"15min":8.0670000000000002},"last":0.84599999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626879,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1689748,"kb_used_data":785612,"kb_used_omap":0,"kb_used_meta":904128,"kb_avail":92055404,"statfs":{"total":95995035648,"available":94264733696,"internally_reserved":0,"allocated":804466688,"data_stored":803663579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925827072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:55:52 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":1.7430000000000001,"15min":2.9159999999999999},"min":{"1min":0.27600000000000002,"5min":0.23100000000000001,"15min":0.20999999999999999},"max":{"1min":2.8769999999999998,"5min":50.427,"15min":162.08799999999999},"last":1.2270000000000001},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.7909999999999999,"15min":2.9049999999999998},"min":{"1min":0.26200000000000001,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":3.5099999999999998,"5min":50.634999999999998,"15min":161.90899999999999},"last":0.48399999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:55:17 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":4.2880000000000003,"15min":3.7109999999999999},"min":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":6.8200000000000003,"5min":165.66399999999999,"15min":228.24299999999999},"last":2.7130000000000001},{"interface":"front","average":{"1min":1.4370000000000001,"5min":4.4770000000000003,"15min":3.6890000000000001},"min":{"1min":0.32200000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.9290000000000003,"5min":165.649,"15min":228.22499999999999},"last":1.266}]},{"osd":2,"last update":"Mon Oct 9 05:55:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":2.1560000000000001,"15min":2.4910000000000001},"min":{"1min":0.39900000000000002,"5min":0.29599999999999999,"15min":0.27200000000000002},"max":{"1min":3.004,"5min":91.049999999999997,"15min":91.049999999999997},"last":1.294},{"interface":"front","average":{"1min":1.0580000000000001,"5min":2.218,"15min":2.6019999999999999},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.29499999999999998},"max":{"1min":3.0350000000000001,"5min":91.105000000000004,"15min":91.105000000000004},"last":2.7610000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:55:17 2023","interfaces":[{"interface":"back","average":{"1min":4.0919999999999996,"5min":6.4160000000000004,"15min":4.4710000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.31},"max":{"1min":51.112000000000002,"5min":152.74700000000001,"15min":152.74700000000001},"last":2.79},{"interface":"front","average":{"1min":3.4180000000000001,"5min":6.1980000000000004,"15min":4.3970000000000002},"min":{"1min":0.35099999999999998,"5min":0.313,"15min":0.313},"max":{"1min":50.728000000000002,"5min":152.78100000000001,"15min":152.78100000000001},"last":1.3160000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:55:39 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.0980000000000001,"15min":1.7729999999999999},"min":{"1min":0.27400000000000002,"5min":0.253,"15min":0.24099999999999999},"max":{"1min":2.8079999999999998,"5min":20.367999999999999,"15min":257.375},"last":1.3640000000000001},{"interface":"front","average":{"1min":1.036,"5min":1.276,"15min":2.0369999999999999},"min":{"1min":0.29499999999999998,"5min":0.26900000000000002,"15min":0.246},"max":{"1min":6.6710000000000003,"5min":19.745000000000001,"15min":257.61200000000002},"last":0.44900000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:55:39 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":2.4660000000000002,"15min":2.3570000000000002},"min":{"1min":0.31900000000000001,"5min":0.28699999999999998,"15min":0.27800000000000002},"max":{"1min":2.903,"5min":87.828999999999994,"15min":256.05700000000002},"last":5.1459999999999999},{"interface":"front","average":{"1min":0.751,"5min":2.6640000000000001,"15min":2.5299999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.27500000000000002},"max":{"1min":3.5139999999999998,"5min":89.352999999999994,"15min":256.08499999999998},"last":0.41499999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:55:17 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":3.4660000000000002,"15min":1.645},"min":{"1min":0.38500000000000001,"5min":0.31,"15min":0.31},"max":{"1min":3.6480000000000001,"5min":204.15299999999999,"15min":204.15299999999999},"last":1.3380000000000001},{"interface":"front","average":{"1min":0.88200000000000001,"5min":3.6320000000000001,"15min":1.8939999999999999},"min":{"1min":0.375,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.6859999999999999,"5min":203.85499999999999,"15min":203.85499999999999},"last":0.747}]}]},{"osd":1,"up_from":93,"seq":399431958840,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3311492,"kb_used_data":2467644,"kb_used_omap":7,"kb_used_meta":843832,"kb_avail":90433660,"statfs":{"total":95995035648,"available":92604067840,"internally_reserved":0,"allocated":2526867456,"data_stored":2525388236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":864084494},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":3.3759999999999999,"15min":3.9540000000000002},"min":{"1min":0.217,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.51600000000000001,"5min":131.91300000000001,"15min":131.91300000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.34499999999999997,"5min":3.8759999999999999,"15min":4.2389999999999999},"min":{"1min":0.26200000000000001,"5min":0.223,"15min":0.182},"max":{"1min":0.52200000000000002,"5min":132.191,"15min":132.191},"last":0.26600000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:55:36 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":3.3719999999999999,"15min":3.194},"min":{"1min":0.24299999999999999,"5min":0.24199999999999999,"15min":0.19800000000000001},"max":{"1min":10.782,"5min":132.35400000000001,"15min":132.35400000000001},"last":0.254},{"interface":"front","average":{"1min":1.383,"5min":3.0630000000000002,"15min":2.9820000000000002},"min":{"1min":0.30199999999999999,"5min":0.24199999999999999,"15min":0.20699999999999999},"max":{"1min":5.0960000000000001,"5min":131.70599999999999,"15min":131.70599999999999},"last":0.33900000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":3.2069999999999999,"15min":3.7749999999999999},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.20200000000000001},"max":{"1min":3.0590000000000002,"5min":135.10900000000001,"15min":135.10900000000001},"last":0.314},{"interface":"front","average":{"1min":0.42399999999999999,"5min":3.2530000000000001,"15min":3.7109999999999999},"min":{"1min":0.28999999999999998,"5min":0.248,"15min":0.248},"max":{"1min":0.60099999999999998,"5min":131.886,"15min":131.886},"last":0.30099999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":3.0840000000000001,"15min":1.889},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.29799999999999999},"max":{"1min":0.98099999999999998,"5min":131.636,"15min":131.636},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":2.7149999999999999,"15min":1.702},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.96199999999999997,"5min":131.75999999999999,"15min":131.75999999999999},"last":0.53100000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":4.1050000000000004,"15min":2.3820000000000001},"min":{"1min":0.36799999999999999,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":2.6819999999999999,"5min":132.291,"15min":132.291},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.54200000000000004,"5min":4.2949999999999999,"15min":2.359},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.32200000000000001},"max":{"1min":0.90800000000000003,"5min":131.738,"15min":131.738},"last":0.41599999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:56:11 2023","interfaces":[{"interface":"back","average":{"1min":8.0950000000000006,"5min":4.4550000000000001,"15min":2.8119999999999998},"min":{"1min":0.34599999999999997,"5min":0.33300000000000002,"15min":0.316},"max":{"1min":158.50399999999999,"5min":158.50399999999999,"15min":158.50399999999999},"last":0.379},{"interface":"front","average":{"1min":8.0830000000000002,"5min":4.6210000000000004,"15min":2.6899999999999999},"min":{"1min":0.441,"5min":0.37,"15min":0.37},"max":{"1min":158.577,"5min":158.577,"15min":158.577},"last":0.46500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":2.7480000000000002,"15min":1.7709999999999999},"min":{"1min":0.40699999999999997,"5min":0.375,"15min":0.37},"max":{"1min":0.95799999999999996,"5min":131.821,"15min":131.821},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.62,"5min":2.9329999999999998,"15min":1.921},"min":{"1min":0.44700000000000001,"5min":0.39200000000000002,"15min":0.35499999999999998},"max":{"1min":0.998,"5min":131.852,"15min":131.852},"last":0.59499999999999997}]}]},{"osd":0,"up_from":0,"seq":377957122364,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":98,"seq":420906795314,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1742188,"kb_used_data":920548,"kb_used_omap":11,"kb_used_meta":821620,"kb_avail":92002964,"statfs":{"total":95995035648,"available":94211035136,"internally_reserved":0,"allocated":942641152,"data_stored":940890053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":841339477},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:55:55 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":2.8399999999999999,"15min":3.4689999999999999},"min":{"1min":0.223,"5min":0.21199999999999999,"15min":0.17999999999999999},"max":{"1min":5.7110000000000003,"5min":140.71100000000001,"15min":162.285},"last":0.26300000000000001},{"interface":"front","average":{"1min":2.5139999999999998,"5min":3.9969999999999999,"15min":4.0679999999999996},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.214},"max":{"1min":17.109999999999999,"5min":142.542,"15min":162.643},"last":0.38200000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:55:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":2.6269999999999998,"15min":2.9199999999999999},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.193},"max":{"1min":8.5150000000000006,"5min":38.677999999999997,"15min":125.77500000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.306,"5min":2.9889999999999999,"15min":2.7090000000000001},"min":{"1min":0.29199999999999998,"5min":0.26700000000000002,"15min":0.20999999999999999},"max":{"1min":9.4000000000000004,"5min":57.715000000000003,"15min":125.857},"last":0.28199999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:55:30 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":2.5569999999999999,"15min":4.0170000000000003},"min":{"1min":0.34399999999999997,"5min":0.25,"15min":0.21099999999999999},"max":{"1min":9.1430000000000007,"5min":29.606999999999999,"15min":159.35499999999999},"last":0.318},{"interface":"front","average":{"1min":1.7410000000000001,"5min":2.3690000000000002,"15min":3.5270000000000001},"min":{"1min":0.29399999999999998,"5min":0.217,"15min":0.217},"max":{"1min":6.7000000000000002,"5min":29.920999999999999,"15min":159.196},"last":0.46899999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:55:55 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":2.2730000000000001,"15min":1.6870000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":9.1539999999999999,"5min":68.052000000000007,"15min":68.052000000000007},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.9419999999999999,"15min":1.482},"min":{"1min":0.36199999999999999,"5min":0.29199999999999998,"15min":0.28699999999999998},"max":{"1min":2.504,"5min":68.727000000000004,"15min":68.727000000000004},"last":0.52000000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:55:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4850000000000001,"5min":6.125,"15min":3.1080000000000001},"min":{"1min":0.40200000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.7550000000000008,"5min":207.80500000000001,"15min":207.80500000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.5069999999999999,"5min":6.3289999999999997,"15min":3.0619999999999998},"min":{"1min":0.40300000000000002,"5min":0.371,"15min":0.30499999999999999},"max":{"1min":9.1259999999999994,"5min":207.54900000000001,"15min":207.54900000000001},"last":0.48999999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:55:55 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":1.1579999999999999,"15min":1.363},"min":{"1min":0.45400000000000001,"5min":0.33200000000000002,"15min":0.32000000000000001},"max":{"1min":2.5339999999999998,"5min":9.9589999999999996,"15min":25.818999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.8839999999999999,"15min":1.839},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.34100000000000003},"max":{"1min":8.7100000000000009,"5min":10.821,"15min":24.481000000000002},"last":0.45100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":2.863,"15min":2.1230000000000002},"min":{"1min":0.44,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":2.569,"5min":162.09200000000001,"15min":162.09200000000001},"last":0.623},{"interface":"front","average":{"1min":2.1240000000000001,"5min":3.2789999999999999,"15min":2.484},"min":{"1min":0.46400000000000002,"5min":0.378,"15min":0.373},"max":{"1min":8.6379999999999999,"5min":162.35400000000001,"15min":162.35400000000001},"last":0.53600000000000003}]}]},{"osd":3,"up_from":103,"seq":442381631788,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3400460,"kb_used_data":2506884,"kb_used_omap":4,"kb_used_meta":893563,"kb_avail":90344692,"statfs":{"total":95995035648,"available":92512964608,"internally_reserved":0,"allocated":2567049216,"data_stored":2565503345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":915009118},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":7.5540000000000003,"15min":4.8959999999999999},"min":{"1min":0.215,"5min":0.20899999999999999,"15min":0.184},"max":{"1min":2.77,"5min":312.56599999999997,"15min":312.56599999999997},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":7.4710000000000001,"15min":5.0830000000000002},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.20499999999999999},"max":{"1min":1.254,"5min":311.02300000000002,"15min":311.02300000000002},"last":0.317}]},{"osd":1,"last update":"Mon Oct 9 05:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":6.3579999999999997,"15min":4.3300000000000001},"min":{"1min":0.29399999999999998,"5min":0.22,"15min":0.214},"max":{"1min":0.97299999999999998,"5min":142.32599999999999,"15min":150.29599999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":6.492,"15min":4.5979999999999999},"min":{"1min":0.30599999999999999,"5min":0.216,"15min":0.19},"max":{"1min":1.861,"5min":139.33199999999999,"15min":151.25700000000001},"last":0.26100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":3.0600000000000001,"15min":2.556},"min":{"1min":0.34499999999999997,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":2.653,"5min":72.602000000000004,"15min":150.006},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.50900000000000001,"5min":2.7349999999999999,"15min":2.456},"min":{"1min":0.32900000000000001,"5min":0.251,"15min":0.20499999999999999},"max":{"1min":1.302,"5min":69.254000000000005,"15min":150.65100000000001},"last":0.40200000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":2.081,"15min":2.448},"min":{"1min":0.35499999999999998,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":2.085,"5min":69.224999999999994,"15min":149.84700000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":2.0739999999999998,"15min":2.7789999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.28999999999999998},"max":{"1min":1.3859999999999999,"5min":70.439999999999998,"15min":149.74700000000001},"last":0.438}]},{"osd":5,"last update":"Mon Oct 9 05:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":2.7069999999999999,"15min":2.4380000000000002},"min":{"1min":0.40799999999999997,"5min":0.307,"15min":0.28199999999999997},"max":{"1min":1.5269999999999999,"5min":71.203999999999994,"15min":149.43000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":2.3780000000000001,"15min":2.2170000000000001},"min":{"1min":0.35499999999999998,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":1.4099999999999999,"5min":70.397000000000006,"15min":149.69300000000001},"last":0.51600000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":2.0289999999999999,"15min":2.0009999999999999},"min":{"1min":0.40899999999999997,"5min":0.35899999999999999,"15min":0.33900000000000002},"max":{"1min":1.3500000000000001,"5min":70.474000000000004,"15min":149.72399999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.66800000000000004,"5min":2.1589999999999998,"15min":2.2789999999999999},"min":{"1min":0.39400000000000002,"5min":0.34599999999999997,"15min":0.307},"max":{"1min":1.5640000000000001,"5min":71.251999999999995,"15min":149.589},"last":0.47799999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":2.0059999999999998,"15min":2.1970000000000001},"min":{"1min":0.42399999999999999,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":1.9339999999999999,"5min":69.277000000000001,"15min":149.77199999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.84699999999999998,"5min":1.9390000000000001,"15min":2.242},"min":{"1min":0.46600000000000003,"5min":0.374,"15min":0.373},"max":{"1min":2.0899999999999999,"5min":69.299000000000007,"15min":149.80799999999999},"last":0.59699999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019335,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2124760,"kb_used_data":1223312,"kb_used_omap":0,"kb_used_meta":901440,"kb_avail":91620392,"statfs":{"total":95995035648,"available":93819281408,"internally_reserved":0,"allocated":1252671488,"data_stored":1251863531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923074560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":4.7560000000000002,"15min":3.8660000000000001},"min":{"1min":0.24099999999999999,"5min":0.23000000000000001,"15min":0.219},"max":{"1min":7.8609999999999998,"5min":95.795000000000002,"15min":132.35599999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.75600000000000001,"5min":5.1799999999999997,"15min":4.1639999999999997},"min":{"1min":0.24299999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.214,"5min":95.784999999999997,"15min":132.501},"last":2.9289999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:55:25 2023","interfaces":[{"interface":"back","average":{"1min":10.279999999999999,"5min":5.4349999999999996,"15min":3.7559999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":96.033000000000001,"5min":96.033000000000001,"15min":187.369},"last":0.32800000000000001},{"interface":"front","average":{"1min":10.411,"5min":5.194,"15min":3.7240000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":96.022000000000006,"5min":96.022000000000006,"15min":187.35300000000001},"last":0.36899999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:55:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.694,"15min":2.5710000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":4.7389999999999999,"5min":51.768000000000001,"15min":100.148},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.389,"5min":2.552,"15min":2.5649999999999999},"min":{"1min":0.318,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.1550000000000002,"5min":51.753999999999998,"15min":100.086},"last":0.34699999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":4.5650000000000004,"15min":3.589},"min":{"1min":0.31900000000000001,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":2.4500000000000002,"5min":100.009,"15min":123.59},"last":3.0960000000000001},{"interface":"front","average":{"1min":1.528,"5min":4.6109999999999998,"15min":3.581},"min":{"1min":0.39600000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9379999999999997,"5min":100.254,"15min":123.566},"last":0.45200000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:56:11 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.86099999999999999,"15min":0.86599999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":3.1589999999999998,"5min":8.5190000000000001,"15min":12.34},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.0960000000000001,"5min":0.92600000000000005,"15min":0.83399999999999996},"min":{"1min":0.34499999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.0129999999999999,"5min":8.0129999999999999,"15min":10.191000000000001},"last":0.53000000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":2.516,"15min":1.3080000000000001},"min":{"1min":0.316,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.4159999999999999,"5min":181.411,"15min":181.411},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.1299999999999999,"5min":2.677,"15min":1.45},"min":{"1min":0.35499999999999998,"5min":0.29799999999999999,"15min":0.28399999999999997},"max":{"1min":7.585,"5min":180.63999999999999,"15min":180.63999999999999},"last":0.41699999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.2889999999999999,"15min":1.224},"min":{"1min":0.375,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.3620000000000001,"5min":36.729999999999997,"15min":96.131},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.2150000000000001,"15min":1.272},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.27200000000000002},"max":{"1min":1.8300000000000001,"5min":36.424999999999997,"15min":96.465000000000003},"last":0.56599999999999995}]}]},{"osd":5,"up_from":36,"seq":154618823107,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3487252,"kb_used_data":2596748,"kb_used_omap":0,"kb_used_meta":890496,"kb_avail":90257900,"statfs":{"total":95995035648,"available":92424089600,"internally_reserved":0,"allocated":2659069952,"data_stored":2658269858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911867904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":5.7220000000000004,"5min":9.0150000000000006,"15min":5.1669999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.217},"max":{"1min":104.842,"5min":210.392,"15min":210.392},"last":0.28100000000000003},{"interface":"front","average":{"1min":5.8609999999999998,"5min":9.2560000000000002,"15min":5.3840000000000003},"min":{"1min":0.254,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":104.813,"5min":208.43600000000001,"15min":208.43600000000001},"last":1.2130000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:56:16 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":9.6829999999999998,"15min":4.5510000000000002},"min":{"1min":0.27900000000000003,"5min":0.247,"15min":0.247},"max":{"1min":11.884,"5min":284.13900000000001,"15min":284.13900000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.872,"5min":9.4459999999999997,"15min":4.5},"min":{"1min":0.29899999999999999,"5min":0.248,"15min":0.248},"max":{"1min":9.4819999999999993,"5min":284.07900000000001,"15min":284.07900000000001},"last":1.1839999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":4.069,"15min":2.3220000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.26700000000000002},"max":{"1min":4.9989999999999997,"5min":195.92099999999999,"15min":195.92099999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.306,"5min":4.1970000000000001,"15min":2.5129999999999999},"min":{"1min":0.32400000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.832000000000001,"5min":195.905,"15min":195.905},"last":0.44700000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:56:16 2023","interfaces":[{"interface":"back","average":{"1min":5.976,"5min":5.6980000000000004,"15min":3.6469999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30399999999999999},"max":{"1min":102.768,"5min":161.82300000000001,"15min":161.82300000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":5.4450000000000003,"5min":5.6379999999999999,"15min":3.6230000000000002},"min":{"1min":0.33800000000000002,"5min":0.308,"15min":0.308},"max":{"1min":102.806,"5min":161.69800000000001,"15min":161.69800000000001},"last":0.34499999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:56:16 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.79800000000000004,"15min":1.218},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.23899999999999999},"max":{"1min":7.0170000000000003,"5min":7.0170000000000003,"15min":130.81},"last":1.1419999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.995,"15min":1.369},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.253},"max":{"1min":2.831,"5min":9.3170000000000002,"15min":133.803},"last":0.36599999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:55:31 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":1.742,"15min":1.119},"min":{"1min":0.30199999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.73999999999999999,"5min":71.426000000000002,"15min":71.426000000000002},"last":1.5069999999999999},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.857,"15min":1.2210000000000001},"min":{"1min":0.34399999999999997,"5min":0.32300000000000001,"15min":0.27300000000000002},"max":{"1min":1.6359999999999999,"5min":71.498999999999995,"15min":71.498999999999995},"last":0.54500000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:55:31 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":2.1139999999999999,"15min":1.365},"min":{"1min":0.35999999999999999,"5min":0.318,"15min":0.27400000000000002},"max":{"1min":1.0049999999999999,"5min":86.506,"15min":86.506},"last":1.242},{"interface":"front","average":{"1min":0.52100000000000002,"5min":2.383,"15min":1.3939999999999999},"min":{"1min":0.36799999999999999,"5min":0.34300000000000003,"15min":0.308},"max":{"1min":0.76700000000000002,"5min":86.924000000000007,"15min":86.924000000000007},"last":1.54}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1387069440,"data_stored":1387069440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2515931136,"data_stored":2515927064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":930938880,"data_stored":930938880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2555969536,"data_stored":2555969536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1248071680,"data_stored":1248067608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2653753344,"data_stored":2653749272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":794886144,"data_stored":794886144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1384251392,"data_stored":1384251392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:56:24.604 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:56:24.605 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 41 2023-10-09T05:56:24.693 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:24.719 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:24.773 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:24.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:24.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:24.795 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:56:24.878 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:24.910 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:24.943 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:24.959 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:24.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:24.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:25.067 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:25.092 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:25.144 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:25.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:25.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:25.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:25.277 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:25.328 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:25.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:25.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:25.335 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:25.434 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:25.460 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:25.504 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:25.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:25.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:25.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:25.634 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:25.683 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:25.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:25.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:25.780 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:25 smithi008 ceph-mon[184418]: pgmap v1263: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 119 op/s; 29004/192981 objects degraded (15.029%); 14446/192981 objects misplaced (7.486%); 2.7 MiB/s, 106 objects/s recovering 2023-10-09T05:56:25.780 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:25 smithi008 ceph-mon[191227]: pgmap v1263: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 119 op/s; 29004/192981 objects degraded (15.029%); 14446/192981 objects misplaced (7.486%); 2.7 MiB/s, 106 objects/s recovering 2023-10-09T05:56:25.789 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:25.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:25 smithi191 ceph-mon[161669]: pgmap v1263: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 119 op/s; 29004/192981 objects degraded (15.029%); 14446/192981 objects misplaced (7.486%); 2.7 MiB/s, 106 objects/s recovering 2023-10-09T05:56:25.825 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:25.838 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:26.598 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:26.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:26.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:26.688 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:56:26.702 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:26.752 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:26.802 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:26.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:26.907 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:26.934 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4058811290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:26.934 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:26 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:26.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4058811290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:26.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:26 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:26.938 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:27.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4058811290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:27.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:26 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:27.523 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:27.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:27.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:27.628 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:27.657 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:27.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[184418]: pgmap v1264: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28953/191856 objects degraded (15.091%); 14295/191856 objects misplaced (7.451%); 3.8 MiB/s, 127 objects/s recovering 2023-10-09T05:56:27.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:27.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[184418]: failed to encode map e425 with expected crc 2023-10-09T05:56:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[184418]: failed to encode map e425 with expected crc 2023-10-09T05:56:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:56:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[184418]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T05:56:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[184418]: failed to encode map e425 with expected crc 2023-10-09T05:56:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[191227]: pgmap v1264: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28953/191856 objects degraded (15.091%); 14295/191856 objects misplaced (7.451%); 3.8 MiB/s, 127 objects/s recovering 2023-10-09T05:56:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[191227]: failed to encode map e425 with expected crc 2023-10-09T05:56:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[191227]: failed to encode map e425 with expected crc 2023-10-09T05:56:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:56:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[191227]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T05:56:27.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:27 smithi008 ceph-mon[191227]: failed to encode map e425 with expected crc 2023-10-09T05:56:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:27 smithi191 ceph-mon[161669]: pgmap v1264: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28953/191856 objects degraded (15.091%); 14295/191856 objects misplaced (7.451%); 3.8 MiB/s, 127 objects/s recovering 2023-10-09T05:56:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:27 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:27 smithi191 ceph-mon[161669]: failed to encode map e425 with expected crc 2023-10-09T05:56:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:27 smithi191 ceph-mon[161669]: failed to encode map e425 with expected crc 2023-10-09T05:56:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:27 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:56:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:27 smithi191 ceph-mon[161669]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T05:56:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:27 smithi191 ceph-mon[161669]: failed to encode map e425 with expected crc 2023-10-09T05:56:28.420 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:28.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:28.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:28.525 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:28.560 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:28.676 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 28953/191856 objects degraded (15.091%), 23 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:28.676 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/124740719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:28.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 28953/191856 objects degraded (15.091%), 23 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:28.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/124740719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:29.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 28953/191856 objects degraded (15.091%), 23 pgs degraded (PG_DEGRADED) 2023-10-09T05:56:29.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/124740719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:29.140 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:56:29.140 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:56:29.140 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:56:29.141 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:56:29.141 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:56:29.141 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:56:29.141 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:56:29.141 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:56:29.141 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:56:29.141 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:56:29.141 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:56:29.141 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:56:29.142 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:56:29.142 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:56:29.142 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:56:29.142 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:56:29.168 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:29.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:29.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:29.272 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:29.304 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:29 smithi008 ceph-mon[184418]: pgmap v1266: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 28953/190833 objects degraded (15.172%); 14246/190833 objects misplaced (7.465%); 3.5 MiB/s, 118 objects/s recovering 2023-10-09T05:56:29.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:29 smithi008 ceph-mon[191227]: pgmap v1266: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 28953/190833 objects degraded (15.172%); 14246/190833 objects misplaced (7.465%); 3.5 MiB/s, 118 objects/s recovering 2023-10-09T05:56:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:30.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:29 smithi191 ceph-mon[161669]: pgmap v1266: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 28953/190833 objects degraded (15.172%); 14246/190833 objects misplaced (7.465%); 3.5 MiB/s, 118 objects/s recovering 2023-10-09T05:56:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:30.166 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:30.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:30.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:30.258 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:56:30.271 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:30.303 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:30.367 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:30.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:30.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:30.476 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:30.512 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:30.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1603701582' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T05:56:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:30 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T05:56:30.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1603701582' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T05:56:30.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:30 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T05:56:30.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:56:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:56:30] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.33.4" 2023-10-09T05:56:31.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1603701582' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T05:56:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:30 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T05:56:31.112 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:56:31.178 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:56:31.180 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:31.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:31.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:31.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:31.311 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:31.363 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:31.364 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:31.364 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:31.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:31.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:31.469 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:31.503 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:31.666 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:31.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:31.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:31.770 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:31.800 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:31.854 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:31.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:31.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:31.959 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:31 smithi008 ceph-mon[184418]: pgmap v1267: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 28904/190161 objects degraded (15.200%); 14143/190161 objects misplaced (7.437%); 3.9 MiB/s, 114 objects/s recovering 2023-10-09T05:56:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:31 smithi008 ceph-mon[191227]: pgmap v1267: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 28904/190161 objects degraded (15.200%); 14143/190161 objects misplaced (7.437%); 3.9 MiB/s, 114 objects/s recovering 2023-10-09T05:56:31.991 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:31 smithi191 ceph-mon[161669]: pgmap v1267: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 28904/190161 objects degraded (15.200%); 14143/190161 objects misplaced (7.437%); 3.9 MiB/s, 114 objects/s recovering 2023-10-09T05:56:32.186 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:32.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:32.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:32.201 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:56:32.287 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:32.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:32.319 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:32.343 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:56:32.365 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:32.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:32.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:32.470 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:32.502 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:32.548 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:32.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:32.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:32.656 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:32.684 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:32.734 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:32.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:32.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:32.840 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:32.865 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:32.908 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:32.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:32.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:33.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:33.043 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:33.112 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:33.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:33.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:33.218 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:33.250 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:33.458 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:33.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:33.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:33.516 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:33.563 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:33.592 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:33.600 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:33.647 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:33.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:33.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:33.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:33.783 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:33.852 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:33.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:33.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:33.858 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:33 smithi191 ceph-mon[161669]: pgmap v1268: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 28862/188814 objects degraded (15.286%); 14096/188814 objects misplaced (7.466%); 2.6 MiB/s, 43 objects/s recovering 2023-10-09T05:56:33.858 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 28862/188814 objects degraded (15.286%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:33.959 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:33 smithi008 ceph-mon[184418]: pgmap v1268: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 28862/188814 objects degraded (15.286%); 14096/188814 objects misplaced (7.466%); 2.6 MiB/s, 43 objects/s recovering 2023-10-09T05:56:33.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 28862/188814 objects degraded (15.286%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:33 smithi008 ceph-mon[191227]: pgmap v1268: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 28862/188814 objects degraded (15.286%); 14096/188814 objects misplaced (7.466%); 2.6 MiB/s, 43 objects/s recovering 2023-10-09T05:56:33.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 28862/188814 objects degraded (15.286%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:33.994 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:34.062 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:34.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:34.168 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:34.204 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:34.250 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:34.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:34.355 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:34.389 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:34.700 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:56:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1402558649' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:34 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:34.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1402558649' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:34.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:34 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1402558649' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:34 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:35.832 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[184418]: pgmap v1269: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 28816/188517 objects degraded (15.286%); 14142/188517 objects misplaced (7.502%); 2.5 MiB/s, 42 objects/s recovering 2023-10-09T05:56:35.832 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:35.832 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[184418]: failed to encode map e426 with expected crc 2023-10-09T05:56:35.832 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[184418]: failed to encode map e426 with expected crc 2023-10-09T05:56:35.832 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[184418]: failed to encode map e426 with expected crc 2023-10-09T05:56:35.832 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:35.832 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[184418]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T05:56:35.833 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[191227]: pgmap v1269: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 28816/188517 objects degraded (15.286%); 14142/188517 objects misplaced (7.502%); 2.5 MiB/s, 42 objects/s recovering 2023-10-09T05:56:35.833 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:35.833 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[191227]: failed to encode map e426 with expected crc 2023-10-09T05:56:35.833 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[191227]: failed to encode map e426 with expected crc 2023-10-09T05:56:35.833 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[191227]: failed to encode map e426 with expected crc 2023-10-09T05:56:35.833 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:35.833 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:35 smithi008 ceph-mon[191227]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T05:56:36.017 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:36.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:36.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:36.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:35 smithi191 ceph-mon[161669]: pgmap v1269: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 28816/188517 objects degraded (15.286%); 14142/188517 objects misplaced (7.502%); 2.5 MiB/s, 42 objects/s recovering 2023-10-09T05:56:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:35 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:35 smithi191 ceph-mon[161669]: failed to encode map e426 with expected crc 2023-10-09T05:56:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:35 smithi191 ceph-mon[161669]: failed to encode map e426 with expected crc 2023-10-09T05:56:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:35 smithi191 ceph-mon[161669]: failed to encode map e426 with expected crc 2023-10-09T05:56:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:35 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:35 smithi191 ceph-mon[161669]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T05:56:36.122 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:36.154 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:36.180 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 5, 6, 1, 7, 3] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:56:36.180 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:56:36.180 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:56:36.180 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T05:56:36.366 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:36.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:36.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:36.472 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:36.498 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:36.542 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:36.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:36.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:36.647 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:36.673 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:36.695 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3538932156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:36.709 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:56:36.728 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:36.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:36.736 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:56:36.759 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:56:36.782 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:56:36.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3538932156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:36.807 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:56:36.832 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:56:36.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:36.854 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:56:36.878 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:36.880 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:56:36.881 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.882 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.882 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.882 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.882 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.882 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.882 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.882 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.932 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:36.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:36.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:36.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3538932156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:37.039 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:56:37.039 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:56:37.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:56:37.039 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:56:37.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.040 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180271 7.1d 7:baeacd59:::benchmark_data_smithi008_354649_object85775:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:56:37.040 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:56:37.040 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 595.90636391199996, 2023-10-09T05:56:37.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.855218754, 2023-10-09T05:56:37.040 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.040 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.040 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.040 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:56:37.040 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:56:37.040 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180271 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129512+0000", 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129509+0000", 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129525+0000", 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5430999999999998e-05 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129553+0000", 2023-10-09T05:56:37.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8175e-05 2023-10-09T05:56:37.043 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.043 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.129568+0000", 2023-10-09T05:56:37.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5267000000000001e-05 2023-10-09T05:56:37.043 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.043 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.333120+0000", 2023-10-09T05:56:37.044 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.20355221500000001 2023-10-09T05:56:37.044 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.044 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.044 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.044 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.333134+0000", 2023-10-09T05:56:37.044 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4073000000000001e-05 2023-10-09T05:56:37.044 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.044 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.044 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.044 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929537+0000", 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5964021320000001 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929548+0000", 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1759999999999999e-05 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.974829+0000", 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.045281066 2023-10-09T05:56:37.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.046 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.974986+0000", 2023-10-09T05:56:37.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000156174 2023-10-09T05:56:37.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.046 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:56:37.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.975339+0000", 2023-10-09T05:56:37.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00035308699999999998 2023-10-09T05:56:37.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977566+0000", 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022271399999999998 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984241+0000", 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066755160000000003 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984422+0000", 2023-10-09T05:56:37.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000180633 2023-10-09T05:56:37.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.048 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984530+0000", 2023-10-09T05:56:37.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010849999999999999 2023-10-09T05:56:37.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.049 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.049 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984609+0000", 2023-10-09T05:56:37.049 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8506999999999999e-05 2023-10-09T05:56:37.049 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.049 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.050 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984731+0000", 2023-10-09T05:56:37.050 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012211599999999999 2023-10-09T05:56:37.050 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.050 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.050 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.050 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.050 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.051 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180280 7.1d 7:b9a77225:::benchmark_data_smithi008_354649_object85784:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:56:37.051 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:56:37.051 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 595.67401620199996, 2023-10-09T05:56:37.051 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6230030800000002, 2023-10-09T05:56:37.051 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.051 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.051 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.051 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:56:37.052 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:56:37.052 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180280 2023-10-09T05:56:37.052 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.052 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.052 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.052 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.052 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:56:37.052 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.053 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.053 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361860+0000", 2023-10-09T05:56:37.053 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.053 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.054 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361859+0000", 2023-10-09T05:56:37.054 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:37.054 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.054 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.054 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.054 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361865+0000", 2023-10-09T05:56:37.054 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9800000000000003e-06 2023-10-09T05:56:37.054 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.055 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361876+0000", 2023-10-09T05:56:37.055 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1178999999999999e-05 2023-10-09T05:56:37.055 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.055 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361882+0000", 2023-10-09T05:56:37.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0070000000000003e-06 2023-10-09T05:56:37.056 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.056 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.056 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.056 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361924+0000", 2023-10-09T05:56:37.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2243999999999997e-05 2023-10-09T05:56:37.056 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.057 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.057 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.057 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.361931+0000", 2023-10-09T05:56:37.057 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.097e-06 2023-10-09T05:56:37.057 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.057 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.057 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.057 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929573+0000", 2023-10-09T05:56:37.058 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.567641397 2023-10-09T05:56:37.058 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.058 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.058 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.058 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929580+0000", 2023-10-09T05:56:37.058 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4569999999999999e-06 2023-10-09T05:56:37.059 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.059 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.059 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.059 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977745+0000", 2023-10-09T05:56:37.059 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0481653790000001 2023-10-09T05:56:37.059 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.060 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.060 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.060 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.977840+0000", 2023-10-09T05:56:37.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4581999999999997e-05 2023-10-09T05:56:37.060 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.060 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.060 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:56:37.060 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.978096+0000", 2023-10-09T05:56:37.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00025544499999999998 2023-10-09T05:56:37.060 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.061 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.061 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984291+0000", 2023-10-09T05:56:37.061 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061956720000000002 2023-10-09T05:56:37.061 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.061 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.061 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984301+0000", 2023-10-09T05:56:37.061 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7710000000000001e-06 2023-10-09T05:56:37.062 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.062 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.062 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.062 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984457+0000", 2023-10-09T05:56:37.062 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000155885 2023-10-09T05:56:37.062 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.063 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.063 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984780+0000", 2023-10-09T05:56:37.063 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00032328700000000001 2023-10-09T05:56:37.063 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.063 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.064 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984806+0000", 2023-10-09T05:56:37.064 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6290000000000001e-05 2023-10-09T05:56:37.064 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.064 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.064 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.065 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.984863+0000", 2023-10-09T05:56:37.065 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6693000000000001e-05 2023-10-09T05:56:37.065 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.065 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.065 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.065 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.066 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.066 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45365.0:180292 7.1f 7:fc8bce44:::benchmark_data_smithi008_354649_object85796:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T05:56:37.066 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:56:37.066 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 595.50839024699997, 2023-10-09T05:56:37.066 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4946860599999998, 2023-10-09T05:56:37.066 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.067 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.067 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.067 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45365", 2023-10-09T05:56:37.067 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/2390798608", 2023-10-09T05:56:37.067 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 180292 2023-10-09T05:56:37.067 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.067 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.068 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.068 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:56:37.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527486+0000", 2023-10-09T05:56:37.069 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.069 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527485+0000", 2023-10-09T05:56:37.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:56:37.070 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.070 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.070 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.071 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527491+0000", 2023-10-09T05:56:37.071 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.139e-06 2023-10-09T05:56:37.071 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.071 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.071 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.071 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527504+0000", 2023-10-09T05:56:37.071 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2964e-05 2023-10-09T05:56:37.071 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.071 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.071 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.071 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527512+0000", 2023-10-09T05:56:37.072 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2230000000000007e-06 2023-10-09T05:56:37.072 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.072 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.072 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.072 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527553+0000", 2023-10-09T05:56:37.072 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1266000000000001e-05 2023-10-09T05:56:37.072 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.072 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.072 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.072 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:41.527560+0000", 2023-10-09T05:56:37.073 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6599999999999998e-06 2023-10-09T05:56:37.073 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.073 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.073 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.073 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929860+0000", 2023-10-09T05:56:37.073 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4023002959999999 2023-10-09T05:56:37.073 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.073 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.073 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.073 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:42.929869+0000", 2023-10-09T05:56:37.074 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9080000000000001e-06 2023-10-09T05:56:37.074 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.074 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.074 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.074 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986248+0000", 2023-10-09T05:56:37.074 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0563782779999999 2023-10-09T05:56:37.074 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.074 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.074 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.074 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986407+0000", 2023-10-09T05:56:37.075 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000159588 2023-10-09T05:56:37.075 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.075 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.075 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:56:37.075 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:43.986738+0000", 2023-10-09T05:56:37.075 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00033078600000000001 2023-10-09T05:56:37.075 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.075 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.075 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.075 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.015271+0000", 2023-10-09T05:56:37.075 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028533060999999998 2023-10-09T05:56:37.076 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.076 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.076 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.076 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.015291+0000", 2023-10-09T05:56:37.076 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0077000000000001e-05 2023-10-09T05:56:37.076 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.076 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.076 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.076 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.021759+0000", 2023-10-09T05:56:37.076 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064680570000000001 2023-10-09T05:56:37.076 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.021808+0000", 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8615000000000001e-05 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:46:44.022172+0000", 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00036433100000000002 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.077 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.078 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.078 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.078 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45727 9.5 9:af6b034d:::benchmark_data_smithi008_502820_object45726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T05:56:37.078 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:45.884586+0000", 2023-10-09T05:56:37.078 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 171.15129021999999, 2023-10-09T05:56:37.078 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.280584508, 2023-10-09T05:56:37.078 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.078 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:56:37.078 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.078 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.078 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45727 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884586+0000", 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884586+0000", 2023-10-09T05:56:37.079 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884585+0000", 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884616+0000", 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0423e-05 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.080 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884625+0000", 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0250000000000008e-06 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884631+0000", 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3840000000000002e-06 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.081 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884669+0000", 2023-10-09T05:56:37.082 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7698999999999999e-05 2023-10-09T05:56:37.082 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.082 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.082 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.082 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884795+0000", 2023-10-09T05:56:37.082 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012634500000000001 2023-10-09T05:56:37.082 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.082 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.082 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:56:37.082 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884920+0000", 2023-10-09T05:56:37.082 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012516900000000001 2023-10-09T05:56:37.083 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.083 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.083 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.083 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.061847+0000", 2023-10-09T05:56:37.083 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17692612699999999 2023-10-09T05:56:37.083 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.083 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.083 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.083 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.061860+0000", 2023-10-09T05:56:37.083 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3361e-05 2023-10-09T05:56:37.083 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933452+0000", 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.87159255700000005 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933460+0000", 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5549999999999997e-06 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.084 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.085 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.120071+0000", 2023-10-09T05:56:37.085 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1866107539999999 2023-10-09T05:56:37.085 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.085 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.085 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:56:37.085 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.120662+0000", 2023-10-09T05:56:37.085 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00059118300000000001 2023-10-09T05:56:37.085 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.085 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.085 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.085 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165062+0000", 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.044399986000000002 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165171+0000", 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010856700000000001 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.086 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45730 9.5 9:a76420fe:::benchmark_data_smithi008_502820_object45729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:45.884645+0000", 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 171.15123078100001, 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2944595059999999, 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45730 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.087 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884645+0000", 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884645+0000", 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.088 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884645+0000", 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884669+0000", 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4326000000000001e-05 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884678+0000", 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8829999999999995e-06 2023-10-09T05:56:37.089 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.090 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.090 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.090 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884684+0000", 2023-10-09T05:56:37.090 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.806e-06 2023-10-09T05:56:37.090 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.090 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.090 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.090 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055897+0000", 2023-10-09T05:56:37.090 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17121303700000001 2023-10-09T05:56:37.090 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.090 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.056018+0000", 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000120831 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.056156+0000", 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013870000000000001 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.091 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066733+0000", 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010576189999999999 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066739+0000", 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4500000000000001e-06 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933477+0000", 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86673776000000002 2023-10-09T05:56:37.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.093 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.093 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.093 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933485+0000", 2023-10-09T05:56:37.093 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2779999999999993e-06 2023-10-09T05:56:37.093 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.093 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.093 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.093 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165241+0000", 2023-10-09T05:56:37.093 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.231755605 2023-10-09T05:56:37.093 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.093 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.094 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:56:37.094 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165788+0000", 2023-10-09T05:56:37.094 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00054761800000000004 2023-10-09T05:56:37.094 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.094 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.094 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.094 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.178974+0000", 2023-10-09T05:56:37.094 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013186164 2023-10-09T05:56:37.094 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.094 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.094 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.179105+0000", 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000130462 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45731 9.9 9:9d7980f6:::benchmark_data_smithi008_502820_object45730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:45.884828+0000", 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 171.15104868, 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.325062725, 2023-10-09T05:56:37.095 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45731 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884828+0000", 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.096 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884828+0000", 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884827+0000", 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.097 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.098 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884851+0000", 2023-10-09T05:56:37.098 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4647999999999998e-05 2023-10-09T05:56:37.098 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.098 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.098 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.098 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884863+0000", 2023-10-09T05:56:37.098 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1389000000000001e-05 2023-10-09T05:56:37.098 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.098 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.098 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.099 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884870+0000", 2023-10-09T05:56:37.099 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1250000000000004e-06 2023-10-09T05:56:37.099 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.099 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.099 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.099 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055752+0000", 2023-10-09T05:56:37.099 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17088223499999999 2023-10-09T05:56:37.099 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.099 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.099 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.099 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055870+0000", 2023-10-09T05:56:37.100 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011758500000000001 2023-10-09T05:56:37.100 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.100 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.100 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T05:56:37.100 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.056006+0000", 2023-10-09T05:56:37.100 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000136243 2023-10-09T05:56:37.100 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.100 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.100 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.100 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064121+0000", 2023-10-09T05:56:37.100 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081152840000000004 2023-10-09T05:56:37.101 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.101 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.101 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.101 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064134+0000", 2023-10-09T05:56:37.101 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2826999999999999e-05 2023-10-09T05:56:37.101 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.101 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.101 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.101 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933097+0000", 2023-10-09T05:56:37.102 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86896320299999996 2023-10-09T05:56:37.102 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.102 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.102 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.102 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933109+0000", 2023-10-09T05:56:37.102 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1987999999999999e-05 2023-10-09T05:56:37.103 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.103 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.103 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.103 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.184026+0000", 2023-10-09T05:56:37.103 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2509170430000001 2023-10-09T05:56:37.103 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.103 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.104 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:56:37.104 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.184835+0000", 2023-10-09T05:56:37.104 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00080872799999999999 2023-10-09T05:56:37.104 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.104 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.104 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.104 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.209763+0000", 2023-10-09T05:56:37.105 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024928049000000001 2023-10-09T05:56:37.105 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.105 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.105 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.105 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.209890+0000", 2023-10-09T05:56:37.105 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000127267 2023-10-09T05:56:37.105 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.105 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.105 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.105 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.106 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.106 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77614 9.15 9:ad1d5133:::benchmark_data_smithi008_502820_object77613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T05:56:37.106 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:36.822615+0000", 2023-10-09T05:56:37.106 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 120.213261609, 2023-10-09T05:56:37.106 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3942615420000002, 2023-10-09T05:56:37.106 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.106 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:56:37.106 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.106 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.106 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.106 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77614 2023-10-09T05:56:37.107 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.107 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.107 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.107 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.107 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822615+0000", 2023-10-09T05:56:37.107 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.107 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.107 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.107 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.107 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822615+0000", 2023-10-09T05:56:37.107 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822614+0000", 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822641+0000", 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6928e-05 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.108 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822648+0000", 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7700000000000004e-06 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822654+0000", 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6119999999999998e-06 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.109 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.879602+0000", 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.056948375000000002 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.879611+0000", 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5470000000000002e-06 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:38.167351+0000", 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2877391039999999 2023-10-09T05:56:37.110 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:38.167362+0000", 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1535999999999999e-05 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:39.204324+0000", 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0369618030000001 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.111 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:39.204880+0000", 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00055560899999999999 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:39.216793+0000", 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011913464 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:39.216876+0000", 2023-10-09T05:56:37.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3181000000000002e-05 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93502 9.e 9:7c9e1516:::benchmark_data_smithi008_502820_object9329:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e392)", 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.257612+0000", 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.778263953999996, 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5408023110000002, 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.113 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.114 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.114 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.114 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93502 2023-10-09T05:56:37.114 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.114 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.114 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.114 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.114 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257612+0000", 2023-10-09T05:56:37.114 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.114 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.114 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257612+0000", 2023-10-09T05:56:37.115 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257612+0000", 2023-10-09T05:56:37.115 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:37.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257615+0000", 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8990000000000001e-06 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257622+0000", 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.08e-06 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257627+0000", 2023-10-09T05:56:37.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3120000000000001e-06 2023-10-09T05:56:37.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.117 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.117 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.117 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257937+0000", 2023-10-09T05:56:37.117 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00030955700000000002 2023-10-09T05:56:37.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.117 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.117 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.117 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257946+0000", 2023-10-09T05:56:37.117 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2639999999999997e-06 2023-10-09T05:56:37.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.343810+0000", 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.085863609 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.347315+0000", 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035057539999999998 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.119 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.372730+0000", 2023-10-09T05:56:37.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.025414923999999998 2023-10-09T05:56:37.119 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.119 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.119 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.119 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.376888+0000", 2023-10-09T05:56:37.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041572459999999999 2023-10-09T05:56:37.119 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.119 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.119 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.763791+0000", 2023-10-09T05:56:37.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.38690379400000002 2023-10-09T05:56:37.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.766585+0000", 2023-10-09T05:56:37.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027940199999999999 2023-10-09T05:56:37.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.771743+0000", 2023-10-09T05:56:37.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051580180000000003 2023-10-09T05:56:37.121 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.121 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.788797+0000", 2023-10-09T05:56:37.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017053295 2023-10-09T05:56:37.122 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.122 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.122 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.122 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.789756+0000", 2023-10-09T05:56:37.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095954999999999999 2023-10-09T05:56:37.123 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.123 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.123 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.123 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.790067+0000", 2023-10-09T05:56:37.123 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000310468 2023-10-09T05:56:37.124 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.124 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.124 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.124 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.791632+0000", 2023-10-09T05:56:37.124 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015649539999999999 2023-10-09T05:56:37.124 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.125 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.125 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.125 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.798415+0000", 2023-10-09T05:56:37.125 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0067829220000000003 2023-10-09T05:56:37.125 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.126 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.126 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.126 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.126 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.126 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93503 9.e 9:722ffbd5:::benchmark_data_smithi008_502820_object9330:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e392)", 2023-10-09T05:56:37.126 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.257654+0000", 2023-10-09T05:56:37.127 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.778222645, 2023-10-09T05:56:37.127 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5884587200000002, 2023-10-09T05:56:37.127 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.127 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.127 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.127 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.127 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.127 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93503 2023-10-09T05:56:37.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.128 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.128 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257654+0000", 2023-10-09T05:56:37.128 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.129 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.129 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257654+0000", 2023-10-09T05:56:37.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.129 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.129 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257653+0000", 2023-10-09T05:56:37.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:37.130 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.130 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257656+0000", 2023-10-09T05:56:37.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1779999999999999e-06 2023-10-09T05:56:37.131 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.131 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.131 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.131 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257663+0000", 2023-10-09T05:56:37.131 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9489999999999999e-06 2023-10-09T05:56:37.131 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.131 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.132 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.132 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257669+0000", 2023-10-09T05:56:37.132 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8549999999999998e-06 2023-10-09T05:56:37.132 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.132 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.132 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.132 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.258141+0000", 2023-10-09T05:56:37.132 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00047206199999999998 2023-10-09T05:56:37.132 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.258149+0000", 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1899999999999995e-06 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.394518+0000", 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1363689040000002 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.134 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.397990+0000", 2023-10-09T05:56:37.134 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034717139999999999 2023-10-09T05:56:37.134 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.134 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.134 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.134 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.443090+0000", 2023-10-09T05:56:37.134 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.045099601000000003 2023-10-09T05:56:37.134 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.134 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.446520+0000", 2023-10-09T05:56:37.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034299270000000001 2023-10-09T05:56:37.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.798754+0000", 2023-10-09T05:56:37.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35223448899999998 2023-10-09T05:56:37.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.801843+0000", 2023-10-09T05:56:37.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030893740000000002 2023-10-09T05:56:37.136 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.136 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.136 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.807614+0000", 2023-10-09T05:56:37.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005771132 2023-10-09T05:56:37.136 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.136 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.136 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.836687+0000", 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.029072210000000001 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.836991+0000", 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00030478900000000003 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.837946+0000", 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095470500000000003 2023-10-09T05:56:37.137 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.138 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.138 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.839570+0000", 2023-10-09T05:56:37.138 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016237630000000001 2023-10-09T05:56:37.138 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.138 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.138 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.846112+0000", 2023-10-09T05:56:37.138 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006542338 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93531 9.e 9:7b068cda:::benchmark_data_smithi008_502820_object9358:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e392)", 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.257964+0000", 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.777912556999993, 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6005753459999998, 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.139 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93531 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257964+0000", 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.141 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257964+0000", 2023-10-09T05:56:37.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.141 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.141 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257963+0000", 2023-10-09T05:56:37.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:37.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257967+0000", 2023-10-09T05:56:37.142 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8739999999999997e-06 2023-10-09T05:56:37.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257976+0000", 2023-10-09T05:56:37.142 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5760000000000006e-06 2023-10-09T05:56:37.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257982+0000", 2023-10-09T05:56:37.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5239999999999997e-06 2023-10-09T05:56:37.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.143 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.258187+0000", 2023-10-09T05:56:37.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000204546 2023-10-09T05:56:37.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.143 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.258195+0000", 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0560000000000005e-06 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.398339+0000", 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1401438540000002 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.401890+0000", 2023-10-09T05:56:37.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035510469999999999 2023-10-09T05:56:37.145 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.145 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.145 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.145 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.460275+0000", 2023-10-09T05:56:37.145 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.058385640000000003 2023-10-09T05:56:37.145 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.145 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.145 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.145 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.463759+0000", 2023-10-09T05:56:37.145 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034834689999999999 2023-10-09T05:56:37.146 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.146 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.146 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.146 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.822402+0000", 2023-10-09T05:56:37.146 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.358643673 2023-10-09T05:56:37.146 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.146 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.146 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.146 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.825300+0000", 2023-10-09T05:56:37.146 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002897929 2023-10-09T05:56:37.146 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.830267+0000", 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049663010000000002 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.847190+0000", 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016923032000000001 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.847522+0000", 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000331952 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.848622+0000", 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001100125 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.850417+0000", 2023-10-09T05:56:37.148 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017952689999999999 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.858539+0000", 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081220330000000007 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93555 9.e 9:73c791c6:::benchmark_data_smithi008_502820_object9382:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T05:56:37.149 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.314621+0000", 2023-10-09T05:56:37.150 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.721254965, 2023-10-09T05:56:37.150 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6041918630000001, 2023-10-09T05:56:37.150 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.150 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.150 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.150 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.150 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.150 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93555 2023-10-09T05:56:37.150 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.150 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314621+0000", 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314621+0000", 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.151 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.152 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314620+0000", 2023-10-09T05:56:37.152 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:37.152 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.152 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.152 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.152 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314628+0000", 2023-10-09T05:56:37.152 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.038e-06 2023-10-09T05:56:37.152 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.152 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314651+0000", 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2671e-05 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314662+0000", 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0829000000000001e-05 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.153 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314731+0000", 2023-10-09T05:56:37.154 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9561999999999996e-05 2023-10-09T05:56:37.154 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.154 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.154 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.154 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314744+0000", 2023-10-09T05:56:37.155 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3275000000000001e-05 2023-10-09T05:56:37.155 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.155 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.155 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.155 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.402420+0000", 2023-10-09T05:56:37.155 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.08767521 2023-10-09T05:56:37.155 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.155 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.155 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.155 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.405867+0000", 2023-10-09T05:56:37.155 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034470909999999998 2023-10-09T05:56:37.156 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.156 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.156 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.156 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.478021+0000", 2023-10-09T05:56:37.156 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.072154274000000004 2023-10-09T05:56:37.156 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.156 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.156 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.156 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.481605+0000", 2023-10-09T05:56:37.156 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035844610000000002 2023-10-09T05:56:37.156 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.157 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.157 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.157 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.867423+0000", 2023-10-09T05:56:37.157 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.385817405 2023-10-09T05:56:37.157 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.157 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.870665+0000", 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032420019999999999 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.875924+0000", 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005258764 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.158 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.907546+0000", 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.031622665000000001 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.908496+0000", 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00094938399999999995 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.911083+0000", 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002586933 2023-10-09T05:56:37.159 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.912705+0000", 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001622823 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.918813+0000", 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061077049999999997 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.160 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93563 9.e 9:7ced5d1d:::benchmark_data_smithi008_502820_object9390:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.328334+0000", 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.707542433, 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5994731469999999, 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.161 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93563 2023-10-09T05:56:37.162 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.162 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.162 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.162 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.162 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328334+0000", 2023-10-09T05:56:37.163 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.163 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.163 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.163 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.163 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328334+0000", 2023-10-09T05:56:37.163 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.163 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.163 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.163 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.163 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328332+0000", 2023-10-09T05:56:37.163 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328339+0000", 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9129999999999997e-06 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328355+0000", 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5471000000000001e-05 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328362+0000", 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0219999999999999e-06 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328425+0000", 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2731999999999995e-05 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.165 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328435+0000", 2023-10-09T05:56:37.166 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0389e-05 2023-10-09T05:56:37.166 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.166 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.166 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.166 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.483156+0000", 2023-10-09T05:56:37.166 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1547212980000001 2023-10-09T05:56:37.166 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.166 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.166 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.166 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.486776+0000", 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036195910000000001 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.511225+0000", 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024448785000000001 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.515218+0000", 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039933829999999997 2023-10-09T05:56:37.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.890925+0000", 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37570650999999999 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.894485+0000", 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035602559999999999 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.168 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.900641+0000", 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006156089 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.909685+0000", 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0090444210000000004 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.910760+0000", 2023-10-09T05:56:37.169 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001074965 2023-10-09T05:56:37.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.170 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.170 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.170 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.919115+0000", 2023-10-09T05:56:37.170 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0083542170000000006 2023-10-09T05:56:37.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.170 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.170 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.170 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.920728+0000", 2023-10-09T05:56:37.170 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00161359 2023-10-09T05:56:37.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.927807+0000", 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070788229999999997 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93576 9.e 9:775af324:::benchmark_data_smithi008_502820_object9403:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.344876+0000", 2023-10-09T05:56:37.171 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.691000007, 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.712756782, 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93576 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.172 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344876+0000", 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344876+0000", 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344874+0000", 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:37.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.174 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.174 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.174 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344882+0000", 2023-10-09T05:56:37.174 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4549999999999998e-06 2023-10-09T05:56:37.174 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.174 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.174 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.174 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344894+0000", 2023-10-09T05:56:37.174 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1877e-05 2023-10-09T05:56:37.174 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.174 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344901+0000", 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4229999999999999e-06 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344960+0000", 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8801000000000002e-05 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.175 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344970+0000", 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9490000000000008e-06 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.487235+0000", 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1422650170000002 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.492031+0000", 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0047961879999999998 2023-10-09T05:56:37.176 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.529617+0000", 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037586040000000001 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.533059+0000", 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034415890000000001 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.177 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.012907+0000", 2023-10-09T05:56:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47984856799999998 2023-10-09T05:56:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.016476+0000", 2023-10-09T05:56:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035688650000000001 2023-10-09T05:56:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.021478+0000", 2023-10-09T05:56:37.178 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050017769999999998 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.047526+0000", 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.026047573000000001 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.048644+0000", 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001118703 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.179 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.049061+0000", 2023-10-09T05:56:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000416425 2023-10-09T05:56:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.050924+0000", 2023-10-09T05:56:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001863504 2023-10-09T05:56:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.180 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.057633+0000", 2023-10-09T05:56:37.181 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0067087539999999999 2023-10-09T05:56:37.181 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.181 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.181 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.181 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.181 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.181 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93584 9.e 9:788be11e:::benchmark_data_smithi008_502820_object9411:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T05:56:37.181 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.353872+0000", 2023-10-09T05:56:37.182 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.682004308000003, 2023-10-09T05:56:37.182 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7661968130000001, 2023-10-09T05:56:37.182 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.182 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.182 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.182 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.182 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.182 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93584 2023-10-09T05:56:37.182 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.182 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.183 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.183 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353872+0000", 2023-10-09T05:56:37.183 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.183 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.183 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.183 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353872+0000", 2023-10-09T05:56:37.183 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.183 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.184 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.185 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353871+0000", 2023-10-09T05:56:37.185 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:37.185 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.185 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.185 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353877+0000", 2023-10-09T05:56:37.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3980000000000004e-06 2023-10-09T05:56:37.186 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.186 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353890+0000", 2023-10-09T05:56:37.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2541000000000001e-05 2023-10-09T05:56:37.186 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.186 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353898+0000", 2023-10-09T05:56:37.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4239999999999993e-06 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353966+0000", 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7978999999999994e-05 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353979+0000", 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.293e-05 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.493408+0000", 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.139429239 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.497116+0000", 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037075620000000002 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.546542+0000", 2023-10-09T05:56:37.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.049425963000000003 2023-10-09T05:56:37.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.549991+0000", 2023-10-09T05:56:37.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034494270000000001 2023-10-09T05:56:37.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.068138+0000", 2023-10-09T05:56:37.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.51814739300000001 2023-10-09T05:56:37.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.190 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.190 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.071757+0000", 2023-10-09T05:56:37.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003618095 2023-10-09T05:56:37.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.190 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.190 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.078191+0000", 2023-10-09T05:56:37.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064343730000000002 2023-10-09T05:56:37.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.094886+0000", 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016694831 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.111048+0000", 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016162398000000001 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.191 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.112084+0000", 2023-10-09T05:56:37.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010355239999999999 2023-10-09T05:56:37.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.192 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.113683+0000", 2023-10-09T05:56:37.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001599222 2023-10-09T05:56:37.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.192 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.120069+0000", 2023-10-09T05:56:37.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0063858350000000003 2023-10-09T05:56:37.193 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.193 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.193 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.193 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.193 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.193 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93589 9.e 9:787a8e5e:::benchmark_data_smithi008_502820_object9416:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T05:56:37.193 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.358349+0000", 2023-10-09T05:56:37.194 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.677527495000007, 2023-10-09T05:56:37.194 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.772108904, 2023-10-09T05:56:37.194 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.194 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.194 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.194 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.194 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.194 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93589 2023-10-09T05:56:37.194 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.194 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.195 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.195 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.195 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358349+0000", 2023-10-09T05:56:37.195 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.195 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.195 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.196 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.196 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358349+0000", 2023-10-09T05:56:37.196 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.196 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.196 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.196 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.197 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358348+0000", 2023-10-09T05:56:37.197 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:37.197 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.197 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.197 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.197 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358351+0000", 2023-10-09T05:56:37.198 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.044e-06 2023-10-09T05:56:37.198 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.198 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.198 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.198 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358361+0000", 2023-10-09T05:56:37.198 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.516e-06 2023-10-09T05:56:37.198 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.199 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.199 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.199 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358367+0000", 2023-10-09T05:56:37.199 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8710000000000002e-06 2023-10-09T05:56:37.199 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.199 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.199 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.199 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358431+0000", 2023-10-09T05:56:37.200 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4703999999999996e-05 2023-10-09T05:56:37.200 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.200 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.200 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.200 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358447+0000", 2023-10-09T05:56:37.200 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5749000000000001e-05 2023-10-09T05:56:37.200 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.201 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.201 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.201 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.550328+0000", 2023-10-09T05:56:37.201 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1918804390000002 2023-10-09T05:56:37.201 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.201 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.201 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.202 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.554496+0000", 2023-10-09T05:56:37.202 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041680650000000003 2023-10-09T05:56:37.202 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.202 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.202 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.202 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.576252+0000", 2023-10-09T05:56:37.202 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021756074 2023-10-09T05:56:37.202 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.203 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.203 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.203 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.579807+0000", 2023-10-09T05:56:37.203 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003555599 2023-10-09T05:56:37.203 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.203 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.203 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.203 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.095351+0000", 2023-10-09T05:56:37.203 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.51554361800000004 2023-10-09T05:56:37.203 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.204 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.204 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.204 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.099028+0000", 2023-10-09T05:56:37.204 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036770129999999998 2023-10-09T05:56:37.204 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.204 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.204 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.204 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.104732+0000", 2023-10-09T05:56:37.204 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0057035779999999999 2023-10-09T05:56:37.204 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.204 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.205 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.205 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.120380+0000", 2023-10-09T05:56:37.205 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015648539 2023-10-09T05:56:37.205 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.205 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.205 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.205 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.121336+0000", 2023-10-09T05:56:37.205 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095552600000000005 2023-10-09T05:56:37.205 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.205 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.205 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.206 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.122385+0000", 2023-10-09T05:56:37.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010494969999999999 2023-10-09T05:56:37.206 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.206 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.206 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.206 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.124126+0000", 2023-10-09T05:56:37.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017405159999999999 2023-10-09T05:56:37.206 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.206 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.206 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.207 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.130458+0000", 2023-10-09T05:56:37.207 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0063319300000000004 2023-10-09T05:56:37.207 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.207 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.207 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.207 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.207 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.207 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93594 9.e 9:7f17079c:::benchmark_data_smithi008_502820_object9421:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T05:56:37.207 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.374295+0000", 2023-10-09T05:56:37.207 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.661580866999998, 2023-10-09T05:56:37.207 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8370609930000001, 2023-10-09T05:56:37.208 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.208 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.208 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.208 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.208 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.208 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93594 2023-10-09T05:56:37.208 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.208 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.208 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.208 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374295+0000", 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374295+0000", 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374293+0000", 2023-10-09T05:56:37.209 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:37.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.210 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.210 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374302+0000", 2023-10-09T05:56:37.210 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8829999999999995e-06 2023-10-09T05:56:37.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.210 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.210 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374317+0000", 2023-10-09T05:56:37.210 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4803999999999999e-05 2023-10-09T05:56:37.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.211 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.211 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.211 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374326+0000", 2023-10-09T05:56:37.211 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5480000000000007e-06 2023-10-09T05:56:37.211 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.211 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.211 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.211 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374387+0000", 2023-10-09T05:56:37.211 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0439000000000002e-05 2023-10-09T05:56:37.211 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.211 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.212 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.212 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374401+0000", 2023-10-09T05:56:37.212 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4321000000000001e-05 2023-10-09T05:56:37.212 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.212 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.212 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.212 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.555902+0000", 2023-10-09T05:56:37.212 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1815005030000001 2023-10-09T05:56:37.212 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.212 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.212 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.559403+0000", 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035008999999999999 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.596637+0000", 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037234872000000002 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.600233+0000", 2023-10-09T05:56:37.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035955890000000002 2023-10-09T05:56:37.214 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.214 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.214 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.214 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.148074+0000", 2023-10-09T05:56:37.214 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.54784091499999998 2023-10-09T05:56:37.214 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.215 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.215 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.215 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.151004+0000", 2023-10-09T05:56:37.215 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029301489999999999 2023-10-09T05:56:37.215 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.215 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.215 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.216 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.156003+0000", 2023-10-09T05:56:37.216 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049993269999999996 2023-10-09T05:56:37.216 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.216 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.216 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.216 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.180475+0000", 2023-10-09T05:56:37.216 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024471809000000001 2023-10-09T05:56:37.217 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.217 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.217 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.217 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.200700+0000", 2023-10-09T05:56:37.217 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020225205 2023-10-09T05:56:37.218 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.218 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.218 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.218 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.202560+0000", 2023-10-09T05:56:37.218 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018592120000000001 2023-10-09T05:56:37.218 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.219 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.219 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.219 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.204613+0000", 2023-10-09T05:56:37.219 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020536410000000001 2023-10-09T05:56:37.219 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.219 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.219 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.220 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.211356+0000", 2023-10-09T05:56:37.220 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0067430190000000003 2023-10-09T05:56:37.220 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.220 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.220 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.220 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.220 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.221 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93595 9.e 9:768fa25c:::benchmark_data_smithi008_502820_object9422:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T05:56:37.221 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.374670+0000", 2023-10-09T05:56:37.221 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.661206042999993, 2023-10-09T05:56:37.221 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9132208020000001, 2023-10-09T05:56:37.221 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.222 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.222 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.222 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.222 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.222 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93595 2023-10-09T05:56:37.222 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.223 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.223 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.223 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.223 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374670+0000", 2023-10-09T05:56:37.223 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.224 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.224 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.224 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.224 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374670+0000", 2023-10-09T05:56:37.224 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.224 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.224 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.224 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374669+0000", 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374675+0000", 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2899999999999999e-06 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374687+0000", 2023-10-09T05:56:37.225 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1705999999999999e-05 2023-10-09T05:56:37.226 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.226 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.226 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374695+0000", 2023-10-09T05:56:37.226 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7400000000000004e-06 2023-10-09T05:56:37.226 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.226 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.226 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374776+0000", 2023-10-09T05:56:37.226 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0828000000000006e-05 2023-10-09T05:56:37.227 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.227 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.227 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.227 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374784+0000", 2023-10-09T05:56:37.227 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8419999999999994e-06 2023-10-09T05:56:37.227 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.227 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.227 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.227 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.601628+0000", 2023-10-09T05:56:37.227 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.226844104 2023-10-09T05:56:37.227 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.605105+0000", 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003476252 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.629729+0000", 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024624511000000002 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.228 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.229 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.633203+0000", 2023-10-09T05:56:37.229 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034742140000000002 2023-10-09T05:56:37.229 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.229 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.229 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.229 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.228786+0000", 2023-10-09T05:56:37.229 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.595582308 2023-10-09T05:56:37.229 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.229 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.229 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.229 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.231835+0000", 2023-10-09T05:56:37.230 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030496080000000001 2023-10-09T05:56:37.230 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.230 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.230 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.230 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.236845+0000", 2023-10-09T05:56:37.230 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050095119999999998 2023-10-09T05:56:37.230 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.230 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.230 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.230 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.273038+0000", 2023-10-09T05:56:37.230 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.036192884000000002 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.278799+0000", 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0057609829999999999 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.279816+0000", 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001016889 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.231 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.281482+0000", 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001666257 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.287891+0000", 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064091240000000004 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.232 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93610 9.e 9:740b9f1d:::benchmark_data_smithi008_502820_object9437:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.406485+0000", 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.629391299999995, 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9398336600000001, 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93610 2023-10-09T05:56:37.233 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.234 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.234 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.234 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.234 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406485+0000", 2023-10-09T05:56:37.234 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.234 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.234 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.234 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.234 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406485+0000", 2023-10-09T05:56:37.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.235 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.235 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.235 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406484+0000", 2023-10-09T05:56:37.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:56:37.235 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.235 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406489+0000", 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.899e-06 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406503+0000", 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3641e-05 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406510+0000", 2023-10-09T05:56:37.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9840000000000004e-06 2023-10-09T05:56:37.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.237 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.237 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.237 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406560+0000", 2023-10-09T05:56:37.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0145999999999997e-05 2023-10-09T05:56:37.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.237 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.237 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.237 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406574+0000", 2023-10-09T05:56:37.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3728e-05 2023-10-09T05:56:37.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.238 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.238 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.238 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.605471+0000", 2023-10-09T05:56:37.238 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1988978069999998 2023-10-09T05:56:37.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.238 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.238 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.238 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.609030+0000", 2023-10-09T05:56:37.238 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035584269999999999 2023-10-09T05:56:37.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.646970+0000", 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037940146000000001 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.650410+0000", 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034402059999999999 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.239 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.298075+0000", 2023-10-09T05:56:37.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.64766473599999996 2023-10-09T05:56:37.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.240 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.301969+0000", 2023-10-09T05:56:37.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038939159999999999 2023-10-09T05:56:37.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.240 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.240 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.307872+0000", 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0059028589999999999 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.335322+0000", 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.027449871000000001 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.336234+0000", 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00091258099999999998 2023-10-09T05:56:37.241 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.337575+0000", 2023-10-09T05:56:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00134044 2023-10-09T05:56:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.339121+0000", 2023-10-09T05:56:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015462939999999999 2023-10-09T05:56:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.243 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.243 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.346319+0000", 2023-10-09T05:56:37.243 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0071977279999999996 2023-10-09T05:56:37.243 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.243 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.243 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.243 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.243 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.243 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93677 9.e 9:78283dde:::benchmark_data_smithi008_502820_object9504:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T05:56:37.243 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.566695+0000", 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.469181289000005, 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7929214409999998, 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93677 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.244 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566695+0000", 2023-10-09T05:56:37.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.245 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566695+0000", 2023-10-09T05:56:37.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.245 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566693+0000", 2023-10-09T05:56:37.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:37.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566701+0000", 2023-10-09T05:56:37.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1240000000000005e-06 2023-10-09T05:56:37.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566720+0000", 2023-10-09T05:56:37.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8777e-05 2023-10-09T05:56:37.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.248 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566729+0000", 2023-10-09T05:56:37.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0990000000000005e-06 2023-10-09T05:56:37.249 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566809+0000", 2023-10-09T05:56:37.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9973000000000002e-05 2023-10-09T05:56:37.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.250 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.250 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566824+0000", 2023-10-09T05:56:37.250 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5009e-05 2023-10-09T05:56:37.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.251 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.610494+0000", 2023-10-09T05:56:37.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0436703280000001 2023-10-09T05:56:37.251 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.251 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.251 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.615455+0000", 2023-10-09T05:56:37.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049606219999999996 2023-10-09T05:56:37.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.663800+0000", 2023-10-09T05:56:37.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.048344564 2023-10-09T05:56:37.253 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.253 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.253 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.667270+0000", 2023-10-09T05:56:37.253 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034707660000000001 2023-10-09T05:56:37.253 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.253 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.321874+0000", 2023-10-09T05:56:37.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.65460370099999998 2023-10-09T05:56:37.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.324634+0000", 2023-10-09T05:56:37.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027600049999999998 2023-10-09T05:56:37.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.329479+0000", 2023-10-09T05:56:37.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048448149999999997 2023-10-09T05:56:37.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.337263+0000", 2023-10-09T05:56:37.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007783852 2023-10-09T05:56:37.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.347338+0000", 2023-10-09T05:56:37.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010074797 2023-10-09T05:56:37.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.351320+0000", 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039819499999999997 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.352893+0000", 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015735219999999999 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.359616+0000", 2023-10-09T05:56:37.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0067233279999999998 2023-10-09T05:56:37.258 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.258 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.258 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.258 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93714 9.e 9:772117e4:::benchmark_data_smithi008_502820_object9541:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T05:56:37.258 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.733175+0000", 2023-10-09T05:56:37.258 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 97.302700912999995, 2023-10-09T05:56:37.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.706776166, 2023-10-09T05:56:37.258 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:37.258 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:37.259 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:37.259 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:37.259 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:37.259 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93714 2023-10-09T05:56:37.259 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.259 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:37.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.260 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:37.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733175+0000", 2023-10-09T05:56:37.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:37.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733175+0000", 2023-10-09T05:56:37.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:37.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:37.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733174+0000", 2023-10-09T05:56:37.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:37.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.262 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.262 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:37.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733183+0000", 2023-10-09T05:56:37.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1949999999999992e-06 2023-10-09T05:56:37.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.263 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:37.263 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733202+0000", 2023-10-09T05:56:37.263 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9624e-05 2023-10-09T05:56:37.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:37.264 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733212+0000", 2023-10-09T05:56:37.264 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7999999999999993e-06 2023-10-09T05:56:37.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.264 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.264 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733269+0000", 2023-10-09T05:56:37.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6560000000000001e-05 2023-10-09T05:56:37.265 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.265 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:37.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733279+0000", 2023-10-09T05:56:37.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0774000000000001e-05 2023-10-09T05:56:37.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.667590+0000", 2023-10-09T05:56:37.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.934310237 2023-10-09T05:56:37.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:37.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.671086+0000", 2023-10-09T05:56:37.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034961910000000001 2023-10-09T05:56:37.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.689198+0000", 2023-10-09T05:56:37.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018112329 2023-10-09T05:56:37.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.692629+0000", 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034307169999999998 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.408630+0000", 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.71600111300000002 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.411913+0000", 2023-10-09T05:56:37.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032832550000000001 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.417307+0000", 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0053933360000000003 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.429677+0000", 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012370188000000001 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.430738+0000", 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001061541 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.431132+0000", 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00039332199999999999 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:37.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.433108+0000", 2023-10-09T05:56:37.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019765020000000002 2023-10-09T05:56:37.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:37.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:37.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:37.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.439951+0000", 2023-10-09T05:56:37.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068432559999999998 2023-10-09T05:56:37.271 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.272 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.272 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.272 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:37.272 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:37.272 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:56:37.272 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:37.305 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:37.695 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:37 smithi008 ceph-mon[191227]: pgmap v1271: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28775/187443 objects degraded (15.351%); 13978/187443 objects misplaced (7.457%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:37.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:37 smithi008 ceph-mon[184418]: pgmap v1271: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28775/187443 objects degraded (15.351%); 13978/187443 objects misplaced (7.457%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:37 smithi191 ceph-mon[161669]: pgmap v1271: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28775/187443 objects degraded (15.351%); 13978/187443 objects misplaced (7.457%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:38.102 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:56:38.118 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:38.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:38.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:38.205 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:38.223 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:38.338 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:38.443 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:38.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:38.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:38.547 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:38.584 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:38.697 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 28775/187443 objects degraded (15.351%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:38.797 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:38.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:38.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:38.901 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:38.930 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 28775/187443 objects degraded (15.351%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:38.981 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:38.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:38.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:38.989 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 28775/187443 objects degraded (15.351%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:39.090 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:39.117 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:39.166 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:39.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:39.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:39.278 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:39.305 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:39.349 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:39.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:39.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:39.455 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:39.486 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:39.600 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:39.646 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:39.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:39.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:39.657 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:39.753 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:39.781 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:39.842 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:39.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:39.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:39.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:39.982 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:40.035 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:40.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:40.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:40.142 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:40.182 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:40.247 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:56:40.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:40 smithi191 ceph-mon[161669]: pgmap v1272: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 28775/186618 objects degraded (15.419%); 13936/186618 objects misplaced (7.468%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:40.391 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:40.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:40.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:40.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:40 smithi008 ceph-mon[184418]: pgmap v1272: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 28775/186618 objects degraded (15.419%); 13936/186618 objects misplaced (7.468%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:40.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:40 smithi008 ceph-mon[191227]: pgmap v1272: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 28775/186618 objects degraded (15.419%); 13936/186618 objects misplaced (7.468%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:40.497 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:40.529 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:40.596 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:40.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:40.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:40.703 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:40.731 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:40.802 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:40.805 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:56:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:56:40] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.33.4" 2023-10-09T05:56:40.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:40.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:40.832 INFO:teuthology.orchestra.run.smithi008.stdout:moved 37 / 297 (12.4579%) 2023-10-09T05:56:40.833 INFO:teuthology.orchestra.run.smithi008.stdout:avg 42.4286 2023-10-09T05:56:40.833 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 10.0691 -> 5.82745 (expected baseline 6.03053) 2023-10-09T05:56:40.833 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.2 with 29 -> 39 pgs (0.683502 -> 0.919192 * mean) 2023-10-09T05:56:40.833 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.1 with 60 -> 48 pgs (1.41414 -> 1.13131 * mean) 2023-10-09T05:56:40.834 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:56:40.834 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T05:56:40.834 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 1 2023-10-09T05:56:40.834 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:56:40.834 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0268 2023-10-09T05:56:40.835 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0295 2023-10-09T05:56:40.835 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 weight 1.0000 -> 0.7413 2023-10-09T05:56:40.835 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 weight 1.0000 -> 0.7770 2023-10-09T05:56:40.835 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 1.0000 -> 0.7548 2023-10-09T05:56:40.836 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T05:56:40.907 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:40.938 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:41.215 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:56:40 smithi008 conmon[177229]: 2023-10-09T05:56:40.825+0000 7f020285b700 -1 mgr.server reply reply (3) No such process no change 2023-10-09T05:56:41.847 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:41.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:41.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:41.951 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:41.991 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:42.209 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:42 smithi008 ceph-mon[191227]: pgmap v1273: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 28725/185880 objects degraded (15.454%); 13837/185880 objects misplaced (7.444%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:42.210 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:42 smithi008 ceph-mon[191227]: from='client.46460 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:42.454 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:42.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:42.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:42.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:42 smithi008 ceph-mon[184418]: pgmap v1273: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 28725/185880 objects degraded (15.454%); 13837/185880 objects misplaced (7.444%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:42.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:42 smithi008 ceph-mon[184418]: from='client.46460 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:42.514 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T05:56:42.515 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T05:56:42.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:42 smithi191 ceph-mon[161669]: pgmap v1273: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 28725/185880 objects degraded (15.454%); 13837/185880 objects misplaced (7.444%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:42 smithi191 ceph-mon[161669]: from='client.46460 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:42.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:42.591 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:42.641 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:42.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:42.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:42.747 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:42.774 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:42.821 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:42.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:42.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:42.926 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:42.964 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:42.995 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:43.100 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:43.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:43.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:43.128 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:56:43.128 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:56:43.128 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T05:56:43.128 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:56:43.173 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:43.205 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:43.222 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:56:43.265 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:43.313 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:43.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:43.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:43.396 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:43 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/243546004' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:43.396 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:43 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:43.396 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 28725/185880 objects degraded (15.454%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:43.397 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:43 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/243546004' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:43.397 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:43 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:43.397 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 28725/185880 objects degraded (15.454%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:43.418 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:43.450 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:43 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/243546004' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:43 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 28725/185880 objects degraded (15.454%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:44.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[191227]: pgmap v1274: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28680/184401 objects degraded (15.553%); 13783/184401 objects misplaced (7.474%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:44.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:44.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[191227]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[191227]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:56:44.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[191227]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T05:56:44.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[191227]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[191227]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[191227]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:44.413 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:44.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:44.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:44.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[184418]: pgmap v1274: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28680/184401 objects degraded (15.553%); 13783/184401 objects misplaced (7.474%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:44.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:44.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[184418]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[184418]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:56:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[184418]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T05:56:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[184418]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[184418]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[184418]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:44.517 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:44.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:44 smithi191 ceph-mon[161669]: pgmap v1274: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28680/184401 objects degraded (15.553%); 13783/184401 objects misplaced (7.474%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:56:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:44 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:44 smithi191 ceph-mon[161669]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:44 smithi191 ceph-mon[161669]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:44 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:56:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:44 smithi191 ceph-mon[161669]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T05:56:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:44 smithi191 ceph-mon[161669]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:44 smithi191 ceph-mon[161669]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:44 smithi191 ceph-mon[161669]: failed to encode map e427 with expected crc 2023-10-09T05:56:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:44.551 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:45.351 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:45.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:45.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:45.433 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:56:45.457 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:45.508 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:45.999 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:46.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:46.104 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:46.130 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:46.176 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:46.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:46.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:46.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:46 smithi008 ceph-mon[191227]: pgmap v1276: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28629/184113 objects degraded (15.550%); 13829/184113 objects misplaced (7.511%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:56:46.281 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:46.319 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:46.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:46 smithi008 ceph-mon[184418]: pgmap v1276: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28629/184113 objects degraded (15.550%); 13829/184113 objects misplaced (7.511%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:56:46.534 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:46.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:46.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:46 smithi191 ceph-mon[161669]: pgmap v1276: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 28629/184113 objects degraded (15.550%); 13829/184113 objects misplaced (7.511%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:56:46.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:46.671 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:46.723 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:46.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:46.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:46.780 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:56:46.809 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:56:46.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:46.835 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:56:46.862 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:56:46.875 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:46.884 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:56:46.906 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:46.920 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:56:46.936 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:46.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:46.952 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:56:46.984 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:56:46.987 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:56:46.987 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:56:46.987 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:56:46.987 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:56:46.987 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:56:46.987 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:56:46.987 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:56:46.987 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:56:47.044 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:47.086 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:47.218 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2845163934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:47.250 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:47.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:47.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:47.355 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:47.357 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:56:47.384 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2845163934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:47.384 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2845163934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:47.387 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:48.073 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:48.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:48.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:48.179 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:48.213 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:48.236 INFO:teuthology.orchestra.run.smithi008.stdout:moved 37 / 297 (12.4579%) 2023-10-09T05:56:48.236 INFO:teuthology.orchestra.run.smithi008.stdout:avg 42.4286 2023-10-09T05:56:48.237 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 10.0691 -> 5.82745 (expected baseline 6.03053) 2023-10-09T05:56:48.237 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.2 with 29 -> 39 pgs (0.683502 -> 0.919192 * mean) 2023-10-09T05:56:48.237 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.1 with 60 -> 48 pgs (1.41414 -> 1.13131 * mean) 2023-10-09T05:56:48.237 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:56:48.237 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T05:56:48.237 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 3 2023-10-09T05:56:48.237 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:56:48.237 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0266 2023-10-09T05:56:48.237 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0292 2023-10-09T05:56:48.237 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5 weight 1.0000 -> 0.7417 2023-10-09T05:56:48.237 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 weight 1.0000 -> 0.7757 2023-10-09T05:56:48.238 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 1.0000 -> 0.7579 2023-10-09T05:56:48.292 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:48 smithi008 ceph-mon[184418]: pgmap v1277: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 28576/183105 objects degraded (15.606%); 13660/183105 objects misplaced (7.460%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:56:48.293 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:48 smithi008 ceph-mon[184418]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:48.293 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:48 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'50836','3':'49672','5':'48605'}"}]: dispatch 2023-10-09T05:56:48.293 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 28576/183105 objects degraded (15.606%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:48.295 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:48 smithi008 ceph-mon[191227]: pgmap v1277: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 28576/183105 objects degraded (15.606%); 13660/183105 objects misplaced (7.460%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:56:48.295 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:48 smithi008 ceph-mon[191227]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:48.295 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:48 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'50836','3':'49672','5':'48605'}"}]: dispatch 2023-10-09T05:56:48.295 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 28576/183105 objects degraded (15.606%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:48.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:48 smithi191 ceph-mon[161669]: pgmap v1277: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 28576/183105 objects degraded (15.606%); 13660/183105 objects misplaced (7.460%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:56:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:48 smithi191 ceph-mon[161669]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:48 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'50836','3':'49672','5':'48605'}"}]: dispatch 2023-10-09T05:56:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 28576/183105 objects degraded (15.606%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:49.046 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:49.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:49.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:49.151 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:49.177 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:49.439 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:49.442 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:49 smithi191 ceph-mon[161669]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.442 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:49 smithi191 ceph-mon[161669]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.442 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:49 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'50836','3':'49672','5':'48605'}"}]': finished 2023-10-09T05:56:49.442 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:49 smithi191 ceph-mon[161669]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T05:56:49.442 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:49 smithi191 ceph-mon[161669]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.443 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:49 smithi191 ceph-mon[161669]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:49.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:49.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[184418]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[184418]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'50836','3':'49672','5':'48605'}"}]': finished 2023-10-09T05:56:49.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[184418]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T05:56:49.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[184418]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[184418]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[191227]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[191227]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'50836','3':'49672','5':'48605'}"}]': finished 2023-10-09T05:56:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[191227]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T05:56:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[191227]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:49 smithi008 ceph-mon[191227]: failed to encode map e428 with expected crc 2023-10-09T05:56:49.532 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:49.544 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:49.551 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:56:49.552 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-10-09T05:56:49.700 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:49.745 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:49.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:49.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:49.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:49.878 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:49.926 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:49.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:49.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:50.032 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:50.036 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:50.037 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:50.058 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:50.096 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:56:50.106 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:50.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:50.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:50.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:50.241 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:50.341 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[184418]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 28576/182301 objects degraded (15.675%); 13614/182301 objects misplaced (7.468%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T05:56:50.341 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[184418]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.342 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[184418]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.342 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[184418]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.342 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[184418]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.342 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[184418]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T05:56:50.342 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[184418]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.342 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[184418]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.342 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[184418]: 9.a continuing backfill to osd.7 from (0'0,427'6813] MIN to 427'6813 2023-10-09T05:56:50.342 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[184418]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.342 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[184418]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.343 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[191227]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 28576/182301 objects degraded (15.675%); 13614/182301 objects misplaced (7.468%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T05:56:50.343 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[191227]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.343 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[191227]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.343 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[191227]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.343 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[191227]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.343 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[191227]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T05:56:50.343 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[191227]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.343 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[191227]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.343 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[191227]: 9.a continuing backfill to osd.7 from (0'0,427'6813] MIN to 427'6813 2023-10-09T05:56:50.343 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[191227]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.344 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:50 smithi008 ceph-mon[191227]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.459 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:50.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:50.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:50.467 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:50 smithi191 ceph-mon[161669]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 12 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 28576/182301 objects degraded (15.675%); 13614/182301 objects misplaced (7.468%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T05:56:50.467 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:50 smithi191 ceph-mon[161669]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.467 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:50 smithi191 ceph-mon[161669]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.467 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:50 smithi191 ceph-mon[161669]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.467 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:50 smithi191 ceph-mon[161669]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.467 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:50 smithi191 ceph-mon[161669]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T05:56:50.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:50 smithi191 ceph-mon[161669]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:50 smithi191 ceph-mon[161669]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:50 smithi191 ceph-mon[161669]: 9.a continuing backfill to osd.7 from (0'0,427'6813] MIN to 427'6813 2023-10-09T05:56:50.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:50 smithi191 ceph-mon[161669]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:50 smithi191 ceph-mon[161669]: failed to encode map e429 with expected crc 2023-10-09T05:56:50.567 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:50.617 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:50.693 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:50.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:50.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:50.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:56:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:56:50] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.33.4" 2023-10-09T05:56:50.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:50.828 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:50.877 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:50.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:50.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:50.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:51.008 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:51.055 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:51.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:51.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:51.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:51.187 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:51.237 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:51.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:51.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:51.344 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:51.370 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:51.420 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:51.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:51.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:51.484 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:51.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[184418]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[184418]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[184418]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T05:56:51.516 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[184418]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[184418]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.517 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[184418]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[191227]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[191227]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[191227]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T05:56:51.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[191227]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[191227]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.517 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:51 smithi008 ceph-mon[191227]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.518 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:56:51.525 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:51.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:51 smithi191 ceph-mon[161669]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:51 smithi191 ceph-mon[161669]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:51 smithi191 ceph-mon[161669]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T05:56:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:51 smithi191 ceph-mon[161669]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:51 smithi191 ceph-mon[161669]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:51 smithi191 ceph-mon[161669]: failed to encode map e430 with expected crc 2023-10-09T05:56:51.590 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:51.747 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:51.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:51.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:51.851 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:51.877 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:51.922 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:51.923 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:51.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:51.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:52.030 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:52.069 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:52.133 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:52.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:52.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:52.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:52.269 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:52.288 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:52.339 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:52.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:52.411 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:52 smithi008 ceph-mon[184418]: pgmap v1282: 99 pgs: 6 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 150 op/s; 13974/181398 objects degraded (7.704%); 15919/181398 objects misplaced (8.776%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:56:52.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:52 smithi008 ceph-mon[184418]: failed to encode map e431 with expected crc 2023-10-09T05:56:52.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:52 smithi008 ceph-mon[184418]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T05:56:52.412 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:52 smithi008 ceph-mon[184418]: failed to encode map e431 with expected crc 2023-10-09T05:56:52.412 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:52 smithi008 ceph-mon[191227]: pgmap v1282: 99 pgs: 6 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 150 op/s; 13974/181398 objects degraded (7.704%); 15919/181398 objects misplaced (8.776%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:56:52.412 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:52 smithi008 ceph-mon[191227]: failed to encode map e431 with expected crc 2023-10-09T05:56:52.412 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:52 smithi008 ceph-mon[191227]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T05:56:52.412 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:52 smithi008 ceph-mon[191227]: failed to encode map e431 with expected crc 2023-10-09T05:56:52.444 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:52.469 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:52.514 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:52.517 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:52 smithi191 ceph-mon[161669]: pgmap v1282: 99 pgs: 6 remapped+peering, 1 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 150 op/s; 13974/181398 objects degraded (7.704%); 15919/181398 objects misplaced (8.776%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:56:52.517 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:52 smithi191 ceph-mon[161669]: failed to encode map e431 with expected crc 2023-10-09T05:56:52.517 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:52 smithi191 ceph-mon[161669]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T05:56:52.517 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:52 smithi191 ceph-mon[161669]: failed to encode map e431 with expected crc 2023-10-09T05:56:52.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:52.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:52.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:52.645 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:52.689 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:52.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:52.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:52.796 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:52.826 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:53.240 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:53.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:53.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:53.343 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:53.369 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:53.413 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:53.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:53.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:53.519 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:53.546 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13974/181398 objects degraded (7.704%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:53.546 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13974/181398 objects degraded (7.704%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:53.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13974/181398 objects degraded (7.704%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:53.550 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:54.260 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:56:54.504 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:54.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:54.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:54.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:54 smithi191 ceph-mon[161669]: pgmap v1284: 99 pgs: 6 remapped+peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 24435/181209 objects degraded (13.484%); 15912/181209 objects misplaced (8.781%); 3.0 MiB/s, 49 objects/s recovering 2023-10-09T05:56:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:54 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/661243978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:54 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:54.556 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:54 smithi008 ceph-mon[184418]: pgmap v1284: 99 pgs: 6 remapped+peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 24435/181209 objects degraded (13.484%); 15912/181209 objects misplaced (8.781%); 3.0 MiB/s, 49 objects/s recovering 2023-10-09T05:56:54.557 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:54 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/661243978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:54.557 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:54 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:54.557 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:54 smithi008 ceph-mon[191227]: pgmap v1284: 99 pgs: 6 remapped+peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 24435/181209 objects degraded (13.484%); 15912/181209 objects misplaced (8.781%); 3.0 MiB/s, 49 objects/s recovering 2023-10-09T05:56:54.557 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:54 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/661243978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:54.557 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:54 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:54.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:54.641 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:54.696 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:54.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:54.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:54.802 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:54.835 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:55.229 INFO:teuthology.orchestra.run.smithi008.stdout:moved 0 / 297 (0%) 2023-10-09T05:56:55.230 INFO:teuthology.orchestra.run.smithi008.stdout:avg 42.4286 2023-10-09T05:56:55.230 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 5.82745 -> 5.82745 (expected baseline 6.03053) 2023-10-09T05:56:55.230 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.5 with 36 -> 36 pgs (0.848485 -> 0.848485 * mean) 2023-10-09T05:56:55.230 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.7 with 54 -> 54 pgs (1.27273 -> 1.27273 * mean) 2023-10-09T05:56:55.230 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:56:55.230 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T05:56:55.230 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 0.05 2023-10-09T05:56:55.230 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:56:55.230 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0264 2023-10-09T05:56:55.230 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.2640 2023-10-09T05:56:55.231 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T05:56:55.231 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:55.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:55.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:55.337 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:55.362 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:55.413 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:55.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:55.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:55.461 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:55 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:56:55.461 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:55 smithi008 ceph-mon[184418]: failed to encode map e432 with expected crc 2023-10-09T05:56:55.461 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:55 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:55.461 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:55 smithi008 ceph-mon[184418]: osdmap e432: 8 total, 8 up, 7 in 2023-10-09T05:56:55.461 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:55 smithi008 ceph-mon[184418]: failed to encode map e432 with expected crc 2023-10-09T05:56:55.462 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:55 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:56:55.462 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:55 smithi008 ceph-mon[191227]: failed to encode map e432 with expected crc 2023-10-09T05:56:55.462 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:55 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:55.462 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:55 smithi008 ceph-mon[191227]: osdmap e432: 8 total, 8 up, 7 in 2023-10-09T05:56:55.462 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:55 smithi008 ceph-mon[191227]: failed to encode map e432 with expected crc 2023-10-09T05:56:55.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:55.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:55 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:56:55.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:55 smithi191 ceph-mon[161669]: failed to encode map e432 with expected crc 2023-10-09T05:56:55.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:55 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:55.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:55 smithi191 ceph-mon[161669]: osdmap e432: 8 total, 8 up, 7 in 2023-10-09T05:56:55.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:55 smithi191 ceph-mon[161669]: failed to encode map e432 with expected crc 2023-10-09T05:56:55.547 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:55.598 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:55.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:55.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:55.704 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:55.728 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:56:55.741 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:55.752 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:56:55.777 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:56:55.797 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:56:55.816 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:56:55.836 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:56:55.854 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:56:56.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:56 smithi008 ceph-mon[191227]: pgmap v1286: 99 pgs: 1 activating+undersized+remapped, 5 remapped+peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 70 op/s; 25529/181167 objects degraded (14.091%); 17502/181167 objects misplaced (9.661%); 2.8 MiB/s, 46 objects/s recovering 2023-10-09T05:56:56.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:56 smithi008 ceph-mon[191227]: from='client.36546 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:56.264 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:56 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2197073770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:56.490 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:56.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:56.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:56.538 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:56 smithi008 ceph-mon[184418]: pgmap v1286: 99 pgs: 1 activating+undersized+remapped, 5 remapped+peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 70 op/s; 25529/181167 objects degraded (14.091%); 17502/181167 objects misplaced (9.661%); 2.8 MiB/s, 46 objects/s recovering 2023-10-09T05:56:56.539 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:56 smithi008 ceph-mon[184418]: from='client.36546 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:56.539 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:56 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2197073770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:56.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:56 smithi191 ceph-mon[161669]: pgmap v1286: 99 pgs: 1 activating+undersized+remapped, 5 remapped+peering, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 70 op/s; 25529/181167 objects degraded (14.091%); 17502/181167 objects misplaced (9.661%); 2.8 MiB/s, 46 objects/s recovering 2023-10-09T05:56:56.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:56 smithi191 ceph-mon[161669]: from='client.36546 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:56 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2197073770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:56.594 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:56.624 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:57.085 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:57.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:57.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:57.168 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T05:56:57.168 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-10-09T05:56:57.189 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:57.242 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:57.354 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:57.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:57.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:57.458 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:57.485 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:57.529 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:57.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:57.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:57.589 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:56:57.589 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:56:57.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:56:57.589 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:56:57.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.590 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108328 8.1f 8:fb066e62:::benchmark_data_smithi008_432048_object5386:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:56:57.590 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:29.984571+0000", 2023-10-09T05:56:57.590 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 447.60133399300003, 2023-10-09T05:56:57.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92517276500000001, 2023-10-09T05:56:57.590 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.590 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.590 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.590 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.590 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.590 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108328 2023-10-09T05:56:57.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.591 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984571+0000", 2023-10-09T05:56:57.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.591 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.591 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.591 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.591 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984571+0000", 2023-10-09T05:56:57.591 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984570+0000", 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984577+0000", 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4800000000000004e-06 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.592 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984588+0000", 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0906e-05 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:29.984594+0000", 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3620000000000002e-06 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.005278+0000", 2023-10-09T05:56:57.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020683634999999999 2023-10-09T05:56:57.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.009272+0000", 2023-10-09T05:56:57.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039940679999999999 2023-10-09T05:56:57.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.062179+0000", 2023-10-09T05:56:57.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.052906448000000002 2023-10-09T05:56:57.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.596 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.066018+0000", 2023-10-09T05:56:57.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003839496 2023-10-09T05:56:57.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.596 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.596 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.827356+0000", 2023-10-09T05:56:57.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.76133771100000003 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.838496+0000", 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011140278 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.856903+0000", 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018407137 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.597 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.878092+0000", 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021188335999999999 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.878887+0000", 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00079575399999999997 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.891744+0000", 2023-10-09T05:56:57.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012856932 2023-10-09T05:56:57.599 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.599 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.599 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.895389+0000", 2023-10-09T05:56:57.599 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003644974 2023-10-09T05:56:57.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.600 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.600 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:30.909744+0000", 2023-10-09T05:56:57.600 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014354693 2023-10-09T05:56:57.600 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.600 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.600 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.600 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.601 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108591 8.1f 8:f826337a:::benchmark_data_smithi008_432048_object5649:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:56:57.601 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:31.239317+0000", 2023-10-09T05:56:57.601 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 446.346588193, 2023-10-09T05:56:57.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94874395700000003, 2023-10-09T05:56:57.601 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.601 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.601 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.601 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.601 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.601 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108591 2023-10-09T05:56:57.601 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.602 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.602 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.602 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239317+0000", 2023-10-09T05:56:57.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.602 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.602 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239317+0000", 2023-10-09T05:56:57.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239315+0000", 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239323+0000", 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8040000000000002e-06 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239336+0000", 2023-10-09T05:56:57.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2957e-05 2023-10-09T05:56:57.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.239345+0000", 2023-10-09T05:56:57.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9830000000000002e-06 2023-10-09T05:56:57.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.277097+0000", 2023-10-09T05:56:57.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037752218999999997 2023-10-09T05:56:57.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.281903+0000", 2023-10-09T05:56:57.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048058729999999996 2023-10-09T05:56:57.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.310019+0000", 2023-10-09T05:56:57.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028116208 2023-10-09T05:56:57.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.314967+0000", 2023-10-09T05:56:57.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049481849999999999 2023-10-09T05:56:57.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.108322+0000", 2023-10-09T05:56:57.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.79335536500000003 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.114934+0000", 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066115519999999997 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.143064+0000", 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028130485 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.156587+0000", 2023-10-09T05:56:57.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013522569999999999 2023-10-09T05:56:57.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.159203+0000", 2023-10-09T05:56:57.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026158650000000002 2023-10-09T05:56:57.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.170337+0000", 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011134412 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.174909+0000", 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045717029999999999 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.188061+0000", 2023-10-09T05:56:57.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013151985 2023-10-09T05:56:57.610 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.610 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.610 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.610 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.610 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108664 8.1f 8:fb1a1b36:::benchmark_data_smithi008_432048_object5722:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:56:57.610 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:31.614117+0000", 2023-10-09T05:56:57.610 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 445.97178830600001, 2023-10-09T05:56:57.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1244621509999999, 2023-10-09T05:56:57.610 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.611 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.611 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.611 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.611 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.611 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108664 2023-10-09T05:56:57.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.611 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614117+0000", 2023-10-09T05:56:57.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614117+0000", 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614115+0000", 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614125+0000", 2023-10-09T05:56:57.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0227000000000001e-05 2023-10-09T05:56:57.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614141+0000", 2023-10-09T05:56:57.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5631999999999999e-05 2023-10-09T05:56:57.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.614152+0000", 2023-10-09T05:56:57.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0627e-05 2023-10-09T05:56:57.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.644859+0000", 2023-10-09T05:56:57.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030707564999999999 2023-10-09T05:56:57.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.650494+0000", 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0056349290000000003 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.686762+0000", 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.036267668000000003 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.693827+0000", 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070651710000000003 2023-10-09T05:56:57.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.512455+0000", 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.81862801299999999 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.517577+0000", 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051220679999999996 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.526918+0000", 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0093413560000000003 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.563730+0000", 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.036811893999999998 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.726330+0000", 2023-10-09T05:56:57.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.16260017500000001 2023-10-09T05:56:57.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.618 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.618 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.726967+0000", 2023-10-09T05:56:57.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000636657 2023-10-09T05:56:57.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.618 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.618 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.729816+0000", 2023-10-09T05:56:57.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028490059999999998 2023-10-09T05:56:57.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.738579+0000", 2023-10-09T05:56:57.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0087629040000000002 2023-10-09T05:56:57.619 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.619 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.619 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.619 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108709 8.1f 8:fc046827:::benchmark_data_smithi008_432048_object5767:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:31.757269+0000", 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 445.82863604800002, 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.998060002, 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108709 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757269+0000", 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757269+0000", 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757267+0000", 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757278+0000", 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0876000000000001e-05 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757298+0000", 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0075999999999999e-05 2023-10-09T05:56:57.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.757309+0000", 2023-10-09T05:56:57.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0896999999999999e-05 2023-10-09T05:56:57.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.817372+0000", 2023-10-09T05:56:57.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.060062786999999999 2023-10-09T05:56:57.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.828286+0000", 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010914316 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.896232+0000", 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.067946329 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:31.901371+0000", 2023-10-09T05:56:57.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051384359999999997 2023-10-09T05:56:57.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.544517+0000", 2023-10-09T05:56:57.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.64314595100000005 2023-10-09T05:56:57.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.548771+0000", 2023-10-09T05:56:57.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042545530000000003 2023-10-09T05:56:57.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.556889+0000", 2023-10-09T05:56:57.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081172039999999994 2023-10-09T05:56:57.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.739031+0000", 2023-10-09T05:56:57.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.182142532 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.740311+0000", 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0012802569999999999 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.741944+0000", 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001632313 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.744193+0000", 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002249418 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.755329+0000", 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011135987999999999 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.629 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108836 8.1f 8:fd9739dd:::benchmark_data_smithi008_432048_object5894:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:56:57.629 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:32.387648+0000", 2023-10-09T05:56:57.629 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 445.19825669099998, 2023-10-09T05:56:57.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92865287500000004, 2023-10-09T05:56:57.629 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.629 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.629 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.629 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.629 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.629 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108836 2023-10-09T05:56:57.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.630 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387648+0000", 2023-10-09T05:56:57.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387648+0000", 2023-10-09T05:56:57.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387648+0000", 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387651+0000", 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8210000000000002e-06 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387657+0000", 2023-10-09T05:56:57.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8000000000000004e-06 2023-10-09T05:56:57.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.387663+0000", 2023-10-09T05:56:57.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0410000000000003e-06 2023-10-09T05:56:57.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.439921+0000", 2023-10-09T05:56:57.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.052258302 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.455650+0000", 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015728783 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.506375+0000", 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.050725721000000001 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.511713+0000", 2023-10-09T05:56:57.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005337713 2023-10-09T05:56:57.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.231814+0000", 2023-10-09T05:56:57.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72010072700000005 2023-10-09T05:56:57.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.238299+0000", 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064853979999999999 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.246417+0000", 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.008117839 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.299253+0000", 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.052835676999999998 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.301785+0000", 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025323110000000002 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.302525+0000", 2023-10-09T05:56:57.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00073951699999999997 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.305906+0000", 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003381481 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.316301+0000", 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010395196000000001 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.637 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.638 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.638 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108877 8.f 8:f5ef4290:::benchmark_data_smithi008_432048_object5935:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:56:57.638 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:32.524792+0000", 2023-10-09T05:56:57.638 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 445.06111317199998, 2023-10-09T05:56:57.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93690308099999997, 2023-10-09T05:56:57.638 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.638 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.638 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.638 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.639 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.639 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108877 2023-10-09T05:56:57.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.639 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524792+0000", 2023-10-09T05:56:57.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524792+0000", 2023-10-09T05:56:57.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524790+0000", 2023-10-09T05:56:57.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:57.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524798+0000", 2023-10-09T05:56:57.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1699999999999997e-06 2023-10-09T05:56:57.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524811+0000", 2023-10-09T05:56:57.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3185e-05 2023-10-09T05:56:57.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.524820+0000", 2023-10-09T05:56:57.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4840000000000004e-06 2023-10-09T05:56:57.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.771712+0000", 2023-10-09T05:56:57.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.246892371 2023-10-09T05:56:57.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.776583+0000", 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004870854 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.816435+0000", 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.039851866999999999 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:32.821386+0000", 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049510520000000001 2023-10-09T05:56:57.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.419677+0000", 2023-10-09T05:56:57.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.59829150799999997 2023-10-09T05:56:57.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.423867+0000", 2023-10-09T05:56:57.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041896390000000002 2023-10-09T05:56:57.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.431852+0000", 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0079849550000000002 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.440239+0000", 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0083871919999999999 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.441156+0000", 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00091646800000000004 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.451403+0000", 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010247166 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.453645+0000", 2023-10-09T05:56:57.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022420090000000001 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.461695+0000", 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0080501570000000005 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:108988 8.1f 8:fb6f108d:::benchmark_data_smithi008_432048_object6046:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T05:56:57.647 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:33.159282+0000", 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 444.42662363900001, 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.92710595600000001, 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 108988 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159282+0000", 2023-10-09T05:56:57.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159282+0000", 2023-10-09T05:56:57.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.649 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159279+0000", 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159290+0000", 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0994e-05 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159310+0000", 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9641e-05 2023-10-09T05:56:57.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.159326+0000", 2023-10-09T05:56:57.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5529e-05 2023-10-09T05:56:57.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.181153+0000", 2023-10-09T05:56:57.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021827215000000001 2023-10-09T05:56:57.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.188840+0000", 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0076866929999999997 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.209278+0000", 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020438587000000001 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.215354+0000", 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006075886 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.986664+0000", 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.771309731 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.990016+0000", 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033521359999999999 2023-10-09T05:56:57.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:33.996527+0000", 2023-10-09T05:56:57.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00651148 2023-10-09T05:56:57.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.051138+0000", 2023-10-09T05:56:57.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.054610341999999999 2023-10-09T05:56:57.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.070103+0000", 2023-10-09T05:56:57.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018965288 2023-10-09T05:56:57.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.072016+0000", 2023-10-09T05:56:57.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019130620000000001 2023-10-09T05:56:57.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.073818+0000", 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018016029999999999 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:34.086387+0000", 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012569800000000001 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:114019 8.f 8:f7f5942a:::benchmark_data_smithi008_432048_object11077:head [delete] snapc 0=[] ondisk+write+known_if_redirected e319)", 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:53.608835+0000", 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 423.97706968199998, 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.98156760300000001, 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 114019 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.657 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608835+0000", 2023-10-09T05:56:57.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608835+0000", 2023-10-09T05:56:57.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608834+0000", 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608840+0000", 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8810000000000001e-06 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608849+0000", 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.054e-06 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.608855+0000", 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8660000000000002e-06 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.709132+0000", 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.100276884 2023-10-09T05:56:57.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.712800+0000", 2023-10-09T05:56:57.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036683029999999999 2023-10-09T05:56:57.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.729698+0000", 2023-10-09T05:56:57.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016897586999999999 2023-10-09T05:56:57.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.732690+0000", 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029921689999999998 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.556908+0000", 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.82421833499999997 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.560831+0000", 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039229139999999996 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.568419+0000", 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0075875789999999997 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.575101+0000", 2023-10-09T05:56:57.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066826059999999998 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.582134+0000", 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070324810000000001 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.582676+0000", 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00054164800000000004 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.584594+0000", 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019186209999999999 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.590403+0000", 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005808957 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.665 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.666 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:114052 8.1f 8:f8538642:::benchmark_data_smithi008_432048_object11110:head [delete] snapc 0=[] ondisk+write+known_if_redirected e319)", 2023-10-09T05:56:57.666 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:53.698035+0000", 2023-10-09T05:56:57.666 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 423.88787037200001, 2023-10-09T05:56:57.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97790464700000002, 2023-10-09T05:56:57.666 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.666 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.666 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.666 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.666 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 114052 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698035+0000", 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698035+0000", 2023-10-09T05:56:57.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698033+0000", 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698041+0000", 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6939999999999995e-06 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698053+0000", 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2361000000000001e-05 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.698063+0000", 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8199999999999992e-06 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.711539+0000", 2023-10-09T05:56:57.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013475941 2023-10-09T05:56:57.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.718579+0000", 2023-10-09T05:56:57.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007040073 2023-10-09T05:56:57.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.773633+0000", 2023-10-09T05:56:57.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.055054084000000003 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.778235+0000", 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0046020050000000002 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.624012+0000", 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.84577629099999996 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.628102+0000", 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004090247 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.636818+0000", 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0087158210000000003 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.664357+0000", 2023-10-09T05:56:57.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.02753889 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.664692+0000", 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00033567800000000001 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.666599+0000", 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001906477 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.668728+0000", 2023-10-09T05:56:57.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021296900000000001 2023-10-09T05:56:57.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.675939+0000", 2023-10-09T05:56:57.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007211065 2023-10-09T05:56:57.674 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.674 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.674 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:114053 8.f 8:f0d5fbe3:::benchmark_data_smithi008_432048_object11111:head [delete] snapc 0=[] ondisk+write+known_if_redirected e319)", 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:53.704034+0000", 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 423.88187102199998, 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.95060431000000001, 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.675 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 114053 2023-10-09T05:56:57.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.676 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704034+0000", 2023-10-09T05:56:57.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704034+0000", 2023-10-09T05:56:57.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704032+0000", 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704042+0000", 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2909999999999996e-06 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704058+0000", 2023-10-09T05:56:57.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6708e-05 2023-10-09T05:56:57.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.704068+0000", 2023-10-09T05:56:57.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6020000000000006e-06 2023-10-09T05:56:57.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.713094+0000", 2023-10-09T05:56:57.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0090258779999999993 2023-10-09T05:56:57.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.716172+0000", 2023-10-09T05:56:57.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003077766 2023-10-09T05:56:57.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.745331+0000", 2023-10-09T05:56:57.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.029158836 2023-10-09T05:56:57.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.748442+0000", 2023-10-09T05:56:57.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031115719999999999 2023-10-09T05:56:57.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.599175+0000", 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.85073269900000004 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.604454+0000", 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0052789769999999998 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.611546+0000", 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070920089999999998 2023-10-09T05:56:57.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.625320+0000", 2023-10-09T05:56:57.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013774036 2023-10-09T05:56:57.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.625757+0000", 2023-10-09T05:56:57.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000436718 2023-10-09T05:56:57.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.645189+0000", 2023-10-09T05:56:57.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.01943224 2023-10-09T05:56:57.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.647314+0000", 2023-10-09T05:56:57.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021256909999999999 2023-10-09T05:56:57.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.654638+0000", 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0073239840000000004 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:114063 8.1f 8:fc8f46ab:::benchmark_data_smithi008_432048_object11121:head [delete] snapc 0=[] ondisk+write+known_if_redirected e319)", 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:53.731006+0000", 2023-10-09T05:56:57.684 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 423.85489963700002, 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0338361009999999, 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 114063 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731006+0000", 2023-10-09T05:56:57.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731006+0000", 2023-10-09T05:56:57.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731004+0000", 2023-10-09T05:56:57.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731011+0000", 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.379e-06 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731027+0000", 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5534999999999999e-05 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.731038+0000", 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1374e-05 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.792521+0000", 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.061482556000000001 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:53.796234+0000", 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037128450000000002 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.040431+0000", 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24419717599999999 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.044129+0000", 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036982790000000001 2023-10-09T05:56:57.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.687635+0000", 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.64350545100000001 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.696095+0000", 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0084602170000000008 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.711633+0000", 2023-10-09T05:56:57.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015537781000000001 2023-10-09T05:56:57.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.723121+0000", 2023-10-09T05:56:57.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011488414000000001 2023-10-09T05:56:57.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.737918+0000", 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014796785 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.756216+0000", 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018297979999999998 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.757979+0000", 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017634949999999999 2023-10-09T05:56:57.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.764842+0000", 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068624489999999996 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:114185 8.1f 8:f801fb94:::benchmark_data_smithi008_432048_object11243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e319)", 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:49:54.429094+0000", 2023-10-09T05:56:57.693 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 423.15681084599998, 2023-10-09T05:56:57.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97138411000000002, 2023-10-09T05:56:57.694 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.694 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.694 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.694 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.694 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.694 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 114185 2023-10-09T05:56:57.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.694 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429094+0000", 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429094+0000", 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429092+0000", 2023-10-09T05:56:57.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:57.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429102+0000", 2023-10-09T05:56:57.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3060000000000003e-06 2023-10-09T05:56:57.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429117+0000", 2023-10-09T05:56:57.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5579000000000001e-05 2023-10-09T05:56:57.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.429127+0000", 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0037999999999999e-05 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.518323+0000", 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.089195351000000006 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.524972+0000", 2023-10-09T05:56:57.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066490619999999999 2023-10-09T05:56:57.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.566402+0000", 2023-10-09T05:56:57.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.041430283999999998 2023-10-09T05:56:57.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T05:56:57.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:54.579147+0000", 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012745127 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.303210+0000", 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.72406284799999998 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.310075+0000", 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068648809999999998 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.320914+0000", 2023-10-09T05:56:57.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010838634999999999 2023-10-09T05:56:57.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.368949+0000", 2023-10-09T05:56:57.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.048035080000000001 2023-10-09T05:56:57.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.370199+0000", 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001250216 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.385700+0000", 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015500978 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.388642+0000", 2023-10-09T05:56:57.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029421830000000001 2023-10-09T05:56:57.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:49:55.400478+0000", 2023-10-09T05:56:57.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0118364 2023-10-09T05:56:57.702 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.702 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.702 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.702 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119611 8.1 8:835429e4:::benchmark_data_smithi008_432048_object16669:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e328)", 2023-10-09T05:56:57.703 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:18.039479+0000", 2023-10-09T05:56:57.703 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 399.54642571099998, 2023-10-09T05:56:57.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.748108534, 2023-10-09T05:56:57.703 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.703 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:56:57.703 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.703 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:56:57.703 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:56:57.703 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119611 2023-10-09T05:56:57.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.703 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039479+0000", 2023-10-09T05:56:57.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039479+0000", 2023-10-09T05:56:57.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039479+0000", 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039482+0000", 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4619999999999999e-06 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039488+0000", 2023-10-09T05:56:57.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2859999999999996e-06 2023-10-09T05:56:57.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.039494+0000", 2023-10-09T05:56:57.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1489999999999999e-06 2023-10-09T05:56:57.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.040837+0000", 2023-10-09T05:56:57.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013427409999999999 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.040848+0000", 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1238e-05 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.765099+0000", 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7242508669999999 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T05:56:57.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.768458+0000", 2023-10-09T05:56:57.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003359169 2023-10-09T05:56:57.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.785927+0000", 2023-10-09T05:56:57.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017468857000000001 2023-10-09T05:56:57.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.787588+0000", 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001661158 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45778 9.3 9:cd1ec889:::benchmark_data_smithi008_502820_object45777:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e373)", 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:46.945197+0000", 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 190.640707972, 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1527857639999999, 2023-10-09T05:56:57.709 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.710 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.710 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.710 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:57.710 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:57.710 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45778 2023-10-09T05:56:57.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.710 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.945197+0000", 2023-10-09T05:56:57.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.945197+0000", 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.945195+0000", 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.945225+0000", 2023-10-09T05:56:57.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.993e-05 2023-10-09T05:56:57.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.945240+0000", 2023-10-09T05:56:57.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4969e-05 2023-10-09T05:56:57.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.945252+0000", 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2279999999999999e-05 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.949089+0000", 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038366369999999999 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.949102+0000", 2023-10-09T05:56:57.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2615e-05 2023-10-09T05:56:57.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.082139+0000", 2023-10-09T05:56:57.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1330372280000001 2023-10-09T05:56:57.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.083504+0000", 2023-10-09T05:56:57.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013650590000000001 2023-10-09T05:56:57.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:56:57.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.083703+0000", 2023-10-09T05:56:57.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00019908600000000001 2023-10-09T05:56:57.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.088298+0000", 2023-10-09T05:56:57.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045952340000000001 2023-10-09T05:56:57.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.093057+0000", 2023-10-09T05:56:57.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0047591680000000003 2023-10-09T05:56:57.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.095347+0000", 2023-10-09T05:56:57.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022898300000000001 2023-10-09T05:56:57.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.095417+0000", 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9726000000000003e-05 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.097983+0000", 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00256609 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.717 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:56930 9.4 9:208f0955:::benchmark_data_smithi008_502820_object56929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:04.180242+0000", 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 173.405663022, 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99321452499999996, 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:57.718 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56930 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180242+0000", 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180242+0000", 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180242+0000", 2023-10-09T05:56:57.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:57.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180263+0000", 2023-10-09T05:56:57.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1599000000000001e-05 2023-10-09T05:56:57.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180270+0000", 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8889999999999996e-06 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180276+0000", 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5199999999999997e-06 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.181919+0000", 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016433559999999999 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.181931+0000", 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1698999999999999e-05 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.166507+0000", 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.98457661200000002 2023-10-09T05:56:57.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.167509+0000", 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010012490000000001 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.167615+0000", 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000106848 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173305+0000", 2023-10-09T05:56:57.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0056890309999999998 2023-10-09T05:56:57.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173358+0000", 2023-10-09T05:56:57.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3087999999999997e-05 2023-10-09T05:56:57.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173383+0000", 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5792e-05 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173420+0000", 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7038e-05 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173457+0000", 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6229000000000002e-05 2023-10-09T05:56:57.725 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.726 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.726 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.726 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:56935 9.4 9:2e5431a0:::benchmark_data_smithi008_502820_object56934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T05:56:57.726 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:04.187552+0000", 2023-10-09T05:56:57.726 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 173.39835327500001, 2023-10-09T05:56:57.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99016123, 2023-10-09T05:56:57.726 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.726 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56935 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187552+0000", 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187552+0000", 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187551+0000", 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.728 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187573+0000", 2023-10-09T05:56:57.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2022e-05 2023-10-09T05:56:57.729 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.729 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187583+0000", 2023-10-09T05:56:57.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0207000000000001e-05 2023-10-09T05:56:57.729 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.729 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187590+0000", 2023-10-09T05:56:57.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2949999999999996e-06 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.189845+0000", 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002255047 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.189856+0000", 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1161999999999999e-05 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.169499+0000", 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97964239200000003 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.169675+0000", 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017621499999999999 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.169851+0000", 2023-10-09T05:56:57.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017612199999999999 2023-10-09T05:56:57.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173502+0000", 2023-10-09T05:56:57.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036512459999999999 2023-10-09T05:56:57.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173540+0000", 2023-10-09T05:56:57.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7245e-05 2023-10-09T05:56:57.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177611+0000", 2023-10-09T05:56:57.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040713750000000003 2023-10-09T05:56:57.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177670+0000", 2023-10-09T05:56:57.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9184000000000001e-05 2023-10-09T05:56:57.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177713+0000", 2023-10-09T05:56:57.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2997000000000002e-05 2023-10-09T05:56:57.734 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.734 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.734 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.734 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:56945 9.4 9:25604571:::benchmark_data_smithi008_502820_object56944:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T05:56:57.734 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:04.198697+0000", 2023-10-09T05:56:57.735 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 173.387207843, 2023-10-09T05:56:57.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97919550899999996, 2023-10-09T05:56:57.735 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.735 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.735 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.735 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:57.735 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:57.735 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56945 2023-10-09T05:56:57.735 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.735 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198697+0000", 2023-10-09T05:56:57.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198697+0000", 2023-10-09T05:56:57.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198697+0000", 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198720+0000", 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3595e-05 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198728+0000", 2023-10-09T05:56:57.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9619999999999993e-06 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198738+0000", 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7280000000000007e-06 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198805+0000", 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7163999999999996e-05 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198816+0000", 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1018e-05 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.171495+0000", 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97267913900000003 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.171653+0000", 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000157152 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.171818+0000", 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016535700000000001 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173547+0000", 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017288850000000001 2023-10-09T05:56:57.740 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173574+0000", 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.756e-05 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177808+0000", 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042333329999999997 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.741 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.742 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177854+0000", 2023-10-09T05:56:57.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6777000000000003e-05 2023-10-09T05:56:57.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.742 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.742 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.742 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177893+0000", 2023-10-09T05:56:57.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8414000000000002e-05 2023-10-09T05:56:57.742 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.742 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.742 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.743 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:56948 9.4 9:2ebab9cb:::benchmark_data_smithi008_502820_object56947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T05:56:57.743 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:04.209232+0000", 2023-10-09T05:56:57.743 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 173.37667341400001, 2023-10-09T05:56:57.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97360594499999997, 2023-10-09T05:56:57.743 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.743 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.743 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.743 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:57.743 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56948 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.209232+0000", 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.209232+0000", 2023-10-09T05:56:57.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.209230+0000", 2023-10-09T05:56:57.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:57.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.209264+0000", 2023-10-09T05:56:57.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3652999999999998e-05 2023-10-09T05:56:57.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.211202+0000", 2023-10-09T05:56:57.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019386049999999999 2023-10-09T05:56:57.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.211216+0000", 2023-10-09T05:56:57.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3991e-05 2023-10-09T05:56:57.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.211358+0000", 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000142161 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.211371+0000", 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2241e-05 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173589+0000", 2023-10-09T05:56:57.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96221823699999998 2023-10-09T05:56:57.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173722+0000", 2023-10-09T05:56:57.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000133311 2023-10-09T05:56:57.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173878+0000", 2023-10-09T05:56:57.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015562099999999999 2023-10-09T05:56:57.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177911+0000", 2023-10-09T05:56:57.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040336369999999996 2023-10-09T05:56:57.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182682+0000", 2023-10-09T05:56:57.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004770364 2023-10-09T05:56:57.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.750 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.750 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.750 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182729+0000", 2023-10-09T05:56:57.750 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6730000000000002e-05 2023-10-09T05:56:57.750 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.750 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.750 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.750 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182791+0000", 2023-10-09T05:56:57.750 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2329999999999994e-05 2023-10-09T05:56:57.750 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.750 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182838+0000", 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6848999999999998e-05 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:56965 9.4 9:22816604:::benchmark_data_smithi008_502820_object56964:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:04.236294+0000", 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 173.34961074899999, 2023-10-09T05:56:57.751 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94675013200000002, 2023-10-09T05:56:57.752 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.752 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:57.752 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.752 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:57.752 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:57.752 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56965 2023-10-09T05:56:57.752 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.752 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.752 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.752 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.752 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236294+0000", 2023-10-09T05:56:57.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.753 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.753 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.753 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236294+0000", 2023-10-09T05:56:57.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.753 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.753 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.753 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236293+0000", 2023-10-09T05:56:57.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236321+0000", 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8558999999999999e-05 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236338+0000", 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7093000000000001e-05 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236348+0000", 2023-10-09T05:56:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9380000000000004e-06 2023-10-09T05:56:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236420+0000", 2023-10-09T05:56:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1601999999999996e-05 2023-10-09T05:56:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236434+0000", 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3636e-05 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.175615+0000", 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93918106800000001 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.175753+0000", 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000138543 2023-10-09T05:56:57.756 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.757 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:56:57.757 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.175954+0000", 2023-10-09T05:56:57.757 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020097299999999999 2023-10-09T05:56:57.757 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.757 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:56:57.757 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.180332+0000", 2023-10-09T05:56:57.757 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0043779500000000002 2023-10-09T05:56:57.757 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182893+0000", 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025605839999999999 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182958+0000", 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5178000000000005e-05 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:56:57.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.183006+0000", 2023-10-09T05:56:57.759 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7645000000000002e-05 2023-10-09T05:56:57.759 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.759 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.759 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.759 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.183045+0000", 2023-10-09T05:56:57.759 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8965999999999999e-05 2023-10-09T05:56:57.759 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.759 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.759 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.759 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.759 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77620 9.14 9:2ff0425c:::benchmark_data_smithi008_502820_object77619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:36.817602+0000", 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 140.76830294000001, 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.935633622, 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77620 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.760 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:56:57.761 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.761 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:56:57.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817602+0000", 2023-10-09T05:56:57.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.761 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.761 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.761 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:56:57.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817602+0000", 2023-10-09T05:56:57.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:56:57.761 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.761 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817600+0000", 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817634+0000", 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4471000000000003e-05 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:56:57.762 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817660+0000", 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5888e-05 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817673+0000", 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2499e-05 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.912688+0000", 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.095015316000000002 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.763 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.764 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:56:57.764 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.912700+0000", 2023-10-09T05:56:57.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.241e-05 2023-10-09T05:56:57.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.764 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.764 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.764 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.749377+0000", 2023-10-09T05:56:57.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83667673600000003 2023-10-09T05:56:57.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.764 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.764 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.749824+0000", 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00044703499999999998 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.753124+0000", 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032999209999999999 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.753236+0000", 2023-10-09T05:56:57.765 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000111821 2023-10-09T05:56:57.766 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.766 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.766 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.766 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:56:57.766 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:56:57.766 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:56:57.766 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:57.767 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:57.799 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:57.853 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:57.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:57.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:57.958 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:57.989 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:58.005 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:56:58.041 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:58.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:58.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:58.147 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:58.173 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:58.219 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:58.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:58.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:58.326 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:58.352 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:58.395 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:58.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:58.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:58.500 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:58.525 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:58 smithi191 ceph-mon[161669]: pgmap v1287: 99 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 46115/181050 objects degraded (25.471%); 22704/181050 objects misplaced (12.540%); 510 KiB/s, 9 objects/s recovering 2023-10-09T05:56:58.526 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 46115/181050 objects degraded (25.471%), 26 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:58.528 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:58.574 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:58.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:58.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:58.679 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:58.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:58 smithi008 ceph-mon[184418]: pgmap v1287: 99 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 46115/181050 objects degraded (25.471%); 22704/181050 objects misplaced (12.540%); 510 KiB/s, 9 objects/s recovering 2023-10-09T05:56:58.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 46115/181050 objects degraded (25.471%), 26 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:58.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:58 smithi008 ceph-mon[191227]: pgmap v1287: 99 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 46115/181050 objects degraded (25.471%); 22704/181050 objects misplaced (12.540%); 510 KiB/s, 9 objects/s recovering 2023-10-09T05:56:58.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 46115/181050 objects degraded (25.471%), 26 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:58.711 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:58.936 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:58.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:59.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:59.066 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:59.111 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:59.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:59.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:59.151 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:56:59.216 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:59.242 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:59.293 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:59.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:59.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:59.308 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:59.400 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:59.438 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:59.490 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:59.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:59.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:59.594 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:59.621 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:59.665 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:59.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:59.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:59.771 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:59.797 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:59.846 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:59.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:59.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:59.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:00.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:00.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:00.151 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:00.196 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:00.200 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:59 smithi191 ceph-mon[161669]: pgmap v1288: 99 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 46115/181008 objects degraded (25.477%); 22700/181008 objects misplaced (12.541%); 408 KiB/s, 7 objects/s recovering 2023-10-09T05:57:00.200 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:00.200 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:56:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:00.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:00.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:59 smithi008 ceph-mon[184418]: pgmap v1288: 99 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 46115/181008 objects degraded (25.477%); 22700/181008 objects misplaced (12.541%); 408 KiB/s, 7 objects/s recovering 2023-10-09T05:57:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:56:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:59 smithi008 ceph-mon[191227]: pgmap v1288: 99 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 46115/181008 objects degraded (25.477%); 22700/181008 objects misplaced (12.541%); 408 KiB/s, 7 objects/s recovering 2023-10-09T05:57:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:56:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:00.302 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:00.332 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:00.684 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:00.688 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:57:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:57:00] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.33.4" 2023-10-09T05:57:00.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:00.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:00.790 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:00.815 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:00.861 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:00.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:00.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:00.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:00.996 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:01.040 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:01.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:01.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:01.146 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:01.181 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:01.233 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:01.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:01.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:01.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:01.366 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:01.416 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:01.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:01.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:01.524 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:01.552 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:01.599 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:01.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:01.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:01.637 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:57:01.638 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:57:01.639 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:57:01.639 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:57:01.639 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:57:01.639 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:57:01.639 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:57:01.639 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:57:01.639 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:57:01.703 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:01.731 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:01.786 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:01.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:01.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:01.892 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:01.928 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:02.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:01 smithi191 ceph-mon[161669]: pgmap v1289: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 15 op/s; 45015/180975 objects degraded (24.874%); 22482/180975 objects misplaced (12.423%); 4.0 MiB/s, 69 objects/s recovering 2023-10-09T05:57:02.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:01 smithi008 ceph-mon[184418]: pgmap v1289: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 15 op/s; 45015/180975 objects degraded (24.874%); 22482/180975 objects misplaced (12.423%); 4.0 MiB/s, 69 objects/s recovering 2023-10-09T05:57:02.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:01 smithi008 ceph-mon[191227]: pgmap v1289: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 15 op/s; 45015/180975 objects degraded (24.874%); 22482/180975 objects misplaced (12.423%); 4.0 MiB/s, 69 objects/s recovering 2023-10-09T05:57:03.723 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:03.805 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:03.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:03.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:03.942 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:03.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:03 smithi008 ceph-mon[184418]: pgmap v1290: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 15 op/s; 44727/180750 objects degraded (24.745%); 24235/180750 objects misplaced (13.408%); 4.6 MiB/s, 79 objects/s recovering 2023-10-09T05:57:03.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 44727/180750 objects degraded (24.745%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:03 smithi008 ceph-mon[191227]: pgmap v1290: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 15 op/s; 44727/180750 objects degraded (24.745%); 24235/180750 objects misplaced (13.408%); 4.6 MiB/s, 79 objects/s recovering 2023-10-09T05:57:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 44727/180750 objects degraded (24.745%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:03.992 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:03.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:03.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:04.076 INFO:teuthology.orchestra.run.smithi008.stdout:moved 0 / 297 (0%) 2023-10-09T05:57:04.076 INFO:teuthology.orchestra.run.smithi008.stdout:avg 42.4286 2023-10-09T05:57:04.076 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 5.82745 -> 5.82745 (expected baseline 6.03053) 2023-10-09T05:57:04.077 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.5 with 36 -> 36 pgs (0.848485 -> 0.848485 * mean) 2023-10-09T05:57:04.077 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.7 with 54 -> 54 pgs (1.27273 -> 1.27273 * mean) 2023-10-09T05:57:04.077 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:57:04.077 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T05:57:04.077 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 3 2023-10-09T05:57:04.077 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:57:04.077 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0265 2023-10-09T05:57:04.077 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.2646 2023-10-09T05:57:04.084 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T05:57:04.098 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:04.142 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:04.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:03 smithi191 ceph-mon[161669]: pgmap v1290: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 15 op/s; 44727/180750 objects degraded (24.745%); 24235/180750 objects misplaced (13.408%); 4.6 MiB/s, 79 objects/s recovering 2023-10-09T05:57:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 44727/180750 objects degraded (24.745%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:05.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:04 smithi008 ceph-mon[184418]: from='client.56386 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:04 smithi008 ceph-mon[191227]: from='client.56386 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:05.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:04 smithi191 ceph-mon[161669]: from='client.56386 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:05.712 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:57:05.954 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:57:05.974 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:05.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:05.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:06.078 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:06.101 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:05 smithi008 ceph-mon[191227]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 22 op/s; 44694/180513 objects degraded (24.759%); 23556/180513 objects misplaced (13.049%); 8.6 MiB/s, 167 objects/s recovering 2023-10-09T05:57:06.101 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:05 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1054455799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:06.101 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:05 smithi008 ceph-mon[184418]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 22 op/s; 44694/180513 objects degraded (24.759%); 23556/180513 objects misplaced (13.049%); 8.6 MiB/s, 167 objects/s recovering 2023-10-09T05:57:06.101 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:05 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1054455799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:06.128 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:06.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:05 smithi191 ceph-mon[161669]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 22 op/s; 44694/180513 objects degraded (24.759%); 23556/180513 objects misplaced (13.049%); 8.6 MiB/s, 167 objects/s recovering 2023-10-09T05:57:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:05 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1054455799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:07.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[184418]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[184418]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1054455799' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:57:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[184418]: osdmap e433: 8 total, 8 up, 7 in 2023-10-09T05:57:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[184418]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[184418]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:07.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[191227]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[191227]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1054455799' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:57:07.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[191227]: osdmap e433: 8 total, 8 up, 7 in 2023-10-09T05:57:07.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[191227]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:06 smithi008 ceph-mon[191227]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.294 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T05:57:07.294 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-10-09T05:57:07.296 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:07.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:07.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:07.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:06 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:06 smithi191 ceph-mon[161669]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:06 smithi191 ceph-mon[161669]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1054455799' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:57:07.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:06 smithi191 ceph-mon[161669]: osdmap e433: 8 total, 8 up, 7 in 2023-10-09T05:57:07.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:06 smithi191 ceph-mon[161669]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:06 smithi191 ceph-mon[161669]: failed to encode map e433 with expected crc 2023-10-09T05:57:07.400 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:07.427 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:07.474 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:07.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:07.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:07.580 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:07.608 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:07.659 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:07.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:07.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:07.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:07.795 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:07.855 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:07.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:07.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:07.964 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:08.002 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:08.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:07 smithi191 ceph-mon[161669]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 50 op/s; 47158/179541 objects degraded (26.266%); 21757/179541 objects misplaced (12.118%); 9.4 MiB/s, 180 objects/s recovering 2023-10-09T05:57:08.078 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:08.080 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:08.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:08.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:08.125 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:08.187 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:08.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:07 smithi008 ceph-mon[184418]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 50 op/s; 47158/179541 objects degraded (26.266%); 21757/179541 objects misplaced (12.118%); 9.4 MiB/s, 180 objects/s recovering 2023-10-09T05:57:08.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:07 smithi008 ceph-mon[191227]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 50 op/s; 47158/179541 objects degraded (26.266%); 21757/179541 objects misplaced (12.118%); 9.4 MiB/s, 180 objects/s recovering 2023-10-09T05:57:08.225 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:08.282 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:57:08.282 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:57:08.283 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T05:57:08.283 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:57:08.568 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:08.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:08.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:08.675 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:08.707 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:08.763 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:08.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:08.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:08.869 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:08.904 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:08.969 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:08.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 47158/179541 objects degraded (26.266%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:08.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:08.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:09.076 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:09.103 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:09.151 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:09.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:09.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 47158/179541 objects degraded (26.266%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 47158/179541 objects degraded (26.266%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:09.257 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:09.292 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:09.450 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:09.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:09.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:09.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:09.583 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:09.628 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:09.632 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:57:09.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:09.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:09.734 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:09.762 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:09.810 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:09.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:09.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:09.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:09.950 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:09.999 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:10.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:10.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:10.108 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:10.137 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:10.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:10 smithi008 ceph-mon[184418]: pgmap v1294: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 75 op/s; 47158/178746 objects degraded (26.383%); 21736/178746 objects misplaced (12.160%); 9.4 MiB/s, 180 objects/s recovering 2023-10-09T05:57:10.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:10 smithi008 ceph-mon[184418]: failed to encode map e434 with expected crc 2023-10-09T05:57:10.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:10 smithi008 ceph-mon[184418]: failed to encode map e434 with expected crc 2023-10-09T05:57:10.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:10 smithi008 ceph-mon[184418]: osdmap e434: 8 total, 8 up, 7 in 2023-10-09T05:57:10.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:10 smithi008 ceph-mon[184418]: failed to encode map e434 with expected crc 2023-10-09T05:57:10.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:10 smithi008 ceph-mon[191227]: pgmap v1294: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 75 op/s; 47158/178746 objects degraded (26.383%); 21736/178746 objects misplaced (12.160%); 9.4 MiB/s, 180 objects/s recovering 2023-10-09T05:57:10.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:10 smithi008 ceph-mon[191227]: failed to encode map e434 with expected crc 2023-10-09T05:57:10.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:10 smithi008 ceph-mon[191227]: failed to encode map e434 with expected crc 2023-10-09T05:57:10.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:10 smithi008 ceph-mon[191227]: osdmap e434: 8 total, 8 up, 7 in 2023-10-09T05:57:10.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:10 smithi008 ceph-mon[191227]: failed to encode map e434 with expected crc 2023-10-09T05:57:10.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:10 smithi191 ceph-mon[161669]: pgmap v1294: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 75 op/s; 47158/178746 objects degraded (26.383%); 21736/178746 objects misplaced (12.160%); 9.4 MiB/s, 180 objects/s recovering 2023-10-09T05:57:10.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:10 smithi191 ceph-mon[161669]: failed to encode map e434 with expected crc 2023-10-09T05:57:10.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:10 smithi191 ceph-mon[161669]: failed to encode map e434 with expected crc 2023-10-09T05:57:10.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:10 smithi191 ceph-mon[161669]: osdmap e434: 8 total, 8 up, 7 in 2023-10-09T05:57:10.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:10 smithi191 ceph-mon[161669]: failed to encode map e434 with expected crc 2023-10-09T05:57:10.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:57:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:57:10] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.33.4" 2023-10-09T05:57:11.220 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:57:11.304 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:11.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:11.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:11.409 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:11.454 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:11 smithi008 ceph-mon[184418]: failed to encode map e435 with expected crc 2023-10-09T05:57:11.455 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:11 smithi008 ceph-mon[184418]: failed to encode map e435 with expected crc 2023-10-09T05:57:11.455 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:11 smithi008 ceph-mon[184418]: osdmap e435: 8 total, 8 up, 7 in 2023-10-09T05:57:11.455 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:11 smithi008 ceph-mon[184418]: failed to encode map e435 with expected crc 2023-10-09T05:57:11.455 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:11 smithi008 ceph-mon[191227]: failed to encode map e435 with expected crc 2023-10-09T05:57:11.455 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:11 smithi008 ceph-mon[191227]: failed to encode map e435 with expected crc 2023-10-09T05:57:11.455 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:11 smithi008 ceph-mon[191227]: osdmap e435: 8 total, 8 up, 7 in 2023-10-09T05:57:11.456 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:11 smithi008 ceph-mon[191227]: failed to encode map e435 with expected crc 2023-10-09T05:57:11.458 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:11 smithi191 ceph-mon[161669]: failed to encode map e435 with expected crc 2023-10-09T05:57:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:11 smithi191 ceph-mon[161669]: failed to encode map e435 with expected crc 2023-10-09T05:57:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:11 smithi191 ceph-mon[161669]: osdmap e435: 8 total, 8 up, 7 in 2023-10-09T05:57:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:11 smithi191 ceph-mon[161669]: failed to encode map e435 with expected crc 2023-10-09T05:57:12.196 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:12.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:12.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:12.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:12.332 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:12.388 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:12.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:12.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:12.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:12 smithi008 ceph-mon[184418]: pgmap v1297: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 170 op/s; 46997/177450 objects degraded (26.485%); 20828/177450 objects misplaced (11.737%); 11 MiB/s, 248 objects/s recovering 2023-10-09T05:57:12.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:12 smithi008 ceph-mon[191227]: pgmap v1297: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 170 op/s; 46997/177450 objects degraded (26.485%); 20828/177450 objects misplaced (11.737%); 11 MiB/s, 248 objects/s recovering 2023-10-09T05:57:12.495 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:12.526 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:12 smithi191 ceph-mon[161669]: pgmap v1297: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 170 op/s; 46997/177450 objects degraded (26.485%); 20828/177450 objects misplaced (11.737%); 11 MiB/s, 248 objects/s recovering 2023-10-09T05:57:12.532 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:12.598 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:12.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:12.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:12.714 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:12.733 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:12.792 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:12.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:12.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:12.884 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:57:12.899 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:12.912 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:57:12.947 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:57:12.948 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:12.978 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:57:12.993 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:57:13.015 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:57:13.039 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:57:13.069 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:57:13.071 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:13.084 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:57:13.084 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:57:13.084 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:57:13.085 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:57:13.085 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:57:13.085 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:57:13.085 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:57:13.085 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:57:13.139 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:13.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:13.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:13.246 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:13.292 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:13.437 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1198398117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:13.437 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 46997/177450 objects degraded (26.485%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:13.438 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:13 smithi008 ceph-mon[184418]: osdmap e436: 8 total, 8 up, 7 in 2023-10-09T05:57:13.442 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1198398117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:13.442 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 46997/177450 objects degraded (26.485%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:13.442 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:13 smithi008 ceph-mon[191227]: osdmap e436: 8 total, 8 up, 7 in 2023-10-09T05:57:13.449 INFO:teuthology.orchestra.run.smithi008.stdout:moved 0 / 297 (0%) 2023-10-09T05:57:13.449 INFO:teuthology.orchestra.run.smithi008.stdout:avg 42.4286 2023-10-09T05:57:13.449 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 5.82745 -> 5.82745 (expected baseline 6.03053) 2023-10-09T05:57:13.449 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.5 with 36 -> 36 pgs (0.848485 -> 0.848485 * mean) 2023-10-09T05:57:13.450 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.7 with 54 -> 54 pgs (1.27273 -> 1.27273 * mean) 2023-10-09T05:57:13.450 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:57:13.450 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T05:57:13.450 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 3 2023-10-09T05:57:13.450 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:57:13.450 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0263 2023-10-09T05:57:13.450 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.2629 2023-10-09T05:57:13.452 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T05:57:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1198398117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 46997/177450 objects degraded (26.485%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:13 smithi191 ceph-mon[161669]: osdmap e436: 8 total, 8 up, 7 in 2023-10-09T05:57:13.760 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:13.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:13.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:13.891 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:13.935 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:13.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:13.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:14.041 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:14.072 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:14.233 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:14 smithi191 ceph-mon[161669]: pgmap v1298: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 170 op/s; 46997/176541 objects degraded (26.621%); 20759/176541 objects misplaced (11.759%); 9.7 MiB/s, 225 objects/s recovering 2023-10-09T05:57:14.233 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:14 smithi191 ceph-mon[161669]: failed to encode map e436 with expected crc 2023-10-09T05:57:14.233 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:14 smithi191 ceph-mon[161669]: failed to encode map e436 with expected crc 2023-10-09T05:57:14.233 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:14.233 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:14.233 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:14 smithi191 ceph-mon[161669]: osdmap e437: 8 total, 8 up, 7 in 2023-10-09T05:57:14.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[184418]: pgmap v1298: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 170 op/s; 46997/176541 objects degraded (26.621%); 20759/176541 objects misplaced (11.759%); 9.7 MiB/s, 225 objects/s recovering 2023-10-09T05:57:14.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[184418]: failed to encode map e436 with expected crc 2023-10-09T05:57:14.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[184418]: failed to encode map e436 with expected crc 2023-10-09T05:57:14.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:14.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:14.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[184418]: osdmap e437: 8 total, 8 up, 7 in 2023-10-09T05:57:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[191227]: pgmap v1298: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 170 op/s; 46997/176541 objects degraded (26.621%); 20759/176541 objects misplaced (11.759%); 9.7 MiB/s, 225 objects/s recovering 2023-10-09T05:57:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[191227]: failed to encode map e436 with expected crc 2023-10-09T05:57:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[191227]: failed to encode map e436 with expected crc 2023-10-09T05:57:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:14.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:14 smithi008 ceph-mon[191227]: osdmap e437: 8 total, 8 up, 7 in 2023-10-09T05:57:14.675 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:14.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:14.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:14.780 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:14.810 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:15.533 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:15 smithi008 ceph-mon[184418]: from='client.46517 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:15.533 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:15 smithi008 ceph-mon[184418]: failed to encode map e437 with expected crc 2023-10-09T05:57:15.533 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:15 smithi008 ceph-mon[184418]: failed to encode map e437 with expected crc 2023-10-09T05:57:15.533 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:15 smithi008 ceph-mon[184418]: failed to encode map e437 with expected crc 2023-10-09T05:57:15.534 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:15 smithi008 ceph-mon[191227]: from='client.46517 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:15.534 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:15 smithi008 ceph-mon[191227]: failed to encode map e437 with expected crc 2023-10-09T05:57:15.534 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:15 smithi008 ceph-mon[191227]: failed to encode map e437 with expected crc 2023-10-09T05:57:15.534 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:15 smithi008 ceph-mon[191227]: failed to encode map e437 with expected crc 2023-10-09T05:57:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:15 smithi191 ceph-mon[161669]: from='client.46517 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:15 smithi191 ceph-mon[161669]: failed to encode map e437 with expected crc 2023-10-09T05:57:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:15 smithi191 ceph-mon[161669]: failed to encode map e437 with expected crc 2023-10-09T05:57:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:15 smithi191 ceph-mon[161669]: failed to encode map e437 with expected crc 2023-10-09T05:57:16.025 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:16.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:16.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:16.131 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:16.168 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:16.216 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:16.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:16.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:16.323 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:16.350 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:16.393 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:16.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:16.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:16.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:16 smithi008 ceph-mon[184418]: pgmap v1301: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 46949/176259 objects degraded (26.636%); 20807/176259 objects misplaced (11.805%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T05:57:16.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:16 smithi008 ceph-mon[191227]: pgmap v1301: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 46949/176259 objects degraded (26.636%); 20807/176259 objects misplaced (11.805%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T05:57:16.498 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:16.521 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:16 smithi191 ceph-mon[161669]: pgmap v1301: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 46949/176259 objects degraded (26.636%); 20807/176259 objects misplaced (11.805%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T05:57:16.524 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:16.573 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:16.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:16.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:16.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:16.679 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:57:16.705 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:16.752 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:16.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:16.855 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:16.897 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:17.156 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:17.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:17.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:17.182 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:57:17.182 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:57:17.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:17.290 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:17.335 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:17.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:17.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:17.441 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:17.475 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:17.630 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:17.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:17.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:17.735 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:17.764 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:17.817 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:17.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:17.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:17.922 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:17.953 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:18.211 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:18.219 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:18.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:18.247 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:57:18.254 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:57:18.326 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:18.354 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:18.409 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:18.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:18 smithi008 ceph-mon[184418]: pgmap v1302: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 46861/174966 objects degraded (26.783%); 18966/174966 objects misplaced (10.840%); 18 MiB/s, 292 objects/s recovering 2023-10-09T05:57:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1156556202' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 46861/174966 objects degraded (26.783%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:18.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:18 smithi008 ceph-mon[191227]: pgmap v1302: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 46861/174966 objects degraded (26.783%); 18966/174966 objects misplaced (10.840%); 18 MiB/s, 292 objects/s recovering 2023-10-09T05:57:18.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1156556202' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:18.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 46861/174966 objects degraded (26.783%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:18.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:18.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:18 smithi191 ceph-mon[161669]: pgmap v1302: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 46861/174966 objects degraded (26.783%); 18966/174966 objects misplaced (10.840%); 18 MiB/s, 292 objects/s recovering 2023-10-09T05:57:18.547 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1156556202' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:18.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 46861/174966 objects degraded (26.783%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:18.550 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:18.595 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:18.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:18.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:18.701 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:18.736 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:19.091 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:19.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:19.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:19.197 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:19.231 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:19.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:19 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:19 smithi191 ceph-mon[161669]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:19 smithi191 ceph-mon[161669]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:19 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1156556202' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:19 smithi191 ceph-mon[161669]: osdmap e438: 8 total, 8 up, 7 in 2023-10-09T05:57:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:19 smithi191 ceph-mon[161669]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:19 smithi191 ceph-mon[161669]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[184418]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[184418]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1156556202' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[184418]: osdmap e438: 8 total, 8 up, 7 in 2023-10-09T05:57:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[184418]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[184418]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[191227]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[191227]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1156556202' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:19.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[191227]: osdmap e438: 8 total, 8 up, 7 in 2023-10-09T05:57:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[191227]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:19 smithi008 ceph-mon[191227]: failed to encode map e438 with expected crc 2023-10-09T05:57:19.937 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:19.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:19.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:20.003 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:20.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:20.069 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:20.117 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:20.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:20.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:20.225 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:20.275 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:20.404 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:20.406 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:20 smithi008 ceph-mon[184418]: pgmap v1304: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 46861/174171 objects degraded (26.905%); 18930/174171 objects misplaced (10.869%); 19 MiB/s, 303 objects/s recovering 2023-10-09T05:57:20.407 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:20 smithi008 ceph-mon[191227]: pgmap v1304: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 46861/174171 objects degraded (26.905%); 18930/174171 objects misplaced (10.869%); 19 MiB/s, 303 objects/s recovering 2023-10-09T05:57:20.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:20.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:20.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:20.536 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:20 smithi191 ceph-mon[161669]: pgmap v1304: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 46861/174171 objects degraded (26.905%); 18930/174171 objects misplaced (10.869%); 19 MiB/s, 303 objects/s recovering 2023-10-09T05:57:20.539 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:20.591 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:20.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:20.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:20.696 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:20.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:57:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:57:20] "GET /metrics HTTP/1.1" 200 36606 "" "Prometheus/2.33.4" 2023-10-09T05:57:20.730 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:20.786 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:20.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:20.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:20.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:20.932 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:20.993 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:20.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:20.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:21.030 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 5, 6, 1, 7, 3] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:57:21.030 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:57:21.030 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T05:57:21.030 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:57:21.030 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:57:21.031 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T05:57:21.098 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:21.131 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:21.348 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:21.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:21.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:21.455 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:21.489 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:21.574 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:57:21.605 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:57:21.628 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:57:21.652 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:57:21.678 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:57:21.711 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:57:21.711 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:21.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:21.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:21.744 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:57:21.751 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:57:21.775 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:57:21.779 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:57:21.779 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:57:21.779 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:57:21.779 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:57:21.780 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:57:21.780 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:57:21.780 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:57:21.780 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:57:21.822 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:21.880 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:21.930 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:22.013 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:57:22.014 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:57:22.014 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:57:22.014 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:57:22.014 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.014 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:75147 8.a 8:5b312b1e:::benchmark_data_smithi008_432048_object75146:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-09T05:57:22.014 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:31.714915+0000", 2023-10-09T05:57:22.014 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.29092754500005, 2023-10-09T05:57:22.014 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30074143399999997, 2023-10-09T05:57:22.014 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.014 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 75147 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714915+0000", 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.015 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714915+0000", 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714914+0000", 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.016 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714936+0000", 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2353000000000001e-05 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714945+0000", 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1670000000000005e-06 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.714951+0000", 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5999999999999997e-06 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.017 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716442+0000", 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001491514 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716558+0000", 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011547799999999999 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:57:22.018 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716652+0000", 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4291999999999998e-05 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.014083+0000", 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.297430892 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015543+0000", 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014593900000000001 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.019 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015556+0000", 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3715e-05 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015613+0000", 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6848000000000002e-05 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015657+0000", 2023-10-09T05:57:22.020 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3412999999999999e-05 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:75150 8.3 e296/285 8:c720d3fb:::benchmark_data_smithi008_432048_object75149:head v 296'4679, mlcod=296'4677)", 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:31.716949+0000", 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.288893981, 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30161086100000001, 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.021 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716949+0000", 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716949+0000", 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.022 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716948+0000", 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.716999+0000", 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0831999999999997e-05 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011201+0000", 2023-10-09T05:57:22.023 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29420217900000001 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011219+0000", 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7883000000000001e-05 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011269+0000", 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9586999999999998e-05 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.024 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.011301+0000", 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1902000000000001e-05 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.018513+0000", 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0072121429999999999 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.018559+0000", 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6836999999999999e-05 2023-10-09T05:57:22.025 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45782.0:75147 8.a e296/285 ondisk, result = 0)", 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:31.718538+0000", 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.28730459600001, 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29700873900000002, 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.026 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718538+0000", 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718538+0000", 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.027 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718537+0000", 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:31.718541+0000", 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7900000000000001e-06 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.012365+0000", 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29382405900000003 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.028 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.029 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.029 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.012376+0000", 2023-10-09T05:57:22.029 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0210000000000001e-05 2023-10-09T05:57:22.029 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.029 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.029 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.029 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015527+0000", 2023-10-09T05:57:22.029 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031513660000000001 2023-10-09T05:57:22.029 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.029 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.029 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015536+0000", 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1689999999999998e-06 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:32.015547+0000", 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0713000000000001e-05 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:90579 8.a 8:55cc0eec:::benchmark_data_smithi008_432048_object90578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:49.809093+0000", 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 512.19674992900002, 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29641294499999998, 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 90579 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.031 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809093+0000", 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809093+0000", 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809091+0000", 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809117+0000", 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6276999999999999e-05 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809142+0000", 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4307e-05 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809153+0000", 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1561e-05 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809289+0000", 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000136398 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809442+0000", 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000152211 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809589+0000", 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014731999999999999 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.819043+0000", 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0094543100000000005 2023-10-09T05:57:22.035 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.828230+0000", 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0091865950000000005 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105413+0000", 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27718348900000001 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.036 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105464+0000", 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0207000000000002e-05 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105506+0000", 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2001e-05 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.037 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:90580 8.a 8:5146ec73:::benchmark_data_smithi008_432048_object90579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:49.809186+0000", 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 512.19665709000003, 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29646979400000001, 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 90580 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809186+0000", 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809186+0000", 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809185+0000", 2023-10-09T05:57:22.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809210+0000", 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5360000000000001e-05 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809224+0000", 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3818000000000001e-05 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.809232+0000", 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7230000000000004e-06 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.812756+0000", 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035239479999999998 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.812903+0000", 2023-10-09T05:57:22.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000147274 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.813046+0000", 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000142492 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.819054+0000", 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0060084450000000003 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:49.828260+0000", 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0092060430000000006 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105560+0000", 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27729968900000002 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105616+0000", 2023-10-09T05:57:22.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6239999999999997e-05 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:50.105655+0000", 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9382999999999999e-05 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop_reply(client.45782.0:168592 8.1a e352/349 ondisk, result = 0)", 2023-10-09T05:57:22.044 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:53.754226+0000", 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 328.25161644500002, 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30718891300000001, 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754226+0000", 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754226+0000", 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754225+0000", 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754233+0000", 2023-10-09T05:57:22.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3259999999999996e-06 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754249+0000", 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5512e-05 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754258+0000", 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3989999999999993e-06 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754302+0000", 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4023999999999997e-05 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.754311+0000", 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5580000000000001e-06 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061415+0000", 2023-10-09T05:57:22.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.307104341 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:168594 8.f e352/339 8:f6b60d9c:::benchmark_data_smithi008_432048_object65652:head v 352'8443, mlcod=293'4003)", 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:53.755882+0000", 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 328.24996053799998, 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.31213674699999999, 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755882+0000", 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755882+0000", 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.050 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755881+0000", 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.755888+0000", 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2779999999999999e-06 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061422+0000", 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30553472300000001 2023-10-09T05:57:22.051 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061434+0000", 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2132e-05 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061482+0000", 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7129999999999998e-05 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.052 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061517+0000", 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5268999999999997e-05 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.067965+0000", 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064480149999999996 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.068019+0000", 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3918000000000001e-05 2023-10-09T05:57:22.053 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.45782.0:168601 8.14 e352/299 8:2b44b62c:::benchmark_data_smithi008_432048_object65659:head v 352'7985, mlcod=352'7983)", 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:51:53.758418+0000", 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 328.24742433300003, 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.305713397, 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.054 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758418+0000", 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758418+0000", 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758417+0000", 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:53.758427+0000", 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6940000000000007e-06 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061398+0000", 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30297090500000001 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.056 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061426+0000", 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7858999999999999e-05 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061482+0000", 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6051000000000003e-05 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.057 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.061510+0000", 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8085999999999998e-05 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.064085+0000", 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025752219999999998 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:51:54.064132+0000", 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6628999999999997e-05 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.058 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:189481 8.1a 8:5d874d6e:::benchmark_data_smithi008_432048_object86539:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:52:23.908117+0000", 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 298.097725744, 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35551341400000003, 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:22.059 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 189481 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908117+0000", 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908117+0000", 2023-10-09T05:57:22.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908116+0000", 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908121+0000", 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4879999999999998e-06 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.061 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908131+0000", 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7969999999999995e-06 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908136+0000", 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5509999999999999e-06 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908196+0000", 2023-10-09T05:57:22.062 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9165e-05 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:23.908548+0000", 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000352345 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.247746+0000", 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33919810900000003 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.247846+0000", 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010041 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.248364+0000", 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00051785399999999999 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.252480+0000", 2023-10-09T05:57:22.064 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041154950000000003 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.259547+0000", 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070675399999999998 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.263254+0000", 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037071130000000002 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.065 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.263306+0000", 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1272e-05 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:52:24.263630+0000", 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00032452000000000002 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.066 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:13236 9.2 9:4c9db24c:::benchmark_data_smithi008_502820_object13235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:23.473276+0000", 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 238.53256618399999, 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37738287399999998, 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 13236 2023-10-09T05:57:22.067 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473276+0000", 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473276+0000", 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473275+0000", 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473299+0000", 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3838000000000001e-05 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473311+0000", 2023-10-09T05:57:22.069 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1341e-05 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.473318+0000", 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2239999999999998e-06 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.474378+0000", 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010603629999999999 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.070 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.474523+0000", 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014455100000000001 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.474663+0000", 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014071900000000001 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475992+0000", 2023-10-09T05:57:22.071 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001328678 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475999+0000", 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.007e-06 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850549+0000", 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37454978100000003 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.072 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850610+0000", 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0807999999999998e-05 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850659+0000", 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9536999999999997e-05 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.073 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:13240 9.2 9:4ac49363:::benchmark_data_smithi008_502820_object13239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T05:57:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:23.475894+0000", 2023-10-09T05:57:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 238.52994812399999, 2023-10-09T05:57:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37490599200000002, 2023-10-09T05:57:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:57:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:57:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 13240 2023-10-09T05:57:22.074 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.075 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475894+0000", 2023-10-09T05:57:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475894+0000", 2023-10-09T05:57:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.077 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475893+0000", 2023-10-09T05:57:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475917+0000", 2023-10-09T05:57:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4465999999999999e-05 2023-10-09T05:57:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.078 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475929+0000", 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1144e-05 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475937+0000", 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9340000000000006e-06 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476020+0000", 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3170999999999994e-05 2023-10-09T05:57:22.079 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476141+0000", 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012152 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476295+0000", 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000153709 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.080 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.811119+0000", 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33482404700000001 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.834120+0000", 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.023000782000000001 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850714+0000", 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016593891 2023-10-09T05:57:22.081 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850764+0000", 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0479999999999998e-05 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.850800+0000", 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6355e-05 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.082 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:13243 9.0 9:0610f3a0:::benchmark_data_smithi008_502820_object13242:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:23.475944+0000", 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 238.52989857, 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37392950800000002, 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 13243 2023-10-09T05:57:22.083 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475944+0000", 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475944+0000", 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.084 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475943+0000", 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475972+0000", 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8776999999999999e-05 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475980+0000", 2023-10-09T05:57:22.085 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3510000000000002e-06 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.475990+0000", 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8600000000000005e-06 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476039+0000", 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9144000000000002e-05 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.086 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476196+0000", 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015608700000000001 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.476325+0000", 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000129407 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.811118+0000", 2023-10-09T05:57:22.087 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33479272500000001 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.811147+0000", 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8900000000000001e-05 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.849758+0000", 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.038611131 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.849825+0000", 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7037000000000002e-05 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.849874+0000", 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8823000000000003e-05 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.089 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:13247 9.2 9:4cd4543f:::benchmark_data_smithi008_502820_object13246:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:23.477800+0000", 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 238.52804271100001, 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37350776499999999, 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 13247 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.090 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.477800+0000", 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.477800+0000", 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.091 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.477799+0000", 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.477826+0000", 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7291e-05 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.809475+0000", 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33164901299999999 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.809508+0000", 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2771000000000002e-05 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.810358+0000", 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00085006899999999995 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.810484+0000", 2023-10-09T05:57:22.093 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000125742 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.810587+0000", 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010355 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.811302+0000", 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00071510300000000003 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.094 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.834210+0000", 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022907375000000001 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.851187+0000", 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016977590000000001 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.851262+0000", 2023-10-09T05:57:22.095 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5185000000000006e-05 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:23.851308+0000", 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5237000000000001e-05 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:62837 9.2 9:4c5dbb04:::benchmark_data_smithi008_502820_object62836:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e379)", 2023-10-09T05:57:22.096 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:13.285461+0000", 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 188.72038171400001, 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27731913600000002, 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 62837 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.097 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285461+0000", 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285461+0000", 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285460+0000", 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:57:22.098 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285481+0000", 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0845000000000001e-05 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285490+0000", 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8800000000000003e-06 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.099 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285496+0000", 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7010000000000001e-06 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285541+0000", 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5210000000000003e-05 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285688+0000", 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000146093 2023-10-09T05:57:22.100 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.285847+0000", 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015936999999999999 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.288768+0000", 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029215479999999999 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.101 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.297607+0000", 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.008838472 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.562687+0000", 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26507981600000002 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.562740+0000", 2023-10-09T05:57:22.102 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3507000000000001e-05 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:13.562780+0000", 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9818999999999999e-05 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77599 9.0 9:0003d9cd:::benchmark_data_smithi008_502820_object77598:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:36.817794+0000", 2023-10-09T05:57:22.103 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 165.188048369, 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.88810962599999999, 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77599 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.104 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817794+0000", 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817794+0000", 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817793+0000", 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.105 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817819+0000", 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6563999999999998e-05 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817840+0000", 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1183999999999999e-05 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.106 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817850+0000", 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5040000000000008e-06 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817907+0000", 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6700000000000003e-05 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817919+0000", 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2755000000000001e-05 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.107 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.705815+0000", 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.88789530699999997 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.705904+0000", 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9301000000000004e-05 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.108 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77622 9.2 9:43e6566f:::benchmark_data_smithi008_502820_object77621:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e385)", 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:36.872539+0000", 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 165.13330335200001, 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86671261899999996, 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77622 2023-10-09T05:57:22.109 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872539+0000", 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872539+0000", 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.110 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872537+0000", 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872566+0000", 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8402999999999999e-05 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872580+0000", 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4102e-05 2023-10-09T05:57:22.111 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872590+0000", 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8509999999999994e-06 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872642+0000", 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2284999999999998e-05 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.112 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872653+0000", 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0428000000000001e-05 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.703374+0000", 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83072101700000001 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.703547+0000", 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000173058 2023-10-09T05:57:22.113 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.703705+0000", 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000158813 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.714288+0000", 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010582225000000001 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.114 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.725463+0000", 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011175243 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.739134+0000", 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013670748 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.739206+0000", 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2875999999999998e-05 2023-10-09T05:57:22.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.739252+0000", 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5386000000000002e-05 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77624 9.10 9:0cfb0a88:::benchmark_data_smithi008_502820_object77623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e385)", 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:37.004849+0000", 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 165.00099353900001, 2023-10-09T05:57:22.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.76289203999999999, 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77624 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004849+0000", 2023-10-09T05:57:22.117 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004849+0000", 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004847+0000", 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004881+0000", 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3456e-05 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004897+0000", 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.594e-05 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004908+0000", 2023-10-09T05:57:22.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1667e-05 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004998+0000", 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9231e-05 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.005010+0000", 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.225e-05 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.747777+0000", 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.742767222 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.747948+0000", 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017099300000000001 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.752637+0000", 2023-10-09T05:57:22.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0046887279999999996 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.763870+0000", 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011232956000000001 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.764548+0000", 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00067862 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.122 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.767630+0000", 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030816730000000001 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.767696+0000", 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5646000000000001e-05 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.123 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.767741+0000", 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5454000000000002e-05 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93493 9.2 9:40721287:::benchmark_data_smithi008_502820_object9320:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:58.119061+0000", 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 143.88678143300001, 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24733692800000001, 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.124 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93493 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119061+0000", 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.125 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119061+0000", 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119060+0000", 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119065+0000", 2023-10-09T05:57:22.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8840000000000002e-06 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119077+0000", 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2122e-05 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119083+0000", 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5169999999999999e-06 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.127 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119670+0000", 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00058677600000000001 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119757+0000", 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7323000000000004e-05 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:57:22.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.119823+0000", 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6447999999999998e-05 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.121014+0000", 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001191089 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.121251+0000", 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00023627199999999999 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.366302+0000", 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.245051305 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.366355+0000", 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3211999999999998e-05 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:58.366398+0000", 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2898000000000002e-05 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:102993 9.b 9.bd9d978b (undecoded) ondisk+write+known_if_redirected e419)", 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:10.583702+0000", 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 71.422140968999997, 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.74320509000000001, 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.131 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102993 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583702+0000", 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.132 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583702+0000", 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583699+0000", 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583715+0000", 2023-10-09T05:57:22.133 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6668999999999999e-05 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583750+0000", 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4641000000000003e-05 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583765+0000", 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5095e-05 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.134 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583831+0000", 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5693999999999998e-05 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583842+0000", 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1454000000000001e-05 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:11.326897+0000", 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.74305512600000001 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:11.326907+0000", 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2520000000000005e-06 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:107455 9.10 9:0df014fd:::benchmark_data_smithi008_502820_object23282:head [delete] snapc 0=[] ondisk+write+known_if_redirected e427)", 2023-10-09T05:57:22.136 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:44.936319+0000", 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 37.069523848999999, 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.101730354, 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 107455 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.137 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:22.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:44.936319+0000", 2023-10-09T05:57:22.138 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.138 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.138 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.138 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:22.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:44.936319+0000", 2023-10-09T05:57:22.138 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:22.138 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.138 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.138 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:22.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:44.936317+0000", 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:44.936325+0000", 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8150000000000006e-06 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:44.936343+0000", 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7781000000000001e-05 2023-10-09T05:57:22.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:44.936352+0000", 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3340000000000008e-06 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:44.936421+0000", 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8638000000000003e-05 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:44.936519+0000", 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7918999999999995e-05 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:44.936591+0000", 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2627999999999996e-05 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:44.938032+0000", 2023-10-09T05:57:22.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001440279 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:44.938396+0000", 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00036430300000000001 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:45.037962+0000", 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.099566047000000005 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:45.038010+0000", 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7987e-05 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:45.038049+0000", 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9031e-05 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.143 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:22.144 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:22.144 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:57:22.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:22.145 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:22.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:22.246 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:22.282 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:22.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:22 smithi008 ceph-mon[184418]: pgmap v1305: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 156 op/s; 46709/173061 objects degraded (26.990%); 18890/173061 objects misplaced (10.915%); 16 MiB/s, 259 objects/s recovering 2023-10-09T05:57:22.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:22 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1425762447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:22.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:22 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:22.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:22 smithi008 ceph-mon[191227]: pgmap v1305: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 156 op/s; 46709/173061 objects degraded (26.990%); 18890/173061 objects misplaced (10.915%); 16 MiB/s, 259 objects/s recovering 2023-10-09T05:57:22.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:22 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1425762447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:22.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:22 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:22.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:22 smithi191 ceph-mon[161669]: pgmap v1305: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 156 op/s; 46709/173061 objects degraded (26.990%); 18890/173061 objects misplaced (10.915%); 16 MiB/s, 259 objects/s recovering 2023-10-09T05:57:22.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:22 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1425762447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:22.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:22 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:23.599 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:23 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:23.599 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:23 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:23.599 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:23 smithi008 ceph-mon[184418]: pgmap v1306: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 46709/171663 objects degraded (27.210%); 18811/171663 objects misplaced (10.958%); 15 MiB/s, 235 objects/s recovering 2023-10-09T05:57:23.599 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 46709/173061 objects degraded (26.990%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:23.599 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:23 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:23.600 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:23 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:23.600 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:23 smithi008 ceph-mon[191227]: pgmap v1306: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 46709/171663 objects degraded (27.210%); 18811/171663 objects misplaced (10.958%); 15 MiB/s, 235 objects/s recovering 2023-10-09T05:57:23.600 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 46709/173061 objects degraded (26.990%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:23.625 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:57:23.801 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:23.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:23.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:23.905 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:23.936 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:24.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:23 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:24.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:23 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:24.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:23 smithi191 ceph-mon[161669]: pgmap v1306: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 46709/171663 objects degraded (27.210%); 18811/171663 objects misplaced (10.958%); 15 MiB/s, 235 objects/s recovering 2023-10-09T05:57:24.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 46709/173061 objects degraded (26.990%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:24.293 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:24.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:24.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:24.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:24.440 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:24.527 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:24.583 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:24.654 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:24.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:24.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:24.759 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:24.797 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:24.939 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:57:24.939 INFO:teuthology.orchestra.run.smithi008.stdout: "success": "" 2023-10-09T05:57:24.939 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:57:25.080 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:25.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:25.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:25.184 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:25.220 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:25.432 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:57:25.696 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:25 smithi008 ceph-mon[191227]: pgmap v1307: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 46709/171663 objects degraded (27.210%); 18811/171663 objects misplaced (10.958%); 12 MiB/s, 193 objects/s recovering 2023-10-09T05:57:25.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:25 smithi008 ceph-mon[184418]: pgmap v1307: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 46709/171663 objects degraded (27.210%); 18811/171663 objects misplaced (10.958%); 12 MiB/s, 193 objects/s recovering 2023-10-09T05:57:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:25 smithi191 ceph-mon[161669]: pgmap v1307: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 46709/171663 objects degraded (27.210%); 18811/171663 objects misplaced (10.958%); 12 MiB/s, 193 objects/s recovering 2023-10-09T05:57:27.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:26 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:27.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:26 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:27.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:26 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:27.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:26 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:27.233 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:27.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:27.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:27.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:26 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:27.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:26 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:27.337 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:27.368 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:27.482 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:27.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:27.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:27.591 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:27.628 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:27.682 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:27.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:27.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:27.788 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:27.817 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:27.870 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:27.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:27 smithi191 ceph-mon[161669]: pgmap v1308: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 46573/170358 objects degraded (27.338%); 18748/170358 objects misplaced (11.005%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:57:27.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:27.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:27.979 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:28.019 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:28.040 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:28.098 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:28.104 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:27 smithi008 ceph-mon[184418]: pgmap v1308: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 46573/170358 objects degraded (27.338%); 18748/170358 objects misplaced (11.005%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:57:28.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:28.104 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:27 smithi008 ceph-mon[191227]: pgmap v1308: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 46573/170358 objects degraded (27.338%); 18748/170358 objects misplaced (11.005%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T05:57:28.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:28.153 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:57:28.153 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:57:28.153 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T05:57:28.153 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:57:28.206 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:28.244 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:28.659 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:28.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:28.765 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:28.808 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:28.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 46573/170358 objects degraded (27.338%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:28.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/259906341' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:28.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:28 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:28.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 46573/170358 objects degraded (27.338%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:28.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/259906341' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:28.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:28 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:29.111 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:29.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:29.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:29.148 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:57:29.225 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:29.237 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:57:29.281 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:29.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 46573/170358 objects degraded (27.338%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:29.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/259906341' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:29.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:28 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[184418]: pgmap v1309: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 46573/169143 objects degraded (27.535%); 18675/169143 objects misplaced (11.041%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T05:57:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:29.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:29.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:57:29.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[184418]: osdmap e439: 8 total, 8 up, 7 in 2023-10-09T05:57:29.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[184418]: failed to encode map e439 with expected crc 2023-10-09T05:57:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[191227]: pgmap v1309: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 46573/169143 objects degraded (27.535%); 18675/169143 objects misplaced (11.041%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T05:57:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:57:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[191227]: osdmap e439: 8 total, 8 up, 7 in 2023-10-09T05:57:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:29 smithi008 ceph-mon[191227]: failed to encode map e439 with expected crc 2023-10-09T05:57:30.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:29 smithi191 ceph-mon[161669]: pgmap v1309: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 46573/169143 objects degraded (27.535%); 18675/169143 objects misplaced (11.041%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T05:57:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:29 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:29 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:57:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:29 smithi191 ceph-mon[161669]: osdmap e439: 8 total, 8 up, 7 in 2023-10-09T05:57:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:29 smithi191 ceph-mon[161669]: failed to encode map e439 with expected crc 2023-10-09T05:57:30.655 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:30.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:30.760 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:30.798 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:57:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:57:30] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.33.4" 2023-10-09T05:57:30.803 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:30.957 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:31.025 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:31.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:31.108 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:30 smithi008 ceph-mon[184418]: failed to encode map e439 with expected crc 2023-10-09T05:57:31.109 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:30 smithi008 ceph-mon[191227]: failed to encode map e439 with expected crc 2023-10-09T05:57:31.131 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:31.158 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:30 smithi191 ceph-mon[161669]: failed to encode map e439 with expected crc 2023-10-09T05:57:31.159 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:31.210 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:31.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:31.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:31.316 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:31.332 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:57:31.366 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:57:31.369 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:31.394 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:57:31.434 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:57:31.443 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:31.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:31.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:31.465 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:57:31.492 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:57:31.516 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:57:31.542 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:57:31.543 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.543 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.543 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.543 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.543 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.543 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.543 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.544 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:31.581 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:31.624 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:31.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:31.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:31.755 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:31.798 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:31.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:31.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:31.900 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 5, 6, 1, 7, 3] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:57:31.900 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:57:31.900 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T05:57:31.900 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:57:31.904 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:31.973 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:31 smithi191 ceph-mon[161669]: pgmap v1311: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 46426/168126 objects degraded (27.614%); 18642/168126 objects misplaced (11.088%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:31 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3953688543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:32.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:31 smithi008 ceph-mon[184418]: pgmap v1311: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 46426/168126 objects degraded (27.614%); 18642/168126 objects misplaced (11.088%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:32.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:31 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3953688543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:32.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:31 smithi008 ceph-mon[191227]: pgmap v1311: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 46426/168126 objects degraded (27.614%); 18642/168126 objects misplaced (11.088%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:32.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:31 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3953688543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:32.670 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:57:33.088 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:33.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:33.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:33.192 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:33.217 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:33.262 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:33.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:33.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:33.368 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:33.394 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:33.420 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:33.439 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:33.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:33.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:33.522 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:57:33.522 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:57:33.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:33.573 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:33.620 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:33.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:33.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:33.728 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:33.766 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:33.823 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:33 smithi008 ceph-mon[191227]: pgmap v1312: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 46426/166710 objects degraded (27.848%); 18559/166710 objects misplaced (11.133%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:33.823 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 46426/166710 objects degraded (27.848%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:34.117 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:34.121 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:33 smithi008 ceph-mon[184418]: pgmap v1312: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 46426/166710 objects degraded (27.848%); 18559/166710 objects misplaced (11.133%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:34.121 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 46426/166710 objects degraded (27.848%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:34.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:34.177 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:34.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:34.247 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:33 smithi191 ceph-mon[161669]: pgmap v1312: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 46426/166710 objects degraded (27.848%); 18559/166710 objects misplaced (11.133%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:34.247 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 46426/166710 objects degraded (27.848%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:34.249 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:34.292 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:34.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:34.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:34.396 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:34.433 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:35.091 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:57:35.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:35.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:35.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:35.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:35.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:35 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:35 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:35.395 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:35.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:35.499 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:35.522 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:35.523 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:35.523 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:35.523 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:35.523 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:35 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:35.525 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:35.570 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:35.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:35.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:35.610 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:57:35.677 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:35.710 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:35.779 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:35.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:35.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:35.886 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:35.913 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:35.957 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:35.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:35.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:36.062 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:36.090 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:36.142 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:36.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:36.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:36.250 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:36.281 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:36.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:36 smithi191 ceph-mon[161669]: pgmap v1313: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 46426/166710 objects degraded (27.848%); 18559/166710 objects misplaced (11.133%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:36.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:36 smithi008 ceph-mon[184418]: pgmap v1313: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 46426/166710 objects degraded (27.848%); 18559/166710 objects misplaced (11.133%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:36.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:36 smithi008 ceph-mon[191227]: pgmap v1313: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 46426/166710 objects degraded (27.848%); 18559/166710 objects misplaced (11.133%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:36.690 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:36.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:36.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:36.734 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:36.793 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:36.821 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:36.888 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:36.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:36.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:36.992 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:37.049 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:37.153 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:37.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:37.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:37.242 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:57:37.244 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:57:37.258 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:37.288 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:37.320 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:37.342 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:37.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:37.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:37.447 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:37.480 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:37.597 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:37.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:37.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:37.702 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:37.730 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:37.781 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:37.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:37.886 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:37.929 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:38.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:38 smithi008 ceph-mon[184418]: pgmap v1314: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 46293/165603 objects degraded (27.954%); 18530/165603 objects misplaced (11.189%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:38 smithi008 ceph-mon[184418]: from='client.46547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:57:38.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:38 smithi008 ceph-mon[191227]: pgmap v1314: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 46293/165603 objects degraded (27.954%); 18530/165603 objects misplaced (11.189%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:38.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:38 smithi008 ceph-mon[191227]: from='client.46547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:57:38.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:38 smithi191 ceph-mon[161669]: pgmap v1314: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 158 op/s; 46293/165603 objects degraded (27.954%); 18530/165603 objects misplaced (11.189%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:38.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:38 smithi191 ceph-mon[161669]: from='client.46547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:57:38.686 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1314,"stamp":"2023-10-09T05:57:36.644082+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3620542411,"num_objects":55201,"num_object_clones":0,"num_object_copies":165603,"num_objects_missing_on_primary":7198,"num_objects_missing":7198,"num_objects_degraded":46293,"num_objects_misplaced":18530,"num_objects_unfound":0,"num_objects_dirty":55201,"num_whiteouts":0,"num_read":6290,"num_read_kb":13713,"num_write":114275,"num_write_kb":5402022,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23490,"num_bytes_recovered":1404518968,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":157977,"ondisk_log_size":157977,"up":297,"acting":280,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16616652,"kb_used_data":10403284,"kb_used_omap":37,"kb_used_meta":6213274,"kb_avail":639599412,"statfs":{"total":671965249536,"available":654949797888,"internally_reserved":0,"allocated":10652962816,"data_stored":10644423683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38373,"internal_metadata":6362393115},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2,0,1,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-103874560,"num_objects":-1585,"num_object_clones":0,"num_object_copies":-4755,"num_objects_missing_on_primary":-219,"num_objects_missing":-219,"num_objects_degraded":-280,"num_objects_misplaced":-218,"num_objects_unfound":0,"num_objects_dirty":-1585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1585,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":304,"num_bytes_recovered":19922944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003409"},"pg_stats":[{"pgid":"9.10","version":"439'5900","reported_seq":6003,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:32.078992+0000","last_change":"2023-10-09T05:56:50.758012+0000","last_active":"2023-10-09T05:57:32.078992+0000","last_peered":"2023-10-09T05:57:32.078992+0000","last_clean":"2023-10-09T05:56:48.221888+0000","last_became_active":"2023-10-09T05:56:50.720149+0000","last_became_peered":"2023-10-09T05:56:50.720149+0000","last_unstale":"2023-10-09T05:57:32.078992+0000","last_undegraded":"2023-10-09T05:56:50.267102+0000","last_fullsized":"2023-10-09T05:56:50.236705+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5900,"log_dups_size":0,"ondisk_log_size":5900,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109969408,"num_objects":1678,"num_object_clones":0,"num_object_copies":5034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2596,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3510,"num_write_kb":166016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2,6","objects":1680},{"shards":"2,6","objects":-162}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":60,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T05:57:29.232926+0000","last_change":"2023-10-09T05:54:59.256087+0000","last_active":"2023-10-09T05:57:29.232926+0000","last_peered":"2023-10-09T05:57:29.232926+0000","last_clean":"2023-10-09T05:57:29.232926+0000","last_became_active":"2023-10-09T05:54:59.255773+0000","last_became_peered":"2023-10-09T05:54:59.255773+0000","last_unstale":"2023-10-09T05:57:29.232926+0000","last_undegraded":"2023-10-09T05:57:29.232926+0000","last_fullsized":"2023-10-09T05:57:29.232926+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:19.545754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":34,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.278606+0000","last_change":"2023-10-09T05:56:00.247579+0000","last_active":"2023-10-09T05:57:18.278606+0000","last_peered":"2023-10-09T05:57:18.278606+0000","last_clean":"2023-10-09T05:57:18.278606+0000","last_became_active":"2023-10-09T05:56:00.238151+0000","last_became_peered":"2023-10-09T05:56:00.238151+0000","last_unstale":"2023-10-09T05:57:18.278606+0000","last_undegraded":"2023-10-09T05:57:18.278606+0000","last_fullsized":"2023-10-09T05:57:18.278606+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:05.692190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"439'5925","reported_seq":11755,"reported_epoch":439,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:57:32.174631+0000","last_change":"2023-10-09T05:56:01.319470+0000","last_active":"2023-10-09T05:57:32.174631+0000","last_peered":"2023-10-09T05:57:32.174631+0000","last_clean":"2023-10-09T05:55:24.768851+0000","last_became_active":"2023-10-09T05:56:01.311996+0000","last_became_peered":"2023-10-09T05:56:01.311996+0000","last_unstale":"2023-10-09T05:57:32.174631+0000","last_undegraded":"2023-10-09T05:57:32.174631+0000","last_fullsized":"2023-10-09T05:57:32.174631+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5925,"log_dups_size":0,"ondisk_log_size":5925,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114688000,"num_objects":1750,"num_object_clones":0,"num_object_copies":5250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1750,"num_objects_unfound":0,"num_objects_dirty":1750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3512,"num_write_kb":168384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1422,"num_bytes_recovered":93192192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1750}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":141,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:27.453074+0000","last_change":"2023-10-09T05:56:00.243822+0000","last_active":"2023-10-09T05:57:27.453074+0000","last_peered":"2023-10-09T05:57:27.453074+0000","last_clean":"2023-10-09T05:57:27.453074+0000","last_became_active":"2023-10-09T05:56:00.243483+0000","last_became_peered":"2023-10-09T05:56:00.243483+0000","last_unstale":"2023-10-09T05:57:27.453074+0000","last_undegraded":"2023-10-09T05:57:27.453074+0000","last_fullsized":"2023-10-09T05:57:27.453074+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:39.811952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":500,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.249222+0000","last_change":"2023-10-09T05:56:49.272001+0000","last_active":"2023-10-09T05:57:18.249222+0000","last_peered":"2023-10-09T05:57:18.249222+0000","last_clean":"2023-10-09T05:57:18.249222+0000","last_became_active":"2023-10-09T05:56:49.271668+0000","last_became_peered":"2023-10-09T05:56:49.271668+0000","last_unstale":"2023-10-09T05:57:18.249222+0000","last_undegraded":"2023-10-09T05:57:18.249222+0000","last_fullsized":"2023-10-09T05:57:18.249222+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.12","version":"439'6001","reported_seq":6191,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:31.840686+0000","last_change":"2023-10-09T05:56:50.476848+0000","last_active":"2023-10-09T05:57:31.840686+0000","last_peered":"2023-10-09T05:57:31.840686+0000","last_clean":"2023-10-09T05:56:48.123677+0000","last_became_active":"2023-10-09T05:56:50.467001+0000","last_became_peered":"2023-10-09T05:56:50.467001+0000","last_unstale":"2023-10-09T05:57:31.840686+0000","last_undegraded":"2023-10-09T05:56:50.269295+0000","last_fullsized":"2023-10-09T05:56:50.237086+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6001,"log_dups_size":0,"ondisk_log_size":6001,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113213440,"num_objects":1728,"num_object_clones":0,"num_object_copies":5184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2656,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3580,"num_write_kb":169824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-169},{"shards":"1,2,3","objects":1730}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":39,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.245788+0000","last_change":"2023-10-09T05:56:49.265409+0000","last_active":"2023-10-09T05:57:18.245788+0000","last_peered":"2023-10-09T05:57:18.245788+0000","last_clean":"2023-10-09T05:57:18.245788+0000","last_became_active":"2023-10-09T05:56:49.265006+0000","last_became_peered":"2023-10-09T05:56:49.265006+0000","last_unstale":"2023-10-09T05:57:18.245788+0000","last_undegraded":"2023-10-09T05:57:18.245788+0000","last_fullsized":"2023-10-09T05:57:18.245788+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:29.506553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":265,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.362486+0000","last_change":"2023-10-09T05:55:11.312896+0000","last_active":"2023-10-09T05:57:18.362486+0000","last_peered":"2023-10-09T05:57:18.362486+0000","last_clean":"2023-10-09T05:57:18.362486+0000","last_became_active":"2023-10-09T05:55:11.312597+0000","last_became_peered":"2023-10-09T05:55:11.312597+0000","last_unstale":"2023-10-09T05:57:18.362486+0000","last_undegraded":"2023-10-09T05:57:18.362486+0000","last_fullsized":"2023-10-09T05:57:18.362486+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":413,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.263749+0000","last_change":"2023-10-09T05:56:11.346423+0000","last_active":"2023-10-09T05:57:18.263749+0000","last_peered":"2023-10-09T05:57:18.263749+0000","last_clean":"2023-10-09T05:57:18.263749+0000","last_became_active":"2023-10-09T05:56:11.346098+0000","last_became_peered":"2023-10-09T05:56:11.346098+0000","last_unstale":"2023-10-09T05:57:18.263749+0000","last_undegraded":"2023-10-09T05:57:18.263749+0000","last_fullsized":"2023-10-09T05:57:18.263749+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":34,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.278085+0000","last_change":"2023-10-09T05:56:00.263498+0000","last_active":"2023-10-09T05:57:18.278085+0000","last_peered":"2023-10-09T05:57:18.278085+0000","last_clean":"2023-10-09T05:57:18.278085+0000","last_became_active":"2023-10-09T05:56:00.262929+0000","last_became_peered":"2023-10-09T05:56:00.262929+0000","last_unstale":"2023-10-09T05:57:18.278085+0000","last_undegraded":"2023-10-09T05:57:18.278085+0000","last_fullsized":"2023-10-09T05:57:18.278085+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:56.874781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":113,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.262314+0000","last_change":"2023-10-09T05:55:39.287521+0000","last_active":"2023-10-09T05:57:18.262314+0000","last_peered":"2023-10-09T05:57:18.262314+0000","last_clean":"2023-10-09T05:57:18.262314+0000","last_became_active":"2023-10-09T05:55:39.287263+0000","last_became_peered":"2023-10-09T05:55:39.287263+0000","last_unstale":"2023-10-09T05:57:18.262314+0000","last_undegraded":"2023-10-09T05:57:18.262314+0000","last_fullsized":"2023-10-09T05:57:18.262314+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.248249+0000","last_change":"2023-10-09T05:56:00.235776+0000","last_active":"2023-10-09T05:57:18.248249+0000","last_peered":"2023-10-09T05:57:18.248249+0000","last_clean":"2023-10-09T05:57:18.248249+0000","last_became_active":"2023-10-09T05:56:00.235383+0000","last_became_peered":"2023-10-09T05:56:00.235383+0000","last_unstale":"2023-10-09T05:57:18.248249+0000","last_undegraded":"2023-10-09T05:57:18.248249+0000","last_fullsized":"2023-10-09T05:57:18.248249+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":76,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.274256+0000","last_change":"2023-10-09T05:56:00.348771+0000","last_active":"2023-10-09T05:57:18.274256+0000","last_peered":"2023-10-09T05:57:18.274256+0000","last_clean":"2023-10-09T05:57:18.274256+0000","last_became_active":"2023-10-09T05:56:00.224347+0000","last_became_peered":"2023-10-09T05:56:00.224347+0000","last_unstale":"2023-10-09T05:57:18.274256+0000","last_undegraded":"2023-10-09T05:57:18.274256+0000","last_fullsized":"2023-10-09T05:57:18.274256+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":190,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.249661+0000","last_change":"2023-10-09T05:56:49.249928+0000","last_active":"2023-10-09T05:57:18.249661+0000","last_peered":"2023-10-09T05:57:18.249661+0000","last_clean":"2023-10-09T05:57:18.249661+0000","last_became_active":"2023-10-09T05:56:49.249596+0000","last_became_peered":"2023-10-09T05:56:49.249596+0000","last_unstale":"2023-10-09T05:57:18.249661+0000","last_undegraded":"2023-10-09T05:57:18.249661+0000","last_fullsized":"2023-10-09T05:57:18.249661+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:52.544891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":57,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.249233+0000","last_change":"2023-10-09T05:55:26.703302+0000","last_active":"2023-10-09T05:57:18.249233+0000","last_peered":"2023-10-09T05:57:18.249233+0000","last_clean":"2023-10-09T05:57:18.249233+0000","last_became_active":"2023-10-09T05:55:26.265755+0000","last_became_peered":"2023-10-09T05:55:26.265755+0000","last_unstale":"2023-10-09T05:57:18.249233+0000","last_undegraded":"2023-10-09T05:57:18.249233+0000","last_fullsized":"2023-10-09T05:57:18.249233+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":415,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:22.578810+0000","last_change":"2023-10-09T05:54:59.255627+0000","last_active":"2023-10-09T05:57:22.578810+0000","last_peered":"2023-10-09T05:57:22.578810+0000","last_clean":"2023-10-09T05:57:22.578810+0000","last_became_active":"2023-10-09T05:54:59.255251+0000","last_became_peered":"2023-10-09T05:54:59.255251+0000","last_unstale":"2023-10-09T05:57:22.578810+0000","last_undegraded":"2023-10-09T05:57:22.578810+0000","last_fullsized":"2023-10-09T05:57:22.578810+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:37:19.719962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.19","version":"439'5923","reported_seq":6986,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:33.552584+0000","last_change":"2023-10-09T05:56:50.640988+0000","last_active":"2023-10-09T05:57:33.552584+0000","last_peered":"2023-10-09T05:57:33.552584+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T05:56:50.627133+0000","last_became_peered":"2023-10-09T05:56:50.627133+0000","last_unstale":"2023-10-09T05:57:33.552584+0000","last_undegraded":"2023-10-09T05:56:50.470148+0000","last_fullsized":"2023-10-09T05:56:50.237480+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5923,"log_dups_size":0,"ondisk_log_size":5923,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113410048,"num_objects":1731,"num_object_clones":0,"num_object_copies":5193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4317,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3517,"num_write_kb":167904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":472,"num_bytes_recovered":30900224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[5,7],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":1217},{"shards":"0,5","objects":1},{"shards":"0,5,7","objects":517},{"shards":"5,7","objects":-171}],"blocked_by":[],"up_primary":7,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":53,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.248988+0000","last_change":"2023-10-09T05:55:27.092744+0000","last_active":"2023-10-09T05:57:18.248988+0000","last_peered":"2023-10-09T05:57:18.248988+0000","last_clean":"2023-10-09T05:57:18.248988+0000","last_became_active":"2023-10-09T05:55:26.271096+0000","last_became_peered":"2023-10-09T05:55:26.271096+0000","last_unstale":"2023-10-09T05:57:18.248988+0000","last_undegraded":"2023-10-09T05:57:18.248988+0000","last_fullsized":"2023-10-09T05:57:18.248988+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":74,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.249145+0000","last_change":"2023-10-09T05:56:49.266384+0000","last_active":"2023-10-09T05:57:18.249145+0000","last_peered":"2023-10-09T05:57:18.249145+0000","last_clean":"2023-10-09T05:57:18.249145+0000","last_became_active":"2023-10-09T05:56:49.266051+0000","last_became_peered":"2023-10-09T05:56:49.266051+0000","last_unstale":"2023-10-09T05:57:18.249145+0000","last_undegraded":"2023-10-09T05:57:18.249145+0000","last_fullsized":"2023-10-09T05:57:18.249145+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"439'5928","reported_seq":6960,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:34.555941+0000","last_change":"2023-10-09T05:56:50.728342+0000","last_active":"2023-10-09T05:57:34.555941+0000","last_peered":"2023-10-09T05:57:34.555941+0000","last_clean":"2023-10-09T05:55:49.156862+0000","last_became_active":"2023-10-09T05:56:50.718720+0000","last_became_peered":"2023-10-09T05:56:50.718720+0000","last_unstale":"2023-10-09T05:57:34.555941+0000","last_undegraded":"2023-10-09T05:56:50.504293+0000","last_fullsized":"2023-10-09T05:56:50.237342+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5928,"log_dups_size":0,"ondisk_log_size":5928,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112001024,"num_objects":1709,"num_object_clones":0,"num_object_copies":5127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4804,"num_objects_misplaced":2475,"num_objects_unfound":0,"num_objects_dirty":1709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3515,"num_write_kb":167168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":312,"num_bytes_recovered":20447232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[5,7],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":1738},{"shards":"5,7","objects":-29}],"blocked_by":[],"up_primary":7,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":474,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.248721+0000","last_change":"2023-10-09T05:56:49.261881+0000","last_active":"2023-10-09T05:57:18.248721+0000","last_peered":"2023-10-09T05:57:18.248721+0000","last_clean":"2023-10-09T05:57:18.248721+0000","last_became_active":"2023-10-09T05:56:49.261440+0000","last_became_peered":"2023-10-09T05:56:49.261440+0000","last_unstale":"2023-10-09T05:57:18.248721+0000","last_undegraded":"2023-10-09T05:57:18.248721+0000","last_fullsized":"2023-10-09T05:57:18.248721+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":120,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.274340+0000","last_change":"2023-10-09T05:56:49.264085+0000","last_active":"2023-10-09T05:57:18.274340+0000","last_peered":"2023-10-09T05:57:18.274340+0000","last_clean":"2023-10-09T05:57:18.274340+0000","last_became_active":"2023-10-09T05:56:49.263267+0000","last_became_peered":"2023-10-09T05:56:49.263267+0000","last_unstale":"2023-10-09T05:57:18.274340+0000","last_undegraded":"2023-10-09T05:57:18.274340+0000","last_fullsized":"2023-10-09T05:57:18.274340+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":47,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.248221+0000","last_change":"2023-10-09T05:55:26.272219+0000","last_active":"2023-10-09T05:57:18.248221+0000","last_peered":"2023-10-09T05:57:18.248221+0000","last_clean":"2023-10-09T05:57:18.248221+0000","last_became_active":"2023-10-09T05:55:26.271861+0000","last_became_peered":"2023-10-09T05:55:26.271861+0000","last_unstale":"2023-10-09T05:57:18.248221+0000","last_undegraded":"2023-10-09T05:57:18.248221+0000","last_fullsized":"2023-10-09T05:57:18.248221+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":462,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.249299+0000","last_change":"2023-10-09T05:56:11.341328+0000","last_active":"2023-10-09T05:57:18.249299+0000","last_peered":"2023-10-09T05:57:18.249299+0000","last_clean":"2023-10-09T05:57:18.249299+0000","last_became_active":"2023-10-09T05:56:11.340361+0000","last_became_peered":"2023-10-09T05:56:11.340361+0000","last_unstale":"2023-10-09T05:57:18.249299+0000","last_undegraded":"2023-10-09T05:57:18.249299+0000","last_fullsized":"2023-10-09T05:57:18.249299+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":37,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.245701+0000","last_change":"2023-10-09T05:56:49.278174+0000","last_active":"2023-10-09T05:57:18.245701+0000","last_peered":"2023-10-09T05:57:18.245701+0000","last_clean":"2023-10-09T05:57:18.245701+0000","last_became_active":"2023-10-09T05:56:49.249012+0000","last_became_peered":"2023-10-09T05:56:49.249012+0000","last_unstale":"2023-10-09T05:57:18.245701+0000","last_undegraded":"2023-10-09T05:57:18.245701+0000","last_fullsized":"2023-10-09T05:57:18.245701+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:39.546507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.249624+0000","last_change":"2023-10-09T05:56:49.263088+0000","last_active":"2023-10-09T05:57:18.249624+0000","last_peered":"2023-10-09T05:57:18.249624+0000","last_clean":"2023-10-09T05:57:18.249624+0000","last_became_active":"2023-10-09T05:56:49.262781+0000","last_became_peered":"2023-10-09T05:56:49.262781+0000","last_unstale":"2023-10-09T05:57:18.249624+0000","last_undegraded":"2023-10-09T05:57:18.249624+0000","last_fullsized":"2023-10-09T05:57:18.249624+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"439'6002","reported_seq":7265,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:34.113787+0000","last_change":"2023-10-09T05:57:01.636835+0000","last_active":"2023-10-09T05:57:34.113787+0000","last_peered":"2023-10-09T05:57:34.113787+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T05:56:12.505348+0000","last_became_peered":"2023-10-09T05:56:12.505348+0000","last_unstale":"2023-10-09T05:57:34.113787+0000","last_undegraded":"2023-10-09T05:56:12.423351+0000","last_fullsized":"2023-10-09T05:56:12.346143+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6002,"log_dups_size":0,"ondisk_log_size":6002,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112099328,"num_objects":1710,"num_object_clones":0,"num_object_copies":5130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2690,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3560,"num_write_kb":168672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":391,"num_bytes_recovered":25624576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,1,5","objects":964},{"shards":"0,5,7","objects":520},{"shards":"5,7","objects":-120}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":64,"reported_epoch":438,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:57:18.249680+0000","last_change":"2023-10-09T05:56:49.278256+0000","last_active":"2023-10-09T05:57:18.249680+0000","last_peered":"2023-10-09T05:57:18.249680+0000","last_clean":"2023-10-09T05:56:02.699477+0000","last_became_active":"2023-10-09T05:56:49.277974+0000","last_became_peered":"2023-10-09T05:56:49.277974+0000","last_unstale":"2023-10-09T05:57:18.249680+0000","last_undegraded":"2023-10-09T05:56:49.251389+0000","last_fullsized":"2023-10-09T05:57:18.249680+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":93,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.248179+0000","last_change":"2023-10-09T05:56:49.277892+0000","last_active":"2023-10-09T05:57:18.248179+0000","last_peered":"2023-10-09T05:57:18.248179+0000","last_clean":"2023-10-09T05:57:18.248179+0000","last_became_active":"2023-10-09T05:56:49.271228+0000","last_became_peered":"2023-10-09T05:56:49.271228+0000","last_unstale":"2023-10-09T05:57:18.248179+0000","last_undegraded":"2023-10-09T05:57:18.248179+0000","last_fullsized":"2023-10-09T05:57:18.248179+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"439'5985","reported_seq":6102,"reported_epoch":439,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:57:32.060892+0000","last_change":"2023-10-09T05:56:48.413650+0000","last_active":"2023-10-09T05:57:32.060892+0000","last_peered":"2023-10-09T05:57:32.060892+0000","last_clean":"2023-10-09T05:55:48.934818+0000","last_became_active":"2023-10-09T05:56:01.246051+0000","last_became_peered":"2023-10-09T05:56:01.246051+0000","last_unstale":"2023-10-09T05:57:32.060892+0000","last_undegraded":"2023-10-09T05:56:48.413475+0000","last_fullsized":"2023-10-09T05:57:32.060892+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5985,"log_dups_size":0,"ondisk_log_size":5985,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113180672,"num_objects":1727,"num_object_clones":0,"num_object_copies":5181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3555,"num_write_kb":169024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1727}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":51,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.249384+0000","last_change":"2023-10-09T05:55:26.358519+0000","last_active":"2023-10-09T05:57:18.249384+0000","last_peered":"2023-10-09T05:57:18.249384+0000","last_clean":"2023-10-09T05:57:18.249384+0000","last_became_active":"2023-10-09T05:55:26.260138+0000","last_became_peered":"2023-10-09T05:55:26.260138+0000","last_unstale":"2023-10-09T05:57:18.249384+0000","last_undegraded":"2023-10-09T05:57:18.249384+0000","last_fullsized":"2023-10-09T05:57:18.249384+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":474,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.269008+0000","last_change":"2023-10-09T05:56:49.252186+0000","last_active":"2023-10-09T05:57:18.269008+0000","last_peered":"2023-10-09T05:57:18.269008+0000","last_clean":"2023-10-09T05:57:18.269008+0000","last_became_active":"2023-10-09T05:56:49.251848+0000","last_became_peered":"2023-10-09T05:56:49.251848+0000","last_unstale":"2023-10-09T05:57:18.269008+0000","last_undegraded":"2023-10-09T05:57:18.269008+0000","last_fullsized":"2023-10-09T05:57:18.269008+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"439'5917","reported_seq":8329,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:33.637936+0000","last_change":"2023-10-09T05:56:50.495067+0000","last_active":"2023-10-09T05:57:33.637936+0000","last_peered":"2023-10-09T05:57:33.637936+0000","last_clean":"2023-10-09T05:53:45.891460+0000","last_became_active":"2023-10-09T05:56:50.472629+0000","last_became_peered":"2023-10-09T05:56:50.472629+0000","last_unstale":"2023-10-09T05:57:33.637936+0000","last_undegraded":"2023-10-09T05:56:50.467715+0000","last_fullsized":"2023-10-09T05:56:50.242023+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5917,"log_dups_size":0,"ondisk_log_size":5917,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113573888,"num_objects":1733,"num_object_clones":0,"num_object_copies":5199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1887,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3493,"num_write_kb":167232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1139,"num_bytes_recovered":72351744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[5,7],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"2,3,5","objects":475},{"shards":"3,5,7","objects":532},{"shards":"5,7","objects":571}],"blocked_by":[],"up_primary":7,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":62,"reported_epoch":438,"state":"active+recovery_wait","last_fresh":"2023-10-09T05:57:18.274456+0000","last_change":"2023-10-09T05:56:50.248775+0000","last_active":"2023-10-09T05:57:18.274456+0000","last_peered":"2023-10-09T05:57:18.274456+0000","last_clean":"2023-10-09T05:56:43.219657+0000","last_became_active":"2023-10-09T05:56:49.263736+0000","last_became_peered":"2023-10-09T05:56:49.263736+0000","last_unstale":"2023-10-09T05:57:18.274456+0000","last_undegraded":"2023-10-09T05:57:18.274456+0000","last_fullsized":"2023-10-09T05:57:18.274456+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":["7","3","5","6"],"object_location_counts":[{"shards":"3,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":47,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.248636+0000","last_change":"2023-10-09T05:55:26.254446+0000","last_active":"2023-10-09T05:57:18.248636+0000","last_peered":"2023-10-09T05:57:18.248636+0000","last_clean":"2023-10-09T05:57:18.248636+0000","last_became_active":"2023-10-09T05:55:26.254088+0000","last_became_peered":"2023-10-09T05:55:26.254088+0000","last_unstale":"2023-10-09T05:57:18.248636+0000","last_undegraded":"2023-10-09T05:57:18.248636+0000","last_fullsized":"2023-10-09T05:57:18.248636+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"439'5849","reported_seq":5951,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T05:57:32.320952+0000","last_change":"2023-10-09T05:56:12.824335+0000","last_active":"2023-10-09T05:57:32.320952+0000","last_peered":"2023-10-09T05:57:32.320952+0000","last_clean":"2023-10-09T05:57:32.320952+0000","last_became_active":"2023-10-09T05:55:11.317275+0000","last_became_peered":"2023-10-09T05:55:11.317275+0000","last_unstale":"2023-10-09T05:57:32.320952+0000","last_undegraded":"2023-10-09T05:57:32.320952+0000","last_fullsized":"2023-10-09T05:57:32.320952+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5849,"log_dups_size":0,"ondisk_log_size":5849,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113377280,"num_objects":1730,"num_object_clones":0,"num_object_copies":5190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3450,"num_write_kb":165760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":424,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.256760+0000","last_change":"2023-10-09T05:56:00.245186+0000","last_active":"2023-10-09T05:57:18.256760+0000","last_peered":"2023-10-09T05:57:18.256760+0000","last_clean":"2023-10-09T05:57:18.256760+0000","last_became_active":"2023-10-09T05:56:00.244871+0000","last_became_peered":"2023-10-09T05:56:00.244871+0000","last_unstale":"2023-10-09T05:57:18.256760+0000","last_undegraded":"2023-10-09T05:57:18.256760+0000","last_fullsized":"2023-10-09T05:57:18.256760+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":301,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.349384+0000","last_change":"2023-10-09T05:56:11.340232+0000","last_active":"2023-10-09T05:57:18.349384+0000","last_peered":"2023-10-09T05:57:18.349384+0000","last_clean":"2023-10-09T05:57:18.349384+0000","last_became_active":"2023-10-09T05:56:11.339675+0000","last_became_peered":"2023-10-09T05:56:11.339675+0000","last_unstale":"2023-10-09T05:57:18.349384+0000","last_undegraded":"2023-10-09T05:57:18.349384+0000","last_fullsized":"2023-10-09T05:57:18.349384+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:50.837847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"439'5986","reported_seq":16770,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T05:57:31.396032+0000","last_change":"2023-10-09T05:56:20.425478+0000","last_active":"2023-10-09T05:57:31.396032+0000","last_peered":"2023-10-09T05:57:31.396032+0000","last_clean":"2023-10-09T05:57:31.396032+0000","last_became_active":"2023-10-09T05:56:20.416446+0000","last_became_peered":"2023-10-09T05:56:20.416446+0000","last_unstale":"2023-10-09T05:57:31.396032+0000","last_undegraded":"2023-10-09T05:57:31.396032+0000","last_fullsized":"2023-10-09T05:57:31.396032+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5986,"log_dups_size":0,"ondisk_log_size":5986,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114720768,"num_objects":1750,"num_object_clones":0,"num_object_copies":5250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3532,"num_write_kb":169056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4116,"num_bytes_recovered":234323968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":301,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.278553+0000","last_change":"2023-10-09T05:56:11.343723+0000","last_active":"2023-10-09T05:57:18.278553+0000","last_peered":"2023-10-09T05:57:18.278553+0000","last_clean":"2023-10-09T05:57:18.278553+0000","last_became_active":"2023-10-09T05:56:11.337217+0000","last_became_peered":"2023-10-09T05:56:11.337217+0000","last_unstale":"2023-10-09T05:57:18.278553+0000","last_undegraded":"2023-10-09T05:57:18.278553+0000","last_fullsized":"2023-10-09T05:57:18.278553+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:55.912242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":422,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.269357+0000","last_change":"2023-10-09T05:56:00.235557+0000","last_active":"2023-10-09T05:57:18.269357+0000","last_peered":"2023-10-09T05:57:18.269357+0000","last_clean":"2023-10-09T05:57:18.269357+0000","last_became_active":"2023-10-09T05:56:00.235199+0000","last_became_peered":"2023-10-09T05:56:00.235199+0000","last_unstale":"2023-10-09T05:57:18.269357+0000","last_undegraded":"2023-10-09T05:57:18.269357+0000","last_fullsized":"2023-10-09T05:57:18.269357+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"439'5870","reported_seq":5976,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:32.335325+0000","last_change":"2023-10-09T05:56:50.757983+0000","last_active":"2023-10-09T05:57:32.335325+0000","last_peered":"2023-10-09T05:57:32.335325+0000","last_clean":"2023-10-09T05:56:47.068306+0000","last_became_active":"2023-10-09T05:56:50.719969+0000","last_became_peered":"2023-10-09T05:56:50.719969+0000","last_unstale":"2023-10-09T05:57:32.335325+0000","last_undegraded":"2023-10-09T05:56:50.270087+0000","last_fullsized":"2023-10-09T05:56:50.236726+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5870,"log_dups_size":0,"ondisk_log_size":5870,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112721920,"num_objects":1720,"num_object_clones":0,"num_object_copies":5160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2598,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3480,"num_write_kb":166400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2,6","objects":1718},{"shards":"2,6","objects":-154}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":279,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.361675+0000","last_change":"2023-10-09T05:56:49.252293+0000","last_active":"2023-10-09T05:57:18.361675+0000","last_peered":"2023-10-09T05:57:18.361675+0000","last_clean":"2023-10-09T05:57:18.361675+0000","last_became_active":"2023-10-09T05:56:49.251985+0000","last_became_peered":"2023-10-09T05:56:49.251985+0000","last_unstale":"2023-10-09T05:57:18.361675+0000","last_undegraded":"2023-10-09T05:57:18.361675+0000","last_fullsized":"2023-10-09T05:57:18.361675+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":368,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.246228+0000","last_change":"2023-10-09T05:54:05.161959+0000","last_active":"2023-10-09T05:57:18.246228+0000","last_peered":"2023-10-09T05:57:18.246228+0000","last_clean":"2023-10-09T05:57:18.246228+0000","last_became_active":"2023-10-09T05:54:05.161671+0000","last_became_peered":"2023-10-09T05:54:05.161671+0000","last_unstale":"2023-10-09T05:57:18.246228+0000","last_undegraded":"2023-10-09T05:57:18.246228+0000","last_fullsized":"2023-10-09T05:57:18.246228+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:47.282176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"439'5914","reported_seq":11829,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:32.197357+0000","last_change":"2023-10-09T05:56:01.447837+0000","last_active":"2023-10-09T05:57:32.197357+0000","last_peered":"2023-10-09T05:57:32.197357+0000","last_clean":"2023-10-09T05:55:25.103026+0000","last_became_active":"2023-10-09T05:56:01.440057+0000","last_became_peered":"2023-10-09T05:56:01.440057+0000","last_unstale":"2023-10-09T05:57:32.197357+0000","last_undegraded":"2023-10-09T05:56:01.309926+0000","last_fullsized":"2023-10-09T05:56:01.207038+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5914,"log_dups_size":0,"ondisk_log_size":5914,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112132096,"num_objects":1711,"num_object_clones":0,"num_object_copies":5133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3501,"num_write_kb":166784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1468,"num_bytes_recovered":96206848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1,3","objects":-364},{"shards":"1,3,5","objects":1740}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":441,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.245675+0000","last_change":"2023-10-09T05:55:26.256379+0000","last_active":"2023-10-09T05:57:18.245675+0000","last_peered":"2023-10-09T05:57:18.245675+0000","last_clean":"2023-10-09T05:57:18.245675+0000","last_became_active":"2023-10-09T05:55:26.256062+0000","last_became_peered":"2023-10-09T05:55:26.256062+0000","last_unstale":"2023-10-09T05:57:18.245675+0000","last_undegraded":"2023-10-09T05:57:18.245675+0000","last_fullsized":"2023-10-09T05:57:18.245675+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:00.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":440,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.269659+0000","last_change":"2023-10-09T05:56:49.252569+0000","last_active":"2023-10-09T05:57:18.269659+0000","last_peered":"2023-10-09T05:57:18.269659+0000","last_clean":"2023-10-09T05:57:18.269659+0000","last_became_active":"2023-10-09T05:56:49.252185+0000","last_became_peered":"2023-10-09T05:56:49.252185+0000","last_unstale":"2023-10-09T05:57:18.269659+0000","last_undegraded":"2023-10-09T05:57:18.269659+0000","last_fullsized":"2023-10-09T05:57:18.269659+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"439'5926","reported_seq":6139,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:31.899625+0000","last_change":"2023-10-09T05:56:50.476851+0000","last_active":"2023-10-09T05:57:31.899625+0000","last_peered":"2023-10-09T05:57:31.899625+0000","last_clean":"2023-10-09T05:56:48.126316+0000","last_became_active":"2023-10-09T05:56:50.466802+0000","last_became_peered":"2023-10-09T05:56:50.466802+0000","last_unstale":"2023-10-09T05:57:31.899625+0000","last_undegraded":"2023-10-09T05:56:50.272069+0000","last_fullsized":"2023-10-09T05:56:50.242739+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5926,"log_dups_size":0,"ondisk_log_size":5926,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113541120,"num_objects":1732,"num_object_clones":0,"num_object_copies":5196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2616,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3504,"num_write_kb":167584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-132},{"shards":"1,2,3","objects":1730}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.251668+0000","last_change":"2023-10-09T05:56:49.287906+0000","last_active":"2023-10-09T05:57:18.251668+0000","last_peered":"2023-10-09T05:57:18.251668+0000","last_clean":"2023-10-09T05:57:18.251668+0000","last_became_active":"2023-10-09T05:56:49.275081+0000","last_became_peered":"2023-10-09T05:56:49.275081+0000","last_unstale":"2023-10-09T05:57:18.251668+0000","last_undegraded":"2023-10-09T05:57:18.251668+0000","last_fullsized":"2023-10-09T05:57:18.251668+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":126,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.352440+0000","last_change":"2023-10-09T05:56:00.225724+0000","last_active":"2023-10-09T05:57:18.352440+0000","last_peered":"2023-10-09T05:57:18.352440+0000","last_clean":"2023-10-09T05:57:18.352440+0000","last_became_active":"2023-10-09T05:56:00.225335+0000","last_became_peered":"2023-10-09T05:56:00.225335+0000","last_unstale":"2023-10-09T05:57:18.352440+0000","last_undegraded":"2023-10-09T05:57:18.352440+0000","last_fullsized":"2023-10-09T05:57:18.352440+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:41.405222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"439'7211","reported_seq":7311,"reported_epoch":439,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:57:34.515949+0000","last_change":"2023-10-09T05:56:01.503330+0000","last_active":"2023-10-09T05:57:34.515949+0000","last_peered":"2023-10-09T05:57:34.515949+0000","last_clean":"2023-10-09T05:55:49.154347+0000","last_became_active":"2023-10-09T05:56:01.480211+0000","last_became_peered":"2023-10-09T05:56:01.480211+0000","last_unstale":"2023-10-09T05:57:34.515949+0000","last_undegraded":"2023-10-09T05:56:01.465289+0000","last_fullsized":"2023-10-09T05:57:34.515949+0000","mapping_epoch":416,"log_start":"368'661","ondisk_log_start":"368'661","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6550,"log_dups_size":0,"ondisk_log_size":6550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227737600,"num_objects":3475,"num_object_clones":0,"num_object_copies":10425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":93,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7211,"num_write_kb":341952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,3,5","objects":3475}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":867,"reported_epoch":438,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:57:18.248986+0000","last_change":"2023-10-09T05:56:12.339160+0000","last_active":"2023-10-09T05:57:18.248986+0000","last_peered":"2023-10-09T05:57:18.248986+0000","last_clean":"2023-10-09T05:55:22.199269+0000","last_became_active":"2023-10-09T05:56:12.338771+0000","last_became_peered":"2023-10-09T05:56:12.338771+0000","last_unstale":"2023-10-09T05:57:18.248986+0000","last_undegraded":"2023-10-09T05:56:12.324800+0000","last_fullsized":"2023-10-09T05:57:18.248986+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":438,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.261921+0000","last_change":"2023-10-09T05:56:00.220470+0000","last_active":"2023-10-09T05:57:18.261921+0000","last_peered":"2023-10-09T05:57:18.261921+0000","last_clean":"2023-10-09T05:57:18.261921+0000","last_became_active":"2023-10-09T05:56:00.220204+0000","last_became_peered":"2023-10-09T05:56:00.220204+0000","last_unstale":"2023-10-09T05:57:18.261921+0000","last_undegraded":"2023-10-09T05:57:18.261921+0000","last_fullsized":"2023-10-09T05:57:18.261921+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":21,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.245826+0000","last_change":"2023-10-09T05:56:49.255402+0000","last_active":"2023-10-09T05:57:18.245826+0000","last_peered":"2023-10-09T05:57:18.245826+0000","last_clean":"2023-10-09T05:57:18.245826+0000","last_became_active":"2023-10-09T05:56:49.255103+0000","last_became_peered":"2023-10-09T05:56:49.255103+0000","last_unstale":"2023-10-09T05:57:18.245826+0000","last_undegraded":"2023-10-09T05:57:18.245826+0000","last_fullsized":"2023-10-09T05:57:18.245826+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:32.251375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"439'5905","reported_seq":6928,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:33.451910+0000","last_change":"2023-10-09T05:56:50.626241+0000","last_active":"2023-10-09T05:57:33.451910+0000","last_peered":"2023-10-09T05:57:33.451910+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T05:56:50.548766+0000","last_became_peered":"2023-10-09T05:56:50.548766+0000","last_unstale":"2023-10-09T05:57:33.451910+0000","last_undegraded":"2023-10-09T05:56:50.381420+0000","last_fullsized":"2023-10-09T05:56:50.237890+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5905,"log_dups_size":0,"ondisk_log_size":5905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112623616,"num_objects":1718,"num_object_clones":0,"num_object_copies":5154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3498,"num_write_kb":166944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":450,"num_bytes_recovered":29523968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[5,7],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":1190},{"shards":"0,5,7","objects":524},{"shards":"5,7","objects":-180}],"blocked_by":[],"up_primary":7,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":929,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.257639+0000","last_change":"2023-10-09T05:48:54.592642+0000","last_active":"2023-10-09T05:57:18.257639+0000","last_peered":"2023-10-09T05:57:18.257639+0000","last_clean":"2023-10-09T05:57:18.257639+0000","last_became_active":"2023-10-09T05:48:54.491917+0000","last_became_peered":"2023-10-09T05:48:54.491917+0000","last_unstale":"2023-10-09T05:57:18.257639+0000","last_undegraded":"2023-10-09T05:57:18.257639+0000","last_fullsized":"2023-10-09T05:57:18.257639+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":386,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.269812+0000","last_change":"2023-10-09T05:56:11.353151+0000","last_active":"2023-10-09T05:57:18.269812+0000","last_peered":"2023-10-09T05:57:18.269812+0000","last_clean":"2023-10-09T05:57:18.269812+0000","last_became_active":"2023-10-09T05:56:11.352842+0000","last_became_peered":"2023-10-09T05:56:11.352842+0000","last_unstale":"2023-10-09T05:57:18.269812+0000","last_undegraded":"2023-10-09T05:57:18.269812+0000","last_fullsized":"2023-10-09T05:57:18.269812+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":265,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.246103+0000","last_change":"2023-10-09T05:56:00.228904+0000","last_active":"2023-10-09T05:57:18.246103+0000","last_peered":"2023-10-09T05:57:18.246103+0000","last_clean":"2023-10-09T05:57:18.246103+0000","last_became_active":"2023-10-09T05:56:00.228568+0000","last_became_peered":"2023-10-09T05:56:00.228568+0000","last_unstale":"2023-10-09T05:57:18.246103+0000","last_undegraded":"2023-10-09T05:57:18.246103+0000","last_fullsized":"2023-10-09T05:57:18.246103+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:18.837848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"439'7134","reported_seq":12905,"reported_epoch":439,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:57:33.772803+0000","last_change":"2023-10-09T05:56:49.277681+0000","last_active":"2023-10-09T05:57:33.772803+0000","last_peered":"2023-10-09T05:57:33.772803+0000","last_clean":"2023-10-09T05:55:25.212985+0000","last_became_active":"2023-10-09T05:56:49.259166+0000","last_became_peered":"2023-10-09T05:56:49.259166+0000","last_unstale":"2023-10-09T05:57:33.772803+0000","last_undegraded":"2023-10-09T05:57:33.772803+0000","last_fullsized":"2023-10-09T05:57:33.772803+0000","mapping_epoch":428,"log_start":"368'592","ondisk_log_start":"368'592","created":364,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6542,"log_dups_size":0,"ondisk_log_size":6542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227540992,"num_objects":3472,"num_object_clones":0,"num_object_copies":10416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3472,"num_objects_unfound":0,"num_objects_dirty":3472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7134,"num_write_kb":339392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2829,"num_bytes_recovered":147193856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[5,1,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":3472}],"blocked_by":[],"up_primary":7,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":785,"reported_epoch":438,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:57:18.245524+0000","last_change":"2023-10-09T05:56:11.337498+0000","last_active":"2023-10-09T05:57:18.245524+0000","last_peered":"2023-10-09T05:57:18.245524+0000","last_clean":"2023-10-09T05:56:02.545242+0000","last_became_active":"2023-10-09T05:56:11.336951+0000","last_became_peered":"2023-10-09T05:56:11.336951+0000","last_unstale":"2023-10-09T05:57:18.245524+0000","last_undegraded":"2023-10-09T05:56:11.323856+0000","last_fullsized":"2023-10-09T05:57:18.245524+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:54.695500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":["2","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":222,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.248297+0000","last_change":"2023-10-09T05:50:18.063466+0000","last_active":"2023-10-09T05:57:18.248297+0000","last_peered":"2023-10-09T05:57:18.248297+0000","last_clean":"2023-10-09T05:57:18.248297+0000","last_became_active":"2023-10-09T05:50:18.063120+0000","last_became_peered":"2023-10-09T05:50:18.063120+0000","last_unstale":"2023-10-09T05:57:18.248297+0000","last_undegraded":"2023-10-09T05:57:18.248297+0000","last_fullsized":"2023-10-09T05:57:18.248297+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:13.683314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":260,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.263829+0000","last_change":"2023-10-09T05:56:49.267961+0000","last_active":"2023-10-09T05:57:18.263829+0000","last_peered":"2023-10-09T05:57:18.263829+0000","last_clean":"2023-10-09T05:57:18.263829+0000","last_became_active":"2023-10-09T05:56:49.267614+0000","last_became_peered":"2023-10-09T05:56:49.267614+0000","last_unstale":"2023-10-09T05:57:18.263829+0000","last_undegraded":"2023-10-09T05:57:18.263829+0000","last_fullsized":"2023-10-09T05:57:18.263829+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"439'5908","reported_seq":6935,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:34.566439+0000","last_change":"2023-10-09T05:56:50.718337+0000","last_active":"2023-10-09T05:57:34.566439+0000","last_peered":"2023-10-09T05:57:34.566439+0000","last_clean":"2023-10-09T05:55:49.071451+0000","last_became_active":"2023-10-09T05:56:50.673096+0000","last_became_peered":"2023-10-09T05:56:50.673096+0000","last_unstale":"2023-10-09T05:57:34.566439+0000","last_undegraded":"2023-10-09T05:56:50.468858+0000","last_fullsized":"2023-10-09T05:56:50.237380+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5908,"log_dups_size":0,"ondisk_log_size":5908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113967104,"num_objects":1739,"num_object_clones":0,"num_object_copies":5217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4707,"num_objects_misplaced":2434,"num_objects_unfound":0,"num_objects_dirty":1739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3495,"num_write_kb":167488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":315,"num_bytes_recovered":20578304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[5,7],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":1710},{"shards":"5,7","objects":29}],"blocked_by":[],"up_primary":7,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":1137,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.257153+0000","last_change":"2023-10-09T05:56:00.632765+0000","last_active":"2023-10-09T05:57:18.257153+0000","last_peered":"2023-10-09T05:57:18.257153+0000","last_clean":"2023-10-09T05:57:18.257153+0000","last_became_active":"2023-10-09T05:55:40.295413+0000","last_became_peered":"2023-10-09T05:55:40.295413+0000","last_unstale":"2023-10-09T05:57:18.257153+0000","last_undegraded":"2023-10-09T05:57:18.257153+0000","last_fullsized":"2023-10-09T05:57:18.257153+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":888,"num_read_kb":803,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":576,"reported_epoch":438,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:18.248382+0000","last_change":"2023-10-09T05:56:16.297651+0000","last_active":"2023-10-09T05:57:18.248382+0000","last_peered":"2023-10-09T05:57:18.248382+0000","last_clean":"2023-10-09T05:56:02.257688+0000","last_became_active":"2023-10-09T05:56:16.262977+0000","last_became_peered":"2023-10-09T05:56:16.262977+0000","last_unstale":"2023-10-09T05:57:18.248382+0000","last_undegraded":"2023-10-09T05:56:16.181686+0000","last_fullsized":"2023-10-09T05:56:16.146296+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","0","6"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":324,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.257537+0000","last_change":"2023-10-09T05:56:00.243281+0000","last_active":"2023-10-09T05:57:18.257537+0000","last_peered":"2023-10-09T05:57:18.257537+0000","last_clean":"2023-10-09T05:57:18.257537+0000","last_became_active":"2023-10-09T05:56:00.242999+0000","last_became_peered":"2023-10-09T05:56:00.242999+0000","last_unstale":"2023-10-09T05:57:18.257537+0000","last_undegraded":"2023-10-09T05:57:18.257537+0000","last_fullsized":"2023-10-09T05:57:18.257537+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.274154+0000","last_change":"2023-10-09T05:55:26.336440+0000","last_active":"2023-10-09T05:57:18.274154+0000","last_peered":"2023-10-09T05:57:18.274154+0000","last_clean":"2023-10-09T05:57:18.274154+0000","last_became_active":"2023-10-09T05:55:26.335530+0000","last_became_peered":"2023-10-09T05:55:26.335530+0000","last_unstale":"2023-10-09T05:57:18.274154+0000","last_undegraded":"2023-10-09T05:57:18.274154+0000","last_fullsized":"2023-10-09T05:57:18.274154+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"439'7150","reported_seq":7334,"reported_epoch":439,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T05:57:34.511876+0000","last_change":"2023-10-09T05:56:48.266044+0000","last_active":"2023-10-09T05:57:34.511876+0000","last_peered":"2023-10-09T05:57:34.511876+0000","last_clean":"2023-10-09T05:56:10.073783+0000","last_became_active":"2023-10-09T05:56:12.516093+0000","last_became_peered":"2023-10-09T05:56:12.516093+0000","last_unstale":"2023-10-09T05:57:34.511876+0000","last_undegraded":"2023-10-09T05:57:34.511876+0000","last_fullsized":"2023-10-09T05:57:34.511876+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7150,"log_dups_size":0,"ondisk_log_size":7150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:46.643480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226885632,"num_objects":3462,"num_object_clones":0,"num_object_copies":10386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5261,"num_objects_unfound":0,"num_objects_dirty":3462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7150,"num_write_kb":339584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":2949120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[0,6,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":-622}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":948,"reported_epoch":438,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:18.248589+0000","last_change":"2023-10-09T05:56:16.297927+0000","last_active":"2023-10-09T05:57:18.248589+0000","last_peered":"2023-10-09T05:57:18.248589+0000","last_clean":"2023-10-09T05:56:02.257755+0000","last_became_active":"2023-10-09T05:56:16.263458+0000","last_became_peered":"2023-10-09T05:56:16.263458+0000","last_unstale":"2023-10-09T05:57:18.248589+0000","last_undegraded":"2023-10-09T05:56:16.199131+0000","last_fullsized":"2023-10-09T05:56:16.146936+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.248069+0000","last_change":"2023-10-09T05:56:49.283609+0000","last_active":"2023-10-09T05:57:18.248069+0000","last_peered":"2023-10-09T05:57:18.248069+0000","last_clean":"2023-10-09T05:57:18.248069+0000","last_became_active":"2023-10-09T05:56:49.283281+0000","last_became_peered":"2023-10-09T05:56:49.283281+0000","last_unstale":"2023-10-09T05:57:18.248069+0000","last_undegraded":"2023-10-09T05:57:18.248069+0000","last_fullsized":"2023-10-09T05:57:18.248069+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":316,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.246952+0000","last_change":"2023-10-09T05:56:11.340751+0000","last_active":"2023-10-09T05:57:18.246952+0000","last_peered":"2023-10-09T05:57:18.246952+0000","last_clean":"2023-10-09T05:57:18.246952+0000","last_became_active":"2023-10-09T05:56:11.340499+0000","last_became_peered":"2023-10-09T05:56:11.340499+0000","last_unstale":"2023-10-09T05:57:18.246952+0000","last_undegraded":"2023-10-09T05:57:18.246952+0000","last_fullsized":"2023-10-09T05:57:18.246952+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:36.120602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"439'7074","reported_seq":9746,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:34.813470+0000","last_change":"2023-10-09T05:56:19.111667+0000","last_active":"2023-10-09T05:57:34.813470+0000","last_peered":"2023-10-09T05:57:34.813470+0000","last_clean":"2023-10-09T05:53:45.896176+0000","last_became_active":"2023-10-09T05:55:38.711627+0000","last_became_peered":"2023-10-09T05:55:38.711627+0000","last_unstale":"2023-10-09T05:57:34.813470+0000","last_undegraded":"2023-10-09T05:56:19.081247+0000","last_fullsized":"2023-10-09T05:55:27.232363+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:26.171175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220856320,"num_objects":3370,"num_object_clones":0,"num_object_copies":10110,"num_objects_missing_on_primary":1574,"num_objects_missing":1574,"num_objects_degraded":1574,"num_objects_misplaced":1713,"num_objects_unfound":0,"num_objects_dirty":3370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7074,"num_write_kb":334208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1270,"num_bytes_recovered":82968576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[0,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1","objects":1},{"shards":"0,1,4","objects":2354},{"shards":"0,1,4,7","objects":12},{"shards":"1,4,7","objects":1003}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":608,"reported_epoch":438,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:18.303928+0000","last_change":"2023-10-09T05:56:12.339234+0000","last_active":"2023-10-09T05:57:18.303928+0000","last_peered":"2023-10-09T05:57:18.303928+0000","last_clean":"2023-10-09T05:56:02.265149+0000","last_became_active":"2023-10-09T05:56:12.338651+0000","last_became_peered":"2023-10-09T05:56:12.338651+0000","last_unstale":"2023-10-09T05:57:18.303928+0000","last_undegraded":"2023-10-09T05:56:12.325644+0000","last_fullsized":"2023-10-09T05:56:12.324625+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":240,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.246252+0000","last_change":"2023-10-09T05:51:43.127835+0000","last_active":"2023-10-09T05:57:18.246252+0000","last_peered":"2023-10-09T05:57:18.246252+0000","last_clean":"2023-10-09T05:57:18.246252+0000","last_became_active":"2023-10-09T05:51:43.127489+0000","last_became_peered":"2023-10-09T05:51:43.127489+0000","last_unstale":"2023-10-09T05:57:18.246252+0000","last_undegraded":"2023-10-09T05:57:18.246252+0000","last_fullsized":"2023-10-09T05:57:18.246252+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:20.191774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":419,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.257063+0000","last_change":"2023-10-09T05:56:11.340202+0000","last_active":"2023-10-09T05:57:18.257063+0000","last_peered":"2023-10-09T05:57:18.257063+0000","last_clean":"2023-10-09T05:57:18.257063+0000","last_became_active":"2023-10-09T05:56:11.339869+0000","last_became_peered":"2023-10-09T05:56:11.339869+0000","last_unstale":"2023-10-09T05:57:18.257063+0000","last_undegraded":"2023-10-09T05:57:18.257063+0000","last_fullsized":"2023-10-09T05:57:18.257063+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"439'7038","reported_seq":8433,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:31.507626+0000","last_change":"2023-10-09T05:57:02.142258+0000","last_active":"2023-10-09T05:57:31.507626+0000","last_peered":"2023-10-09T05:57:31.507626+0000","last_clean":"2023-10-09T05:53:45.883404+0000","last_became_active":"2023-10-09T05:57:01.676028+0000","last_became_peered":"2023-10-09T05:57:01.676028+0000","last_unstale":"2023-10-09T05:57:31.507626+0000","last_undegraded":"2023-10-09T05:57:01.577874+0000","last_fullsized":"2023-10-09T05:56:50.245097+0000","mapping_epoch":429,"log_start":"369'1961","ondisk_log_start":"369'1961","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5077,"log_dups_size":0,"ondisk_log_size":5077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224854040,"num_objects":3432,"num_object_clones":0,"num_object_copies":10296,"num_objects_missing_on_primary":3238,"num_objects_missing":3238,"num_objects_degraded":5891,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3432,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7038,"num_write_kb":334977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":641,"num_bytes_recovered":42008576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,4,5","objects":2308},{"shards":"1,5,7","objects":1052},{"shards":"5,7","objects":-240}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":89,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.249798+0000","last_change":"2023-10-09T05:56:48.323936+0000","last_active":"2023-10-09T05:57:18.249798+0000","last_peered":"2023-10-09T05:57:18.249798+0000","last_clean":"2023-10-09T05:57:18.249798+0000","last_became_active":"2023-10-09T05:56:11.340483+0000","last_became_peered":"2023-10-09T05:56:11.340483+0000","last_unstale":"2023-10-09T05:57:18.249798+0000","last_undegraded":"2023-10-09T05:57:18.249798+0000","last_fullsized":"2023-10-09T05:57:18.249798+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":523,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.352078+0000","last_change":"2023-10-09T05:54:59.255889+0000","last_active":"2023-10-09T05:57:18.352078+0000","last_peered":"2023-10-09T05:57:18.352078+0000","last_clean":"2023-10-09T05:57:18.352078+0000","last_became_active":"2023-10-09T05:54:59.255366+0000","last_became_peered":"2023-10-09T05:54:59.255366+0000","last_unstale":"2023-10-09T05:57:18.352078+0000","last_undegraded":"2023-10-09T05:57:18.352078+0000","last_fullsized":"2023-10-09T05:57:18.352078+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:22.233818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":82,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.248095+0000","last_change":"2023-10-09T05:56:49.289178+0000","last_active":"2023-10-09T05:57:18.248095+0000","last_peered":"2023-10-09T05:57:18.248095+0000","last_clean":"2023-10-09T05:57:18.248095+0000","last_became_active":"2023-10-09T05:56:49.286277+0000","last_became_peered":"2023-10-09T05:56:49.286277+0000","last_unstale":"2023-10-09T05:57:18.248095+0000","last_undegraded":"2023-10-09T05:57:18.248095+0000","last_fullsized":"2023-10-09T05:57:18.248095+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"439'7109","reported_seq":10493,"reported_epoch":439,"state":"active+recovering","last_fresh":"2023-10-09T05:57:31.566048+0000","last_change":"2023-10-09T05:56:49.370077+0000","last_active":"2023-10-09T05:57:31.566048+0000","last_peered":"2023-10-09T05:57:31.566048+0000","last_clean":"2023-10-09T05:53:45.907519+0000","last_became_active":"2023-10-09T05:56:03.326563+0000","last_became_peered":"2023-10-09T05:56:03.326563+0000","last_unstale":"2023-10-09T05:57:31.566048+0000","last_undegraded":"2023-10-09T05:57:31.566048+0000","last_fullsized":"2023-10-09T05:57:31.566048+0000","mapping_epoch":416,"log_start":"369'1532","ondisk_log_start":"369'1532","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5577,"log_dups_size":0,"ondisk_log_size":5577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228130816,"num_objects":3481,"num_object_clones":0,"num_object_copies":10443,"num_objects_missing_on_primary":2374,"num_objects_missing":2374,"num_objects_degraded":0,"num_objects_misplaced":1425,"num_objects_unfound":0,"num_objects_dirty":3481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7109,"num_write_kb":338880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1627,"num_bytes_recovered":106561536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["4"],"object_location_counts":[{"shards":"0,3,4","objects":1425},{"shards":"0,3,4,7","objects":1},{"shards":"3,4,7","objects":2055}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":609,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.249709+0000","last_change":"2023-10-09T05:55:55.247195+0000","last_active":"2023-10-09T05:57:18.249709+0000","last_peered":"2023-10-09T05:57:18.249709+0000","last_clean":"2023-10-09T05:57:18.249709+0000","last_became_active":"2023-10-09T05:55:55.246499+0000","last_became_peered":"2023-10-09T05:55:55.246499+0000","last_unstale":"2023-10-09T05:57:18.249709+0000","last_undegraded":"2023-10-09T05:57:18.249709+0000","last_fullsized":"2023-10-09T05:57:18.249709+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":33,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.249137+0000","last_change":"2023-10-09T05:56:48.410914+0000","last_active":"2023-10-09T05:57:18.249137+0000","last_peered":"2023-10-09T05:57:18.249137+0000","last_clean":"2023-10-09T05:57:18.249137+0000","last_became_active":"2023-10-09T05:56:11.349646+0000","last_became_peered":"2023-10-09T05:56:11.349646+0000","last_unstale":"2023-10-09T05:57:18.249137+0000","last_undegraded":"2023-10-09T05:57:18.249137+0000","last_fullsized":"2023-10-09T05:57:18.249137+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":374,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.259439+0000","last_change":"2023-10-09T05:55:39.368525+0000","last_active":"2023-10-09T05:57:18.259439+0000","last_peered":"2023-10-09T05:57:18.259439+0000","last_clean":"2023-10-09T05:57:18.259439+0000","last_became_active":"2023-10-09T05:55:39.368203+0000","last_became_peered":"2023-10-09T05:55:39.368203+0000","last_unstale":"2023-10-09T05:57:18.259439+0000","last_undegraded":"2023-10-09T05:57:18.259439+0000","last_fullsized":"2023-10-09T05:57:18.259439+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.11","version":"439'6017","reported_seq":15089,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T05:57:31.498240+0000","last_change":"2023-10-09T05:57:14.256062+0000","last_active":"2023-10-09T05:57:31.498240+0000","last_peered":"2023-10-09T05:57:31.498240+0000","last_clean":"2023-10-09T05:57:31.498240+0000","last_became_active":"2023-10-09T05:57:14.235091+0000","last_became_peered":"2023-10-09T05:57:14.235091+0000","last_unstale":"2023-10-09T05:57:31.498240+0000","last_undegraded":"2023-10-09T05:57:31.498240+0000","last_fullsized":"2023-10-09T05:57:31.498240+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6017,"log_dups_size":0,"ondisk_log_size":6017,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114524160,"num_objects":1748,"num_object_clones":0,"num_object_copies":5244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3564,"num_write_kb":169952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3255,"num_bytes_recovered":213286912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":429,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.249080+0000","last_change":"2023-10-09T05:56:49.265734+0000","last_active":"2023-10-09T05:57:18.249080+0000","last_peered":"2023-10-09T05:57:18.249080+0000","last_clean":"2023-10-09T05:57:18.249080+0000","last_became_active":"2023-10-09T05:56:49.264595+0000","last_became_peered":"2023-10-09T05:56:49.264595+0000","last_unstale":"2023-10-09T05:57:18.249080+0000","last_undegraded":"2023-10-09T05:57:18.249080+0000","last_fullsized":"2023-10-09T05:57:18.249080+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":130,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.246176+0000","last_change":"2023-10-09T05:56:00.226695+0000","last_active":"2023-10-09T05:57:18.246176+0000","last_peered":"2023-10-09T05:57:18.246176+0000","last_clean":"2023-10-09T05:57:18.246176+0000","last_became_active":"2023-10-09T05:56:00.226359+0000","last_became_peered":"2023-10-09T05:56:00.226359+0000","last_unstale":"2023-10-09T05:57:18.246176+0000","last_undegraded":"2023-10-09T05:57:18.246176+0000","last_fullsized":"2023-10-09T05:57:18.246176+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:20.323704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"439'5929","reported_seq":6028,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T05:57:32.302628+0000","last_change":"2023-10-09T05:55:11.328085+0000","last_active":"2023-10-09T05:57:32.302628+0000","last_peered":"2023-10-09T05:57:32.302628+0000","last_clean":"2023-10-09T05:57:32.302628+0000","last_became_active":"2023-10-09T05:55:11.322136+0000","last_became_peered":"2023-10-09T05:55:11.322136+0000","last_unstale":"2023-10-09T05:57:32.302628+0000","last_undegraded":"2023-10-09T05:57:32.302628+0000","last_fullsized":"2023-10-09T05:57:32.302628+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5929,"log_dups_size":0,"ondisk_log_size":5929,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109969408,"num_objects":1678,"num_object_clones":0,"num_object_copies":5034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3530,"num_write_kb":166656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":434,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.256676+0000","last_change":"2023-10-09T05:48:54.583404+0000","last_active":"2023-10-09T05:57:18.256676+0000","last_peered":"2023-10-09T05:57:18.256676+0000","last_clean":"2023-10-09T05:57:18.256676+0000","last_became_active":"2023-10-09T05:48:54.492565+0000","last_became_peered":"2023-10-09T05:48:54.492565+0000","last_unstale":"2023-10-09T05:57:18.256676+0000","last_undegraded":"2023-10-09T05:57:18.256676+0000","last_fullsized":"2023-10-09T05:57:18.256676+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":21,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.247207+0000","last_change":"2023-10-09T05:56:49.252225+0000","last_active":"2023-10-09T05:57:18.247207+0000","last_peered":"2023-10-09T05:57:18.247207+0000","last_clean":"2023-10-09T05:57:18.247207+0000","last_became_active":"2023-10-09T05:56:49.251880+0000","last_became_peered":"2023-10-09T05:56:49.251880+0000","last_unstale":"2023-10-09T05:57:18.247207+0000","last_undegraded":"2023-10-09T05:57:18.247207+0000","last_fullsized":"2023-10-09T05:57:18.247207+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:34.079170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.17","version":"439'5986","reported_seq":12544,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T05:57:31.446465+0000","last_change":"2023-10-09T05:57:10.249804+0000","last_active":"2023-10-09T05:57:31.446465+0000","last_peered":"2023-10-09T05:57:31.446465+0000","last_clean":"2023-10-09T05:57:31.446465+0000","last_became_active":"2023-10-09T05:57:10.235053+0000","last_became_peered":"2023-10-09T05:57:10.235053+0000","last_unstale":"2023-10-09T05:57:31.446465+0000","last_undegraded":"2023-10-09T05:57:31.446465+0000","last_fullsized":"2023-10-09T05:57:31.446465+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5986,"log_dups_size":0,"ondisk_log_size":5986,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111017984,"num_objects":1694,"num_object_clones":0,"num_object_copies":5082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3562,"num_write_kb":168192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3182,"num_bytes_recovered":157876224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":393,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.261849+0000","last_change":"2023-10-09T05:21:16.776971+0000","last_active":"2023-10-09T05:57:18.261849+0000","last_peered":"2023-10-09T05:57:18.261849+0000","last_clean":"2023-10-09T05:57:18.261849+0000","last_became_active":"2023-10-09T05:21:16.776040+0000","last_became_peered":"2023-10-09T05:21:16.776040+0000","last_unstale":"2023-10-09T05:57:18.261849+0000","last_undegraded":"2023-10-09T05:57:18.261849+0000","last_fullsized":"2023-10-09T05:57:18.261849+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":30,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.245907+0000","last_change":"2023-10-09T05:56:11.337104+0000","last_active":"2023-10-09T05:57:18.245907+0000","last_peered":"2023-10-09T05:57:18.245907+0000","last_clean":"2023-10-09T05:57:18.245907+0000","last_became_active":"2023-10-09T05:56:11.336690+0000","last_became_peered":"2023-10-09T05:56:11.336690+0000","last_unstale":"2023-10-09T05:57:18.245907+0000","last_undegraded":"2023-10-09T05:57:18.245907+0000","last_fullsized":"2023-10-09T05:57:18.245907+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:56.369633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.14","version":"439'5954","reported_seq":6070,"reported_epoch":439,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:57:32.215609+0000","last_change":"2023-10-09T05:56:01.281023+0000","last_active":"2023-10-09T05:57:32.215609+0000","last_peered":"2023-10-09T05:57:32.215609+0000","last_clean":"2023-10-09T05:55:48.964110+0000","last_became_active":"2023-10-09T05:56:01.245699+0000","last_became_peered":"2023-10-09T05:56:01.245699+0000","last_unstale":"2023-10-09T05:57:32.215609+0000","last_undegraded":"2023-10-09T05:56:01.216362+0000","last_fullsized":"2023-10-09T05:57:32.215609+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5954,"log_dups_size":0,"ondisk_log_size":5954,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114688000,"num_objects":1750,"num_object_clones":0,"num_object_copies":5250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":40,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3524,"num_write_kb":168768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1750}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":178,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.247105+0000","last_change":"2023-10-09T05:56:00.221032+0000","last_active":"2023-10-09T05:57:18.247105+0000","last_peered":"2023-10-09T05:57:18.247105+0000","last_clean":"2023-10-09T05:57:18.247105+0000","last_became_active":"2023-10-09T05:56:00.220693+0000","last_became_peered":"2023-10-09T05:56:00.220693+0000","last_unstale":"2023-10-09T05:57:18.247105+0000","last_undegraded":"2023-10-09T05:57:18.247105+0000","last_fullsized":"2023-10-09T05:57:18.247105+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:55.448253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":269,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.245411+0000","last_change":"2023-10-09T05:56:11.354072+0000","last_active":"2023-10-09T05:57:18.245411+0000","last_peered":"2023-10-09T05:57:18.245411+0000","last_clean":"2023-10-09T05:57:18.245411+0000","last_became_active":"2023-10-09T05:56:11.353547+0000","last_became_peered":"2023-10-09T05:56:11.353547+0000","last_unstale":"2023-10-09T05:57:18.245411+0000","last_undegraded":"2023-10-09T05:57:18.245411+0000","last_fullsized":"2023-10-09T05:57:18.245411+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:42.629830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"439'6012","reported_seq":7064,"reported_epoch":439,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:57:33.362191+0000","last_change":"2023-10-09T05:56:12.557070+0000","last_active":"2023-10-09T05:57:33.362191+0000","last_peered":"2023-10-09T05:57:33.362191+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T05:56:12.536142+0000","last_became_peered":"2023-10-09T05:56:12.536142+0000","last_unstale":"2023-10-09T05:57:33.362191+0000","last_undegraded":"2023-10-09T05:56:12.474066+0000","last_fullsized":"2023-10-09T05:56:12.424748+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6012,"log_dups_size":0,"ondisk_log_size":6012,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114327552,"num_objects":1745,"num_object_clones":0,"num_object_copies":5235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2864,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3571,"num_write_kb":170080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":287,"num_bytes_recovered":18808832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,1,5","objects":1127},{"shards":"0,5,7","objects":527},{"shards":"5,7","objects":-211}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":352,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.246632+0000","last_change":"2023-10-09T05:56:11.384692+0000","last_active":"2023-10-09T05:57:18.246632+0000","last_peered":"2023-10-09T05:57:18.246632+0000","last_clean":"2023-10-09T05:57:18.246632+0000","last_became_active":"2023-10-09T05:56:11.346444+0000","last_became_peered":"2023-10-09T05:56:11.346444+0000","last_unstale":"2023-10-09T05:57:18.246632+0000","last_undegraded":"2023-10-09T05:57:18.246632+0000","last_fullsized":"2023-10-09T05:57:18.246632+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:10.451624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":422,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T05:57:18.267949+0000","last_change":"2023-10-09T05:48:54.492580+0000","last_active":"2023-10-09T05:57:18.267949+0000","last_peered":"2023-10-09T05:57:18.267949+0000","last_clean":"2023-10-09T05:57:18.267949+0000","last_became_active":"2023-10-09T05:48:54.492259+0000","last_became_peered":"2023-10-09T05:48:54.492259+0000","last_unstale":"2023-10-09T05:57:18.267949+0000","last_undegraded":"2023-10-09T05:57:18.267949+0000","last_fullsized":"2023-10-09T05:57:18.267949+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":15826,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17226,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":4},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":3615752216,"num_objects":55173,"num_object_clones":0,"num_object_copies":165519,"num_objects_missing_on_primary":7186,"num_objects_missing":7186,"num_objects_degraded":46244,"num_objects_misplaced":18530,"num_objects_unfound":0,"num_objects_dirty":55173,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":113169,"num_write_kb":5386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23300,"num_bytes_recovered":1394933760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11940671488,"data_stored":11940659272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156807,"ondisk_log_size":156807,"up":78,"acting":64,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":46,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846865,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2273896,"kb_used_data":1355872,"kb_used_omap":14,"kb_used_meta":918001,"kb_avail":91471256,"statfs":{"total":95995035648,"available":93666566144,"internally_reserved":0,"allocated":1388412928,"data_stored":1387535600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":940033882},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,0,1,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:23 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":5.274,"15min":4.8200000000000003},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.20699999999999999},"max":{"1min":1.1180000000000001,"5min":212.64400000000001,"15min":212.64400000000001},"last":0.316},{"interface":"front","average":{"1min":0.82599999999999996,"5min":5.3920000000000003,"15min":4.8230000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.245},"max":{"1min":4.9370000000000003,"5min":212.655,"15min":212.655},"last":0.38800000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:57:23 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":2.7010000000000001,"15min":2.581},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":5.2300000000000004,"5min":78.013000000000005,"15min":114.422},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.46600000000000003,"5min":2.6960000000000002,"15min":2.6629999999999998},"min":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.255},"max":{"1min":0.70599999999999996,"5min":78.174999999999997,"15min":111.708},"last":0.34499999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:57:23 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":3.6989999999999998,"15min":2.5489999999999999},"min":{"1min":0.311,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":7.1369999999999996,"5min":205.863,"15min":205.863},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.66300000000000003,"5min":3.4420000000000002,"15min":2.415},"min":{"1min":0.311,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":3.1989999999999998,"5min":205.88399999999999,"15min":205.88399999999999},"last":0.47999999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:57:23 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":2.895,"15min":3.819},"min":{"1min":0.373,"5min":0.315,"15min":0.28399999999999997},"max":{"1min":3.0209999999999999,"5min":115.69799999999999,"15min":115.69799999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.93700000000000006,"5min":3.024,"15min":4.0250000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29299999999999998},"max":{"1min":4.8680000000000003,"5min":115.75700000000001,"15min":115.75700000000001},"last":0.46000000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:57:23 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":1.6679999999999999,"15min":1.909},"min":{"1min":0.312,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":0.67500000000000004,"5min":78.963999999999999,"15min":159.50299999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.5980000000000001,"15min":1.9219999999999999},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.27800000000000002},"max":{"1min":4.9619999999999997,"5min":79.099999999999994,"15min":159.483},"last":0.35499999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:57:23 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":2.7010000000000001,"15min":1.6870000000000001},"min":{"1min":0.29699999999999999,"5min":0.29399999999999998,"15min":0.27800000000000002},"max":{"1min":4.9889999999999999,"5min":94.093999999999994,"15min":94.093999999999994},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":2.863,"15min":1.8140000000000001},"min":{"1min":0.36199999999999999,"5min":0.312,"15min":0.308},"max":{"1min":4.6909999999999998,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.63600000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:57:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":3.1080000000000001,"15min":1.554},"min":{"1min":0.35099999999999998,"5min":0.32900000000000001,"15min":0.29299999999999998},"max":{"1min":6.2779999999999996,"5min":209.322,"15min":209.322},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":3.085,"15min":1.649},"min":{"1min":0.38,"5min":0.30599999999999999,"15min":0.26700000000000002},"max":{"1min":4.5190000000000001,"5min":209.19800000000001,"15min":209.19800000000001},"last":0.67000000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626894,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1570528,"kb_used_data":670488,"kb_used_omap":0,"kb_used_meta":900032,"kb_avail":92174624,"statfs":{"total":95995035648,"available":94386814976,"internally_reserved":0,"allocated":686579712,"data_stored":685710738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921632768},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:54 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.744,"15min":2.8220000000000001},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.20999999999999999},"max":{"1min":3.8029999999999999,"5min":50.427,"15min":162.08799999999999},"last":0.254},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.7450000000000001,"15min":2.8319999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.22700000000000001},"max":{"1min":6.8129999999999997,"5min":50.634999999999998,"15min":161.90899999999999},"last":0.82299999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:57:23 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":5.202,"15min":2.9780000000000002},"min":{"1min":0.30399999999999999,"5min":0.28199999999999997,"15min":0.26000000000000001},"max":{"1min":1.0629999999999999,"5min":165.66399999999999,"15min":165.66399999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.45400000000000001,"5min":5.173,"15min":2.9980000000000002},"min":{"1min":0.317,"5min":0.308,"15min":0.26000000000000001},"max":{"1min":1.0700000000000001,"5min":165.649,"15min":165.649},"last":0.309}]},{"osd":2,"last update":"Mon Oct 9 05:56:54 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":2.1179999999999999,"15min":2.016},"min":{"1min":0.35499999999999998,"5min":0.29599999999999999,"15min":0.27200000000000002},"max":{"1min":3.6880000000000002,"5min":91.049999999999997,"15min":91.049999999999997},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.75600000000000001,"5min":2.1760000000000002,"15min":2.0880000000000001},"min":{"1min":0.35599999999999998,"5min":0.33600000000000002,"15min":0.29499999999999998},"max":{"1min":3.6400000000000001,"5min":91.105000000000004,"15min":91.105000000000004},"last":0.54900000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:57:23 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":4.4530000000000003,"15min":3.673},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.2690000000000001,"5min":152.74700000000001,"15min":152.74700000000001},"last":1.1830000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":4.0019999999999998,"15min":3.714},"min":{"1min":0.38900000000000001,"5min":0.313,"15min":0.313},"max":{"1min":1.2430000000000001,"5min":152.78100000000001,"15min":152.78100000000001},"last":0.499}]},{"osd":4,"last update":"Mon Oct 9 05:56:41 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.073,"15min":1.778},"min":{"1min":0.315,"5min":0.27400000000000002,"15min":0.248},"max":{"1min":1.3640000000000001,"5min":20.367999999999999,"15min":257.375},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.3129999999999999,"15min":2.052},"min":{"1min":0.32700000000000001,"5min":0.26900000000000002,"15min":0.246},"max":{"1min":6.9279999999999999,"5min":19.745000000000001,"15min":257.61200000000002},"last":0.89200000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:56:41 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":2.484,"15min":2.3359999999999999},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.27800000000000002},"max":{"1min":5.1459999999999999,"5min":87.828999999999994,"15min":256.05700000000002},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.98599999999999999,"5min":2.702,"15min":2.524},"min":{"1min":0.36599999999999999,"5min":0.30399999999999999,"15min":0.27500000000000002},"max":{"1min":6.867,"5min":89.352999999999994,"15min":256.08499999999998},"last":0.86299999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:57:23 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":3.3919999999999999,"15min":1.6599999999999999},"min":{"1min":0.33000000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.73599999999999999,"5min":204.15299999999999,"15min":204.15299999999999},"last":0.75},{"interface":"front","average":{"1min":0.50900000000000001,"5min":3.4129999999999998,"15min":1.8979999999999999},"min":{"1min":0.34200000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.0840000000000001,"5min":203.85499999999999,"15min":203.85499999999999},"last":5.5179999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958854,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3053212,"kb_used_data":2187220,"kb_used_omap":7,"kb_used_meta":865976,"kb_avail":90691940,"statfs":{"total":95995035648,"available":92868546560,"internally_reserved":0,"allocated":2239713280,"data_stored":2238143027,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":886759950},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":2.597,"15min":3.786},"min":{"1min":0.22900000000000001,"5min":0.217,"15min":0.17499999999999999},"max":{"1min":1.2989999999999999,"5min":131.91300000000001,"15min":131.91300000000001},"last":2.052},{"interface":"front","average":{"1min":0.40899999999999997,"5min":3.1240000000000001,"15min":4.0410000000000004},"min":{"1min":0.248,"5min":0.24399999999999999,"15min":0.182},"max":{"1min":1.282,"5min":132.191,"15min":132.191},"last":2.96}]},{"osd":2,"last update":"Mon Oct 9 05:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":3.1539999999999999,"15min":2.8069999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.19800000000000001},"max":{"1min":1.841,"5min":132.35400000000001,"15min":132.35400000000001},"last":1.5389999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":2.839,"15min":2.6520000000000001},"min":{"1min":0.28399999999999997,"5min":0.27800000000000002,"15min":0.20699999999999999},"max":{"1min":0.66400000000000003,"5min":131.70599999999999,"15min":131.70599999999999},"last":1.224}]},{"osd":3,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":2.9689999999999999,"15min":3.8239999999999998},"min":{"1min":0.26400000000000001,"5min":0.25700000000000001,"15min":0.20200000000000001},"max":{"1min":22.236000000000001,"5min":135.10900000000001,"15min":135.10900000000001},"last":2.4870000000000001},{"interface":"front","average":{"1min":2.2429999999999999,"5min":2.9289999999999998,"15min":3.782},"min":{"1min":0.29199999999999998,"5min":0.248,"15min":0.248},"max":{"1min":22.913,"5min":131.886,"15min":131.886},"last":1.1870000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":2.8370000000000002,"15min":1.8680000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":0.72199999999999998,"5min":131.636,"15min":131.636},"last":2.8610000000000002},{"interface":"front","average":{"1min":0.47999999999999998,"5min":2.3809999999999998,"15min":1.6799999999999999},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67100000000000004,"5min":131.75999999999999,"15min":131.75999999999999},"last":1.2569999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":3.722,"15min":2.3620000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28599999999999998},"max":{"1min":2.2930000000000001,"5min":132.291,"15min":132.291},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.9790000000000001,"5min":4.3440000000000003,"15min":2.431},"min":{"1min":0.38500000000000001,"5min":0.38200000000000001,"15min":0.32200000000000001},"max":{"1min":22.268999999999998,"5min":131.738,"15min":131.738},"last":2.9260000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":4.0659999999999998,"15min":2.7909999999999999},"min":{"1min":0.379,"5min":0.34599999999999997,"15min":0.316},"max":{"1min":2.137,"5min":158.50399999999999,"15min":158.50399999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.71999999999999997,"5min":4.4249999999999998,"15min":2.6739999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.37},"max":{"1min":2.198,"5min":158.577,"15min":158.577},"last":2.895}]},{"osd":7,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":2.431,"15min":1.7509999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.256,"5min":131.821,"15min":131.821},"last":1.325},{"interface":"front","average":{"1min":0.77800000000000002,"5min":2.6469999999999998,"15min":1.897},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.6639999999999997,"5min":131.852,"15min":131.852},"last":1.2929999999999999}]}]},{"osd":0,"up_from":0,"seq":377957122379,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":98,"seq":420906795328,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1693524,"kb_used_data":853836,"kb_used_omap":11,"kb_used_meta":839668,"kb_avail":92051628,"statfs":{"total":95995035648,"available":94260867072,"internally_reserved":0,"allocated":874328064,"data_stored":872470809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":859820629},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:00 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":2.54,"15min":3.2040000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.17999999999999999},"max":{"1min":1.2110000000000001,"5min":140.71100000000001,"15min":162.285},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.374,"5min":3.4249999999999998,"15min":3.847},"min":{"1min":0.219,"5min":0.219,"15min":0.217},"max":{"1min":0.79700000000000004,"5min":142.542,"15min":162.643},"last":0.307}]},{"osd":1,"last update":"Mon Oct 9 05:57:00 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":2.0630000000000002,"15min":2.5750000000000002},"min":{"1min":0.247,"5min":0.23599999999999999,"15min":0.193},"max":{"1min":1.0620000000000001,"5min":38.677999999999997,"15min":125.77500000000001},"last":0.253},{"interface":"front","average":{"1min":0.64100000000000001,"5min":2.593,"15min":2.4470000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20999999999999999},"max":{"1min":4.6589999999999998,"5min":57.715000000000003,"15min":125.857},"last":0.33200000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:57:31 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":2.1840000000000002,"15min":3.6659999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.21099999999999999},"max":{"1min":4.6989999999999998,"5min":29.606999999999999,"15min":159.35499999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.5349999999999999,"5min":2.0019999999999998,"15min":3.153},"min":{"1min":0.28899999999999998,"5min":0.28499999999999998,"15min":0.217},"max":{"1min":8.907,"5min":29.920999999999999,"15min":159.196},"last":0.46999999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:57:00 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":1.429,"15min":1.4890000000000001},"min":{"1min":0.33300000000000002,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":4.7190000000000003,"5min":9.8599999999999994,"15min":68.052000000000007},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":1.081,"15min":1.3440000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.28699999999999998},"max":{"1min":1.4710000000000001,"5min":10.013,"15min":68.727000000000004},"last":0.52300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:57:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":5.46,"15min":2.9980000000000002},"min":{"1min":0.36399999999999999,"5min":0.36099999999999999,"15min":0.29099999999999998},"max":{"1min":18.350000000000001,"5min":207.80500000000001,"15min":207.80500000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.0840000000000001,"5min":5.5330000000000004,"15min":2.9199999999999999},"min":{"1min":0.378,"5min":0.371,"15min":0.30499999999999999},"max":{"1min":5.9710000000000001,"5min":207.54900000000001,"15min":207.54900000000001},"last":0.56100000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:57:00 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":1.014,"15min":1.226},"min":{"1min":0.372,"5min":0.35799999999999998,"15min":0.32000000000000001},"max":{"1min":1.278,"5min":9.9589999999999996,"15min":25.818999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.63500000000000001,"5min":1.6339999999999999,"15min":1.7},"min":{"1min":0.41899999999999998,"5min":0.40100000000000002,"15min":0.34100000000000003},"max":{"1min":1.776,"5min":8.7599999999999998,"15min":24.481000000000002},"last":0.53700000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:57:31 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":2.4649999999999999,"15min":1.9419999999999999},"min":{"1min":0.42399999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":3.4049999999999998,"5min":162.09200000000001,"15min":162.09200000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.2969999999999999,"5min":2.9460000000000002,"15min":2.327},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.373},"max":{"1min":8.7959999999999994,"5min":162.35400000000001,"15min":162.35400000000001},"last":0.48899999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631802,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3085468,"kb_used_data":2187796,"kb_used_omap":4,"kb_used_meta":897659,"kb_avail":90659684,"statfs":{"total":95995035648,"available":92835516416,"internally_reserved":0,"allocated":2240303104,"data_stored":2238673593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":919203422},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":7.0869999999999997,"15min":4.8700000000000001},"min":{"1min":0.22800000000000001,"5min":0.20899999999999999,"15min":0.184},"max":{"1min":1.47,"5min":312.56599999999997,"15min":312.56599999999997},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":7.1529999999999996,"15min":5.0359999999999996},"min":{"1min":0.216,"5min":0.216,"15min":0.20499999999999999},"max":{"1min":4.0810000000000004,"5min":311.02300000000002,"15min":311.02300000000002},"last":0.29399999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:57:15 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":6.1500000000000004,"15min":4.2640000000000002},"min":{"1min":0.24199999999999999,"5min":0.22,"15min":0.214},"max":{"1min":4.0309999999999997,"5min":142.32599999999999,"15min":150.29599999999999},"last":0.312},{"interface":"front","average":{"1min":0.754,"5min":6.2640000000000002,"15min":4.5579999999999998},"min":{"1min":0.23799999999999999,"5min":0.224,"15min":0.19},"max":{"1min":5.5890000000000004,"5min":139.33199999999999,"15min":151.25700000000001},"last":0.70999999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:57:15 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":2.6899999999999999,"15min":2.48},"min":{"1min":0.25900000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.84199999999999997,"5min":72.602000000000004,"15min":150.006},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.70099999999999996,"5min":2.395,"15min":2.3690000000000002},"min":{"1min":0.26400000000000001,"5min":0.255,"15min":0.20499999999999999},"max":{"1min":3.0649999999999999,"5min":69.254000000000005,"15min":150.65100000000001},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:57:15 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.875,"15min":2.3999999999999999},"min":{"1min":0.35799999999999998,"5min":0.35499999999999998,"15min":0.25600000000000001},"max":{"1min":1.8089999999999999,"5min":69.224999999999994,"15min":149.84700000000001},"last":0.434},{"interface":"front","average":{"1min":0.51600000000000001,"5min":1.889,"15min":2.7490000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28999999999999998},"max":{"1min":0.68700000000000006,"5min":70.439999999999998,"15min":149.74700000000001},"last":0.58199999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:57:15 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":2.5230000000000001,"15min":2.4009999999999998},"min":{"1min":0.40899999999999997,"5min":0.309,"15min":0.28199999999999997},"max":{"1min":2.48,"5min":71.203999999999994,"15min":149.43000000000001},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.63600000000000001,"5min":2.2109999999999999,"15min":2.1589999999999998},"min":{"1min":0.36599999999999999,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":2.4969999999999999,"5min":70.397000000000006,"15min":149.69300000000001},"last":0.56399999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:57:15 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":1.8340000000000001,"15min":1.974},"min":{"1min":0.42099999999999999,"5min":0.38600000000000001,"15min":0.33900000000000002},"max":{"1min":0.69499999999999995,"5min":70.474000000000004,"15min":149.72399999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.504,"5min":1.952,"15min":2.2250000000000001},"min":{"1min":0.44600000000000001,"5min":0.34599999999999997,"15min":0.307},"max":{"1min":0.59299999999999997,"5min":71.251999999999995,"15min":149.589},"last":0.74099999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:56:32 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.9770000000000001,"15min":1.986},"min":{"1min":0.42099999999999999,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":4.0949999999999998,"5min":69.277000000000001,"15min":149.77199999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.8420000000000001,"15min":2.0209999999999999},"min":{"1min":0.47699999999999998,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":3.71,"5min":69.299000000000007,"15min":149.80799999999999},"last":0.64600000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019350,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1808356,"kb_used_data":915100,"kb_used_omap":0,"kb_used_meta":893248,"kb_avail":91936796,"statfs":{"total":95995035648,"available":94143279104,"internally_reserved":0,"allocated":937062400,"data_stored":936188578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914685952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:02 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":3.5099999999999998,"15min":3.46},"min":{"1min":0.24299999999999999,"5min":0.23000000000000001,"15min":0.219},"max":{"1min":0.89600000000000002,"5min":95.466999999999999,"15min":132.35599999999999},"last":0.432},{"interface":"front","average":{"1min":1.2290000000000001,"5min":3.8740000000000001,"15min":3.7909999999999999},"min":{"1min":0.28699999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":12.462,"5min":95.367999999999995,"15min":132.501},"last":0.46600000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":3.677,"15min":2.702},"min":{"1min":0.29299999999999998,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":1.548,"5min":96.033000000000001,"15min":96.033000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.47299999999999998,"5min":3.3069999999999999,"15min":2.577},"min":{"1min":0.29099999999999998,"5min":0.255,"15min":0.255},"max":{"1min":1.405,"5min":96.022000000000006,"15min":96.022000000000006},"last":0.51600000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:56:39 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":2.5070000000000001,"15min":2.036},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":5.1639999999999997,"5min":51.768000000000001,"15min":71.748000000000005},"last":0.318},{"interface":"front","average":{"1min":4.8049999999999997,"5min":3.2480000000000002,"15min":2.2829999999999999},"min":{"1min":0.32400000000000001,"5min":0.315,"15min":0.29499999999999998},"max":{"1min":84.123999999999995,"5min":84.123999999999995,"15min":84.123999999999995},"last":0.28399999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:57:13 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":4.2469999999999999,"15min":3.2749999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":3.0960000000000001,"5min":100.009,"15min":123.59},"last":0.501},{"interface":"front","average":{"1min":0.57299999999999995,"5min":4.407,"15min":3.2269999999999999},"min":{"1min":0.34000000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.1299999999999999,"5min":100.254,"15min":123.566},"last":0.372}]},{"osd":5,"last update":"Mon Oct 9 05:57:13 2023","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.77400000000000002,"15min":0.86399999999999999},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":0.58399999999999996,"5min":8.5190000000000001,"15min":12.34},"last":0.436},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.879,"15min":0.83699999999999997},"min":{"1min":0.34599999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.91600000000000004,"5min":8.0129999999999999,"15min":10.191000000000001},"last":0.40500000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":2.9329999999999998,"15min":1.4870000000000001},"min":{"1min":0.314,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":2.1379999999999999,"5min":181.411,"15min":181.411},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":2.9399999999999999,"15min":1.5800000000000001},"min":{"1min":0.36099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.5189999999999999,"5min":180.63999999999999,"15min":180.63999999999999},"last":0.48499999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.3560000000000001,"15min":1.2629999999999999},"min":{"1min":0.33100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.1659999999999999,"5min":36.729999999999997,"15min":96.131},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.153,"15min":1.2869999999999999},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.27200000000000002},"max":{"1min":1.8779999999999999,"5min":36.424999999999997,"15min":96.465000000000003},"last":0.35799999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823122,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3131668,"kb_used_data":2232972,"kb_used_omap":0,"kb_used_meta":898688,"kb_avail":90613484,"statfs":{"total":95995035648,"available":92788207616,"internally_reserved":0,"allocated":2286563328,"data_stored":2285701338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920256512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:01 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":5.6849999999999996,"15min":5.0880000000000001},"min":{"1min":0.24399999999999999,"5min":0.22800000000000001,"15min":0.224},"max":{"1min":0.92900000000000005,"5min":167.566,"15min":210.392},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":5.9779999999999998,"15min":5.2960000000000003},"min":{"1min":0.26600000000000001,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":1.754,"5min":167.62100000000001,"15min":208.43600000000001},"last":0.5}]},{"osd":1,"last update":"Mon Oct 9 05:57:18 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":7.3860000000000001,"15min":4.5119999999999996},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.247},"max":{"1min":2.54,"5min":284.13900000000001,"15min":284.13900000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":7.2789999999999999,"15min":4.4329999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.248},"max":{"1min":0.50900000000000001,"5min":284.07900000000001,"15min":284.07900000000001},"last":0.41599999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:57:01 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":3.7599999999999998,"15min":2.258},"min":{"1min":0.29599999999999999,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":0.96099999999999997,"5min":195.92099999999999,"15min":195.92099999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":3.774,"15min":2.335},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.1319999999999999,"5min":195.905,"15min":195.905},"last":0.51900000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:57:18 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":4.6520000000000001,"15min":3.4980000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30399999999999999},"max":{"1min":2.5720000000000001,"5min":161.82300000000001,"15min":161.82300000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":4.718,"15min":3.4769999999999999},"min":{"1min":0.377,"5min":0.33800000000000002,"15min":0.308},"max":{"1min":1.9299999999999999,"5min":161.69800000000001,"15min":161.69800000000001},"last":0.47099999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:57:18 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.64500000000000002,"15min":1.214},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.23899999999999999},"max":{"1min":0.57599999999999996,"5min":7.0170000000000003,"15min":130.81},"last":0.443},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.81299999999999994,"15min":1.375},"min":{"1min":0.35799999999999998,"5min":0.27000000000000002,"15min":0.253},"max":{"1min":1.8700000000000001,"5min":9.3170000000000002,"15min":133.803},"last":0.376}]},{"osd":6,"last update":"Mon Oct 9 05:56:32 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.92400000000000004,"15min":1.127},"min":{"1min":0.35999999999999999,"5min":0.29799999999999999,"15min":0.27100000000000002},"max":{"1min":2.9540000000000002,"5min":6.8639999999999999,"15min":71.426000000000002},"last":4.1920000000000002},{"interface":"front","average":{"1min":1.212,"5min":1.032,"15min":1.2370000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.27300000000000002},"max":{"1min":11.952,"5min":11.952,"15min":71.498999999999995},"last":0.55300000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:56:32 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":1.778,"15min":1.3160000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.27400000000000002},"max":{"1min":6.3380000000000001,"5min":86.506,"15min":86.506},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.71899999999999997,"5min":2.0139999999999998,"15min":1.361},"min":{"1min":0.38700000000000001,"5min":0.34300000000000003,"15min":0.308},"max":{"1min":3.77,"5min":86.924000000000007,"15min":86.924000000000007},"last":0.42599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1345978368,"data_stored":1345978368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2231046144,"data_stored":2231042072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":862257152,"data_stored":862257152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2235301888,"data_stored":2235301888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":932843520,"data_stored":932839448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2281115648,"data_stored":2281111576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":676986880,"data_stored":676986880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1375141888,"data_stored":1375141888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:57:38.694 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:57:38.695 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 51 2023-10-09T05:57:38.735 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:38.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:38.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:38.839 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:38.867 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:38.917 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:38.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:38.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:39.022 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:39.049 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:39.103 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:39.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:39.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:39.195 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:39 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 46293/165603 objects degraded (27.954%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:39.195 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/394955859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:39.196 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:39 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 46293/165603 objects degraded (27.954%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:39.196 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/394955859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:39.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:39.241 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:57:39.249 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:39 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 46293/165603 objects degraded (27.954%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:39.249 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/394955859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:39.265 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:39.325 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:39.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:39.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:39.432 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:39.467 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:39.469 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:57:39.498 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:57:39.520 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:57:39.523 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:39.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:39.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:39.544 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:57:39.565 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:57:39.583 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:57:39.601 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:57:39.621 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:57:39.622 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:57:39.622 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:57:39.622 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:57:39.622 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:57:39.623 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:57:39.623 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:57:39.623 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:57:39.623 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:57:39.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:39.655 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:39.706 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:39.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:39.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:39.812 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:39.845 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:40.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[184418]: pgmap v1315: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 150 op/s; 46293/164622 objects degraded (28.121%); 18455/164622 objects misplaced (11.211%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:57:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[184418]: failed to encode map e440 with expected crc 2023-10-09T05:57:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[184418]: failed to encode map e440 with expected crc 2023-10-09T05:57:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/394955859' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[184418]: osdmap e440: 8 total, 8 up, 7 in 2023-10-09T05:57:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[184418]: failed to encode map e440 with expected crc 2023-10-09T05:57:40.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3973848230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:40.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[191227]: pgmap v1315: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 150 op/s; 46293/164622 objects degraded (28.121%); 18455/164622 objects misplaced (11.211%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:40.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:57:40.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[191227]: failed to encode map e440 with expected crc 2023-10-09T05:57:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[191227]: failed to encode map e440 with expected crc 2023-10-09T05:57:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/394955859' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[191227]: osdmap e440: 8 total, 8 up, 7 in 2023-10-09T05:57:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[191227]: failed to encode map e440 with expected crc 2023-10-09T05:57:40.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:40 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3973848230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:40 smithi191 ceph-mon[161669]: pgmap v1315: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 150 op/s; 46293/164622 objects degraded (28.121%); 18455/164622 objects misplaced (11.211%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:40 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:57:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:40 smithi191 ceph-mon[161669]: failed to encode map e440 with expected crc 2023-10-09T05:57:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:40 smithi191 ceph-mon[161669]: failed to encode map e440 with expected crc 2023-10-09T05:57:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:40 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/394955859' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:40 smithi191 ceph-mon[161669]: osdmap e440: 8 total, 8 up, 7 in 2023-10-09T05:57:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:40 smithi191 ceph-mon[161669]: failed to encode map e440 with expected crc 2023-10-09T05:57:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:40 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3973848230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:40.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:57:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:57:40] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.33.4" 2023-10-09T05:57:41.035 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:41.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:41.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:41.140 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:41.179 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:41.731 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:41.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:41.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:41.835 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:41.850 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:41.893 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:42.084 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:42 smithi008 ceph-mon[191227]: pgmap v1317: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 151 op/s; 46161/163572 objects degraded (28.221%); 18437/163572 objects misplaced (11.271%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T05:57:42.100 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:42.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:42.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:42.115 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:57:42.116 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T05:57:42.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T05:57:42.116 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:57:42.116 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.116 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:91089 8.10 8:0e1805e1:::benchmark_data_smithi008_432048_object91088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e303)", 2023-10-09T05:57:42.116 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:51.427594+0000", 2023-10-09T05:57:42.116 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.68514726199999, 2023-10-09T05:57:42.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.379546666, 2023-10-09T05:57:42.116 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.116 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.116 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 91089 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427594+0000", 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.117 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.118 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427594+0000", 2023-10-09T05:57:42.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.118 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427593+0000", 2023-10-09T05:57:42.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.118 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427621+0000", 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8181999999999999e-05 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427629+0000", 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3459999999999996e-06 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427637+0000", 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2560000000000002e-06 2023-10-09T05:57:42.119 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429329+0000", 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016914300000000001 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429333+0000", 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7330000000000003e-06 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.784547+0000", 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3552137100000001 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.784707+0000", 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016003999999999999 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.784908+0000", 2023-10-09T05:57:42.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000200738 2023-10-09T05:57:42.122 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.122 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.122 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.122 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.787520+0000", 2023-10-09T05:57:42.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026119979999999999 2023-10-09T05:57:42.122 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.122 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.122 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.122 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.787534+0000", 2023-10-09T05:57:42.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4293e-05 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.804953+0000", 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017419165 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.805482+0000", 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052865200000000001 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.123 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.124 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.124 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.805549+0000", 2023-10-09T05:57:42.124 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7589000000000006e-05 2023-10-09T05:57:42.124 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.124 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.124 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.124 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.807140+0000", 2023-10-09T05:57:42.124 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001590957 2023-10-09T05:57:42.124 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.124 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.124 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:91095 8.10 8:0d58a216:::benchmark_data_smithi008_432048_object91094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e303)", 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:51.427643+0000", 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.68509816200003, 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3810022559999999, 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.125 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 91095 2023-10-09T05:57:42.126 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.126 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.126 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.126 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.126 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427643+0000", 2023-10-09T05:57:42.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.126 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.126 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.126 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.126 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427643+0000", 2023-10-09T05:57:42.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.127 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.127 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.127 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.127 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427642+0000", 2023-10-09T05:57:42.127 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.127 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.127 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.127 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.127 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427670+0000", 2023-10-09T05:57:42.127 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7481999999999999e-05 2023-10-09T05:57:42.127 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427677+0000", 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5870000000000004e-06 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427683+0000", 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8379999999999998e-06 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.129 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429350+0000", 2023-10-09T05:57:42.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016668099999999999 2023-10-09T05:57:42.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.129 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.129 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429354+0000", 2023-10-09T05:57:42.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.568e-06 2023-10-09T05:57:42.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.786011+0000", 2023-10-09T05:57:42.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3566574820000001 2023-10-09T05:57:42.130 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.130 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.786154+0000", 2023-10-09T05:57:42.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000142416 2023-10-09T05:57:42.131 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.131 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.131 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T05:57:42.131 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.786313+0000", 2023-10-09T05:57:42.131 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015937500000000001 2023-10-09T05:57:42.131 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.132 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.132 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.134 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.789797+0000", 2023-10-09T05:57:42.134 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034837729999999999 2023-10-09T05:57:42.134 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.134 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.134 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.134 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.789860+0000", 2023-10-09T05:57:42.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3681000000000002e-05 2023-10-09T05:57:42.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.805106+0000", 2023-10-09T05:57:42.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015245702 2023-10-09T05:57:42.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.136 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.136 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.807205+0000", 2023-10-09T05:57:42.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020989340000000002 2023-10-09T05:57:42.136 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.136 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.136 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.137 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.807255+0000", 2023-10-09T05:57:42.137 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9778999999999999e-05 2023-10-09T05:57:42.137 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.137 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.137 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.137 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.808645+0000", 2023-10-09T05:57:42.137 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00139029 2023-10-09T05:57:42.138 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.138 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.138 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.138 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.138 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.138 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:91120 8.10 8:0f9502b6:::benchmark_data_smithi008_432048_object91119:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e303)", 2023-10-09T05:57:42.139 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:51.427690+0000", 2023-10-09T05:57:42.139 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.68505078299995, 2023-10-09T05:57:42.139 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.38482328, 2023-10-09T05:57:42.139 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.139 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.139 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.139 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.140 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.140 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 91120 2023-10-09T05:57:42.140 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.140 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.140 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427690+0000", 2023-10-09T05:57:42.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.141 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.141 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427690+0000", 2023-10-09T05:57:42.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427690+0000", 2023-10-09T05:57:42.142 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427721+0000", 2023-10-09T05:57:42.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.154e-05 2023-10-09T05:57:42.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.143 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427729+0000", 2023-10-09T05:57:42.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2290000000000003e-06 2023-10-09T05:57:42.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.143 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427736+0000", 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.161e-06 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429366+0000", 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016300349999999999 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429369+0000", 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3680000000000002e-06 2023-10-09T05:57:42.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.145 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.145 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.145 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.787560+0000", 2023-10-09T05:57:42.145 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3581907280000001 2023-10-09T05:57:42.145 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.145 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.145 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.145 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.787708+0000", 2023-10-09T05:57:42.145 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000148734 2023-10-09T05:57:42.145 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.145 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.146 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T05:57:42.146 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.787868+0000", 2023-10-09T05:57:42.146 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015949900000000001 2023-10-09T05:57:42.146 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.146 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.146 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.146 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.790807+0000", 2023-10-09T05:57:42.146 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029387079999999999 2023-10-09T05:57:42.146 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.147 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.791852+0000", 2023-10-09T05:57:42.147 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010451 2023-10-09T05:57:42.147 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.150 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.809695+0000", 2023-10-09T05:57:42.150 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017843562 2023-10-09T05:57:42.150 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.150 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.151 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.151 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.810535+0000", 2023-10-09T05:57:42.151 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00084004700000000002 2023-10-09T05:57:42.151 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.151 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.151 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.151 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.810577+0000", 2023-10-09T05:57:42.151 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1857000000000003e-05 2023-10-09T05:57:42.151 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.151 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.151 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.812513+0000", 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019362400000000001 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:91141 8.10 8:0f700754:::benchmark_data_smithi008_432048_object91140:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e303)", 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:51.427741+0000", 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.68500008299998, 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.386614518, 2023-10-09T05:57:42.152 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 91141 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427741+0000", 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.153 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.154 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.154 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.154 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427741+0000", 2023-10-09T05:57:42.154 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.154 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.154 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.154 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.154 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427740+0000", 2023-10-09T05:57:42.154 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.154 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.154 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.155 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.155 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427769+0000", 2023-10-09T05:57:42.155 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8952e-05 2023-10-09T05:57:42.155 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.155 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.155 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.155 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427781+0000", 2023-10-09T05:57:42.155 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1545e-05 2023-10-09T05:57:42.155 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427787+0000", 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0109999999999997e-06 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429381+0000", 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015937150000000001 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.156 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.157 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429384+0000", 2023-10-09T05:57:42.157 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5599999999999998e-06 2023-10-09T05:57:42.157 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.157 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.157 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.157 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.788813+0000", 2023-10-09T05:57:42.157 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3594289210000001 2023-10-09T05:57:42.157 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.157 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.157 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.157 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.788936+0000", 2023-10-09T05:57:42.158 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000122557 2023-10-09T05:57:42.158 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.158 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.158 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T05:57:42.158 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.789034+0000", 2023-10-09T05:57:42.158 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8590000000000003e-05 2023-10-09T05:57:42.158 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.158 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.158 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.159 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.791867+0000", 2023-10-09T05:57:42.159 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028327339999999999 2023-10-09T05:57:42.159 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.159 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.159 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.159 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.798579+0000", 2023-10-09T05:57:42.159 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0067117790000000002 2023-10-09T05:57:42.159 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.159 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.159 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.809731+0000", 2023-10-09T05:57:42.160 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011151805000000001 2023-10-09T05:57:42.160 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.160 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.160 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.812623+0000", 2023-10-09T05:57:42.160 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028926149999999999 2023-10-09T05:57:42.160 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.160 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.160 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.161 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.812693+0000", 2023-10-09T05:57:42.161 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0110999999999993e-05 2023-10-09T05:57:42.161 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.161 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.161 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.161 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.814355+0000", 2023-10-09T05:57:42.161 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001662164 2023-10-09T05:57:42.161 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.161 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.161 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.161 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.162 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.162 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:91178 8.10 8:0996003c:::benchmark_data_smithi008_432048_object91177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e303)", 2023-10-09T05:57:42.162 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:51.427792+0000", 2023-10-09T05:57:42.162 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.684948921, 2023-10-09T05:57:42.162 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3949157400000001, 2023-10-09T05:57:42.162 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.162 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.162 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.162 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.162 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.162 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 91178 2023-10-09T05:57:42.163 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.163 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.163 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.163 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.163 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427792+0000", 2023-10-09T05:57:42.163 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.163 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.163 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.163 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.163 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427792+0000", 2023-10-09T05:57:42.163 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.164 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.164 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.164 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427792+0000", 2023-10-09T05:57:42.164 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.164 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.164 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.164 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427821+0000", 2023-10-09T05:57:42.164 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9076000000000001e-05 2023-10-09T05:57:42.164 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427830+0000", 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1640000000000007e-06 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427836+0000", 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1689999999999997e-06 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.165 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.166 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429409+0000", 2023-10-09T05:57:42.166 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015732470000000001 2023-10-09T05:57:42.166 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.166 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.166 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.166 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429414+0000", 2023-10-09T05:57:42.166 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0459999999999996e-06 2023-10-09T05:57:42.166 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.166 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.166 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.789898+0000", 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3604835129999999 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.790012+0000", 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011454400000000001 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.790108+0000", 2023-10-09T05:57:42.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5960999999999998e-05 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.793079+0000", 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029701910000000001 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.799918+0000", 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068393129999999996 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.168 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.817680+0000", 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017762512000000001 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.820795+0000", 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031151600000000001 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.820880+0000", 2023-10-09T05:57:42.170 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4622000000000004e-05 2023-10-09T05:57:42.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.170 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.170 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.170 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.822708+0000", 2023-10-09T05:57:42.170 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018276919999999999 2023-10-09T05:57:42.170 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.170 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.170 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.170 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.171 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:91251 8.10 8:0c34f7ee:::benchmark_data_smithi008_432048_object91250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e303)", 2023-10-09T05:57:42.171 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:51.427841+0000", 2023-10-09T05:57:42.171 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.68489995799996, 2023-10-09T05:57:42.171 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.399802786, 2023-10-09T05:57:42.171 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.171 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.171 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.171 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.172 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.172 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 91251 2023-10-09T05:57:42.172 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.172 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.172 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.173 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.173 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427841+0000", 2023-10-09T05:57:42.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.173 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.173 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.173 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427841+0000", 2023-10-09T05:57:42.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.174 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.174 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.174 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427841+0000", 2023-10-09T05:57:42.174 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.174 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.175 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.175 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427879+0000", 2023-10-09T05:57:42.175 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8448000000000002e-05 2023-10-09T05:57:42.175 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.175 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.175 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427887+0000", 2023-10-09T05:57:42.175 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1340000000000004e-06 2023-10-09T05:57:42.176 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.176 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.176 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.176 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427894+0000", 2023-10-09T05:57:42.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6089999999999997e-06 2023-10-09T05:57:42.176 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.176 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.176 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.176 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429453+0000", 2023-10-09T05:57:42.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015590039999999999 2023-10-09T05:57:42.177 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.177 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429457+0000", 2023-10-09T05:57:42.177 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7610000000000001e-06 2023-10-09T05:57:42.177 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.177 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.790850+0000", 2023-10-09T05:57:42.177 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3613931349999999 2023-10-09T05:57:42.177 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.790970+0000", 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000120222 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.791086+0000", 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011642 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.178 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.794048+0000", 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029619300000000002 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.799978+0000", 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0059299330000000001 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.822758+0000", 2023-10-09T05:57:42.179 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022780205000000001 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.825547+0000", 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027883299999999999 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.825606+0000", 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9497999999999999e-05 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.827644+0000", 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020375929999999999 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:91272 8.10 8:0eb654a7:::benchmark_data_smithi008_432048_object91271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e303)", 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:51.427899+0000", 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.68484199500006, 2023-10-09T05:57:42.181 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.401248117, 2023-10-09T05:57:42.182 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.182 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.182 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.182 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.182 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.182 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 91272 2023-10-09T05:57:42.182 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.182 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.182 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.182 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.182 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427899+0000", 2023-10-09T05:57:42.183 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.183 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.183 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.183 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427899+0000", 2023-10-09T05:57:42.183 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.183 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.183 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.183 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427899+0000", 2023-10-09T05:57:42.184 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.184 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.184 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.184 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.184 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427929+0000", 2023-10-09T05:57:42.184 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0454999999999999e-05 2023-10-09T05:57:42.184 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.184 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.184 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.184 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427937+0000", 2023-10-09T05:57:42.184 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1249999999999993e-06 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427945+0000", 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.763e-06 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429488+0000", 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001543474 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429494+0000", 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8749999999999997e-06 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.791901+0000", 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.362406537 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.792066+0000", 2023-10-09T05:57:42.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016477700000000001 2023-10-09T05:57:42.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.187 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T05:57:42.187 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.792269+0000", 2023-10-09T05:57:42.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020384 2023-10-09T05:57:42.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.187 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.187 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.795900+0000", 2023-10-09T05:57:42.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036308239999999999 2023-10-09T05:57:42.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.188 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.188 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.804874+0000", 2023-10-09T05:57:42.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0089739910000000006 2023-10-09T05:57:42.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.822795+0000", 2023-10-09T05:57:42.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017920456000000001 2023-10-09T05:57:42.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.190 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.190 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.827692+0000", 2023-10-09T05:57:42.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048971609999999997 2023-10-09T05:57:42.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.191 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.827746+0000", 2023-10-09T05:57:42.191 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4400000000000001e-05 2023-10-09T05:57:42.191 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.191 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.829147+0000", 2023-10-09T05:57:42.191 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001400912 2023-10-09T05:57:42.192 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.192 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.192 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.192 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:91385 8.10 8:0d8ba84c:::benchmark_data_smithi008_432048_object91384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e303)", 2023-10-09T05:57:42.192 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:51.427950+0000", 2023-10-09T05:57:42.192 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.68479050799999, 2023-10-09T05:57:42.193 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.403224523, 2023-10-09T05:57:42.193 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.193 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.193 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.193 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.194 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.194 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 91385 2023-10-09T05:57:42.194 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.194 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.194 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.194 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.194 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427950+0000", 2023-10-09T05:57:42.194 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.194 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.195 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.195 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.195 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427950+0000", 2023-10-09T05:57:42.195 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.195 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.195 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.196 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.196 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.427950+0000", 2023-10-09T05:57:42.196 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.196 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.196 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.196 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.197 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428008+0000", 2023-10-09T05:57:42.197 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7989000000000003e-05 2023-10-09T05:57:42.197 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.197 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.197 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.197 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428017+0000", 2023-10-09T05:57:42.197 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8570000000000001e-06 2023-10-09T05:57:42.197 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.197 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.197 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.197 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428024+0000", 2023-10-09T05:57:42.198 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.401e-06 2023-10-09T05:57:42.198 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.198 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.198 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.198 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429516+0000", 2023-10-09T05:57:42.198 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014914069999999999 2023-10-09T05:57:42.198 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.198 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.198 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.198 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429519+0000", 2023-10-09T05:57:42.198 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4989999999999999e-06 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.793113+0000", 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.363593713 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.793254+0000", 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014146499999999999 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.199 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.200 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T05:57:42.200 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.793364+0000", 2023-10-09T05:57:42.200 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010948200000000001 2023-10-09T05:57:42.200 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.200 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.200 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.200 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.798512+0000", 2023-10-09T05:57:42.200 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051486639999999998 2023-10-09T05:57:42.201 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.201 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.201 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.201 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.805040+0000", 2023-10-09T05:57:42.201 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006527901 2023-10-09T05:57:42.202 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.202 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.202 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.202 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.822821+0000", 2023-10-09T05:57:42.202 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017780793999999999 2023-10-09T05:57:42.202 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.202 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.202 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.202 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.829345+0000", 2023-10-09T05:57:42.203 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0065241539999999999 2023-10-09T05:57:42.203 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.203 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.203 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.203 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.829400+0000", 2023-10-09T05:57:42.203 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4518000000000002e-05 2023-10-09T05:57:42.203 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.204 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.204 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.204 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.831175+0000", 2023-10-09T05:57:42.204 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001775186 2023-10-09T05:57:42.204 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.204 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.204 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.204 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.204 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.204 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:91395 8.10 8:083f0394:::benchmark_data_smithi008_432048_object91394:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e303)", 2023-10-09T05:57:42.205 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:51.428031+0000", 2023-10-09T05:57:42.205 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.68471046000002, 2023-10-09T05:57:42.205 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.404684942, 2023-10-09T05:57:42.205 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.205 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.205 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.205 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.205 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.205 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 91395 2023-10-09T05:57:42.205 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.205 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.206 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.206 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.206 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428031+0000", 2023-10-09T05:57:42.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.206 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.206 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.206 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.206 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428031+0000", 2023-10-09T05:57:42.206 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.206 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.206 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.207 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.207 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428030+0000", 2023-10-09T05:57:42.207 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.207 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.207 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.207 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.207 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428076+0000", 2023-10-09T05:57:42.207 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5886000000000001e-05 2023-10-09T05:57:42.207 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.207 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.207 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428082+0000", 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1920000000000002e-06 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428090+0000", 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5970000000000003e-06 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429606+0000", 2023-10-09T05:57:42.208 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015161280000000001 2023-10-09T05:57:42.209 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.209 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.209 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.209 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429613+0000", 2023-10-09T05:57:42.209 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2749999999999998e-06 2023-10-09T05:57:42.209 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.209 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.209 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.209 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.794084+0000", 2023-10-09T05:57:42.209 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3644705880000001 2023-10-09T05:57:42.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.210 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.210 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.794208+0000", 2023-10-09T05:57:42.210 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000124797 2023-10-09T05:57:42.210 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.210 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.210 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T05:57:42.210 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.794310+0000", 2023-10-09T05:57:42.210 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000101492 2023-10-09T05:57:42.211 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.211 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.211 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.211 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.798519+0000", 2023-10-09T05:57:42.211 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042087510000000002 2023-10-09T05:57:42.211 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.211 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.211 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.211 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.805183+0000", 2023-10-09T05:57:42.211 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006664306 2023-10-09T05:57:42.211 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.212 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.212 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.212 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.829211+0000", 2023-10-09T05:57:42.212 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024028073 2023-10-09T05:57:42.212 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.212 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.212 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.212 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.831251+0000", 2023-10-09T05:57:42.212 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002040418 2023-10-09T05:57:42.212 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.831322+0000", 2023-10-09T05:57:42.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0647000000000003e-05 2023-10-09T05:57:42.213 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.213 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.213 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.213 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.832715+0000", 2023-10-09T05:57:42.213 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013934069999999999 2023-10-09T05:57:42.213 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.214 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.214 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.214 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.214 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.214 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:91428 8.10 8:0d69395f:::benchmark_data_smithi008_432048_object91427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e303)", 2023-10-09T05:57:42.214 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:48:51.428099+0000", 2023-10-09T05:57:42.214 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 530.68464172999995, 2023-10-09T05:57:42.214 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.407996102, 2023-10-09T05:57:42.214 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.214 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.214 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.215 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.215 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.215 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 91428 2023-10-09T05:57:42.215 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.215 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.215 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.215 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.215 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428099+0000", 2023-10-09T05:57:42.215 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.215 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.215 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.216 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.216 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428099+0000", 2023-10-09T05:57:42.216 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.216 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.216 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.216 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.216 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428099+0000", 2023-10-09T05:57:42.216 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.216 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.217 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.217 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.217 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428154+0000", 2023-10-09T05:57:42.217 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5318999999999997e-05 2023-10-09T05:57:42.217 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.217 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.217 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.218 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428191+0000", 2023-10-09T05:57:42.218 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6798000000000002e-05 2023-10-09T05:57:42.218 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.218 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.218 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.218 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.428202+0000", 2023-10-09T05:57:42.218 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1574e-05 2023-10-09T05:57:42.219 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.219 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.219 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.219 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429640+0000", 2023-10-09T05:57:42.219 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014373070000000001 2023-10-09T05:57:42.219 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.220 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.220 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.220 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:51.429644+0000", 2023-10-09T05:57:42.220 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6179999999999997e-06 2023-10-09T05:57:42.220 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.220 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.220 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.221 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.795002+0000", 2023-10-09T05:57:42.221 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3653574210000001 2023-10-09T05:57:42.221 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.221 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.221 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.221 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.795117+0000", 2023-10-09T05:57:42.221 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000114863 2023-10-09T05:57:42.222 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.222 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.222 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T05:57:42.222 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.795226+0000", 2023-10-09T05:57:42.222 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010966799999999999 2023-10-09T05:57:42.222 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.223 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.223 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.223 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.799848+0000", 2023-10-09T05:57:42.223 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0046219349999999998 2023-10-09T05:57:42.223 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.223 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.223 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.224 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.805252+0000", 2023-10-09T05:57:42.224 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0054042070000000003 2023-10-09T05:57:42.224 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.224 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.224 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.224 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.832785+0000", 2023-10-09T05:57:42.225 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.027532919999999999 2023-10-09T05:57:42.225 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.225 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.225 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.225 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.834111+0000", 2023-10-09T05:57:42.225 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001325915 2023-10-09T05:57:42.225 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.226 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.226 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.834203+0000", 2023-10-09T05:57:42.226 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1643000000000003e-05 2023-10-09T05:57:42.226 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.226 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.226 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.227 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:48:52.836095+0000", 2023-10-09T05:57:42.227 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018924499999999999 2023-10-09T05:57:42.227 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.227 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.227 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.227 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.227 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.228 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119513 8.f 8:f6feaa9b:::benchmark_data_smithi008_432048_object16571:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e328)", 2023-10-09T05:57:42.228 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:18.038143+0000", 2023-10-09T05:57:42.228 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 444.07459807599997, 2023-10-09T05:57:42.228 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3344732289999999, 2023-10-09T05:57:42.228 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.228 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.228 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.229 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.229 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.229 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119513 2023-10-09T05:57:42.229 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.229 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.229 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.229 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.230 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038143+0000", 2023-10-09T05:57:42.230 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.230 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.230 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.230 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.230 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038143+0000", 2023-10-09T05:57:42.231 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.231 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.231 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.231 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.231 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038141+0000", 2023-10-09T05:57:42.231 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:42.231 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.232 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.232 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.232 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038149+0000", 2023-10-09T05:57:42.232 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5680000000000002e-06 2023-10-09T05:57:42.232 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.232 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.232 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.233 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038182+0000", 2023-10-09T05:57:42.233 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3359999999999999e-05 2023-10-09T05:57:42.233 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.233 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.233 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.233 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038194+0000", 2023-10-09T05:57:42.233 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1936999999999999e-05 2023-10-09T05:57:42.234 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.234 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.234 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.234 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038592+0000", 2023-10-09T05:57:42.234 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00039797600000000003 2023-10-09T05:57:42.234 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.235 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.235 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038605+0000", 2023-10-09T05:57:42.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3249e-05 2023-10-09T05:57:42.235 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.235 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.235 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.235 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.194227+0000", 2023-10-09T05:57:42.235 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1556221190000002 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.194397+0000", 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000169951 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.194796+0000", 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00039891500000000001 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.236 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.237 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.237 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.199646+0000", 2023-10-09T05:57:42.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048497929999999998 2023-10-09T05:57:42.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.237 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.237 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.237 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.201124+0000", 2023-10-09T05:57:42.237 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001477674 2023-10-09T05:57:42.237 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.237 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.237 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.238 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.370854+0000", 2023-10-09T05:57:42.238 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.16973037299999999 2023-10-09T05:57:42.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.238 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.238 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.238 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.370930+0000", 2023-10-09T05:57:42.238 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5791999999999995e-05 2023-10-09T05:57:42.238 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.238 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.239 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.239 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.372616+0000", 2023-10-09T05:57:42.239 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001686182 2023-10-09T05:57:42.239 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.239 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.239 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.239 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.239 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.239 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119536 8.f 8:f4127ade:::benchmark_data_smithi008_432048_object16594:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e328)", 2023-10-09T05:57:42.239 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:18.038201+0000", 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 444.074539781, 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3361112149999999, 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119536 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.240 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038201+0000", 2023-10-09T05:57:42.241 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.241 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.241 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.241 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.241 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038201+0000", 2023-10-09T05:57:42.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.242 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.242 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038201+0000", 2023-10-09T05:57:42.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.242 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.242 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.242 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.242 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038204+0000", 2023-10-09T05:57:42.242 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9129999999999999e-06 2023-10-09T05:57:42.243 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.243 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.243 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.243 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038211+0000", 2023-10-09T05:57:42.243 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9419999999999998e-06 2023-10-09T05:57:42.243 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.243 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.243 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.243 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038218+0000", 2023-10-09T05:57:42.243 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5580000000000003e-06 2023-10-09T05:57:42.243 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038632+0000", 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00041360799999999999 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038641+0000", 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6940000000000001e-06 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.244 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.197894+0000", 2023-10-09T05:57:42.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1592536629999999 2023-10-09T05:57:42.245 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.245 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.245 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.245 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.198003+0000", 2023-10-09T05:57:42.245 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000108411 2023-10-09T05:57:42.245 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:57:42.246 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.198271+0000", 2023-10-09T05:57:42.246 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00026847900000000002 2023-10-09T05:57:42.246 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.246 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.246 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.201271+0000", 2023-10-09T05:57:42.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029996929999999999 2023-10-09T05:57:42.247 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.247 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.247 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.247 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.203186+0000", 2023-10-09T05:57:42.247 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001915094 2023-10-09T05:57:42.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.248 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.372746+0000", 2023-10-09T05:57:42.248 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.16956000299999999 2023-10-09T05:57:42.248 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.248 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.248 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.372796+0000", 2023-10-09T05:57:42.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9733e-05 2023-10-09T05:57:42.249 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.249 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.249 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.249 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.374312+0000", 2023-10-09T05:57:42.249 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00151684 2023-10-09T05:57:42.250 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.250 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.250 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.250 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.250 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.250 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119561 8.f 8:f5372cc7:::benchmark_data_smithi008_432048_object16619:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e328)", 2023-10-09T05:57:42.250 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:18.038223+0000", 2023-10-09T05:57:42.251 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 444.07451798699998, 2023-10-09T05:57:42.251 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3394108180000002, 2023-10-09T05:57:42.251 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.251 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.251 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.251 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.251 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.251 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119561 2023-10-09T05:57:42.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.252 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.252 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.252 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.252 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038223+0000", 2023-10-09T05:57:42.252 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.252 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.253 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.253 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038223+0000", 2023-10-09T05:57:42.253 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.253 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.253 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.253 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.253 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038223+0000", 2023-10-09T05:57:42.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.254 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.254 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.254 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038225+0000", 2023-10-09T05:57:42.254 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4219999999999999e-06 2023-10-09T05:57:42.254 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.255 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038231+0000", 2023-10-09T05:57:42.255 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2999999999999998e-06 2023-10-09T05:57:42.255 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.255 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.255 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038237+0000", 2023-10-09T05:57:42.256 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.446e-06 2023-10-09T05:57:42.256 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.256 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.256 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.256 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038664+0000", 2023-10-09T05:57:42.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00042669499999999999 2023-10-09T05:57:42.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.257 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.257 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.257 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038674+0000", 2023-10-09T05:57:42.257 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0910999999999999e-05 2023-10-09T05:57:42.257 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.258 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.258 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.199665+0000", 2023-10-09T05:57:42.258 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1609910640000001 2023-10-09T05:57:42.258 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.258 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.259 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.199775+0000", 2023-10-09T05:57:42.259 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010922899999999999 2023-10-09T05:57:42.259 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.259 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.259 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:57:42.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.199982+0000", 2023-10-09T05:57:42.260 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020698400000000001 2023-10-09T05:57:42.260 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.260 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.260 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.260 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.203264+0000", 2023-10-09T05:57:42.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032822900000000002 2023-10-09T05:57:42.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.203338+0000", 2023-10-09T05:57:42.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3665999999999996e-05 2023-10-09T05:57:42.261 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.261 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.261 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.261 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.375802+0000", 2023-10-09T05:57:42.261 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.172464741 2023-10-09T05:57:42.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.262 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.262 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.375857+0000", 2023-10-09T05:57:42.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4373999999999998e-05 2023-10-09T05:57:42.262 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.262 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.262 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.262 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.377634+0000", 2023-10-09T05:57:42.262 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001777034 2023-10-09T05:57:42.263 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.263 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.263 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.263 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.263 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.263 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119582 8.f 8:f18ab547:::benchmark_data_smithi008_432048_object16640:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e328)", 2023-10-09T05:57:42.263 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:18.038242+0000", 2023-10-09T05:57:42.263 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 444.074499473, 2023-10-09T05:57:42.263 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3410004390000001, 2023-10-09T05:57:42.263 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.263 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.264 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.264 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.264 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.264 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119582 2023-10-09T05:57:42.264 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.264 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038242+0000", 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038242+0000", 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.265 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038241+0000", 2023-10-09T05:57:42.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038244+0000", 2023-10-09T05:57:42.266 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3889999999999999e-06 2023-10-09T05:57:42.266 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.266 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.266 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.266 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038250+0000", 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5950000000000004e-06 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038256+0000", 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6250000000000004e-06 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038698+0000", 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00044198300000000001 2023-10-09T05:57:42.267 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.268 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038707+0000", 2023-10-09T05:57:42.268 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4180000000000003e-06 2023-10-09T05:57:42.268 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.268 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.268 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.268 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.201321+0000", 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1626139850000001 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.201476+0000", 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015470800000000001 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.201769+0000", 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00029359699999999999 2023-10-09T05:57:42.269 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.203325+0000", 2023-10-09T05:57:42.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015555479999999999 2023-10-09T05:57:42.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.270 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.270 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.206284+0000", 2023-10-09T05:57:42.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029586790000000001 2023-10-09T05:57:42.270 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.270 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.377686+0000", 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17140230200000001 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.377732+0000", 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5540000000000001e-05 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.271 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.379242+0000", 2023-10-09T05:57:42.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001510419 2023-10-09T05:57:42.272 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.272 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.272 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.272 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119606 8.f 8:f3efa860:::benchmark_data_smithi008_432048_object16664:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e328)", 2023-10-09T05:57:42.272 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:18.038260+0000", 2023-10-09T05:57:42.272 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 444.074480709, 2023-10-09T05:57:42.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.342543751, 2023-10-09T05:57:42.273 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.273 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.273 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.273 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.273 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.273 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119606 2023-10-09T05:57:42.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.273 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.274 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.274 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038260+0000", 2023-10-09T05:57:42.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.274 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.274 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.274 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038260+0000", 2023-10-09T05:57:42.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.274 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.274 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038260+0000", 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038262+0000", 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5299999999999999e-06 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038267+0000", 2023-10-09T05:57:42.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0119999999999996e-06 2023-10-09T05:57:42.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038274+0000", 2023-10-09T05:57:42.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8299999999999998e-06 2023-10-09T05:57:42.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038731+0000", 2023-10-09T05:57:42.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00045686199999999999 2023-10-09T05:57:42.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038741+0000", 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5289999999999997e-06 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.203377+0000", 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.164635863 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.203545+0000", 2023-10-09T05:57:42.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000168152 2023-10-09T05:57:42.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:57:42.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.203948+0000", 2023-10-09T05:57:42.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000403218 2023-10-09T05:57:42.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.206356+0000", 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0024079069999999999 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.206692+0000", 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00033650800000000001 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.379308+0000", 2023-10-09T05:57:42.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.172615828 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.379364+0000", 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.609e-05 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.380804+0000", 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001439818 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.280 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119614 8.f 8:f250265f:::benchmark_data_smithi008_432048_object16672:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e328)", 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:18.038278+0000", 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 444.07446262799999, 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3442649219999998, 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.281 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.282 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119614 2023-10-09T05:57:42.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.282 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.282 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.282 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038278+0000", 2023-10-09T05:57:42.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038278+0000", 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038278+0000", 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038280+0000", 2023-10-09T05:57:42.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3309999999999998e-06 2023-10-09T05:57:42.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038286+0000", 2023-10-09T05:57:42.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1210000000000003e-06 2023-10-09T05:57:42.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038292+0000", 2023-10-09T05:57:42.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3569999999999997e-06 2023-10-09T05:57:42.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038764+0000", 2023-10-09T05:57:42.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000472132 2023-10-09T05:57:42.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:18.038773+0000", 2023-10-09T05:57:42.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5189999999999998e-06 2023-10-09T05:57:42.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.205931+0000", 2023-10-09T05:57:42.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1671570400000002 2023-10-09T05:57:42.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:42.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.206265+0000", 2023-10-09T05:57:42.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00033399800000000001 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.340771+0000", 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.13450684800000001 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.340910+0000", 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000138968 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.341427+0000", 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00051669699999999999 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.344563+0000", 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031357310000000001 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.344650+0000", 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6923999999999996e-05 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.380984+0000", 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.036334090999999999 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.381037+0000", 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3582000000000003e-05 2023-10-09T05:57:42.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.382543+0000", 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001505922 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119641 8.f 8:f0308257:::benchmark_data_smithi008_432048_object16699:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:19.797349+0000", 2023-10-09T05:57:42.291 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 442.31539179599997, 2023-10-09T05:57:42.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6140632130000001, 2023-10-09T05:57:42.291 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.291 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.291 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.291 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.291 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.291 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119641 2023-10-09T05:57:42.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.291 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.797349+0000", 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.797349+0000", 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.797347+0000", 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.797357+0000", 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1810000000000007e-06 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.797378+0000", 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1851000000000001e-05 2023-10-09T05:57:42.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.797388+0000", 2023-10-09T05:57:42.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6350000000000001e-06 2023-10-09T05:57:42.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.797453+0000", 2023-10-09T05:57:42.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4472000000000002e-05 2023-10-09T05:57:42.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.797467+0000", 2023-10-09T05:57:42.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4493e-05 2023-10-09T05:57:42.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.206380+0000", 2023-10-09T05:57:42.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4089128479999999 2023-10-09T05:57:42.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.206674+0000", 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00029387700000000001 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.358335+0000", 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.15166090900000001 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.358466+0000", 2023-10-09T05:57:42.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000130934 2023-10-09T05:57:42.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:57:42.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.358880+0000", 2023-10-09T05:57:42.297 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00041435499999999999 2023-10-09T05:57:42.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.298 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.298 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.298 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.361242+0000", 2023-10-09T05:57:42.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023616090000000002 2023-10-09T05:57:42.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.361323+0000", 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1481000000000001e-05 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.409532+0000", 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.048209228999999999 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.300 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.409591+0000", 2023-10-09T05:57:42.300 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9092000000000003e-05 2023-10-09T05:57:42.300 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:42.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.411412+0000", 2023-10-09T05:57:42.302 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018210360000000001 2023-10-09T05:57:42.302 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.302 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.302 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.302 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119644 8.f 8:f0c05df0:::benchmark_data_smithi008_432048_object16702:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-10-09T05:57:42.303 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:19.803388+0000", 2023-10-09T05:57:42.303 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 442.309352671, 2023-10-09T05:57:42.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.609917531, 2023-10-09T05:57:42.303 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.303 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.303 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.303 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.303 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.303 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119644 2023-10-09T05:57:42.303 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.303 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.803388+0000", 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.803388+0000", 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.803386+0000", 2023-10-09T05:57:42.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T05:57:42.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.803396+0000", 2023-10-09T05:57:42.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0295999999999999e-05 2023-10-09T05:57:42.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.803412+0000", 2023-10-09T05:57:42.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5194999999999999e-05 2023-10-09T05:57:42.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.803422+0000", 2023-10-09T05:57:42.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0403e-05 2023-10-09T05:57:42.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.803483+0000", 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1050000000000007e-05 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.803495+0000", 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2089999999999999e-05 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.206728+0000", 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4032331980000001 2023-10-09T05:57:42.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.207132+0000", 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00040386399999999998 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.367467+0000", 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.16033435400000001 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.309 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.367590+0000", 2023-10-09T05:57:42.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012382800000000001 2023-10-09T05:57:42.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.309 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:57:42.309 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.368118+0000", 2023-10-09T05:57:42.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052799699999999999 2023-10-09T05:57:42.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.309 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.374329+0000", 2023-10-09T05:57:42.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0062105720000000001 2023-10-09T05:57:42.310 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.375336+0000", 2023-10-09T05:57:42.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001006917 2023-10-09T05:57:42.310 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.411481+0000", 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.036144923000000002 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.411548+0000", 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7136000000000002e-05 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.413306+0000", 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017579290000000001 2023-10-09T05:57:42.311 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.312 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.312 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.312 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.45782.0:119648 8.17 8:eb9f50cd:::benchmark_data_smithi008_432048_object16706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-10-09T05:57:42.312 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:50:19.811797+0000", 2023-10-09T05:57:42.312 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 442.30094419300002, 2023-10-09T05:57:42.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.554293959, 2023-10-09T05:57:42.312 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.312 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:42.312 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.45782", 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 119648 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.811797+0000", 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.313 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.811797+0000", 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.811796+0000", 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.811802+0000", 2023-10-09T05:57:42.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.6749999999999996e-06 2023-10-09T05:57:42.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.811813+0000", 2023-10-09T05:57:42.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1245e-05 2023-10-09T05:57:42.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.811820+0000", 2023-10-09T05:57:42.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5010000000000001e-06 2023-10-09T05:57:42.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.316 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.316 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.811877+0000", 2023-10-09T05:57:42.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6981000000000002e-05 2023-10-09T05:57:42.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.316 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.316 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.812081+0000", 2023-10-09T05:57:42.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020405900000000001 2023-10-09T05:57:42.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.812559+0000", 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000477586 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.814190+0000", 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001631167 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:19.814628+0000", 2023-10-09T05:57:42.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000437747 2023-10-09T05:57:42.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:42.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.364452+0000", 2023-10-09T05:57:42.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5498246 2023-10-09T05:57:42.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T05:57:42.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.364491+0000", 2023-10-09T05:57:42.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8909999999999998e-05 2023-10-09T05:57:42.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:50:21.366091+0000", 2023-10-09T05:57:42.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001599739 2023-10-09T05:57:42.319 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.319 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.319 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "MOSDPGRecoveryDelete(9.a e398,373 9:5d265ad6:::benchmark_data_smithi008_502820_object1769:head,389'5413)", 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:55:18.315220+0000", 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 143.79752076899999, 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14534585799999999, 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:18.315220+0000", 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.320 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:18.315220+0000", 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:18.315219+0000", 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.321 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:57:42.322 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:18.315225+0000", 2023-10-09T05:57:42.322 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6280000000000002e-06 2023-10-09T05:57:42.322 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.322 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.322 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:57:42.322 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:18.315233+0000", 2023-10-09T05:57:42.322 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3860000000000007e-06 2023-10-09T05:57:42.322 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.322 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.323 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:57:42.323 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:18.315241+0000", 2023-10-09T05:57:42.323 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2269999999999996e-06 2023-10-09T05:57:42.323 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.323 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.323 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:57:42.323 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:18.460402+0000", 2023-10-09T05:57:42.323 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14516090200000001 2023-10-09T05:57:42.323 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.323 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.323 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:18.460432+0000", 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0369e-05 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:18.460566+0000", 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013416900000000001 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:57:42.324 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:57:42.325 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:57:42.335 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:42.336 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:42 smithi191 ceph-mon[161669]: pgmap v1317: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 151 op/s; 46161/163572 objects degraded (28.221%); 18437/163572 objects misplaced (11.271%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T05:57:42.387 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:42 smithi008 ceph-mon[184418]: pgmap v1317: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 151 op/s; 46161/163572 objects degraded (28.221%); 18437/163572 objects misplaced (11.271%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T05:57:42.396 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:42.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:42.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:42.501 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:42.535 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:42.591 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:42.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:42.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:42.695 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:42.726 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:43.063 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:57:43.628 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:43.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:43.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:43.700 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:57:43.732 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:43.770 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:43.782 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:43.815 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:43.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:43.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:43.886 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:57:43.923 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:43.967 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:44.086 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:57:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:44 smithi008 ceph-mon[184418]: pgmap v1318: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 156 op/s; 46161/162027 objects degraded (28.490%); 18355/162027 objects misplaced (11.328%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:44 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 46161/162027 objects degraded (28.490%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:44.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:44.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:44 smithi008 ceph-mon[191227]: pgmap v1318: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 156 op/s; 46161/162027 objects degraded (28.490%); 18355/162027 objects misplaced (11.328%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:44.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:44 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 46161/162027 objects degraded (28.490%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:44.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:44.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:44.248 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:44.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:44.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:44.353 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:44 smithi191 ceph-mon[161669]: pgmap v1318: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 156 op/s; 46161/162027 objects degraded (28.490%); 18355/162027 objects misplaced (11.328%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:44.353 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:44 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 46161/162027 objects degraded (28.490%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:44.353 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:44.354 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:44.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:44.387 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:44.493 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:44.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:44.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:44.512 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:57:44.597 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:44.629 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:45.195 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2980864021' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T05:57:45.195 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:45 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T05:57:45.195 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2980864021' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T05:57:45.196 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:45 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T05:57:45.500 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:45.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:45.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:45.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2980864021' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T05:57:45.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:45 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T05:57:45.605 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:45.636 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:45.679 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:45.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:45.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:45.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:45.785 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:57:45.807 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:45.810 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:45.854 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:45.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:45.962 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:46.020 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:46.249 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:46.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:46.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:46.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:46 smithi191 ceph-mon[161669]: pgmap v1319: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 156 op/s; 46161/162027 objects degraded (28.490%); 18355/162027 objects misplaced (11.328%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:46.354 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:46.383 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:46 smithi008 ceph-mon[184418]: pgmap v1319: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 156 op/s; 46161/162027 objects degraded (28.490%); 18355/162027 objects misplaced (11.328%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:46.383 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:46 smithi008 ceph-mon[191227]: pgmap v1319: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 156 op/s; 46161/162027 objects degraded (28.490%); 18355/162027 objects misplaced (11.328%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:57:46.387 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:46.650 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:46.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:46.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:46.759 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:57:46.759 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:57:46.759 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T05:57:46.759 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:57:46.788 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:46.993 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:47.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:47.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:47.103 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:57:47.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:47.132 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:47.175 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:47.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:47.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:47.279 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:47.319 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:47.387 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/913436004' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:47.387 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:47 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:47.387 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/913436004' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:47.388 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:47 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:47.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/913436004' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:47 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:47.588 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:57:47.634 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:47.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:47.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:47.738 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:47.768 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:47.952 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:47.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:48.056 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:48.064 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:57:48.091 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:48.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[184418]: pgmap v1320: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 45960/160671 objects degraded (28.605%); 18282/160671 objects misplaced (11.379%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T05:57:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:57:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[184418]: osdmap e441: 8 total, 8 up, 7 in 2023-10-09T05:57:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[184418]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[184418]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[184418]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[184418]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[184418]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[191227]: pgmap v1320: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 45960/160671 objects degraded (28.605%); 18282/160671 objects misplaced (11.379%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T05:57:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:57:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[191227]: osdmap e441: 8 total, 8 up, 7 in 2023-10-09T05:57:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[191227]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[191227]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[191227]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[191227]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:48 smithi008 ceph-mon[191227]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.256 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:48.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:48.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:48.312 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:57:48.334 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:57:48.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:48 smithi191 ceph-mon[161669]: pgmap v1320: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 45960/160671 objects degraded (28.605%); 18282/160671 objects misplaced (11.379%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T05:57:48.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:48 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:48.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:48 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:57:48.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:48 smithi191 ceph-mon[161669]: osdmap e441: 8 total, 8 up, 7 in 2023-10-09T05:57:48.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:48 smithi191 ceph-mon[161669]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:48 smithi191 ceph-mon[161669]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:48 smithi191 ceph-mon[161669]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:48 smithi191 ceph-mon[161669]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.352 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:48 smithi191 ceph-mon[161669]: failed to encode map e441 with expected crc 2023-10-09T05:57:48.355 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:57:48.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:48.377 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:57:48.399 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:57:48.400 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:48.420 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:57:48.442 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:57:48.460 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:48.461 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:57:48.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:48.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:48.463 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:57:48.463 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:57:48.463 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:57:48.463 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:57:48.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:57:48.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:57:48.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:57:48.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:57:48.563 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:48.597 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:48.709 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:48.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:48.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:48.814 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:48.842 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:49.113 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:49.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:49.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:49.217 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:49.259 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:49.325 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:49.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:49.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:49.415 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:49 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 45960/160671 objects degraded (28.605%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:49.416 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2357569645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:49.430 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:49.463 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:49.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:49 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 45960/160671 objects degraded (28.605%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:49.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2357569645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:49.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:49 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 45960/160671 objects degraded (28.605%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:49.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2357569645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:49.566 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:49.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:49.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:49.644 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:57:49.644 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:57:49.670 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:49.701 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:49.757 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:49.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:49.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:49.863 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:49.902 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:49.947 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:49.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:49.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:49.954 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:57:50.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:50.077 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:50.331 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:50.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:50.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:50.371 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:50.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:57:50.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:50 smithi008 ceph-mon[184418]: pgmap v1322: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 198 op/s; 45960/159018 objects degraded (28.902%); 18197/159018 objects misplaced (11.443%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:57:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:50 smithi008 ceph-mon[191227]: pgmap v1322: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 198 op/s; 45960/159018 objects degraded (28.902%); 18197/159018 objects misplaced (11.443%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:57:50.466 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:50.526 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:50.536 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:50 smithi191 ceph-mon[161669]: pgmap v1322: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 198 op/s; 45960/159018 objects degraded (28.902%); 18197/159018 objects misplaced (11.443%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:57:50.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:50.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:50.637 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:50.673 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:50.767 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:57:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:57:50] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.33.4" 2023-10-09T05:57:50.786 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 5, 6, 1, 7, 3] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:57:50.786 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:57:50.786 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [2, 4, 5, 6, 1, 7, 3] 2023-10-09T05:57:50.787 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd out 5 2023-10-09T05:57:50.822 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:57:50.881 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:50.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:50.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:50.986 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:51.020 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:51.077 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:51.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:51.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:51.181 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:51.220 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:51.282 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:51.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:51.311 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:57:51.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:51.418 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:51.483 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:51.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:51.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:51.586 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:51.625 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:51.919 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:51.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:51.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:52.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:52.099 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:52.177 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:52.179 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:52.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:52.228 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:52 smithi008 ceph-mon[191227]: pgmap v1323: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 45759/157554 objects degraded (29.043%); 18126/157554 objects misplaced (11.505%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:57:52.282 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:52.314 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:52.390 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:52.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:52.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:52.403 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:57:52.403 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:57:52.430 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:52 smithi191 ceph-mon[161669]: pgmap v1323: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 45759/157554 objects degraded (29.043%); 18126/157554 objects misplaced (11.505%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:57:52.483 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:52.498 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:52.548 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:52 smithi008 ceph-mon[184418]: pgmap v1323: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 45759/157554 objects degraded (29.043%); 18126/157554 objects misplaced (11.505%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:57:52.552 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:52.614 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:52.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:52.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:52.718 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:52.748 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:52.955 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:52.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:52.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:53.059 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:53.097 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:53.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:57:53.248 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:57:53.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/213878809' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:53.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:53 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:53.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 45759/157554 objects degraded (29.043%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/213878809' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:53 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 45759/157554 objects degraded (29.043%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/213878809' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:53 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 45759/157554 objects degraded (29.043%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:57:54.163 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:54.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:54.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:54.266 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:54.301 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[184418]: pgmap v1324: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 45759/155961 objects degraded (29.340%); 18043/155961 objects misplaced (11.569%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:57:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[184418]: failed to encode map e442 with expected crc 2023-10-09T05:57:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[184418]: failed to encode map e442 with expected crc 2023-10-09T05:57:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[184418]: osdmap e442: 8 total, 8 up, 7 in 2023-10-09T05:57:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[184418]: failed to encode map e442 with expected crc 2023-10-09T05:57:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[191227]: pgmap v1324: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 45759/155961 objects degraded (29.340%); 18043/155961 objects misplaced (11.569%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:57:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[191227]: failed to encode map e442 with expected crc 2023-10-09T05:57:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[191227]: failed to encode map e442 with expected crc 2023-10-09T05:57:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[191227]: osdmap e442: 8 total, 8 up, 7 in 2023-10-09T05:57:54.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:54 smithi008 ceph-mon[191227]: failed to encode map e442 with expected crc 2023-10-09T05:57:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:54 smithi191 ceph-mon[161669]: pgmap v1324: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 45759/155961 objects degraded (29.340%); 18043/155961 objects misplaced (11.569%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T05:57:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:54 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:54 smithi191 ceph-mon[161669]: failed to encode map e442 with expected crc 2023-10-09T05:57:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:54 smithi191 ceph-mon[161669]: failed to encode map e442 with expected crc 2023-10-09T05:57:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:54 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:54 smithi191 ceph-mon[161669]: osdmap e442: 8 total, 8 up, 7 in 2023-10-09T05:57:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:54 smithi191 ceph-mon[161669]: failed to encode map e442 with expected crc 2023-10-09T05:57:54.557 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:54.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:54.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:54.663 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:54.698 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:55.130 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:57:55.130 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:57:55.242 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:55.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:55.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:55.347 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:55.386 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:55.464 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:57:55.483 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:57:55.503 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:57:55.527 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:57:55.548 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:55.556 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:57:55.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:55.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:55.581 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:57:55.612 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:57:55.638 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:57:55.640 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:57:55.640 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:57:55.640 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:57:55.640 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:57:55.640 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:57:55.640 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:57:55.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:57:55.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:57:55.658 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:55.691 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:55.868 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:57:55.900 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:55.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:55.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:56.003 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:56.033 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:56.089 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:56.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:56.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:56.196 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:56.235 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:56.253 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:56 smithi008 conmon[184394]: 2023-10-09T05:57:56.000+0000 7f8a66521700 -1 mon.a@0(leader).osd e442 definitely_dead 0 2023-10-09T05:57:56.267 INFO:teuthology.orchestra.run.smithi008.stderr:marked out osd.5. 2023-10-09T05:57:56.318 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:56.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:56.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:56.348 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:57:56.424 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:56.450 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:56.495 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:56.497 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:56 smithi191 ceph-mon[161669]: pgmap v1326: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 45759/155961 objects degraded (29.340%); 18043/155961 objects misplaced (11.569%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T05:57:56.497 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:56 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3160013013' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:56.498 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:56 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/544728069' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T05:57:56.498 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:56 smithi191 ceph-mon[161669]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T05:57:56.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:56.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:56.594 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:56 smithi008 ceph-mon[184418]: pgmap v1326: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 45759/155961 objects degraded (29.340%); 18043/155961 objects misplaced (11.569%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T05:57:56.594 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:56 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3160013013' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:56.594 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:56 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/544728069' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T05:57:56.594 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:56 smithi008 ceph-mon[184418]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T05:57:56.594 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:56 smithi008 ceph-mon[191227]: pgmap v1326: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 45759/155961 objects degraded (29.340%); 18043/155961 objects misplaced (11.569%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T05:57:56.594 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:56 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3160013013' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:56.594 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:56 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/544728069' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T05:57:56.595 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:56 smithi008 ceph-mon[191227]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T05:57:56.599 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:56.631 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:57.110 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:57.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:57.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:57.169 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:57:57.213 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:57.326 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:57.409 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:57.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:57.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:57.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[184418]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/544728069' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T05:57:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[184418]: osdmap e443: 8 total, 8 up, 6 in 2023-10-09T05:57:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[184418]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[184418]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[184418]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[184418]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[184418]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[184418]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[184418]: osdmap e444: 8 total, 8 up, 6 in 2023-10-09T05:57:57.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[191227]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/544728069' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T05:57:57.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[191227]: osdmap e443: 8 total, 8 up, 6 in 2023-10-09T05:57:57.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[191227]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[191227]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[191227]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[191227]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[191227]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[191227]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:57 smithi008 ceph-mon[191227]: osdmap e444: 8 total, 8 up, 6 in 2023-10-09T05:57:57.514 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:57.546 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:57 smithi191 ceph-mon[161669]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:57 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/544728069' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T05:57:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:57 smithi191 ceph-mon[161669]: osdmap e443: 8 total, 8 up, 6 in 2023-10-09T05:57:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:57 smithi191 ceph-mon[161669]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:57 smithi191 ceph-mon[161669]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:57 smithi191 ceph-mon[161669]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:57 smithi191 ceph-mon[161669]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:57 smithi191 ceph-mon[161669]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:57 smithi191 ceph-mon[161669]: failed to encode map e443 with expected crc 2023-10-09T05:57:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:57 smithi191 ceph-mon[161669]: osdmap e444: 8 total, 8 up, 6 in 2023-10-09T05:57:57.600 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:57.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:57.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:57.609 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:57.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:57.748 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:57.849 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:57.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:57.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:57.888 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:57:57.888 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:57:57.888 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:57:57.889 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:57:57.889 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:57:57.952 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:57.983 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:58.245 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:58.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:58.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:58.354 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:58.395 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[184418]: pgmap v1328: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 45585/154644 objects degraded (29.477%); 17980/154644 objects misplaced (11.627%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T05:57:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[184418]: failed to encode map e444 with expected crc 2023-10-09T05:57:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[184418]: failed to encode map e444 with expected crc 2023-10-09T05:57:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[184418]: failed to encode map e444 with expected crc 2023-10-09T05:57:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 45585/154644 objects degraded (29.477%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[184418]: osdmap e445: 8 total, 8 up, 6 in 2023-10-09T05:57:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[191227]: pgmap v1328: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 45585/154644 objects degraded (29.477%); 17980/154644 objects misplaced (11.627%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T05:57:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[191227]: failed to encode map e444 with expected crc 2023-10-09T05:57:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[191227]: failed to encode map e444 with expected crc 2023-10-09T05:57:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[191227]: failed to encode map e444 with expected crc 2023-10-09T05:57:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 45585/154644 objects degraded (29.477%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:58.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:58 smithi008 ceph-mon[191227]: osdmap e445: 8 total, 8 up, 6 in 2023-10-09T05:57:58.492 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:58.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:58.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:58.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:58 smithi191 ceph-mon[161669]: pgmap v1328: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 45585/154644 objects degraded (29.477%); 17980/154644 objects misplaced (11.627%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T05:57:58.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:58 smithi191 ceph-mon[161669]: failed to encode map e444 with expected crc 2023-10-09T05:57:58.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:58 smithi191 ceph-mon[161669]: failed to encode map e444 with expected crc 2023-10-09T05:57:58.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:58 smithi191 ceph-mon[161669]: failed to encode map e444 with expected crc 2023-10-09T05:57:58.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 45585/154644 objects degraded (29.477%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:58.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:58 smithi191 ceph-mon[161669]: osdmap e445: 8 total, 8 up, 6 in 2023-10-09T05:57:58.599 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:58.630 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:58.682 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:58.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:58.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:58.745 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:57:58.786 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:58.828 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:58.917 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:58.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:58.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:58.925 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:57:59.022 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:59.062 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:59.136 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:59.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:59.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:59.243 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:59.293 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:59.367 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:57:59.370 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:59.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:59.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:59 smithi191 ceph-mon[161669]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:59 smithi191 ceph-mon[161669]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:59 smithi191 ceph-mon[161669]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:59 smithi191 ceph-mon[161669]: 9.b starting backfill to osd.1 from (0'0,0'0] MAX to 442'7407 2023-10-09T05:57:59.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:59 smithi191 ceph-mon[161669]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:59 smithi191 ceph-mon[161669]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:59 smithi191 ceph-mon[161669]: 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 442'7321 2023-10-09T05:57:59.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:59 smithi191 ceph-mon[161669]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:59.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:57:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:59.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:59.514 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:59.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[184418]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[184418]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[184418]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[184418]: 9.b starting backfill to osd.1 from (0'0,0'0] MAX to 442'7407 2023-10-09T05:57:59.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[184418]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[184418]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[184418]: 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 442'7321 2023-10-09T05:57:59.567 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[184418]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.567 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:59.567 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:59.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[191227]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[191227]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[191227]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[191227]: 9.b starting backfill to osd.1 from (0'0,0'0] MAX to 442'7407 2023-10-09T05:57:59.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[191227]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[191227]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[191227]: 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 442'7321 2023-10-09T05:57:59.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[191227]: failed to encode map e445 with expected crc 2023-10-09T05:57:59.568 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:57:59.568 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:57:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:59.628 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:59.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:59.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:59.679 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:59.733 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:59.773 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:00.004 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:00.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:00.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:00.109 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:00.141 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:00.248 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:00.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:00.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:00.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:00.383 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:00.440 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:00 smithi008 ceph-mon[184418]: pgmap v1331: 99 pgs: 6 remapped+peering, 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 37216/153459 objects degraded (24.251%); 14242/153459 objects misplaced (9.281%); 2.3 MiB/s, 27 objects/s recovering 2023-10-09T05:58:00.440 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:00 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:58:00.440 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:00 smithi008 ceph-mon[191227]: pgmap v1331: 99 pgs: 6 remapped+peering, 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 37216/153459 objects degraded (24.251%); 14242/153459 objects misplaced (9.281%); 2.3 MiB/s, 27 objects/s recovering 2023-10-09T05:58:00.440 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:00 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:58:00.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:00 smithi191 ceph-mon[161669]: pgmap v1331: 99 pgs: 6 remapped+peering, 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 37216/153459 objects degraded (24.251%); 14242/153459 objects misplaced (9.281%); 2.3 MiB/s, 27 objects/s recovering 2023-10-09T05:58:00.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:00 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:58:00.661 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:00.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:00.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:00.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:58:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:58:00] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.33.4" 2023-10-09T05:58:00.769 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:00.804 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:00.858 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:00.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:00.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:00.962 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:00.991 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:01.043 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:01.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:01.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:01.147 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:01.166 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:58:01.166 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T05:58:01.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T05:58:01.166 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T05:58:01.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.166 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86696 8.14 8:2cb5a248:::benchmark_data_smithi008_432048_object86695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:58:01.167 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078540+0000", 2023-10-09T05:58:01.167 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 556.08418602500001, 2023-10-09T05:58:01.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.192233133, 2023-10-09T05:58:01.167 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.167 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:58:01.167 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.167 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:58:01.167 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:58:01.167 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86696 2023-10-09T05:58:01.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.167 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078540+0000", 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078540+0000", 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078538+0000", 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5841000000000003e-05 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078594+0000", 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4001e-05 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078608+0000", 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3899999999999998e-06 2023-10-09T05:58:01.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078614+0000", 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21596270400000001 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294577+0000", 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7299999999999994e-06 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294586+0000", 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97611344200000005 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.270699+0000", 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4124999999999999e-05 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.270773+0000", 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.192233133 2023-10-09T05:58:01.171 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.172 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.172 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.172 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86699 8.3 8:c3d6f3c4:::benchmark_data_smithi008_432048_object86698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:58:01.172 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:58:01.172 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 556.08410094299995, 2023-10-09T05:58:01.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650868249999999, 2023-10-09T05:58:01.172 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.173 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:58:01.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:58:01.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:58:01.173 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86699 2023-10-09T05:58:01.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.173 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:58:01.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:58:01.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078625+0000", 2023-10-09T05:58:01.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0989999999999999e-05 2023-10-09T05:58:01.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078666+0000", 2023-10-09T05:58:01.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7989999999999996e-06 2023-10-09T05:58:01.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078672+0000", 2023-10-09T05:58:01.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0949999999999998e-06 2023-10-09T05:58:01.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078676+0000", 2023-10-09T05:58:01.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15420414599999999 2023-10-09T05:58:01.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.232881+0000", 2023-10-09T05:58:01.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6739999999999992e-06 2023-10-09T05:58:01.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.232890+0000", 2023-10-09T05:58:01.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0107454360000001 2023-10-09T05:58:01.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.243636+0000", 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6348999999999997e-05 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.243712+0000", 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650868249999999 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86700 8.18 8:1e3af286:::benchmark_data_smithi008_432048_object86699:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078684+0000", 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 556.08404234, 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1857459349999999, 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86700 2023-10-09T05:58:01.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.181 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078684+0000", 2023-10-09T05:58:01.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078684+0000", 2023-10-09T05:58:01.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078683+0000", 2023-10-09T05:58:01.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9190999999999998e-05 2023-10-09T05:58:01.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078723+0000", 2023-10-09T05:58:01.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0850000000000002e-06 2023-10-09T05:58:01.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078729+0000", 2023-10-09T05:58:01.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7340000000000002e-06 2023-10-09T05:58:01.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078732+0000", 2023-10-09T05:58:01.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.179763071 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258495+0000", 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8499999999999999e-06 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258501+0000", 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0058671960000001 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.264369+0000", 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1284999999999993e-05 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.264430+0000", 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1857459349999999 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.186 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86707 8.18 8:1d2a32f3:::benchmark_data_smithi008_432048_object86706:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:58:01.186 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078737+0000", 2023-10-09T05:58:01.186 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 556.08398944099997, 2023-10-09T05:58:01.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.195960744, 2023-10-09T05:58:01.186 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.186 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86707 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078737+0000", 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078737+0000", 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078736+0000", 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9480999999999997e-05 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078776+0000", 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9616000000000002e-05 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078815+0000", 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2260000000000005e-06 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078824+0000", 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.179694931 2023-10-09T05:58:01.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258519+0000", 2023-10-09T05:58:01.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3319999999999999e-06 2023-10-09T05:58:01.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258524+0000", 2023-10-09T05:58:01.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0062883300000001 2023-10-09T05:58:01.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.264812+0000", 2023-10-09T05:58:01.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002849922 2023-10-09T05:58:01.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:58:01.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267662+0000", 2023-10-09T05:58:01.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0069641929999999996 2023-10-09T05:58:01.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274626+0000", 2023-10-09T05:58:01.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1110000000000002e-05 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274697+0000", 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.195960744 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86710 8.14 8:28e4dd35:::benchmark_data_smithi008_432048_object86709:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:58:01.194 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 556.08389783099994, 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2037845970000001, 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86710 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:58:01.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:58:01.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078828+0000", 2023-10-09T05:58:01.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5683000000000001e-05 2023-10-09T05:58:01.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078894+0000", 2023-10-09T05:58:01.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5679999999999996e-06 2023-10-09T05:58:01.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078900+0000", 2023-10-09T05:58:01.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0676000000000001e-05 2023-10-09T05:58:01.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078911+0000", 2023-10-09T05:58:01.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.215693774 2023-10-09T05:58:01.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294605+0000", 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2909999999999998e-06 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294610+0000", 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97619070799999996 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.270801+0000", 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037587950000000001 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274559+0000", 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079496329999999994 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282509+0000", 2023-10-09T05:58:01.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103909 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282613+0000", 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2037845970000001 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86715 8.14 8:2d9c7f0c:::benchmark_data_smithi008_432048_object86714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:58:01.201 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:58:01.202 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 556.08381012100006, 2023-10-09T05:58:01.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.203914588, 2023-10-09T05:58:01.202 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.202 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:01.202 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.202 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:58:01.202 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:58:01.202 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86715 2023-10-09T05:58:01.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.202 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:58:01.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:58:01.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078916+0000", 2023-10-09T05:58:01.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2407999999999997e-05 2023-10-09T05:58:01.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078968+0000", 2023-10-09T05:58:01.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4019999999999999e-06 2023-10-09T05:58:01.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078974+0000", 2023-10-09T05:58:01.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1539000000000001e-05 2023-10-09T05:58:01.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078986+0000", 2023-10-09T05:58:01.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.215643644 2023-10-09T05:58:01.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294630+0000", 2023-10-09T05:58:01.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5470000000000004e-06 2023-10-09T05:58:01.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.294635+0000", 2023-10-09T05:58:01.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.979978393 2023-10-09T05:58:01.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274614+0000", 2023-10-09T05:58:01.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014625400000000001 2023-10-09T05:58:01.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:58:01.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274760+0000", 2023-10-09T05:58:01.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017972999999999999 2023-10-09T05:58:01.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T05:58:01.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.274940+0000", 2023-10-09T05:58:01.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017525329999999999 2023-10-09T05:58:01.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:58:01.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.276692+0000", 2023-10-09T05:58:01.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003165429 2023-10-09T05:58:01.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.279857+0000", 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028214580000000002 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282679+0000", 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9833000000000002e-05 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282729+0000", 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010188600000000001 2023-10-09T05:58:01.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.282831+0000", 2023-10-09T05:58:01.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.203914588 2023-10-09T05:58:01.212 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.212 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.212 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.212 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86716 8.8 8:1416e0db:::benchmark_data_smithi008_432048_object86715:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e299)", 2023-10-09T05:58:01.212 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 556.08373579500005, 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1828720580000001, 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86716 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.078990+0000", 2023-10-09T05:58:01.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0300000000000002e-05 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.079050+0000", 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0120000000000002e-06 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.079056+0000", 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.552e-06 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.079060+0000", 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17917138099999999 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258231+0000", 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0326000000000001e-05 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258242+0000", 2023-10-09T05:58:01.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99156280299999999 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.249804+0000", 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000172201 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.249977+0000", 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000205291 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.250182+0000", 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0089602589999999999 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.259142+0000", 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025977299999999999 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261740+0000", 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1816999999999997e-05 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261772+0000", 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1427000000000001e-05 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261823+0000", 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9397999999999998e-05 2023-10-09T05:58:01.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.261862+0000", 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1828720580000001 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45782.0:86719 8.8 8:14442d3e:::benchmark_data_smithi008_432048_object86718:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:48:45.102694+0000", 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 556.060032181, 2023-10-09T05:58:01.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650766299999999, 2023-10-09T05:58:01.221 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.221 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:01.221 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.221 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45782", 2023-10-09T05:58:01.221 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/223202577", 2023-10-09T05:58:01.221 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86719 2023-10-09T05:58:01.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.221 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102694+0000", 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102694+0000", 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999962 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102690+0000", 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0977000000000002e-05 2023-10-09T05:58:01.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102751+0000", 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2374e-05 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102774+0000", 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2845e-05 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.102786+0000", 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15555860099999999 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258345+0000", 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0730000000000005e-06 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:45.258353+0000", 2023-10-09T05:58:01.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0014172180000001 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.259770+0000", 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000128658 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.259899+0000", 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016774100000000001 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:58:01.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.260067+0000", 2023-10-09T05:58:01.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039100710000000002 2023-10-09T05:58:01.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.263977+0000", 2023-10-09T05:58:01.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029711770000000002 2023-10-09T05:58:01.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.266948+0000", 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00073392599999999998 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267682+0000", 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3998e-05 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267736+0000", 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4730000000000001e-05 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:48:46.267771+0000", 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1650766299999999 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:45770 9.a 9:54ab40b6:::benchmark_data_smithi008_502820_object45769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e372)", 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:53:46.056257+0000", 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 255.10646920900001, 2023-10-09T05:58:01.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0314687650000001, 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 45770 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056257+0000", 2023-10-09T05:58:01.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056257+0000", 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999952 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056252+0000", 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2493e-05 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056305+0000", 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6889e-05 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056322+0000", 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3180000000000001e-06 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056329+0000", 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6201e-05 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056375+0000", 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7790000000000007e-06 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056385+0000", 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88098339699999995 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.937368+0000", 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1769e-05 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.937380+0000", 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1403479299999999 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.077728+0000", 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000195169 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.077923+0000", 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015467500000000001 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.078078+0000", 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0036624750000000001 2023-10-09T05:58:01.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:58:01.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.081740+0000", 2023-10-09T05:58:01.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003406278 2023-10-09T05:58:01.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.085147+0000", 2023-10-09T05:58:01.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00062187400000000002 2023-10-09T05:58:01.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.236 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.085768+0000", 2023-10-09T05:58:01.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7609999999999997e-05 2023-10-09T05:58:01.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.237 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:58:01.237 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.085846+0000", 2023-10-09T05:58:01.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018796590000000001 2023-10-09T05:58:01.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.237 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.237 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.087726+0000", 2023-10-09T05:58:01.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0314687650000001 2023-10-09T05:58:01.237 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.238 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.238 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.238 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:77607 9.7 9:e59624ae:::benchmark_data_smithi008_502820_object77606:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T05:58:01.238 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.819106+0000", 2023-10-09T05:58:01.239 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.343619851, 2023-10-09T05:58:01.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585100272, 2023-10-09T05:58:01.239 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.239 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:58:01.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.239 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 77607 2023-10-09T05:58:01.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.239 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819106+0000", 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819106+0000", 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819106+0000", 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1301000000000002e-05 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819157+0000", 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6730000000000003e-06 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819164+0000", 2023-10-09T05:58:01.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6500000000000004e-06 2023-10-09T05:58:01.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819168+0000", 2023-10-09T05:58:01.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.052763364 2023-10-09T05:58:01.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871932+0000", 2023-10-09T05:58:01.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.26e-05 2023-10-09T05:58:01.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871944+0000", 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81505207000000002 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.686996+0000", 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.464e-05 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.687011+0000", 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.70573758499999995 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.392749+0000", 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00064714099999999995 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393396+0000", 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010737744 2023-10-09T05:58:01.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404133+0000", 2023-10-09T05:58:01.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3168000000000001e-05 2023-10-09T05:58:01.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404207+0000", 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585100272 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:77621 9.7 9:e7b7c537:::benchmark_data_smithi008_502820_object77620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.819295+0000", 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.34343153, 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585638342, 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.246 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:01.247 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.247 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.247 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.247 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 77621 2023-10-09T05:58:01.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.247 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819295+0000", 2023-10-09T05:58:01.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819295+0000", 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819294+0000", 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1934000000000002e-05 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819336+0000", 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5550000000000002e-06 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819342+0000", 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.766e-06 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819345+0000", 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.052626693000000002 2023-10-09T05:58:01.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871972+0000", 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2029999999999998e-06 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871981+0000", 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81507707100000004 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.687058+0000", 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7999999999999993e-06 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.687068+0000", 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.70636606700000004 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393434+0000", 2023-10-09T05:58:01.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011293 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393547+0000", 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000142881 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393690+0000", 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0044343000000000004 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.398124+0000", 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6621e-05 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.398171+0000", 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060740669999999998 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404245+0000", 2023-10-09T05:58:01.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8288999999999996e-05 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404313+0000", 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00061965000000000002 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404933+0000", 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585638342 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.254 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.255 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:99261 9.a 9:56a2e757:::benchmark_data_smithi008_502820_object15088:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e406)", 2023-10-09T05:58:01.255 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:39.286092+0000", 2023-10-09T05:58:01.255 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 141.87663417100001, 2023-10-09T05:58:01.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.036949087, 2023-10-09T05:58:01.255 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.255 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:01.255 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.255 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.256 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.256 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99261 2023-10-09T05:58:01.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.256 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286092+0000", 2023-10-09T05:58:01.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286092+0000", 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286092+0000", 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.79e-06 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286094+0000", 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9939999999999997e-06 2023-10-09T05:58:01.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286100+0000", 2023-10-09T05:58:01.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.138e-06 2023-10-09T05:58:01.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286105+0000", 2023-10-09T05:58:01.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5616000000000001e-05 2023-10-09T05:58:01.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286200+0000", 2023-10-09T05:58:01.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0020000000000003e-06 2023-10-09T05:58:01.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286206+0000", 2023-10-09T05:58:01.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0250316589999999 2023-10-09T05:58:01.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.311238+0000", 2023-10-09T05:58:01.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000140032 2023-10-09T05:58:01.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:58:01.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.311378+0000", 2023-10-09T05:58:01.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000164034 2023-10-09T05:58:01.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T05:58:01.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.311542+0000", 2023-10-09T05:58:01.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011212313999999999 2023-10-09T05:58:01.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:58:01.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322754+0000", 2023-10-09T05:58:01.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3466e-05 2023-10-09T05:58:01.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322798+0000", 2023-10-09T05:58:01.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4289000000000001e-05 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322852+0000", 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9132999999999999e-05 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322901+0000", 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9549999999999999e-05 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:58:01.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322941+0000", 2023-10-09T05:58:01.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100512 2023-10-09T05:58:01.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323041+0000", 2023-10-09T05:58:01.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.036949087 2023-10-09T05:58:01.265 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.265 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.266 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.266 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:99272 9.a 9:54aba215:::benchmark_data_smithi008_502820_object15099:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e406)", 2023-10-09T05:58:01.266 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:39.286108+0000", 2023-10-09T05:58:01.267 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 141.87661772199999, 2023-10-09T05:58:01.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0372746719999999, 2023-10-09T05:58:01.267 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.267 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:01.267 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.267 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.267 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.267 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99272 2023-10-09T05:58:01.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.267 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286108+0000", 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286108+0000", 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286108+0000", 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2369999999999999e-06 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286110+0000", 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.49e-06 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286116+0000", 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7799999999999998e-06 2023-10-09T05:58:01.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286120+0000", 2023-10-09T05:58:01.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106143 2023-10-09T05:58:01.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286226+0000", 2023-10-09T05:58:01.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6489999999999996e-06 2023-10-09T05:58:01.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286231+0000", 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0284209600000001 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.314652+0000", 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016349299999999999 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.314816+0000", 2023-10-09T05:58:01.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019973899999999999 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.315016+0000", 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0077500399999999997 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322766+0000", 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000319585 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323085+0000", 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9708e-05 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323125+0000", 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2876e-05 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323178+0000", 2023-10-09T05:58:01.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2748000000000003e-05 2023-10-09T05:58:01.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:58:01.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323241+0000", 2023-10-09T05:58:01.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014258200000000001 2023-10-09T05:58:01.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323383+0000", 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0372746719999999 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:103011 9.15 9:ad5520b2:::benchmark_data_smithi008_502820_object18838:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e420)", 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:56:11.329331+0000", 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 109.833395572, 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.236479597, 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 103011 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329331+0000", 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329331+0000", 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329328+0000", 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.61e-06 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329336+0000", 2023-10-09T05:58:01.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1816e-05 2023-10-09T05:58:01.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329348+0000", 2023-10-09T05:58:01.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9629999999999998e-06 2023-10-09T05:58:01.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329356+0000", 2023-10-09T05:58:01.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033304369999999999 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332686+0000", 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0757000000000001e-05 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332697+0000", 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2147822829999999 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.547479+0000", 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014378700000000001 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.547623+0000", 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118553 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.547742+0000", 2023-10-09T05:58:01.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00508282 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.552824+0000", 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011131831 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.563956+0000", 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3128999999999998e-05 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.564019+0000", 2023-10-09T05:58:01.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9806999999999998e-05 2023-10-09T05:58:01.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:58:01.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.564079+0000", 2023-10-09T05:58:01.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017310330000000001 2023-10-09T05:58:01.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.565810+0000", 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.236479597 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:103018 9.15 9:a99fcd49:::benchmark_data_smithi008_502820_object18845:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e420)", 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:56:11.332182+0000", 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 109.830543914, 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2352073649999999, 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:01.283 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 103018 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332182+0000", 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332182+0000", 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332181+0000", 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.511e-06 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332188+0000", 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1018e-05 2023-10-09T05:58:01.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332199+0000", 2023-10-09T05:58:01.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3490000000000003e-06 2023-10-09T05:58:01.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332206+0000", 2023-10-09T05:58:01.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00051741500000000002 2023-10-09T05:58:01.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332723+0000", 2023-10-09T05:58:01.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6009999999999997e-06 2023-10-09T05:58:01.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332731+0000", 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.216443433 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.549174+0000", 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016414700000000001 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.549339+0000", 2023-10-09T05:58:01.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111173 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.549450+0000", 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014515375 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.563965+0000", 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001919088 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.565884+0000", 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017655200000000001 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.566061+0000", 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9567000000000007e-05 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:58:01.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.566130+0000", 2023-10-09T05:58:01.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012592809999999999 2023-10-09T05:58:01.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.567390+0000", 2023-10-09T05:58:01.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2352073649999999 2023-10-09T05:58:01.291 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.291 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.291 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:103025 9.15 9:aa461274:::benchmark_data_smithi008_502820_object18852:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e420)", 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:56:11.332212+0000", 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 109.830514017, 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.236778771, 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 103025 2023-10-09T05:58:01.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332212+0000", 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332212+0000", 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332212+0000", 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.534e-06 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332215+0000", 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5549999999999999e-06 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332222+0000", 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4190000000000004e-06 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332226+0000", 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00089926800000000005 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.333125+0000", 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1258e-05 2023-10-09T05:58:01.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.333137+0000", 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2177126110000001 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.550849+0000", 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014004999999999999 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:58:01.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.550989+0000", 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015518300000000001 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.551145+0000", 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012827781999999999 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.563972+0000", 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019446349999999999 2023-10-09T05:58:01.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.565917+0000", 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001559324 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.567476+0000", 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3046000000000006e-05 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.567539+0000", 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014516080000000001 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.568991+0000", 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.236778771 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:103039 9.5 9:a61879d9:::benchmark_data_smithi008_502820_object18866:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e420)", 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:56:11.332231+0000", 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 109.83049488899999, 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2042948360000001, 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 103039 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332231+0000", 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332231+0000", 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332231+0000", 2023-10-09T05:58:01.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9050000000000001e-06 2023-10-09T05:58:01.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332234+0000", 2023-10-09T05:58:01.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0390000000000001e-06 2023-10-09T05:58:01.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332240+0000", 2023-10-09T05:58:01.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9910000000000004e-06 2023-10-09T05:58:01.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332244+0000", 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011900470000000001 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.333434+0000", 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3599999999999996e-06 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.333442+0000", 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.189327544 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.522770+0000", 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015325399999999999 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.522923+0000", 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012667599999999999 2023-10-09T05:58:01.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.523050+0000", 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0097749440000000007 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.532825+0000", 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021428039999999999 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.306 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.534967+0000", 2023-10-09T05:58:01.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7425000000000001e-05 2023-10-09T05:58:01.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.306 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:01.306 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.535015+0000", 2023-10-09T05:58:01.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6205999999999995e-05 2023-10-09T05:58:01.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.306 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.535091+0000", 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001435052 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.536526+0000", 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2042948360000001 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117308 9.7 9.dae134a7 (undecoded) ondisk+write+known_if_redirected e443)", 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:56.440558+0000", 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 4.7221682630000004, 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81392114900000001, 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117308 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.308 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.440558+0000", 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.440558+0000", 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.440555+0000", 2023-10-09T05:58:01.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3183999999999999e-05 2023-10-09T05:58:01.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.310 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.440569+0000", 2023-10-09T05:58:01.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3480000000000003e-05 2023-10-09T05:58:01.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.310 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.440622+0000", 2023-10-09T05:58:01.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9729999999999992e-06 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.440632+0000", 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1914000000000003e-05 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.440704+0000", 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4011e-05 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.440718+0000", 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81374861700000001 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.254467+0000", 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2434e-05 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.254479+0000", 2023-10-09T05:58:01.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81392114900000001 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117312 9.9 9.54707909 (undecoded) ondisk+write+known_if_redirected e443)", 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:56.591251+0000", 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 4.571475242, 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65641013000000004, 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:58:01.313 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117312 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.591251+0000", 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.591251+0000", 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999967 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.591248+0000", 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4025999999999999e-05 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.591262+0000", 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2218e-05 2023-10-09T05:58:01.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.591284+0000", 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3119999999999994e-06 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.591292+0000", 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5362000000000003e-05 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.591347+0000", 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4477e-05 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.591362+0000", 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65629019700000002 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.247652+0000", 2023-10-09T05:58:01.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9849999999999995e-06 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.247661+0000", 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65641013000000004 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117314 9.19 9.ba8468b9 (undecoded) ondisk+write+known_if_redirected e443)", 2023-10-09T05:58:01.318 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:56.690099+0000", 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 4.4726275429999998, 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.55746747100000005, 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117314 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.690099+0000", 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.690099+0000", 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.690097+0000", 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2169999999999997e-06 2023-10-09T05:58:01.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T05:58:01.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.690104+0000", 2023-10-09T05:58:01.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2962000000000001e-05 2023-10-09T05:58:01.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T05:58:01.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.690117+0000", 2023-10-09T05:58:01.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3924999999999999e-05 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.690141+0000", 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1841999999999995e-05 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.690203+0000", 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3739e-05 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:56.690217+0000", 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.55733759299999996 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.247554+0000", 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1667e-05 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T05:58:01.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.247566+0000", 2023-10-09T05:58:01.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.55746747100000005 2023-10-09T05:58:01.324 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.324 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.324 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.324 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T05:58:01.324 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T05:58:01.324 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:58:01.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:01.325 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:01.326 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:01.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:01.326 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:58:01.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:01.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:01 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2887416463' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:01.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:01 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2887416463' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:01.471 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:01 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2887416463' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:01.474 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:01.531 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:01.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:01.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:01.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:01.665 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:02.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:58:02.008 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:02.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:02.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:02.115 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:02.146 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:02.204 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 6, 1, 7, 3] out_osds: [0, 5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:58:02.205 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:58:02.205 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [2, 4, 6, 1, 7, 3] 2023-10-09T05:58:02.205 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T05:58:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:02 smithi191 ceph-mon[161669]: pgmap v1332: 99 pgs: 1 activating+undersized+degraded+remapped, 5 remapped+peering, 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 42959/153270 objects degraded (28.028%); 17459/153270 objects misplaced (11.391%); 2.6 MiB/s, 33 objects/s recovering 2023-10-09T05:58:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:02 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:02 smithi191 ceph-mon[161669]: failed to encode map e446 with expected crc 2023-10-09T05:58:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:02 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2887416463' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:58:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:02 smithi191 ceph-mon[161669]: osdmap e446: 8 total, 8 up, 6 in 2023-10-09T05:58:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:02 smithi191 ceph-mon[161669]: failed to encode map e446 with expected crc 2023-10-09T05:58:02.486 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:58:02.709 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[184418]: pgmap v1332: 99 pgs: 1 activating+undersized+degraded+remapped, 5 remapped+peering, 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 42959/153270 objects degraded (28.028%); 17459/153270 objects misplaced (11.391%); 2.6 MiB/s, 33 objects/s recovering 2023-10-09T05:58:02.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:02.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[184418]: failed to encode map e446 with expected crc 2023-10-09T05:58:02.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2887416463' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:58:02.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[184418]: osdmap e446: 8 total, 8 up, 6 in 2023-10-09T05:58:02.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[184418]: failed to encode map e446 with expected crc 2023-10-09T05:58:02.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[191227]: pgmap v1332: 99 pgs: 1 activating+undersized+degraded+remapped, 5 remapped+peering, 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 42959/153270 objects degraded (28.028%); 17459/153270 objects misplaced (11.391%); 2.6 MiB/s, 33 objects/s recovering 2023-10-09T05:58:02.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:02.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[191227]: failed to encode map e446 with expected crc 2023-10-09T05:58:02.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2887416463' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:58:02.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[191227]: osdmap e446: 8 total, 8 up, 6 in 2023-10-09T05:58:02.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:02 smithi008 ceph-mon[191227]: failed to encode map e446 with expected crc 2023-10-09T05:58:02.968 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:02.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:03.071 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:03.103 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:03.220 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:03.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:03.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:03.327 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:03.358 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:03.363 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:03.434 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:03.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:03.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:03.483 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:58:03.509 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:58:03.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:03.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:03 smithi191 ceph-mon[161669]: failed to encode map e446 with expected crc 2023-10-09T05:58:03.569 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 42959/153270 objects degraded (28.028%), 22 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:03.572 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:03.645 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:03 smithi008 ceph-mon[184418]: failed to encode map e446 with expected crc 2023-10-09T05:58:03.645 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 42959/153270 objects degraded (28.028%), 22 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:03.645 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:03 smithi008 ceph-mon[191227]: failed to encode map e446 with expected crc 2023-10-09T05:58:03.645 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 42959/153270 objects degraded (28.028%), 22 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:03.681 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:03.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:03.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:03.788 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:03.822 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:03.877 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:03.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:03.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:03.981 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:04.016 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:04.086 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:04.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:04.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:04.169 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:58:04.169 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:58:04.169 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T05:58:04.169 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:58:04.191 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:04.222 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:04.277 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:04.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:04.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:04.380 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:04.411 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:04.468 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:04.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:04.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:04 smithi191 ceph-mon[161669]: pgmap v1334: 99 pgs: 1 activating+undersized+degraded+remapped, 4 remapped+peering, 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 48395/153261 objects degraded (31.577%); 17458/153261 objects misplaced (11.391%); 1.4 MiB/s, 13 objects/s recovering 2023-10-09T05:58:04.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3110287553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:04.572 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:04.598 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:04 smithi008 ceph-mon[184418]: pgmap v1334: 99 pgs: 1 activating+undersized+degraded+remapped, 4 remapped+peering, 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 48395/153261 objects degraded (31.577%); 17458/153261 objects misplaced (11.391%); 1.4 MiB/s, 13 objects/s recovering 2023-10-09T05:58:04.598 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3110287553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:04.598 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:04 smithi008 ceph-mon[191227]: pgmap v1334: 99 pgs: 1 activating+undersized+degraded+remapped, 4 remapped+peering, 12 peering, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 48395/153261 objects degraded (31.577%); 17458/153261 objects misplaced (11.391%); 1.4 MiB/s, 13 objects/s recovering 2023-10-09T05:58:04.598 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3110287553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:04.601 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:04.650 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:04.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:04.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:04.754 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:04.784 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:04.834 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:04.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:04.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:04.886 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:58:04.938 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:04.968 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:05.067 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:58:05.256 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:05.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:05.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:05.341 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:58:05.360 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:05.398 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:05.501 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:58:05.504 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:05.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:05.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:05.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:05 smithi008 ceph-mon[184418]: pgmap v1335: 99 pgs: 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 4 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 64 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 264 KiB/s rd, 165 KiB/s wr, 145 op/s; 64167/153156 objects degraded (41.896%); 17381/153156 objects misplaced (11.349%); 1.3 MiB/s, 17 objects/s recovering 2023-10-09T05:58:05.513 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:05 smithi008 ceph-mon[184418]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:58:05.515 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:05 smithi008 ceph-mon[191227]: pgmap v1335: 99 pgs: 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 4 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 64 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 264 KiB/s rd, 165 KiB/s wr, 145 op/s; 64167/153156 objects degraded (41.896%); 17381/153156 objects misplaced (11.349%); 1.3 MiB/s, 17 objects/s recovering 2023-10-09T05:58:05.515 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:05 smithi008 ceph-mon[191227]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:58:05.531 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:58:05.553 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:58:05.572 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:58:05.593 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:58:05.614 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:05.618 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:58:05.619 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:05 smithi191 ceph-mon[161669]: pgmap v1335: 99 pgs: 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 4 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 64 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 264 KiB/s rd, 165 KiB/s wr, 145 op/s; 64167/153156 objects degraded (41.896%); 17381/153156 objects misplaced (11.349%); 1.3 MiB/s, 17 objects/s recovering 2023-10-09T05:58:05.619 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:05 smithi191 ceph-mon[161669]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:58:05.622 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:58:05.622 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:58:05.622 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:58:05.622 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:58:05.622 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:58:05.622 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:58:05.622 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:58:05.623 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:58:05.645 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:05.672 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:05.947 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:05.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:05.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:06.051 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:06.077 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:06.120 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:06.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:06.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:06.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:06 smithi008 conmon[184394]: 2023-10-09T05:58:06.108+0000 7f8a66521700 -1 mon.a@0(leader).osd e446 definitely_dead 0 2023-10-09T05:58:06.224 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:06.251 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:06.375 INFO:teuthology.orchestra.run.smithi008.stderr:marked out osd.6. 2023-10-09T05:58:06.493 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:06.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:06.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:06.535 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4021162370' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.535 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:06 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.535 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:06 smithi191 ceph-mon[161669]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T05:58:06.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:06.630 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:06.683 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:06.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:06.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:06.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4021162370' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:06 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:06 smithi008 ceph-mon[184418]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T05:58:06.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4021162370' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:06 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:06 smithi008 ceph-mon[191227]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T05:58:06.787 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:06.825 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:07.003 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:58:07.003 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:58:07.412 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:07.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:07.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:07.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[184418]: failed to encode map e447 with expected crc 2023-10-09T05:58:07.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[184418]: failed to encode map e447 with expected crc 2023-10-09T05:58:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T05:58:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[184418]: osdmap e447: 8 total, 8 up, 5 in 2023-10-09T05:58:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[184418]: failed to encode map e447 with expected crc 2023-10-09T05:58:07.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[184418]: pgmap v1337: 99 pgs: 1 activating+degraded+remapped, 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 64 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 244 KiB/s rd, 153 KiB/s wr, 107 op/s; 72567/153138 objects degraded (47.387%); 21552/153138 objects misplaced (14.074%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T05:58:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[191227]: failed to encode map e447 with expected crc 2023-10-09T05:58:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[191227]: failed to encode map e447 with expected crc 2023-10-09T05:58:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T05:58:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[191227]: osdmap e447: 8 total, 8 up, 5 in 2023-10-09T05:58:07.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[191227]: failed to encode map e447 with expected crc 2023-10-09T05:58:07.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:07 smithi008 ceph-mon[191227]: pgmap v1337: 99 pgs: 1 activating+degraded+remapped, 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 64 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 244 KiB/s rd, 153 KiB/s wr, 107 op/s; 72567/153138 objects degraded (47.387%); 21552/153138 objects misplaced (14.074%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T05:58:07.518 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:07 smithi191 ceph-mon[161669]: failed to encode map e447 with expected crc 2023-10-09T05:58:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:07 smithi191 ceph-mon[161669]: failed to encode map e447 with expected crc 2023-10-09T05:58:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:07 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T05:58:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:07 smithi191 ceph-mon[161669]: osdmap e447: 8 total, 8 up, 5 in 2023-10-09T05:58:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:07 smithi191 ceph-mon[161669]: failed to encode map e447 with expected crc 2023-10-09T05:58:07.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:07 smithi191 ceph-mon[161669]: pgmap v1337: 99 pgs: 1 activating+degraded+remapped, 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 64 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 244 KiB/s rd, 153 KiB/s wr, 107 op/s; 72567/153138 objects degraded (47.387%); 21552/153138 objects misplaced (14.074%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T05:58:07.581 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:07.703 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:58:07.709 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:07.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:07.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:07.815 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:07.849 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:08.098 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:58:08.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[184418]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[184418]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[184418]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[184418]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[184418]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[184418]: osdmap e448: 8 total, 8 up, 5 in 2023-10-09T05:58:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[184418]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 72567/153138 objects degraded (47.387%), 30 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:08.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[184418]: osdmap e449: 8 total, 8 up, 5 in 2023-10-09T05:58:08.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[191227]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[191227]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[191227]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[191227]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[191227]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[191227]: osdmap e448: 8 total, 8 up, 5 in 2023-10-09T05:58:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[191227]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 72567/153138 objects degraded (47.387%), 30 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:08.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:08 smithi008 ceph-mon[191227]: osdmap e449: 8 total, 8 up, 5 in 2023-10-09T05:58:08.609 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:08 smithi191 ceph-mon[161669]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.609 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:08 smithi191 ceph-mon[161669]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.609 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:08 smithi191 ceph-mon[161669]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.609 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:08 smithi191 ceph-mon[161669]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.609 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:08 smithi191 ceph-mon[161669]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.610 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:08 smithi191 ceph-mon[161669]: osdmap e448: 8 total, 8 up, 5 in 2023-10-09T05:58:08.610 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:08 smithi191 ceph-mon[161669]: failed to encode map e448 with expected crc 2023-10-09T05:58:08.610 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 72567/153138 objects degraded (47.387%), 30 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:08.610 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:08 smithi191 ceph-mon[161669]: osdmap e449: 8 total, 8 up, 5 in 2023-10-09T05:58:08.653 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:08.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:08.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:08.756 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:08.786 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:09.223 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:58:09.288 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:09.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:09.393 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:09.439 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:09.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:09 smithi191 ceph-mon[161669]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:09 smithi191 ceph-mon[161669]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:09 smithi191 ceph-mon[161669]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:09 smithi191 ceph-mon[161669]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:09 smithi191 ceph-mon[161669]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:09 smithi191 ceph-mon[161669]: 9.d starting backfill to osd.2 from (0'0,0'0] MAX to 446'7325 2023-10-09T05:58:09.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:09 smithi191 ceph-mon[161669]: pgmap v1340: 99 pgs: 1 remapped+peering, 1 activating+degraded+remapped, 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 326 KiB/s rd, 204 KiB/s wr, 132 op/s; 72567/153126 objects degraded (47.390%); 21551/153126 objects misplaced (14.074%); 12 MiB/s, 197 objects/s recovering 2023-10-09T05:58:09.466 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:09 smithi191 ceph-mon[161669]: osdmap e450: 8 total, 8 up, 5 in 2023-10-09T05:58:09.505 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:09.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:09.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:09.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:09.650 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:09.712 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[184418]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[184418]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[184418]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[184418]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[184418]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[184418]: 9.d starting backfill to osd.2 from (0'0,0'0] MAX to 446'7325 2023-10-09T05:58:09.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[184418]: pgmap v1340: 99 pgs: 1 remapped+peering, 1 activating+degraded+remapped, 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 326 KiB/s rd, 204 KiB/s wr, 132 op/s; 72567/153126 objects degraded (47.390%); 21551/153126 objects misplaced (14.074%); 12 MiB/s, 197 objects/s recovering 2023-10-09T05:58:09.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[184418]: osdmap e450: 8 total, 8 up, 5 in 2023-10-09T05:58:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[191227]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[191227]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[191227]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[191227]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[191227]: failed to encode map e449 with expected crc 2023-10-09T05:58:09.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[191227]: 9.d starting backfill to osd.2 from (0'0,0'0] MAX to 446'7325 2023-10-09T05:58:09.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[191227]: pgmap v1340: 99 pgs: 1 remapped+peering, 1 activating+degraded+remapped, 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 326 KiB/s rd, 204 KiB/s wr, 132 op/s; 72567/153126 objects degraded (47.390%); 21551/153126 objects misplaced (14.074%); 12 MiB/s, 197 objects/s recovering 2023-10-09T05:58:09.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:09 smithi008 ceph-mon[191227]: osdmap e450: 8 total, 8 up, 5 in 2023-10-09T05:58:09.716 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:58:09.716 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:58:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:09.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:09.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:09.873 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:09.939 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:09.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:10.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:10.069 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:10.129 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:10.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:10.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:10.235 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:10.272 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:10.369 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:10.378 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:10 smithi191 ceph-mon[161669]: failed to encode map e450 with expected crc 2023-10-09T05:58:10.378 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:10 smithi191 ceph-mon[161669]: 9.b continuing backfill to osd.1 from (367'200,442'7407] MIN to 442'7407 2023-10-09T05:58:10.378 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:10 smithi191 ceph-mon[161669]: 9.b starting backfill to osd.3 from (0'0,0'0] MAX to 442'7407 2023-10-09T05:58:10.378 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:10 smithi191 ceph-mon[161669]: failed to encode map e450 with expected crc 2023-10-09T05:58:10.379 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:10.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:10.452 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:10 smithi008 ceph-mon[184418]: failed to encode map e450 with expected crc 2023-10-09T05:58:10.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:10 smithi008 ceph-mon[184418]: 9.b continuing backfill to osd.1 from (367'200,442'7407] MIN to 442'7407 2023-10-09T05:58:10.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:10 smithi008 ceph-mon[184418]: 9.b starting backfill to osd.3 from (0'0,0'0] MAX to 442'7407 2023-10-09T05:58:10.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:10 smithi008 ceph-mon[184418]: failed to encode map e450 with expected crc 2023-10-09T05:58:10.453 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:10 smithi008 ceph-mon[191227]: failed to encode map e450 with expected crc 2023-10-09T05:58:10.453 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:10 smithi008 ceph-mon[191227]: 9.b continuing backfill to osd.1 from (367'200,442'7407] MIN to 442'7407 2023-10-09T05:58:10.453 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:10 smithi008 ceph-mon[191227]: 9.b starting backfill to osd.3 from (0'0,0'0] MAX to 442'7407 2023-10-09T05:58:10.453 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:10 smithi008 ceph-mon[191227]: failed to encode map e450 with expected crc 2023-10-09T05:58:10.472 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:58:10.484 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:10.552 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:10.620 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:10.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:10.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:10.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:58:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:58:10] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.33.4" 2023-10-09T05:58:10.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:10.761 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:10.835 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:10.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:10.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:10.937 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:58:10.945 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:10.976 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:11.176 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:11.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:11.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:11.281 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:11.318 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:11.392 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:58:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:11 smithi008 ceph-mon[184418]: 9.11 starting backfill to osd.4 from (0'0,435'5920] MIN to 442'6133 2023-10-09T05:58:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:11 smithi008 ceph-mon[184418]: pgmap v1342: 99 pgs: 1 remapped+peering, 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 3 op/s; 77716/153093 objects degraded (50.764%); 24247/153093 objects misplaced (15.838%); 7.5 MiB/s, 306 objects/s recovering 2023-10-09T05:58:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1855515445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:11.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:11 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:11.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:11 smithi008 ceph-mon[191227]: 9.11 starting backfill to osd.4 from (0'0,435'5920] MIN to 442'6133 2023-10-09T05:58:11.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:11 smithi008 ceph-mon[191227]: pgmap v1342: 99 pgs: 1 remapped+peering, 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 3 op/s; 77716/153093 objects degraded (50.764%); 24247/153093 objects misplaced (15.838%); 7.5 MiB/s, 306 objects/s recovering 2023-10-09T05:58:11.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1855515445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:11.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:11 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:11 smithi191 ceph-mon[161669]: 9.11 starting backfill to osd.4 from (0'0,435'5920] MIN to 442'6133 2023-10-09T05:58:11.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:11 smithi191 ceph-mon[161669]: pgmap v1342: 99 pgs: 1 remapped+peering, 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 3 op/s; 77716/153093 objects degraded (50.764%); 24247/153093 objects misplaced (15.838%); 7.5 MiB/s, 306 objects/s recovering 2023-10-09T05:58:11.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1855515445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:11.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:11 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:11.592 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:11.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:11.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:11.696 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:11.725 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:12.530 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T05:58:12.530 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:12.530 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[184418]: failed to encode map e451 with expected crc 2023-10-09T05:58:12.530 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[184418]: failed to encode map e451 with expected crc 2023-10-09T05:58:12.530 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:12.532 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[184418]: osdmap e451: 8 total, 8 up, 5 in 2023-10-09T05:58:12.532 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T05:58:12.533 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:12.533 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[191227]: failed to encode map e451 with expected crc 2023-10-09T05:58:12.533 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[191227]: failed to encode map e451 with expected crc 2023-10-09T05:58:12.533 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:12.533 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:12 smithi008 ceph-mon[191227]: osdmap e451: 8 total, 8 up, 5 in 2023-10-09T05:58:12.705 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:12.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:12.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:12 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T05:58:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:12 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:12 smithi191 ceph-mon[161669]: failed to encode map e451 with expected crc 2023-10-09T05:58:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:12 smithi191 ceph-mon[161669]: failed to encode map e451 with expected crc 2023-10-09T05:58:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:12 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:12 smithi191 ceph-mon[161669]: osdmap e451: 8 total, 8 up, 5 in 2023-10-09T05:58:12.808 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:12.837 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:12.939 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:12.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:12.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:13.043 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:13.073 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:13.123 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:13.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:13.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:13.154 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:58:13.165 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:58:13.166 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:58:13.174 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:58:13.193 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:58:13.213 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:58:13.227 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:13.232 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:58:13.252 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:58:13.256 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:13.273 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:58:13.303 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:13.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:13.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:13.407 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:13.442 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:13 smithi191 ceph-mon[161669]: failed to encode map e452 with expected crc 2023-10-09T05:58:13.442 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:13 smithi191 ceph-mon[161669]: osdmap e452: 8 total, 8 up, 5 in 2023-10-09T05:58:13.442 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:13 smithi191 ceph-mon[161669]: pgmap v1345: 99 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 2 op/s; 77726/153093 objects degraded (50.770%); 24247/153093 objects misplaced (15.838%); 7.2 MiB/s, 297 objects/s recovering 2023-10-09T05:58:13.443 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/392251429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:13.443 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 77716/153093 objects degraded (50.764%), 31 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:13.447 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:13.509 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:13.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:13.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:13.614 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:13.641 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:13.687 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:13.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:13 smithi008 ceph-mon[184418]: failed to encode map e452 with expected crc 2023-10-09T05:58:13.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:13 smithi008 ceph-mon[184418]: osdmap e452: 8 total, 8 up, 5 in 2023-10-09T05:58:13.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:13 smithi008 ceph-mon[184418]: pgmap v1345: 99 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 2 op/s; 77726/153093 objects degraded (50.770%); 24247/153093 objects misplaced (15.838%); 7.2 MiB/s, 297 objects/s recovering 2023-10-09T05:58:13.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/392251429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:13.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 77716/153093 objects degraded (50.764%), 31 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:13.689 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:13 smithi008 ceph-mon[191227]: failed to encode map e452 with expected crc 2023-10-09T05:58:13.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:13 smithi008 ceph-mon[191227]: osdmap e452: 8 total, 8 up, 5 in 2023-10-09T05:58:13.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:13 smithi008 ceph-mon[191227]: pgmap v1345: 99 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 2 op/s; 77726/153093 objects degraded (50.770%); 24247/153093 objects misplaced (15.838%); 7.2 MiB/s, 297 objects/s recovering 2023-10-09T05:58:13.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/392251429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:13.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 77716/153093 objects degraded (50.764%), 31 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:13.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:13.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:13.792 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:13.819 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:13.864 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:13.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:13.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:13.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:14.000 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:14.054 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:14.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:14.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:14.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:14.190 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:14.248 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:14.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:14.354 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:14.381 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:14.427 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:14.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:14.531 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:14.558 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:14.604 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:14.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:14.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:14.707 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:14.708 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:58:14.737 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:14.786 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:14.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:14.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:14.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:58:14.889 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:14.918 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:14.973 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:14.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:14.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:15.077 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:15.087 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:15.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:15.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:15.115 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:15.180 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3] out_osds: [0, 5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:58:15.180 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:58:15.180 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T05:58:15.180 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:58:15.180 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:58:15.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T05:58:15.188 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:15.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:15.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:15.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:15.294 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:15.325 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:15.375 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:15.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:15.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:15.438 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:58:15.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:15.540 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:15.613 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:15.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:15.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:15.718 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:15.748 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:15.798 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:15.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:15.888 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:58:15.901 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:15.977 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:16.094 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:16.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:16.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:16.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:15 smithi008 ceph-mon[184418]: pgmap v1346: 99 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 77726/153093 objects degraded (50.770%); 24247/153093 objects misplaced (15.838%); 5.3 MiB/s, 219 objects/s recovering 2023-10-09T05:58:16.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:15 smithi008 ceph-mon[191227]: pgmap v1346: 99 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 77726/153093 objects degraded (50.770%); 24247/153093 objects misplaced (15.838%); 5.3 MiB/s, 219 objects/s recovering 2023-10-09T05:58:16.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:15 smithi191 ceph-mon[161669]: pgmap v1346: 99 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 62 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 77726/153093 objects degraded (50.770%); 24247/153093 objects misplaced (15.838%); 5.3 MiB/s, 219 objects/s recovering 2023-10-09T05:58:16.234 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:17.633 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:17.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:17.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:17.639 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:58:17.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:58:17.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:58:17.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:58:17.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:58:17.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:58:17.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:58:17.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:58:17.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:58:17.739 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:17.778 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:17.860 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:17.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:17.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:17.949 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:17 smithi191 ceph-mon[161669]: pgmap v1347: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 73067/153093 objects degraded (47.727%); 25960/153093 objects misplaced (16.957%); 6.3 MiB/s, 239 objects/s recovering 2023-10-09T05:58:17.949 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1236510795' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:17.950 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:17 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:17.959 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:58:17.965 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:17.999 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:18.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:17 smithi008 ceph-mon[184418]: pgmap v1347: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 73067/153093 objects degraded (47.727%); 25960/153093 objects misplaced (16.957%); 6.3 MiB/s, 239 objects/s recovering 2023-10-09T05:58:18.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1236510795' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:18.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:17 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:18.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:17 smithi008 ceph-mon[191227]: pgmap v1347: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 73067/153093 objects degraded (47.727%); 25960/153093 objects misplaced (16.957%); 6.3 MiB/s, 239 objects/s recovering 2023-10-09T05:58:18.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1236510795' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:18.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:17 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:18.253 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:18.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:18.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:18.334 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:58:18.334 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T05:58:18.334 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T05:58:18.335 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T05:58:18.335 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:58:18.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:18.391 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:18.441 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:18.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:18.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:18.545 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:18.577 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:18.628 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:18.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:18.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:18.733 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:18.763 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:19.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:19.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[191227]: failed to encode map e453 with expected crc 2023-10-09T05:58:19.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:58:19.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[191227]: osdmap e453: 8 total, 8 up, 5 in 2023-10-09T05:58:19.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[191227]: failed to encode map e453 with expected crc 2023-10-09T05:58:19.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 73067/153093 objects degraded (47.727%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[184418]: failed to encode map e453 with expected crc 2023-10-09T05:58:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:58:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[184418]: osdmap e453: 8 total, 8 up, 5 in 2023-10-09T05:58:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[184418]: failed to encode map e453 with expected crc 2023-10-09T05:58:19.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 73067/153093 objects degraded (47.727%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:19.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:18 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:18 smithi191 ceph-mon[161669]: failed to encode map e453 with expected crc 2023-10-09T05:58:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:58:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:18 smithi191 ceph-mon[161669]: osdmap e453: 8 total, 8 up, 5 in 2023-10-09T05:58:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:18 smithi191 ceph-mon[161669]: failed to encode map e453 with expected crc 2023-10-09T05:58:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 73067/153093 objects degraded (47.727%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:20.014 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:20.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:20.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:20.119 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:20.148 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:20.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:20 smithi191 ceph-mon[161669]: pgmap v1349: 99 pgs: 1 activating+remapped, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 75742/153066 objects degraded (49.483%); 27443/153066 objects misplaced (17.929%); 3.4 MiB/s, 84 objects/s recovering 2023-10-09T05:58:20.505 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:20 smithi008 ceph-mon[184418]: pgmap v1349: 99 pgs: 1 activating+remapped, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 75742/153066 objects degraded (49.483%); 27443/153066 objects misplaced (17.929%); 3.4 MiB/s, 84 objects/s recovering 2023-10-09T05:58:20.505 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:20 smithi008 ceph-mon[191227]: pgmap v1349: 99 pgs: 1 activating+remapped, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 75742/153066 objects degraded (49.483%); 27443/153066 objects misplaced (17.929%); 3.4 MiB/s, 84 objects/s recovering 2023-10-09T05:58:20.833 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:58:20.865 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:58:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:58:20] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.33.4" 2023-10-09T05:58:20.955 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:20.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:20.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:21.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:58:21.063 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:21.068 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:58:21.095 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:21.176 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:21.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:21.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:21.287 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:21.322 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:21.324 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:21.391 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:21.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:21.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:21.498 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:21.537 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T05:58:21.537 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "" 2023-10-09T05:58:21.537 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T05:58:21.564 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:21.782 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:21.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:21.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:21.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:21.920 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:21.976 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:21.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:21.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:22.081 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:22.111 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:22.158 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:22.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:22.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:22.232 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:22 smithi191 ceph-mon[161669]: pgmap v1350: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 75742/153066 objects degraded (49.483%); 26345/153066 objects misplaced (17.212%); 5.7 MiB/s, 118 objects/s recovering 2023-10-09T05:58:22.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:22.290 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:22.340 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:22.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:22.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:22.447 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:22.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:22 smithi008 ceph-mon[184418]: pgmap v1350: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 75742/153066 objects degraded (49.483%); 26345/153066 objects misplaced (17.212%); 5.7 MiB/s, 118 objects/s recovering 2023-10-09T05:58:22.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:22 smithi008 ceph-mon[191227]: pgmap v1350: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 75742/153066 objects degraded (49.483%); 26345/153066 objects misplaced (17.212%); 5.7 MiB/s, 118 objects/s recovering 2023-10-09T05:58:22.478 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:22.533 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:22.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:22.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:22.637 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:22.669 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:22.689 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:22.727 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:22.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:22.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:22.832 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:22.861 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:22.949 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:58:22.962 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:22.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:22.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:23.070 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:23.101 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:23.179 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:58:23.204 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:58:23.207 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:23.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:23.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:23.231 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:58:23.252 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:58:23.271 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:58:23.289 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:58:23.308 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:58:23.314 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:23.343 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:23.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1814230734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:23.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 75742/153066 objects degraded (49.483%), 36 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:23.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1814230734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:23.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 75742/153066 objects degraded (49.483%), 36 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:23.540 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1814230734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:23.540 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 75742/153066 objects degraded (49.483%), 36 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:23.595 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:23.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:23.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:23.699 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:23.728 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:23.776 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:23.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:23.883 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:23.911 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:24.178 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:24.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:24.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:24.282 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:24.312 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:24.319 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:58:24.320 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:58:24.320 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:58:24.321 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:58:24.321 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:58:24.321 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:58:24.321 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:58:24.321 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:58:24.321 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:58:24.426 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:24 smithi008 ceph-mon[184418]: pgmap v1351: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 5 op/s; 75742/152931 objects degraded (49.527%); 26341/152931 objects misplaced (17.224%); 4.7 MiB/s, 98 objects/s recovering 2023-10-09T05:58:24.427 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:24 smithi008 ceph-mon[191227]: pgmap v1351: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 5 op/s; 75742/152931 objects degraded (49.527%); 26341/152931 objects misplaced (17.224%); 4.7 MiB/s, 98 objects/s recovering 2023-10-09T05:58:24.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:24 smithi191 ceph-mon[161669]: pgmap v1351: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 5 op/s; 75742/152931 objects degraded (49.527%); 26341/152931 objects misplaced (17.224%); 4.7 MiB/s, 98 objects/s recovering 2023-10-09T05:58:24.872 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:24.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:24.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:25.032 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:25.111 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:25.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:25.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:25.227 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:25.260 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:58:25.308 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:25.408 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T05:58:25.408 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:58:25.446 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:25 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4148771119' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:25.446 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:25 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:25.462 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:25.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:25.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:25.577 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:25.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:25 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4148771119' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:25.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:25 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:25.608 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:25 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4148771119' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:25.608 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:25 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:25.611 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:25.665 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:25.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:25.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:25.769 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:25.808 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:26.053 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:26.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:26.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:26.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:26.186 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:26.228 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:26.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:26.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:26.331 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:26.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[184418]: pgmap v1352: 99 pgs: 1 active+degraded+remapped, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 11 op/s; 75690/152733 objects degraded (49.557%); 26290/152733 objects misplaced (17.213%); 5.2 MiB/s, 105 objects/s recovering 2023-10-09T05:58:26.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:26.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[184418]: failed to encode map e454 with expected crc 2023-10-09T05:58:26.368 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:26.368 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[184418]: osdmap e454: 8 total, 8 up, 5 in 2023-10-09T05:58:26.368 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[184418]: failed to encode map e454 with expected crc 2023-10-09T05:58:26.368 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[184418]: failed to encode map e454 with expected crc 2023-10-09T05:58:26.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[191227]: pgmap v1352: 99 pgs: 1 active+degraded+remapped, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 11 op/s; 75690/152733 objects degraded (49.557%); 26290/152733 objects misplaced (17.213%); 5.2 MiB/s, 105 objects/s recovering 2023-10-09T05:58:26.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:26.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[191227]: failed to encode map e454 with expected crc 2023-10-09T05:58:26.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:26.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[191227]: osdmap e454: 8 total, 8 up, 5 in 2023-10-09T05:58:26.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[191227]: failed to encode map e454 with expected crc 2023-10-09T05:58:26.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:26 smithi008 ceph-mon[191227]: failed to encode map e454 with expected crc 2023-10-09T05:58:26.370 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:26.438 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:26.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:26.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:26.544 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:26 smithi191 ceph-mon[161669]: pgmap v1352: 99 pgs: 1 active+degraded+remapped, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 11 op/s; 75690/152733 objects degraded (49.557%); 26290/152733 objects misplaced (17.213%); 5.2 MiB/s, 105 objects/s recovering 2023-10-09T05:58:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:26 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:26 smithi191 ceph-mon[161669]: failed to encode map e454 with expected crc 2023-10-09T05:58:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:26 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:26 smithi191 ceph-mon[161669]: osdmap e454: 8 total, 8 up, 5 in 2023-10-09T05:58:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:26 smithi191 ceph-mon[161669]: failed to encode map e454 with expected crc 2023-10-09T05:58:26.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:26 smithi191 ceph-mon[161669]: failed to encode map e454 with expected crc 2023-10-09T05:58:26.575 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:26.634 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:26.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:26.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:26.741 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:26.774 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:26.809 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:58:26.827 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:26.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:26.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:26.935 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:27.000 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:27.071 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:27.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:27.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:27.177 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:27.221 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:27.274 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:27.282 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:58:27.292 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:27.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:27.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:27.398 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:27.435 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:27.495 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:27.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:27.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:27.601 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:27.639 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:27.702 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:27.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:27.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:27.809 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:27.863 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:27.979 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3] out_osds: [0, 5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:58:27.980 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:58:27.980 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T05:58:27.980 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:58:28.023 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:28.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:28.050 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:58:28.131 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:28.163 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:28.216 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:28.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:28.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:28.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:28.370 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:28.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:28 smithi008 ceph-mon[184418]: pgmap v1354: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 26 op/s; 75601/152376 objects degraded (49.615%); 24936/152376 objects misplaced (16.365%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T05:58:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 75601/152376 objects degraded (49.615%), 37 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:28.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:28 smithi008 ceph-mon[191227]: pgmap v1354: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 26 op/s; 75601/152376 objects degraded (49.615%); 24936/152376 objects misplaced (16.365%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T05:58:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 75601/152376 objects degraded (49.615%), 37 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:28.480 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:28.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:28.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:28.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:28 smithi191 ceph-mon[161669]: pgmap v1354: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 26 op/s; 75601/152376 objects degraded (49.615%); 24936/152376 objects misplaced (16.365%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T05:58:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 75601/152376 objects degraded (49.615%), 37 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:28.586 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:28.616 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:28.791 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:28.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:28.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:28.821 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:28.896 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:28.933 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:28.955 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:28.985 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:28.987 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:28.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:28.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:29.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:29.125 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:29.186 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:29.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:29.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:29.293 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:29.338 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:29.365 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:58:29.365 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:58:30.052 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:29 smithi008 ceph-mon[184418]: pgmap v1355: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 29 op/s; 75601/152175 objects degraded (49.680%); 24930/152175 objects misplaced (16.382%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T05:58:30.052 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:30.052 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:30.053 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:29 smithi008 ceph-mon[191227]: pgmap v1355: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 29 op/s; 75601/152175 objects degraded (49.680%); 24930/152175 objects misplaced (16.382%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T05:58:30.053 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:30.053 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:29 smithi191 ceph-mon[161669]: pgmap v1355: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 29 op/s; 75601/152175 objects degraded (49.680%); 24930/152175 objects misplaced (16.382%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T05:58:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:30.881 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:58:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:58:30] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.33.4" 2023-10-09T05:58:31.317 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:31.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:31.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:31.425 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:31.480 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:31 smithi008 ceph-mon[184418]: pgmap v1356: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 78 op/s; 75129/150708 objects degraded (49.851%); 23309/150708 objects misplaced (15.466%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:58:31.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:31 smithi008 ceph-mon[191227]: pgmap v1356: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 78 op/s; 75129/150708 objects degraded (49.851%); 23309/150708 objects misplaced (15.466%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:58:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:31 smithi191 ceph-mon[161669]: pgmap v1356: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 78 op/s; 75129/150708 objects degraded (49.851%); 23309/150708 objects misplaced (15.466%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:58:32.683 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:58:32.766 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:32.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:32.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:32.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:32.915 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:32.973 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:32.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:32.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:33.078 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:33.134 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:33.787 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:33.819 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:33.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:33.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:33.870 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:58:33.898 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:58:33.925 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:33.926 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:58:33.956 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:58:33.981 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:33.985 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:58:34.024 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:58:34.055 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:58:34.058 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:58:34.063 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:58:34.088 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:33 smithi008 ceph-mon[184418]: pgmap v1357: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 87 op/s; 75129/150294 objects degraded (49.988%); 23284/150294 objects misplaced (15.492%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:58:34.089 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 75129/150294 objects degraded (49.988%), 37 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:34.089 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:33 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2424437423' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:34.089 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:33 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3448378081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:34.089 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:33 smithi008 ceph-mon[191227]: pgmap v1357: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 87 op/s; 75129/150294 objects degraded (49.988%); 23284/150294 objects misplaced (15.492%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:58:34.089 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 75129/150294 objects degraded (49.988%), 37 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:34.089 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:33 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2424437423' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:34.089 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:33 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3448378081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:34.099 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:58:34.103 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:58:34.103 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:58:34.103 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:58:34.103 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:58:34.103 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:58:34.103 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:58:34.103 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:58:34.103 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:58:34.104 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:34.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:34.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:34.209 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:34.251 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:34.257 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:58:34.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:33 smithi191 ceph-mon[161669]: pgmap v1357: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 87 op/s; 75129/150294 objects degraded (49.988%); 23284/150294 objects misplaced (15.492%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:58:34.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 75129/150294 objects degraded (49.988%), 37 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:34.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:33 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2424437423' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:34.303 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:33 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3448378081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:35.188 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:35.190 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[184418]: from='client.36699 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:35.190 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:35.190 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[184418]: failed to encode map e455 with expected crc 2023-10-09T05:58:35.190 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[184418]: failed to encode map e455 with expected crc 2023-10-09T05:58:35.190 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2424437423' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:58:35.190 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[184418]: osdmap e455: 8 total, 8 up, 5 in 2023-10-09T05:58:35.190 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:35.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[191227]: from='client.36699 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:35.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:35.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[191227]: failed to encode map e455 with expected crc 2023-10-09T05:58:35.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[191227]: failed to encode map e455 with expected crc 2023-10-09T05:58:35.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2424437423' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:58:35.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[191227]: osdmap e455: 8 total, 8 up, 5 in 2023-10-09T05:58:35.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:34 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:35.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:35.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:35.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:34 smithi191 ceph-mon[161669]: from='client.36699 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:34 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:34 smithi191 ceph-mon[161669]: failed to encode map e455 with expected crc 2023-10-09T05:58:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:34 smithi191 ceph-mon[161669]: failed to encode map e455 with expected crc 2023-10-09T05:58:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2424437423' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:58:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:34 smithi191 ceph-mon[161669]: osdmap e455: 8 total, 8 up, 5 in 2023-10-09T05:58:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:34 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:35.383 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:35.511 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:35.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:35.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:35.623 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:35.653 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:35.767 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:35.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:35.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:35.872 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:35.902 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:35.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:35 smithi191 ceph-mon[161669]: failed to encode map e455 with expected crc 2023-10-09T05:58:35.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:35 smithi191 ceph-mon[161669]: pgmap v1359: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 109 op/s; 74803/149442 objects degraded (50.055%); 23200/149442 objects misplaced (15.524%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:58:35.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:35 smithi191 ceph-mon[161669]: osdmap e456: 8 total, 8 up, 5 in 2023-10-09T05:58:35.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:35 smithi191 ceph-mon[161669]: failed to encode map e456 with expected crc 2023-10-09T05:58:35.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:35 smithi191 ceph-mon[161669]: failed to encode map e456 with expected crc 2023-10-09T05:58:35.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:35 smithi191 ceph-mon[161669]: failed to encode map e456 with expected crc 2023-10-09T05:58:36.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[184418]: failed to encode map e455 with expected crc 2023-10-09T05:58:36.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[184418]: pgmap v1359: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 109 op/s; 74803/149442 objects degraded (50.055%); 23200/149442 objects misplaced (15.524%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:58:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[184418]: osdmap e456: 8 total, 8 up, 5 in 2023-10-09T05:58:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[184418]: failed to encode map e456 with expected crc 2023-10-09T05:58:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[184418]: failed to encode map e456 with expected crc 2023-10-09T05:58:36.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[184418]: failed to encode map e456 with expected crc 2023-10-09T05:58:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[191227]: failed to encode map e455 with expected crc 2023-10-09T05:58:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[191227]: pgmap v1359: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 23 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 109 op/s; 74803/149442 objects degraded (50.055%); 23200/149442 objects misplaced (15.524%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:58:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[191227]: osdmap e456: 8 total, 8 up, 5 in 2023-10-09T05:58:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[191227]: failed to encode map e456 with expected crc 2023-10-09T05:58:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[191227]: failed to encode map e456 with expected crc 2023-10-09T05:58:36.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:35 smithi008 ceph-mon[191227]: failed to encode map e456 with expected crc 2023-10-09T05:58:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[184418]: failed to encode map e456 with expected crc 2023-10-09T05:58:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[184418]: failed to encode map e456 with expected crc 2023-10-09T05:58:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[184418]: failed to encode map e457 with expected crc 2023-10-09T05:58:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[184418]: osdmap e457: 8 total, 8 up, 5 in 2023-10-09T05:58:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[184418]: failed to encode map e457 with expected crc 2023-10-09T05:58:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[184418]: failed to encode map e457 with expected crc 2023-10-09T05:58:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[191227]: failed to encode map e456 with expected crc 2023-10-09T05:58:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[191227]: failed to encode map e456 with expected crc 2023-10-09T05:58:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[191227]: failed to encode map e457 with expected crc 2023-10-09T05:58:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[191227]: osdmap e457: 8 total, 8 up, 5 in 2023-10-09T05:58:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[191227]: failed to encode map e457 with expected crc 2023-10-09T05:58:37.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:36 smithi008 ceph-mon[191227]: failed to encode map e457 with expected crc 2023-10-09T05:58:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:36 smithi191 ceph-mon[161669]: failed to encode map e456 with expected crc 2023-10-09T05:58:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:36 smithi191 ceph-mon[161669]: failed to encode map e456 with expected crc 2023-10-09T05:58:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:36 smithi191 ceph-mon[161669]: failed to encode map e457 with expected crc 2023-10-09T05:58:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:36 smithi191 ceph-mon[161669]: osdmap e457: 8 total, 8 up, 5 in 2023-10-09T05:58:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:36 smithi191 ceph-mon[161669]: failed to encode map e457 with expected crc 2023-10-09T05:58:37.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:36 smithi191 ceph-mon[161669]: failed to encode map e457 with expected crc 2023-10-09T05:58:37.310 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:37.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:37.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:37.414 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:37.472 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:37.591 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:37.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:37.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:37.678 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1357,"stamp":"2023-10-09T05:58:32.664305+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3286112203,"num_objects":50098,"num_object_clones":0,"num_object_copies":150294,"num_objects_missing_on_primary":13589,"num_objects_missing":13589,"num_objects_degraded":75129,"num_objects_misplaced":23284,"num_objects_unfound":0,"num_objects_dirty":50098,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":119378,"num_write_kb":5402022,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29106,"num_bytes_recovered":1707819557,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":157016,"ondisk_log_size":157016,"up":297,"acting":273,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":319,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":11692940,"kb_used_data":7238116,"kb_used_omap":37,"kb_used_meta":4454746,"kb_avail":457032820,"statfs":{"total":479975178240,"available":468001607680,"internally_reserved":0,"allocated":7411830784,"data_stored":7404640775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38373,"internal_metadata":4561660443},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-57606144,"num_objects":-879,"num_object_clones":0,"num_object_copies":-2637,"num_objects_missing_on_primary":-160,"num_objects_missing":-160,"num_objects_degraded":-613,"num_objects_misplaced":-3057,"num_objects_unfound":0,"num_objects_dirty":-879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":879,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1358,"num_bytes_recovered":88997888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003900"},"pg_stats":[{"pgid":"9.10","version":"454'6073","reported_seq":6309,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:30.225011+0000","last_change":"2023-10-09T05:58:08.524814+0000","last_active":"2023-10-09T05:58:30.225011+0000","last_peered":"2023-10-09T05:58:30.225011+0000","last_clean":"2023-10-09T05:56:48.221888+0000","last_became_active":"2023-10-09T05:58:08.514640+0000","last_became_peered":"2023-10-09T05:58:08.514640+0000","last_unstale":"2023-10-09T05:58:30.225011+0000","last_undegraded":"2023-10-09T05:58:08.398419+0000","last_fullsized":"2023-10-09T05:58:08.325336+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6073,"log_dups_size":0,"ondisk_log_size":6073,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98631680,"num_objects":1505,"num_object_clones":0,"num_object_copies":4515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5124,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3683,"num_write_kb":166016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":2228224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2,6","objects":1507},{"shards":"2,4","objects":-2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":75,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.954865+0000","last_change":"2023-10-09T05:54:59.256087+0000","last_active":"2023-10-09T05:58:17.954865+0000","last_peered":"2023-10-09T05:58:17.954865+0000","last_clean":"2023-10-09T05:58:17.954865+0000","last_became_active":"2023-10-09T05:54:59.255773+0000","last_became_peered":"2023-10-09T05:54:59.255773+0000","last_unstale":"2023-10-09T05:58:17.954865+0000","last_undegraded":"2023-10-09T05:58:17.954865+0000","last_fullsized":"2023-10-09T05:58:17.954865+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:19.545754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":49,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.087071+0000","last_change":"2023-10-09T05:56:00.247579+0000","last_active":"2023-10-09T05:58:18.087071+0000","last_peered":"2023-10-09T05:58:18.087071+0000","last_clean":"2023-10-09T05:58:18.087071+0000","last_became_active":"2023-10-09T05:56:00.238151+0000","last_became_peered":"2023-10-09T05:56:00.238151+0000","last_unstale":"2023-10-09T05:58:18.087071+0000","last_undegraded":"2023-10-09T05:58:18.087071+0000","last_fullsized":"2023-10-09T05:58:18.087071+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:05.692190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"454'6084","reported_seq":11930,"reported_epoch":454,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:32.217457+0000","last_change":"2023-10-09T05:56:01.319470+0000","last_active":"2023-10-09T05:58:32.217457+0000","last_peered":"2023-10-09T05:58:32.217457+0000","last_clean":"2023-10-09T05:55:24.768851+0000","last_became_active":"2023-10-09T05:56:01.311996+0000","last_became_peered":"2023-10-09T05:56:01.311996+0000","last_unstale":"2023-10-09T05:58:32.217457+0000","last_undegraded":"2023-10-09T05:58:32.217457+0000","last_fullsized":"2023-10-09T05:58:32.217457+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6084,"log_dups_size":0,"ondisk_log_size":6084,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104267776,"num_objects":1591,"num_object_clones":0,"num_object_copies":4773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1591,"num_objects_unfound":0,"num_objects_dirty":1591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3671,"num_write_kb":168384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1422,"num_bytes_recovered":93192192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1591}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":156,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.973003+0000","last_change":"2023-10-09T05:56:00.243822+0000","last_active":"2023-10-09T05:58:17.973003+0000","last_peered":"2023-10-09T05:58:17.973003+0000","last_clean":"2023-10-09T05:58:17.973003+0000","last_became_active":"2023-10-09T05:56:00.243483+0000","last_became_peered":"2023-10-09T05:56:00.243483+0000","last_unstale":"2023-10-09T05:58:17.973003+0000","last_undegraded":"2023-10-09T05:58:17.973003+0000","last_fullsized":"2023-10-09T05:58:17.973003+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:39.811952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":524,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961808+0000","last_change":"2023-10-09T05:57:57.277222+0000","last_active":"2023-10-09T05:58:17.961808+0000","last_peered":"2023-10-09T05:58:17.961808+0000","last_clean":"2023-10-09T05:58:17.961808+0000","last_became_active":"2023-10-09T05:57:57.276994+0000","last_became_peered":"2023-10-09T05:57:57.276994+0000","last_unstale":"2023-10-09T05:58:17.961808+0000","last_undegraded":"2023-10-09T05:58:17.961808+0000","last_fullsized":"2023-10-09T05:58:17.961808+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.12","version":"454'6135","reported_seq":11663,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T05:58:29.839294+0000","last_change":"2023-10-09T05:58:12.409601+0000","last_active":"2023-10-09T05:58:29.839294+0000","last_peered":"2023-10-09T05:58:29.839294+0000","last_clean":"2023-10-09T05:58:29.839294+0000","last_became_active":"2023-10-09T05:58:12.403280+0000","last_became_peered":"2023-10-09T05:58:12.403280+0000","last_unstale":"2023-10-09T05:58:29.839294+0000","last_undegraded":"2023-10-09T05:58:29.839294+0000","last_fullsized":"2023-10-09T05:58:29.839294+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6135,"log_dups_size":0,"ondisk_log_size":6135,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104431616,"num_objects":1594,"num_object_clones":0,"num_object_copies":4782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3714,"num_write_kb":169824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2689,"num_bytes_recovered":106627072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":54,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.954505+0000","last_change":"2023-10-09T05:56:49.265409+0000","last_active":"2023-10-09T05:58:17.954505+0000","last_peered":"2023-10-09T05:58:17.954505+0000","last_clean":"2023-10-09T05:58:17.954505+0000","last_became_active":"2023-10-09T05:56:49.265006+0000","last_became_peered":"2023-10-09T05:56:49.265006+0000","last_unstale":"2023-10-09T05:58:17.954505+0000","last_undegraded":"2023-10-09T05:58:17.954505+0000","last_fullsized":"2023-10-09T05:58:17.954505+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:29.506553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":29,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961856+0000","last_change":"2023-10-09T05:58:07.465790+0000","last_active":"2023-10-09T05:58:17.961856+0000","last_peered":"2023-10-09T05:58:17.961856+0000","last_clean":"2023-10-09T05:58:17.961856+0000","last_became_active":"2023-10-09T05:58:07.430785+0000","last_became_peered":"2023-10-09T05:58:07.430785+0000","last_unstale":"2023-10-09T05:58:17.961856+0000","last_undegraded":"2023-10-09T05:58:17.961856+0000","last_fullsized":"2023-10-09T05:58:17.961856+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":18,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.065314+0000","last_change":"2023-10-09T05:58:07.412070+0000","last_active":"2023-10-09T05:58:18.065314+0000","last_peered":"2023-10-09T05:58:18.065314+0000","last_clean":"2023-10-09T05:58:18.065314+0000","last_became_active":"2023-10-09T05:58:07.411692+0000","last_became_peered":"2023-10-09T05:58:07.411692+0000","last_unstale":"2023-10-09T05:58:18.065314+0000","last_undegraded":"2023-10-09T05:58:18.065314+0000","last_fullsized":"2023-10-09T05:58:18.065314+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:36.457810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":59,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:19.130627+0000","last_change":"2023-10-09T05:58:07.387601+0000","last_active":"2023-10-09T05:58:19.130627+0000","last_peered":"2023-10-09T05:58:19.130627+0000","last_clean":"2023-10-09T05:58:19.130627+0000","last_became_active":"2023-10-09T05:58:07.387307+0000","last_became_peered":"2023-10-09T05:58:07.387307+0000","last_unstale":"2023-10-09T05:58:19.130627+0000","last_undegraded":"2023-10-09T05:58:19.130627+0000","last_fullsized":"2023-10-09T05:58:19.130627+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:33:56.748928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":93,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961961+0000","last_change":"2023-10-09T05:57:57.266674+0000","last_active":"2023-10-09T05:58:17.961961+0000","last_peered":"2023-10-09T05:58:17.961961+0000","last_clean":"2023-10-09T05:58:17.961961+0000","last_became_active":"2023-10-09T05:57:57.266209+0000","last_became_peered":"2023-10-09T05:57:57.266209+0000","last_unstale":"2023-10-09T05:58:17.961961+0000","last_undegraded":"2023-10-09T05:58:17.961961+0000","last_fullsized":"2023-10-09T05:58:17.961961+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":87,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:20.261437+0000","last_change":"2023-10-09T05:58:07.427310+0000","last_active":"2023-10-09T05:58:20.261437+0000","last_peered":"2023-10-09T05:58:20.261437+0000","last_clean":"2023-10-09T05:58:20.261437+0000","last_became_active":"2023-10-09T05:58:07.426813+0000","last_became_peered":"2023-10-09T05:58:07.426813+0000","last_unstale":"2023-10-09T05:58:20.261437+0000","last_undegraded":"2023-10-09T05:58:20.261437+0000","last_fullsized":"2023-10-09T05:58:20.261437+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":87,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:23.676202+0000","last_change":"2023-10-09T05:56:00.348771+0000","last_active":"2023-10-09T05:58:23.676202+0000","last_peered":"2023-10-09T05:58:23.676202+0000","last_clean":"2023-10-09T05:58:23.676202+0000","last_became_active":"2023-10-09T05:56:00.224347+0000","last_became_peered":"2023-10-09T05:56:00.224347+0000","last_unstale":"2023-10-09T05:58:23.676202+0000","last_undegraded":"2023-10-09T05:58:23.676202+0000","last_fullsized":"2023-10-09T05:58:23.676202+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":223,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.954938+0000","last_change":"2023-10-09T05:58:07.401823+0000","last_active":"2023-10-09T05:58:17.954938+0000","last_peered":"2023-10-09T05:58:17.954938+0000","last_clean":"2023-10-09T05:58:17.954938+0000","last_became_active":"2023-10-09T05:58:07.401454+0000","last_became_peered":"2023-10-09T05:58:07.401454+0000","last_unstale":"2023-10-09T05:58:17.954938+0000","last_undegraded":"2023-10-09T05:58:17.954938+0000","last_fullsized":"2023-10-09T05:58:17.954938+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:04.542473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":79,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:17.961882+0000","last_change":"2023-10-09T05:58:07.410626+0000","last_active":"2023-10-09T05:58:17.961882+0000","last_peered":"2023-10-09T05:58:17.961882+0000","last_clean":"2023-10-09T05:58:01.307429+0000","last_became_active":"2023-10-09T05:58:07.410375+0000","last_became_peered":"2023-10-09T05:58:07.410375+0000","last_unstale":"2023-10-09T05:58:17.961882+0000","last_undegraded":"2023-10-09T05:58:07.381410+0000","last_fullsized":"2023-10-09T05:58:17.961882+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":["7","4","6"],"object_location_counts":[{"shards":"4,6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":438,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.954900+0000","last_change":"2023-10-09T05:58:07.913883+0000","last_active":"2023-10-09T05:58:17.954900+0000","last_peered":"2023-10-09T05:58:17.954900+0000","last_clean":"2023-10-09T05:58:17.954900+0000","last_became_active":"2023-10-09T05:58:07.913605+0000","last_became_peered":"2023-10-09T05:58:07.913605+0000","last_unstale":"2023-10-09T05:58:17.954900+0000","last_undegraded":"2023-10-09T05:58:17.954900+0000","last_fullsized":"2023-10-09T05:58:17.954900+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:46.780446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.19","version":"454'6062","reported_seq":7223,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:28.801823+0000","last_change":"2023-10-09T05:58:17.644481+0000","last_active":"2023-10-09T05:58:28.801823+0000","last_peered":"2023-10-09T05:58:28.801823+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T05:58:16.616896+0000","last_became_peered":"2023-10-09T05:58:16.616896+0000","last_unstale":"2023-10-09T05:58:28.801823+0000","last_undegraded":"2023-10-09T05:58:11.593270+0000","last_fullsized":"2023-10-09T05:58:09.843164+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6062,"log_dups_size":0,"ondisk_log_size":6062,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104300544,"num_objects":1592,"num_object_clones":0,"num_object_copies":4776,"num_objects_missing_on_primary":1689,"num_objects_missing":1689,"num_objects_degraded":6898,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3656,"num_write_kb":167904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":494,"num_bytes_recovered":32342016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1217},{"shards":"0,5","objects":1},{"shards":"0,5,7","objects":378},{"shards":"4,7","objects":-7}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":68,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961367+0000","last_change":"2023-10-09T05:55:27.092744+0000","last_active":"2023-10-09T05:58:17.961367+0000","last_peered":"2023-10-09T05:58:17.961367+0000","last_clean":"2023-10-09T05:58:17.961367+0000","last_became_active":"2023-10-09T05:55:26.271096+0000","last_became_peered":"2023-10-09T05:55:26.271096+0000","last_unstale":"2023-10-09T05:58:17.961367+0000","last_undegraded":"2023-10-09T05:58:17.961367+0000","last_fullsized":"2023-10-09T05:58:17.961367+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":89,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961769+0000","last_change":"2023-10-09T05:56:49.266384+0000","last_active":"2023-10-09T05:58:17.961769+0000","last_peered":"2023-10-09T05:58:17.961769+0000","last_clean":"2023-10-09T05:58:17.961769+0000","last_became_active":"2023-10-09T05:56:49.266051+0000","last_became_peered":"2023-10-09T05:56:49.266051+0000","last_unstale":"2023-10-09T05:58:17.961769+0000","last_undegraded":"2023-10-09T05:58:17.961769+0000","last_fullsized":"2023-10-09T05:58:17.961769+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"454'6072","reported_seq":7695,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:28.917632+0000","last_change":"2023-10-09T05:56:50.728342+0000","last_active":"2023-10-09T05:58:28.917632+0000","last_peered":"2023-10-09T05:58:28.917632+0000","last_clean":"2023-10-09T05:55:49.156862+0000","last_became_active":"2023-10-09T05:56:50.718720+0000","last_became_peered":"2023-10-09T05:56:50.718720+0000","last_unstale":"2023-10-09T05:58:28.917632+0000","last_undegraded":"2023-10-09T05:56:50.504293+0000","last_fullsized":"2023-10-09T05:56:50.237342+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6072,"log_dups_size":0,"ondisk_log_size":6072,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102563840,"num_objects":1565,"num_object_clones":0,"num_object_copies":4695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4516,"num_objects_misplaced":2331,"num_objects_unfound":0,"num_objects_dirty":1565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3659,"num_write_kb":167168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":456,"num_bytes_recovered":29884416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[5,7],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":1594},{"shards":"5,7","objects":-29}],"blocked_by":[],"up_primary":7,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":498,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961139+0000","last_change":"2023-10-09T05:58:07.410995+0000","last_active":"2023-10-09T05:58:17.961139+0000","last_peered":"2023-10-09T05:58:17.961139+0000","last_clean":"2023-10-09T05:58:17.961139+0000","last_became_active":"2023-10-09T05:58:07.410682+0000","last_became_peered":"2023-10-09T05:58:07.410682+0000","last_unstale":"2023-10-09T05:58:17.961139+0000","last_undegraded":"2023-10-09T05:58:17.961139+0000","last_fullsized":"2023-10-09T05:58:17.961139+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":140,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:23.676760+0000","last_change":"2023-10-09T05:58:07.404172+0000","last_active":"2023-10-09T05:58:23.676760+0000","last_peered":"2023-10-09T05:58:23.676760+0000","last_clean":"2023-10-09T05:58:23.676760+0000","last_became_active":"2023-10-09T05:58:07.403800+0000","last_became_peered":"2023-10-09T05:58:07.403800+0000","last_unstale":"2023-10-09T05:58:23.676760+0000","last_undegraded":"2023-10-09T05:58:23.676760+0000","last_fullsized":"2023-10-09T05:58:23.676760+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":69,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:20.261382+0000","last_change":"2023-10-09T05:58:07.405183+0000","last_active":"2023-10-09T05:58:20.261382+0000","last_peered":"2023-10-09T05:58:20.261382+0000","last_clean":"2023-10-09T05:58:20.261382+0000","last_became_active":"2023-10-09T05:58:07.404785+0000","last_became_peered":"2023-10-09T05:58:07.404785+0000","last_unstale":"2023-10-09T05:58:20.261382+0000","last_undegraded":"2023-10-09T05:58:20.261382+0000","last_fullsized":"2023-10-09T05:58:20.261382+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":477,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961876+0000","last_change":"2023-10-09T05:56:11.341328+0000","last_active":"2023-10-09T05:58:17.961876+0000","last_peered":"2023-10-09T05:58:17.961876+0000","last_clean":"2023-10-09T05:58:17.961876+0000","last_became_active":"2023-10-09T05:56:11.340361+0000","last_became_peered":"2023-10-09T05:56:11.340361+0000","last_unstale":"2023-10-09T05:58:17.961876+0000","last_undegraded":"2023-10-09T05:58:17.961876+0000","last_fullsized":"2023-10-09T05:58:17.961876+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":59,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:17.954483+0000","last_change":"2023-10-09T05:58:07.401970+0000","last_active":"2023-10-09T05:58:17.954483+0000","last_peered":"2023-10-09T05:58:17.954483+0000","last_clean":"2023-10-09T05:58:01.300638+0000","last_became_active":"2023-10-09T05:58:07.401722+0000","last_became_peered":"2023-10-09T05:58:07.401722+0000","last_unstale":"2023-10-09T05:58:17.954483+0000","last_undegraded":"2023-10-09T05:58:07.367598+0000","last_fullsized":"2023-10-09T05:58:17.954483+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:31.363237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":["2","0","3","5","6"],"object_location_counts":[{"shards":"0,2,3,5,6","objects":4}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":88,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.962023+0000","last_change":"2023-10-09T05:56:49.263088+0000","last_active":"2023-10-09T05:58:17.962023+0000","last_peered":"2023-10-09T05:58:17.962023+0000","last_clean":"2023-10-09T05:58:17.962023+0000","last_became_active":"2023-10-09T05:56:49.262781+0000","last_became_peered":"2023-10-09T05:56:49.262781+0000","last_unstale":"2023-10-09T05:58:17.962023+0000","last_undegraded":"2023-10-09T05:58:17.962023+0000","last_fullsized":"2023-10-09T05:58:17.962023+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"454'6161","reported_seq":7508,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:29.078742+0000","last_change":"2023-10-09T05:58:05.458315+0000","last_active":"2023-10-09T05:58:29.078742+0000","last_peered":"2023-10-09T05:58:29.078742+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T05:58:04.631574+0000","last_became_peered":"2023-10-09T05:58:04.631574+0000","last_unstale":"2023-10-09T05:58:29.078742+0000","last_undegraded":"2023-10-09T05:58:03.836605+0000","last_fullsized":"2023-10-09T05:58:01.433430+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101679104,"num_objects":1551,"num_object_clones":0,"num_object_copies":4653,"num_objects_missing_on_primary":1419,"num_objects_missing":1419,"num_objects_degraded":5266,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3719,"num_write_kb":168672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":413,"num_bytes_recovered":27066368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":964},{"shards":"0,1,5,7","objects":211},{"shards":"0,5,7","objects":361},{"shards":"1,7","objects":15}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":86,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:17.960492+0000","last_change":"2023-10-09T05:57:57.271964+0000","last_active":"2023-10-09T05:58:17.960492+0000","last_peered":"2023-10-09T05:58:17.960492+0000","last_clean":"2023-10-09T05:56:02.699477+0000","last_became_active":"2023-10-09T05:57:57.268930+0000","last_became_peered":"2023-10-09T05:57:57.268930+0000","last_unstale":"2023-10-09T05:58:17.960492+0000","last_undegraded":"2023-10-09T05:57:57.243041+0000","last_fullsized":"2023-10-09T05:58:17.960492+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":117,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.960731+0000","last_change":"2023-10-09T05:58:07.397628+0000","last_active":"2023-10-09T05:58:17.960731+0000","last_peered":"2023-10-09T05:58:17.960731+0000","last_clean":"2023-10-09T05:58:17.960731+0000","last_became_active":"2023-10-09T05:58:07.397010+0000","last_became_peered":"2023-10-09T05:58:07.397010+0000","last_unstale":"2023-10-09T05:58:17.960731+0000","last_undegraded":"2023-10-09T05:58:17.960731+0000","last_fullsized":"2023-10-09T05:58:17.960731+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"454'6169","reported_seq":6302,"reported_epoch":454,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:32.011505+0000","last_change":"2023-10-09T05:56:48.413650+0000","last_active":"2023-10-09T05:58:32.011505+0000","last_peered":"2023-10-09T05:58:32.011505+0000","last_clean":"2023-10-09T05:55:48.934818+0000","last_became_active":"2023-10-09T05:56:01.246051+0000","last_became_peered":"2023-10-09T05:56:01.246051+0000","last_unstale":"2023-10-09T05:58:32.011505+0000","last_undegraded":"2023-10-09T05:56:48.413475+0000","last_fullsized":"2023-10-09T05:58:32.011505+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6169,"log_dups_size":0,"ondisk_log_size":6169,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101122048,"num_objects":1543,"num_object_clones":0,"num_object_copies":4629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3739,"num_write_kb":169024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1543}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":66,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961898+0000","last_change":"2023-10-09T05:55:26.358519+0000","last_active":"2023-10-09T05:58:17.961898+0000","last_peered":"2023-10-09T05:58:17.961898+0000","last_clean":"2023-10-09T05:58:17.961898+0000","last_became_active":"2023-10-09T05:55:26.260138+0000","last_became_peered":"2023-10-09T05:55:26.260138+0000","last_unstale":"2023-10-09T05:58:17.961898+0000","last_undegraded":"2023-10-09T05:58:17.961898+0000","last_fullsized":"2023-10-09T05:58:17.961898+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:20.243061+0000","last_change":"2023-10-09T05:58:07.397638+0000","last_active":"2023-10-09T05:58:20.243061+0000","last_peered":"2023-10-09T05:58:20.243061+0000","last_clean":"2023-10-09T05:58:20.243061+0000","last_became_active":"2023-10-09T05:58:07.397242+0000","last_became_peered":"2023-10-09T05:58:07.397242+0000","last_unstale":"2023-10-09T05:58:20.243061+0000","last_undegraded":"2023-10-09T05:58:20.243061+0000","last_fullsized":"2023-10-09T05:58:20.243061+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.7","version":"454'6065","reported_seq":8592,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:28.414956+0000","last_change":"2023-10-09T05:58:02.333623+0000","last_active":"2023-10-09T05:58:28.414956+0000","last_peered":"2023-10-09T05:58:28.414956+0000","last_clean":"2023-10-09T05:53:45.891460+0000","last_became_active":"2023-10-09T05:58:02.118107+0000","last_became_peered":"2023-10-09T05:58:02.118107+0000","last_unstale":"2023-10-09T05:58:28.414956+0000","last_undegraded":"2023-10-09T05:58:01.208593+0000","last_fullsized":"2023-10-09T05:57:58.248025+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6065,"log_dups_size":0,"ondisk_log_size":6065,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103874560,"num_objects":1585,"num_object_clones":0,"num_object_copies":4755,"num_objects_missing_on_primary":937,"num_objects_missing":937,"num_objects_degraded":4353,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3641,"num_write_kb":167232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1170,"num_bytes_recovered":74383360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"2,3,5","objects":475},{"shards":"2,3,5,7","objects":641},{"shards":"2,7","objects":85},{"shards":"3,5,7","objects":384}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":80,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:23.676923+0000","last_change":"2023-10-09T05:58:07.406363+0000","last_active":"2023-10-09T05:58:23.676923+0000","last_peered":"2023-10-09T05:58:23.676923+0000","last_clean":"2023-10-09T05:56:43.219657+0000","last_became_active":"2023-10-09T05:58:07.404369+0000","last_became_peered":"2023-10-09T05:58:07.404369+0000","last_unstale":"2023-10-09T05:58:23.676923+0000","last_undegraded":"2023-10-09T05:58:07.387150+0000","last_fullsized":"2023-10-09T05:58:23.676923+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["7","3","5","6"],"object_location_counts":[{"shards":"3,5,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":71,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961180+0000","last_change":"2023-10-09T05:58:07.400222+0000","last_active":"2023-10-09T05:58:17.961180+0000","last_peered":"2023-10-09T05:58:17.961180+0000","last_clean":"2023-10-09T05:58:17.961180+0000","last_became_active":"2023-10-09T05:58:07.399872+0000","last_became_peered":"2023-10-09T05:58:07.399872+0000","last_unstale":"2023-10-09T05:58:17.961180+0000","last_undegraded":"2023-10-09T05:58:17.961180+0000","last_fullsized":"2023-10-09T05:58:17.961180+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"454'6016","reported_seq":6148,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:29.483939+0000","last_change":"2023-10-09T05:58:07.640003+0000","last_active":"2023-10-09T05:58:29.483939+0000","last_peered":"2023-10-09T05:58:29.483939+0000","last_clean":"2023-10-09T05:57:55.996837+0000","last_became_active":"2023-10-09T05:58:07.625633+0000","last_became_peered":"2023-10-09T05:58:07.625633+0000","last_unstale":"2023-10-09T05:58:29.483939+0000","last_undegraded":"2023-10-09T05:58:07.405708+0000","last_fullsized":"2023-10-09T05:58:07.368268+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6016,"log_dups_size":0,"ondisk_log_size":6016,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102432768,"num_objects":1563,"num_object_clones":0,"num_object_copies":4689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2570,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3617,"num_write_kb":165760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","5"],"object_location_counts":[{"shards":"1,2","objects":-8},{"shards":"1,2,5","objects":1543}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":18,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.954605+0000","last_change":"2023-10-09T05:58:07.413999+0000","last_active":"2023-10-09T05:58:17.954605+0000","last_peered":"2023-10-09T05:58:17.954605+0000","last_clean":"2023-10-09T05:58:17.954605+0000","last_became_active":"2023-10-09T05:58:07.412998+0000","last_became_peered":"2023-10-09T05:58:07.412998+0000","last_unstale":"2023-10-09T05:58:17.954605+0000","last_undegraded":"2023-10-09T05:58:17.954605+0000","last_fullsized":"2023-10-09T05:58:17.954605+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:57.127693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":317,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T05:58:25.256832+0000","last_change":"2023-10-09T05:56:11.340232+0000","last_active":"2023-10-09T05:58:25.256832+0000","last_peered":"2023-10-09T05:58:25.256832+0000","last_clean":"2023-10-09T05:58:25.256832+0000","last_became_active":"2023-10-09T05:56:11.339675+0000","last_became_peered":"2023-10-09T05:56:11.339675+0000","last_unstale":"2023-10-09T05:58:25.256832+0000","last_undegraded":"2023-10-09T05:58:25.256832+0000","last_fullsized":"2023-10-09T05:58:25.256832+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:50.837847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"454'6162","reported_seq":16967,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:29.015070+0000","last_change":"2023-10-09T05:58:20.540189+0000","last_active":"2023-10-09T05:58:29.015070+0000","last_peered":"2023-10-09T05:58:29.015070+0000","last_clean":"2023-10-09T05:58:03.790253+0000","last_became_active":"2023-10-09T05:58:20.243902+0000","last_became_peered":"2023-10-09T05:58:20.243902+0000","last_unstale":"2023-10-09T05:58:29.015070+0000","last_undegraded":"2023-10-09T05:58:11.625534+0000","last_fullsized":"2023-10-09T05:58:11.609771+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6162,"log_dups_size":0,"ondisk_log_size":6162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103186432,"num_objects":1574,"num_object_clones":0,"num_object_copies":4722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2666,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3708,"num_write_kb":169056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4116,"num_bytes_recovered":234323968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":1599},{"shards":"3,7","objects":-55}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":317,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:24.131367+0000","last_change":"2023-10-09T05:56:11.343723+0000","last_active":"2023-10-09T05:58:24.131367+0000","last_peered":"2023-10-09T05:58:24.131367+0000","last_clean":"2023-10-09T05:58:24.131367+0000","last_became_active":"2023-10-09T05:56:11.337217+0000","last_became_peered":"2023-10-09T05:56:11.337217+0000","last_unstale":"2023-10-09T05:58:24.131367+0000","last_undegraded":"2023-10-09T05:58:24.131367+0000","last_fullsized":"2023-10-09T05:58:24.131367+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:55.912242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":446,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.087694+0000","last_change":"2023-10-09T05:58:07.394333+0000","last_active":"2023-10-09T05:58:18.087694+0000","last_peered":"2023-10-09T05:58:18.087694+0000","last_clean":"2023-10-09T05:58:18.087694+0000","last_became_active":"2023-10-09T05:58:07.394002+0000","last_became_peered":"2023-10-09T05:58:07.394002+0000","last_unstale":"2023-10-09T05:58:18.087694+0000","last_undegraded":"2023-10-09T05:58:18.087694+0000","last_fullsized":"2023-10-09T05:58:18.087694+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"454'6027","reported_seq":6248,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:30.343989+0000","last_change":"2023-10-09T05:58:08.524816+0000","last_active":"2023-10-09T05:58:30.343989+0000","last_peered":"2023-10-09T05:58:30.343989+0000","last_clean":"2023-10-09T05:56:47.068306+0000","last_became_active":"2023-10-09T05:58:08.515147+0000","last_became_peered":"2023-10-09T05:58:08.515147+0000","last_unstale":"2023-10-09T05:58:30.343989+0000","last_undegraded":"2023-10-09T05:58:08.323807+0000","last_fullsized":"2023-10-09T05:58:08.237221+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6027,"log_dups_size":0,"ondisk_log_size":6027,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102432768,"num_objects":1563,"num_object_clones":0,"num_object_copies":4689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5140,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3637,"num_write_kb":166400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1835008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2,6","objects":1561},{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":16,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.960735+0000","last_change":"2023-10-09T05:58:07.400025+0000","last_active":"2023-10-09T05:58:17.960735+0000","last_peered":"2023-10-09T05:58:17.960735+0000","last_clean":"2023-10-09T05:58:17.960735+0000","last_became_active":"2023-10-09T05:58:07.399511+0000","last_became_peered":"2023-10-09T05:58:07.399511+0000","last_unstale":"2023-10-09T05:58:17.960735+0000","last_undegraded":"2023-10-09T05:58:17.960735+0000","last_fullsized":"2023-10-09T05:58:17.960735+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":383,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.065264+0000","last_change":"2023-10-09T05:54:05.161959+0000","last_active":"2023-10-09T05:58:18.065264+0000","last_peered":"2023-10-09T05:58:18.065264+0000","last_clean":"2023-10-09T05:58:18.065264+0000","last_became_active":"2023-10-09T05:54:05.161671+0000","last_became_peered":"2023-10-09T05:54:05.161671+0000","last_unstale":"2023-10-09T05:58:18.065264+0000","last_undegraded":"2023-10-09T05:58:18.065264+0000","last_fullsized":"2023-10-09T05:58:18.065264+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:47.282176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"454'6086","reported_seq":12017,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:32.333081+0000","last_change":"2023-10-09T05:56:01.447837+0000","last_active":"2023-10-09T05:58:32.333081+0000","last_peered":"2023-10-09T05:58:32.333081+0000","last_clean":"2023-10-09T05:55:25.103026+0000","last_became_active":"2023-10-09T05:56:01.440057+0000","last_became_peered":"2023-10-09T05:56:01.440057+0000","last_unstale":"2023-10-09T05:58:32.333081+0000","last_undegraded":"2023-10-09T05:56:01.309926+0000","last_fullsized":"2023-10-09T05:56:01.207038+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6086,"log_dups_size":0,"ondisk_log_size":6086,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100859904,"num_objects":1539,"num_object_clones":0,"num_object_copies":4617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3673,"num_write_kb":166784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1468,"num_bytes_recovered":96206848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1,3","objects":-536},{"shards":"1,3,5","objects":1568}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":456,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.065051+0000","last_change":"2023-10-09T05:55:26.256379+0000","last_active":"2023-10-09T05:58:18.065051+0000","last_peered":"2023-10-09T05:58:18.065051+0000","last_clean":"2023-10-09T05:58:18.065051+0000","last_became_active":"2023-10-09T05:55:26.256062+0000","last_became_peered":"2023-10-09T05:55:26.256062+0000","last_unstale":"2023-10-09T05:58:18.065051+0000","last_undegraded":"2023-10-09T05:58:18.065051+0000","last_fullsized":"2023-10-09T05:58:18.065051+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:00.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":16,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961723+0000","last_change":"2023-10-09T05:58:07.397389+0000","last_active":"2023-10-09T05:58:17.961723+0000","last_peered":"2023-10-09T05:58:17.961723+0000","last_clean":"2023-10-09T05:58:17.961723+0000","last_became_active":"2023-10-09T05:58:07.397034+0000","last_became_peered":"2023-10-09T05:58:07.397034+0000","last_unstale":"2023-10-09T05:58:17.961723+0000","last_undegraded":"2023-10-09T05:58:17.961723+0000","last_fullsized":"2023-10-09T05:58:17.961723+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2","version":"454'6090","reported_seq":6318,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:30.321599+0000","last_change":"2023-10-09T05:56:50.476851+0000","last_active":"2023-10-09T05:58:30.321599+0000","last_peered":"2023-10-09T05:58:30.321599+0000","last_clean":"2023-10-09T05:56:48.126316+0000","last_became_active":"2023-10-09T05:56:50.466802+0000","last_became_peered":"2023-10-09T05:56:50.466802+0000","last_unstale":"2023-10-09T05:58:30.321599+0000","last_undegraded":"2023-10-09T05:56:50.272069+0000","last_fullsized":"2023-10-09T05:56:50.242739+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6090,"log_dups_size":0,"ondisk_log_size":6090,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102793216,"num_objects":1568,"num_object_clones":0,"num_object_copies":4704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2616,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3668,"num_write_kb":167584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-296},{"shards":"1,2,3","objects":1566}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":88,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.960642+0000","last_change":"2023-10-09T05:56:49.287906+0000","last_active":"2023-10-09T05:58:17.960642+0000","last_peered":"2023-10-09T05:58:17.960642+0000","last_clean":"2023-10-09T05:58:17.960642+0000","last_became_active":"2023-10-09T05:56:49.275081+0000","last_became_peered":"2023-10-09T05:56:49.275081+0000","last_unstale":"2023-10-09T05:58:17.960642+0000","last_undegraded":"2023-10-09T05:58:17.960642+0000","last_fullsized":"2023-10-09T05:58:17.960642+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":141,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.065054+0000","last_change":"2023-10-09T05:56:00.225724+0000","last_active":"2023-10-09T05:58:18.065054+0000","last_peered":"2023-10-09T05:58:18.065054+0000","last_clean":"2023-10-09T05:58:18.065054+0000","last_became_active":"2023-10-09T05:56:00.225335+0000","last_became_peered":"2023-10-09T05:56:00.225335+0000","last_unstale":"2023-10-09T05:58:18.065054+0000","last_undegraded":"2023-10-09T05:58:18.065054+0000","last_fullsized":"2023-10-09T05:58:18.065054+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:41.405222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"454'7539","reported_seq":7660,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:32.369018+0000","last_change":"2023-10-09T05:57:58.501288+0000","last_active":"2023-10-09T05:58:32.369018+0000","last_peered":"2023-10-09T05:58:32.369018+0000","last_clean":"2023-10-09T05:55:49.154347+0000","last_became_active":"2023-10-09T05:57:58.384024+0000","last_became_peered":"2023-10-09T05:57:58.384024+0000","last_unstale":"2023-10-09T05:58:32.369018+0000","last_undegraded":"2023-10-09T05:57:58.373205+0000","last_fullsized":"2023-10-09T05:57:58.246583+0000","mapping_epoch":444,"log_start":"377'3502","ondisk_log_start":"377'3502","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4037,"log_dups_size":0,"ondisk_log_size":4037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:58.015603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206241792,"num_objects":3147,"num_object_clones":0,"num_object_copies":9441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5436,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7539,"num_write_kb":341952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[1,3],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,3","objects":-94},{"shards":"1,3,5","objects":3147}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":882,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:17.961605+0000","last_change":"2023-10-09T05:56:12.339160+0000","last_active":"2023-10-09T05:58:17.961605+0000","last_peered":"2023-10-09T05:58:17.961605+0000","last_clean":"2023-10-09T05:55:22.199269+0000","last_became_active":"2023-10-09T05:56:12.338771+0000","last_became_peered":"2023-10-09T05:56:12.338771+0000","last_unstale":"2023-10-09T05:58:17.961605+0000","last_undegraded":"2023-10-09T05:56:12.324800+0000","last_fullsized":"2023-10-09T05:58:17.961605+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":437,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.065340+0000","last_change":"2023-10-09T05:58:07.410383+0000","last_active":"2023-10-09T05:58:18.065340+0000","last_peered":"2023-10-09T05:58:18.065340+0000","last_clean":"2023-10-09T05:58:18.065340+0000","last_became_active":"2023-10-09T05:58:07.410041+0000","last_became_peered":"2023-10-09T05:58:07.410041+0000","last_unstale":"2023-10-09T05:58:18.065340+0000","last_undegraded":"2023-10-09T05:58:18.065340+0000","last_fullsized":"2023-10-09T05:58:18.065340+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:30.848859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":36,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.954656+0000","last_change":"2023-10-09T05:56:49.255402+0000","last_active":"2023-10-09T05:58:17.954656+0000","last_peered":"2023-10-09T05:58:17.954656+0000","last_clean":"2023-10-09T05:58:17.954656+0000","last_became_active":"2023-10-09T05:56:49.255103+0000","last_became_peered":"2023-10-09T05:56:49.255103+0000","last_unstale":"2023-10-09T05:58:17.954656+0000","last_undegraded":"2023-10-09T05:58:17.954656+0000","last_fullsized":"2023-10-09T05:58:17.954656+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:32.251375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"454'6073","reported_seq":7234,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:28.851615+0000","last_change":"2023-10-09T05:58:20.270966+0000","last_active":"2023-10-09T05:58:28.851615+0000","last_peered":"2023-10-09T05:58:28.851615+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T05:58:15.670784+0000","last_became_peered":"2023-10-09T05:58:15.670784+0000","last_unstale":"2023-10-09T05:58:28.851615+0000","last_undegraded":"2023-10-09T05:58:09.828556+0000","last_fullsized":"2023-10-09T05:58:08.235951+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6073,"log_dups_size":0,"ondisk_log_size":6073,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101613568,"num_objects":1550,"num_object_clones":0,"num_object_copies":4650,"num_objects_missing_on_primary":1639,"num_objects_missing":1639,"num_objects_degraded":6779,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3666,"num_write_kb":166944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":485,"num_bytes_recovered":31817728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1190},{"shards":"0,5,7","objects":356},{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":26,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:17.961007+0000","last_change":"2023-10-09T05:58:07.426001+0000","last_active":"2023-10-09T05:58:17.961007+0000","last_peered":"2023-10-09T05:58:17.961007+0000","last_clean":"2023-10-09T05:48:52.714098+0000","last_became_active":"2023-10-09T05:58:07.425667+0000","last_became_peered":"2023-10-09T05:58:07.425667+0000","last_unstale":"2023-10-09T05:58:17.961007+0000","last_undegraded":"2023-10-09T05:58:07.383477+0000","last_fullsized":"2023-10-09T05:58:17.961007+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":410,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.086601+0000","last_change":"2023-10-09T05:58:07.393596+0000","last_active":"2023-10-09T05:58:18.086601+0000","last_peered":"2023-10-09T05:58:18.086601+0000","last_clean":"2023-10-09T05:58:18.086601+0000","last_became_active":"2023-10-09T05:58:07.393244+0000","last_became_peered":"2023-10-09T05:58:07.393244+0000","last_unstale":"2023-10-09T05:58:18.086601+0000","last_undegraded":"2023-10-09T05:58:18.086601+0000","last_fullsized":"2023-10-09T05:58:18.086601+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":289,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.067942+0000","last_change":"2023-10-09T05:57:57.269966+0000","last_active":"2023-10-09T05:58:18.067942+0000","last_peered":"2023-10-09T05:58:18.067942+0000","last_clean":"2023-10-09T05:58:18.067942+0000","last_became_active":"2023-10-09T05:57:57.269658+0000","last_became_peered":"2023-10-09T05:57:57.269658+0000","last_unstale":"2023-10-09T05:58:18.067942+0000","last_undegraded":"2023-10-09T05:58:18.067942+0000","last_fullsized":"2023-10-09T05:58:18.067942+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:31.515794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"454'7404","reported_seq":13190,"reported_epoch":454,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:29.109740+0000","last_change":"2023-10-09T05:56:49.277681+0000","last_active":"2023-10-09T05:58:29.109740+0000","last_peered":"2023-10-09T05:58:29.109740+0000","last_clean":"2023-10-09T05:55:25.212985+0000","last_became_active":"2023-10-09T05:56:49.259166+0000","last_became_peered":"2023-10-09T05:56:49.259166+0000","last_unstale":"2023-10-09T05:58:29.109740+0000","last_undegraded":"2023-10-09T05:58:29.109740+0000","last_fullsized":"2023-10-09T05:58:29.109740+0000","mapping_epoch":428,"log_start":"368'792","ondisk_log_start":"368'792","created":364,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6612,"log_dups_size":0,"ondisk_log_size":6612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209846272,"num_objects":3202,"num_object_clones":0,"num_object_copies":9606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3202,"num_objects_unfound":0,"num_objects_dirty":3202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7404,"num_write_kb":339392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2829,"num_bytes_recovered":147193856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[5,1,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":3202}],"blocked_by":[],"up_primary":7,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":801,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:17.954606+0000","last_change":"2023-10-09T05:56:11.337498+0000","last_active":"2023-10-09T05:58:17.954606+0000","last_peered":"2023-10-09T05:58:17.954606+0000","last_clean":"2023-10-09T05:56:02.545242+0000","last_became_active":"2023-10-09T05:56:11.336951+0000","last_became_peered":"2023-10-09T05:56:11.336951+0000","last_unstale":"2023-10-09T05:58:17.954606+0000","last_undegraded":"2023-10-09T05:56:11.323856+0000","last_fullsized":"2023-10-09T05:58:17.954606+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:54.695500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":["2","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":247,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T05:58:25.246585+0000","last_change":"2023-10-09T05:58:07.446709+0000","last_active":"2023-10-09T05:58:25.246585+0000","last_peered":"2023-10-09T05:58:25.246585+0000","last_clean":"2023-10-09T05:58:25.246585+0000","last_became_active":"2023-10-09T05:58:07.446340+0000","last_became_peered":"2023-10-09T05:58:07.446340+0000","last_unstale":"2023-10-09T05:58:25.246585+0000","last_undegraded":"2023-10-09T05:58:25.246585+0000","last_fullsized":"2023-10-09T05:58:25.246585+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:37.041432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":263,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961942+0000","last_change":"2023-10-09T05:57:57.269800+0000","last_active":"2023-10-09T05:58:17.961942+0000","last_peered":"2023-10-09T05:58:17.961942+0000","last_clean":"2023-10-09T05:58:17.961942+0000","last_became_active":"2023-10-09T05:57:57.268007+0000","last_became_peered":"2023-10-09T05:57:57.268007+0000","last_unstale":"2023-10-09T05:58:17.961942+0000","last_undegraded":"2023-10-09T05:58:17.961942+0000","last_fullsized":"2023-10-09T05:58:17.961942+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.8","version":"454'6071","reported_seq":13868,"reported_epoch":454,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:58:29.767872+0000","last_change":"2023-10-09T05:58:11.034970+0000","last_active":"2023-10-09T05:58:29.767872+0000","last_peered":"2023-10-09T05:58:29.767872+0000","last_clean":"2023-10-09T05:55:49.071451+0000","last_became_active":"2023-10-09T05:56:50.673096+0000","last_became_peered":"2023-10-09T05:56:50.673096+0000","last_unstale":"2023-10-09T05:58:29.767872+0000","last_undegraded":"2023-10-09T05:58:29.767872+0000","last_fullsized":"2023-10-09T05:56:50.237380+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6071,"log_dups_size":0,"ondisk_log_size":6071,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103284736,"num_objects":1576,"num_object_clones":0,"num_object_copies":4728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":65,"num_objects_unfound":0,"num_objects_dirty":1576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3658,"num_write_kb":167488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2003,"num_bytes_recovered":131203072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[5,7],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":21},{"shards":"0,3,5,7","objects":1},{"shards":"5,7","objects":1554}],"blocked_by":[],"up_primary":7,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":667,"reported_epoch":453,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:18.065159+0000","last_change":"2023-10-09T05:58:07.894313+0000","last_active":"2023-10-09T05:58:18.065159+0000","last_peered":"2023-10-09T05:58:18.065159+0000","last_clean":"2023-10-09T05:55:22.245899+0000","last_became_active":"2023-10-09T05:58:05.515736+0000","last_became_peered":"2023-10-09T05:58:05.515736+0000","last_unstale":"2023-10-09T05:58:18.065159+0000","last_undegraded":"2023-10-09T05:57:58.245342+0000","last_fullsized":"2023-10-09T05:57:58.244553+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:13.491224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":887,"num_read_kb":802,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":600,"reported_epoch":453,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:17.960856+0000","last_change":"2023-10-09T05:58:08.513222+0000","last_active":"2023-10-09T05:58:17.960856+0000","last_peered":"2023-10-09T05:58:17.960856+0000","last_clean":"2023-10-09T05:56:02.257688+0000","last_became_active":"2023-10-09T05:58:08.512938+0000","last_became_peered":"2023-10-09T05:58:08.512938+0000","last_unstale":"2023-10-09T05:58:17.960856+0000","last_undegraded":"2023-10-09T05:58:08.236858+0000","last_fullsized":"2023-10-09T05:58:08.236036+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1],"avail_no_missing":["7","0","6"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":323,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.086748+0000","last_change":"2023-10-09T05:58:07.387935+0000","last_active":"2023-10-09T05:58:18.086748+0000","last_peered":"2023-10-09T05:58:18.086748+0000","last_clean":"2023-10-09T05:58:18.086748+0000","last_became_active":"2023-10-09T05:58:07.386962+0000","last_became_peered":"2023-10-09T05:58:07.386962+0000","last_unstale":"2023-10-09T05:58:18.086748+0000","last_undegraded":"2023-10-09T05:58:18.086748+0000","last_fullsized":"2023-10-09T05:58:18.086748+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:02.334925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":66,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:23.676332+0000","last_change":"2023-10-09T05:58:07.410680+0000","last_active":"2023-10-09T05:58:23.676332+0000","last_peered":"2023-10-09T05:58:23.676332+0000","last_clean":"2023-10-09T05:58:23.676332+0000","last_became_active":"2023-10-09T05:58:07.409894+0000","last_became_peered":"2023-10-09T05:58:07.409894+0000","last_unstale":"2023-10-09T05:58:23.676332+0000","last_undegraded":"2023-10-09T05:58:23.676332+0000","last_fullsized":"2023-10-09T05:58:23.676332+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"454'7461","reported_seq":7666,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:30.127474+0000","last_change":"2023-10-09T05:58:09.628087+0000","last_active":"2023-10-09T05:58:30.127474+0000","last_peered":"2023-10-09T05:58:30.127474+0000","last_clean":"2023-10-09T05:56:10.073783+0000","last_became_active":"2023-10-09T05:58:09.541128+0000","last_became_peered":"2023-10-09T05:58:09.541128+0000","last_unstale":"2023-10-09T05:58:30.127474+0000","last_undegraded":"2023-10-09T05:58:09.535378+0000","last_fullsized":"2023-10-09T05:58:09.236027+0000","mapping_epoch":449,"log_start":"367'300","ondisk_log_start":"367'300","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7161,"log_dups_size":0,"ondisk_log_size":7161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:19.154205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206503936,"num_objects":3151,"num_object_clones":0,"num_object_copies":9453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3151,"num_objects_misplaced":8412,"num_objects_unfound":0,"num_objects_dirty":3151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7461,"num_write_kb":339584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":2949120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[0,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":3151},{"shards":"0,6","objects":-54}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":963,"reported_epoch":453,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:17.960966+0000","last_change":"2023-10-09T05:56:16.297927+0000","last_active":"2023-10-09T05:58:17.960966+0000","last_peered":"2023-10-09T05:58:17.960966+0000","last_clean":"2023-10-09T05:56:02.257755+0000","last_became_active":"2023-10-09T05:56:16.263458+0000","last_became_peered":"2023-10-09T05:56:16.263458+0000","last_unstale":"2023-10-09T05:58:17.960966+0000","last_undegraded":"2023-10-09T05:56:16.199131+0000","last_fullsized":"2023-10-09T05:56:16.146936+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":43,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:20.243177+0000","last_change":"2023-10-09T05:57:57.261895+0000","last_active":"2023-10-09T05:58:20.243177+0000","last_peered":"2023-10-09T05:58:20.243177+0000","last_clean":"2023-10-09T05:58:20.243177+0000","last_became_active":"2023-10-09T05:57:57.261556+0000","last_became_peered":"2023-10-09T05:57:57.261556+0000","last_unstale":"2023-10-09T05:58:20.243177+0000","last_undegraded":"2023-10-09T05:58:20.243177+0000","last_fullsized":"2023-10-09T05:58:20.243177+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":332,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:20.471883+0000","last_change":"2023-10-09T05:56:11.340751+0000","last_active":"2023-10-09T05:58:20.471883+0000","last_peered":"2023-10-09T05:58:20.471883+0000","last_clean":"2023-10-09T05:58:20.471883+0000","last_became_active":"2023-10-09T05:56:11.340499+0000","last_became_peered":"2023-10-09T05:56:11.340499+0000","last_unstale":"2023-10-09T05:58:20.471883+0000","last_undegraded":"2023-10-09T05:58:20.471883+0000","last_fullsized":"2023-10-09T05:58:20.471883+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:36.120602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"454'7391","reported_seq":10699,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:30.186563+0000","last_change":"2023-10-09T05:56:19.111667+0000","last_active":"2023-10-09T05:58:30.186563+0000","last_peered":"2023-10-09T05:58:30.186563+0000","last_clean":"2023-10-09T05:53:45.896176+0000","last_became_active":"2023-10-09T05:55:38.711627+0000","last_became_peered":"2023-10-09T05:55:38.711627+0000","last_unstale":"2023-10-09T05:58:30.186563+0000","last_undegraded":"2023-10-09T05:56:19.081247+0000","last_fullsized":"2023-10-09T05:55:27.232363+0000","mapping_epoch":402,"log_start":"367'200","ondisk_log_start":"367'200","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7191,"log_dups_size":0,"ondisk_log_size":7191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:26.171175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200081408,"num_objects":3053,"num_object_clones":0,"num_object_copies":9159,"num_objects_missing_on_primary":1269,"num_objects_missing":1269,"num_objects_degraded":1269,"num_objects_misplaced":2018,"num_objects_unfound":0,"num_objects_dirty":3053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7391,"num_write_kb":334208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1586,"num_bytes_recovered":103677952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[0,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2354},{"shards":"0,1,4,7","objects":1},{"shards":"1,4,7","objects":698}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":620,"reported_epoch":453,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:23.677177+0000","last_change":"2023-10-09T05:56:12.339234+0000","last_active":"2023-10-09T05:58:23.677177+0000","last_peered":"2023-10-09T05:58:23.677177+0000","last_clean":"2023-10-09T05:56:02.265149+0000","last_became_active":"2023-10-09T05:56:12.338651+0000","last_became_peered":"2023-10-09T05:56:12.338651+0000","last_unstale":"2023-10-09T05:58:23.677177+0000","last_undegraded":"2023-10-09T05:56:12.325644+0000","last_fullsized":"2023-10-09T05:56:12.324625+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":273,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.068099+0000","last_change":"2023-10-09T05:58:07.414234+0000","last_active":"2023-10-09T05:58:18.068099+0000","last_peered":"2023-10-09T05:58:18.068099+0000","last_clean":"2023-10-09T05:58:18.068099+0000","last_became_active":"2023-10-09T05:58:07.413481+0000","last_became_peered":"2023-10-09T05:58:07.413481+0000","last_unstale":"2023-10-09T05:58:18.068099+0000","last_undegraded":"2023-10-09T05:58:18.068099+0000","last_fullsized":"2023-10-09T05:58:18.068099+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:13.780593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":413,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.960708+0000","last_change":"2023-10-09T05:57:57.268263+0000","last_active":"2023-10-09T05:58:17.960708+0000","last_peered":"2023-10-09T05:58:17.960708+0000","last_clean":"2023-10-09T05:58:17.960708+0000","last_became_active":"2023-10-09T05:57:57.267885+0000","last_became_peered":"2023-10-09T05:57:57.267885+0000","last_unstale":"2023-10-09T05:58:17.960708+0000","last_undegraded":"2023-10-09T05:58:17.960708+0000","last_fullsized":"2023-10-09T05:58:17.960708+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"454'7369","reported_seq":8886,"reported_epoch":454,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:58:29.272356+0000","last_change":"2023-10-09T05:58:24.310761+0000","last_active":"2023-10-09T05:58:29.272356+0000","last_peered":"2023-10-09T05:58:29.272356+0000","last_clean":"2023-10-09T05:53:45.883404+0000","last_became_active":"2023-10-09T05:58:23.671545+0000","last_became_peered":"2023-10-09T05:58:23.671545+0000","last_unstale":"2023-10-09T05:58:29.272356+0000","last_undegraded":"2023-10-09T05:58:17.604954+0000","last_fullsized":"2023-10-09T05:58:29.272356+0000","mapping_epoch":448,"log_start":"376'2978","ondisk_log_start":"376'2978","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4391,"log_dups_size":0,"ondisk_log_size":4391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203161624,"num_objects":3101,"num_object_clones":0,"num_object_copies":9303,"num_objects_missing_on_primary":3238,"num_objects_missing":3238,"num_objects_degraded":5706,"num_objects_misplaced":3238,"num_objects_unfound":0,"num_objects_dirty":3101,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7369,"num_write_kb":334977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":689,"num_bytes_recovered":45154304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,4,5","objects":2308},{"shards":"1,5,7","objects":721},{"shards":"4,5,7","objects":72}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":111,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:17.962342+0000","last_change":"2023-10-09T05:57:57.269997+0000","last_active":"2023-10-09T05:58:17.962342+0000","last_peered":"2023-10-09T05:58:17.962342+0000","last_clean":"2023-10-09T05:57:53.238744+0000","last_became_active":"2023-10-09T05:57:57.268391+0000","last_became_peered":"2023-10-09T05:57:57.268391+0000","last_unstale":"2023-10-09T05:58:17.962342+0000","last_undegraded":"2023-10-09T05:57:57.242535+0000","last_fullsized":"2023-10-09T05:58:17.962342+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["7","0","3","5"],"object_location_counts":[{"shards":"0,3,5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":546,"reported_epoch":453,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:18.066009+0000","last_change":"2023-10-09T05:58:08.247059+0000","last_active":"2023-10-09T05:58:18.066009+0000","last_peered":"2023-10-09T05:58:18.066009+0000","last_clean":"2023-10-09T05:58:01.663036+0000","last_became_active":"2023-10-09T05:58:08.246587+0000","last_became_peered":"2023-10-09T05:58:08.246587+0000","last_unstale":"2023-10-09T05:58:18.066009+0000","last_undegraded":"2023-10-09T05:58:08.236225+0000","last_fullsized":"2023-10-09T05:58:08.235639+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:37.664299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4],"avail_no_missing":["1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":115,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.960697+0000","last_change":"2023-10-09T05:58:07.397986+0000","last_active":"2023-10-09T05:58:17.960697+0000","last_peered":"2023-10-09T05:58:17.960697+0000","last_clean":"2023-10-09T05:58:17.960697+0000","last_became_active":"2023-10-09T05:58:07.397576+0000","last_became_peered":"2023-10-09T05:58:07.397576+0000","last_unstale":"2023-10-09T05:58:17.960697+0000","last_undegraded":"2023-10-09T05:58:17.960697+0000","last_fullsized":"2023-10-09T05:58:17.960697+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"454'7433","reported_seq":12046,"reported_epoch":454,"state":"active+recovering","last_fresh":"2023-10-09T05:58:29.301127+0000","last_change":"2023-10-09T05:56:49.370077+0000","last_active":"2023-10-09T05:58:29.301127+0000","last_peered":"2023-10-09T05:58:29.301127+0000","last_clean":"2023-10-09T05:53:45.907519+0000","last_became_active":"2023-10-09T05:56:03.326563+0000","last_became_peered":"2023-10-09T05:56:03.326563+0000","last_unstale":"2023-10-09T05:58:29.301127+0000","last_undegraded":"2023-10-09T05:58:29.301127+0000","last_fullsized":"2023-10-09T05:58:29.301127+0000","mapping_epoch":416,"log_start":"376'3038","ondisk_log_start":"376'3038","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4395,"log_dups_size":0,"ondisk_log_size":4395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206897152,"num_objects":3157,"num_object_clones":0,"num_object_copies":9471,"num_objects_missing_on_primary":1767,"num_objects_missing":1767,"num_objects_degraded":0,"num_objects_misplaced":818,"num_objects_unfound":0,"num_objects_dirty":3157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7433,"num_write_kb":338880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2234,"num_bytes_recovered":146341888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["4"],"object_location_counts":[{"shards":"0,3,4","objects":818},{"shards":"0,3,4,7","objects":1},{"shards":"3,4,7","objects":2338}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":632,"reported_epoch":453,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:17.962150+0000","last_change":"2023-10-09T05:58:08.253520+0000","last_active":"2023-10-09T05:58:17.962150+0000","last_peered":"2023-10-09T05:58:17.962150+0000","last_clean":"2023-10-09T05:58:02.145081+0000","last_became_active":"2023-10-09T05:58:08.253168+0000","last_became_peered":"2023-10-09T05:58:08.253168+0000","last_unstale":"2023-10-09T05:58:17.962150+0000","last_undegraded":"2023-10-09T05:58:08.242171+0000","last_fullsized":"2023-10-09T05:58:08.239234+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":48,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961794+0000","last_change":"2023-10-09T05:56:48.410914+0000","last_active":"2023-10-09T05:58:17.961794+0000","last_peered":"2023-10-09T05:58:17.961794+0000","last_clean":"2023-10-09T05:58:17.961794+0000","last_became_active":"2023-10-09T05:56:11.349646+0000","last_became_peered":"2023-10-09T05:56:11.349646+0000","last_unstale":"2023-10-09T05:58:17.961794+0000","last_undegraded":"2023-10-09T05:58:17.961794+0000","last_fullsized":"2023-10-09T05:58:17.961794+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":27,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:20.243117+0000","last_change":"2023-10-09T05:58:07.427135+0000","last_active":"2023-10-09T05:58:20.243117+0000","last_peered":"2023-10-09T05:58:20.243117+0000","last_clean":"2023-10-09T05:58:20.243117+0000","last_became_active":"2023-10-09T05:58:07.426364+0000","last_became_peered":"2023-10-09T05:58:07.426364+0000","last_unstale":"2023-10-09T05:58:20.243117+0000","last_undegraded":"2023-10-09T05:58:20.243117+0000","last_fullsized":"2023-10-09T05:58:20.243117+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.11","version":"454'6156","reported_seq":15250,"reported_epoch":454,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:29.272953+0000","last_change":"2023-10-09T05:58:20.369743+0000","last_active":"2023-10-09T05:58:29.272953+0000","last_peered":"2023-10-09T05:58:29.272953+0000","last_clean":"2023-10-09T05:58:02.997585+0000","last_became_active":"2023-10-09T05:58:15.913938+0000","last_became_peered":"2023-10-09T05:58:15.913938+0000","last_unstale":"2023-10-09T05:58:29.272953+0000","last_undegraded":"2023-10-09T05:58:29.272953+0000","last_fullsized":"2023-10-09T05:58:29.272953+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6156,"log_dups_size":0,"ondisk_log_size":6156,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105414656,"num_objects":1609,"num_object_clones":0,"num_object_copies":4827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1609,"num_objects_unfound":0,"num_objects_dirty":1609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3703,"num_write_kb":169952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3255,"num_bytes_recovered":213286912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,6],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":1609}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":444,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.961390+0000","last_change":"2023-10-09T05:56:49.265734+0000","last_active":"2023-10-09T05:58:17.961390+0000","last_peered":"2023-10-09T05:58:17.961390+0000","last_clean":"2023-10-09T05:58:17.961390+0000","last_became_active":"2023-10-09T05:56:49.264595+0000","last_became_peered":"2023-10-09T05:56:49.264595+0000","last_unstale":"2023-10-09T05:58:17.961390+0000","last_undegraded":"2023-10-09T05:58:17.961390+0000","last_fullsized":"2023-10-09T05:58:17.961390+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":145,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.068021+0000","last_change":"2023-10-09T05:56:00.226695+0000","last_active":"2023-10-09T05:58:18.068021+0000","last_peered":"2023-10-09T05:58:18.068021+0000","last_clean":"2023-10-09T05:58:18.068021+0000","last_became_active":"2023-10-09T05:56:00.226359+0000","last_became_peered":"2023-10-09T05:56:00.226359+0000","last_unstale":"2023-10-09T05:58:18.068021+0000","last_undegraded":"2023-10-09T05:58:18.068021+0000","last_fullsized":"2023-10-09T05:58:18.068021+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:20.323704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"454'6086","reported_seq":6216,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:30.192869+0000","last_change":"2023-10-09T05:58:07.639997+0000","last_active":"2023-10-09T05:58:30.192869+0000","last_peered":"2023-10-09T05:58:30.192869+0000","last_clean":"2023-10-09T05:57:55.523800+0000","last_became_active":"2023-10-09T05:58:07.626014+0000","last_became_peered":"2023-10-09T05:58:07.626014+0000","last_unstale":"2023-10-09T05:58:30.192869+0000","last_undegraded":"2023-10-09T05:58:07.410678+0000","last_fullsized":"2023-10-09T05:58:07.367850+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6086,"log_dups_size":0,"ondisk_log_size":6086,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99680256,"num_objects":1521,"num_object_clones":0,"num_object_copies":4563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2624,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3687,"num_write_kb":166656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","5"],"object_location_counts":[{"shards":"1,2","objects":-49},{"shards":"1,2,5","objects":1541}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":14,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:17.961394+0000","last_change":"2023-10-09T05:58:07.399584+0000","last_active":"2023-10-09T05:58:17.961394+0000","last_peered":"2023-10-09T05:58:17.961394+0000","last_clean":"2023-10-09T05:48:52.714337+0000","last_became_active":"2023-10-09T05:58:07.399203+0000","last_became_peered":"2023-10-09T05:58:07.399203+0000","last_unstale":"2023-10-09T05:58:17.961394+0000","last_undegraded":"2023-10-09T05:58:07.381974+0000","last_fullsized":"2023-10-09T05:58:17.961394+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["1","4","5","6"],"object_location_counts":[{"shards":"1,4,5,6","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":35,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.064911+0000","last_change":"2023-10-09T05:56:49.252225+0000","last_active":"2023-10-09T05:58:18.064911+0000","last_peered":"2023-10-09T05:58:18.064911+0000","last_clean":"2023-10-09T05:58:18.064911+0000","last_became_active":"2023-10-09T05:56:49.251880+0000","last_became_peered":"2023-10-09T05:56:49.251880+0000","last_unstale":"2023-10-09T05:58:18.064911+0000","last_undegraded":"2023-10-09T05:58:18.064911+0000","last_fullsized":"2023-10-09T05:58:18.064911+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:34.079170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.17","version":"454'6181","reported_seq":12762,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:28.737778+0000","last_change":"2023-10-09T05:58:02.382245+0000","last_active":"2023-10-09T05:58:28.737778+0000","last_peered":"2023-10-09T05:58:28.737778+0000","last_clean":"2023-10-09T05:57:55.399317+0000","last_became_active":"2023-10-09T05:58:02.319859+0000","last_became_peered":"2023-10-09T05:58:02.319859+0000","last_unstale":"2023-10-09T05:58:28.737778+0000","last_undegraded":"2023-10-09T05:58:01.234664+0000","last_fullsized":"2023-10-09T05:58:01.215806+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6181,"log_dups_size":0,"ondisk_log_size":6181,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98238464,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2713,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3757,"num_write_kb":168192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3182,"num_bytes_recovered":157876224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["7","2","3","5"],"object_location_counts":[{"shards":"2,5,7","objects":1584},{"shards":"2,7","objects":-110}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":11,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:23.676507+0000","last_change":"2023-10-09T05:58:07.410770+0000","last_active":"2023-10-09T05:58:23.676507+0000","last_peered":"2023-10-09T05:58:23.676507+0000","last_clean":"2023-10-09T05:21:17.840829+0000","last_became_active":"2023-10-09T05:58:07.410323+0000","last_became_peered":"2023-10-09T05:58:07.410323+0000","last_unstale":"2023-10-09T05:58:23.676507+0000","last_undegraded":"2023-10-09T05:58:07.390791+0000","last_fullsized":"2023-10-09T05:58:23.676507+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["1","4","5","6"],"object_location_counts":[{"shards":"1,4,5,6","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":45,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.954683+0000","last_change":"2023-10-09T05:56:11.337104+0000","last_active":"2023-10-09T05:58:17.954683+0000","last_peered":"2023-10-09T05:58:17.954683+0000","last_clean":"2023-10-09T05:58:17.954683+0000","last_became_active":"2023-10-09T05:56:11.336690+0000","last_became_peered":"2023-10-09T05:56:11.336690+0000","last_unstale":"2023-10-09T05:58:17.954683+0000","last_undegraded":"2023-10-09T05:58:17.954683+0000","last_fullsized":"2023-10-09T05:58:17.954683+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:56.369633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.14","version":"454'6133","reported_seq":6264,"reported_epoch":454,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:32.209493+0000","last_change":"2023-10-09T05:56:01.281023+0000","last_active":"2023-10-09T05:58:32.209493+0000","last_peered":"2023-10-09T05:58:32.209493+0000","last_clean":"2023-10-09T05:55:48.964110+0000","last_became_active":"2023-10-09T05:56:01.245699+0000","last_became_peered":"2023-10-09T05:56:01.245699+0000","last_unstale":"2023-10-09T05:58:32.209493+0000","last_undegraded":"2023-10-09T05:56:01.216362+0000","last_fullsized":"2023-10-09T05:58:32.209493+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6133,"log_dups_size":0,"ondisk_log_size":6133,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102957056,"num_objects":1571,"num_object_clones":0,"num_object_copies":4713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":40,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3703,"num_write_kb":168768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1571}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":203,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T05:58:25.249080+0000","last_change":"2023-10-09T05:58:07.415396+0000","last_active":"2023-10-09T05:58:25.249080+0000","last_peered":"2023-10-09T05:58:25.249080+0000","last_clean":"2023-10-09T05:58:25.249080+0000","last_became_active":"2023-10-09T05:58:07.414511+0000","last_became_peered":"2023-10-09T05:58:07.414511+0000","last_unstale":"2023-10-09T05:58:25.249080+0000","last_undegraded":"2023-10-09T05:58:25.249080+0000","last_fullsized":"2023-10-09T05:58:25.249080+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:10.158875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":294,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.954784+0000","last_change":"2023-10-09T05:57:57.272049+0000","last_active":"2023-10-09T05:58:17.954784+0000","last_peered":"2023-10-09T05:58:17.954784+0000","last_clean":"2023-10-09T05:58:17.954784+0000","last_became_active":"2023-10-09T05:57:57.271709+0000","last_became_peered":"2023-10-09T05:57:57.271709+0000","last_unstale":"2023-10-09T05:58:17.954784+0000","last_undegraded":"2023-10-09T05:58:17.954784+0000","last_fullsized":"2023-10-09T05:58:17.954784+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:30.502975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"454'6157","reported_seq":7306,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:28.698927+0000","last_change":"2023-10-09T05:58:05.481038+0000","last_active":"2023-10-09T05:58:28.698927+0000","last_peered":"2023-10-09T05:58:28.698927+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T05:58:04.624669+0000","last_became_peered":"2023-10-09T05:58:04.624669+0000","last_unstale":"2023-10-09T05:58:28.698927+0000","last_undegraded":"2023-10-09T05:58:01.427162+0000","last_fullsized":"2023-10-09T05:57:58.378378+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6157,"log_dups_size":0,"ondisk_log_size":6157,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104824832,"num_objects":1600,"num_object_clones":0,"num_object_copies":4800,"num_objects_missing_on_primary":1609,"num_objects_missing":1609,"num_objects_degraded":5485,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3716,"num_write_kb":170080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":313,"num_bytes_recovered":20512768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":1127},{"shards":"0,1,5,7","objects":106},{"shards":"0,5,7","objects":382},{"shards":"1,7","objects":-15}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":383,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:58:18.065523+0000","last_change":"2023-10-09T05:58:07.410502+0000","last_active":"2023-10-09T05:58:18.065523+0000","last_peered":"2023-10-09T05:58:18.065523+0000","last_clean":"2023-10-09T05:57:53.237962+0000","last_became_active":"2023-10-09T05:58:07.410010+0000","last_became_peered":"2023-10-09T05:58:07.410010+0000","last_unstale":"2023-10-09T05:58:18.065523+0000","last_undegraded":"2023-10-09T05:58:07.372990+0000","last_fullsized":"2023-10-09T05:58:18.065523+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:54.252736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":["2","0","5","6"],"object_location_counts":[{"shards":"0,2,5,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":446,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-10-09T05:58:18.086261+0000","last_change":"2023-10-09T05:58:07.393357+0000","last_active":"2023-10-09T05:58:18.086261+0000","last_peered":"2023-10-09T05:58:18.086261+0000","last_clean":"2023-10-09T05:58:18.086261+0000","last_became_active":"2023-10-09T05:58:07.392929+0000","last_became_peered":"2023-10-09T05:58:07.392929+0000","last_unstale":"2023-10-09T05:58:18.086261+0000","last_undegraded":"2023-10-09T05:58:18.086261+0000","last_fullsized":"2023-10-09T05:58:18.086261+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":17,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":15826,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17226,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":3281322008,"num_objects":50070,"num_object_clones":0,"num_object_copies":150210,"num_objects_missing_on_primary":13567,"num_objects_missing":13567,"num_objects_degraded":75020,"num_objects_misplaced":23284,"num_objects_unfound":0,"num_objects_dirty":50070,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":118272,"num_write_kb":5386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28957,"num_bytes_recovered":1698234368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11440893952,"data_stored":11440881736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155846,"ondisk_log_size":155846,"up":78,"acting":60,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":20,"num_objects_missing":20,"num_objects_degraded":88,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13211,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":98,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846874,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2344284,"kb_used_data":1434452,"kb_used_omap":14,"kb_used_meta":909809,"kb_avail":91400868,"statfs":{"total":95995035648,"available":93594488832,"internally_reserved":0,"allocated":1468878848,"data_stored":1467945640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":931645274},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":5.1239999999999997,"15min":4.1449999999999996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.20699999999999999},"max":{"1min":1.419,"5min":212.64400000000001,"15min":212.64400000000001},"last":1.0229999999999999},{"interface":"front","average":{"1min":0.89200000000000002,"5min":5.3310000000000004,"15min":4.1699999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.245},"max":{"1min":5.0549999999999997,"5min":212.655,"15min":212.655},"last":1.0289999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:58:27 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":2.5680000000000001,"15min":2.2799999999999998},"min":{"1min":0.27300000000000002,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":5.0899999999999999,"5min":78.013000000000005,"15min":114.422},"last":5.0899999999999999},{"interface":"front","average":{"1min":0.70899999999999996,"5min":2.5649999999999999,"15min":2.379},"min":{"1min":0.34399999999999997,"5min":0.26700000000000002,"15min":0.255},"max":{"1min":3.1339999999999999,"5min":78.174999999999997,"15min":111.708},"last":3.1339999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:58:27 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":3.528,"15min":2.4969999999999999},"min":{"1min":0.314,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":2.254,"5min":205.863,"15min":205.863},"last":2.254},{"interface":"front","average":{"1min":0.64500000000000002,"5min":3.29,"15min":2.3719999999999999},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":1.875,"5min":205.88399999999999,"15min":205.88399999999999},"last":0.32800000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:58:27 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":2.7170000000000001,"15min":3.1989999999999998},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.28399999999999997},"max":{"1min":1.9019999999999999,"5min":115.69799999999999,"15min":115.69799999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.91300000000000003,"5min":2.8679999999999999,"15min":3.3769999999999998},"min":{"1min":0.35899999999999999,"5min":0.29699999999999999,"15min":0.29299999999999998},"max":{"1min":5.0890000000000004,"5min":115.75700000000001,"15min":115.75700000000001},"last":0.628}]},{"osd":4,"last update":"Mon Oct 9 05:58:27 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.5740000000000001,"15min":1.9279999999999999},"min":{"1min":0.28899999999999998,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":2.298,"5min":78.963999999999999,"15min":159.50299999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.01,"5min":1.6559999999999999,"15min":1.958},"min":{"1min":0.28499999999999998,"5min":0.28199999999999997,"15min":0.27800000000000002},"max":{"1min":4.9800000000000004,"5min":79.099999999999994,"15min":159.483},"last":1.367}]},{"osd":5,"last update":"Mon Oct 9 05:58:27 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":2.7250000000000001,"15min":1.7350000000000001},"min":{"1min":0.32600000000000001,"5min":0.29399999999999998,"15min":0.27800000000000002},"max":{"1min":5.0250000000000004,"5min":94.093999999999994,"15min":94.093999999999994},"last":1.4159999999999999},{"interface":"front","average":{"1min":0.80800000000000005,"5min":2.77,"15min":1.8180000000000001},"min":{"1min":0.35999999999999999,"5min":0.312,"15min":0.308},"max":{"1min":4.8360000000000003,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.55400000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:58:27 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":3.0979999999999999,"15min":1.5840000000000001},"min":{"1min":0.34200000000000003,"5min":0.32900000000000001,"15min":0.29299999999999998},"max":{"1min":5.117,"5min":209.322,"15min":209.322},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.76300000000000001,"5min":2.9780000000000002,"15min":1.649},"min":{"1min":0.308,"5min":0.30599999999999999,"15min":0.26700000000000002},"max":{"1min":3.2010000000000001,"5min":209.19800000000001,"15min":209.19800000000001},"last":0.60299999999999998}]}]},{"osd":6,"up_from":0,"seq":180388626905,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":93,"seq":399431958866,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3022640,"kb_used_data":2130408,"kb_used_omap":7,"kb_used_meta":892216,"kb_avail":90722512,"statfs":{"total":95995035648,"available":92899852288,"internally_reserved":0,"allocated":2181537792,"data_stored":2179882593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":913629710},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":2.4929999999999999,"15min":2.9750000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.17499999999999999},"max":{"1min":3.6789999999999998,"5min":131.91300000000001,"15min":131.91300000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.80200000000000005,"5min":2.9489999999999998,"15min":3.266},"min":{"1min":0.249,"5min":0.248,"15min":0.219},"max":{"1min":3.6930000000000001,"5min":132.191,"15min":132.191},"last":0.29299999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":2.7709999999999999,"15min":2.7869999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.19800000000000001},"max":{"1min":2.3140000000000001,"5min":132.35400000000001,"15min":132.35400000000001},"last":0.63},{"interface":"front","average":{"1min":0.44500000000000001,"5min":2.653,"15min":2.6219999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.20699999999999999},"max":{"1min":1.224,"5min":131.70599999999999,"15min":131.70599999999999},"last":0.314}]},{"osd":3,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":2.8540000000000001,"15min":2.6099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.20200000000000001},"max":{"1min":2.4870000000000001,"5min":135.10900000000001,"15min":135.10900000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.77300000000000002,"5min":2.8479999999999999,"15min":2.6269999999999998},"min":{"1min":0.29599999999999999,"5min":0.248,"15min":0.248},"max":{"1min":4.4180000000000001,"5min":131.886,"15min":131.886},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":2.6669999999999998,"15min":1.8620000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29799999999999999},"max":{"1min":3.4580000000000002,"5min":131.636,"15min":131.636},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.67500000000000004,"5min":2.2879999999999998,"15min":1.653},"min":{"1min":0.372,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.347,"5min":131.75999999999999,"15min":131.75999999999999},"last":0.58199999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":3.653,"15min":2.3290000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":1.3580000000000001,"5min":132.291,"15min":132.291},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.032,"5min":4.25,"15min":2.4390000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.32200000000000001},"max":{"1min":4.2839999999999998,"5min":131.738,"15min":131.738},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":3.9340000000000002,"15min":2.754},"min":{"1min":0.36599999999999999,"5min":0.34599999999999997,"15min":0.316},"max":{"1min":1.446,"5min":158.50399999999999,"15min":158.50399999999999},"last":2.2320000000000002},{"interface":"front","average":{"1min":0.97799999999999998,"5min":4.2409999999999997,"15min":2.6629999999999998},"min":{"1min":0.437,"5min":0.377,"15min":0.37},"max":{"1min":3.4910000000000001,"5min":158.577,"15min":158.577},"last":0.52700000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":2.3300000000000001,"15min":1.7270000000000001},"min":{"1min":0.40000000000000002,"5min":0.31,"15min":0.31},"max":{"1min":3.2869999999999999,"5min":131.821,"15min":131.821},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.2410000000000001,"5min":2.6040000000000001,"15min":1.9059999999999999},"min":{"1min":0.40000000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":11.276,"5min":131.852,"15min":131.852},"last":0.54800000000000004}]}]},{"osd":0,"up_from":0,"seq":377957122390,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":98,"seq":420906795339,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1718716,"kb_used_data":874932,"kb_used_omap":11,"kb_used_meta":843764,"kb_avail":92026436,"statfs":{"total":95995035648,"available":94235070464,"internally_reserved":0,"allocated":895930368,"data_stored":893969110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":864014933},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":2.2850000000000001,"15min":3.1749999999999998},"min":{"1min":0.215,"5min":0.20799999999999999,"15min":0.17999999999999999},"max":{"1min":3.3639999999999999,"5min":140.71100000000001,"15min":162.285},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.50900000000000001,"5min":3.1320000000000001,"15min":3.7349999999999999},"min":{"1min":0.252,"5min":0.219,"15min":0.217},"max":{"1min":1.956,"5min":142.542,"15min":162.643},"last":0.55100000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":3.9740000000000002,"5min":2.274,"15min":2.7109999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.193},"max":{"1min":75.260000000000005,"5min":75.260000000000005,"15min":125.77500000000001},"last":1.143},{"interface":"front","average":{"1min":4.1390000000000002,"5min":2.9889999999999999,"15min":2.6259999999999999},"min":{"1min":0.25700000000000001,"5min":0.23599999999999999,"15min":0.20999999999999999},"max":{"1min":75.114999999999995,"5min":75.114999999999995,"15min":125.857},"last":1.157}]},{"osd":3,"last update":"Mon Oct 9 05:57:31 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":2.1840000000000002,"15min":3.6659999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.21099999999999999},"max":{"1min":4.6989999999999998,"5min":29.606999999999999,"15min":159.35499999999999},"last":1.587},{"interface":"front","average":{"1min":1.5349999999999999,"5min":2.0019999999999998,"15min":3.153},"min":{"1min":0.28899999999999998,"5min":0.28499999999999998,"15min":0.217},"max":{"1min":8.907,"5min":29.920999999999999,"15min":159.196},"last":0.51800000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":1.1930000000000001,"15min":1.4299999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28699999999999998},"max":{"1min":1.1679999999999999,"5min":9.8599999999999994,"15min":68.052000000000007},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.82299999999999995,"15min":1.335},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.29199999999999998},"max":{"1min":3.4369999999999998,"5min":10.013,"15min":68.727000000000004},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":5.234,"15min":2.948},"min":{"1min":0.36899999999999999,"5min":0.36399999999999999,"15min":0.29099999999999998},"max":{"1min":2.1190000000000002,"5min":207.80500000000001,"15min":207.80500000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.98599999999999999,"5min":5.3630000000000004,"15min":2.891},"min":{"1min":0.39800000000000002,"5min":0.378,"15min":0.30499999999999999},"max":{"1min":6.9589999999999996,"5min":207.54900000000001,"15min":207.54900000000001},"last":0.58399999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":0.98099999999999998,"15min":1.248},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":9.9320000000000004,"5min":9.9589999999999996,"15min":25.818999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.369,"15min":1.647},"min":{"1min":0.41699999999999998,"5min":0.41399999999999998,"15min":0.34100000000000003},"max":{"1min":3.0619999999999998,"5min":8.7100000000000009,"15min":24.481000000000002},"last":0.60299999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:57:31 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":2.4649999999999999,"15min":1.9419999999999999},"min":{"1min":0.42399999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":3.4049999999999998,"5min":162.09200000000001,"15min":162.09200000000001},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.2969999999999999,"5min":2.9460000000000002,"15min":2.327},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.373},"max":{"1min":8.7959999999999994,"5min":162.35400000000001,"15min":162.35400000000001},"last":0.501}]}]},{"osd":3,"up_from":103,"seq":442381631813,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2912300,"kb_used_data":2004772,"kb_used_omap":4,"kb_used_meta":907515,"kb_avail":90832852,"statfs":{"total":95995035648,"available":93012840448,"internally_reserved":0,"allocated":2052886528,"data_stored":2051175886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":929295966},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":2.5419999999999998,"15min":4.758},"min":{"1min":0.218,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":0.66200000000000003,"5min":142.37700000000001,"15min":312.56599999999997},"last":0.318},{"interface":"front","average":{"1min":0.5,"5min":2.859,"15min":4.8579999999999997},"min":{"1min":0.25,"5min":0.216,"15min":0.20499999999999999},"max":{"1min":2.6459999999999999,"5min":142.75800000000001,"15min":311.02300000000002},"last":0.29199999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":4.6710000000000003,"15min":4.1210000000000004},"min":{"1min":0.27400000000000002,"5min":0.24199999999999999,"15min":0.214},"max":{"1min":0.877,"5min":142.32599999999999,"15min":150.29599999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.432,"5min":4.7329999999999997,"15min":4.3879999999999999},"min":{"1min":0.30399999999999999,"5min":0.23799999999999999,"15min":0.19},"max":{"1min":0.70999999999999996,"5min":139.33199999999999,"15min":151.25700000000001},"last":0.46500000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":1.163,"15min":2.3370000000000002},"min":{"1min":0.31,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.81000000000000005,"5min":9.5039999999999996,"15min":150.006},"last":0.503},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.0589999999999999,"15min":2.2519999999999998},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.20499999999999999},"max":{"1min":2.6269999999999998,"5min":12.807,"15min":150.65100000000001},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":1.004,"15min":2.355},"min":{"1min":0.36199999999999999,"5min":0.35499999999999998,"15min":0.28999999999999998},"max":{"1min":2.5800000000000001,"5min":16.713000000000001,"15min":149.84700000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.86599999999999999,"15min":2.6739999999999999},"min":{"1min":0.38600000000000001,"5min":0.32500000000000001,"15min":0.28999999999999998},"max":{"1min":0.82699999999999996,"5min":16.349,"15min":149.74700000000001},"last":1.2490000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.4139999999999999,"15min":2.282},"min":{"1min":0.32700000000000001,"5min":0.309,"15min":0.28199999999999997},"max":{"1min":0.67500000000000004,"5min":35.648000000000003,"15min":149.43000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.137,"15min":2.0840000000000001},"min":{"1min":0.34399999999999997,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":0.84399999999999997,"5min":35.686999999999998,"15min":149.69300000000001},"last":0.54300000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.77800000000000002,"15min":1.901},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.34499999999999997},"max":{"1min":1.2609999999999999,"5min":3.556,"15min":149.72399999999999},"last":1.0740000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.95999999999999996,"15min":2.1070000000000002},"min":{"1min":0.433,"5min":0.34599999999999997,"15min":0.307},"max":{"1min":0.74099999999999999,"5min":5.7480000000000002,"15min":149.589},"last":1.151}]},{"osd":7,"last update":"Mon Oct 9 05:57:34 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":1.147,"15min":1.958},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.33700000000000002},"max":{"1min":2.5419999999999998,"5min":10.775,"15min":149.77199999999999},"last":1.3320000000000001},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.127,"15min":1.984},"min":{"1min":0.46200000000000002,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":2.883,"5min":10.913,"15min":149.80799999999999},"last":0.53000000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019361,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1695000,"kb_used_data":793552,"kb_used_omap":0,"kb_used_meta":901440,"kb_avail":92050152,"statfs":{"total":95995035648,"available":94259355648,"internally_reserved":0,"allocated":812597248,"data_stored":811667546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923074560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":3.2490000000000001,"15min":3.4279999999999999},"min":{"1min":0.26200000000000001,"5min":0.23000000000000001,"15min":0.219},"max":{"1min":2.1040000000000001,"5min":95.466999999999999,"15min":132.35599999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.92700000000000005,"5min":3.597,"15min":3.7909999999999999},"min":{"1min":0.29599999999999999,"5min":0.24299999999999999,"15min":0.23699999999999999},"max":{"1min":6.3890000000000002,"5min":95.367999999999995,"15min":132.501},"last":0.69599999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":3.677,"15min":2.702},"min":{"1min":0.29299999999999998,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":1.548,"5min":96.033000000000001,"15min":96.033000000000001},"last":0.999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":3.3069999999999999,"15min":2.577},"min":{"1min":0.29099999999999998,"5min":0.255,"15min":0.255},"max":{"1min":1.405,"5min":96.022000000000006,"15min":96.022000000000006},"last":0.67400000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:57:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.45,"15min":2.0219999999999998},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":1.589,"5min":49.405000000000001,"15min":71.748000000000005},"last":1.03},{"interface":"front","average":{"1min":0.80900000000000005,"5min":2.2970000000000002,"15min":2.2559999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.2959999999999998,"5min":84.123999999999995,"15min":84.123999999999995},"last":0.56499999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:58:15 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":4.2990000000000004,"15min":2.9740000000000002},"min":{"1min":0.377,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":11.055999999999999,"5min":100.009,"15min":123.59},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":4.2089999999999996,"15min":2.8889999999999998},"min":{"1min":0.32900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.1930000000000001,"5min":100.254,"15min":123.566},"last":0.53400000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:58:15 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":1.04,"15min":0.997},"min":{"1min":0.34499999999999997,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":15.597,"5min":15.597,"15min":15.597},"last":0.62},{"interface":"front","average":{"1min":1.6140000000000001,"5min":0.91700000000000004,"15min":0.90700000000000003},"min":{"1min":0.39500000000000002,"5min":0.33700000000000002,"15min":0.29799999999999999},"max":{"1min":17.882000000000001,"5min":17.882000000000001,"15min":17.882000000000001},"last":0.495}]},{"osd":6,"last update":"Mon Oct 9 05:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":2.9329999999999998,"15min":1.4870000000000001},"min":{"1min":0.314,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":2.1379999999999999,"5min":181.411,"15min":181.411},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.53600000000000003,"5min":2.9399999999999999,"15min":1.5800000000000001},"min":{"1min":0.36099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.5189999999999999,"5min":180.63999999999999,"15min":180.63999999999999},"last":0.97699999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.3560000000000001,"15min":1.2629999999999999},"min":{"1min":0.33100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.1659999999999999,"5min":36.729999999999997,"15min":96.131},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.153,"15min":1.2869999999999999},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.27200000000000002},"max":{"1min":1.8779999999999999,"5min":36.424999999999997,"15min":96.465000000000003},"last":0.95799999999999996}]}]},{"osd":5,"up_from":0,"seq":154618823133,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1324875776,"data_stored":1324875776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2175668224,"data_stored":2175664152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":888537088,"data_stored":888537088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2043084800,"data_stored":2043084800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":807997440,"data_stored":807993368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2114457600,"data_stored":2114453528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":622526464,"data_stored":622526464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1463746560,"data_stored":1463746560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:58:37.684 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:58:37.685 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 61 2023-10-09T05:58:37.700 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:37.738 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:37.810 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:37.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:37.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:37.932 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:37.944 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:37 smithi008 ceph-mon[191227]: failed to encode map e457 with expected crc 2023-10-09T05:58:37.945 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:37 smithi008 ceph-mon[191227]: pgmap v1362: 99 pgs: 13 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 128 op/s; 72251/148392 objects degraded (48.689%); 24551/148392 objects misplaced (16.545%); 4.3 MiB/s, 186 objects/s recovering 2023-10-09T05:58:37.973 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:38.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:37 smithi191 ceph-mon[161669]: failed to encode map e457 with expected crc 2023-10-09T05:58:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:37 smithi191 ceph-mon[161669]: pgmap v1362: 99 pgs: 13 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 128 op/s; 72251/148392 objects degraded (48.689%); 24551/148392 objects misplaced (16.545%); 4.3 MiB/s, 186 objects/s recovering 2023-10-09T05:58:38.143 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:38.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:38.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:38.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:37 smithi008 ceph-mon[184418]: failed to encode map e457 with expected crc 2023-10-09T05:58:38.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:37 smithi008 ceph-mon[184418]: pgmap v1362: 99 pgs: 13 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 128 op/s; 72251/148392 objects degraded (48.689%); 24551/148392 objects misplaced (16.545%); 4.3 MiB/s, 186 objects/s recovering 2023-10-09T05:58:38.264 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:38.346 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:38.352 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:58:38.352 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:58:38.352 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:58:38.352 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:58:38.352 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:58:38.410 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:38.446 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:58:38.508 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:58:38.581 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:38.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:38.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:38.700 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:38.731 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:38.819 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:38.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:38.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:38.924 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:38.962 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:39.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 72251/148392 objects degraded (48.689%), 38 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:39.072 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:39.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:39.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:39.178 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:39.211 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 72251/148392 objects degraded (48.689%), 38 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:39.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 72251/148392 objects degraded (48.689%), 38 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:39.215 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:39.791 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:39.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:39.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:39.896 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:39.937 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:39.969 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:58:40.299 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:40 smithi008 ceph-mon[191227]: pgmap v1363: 99 pgs: 13 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 130 op/s; 72251/147945 objects degraded (48.836%); 24530/147945 objects misplaced (16.580%); 4.3 MiB/s, 186 objects/s recovering 2023-10-09T05:58:40.300 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:40.300 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:40.305 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:40 smithi008 ceph-mon[184418]: pgmap v1363: 99 pgs: 13 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 130 op/s; 72251/147945 objects degraded (48.836%); 24530/147945 objects misplaced (16.580%); 4.3 MiB/s, 186 objects/s recovering 2023-10-09T05:58:40.305 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:40.305 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:40.320 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:40.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:40.430 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:40.464 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:40 smithi191 ceph-mon[161669]: pgmap v1363: 99 pgs: 13 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 130 op/s; 72251/147945 objects degraded (48.836%); 24530/147945 objects misplaced (16.580%); 4.3 MiB/s, 186 objects/s recovering 2023-10-09T05:58:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:40.612 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:58:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:58:40] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.33.4" 2023-10-09T05:58:40.706 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:40.747 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:40.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:40.859 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:40.925 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:40.994 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:40.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:40.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:41.099 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:41.138 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:41.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:41 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:41.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:41 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:41.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:41 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:41.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:41 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:41.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:41 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:41.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:41 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:42.031 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:42.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:42.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:42.137 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:42.199 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:42.282 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:42.282 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:42.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:42.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:42.392 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:42.433 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:42.488 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:42.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:42.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:42.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:42.633 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:42 smithi191 ceph-mon[161669]: pgmap v1364: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 227 op/s; 71644/145905 objects degraded (49.103%); 23887/145905 objects misplaced (16.372%); 12 MiB/s, 308 objects/s recovering 2023-10-09T05:58:42.640 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:42.650 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:42 smithi008 ceph-mon[184418]: pgmap v1364: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 227 op/s; 71644/145905 objects degraded (49.103%); 23887/145905 objects misplaced (16.372%); 12 MiB/s, 308 objects/s recovering 2023-10-09T05:58:42.650 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:42 smithi008 ceph-mon[191227]: pgmap v1364: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 227 op/s; 71644/145905 objects degraded (49.103%); 23887/145905 objects misplaced (16.372%); 12 MiB/s, 308 objects/s recovering 2023-10-09T05:58:42.700 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:42.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:42.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:42.805 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:42.886 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:43.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 71644/145905 objects degraded (49.103%), 37 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:43.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 71644/145905 objects degraded (49.103%), 37 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:43.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 71644/145905 objects degraded (49.103%), 37 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:44.313 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:58:44.322 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:44.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:44.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:44.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:44.453 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:44.669 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:44.678 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:44 smithi191 ceph-mon[161669]: pgmap v1365: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 165 op/s; 71644/145479 objects degraded (49.247%); 23867/145479 objects misplaced (16.406%); 8.1 MiB/s, 220 objects/s recovering 2023-10-09T05:58:44.678 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:44.678 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2144517096' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T05:58:44.678 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:44 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T05:58:44.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:44.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:44 smithi008 ceph-mon[184418]: pgmap v1365: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 165 op/s; 71644/145479 objects degraded (49.247%); 23867/145479 objects misplaced (16.406%); 8.1 MiB/s, 220 objects/s recovering 2023-10-09T05:58:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2144517096' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T05:58:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:44 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T05:58:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:44 smithi008 ceph-mon[191227]: pgmap v1365: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 165 op/s; 71644/145479 objects degraded (49.247%); 23867/145479 objects misplaced (16.406%); 8.1 MiB/s, 220 objects/s recovering 2023-10-09T05:58:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2144517096' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T05:58:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:44 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T05:58:44.744 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:44.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:44.823 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:44.931 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:44.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:44.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:45.036 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:45.096 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:45.270 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:45.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:45.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:45.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:45.418 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:45.485 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:45.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:45.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:45.589 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:45.628 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:45.709 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:45.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:45.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:45.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:45.869 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:46.045 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:46.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:46.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:46.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:46.180 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:46.222 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:46.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:46.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:46.325 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:46.362 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:46.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:46 smithi008 ceph-mon[184418]: pgmap v1366: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 163 op/s; 71302/144519 objects degraded (49.337%); 23788/144519 objects misplaced (16.460%); 8.1 MiB/s, 131 objects/s recovering 2023-10-09T05:58:46.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:46 smithi008 ceph-mon[191227]: pgmap v1366: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 163 op/s; 71302/144519 objects degraded (49.337%); 23788/144519 objects misplaced (16.460%); 8.1 MiB/s, 131 objects/s recovering 2023-10-09T05:58:46.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:46 smithi191 ceph-mon[161669]: pgmap v1366: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 163 op/s; 71302/144519 objects degraded (49.337%); 23788/144519 objects misplaced (16.460%); 8.1 MiB/s, 131 objects/s recovering 2023-10-09T05:58:46.740 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:46.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:46.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:46.845 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:46.892 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:47.035 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:47.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:47.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:47.071 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:58:47.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:47.185 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:47.240 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:47.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:47.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:47.347 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:47.389 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:47.608 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:47.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:47.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:47.716 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:47.752 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:47.802 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:58:47.835 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:58:47.868 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:58:47.892 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:58:47.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:47 smithi008 ceph-mon[184418]: pgmap v1367: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 149 op/s; 71092/143724 objects degraded (49.464%); 23018/143724 objects misplaced (16.015%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:58:47.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:47.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:47.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4292817718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:47.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:47 smithi008 ceph-mon[191227]: pgmap v1367: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 149 op/s; 71092/143724 objects degraded (49.464%); 23018/143724 objects misplaced (16.015%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:58:47.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:47.902 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:47.902 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4292817718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:47.920 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:58:47.950 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:58:47.976 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:58:48.001 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:58:48.002 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:58:48.003 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:58:48.003 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:58:48.003 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:58:48.003 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:58:48.003 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:58:48.003 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:58:48.003 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:58:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:47 smithi191 ceph-mon[161669]: pgmap v1367: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 149 op/s; 71092/143724 objects degraded (49.464%); 23018/143724 objects misplaced (16.015%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:58:48.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:48.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:48.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4292817718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:49.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 71092/143724 objects degraded (49.464%), 37 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:49.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:48 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2363631335' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:49.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 71092/143724 objects degraded (49.464%), 37 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:49.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:48 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2363631335' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:49.140 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:49.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:49.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:49.247 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:58:49.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:49.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 71092/143724 objects degraded (49.464%), 37 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:49.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:48 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2363631335' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:49.312 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:49.397 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:49.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:49.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:49.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:49.546 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:49.640 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:49.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:49.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:49.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:49.802 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:49.876 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:49.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:49.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:49.982 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:50.025 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:50.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:49 smithi191 ceph-mon[161669]: pgmap v1368: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 71092/143301 objects degraded (49.610%); 23003/143301 objects misplaced (16.052%); 9.9 MiB/s, 159 objects/s recovering 2023-10-09T05:58:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:49 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:49 smithi191 ceph-mon[161669]: failed to encode map e458 with expected crc 2023-10-09T05:58:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2363631335' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:49 smithi191 ceph-mon[161669]: osdmap e458: 8 total, 8 up, 5 in 2023-10-09T05:58:50.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:49 smithi191 ceph-mon[161669]: failed to encode map e458 with expected crc 2023-10-09T05:58:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[184418]: pgmap v1368: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 71092/143301 objects degraded (49.610%); 23003/143301 objects misplaced (16.052%); 9.9 MiB/s, 159 objects/s recovering 2023-10-09T05:58:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[184418]: failed to encode map e458 with expected crc 2023-10-09T05:58:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2363631335' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[184418]: osdmap e458: 8 total, 8 up, 5 in 2023-10-09T05:58:50.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[184418]: failed to encode map e458 with expected crc 2023-10-09T05:58:50.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[191227]: pgmap v1368: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 71092/143301 objects degraded (49.610%); 23003/143301 objects misplaced (16.052%); 9.9 MiB/s, 159 objects/s recovering 2023-10-09T05:58:50.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:50.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[191227]: failed to encode map e458 with expected crc 2023-10-09T05:58:50.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2363631335' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:50.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[191227]: osdmap e458: 8 total, 8 up, 5 in 2023-10-09T05:58:50.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:49 smithi008 ceph-mon[191227]: failed to encode map e458 with expected crc 2023-10-09T05:58:50.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:58:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:58:50] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.33.4" 2023-10-09T05:58:51.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:50 smithi008 ceph-mon[184418]: failed to encode map e458 with expected crc 2023-10-09T05:58:51.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:50 smithi008 ceph-mon[191227]: failed to encode map e458 with expected crc 2023-10-09T05:58:51.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:50 smithi191 ceph-mon[161669]: failed to encode map e458 with expected crc 2023-10-09T05:58:51.743 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:51.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:51.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:51.849 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:51.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:51 smithi008 ceph-mon[184418]: pgmap v1370: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 70484/141660 objects degraded (49.756%); 22569/141660 objects misplaced (15.932%); 10 MiB/s, 248 objects/s recovering 2023-10-09T05:58:51.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:51 smithi008 ceph-mon[191227]: pgmap v1370: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 70484/141660 objects degraded (49.756%); 22569/141660 objects misplaced (15.932%); 10 MiB/s, 248 objects/s recovering 2023-10-09T05:58:51.971 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:52.044 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:52.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:52.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:52.072 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3] out_osds: [0, 5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:58:52.072 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:58:52.072 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:58:52.072 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T05:58:52.152 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:52.199 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:51 smithi191 ceph-mon[161669]: pgmap v1370: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 70484/141660 objects degraded (49.756%); 22569/141660 objects misplaced (15.932%); 10 MiB/s, 248 objects/s recovering 2023-10-09T05:58:52.201 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:52.266 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:52.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:52.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:52.377 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:52.405 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:52.476 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:52.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:52.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:52.510 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:58:52.510 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:58:52.582 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:52.624 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:52.838 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[184418]: osdmap e459: 8 total, 8 up, 5 in 2023-10-09T05:58:52.838 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[184418]: failed to encode map e459 with expected crc 2023-10-09T05:58:52.838 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[184418]: failed to encode map e459 with expected crc 2023-10-09T05:58:52.838 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[184418]: failed to encode map e459 with expected crc 2023-10-09T05:58:52.838 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[184418]: failed to encode map e459 with expected crc 2023-10-09T05:58:52.838 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[184418]: failed to encode map e459 with expected crc 2023-10-09T05:58:53.190 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[191227]: osdmap e459: 8 total, 8 up, 5 in 2023-10-09T05:58:53.190 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[191227]: failed to encode map e459 with expected crc 2023-10-09T05:58:53.190 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[191227]: failed to encode map e459 with expected crc 2023-10-09T05:58:53.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[191227]: failed to encode map e459 with expected crc 2023-10-09T05:58:53.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[191227]: failed to encode map e459 with expected crc 2023-10-09T05:58:53.191 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:52 smithi008 ceph-mon[191227]: failed to encode map e459 with expected crc 2023-10-09T05:58:53.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:52 smithi191 ceph-mon[161669]: osdmap e459: 8 total, 8 up, 5 in 2023-10-09T05:58:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:52 smithi191 ceph-mon[161669]: failed to encode map e459 with expected crc 2023-10-09T05:58:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:52 smithi191 ceph-mon[161669]: failed to encode map e459 with expected crc 2023-10-09T05:58:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:52 smithi191 ceph-mon[161669]: failed to encode map e459 with expected crc 2023-10-09T05:58:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:52 smithi191 ceph-mon[161669]: failed to encode map e459 with expected crc 2023-10-09T05:58:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:52 smithi191 ceph-mon[161669]: failed to encode map e459 with expected crc 2023-10-09T05:58:53.549 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:53.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:53.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:53.635 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:58:53.656 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:53.705 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:58:53.710 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:53.736 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:53.769 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:58:53.888 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:53.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:53.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:54.001 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:54.109 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[184418]: pgmap v1372: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 136 op/s; 70484/141252 objects degraded (49.899%); 22546/141252 objects misplaced (15.962%); 11 MiB/s, 282 objects/s recovering 2023-10-09T05:58:54.109 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[184418]: osdmap e460: 8 total, 8 up, 5 in 2023-10-09T05:58:54.110 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[184418]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.110 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[184418]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.110 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[184418]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.110 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[184418]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.110 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[184418]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.110 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 70484/141252 objects degraded (49.899%), 37 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:54.110 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:54.110 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:54.111 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:54.113 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[191227]: pgmap v1372: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 136 op/s; 70484/141252 objects degraded (49.899%); 22546/141252 objects misplaced (15.962%); 11 MiB/s, 282 objects/s recovering 2023-10-09T05:58:54.113 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[191227]: osdmap e460: 8 total, 8 up, 5 in 2023-10-09T05:58:54.113 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[191227]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.114 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[191227]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.114 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[191227]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.114 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[191227]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.114 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[191227]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.114 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 70484/141252 objects degraded (49.899%), 37 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:54.114 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:54.114 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:54.262 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:54.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:54.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:53 smithi191 ceph-mon[161669]: pgmap v1372: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 136 op/s; 70484/141252 objects degraded (49.899%); 22546/141252 objects misplaced (15.962%); 11 MiB/s, 282 objects/s recovering 2023-10-09T05:58:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:53 smithi191 ceph-mon[161669]: osdmap e460: 8 total, 8 up, 5 in 2023-10-09T05:58:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:53 smithi191 ceph-mon[161669]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:53 smithi191 ceph-mon[161669]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:53 smithi191 ceph-mon[161669]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:53 smithi191 ceph-mon[161669]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:53 smithi191 ceph-mon[161669]: failed to encode map e460 with expected crc 2023-10-09T05:58:54.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 70484/141252 objects degraded (49.899%), 37 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:54.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:54.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:54.368 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:54.402 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:54.634 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:54.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:54.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:54.739 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:54.774 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:54.829 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:54.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:54.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:54.933 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:54.963 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:55.021 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:55.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:55.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:55.134 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:55.182 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:54 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:55.182 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:54 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:55.182 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:54 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:55.193 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:55.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:54 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:55.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:54 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:55.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:54 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:54 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:54 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:54 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:58:55.256 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:55.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:55.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:55.363 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:55.383 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:55.420 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:55.425 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:55.441 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:55.550 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:55.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:55.655 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:55.685 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:55.770 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:55.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:55.876 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:55.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:55 smithi008 ceph-mon[184418]: pgmap v1374: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 169 op/s; 70119/140256 objects degraded (49.994%); 22467/140256 objects misplaced (16.019%); 8.9 MiB/s, 279 objects/s recovering 2023-10-09T05:58:55.900 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:55 smithi008 ceph-mon[191227]: pgmap v1374: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 169 op/s; 70119/140256 objects degraded (49.994%); 22467/140256 objects misplaced (16.019%); 8.9 MiB/s, 279 objects/s recovering 2023-10-09T05:58:55.907 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:55.969 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:55.977 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:55 smithi191 ceph-mon[161669]: pgmap v1374: 99 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 22 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 169 op/s; 70119/140256 objects degraded (49.994%); 22467/140256 objects misplaced (16.019%); 8.9 MiB/s, 279 objects/s recovering 2023-10-09T05:58:55.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:55.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:56.078 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:56.124 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:56.185 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:56.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:56.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:56.291 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:56.336 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:56.877 INFO:teuthology.orchestra.run.smithi008.stdout:moved 0 / 297 (0%) 2023-10-09T05:58:56.878 INFO:teuthology.orchestra.run.smithi008.stdout:avg 59.4 2023-10-09T05:58:56.878 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 5.64269 -> 5.64269 (expected baseline 6.89348) 2023-10-09T05:58:56.878 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.2 with 54 -> 54 pgs (0.909091 -> 0.909091 * mean) 2023-10-09T05:58:56.878 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.7 with 69 -> 69 pgs (1.16162 -> 1.16162 * mean) 2023-10-09T05:58:56.878 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:58:56.878 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T05:58:56.878 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 1 2023-10-09T05:58:56.878 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:58:56.879 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0243 2023-10-09T05:58:56.879 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.2425 2023-10-09T05:58:56.879 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T05:58:57.612 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:57.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:57.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:57.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:57.749 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:57.809 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:57.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:57.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:57.856 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:57 smithi008 ceph-mon[191227]: pgmap v1375: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 184 op/s; 65366/139191 objects degraded (46.961%); 24067/139191 objects misplaced (17.291%); 9.6 MiB/s, 297 objects/s recovering 2023-10-09T05:58:57.915 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:57.953 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:57 smithi191 ceph-mon[161669]: pgmap v1375: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 184 op/s; 65366/139191 objects degraded (46.961%); 24067/139191 objects misplaced (17.291%); 9.6 MiB/s, 297 objects/s recovering 2023-10-09T05:58:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:57 smithi191 ceph-mon[161669]: from='client.56563 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:58:58.132 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:58.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:57 smithi008 ceph-mon[184418]: pgmap v1375: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 184 op/s; 65366/139191 objects degraded (46.961%); 24067/139191 objects misplaced (17.291%); 9.6 MiB/s, 297 objects/s recovering 2023-10-09T05:58:58.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:57 smithi008 ceph-mon[184418]: from='client.56563 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:58:58.134 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:57 smithi008 ceph-mon[191227]: from='client.56563 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:58:58.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:58.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:58.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:58.287 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:58.348 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:58.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:58.455 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:58.481 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:58.544 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:58.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:58.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:58.655 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:58.686 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:58.796 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:58:58.797 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T05:58:58.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 65366/139191 objects degraded (46.961%), 35 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:58.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 65366/139191 objects degraded (46.961%), 35 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:59.058 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:59.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:59.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:59.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:59.200 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 65366/139191 objects degraded (46.961%), 35 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:59.201 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:59.267 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:59.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:59.376 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:59.442 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:59.561 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:58:59.586 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:58:59.609 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:58:59.631 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:58:59.652 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:58:59.673 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:58:59.712 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:58:59.738 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:58:59.740 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:58:59.740 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:58:59.740 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:58:59.740 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:58:59.740 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:58:59.740 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:58:59.740 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:58:59.740 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:59:00.119 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:00.123 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:00.128 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:59 smithi008 ceph-mon[191227]: pgmap v1376: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 120 op/s; 65366/138765 objects degraded (47.106%); 24044/138765 objects misplaced (17.327%); 4.1 MiB/s, 96 objects/s recovering 2023-10-09T05:59:00.128 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:00.128 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:00.128 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:58:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2906926814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:00.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:00.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:00.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:59 smithi008 ceph-mon[184418]: pgmap v1376: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 120 op/s; 65366/138765 objects degraded (47.106%); 24044/138765 objects misplaced (17.327%); 4.1 MiB/s, 96 objects/s recovering 2023-10-09T05:59:00.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:00.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:00.130 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:58:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2906926814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:00.231 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:00.278 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:59 smithi191 ceph-mon[161669]: pgmap v1376: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 120 op/s; 65366/138765 objects degraded (47.106%); 24044/138765 objects misplaced (17.327%); 4.1 MiB/s, 96 objects/s recovering 2023-10-09T05:59:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:58:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2906926814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:00.549 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:00.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:00.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:00.657 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:00.691 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:00.927 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:59:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:59:00] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.33.4" 2023-10-09T05:59:00.945 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:59:01.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1328701078' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:01.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:00 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1328701078' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:00 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:01.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1328701078' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:01.304 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:00 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:01.426 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:01.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:01.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:01.530 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:01.581 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:01.635 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:01.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:01.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:01.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:01.774 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:01.861 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:01.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:01.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:01.974 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:02.010 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:02.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[191227]: pgmap v1377: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 183 op/s; 64833/136791 objects degraded (47.396%); 22934/136791 objects misplaced (16.766%); 8.2 MiB/s, 159 objects/s recovering 2023-10-09T05:59:02.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:02.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:59:02.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[191227]: osdmap e461: 8 total, 8 up, 5 in 2023-10-09T05:59:02.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[191227]: failed to encode map e461 with expected crc 2023-10-09T05:59:02.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[191227]: failed to encode map e461 with expected crc 2023-10-09T05:59:02.043 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[191227]: failed to encode map e461 with expected crc 2023-10-09T05:59:02.044 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[184418]: pgmap v1377: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 183 op/s; 64833/136791 objects degraded (47.396%); 22934/136791 objects misplaced (16.766%); 8.2 MiB/s, 159 objects/s recovering 2023-10-09T05:59:02.044 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:02.044 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:59:02.044 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[184418]: osdmap e461: 8 total, 8 up, 5 in 2023-10-09T05:59:02.044 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[184418]: failed to encode map e461 with expected crc 2023-10-09T05:59:02.044 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[184418]: failed to encode map e461 with expected crc 2023-10-09T05:59:02.044 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:01 smithi008 ceph-mon[184418]: failed to encode map e461 with expected crc 2023-10-09T05:59:02.098 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:02.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:02.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:02.209 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:02.270 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:01 smithi191 ceph-mon[161669]: pgmap v1377: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 183 op/s; 64833/136791 objects degraded (47.396%); 22934/136791 objects misplaced (16.766%); 8.2 MiB/s, 159 objects/s recovering 2023-10-09T05:59:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:01 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:01 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:59:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:01 smithi191 ceph-mon[161669]: osdmap e461: 8 total, 8 up, 5 in 2023-10-09T05:59:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:01 smithi191 ceph-mon[161669]: failed to encode map e461 with expected crc 2023-10-09T05:59:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:01 smithi191 ceph-mon[161669]: failed to encode map e461 with expected crc 2023-10-09T05:59:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:01 smithi191 ceph-mon[161669]: failed to encode map e461 with expected crc 2023-10-09T05:59:02.808 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:59:02.808 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:59:02.808 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T05:59:02.809 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:59:03.137 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:03.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:03.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:03.244 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:03.297 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:03.368 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:03.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:03.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:03.472 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:03.498 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:03.833 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:03.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:03.940 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:59:03.952 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:04.022 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:04.104 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:59:04.149 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:04.156 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:03 smithi008 ceph-mon[184418]: pgmap v1379: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 64833/136308 objects degraded (47.564%); 22905/136308 objects misplaced (16.804%); 7.4 MiB/s, 143 objects/s recovering 2023-10-09T05:59:04.156 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 64833/136308 objects degraded (47.564%), 35 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:04.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:04.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:04.165 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:03 smithi008 ceph-mon[191227]: pgmap v1379: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 64833/136308 objects degraded (47.564%); 22905/136308 objects misplaced (16.804%); 7.4 MiB/s, 143 objects/s recovering 2023-10-09T05:59:04.165 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 64833/136308 objects degraded (47.564%), 35 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:04.259 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:04.263 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:59:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:03 smithi191 ceph-mon[161669]: pgmap v1379: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 64833/136308 objects degraded (47.564%); 22905/136308 objects misplaced (16.804%); 7.4 MiB/s, 143 objects/s recovering 2023-10-09T05:59:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 64833/136308 objects degraded (47.564%), 35 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:04.349 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:04.530 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:04.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:04.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:04.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:04.695 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:04.696 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:04.768 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:04.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:04.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:04.784 INFO:teuthology.orchestra.run.smithi008.stdout:moved 13 / 297 (4.3771%) 2023-10-09T05:59:04.784 INFO:teuthology.orchestra.run.smithi008.stdout:avg 59.4 2023-10-09T05:59:04.784 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 5.64269 -> 7.52596 (expected baseline 6.89348) 2023-10-09T05:59:04.784 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.2 with 54 -> 59 pgs (0.909091 -> 0.993266 * mean) 2023-10-09T05:59:04.784 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.7 with 69 -> 72 pgs (1.16162 -> 1.21212 * mean) 2023-10-09T05:59:04.785 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:59:04.785 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T05:59:04.785 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 1 2023-10-09T05:59:04.785 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:59:04.785 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0240 2023-10-09T05:59:04.785 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0264 2023-10-09T05:59:04.785 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 weight 0.7757 -> 0.5988 2023-10-09T05:59:04.785 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 0.7579 -> 0.6450 2023-10-09T05:59:04.798 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T05:59:04.873 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:04.933 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:05.110 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:59:04 smithi008 conmon[177229]: 2023-10-09T05:59:04.781+0000 7f020285b700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T05:59:05.898 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:05.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:05.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:05.964 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:59:05.964 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-10-09T05:59:06.003 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:06.044 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:06.147 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:06.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:05 smithi008 ceph-mon[184418]: pgmap v1380: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 131 op/s; 64833/136308 objects degraded (47.564%); 22905/136308 objects misplaced (16.804%); 5.8 MiB/s, 117 objects/s recovering 2023-10-09T05:59:06.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:05 smithi008 ceph-mon[184418]: from='client.56578 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:05 smithi008 ceph-mon[191227]: pgmap v1380: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 131 op/s; 64833/136308 objects degraded (47.564%); 22905/136308 objects misplaced (16.804%); 5.8 MiB/s, 117 objects/s recovering 2023-10-09T05:59:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:05 smithi008 ceph-mon[191227]: from='client.56578 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:06.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:05 smithi191 ceph-mon[161669]: pgmap v1380: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 131 op/s; 64833/136308 objects degraded (47.564%); 22905/136308 objects misplaced (16.804%); 5.8 MiB/s, 117 objects/s recovering 2023-10-09T05:59:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:05 smithi191 ceph-mon[161669]: from='client.56578 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:06.341 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:06.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:06.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:06.445 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:06.475 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:06.522 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:06.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:06.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:06.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:06.666 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:06.742 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:06.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:06.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:06.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:06.896 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:06.957 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:06.963 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:06.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:06.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:07.069 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:07.108 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:07.171 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:07.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:07.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:07.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:07.312 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:07.382 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:07.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:07.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:07.397 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:07.489 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:07.532 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:07.604 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:07.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:07.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:07.711 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:07.751 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:07.821 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:07.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:07.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:07.883 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:59:07.883 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:59:07.883 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:59:07.883 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:59:07.883 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:59:07.931 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:07.972 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:08.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:07 smithi008 ceph-mon[184418]: pgmap v1381: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 171 op/s; 64207/134046 objects degraded (47.899%); 21798/134046 objects misplaced (16.262%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:59:08.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:07 smithi008 ceph-mon[191227]: pgmap v1381: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 171 op/s; 64207/134046 objects degraded (47.899%); 21798/134046 objects misplaced (16.262%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:59:08.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:07 smithi191 ceph-mon[161669]: pgmap v1381: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 171 op/s; 64207/134046 objects degraded (47.899%); 21798/134046 objects misplaced (16.262%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:59:09.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 64207/134046 objects degraded (47.899%), 35 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 64207/134046 objects degraded (47.899%), 35 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 64207/134046 objects degraded (47.899%), 35 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:09.536 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:09.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:09.641 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:09.690 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:09.756 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:09.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:09.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:09.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:09.931 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:10.032 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:10.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:10.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:10.139 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:10.194 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:10.222 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:59:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:10 smithi191 ceph-mon[161669]: pgmap v1382: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 174 op/s; 64207/133515 objects degraded (48.090%); 21754/133515 objects misplaced (16.293%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:59:10.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:10 smithi008 ceph-mon[184418]: pgmap v1382: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 174 op/s; 64207/133515 objects degraded (48.090%); 21754/133515 objects misplaced (16.293%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:59:10.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:10 smithi008 ceph-mon[191227]: pgmap v1382: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 174 op/s; 64207/133515 objects degraded (48.090%); 21754/133515 objects misplaced (16.293%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:59:10.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:59:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:59:10] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.33.4" 2023-10-09T05:59:11.265 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:11.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:11.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:11.297 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:59:11.370 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:11.415 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:11.544 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1254245477' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:11.544 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1254245477' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:11.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1254245477' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:11.868 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:11.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:11.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:11.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:12.037 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:59:12.061 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:12.064 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:59:12.086 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:59:12.107 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:59:12.133 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:59:12.146 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:12.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:12.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:12.157 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:59:12.183 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:59:12.209 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:59:12.211 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:59:12.211 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:59:12.211 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:59:12.211 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:59:12.211 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:59:12.211 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:59:12.211 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:59:12.211 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:59:12.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:12.296 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:12.410 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:12.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:12.419 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:12 smithi191 ceph-mon[161669]: pgmap v1383: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 172 op/s; 63704/131616 objects degraded (48.401%); 20335/131616 objects misplaced (15.450%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:59:12.419 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:12 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:12.419 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:12 smithi191 ceph-mon[161669]: failed to encode map e462 with expected crc 2023-10-09T05:59:12.419 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1254245477' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:12.419 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:12 smithi191 ceph-mon[161669]: osdmap e462: 8 total, 8 up, 5 in 2023-10-09T05:59:12.419 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:12 smithi191 ceph-mon[161669]: failed to encode map e462 with expected crc 2023-10-09T05:59:12.419 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/157475775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:12.519 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:12.557 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[184418]: pgmap v1383: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 172 op/s; 63704/131616 objects degraded (48.401%); 20335/131616 objects misplaced (15.450%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:59:12.557 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:12.557 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[184418]: failed to encode map e462 with expected crc 2023-10-09T05:59:12.557 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1254245477' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:12.557 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[184418]: osdmap e462: 8 total, 8 up, 5 in 2023-10-09T05:59:12.557 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[184418]: failed to encode map e462 with expected crc 2023-10-09T05:59:12.557 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/157475775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:12.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[191227]: pgmap v1383: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 172 op/s; 63704/131616 objects degraded (48.401%); 20335/131616 objects misplaced (15.450%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:59:12.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:12.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[191227]: failed to encode map e462 with expected crc 2023-10-09T05:59:12.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1254245477' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:12.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[191227]: osdmap e462: 8 total, 8 up, 5 in 2023-10-09T05:59:12.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[191227]: failed to encode map e462 with expected crc 2023-10-09T05:59:12.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/157475775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:12.561 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:13.014 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:13.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:13.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:13.119 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:13.157 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:13.272 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:13.282 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:13.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:13.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:13.392 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:13.434 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:13.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 63704/131616 objects degraded (48.401%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:13.563 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:13.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 63704/131616 objects degraded (48.401%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:13.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 63704/131616 objects degraded (48.401%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:13.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:13.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:13.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:13.701 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:13.761 INFO:teuthology.orchestra.run.smithi008.stdout:moved 11 / 297 (3.7037%) 2023-10-09T05:59:13.762 INFO:teuthology.orchestra.run.smithi008.stdout:avg 59.4 2023-10-09T05:59:13.762 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 5.64269 -> 6.65132 (expected baseline 6.89348) 2023-10-09T05:59:13.762 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.2 with 54 -> 59 pgs (0.909091 -> 0.993266 * mean) 2023-10-09T05:59:13.762 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.7 with 69 -> 71 pgs (1.16162 -> 1.19529 * mean) 2023-10-09T05:59:13.762 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:59:13.762 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T05:59:13.762 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 3 2023-10-09T05:59:13.762 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:59:13.762 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0239 2023-10-09T05:59:13.762 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0262 2023-10-09T05:59:13.762 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 weight 0.7757 -> 0.5943 2023-10-09T05:59:13.763 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 0.7579 -> 0.6545 2023-10-09T05:59:13.765 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T05:59:13.774 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:13.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:13.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:13.891 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:13.956 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:13.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:59:13 smithi008 conmon[177229]: 2023-10-09T05:59:13.760+0000 7f020285b700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T05:59:14.042 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:14.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:14.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:14.155 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:14.197 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:14.277 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:14.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:14.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:14.385 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:59:14.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:14.416 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:14.477 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:14.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:14 smithi191 ceph-mon[161669]: pgmap v1385: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 174 op/s; 63704/131082 objects degraded (48.599%); 20305/131082 objects misplaced (15.490%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:59:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:14.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:14.579 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:14 smithi008 ceph-mon[184418]: pgmap v1385: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 174 op/s; 63704/131082 objects degraded (48.599%); 20305/131082 objects misplaced (15.490%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:59:14.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:14.580 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:14.580 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:14 smithi008 ceph-mon[191227]: pgmap v1385: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 174 op/s; 63704/131082 objects degraded (48.599%); 20305/131082 objects misplaced (15.490%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:59:14.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:14.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:14.584 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:14.614 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:14.674 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:14.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:14.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:14.782 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:14.821 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:14.895 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:14.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:14.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:15.002 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:15.040 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:15.120 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:15.128 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:59:15.128 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-10-09T05:59:15.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:15.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:15.238 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:15.294 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:15.307 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:59:15.398 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:15.402 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:15 smithi008 ceph-mon[191227]: from='client.36765 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:15.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:15.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:15.403 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:15 smithi008 ceph-mon[184418]: from='client.36765 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:15.417 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:15.504 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:15.569 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:15.796 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:15.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:15.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:15 smithi191 ceph-mon[161669]: from='client.36765 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:15.901 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:15.911 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:59:15.911 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:59:15.961 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:16.033 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:16.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:16.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:16.146 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:16.188 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:16.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:16 smithi008 ceph-mon[184418]: pgmap v1386: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 174 op/s; 63704/131082 objects degraded (48.599%); 20305/131082 objects misplaced (15.490%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:59:16.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:16 smithi008 ceph-mon[191227]: pgmap v1386: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 174 op/s; 63704/131082 objects degraded (48.599%); 20305/131082 objects misplaced (15.490%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:59:16.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:16 smithi191 ceph-mon[161669]: pgmap v1386: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 174 op/s; 63704/131082 objects degraded (48.599%); 20305/131082 objects misplaced (15.490%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:59:17.198 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:17.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:17.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:17.302 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:17.347 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:17.425 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:59:17.505 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:17.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:17.541 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:17.592 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:17.610 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:17.653 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:18.269 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:18.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:18.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:18.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[184418]: pgmap v1387: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 169 op/s; 63119/128973 objects degraded (48.940%); 19469/128973 objects misplaced (15.095%); 8.3 MiB/s, 167 objects/s recovering 2023-10-09T05:59:18.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[184418]: failed to encode map e463 with expected crc 2023-10-09T05:59:18.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[184418]: failed to encode map e463 with expected crc 2023-10-09T05:59:18.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[184418]: osdmap e463: 8 total, 8 up, 5 in 2023-10-09T05:59:18.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[184418]: failed to encode map e463 with expected crc 2023-10-09T05:59:18.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 63119/128973 objects degraded (48.940%), 35 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:18.366 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[191227]: pgmap v1387: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 169 op/s; 63119/128973 objects degraded (48.940%); 19469/128973 objects misplaced (15.095%); 8.3 MiB/s, 167 objects/s recovering 2023-10-09T05:59:18.366 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[191227]: failed to encode map e463 with expected crc 2023-10-09T05:59:18.366 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[191227]: failed to encode map e463 with expected crc 2023-10-09T05:59:18.367 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[191227]: osdmap e463: 8 total, 8 up, 5 in 2023-10-09T05:59:18.367 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[191227]: failed to encode map e463 with expected crc 2023-10-09T05:59:18.367 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 63119/128973 objects degraded (48.940%), 35 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:18.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:18.476 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:18.583 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:18.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:18.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:18.595 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:18 smithi191 ceph-mon[161669]: pgmap v1387: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 169 op/s; 63119/128973 objects degraded (48.940%); 19469/128973 objects misplaced (15.095%); 8.3 MiB/s, 167 objects/s recovering 2023-10-09T05:59:18.595 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:18 smithi191 ceph-mon[161669]: failed to encode map e463 with expected crc 2023-10-09T05:59:18.595 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:18 smithi191 ceph-mon[161669]: failed to encode map e463 with expected crc 2023-10-09T05:59:18.595 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:18 smithi191 ceph-mon[161669]: osdmap e463: 8 total, 8 up, 5 in 2023-10-09T05:59:18.595 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:18 smithi191 ceph-mon[161669]: failed to encode map e463 with expected crc 2023-10-09T05:59:18.595 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 63119/128973 objects degraded (48.940%), 35 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:18.694 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:18.729 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:19.333 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[191227]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[191227]: osdmap e464: 8 total, 8 up, 5 in 2023-10-09T05:59:19.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[191227]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[191227]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[191227]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/411863814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:19.334 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:19.340 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:59:19.358 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:19.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:19.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:19.461 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:19.503 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[184418]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[184418]: osdmap e464: 8 total, 8 up, 5 in 2023-10-09T05:59:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[184418]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[184418]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[184418]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/411863814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:19.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:19 smithi191 ceph-mon[161669]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:19 smithi191 ceph-mon[161669]: osdmap e464: 8 total, 8 up, 5 in 2023-10-09T05:59:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:19 smithi191 ceph-mon[161669]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:19 smithi191 ceph-mon[161669]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:19 smithi191 ceph-mon[161669]: failed to encode map e464 with expected crc 2023-10-09T05:59:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:19 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/411863814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:20.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[184418]: pgmap v1390: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 139 op/s; 63119/128505 objects degraded (49.118%); 19450/128505 objects misplaced (15.136%); 5.0 MiB/s, 126 objects/s recovering 2023-10-09T05:59:20.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[184418]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[184418]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[184418]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[184418]: osdmap e465: 8 total, 8 up, 5 in 2023-10-09T05:59:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[184418]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[191227]: pgmap v1390: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 139 op/s; 63119/128505 objects degraded (49.118%); 19450/128505 objects misplaced (15.136%); 5.0 MiB/s, 126 objects/s recovering 2023-10-09T05:59:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[191227]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[191227]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[191227]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:20.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[191227]: osdmap e465: 8 total, 8 up, 5 in 2023-10-09T05:59:20.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:20 smithi008 ceph-mon[191227]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.758 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:59:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:59:20] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.33.4" 2023-10-09T05:59:20.760 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:20.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:20 smithi191 ceph-mon[161669]: pgmap v1390: 99 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 139 op/s; 63119/128505 objects degraded (49.118%); 19450/128505 objects misplaced (15.136%); 5.0 MiB/s, 126 objects/s recovering 2023-10-09T05:59:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:20 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:20.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:20 smithi191 ceph-mon[161669]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:20 smithi191 ceph-mon[161669]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:20 smithi191 ceph-mon[161669]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:20 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:20.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:20 smithi191 ceph-mon[161669]: osdmap e465: 8 total, 8 up, 5 in 2023-10-09T05:59:20.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:20 smithi191 ceph-mon[161669]: failed to encode map e465 with expected crc 2023-10-09T05:59:20.821 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:20.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:20.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:20.930 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:20.974 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:21.091 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:21.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:21.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:21.196 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:21.237 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:59:21.250 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:21.256 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:59:21.280 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:59:21.300 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:59:21.333 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:59:21.333 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:21.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:21.358 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:21 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3257748510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:21.358 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:21 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3257748510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:21.360 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:59:21.383 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:59:21.415 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:59:21.417 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:59:21.417 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:59:21.417 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:59:21.417 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:59:21.417 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:59:21.417 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:59:21.418 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:59:21.418 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:59:21.439 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:21.479 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:21.548 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:21.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:21 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3257748510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:21.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:21.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:21.657 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:21.690 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:21.747 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:21.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:21.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:21.856 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:21.910 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:22.029 INFO:teuthology.orchestra.run.smithi008.stdout:moved 0 / 297 (0%) 2023-10-09T05:59:22.029 INFO:teuthology.orchestra.run.smithi008.stdout:avg 59.4 2023-10-09T05:59:22.029 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 5.64269 -> 5.64269 (expected baseline 6.89348) 2023-10-09T05:59:22.030 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.2 with 54 -> 54 pgs (0.909091 -> 0.909091 * mean) 2023-10-09T05:59:22.030 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.7 with 69 -> 69 pgs (1.16162 -> 1.16162 * mean) 2023-10-09T05:59:22.030 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:59:22.030 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T05:59:22.030 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 0.05 2023-10-09T05:59:22.030 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:59:22.030 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0236 2023-10-09T05:59:22.030 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.2358 2023-10-09T05:59:22.032 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T05:59:22.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:22 smithi008 ceph-mon[184418]: pgmap v1392: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 62608/126732 objects degraded (49.402%); 18586/126732 objects misplaced (14.666%); 16 MiB/s, 386 objects/s recovering 2023-10-09T05:59:22.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:22 smithi008 ceph-mon[191227]: pgmap v1392: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 62608/126732 objects degraded (49.402%); 18586/126732 objects misplaced (14.666%); 16 MiB/s, 386 objects/s recovering 2023-10-09T05:59:22.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:22 smithi191 ceph-mon[161669]: pgmap v1392: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 62608/126732 objects degraded (49.402%); 18586/126732 objects misplaced (14.666%); 16 MiB/s, 386 objects/s recovering 2023-10-09T05:59:22.833 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:59:23.091 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:23.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:23.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:23.201 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:23.232 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:23.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:23 smithi008 ceph-mon[184418]: from='client.36783 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:23.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 62608/126732 objects degraded (49.402%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:23.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:23 smithi008 ceph-mon[191227]: from='client.36783 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:23.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 62608/126732 objects degraded (49.402%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:23.646 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:23.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:23.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:23.752 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:23.787 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:23.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:23 smithi191 ceph-mon[161669]: from='client.36783 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 62608/126732 objects degraded (49.402%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:23.845 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:23.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:23.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:23.913 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T05:59:23.913 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-10-09T05:59:23.949 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:23.995 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:24.083 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:59:24.084 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:59:24.084 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.084 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:126989 9.10 9:091e34e3:::benchmark_data_smithi008_502820_object42816:head [delete] snapc 0=[] ondisk+write+known_if_redirected e465)", 2023-10-09T05:59:24.084 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:59:24.068175+0000", 2023-10-09T05:59:24.084 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.015679238000000002, 2023-10-09T05:59:24.084 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015724037999999999, 2023-10-09T05:59:24.084 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:59:24.084 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:59:24.084 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T05:59:24.084 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 126989 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.068175+0000", 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:59:24.085 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.068175+0000", 2023-10-09T05:59:24.086 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:59:24.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.086 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.086 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:59:24.086 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.068159+0000", 2023-10-09T05:59:24.086 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999843 2023-10-09T05:59:24.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.086 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.086 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:59:24.086 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.068182+0000", 2023-10-09T05:59:24.086 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2272999999999998e-05 2023-10-09T05:59:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:59:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.068199+0000", 2023-10-09T05:59:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7150999999999999e-05 2023-10-09T05:59:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:59:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.068208+0000", 2023-10-09T05:59:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9700000000000005e-06 2023-10-09T05:59:24.087 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.068261+0000", 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.321e-05 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.068824+0000", 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00056274799999999996 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.074351+0000", 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0055276689999999998 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.074478+0000", 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000126258 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.074954+0000", 2023-10-09T05:59:24.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00047655499999999998 2023-10-09T05:59:24.090 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.090 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.090 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T05:59:24.090 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.077177+0000", 2023-10-09T05:59:24.090 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00222236 2023-10-09T05:59:24.090 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.090 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.090 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:24.090 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.079738+0000", 2023-10-09T05:59:24.090 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002561415 2023-10-09T05:59:24.090 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:59:24.091 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:59:24.091 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:59:24.091 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.091 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.091 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.36258.0:126992 9.b e465/449 9:dd1a90f8:::benchmark_data_smithi008_502820_object42819:head v 465'8048, mlcod=369'2101)", 2023-10-09T05:59:24.091 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:59:24.080808+0000", 2023-10-09T05:59:24.091 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.003046605, 2023-10-09T05:59:24.091 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032057740000000002, 2023-10-09T05:59:24.091 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:59:24.091 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:59:24.091 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.080808+0000", 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.080808+0000", 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.092 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.080807+0000", 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.080816+0000", 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9630000000000004e-06 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.080833+0000", 2023-10-09T05:59:24.093 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7139999999999999e-05 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.080841+0000", 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3520000000000007e-06 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.080915+0000", 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3768000000000003e-05 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:24.094 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:24.095 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:59:24.095 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:24.080944+0000", 2023-10-09T05:59:24.095 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8847e-05 2023-10-09T05:59:24.095 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:59:24.095 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:59:24.095 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:59:24.095 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:59:24.095 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:59:24.095 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 2 2023-10-09T05:59:24.095 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:59:24.104 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:24.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:24.208 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:24.277 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:24.286 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:24.366 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:24.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:24.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:24.475 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:24.508 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:24.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:24 smithi191 ceph-mon[161669]: pgmap v1393: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 170 op/s; 62504/125904 objects degraded (49.644%); 18540/125904 objects misplaced (14.726%); 11 MiB/s, 237 objects/s recovering 2023-10-09T05:59:24.609 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:24.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:24.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:24.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:24 smithi008 ceph-mon[191227]: pgmap v1393: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 170 op/s; 62504/125904 objects degraded (49.644%); 18540/125904 objects misplaced (14.726%); 11 MiB/s, 237 objects/s recovering 2023-10-09T05:59:24.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:24 smithi008 ceph-mon[184418]: pgmap v1393: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 170 op/s; 62504/125904 objects degraded (49.644%); 18540/125904 objects misplaced (14.726%); 11 MiB/s, 237 objects/s recovering 2023-10-09T05:59:24.727 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:24.767 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:25.024 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:25.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:25.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:25.052 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:59:25.128 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:59:25.135 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:25.217 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:25.388 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:25.409 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:25.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:25.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:25.514 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:25.547 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:25.605 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:25.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:25.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:25.710 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:25.738 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:26.389 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:26.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:26.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:26.496 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:26.532 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:26 smithi008 ceph-mon[184418]: pgmap v1394: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 138 op/s; 62504/125904 objects degraded (49.644%); 18540/125904 objects misplaced (14.726%); 8.8 MiB/s, 193 objects/s recovering 2023-10-09T05:59:26.532 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:26 smithi008 ceph-mon[191227]: pgmap v1394: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 138 op/s; 62504/125904 objects degraded (49.644%); 18540/125904 objects misplaced (14.726%); 8.8 MiB/s, 193 objects/s recovering 2023-10-09T05:59:26.533 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:26.636 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:26.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:26 smithi191 ceph-mon[161669]: pgmap v1394: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 138 op/s; 62504/125904 objects degraded (49.644%); 18540/125904 objects misplaced (14.726%); 8.8 MiB/s, 193 objects/s recovering 2023-10-09T05:59:26.826 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:26.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:26.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:26.931 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:26.961 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:27.501 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:27.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:27.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:27.606 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:27.616 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:27 smithi008 ceph-mon[184418]: failed to encode map e466 with expected crc 2023-10-09T05:59:27.668 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:27 smithi008 ceph-mon[184418]: failed to encode map e466 with expected crc 2023-10-09T05:59:27.669 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:27 smithi008 ceph-mon[184418]: osdmap e466: 8 total, 8 up, 5 in 2023-10-09T05:59:27.669 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:27 smithi008 ceph-mon[184418]: failed to encode map e466 with expected crc 2023-10-09T05:59:27.669 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:27 smithi008 ceph-mon[184418]: failed to encode map e466 with expected crc 2023-10-09T05:59:27.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:27 smithi008 ceph-mon[191227]: failed to encode map e466 with expected crc 2023-10-09T05:59:27.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:27 smithi008 ceph-mon[191227]: failed to encode map e466 with expected crc 2023-10-09T05:59:27.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:27 smithi008 ceph-mon[191227]: osdmap e466: 8 total, 8 up, 5 in 2023-10-09T05:59:27.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:27 smithi008 ceph-mon[191227]: failed to encode map e466 with expected crc 2023-10-09T05:59:27.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:27 smithi008 ceph-mon[191227]: failed to encode map e466 with expected crc 2023-10-09T05:59:27.670 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:27.745 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:27 smithi191 ceph-mon[161669]: failed to encode map e466 with expected crc 2023-10-09T05:59:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:27 smithi191 ceph-mon[161669]: failed to encode map e466 with expected crc 2023-10-09T05:59:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:27 smithi191 ceph-mon[161669]: osdmap e466: 8 total, 8 up, 5 in 2023-10-09T05:59:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:27 smithi191 ceph-mon[161669]: failed to encode map e466 with expected crc 2023-10-09T05:59:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:27 smithi191 ceph-mon[161669]: failed to encode map e466 with expected crc 2023-10-09T05:59:28.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[191227]: pgmap v1396: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 192 op/s; 61887/123894 objects degraded (49.952%); 16888/123894 objects misplaced (13.631%); 22 MiB/s, 398 objects/s recovering 2023-10-09T05:59:28.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[191227]: failed to encode map e467 with expected crc 2023-10-09T05:59:28.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[191227]: failed to encode map e467 with expected crc 2023-10-09T05:59:28.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[191227]: failed to encode map e467 with expected crc 2023-10-09T05:59:28.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[191227]: osdmap e467: 8 total, 8 up, 5 in 2023-10-09T05:59:28.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4131335558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:28.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:28.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 61887/123894 objects degraded (49.952%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:28.381 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T05:59:28.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[184418]: pgmap v1396: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 192 op/s; 61887/123894 objects degraded (49.952%); 16888/123894 objects misplaced (13.631%); 22 MiB/s, 398 objects/s recovering 2023-10-09T05:59:28.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[184418]: failed to encode map e467 with expected crc 2023-10-09T05:59:28.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[184418]: failed to encode map e467 with expected crc 2023-10-09T05:59:28.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[184418]: failed to encode map e467 with expected crc 2023-10-09T05:59:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[184418]: osdmap e467: 8 total, 8 up, 5 in 2023-10-09T05:59:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4131335558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:28.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 61887/123894 objects degraded (49.952%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:28 smithi191 ceph-mon[161669]: pgmap v1396: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 192 op/s; 61887/123894 objects degraded (49.952%); 16888/123894 objects misplaced (13.631%); 22 MiB/s, 398 objects/s recovering 2023-10-09T05:59:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:28 smithi191 ceph-mon[161669]: failed to encode map e467 with expected crc 2023-10-09T05:59:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:28 smithi191 ceph-mon[161669]: failed to encode map e467 with expected crc 2023-10-09T05:59:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:28 smithi191 ceph-mon[161669]: failed to encode map e467 with expected crc 2023-10-09T05:59:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:28 smithi191 ceph-mon[161669]: osdmap e467: 8 total, 8 up, 5 in 2023-10-09T05:59:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4131335558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:28.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:28 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:28.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 61887/123894 objects degraded (49.952%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:29.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:29.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:29.629 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:59:29.629 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[184418]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.629 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:29.629 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[184418]: osdmap e468: 8 total, 8 up, 5 in 2023-10-09T05:59:29.630 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[184418]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.630 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[184418]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.630 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[184418]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.630 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:29.630 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:29.630 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:59:29.630 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[191227]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.630 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:29.630 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[191227]: osdmap e468: 8 total, 8 up, 5 in 2023-10-09T05:59:29.631 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[191227]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.631 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[191227]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.631 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[191227]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.631 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:29.631 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:29.714 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:29.771 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:29.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:29 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:59:29.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:29 smithi191 ceph-mon[161669]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:29 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:29.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:29 smithi191 ceph-mon[161669]: osdmap e468: 8 total, 8 up, 5 in 2023-10-09T05:59:29.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:29 smithi191 ceph-mon[161669]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:29 smithi191 ceph-mon[161669]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:29 smithi191 ceph-mon[161669]: failed to encode map e468 with expected crc 2023-10-09T05:59:29.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:29.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:29.844 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:29.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:29.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:29.987 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:30.047 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:30.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:30.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:30.153 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:30.188 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:30.346 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:30.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:30.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:30.388 INFO:teuthology.orchestra.run.smithi008.stdout:moved 11 / 297 (3.7037%) 2023-10-09T05:59:30.389 INFO:teuthology.orchestra.run.smithi008.stdout:avg 59.4 2023-10-09T05:59:30.389 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 5.64269 -> 6.65132 (expected baseline 6.89348) 2023-10-09T05:59:30.389 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.2 with 54 -> 59 pgs (0.909091 -> 0.993266 * mean) 2023-10-09T05:59:30.389 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.7 with 69 -> 71 pgs (1.16162 -> 1.19529 * mean) 2023-10-09T05:59:30.389 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:59:30.389 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T05:59:30.389 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 1 2023-10-09T05:59:30.389 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T05:59:30.389 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0233 2023-10-09T05:59:30.389 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0256 2023-10-09T05:59:30.389 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 weight 0.7757 -> 0.5961 2023-10-09T05:59:30.390 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 0.7579 -> 0.6602 2023-10-09T05:59:30.453 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:30.484 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:30.642 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:30.643 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:30 smithi008 ceph-mon[184418]: pgmap v1399: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 135 op/s; 61887/123456 objects degraded (50.129%); 16855/123456 objects misplaced (13.653%); 18 MiB/s, 294 objects/s recovering 2023-10-09T05:59:30.643 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:30 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'39067','3':'43265'}"}]: dispatch 2023-10-09T05:59:30.644 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:59:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:59:30] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.33.4" 2023-10-09T05:59:30.644 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:30 smithi008 ceph-mon[191227]: pgmap v1399: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 135 op/s; 61887/123456 objects degraded (50.129%); 16855/123456 objects misplaced (13.653%); 18 MiB/s, 294 objects/s recovering 2023-10-09T05:59:30.644 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:30 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'39067','3':'43265'}"}]: dispatch 2023-10-09T05:59:30.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:30.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:30.747 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:30.771 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:30 smithi191 ceph-mon[161669]: pgmap v1399: 99 pgs: 1 active+remapped+backfilling, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 135 op/s; 61887/123456 objects degraded (50.129%); 16855/123456 objects misplaced (13.653%); 18 MiB/s, 294 objects/s recovering 2023-10-09T05:59:30.771 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:30 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'39067','3':'43265'}"}]: dispatch 2023-10-09T05:59:30.774 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:30.824 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:30.832 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:59:30.832 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:59:30.832 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:59:30.832 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:59:30.832 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:59:30.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:30.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:30.934 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:30.965 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:31.477 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:59:31.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: from='client.46733 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'39067','3':'43265'}"}]': finished 2023-10-09T05:59:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: osdmap e469: 8 total, 8 up, 5 in 2023-10-09T05:59:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[191227]: pgmap v1401: 99 pgs: 4 peering, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 170 op/s; 61392/121680 objects degraded (50.454%); 16367/121680 objects misplaced (13.451%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T05:59:31.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: from='client.46733 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:31.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'39067','3':'43265'}"}]': finished 2023-10-09T05:59:31.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: osdmap e469: 8 total, 8 up, 5 in 2023-10-09T05:59:31.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:31 smithi008 ceph-mon[184418]: pgmap v1401: 99 pgs: 4 peering, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 170 op/s; 61392/121680 objects degraded (50.454%); 16367/121680 objects misplaced (13.451%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T05:59:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: from='client.46733 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'39067','3':'43265'}"}]': finished 2023-10-09T05:59:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: osdmap e469: 8 total, 8 up, 5 in 2023-10-09T05:59:31.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: failed to encode map e469 with expected crc 2023-10-09T05:59:31.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:31 smithi191 ceph-mon[161669]: pgmap v1401: 99 pgs: 4 peering, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 57 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 170 op/s; 61392/121680 objects degraded (50.454%); 16367/121680 objects misplaced (13.451%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T05:59:32.126 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:32.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:32.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:32.137 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:59:32.215 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:32.232 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:32.258 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:32.299 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:32.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:32.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:32.405 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:32.454 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:32.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:32 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:59:32.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:32 smithi191 ceph-mon[161669]: failed to encode map e470 with expected crc 2023-10-09T05:59:32.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:32 smithi191 ceph-mon[161669]: failed to encode map e470 with expected crc 2023-10-09T05:59:32.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:32 smithi191 ceph-mon[161669]: osdmap e470: 8 total, 8 up, 5 in 2023-10-09T05:59:32.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:32 smithi191 ceph-mon[161669]: failed to encode map e470 with expected crc 2023-10-09T05:59:32.575 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:32.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:32.583 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:32 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:59:32.583 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:32 smithi008 ceph-mon[184418]: failed to encode map e470 with expected crc 2023-10-09T05:59:32.583 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:32 smithi008 ceph-mon[184418]: failed to encode map e470 with expected crc 2023-10-09T05:59:32.583 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:32 smithi008 ceph-mon[184418]: osdmap e470: 8 total, 8 up, 5 in 2023-10-09T05:59:32.583 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:32 smithi008 ceph-mon[184418]: failed to encode map e470 with expected crc 2023-10-09T05:59:32.583 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:32 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:59:32.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:32 smithi008 ceph-mon[191227]: failed to encode map e470 with expected crc 2023-10-09T05:59:32.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:32 smithi008 ceph-mon[191227]: failed to encode map e470 with expected crc 2023-10-09T05:59:32.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:32 smithi008 ceph-mon[191227]: osdmap e470: 8 total, 8 up, 5 in 2023-10-09T05:59:32.584 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:32 smithi008 ceph-mon[191227]: failed to encode map e470 with expected crc 2023-10-09T05:59:32.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:32.634 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:59:32.662 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:59:32.686 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:32.694 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:59:32.725 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:59:32.744 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:32.752 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:59:32.785 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:59:32.814 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:59:32.845 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:59:32.847 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:59:32.847 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:59:32.847 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:59:32.847 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:59:32.847 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:59:32.848 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:59:32.848 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:59:32.848 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:59:32.966 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:32.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:32.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:33.072 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:33.104 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:33.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[184418]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[184418]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[184418]: osdmap e471: 8 total, 8 up, 5 in 2023-10-09T05:59:33.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[184418]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[184418]: 9.e starting backfill to osd.3 from (0'0,0'0] MAX to 468'7985 2023-10-09T05:59:33.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[184418]: 9.a starting backfill to osd.2 from (0'0,0'0] MAX to 468'8001 2023-10-09T05:59:33.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[184418]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2876278419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:33.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[184418]: pgmap v1404: 99 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 57 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 197 op/s; 63058/120900 objects degraded (52.157%); 17032/120900 objects misplaced (14.088%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:59:33.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 61392/121680 objects degraded (50.454%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:33.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[191227]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[191227]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[191227]: osdmap e471: 8 total, 8 up, 5 in 2023-10-09T05:59:33.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[191227]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[191227]: 9.e starting backfill to osd.3 from (0'0,0'0] MAX to 468'7985 2023-10-09T05:59:33.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[191227]: 9.a starting backfill to osd.2 from (0'0,0'0] MAX to 468'8001 2023-10-09T05:59:33.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[191227]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2876278419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:33.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[191227]: pgmap v1404: 99 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 57 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 197 op/s; 63058/120900 objects degraded (52.157%); 17032/120900 objects misplaced (14.088%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:59:33.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 61392/121680 objects degraded (50.454%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:33.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:33 smithi191 ceph-mon[161669]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:33 smithi191 ceph-mon[161669]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:33 smithi191 ceph-mon[161669]: osdmap e471: 8 total, 8 up, 5 in 2023-10-09T05:59:33.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:33 smithi191 ceph-mon[161669]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:33 smithi191 ceph-mon[161669]: 9.e starting backfill to osd.3 from (0'0,0'0] MAX to 468'7985 2023-10-09T05:59:33.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:33 smithi191 ceph-mon[161669]: 9.a starting backfill to osd.2 from (0'0,0'0] MAX to 468'8001 2023-10-09T05:59:33.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:33 smithi191 ceph-mon[161669]: failed to encode map e471 with expected crc 2023-10-09T05:59:33.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:33 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2876278419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:33.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:33 smithi191 ceph-mon[161669]: pgmap v1404: 99 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 57 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 197 op/s; 63058/120900 objects degraded (52.157%); 17032/120900 objects misplaced (14.088%); 11 MiB/s, 174 objects/s recovering 2023-10-09T05:59:33.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 61392/121680 objects degraded (50.454%), 35 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:34.224 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:34.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:34.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:34.326 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:34.328 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:34.366 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:34.437 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:34.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:34.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:34.554 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:34.597 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:34.874 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:34.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:34.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:34.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:35.011 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:35.076 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:35.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:35.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:35.183 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:35.226 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:35.315 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:35.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:35.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:35.421 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:35.451 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:35.503 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:35.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:35.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:35.607 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:35.637 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:35.745 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:35 smithi191 ceph-mon[161669]: pgmap v1405: 99 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 57 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 63058/120900 objects degraded (52.157%); 17032/120900 objects misplaced (14.088%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T05:59:35.745 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3679644219' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:35.757 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T05:59:35.779 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:35 smithi008 ceph-mon[184418]: pgmap v1405: 99 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 57 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 63058/120900 objects degraded (52.157%); 17032/120900 objects misplaced (14.088%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T05:59:35.779 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3679644219' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:35.780 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:35 smithi008 ceph-mon[191227]: pgmap v1405: 99 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 13 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 57 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 63058/120900 objects degraded (52.157%); 17032/120900 objects misplaced (14.088%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T05:59:35.780 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3679644219' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:36.372 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:59:36.564 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:36.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:36.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:36.668 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:36.755 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:36.876 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:36.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:36.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:36.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:36.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[184418]: failed to encode map e472 with expected crc 2023-10-09T05:59:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[184418]: failed to encode map e472 with expected crc 2023-10-09T05:59:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3679644219' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:59:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[184418]: osdmap e472: 8 total, 8 up, 5 in 2023-10-09T05:59:36.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[184418]: failed to encode map e472 with expected crc 2023-10-09T05:59:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[191227]: failed to encode map e472 with expected crc 2023-10-09T05:59:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[191227]: failed to encode map e472 with expected crc 2023-10-09T05:59:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3679644219' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:59:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[191227]: osdmap e472: 8 total, 8 up, 5 in 2023-10-09T05:59:36.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:36 smithi008 ceph-mon[191227]: failed to encode map e472 with expected crc 2023-10-09T05:59:36.980 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:37.025 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:37.043 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:37.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:36 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:37.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:36 smithi191 ceph-mon[161669]: failed to encode map e472 with expected crc 2023-10-09T05:59:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:36 smithi191 ceph-mon[161669]: failed to encode map e472 with expected crc 2023-10-09T05:59:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3679644219' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:59:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:36 smithi191 ceph-mon[161669]: osdmap e472: 8 total, 8 up, 5 in 2023-10-09T05:59:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:36 smithi191 ceph-mon[161669]: failed to encode map e472 with expected crc 2023-10-09T05:59:37.083 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3] out_osds: [0, 5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:59:37.084 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:59:37.084 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T05:59:37.084 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:59:37.108 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:37.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:37.148 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:59:37.148 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:59:37.213 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:37.247 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:38.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:37 smithi008 ceph-mon[184418]: failed to encode map e472 with expected crc 2023-10-09T05:59:38.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:37 smithi008 ceph-mon[184418]: pgmap v1407: 99 pgs: 1 activating+undersized+degraded+remapped, 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 146 op/s; 62540/119046 objects degraded (52.534%); 19468/119046 objects misplaced (16.353%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:59:38.010 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:37 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T05:59:38.010 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:37 smithi008 ceph-mon[191227]: failed to encode map e472 with expected crc 2023-10-09T05:59:38.011 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:37 smithi008 ceph-mon[191227]: pgmap v1407: 99 pgs: 1 activating+undersized+degraded+remapped, 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 146 op/s; 62540/119046 objects degraded (52.534%); 19468/119046 objects misplaced (16.353%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:59:38.011 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:37 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T05:59:38.038 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:38.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:38.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:37 smithi191 ceph-mon[161669]: failed to encode map e472 with expected crc 2023-10-09T05:59:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:37 smithi191 ceph-mon[161669]: pgmap v1407: 99 pgs: 1 activating+undersized+degraded+remapped, 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 146 op/s; 62540/119046 objects degraded (52.534%); 19468/119046 objects misplaced (16.353%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:59:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:37 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T05:59:38.061 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:59:38.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:38.184 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:38.252 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:38.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:38.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:38.358 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:38.391 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:38.670 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:59:38.841 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:38.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:38.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:38.945 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:38.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 62540/119046 objects degraded (52.534%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:38.956 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 62540/119046 objects degraded (52.534%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:38.962 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:39.007 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:39.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 62540/119046 objects degraded (52.534%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:39.115 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:39.151 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:39.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:39.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:39.255 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:39.300 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:39.363 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:39.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:39.470 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:39.510 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:39.717 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:59:39.723 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:39.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:39.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:39.745 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:59:39.771 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:59:39.794 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:59:39.813 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:59:39.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:39.831 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:39.831 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:59:39.853 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:59:39.859 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:39.883 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:59:39.888 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:59:39.888 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:59:39.888 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:59:39.888 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:59:39.888 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:59:39.888 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:59:39.889 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:59:39.889 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:59:39.924 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:39.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:39.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:40.030 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:40.061 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:40.261 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:40 smithi191 ceph-mon[161669]: pgmap v1408: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 140 op/s; 62472/118236 objects degraded (52.837%); 19435/118236 objects misplaced (16.437%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:59:40.261 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:40 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3694330307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:40.323 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:40 smithi008 ceph-mon[184418]: pgmap v1408: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 140 op/s; 62472/118236 objects degraded (52.837%); 19435/118236 objects misplaced (16.437%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:59:40.323 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:40 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3694330307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:40.323 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:40 smithi008 ceph-mon[191227]: pgmap v1408: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 140 op/s; 62472/118236 objects degraded (52.837%); 19435/118236 objects misplaced (16.437%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:59:40.323 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:40 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3694330307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:40.717 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:59:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:59:40] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.33.4" 2023-10-09T05:59:40.911 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:40.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:40.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:41.017 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:41.054 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:41.432 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:41.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:41.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:41.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:41.580 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:41.658 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:41.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:41.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:41.769 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:41.811 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:41.886 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:41.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:41.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:41.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:42.025 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:42.086 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:42.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:42.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:42.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:42.233 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:42.296 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:42.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:42.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:42.405 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:42.443 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:42.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:42 smithi191 ceph-mon[161669]: pgmap v1409: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 210 op/s; 61903/116061 objects degraded (53.337%); 19254/116061 objects misplaced (16.590%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:59:42.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:42 smithi008 ceph-mon[184418]: pgmap v1409: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 210 op/s; 61903/116061 objects degraded (53.337%); 19254/116061 objects misplaced (16.590%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:59:42.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:42 smithi008 ceph-mon[191227]: pgmap v1409: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 210 op/s; 61903/116061 objects degraded (53.337%); 19254/116061 objects misplaced (16.590%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:59:42.779 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:42.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:42.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:42.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:42.912 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:42.959 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:42.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:42.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:43.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:43.090 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:43.135 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:43.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:43.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:43.243 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:43.274 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 61903/116061 objects degraded (53.337%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:43.277 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:43.343 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:43.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:43.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:43.451 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:43.485 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:43.547 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 61903/116061 objects degraded (53.337%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:43.548 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 61903/116061 objects degraded (53.337%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:43.625 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:43.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:43.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:43.769 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:43.829 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:43.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:43.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:43.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:43.986 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:43.999 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:59:43.999 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T05:59:43.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:43.999 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.36258.0:131148 9.b e472/449 9:d765128d:::benchmark_data_smithi008_502820_object46975:head v 472'8305, mlcod=369'2101)", 2023-10-09T05:59:43.999 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:59:43.996628+0000", 2023-10-09T05:59:43.999 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0023919900000000001, 2023-10-09T05:59:43.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0024187190000000002, 2023-10-09T05:59:43.999 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:43.996628+0000", 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:43.996628+0000", 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T05:59:44.000 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:43.996627+0000", 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:43.996636+0000", 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1209999999999996e-06 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:44.001 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:43.996655+0000", 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8664e-05 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:43.996664+0000", 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5789999999999993e-06 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:43.996717+0000", 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2995999999999999e-05 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:59:43.996738+0000", 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0820999999999999e-05 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 1 2023-10-09T05:59:44.003 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:59:44.057 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:44.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:44.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:44.163 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:44.203 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:44.253 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T05:59:44.255 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T05:59:44.284 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:44 smithi008 ceph-mon[184418]: pgmap v1410: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 203 op/s; 61757/114780 objects degraded (53.805%); 19150/114780 objects misplaced (16.684%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T05:59:44.284 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:44.284 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:44.284 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:44 smithi008 ceph-mon[191227]: pgmap v1410: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 203 op/s; 61757/114780 objects degraded (53.805%); 19150/114780 objects misplaced (16.684%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T05:59:44.284 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:44.285 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:44.362 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:59:44.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:44 smithi191 ceph-mon[161669]: pgmap v1410: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 203 op/s; 61757/114780 objects degraded (53.805%); 19150/114780 objects misplaced (16.684%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T05:59:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:45.128 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:45.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:45.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:45.232 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:45.266 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:45.283 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T05:59:45.326 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:45.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:45.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:45.401 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:45 smithi008 ceph-mon[184418]: from='client.46754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:59:45.401 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1643333815' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:45.401 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:45 smithi008 ceph-mon[191227]: from='client.46754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:59:45.401 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1643333815' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:45.432 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:45.435 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:59:45.462 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:45.534 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:45.540 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:45 smithi191 ceph-mon[161669]: from='client.46754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:59:45.540 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1643333815' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:45.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:45.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:45.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:45.676 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:45.727 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:45.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:45.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:45.833 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:45.861 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:46.052 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1410,"stamp":"2023-10-09T05:59:42.694201+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2510297035,"num_objects":38260,"num_object_clones":0,"num_object_copies":114780,"num_objects_missing_on_primary":11538,"num_objects_missing":11538,"num_objects_degraded":61757,"num_objects_misplaced":19150,"num_objects_unfound":0,"num_objects_dirty":38260,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":131216,"num_write_kb":5402022,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41912,"num_bytes_recovered":2381005349,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":157582,"ondisk_log_size":157582,"up":297,"acting":277,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":10470772,"kb_used_data":5933900,"kb_used_omap":37,"kb_used_meta":4536794,"kb_avail":458254988,"statfs":{"total":479975178240,"available":469253107712,"internally_reserved":0,"allocated":6076313600,"data_stored":6068682641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38373,"internal_metadata":4645677595},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-133693440,"num_objects":-2040,"num_object_clones":0,"num_object_copies":-6120,"num_objects_missing_on_primary":-109,"num_objects_missing":-109,"num_objects_degraded":-1301,"num_objects_misplaced":2118,"num_objects_unfound":0,"num_objects_dirty":-2040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2040,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":631,"num_bytes_recovered":41418752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004023"},"pg_stats":[{"pgid":"9.10","version":"472'6403","reported_seq":7648,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:40.028623+0000","last_change":"2023-10-09T05:58:08.524814+0000","last_active":"2023-10-09T05:59:40.028623+0000","last_peered":"2023-10-09T05:59:40.028623+0000","last_clean":"2023-10-09T05:56:48.221888+0000","last_became_active":"2023-10-09T05:58:08.514640+0000","last_became_peered":"2023-10-09T05:58:08.514640+0000","last_unstale":"2023-10-09T05:59:40.028623+0000","last_undegraded":"2023-10-09T05:58:08.398419+0000","last_fullsized":"2023-10-09T05:58:08.325336+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6403,"log_dups_size":0,"ondisk_log_size":6403,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77004800,"num_objects":1175,"num_object_clones":0,"num_object_copies":3525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4464,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4013,"num_write_kb":166016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23855104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2,6","objects":1177},{"shards":"2,4","objects":-2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":93,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.408317+0000","last_change":"2023-10-09T05:54:59.256087+0000","last_active":"2023-10-09T05:59:32.408317+0000","last_peered":"2023-10-09T05:59:32.408317+0000","last_clean":"2023-10-09T05:59:32.408317+0000","last_became_active":"2023-10-09T05:54:59.255773+0000","last_became_peered":"2023-10-09T05:54:59.255773+0000","last_unstale":"2023-10-09T05:59:32.408317+0000","last_undegraded":"2023-10-09T05:59:32.408317+0000","last_fullsized":"2023-10-09T05:59:32.408317+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:19.545754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":68,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.406310+0000","last_change":"2023-10-09T05:56:00.247579+0000","last_active":"2023-10-09T05:59:32.406310+0000","last_peered":"2023-10-09T05:59:32.406310+0000","last_clean":"2023-10-09T05:59:32.406310+0000","last_became_active":"2023-10-09T05:56:00.238151+0000","last_became_peered":"2023-10-09T05:56:00.238151+0000","last_unstale":"2023-10-09T05:59:32.406310+0000","last_undegraded":"2023-10-09T05:59:32.406310+0000","last_fullsized":"2023-10-09T05:59:32.406310+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:05.692190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"472'6458","reported_seq":12322,"reported_epoch":472,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:59:42.440666+0000","last_change":"2023-10-09T05:56:01.319470+0000","last_active":"2023-10-09T05:59:42.440666+0000","last_peered":"2023-10-09T05:59:42.440666+0000","last_clean":"2023-10-09T05:55:24.768851+0000","last_became_active":"2023-10-09T05:56:01.311996+0000","last_became_peered":"2023-10-09T05:56:01.311996+0000","last_unstale":"2023-10-09T05:59:42.440666+0000","last_undegraded":"2023-10-09T05:59:42.440666+0000","last_fullsized":"2023-10-09T05:59:42.440666+0000","mapping_epoch":416,"log_start":"367'129","ondisk_log_start":"367'129","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6329,"log_dups_size":0,"ondisk_log_size":6329,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79757312,"num_objects":1217,"num_object_clones":0,"num_object_copies":3651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1217,"num_objects_unfound":0,"num_objects_dirty":1217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4045,"num_write_kb":168384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1422,"num_bytes_recovered":93192192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1217}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":174,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.399148+0000","last_change":"2023-10-09T05:56:00.243822+0000","last_active":"2023-10-09T05:59:32.399148+0000","last_peered":"2023-10-09T05:59:32.399148+0000","last_clean":"2023-10-09T05:59:32.399148+0000","last_became_active":"2023-10-09T05:56:00.243483+0000","last_became_peered":"2023-10-09T05:56:00.243483+0000","last_unstale":"2023-10-09T05:59:32.399148+0000","last_undegraded":"2023-10-09T05:59:32.399148+0000","last_fullsized":"2023-10-09T05:59:32.399148+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:39.811952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":542,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.402359+0000","last_change":"2023-10-09T05:57:57.277222+0000","last_active":"2023-10-09T05:59:32.402359+0000","last_peered":"2023-10-09T05:59:32.402359+0000","last_clean":"2023-10-09T05:59:32.402359+0000","last_became_active":"2023-10-09T05:57:57.276994+0000","last_became_peered":"2023-10-09T05:57:57.276994+0000","last_unstale":"2023-10-09T05:59:32.402359+0000","last_undegraded":"2023-10-09T05:59:32.402359+0000","last_fullsized":"2023-10-09T05:59:32.402359+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.12","version":"472'6490","reported_seq":12037,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-10-09T05:59:40.179987+0000","last_change":"2023-10-09T05:58:12.409601+0000","last_active":"2023-10-09T05:59:40.179987+0000","last_peered":"2023-10-09T05:59:40.179987+0000","last_clean":"2023-10-09T05:59:40.179987+0000","last_became_active":"2023-10-09T05:58:12.403280+0000","last_became_peered":"2023-10-09T05:58:12.403280+0000","last_unstale":"2023-10-09T05:59:40.179987+0000","last_undegraded":"2023-10-09T05:59:40.179987+0000","last_fullsized":"2023-10-09T05:59:40.179987+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6490,"log_dups_size":0,"ondisk_log_size":6490,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81166336,"num_objects":1239,"num_object_clones":0,"num_object_copies":3717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4069,"num_write_kb":169824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2689,"num_bytes_recovered":106627072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":73,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.403781+0000","last_change":"2023-10-09T05:56:49.265409+0000","last_active":"2023-10-09T05:59:32.403781+0000","last_peered":"2023-10-09T05:59:32.403781+0000","last_clean":"2023-10-09T05:59:32.403781+0000","last_became_active":"2023-10-09T05:56:49.265006+0000","last_became_peered":"2023-10-09T05:56:49.265006+0000","last_unstale":"2023-10-09T05:59:32.403781+0000","last_undegraded":"2023-10-09T05:59:32.403781+0000","last_fullsized":"2023-10-09T05:59:32.403781+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:29.506553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":47,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.402202+0000","last_change":"2023-10-09T05:58:07.465790+0000","last_active":"2023-10-09T05:59:32.402202+0000","last_peered":"2023-10-09T05:59:32.402202+0000","last_clean":"2023-10-09T05:59:32.402202+0000","last_became_active":"2023-10-09T05:58:07.430785+0000","last_became_peered":"2023-10-09T05:58:07.430785+0000","last_unstale":"2023-10-09T05:59:32.402202+0000","last_undegraded":"2023-10-09T05:59:32.402202+0000","last_fullsized":"2023-10-09T05:59:32.402202+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":36,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.407603+0000","last_change":"2023-10-09T05:58:07.412070+0000","last_active":"2023-10-09T05:59:32.407603+0000","last_peered":"2023-10-09T05:59:32.407603+0000","last_clean":"2023-10-09T05:59:32.407603+0000","last_became_active":"2023-10-09T05:58:07.411692+0000","last_became_peered":"2023-10-09T05:58:07.411692+0000","last_unstale":"2023-10-09T05:59:32.407603+0000","last_undegraded":"2023-10-09T05:59:32.407603+0000","last_fullsized":"2023-10-09T05:59:32.407603+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:36.457810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":77,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.412005+0000","last_change":"2023-10-09T05:58:07.387601+0000","last_active":"2023-10-09T05:59:32.412005+0000","last_peered":"2023-10-09T05:59:32.412005+0000","last_clean":"2023-10-09T05:59:32.412005+0000","last_became_active":"2023-10-09T05:58:07.387307+0000","last_became_peered":"2023-10-09T05:58:07.387307+0000","last_unstale":"2023-10-09T05:59:32.412005+0000","last_undegraded":"2023-10-09T05:59:32.412005+0000","last_fullsized":"2023-10-09T05:59:32.412005+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:33:56.748928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":120,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.402653+0000","last_change":"2023-10-09T05:59:31.413549+0000","last_active":"2023-10-09T05:59:32.402653+0000","last_peered":"2023-10-09T05:59:32.402653+0000","last_clean":"2023-10-09T05:59:32.402653+0000","last_became_active":"2023-10-09T05:59:31.413313+0000","last_became_peered":"2023-10-09T05:59:31.413313+0000","last_unstale":"2023-10-09T05:59:32.402653+0000","last_undegraded":"2023-10-09T05:59:32.402653+0000","last_fullsized":"2023-10-09T05:59:32.402653+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":105,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.422658+0000","last_change":"2023-10-09T05:58:07.427310+0000","last_active":"2023-10-09T05:59:32.422658+0000","last_peered":"2023-10-09T05:59:32.422658+0000","last_clean":"2023-10-09T05:59:32.422658+0000","last_became_active":"2023-10-09T05:58:07.426813+0000","last_became_peered":"2023-10-09T05:58:07.426813+0000","last_unstale":"2023-10-09T05:59:32.422658+0000","last_undegraded":"2023-10-09T05:59:32.422658+0000","last_fullsized":"2023-10-09T05:59:32.422658+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":105,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.404903+0000","last_change":"2023-10-09T05:56:00.348771+0000","last_active":"2023-10-09T05:59:32.404903+0000","last_peered":"2023-10-09T05:59:32.404903+0000","last_clean":"2023-10-09T05:59:32.404903+0000","last_became_active":"2023-10-09T05:56:00.224347+0000","last_became_peered":"2023-10-09T05:56:00.224347+0000","last_unstale":"2023-10-09T05:59:32.404903+0000","last_undegraded":"2023-10-09T05:59:32.404903+0000","last_fullsized":"2023-10-09T05:59:32.404903+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":242,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.408357+0000","last_change":"2023-10-09T05:58:07.401823+0000","last_active":"2023-10-09T05:59:32.408357+0000","last_peered":"2023-10-09T05:59:32.408357+0000","last_clean":"2023-10-09T05:59:32.408357+0000","last_became_active":"2023-10-09T05:58:07.401454+0000","last_became_peered":"2023-10-09T05:58:07.401454+0000","last_unstale":"2023-10-09T05:59:32.408357+0000","last_undegraded":"2023-10-09T05:59:32.408357+0000","last_fullsized":"2023-10-09T05:59:32.408357+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:04.542473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":97,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.401806+0000","last_change":"2023-10-09T05:58:07.410626+0000","last_active":"2023-10-09T05:59:32.401806+0000","last_peered":"2023-10-09T05:59:32.401806+0000","last_clean":"2023-10-09T05:58:01.307429+0000","last_became_active":"2023-10-09T05:58:07.410375+0000","last_became_peered":"2023-10-09T05:58:07.410375+0000","last_unstale":"2023-10-09T05:59:32.401806+0000","last_undegraded":"2023-10-09T05:58:07.381410+0000","last_fullsized":"2023-10-09T05:59:32.401806+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":["7","4","6"],"object_location_counts":[{"shards":"4,6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":456,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.408569+0000","last_change":"2023-10-09T05:58:07.913883+0000","last_active":"2023-10-09T05:59:32.408569+0000","last_peered":"2023-10-09T05:59:32.408569+0000","last_clean":"2023-10-09T05:59:32.408569+0000","last_became_active":"2023-10-09T05:58:07.913605+0000","last_became_peered":"2023-10-09T05:58:07.913605+0000","last_unstale":"2023-10-09T05:59:32.408569+0000","last_undegraded":"2023-10-09T05:59:32.408569+0000","last_fullsized":"2023-10-09T05:59:32.408569+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:46.780446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.19","version":"472'6439","reported_seq":8749,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:39.766143+0000","last_change":"2023-10-09T05:58:17.644481+0000","last_active":"2023-10-09T05:59:39.766143+0000","last_peered":"2023-10-09T05:59:39.766143+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T05:58:16.616896+0000","last_became_peered":"2023-10-09T05:58:16.616896+0000","last_unstale":"2023-10-09T05:59:39.766143+0000","last_undegraded":"2023-10-09T05:58:11.593270+0000","last_fullsized":"2023-10-09T05:58:09.843164+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6439,"log_dups_size":0,"ondisk_log_size":6439,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79593472,"num_objects":1215,"num_object_clones":0,"num_object_copies":3645,"num_objects_missing_on_primary":1689,"num_objects_missing":1689,"num_objects_degraded":6144,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4033,"num_write_kb":167904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":871,"num_bytes_recovered":57049088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1217},{"shards":"0,5","objects":1},{"shards":"0,5,7","objects":1},{"shards":"4,7","objects":-7}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":86,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.401439+0000","last_change":"2023-10-09T05:55:27.092744+0000","last_active":"2023-10-09T05:59:32.401439+0000","last_peered":"2023-10-09T05:59:32.401439+0000","last_clean":"2023-10-09T05:59:32.401439+0000","last_became_active":"2023-10-09T05:55:26.271096+0000","last_became_peered":"2023-10-09T05:55:26.271096+0000","last_unstale":"2023-10-09T05:59:32.401439+0000","last_undegraded":"2023-10-09T05:59:32.401439+0000","last_fullsized":"2023-10-09T05:59:32.401439+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":107,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.401723+0000","last_change":"2023-10-09T05:56:49.266384+0000","last_active":"2023-10-09T05:59:32.401723+0000","last_peered":"2023-10-09T05:59:32.401723+0000","last_clean":"2023-10-09T05:59:32.401723+0000","last_became_active":"2023-10-09T05:56:49.266051+0000","last_became_peered":"2023-10-09T05:56:49.266051+0000","last_unstale":"2023-10-09T05:59:32.401723+0000","last_undegraded":"2023-10-09T05:59:32.401723+0000","last_fullsized":"2023-10-09T05:59:32.401723+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"472'6434","reported_seq":17263,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-10-09T05:59:39.647336+0000","last_change":"2023-10-09T05:59:18.379327+0000","last_active":"2023-10-09T05:59:39.647336+0000","last_peered":"2023-10-09T05:59:39.647336+0000","last_clean":"2023-10-09T05:59:39.647336+0000","last_became_active":"2023-10-09T05:59:18.353678+0000","last_became_peered":"2023-10-09T05:59:18.353678+0000","last_unstale":"2023-10-09T05:59:39.647336+0000","last_undegraded":"2023-10-09T05:59:39.647336+0000","last_fullsized":"2023-10-09T05:59:39.647336+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6434,"log_dups_size":0,"ondisk_log_size":6434,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78839808,"num_objects":1203,"num_object_clones":0,"num_object_copies":3609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4021,"num_write_kb":167168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2787,"num_bytes_recovered":134348800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":516,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.415682+0000","last_change":"2023-10-09T05:58:07.410995+0000","last_active":"2023-10-09T05:59:32.415682+0000","last_peered":"2023-10-09T05:59:32.415682+0000","last_clean":"2023-10-09T05:59:32.415682+0000","last_became_active":"2023-10-09T05:58:07.410682+0000","last_became_peered":"2023-10-09T05:58:07.410682+0000","last_unstale":"2023-10-09T05:59:32.415682+0000","last_undegraded":"2023-10-09T05:59:32.415682+0000","last_fullsized":"2023-10-09T05:59:32.415682+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":158,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.404956+0000","last_change":"2023-10-09T05:58:07.404172+0000","last_active":"2023-10-09T05:59:32.404956+0000","last_peered":"2023-10-09T05:59:32.404956+0000","last_clean":"2023-10-09T05:59:32.404956+0000","last_became_active":"2023-10-09T05:58:07.403800+0000","last_became_peered":"2023-10-09T05:58:07.403800+0000","last_unstale":"2023-10-09T05:59:32.404956+0000","last_undegraded":"2023-10-09T05:59:32.404956+0000","last_fullsized":"2023-10-09T05:59:32.404956+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":87,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.422611+0000","last_change":"2023-10-09T05:58:07.405183+0000","last_active":"2023-10-09T05:59:32.422611+0000","last_peered":"2023-10-09T05:59:32.422611+0000","last_clean":"2023-10-09T05:59:32.422611+0000","last_became_active":"2023-10-09T05:58:07.404785+0000","last_became_peered":"2023-10-09T05:58:07.404785+0000","last_unstale":"2023-10-09T05:59:32.422611+0000","last_undegraded":"2023-10-09T05:59:32.422611+0000","last_fullsized":"2023-10-09T05:59:32.422611+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":495,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.402451+0000","last_change":"2023-10-09T05:56:11.341328+0000","last_active":"2023-10-09T05:59:32.402451+0000","last_peered":"2023-10-09T05:59:32.402451+0000","last_clean":"2023-10-09T05:59:32.402451+0000","last_became_active":"2023-10-09T05:56:11.340361+0000","last_became_peered":"2023-10-09T05:56:11.340361+0000","last_unstale":"2023-10-09T05:59:32.402451+0000","last_undegraded":"2023-10-09T05:59:32.402451+0000","last_fullsized":"2023-10-09T05:59:32.402451+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":78,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.403799+0000","last_change":"2023-10-09T05:58:07.401970+0000","last_active":"2023-10-09T05:59:32.403799+0000","last_peered":"2023-10-09T05:59:32.403799+0000","last_clean":"2023-10-09T05:58:01.300638+0000","last_became_active":"2023-10-09T05:58:07.401722+0000","last_became_peered":"2023-10-09T05:58:07.401722+0000","last_unstale":"2023-10-09T05:59:32.403799+0000","last_undegraded":"2023-10-09T05:58:07.367598+0000","last_fullsized":"2023-10-09T05:59:32.403799+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:31.363237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":["2","0","3","5","6"],"object_location_counts":[{"shards":"0,2,3,5,6","objects":4}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":106,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.403624+0000","last_change":"2023-10-09T05:56:49.263088+0000","last_active":"2023-10-09T05:59:32.403624+0000","last_peered":"2023-10-09T05:59:32.403624+0000","last_clean":"2023-10-09T05:59:32.403624+0000","last_became_active":"2023-10-09T05:56:49.262781+0000","last_became_peered":"2023-10-09T05:56:49.262781+0000","last_unstale":"2023-10-09T05:59:32.403624+0000","last_undegraded":"2023-10-09T05:59:32.403624+0000","last_fullsized":"2023-10-09T05:59:32.403624+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"472'6517","reported_seq":8953,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:39.905096+0000","last_change":"2023-10-09T05:58:05.458315+0000","last_active":"2023-10-09T05:59:39.905096+0000","last_peered":"2023-10-09T05:59:39.905096+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T05:58:04.631574+0000","last_became_peered":"2023-10-09T05:58:04.631574+0000","last_unstale":"2023-10-09T05:59:39.905096+0000","last_undegraded":"2023-10-09T05:58:03.836605+0000","last_fullsized":"2023-10-09T05:58:01.433430+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6517,"log_dups_size":0,"ondisk_log_size":6517,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78348288,"num_objects":1195,"num_object_clones":0,"num_object_copies":3585,"num_objects_missing_on_primary":1419,"num_objects_missing":1419,"num_objects_degraded":4552,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4075,"num_write_kb":168672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":770,"num_bytes_recovered":50462720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":964},{"shards":"0,1,5,7","objects":211},{"shards":"0,5,7","objects":4},{"shards":"1,7","objects":16}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":104,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.401925+0000","last_change":"2023-10-09T05:57:57.271964+0000","last_active":"2023-10-09T05:59:32.401925+0000","last_peered":"2023-10-09T05:59:32.401925+0000","last_clean":"2023-10-09T05:56:02.699477+0000","last_became_active":"2023-10-09T05:57:57.268930+0000","last_became_peered":"2023-10-09T05:57:57.268930+0000","last_unstale":"2023-10-09T05:59:32.401925+0000","last_undegraded":"2023-10-09T05:57:57.243041+0000","last_fullsized":"2023-10-09T05:59:32.401925+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":135,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.402980+0000","last_change":"2023-10-09T05:58:07.397628+0000","last_active":"2023-10-09T05:59:32.402980+0000","last_peered":"2023-10-09T05:59:32.402980+0000","last_clean":"2023-10-09T05:59:32.402980+0000","last_became_active":"2023-10-09T05:58:07.397010+0000","last_became_peered":"2023-10-09T05:58:07.397010+0000","last_unstale":"2023-10-09T05:59:32.402980+0000","last_undegraded":"2023-10-09T05:59:32.402980+0000","last_fullsized":"2023-10-09T05:59:32.402980+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"472'6547","reported_seq":6698,"reported_epoch":472,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:42.405715+0000","last_change":"2023-10-09T05:56:48.413650+0000","last_active":"2023-10-09T05:59:42.405715+0000","last_peered":"2023-10-09T05:59:42.405715+0000","last_clean":"2023-10-09T05:55:48.934818+0000","last_became_active":"2023-10-09T05:56:01.246051+0000","last_became_peered":"2023-10-09T05:56:01.246051+0000","last_unstale":"2023-10-09T05:59:42.405715+0000","last_undegraded":"2023-10-09T05:56:48.413475+0000","last_fullsized":"2023-10-09T05:59:42.405715+0000","mapping_epoch":416,"log_start":"367'362","ondisk_log_start":"367'362","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6185,"log_dups_size":0,"ondisk_log_size":6185,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76349440,"num_objects":1165,"num_object_clones":0,"num_object_copies":3495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4117,"num_write_kb":169024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1165}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":84,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.402544+0000","last_change":"2023-10-09T05:55:26.358519+0000","last_active":"2023-10-09T05:59:32.402544+0000","last_peered":"2023-10-09T05:59:32.402544+0000","last_clean":"2023-10-09T05:59:32.402544+0000","last_became_active":"2023-10-09T05:55:26.260138+0000","last_became_peered":"2023-10-09T05:55:26.260138+0000","last_unstale":"2023-10-09T05:59:32.402544+0000","last_undegraded":"2023-10-09T05:59:32.402544+0000","last_fullsized":"2023-10-09T05:59:32.402544+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":32,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.421359+0000","last_change":"2023-10-09T05:58:07.397638+0000","last_active":"2023-10-09T05:59:32.421359+0000","last_peered":"2023-10-09T05:59:32.421359+0000","last_clean":"2023-10-09T05:59:32.421359+0000","last_became_active":"2023-10-09T05:58:07.397242+0000","last_became_peered":"2023-10-09T05:58:07.397242+0000","last_unstale":"2023-10-09T05:59:32.421359+0000","last_undegraded":"2023-10-09T05:59:32.421359+0000","last_fullsized":"2023-10-09T05:59:32.421359+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.7","version":"472'6448","reported_seq":10142,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:39.875052+0000","last_change":"2023-10-09T05:58:02.333623+0000","last_active":"2023-10-09T05:59:39.875052+0000","last_peered":"2023-10-09T05:59:39.875052+0000","last_clean":"2023-10-09T05:53:45.891460+0000","last_became_active":"2023-10-09T05:58:02.118107+0000","last_became_peered":"2023-10-09T05:58:02.118107+0000","last_unstale":"2023-10-09T05:59:39.875052+0000","last_undegraded":"2023-10-09T05:58:01.208593+0000","last_fullsized":"2023-10-09T05:57:58.248025+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6448,"log_dups_size":0,"ondisk_log_size":6448,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78774272,"num_objects":1202,"num_object_clones":0,"num_object_copies":3606,"num_objects_missing_on_primary":937,"num_objects_missing":937,"num_objects_degraded":3587,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4024,"num_write_kb":167232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1553,"num_bytes_recovered":99483648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"2,3,5","objects":475},{"shards":"2,3,5,7","objects":641},{"shards":"2,7","objects":85},{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":98,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.405020+0000","last_change":"2023-10-09T05:58:07.406363+0000","last_active":"2023-10-09T05:59:32.405020+0000","last_peered":"2023-10-09T05:59:32.405020+0000","last_clean":"2023-10-09T05:56:43.219657+0000","last_became_active":"2023-10-09T05:58:07.404369+0000","last_became_peered":"2023-10-09T05:58:07.404369+0000","last_unstale":"2023-10-09T05:59:32.405020+0000","last_undegraded":"2023-10-09T05:58:07.387150+0000","last_fullsized":"2023-10-09T05:59:32.405020+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["7","3","5","6"],"object_location_counts":[{"shards":"3,5,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":98,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.415770+0000","last_change":"2023-10-09T05:59:31.413287+0000","last_active":"2023-10-09T05:59:32.415770+0000","last_peered":"2023-10-09T05:59:32.415770+0000","last_clean":"2023-10-09T05:59:32.415770+0000","last_became_active":"2023-10-09T05:59:31.412973+0000","last_became_peered":"2023-10-09T05:59:31.412973+0000","last_unstale":"2023-10-09T05:59:32.415770+0000","last_undegraded":"2023-10-09T05:59:32.415770+0000","last_fullsized":"2023-10-09T05:59:32.415770+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"472'6377","reported_seq":6528,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:40.483329+0000","last_change":"2023-10-09T05:58:07.640003+0000","last_active":"2023-10-09T05:59:40.483329+0000","last_peered":"2023-10-09T05:59:40.483329+0000","last_clean":"2023-10-09T05:57:55.996837+0000","last_became_active":"2023-10-09T05:58:07.625633+0000","last_became_peered":"2023-10-09T05:58:07.625633+0000","last_unstale":"2023-10-09T05:59:40.483329+0000","last_undegraded":"2023-10-09T05:58:07.405708+0000","last_fullsized":"2023-10-09T05:58:07.368268+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6377,"log_dups_size":0,"ondisk_log_size":6377,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78774272,"num_objects":1202,"num_object_clones":0,"num_object_copies":3606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2570,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3978,"num_write_kb":165760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","5"],"object_location_counts":[{"shards":"1,2","objects":-369},{"shards":"1,2,5","objects":1182}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":36,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.404698+0000","last_change":"2023-10-09T05:58:07.413999+0000","last_active":"2023-10-09T05:59:32.404698+0000","last_peered":"2023-10-09T05:59:32.404698+0000","last_clean":"2023-10-09T05:59:32.404698+0000","last_became_active":"2023-10-09T05:58:07.412998+0000","last_became_peered":"2023-10-09T05:58:07.412998+0000","last_unstale":"2023-10-09T05:59:32.404698+0000","last_undegraded":"2023-10-09T05:59:32.404698+0000","last_fullsized":"2023-10-09T05:59:32.404698+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:57.127693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":335,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.400294+0000","last_change":"2023-10-09T05:56:11.340232+0000","last_active":"2023-10-09T05:59:32.400294+0000","last_peered":"2023-10-09T05:59:32.400294+0000","last_clean":"2023-10-09T05:59:32.400294+0000","last_became_active":"2023-10-09T05:56:11.339675+0000","last_became_peered":"2023-10-09T05:56:11.339675+0000","last_unstale":"2023-10-09T05:59:32.400294+0000","last_undegraded":"2023-10-09T05:59:32.400294+0000","last_fullsized":"2023-10-09T05:59:32.400294+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:50.837847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"472'6534","reported_seq":17357,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:39.249108+0000","last_change":"2023-10-09T05:58:20.540189+0000","last_active":"2023-10-09T05:59:39.249108+0000","last_peered":"2023-10-09T05:59:39.249108+0000","last_clean":"2023-10-09T05:58:03.790253+0000","last_became_active":"2023-10-09T05:58:20.243902+0000","last_became_peered":"2023-10-09T05:58:20.243902+0000","last_unstale":"2023-10-09T05:59:39.249108+0000","last_undegraded":"2023-10-09T05:58:11.625534+0000","last_fullsized":"2023-10-09T05:58:11.609771+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6534,"log_dups_size":0,"ondisk_log_size":6534,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78807040,"num_objects":1202,"num_object_clones":0,"num_object_copies":3606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2666,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4080,"num_write_kb":169056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4116,"num_bytes_recovered":234323968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":1227},{"shards":"3,7","objects":-427}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":10,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.401237+0000","last_change":"2023-10-09T05:59:31.411309+0000","last_active":"2023-10-09T05:59:32.401237+0000","last_peered":"2023-10-09T05:59:32.401237+0000","last_clean":"2023-10-09T05:55:48.328137+0000","last_became_active":"2023-10-09T05:59:31.411011+0000","last_became_peered":"2023-10-09T05:59:31.411011+0000","last_unstale":"2023-10-09T05:59:32.401237+0000","last_undegraded":"2023-10-09T05:59:31.396595+0000","last_fullsized":"2023-10-09T05:59:32.401237+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["0","1","3","4"],"object_location_counts":[{"shards":"0,1,3,4","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":464,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.400723+0000","last_change":"2023-10-09T05:58:07.394333+0000","last_active":"2023-10-09T05:59:32.400723+0000","last_peered":"2023-10-09T05:59:32.400723+0000","last_clean":"2023-10-09T05:59:32.400723+0000","last_became_active":"2023-10-09T05:58:07.394002+0000","last_became_peered":"2023-10-09T05:58:07.394002+0000","last_unstale":"2023-10-09T05:59:32.400723+0000","last_undegraded":"2023-10-09T05:59:32.400723+0000","last_fullsized":"2023-10-09T05:59:32.400723+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"472'6393","reported_seq":7731,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:40.480568+0000","last_change":"2023-10-09T05:58:08.524816+0000","last_active":"2023-10-09T05:59:40.480568+0000","last_peered":"2023-10-09T05:59:40.480568+0000","last_clean":"2023-10-09T05:56:47.068306+0000","last_became_active":"2023-10-09T05:58:08.515147+0000","last_became_peered":"2023-10-09T05:58:08.515147+0000","last_unstale":"2023-10-09T05:59:40.480568+0000","last_undegraded":"2023-10-09T05:58:08.323807+0000","last_fullsized":"2023-10-09T05:58:08.237221+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6393,"log_dups_size":0,"ondisk_log_size":6393,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78446592,"num_objects":1197,"num_object_clones":0,"num_object_copies":3591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4408,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4003,"num_write_kb":166400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":394,"num_bytes_recovered":25821184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2,6","objects":1195},{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":34,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.401156+0000","last_change":"2023-10-09T05:58:07.400025+0000","last_active":"2023-10-09T05:59:32.401156+0000","last_peered":"2023-10-09T05:59:32.401156+0000","last_clean":"2023-10-09T05:59:32.401156+0000","last_became_active":"2023-10-09T05:58:07.399511+0000","last_became_peered":"2023-10-09T05:58:07.399511+0000","last_unstale":"2023-10-09T05:59:32.401156+0000","last_undegraded":"2023-10-09T05:59:32.401156+0000","last_fullsized":"2023-10-09T05:59:32.401156+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":402,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.405270+0000","last_change":"2023-10-09T05:54:05.161959+0000","last_active":"2023-10-09T05:59:32.405270+0000","last_peered":"2023-10-09T05:59:32.405270+0000","last_clean":"2023-10-09T05:59:32.405270+0000","last_became_active":"2023-10-09T05:54:05.161671+0000","last_became_peered":"2023-10-09T05:54:05.161671+0000","last_unstale":"2023-10-09T05:59:32.405270+0000","last_undegraded":"2023-10-09T05:59:32.405270+0000","last_fullsized":"2023-10-09T05:59:32.405270+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:47.282176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"472'6513","reported_seq":12462,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:42.458316+0000","last_change":"2023-10-09T05:56:01.447837+0000","last_active":"2023-10-09T05:59:42.458316+0000","last_peered":"2023-10-09T05:59:42.458316+0000","last_clean":"2023-10-09T05:55:25.103026+0000","last_became_active":"2023-10-09T05:56:01.440057+0000","last_became_peered":"2023-10-09T05:56:01.440057+0000","last_unstale":"2023-10-09T05:59:42.458316+0000","last_undegraded":"2023-10-09T05:56:01.309926+0000","last_fullsized":"2023-10-09T05:56:01.207038+0000","mapping_epoch":416,"log_start":"367'342","ondisk_log_start":"367'342","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6171,"log_dups_size":0,"ondisk_log_size":6171,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72876032,"num_objects":1112,"num_object_clones":0,"num_object_copies":3336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4100,"num_write_kb":166784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1468,"num_bytes_recovered":96206848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1,3","objects":-963},{"shards":"1,3,5","objects":1141}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":475,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.404094+0000","last_change":"2023-10-09T05:55:26.256379+0000","last_active":"2023-10-09T05:59:32.404094+0000","last_peered":"2023-10-09T05:59:32.404094+0000","last_clean":"2023-10-09T05:59:32.404094+0000","last_became_active":"2023-10-09T05:55:26.256062+0000","last_became_peered":"2023-10-09T05:55:26.256062+0000","last_unstale":"2023-10-09T05:59:32.404094+0000","last_undegraded":"2023-10-09T05:59:32.404094+0000","last_fullsized":"2023-10-09T05:59:32.404094+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:00.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":34,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.402090+0000","last_change":"2023-10-09T05:58:07.397389+0000","last_active":"2023-10-09T05:59:32.402090+0000","last_peered":"2023-10-09T05:59:32.402090+0000","last_clean":"2023-10-09T05:59:32.402090+0000","last_became_active":"2023-10-09T05:58:07.397034+0000","last_became_peered":"2023-10-09T05:58:07.397034+0000","last_unstale":"2023-10-09T05:59:32.402090+0000","last_undegraded":"2023-10-09T05:59:32.402090+0000","last_fullsized":"2023-10-09T05:59:32.402090+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2","version":"472'6481","reported_seq":11971,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-10-09T05:59:40.206139+0000","last_change":"2023-10-09T05:58:52.969080+0000","last_active":"2023-10-09T05:59:40.206139+0000","last_peered":"2023-10-09T05:59:40.206139+0000","last_clean":"2023-10-09T05:59:40.206139+0000","last_became_active":"2023-10-09T05:58:52.906218+0000","last_became_peered":"2023-10-09T05:58:52.906218+0000","last_unstale":"2023-10-09T05:59:40.206139+0000","last_undegraded":"2023-10-09T05:59:40.206139+0000","last_fullsized":"2023-10-09T05:59:40.206139+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6481,"log_dups_size":0,"ondisk_log_size":6481,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77168640,"num_objects":1177,"num_object_clones":0,"num_object_copies":3531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4059,"num_write_kb":167584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2659,"num_bytes_recovered":101056512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":106,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.402896+0000","last_change":"2023-10-09T05:56:49.287906+0000","last_active":"2023-10-09T05:59:32.402896+0000","last_peered":"2023-10-09T05:59:32.402896+0000","last_clean":"2023-10-09T05:59:32.402896+0000","last_became_active":"2023-10-09T05:56:49.275081+0000","last_became_peered":"2023-10-09T05:56:49.275081+0000","last_unstale":"2023-10-09T05:59:32.402896+0000","last_undegraded":"2023-10-09T05:59:32.402896+0000","last_fullsized":"2023-10-09T05:59:32.402896+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":160,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.510627+0000","last_change":"2023-10-09T05:56:00.225724+0000","last_active":"2023-10-09T05:59:32.510627+0000","last_peered":"2023-10-09T05:59:32.510627+0000","last_clean":"2023-10-09T05:59:32.510627+0000","last_became_active":"2023-10-09T05:56:00.225335+0000","last_became_peered":"2023-10-09T05:56:00.225335+0000","last_unstale":"2023-10-09T05:59:32.510627+0000","last_undegraded":"2023-10-09T05:59:32.510627+0000","last_fullsized":"2023-10-09T05:59:32.510627+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:41.405222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"472'8293","reported_seq":8432,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:42.485823+0000","last_change":"2023-10-09T05:57:58.501288+0000","last_active":"2023-10-09T05:59:42.485823+0000","last_peered":"2023-10-09T05:59:42.485823+0000","last_clean":"2023-10-09T05:55:49.154347+0000","last_became_active":"2023-10-09T05:57:58.384024+0000","last_became_peered":"2023-10-09T05:57:58.384024+0000","last_unstale":"2023-10-09T05:59:42.485823+0000","last_undegraded":"2023-10-09T05:57:58.373205+0000","last_fullsized":"2023-10-09T05:57:58.246583+0000","mapping_epoch":444,"log_start":"380'4302","ondisk_log_start":"380'4302","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":3991,"log_dups_size":0,"ondisk_log_size":3991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:58.015603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156827648,"num_objects":2393,"num_object_clones":0,"num_object_copies":7179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5436,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8293,"num_write_kb":341952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[1,3],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,3","objects":-848},{"shards":"1,3,5","objects":2393}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":900,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.401712+0000","last_change":"2023-10-09T05:56:12.339160+0000","last_active":"2023-10-09T05:59:32.401712+0000","last_peered":"2023-10-09T05:59:32.401712+0000","last_clean":"2023-10-09T05:55:22.199269+0000","last_became_active":"2023-10-09T05:56:12.338771+0000","last_became_peered":"2023-10-09T05:56:12.338771+0000","last_unstale":"2023-10-09T05:59:32.401712+0000","last_undegraded":"2023-10-09T05:56:12.324800+0000","last_fullsized":"2023-10-09T05:59:32.401712+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":455,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.405353+0000","last_change":"2023-10-09T05:58:07.410383+0000","last_active":"2023-10-09T05:59:32.405353+0000","last_peered":"2023-10-09T05:59:32.405353+0000","last_clean":"2023-10-09T05:59:32.405353+0000","last_became_active":"2023-10-09T05:58:07.410041+0000","last_became_peered":"2023-10-09T05:58:07.410041+0000","last_unstale":"2023-10-09T05:59:32.405353+0000","last_undegraded":"2023-10-09T05:59:32.405353+0000","last_fullsized":"2023-10-09T05:59:32.405353+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:30.848859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":55,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.404228+0000","last_change":"2023-10-09T05:56:49.255402+0000","last_active":"2023-10-09T05:59:32.404228+0000","last_peered":"2023-10-09T05:59:32.404228+0000","last_clean":"2023-10-09T05:59:32.404228+0000","last_became_active":"2023-10-09T05:56:49.255103+0000","last_became_peered":"2023-10-09T05:56:49.255103+0000","last_unstale":"2023-10-09T05:59:32.404228+0000","last_undegraded":"2023-10-09T05:59:32.404228+0000","last_fullsized":"2023-10-09T05:59:32.404228+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:32.251375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"472'6426","reported_seq":8664,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:39.856782+0000","last_change":"2023-10-09T05:58:20.270966+0000","last_active":"2023-10-09T05:59:39.856782+0000","last_peered":"2023-10-09T05:59:39.856782+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T05:58:15.670784+0000","last_became_peered":"2023-10-09T05:58:15.670784+0000","last_unstale":"2023-10-09T05:59:39.856782+0000","last_undegraded":"2023-10-09T05:58:09.828556+0000","last_fullsized":"2023-10-09T05:58:08.235951+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6426,"log_dups_size":0,"ondisk_log_size":6426,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78479360,"num_objects":1197,"num_object_clones":0,"num_object_copies":3591,"num_objects_missing_on_primary":1639,"num_objects_missing":1639,"num_objects_degraded":6073,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4019,"num_write_kb":166944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54951936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1190},{"shards":"0,5,7","objects":3},{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":44,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.415475+0000","last_change":"2023-10-09T05:58:07.426001+0000","last_active":"2023-10-09T05:59:32.415475+0000","last_peered":"2023-10-09T05:59:32.415475+0000","last_clean":"2023-10-09T05:48:52.714098+0000","last_became_active":"2023-10-09T05:58:07.425667+0000","last_became_peered":"2023-10-09T05:58:07.425667+0000","last_unstale":"2023-10-09T05:59:32.415475+0000","last_undegraded":"2023-10-09T05:58:07.383477+0000","last_fullsized":"2023-10-09T05:59:32.415475+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.403885+0000","last_change":"2023-10-09T05:59:31.411700+0000","last_active":"2023-10-09T05:59:32.403885+0000","last_peered":"2023-10-09T05:59:32.403885+0000","last_clean":"2023-10-09T05:59:32.403885+0000","last_became_active":"2023-10-09T05:59:31.411349+0000","last_became_peered":"2023-10-09T05:59:31.411349+0000","last_unstale":"2023-10-09T05:59:32.403885+0000","last_undegraded":"2023-10-09T05:59:32.403885+0000","last_fullsized":"2023-10-09T05:59:32.403885+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:08.374653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":317,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.404946+0000","last_change":"2023-10-09T05:59:31.402401+0000","last_active":"2023-10-09T05:59:32.404946+0000","last_peered":"2023-10-09T05:59:32.404946+0000","last_clean":"2023-10-09T05:59:32.404946+0000","last_became_active":"2023-10-09T05:59:31.402040+0000","last_became_peered":"2023-10-09T05:59:31.402040+0000","last_unstale":"2023-10-09T05:59:32.404946+0000","last_undegraded":"2023-10-09T05:59:32.404946+0000","last_fullsized":"2023-10-09T05:59:32.404946+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:45.087787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"472'8125","reported_seq":18917,"reported_epoch":472,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:59:39.903392+0000","last_change":"2023-10-09T05:59:32.537556+0000","last_active":"2023-10-09T05:59:39.903392+0000","last_peered":"2023-10-09T05:59:39.903392+0000","last_clean":"2023-10-09T05:59:30.366736+0000","last_became_active":"2023-10-09T05:59:32.493000+0000","last_became_peered":"2023-10-09T05:59:32.493000+0000","last_unstale":"2023-10-09T05:59:39.903392+0000","last_undegraded":"2023-10-09T05:59:39.903392+0000","last_fullsized":"2023-10-09T05:59:39.903392+0000","mapping_epoch":470,"log_start":"379'3876","ondisk_log_start":"379'3876","created":364,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4249,"log_dups_size":0,"ondisk_log_size":4249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162594816,"num_objects":2481,"num_object_clones":0,"num_object_copies":7443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2481,"num_objects_unfound":0,"num_objects_dirty":2481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8125,"num_write_kb":339392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5309,"num_bytes_recovered":309723136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,3],"avail_no_missing":["7","1","3","5"],"object_location_counts":[{"shards":"1,3,7","objects":2481}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":819,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.404199+0000","last_change":"2023-10-09T05:56:11.337498+0000","last_active":"2023-10-09T05:59:32.404199+0000","last_peered":"2023-10-09T05:59:32.404199+0000","last_clean":"2023-10-09T05:56:02.545242+0000","last_became_active":"2023-10-09T05:56:11.336951+0000","last_became_peered":"2023-10-09T05:56:11.336951+0000","last_unstale":"2023-10-09T05:59:32.404199+0000","last_undegraded":"2023-10-09T05:56:11.323856+0000","last_fullsized":"2023-10-09T05:59:32.404199+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:54.695500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":["2","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":265,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.411308+0000","last_change":"2023-10-09T05:58:07.446709+0000","last_active":"2023-10-09T05:59:32.411308+0000","last_peered":"2023-10-09T05:59:32.411308+0000","last_clean":"2023-10-09T05:59:32.411308+0000","last_became_active":"2023-10-09T05:58:07.446340+0000","last_became_peered":"2023-10-09T05:58:07.446340+0000","last_unstale":"2023-10-09T05:59:32.411308+0000","last_undegraded":"2023-10-09T05:59:32.411308+0000","last_fullsized":"2023-10-09T05:59:32.411308+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:37.041432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":281,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.402343+0000","last_change":"2023-10-09T05:57:57.269800+0000","last_active":"2023-10-09T05:59:32.402343+0000","last_peered":"2023-10-09T05:59:32.402343+0000","last_clean":"2023-10-09T05:59:32.402343+0000","last_became_active":"2023-10-09T05:57:57.268007+0000","last_became_peered":"2023-10-09T05:57:57.268007+0000","last_unstale":"2023-10-09T05:59:32.402343+0000","last_undegraded":"2023-10-09T05:59:32.402343+0000","last_fullsized":"2023-10-09T05:59:32.402343+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.8","version":"472'6411","reported_seq":17056,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-10-09T05:59:39.626138+0000","last_change":"2023-10-09T05:58:36.300511+0000","last_active":"2023-10-09T05:59:39.626138+0000","last_peered":"2023-10-09T05:59:39.626138+0000","last_clean":"2023-10-09T05:59:39.626138+0000","last_became_active":"2023-10-09T05:58:36.279322+0000","last_became_peered":"2023-10-09T05:58:36.279322+0000","last_unstale":"2023-10-09T05:59:39.626138+0000","last_undegraded":"2023-10-09T05:59:39.626138+0000","last_fullsized":"2023-10-09T05:59:39.626138+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6411,"log_dups_size":0,"ondisk_log_size":6411,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81002496,"num_objects":1236,"num_object_clones":0,"num_object_copies":3708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3998,"num_write_kb":167488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2749,"num_bytes_recovered":132644864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":685,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:32.408669+0000","last_change":"2023-10-09T05:58:07.894313+0000","last_active":"2023-10-09T05:59:32.408669+0000","last_peered":"2023-10-09T05:59:32.408669+0000","last_clean":"2023-10-09T05:55:22.245899+0000","last_became_active":"2023-10-09T05:58:05.515736+0000","last_became_peered":"2023-10-09T05:58:05.515736+0000","last_unstale":"2023-10-09T05:59:32.408669+0000","last_undegraded":"2023-10-09T05:57:58.245342+0000","last_fullsized":"2023-10-09T05:57:58.244553+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:13.491224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":887,"num_read_kb":802,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":618,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:32.403182+0000","last_change":"2023-10-09T05:58:08.513222+0000","last_active":"2023-10-09T05:59:32.403182+0000","last_peered":"2023-10-09T05:59:32.403182+0000","last_clean":"2023-10-09T05:56:02.257688+0000","last_became_active":"2023-10-09T05:58:08.512938+0000","last_became_peered":"2023-10-09T05:58:08.512938+0000","last_unstale":"2023-10-09T05:59:32.403182+0000","last_undegraded":"2023-10-09T05:58:08.236858+0000","last_fullsized":"2023-10-09T05:58:08.236036+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1],"avail_no_missing":["7","0","6"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":341,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.405691+0000","last_change":"2023-10-09T05:58:07.387935+0000","last_active":"2023-10-09T05:59:32.405691+0000","last_peered":"2023-10-09T05:59:32.405691+0000","last_clean":"2023-10-09T05:59:32.405691+0000","last_became_active":"2023-10-09T05:58:07.386962+0000","last_became_peered":"2023-10-09T05:58:07.386962+0000","last_unstale":"2023-10-09T05:59:32.405691+0000","last_undegraded":"2023-10-09T05:59:32.405691+0000","last_fullsized":"2023-10-09T05:59:32.405691+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:02.334925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":84,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.402259+0000","last_change":"2023-10-09T05:58:07.410680+0000","last_active":"2023-10-09T05:59:32.402259+0000","last_peered":"2023-10-09T05:59:32.402259+0000","last_clean":"2023-10-09T05:59:32.402259+0000","last_became_active":"2023-10-09T05:58:07.409894+0000","last_became_peered":"2023-10-09T05:58:07.409894+0000","last_unstale":"2023-10-09T05:59:32.402259+0000","last_undegraded":"2023-10-09T05:59:32.402259+0000","last_fullsized":"2023-10-09T05:59:32.402259+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"472'8240","reported_seq":8463,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:40.673550+0000","last_change":"2023-10-09T05:58:09.628087+0000","last_active":"2023-10-09T05:59:40.673550+0000","last_peered":"2023-10-09T05:59:40.673550+0000","last_clean":"2023-10-09T05:56:10.073783+0000","last_became_active":"2023-10-09T05:58:09.541128+0000","last_became_peered":"2023-10-09T05:58:09.541128+0000","last_unstale":"2023-10-09T05:59:40.673550+0000","last_undegraded":"2023-10-09T05:58:09.535378+0000","last_fullsized":"2023-10-09T05:58:09.236027+0000","mapping_epoch":449,"log_start":"368'1000","ondisk_log_start":"368'1000","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7240,"log_dups_size":0,"ondisk_log_size":7240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:19.154205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155451392,"num_objects":2372,"num_object_clones":0,"num_object_copies":7116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2372,"num_objects_misplaced":7633,"num_objects_unfound":0,"num_objects_dirty":2372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8240,"num_write_kb":339584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":2949120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[0,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":2372},{"shards":"0,6","objects":-833}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":990,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:32.411071+0000","last_change":"2023-10-09T05:59:32.411071+0000","last_active":"2023-10-09T05:59:32.411071+0000","last_peered":"2023-10-09T05:59:32.411071+0000","last_clean":"2023-10-09T05:56:02.257755+0000","last_became_active":"2023-10-09T05:59:32.410596+0000","last_became_peered":"2023-10-09T05:59:32.410596+0000","last_unstale":"2023-10-09T05:59:32.411071+0000","last_undegraded":"2023-10-09T05:59:32.401690+0000","last_fullsized":"2023-10-09T05:59:32.400900+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":61,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.421471+0000","last_change":"2023-10-09T05:57:57.261895+0000","last_active":"2023-10-09T05:59:32.421471+0000","last_peered":"2023-10-09T05:59:32.421471+0000","last_clean":"2023-10-09T05:59:32.421471+0000","last_became_active":"2023-10-09T05:57:57.261556+0000","last_became_peered":"2023-10-09T05:57:57.261556+0000","last_unstale":"2023-10-09T05:59:32.421471+0000","last_undegraded":"2023-10-09T05:59:32.421471+0000","last_fullsized":"2023-10-09T05:59:32.421471+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":350,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.408428+0000","last_change":"2023-10-09T05:56:11.340751+0000","last_active":"2023-10-09T05:59:32.408428+0000","last_peered":"2023-10-09T05:59:32.408428+0000","last_clean":"2023-10-09T05:59:32.408428+0000","last_became_active":"2023-10-09T05:56:11.340499+0000","last_became_peered":"2023-10-09T05:56:11.340499+0000","last_unstale":"2023-10-09T05:59:32.408428+0000","last_undegraded":"2023-10-09T05:59:32.408428+0000","last_fullsized":"2023-10-09T05:59:32.408428+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:36.120602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"472'8133","reported_seq":12654,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:42.504544+0000","last_change":"2023-10-09T05:59:32.621465+0000","last_active":"2023-10-09T05:59:42.504544+0000","last_peered":"2023-10-09T05:59:42.504544+0000","last_clean":"2023-10-09T05:53:45.896176+0000","last_became_active":"2023-10-09T05:59:32.569578+0000","last_became_peered":"2023-10-09T05:59:32.569578+0000","last_unstale":"2023-10-09T05:59:42.504544+0000","last_undegraded":"2023-10-09T05:59:32.503770+0000","last_fullsized":"2023-10-09T05:59:32.400407+0000","mapping_epoch":470,"log_start":"380'4138","ondisk_log_start":"380'4138","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":3995,"log_dups_size":0,"ondisk_log_size":3995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:59:27.961543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151453696,"num_objects":2311,"num_object_clones":0,"num_object_copies":6933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2311,"num_objects_misplaced":3287,"num_objects_unfound":0,"num_objects_dirty":2311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8133,"num_write_kb":334208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2180,"num_bytes_recovered":142606336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2311},{"shards":"1,4","objects":-43}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":638,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:32.424052+0000","last_change":"2023-10-09T05:56:12.339234+0000","last_active":"2023-10-09T05:59:32.424052+0000","last_peered":"2023-10-09T05:59:32.424052+0000","last_clean":"2023-10-09T05:56:02.265149+0000","last_became_active":"2023-10-09T05:56:12.338651+0000","last_became_peered":"2023-10-09T05:56:12.338651+0000","last_unstale":"2023-10-09T05:59:32.424052+0000","last_undegraded":"2023-10-09T05:56:12.325644+0000","last_fullsized":"2023-10-09T05:56:12.324625+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":292,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.405092+0000","last_change":"2023-10-09T05:58:07.414234+0000","last_active":"2023-10-09T05:59:32.405092+0000","last_peered":"2023-10-09T05:59:32.405092+0000","last_clean":"2023-10-09T05:59:32.405092+0000","last_became_active":"2023-10-09T05:58:07.413481+0000","last_became_peered":"2023-10-09T05:58:07.413481+0000","last_unstale":"2023-10-09T05:59:32.405092+0000","last_undegraded":"2023-10-09T05:59:32.405092+0000","last_fullsized":"2023-10-09T05:59:32.405092+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:13.780593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":431,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.401520+0000","last_change":"2023-10-09T05:57:57.268263+0000","last_active":"2023-10-09T05:59:32.401520+0000","last_peered":"2023-10-09T05:59:32.401520+0000","last_clean":"2023-10-09T05:59:32.401520+0000","last_became_active":"2023-10-09T05:57:57.267885+0000","last_became_peered":"2023-10-09T05:57:57.267885+0000","last_unstale":"2023-10-09T05:59:32.401520+0000","last_undegraded":"2023-10-09T05:59:32.401520+0000","last_fullsized":"2023-10-09T05:59:32.401520+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"472'8082","reported_seq":11045,"reported_epoch":472,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:59:39.856021+0000","last_change":"2023-10-09T05:58:24.310761+0000","last_active":"2023-10-09T05:59:39.856021+0000","last_peered":"2023-10-09T05:59:39.856021+0000","last_clean":"2023-10-09T05:53:45.883404+0000","last_became_active":"2023-10-09T05:58:23.671545+0000","last_became_peered":"2023-10-09T05:58:23.671545+0000","last_unstale":"2023-10-09T05:59:39.856021+0000","last_undegraded":"2023-10-09T05:58:17.604954+0000","last_fullsized":"2023-10-09T05:59:39.856021+0000","mapping_epoch":448,"log_start":"379'3796","ondisk_log_start":"379'3796","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4286,"log_dups_size":0,"ondisk_log_size":4286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156434456,"num_objects":2388,"num_object_clones":0,"num_object_copies":7164,"num_objects_missing_on_primary":3238,"num_objects_missing":3238,"num_objects_degraded":4280,"num_objects_misplaced":3238,"num_objects_unfound":0,"num_objects_dirty":2388,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8082,"num_write_kb":334977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":91881472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,4,5","objects":2308},{"shards":"1,5,7","objects":8},{"shards":"4,5,7","objects":72}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":129,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.402815+0000","last_change":"2023-10-09T05:57:57.269997+0000","last_active":"2023-10-09T05:59:32.402815+0000","last_peered":"2023-10-09T05:59:32.402815+0000","last_clean":"2023-10-09T05:57:53.238744+0000","last_became_active":"2023-10-09T05:57:57.268391+0000","last_became_peered":"2023-10-09T05:57:57.268391+0000","last_unstale":"2023-10-09T05:59:32.402815+0000","last_undegraded":"2023-10-09T05:57:57.242535+0000","last_fullsized":"2023-10-09T05:59:32.402815+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["7","0","3","5"],"object_location_counts":[{"shards":"0,3,5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":575,"reported_epoch":471,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:59:32.508382+0000","last_change":"2023-10-09T05:59:32.508382+0000","last_active":"2023-10-09T05:59:32.508382+0000","last_peered":"2023-10-09T05:59:32.508382+0000","last_clean":"2023-10-09T05:58:01.663036+0000","last_became_active":"2023-10-09T05:59:32.507750+0000","last_became_peered":"2023-10-09T05:59:32.507750+0000","last_unstale":"2023-10-09T05:59:32.508382+0000","last_undegraded":"2023-10-09T05:59:32.400411+0000","last_fullsized":"2023-10-09T05:59:32.508382+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:34.806870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[1,4,3],"avail_no_missing":["1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":133,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.403444+0000","last_change":"2023-10-09T05:58:07.397986+0000","last_active":"2023-10-09T05:59:32.403444+0000","last_peered":"2023-10-09T05:59:32.403444+0000","last_clean":"2023-10-09T05:59:32.403444+0000","last_became_active":"2023-10-09T05:58:07.397576+0000","last_became_peered":"2023-10-09T05:58:07.397576+0000","last_unstale":"2023-10-09T05:59:32.403444+0000","last_undegraded":"2023-10-09T05:59:32.403444+0000","last_fullsized":"2023-10-09T05:59:32.403444+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"472'8165","reported_seq":14362,"reported_epoch":472,"state":"active+recovering","last_fresh":"2023-10-09T05:59:39.849974+0000","last_change":"2023-10-09T05:56:49.370077+0000","last_active":"2023-10-09T05:59:39.849974+0000","last_peered":"2023-10-09T05:59:39.849974+0000","last_clean":"2023-10-09T05:53:45.907519+0000","last_became_active":"2023-10-09T05:56:03.326563+0000","last_became_peered":"2023-10-09T05:56:03.326563+0000","last_unstale":"2023-10-09T05:59:39.849974+0000","last_undegraded":"2023-10-09T05:59:39.849974+0000","last_fullsized":"2023-10-09T05:59:39.849974+0000","mapping_epoch":416,"log_start":"379'3871","ondisk_log_start":"379'3871","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4294,"log_dups_size":0,"ondisk_log_size":4294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158924800,"num_objects":2425,"num_object_clones":0,"num_object_copies":7275,"num_objects_missing_on_primary":984,"num_objects_missing":984,"num_objects_degraded":0,"num_objects_misplaced":35,"num_objects_unfound":0,"num_objects_dirty":2425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8165,"num_write_kb":338880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3017,"num_bytes_recovered":197656576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["4"],"object_location_counts":[{"shards":"0,3,4","objects":35},{"shards":"0,3,4,7","objects":1},{"shards":"3,4,7","objects":2389}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":650,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:32.402470+0000","last_change":"2023-10-09T05:58:08.253520+0000","last_active":"2023-10-09T05:59:32.402470+0000","last_peered":"2023-10-09T05:59:32.402470+0000","last_clean":"2023-10-09T05:58:02.145081+0000","last_became_active":"2023-10-09T05:58:08.253168+0000","last_became_peered":"2023-10-09T05:58:08.253168+0000","last_unstale":"2023-10-09T05:59:32.402470+0000","last_undegraded":"2023-10-09T05:58:08.242171+0000","last_fullsized":"2023-10-09T05:58:08.239234+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":66,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.402262+0000","last_change":"2023-10-09T05:56:48.410914+0000","last_active":"2023-10-09T05:59:32.402262+0000","last_peered":"2023-10-09T05:59:32.402262+0000","last_clean":"2023-10-09T05:59:32.402262+0000","last_became_active":"2023-10-09T05:56:11.349646+0000","last_became_peered":"2023-10-09T05:56:11.349646+0000","last_unstale":"2023-10-09T05:59:32.402262+0000","last_undegraded":"2023-10-09T05:59:32.402262+0000","last_fullsized":"2023-10-09T05:59:32.402262+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.421411+0000","last_change":"2023-10-09T05:58:07.427135+0000","last_active":"2023-10-09T05:59:32.421411+0000","last_peered":"2023-10-09T05:59:32.421411+0000","last_clean":"2023-10-09T05:59:32.421411+0000","last_became_active":"2023-10-09T05:58:07.426364+0000","last_became_peered":"2023-10-09T05:58:07.426364+0000","last_unstale":"2023-10-09T05:59:32.421411+0000","last_undegraded":"2023-10-09T05:59:32.421411+0000","last_fullsized":"2023-10-09T05:59:32.421411+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.11","version":"472'6516","reported_seq":15628,"reported_epoch":472,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:59:39.765949+0000","last_change":"2023-10-09T05:58:20.369743+0000","last_active":"2023-10-09T05:59:39.765949+0000","last_peered":"2023-10-09T05:59:39.765949+0000","last_clean":"2023-10-09T05:58:02.997585+0000","last_became_active":"2023-10-09T05:58:15.913938+0000","last_became_peered":"2023-10-09T05:58:15.913938+0000","last_unstale":"2023-10-09T05:59:39.765949+0000","last_undegraded":"2023-10-09T05:59:39.765949+0000","last_fullsized":"2023-10-09T05:59:39.765949+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6516,"log_dups_size":0,"ondisk_log_size":6516,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81821696,"num_objects":1249,"num_object_clones":0,"num_object_copies":3747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1249,"num_objects_unfound":0,"num_objects_dirty":1249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4063,"num_write_kb":169952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3255,"num_bytes_recovered":213286912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,6],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":1249}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":462,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.401521+0000","last_change":"2023-10-09T05:56:49.265734+0000","last_active":"2023-10-09T05:59:32.401521+0000","last_peered":"2023-10-09T05:59:32.401521+0000","last_clean":"2023-10-09T05:59:32.401521+0000","last_became_active":"2023-10-09T05:56:49.264595+0000","last_became_peered":"2023-10-09T05:56:49.264595+0000","last_unstale":"2023-10-09T05:59:32.401521+0000","last_undegraded":"2023-10-09T05:59:32.401521+0000","last_fullsized":"2023-10-09T05:59:32.401521+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":164,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.405015+0000","last_change":"2023-10-09T05:56:00.226695+0000","last_active":"2023-10-09T05:59:32.405015+0000","last_peered":"2023-10-09T05:59:32.405015+0000","last_clean":"2023-10-09T05:59:32.405015+0000","last_became_active":"2023-10-09T05:56:00.226359+0000","last_became_peered":"2023-10-09T05:56:00.226359+0000","last_unstale":"2023-10-09T05:59:32.405015+0000","last_undegraded":"2023-10-09T05:59:32.405015+0000","last_fullsized":"2023-10-09T05:59:32.405015+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:20.323704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"472'6452","reported_seq":6600,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:40.229214+0000","last_change":"2023-10-09T05:58:07.639997+0000","last_active":"2023-10-09T05:59:40.229214+0000","last_peered":"2023-10-09T05:59:40.229214+0000","last_clean":"2023-10-09T05:57:55.523800+0000","last_became_active":"2023-10-09T05:58:07.626014+0000","last_became_peered":"2023-10-09T05:58:07.626014+0000","last_unstale":"2023-10-09T05:59:40.229214+0000","last_undegraded":"2023-10-09T05:58:07.410678+0000","last_fullsized":"2023-10-09T05:58:07.367850+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6452,"log_dups_size":0,"ondisk_log_size":6452,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75694080,"num_objects":1155,"num_object_clones":0,"num_object_copies":3465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2624,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4053,"num_write_kb":166656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","5"],"object_location_counts":[{"shards":"1,2","objects":-415},{"shards":"1,2,5","objects":1175}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":32,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.401459+0000","last_change":"2023-10-09T05:58:07.399584+0000","last_active":"2023-10-09T05:59:32.401459+0000","last_peered":"2023-10-09T05:59:32.401459+0000","last_clean":"2023-10-09T05:48:52.714337+0000","last_became_active":"2023-10-09T05:58:07.399203+0000","last_became_peered":"2023-10-09T05:58:07.399203+0000","last_unstale":"2023-10-09T05:59:32.401459+0000","last_undegraded":"2023-10-09T05:58:07.381974+0000","last_fullsized":"2023-10-09T05:59:32.401459+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["1","4","5","6"],"object_location_counts":[{"shards":"1,4,5,6","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":54,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.408437+0000","last_change":"2023-10-09T05:56:49.252225+0000","last_active":"2023-10-09T05:59:32.408437+0000","last_peered":"2023-10-09T05:59:32.408437+0000","last_clean":"2023-10-09T05:59:32.408437+0000","last_became_active":"2023-10-09T05:56:49.251880+0000","last_became_peered":"2023-10-09T05:56:49.251880+0000","last_unstale":"2023-10-09T05:59:32.408437+0000","last_undegraded":"2023-10-09T05:59:32.408437+0000","last_fullsized":"2023-10-09T05:59:32.408437+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:34.079170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.17","version":"472'6568","reported_seq":13167,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:39.623860+0000","last_change":"2023-10-09T05:58:02.382245+0000","last_active":"2023-10-09T05:59:39.623860+0000","last_peered":"2023-10-09T05:59:39.623860+0000","last_clean":"2023-10-09T05:57:55.399317+0000","last_became_active":"2023-10-09T05:58:02.319859+0000","last_became_peered":"2023-10-09T05:58:02.319859+0000","last_unstale":"2023-10-09T05:59:39.623860+0000","last_undegraded":"2023-10-09T05:58:01.234664+0000","last_fullsized":"2023-10-09T05:58:01.215806+0000","mapping_epoch":444,"log_start":"367'6","ondisk_log_start":"367'6","created":385,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6562,"log_dups_size":0,"ondisk_log_size":6562,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72876032,"num_objects":1112,"num_object_clones":0,"num_object_copies":3336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2713,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4144,"num_write_kb":168192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3182,"num_bytes_recovered":157876224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["7","2","3","5"],"object_location_counts":[{"shards":"2,5,7","objects":1197},{"shards":"2,7","objects":-497}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":29,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.402016+0000","last_change":"2023-10-09T05:58:07.410770+0000","last_active":"2023-10-09T05:59:32.402016+0000","last_peered":"2023-10-09T05:59:32.402016+0000","last_clean":"2023-10-09T05:21:17.840829+0000","last_became_active":"2023-10-09T05:58:07.410323+0000","last_became_peered":"2023-10-09T05:58:07.410323+0000","last_unstale":"2023-10-09T05:59:32.402016+0000","last_undegraded":"2023-10-09T05:58:07.390791+0000","last_fullsized":"2023-10-09T05:59:32.402016+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["1","4","5","6"],"object_location_counts":[{"shards":"1,4,5,6","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":64,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.404300+0000","last_change":"2023-10-09T05:56:11.337104+0000","last_active":"2023-10-09T05:59:32.404300+0000","last_peered":"2023-10-09T05:59:32.404300+0000","last_clean":"2023-10-09T05:59:32.404300+0000","last_became_active":"2023-10-09T05:56:11.336690+0000","last_became_peered":"2023-10-09T05:56:11.336690+0000","last_unstale":"2023-10-09T05:59:32.404300+0000","last_undegraded":"2023-10-09T05:59:32.404300+0000","last_fullsized":"2023-10-09T05:59:32.404300+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:56.369633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.14","version":"472'6515","reported_seq":6665,"reported_epoch":472,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:42.286898+0000","last_change":"2023-10-09T05:56:01.281023+0000","last_active":"2023-10-09T05:59:42.286898+0000","last_peered":"2023-10-09T05:59:42.286898+0000","last_clean":"2023-10-09T05:55:48.964110+0000","last_became_active":"2023-10-09T05:56:01.245699+0000","last_became_peered":"2023-10-09T05:56:01.245699+0000","last_unstale":"2023-10-09T05:59:42.286898+0000","last_undegraded":"2023-10-09T05:56:01.216362+0000","last_fullsized":"2023-10-09T05:59:42.286898+0000","mapping_epoch":416,"log_start":"367'260","ondisk_log_start":"367'260","created":385,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6255,"log_dups_size":0,"ondisk_log_size":6255,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77922304,"num_objects":1189,"num_object_clones":0,"num_object_copies":3567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":40,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4085,"num_write_kb":168768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1189}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":221,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.408327+0000","last_change":"2023-10-09T05:58:07.415396+0000","last_active":"2023-10-09T05:59:32.408327+0000","last_peered":"2023-10-09T05:59:32.408327+0000","last_clean":"2023-10-09T05:59:32.408327+0000","last_became_active":"2023-10-09T05:58:07.414511+0000","last_became_peered":"2023-10-09T05:58:07.414511+0000","last_unstale":"2023-10-09T05:59:32.408327+0000","last_undegraded":"2023-10-09T05:59:32.408327+0000","last_fullsized":"2023-10-09T05:59:32.408327+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:10.158875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":321,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.404358+0000","last_change":"2023-10-09T05:59:31.411306+0000","last_active":"2023-10-09T05:59:32.404358+0000","last_peered":"2023-10-09T05:59:32.404358+0000","last_clean":"2023-10-09T05:59:32.404358+0000","last_became_active":"2023-10-09T05:59:31.410962+0000","last_became_peered":"2023-10-09T05:59:31.410962+0000","last_unstale":"2023-10-09T05:59:32.404358+0000","last_undegraded":"2023-10-09T05:59:32.404358+0000","last_fullsized":"2023-10-09T05:59:32.404358+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:20.164431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"472'6534","reported_seq":8835,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:59:39.892682+0000","last_change":"2023-10-09T05:58:05.481038+0000","last_active":"2023-10-09T05:59:39.892682+0000","last_peered":"2023-10-09T05:59:39.892682+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T05:58:04.624669+0000","last_became_peered":"2023-10-09T05:58:04.624669+0000","last_unstale":"2023-10-09T05:59:39.892682+0000","last_undegraded":"2023-10-09T05:58:01.427162+0000","last_fullsized":"2023-10-09T05:57:58.378378+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6534,"log_dups_size":0,"ondisk_log_size":6534,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80117760,"num_objects":1223,"num_object_clones":0,"num_object_copies":3669,"num_objects_missing_on_primary":1609,"num_objects_missing":1609,"num_objects_degraded":4729,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4093,"num_write_kb":170080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":691,"num_bytes_recovered":45285376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":1127},{"shards":"0,1,5,7","objects":106},{"shards":"0,5,7","objects":4},{"shards":"1,7","objects":-14}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":401,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:59:32.407883+0000","last_change":"2023-10-09T05:58:07.410502+0000","last_active":"2023-10-09T05:59:32.407883+0000","last_peered":"2023-10-09T05:59:32.407883+0000","last_clean":"2023-10-09T05:57:53.237962+0000","last_became_active":"2023-10-09T05:58:07.410010+0000","last_became_peered":"2023-10-09T05:58:07.410010+0000","last_unstale":"2023-10-09T05:59:32.407883+0000","last_undegraded":"2023-10-09T05:58:07.372990+0000","last_fullsized":"2023-10-09T05:59:32.407883+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:54.252736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":["2","0","5","6"],"object_location_counts":[{"shards":"0,2,5,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-10-09T05:59:32.403543+0000","last_change":"2023-10-09T05:59:31.411398+0000","last_active":"2023-10-09T05:59:32.403543+0000","last_peered":"2023-10-09T05:59:32.403543+0000","last_clean":"2023-10-09T05:59:32.403543+0000","last_became_active":"2023-10-09T05:59:31.411060+0000","last_became_peered":"2023-10-09T05:59:31.411060+0000","last_unstale":"2023-10-09T05:59:32.403543+0000","last_undegraded":"2023-10-09T05:59:32.403543+0000","last_fullsized":"2023-10-09T05:59:32.403543+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:35.484072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":3,"num_objects_missing":3,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":15826,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17226,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":2505506840,"num_objects":38232,"num_object_clones":0,"num_object_copies":114696,"num_objects_missing_on_primary":11515,"num_objects_missing":11515,"num_objects_degraded":61637,"num_objects_misplaced":19140,"num_objects_unfound":0,"num_objects_dirty":38232,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":130110,"num_write_kb":5386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41764,"num_bytes_recovered":2371420160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9736761344,"data_stored":9736749128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156412,"ondisk_log_size":156412,"up":78,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":20,"num_objects_missing":20,"num_objects_degraded":98,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13211,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":98,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846888,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2184372,"kb_used_data":1266348,"kb_used_omap":14,"kb_used_meta":918001,"kb_avail":91560780,"statfs":{"total":95995035648,"available":93758238720,"internally_reserved":0,"allocated":1296740352,"data_stored":1295740135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":940033882},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.875,"15min":4.1150000000000002},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.20699999999999999},"max":{"1min":5.0960000000000001,"5min":117.756,"15min":212.64400000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.1299999999999999,"5min":2.0510000000000002,"15min":4.1609999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.245},"max":{"1min":4.6769999999999996,"5min":117.73,"15min":212.655},"last":3.7530000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":4.2720000000000002,"5min":2.5430000000000001,"15min":2.1699999999999999},"min":{"1min":0.33800000000000002,"5min":0.26900000000000002,"15min":0.24199999999999999},"max":{"1min":55.212000000000003,"5min":78.013000000000005,"15min":114.422},"last":0.317},{"interface":"front","average":{"1min":4.0229999999999997,"5min":2.456,"15min":2.2480000000000002},"min":{"1min":0.36299999999999999,"5min":0.26800000000000002,"15min":0.255},"max":{"1min":55.430999999999997,"5min":78.174999999999997,"15min":111.708},"last":0.42499999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.155,"15min":2.444},"min":{"1min":0.30599999999999999,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":8.0559999999999992,"5min":12.047000000000001,"15min":205.863},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.171,"5min":0.86799999999999999,"15min":2.2890000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.28299999999999997},"max":{"1min":5.1909999999999998,"5min":5.5010000000000003,"15min":205.88399999999999},"last":2.2789999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":2.2210000000000001,"15min":3.0840000000000001},"min":{"1min":0.34100000000000003,"5min":0.315,"15min":0.28399999999999997},"max":{"1min":5.2469999999999999,"5min":115.69799999999999,"15min":115.69799999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.292,"5min":2.327,"15min":3.3090000000000002},"min":{"1min":0.40200000000000002,"5min":0.29699999999999999,"15min":0.29299999999999998},"max":{"1min":4.1879999999999997,"5min":115.75700000000001,"15min":115.75700000000001},"last":3.8340000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":0.71699999999999997,"15min":1.958},"min":{"1min":0.29499999999999998,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":4.8390000000000004,"5min":5.6130000000000004,"15min":159.50299999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.2909999999999999,"5min":0.89000000000000001,"15min":2.0030000000000001},"min":{"1min":0.28499999999999998,"5min":0.28199999999999997,"15min":0.27800000000000002},"max":{"1min":4.0899999999999999,"5min":4.9800000000000004,"15min":159.483},"last":3.7170000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":1.9159999999999999,"15min":1.782},"min":{"1min":0.34399999999999997,"5min":0.29399999999999998,"15min":0.27800000000000002},"max":{"1min":8.968,"5min":94.093999999999994,"15min":94.093999999999994},"last":3.802},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.8779999999999999,"15min":1.8560000000000001},"min":{"1min":0.40999999999999998,"5min":0.312,"15min":0.308},"max":{"1min":8.1010000000000009,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.50900000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":3.2109999999999999,"15min":1.663},"min":{"1min":0.40200000000000002,"5min":0.32900000000000001,"15min":0.29299999999999998},"max":{"1min":5.7590000000000003,"5min":209.322,"15min":209.322},"last":3.7799999999999998},{"interface":"front","average":{"1min":2.5249999999999999,"5min":3.3420000000000001,"15min":1.7569999999999999},"min":{"1min":0.35599999999999998,"5min":0.30599999999999999,"15min":0.26700000000000002},"max":{"1min":8.1289999999999996,"5min":209.19800000000001,"15min":209.19800000000001},"last":0.53700000000000003}]}]},{"osd":6,"up_from":0,"seq":180388626919,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":93,"seq":399431958880,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2741360,"kb_used_data":1843368,"kb_used_omap":7,"kb_used_meta":897976,"kb_avail":91003792,"statfs":{"total":95995035648,"available":93187883008,"internally_reserved":0,"allocated":1887608832,"data_stored":1885848816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":919527950},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":2.2919999999999998,"15min":2.8730000000000002},"min":{"1min":0.214,"5min":0.214,"15min":0.17499999999999999},"max":{"1min":6.96,"5min":131.91300000000001,"15min":131.91300000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.46700000000000003,"5min":2.3969999999999998,"15min":3.0920000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.223},"max":{"1min":2.327,"5min":132.191,"15min":132.191},"last":1.956}]},{"osd":2,"last update":"Mon Oct 9 05:58:41 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":2.653,"15min":2.3029999999999999},"min":{"1min":0.23799999999999999,"5min":0.221,"15min":0.19800000000000001},"max":{"1min":1.516,"5min":132.35400000000001,"15min":132.35400000000001},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.751,"5min":2.6070000000000002,"15min":2.2309999999999999},"min":{"1min":0.29399999999999998,"5min":0.23699999999999999,"15min":0.20699999999999999},"max":{"1min":4.1989999999999998,"5min":131.70599999999999,"15min":131.70599999999999},"last":0.67200000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:59:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3180000000000001,"5min":2.7109999999999999,"15min":2.5379999999999998},"min":{"1min":0.27300000000000002,"5min":0.251,"15min":0.20200000000000001},"max":{"1min":8.843,"5min":135.10900000000001,"15min":135.10900000000001},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.98799999999999999,"5min":2.7599999999999998,"15min":2.452},"min":{"1min":0.315,"5min":0.248,"15min":0.248},"max":{"1min":6.6289999999999996,"5min":131.886,"15min":131.886},"last":0.64800000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":2.4489999999999998,"15min":1.7450000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":3.3479999999999999,"5min":131.636,"15min":131.636},"last":2.0219999999999998},{"interface":"front","average":{"1min":1.724,"5min":2.3849999999999998,"15min":1.6140000000000001},"min":{"1min":0.36299999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":9.1010000000000009,"5min":131.75999999999999,"15min":131.75999999999999},"last":0.59999999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:59:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":2.573,"15min":2.3460000000000001},"min":{"1min":0.371,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":8.7300000000000004,"5min":132.291,"15min":132.291},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.627,"5min":2.988,"15min":2.3620000000000001},"min":{"1min":0.36099999999999999,"5min":0.33700000000000002,"15min":0.32200000000000001},"max":{"1min":9.7919999999999998,"5min":131.738,"15min":131.738},"last":1.994}]},{"osd":6,"last update":"Mon Oct 9 05:59:16 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":3.9910000000000001,"15min":2.706},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.316},"max":{"1min":8.7769999999999992,"5min":158.50399999999999,"15min":158.50399999999999},"last":0.749},{"interface":"front","average":{"1min":1.7949999999999999,"5min":4.2050000000000001,"15min":2.6400000000000001},"min":{"1min":0.39100000000000001,"5min":0.377,"15min":0.37},"max":{"1min":9.8559999999999999,"5min":158.577,"15min":158.577},"last":2.0499999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:59:16 2023","interfaces":[{"interface":"back","average":{"1min":5.851,"5min":3.2810000000000001,"15min":2.016},"min":{"1min":0.46000000000000002,"5min":0.31,"15min":0.31},"max":{"1min":89.787000000000006,"5min":131.821,"15min":131.821},"last":0.54400000000000004},{"interface":"front","average":{"1min":5.4299999999999997,"5min":3.4340000000000002,"15min":2.1389999999999998},"min":{"1min":0.47499999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":89.869,"5min":131.852,"15min":131.852},"last":0.622}]}]},{"osd":0,"up_from":0,"seq":377957122404,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":98,"seq":420906795353,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1634556,"kb_used_data":772724,"kb_used_omap":11,"kb_used_meta":861812,"kb_avail":92110596,"statfs":{"total":95995035648,"available":94321250304,"internally_reserved":0,"allocated":791269376,"data_stored":789203322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":882496085},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:02 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":0.88900000000000001,"15min":2.9409999999999998},"min":{"1min":0.20899999999999999,"5min":0.20799999999999999,"15min":0.17999999999999999},"max":{"1min":9.7070000000000007,"5min":9.9100000000000001,"15min":162.285},"last":1.032},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.5249999999999999,"15min":3.3300000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.8340000000000001,"5min":17.109999999999999,"15min":162.643},"last":2.6230000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:59:02 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.7929999999999999,"15min":2.5739999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.193},"max":{"1min":2.7210000000000001,"5min":75.260000000000005,"15min":125.77500000000001},"last":2.6659999999999999},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.968,"15min":2.5550000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.20999999999999999},"max":{"1min":2.452,"5min":75.114999999999995,"15min":125.857},"last":2.3220000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":1.1859999999999999,"15min":2.4140000000000001},"min":{"1min":0.25700000000000001,"5min":0.246,"15min":0.21099999999999999},"max":{"1min":2.351,"5min":15.193,"15min":159.35499999999999},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.079,"15min":2.3340000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.217},"max":{"1min":9.7680000000000007,"5min":9.7680000000000007,"15min":159.196},"last":0.41999999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:59:02 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.181,"15min":1.377},"min":{"1min":0.35099999999999998,"5min":0.32200000000000001,"15min":0.28699999999999998},"max":{"1min":2.4049999999999998,"5min":9.8599999999999994,"15min":68.052000000000007},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.381,"5min":0.89400000000000002,"15min":1.365},"min":{"1min":0.36899999999999999,"5min":0.313,"15min":0.29199999999999998},"max":{"1min":9.5800000000000001,"5min":10.013,"15min":68.727000000000004},"last":0.50700000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:59:02 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":1.575,"15min":2.851},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.29099999999999998},"max":{"1min":2.7589999999999999,"5min":28.794,"15min":207.80500000000001},"last":2.7080000000000002},{"interface":"front","average":{"1min":1.095,"5min":1.7629999999999999,"15min":2.8500000000000001},"min":{"1min":0.38400000000000001,"5min":0.378,"15min":0.30499999999999999},"max":{"1min":3.1949999999999998,"5min":28.584,"15min":207.54900000000001},"last":0.57599999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:59:02 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.073,"15min":1.2789999999999999},"min":{"1min":0.38300000000000001,"5min":0.31,"15min":0.31},"max":{"1min":9.6739999999999995,"5min":9.9589999999999996,"15min":25.818999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.25,"15min":1.5600000000000001},"min":{"1min":0.46000000000000002,"5min":0.41399999999999998,"15min":0.34100000000000003},"max":{"1min":3.4380000000000002,"5min":8.7100000000000009,"15min":24.481000000000002},"last":2.7490000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:59:37 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":2.2229999999999999,"15min":2.3210000000000002},"min":{"1min":0.42999999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":9.6359999999999992,"5min":122.827,"15min":162.09200000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.296,"5min":2.4039999999999999,"15min":2.4940000000000002},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.373},"max":{"1min":5.6619999999999999,"5min":123.56399999999999,"15min":162.35400000000001},"last":2.7850000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631827,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2355428,"kb_used_data":1393756,"kb_used_omap":4,"kb_used_meta":961659,"kb_avail":91389724,"statfs":{"total":95995035648,"available":93583077376,"internally_reserved":0,"allocated":1427206144,"data_stored":1425398183,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":984739422},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":2.6120000000000001,"15min":4.3170000000000002},"min":{"1min":0.22700000000000001,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":12.728,"5min":142.37700000000001,"15min":312.56599999999997},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.1439999999999999,"5min":2.6749999999999998,"15min":4.2789999999999999},"min":{"1min":0.23499999999999999,"5min":0.216,"15min":0.20499999999999999},"max":{"1min":7.6360000000000001,"5min":142.75800000000001,"15min":311.02300000000002},"last":0.30099999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":2.423,"15min":4.0129999999999999},"min":{"1min":0.28000000000000003,"5min":0.24199999999999999,"15min":0.214},"max":{"1min":7.0220000000000002,"5min":142.32599999999999,"15min":150.29599999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.81799999999999995,"5min":2.5249999999999999,"15min":4.2169999999999996},"min":{"1min":0.27500000000000002,"5min":0.23799999999999999,"15min":0.216},"max":{"1min":3.8860000000000001,"5min":139.33199999999999,"15min":151.25700000000001},"last":0.374}]},{"osd":2,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":3.8029999999999999,"5min":1.4079999999999999,"15min":2.359},"min":{"1min":0.29199999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":50.579000000000001,"5min":50.579000000000001,"15min":150.006},"last":0.39300000000000002},{"interface":"front","average":{"1min":3.266,"5min":1.3240000000000001,"15min":2.2749999999999999},"min":{"1min":0.29099999999999998,"5min":0.255,"15min":0.20499999999999999},"max":{"1min":50.628999999999998,"5min":50.628999999999998,"15min":150.65100000000001},"last":0.35899999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.78200000000000003,"15min":2.274},"min":{"1min":0.36199999999999999,"5min":0.35499999999999998,"15min":0.28999999999999998},"max":{"1min":3.278,"5min":4.7510000000000003,"15min":149.84700000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.167,"5min":0.747,"15min":2.613},"min":{"1min":0.39100000000000001,"5min":0.32500000000000001,"15min":0.28999999999999998},"max":{"1min":7.1390000000000002,"5min":7.1390000000000002,"15min":149.74700000000001},"last":1.208}]},{"osd":5,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.0189999999999999,"15min":2.1989999999999998},"min":{"1min":0.36499999999999999,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":4.3540000000000001,"5min":13.380000000000001,"15min":149.43000000000001},"last":0.432},{"interface":"front","average":{"1min":1.8819999999999999,"5min":0.96299999999999997,"15min":2.0600000000000001},"min":{"1min":0.433,"5min":0.33400000000000002,"15min":0.29399999999999998},"max":{"1min":12.778,"5min":12.778,"15min":149.69300000000001},"last":0.46899999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":0.83599999999999997,"15min":1.8480000000000001},"min":{"1min":0.53700000000000003,"5min":0.38200000000000001,"15min":0.34499999999999997},"max":{"1min":7.1719999999999997,"5min":7.1719999999999997,"15min":149.72399999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.0669999999999999,"5min":0.89800000000000002,"15min":2.0049999999999999},"min":{"1min":0.45600000000000002,"5min":0.39400000000000002,"15min":0.33200000000000002},"max":{"1min":3.3279999999999998,"5min":5.7480000000000002,"15min":149.589},"last":0.48199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:58:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.0209999999999999,"15min":1.931},"min":{"1min":0.44,"5min":0.38300000000000001,"15min":0.35699999999999998},"max":{"1min":2.738,"5min":6.4850000000000003,"15min":149.77199999999999},"last":1.375},{"interface":"front","average":{"1min":0.92200000000000004,"5min":1.02,"15min":1.952},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.374},"max":{"1min":5.3019999999999996,"5min":5.3019999999999996,"15min":149.80799999999999},"last":0.72099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019375,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1555056,"kb_used_data":657704,"kb_used_omap":0,"kb_used_meta":897344,"kb_avail":92190096,"statfs":{"total":95995035648,"available":94402658304,"internally_reserved":0,"allocated":673488896,"data_stored":672492185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918880256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.8069999999999999,"15min":3.1099999999999999},"min":{"1min":0.27000000000000002,"5min":0.23000000000000001,"15min":0.219},"max":{"1min":6.0019999999999998,"5min":95.466999999999999,"15min":132.35599999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.363,"5min":3.1819999999999999,"15min":3.508},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.23699999999999999},"max":{"1min":11.028,"5min":95.367999999999995,"15min":132.501},"last":0.439}]},{"osd":1,"last update":"Mon Oct 9 05:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":2.7429999999999999,"15min":2.569},"min":{"1min":0.30599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.0419999999999998,"5min":96.033000000000001,"15min":96.033000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.3720000000000001,"5min":3.0030000000000001,"15min":2.5510000000000002},"min":{"1min":0.41399999999999998,"5min":0.255,"15min":0.255},"max":{"1min":4.5629999999999997,"5min":96.022000000000006,"15min":96.022000000000006},"last":0.41699999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.94199999999999995,"15min":1.6779999999999999},"min":{"1min":0.314,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":1.03,"5min":8.7880000000000003,"15min":51.768000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.788,"15min":1.9419999999999999},"min":{"1min":0.33400000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.496,"5min":84.123999999999995,"15min":84.123999999999995},"last":0.36899999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":3.278,"15min":2.8700000000000001},"min":{"1min":0.33400000000000002,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":2.8650000000000002,"5min":100.009,"15min":123.59},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.3500000000000001,"5min":3.387,"15min":2.8100000000000001},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.30499999999999999},"max":{"1min":5.9729999999999999,"5min":100.254,"15min":123.566},"last":0.40100000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:59:17 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.119,"15min":1.0189999999999999},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":3.0030000000000001,"5min":15.597,"15min":15.597},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.5880000000000001,"5min":1.0820000000000001,"15min":0.96699999999999997},"min":{"1min":0.42299999999999999,"5min":0.33700000000000002,"15min":0.29799999999999999},"max":{"1min":6.0289999999999999,"5min":17.882000000000001,"15min":17.882000000000001},"last":1.0109999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:59:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":1.4930000000000001,"15min":1.625},"min":{"1min":0.40899999999999997,"5min":0.314,"15min":0.27600000000000002},"max":{"1min":6.0549999999999997,"5min":38.676000000000002,"15min":181.411},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.772,"5min":1.504,"15min":1.722},"min":{"1min":0.40899999999999997,"5min":0.32200000000000001,"15min":0.29799999999999999},"max":{"1min":6.3440000000000003,"5min":38.438000000000002,"15min":180.63999999999999},"last":0.46800000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:59:36 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":1.2989999999999999,"15min":1.4419999999999999},"min":{"1min":0.35199999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.0810000000000004,"5min":17.832000000000001,"15min":96.131},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.25,"5min":1.0569999999999999,"15min":1.4039999999999999},"min":{"1min":0.35699999999999998,"5min":0.32900000000000001,"15min":0.27200000000000002},"max":{"1min":4.9950000000000001,"5min":17.742999999999999,"15min":96.465000000000003},"last":0.65300000000000002}]}]},{"osd":5,"up_from":0,"seq":154618823147,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1273757696,"data_stored":1273757696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1879052288,"data_stored":1879048216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":779354112,"data_stored":779354112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1410727936,"data_stored":1410727936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":666570752,"data_stored":666566680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1896353792,"data_stored":1896349720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":548667392,"data_stored":548667392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1282277376,"data_stored":1282277376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:59:46.059 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 61 2023-10-09T05:59:46.374 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:46.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:46.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:46.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:46.510 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:46.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:46 smithi191 ceph-mon[161669]: pgmap v1411: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 203 op/s; 61757/114780 objects degraded (53.805%); 19150/114780 objects misplaced (16.684%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T05:59:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:46 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:46 smithi191 ceph-mon[161669]: failed to encode map e473 with expected crc 2023-10-09T05:59:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1643333815' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:46 smithi191 ceph-mon[161669]: osdmap e473: 8 total, 8 up, 5 in 2023-10-09T05:59:46.557 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:46.559 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:46.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:46.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:46.603 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:46.658 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:46.662 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:46.705 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:46 smithi008 ceph-mon[184418]: pgmap v1411: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 203 op/s; 61757/114780 objects degraded (53.805%); 19150/114780 objects misplaced (16.684%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T05:59:46.705 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:46 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:46.705 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:46 smithi008 ceph-mon[184418]: failed to encode map e473 with expected crc 2023-10-09T05:59:46.705 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1643333815' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:46.705 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:46 smithi008 ceph-mon[184418]: osdmap e473: 8 total, 8 up, 5 in 2023-10-09T05:59:46.705 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:46 smithi008 ceph-mon[191227]: pgmap v1411: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 203 op/s; 61757/114780 objects degraded (53.805%); 19150/114780 objects misplaced (16.684%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T05:59:46.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:46 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:46.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:46 smithi008 ceph-mon[191227]: failed to encode map e473 with expected crc 2023-10-09T05:59:46.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1643333815' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:46.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:46 smithi008 ceph-mon[191227]: osdmap e473: 8 total, 8 up, 5 in 2023-10-09T05:59:46.708 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:46.757 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:46.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:46.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:46.862 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:46.893 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:47.098 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:47.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:47.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:47.202 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:47.234 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:47.305 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:47.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:47.414 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:47.449 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:48.363 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:48.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:48.388 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:59:48.469 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:48.506 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:48 smithi191 ceph-mon[161669]: pgmap v1413: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 256 op/s; 61061/111357 objects degraded (54.834%); 18865/111357 objects misplaced (16.941%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:59:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 61061/111357 objects degraded (54.834%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:48.581 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:48 smithi008 ceph-mon[184418]: pgmap v1413: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 256 op/s; 61061/111357 objects degraded (54.834%); 18865/111357 objects misplaced (16.941%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:59:48.581 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 61061/111357 objects degraded (54.834%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:48.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:48 smithi008 ceph-mon[191227]: pgmap v1413: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 256 op/s; 61061/111357 objects degraded (54.834%); 18865/111357 objects misplaced (16.941%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:59:48.581 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 61061/111357 objects degraded (54.834%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:48.939 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:48.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:48.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:49.049 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:49.079 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:49.128 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:49.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:49.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:49.233 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:49.262 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:49.312 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:49.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:49.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:49.418 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:49.452 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:49.556 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:49.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:49.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:49.602 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T05:59:49.602 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T05:59:49.602 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T05:59:49.603 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T05:59:49.603 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T05:59:49.661 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:49.711 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:49.764 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:49.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:49.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:49.868 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:49.900 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:50.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:50 smithi191 ceph-mon[161669]: pgmap v1414: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 256 op/s; 60999/110535 objects degraded (55.185%); 18829/110535 objects misplaced (17.034%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:59:50.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:50 smithi008 ceph-mon[184418]: pgmap v1414: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 256 op/s; 60999/110535 objects degraded (55.185%); 18829/110535 objects misplaced (17.034%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:59:50.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 05:59:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:05:59:50] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.33.4" 2023-10-09T05:59:50.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:50 smithi008 ceph-mon[191227]: pgmap v1414: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 256 op/s; 60999/110535 objects degraded (55.185%); 18829/110535 objects misplaced (17.034%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:59:51.029 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:59:51.168 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:51.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:51.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:51.271 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:51.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:51.312 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:51.372 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:51.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:51.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:51.418 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T05:59:51.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:51.506 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:51.556 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:51.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:51.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:51.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:51.705 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:51.771 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:51.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:51.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:51.817 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:59:51.837 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:59:51.861 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:59:51.877 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:51.884 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:59:51.906 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:59:51.910 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:51.928 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:59:51.957 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T05:59:51.967 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:51.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:51.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:51.980 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T05:59:51.982 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:59:51.982 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:59:51.982 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:59:51.982 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:59:51.982 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:59:51.982 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:59:51.982 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:59:51.982 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:59:52.078 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:52.104 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:52.150 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:52.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:52.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:52.255 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:52.293 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:52 smithi191 ceph-mon[161669]: pgmap v1415: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 256 op/s; 60655/108372 objects degraded (55.969%); 18658/108372 objects misplaced (17.217%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:59:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/825258824' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "61"}]: dispatch 2023-10-09T05:59:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1744416986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:52 smithi008 ceph-mon[184418]: pgmap v1415: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 256 op/s; 60655/108372 objects degraded (55.969%); 18658/108372 objects misplaced (17.217%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:59:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/825258824' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "61"}]: dispatch 2023-10-09T05:59:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1744416986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:52.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:52 smithi008 ceph-mon[191227]: pgmap v1415: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 256 op/s; 60655/108372 objects degraded (55.969%); 18658/108372 objects misplaced (17.217%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:59:52.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/825258824' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "61"}]: dispatch 2023-10-09T05:59:52.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1744416986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:53.166 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T05:59:53.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 60655/108372 objects degraded (55.969%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:53.691 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 60655/108372 objects degraded (55.969%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:53.692 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 60655/108372 objects degraded (55.969%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:53.868 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:53.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:53.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:53.972 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:54.013 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:54.048 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:54.077 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:54.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:54.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:54.185 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:54.220 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:54.296 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:54 smithi008 ceph-mon[191227]: pgmap v1416: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 242 op/s; 60541/107517 objects degraded (56.308%); 18563/107517 objects misplaced (17.265%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:59:54.298 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:54.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:54.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:54.402 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:54.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:54.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:54 smithi191 ceph-mon[161669]: pgmap v1416: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 242 op/s; 60541/107517 objects degraded (56.308%); 18563/107517 objects misplaced (17.265%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:59:54.590 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:54.630 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:54 smithi008 ceph-mon[184418]: pgmap v1416: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 242 op/s; 60541/107517 objects degraded (56.308%); 18563/107517 objects misplaced (17.265%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:59:54.639 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:54.891 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:54.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:54.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:54.995 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:55.023 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:55.073 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:55.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:55.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:55.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:55.233 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:55.301 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:55 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:55.302 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:55 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3791844218' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:55.302 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:55 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:55.302 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:55 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:55.302 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:55 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:55.302 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:55 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3791844218' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:55.302 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:55 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:55.302 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:55 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:55.306 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T05:59:55.312 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:55.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:55.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:55.421 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:55.458 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:55.528 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:55.532 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:55 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:55.532 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:55 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3791844218' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:55.532 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:55 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:55.532 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:55 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:55.533 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:55 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:55.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:55.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:55.622 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:55 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:55.622 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:55 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T05:59:55.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:55.679 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:55.714 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:59:55.739 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:55.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:55.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:55.842 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:55.879 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:56.243 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:56.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:56.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:56.348 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:56.386 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:56.443 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:56.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:56.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:56.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:56.592 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:56.603 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:56 smithi191 ceph-mon[161669]: pgmap v1417: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 242 op/s; 60541/107517 objects degraded (56.308%); 18563/107517 objects misplaced (17.265%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:59:56.603 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:56 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:56.603 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:56 smithi191 ceph-mon[161669]: failed to encode map e474 with expected crc 2023-10-09T05:59:56.603 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:56 smithi191 ceph-mon[161669]: failed to encode map e474 with expected crc 2023-10-09T05:59:56.603 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:56 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:56.603 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:56 smithi191 ceph-mon[161669]: osdmap e474: 8 total, 8 up, 5 in 2023-10-09T05:59:56.605 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:56.663 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T05:59:56.663 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:59:56.694 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[184418]: pgmap v1417: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 242 op/s; 60541/107517 objects degraded (56.308%); 18563/107517 objects misplaced (17.265%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:59:56.694 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:56.694 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[184418]: failed to encode map e474 with expected crc 2023-10-09T05:59:56.694 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[184418]: failed to encode map e474 with expected crc 2023-10-09T05:59:56.694 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:56.694 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[184418]: osdmap e474: 8 total, 8 up, 5 in 2023-10-09T05:59:56.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[191227]: pgmap v1417: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 242 op/s; 60541/107517 objects degraded (56.308%); 18563/107517 objects misplaced (17.265%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:59:56.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:56.694 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[191227]: failed to encode map e474 with expected crc 2023-10-09T05:59:56.695 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[191227]: failed to encode map e474 with expected crc 2023-10-09T05:59:56.695 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:56.695 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:56 smithi008 ceph-mon[191227]: osdmap e474: 8 total, 8 up, 5 in 2023-10-09T05:59:56.713 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:56.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:56.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:56.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:56.858 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:57.034 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:57.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:57.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:57.144 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:57.174 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:57.837 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:59:58.129 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:58.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:58.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:58.168 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3] out_osds: [0, 5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:59:58.168 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:59:58.168 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-10-09T05:59:58.168 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd in 0 2023-10-09T05:59:58.233 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:58.255 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:59:58.263 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:58.416 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:58.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:58.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:58.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:58 smithi008 ceph-mon[184418]: pgmap v1419: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 209 op/s; 60000/105078 objects degraded (57.100%); 18307/105078 objects misplaced (17.422%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:59:58.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 60000/105078 objects degraded (57.100%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:58.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:58 smithi008 ceph-mon[191227]: pgmap v1419: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 209 op/s; 60000/105078 objects degraded (57.100%); 18307/105078 objects misplaced (17.422%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:59:58.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 60000/105078 objects degraded (57.100%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:58.518 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:59:58.523 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:58.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:58 smithi191 ceph-mon[161669]: pgmap v1419: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 209 op/s; 60000/105078 objects degraded (57.100%); 18307/105078 objects misplaced (17.422%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:59:58.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 60000/105078 objects degraded (57.100%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:59:58.577 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:58.966 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:58.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:58.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:59.071 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:59.108 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:59.113 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:59.188 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:59.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:59.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:59.293 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:59.313 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 05:59:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:59.314 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 05:59:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:59.338 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:59.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 05:59:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:59.681 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:59.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:59.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:59.734 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T05:59:59.734 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T05:59:59.741 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:59.744 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T05:59:59.766 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T05:59:59.787 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:59.788 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T05:59:59.806 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T05:59:59.816 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:59.825 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:00:00.068 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:00:00.069 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:00:00.070 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:00:00.070 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:00:00.070 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:00:00.070 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:00:00.070 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:00:00.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:00:00.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:00:00.311 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:00.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:00.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:00.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pgmap v1420: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 210 op/s; 59913/104226 objects degraded (57.484%); 18245/104226 objects misplaced (17.505%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:00:00.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2339030070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:00.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 59913/104226 objects degraded (57.484%), 36 pgs degraded, 18 pgs undersized 2023-10-09T06:00:00.363 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:00:00.363 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: [WRN] PG_DEGRADED: Degraded data redundancy: 59913/104226 objects degraded (57.484%), 36 pgs degraded, 18 pgs undersized 2023-10-09T06:00:00.363 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 1.0 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2023-10-09T06:00:00.363 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 2.0 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-10-09T06:00:00.363 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 2.1 is active+recovery_wait+degraded, acting [2,3,1] 2023-10-09T06:00:00.363 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 2.2 is active+recovery_wait+degraded, acting [7,1,2] 2023-10-09T06:00:00.364 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 2.3 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,7] 2023-10-09T06:00:00.364 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 2.4 is active+recovery_wait+degraded, acting [7,1,2] 2023-10-09T06:00:00.364 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 2.5 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,4] 2023-10-09T06:00:00.364 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 2.6 is active+recovery_wait+degraded+remapped, acting [1,4,3] 2023-10-09T06:00:00.364 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 2.7 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:00:00.364 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 3.7 is active+recovery_wait+degraded, acting [7,3,4] 2023-10-09T06:00:00.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 3.b is active+recovery_wait+degraded, acting [7,3,4] 2023-10-09T06:00:00.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 3.d is active+recovery_wait+degraded, acting [7,3,4] 2023-10-09T06:00:00.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 3.f is active+recovery_wait+degraded, acting [7,4,2] 2023-10-09T06:00:00.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 3.10 is active+recovery_wait+degraded, acting [2,3,7] 2023-10-09T06:00:00.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 3.14 is active+recovery_wait+degraded, acting [7,4,2] 2023-10-09T06:00:00.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 3.1c is active+recovery_wait+degraded, acting [7,4,1] 2023-10-09T06:00:00.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 3.1d is active+recovery_wait+degraded, acting [7,4,1] 2023-10-09T06:00:00.365 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 3.1f is active+recovery_wait+degraded, acting [2,4,7] 2023-10-09T06:00:00.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.0 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,4] 2023-10-09T06:00:00.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.1 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2023-10-09T06:00:00.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.3 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,3] 2023-10-09T06:00:00.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.4 is active+recovery_wait+degraded, acting [1,4,3] 2023-10-09T06:00:00.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.5 is stuck undersized for 117s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2023-10-09T06:00:00.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.6 is stuck undersized for 111s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,1] 2023-10-09T06:00:00.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:00:00.366 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.9 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,4] 2023-10-09T06:00:00.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.b is stuck undersized for 109s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,6] 2023-10-09T06:00:00.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.d is active+recovery_wait+degraded+remapped, acting [7,4,5] 2023-10-09T06:00:00.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.e is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2023-10-09T06:00:00.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,3] 2023-10-09T06:00:00.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.10 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,4] 2023-10-09T06:00:00.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.14 is active+recovery_wait+degraded, acting [1,4,3] 2023-10-09T06:00:00.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.15 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2023-10-09T06:00:00.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.16 is stuck undersized for 111s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,1] 2023-10-09T06:00:00.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.17 is stuck undersized for 117s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:00:00.367 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[184418]: pg 9.19 is stuck undersized for 108s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,4] 2023-10-09T06:00:00.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pgmap v1420: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 210 op/s; 59913/104226 objects degraded (57.484%); 18245/104226 objects misplaced (17.505%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:00:00.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2339030070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:00.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 59913/104226 objects degraded (57.484%), 36 pgs degraded, 18 pgs undersized 2023-10-09T06:00:00.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:00:00.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: [WRN] PG_DEGRADED: Degraded data redundancy: 59913/104226 objects degraded (57.484%), 36 pgs degraded, 18 pgs undersized 2023-10-09T06:00:00.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 1.0 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2023-10-09T06:00:00.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 2.0 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-10-09T06:00:00.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 2.1 is active+recovery_wait+degraded, acting [2,3,1] 2023-10-09T06:00:00.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 2.2 is active+recovery_wait+degraded, acting [7,1,2] 2023-10-09T06:00:00.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 2.3 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,7] 2023-10-09T06:00:00.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 2.4 is active+recovery_wait+degraded, acting [7,1,2] 2023-10-09T06:00:00.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 2.5 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,4] 2023-10-09T06:00:00.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 2.6 is active+recovery_wait+degraded+remapped, acting [1,4,3] 2023-10-09T06:00:00.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 2.7 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:00:00.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 3.7 is active+recovery_wait+degraded, acting [7,3,4] 2023-10-09T06:00:00.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 3.b is active+recovery_wait+degraded, acting [7,3,4] 2023-10-09T06:00:00.370 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 3.d is active+recovery_wait+degraded, acting [7,3,4] 2023-10-09T06:00:00.371 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 3.f is active+recovery_wait+degraded, acting [7,4,2] 2023-10-09T06:00:00.371 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 3.10 is active+recovery_wait+degraded, acting [2,3,7] 2023-10-09T06:00:00.371 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 3.14 is active+recovery_wait+degraded, acting [7,4,2] 2023-10-09T06:00:00.371 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 3.1c is active+recovery_wait+degraded, acting [7,4,1] 2023-10-09T06:00:00.371 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 3.1d is active+recovery_wait+degraded, acting [7,4,1] 2023-10-09T06:00:00.372 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 3.1f is active+recovery_wait+degraded, acting [2,4,7] 2023-10-09T06:00:00.373 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.0 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,4] 2023-10-09T06:00:00.373 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.1 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2023-10-09T06:00:00.373 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.3 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,3] 2023-10-09T06:00:00.373 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.4 is active+recovery_wait+degraded, acting [1,4,3] 2023-10-09T06:00:00.373 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.5 is stuck undersized for 117s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2023-10-09T06:00:00.373 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.6 is stuck undersized for 111s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,1] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.9 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,4] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.b is stuck undersized for 109s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,6] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.d is active+recovery_wait+degraded+remapped, acting [7,4,5] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.e is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,3] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.10 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,4] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.14 is active+recovery_wait+degraded, acting [1,4,3] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.15 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.16 is stuck undersized for 111s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,1] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.17 is stuck undersized for 117s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:00:00.374 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:00 smithi008 ceph-mon[191227]: pg 9.19 is stuck undersized for 108s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,4] 2023-10-09T06:00:00.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:00.464 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:00.547 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:00.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:00.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:00.652 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:00.688 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:00:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:00:00] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.33.4" 2023-10-09T06:00:00.691 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:00.721 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pgmap v1420: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 210 op/s; 59913/104226 objects degraded (57.484%); 18245/104226 objects misplaced (17.505%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:00:00.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2339030070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:00.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 59913/104226 objects degraded (57.484%), 36 pgs degraded, 18 pgs undersized 2023-10-09T06:00:00.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:00:00.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: [WRN] PG_DEGRADED: Degraded data redundancy: 59913/104226 objects degraded (57.484%), 36 pgs degraded, 18 pgs undersized 2023-10-09T06:00:00.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 1.0 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2023-10-09T06:00:00.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 2.0 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-10-09T06:00:00.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 2.1 is active+recovery_wait+degraded, acting [2,3,1] 2023-10-09T06:00:00.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 2.2 is active+recovery_wait+degraded, acting [7,1,2] 2023-10-09T06:00:00.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 2.3 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,7] 2023-10-09T06:00:00.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 2.4 is active+recovery_wait+degraded, acting [7,1,2] 2023-10-09T06:00:00.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 2.5 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,4] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 2.6 is active+recovery_wait+degraded+remapped, acting [1,4,3] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 2.7 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 3.7 is active+recovery_wait+degraded, acting [7,3,4] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 3.b is active+recovery_wait+degraded, acting [7,3,4] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 3.d is active+recovery_wait+degraded, acting [7,3,4] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 3.f is active+recovery_wait+degraded, acting [7,4,2] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 3.10 is active+recovery_wait+degraded, acting [2,3,7] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 3.14 is active+recovery_wait+degraded, acting [7,4,2] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 3.1c is active+recovery_wait+degraded, acting [7,4,1] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 3.1d is active+recovery_wait+degraded, acting [7,4,1] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 3.1f is active+recovery_wait+degraded, acting [2,4,7] 2023-10-09T06:00:00.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.0 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,4] 2023-10-09T06:00:00.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.1 is stuck undersized for 107s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2023-10-09T06:00:00.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.3 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,3] 2023-10-09T06:00:00.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.4 is active+recovery_wait+degraded, acting [1,4,3] 2023-10-09T06:00:00.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.5 is stuck undersized for 117s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2023-10-09T06:00:00.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.6 is stuck undersized for 111s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,1] 2023-10-09T06:00:00.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:00:00.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.9 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,4] 2023-10-09T06:00:00.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.b is stuck undersized for 109s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,6] 2023-10-09T06:00:00.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.d is active+recovery_wait+degraded+remapped, acting [7,4,5] 2023-10-09T06:00:00.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.e is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2023-10-09T06:00:00.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,3] 2023-10-09T06:00:00.725 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.10 is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,4] 2023-10-09T06:00:00.725 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.14 is active+recovery_wait+degraded, acting [1,4,3] 2023-10-09T06:00:00.725 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.15 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2023-10-09T06:00:00.725 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.16 is stuck undersized for 111s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,1] 2023-10-09T06:00:00.725 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.17 is stuck undersized for 117s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:00:00.725 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:00 smithi191 ceph-mon[161669]: pg 9.19 is stuck undersized for 108s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,4] 2023-10-09T06:00:00.985 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:00:00.985 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:00:00.985 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T06:00:00.985 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:00:01.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:01 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:01 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:01.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:01 smithi008 ceph-mon[184418]: pgmap v1421: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 202 op/s; 59490/102291 objects degraded (58.158%); 18086/102291 objects misplaced (17.681%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:00:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:01 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:01 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:01.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:01 smithi008 ceph-mon[191227]: pgmap v1421: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 202 op/s; 59490/102291 objects degraded (58.158%); 18086/102291 objects misplaced (17.681%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:00:01.792 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:01 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:01.792 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:01 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:01.792 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:01 smithi191 ceph-mon[161669]: pgmap v1421: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 202 op/s; 59490/102291 objects degraded (58.158%); 18086/102291 objects misplaced (17.681%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:00:01.811 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:00:02.266 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:02.708 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:02.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:02.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:02.813 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:02.848 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:02.924 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:02.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:02.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:03.029 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:03.063 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:03.188 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:03.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:03.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:03.294 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:03.307 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:03 smithi008 conmon[184394]: 2023-10-09T06:00:03.304+0000 7f8a66521700 -1 mon.a@0(leader).osd e474 definitely_dead 0 2023-10-09T06:00:03.358 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:03.690 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:03.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:03.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:03.796 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:03.832 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:03.882 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:03.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:03.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:03.953 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:00:03.953 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:00:03.953 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T06:00:03.953 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T06:00:03.953 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:00:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:03 smithi008 ceph-mon[191227]: pgmap v1422: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 203 op/s; 59378/101406 objects degraded (58.555%); 17981/101406 objects misplaced (17.732%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:00:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 59378/101406 objects degraded (58.555%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:03.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:03 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2676591960' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:00:03.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:03 smithi008 ceph-mon[184418]: pgmap v1422: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 203 op/s; 59378/101406 objects degraded (58.555%); 17981/101406 objects misplaced (17.732%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:00:03.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 59378/101406 objects degraded (58.555%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:03.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:03 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2676591960' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:00:03.981 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:03 smithi191 ceph-mon[161669]: pgmap v1422: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 203 op/s; 59378/101406 objects degraded (58.555%); 17981/101406 objects misplaced (17.732%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:00:03.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 59378/101406 objects degraded (58.555%), 36 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:03.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:03 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2676591960' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:00:03.985 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:04.016 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:04.092 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:04.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:04.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:04.196 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:04.226 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:04.303 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:04.309 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:00:04.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:04.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:04.316 INFO:teuthology.orchestra.run.smithi008.stderr:marked in osd.0. 2023-10-09T06:00:04.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:04.443 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:04.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:00:04.731 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:04.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:04.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:04.806 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:00:04.839 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:04.864 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:04.912 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:04.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:04.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:04.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1547844178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:04.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2676591960' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T06:00:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[184418]: osdmap e475: 8 total, 8 up, 6 in 2023-10-09T06:00:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[184418]: failed to encode map e475 with expected crc 2023-10-09T06:00:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[184418]: failed to encode map e475 with expected crc 2023-10-09T06:00:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:04.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:04.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1547844178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:04.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:04.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:04.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2676591960' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T06:00:04.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:04.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[191227]: osdmap e475: 8 total, 8 up, 6 in 2023-10-09T06:00:04.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[191227]: failed to encode map e475 with expected crc 2023-10-09T06:00:04.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[191227]: failed to encode map e475 with expected crc 2023-10-09T06:00:04.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:04.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:04 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:05.020 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:05.047 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1547844178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:04 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:04 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2676591960' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T06:00:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:04 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:04 smithi191 ceph-mon[161669]: osdmap e475: 8 total, 8 up, 6 in 2023-10-09T06:00:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:04 smithi191 ceph-mon[161669]: failed to encode map e475 with expected crc 2023-10-09T06:00:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:04 smithi191 ceph-mon[161669]: failed to encode map e475 with expected crc 2023-10-09T06:00:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:04 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:05.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:04 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:05.088 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:05.759 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:05.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:05.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:05.863 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:05.891 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:05.935 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:05.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:05 smithi191 ceph-mon[161669]: failed to encode map e475 with expected crc 2023-10-09T06:00:05.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:05 smithi191 ceph-mon[161669]: failed to encode map e475 with expected crc 2023-10-09T06:00:05.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:05 smithi191 ceph-mon[161669]: pgmap v1424: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 160 op/s; 59378/101406 objects degraded (58.555%); 17981/101406 objects misplaced (17.732%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:00:05.937 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:05 smithi191 ceph-mon[161669]: failed to encode map e476 with expected crc 2023-10-09T06:00:05.938 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:05 smithi191 ceph-mon[161669]: osdmap e476: 8 total, 8 up, 6 in 2023-10-09T06:00:05.938 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:05 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:05.938 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:05 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:05.938 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:05 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:05.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:05.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:06.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:06.065 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:06.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[191227]: failed to encode map e475 with expected crc 2023-10-09T06:00:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[191227]: failed to encode map e475 with expected crc 2023-10-09T06:00:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[191227]: pgmap v1424: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 160 op/s; 59378/101406 objects degraded (58.555%); 17981/101406 objects misplaced (17.732%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:00:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[191227]: failed to encode map e476 with expected crc 2023-10-09T06:00:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[191227]: osdmap e476: 8 total, 8 up, 6 in 2023-10-09T06:00:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:06.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[184418]: failed to encode map e475 with expected crc 2023-10-09T06:00:06.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[184418]: failed to encode map e475 with expected crc 2023-10-09T06:00:06.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[184418]: pgmap v1424: 99 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 20 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 2.1 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 160 op/s; 59378/101406 objects degraded (58.555%); 17981/101406 objects misplaced (17.732%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:00:06.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[184418]: failed to encode map e476 with expected crc 2023-10-09T06:00:06.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[184418]: osdmap e476: 8 total, 8 up, 6 in 2023-10-09T06:00:06.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:06.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:06.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:05 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:00:06.264 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:06.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:06.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:06.370 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:06.432 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:06.650 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-10-09T06:00:06.704 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:06.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:06.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:06.726 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:06.771 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:00:06.771 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T06:00:06.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T06:00:06.771 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T06:00:06.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.772 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:45782 9.e 9:7b605443:::benchmark_data_smithi008_502820_object45781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e372)", 2023-10-09T06:00:06.772 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:53:46.078272+0000", 2023-10-09T06:00:06.772 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 380.68985857799998, 2023-10-09T06:00:06.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0922387410000001, 2023-10-09T06:00:06.772 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.772 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:06.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.772 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 45782 2023-10-09T06:00:06.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.773 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078272+0000", 2023-10-09T06:00:06.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078272+0000", 2023-10-09T06:00:06.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078270+0000", 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8878e-05 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078299+0000", 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2486999999999999e-05 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078311+0000", 2023-10-09T06:00:06.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0040000000000002e-06 2023-10-09T06:00:06.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078318+0000", 2023-10-09T06:00:06.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6557000000000002e-05 2023-10-09T06:00:06.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078365+0000", 2023-10-09T06:00:06.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0165e-05 2023-10-09T06:00:06.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.777 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078375+0000", 2023-10-09T06:00:06.777 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85827073899999995 2023-10-09T06:00:06.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.777 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.777 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.936646+0000", 2023-10-09T06:00:06.777 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1354e-05 2023-10-09T06:00:06.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.777 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.777 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.936657+0000", 2023-10-09T06:00:06.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2267288569999999 2023-10-09T06:00:06.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.778 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.778 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.163386+0000", 2023-10-09T06:00:06.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091583599999999995 2023-10-09T06:00:06.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.778 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.778 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.164302+0000", 2023-10-09T06:00:06.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000143702 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.164446+0000", 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003236381 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.167682+0000", 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001185424 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.168867+0000", 2023-10-09T06:00:06.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00053191999999999996 2023-10-09T06:00:06.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.169399+0000", 2023-10-09T06:00:06.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4771000000000002e-05 2023-10-09T06:00:06.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:06.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.169444+0000", 2023-10-09T06:00:06.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010662359999999999 2023-10-09T06:00:06.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.170510+0000", 2023-10-09T06:00:06.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0922387410000001 2023-10-09T06:00:06.782 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:77617 9.1 9:84bc2af6:::benchmark_data_smithi008_502820_object77616:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.819004+0000", 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 329.949126105, 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92076191299999999, 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.783 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.784 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.784 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 77617 2023-10-09T06:00:06.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.784 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.784 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.784 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819004+0000", 2023-10-09T06:00:06.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.784 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819004+0000", 2023-10-09T06:00:06.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:00:06.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819002+0000", 2023-10-09T06:00:06.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6029999999999997e-05 2023-10-09T06:00:06.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819058+0000", 2023-10-09T06:00:06.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4360999999999999e-05 2023-10-09T06:00:06.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819082+0000", 2023-10-09T06:00:06.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7680000000000003e-06 2023-10-09T06:00:06.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819092+0000", 2023-10-09T06:00:06.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.045903408999999999 2023-10-09T06:00:06.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.864995+0000", 2023-10-09T06:00:06.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5949999999999999e-06 2023-10-09T06:00:06.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.865004+0000", 2023-10-09T06:00:06.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.87466948200000005 2023-10-09T06:00:06.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.739673+0000", 2023-10-09T06:00:06.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2732000000000006e-05 2023-10-09T06:00:06.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.739766+0000", 2023-10-09T06:00:06.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92076191299999999 2023-10-09T06:00:06.788 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:77618 9.11 9:8a9bea20:::benchmark_data_smithi008_502820_object77617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.819101+0000", 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 329.94902862200001, 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.928982839, 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.789 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.790 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.790 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 77618 2023-10-09T06:00:06.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.790 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819101+0000", 2023-10-09T06:00:06.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819101+0000", 2023-10-09T06:00:06.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:06.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819101+0000", 2023-10-09T06:00:06.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2029e-05 2023-10-09T06:00:06.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819143+0000", 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6980000000000001e-06 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819150+0000", 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9419999999999997e-06 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819154+0000", 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.045857321999999999 2023-10-09T06:00:06.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.865011+0000", 2023-10-09T06:00:06.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5409999999999997e-06 2023-10-09T06:00:06.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.865017+0000", 2023-10-09T06:00:06.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88297774600000001 2023-10-09T06:00:06.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.747995+0000", 2023-10-09T06:00:06.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9123999999999996e-05 2023-10-09T06:00:06.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.748084+0000", 2023-10-09T06:00:06.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.928982839 2023-10-09T06:00:06.794 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.794 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.794 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:90125 9.c 9:3fead019:::benchmark_data_smithi008_502820_object5952:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:54.013383+0000", 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 312.75474746100002, 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.394155001, 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.795 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 90125 2023-10-09T06:00:06.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.796 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013383+0000", 2023-10-09T06:00:06.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013383+0000", 2023-10-09T06:00:06.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013382+0000", 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.937e-06 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013385+0000", 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1168e-05 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013396+0000", 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.844e-06 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013400+0000", 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003200779 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.016601+0000", 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000143905 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.016745+0000", 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000168927 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.016914+0000", 2023-10-09T06:00:06.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031704670000000002 2023-10-09T06:00:06.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.020084+0000", 2023-10-09T06:00:06.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002970664 2023-10-09T06:00:06.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:06.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.023055+0000", 2023-10-09T06:00:06.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.38235074600000002 2023-10-09T06:00:06.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.405406+0000", 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4887000000000004e-05 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.405470+0000", 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020671679999999999 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.407538+0000", 2023-10-09T06:00:06.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.394155001 2023-10-09T06:00:06.802 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.802 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.802 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.802 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93474 9.e 9:77e94385:::benchmark_data_smithi008_502820_object9301:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:00:06.802 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.096743+0000", 2023-10-09T06:00:06.802 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 308.67138749200001, 2023-10-09T06:00:06.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.154875152, 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93474 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096743+0000", 2023-10-09T06:00:06.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096743+0000", 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096741+0000", 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9869999999999999e-06 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096749+0000", 2023-10-09T06:00:06.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1752e-05 2023-10-09T06:00:06.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096761+0000", 2023-10-09T06:00:06.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.708e-06 2023-10-09T06:00:06.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096768+0000", 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000324956 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.097092+0000", 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016051600000000001 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.097253+0000", 2023-10-09T06:00:06.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000160118 2023-10-09T06:00:06.807 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.807 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:00:06.807 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.097413+0000", 2023-10-09T06:00:06.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035366690000000001 2023-10-09T06:00:06.807 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.807 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:06.808 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100950+0000", 2023-10-09T06:00:06.808 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021417300000000001 2023-10-09T06:00:06.808 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.808 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.808 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.101164+0000", 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14614434300000001 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247308+0000", 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1691e-05 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247320+0000", 2023-10-09T06:00:06.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0042787399999999 2023-10-09T06:00:06.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251599+0000", 2023-10-09T06:00:06.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9018e-05 2023-10-09T06:00:06.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251618+0000", 2023-10-09T06:00:06.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.154875152 2023-10-09T06:00:06.810 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.811 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.811 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.811 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93475 9.c 9:3e40594a:::benchmark_data_smithi008_502820_object9302:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:00:06.811 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.100682+0000", 2023-10-09T06:00:06.811 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 308.66744850499998, 2023-10-09T06:00:06.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2206704390000001, 2023-10-09T06:00:06.811 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.811 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:06.811 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93475 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100682+0000", 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100682+0000", 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100680+0000", 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5109999999999997e-06 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100688+0000", 2023-10-09T06:00:06.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.63e-05 2023-10-09T06:00:06.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100704+0000", 2023-10-09T06:00:06.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6109999999999999e-06 2023-10-09T06:00:06.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100711+0000", 2023-10-09T06:00:06.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6734000000000003e-05 2023-10-09T06:00:06.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100767+0000", 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015612699999999999 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100923+0000", 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016590299999999999 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:00:06.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.101089+0000", 2023-10-09T06:00:06.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030346919999999999 2023-10-09T06:00:06.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.104124+0000", 2023-10-09T06:00:06.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102549 2023-10-09T06:00:06.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:06.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.104227+0000", 2023-10-09T06:00:06.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.143401417 2023-10-09T06:00:06.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247628+0000", 2023-10-09T06:00:06.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9989999999999997e-06 2023-10-09T06:00:06.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247637+0000", 2023-10-09T06:00:06.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0299526569999999 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.277590+0000", 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9399000000000002e-05 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.277679+0000", 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.043604728000000002 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.321284+0000", 2023-10-09T06:00:06.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8312000000000006e-05 2023-10-09T06:00:06.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.321352+0000", 2023-10-09T06:00:06.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2206704390000001 2023-10-09T06:00:06.819 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.819 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.819 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.820 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.820 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.820 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93483 9.e 9:7fc9cbe3:::benchmark_data_smithi008_502820_object9310:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:00:06.820 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.108830+0000", 2023-10-09T06:00:06.820 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 308.65929996599999, 2023-10-09T06:00:06.820 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1428273499999999, 2023-10-09T06:00:06.820 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.820 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:06.820 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.820 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.820 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93483 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108830+0000", 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108830+0000", 2023-10-09T06:00:06.821 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.822 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.822 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.822 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.822 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108829+0000", 2023-10-09T06:00:06.822 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7689999999999999e-06 2023-10-09T06:00:06.822 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.822 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.822 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.822 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108835+0000", 2023-10-09T06:00:06.822 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2876e-05 2023-10-09T06:00:06.822 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108848+0000", 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8799000000000001e-05 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108867+0000", 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6572e-05 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.823 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.824 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108924+0000", 2023-10-09T06:00:06.824 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000126241 2023-10-09T06:00:06.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.824 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.824 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.109050+0000", 2023-10-09T06:00:06.824 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012856799999999999 2023-10-09T06:00:06.824 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.824 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:00:06.824 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.109178+0000", 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003197222 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.112376+0000", 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001610319 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.113986+0000", 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13337320899999999 2023-10-09T06:00:06.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247359+0000", 2023-10-09T06:00:06.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8019999999999992e-06 2023-10-09T06:00:06.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247367+0000", 2023-10-09T06:00:06.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0042815700000001 2023-10-09T06:00:06.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251648+0000", 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0310000000000004e-06 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251657+0000", 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1428273499999999 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.828 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.828 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93490 9.e 9:7c8d93dd:::benchmark_data_smithi008_502820_object9317:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:00:06.828 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.117443+0000", 2023-10-09T06:00:06.828 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 308.65068682999998, 2023-10-09T06:00:06.828 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.134247681, 2023-10-09T06:00:06.828 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.828 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:06.828 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.828 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.828 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.828 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93490 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117443+0000", 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117443+0000", 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.829 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117442+0000", 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7300000000000002e-06 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117448+0000", 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0936e-05 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.830 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.831 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117459+0000", 2023-10-09T06:00:06.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3310000000000004e-06 2023-10-09T06:00:06.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.831 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.831 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.831 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117467+0000", 2023-10-09T06:00:06.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7256000000000003e-05 2023-10-09T06:00:06.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.831 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.831 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.831 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117514+0000", 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019338100000000001 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117708+0000", 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016922399999999999 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117877+0000", 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035029470000000002 2023-10-09T06:00:06.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.121380+0000", 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0540999999999999e-05 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.121400+0000", 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12599297000000001 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.833 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247393+0000", 2023-10-09T06:00:06.834 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1810000000000004e-06 2023-10-09T06:00:06.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247400+0000", 2023-10-09T06:00:06.834 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.004281518 2023-10-09T06:00:06.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251682+0000", 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0119999999999994e-06 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251691+0000", 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.134247681 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.835 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93496 9.c 9:33b252c8:::benchmark_data_smithi008_502820_object9323:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:00:06.836 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.123753+0000", 2023-10-09T06:00:06.836 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 308.64437735400003, 2023-10-09T06:00:06.836 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.192581734, 2023-10-09T06:00:06.836 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.836 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:06.836 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.836 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.836 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.836 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93496 2023-10-09T06:00:06.836 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.836 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123753+0000", 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123753+0000", 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.837 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123751+0000", 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2879999999999998e-06 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123758+0000", 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1221e-05 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123769+0000", 2023-10-09T06:00:06.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7299999999999996e-06 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123774+0000", 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2144000000000001e-05 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123826+0000", 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000124738 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123951+0000", 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.123715246 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247666+0000", 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3069999999999997e-06 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247673+0000", 2023-10-09T06:00:06.841 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0300421200000001 2023-10-09T06:00:06.841 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.841 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.841 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.841 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.277715+0000", 2023-10-09T06:00:06.841 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7616999999999998e-05 2023-10-09T06:00:06.841 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.841 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.841 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.841 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.277803+0000", 2023-10-09T06:00:06.841 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0111000000000005e-05 2023-10-09T06:00:06.842 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.842 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.842 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:00:06.842 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.277893+0000", 2023-10-09T06:00:06.842 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033372605 2023-10-09T06:00:06.842 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.842 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.842 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:06.842 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.311266+0000", 2023-10-09T06:00:06.842 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002796991 2023-10-09T06:00:06.842 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314063+0000", 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017246500000000001 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314235+0000", 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8576999999999999e-05 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.843 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314284+0000", 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020508919999999999 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.316334+0000", 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.192581734 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.844 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.845 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93502 9.e 9.68a8793e (undecoded) ondisk+write+known_if_redirected e391)", 2023-10-09T06:00:06.845 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.383448+0000", 2023-10-09T06:00:06.845 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 308.38468175600002, 2023-10-09T06:00:06.845 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86827123699999997, 2023-10-09T06:00:06.845 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.845 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:06.845 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.845 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.845 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.845 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93502 2023-10-09T06:00:06.845 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383448+0000", 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383448+0000", 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.846 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383447+0000", 2023-10-09T06:00:06.847 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4139999999999997e-06 2023-10-09T06:00:06.847 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383454+0000", 2023-10-09T06:00:06.847 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4278e-05 2023-10-09T06:00:06.847 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383468+0000", 2023-10-09T06:00:06.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5860000000000001e-06 2023-10-09T06:00:06.848 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.848 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.848 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383475+0000", 2023-10-09T06:00:06.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3341999999999999e-05 2023-10-09T06:00:06.848 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.848 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.848 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383528+0000", 2023-10-09T06:00:06.849 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.009e-05 2023-10-09T06:00:06.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.849 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383539+0000", 2023-10-09T06:00:06.849 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86817271100000004 2023-10-09T06:00:06.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.849 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251711+0000", 2023-10-09T06:00:06.849 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3489999999999992e-06 2023-10-09T06:00:06.850 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.850 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.850 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.850 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251720+0000", 2023-10-09T06:00:06.850 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86827123699999997 2023-10-09T06:00:06.850 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.850 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.850 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.850 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.850 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.850 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93503 9.e 9.abdff44e (undecoded) ondisk+write+known_if_redirected e391)", 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.383488+0000", 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 308.38464198600002, 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86825971800000001, 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93503 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.851 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383488+0000", 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383488+0000", 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383488+0000", 2023-10-09T06:00:06.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0340000000000001e-06 2023-10-09T06:00:06.853 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.853 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383491+0000", 2023-10-09T06:00:06.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3049999999999998e-06 2023-10-09T06:00:06.853 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.853 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383497+0000", 2023-10-09T06:00:06.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.172e-06 2023-10-09T06:00:06.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.854 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383501+0000", 2023-10-09T06:00:06.854 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00046004500000000002 2023-10-09T06:00:06.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.854 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383961+0000", 2023-10-09T06:00:06.854 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3599999999999998e-06 2023-10-09T06:00:06.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.855 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.855 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.855 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383969+0000", 2023-10-09T06:00:06.855 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86777167399999999 2023-10-09T06:00:06.855 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.855 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.855 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.855 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251740+0000", 2023-10-09T06:00:06.855 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6320000000000008e-06 2023-10-09T06:00:06.855 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.855 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251748+0000", 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86825971800000001 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93512 9.c 9:347bba79:::benchmark_data_smithi008_502820_object9339:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.396258+0000", 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 308.371872344, 2023-10-09T06:00:06.856 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92175368199999996, 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93512 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396258+0000", 2023-10-09T06:00:06.857 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.858 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.858 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.858 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.858 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396258+0000", 2023-10-09T06:00:06.858 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.858 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.858 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.858 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.858 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396256+0000", 2023-10-09T06:00:06.858 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1110000000000002e-06 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396264+0000", 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2522e-05 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396276+0000", 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0119999999999996e-06 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.859 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.860 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.860 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396281+0000", 2023-10-09T06:00:06.860 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5559000000000002e-05 2023-10-09T06:00:06.860 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.860 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.860 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.860 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396327+0000", 2023-10-09T06:00:06.860 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3000000000000007e-06 2023-10-09T06:00:06.860 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.860 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.860 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396336+0000", 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88407117199999996 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.280407+0000", 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012818199999999999 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.280535+0000", 2023-10-09T06:00:06.861 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010833800000000001 2023-10-09T06:00:06.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.862 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.862 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:00:06.862 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.280644+0000", 2023-10-09T06:00:06.862 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033468772000000001 2023-10-09T06:00:06.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.862 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.862 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.862 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314112+0000", 2023-10-09T06:00:06.862 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0919999999999999e-05 2023-10-09T06:00:06.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314173+0000", 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002239053 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.316412+0000", 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9391999999999997e-05 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.863 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.316462+0000", 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00154963 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.318011+0000", 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92175368199999996 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.864 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93526 9.c 9:38adb9cd:::benchmark_data_smithi008_502820_object9353:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.420062+0000", 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 308.348068431, 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89930828500000004, 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93526 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.865 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.866 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.866 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.866 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420062+0000", 2023-10-09T06:00:06.866 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.866 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.866 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.866 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.866 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420062+0000", 2023-10-09T06:00:06.866 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.866 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.866 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.867 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.867 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420060+0000", 2023-10-09T06:00:06.867 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.994e-06 2023-10-09T06:00:06.867 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.867 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.867 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.867 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420065+0000", 2023-10-09T06:00:06.867 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0139000000000001e-05 2023-10-09T06:00:06.867 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.867 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.867 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420075+0000", 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8459999999999999e-06 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420080+0000", 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073188280000000003 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.427399+0000", 2023-10-09T06:00:06.868 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4809999999999995e-06 2023-10-09T06:00:06.869 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.869 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.869 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.869 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.427409+0000", 2023-10-09T06:00:06.869 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85476691800000004 2023-10-09T06:00:06.869 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.869 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.869 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.869 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.282176+0000", 2023-10-09T06:00:06.869 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012473900000000001 2023-10-09T06:00:06.869 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.870 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.870 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.870 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.282300+0000", 2023-10-09T06:00:06.870 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000178133 2023-10-09T06:00:06.870 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.870 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.870 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:00:06.870 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.282478+0000", 2023-10-09T06:00:06.870 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.031680262000000001 2023-10-09T06:00:06.870 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.870 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.871 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.871 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314159+0000", 2023-10-09T06:00:06.871 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3935e-05 2023-10-09T06:00:06.871 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.871 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.871 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:06.871 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314183+0000", 2023-10-09T06:00:06.871 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038877590000000002 2023-10-09T06:00:06.871 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.871 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.871 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.318070+0000", 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3279000000000002e-05 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.318134+0000", 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00123628 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.319370+0000", 2023-10-09T06:00:06.872 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89930828500000004 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93531 9.e 9.5b3160de (undecoded) ondisk+write+known_if_redirected e391)", 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.434836+0000", 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 308.33329448500001, 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81693827500000005, 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:06.873 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:06.874 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:06.874 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:06.874 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93531 2023-10-09T06:00:06.874 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.874 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.874 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.874 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.874 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434836+0000", 2023-10-09T06:00:06.874 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.874 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.874 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.875 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.875 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434836+0000", 2023-10-09T06:00:06.875 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:06.875 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.875 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.875 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.875 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434834+0000", 2023-10-09T06:00:06.875 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8570000000000001e-06 2023-10-09T06:00:06.875 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.875 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.875 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.876 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434843+0000", 2023-10-09T06:00:06.876 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6622000000000001e-05 2023-10-09T06:00:06.876 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.876 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.876 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.876 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434859+0000", 2023-10-09T06:00:06.876 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1670000000000004e-06 2023-10-09T06:00:06.876 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.876 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.876 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.876 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434866+0000", 2023-10-09T06:00:06.877 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7221000000000001e-05 2023-10-09T06:00:06.877 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.877 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.877 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.877 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434913+0000", 2023-10-09T06:00:06.877 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2180000000000005e-06 2023-10-09T06:00:06.877 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.877 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.877 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:06.877 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434922+0000", 2023-10-09T06:00:06.877 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81684441699999999 2023-10-09T06:00:06.878 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.878 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.878 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.878 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251766+0000", 2023-10-09T06:00:06.878 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5000000000000002e-06 2023-10-09T06:00:06.878 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.878 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.878 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.878 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251774+0000", 2023-10-09T06:00:06.878 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81693827500000005 2023-10-09T06:00:06.878 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.879 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.879 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.879 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.879 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.879 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "MOSDPGPush(9.d 457/448 [PushOp(9:ba5d9b80:::benchmark_data_smithi008_502820_object36229:head, version: 370'2270, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:ba5d9b80:::benchmark_data_smithi008_502820_object36229:head@370'2270, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:00:06.879 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:44.135612+0000", 2023-10-09T06:00:06.879 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 82.632518102999995, 2023-10-09T06:00:06.879 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028247918, 2023-10-09T06:00:06.879 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.879 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T06:00:06.879 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:44.135612+0000", 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:44.135612+0000", 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.880 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.881 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:44.135611+0000", 2023-10-09T06:00:06.881 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8401e-05 2023-10-09T06:00:06.881 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.881 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.881 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.881 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:44.135649+0000", 2023-10-09T06:00:06.881 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4071999999999999e-05 2023-10-09T06:00:06.881 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.881 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.881 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.881 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:44.135673+0000", 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3135e-05 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:44.135686+0000", 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4803999999999997e-05 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:44.135761+0000", 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0969999999999995e-06 2023-10-09T06:00:06.882 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.883 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.883 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.883 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:44.135770+0000", 2023-10-09T06:00:06.883 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028089879000000002 2023-10-09T06:00:06.883 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.883 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.883 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.883 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:44.163860+0000", 2023-10-09T06:00:06.883 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028247918 2023-10-09T06:00:06.883 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.883 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:127406 9.e e466/402 9:7a939fb4:::benchmark_data_smithi008_502820_object43233:head v 466'7942, mlcod=370'2727)", 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:27.392905+0000", 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 39.375224920000001, 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017931092999999999, 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.884 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.392905+0000", 2023-10-09T06:00:06.885 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.885 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.885 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.392905+0000", 2023-10-09T06:00:06.885 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:06.885 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.885 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.392904+0000", 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3889999999999994e-06 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.392914+0000", 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001403559 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.394317+0000", 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2350000000000005e-06 2023-10-09T06:00:06.886 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.887 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.887 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.394327+0000", 2023-10-09T06:00:06.887 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001232438 2023-10-09T06:00:06.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.887 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.887 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.395559+0000", 2023-10-09T06:00:06.887 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2920999999999998e-05 2023-10-09T06:00:06.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.395602+0000", 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015189655 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.410792+0000", 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4595999999999998e-05 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.888 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.410836+0000", 2023-10-09T06:00:06.889 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017931092999999999 2023-10-09T06:00:06.889 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.889 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.889 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.889 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.889 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.889 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:127493 9.10 e467/448 9:0d8a52d7:::benchmark_data_smithi008_502820_object43320:head v 467'6340, mlcod=370'2678)", 2023-10-09T06:00:06.889 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:27.454495+0000", 2023-10-09T06:00:06.889 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 39.313635503999997, 2023-10-09T06:00:06.889 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019148958000000001, 2023-10-09T06:00:06.889 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.890 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:06.890 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.890 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.890 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.890 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.454495+0000", 2023-10-09T06:00:06.890 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.890 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.890 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.890 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.890 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.454495+0000", 2023-10-09T06:00:06.890 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.454494+0000", 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0125999999999999e-05 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.454504+0000", 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6928000000000001e-05 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.891 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.454521+0000", 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7989999999999996e-06 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.454528+0000", 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030232869999999999 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.457551+0000", 2023-10-09T06:00:06.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2262999999999999e-05 2023-10-09T06:00:06.893 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.893 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.893 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.893 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.457583+0000", 2023-10-09T06:00:06.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016002925000000001 2023-10-09T06:00:06.893 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.893 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.893 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:06.893 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.473586+0000", 2023-10-09T06:00:06.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7439999999999999e-05 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:27.473644+0000", 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019148958000000001 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:127919 9.1 e468/448 9:81f403b6:::benchmark_data_smithi008_502820_object43746:head v 468'6482, mlcod=369'2061)", 2023-10-09T06:00:06.894 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:29.609378+0000", 2023-10-09T06:00:06.895 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 37.158752428, 2023-10-09T06:00:06.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018280666000000001, 2023-10-09T06:00:06.895 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.895 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:06.895 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.895 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.895 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:29.609378+0000", 2023-10-09T06:00:06.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.895 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.895 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.896 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.896 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:29.609378+0000", 2023-10-09T06:00:06.896 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.896 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.896 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.896 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.896 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:29.609376+0000", 2023-10-09T06:00:06.896 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1643e-05 2023-10-09T06:00:06.896 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.896 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.896 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:29.609388+0000", 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5398000000000001e-05 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:29.609403+0000", 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9810000000000003e-06 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:29.609411+0000", 2023-10-09T06:00:06.897 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060216000000000002 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:29.615433+0000", 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8903000000000001e-05 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:29.615462+0000", 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012154673 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:29.627616+0000", 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1956999999999998e-05 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:29.627658+0000", 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018280666000000001 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.899 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:128121 9.e e471/470 9:7c9069ca:::benchmark_data_smithi008_502820_object43948:head v 471'7986, mlcod=0'0)", 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:32.599934+0000", 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 34.168196064999997, 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.041291387999999998, 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599934+0000", 2023-10-09T06:00:06.900 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599934+0000", 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599933+0000", 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7060000000000004e-06 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.901 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599941+0000", 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4945e-05 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599956+0000", 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8809999999999998e-06 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.902 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599962+0000", 2023-10-09T06:00:06.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7352000000000002e-05 2023-10-09T06:00:06.903 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.903 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.903 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.903 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.600020+0000", 2023-10-09T06:00:06.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6948999999999999e-05 2023-10-09T06:00:06.903 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.903 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.903 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.903 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.600047+0000", 2023-10-09T06:00:06.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.041137582999999998 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.641184+0000", 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1075999999999998e-05 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.641225+0000", 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.041291387999999998 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.904 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:129962 9.0 e472/448 9:05a64494:::benchmark_data_smithi008_502820_object45789:head v 472'6393, mlcod=372'2821)", 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:40.482464+0000", 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 26.285665979000001, 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023961679999999999, 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.905 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:06.906 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:40.482464+0000", 2023-10-09T06:00:06.906 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:06.906 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.906 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.906 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:06.906 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:40.482464+0000", 2023-10-09T06:00:06.906 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:06.906 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.906 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.906 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:06.906 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:40.482463+0000", 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0583e-05 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:40.482473+0000", 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5607000000000002e-05 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:40.482489+0000", 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6830000000000002e-06 2023-10-09T06:00:06.907 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:40.482496+0000", 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012134447 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:40.494630+0000", 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1427000000000003e-05 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.908 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:06.909 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:40.494662+0000", 2023-10-09T06:00:06.909 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011716603000000001 2023-10-09T06:00:06.909 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.909 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.909 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:06.909 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:40.506378+0000", 2023-10-09T06:00:06.909 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7565999999999998e-05 2023-10-09T06:00:06.909 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:06.909 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:06.909 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:06.909 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:40.506426+0000", 2023-10-09T06:00:06.910 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023961679999999999 2023-10-09T06:00:06.910 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.910 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.910 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.910 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:06.910 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:06.910 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:00:06.910 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:06.911 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:06 smithi191 ceph-mon[161669]: failed to encode map e476 with expected crc 2023-10-09T06:00:06.911 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:06 smithi191 ceph-mon[161669]: failed to encode map e476 with expected crc 2023-10-09T06:00:06.911 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:06 smithi191 ceph-mon[161669]: failed to encode map e477 with expected crc 2023-10-09T06:00:06.912 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:06 smithi191 ceph-mon[161669]: failed to encode map e477 with expected crc 2023-10-09T06:00:06.912 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:06 smithi191 ceph-mon[161669]: osdmap e477: 8 total, 8 up, 6 in 2023-10-09T06:00:06.912 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:06 smithi191 ceph-mon[161669]: 9.b continuing backfill to osd.1 from (368'1400,474'8613] MIN to 474'8613 2023-10-09T06:00:06.936 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[184418]: failed to encode map e476 with expected crc 2023-10-09T06:00:06.936 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[184418]: failed to encode map e476 with expected crc 2023-10-09T06:00:06.936 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[184418]: failed to encode map e477 with expected crc 2023-10-09T06:00:06.936 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[184418]: failed to encode map e477 with expected crc 2023-10-09T06:00:06.936 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[184418]: osdmap e477: 8 total, 8 up, 6 in 2023-10-09T06:00:06.937 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[184418]: 9.b continuing backfill to osd.1 from (368'1400,474'8613] MIN to 474'8613 2023-10-09T06:00:06.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[191227]: failed to encode map e476 with expected crc 2023-10-09T06:00:06.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[191227]: failed to encode map e476 with expected crc 2023-10-09T06:00:06.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[191227]: failed to encode map e477 with expected crc 2023-10-09T06:00:06.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[191227]: failed to encode map e477 with expected crc 2023-10-09T06:00:06.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[191227]: osdmap e477: 8 total, 8 up, 6 in 2023-10-09T06:00:06.937 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:06 smithi008 ceph-mon[191227]: 9.b continuing backfill to osd.1 from (368'1400,474'8613] MIN to 474'8613 2023-10-09T06:00:06.941 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:06.987 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:06.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:06.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:07.092 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:07.120 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:07.166 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:07.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:07.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:07.270 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:07.330 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:07.577 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:00:07.581 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:07.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:07.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:07.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:07.726 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:07.854 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:07.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:07.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:07.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:07 smithi191 ceph-mon[161669]: failed to encode map e477 with expected crc 2023-10-09T06:00:07.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:07 smithi191 ceph-mon[161669]: failed to encode map e477 with expected crc 2023-10-09T06:00:07.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:07 smithi191 ceph-mon[161669]: 4.5 continuing backfill to osd.0 from (0'0,0'0] MIN to 0'0 2023-10-09T06:00:07.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:07 smithi191 ceph-mon[161669]: 9.d starting backfill to osd.0 from (0'0,0'0] MAX to 474'8439 2023-10-09T06:00:07.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:07 smithi191 ceph-mon[161669]: pgmap v1427: 99 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 9 peering, 8 remapped+peering, 11 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 53 active+clean; 2.0 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 149 op/s; 35796/99606 objects degraded (35.938%); 14863/99606 objects misplaced (14.922%); 2.1 MiB/s, 38 objects/s recovering 2023-10-09T06:00:07.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:07 smithi191 ceph-mon[161669]: osdmap e478: 8 total, 8 up, 6 in 2023-10-09T06:00:07.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:07 smithi191 ceph-mon[161669]: failed to encode map e478 with expected crc 2023-10-09T06:00:07.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:07 smithi191 ceph-mon[161669]: failed to encode map e478 with expected crc 2023-10-09T06:00:07.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:07 smithi191 ceph-mon[161669]: failed to encode map e478 with expected crc 2023-10-09T06:00:07.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:07 smithi191 ceph-mon[161669]: failed to encode map e478 with expected crc 2023-10-09T06:00:07.963 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:07.993 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:08.137 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:08.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[184418]: failed to encode map e477 with expected crc 2023-10-09T06:00:08.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[184418]: failed to encode map e477 with expected crc 2023-10-09T06:00:08.157 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[184418]: 4.5 continuing backfill to osd.0 from (0'0,0'0] MIN to 0'0 2023-10-09T06:00:08.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[184418]: 9.d starting backfill to osd.0 from (0'0,0'0] MAX to 474'8439 2023-10-09T06:00:08.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[184418]: pgmap v1427: 99 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 9 peering, 8 remapped+peering, 11 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 53 active+clean; 2.0 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 149 op/s; 35796/99606 objects degraded (35.938%); 14863/99606 objects misplaced (14.922%); 2.1 MiB/s, 38 objects/s recovering 2023-10-09T06:00:08.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[184418]: osdmap e478: 8 total, 8 up, 6 in 2023-10-09T06:00:08.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[184418]: failed to encode map e478 with expected crc 2023-10-09T06:00:08.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[184418]: failed to encode map e478 with expected crc 2023-10-09T06:00:08.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[184418]: failed to encode map e478 with expected crc 2023-10-09T06:00:08.158 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[184418]: failed to encode map e478 with expected crc 2023-10-09T06:00:08.158 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[191227]: failed to encode map e477 with expected crc 2023-10-09T06:00:08.159 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[191227]: failed to encode map e477 with expected crc 2023-10-09T06:00:08.159 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[191227]: 4.5 continuing backfill to osd.0 from (0'0,0'0] MIN to 0'0 2023-10-09T06:00:08.159 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[191227]: 9.d starting backfill to osd.0 from (0'0,0'0] MAX to 474'8439 2023-10-09T06:00:08.159 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[191227]: pgmap v1427: 99 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 9 peering, 8 remapped+peering, 11 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 53 active+clean; 2.0 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 149 op/s; 35796/99606 objects degraded (35.938%); 14863/99606 objects misplaced (14.922%); 2.1 MiB/s, 38 objects/s recovering 2023-10-09T06:00:08.159 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[191227]: osdmap e478: 8 total, 8 up, 6 in 2023-10-09T06:00:08.159 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[191227]: failed to encode map e478 with expected crc 2023-10-09T06:00:08.159 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[191227]: failed to encode map e478 with expected crc 2023-10-09T06:00:08.159 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[191227]: failed to encode map e478 with expected crc 2023-10-09T06:00:08.159 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:07 smithi008 ceph-mon[191227]: failed to encode map e478 with expected crc 2023-10-09T06:00:08.183 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:08.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:08.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:08.289 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:08.334 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:08.395 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:00:08.421 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:08.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:08.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:08.531 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:08.568 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:08.722 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:00:08.743 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:00:08.765 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:00:08.777 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:08.791 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:00:08.808 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:08.813 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:00:08.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:08.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:08.845 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:00:08.864 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:00:08.882 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:00:08.884 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:00:08.884 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:00:08.884 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:00:08.884 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:00:08.884 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:00:08.884 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:00:08.884 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:00:08.884 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:00:08.916 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:08.946 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:09.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:08 smithi191 ceph-mon[161669]: failed to encode map e478 with expected crc 2023-10-09T06:00:09.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:08 smithi191 ceph-mon[161669]: failed to encode map e478 with expected crc 2023-10-09T06:00:09.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 35796/99606 objects degraded (35.938%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:09.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/532784811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:09.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:08 smithi008 ceph-mon[184418]: failed to encode map e478 with expected crc 2023-10-09T06:00:09.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:08 smithi008 ceph-mon[184418]: failed to encode map e478 with expected crc 2023-10-09T06:00:09.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 35796/99606 objects degraded (35.938%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:09.095 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/532784811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:09.096 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:08 smithi008 ceph-mon[191227]: failed to encode map e478 with expected crc 2023-10-09T06:00:09.096 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:08 smithi008 ceph-mon[191227]: failed to encode map e478 with expected crc 2023-10-09T06:00:09.096 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 35796/99606 objects degraded (35.938%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:09.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/532784811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:09.097 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:09.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:09.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:09.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:09.240 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:09.431 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:09.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:09.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:09.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:09.590 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:09.791 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:09.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:09.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:09.895 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:09.929 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:09.982 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:09.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:09.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:10.054 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:00:10.054 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:00:10.085 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:10.113 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:10.158 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:10.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:10.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:10.262 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:10.290 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:10.522 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:10 smithi008 ceph-mon[184418]: pgmap v1429: 99 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 9 peering, 8 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 174 op/s; 37722/99081 objects degraded (38.072%); 12812/99081 objects misplaced (12.931%); 2.9 MiB/s, 51 objects/s recovering 2023-10-09T06:00:10.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:10 smithi008 ceph-mon[191227]: pgmap v1429: 99 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 9 peering, 8 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 174 op/s; 37722/99081 objects degraded (38.072%); 12812/99081 objects misplaced (12.931%); 2.9 MiB/s, 51 objects/s recovering 2023-10-09T06:00:10.523 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:00:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:00:10] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.33.4" 2023-10-09T06:00:10.542 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:10.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:10.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:10 smithi191 ceph-mon[161669]: pgmap v1429: 99 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 9 peering, 8 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 174 op/s; 37722/99081 objects degraded (38.072%); 12812/99081 objects misplaced (12.931%); 2.9 MiB/s, 51 objects/s recovering 2023-10-09T06:00:10.650 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:10.679 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:10.777 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:10.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:10.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:10.839 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:00:10.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:10.940 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:11.035 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:11.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:11.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:11.200 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:11.295 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:00:11.407 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:11.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:11.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:11.453 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:11.516 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:11.542 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2690318208' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:11.542 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:11 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:11.542 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2690318208' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:11.543 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:11 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:11.545 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:11.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2690318208' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:11.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:11 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:11.598 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:11.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:11.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:11.651 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:00:11.651 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:00:11.651 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:00:11.652 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.997197 2023-10-09T06:00:11.652 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0.9971974510136009 2023-10-09T06:00:11.702 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:11.744 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:11.799 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:11.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:11.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:11.903 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:11.940 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:12.040 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:00:12.267 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:12.316 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:12.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:12.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:12.422 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:12.448 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[184418]: pgmap v1430: 99 pgs: 9 peering, 8 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 37722/99081 objects degraded (38.072%); 12812/99081 objects misplaced (12.931%); 2.1 MiB/s, 37 objects/s recovering 2023-10-09T06:00:12.448 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:12.448 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:00:12.448 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[184418]: osdmap e479: 8 total, 8 up, 6 in 2023-10-09T06:00:12.448 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[184418]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.448 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[184418]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.449 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[184418]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.449 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[184418]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.449 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[191227]: pgmap v1430: 99 pgs: 9 peering, 8 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 37722/99081 objects degraded (38.072%); 12812/99081 objects misplaced (12.931%); 2.1 MiB/s, 37 objects/s recovering 2023-10-09T06:00:12.449 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:12.449 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:00:12.449 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[191227]: osdmap e479: 8 total, 8 up, 6 in 2023-10-09T06:00:12.449 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[191227]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.449 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[191227]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.449 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[191227]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.449 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:12 smithi008 ceph-mon[191227]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.451 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:12.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:12 smithi191 ceph-mon[161669]: pgmap v1430: 99 pgs: 9 peering, 8 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 37722/99081 objects degraded (38.072%); 12812/99081 objects misplaced (12.931%); 2.1 MiB/s, 37 objects/s recovering 2023-10-09T06:00:12.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:12 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:12.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:12 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:00:12.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:12 smithi191 ceph-mon[161669]: osdmap e479: 8 total, 8 up, 6 in 2023-10-09T06:00:12.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:12 smithi191 ceph-mon[161669]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:12 smithi191 ceph-mon[161669]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:12 smithi191 ceph-mon[161669]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:12 smithi191 ceph-mon[161669]: failed to encode map e479 with expected crc 2023-10-09T06:00:12.611 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:12.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:12.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:12.715 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:12.754 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:12.819 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:12.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:12.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:12.931 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:12.973 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:13.094 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:13.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:13.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:13.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:13.241 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:13.314 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:00:13.314 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:00:13.316 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:13.322 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 37722/99081 objects degraded (38.072%), 24 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:13.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:13.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:13.423 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:13.453 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:13.508 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:13.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:13.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:13.548 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:13.613 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:13.639 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:13.682 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:13.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:13.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:13.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 37722/99081 objects degraded (38.072%), 24 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:13.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 37722/99081 objects degraded (38.072%), 24 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:13.786 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:13.821 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:13.924 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:13.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:13.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:14.028 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:14.060 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:14.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:00:14.112 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:14.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:14.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:14.194 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:00:14.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:14.249 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:14.316 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.0 primary-affinity to 0.997197 (8653522) 2023-10-09T06:00:14.365 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:14.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:14.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:14.389 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:14 smithi191 ceph-mon[161669]: pgmap v1432: 99 pgs: 1 activating+undersized+degraded+remapped, 9 peering, 7 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 27 op/s; 42292/99081 objects degraded (42.684%); 12812/99081 objects misplaced (12.931%) 2023-10-09T06:00:14.389 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:14.389 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:14 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4019788767' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9971974510136009}]: dispatch 2023-10-09T06:00:14.489 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:14.523 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:14.539 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:14.580 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:14.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:14.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:14.686 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:14.713 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:14 smithi008 ceph-mon[184418]: pgmap v1432: 99 pgs: 1 activating+undersized+degraded+remapped, 9 peering, 7 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 27 op/s; 42292/99081 objects degraded (42.684%); 12812/99081 objects misplaced (12.931%) 2023-10-09T06:00:14.713 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:14.713 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:14 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4019788767' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9971974510136009}]: dispatch 2023-10-09T06:00:14.713 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:14 smithi008 ceph-mon[191227]: pgmap v1432: 99 pgs: 1 activating+undersized+degraded+remapped, 9 peering, 7 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 27 op/s; 42292/99081 objects degraded (42.684%); 12812/99081 objects misplaced (12.931%) 2023-10-09T06:00:14.713 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:14.713 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:14 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4019788767' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9971974510136009}]: dispatch 2023-10-09T06:00:14.715 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:15.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:15 smithi008 ceph-mon[184418]: failed to encode map e480 with expected crc 2023-10-09T06:00:15.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4019788767' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9971974510136009}]': finished 2023-10-09T06:00:15.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:15 smithi008 ceph-mon[184418]: osdmap e480: 8 total, 8 up, 6 in 2023-10-09T06:00:15.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:15 smithi008 ceph-mon[184418]: failed to encode map e480 with expected crc 2023-10-09T06:00:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:15 smithi008 ceph-mon[191227]: failed to encode map e480 with expected crc 2023-10-09T06:00:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4019788767' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9971974510136009}]': finished 2023-10-09T06:00:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:15 smithi008 ceph-mon[191227]: osdmap e480: 8 total, 8 up, 6 in 2023-10-09T06:00:15.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:15 smithi008 ceph-mon[191227]: failed to encode map e480 with expected crc 2023-10-09T06:00:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:15 smithi191 ceph-mon[161669]: failed to encode map e480 with expected crc 2023-10-09T06:00:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4019788767' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9971974510136009}]': finished 2023-10-09T06:00:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:15 smithi191 ceph-mon[161669]: osdmap e480: 8 total, 8 up, 6 in 2023-10-09T06:00:15.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:15 smithi191 ceph-mon[161669]: failed to encode map e480 with expected crc 2023-10-09T06:00:15.571 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:15.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:15.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:15.676 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:15.712 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:15.766 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:15.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:15.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:15.871 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:15.905 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:15.929 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:15.965 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:15.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:15.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:16.030 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:00:16.048 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:00:16.066 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:00:16.071 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:16.086 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:00:16.103 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:16.109 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:00:16.129 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:00:16.147 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:00:16.147 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T06:00:16.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T06:00:16.147 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T06:00:16.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.147 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:45770 9.a 9:54ab40b6:::benchmark_data_smithi008_502820_object45769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e372)", 2023-10-09T06:00:16.147 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:53:46.056257+0000", 2023-10-09T06:00:16.147 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 390.08740042099998, 2023-10-09T06:00:16.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0314687650000001, 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 45770 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056257+0000", 2023-10-09T06:00:16.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056257+0000", 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999952 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056252+0000", 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2493e-05 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056305+0000", 2023-10-09T06:00:16.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6889e-05 2023-10-09T06:00:16.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056322+0000", 2023-10-09T06:00:16.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3180000000000001e-06 2023-10-09T06:00:16.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056329+0000", 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6201e-05 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056375+0000", 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7790000000000007e-06 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.056385+0000", 2023-10-09T06:00:16.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88098339699999995 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.937368+0000", 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1769e-05 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.937380+0000", 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1403479299999999 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.077728+0000", 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000195169 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.077923+0000", 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015467500000000001 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:00:16.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.078078+0000", 2023-10-09T06:00:16.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0036624750000000001 2023-10-09T06:00:16.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.081740+0000", 2023-10-09T06:00:16.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003406278 2023-10-09T06:00:16.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.085147+0000", 2023-10-09T06:00:16.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00062187400000000002 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.085768+0000", 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7609999999999997e-05 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.085846+0000", 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018796590000000001 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.087726+0000", 2023-10-09T06:00:16.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0314687650000001 2023-10-09T06:00:16.156 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.156 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.156 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.156 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:77607 9.7 9:e59624ae:::benchmark_data_smithi008_502820_object77606:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T06:00:16.156 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.819106+0000", 2023-10-09T06:00:16.156 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 339.324551063, 2023-10-09T06:00:16.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585100272, 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 77607 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819106+0000", 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819106+0000", 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819106+0000", 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1301000000000002e-05 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819157+0000", 2023-10-09T06:00:16.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6730000000000003e-06 2023-10-09T06:00:16.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819164+0000", 2023-10-09T06:00:16.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6500000000000004e-06 2023-10-09T06:00:16.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819168+0000", 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.052763364 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871932+0000", 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.26e-05 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871944+0000", 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81505207000000002 2023-10-09T06:00:16.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.686996+0000", 2023-10-09T06:00:16.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.464e-05 2023-10-09T06:00:16.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.687011+0000", 2023-10-09T06:00:16.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.70573758499999995 2023-10-09T06:00:16.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.392749+0000", 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00064714099999999995 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393396+0000", 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010737744 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404133+0000", 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3168000000000001e-05 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404207+0000", 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585100272 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.163 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:77621 9.7 9:e7b7c537:::benchmark_data_smithi008_502820_object77620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.819295+0000", 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 339.32436274200001, 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585638342, 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 77621 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819295+0000", 2023-10-09T06:00:16.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819295+0000", 2023-10-09T06:00:16.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:16.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819294+0000", 2023-10-09T06:00:16.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1934000000000002e-05 2023-10-09T06:00:16.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819336+0000", 2023-10-09T06:00:16.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5550000000000002e-06 2023-10-09T06:00:16.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819342+0000", 2023-10-09T06:00:16.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.766e-06 2023-10-09T06:00:16.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819345+0000", 2023-10-09T06:00:16.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.052626693000000002 2023-10-09T06:00:16.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871972+0000", 2023-10-09T06:00:16.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2029999999999998e-06 2023-10-09T06:00:16.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.871981+0000", 2023-10-09T06:00:16.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81507707100000004 2023-10-09T06:00:16.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.687058+0000", 2023-10-09T06:00:16.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7999999999999993e-06 2023-10-09T06:00:16.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.687068+0000", 2023-10-09T06:00:16.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.70636606700000004 2023-10-09T06:00:16.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393434+0000", 2023-10-09T06:00:16.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011293 2023-10-09T06:00:16.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393547+0000", 2023-10-09T06:00:16.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000142881 2023-10-09T06:00:16.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:00:16.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.393690+0000", 2023-10-09T06:00:16.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0044343000000000004 2023-10-09T06:00:16.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.398124+0000", 2023-10-09T06:00:16.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6621e-05 2023-10-09T06:00:16.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.398171+0000", 2023-10-09T06:00:16.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060740669999999998 2023-10-09T06:00:16.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404245+0000", 2023-10-09T06:00:16.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8288999999999996e-05 2023-10-09T06:00:16.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404313+0000", 2023-10-09T06:00:16.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00061965000000000002 2023-10-09T06:00:16.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:38.404933+0000", 2023-10-09T06:00:16.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.585638342 2023-10-09T06:00:16.173 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.173 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.173 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.173 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:99261 9.a 9:56a2e757:::benchmark_data_smithi008_502820_object15088:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e406)", 2023-10-09T06:00:16.173 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:39.286092+0000", 2023-10-09T06:00:16.173 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 276.85756538300001, 2023-10-09T06:00:16.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.036949087, 2023-10-09T06:00:16.174 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.174 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.174 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.174 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.174 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.174 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99261 2023-10-09T06:00:16.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.174 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286092+0000", 2023-10-09T06:00:16.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286092+0000", 2023-10-09T06:00:16.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:16.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286092+0000", 2023-10-09T06:00:16.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.79e-06 2023-10-09T06:00:16.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286094+0000", 2023-10-09T06:00:16.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9939999999999997e-06 2023-10-09T06:00:16.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286100+0000", 2023-10-09T06:00:16.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.138e-06 2023-10-09T06:00:16.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286105+0000", 2023-10-09T06:00:16.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5616000000000001e-05 2023-10-09T06:00:16.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286200+0000", 2023-10-09T06:00:16.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0020000000000003e-06 2023-10-09T06:00:16.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286206+0000", 2023-10-09T06:00:16.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0250316589999999 2023-10-09T06:00:16.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.311238+0000", 2023-10-09T06:00:16.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000140032 2023-10-09T06:00:16.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.311378+0000", 2023-10-09T06:00:16.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000164034 2023-10-09T06:00:16.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T06:00:16.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.311542+0000", 2023-10-09T06:00:16.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011212313999999999 2023-10-09T06:00:16.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322754+0000", 2023-10-09T06:00:16.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3466e-05 2023-10-09T06:00:16.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322798+0000", 2023-10-09T06:00:16.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4289000000000001e-05 2023-10-09T06:00:16.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322852+0000", 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9132999999999999e-05 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322901+0000", 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9549999999999999e-05 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322941+0000", 2023-10-09T06:00:16.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100512 2023-10-09T06:00:16.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323041+0000", 2023-10-09T06:00:16.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.036949087 2023-10-09T06:00:16.182 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.182 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.182 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.183 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:99272 9.a 9:54aba215:::benchmark_data_smithi008_502820_object15099:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e406)", 2023-10-09T06:00:16.183 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:39.286108+0000", 2023-10-09T06:00:16.183 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 276.85754893400002, 2023-10-09T06:00:16.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0372746719999999, 2023-10-09T06:00:16.183 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.183 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.183 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.183 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.183 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99272 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286108+0000", 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286108+0000", 2023-10-09T06:00:16.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:16.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286108+0000", 2023-10-09T06:00:16.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2369999999999999e-06 2023-10-09T06:00:16.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286110+0000", 2023-10-09T06:00:16.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.49e-06 2023-10-09T06:00:16.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286116+0000", 2023-10-09T06:00:16.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7799999999999998e-06 2023-10-09T06:00:16.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286120+0000", 2023-10-09T06:00:16.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106143 2023-10-09T06:00:16.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286226+0000", 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6489999999999996e-06 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.286231+0000", 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0284209600000001 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.314652+0000", 2023-10-09T06:00:16.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016349299999999999 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.314816+0000", 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019973899999999999 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.315016+0000", 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0077500399999999997 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.322766+0000", 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000319585 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323085+0000", 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9708e-05 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323125+0000", 2023-10-09T06:00:16.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2876e-05 2023-10-09T06:00:16.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323178+0000", 2023-10-09T06:00:16.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2748000000000003e-05 2023-10-09T06:00:16.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323241+0000", 2023-10-09T06:00:16.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014258200000000001 2023-10-09T06:00:16.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:40.323383+0000", 2023-10-09T06:00:16.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0372746719999999 2023-10-09T06:00:16.191 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.191 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.191 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.192 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:103011 9.15 9:ad5520b2:::benchmark_data_smithi008_502820_object18838:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e420)", 2023-10-09T06:00:16.192 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:56:11.329331+0000", 2023-10-09T06:00:16.192 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 244.814326784, 2023-10-09T06:00:16.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.236479597, 2023-10-09T06:00:16.192 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.192 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.192 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 103011 2023-10-09T06:00:16.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.193 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329331+0000", 2023-10-09T06:00:16.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329331+0000", 2023-10-09T06:00:16.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:00:16.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329328+0000", 2023-10-09T06:00:16.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.61e-06 2023-10-09T06:00:16.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329336+0000", 2023-10-09T06:00:16.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1816e-05 2023-10-09T06:00:16.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329348+0000", 2023-10-09T06:00:16.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9629999999999998e-06 2023-10-09T06:00:16.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.329356+0000", 2023-10-09T06:00:16.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033304369999999999 2023-10-09T06:00:16.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332686+0000", 2023-10-09T06:00:16.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0757000000000001e-05 2023-10-09T06:00:16.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332697+0000", 2023-10-09T06:00:16.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2147822829999999 2023-10-09T06:00:16.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.547479+0000", 2023-10-09T06:00:16.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014378700000000001 2023-10-09T06:00:16.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.547623+0000", 2023-10-09T06:00:16.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118553 2023-10-09T06:00:16.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:00:16.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.547742+0000", 2023-10-09T06:00:16.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00508282 2023-10-09T06:00:16.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.552824+0000", 2023-10-09T06:00:16.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011131831 2023-10-09T06:00:16.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.563956+0000", 2023-10-09T06:00:16.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3128999999999998e-05 2023-10-09T06:00:16.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.564019+0000", 2023-10-09T06:00:16.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9806999999999998e-05 2023-10-09T06:00:16.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.564079+0000", 2023-10-09T06:00:16.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017310330000000001 2023-10-09T06:00:16.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.565810+0000", 2023-10-09T06:00:16.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.236479597 2023-10-09T06:00:16.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.200 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.201 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.201 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:103018 9.15 9:a99fcd49:::benchmark_data_smithi008_502820_object18845:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e420)", 2023-10-09T06:00:16.201 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:56:11.332182+0000", 2023-10-09T06:00:16.201 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 244.811475126, 2023-10-09T06:00:16.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2352073649999999, 2023-10-09T06:00:16.201 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.201 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.202 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.202 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.202 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 103018 2023-10-09T06:00:16.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.202 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332182+0000", 2023-10-09T06:00:16.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332182+0000", 2023-10-09T06:00:16.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:00:16.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332181+0000", 2023-10-09T06:00:16.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.511e-06 2023-10-09T06:00:16.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332188+0000", 2023-10-09T06:00:16.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1018e-05 2023-10-09T06:00:16.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332199+0000", 2023-10-09T06:00:16.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3490000000000003e-06 2023-10-09T06:00:16.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332206+0000", 2023-10-09T06:00:16.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00051741500000000002 2023-10-09T06:00:16.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332723+0000", 2023-10-09T06:00:16.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6009999999999997e-06 2023-10-09T06:00:16.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332731+0000", 2023-10-09T06:00:16.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.216443433 2023-10-09T06:00:16.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.549174+0000", 2023-10-09T06:00:16.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016414700000000001 2023-10-09T06:00:16.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.549339+0000", 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111173 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.549450+0000", 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014515375 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.563965+0000", 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001919088 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.565884+0000", 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017655200000000001 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.566061+0000", 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9567000000000007e-05 2023-10-09T06:00:16.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.566130+0000", 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012592809999999999 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.567390+0000", 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2352073649999999 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.210 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:103025 9.15 9:aa461274:::benchmark_data_smithi008_502820_object18852:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e420)", 2023-10-09T06:00:16.210 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:56:11.332212+0000", 2023-10-09T06:00:16.210 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 244.81144522899999, 2023-10-09T06:00:16.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.236778771, 2023-10-09T06:00:16.210 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.210 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.210 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.210 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.210 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.210 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 103025 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332212+0000", 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332212+0000", 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:16.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332212+0000", 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.534e-06 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332215+0000", 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5549999999999999e-06 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332222+0000", 2023-10-09T06:00:16.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4190000000000004e-06 2023-10-09T06:00:16.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332226+0000", 2023-10-09T06:00:16.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00089926800000000005 2023-10-09T06:00:16.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.333125+0000", 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1258e-05 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.333137+0000", 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2177126110000001 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.550849+0000", 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014004999999999999 2023-10-09T06:00:16.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.550989+0000", 2023-10-09T06:00:16.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015518300000000001 2023-10-09T06:00:16.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:00:16.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.551145+0000", 2023-10-09T06:00:16.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012827781999999999 2023-10-09T06:00:16.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.563972+0000", 2023-10-09T06:00:16.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019446349999999999 2023-10-09T06:00:16.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.565917+0000", 2023-10-09T06:00:16.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001559324 2023-10-09T06:00:16.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.567476+0000", 2023-10-09T06:00:16.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3046000000000006e-05 2023-10-09T06:00:16.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.567539+0000", 2023-10-09T06:00:16.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014516080000000001 2023-10-09T06:00:16.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.568991+0000", 2023-10-09T06:00:16.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.236778771 2023-10-09T06:00:16.218 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.218 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.218 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.218 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:103039 9.5 9:a61879d9:::benchmark_data_smithi008_502820_object18866:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e420)", 2023-10-09T06:00:16.218 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:56:11.332231+0000", 2023-10-09T06:00:16.218 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 244.811426101, 2023-10-09T06:00:16.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2042948360000001, 2023-10-09T06:00:16.219 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.219 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.219 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.219 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.219 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.219 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 103039 2023-10-09T06:00:16.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.219 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332231+0000", 2023-10-09T06:00:16.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332231+0000", 2023-10-09T06:00:16.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:16.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332231+0000", 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9050000000000001e-06 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332234+0000", 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0390000000000001e-06 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332240+0000", 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9910000000000004e-06 2023-10-09T06:00:16.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.332244+0000", 2023-10-09T06:00:16.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011900470000000001 2023-10-09T06:00:16.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.333434+0000", 2023-10-09T06:00:16.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3599999999999996e-06 2023-10-09T06:00:16.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:11.333442+0000", 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.189327544 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.522770+0000", 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015325399999999999 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.522923+0000", 2023-10-09T06:00:16.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012667599999999999 2023-10-09T06:00:16.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:00:16.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.523050+0000", 2023-10-09T06:00:16.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0097749440000000007 2023-10-09T06:00:16.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.532825+0000", 2023-10-09T06:00:16.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021428039999999999 2023-10-09T06:00:16.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.534967+0000", 2023-10-09T06:00:16.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7425000000000001e-05 2023-10-09T06:00:16.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.535015+0000", 2023-10-09T06:00:16.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6205999999999995e-05 2023-10-09T06:00:16.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.535091+0000", 2023-10-09T06:00:16.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001435052 2023-10-09T06:00:16.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:56:12.536526+0000", 2023-10-09T06:00:16.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2042948360000001 2023-10-09T06:00:16.226 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.226 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.227 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.227 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117299 9.d 9:b9332d7c:::benchmark_data_smithi008_502820_object33126:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:00:16.227 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.239953+0000", 2023-10-09T06:00:16.227 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 138.903704522, 2023-10-09T06:00:16.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2374957609999999, 2023-10-09T06:00:16.227 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.227 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.227 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.227 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.228 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117299 2023-10-09T06:00:16.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.228 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.239953+0000", 2023-10-09T06:00:16.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.239953+0000", 2023-10-09T06:00:16.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:16.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.239951+0000", 2023-10-09T06:00:16.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9929999999999998e-06 2023-10-09T06:00:16.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.239958+0000", 2023-10-09T06:00:16.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4627000000000001e-05 2023-10-09T06:00:16.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.239973+0000", 2023-10-09T06:00:16.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.242e-06 2023-10-09T06:00:16.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.239981+0000", 2023-10-09T06:00:16.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7547999999999998e-05 2023-10-09T06:00:16.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240049+0000", 2023-10-09T06:00:16.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9760000000000001e-06 2023-10-09T06:00:16.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T06:00:16.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240058+0000", 2023-10-09T06:00:16.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010402211999999999 2023-10-09T06:00:16.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.250460+0000", 2023-10-09T06:00:16.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0801999999999999e-05 2023-10-09T06:00:16.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.250471+0000", 2023-10-09T06:00:16.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.39885327 2023-10-09T06:00:16.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.649324+0000", 2023-10-09T06:00:16.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00081378699999999995 2023-10-09T06:00:16.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:16.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.650138+0000", 2023-10-09T06:00:16.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014150072 2023-10-09T06:00:16.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.664288+0000", 2023-10-09T06:00:16.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000133044 2023-10-09T06:00:16.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.664421+0000", 2023-10-09T06:00:16.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00022372900000000001 2023-10-09T06:00:16.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6,7", 2023-10-09T06:00:16.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.664645+0000", 2023-10-09T06:00:16.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012753006000000001 2023-10-09T06:00:16.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.677398+0000", 2023-10-09T06:00:16.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0093915460000000006 2023-10-09T06:00:16.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.236 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.686789+0000", 2023-10-09T06:00:16.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0630000000000006e-05 2023-10-09T06:00:16.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.236 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.686860+0000", 2023-10-09T06:00:16.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78747541399999998 2023-10-09T06:00:16.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.237 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.237 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.474335+0000", 2023-10-09T06:00:16.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7182999999999998e-05 2023-10-09T06:00:16.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.237 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.237 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.474393+0000", 2023-10-09T06:00:16.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003056072 2023-10-09T06:00:16.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.477449+0000", 2023-10-09T06:00:16.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2374957609999999 2023-10-09T06:00:16.238 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.238 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.238 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.239 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117309 9.d 9:bba9f6eb:::benchmark_data_smithi008_502820_object33136:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:00:16.239 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.240000+0000", 2023-10-09T06:00:16.239 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 138.903657724, 2023-10-09T06:00:16.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2408529690000005, 2023-10-09T06:00:16.239 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.239 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.239 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.239 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117309 2023-10-09T06:00:16.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240000+0000", 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240000+0000", 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.239999+0000", 2023-10-09T06:00:16.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6289999999999999e-06 2023-10-09T06:00:16.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240003+0000", 2023-10-09T06:00:16.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8720000000000002e-06 2023-10-09T06:00:16.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240011+0000", 2023-10-09T06:00:16.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5940000000000004e-06 2023-10-09T06:00:16.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240015+0000", 2023-10-09T06:00:16.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00047225700000000002 2023-10-09T06:00:16.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240487+0000", 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1701e-05 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting_for_map not empty", 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240499+0000", 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099933490000000003 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.250493+0000", 2023-10-09T06:00:16.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6850000000000001e-06 2023-10-09T06:00:16.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.250500+0000", 2023-10-09T06:00:16.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3996988879999996 2023-10-09T06:00:16.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.650199+0000", 2023-10-09T06:00:16.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00071234200000000003 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.650911+0000", 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022619209000000001 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.673531+0000", 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011534100000000001 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.673646+0000", 2023-10-09T06:00:16.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000182163 2023-10-09T06:00:16.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6,7", 2023-10-09T06:00:16.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.673828+0000", 2023-10-09T06:00:16.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012998302999999999 2023-10-09T06:00:16.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.686826+0000", 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9811e-05 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.686896+0000", 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024109829999999998 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.689307+0000", 2023-10-09T06:00:16.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.78822582500000005 2023-10-09T06:00:16.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.477533+0000", 2023-10-09T06:00:16.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2354000000000005e-05 2023-10-09T06:00:16.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.477615+0000", 2023-10-09T06:00:16.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032372429999999999 2023-10-09T06:00:16.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.480853+0000", 2023-10-09T06:00:16.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2408529690000005 2023-10-09T06:00:16.249 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.249 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.249 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.249 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117311 9.d 9:b6be4fdf:::benchmark_data_smithi008_502820_object33138:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:00:16.249 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.240027+0000", 2023-10-09T06:00:16.250 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 138.90362998500001, 2023-10-09T06:00:16.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2537092140000006, 2023-10-09T06:00:16.250 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.250 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.250 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.250 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.250 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.250 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117311 2023-10-09T06:00:16.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.250 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240027+0000", 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240027+0000", 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240027+0000", 2023-10-09T06:00:16.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0280000000000001e-06 2023-10-09T06:00:16.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240030+0000", 2023-10-09T06:00:16.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9289999999999997e-06 2023-10-09T06:00:16.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240035+0000", 2023-10-09T06:00:16.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8419999999999998e-06 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240039+0000", 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091953800000000002 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240958+0000", 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6215999999999999e-05 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting_for_map not empty", 2023-10-09T06:00:16.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240975+0000", 2023-10-09T06:00:16.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0095457240000000002 2023-10-09T06:00:16.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.250520+0000", 2023-10-09T06:00:16.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3939999999999998e-06 2023-10-09T06:00:16.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:16.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.250528+0000", 2023-10-09T06:00:16.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4004196110000002 2023-10-09T06:00:16.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.650947+0000", 2023-10-09T06:00:16.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000668022 2023-10-09T06:00:16.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:16.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.651615+0000", 2023-10-09T06:00:16.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.031787582000000002 2023-10-09T06:00:16.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.683403+0000", 2023-10-09T06:00:16.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111871 2023-10-09T06:00:16.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.683515+0000", 2023-10-09T06:00:16.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020398200000000001 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6,7", 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.683719+0000", 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0056430760000000003 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.689362+0000", 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8331000000000001e-05 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.689380+0000", 2023-10-09T06:00:16.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00043203399999999998 2023-10-09T06:00:16.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:04.689812+0000", 2023-10-09T06:00:16.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.80060488500000004 2023-10-09T06:00:16.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.490417+0000", 2023-10-09T06:00:16.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7280000000000005e-05 2023-10-09T06:00:16.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.490504+0000", 2023-10-09T06:00:16.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032322689999999999 2023-10-09T06:00:16.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.493737+0000", 2023-10-09T06:00:16.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2537092140000006 2023-10-09T06:00:16.260 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.260 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.260 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.260 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:120723 9.18 9:1a3ebf5d:::benchmark_data_smithi008_502820_object36550:head [delete] snapc 0=[] ondisk+write+known_if_redirected e457)", 2023-10-09T06:00:16.260 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:46.152494+0000", 2023-10-09T06:00:16.260 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 89.991163224999994, 2023-10-09T06:00:16.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23981081400000001, 2023-10-09T06:00:16.260 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 120723 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.152494+0000", 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.152494+0000", 2023-10-09T06:00:16.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:16.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.152492+0000", 2023-10-09T06:00:16.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8980000000000008e-06 2023-10-09T06:00:16.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.152502+0000", 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5447999999999998e-05 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.152518+0000", 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8530000000000003e-06 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.152525+0000", 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9047999999999993e-05 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.152594+0000", 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000959592 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.153553+0000", 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23170021599999999 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.385253+0000", 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012529199999999999 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.385379+0000", 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000210919 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.385590+0000", 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003236048 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.388826+0000", 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001068619 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.389894+0000", 2023-10-09T06:00:16.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021291699999999999 2023-10-09T06:00:16.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.267 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.390107+0000", 2023-10-09T06:00:16.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001206497 2023-10-09T06:00:16.267 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.267 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.267 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.391314+0000", 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7663e-05 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.391371+0000", 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00093367999999999999 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.392305+0000", 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23981081400000001 2023-10-09T06:00:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:120821 9.18 9:1fa9e708:::benchmark_data_smithi008_502820_object36648:head [delete] snapc 0=[] ondisk+write+known_if_redirected e457)", 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:46.928258+0000", 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 89.215399497000007, 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024973414999999999, 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.269 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.270 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.270 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 120821 2023-10-09T06:00:16.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.270 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.928258+0000", 2023-10-09T06:00:16.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.928258+0000", 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.928256+0000", 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5930000000000006e-06 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.928265+0000", 2023-10-09T06:00:16.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6163000000000001e-05 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.928281+0000", 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4599999999999997e-06 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.928288+0000", 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9182999999999999e-05 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.928347+0000", 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00092315400000000003 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.929271+0000", 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015834721999999999 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.945105+0000", 2023-10-09T06:00:16.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012255400000000001 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.945228+0000", 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018744000000000001 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.945415+0000", 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028861440000000002 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.948301+0000", 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000503745 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.948805+0000", 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001038892 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.949844+0000", 2023-10-09T06:00:16.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025156570000000001 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.952360+0000", 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9152000000000001e-05 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.952399+0000", 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00083242899999999998 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:46.953231+0000", 2023-10-09T06:00:16.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024973414999999999 2023-10-09T06:00:16.277 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.277 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.277 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.277 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:123023 9.a 9:505bb0dd:::benchmark_data_smithi008_502820_object38850:head [delete] snapc 0=[] ondisk+write+known_if_redirected e460)", 2023-10-09T06:00:16.277 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:00.940597+0000", 2023-10-09T06:00:16.277 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 75.203059933000006, 2023-10-09T06:00:16.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.089505711000000002, 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 123023 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.940597+0000", 2023-10-09T06:00:16.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.940597+0000", 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.940596+0000", 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8084000000000002e-05 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.940614+0000", 2023-10-09T06:00:16.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4202999999999999e-05 2023-10-09T06:00:16.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.940628+0000", 2023-10-09T06:00:16.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7000000000000002e-06 2023-10-09T06:00:16.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.940635+0000", 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9931000000000004e-05 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.940715+0000", 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016388700000000001 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.940878+0000", 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019279599999999999 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T06:00:16.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.941071+0000", 2023-10-09T06:00:16.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038054 2023-10-09T06:00:16.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.944877+0000", 2023-10-09T06:00:16.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00044292700000000002 2023-10-09T06:00:16.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.945320+0000", 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037519239999999998 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:00.949071+0000", 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.080801810000000002 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.029873+0000", 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5421999999999998e-05 2023-10-09T06:00:16.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.029939+0000", 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016447200000000001 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.030103+0000", 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.089505711000000002 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:123133 9.a 9:5897ce88:::benchmark_data_smithi008_502820_object38960:head [delete] snapc 0=[] ondisk+write+known_if_redirected e461)", 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:01.569114+0000", 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 74.574543495, 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.107822049, 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 123133 2023-10-09T06:00:16.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.569114+0000", 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.569114+0000", 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.569112+0000", 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3810000000000005e-06 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.569121+0000", 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5739999999999998e-05 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.569137+0000", 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8140000000000001e-06 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.569145+0000", 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9509000000000003e-05 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.569194+0000", 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119002 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.569313+0000", 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000121426 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.569435+0000", 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032515539999999998 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.572686+0000", 2023-10-09T06:00:16.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115858 2023-10-09T06:00:16.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.572802+0000", 2023-10-09T06:00:16.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018558279999999999 2023-10-09T06:00:16.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.574658+0000", 2023-10-09T06:00:16.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.10203684 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.676695+0000", 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4875000000000004e-05 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.676770+0000", 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016629000000000001 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:01.676936+0000", 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.107822049 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:123228 9.18 9:1a7393c3:::benchmark_data_smithi008_502820_object39055:head [delete] snapc 0=[] ondisk+write+known_if_redirected e461)", 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:02.109404+0000", 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 74.034253441000004, 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.026269019000000001, 2023-10-09T06:00:16.292 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 123228 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.109404+0000", 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.109404+0000", 2023-10-09T06:00:16.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:16.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.109402+0000", 2023-10-09T06:00:16.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.645e-06 2023-10-09T06:00:16.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.109412+0000", 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2918999999999999e-05 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.109425+0000", 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4710000000000004e-06 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.109431+0000", 2023-10-09T06:00:16.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0065338779999999999 2023-10-09T06:00:16.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.115965+0000", 2023-10-09T06:00:16.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0001817 2023-10-09T06:00:16.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.116147+0000", 2023-10-09T06:00:16.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020752800000000001 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.116354+0000", 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0054816839999999997 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.121836+0000", 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00083925999999999998 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.122675+0000", 2023-10-09T06:00:16.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01097393 2023-10-09T06:00:16.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.133649+0000", 2023-10-09T06:00:16.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010052489999999999 2023-10-09T06:00:16.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.134654+0000", 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4177999999999994e-05 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.134729+0000", 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094443299999999995 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:02.135673+0000", 2023-10-09T06:00:16.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.026269019000000001 2023-10-09T06:00:16.300 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.300 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.300 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.300 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:123653 9.18 9:18626112:::benchmark_data_smithi008_502820_object39480:head [delete] snapc 0=[] ondisk+write+known_if_redirected e461)", 2023-10-09T06:00:16.300 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:04.342686+0000", 2023-10-09T06:00:16.300 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 71.800971656000002, 2023-10-09T06:00:16.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024170903000000001, 2023-10-09T06:00:16.300 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.301 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.301 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.301 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.301 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.301 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 123653 2023-10-09T06:00:16.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.301 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.342686+0000", 2023-10-09T06:00:16.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.342686+0000", 2023-10-09T06:00:16.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:16.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.342684+0000", 2023-10-09T06:00:16.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0289999999999994e-06 2023-10-09T06:00:16.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.342693+0000", 2023-10-09T06:00:16.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6217000000000001e-05 2023-10-09T06:00:16.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.342709+0000", 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3710000000000002e-06 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.342717+0000", 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7170999999999999e-05 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.342774+0000", 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018723399999999999 2023-10-09T06:00:16.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.342961+0000", 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000204699 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.343166+0000", 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002452339 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.345618+0000", 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001372065 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.346990+0000", 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015733213999999999 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.362723+0000", 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035467910000000001 2023-10-09T06:00:16.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.366270+0000", 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7316000000000004e-05 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.366337+0000", 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00051922399999999999 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:04.366857+0000", 2023-10-09T06:00:16.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024170903000000001 2023-10-09T06:00:16.310 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.310 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.310 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.310 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:124042 9.a 9:53b66a2a:::benchmark_data_smithi008_502820_object39869:head [delete] snapc 0=[] ondisk+write+known_if_redirected e461)", 2023-10-09T06:00:16.310 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:06.462599+0000", 2023-10-09T06:00:16.310 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 69.681058132000004, 2023-10-09T06:00:16.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.070517758999999999, 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124042 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.462599+0000", 2023-10-09T06:00:16.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.462599+0000", 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.462597+0000", 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3759999999999997e-06 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.462607+0000", 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3256e-05 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.462620+0000", 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7089999999999996e-06 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.462627+0000", 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9864999999999997e-05 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.462687+0000", 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000173549 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.462860+0000", 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015842100000000001 2023-10-09T06:00:16.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T06:00:16.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.463019+0000", 2023-10-09T06:00:16.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037917390000000001 2023-10-09T06:00:16.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.466810+0000", 2023-10-09T06:00:16.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7995999999999998e-05 2023-10-09T06:00:16.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.466828+0000", 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017785899999999999 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.467006+0000", 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.065880994999999998 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.532887+0000", 2023-10-09T06:00:16.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2668000000000003e-05 2023-10-09T06:00:16.317 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.317 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.317 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.532960+0000", 2023-10-09T06:00:16.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000157248 2023-10-09T06:00:16.317 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.317 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.317 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:06.533117+0000", 2023-10-09T06:00:16.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.070517758999999999 2023-10-09T06:00:16.317 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:124958 9.a 9:527ed9dc:::benchmark_data_smithi008_502820_object40785:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:12.000603+0000", 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 64.143054816000003, 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.029084117, 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:16.318 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:16.319 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:16.319 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 124958 2023-10-09T06:00:16.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.319 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:16.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:16.319 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.000603+0000", 2023-10-09T06:00:16.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:16.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:16.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.000603+0000", 2023-10-09T06:00:16.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:16.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:16.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.000601+0000", 2023-10-09T06:00:16.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0000000000000002e-06 2023-10-09T06:00:16.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:16.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.000610+0000", 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6226e-05 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.000626+0000", 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5900000000000002e-06 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.000634+0000", 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2950999999999998e-05 2023-10-09T06:00:16.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:16.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.000727+0000", 2023-10-09T06:00:16.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000144096 2023-10-09T06:00:16.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:16.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.000871+0000", 2023-10-09T06:00:16.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000157059 2023-10-09T06:00:16.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.001028+0000", 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010019676 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.011048+0000", 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000428885 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.011476+0000", 2023-10-09T06:00:16.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7936000000000002e-05 2023-10-09T06:00:16.324 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.324 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.324 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:16.324 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.011504+0000", 2023-10-09T06:00:16.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018007123999999999 2023-10-09T06:00:16.324 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.324 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.324 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:16.324 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.029512+0000", 2023-10-09T06:00:16.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9273000000000001e-05 2023-10-09T06:00:16.324 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.325 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.325 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:16.325 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.029561+0000", 2023-10-09T06:00:16.325 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012591800000000001 2023-10-09T06:00:16.325 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:16.325 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:16.325 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:16.325 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:12.029687+0000", 2023-10-09T06:00:16.325 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.029084117 2023-10-09T06:00:16.325 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.325 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.326 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.326 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:16.326 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:16.326 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:00:16.326 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:00:16.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:16.327 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:16.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:16.328 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:00:16.328 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:00:16.328 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:00:16.328 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:00:16.328 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:00:16.329 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:00:16.329 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:00:16.329 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:00:16.329 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:00:16.410 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:16 smithi008 ceph-mon[184418]: pgmap v1434: 99 pgs: 1 activating, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 10 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 52908/98961 objects degraded (53.463%); 12471/98961 objects misplaced (12.602%); 34 KiB/s, 28 objects/s recovering 2023-10-09T06:00:16.410 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:16 smithi008 ceph-mon[184418]: failed to encode map e480 with expected crc 2023-10-09T06:00:16.410 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:16 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2625054950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:16.410 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:16 smithi008 ceph-mon[191227]: pgmap v1434: 99 pgs: 1 activating, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 10 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 52908/98961 objects degraded (53.463%); 12471/98961 objects misplaced (12.602%); 34 KiB/s, 28 objects/s recovering 2023-10-09T06:00:16.410 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:16 smithi008 ceph-mon[191227]: failed to encode map e480 with expected crc 2023-10-09T06:00:16.410 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:16 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2625054950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:16.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:16.455 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:16 smithi191 ceph-mon[161669]: pgmap v1434: 99 pgs: 1 activating, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 10 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 52908/98961 objects degraded (53.463%); 12471/98961 objects misplaced (12.602%); 34 KiB/s, 28 objects/s recovering 2023-10-09T06:00:16.456 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:16 smithi191 ceph-mon[161669]: failed to encode map e480 with expected crc 2023-10-09T06:00:16.456 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:16 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2625054950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:16.458 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:16.508 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:16.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:16.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:16.611 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:16.641 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:16.962 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:00:17.091 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:17.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:17.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:17.194 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:17.223 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:17.319 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:17.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:17.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:17.368 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:17.423 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:17.461 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:17.603 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:17.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:17.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:17.725 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:17.758 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:17.823 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:17.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:17.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:17.932 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:17.980 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:18.062 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:18.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:18.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:18.167 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:18.220 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:18.280 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:18.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:18.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:18.385 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:18.430 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:18.521 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:18.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:18.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:18.632 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:18.665 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:18 smithi191 ceph-mon[161669]: pgmap v1435: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating, 3 activating+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 55060/98910 objects degraded (55.667%); 14383/98910 objects misplaced (14.542%); 752 KiB/s, 37 objects/s recovering 2023-10-09T06:00:18.665 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 55060/98910 objects degraded (55.667%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:18.668 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:18.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:18 smithi008 ceph-mon[184418]: pgmap v1435: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating, 3 activating+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 55060/98910 objects degraded (55.667%); 14383/98910 objects misplaced (14.542%); 752 KiB/s, 37 objects/s recovering 2023-10-09T06:00:18.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 55060/98910 objects degraded (55.667%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:18 smithi008 ceph-mon[191227]: pgmap v1435: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating, 3 activating+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 55060/98910 objects degraded (55.667%); 14383/98910 objects misplaced (14.542%); 752 KiB/s, 37 objects/s recovering 2023-10-09T06:00:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 55060/98910 objects degraded (55.667%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:18.793 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:18.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:18.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:18.900 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:18.930 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:18.979 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:18.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:18.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:19.022 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:00:19.047 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:00:19.047 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:00:19.047 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T06:00:19.047 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:00:19.085 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:19.117 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:19.186 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:19.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:19.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:19.291 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:19.319 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:19.342 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:00:19.363 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:19.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:19 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4014657172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:19.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:19.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:19.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:19 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4014657172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:19.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:19 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4014657172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:19.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:19.514 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:19.825 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:19.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:19.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:19.849 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:00:19.932 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:19.963 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:20.026 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:20.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:20.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:20.134 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:20.165 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:20.349 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:20.360 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:20.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:20.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:20.409 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:20.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[184418]: pgmap v1436: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating, 3 activating+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 55054/98892 objects degraded (55.671%); 14377/98892 objects misplaced (14.538%); 800 KiB/s, 38 objects/s recovering 2023-10-09T06:00:20.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:20.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[184418]: failed to encode map e481 with expected crc 2023-10-09T06:00:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[184418]: failed to encode map e481 with expected crc 2023-10-09T06:00:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4014657172' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[184418]: osdmap e481: 8 total, 8 up, 6 in 2023-10-09T06:00:20.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[184418]: failed to encode map e481 with expected crc 2023-10-09T06:00:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[191227]: pgmap v1436: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating, 3 activating+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 55054/98892 objects degraded (55.671%); 14377/98892 objects misplaced (14.538%); 800 KiB/s, 38 objects/s recovering 2023-10-09T06:00:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[191227]: failed to encode map e481 with expected crc 2023-10-09T06:00:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[191227]: failed to encode map e481 with expected crc 2023-10-09T06:00:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4014657172' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:20.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[191227]: osdmap e481: 8 total, 8 up, 6 in 2023-10-09T06:00:20.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:20 smithi008 ceph-mon[191227]: failed to encode map e481 with expected crc 2023-10-09T06:00:20.466 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:20.494 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:20.604 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:20.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:20.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:20.657 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:20 smithi191 ceph-mon[161669]: pgmap v1436: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating, 3 activating+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 55054/98892 objects degraded (55.671%); 14377/98892 objects misplaced (14.538%); 800 KiB/s, 38 objects/s recovering 2023-10-09T06:00:20.657 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:20 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:20.657 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:20 smithi191 ceph-mon[161669]: failed to encode map e481 with expected crc 2023-10-09T06:00:20.657 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:20 smithi191 ceph-mon[161669]: failed to encode map e481 with expected crc 2023-10-09T06:00:20.657 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:20 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4014657172' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:20.658 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:20 smithi191 ceph-mon[161669]: osdmap e481: 8 total, 8 up, 6 in 2023-10-09T06:00:20.658 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:20 smithi191 ceph-mon[161669]: failed to encode map e481 with expected crc 2023-10-09T06:00:20.709 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:20.738 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:20.772 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:00:20.773 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:00:20.773 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T06:00:20.773 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:00:20.779 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:00:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:00:20] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.33.4" 2023-10-09T06:00:21.047 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:21.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:21.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:21.152 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:21.190 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:21.253 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:21.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:21.357 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:21.385 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:21.692 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:21.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:21.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:21.776 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:21.800 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:21.835 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:22.009 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:22.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:22.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:22.115 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:00:22.115 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:00:22.115 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T06:00:22.115 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T06:00:22.116 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:00:22.118 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:22.163 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:22.289 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:00:22.315 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:00:22.346 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:00:22.350 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:22.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:22.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:22.369 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:00:22.370 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:00:22.388 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:00:22.407 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:00:22.426 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:00:22.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:22.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:22 smithi191 ceph-mon[161669]: pgmap v1438: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 54871/97266 objects degraded (56.413%); 14277/97266 objects misplaced (14.678%); 1.6 MiB/s, 62 objects/s recovering 2023-10-09T06:00:22.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:22 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3310187201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:22.483 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:22.534 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:00:22.537 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:00:22.537 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:00:22.537 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:00:22.537 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:00:22.537 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:00:22.538 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:00:22.538 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:00:22.538 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:00:22.544 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:22.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:22.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:22.560 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:22 smithi008 ceph-mon[184418]: pgmap v1438: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 54871/97266 objects degraded (56.413%); 14277/97266 objects misplaced (14.678%); 1.6 MiB/s, 62 objects/s recovering 2023-10-09T06:00:22.560 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:22 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3310187201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:22.560 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:22 smithi008 ceph-mon[191227]: pgmap v1438: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 54871/97266 objects degraded (56.413%); 14277/97266 objects misplaced (14.678%); 1.6 MiB/s, 62 objects/s recovering 2023-10-09T06:00:22.561 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:22 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3310187201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:22.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:22.679 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:22.866 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:00:22.876 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:22.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:22.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:22.983 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:23.016 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:23.093 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:23.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:23.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:23.234 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:23.304 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:23.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:23.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:23.394 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:23.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:23.451 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 54871/97266 objects degraded (56.413%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:23.455 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:23.548 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:23.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:23.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:23.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:23.705 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:23.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 54871/97266 objects degraded (56.413%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:23.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 54871/97266 objects degraded (56.413%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:23.775 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:23.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:23.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:23.882 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:23.924 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:23.931 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:24.034 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:24.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:24.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:24.138 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:24.174 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:24.229 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:24.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:24.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:24.333 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:24.338 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:24 smithi008 ceph-mon[191227]: pgmap v1439: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 54811/97002 objects degraded (56.505%); 13934/97002 objects misplaced (14.365%); 3.0 MiB/s, 83 objects/s recovering 2023-10-09T06:00:24.366 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:24.593 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:24 smithi191 ceph-mon[161669]: pgmap v1439: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 54811/97002 objects degraded (56.505%); 13934/97002 objects misplaced (14.365%); 3.0 MiB/s, 83 objects/s recovering 2023-10-09T06:00:24.631 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:24.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:24.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:24.682 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:00:24.694 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T06:00:24.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:24 smithi008 ceph-mon[184418]: pgmap v1439: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 54811/97002 objects degraded (56.505%); 13934/97002 objects misplaced (14.365%); 3.0 MiB/s, 83 objects/s recovering 2023-10-09T06:00:24.750 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:24.796 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:25.179 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:00:25.179 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:00:25.732 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:25.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:25.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:25.768 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:00:25.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:25.886 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:25.959 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:00:25.964 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:25.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:25.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:26.078 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:26.085 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1439,"stamp":"2023-10-09T06:00:22.709842+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2121930699,"num_objects":32334,"num_object_clones":0,"num_object_copies":97002,"num_objects_missing_on_primary":12763,"num_objects_missing":12763,"num_objects_degraded":54811,"num_objects_misplaced":13934,"num_objects_unfound":0,"num_objects_dirty":32334,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":137142,"num_write_kb":5402022,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43733,"num_bytes_recovered":2479047205,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155136,"ondisk_log_size":155136,"up":297,"acting":278,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11661564,"kb_used_data":6328652,"kb_used_omap":45,"kb_used_meta":5332818,"kb_avail":550809348,"statfs":{"total":575970213888,"available":564028772352,"internally_reserved":0,"allocated":6480539648,"data_stored":6470811123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":5460805923},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,0,0,2,5,2,2,6,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-45416448,"num_objects":-693,"num_object_clones":0,"num_object_copies":-2079,"num_objects_missing_on_primary":1472,"num_objects_missing":1472,"num_objects_degraded":12519,"num_objects_misplaced":1122,"num_objects_unfound":0,"num_objects_dirty":-693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":693,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":706,"num_bytes_recovered":26542080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.417464"},"pg_stats":[{"pgid":"9.10","version":"481'6583","reported_seq":8385,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:20.150778+0000","last_change":"2023-10-09T06:00:05.663826+0000","last_active":"2023-10-09T06:00:20.150778+0000","last_peered":"2023-10-09T06:00:20.150778+0000","last_clean":"2023-10-09T05:56:48.221888+0000","last_became_active":"2023-10-09T06:00:05.653883+0000","last_became_peered":"2023-10-09T06:00:05.653883+0000","last_unstale":"2023-10-09T06:00:20.150778+0000","last_undegraded":"2023-10-09T06:00:05.409198+0000","last_fullsized":"2023-10-09T06:00:05.282568+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6583,"log_dups_size":0,"ondisk_log_size":6583,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65208320,"num_objects":995,"num_object_clones":0,"num_object_copies":2985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4628,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4193,"num_write_kb":166016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":544,"num_bytes_recovered":35651584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,4],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2,6","objects":997},{"shards":"2,4","objects":-2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":103,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.297033+0000","last_change":"2023-10-09T05:54:59.256087+0000","last_active":"2023-10-09T06:00:14.297033+0000","last_peered":"2023-10-09T06:00:14.297033+0000","last_clean":"2023-10-09T06:00:14.297033+0000","last_became_active":"2023-10-09T05:54:59.255773+0000","last_became_peered":"2023-10-09T05:54:59.255773+0000","last_unstale":"2023-10-09T06:00:14.297033+0000","last_undegraded":"2023-10-09T06:00:14.297033+0000","last_fullsized":"2023-10-09T06:00:14.297033+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:19.545754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":76,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:00:11.366429+0000","last_change":"2023-10-09T05:56:00.247579+0000","last_active":"2023-10-09T06:00:11.366429+0000","last_peered":"2023-10-09T06:00:11.366429+0000","last_clean":"2023-10-09T06:00:11.366429+0000","last_became_active":"2023-10-09T05:56:00.238151+0000","last_became_peered":"2023-10-09T05:56:00.238151+0000","last_unstale":"2023-10-09T06:00:11.366429+0000","last_undegraded":"2023-10-09T06:00:11.366429+0000","last_fullsized":"2023-10-09T06:00:11.366429+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:05.692190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"481'6637","reported_seq":12511,"reported_epoch":481,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:00:20.211336+0000","last_change":"2023-10-09T05:56:01.319470+0000","last_active":"2023-10-09T06:00:20.211336+0000","last_peered":"2023-10-09T06:00:20.211336+0000","last_clean":"2023-10-09T05:55:24.768851+0000","last_became_active":"2023-10-09T05:56:01.311996+0000","last_became_peered":"2023-10-09T05:56:01.311996+0000","last_unstale":"2023-10-09T06:00:20.211336+0000","last_undegraded":"2023-10-09T06:00:20.211336+0000","last_fullsized":"2023-10-09T06:00:20.211336+0000","mapping_epoch":416,"log_start":"368'503","ondisk_log_start":"368'503","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6134,"log_dups_size":0,"ondisk_log_size":6134,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68026368,"num_objects":1038,"num_object_clones":0,"num_object_copies":3114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1038,"num_objects_unfound":0,"num_objects_dirty":1038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4224,"num_write_kb":168384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1422,"num_bytes_recovered":93192192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1038}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":184,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:15.229227+0000","last_change":"2023-10-09T05:56:00.243822+0000","last_active":"2023-10-09T06:00:15.229227+0000","last_peered":"2023-10-09T06:00:15.229227+0000","last_clean":"2023-10-09T06:00:15.229227+0000","last_became_active":"2023-10-09T05:56:00.243483+0000","last_became_peered":"2023-10-09T05:56:00.243483+0000","last_unstale":"2023-10-09T06:00:15.229227+0000","last_undegraded":"2023-10-09T06:00:15.229227+0000","last_fullsized":"2023-10-09T06:00:15.229227+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:39.811952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":490,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.304499+0000","last_change":"2023-10-09T06:00:05.354195+0000","last_active":"2023-10-09T06:00:14.304499+0000","last_peered":"2023-10-09T06:00:14.304499+0000","last_clean":"2023-10-09T06:00:14.304499+0000","last_became_active":"2023-10-09T06:00:05.353827+0000","last_became_peered":"2023-10-09T06:00:05.353827+0000","last_unstale":"2023-10-09T06:00:14.304499+0000","last_undegraded":"2023-10-09T06:00:14.304499+0000","last_fullsized":"2023-10-09T06:00:14.304499+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:44.449493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.12","version":"481'6702","reported_seq":12258,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-10-09T06:00:20.551661+0000","last_change":"2023-10-09T05:58:12.409601+0000","last_active":"2023-10-09T06:00:20.551661+0000","last_peered":"2023-10-09T06:00:20.551661+0000","last_clean":"2023-10-09T06:00:20.551661+0000","last_became_active":"2023-10-09T05:58:12.403280+0000","last_became_peered":"2023-10-09T05:58:12.403280+0000","last_unstale":"2023-10-09T06:00:20.551661+0000","last_undegraded":"2023-10-09T06:00:20.551661+0000","last_fullsized":"2023-10-09T06:00:20.551661+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6702,"log_dups_size":0,"ondisk_log_size":6702,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67272704,"num_objects":1027,"num_object_clones":0,"num_object_copies":3081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4281,"num_write_kb":169824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2689,"num_bytes_recovered":106627072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":425,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.303847+0000","last_change":"2023-10-09T06:00:05.620750+0000","last_active":"2023-10-09T06:00:14.303847+0000","last_peered":"2023-10-09T06:00:14.303847+0000","last_clean":"2023-10-09T06:00:14.303847+0000","last_became_active":"2023-10-09T06:00:05.620298+0000","last_became_peered":"2023-10-09T06:00:05.620298+0000","last_unstale":"2023-10-09T06:00:14.303847+0000","last_undegraded":"2023-10-09T06:00:14.303847+0000","last_fullsized":"2023-10-09T06:00:14.303847+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:29.560982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":16,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.304283+0000","last_change":"2023-10-09T06:00:05.329975+0000","last_active":"2023-10-09T06:00:14.304283+0000","last_peered":"2023-10-09T06:00:14.304283+0000","last_clean":"2023-10-09T06:00:14.304283+0000","last_became_active":"2023-10-09T06:00:05.329670+0000","last_became_peered":"2023-10-09T06:00:05.329670+0000","last_unstale":"2023-10-09T06:00:14.304283+0000","last_undegraded":"2023-10-09T06:00:14.304283+0000","last_fullsized":"2023-10-09T06:00:14.304283+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:40.087262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":411,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.304459+0000","last_change":"2023-10-09T06:00:05.337715+0000","last_active":"2023-10-09T06:00:14.304459+0000","last_peered":"2023-10-09T06:00:14.304459+0000","last_clean":"2023-10-09T06:00:14.304459+0000","last_became_active":"2023-10-09T06:00:05.330288+0000","last_became_peered":"2023-10-09T06:00:05.330288+0000","last_unstale":"2023-10-09T06:00:14.304459+0000","last_undegraded":"2023-10-09T06:00:14.304459+0000","last_fullsized":"2023-10-09T06:00:14.304459+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:01.651785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:00:11.364003+0000","last_change":"2023-10-09T05:58:07.387601+0000","last_active":"2023-10-09T06:00:11.364003+0000","last_peered":"2023-10-09T06:00:11.364003+0000","last_clean":"2023-10-09T06:00:11.364003+0000","last_became_active":"2023-10-09T05:58:07.387307+0000","last_became_peered":"2023-10-09T05:58:07.387307+0000","last_unstale":"2023-10-09T06:00:11.364003+0000","last_undegraded":"2023-10-09T06:00:11.364003+0000","last_fullsized":"2023-10-09T06:00:11.364003+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:33:56.748928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":138,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.300925+0000","last_change":"2023-10-09T06:00:05.346651+0000","last_active":"2023-10-09T06:00:14.300925+0000","last_peered":"2023-10-09T06:00:14.300925+0000","last_clean":"2023-10-09T06:00:14.300925+0000","last_became_active":"2023-10-09T06:00:05.346292+0000","last_became_peered":"2023-10-09T06:00:05.346292+0000","last_unstale":"2023-10-09T06:00:14.300925+0000","last_undegraded":"2023-10-09T06:00:14.300925+0000","last_fullsized":"2023-10-09T06:00:14.300925+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":123,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.308824+0000","last_change":"2023-10-09T06:00:05.332503+0000","last_active":"2023-10-09T06:00:14.308824+0000","last_peered":"2023-10-09T06:00:14.308824+0000","last_clean":"2023-10-09T06:00:14.308824+0000","last_became_active":"2023-10-09T06:00:05.331773+0000","last_became_peered":"2023-10-09T06:00:05.331773+0000","last_unstale":"2023-10-09T06:00:14.308824+0000","last_undegraded":"2023-10-09T06:00:14.308824+0000","last_fullsized":"2023-10-09T06:00:14.308824+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":113,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:16.205933+0000","last_change":"2023-10-09T05:56:00.348771+0000","last_active":"2023-10-09T06:00:16.205933+0000","last_peered":"2023-10-09T06:00:16.205933+0000","last_clean":"2023-10-09T06:00:16.205933+0000","last_became_active":"2023-10-09T05:56:00.224347+0000","last_became_peered":"2023-10-09T05:56:00.224347+0000","last_unstale":"2023-10-09T06:00:16.205933+0000","last_undegraded":"2023-10-09T06:00:16.205933+0000","last_fullsized":"2023-10-09T06:00:16.205933+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":260,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.297112+0000","last_change":"2023-10-09T06:00:05.325283+0000","last_active":"2023-10-09T06:00:14.297112+0000","last_peered":"2023-10-09T06:00:14.297112+0000","last_clean":"2023-10-09T06:00:14.297112+0000","last_became_active":"2023-10-09T06:00:05.324971+0000","last_became_peered":"2023-10-09T06:00:05.324971+0000","last_unstale":"2023-10-09T06:00:14.297112+0000","last_undegraded":"2023-10-09T06:00:14.297112+0000","last_fullsized":"2023-10-09T06:00:14.297112+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:14.747264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":106,"reported_epoch":480,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:00:14.300332+0000","last_change":"2023-10-09T05:58:07.410626+0000","last_active":"2023-10-09T06:00:14.300332+0000","last_peered":"2023-10-09T06:00:14.300332+0000","last_clean":"2023-10-09T05:58:01.307429+0000","last_became_active":"2023-10-09T05:58:07.410375+0000","last_became_peered":"2023-10-09T05:58:07.410375+0000","last_unstale":"2023-10-09T06:00:14.300332+0000","last_undegraded":"2023-10-09T05:58:07.381410+0000","last_fullsized":"2023-10-09T06:00:14.300332+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":["7","4","6"],"object_location_counts":[{"shards":"4,6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":466,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.299511+0000","last_change":"2023-10-09T05:58:07.913883+0000","last_active":"2023-10-09T06:00:14.299511+0000","last_peered":"2023-10-09T06:00:14.299511+0000","last_clean":"2023-10-09T06:00:14.299511+0000","last_became_active":"2023-10-09T05:58:07.913605+0000","last_became_peered":"2023-10-09T05:58:07.913605+0000","last_unstale":"2023-10-09T06:00:14.299511+0000","last_undegraded":"2023-10-09T06:00:14.299511+0000","last_fullsized":"2023-10-09T06:00:14.299511+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:46.780446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.19","version":"481'6644","reported_seq":10351,"reported_epoch":481,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:00:20.989873+0000","last_change":"2023-10-09T06:00:13.299489+0000","last_active":"2023-10-09T06:00:20.989873+0000","last_peered":"2023-10-09T06:00:20.989873+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T06:00:13.101688+0000","last_became_peered":"2023-10-09T06:00:13.101688+0000","last_unstale":"2023-10-09T06:00:20.989873+0000","last_undegraded":"2023-10-09T06:00:20.989873+0000","last_fullsized":"2023-10-09T06:00:07.297179+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6644,"log_dups_size":0,"ondisk_log_size":6644,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66158592,"num_objects":1010,"num_object_clones":0,"num_object_copies":3030,"num_objects_missing_on_primary":1823,"num_objects_missing":1823,"num_objects_degraded":0,"num_objects_misplaced":1632,"num_objects_unfound":0,"num_objects_dirty":1010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4238,"num_write_kb":167904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1274,"num_bytes_recovered":83460096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[4,0],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,3,5","objects":816},{"shards":"0,4","objects":194}],"blocked_by":[],"up_primary":0,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":95,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.299583+0000","last_change":"2023-10-09T05:55:27.092744+0000","last_active":"2023-10-09T06:00:14.299583+0000","last_peered":"2023-10-09T06:00:14.299583+0000","last_clean":"2023-10-09T06:00:14.299583+0000","last_became_active":"2023-10-09T05:55:26.271096+0000","last_became_peered":"2023-10-09T05:55:26.271096+0000","last_unstale":"2023-10-09T06:00:14.299583+0000","last_undegraded":"2023-10-09T06:00:14.299583+0000","last_fullsized":"2023-10-09T06:00:14.299583+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":116,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.300208+0000","last_change":"2023-10-09T05:56:49.266384+0000","last_active":"2023-10-09T06:00:14.300208+0000","last_peered":"2023-10-09T06:00:14.300208+0000","last_clean":"2023-10-09T06:00:14.300208+0000","last_became_active":"2023-10-09T05:56:49.266051+0000","last_became_peered":"2023-10-09T05:56:49.266051+0000","last_unstale":"2023-10-09T06:00:14.300208+0000","last_undegraded":"2023-10-09T06:00:14.300208+0000","last_fullsized":"2023-10-09T06:00:14.300208+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"481'6630","reported_seq":17474,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:19.483289+0000","last_change":"2023-10-09T06:00:06.395295+0000","last_active":"2023-10-09T06:00:19.483289+0000","last_peered":"2023-10-09T06:00:19.483289+0000","last_clean":"2023-10-09T06:00:03.898278+0000","last_became_active":"2023-10-09T06:00:06.375058+0000","last_became_peered":"2023-10-09T06:00:06.375058+0000","last_unstale":"2023-10-09T06:00:19.483289+0000","last_undegraded":"2023-10-09T06:00:06.297529+0000","last_fullsized":"2023-10-09T06:00:06.282061+0000","mapping_epoch":476,"log_start":"368'587","ondisk_log_start":"368'587","created":385,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6043,"log_dups_size":0,"ondisk_log_size":6043,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65994752,"num_objects":1007,"num_object_clones":0,"num_object_copies":3021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":993,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4217,"num_write_kb":167168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2787,"num_bytes_recovered":134348800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[1,7],"avail_no_missing":["1","2","3","5","7"],"object_location_counts":[{"shards":"1,7","objects":1007}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":463,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.303599+0000","last_change":"2023-10-09T06:00:05.355525+0000","last_active":"2023-10-09T06:00:14.303599+0000","last_peered":"2023-10-09T06:00:14.303599+0000","last_clean":"2023-10-09T06:00:14.303599+0000","last_became_active":"2023-10-09T06:00:05.354517+0000","last_became_peered":"2023-10-09T06:00:05.354517+0000","last_unstale":"2023-10-09T06:00:14.303599+0000","last_undegraded":"2023-10-09T06:00:14.303599+0000","last_fullsized":"2023-10-09T06:00:14.303599+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:44.826046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":175,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:17.331512+0000","last_change":"2023-10-09T06:00:05.330325+0000","last_active":"2023-10-09T06:00:17.331512+0000","last_peered":"2023-10-09T06:00:17.331512+0000","last_clean":"2023-10-09T06:00:17.331512+0000","last_became_active":"2023-10-09T06:00:05.324423+0000","last_became_peered":"2023-10-09T06:00:05.324423+0000","last_unstale":"2023-10-09T06:00:17.331512+0000","last_undegraded":"2023-10-09T06:00:17.331512+0000","last_fullsized":"2023-10-09T06:00:17.331512+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":105,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.308734+0000","last_change":"2023-10-09T06:00:05.332331+0000","last_active":"2023-10-09T06:00:14.308734+0000","last_peered":"2023-10-09T06:00:14.308734+0000","last_clean":"2023-10-09T06:00:14.308734+0000","last_became_active":"2023-10-09T06:00:05.331368+0000","last_became_peered":"2023-10-09T06:00:05.331368+0000","last_unstale":"2023-10-09T06:00:14.308734+0000","last_undegraded":"2023-10-09T06:00:14.308734+0000","last_fullsized":"2023-10-09T06:00:14.308734+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":460,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.304538+0000","last_change":"2023-10-09T06:00:05.322793+0000","last_active":"2023-10-09T06:00:14.304538+0000","last_peered":"2023-10-09T06:00:14.304538+0000","last_clean":"2023-10-09T06:00:14.304538+0000","last_became_active":"2023-10-09T06:00:05.322462+0000","last_became_peered":"2023-10-09T06:00:05.322462+0000","last_unstale":"2023-10-09T06:00:14.304538+0000","last_undegraded":"2023-10-09T06:00:14.304538+0000","last_fullsized":"2023-10-09T06:00:14.304538+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:11.349003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":252,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.303741+0000","last_change":"2023-10-09T06:00:05.621144+0000","last_active":"2023-10-09T06:00:14.303741+0000","last_peered":"2023-10-09T06:00:14.303741+0000","last_clean":"2023-10-09T06:00:14.303741+0000","last_became_active":"2023-10-09T06:00:05.620329+0000","last_became_peered":"2023-10-09T06:00:05.620329+0000","last_unstale":"2023-10-09T06:00:14.303741+0000","last_undegraded":"2023-10-09T06:00:14.303741+0000","last_fullsized":"2023-10-09T06:00:14.303741+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:59.684753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":115,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.300518+0000","last_change":"2023-10-09T05:56:49.263088+0000","last_active":"2023-10-09T06:00:14.300518+0000","last_peered":"2023-10-09T06:00:14.300518+0000","last_clean":"2023-10-09T06:00:14.300518+0000","last_became_active":"2023-10-09T05:56:49.262781+0000","last_became_peered":"2023-10-09T05:56:49.262781+0000","last_unstale":"2023-10-09T06:00:14.300518+0000","last_undegraded":"2023-10-09T06:00:14.300518+0000","last_fullsized":"2023-10-09T06:00:14.300518+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"481'6708","reported_seq":9169,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:20.176910+0000","last_change":"2023-10-09T06:00:17.809415+0000","last_active":"2023-10-09T06:00:20.176910+0000","last_peered":"2023-10-09T06:00:20.176910+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T06:00:17.617011+0000","last_became_peered":"2023-10-09T06:00:17.617011+0000","last_unstale":"2023-10-09T06:00:20.176910+0000","last_undegraded":"2023-10-09T06:00:12.226744+0000","last_fullsized":"2023-10-09T06:00:07.517595+0000","mapping_epoch":477,"log_start":"368'628","ondisk_log_start":"368'628","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6080,"log_dups_size":0,"ondisk_log_size":6080,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65830912,"num_objects":1004,"num_object_clones":0,"num_object_copies":3012,"num_objects_missing_on_primary":888,"num_objects_missing":888,"num_objects_degraded":3338,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4266,"num_write_kb":168672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":774,"num_bytes_recovered":50724864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[1,0],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,1","objects":40},{"shards":"0,1,5","objects":964}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":122,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.302204+0000","last_change":"2023-10-09T06:00:05.328315+0000","last_active":"2023-10-09T06:00:14.302204+0000","last_peered":"2023-10-09T06:00:14.302204+0000","last_clean":"2023-10-09T06:00:14.302204+0000","last_became_active":"2023-10-09T06:00:05.327973+0000","last_became_peered":"2023-10-09T06:00:05.327973+0000","last_unstale":"2023-10-09T06:00:14.302204+0000","last_undegraded":"2023-10-09T06:00:14.302204+0000","last_fullsized":"2023-10-09T06:00:14.302204+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":144,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.299777+0000","last_change":"2023-10-09T05:58:07.397628+0000","last_active":"2023-10-09T06:00:14.299777+0000","last_peered":"2023-10-09T06:00:14.299777+0000","last_clean":"2023-10-09T06:00:14.299777+0000","last_became_active":"2023-10-09T05:58:07.397010+0000","last_became_peered":"2023-10-09T05:58:07.397010+0000","last_unstale":"2023-10-09T06:00:14.299777+0000","last_undegraded":"2023-10-09T06:00:14.299777+0000","last_fullsized":"2023-10-09T06:00:14.299777+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"481'6707","reported_seq":6867,"reported_epoch":481,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:00:19.330424+0000","last_change":"2023-10-09T05:56:48.413650+0000","last_active":"2023-10-09T06:00:19.330424+0000","last_peered":"2023-10-09T06:00:19.330424+0000","last_clean":"2023-10-09T05:55:48.934818+0000","last_became_active":"2023-10-09T05:56:01.246051+0000","last_became_peered":"2023-10-09T05:56:01.246051+0000","last_unstale":"2023-10-09T06:00:19.330424+0000","last_undegraded":"2023-10-09T05:56:48.413475+0000","last_fullsized":"2023-10-09T06:00:19.330424+0000","mapping_epoch":416,"log_start":"368'691","ondisk_log_start":"368'691","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6016,"log_dups_size":0,"ondisk_log_size":6016,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65863680,"num_objects":1005,"num_object_clones":0,"num_object_copies":3015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4277,"num_write_kb":169024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1005}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":93,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.300844+0000","last_change":"2023-10-09T05:55:26.358519+0000","last_active":"2023-10-09T06:00:14.300844+0000","last_peered":"2023-10-09T06:00:14.300844+0000","last_clean":"2023-10-09T06:00:14.300844+0000","last_became_active":"2023-10-09T05:55:26.260138+0000","last_became_peered":"2023-10-09T05:55:26.260138+0000","last_unstale":"2023-10-09T06:00:14.300844+0000","last_undegraded":"2023-10-09T06:00:14.300844+0000","last_fullsized":"2023-10-09T06:00:14.300844+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":16,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.303432+0000","last_change":"2023-10-09T06:00:05.352127+0000","last_active":"2023-10-09T06:00:14.303432+0000","last_peered":"2023-10-09T06:00:14.303432+0000","last_clean":"2023-10-09T06:00:14.303432+0000","last_became_active":"2023-10-09T06:00:05.351767+0000","last_became_peered":"2023-10-09T06:00:05.351767+0000","last_unstale":"2023-10-09T06:00:14.303432+0000","last_undegraded":"2023-10-09T06:00:14.303432+0000","last_fullsized":"2023-10-09T06:00:14.303432+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:42.100093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.7","version":"481'6619","reported_seq":10325,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:19.511492+0000","last_change":"2023-10-09T05:58:02.333623+0000","last_active":"2023-10-09T06:00:19.511492+0000","last_peered":"2023-10-09T06:00:19.511492+0000","last_clean":"2023-10-09T05:53:45.891460+0000","last_became_active":"2023-10-09T05:58:02.118107+0000","last_became_peered":"2023-10-09T05:58:02.118107+0000","last_unstale":"2023-10-09T06:00:19.511492+0000","last_undegraded":"2023-10-09T05:58:01.208593+0000","last_fullsized":"2023-10-09T05:57:58.248025+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6619,"log_dups_size":0,"ondisk_log_size":6619,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67567616,"num_objects":1031,"num_object_clones":0,"num_object_copies":3093,"num_objects_missing_on_primary":937,"num_objects_missing":937,"num_objects_degraded":3585,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4195,"num_write_kb":167232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1554,"num_bytes_recovered":99549184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"2,3,5","objects":475},{"shards":"2,3,5,7","objects":471},{"shards":"2,7","objects":-85}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":106,"reported_epoch":480,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:00:16.206028+0000","last_change":"2023-10-09T05:58:07.406363+0000","last_active":"2023-10-09T06:00:16.206028+0000","last_peered":"2023-10-09T06:00:16.206028+0000","last_clean":"2023-10-09T05:56:43.219657+0000","last_became_active":"2023-10-09T05:58:07.404369+0000","last_became_peered":"2023-10-09T05:58:07.404369+0000","last_unstale":"2023-10-09T06:00:16.206028+0000","last_undegraded":"2023-10-09T05:58:07.387150+0000","last_fullsized":"2023-10-09T06:00:16.206028+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["7","3","5","6"],"object_location_counts":[{"shards":"3,5,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":116,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.299786+0000","last_change":"2023-10-09T06:00:05.329414+0000","last_active":"2023-10-09T06:00:14.299786+0000","last_peered":"2023-10-09T06:00:14.299786+0000","last_clean":"2023-10-09T06:00:14.299786+0000","last_became_active":"2023-10-09T06:00:05.329135+0000","last_became_peered":"2023-10-09T06:00:05.329135+0000","last_unstale":"2023-10-09T06:00:14.299786+0000","last_undegraded":"2023-10-09T06:00:14.299786+0000","last_fullsized":"2023-10-09T06:00:14.299786+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"481'6547","reported_seq":6714,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:20.451722+0000","last_change":"2023-10-09T06:00:05.689392+0000","last_active":"2023-10-09T06:00:20.451722+0000","last_peered":"2023-10-09T06:00:20.451722+0000","last_clean":"2023-10-09T05:57:55.996837+0000","last_became_active":"2023-10-09T06:00:05.679302+0000","last_became_peered":"2023-10-09T06:00:05.679302+0000","last_unstale":"2023-10-09T06:00:20.451722+0000","last_undegraded":"2023-10-09T06:00:05.327366+0000","last_fullsized":"2023-10-09T06:00:05.282693+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6547,"log_dups_size":0,"ondisk_log_size":6547,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67633152,"num_objects":1032,"num_object_clones":0,"num_object_copies":3096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2570,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4148,"num_write_kb":165760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1],"avail_no_missing":["2","1","5"],"object_location_counts":[{"shards":"1,2","objects":-4},{"shards":"1,2,5","objects":1012}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":16,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.304025+0000","last_change":"2023-10-09T06:00:05.339341+0000","last_active":"2023-10-09T06:00:14.304025+0000","last_peered":"2023-10-09T06:00:14.304025+0000","last_clean":"2023-10-09T06:00:14.304025+0000","last_became_active":"2023-10-09T06:00:05.339005+0000","last_became_peered":"2023-10-09T06:00:05.339005+0000","last_unstale":"2023-10-09T06:00:14.304025+0000","last_undegraded":"2023-10-09T06:00:14.304025+0000","last_fullsized":"2023-10-09T06:00:14.304025+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:55.788842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":412,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.305359+0000","last_change":"2023-10-09T06:00:05.352723+0000","last_active":"2023-10-09T06:00:14.305359+0000","last_peered":"2023-10-09T06:00:14.305359+0000","last_clean":"2023-10-09T06:00:14.305359+0000","last_became_active":"2023-10-09T06:00:05.352008+0000","last_became_peered":"2023-10-09T06:00:05.352008+0000","last_unstale":"2023-10-09T06:00:14.305359+0000","last_undegraded":"2023-10-09T06:00:14.305359+0000","last_fullsized":"2023-10-09T06:00:14.305359+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:25.302918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"481'6720","reported_seq":17552,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:19.549428+0000","last_change":"2023-10-09T05:58:20.540189+0000","last_active":"2023-10-09T06:00:19.549428+0000","last_peered":"2023-10-09T06:00:19.549428+0000","last_clean":"2023-10-09T05:58:03.790253+0000","last_became_active":"2023-10-09T05:58:20.243902+0000","last_became_peered":"2023-10-09T05:58:20.243902+0000","last_unstale":"2023-10-09T06:00:19.549428+0000","last_undegraded":"2023-10-09T05:58:11.625534+0000","last_fullsized":"2023-10-09T05:58:11.609771+0000","mapping_epoch":448,"log_start":"367'133","ondisk_log_start":"367'133","created":364,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6587,"log_dups_size":0,"ondisk_log_size":6587,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66617344,"num_objects":1016,"num_object_clones":0,"num_object_copies":3048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2666,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4266,"num_write_kb":169056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4116,"num_bytes_recovered":234323968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":1041},{"shards":"3,7","objects":-613}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":191,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.305413+0000","last_change":"2023-10-09T06:00:05.340070+0000","last_active":"2023-10-09T06:00:14.305413+0000","last_peered":"2023-10-09T06:00:14.305413+0000","last_clean":"2023-10-09T06:00:14.305413+0000","last_became_active":"2023-10-09T06:00:05.329639+0000","last_became_peered":"2023-10-09T06:00:05.329639+0000","last_unstale":"2023-10-09T06:00:14.305413+0000","last_undegraded":"2023-10-09T06:00:14.305413+0000","last_fullsized":"2023-10-09T06:00:14.305413+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:36.059049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":473,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.317534+0000","last_change":"2023-10-09T05:58:07.394333+0000","last_active":"2023-10-09T06:00:14.317534+0000","last_peered":"2023-10-09T06:00:14.317534+0000","last_clean":"2023-10-09T06:00:14.317534+0000","last_became_active":"2023-10-09T05:58:07.394002+0000","last_became_peered":"2023-10-09T05:58:07.394002+0000","last_unstale":"2023-10-09T06:00:14.317534+0000","last_undegraded":"2023-10-09T06:00:14.317534+0000","last_fullsized":"2023-10-09T06:00:14.317534+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"481'6598","reported_seq":8568,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:20.409469+0000","last_change":"2023-10-09T06:00:05.663828+0000","last_active":"2023-10-09T06:00:20.409469+0000","last_peered":"2023-10-09T06:00:20.409469+0000","last_clean":"2023-10-09T05:56:47.068306+0000","last_became_active":"2023-10-09T06:00:05.653527+0000","last_became_peered":"2023-10-09T06:00:05.653527+0000","last_unstale":"2023-10-09T06:00:20.409469+0000","last_undegraded":"2023-10-09T06:00:05.414414+0000","last_fullsized":"2023-10-09T06:00:05.291358+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6598,"log_dups_size":0,"ondisk_log_size":6598,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65011712,"num_objects":992,"num_object_clones":0,"num_object_copies":2976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4570,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4208,"num_write_kb":166400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":599,"num_bytes_recovered":39256064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,4],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2,6","objects":990},{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":43,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.299398+0000","last_change":"2023-10-09T05:58:07.400025+0000","last_active":"2023-10-09T06:00:14.299398+0000","last_peered":"2023-10-09T06:00:14.299398+0000","last_clean":"2023-10-09T06:00:14.299398+0000","last_became_active":"2023-10-09T05:58:07.399511+0000","last_became_peered":"2023-10-09T05:58:07.399511+0000","last_unstale":"2023-10-09T06:00:14.299398+0000","last_undegraded":"2023-10-09T06:00:14.299398+0000","last_fullsized":"2023-10-09T06:00:14.299398+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":411,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.296185+0000","last_change":"2023-10-09T05:54:05.161959+0000","last_active":"2023-10-09T06:00:14.296185+0000","last_peered":"2023-10-09T06:00:14.296185+0000","last_clean":"2023-10-09T06:00:14.296185+0000","last_became_active":"2023-10-09T05:54:05.161671+0000","last_became_peered":"2023-10-09T05:54:05.161671+0000","last_unstale":"2023-10-09T06:00:14.296185+0000","last_undegraded":"2023-10-09T06:00:14.296185+0000","last_fullsized":"2023-10-09T06:00:14.296185+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:47.282176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"481'6672","reported_seq":12631,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:19.786222+0000","last_change":"2023-10-09T05:56:01.447837+0000","last_active":"2023-10-09T06:00:19.786222+0000","last_peered":"2023-10-09T06:00:19.786222+0000","last_clean":"2023-10-09T05:55:25.103026+0000","last_became_active":"2023-10-09T05:56:01.440057+0000","last_became_peered":"2023-10-09T05:56:01.440057+0000","last_unstale":"2023-10-09T06:00:19.786222+0000","last_undegraded":"2023-10-09T05:56:01.309926+0000","last_fullsized":"2023-10-09T05:56:01.207038+0000","mapping_epoch":416,"log_start":"368'681","ondisk_log_start":"368'681","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5991,"log_dups_size":0,"ondisk_log_size":5991,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62455808,"num_objects":953,"num_object_clones":0,"num_object_copies":2859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4259,"num_write_kb":166784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1468,"num_bytes_recovered":96206848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1,3","objects":-1122},{"shards":"1,3,5","objects":982}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":484,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.295380+0000","last_change":"2023-10-09T05:55:26.256379+0000","last_active":"2023-10-09T06:00:14.295380+0000","last_peered":"2023-10-09T06:00:14.295380+0000","last_clean":"2023-10-09T06:00:14.295380+0000","last_became_active":"2023-10-09T05:55:26.256062+0000","last_became_peered":"2023-10-09T05:55:26.256062+0000","last_unstale":"2023-10-09T06:00:14.295380+0000","last_undegraded":"2023-10-09T06:00:14.295380+0000","last_fullsized":"2023-10-09T06:00:14.295380+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:00.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":43,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.300326+0000","last_change":"2023-10-09T05:58:07.397389+0000","last_active":"2023-10-09T06:00:14.300326+0000","last_peered":"2023-10-09T06:00:14.300326+0000","last_clean":"2023-10-09T06:00:14.300326+0000","last_became_active":"2023-10-09T05:58:07.397034+0000","last_became_peered":"2023-10-09T05:58:07.397034+0000","last_unstale":"2023-10-09T06:00:14.300326+0000","last_undegraded":"2023-10-09T06:00:14.300326+0000","last_fullsized":"2023-10-09T06:00:14.300326+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2","version":"481'6663","reported_seq":12162,"reported_epoch":481,"state":"active+clean","last_fresh":"2023-10-09T06:00:20.142343+0000","last_change":"2023-10-09T05:58:52.969080+0000","last_active":"2023-10-09T06:00:20.142343+0000","last_peered":"2023-10-09T06:00:20.142343+0000","last_clean":"2023-10-09T06:00:20.142343+0000","last_became_active":"2023-10-09T05:58:52.906218+0000","last_became_peered":"2023-10-09T05:58:52.906218+0000","last_unstale":"2023-10-09T06:00:20.142343+0000","last_undegraded":"2023-10-09T06:00:20.142343+0000","last_fullsized":"2023-10-09T06:00:20.142343+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6663,"log_dups_size":0,"ondisk_log_size":6663,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65241088,"num_objects":995,"num_object_clones":0,"num_object_copies":2985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4241,"num_write_kb":167584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2659,"num_bytes_recovered":101056512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":115,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.299624+0000","last_change":"2023-10-09T05:56:49.287906+0000","last_active":"2023-10-09T06:00:14.299624+0000","last_peered":"2023-10-09T06:00:14.299624+0000","last_clean":"2023-10-09T06:00:14.299624+0000","last_became_active":"2023-10-09T05:56:49.275081+0000","last_became_peered":"2023-10-09T05:56:49.275081+0000","last_unstale":"2023-10-09T06:00:14.299624+0000","last_undegraded":"2023-10-09T06:00:14.299624+0000","last_fullsized":"2023-10-09T06:00:14.299624+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":169,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:15.231427+0000","last_change":"2023-10-09T05:56:00.225724+0000","last_active":"2023-10-09T06:00:15.231427+0000","last_peered":"2023-10-09T06:00:15.231427+0000","last_clean":"2023-10-09T06:00:15.231427+0000","last_became_active":"2023-10-09T05:56:00.225335+0000","last_became_peered":"2023-10-09T05:56:00.225335+0000","last_unstale":"2023-10-09T06:00:15.231427+0000","last_undegraded":"2023-10-09T06:00:15.231427+0000","last_fullsized":"2023-10-09T06:00:15.231427+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:41.405222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"481'8606","reported_seq":8755,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:19.983662+0000","last_change":"2023-10-09T05:57:58.501288+0000","last_active":"2023-10-09T06:00:19.983662+0000","last_peered":"2023-10-09T06:00:19.983662+0000","last_clean":"2023-10-09T05:55:49.154347+0000","last_became_active":"2023-10-09T05:57:58.384024+0000","last_became_peered":"2023-10-09T05:57:58.384024+0000","last_unstale":"2023-10-09T06:00:19.983662+0000","last_undegraded":"2023-10-09T05:57:58.373205+0000","last_fullsized":"2023-10-09T05:57:58.246583+0000","mapping_epoch":444,"log_start":"380'4602","ondisk_log_start":"380'4602","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4004,"log_dups_size":0,"ondisk_log_size":4004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:58.015603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136314880,"num_objects":2080,"num_object_clones":0,"num_object_copies":6240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5436,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8606,"num_write_kb":341952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[1,3],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,3","objects":-1161},{"shards":"1,3,5","objects":2080}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":908,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.303707+0000","last_change":"2023-10-09T06:00:05.523981+0000","last_active":"2023-10-09T06:00:14.303707+0000","last_peered":"2023-10-09T06:00:14.303707+0000","last_clean":"2023-10-09T06:00:14.303707+0000","last_became_active":"2023-10-09T06:00:05.338627+0000","last_became_peered":"2023-10-09T06:00:05.338627+0000","last_unstale":"2023-10-09T06:00:14.303707+0000","last_undegraded":"2023-10-09T06:00:14.303707+0000","last_fullsized":"2023-10-09T06:00:14.303707+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:25.623236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":465,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.296268+0000","last_change":"2023-10-09T05:58:07.410383+0000","last_active":"2023-10-09T06:00:14.296268+0000","last_peered":"2023-10-09T06:00:14.296268+0000","last_clean":"2023-10-09T06:00:14.296268+0000","last_became_active":"2023-10-09T05:58:07.410041+0000","last_became_peered":"2023-10-09T05:58:07.410041+0000","last_unstale":"2023-10-09T06:00:14.296268+0000","last_undegraded":"2023-10-09T06:00:14.296268+0000","last_fullsized":"2023-10-09T06:00:14.296268+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:30.848859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":16,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.303394+0000","last_change":"2023-10-09T06:00:05.355057+0000","last_active":"2023-10-09T06:00:14.303394+0000","last_peered":"2023-10-09T06:00:14.303394+0000","last_clean":"2023-10-09T06:00:14.303394+0000","last_became_active":"2023-10-09T06:00:05.352475+0000","last_became_peered":"2023-10-09T06:00:05.352475+0000","last_unstale":"2023-10-09T06:00:14.303394+0000","last_undegraded":"2023-10-09T06:00:14.303394+0000","last_fullsized":"2023-10-09T06:00:14.303394+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:50.255773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.9","version":"481'6619","reported_seq":9622,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:19.656203+0000","last_change":"2023-10-09T06:00:09.349262+0000","last_active":"2023-10-09T06:00:19.656203+0000","last_peered":"2023-10-09T06:00:19.656203+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T06:00:08.202686+0000","last_became_peered":"2023-10-09T06:00:08.202686+0000","last_unstale":"2023-10-09T06:00:19.656203+0000","last_undegraded":"2023-10-09T06:00:07.600855+0000","last_fullsized":"2023-10-09T06:00:06.282656+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6619,"log_dups_size":0,"ondisk_log_size":6619,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65830912,"num_objects":1004,"num_object_clones":0,"num_object_copies":3012,"num_objects_missing_on_primary":1448,"num_objects_missing":1448,"num_objects_degraded":4500,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4212,"num_write_kb":166944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1029,"num_bytes_recovered":67469312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[7,0],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,3,5","objects":999},{"shards":"0,7","objects":3}],"blocked_by":[],"up_primary":0,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":15,"reported_epoch":480,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:00:14.303090+0000","last_change":"2023-10-09T06:00:05.354800+0000","last_active":"2023-10-09T06:00:14.303090+0000","last_peered":"2023-10-09T06:00:14.303090+0000","last_clean":"2023-10-09T05:48:52.714098+0000","last_became_active":"2023-10-09T06:00:05.351822+0000","last_became_peered":"2023-10-09T06:00:05.351822+0000","last_unstale":"2023-10-09T06:00:14.303090+0000","last_undegraded":"2023-10-09T06:00:05.281772+0000","last_fullsized":"2023-10-09T06:00:14.303090+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:20.567186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":103,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.305566+0000","last_change":"2023-10-09T06:00:05.329822+0000","last_active":"2023-10-09T06:00:14.305566+0000","last_peered":"2023-10-09T06:00:14.305566+0000","last_clean":"2023-10-09T06:00:14.305566+0000","last_became_active":"2023-10-09T06:00:05.329453+0000","last_became_peered":"2023-10-09T06:00:05.329453+0000","last_unstale":"2023-10-09T06:00:14.305566+0000","last_undegraded":"2023-10-09T06:00:14.305566+0000","last_fullsized":"2023-10-09T06:00:14.305566+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:22.364221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":326,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.295804+0000","last_change":"2023-10-09T05:59:31.402401+0000","last_active":"2023-10-09T06:00:14.295804+0000","last_peered":"2023-10-09T06:00:14.295804+0000","last_clean":"2023-10-09T06:00:14.295804+0000","last_became_active":"2023-10-09T05:59:31.402040+0000","last_became_peered":"2023-10-09T05:59:31.402040+0000","last_unstale":"2023-10-09T06:00:14.295804+0000","last_undegraded":"2023-10-09T06:00:14.295804+0000","last_fullsized":"2023-10-09T06:00:14.295804+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:45.087787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"481'8555","reported_seq":19356,"reported_epoch":481,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:00:19.659854+0000","last_change":"2023-10-09T05:59:32.537556+0000","last_active":"2023-10-09T06:00:19.659854+0000","last_peered":"2023-10-09T06:00:19.659854+0000","last_clean":"2023-10-09T05:59:30.366736+0000","last_became_active":"2023-10-09T05:59:32.493000+0000","last_became_peered":"2023-10-09T05:59:32.493000+0000","last_unstale":"2023-10-09T06:00:19.659854+0000","last_undegraded":"2023-10-09T06:00:19.659854+0000","last_fullsized":"2023-10-09T06:00:19.659854+0000","mapping_epoch":470,"log_start":"380'4276","ondisk_log_start":"380'4276","created":364,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4279,"log_dups_size":0,"ondisk_log_size":4279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134414336,"num_objects":2051,"num_object_clones":0,"num_object_copies":6153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2051,"num_objects_unfound":0,"num_objects_dirty":2051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8555,"num_write_kb":339392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5309,"num_bytes_recovered":309723136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,3],"avail_no_missing":["7","1","3","5"],"object_location_counts":[{"shards":"1,3,7","objects":2051}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":838,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.295445+0000","last_change":"2023-10-09T06:00:05.324594+0000","last_active":"2023-10-09T06:00:14.295445+0000","last_peered":"2023-10-09T06:00:14.295445+0000","last_clean":"2023-10-09T06:00:14.295445+0000","last_became_active":"2023-10-09T06:00:05.323845+0000","last_became_peered":"2023-10-09T06:00:05.323845+0000","last_unstale":"2023-10-09T06:00:14.295445+0000","last_undegraded":"2023-10-09T06:00:14.295445+0000","last_fullsized":"2023-10-09T06:00:14.295445+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:02.539703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":283,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.297275+0000","last_change":"2023-10-09T06:00:05.417222+0000","last_active":"2023-10-09T06:00:14.297275+0000","last_peered":"2023-10-09T06:00:14.297275+0000","last_clean":"2023-10-09T06:00:14.297275+0000","last_became_active":"2023-10-09T06:00:05.416934+0000","last_became_peered":"2023-10-09T06:00:05.416934+0000","last_unstale":"2023-10-09T06:00:14.297275+0000","last_undegraded":"2023-10-09T06:00:14.297275+0000","last_fullsized":"2023-10-09T06:00:14.297275+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:54.278229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":219,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.304375+0000","last_change":"2023-10-09T06:00:05.328910+0000","last_active":"2023-10-09T06:00:14.304375+0000","last_peered":"2023-10-09T06:00:14.304375+0000","last_clean":"2023-10-09T06:00:14.304375+0000","last_became_active":"2023-10-09T06:00:05.328512+0000","last_became_peered":"2023-10-09T06:00:05.328512+0000","last_unstale":"2023-10-09T06:00:14.304375+0000","last_undegraded":"2023-10-09T06:00:14.304375+0000","last_fullsized":"2023-10-09T06:00:14.304375+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:27.757193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"481'6601","reported_seq":17261,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:19.619997+0000","last_change":"2023-10-09T06:00:06.413710+0000","last_active":"2023-10-09T06:00:19.619997+0000","last_peered":"2023-10-09T06:00:19.619997+0000","last_clean":"2023-10-09T06:00:04.116692+0000","last_became_active":"2023-10-09T06:00:06.395225+0000","last_became_peered":"2023-10-09T06:00:06.395225+0000","last_unstale":"2023-10-09T06:00:19.619997+0000","last_undegraded":"2023-10-09T06:00:06.304668+0000","last_fullsized":"2023-10-09T06:00:06.297952+0000","mapping_epoch":476,"log_start":"367'435","ondisk_log_start":"367'435","created":364,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6166,"log_dups_size":0,"ondisk_log_size":6166,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68550656,"num_objects":1046,"num_object_clones":0,"num_object_copies":3138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":991,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4188,"num_write_kb":167488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2749,"num_bytes_recovered":132644864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[1,7],"avail_no_missing":["1","2","5","7"],"object_location_counts":[{"shards":"1,7","objects":1046}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":695,"reported_epoch":480,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:14.299612+0000","last_change":"2023-10-09T05:58:07.894313+0000","last_active":"2023-10-09T06:00:14.299612+0000","last_peered":"2023-10-09T06:00:14.299612+0000","last_clean":"2023-10-09T05:55:22.245899+0000","last_became_active":"2023-10-09T05:58:05.515736+0000","last_became_peered":"2023-10-09T05:58:05.515736+0000","last_unstale":"2023-10-09T06:00:14.299612+0000","last_undegraded":"2023-10-09T05:57:58.245342+0000","last_fullsized":"2023-10-09T05:57:58.244553+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:13.491224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":887,"num_read_kb":802,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":636,"reported_epoch":480,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:00:14.299198+0000","last_change":"2023-10-09T06:00:06.312906+0000","last_active":"2023-10-09T06:00:14.299198+0000","last_peered":"2023-10-09T06:00:14.299198+0000","last_clean":"2023-10-09T05:56:02.257688+0000","last_became_active":"2023-10-09T06:00:06.312584+0000","last_became_peered":"2023-10-09T06:00:06.312584+0000","last_unstale":"2023-10-09T06:00:14.299198+0000","last_undegraded":"2023-10-09T06:00:06.284085+0000","last_fullsized":"2023-10-09T06:00:14.299198+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":["7","0","6"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":350,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:00:11.364845+0000","last_change":"2023-10-09T05:58:07.387935+0000","last_active":"2023-10-09T06:00:11.364845+0000","last_peered":"2023-10-09T06:00:11.364845+0000","last_clean":"2023-10-09T06:00:11.364845+0000","last_became_active":"2023-10-09T05:58:07.386962+0000","last_became_peered":"2023-10-09T05:58:07.386962+0000","last_unstale":"2023-10-09T06:00:11.364845+0000","last_undegraded":"2023-10-09T06:00:11.364845+0000","last_fullsized":"2023-10-09T06:00:11.364845+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:02.334925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":101,"reported_epoch":480,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:00:16.205780+0000","last_change":"2023-10-09T06:00:16.205780+0000","last_active":"2023-10-09T06:00:16.205780+0000","last_peered":"2023-10-09T06:00:16.205780+0000","last_clean":"2023-10-09T05:59:55.357888+0000","last_became_active":"2023-10-09T06:00:10.873679+0000","last_became_peered":"2023-10-09T06:00:10.873679+0000","last_unstale":"2023-10-09T06:00:16.205780+0000","last_undegraded":"2023-10-09T06:00:16.205780+0000","last_fullsized":"2023-10-09T06:00:16.205780+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,1],"avail_no_missing":["7","0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"481'8679","reported_seq":9054,"reported_epoch":481,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T06:00:22.339454+0000","last_change":"2023-10-09T06:00:06.522003+0000","last_active":"2023-10-09T06:00:22.339454+0000","last_peered":"2023-10-09T06:00:22.339454+0000","last_clean":"2023-10-09T05:56:10.073783+0000","last_became_active":"2023-10-09T06:00:06.462342+0000","last_became_peered":"2023-10-09T06:00:06.462342+0000","last_unstale":"2023-10-09T06:00:22.339454+0000","last_undegraded":"2023-10-09T06:00:06.454228+0000","last_fullsized":"2023-10-09T06:00:22.339454+0000","mapping_epoch":476,"log_start":"377'3441","ondisk_log_start":"377'3441","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5238,"log_dups_size":0,"ondisk_log_size":5238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:15.032961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126681088,"num_objects":1933,"num_object_clones":0,"num_object_copies":5799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1933,"num_objects_misplaced":5195,"num_objects_unfound":0,"num_objects_dirty":1933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8679,"num_write_kb":339584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":111,"num_bytes_recovered":7274496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,6],"avail_no_missing":["0","3","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":1933}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":1008,"reported_epoch":480,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:00:14.305903+0000","last_change":"2023-10-09T06:00:06.308995+0000","last_active":"2023-10-09T06:00:14.305903+0000","last_peered":"2023-10-09T06:00:14.305903+0000","last_clean":"2023-10-09T05:56:02.257755+0000","last_became_active":"2023-10-09T06:00:06.308176+0000","last_became_peered":"2023-10-09T06:00:06.308176+0000","last_unstale":"2023-10-09T06:00:14.305903+0000","last_undegraded":"2023-10-09T06:00:06.282859+0000","last_fullsized":"2023-10-09T06:00:14.305903+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":449,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.303895+0000","last_change":"2023-10-09T06:00:05.314057+0000","last_active":"2023-10-09T06:00:14.303895+0000","last_peered":"2023-10-09T06:00:14.303895+0000","last_clean":"2023-10-09T06:00:14.303895+0000","last_became_active":"2023-10-09T06:00:05.313743+0000","last_became_peered":"2023-10-09T06:00:05.313743+0000","last_unstale":"2023-10-09T06:00:14.303895+0000","last_undegraded":"2023-10-09T06:00:14.303895+0000","last_fullsized":"2023-10-09T06:00:14.303895+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:29.568391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":427,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.304407+0000","last_change":"2023-10-09T06:00:05.590816+0000","last_active":"2023-10-09T06:00:14.304407+0000","last_peered":"2023-10-09T06:00:14.304407+0000","last_clean":"2023-10-09T06:00:14.304407+0000","last_became_active":"2023-10-09T06:00:05.590227+0000","last_became_peered":"2023-10-09T06:00:05.590227+0000","last_unstale":"2023-10-09T06:00:14.304407+0000","last_undegraded":"2023-10-09T06:00:14.304407+0000","last_fullsized":"2023-10-09T06:00:14.304407+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:51.060693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"481'8402","reported_seq":12995,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:19.643355+0000","last_change":"2023-10-09T06:00:13.414491+0000","last_active":"2023-10-09T06:00:19.643355+0000","last_peered":"2023-10-09T06:00:19.643355+0000","last_clean":"2023-10-09T05:53:45.896176+0000","last_became_active":"2023-10-09T06:00:12.549930+0000","last_became_peered":"2023-10-09T06:00:12.549930+0000","last_unstale":"2023-10-09T06:00:19.643355+0000","last_undegraded":"2023-10-09T06:00:11.004109+0000","last_fullsized":"2023-10-09T06:00:06.282679+0000","mapping_epoch":476,"log_start":"380'4338","ondisk_log_start":"380'4338","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4064,"log_dups_size":0,"ondisk_log_size":4064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133824512,"num_objects":2042,"num_object_clones":0,"num_object_copies":6126,"num_objects_missing_on_primary":3256,"num_objects_missing":3256,"num_objects_degraded":4219,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8402,"num_write_kb":334208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2205,"num_bytes_recovered":144244736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"0,1,4","objects":2036},{"shards":"0,1,4,7","objects":6}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":656,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:17.338594+0000","last_change":"2023-10-09T06:00:17.336621+0000","last_active":"2023-10-09T06:00:17.338594+0000","last_peered":"2023-10-09T06:00:17.338594+0000","last_clean":"2023-10-09T06:00:17.338594+0000","last_became_active":"2023-10-09T06:00:14.622413+0000","last_became_peered":"2023-10-09T06:00:14.622413+0000","last_unstale":"2023-10-09T06:00:17.338594+0000","last_undegraded":"2023-10-09T06:00:17.338594+0000","last_fullsized":"2023-10-09T06:00:17.338594+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":310,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.295967+0000","last_change":"2023-10-09T06:00:05.324944+0000","last_active":"2023-10-09T06:00:14.295967+0000","last_peered":"2023-10-09T06:00:14.295967+0000","last_clean":"2023-10-09T06:00:14.295967+0000","last_became_active":"2023-10-09T06:00:05.324533+0000","last_became_peered":"2023-10-09T06:00:05.324533+0000","last_unstale":"2023-10-09T06:00:14.295967+0000","last_undegraded":"2023-10-09T06:00:14.295967+0000","last_fullsized":"2023-10-09T06:00:14.295967+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:33:33.376129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":431,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.305519+0000","last_change":"2023-10-09T06:00:05.352906+0000","last_active":"2023-10-09T06:00:14.305519+0000","last_peered":"2023-10-09T06:00:14.305519+0000","last_clean":"2023-10-09T06:00:14.305519+0000","last_became_active":"2023-10-09T06:00:05.352436+0000","last_became_peered":"2023-10-09T06:00:05.352436+0000","last_unstale":"2023-10-09T06:00:14.305519+0000","last_undegraded":"2023-10-09T06:00:14.305519+0000","last_fullsized":"2023-10-09T06:00:14.305519+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:20.223563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"481'8470","reported_seq":12078,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:19.622195+0000","last_change":"2023-10-09T06:00:17.342081+0000","last_active":"2023-10-09T06:00:19.622195+0000","last_peered":"2023-10-09T06:00:19.622195+0000","last_clean":"2023-10-09T05:53:45.883404+0000","last_became_active":"2023-10-09T06:00:16.216632+0000","last_became_peered":"2023-10-09T06:00:16.216632+0000","last_unstale":"2023-10-09T06:00:19.622195+0000","last_undegraded":"2023-10-09T06:00:10.815413+0000","last_fullsized":"2023-10-09T06:00:06.285511+0000","mapping_epoch":476,"log_start":"380'4196","ondisk_log_start":"380'4196","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4274,"log_dups_size":0,"ondisk_log_size":4274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131006488,"num_objects":2000,"num_object_clones":0,"num_object_copies":6000,"num_objects_missing_on_primary":2929,"num_objects_missing":2929,"num_objects_degraded":3884,"num_objects_misplaced":2929,"num_objects_unfound":0,"num_objects_dirty":2000,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8470,"num_write_kb":334977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1710,"num_bytes_recovered":111935488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,5],"avail_no_missing":["1","2","5","6"],"object_location_counts":[{"shards":"1,4,5","objects":1999},{"shards":"1,4,5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":147,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.302512+0000","last_change":"2023-10-09T06:00:05.329139+0000","last_active":"2023-10-09T06:00:14.302512+0000","last_peered":"2023-10-09T06:00:14.302512+0000","last_clean":"2023-10-09T06:00:14.302512+0000","last_became_active":"2023-10-09T06:00:05.328811+0000","last_became_peered":"2023-10-09T06:00:05.328811+0000","last_unstale":"2023-10-09T06:00:14.302512+0000","last_undegraded":"2023-10-09T06:00:14.302512+0000","last_fullsized":"2023-10-09T06:00:14.302512+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":593,"reported_epoch":480,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T06:00:15.227720+0000","last_change":"2023-10-09T06:00:05.402696+0000","last_active":"2023-10-09T06:00:15.227720+0000","last_peered":"2023-10-09T06:00:15.227720+0000","last_clean":"2023-10-09T05:58:01.663036+0000","last_became_active":"2023-10-09T06:00:05.402085+0000","last_became_peered":"2023-10-09T06:00:05.402085+0000","last_unstale":"2023-10-09T06:00:15.227720+0000","last_undegraded":"2023-10-09T06:00:05.299683+0000","last_fullsized":"2023-10-09T06:00:15.227720+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:30.838702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[1,4,3],"avail_no_missing":["1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":151,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.299699+0000","last_change":"2023-10-09T06:00:05.328236+0000","last_active":"2023-10-09T06:00:14.299699+0000","last_peered":"2023-10-09T06:00:14.299699+0000","last_clean":"2023-10-09T06:00:14.299699+0000","last_became_active":"2023-10-09T06:00:05.327876+0000","last_became_peered":"2023-10-09T06:00:05.327876+0000","last_unstale":"2023-10-09T06:00:14.299699+0000","last_undegraded":"2023-10-09T06:00:14.299699+0000","last_fullsized":"2023-10-09T06:00:14.299699+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"481'8553","reported_seq":15478,"reported_epoch":481,"state":"active+recovering","last_fresh":"2023-10-09T06:00:19.555137+0000","last_change":"2023-10-09T05:56:49.370077+0000","last_active":"2023-10-09T06:00:19.555137+0000","last_peered":"2023-10-09T06:00:19.555137+0000","last_clean":"2023-10-09T05:53:45.907519+0000","last_became_active":"2023-10-09T05:56:03.326563+0000","last_became_peered":"2023-10-09T05:56:03.326563+0000","last_unstale":"2023-10-09T06:00:19.555137+0000","last_undegraded":"2023-10-09T06:00:19.555137+0000","last_fullsized":"2023-10-09T06:00:19.555137+0000","mapping_epoch":416,"log_start":"380'4271","ondisk_log_start":"380'4271","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4282,"log_dups_size":0,"ondisk_log_size":4282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133496832,"num_objects":2037,"num_object_clones":0,"num_object_copies":6111,"num_objects_missing_on_primary":625,"num_objects_missing":625,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8553,"num_write_kb":338880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3377,"num_bytes_recovered":200015872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["4"],"object_location_counts":[{"shards":"3,4,7","objects":2037}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":659,"reported_epoch":480,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:14.302336+0000","last_change":"2023-10-09T05:58:08.253520+0000","last_active":"2023-10-09T06:00:14.302336+0000","last_peered":"2023-10-09T06:00:14.302336+0000","last_clean":"2023-10-09T05:58:02.145081+0000","last_became_active":"2023-10-09T05:58:08.253168+0000","last_became_peered":"2023-10-09T05:58:08.253168+0000","last_unstale":"2023-10-09T06:00:14.302336+0000","last_undegraded":"2023-10-09T05:58:08.242171+0000","last_fullsized":"2023-10-09T05:58:08.239234+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":422,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.304457+0000","last_change":"2023-10-09T06:00:05.352581+0000","last_active":"2023-10-09T06:00:14.304457+0000","last_peered":"2023-10-09T06:00:14.304457+0000","last_clean":"2023-10-09T06:00:14.304457+0000","last_became_active":"2023-10-09T06:00:05.351836+0000","last_became_peered":"2023-10-09T06:00:05.351836+0000","last_unstale":"2023-10-09T06:00:14.304457+0000","last_undegraded":"2023-10-09T06:00:14.304457+0000","last_fullsized":"2023-10-09T06:00:14.304457+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:31.576840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":16,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.303808+0000","last_change":"2023-10-09T06:00:05.314490+0000","last_active":"2023-10-09T06:00:14.303808+0000","last_peered":"2023-10-09T06:00:14.303808+0000","last_clean":"2023-10-09T06:00:14.303808+0000","last_became_active":"2023-10-09T06:00:05.314201+0000","last_became_peered":"2023-10-09T06:00:05.314201+0000","last_unstale":"2023-10-09T06:00:14.303808+0000","last_undegraded":"2023-10-09T06:00:14.303808+0000","last_fullsized":"2023-10-09T06:00:14.303808+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:15.664170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.11","version":"481'6686","reported_seq":15807,"reported_epoch":481,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:00:19.534341+0000","last_change":"2023-10-09T05:58:20.369743+0000","last_active":"2023-10-09T06:00:19.534341+0000","last_peered":"2023-10-09T06:00:19.534341+0000","last_clean":"2023-10-09T05:58:02.997585+0000","last_became_active":"2023-10-09T05:58:15.913938+0000","last_became_peered":"2023-10-09T05:58:15.913938+0000","last_unstale":"2023-10-09T06:00:19.534341+0000","last_undegraded":"2023-10-09T06:00:19.534341+0000","last_fullsized":"2023-10-09T06:00:19.534341+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6686,"log_dups_size":0,"ondisk_log_size":6686,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70680576,"num_objects":1079,"num_object_clones":0,"num_object_copies":3237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1079,"num_objects_unfound":0,"num_objects_dirty":1079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4233,"num_write_kb":169952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3255,"num_bytes_recovered":213286912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,6],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":1079}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":432,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.305471+0000","last_change":"2023-10-09T06:00:05.416922+0000","last_active":"2023-10-09T06:00:14.305471+0000","last_peered":"2023-10-09T06:00:14.305471+0000","last_clean":"2023-10-09T06:00:14.305471+0000","last_became_active":"2023-10-09T06:00:05.416444+0000","last_became_peered":"2023-10-09T06:00:05.416444+0000","last_unstale":"2023-10-09T06:00:14.305471+0000","last_undegraded":"2023-10-09T06:00:14.305471+0000","last_fullsized":"2023-10-09T06:00:14.305471+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:26.775246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":173,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.295872+0000","last_change":"2023-10-09T05:56:00.226695+0000","last_active":"2023-10-09T06:00:14.295872+0000","last_peered":"2023-10-09T06:00:14.295872+0000","last_clean":"2023-10-09T06:00:14.295872+0000","last_became_active":"2023-10-09T05:56:00.226359+0000","last_became_peered":"2023-10-09T05:56:00.226359+0000","last_unstale":"2023-10-09T06:00:14.295872+0000","last_undegraded":"2023-10-09T06:00:14.295872+0000","last_fullsized":"2023-10-09T06:00:14.295872+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:20.323704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"481'6647","reported_seq":6812,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:19.713259+0000","last_change":"2023-10-09T06:00:05.607274+0000","last_active":"2023-10-09T06:00:19.713259+0000","last_peered":"2023-10-09T06:00:19.713259+0000","last_clean":"2023-10-09T05:57:55.523800+0000","last_became_active":"2023-10-09T06:00:05.593640+0000","last_became_peered":"2023-10-09T06:00:05.593640+0000","last_unstale":"2023-10-09T06:00:19.713259+0000","last_undegraded":"2023-10-09T06:00:05.320966+0000","last_fullsized":"2023-10-09T06:00:05.295530+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6647,"log_dups_size":0,"ondisk_log_size":6647,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62914560,"num_objects":960,"num_object_clones":0,"num_object_copies":2880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2624,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4248,"num_write_kb":166656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1],"avail_no_missing":["2","1","5"],"object_location_counts":[{"shards":"1,2","objects":-38},{"shards":"1,2,5","objects":980}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":41,"reported_epoch":480,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:00:14.299948+0000","last_change":"2023-10-09T05:58:07.399584+0000","last_active":"2023-10-09T06:00:14.299948+0000","last_peered":"2023-10-09T06:00:14.299948+0000","last_clean":"2023-10-09T05:48:52.714337+0000","last_became_active":"2023-10-09T05:58:07.399203+0000","last_became_peered":"2023-10-09T05:58:07.399203+0000","last_unstale":"2023-10-09T06:00:14.299948+0000","last_undegraded":"2023-10-09T05:58:07.381974+0000","last_fullsized":"2023-10-09T06:00:14.299948+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["1","4","5","6"],"object_location_counts":[{"shards":"1,4,5,6","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":63,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.299375+0000","last_change":"2023-10-09T05:56:49.252225+0000","last_active":"2023-10-09T06:00:14.299375+0000","last_peered":"2023-10-09T06:00:14.299375+0000","last_clean":"2023-10-09T06:00:14.299375+0000","last_became_active":"2023-10-09T05:56:49.251880+0000","last_became_peered":"2023-10-09T05:56:49.251880+0000","last_unstale":"2023-10-09T06:00:14.299375+0000","last_undegraded":"2023-10-09T06:00:14.299375+0000","last_fullsized":"2023-10-09T06:00:14.299375+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:34.079170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.17","version":"481'6754","reported_seq":13362,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:19.509105+0000","last_change":"2023-10-09T05:58:02.382245+0000","last_active":"2023-10-09T06:00:19.509105+0000","last_peered":"2023-10-09T06:00:19.509105+0000","last_clean":"2023-10-09T05:57:55.399317+0000","last_became_active":"2023-10-09T05:58:02.319859+0000","last_became_peered":"2023-10-09T05:58:02.319859+0000","last_unstale":"2023-10-09T06:00:19.509105+0000","last_undegraded":"2023-10-09T05:58:01.234664+0000","last_fullsized":"2023-10-09T05:58:01.215806+0000","mapping_epoch":444,"log_start":"367'380","ondisk_log_start":"367'380","created":385,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6374,"log_dups_size":0,"ondisk_log_size":6374,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60686336,"num_objects":926,"num_object_clones":0,"num_object_copies":2778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2713,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4330,"num_write_kb":168192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3182,"num_bytes_recovered":157876224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["7","2","3","5"],"object_location_counts":[{"shards":"2,5,7","objects":1011},{"shards":"2,7","objects":-683}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":37,"reported_epoch":480,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:00:16.206194+0000","last_change":"2023-10-09T05:58:07.410770+0000","last_active":"2023-10-09T06:00:16.206194+0000","last_peered":"2023-10-09T06:00:16.206194+0000","last_clean":"2023-10-09T05:21:17.840829+0000","last_became_active":"2023-10-09T05:58:07.410323+0000","last_became_peered":"2023-10-09T05:58:07.410323+0000","last_unstale":"2023-10-09T06:00:16.206194+0000","last_undegraded":"2023-10-09T05:58:07.390791+0000","last_fullsized":"2023-10-09T06:00:16.206194+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["1","4","5","6"],"object_location_counts":[{"shards":"1,4,5,6","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":227,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.303495+0000","last_change":"2023-10-09T06:00:05.340542+0000","last_active":"2023-10-09T06:00:14.303495+0000","last_peered":"2023-10-09T06:00:14.303495+0000","last_clean":"2023-10-09T06:00:14.303495+0000","last_became_active":"2023-10-09T06:00:05.340213+0000","last_became_peered":"2023-10-09T06:00:05.340213+0000","last_unstale":"2023-10-09T06:00:14.303495+0000","last_undegraded":"2023-10-09T06:00:14.303495+0000","last_fullsized":"2023-10-09T06:00:14.303495+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:34.412535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.14","version":"481'6672","reported_seq":6831,"reported_epoch":481,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:00:20.032701+0000","last_change":"2023-10-09T05:56:01.281023+0000","last_active":"2023-10-09T06:00:20.032701+0000","last_peered":"2023-10-09T06:00:20.032701+0000","last_clean":"2023-10-09T05:55:48.964110+0000","last_became_active":"2023-10-09T05:56:01.245699+0000","last_became_peered":"2023-10-09T05:56:01.245699+0000","last_unstale":"2023-10-09T06:00:20.032701+0000","last_undegraded":"2023-10-09T05:56:01.216362+0000","last_fullsized":"2023-10-09T06:00:20.032701+0000","mapping_epoch":416,"log_start":"368'556","ondisk_log_start":"368'556","created":385,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6116,"log_dups_size":0,"ondisk_log_size":6116,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67633152,"num_objects":1032,"num_object_clones":0,"num_object_copies":3096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":40,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4242,"num_write_kb":168768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1032}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":230,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.297244+0000","last_change":"2023-10-09T05:58:07.415396+0000","last_active":"2023-10-09T06:00:14.297244+0000","last_peered":"2023-10-09T06:00:14.297244+0000","last_clean":"2023-10-09T06:00:14.297244+0000","last_became_active":"2023-10-09T05:58:07.414511+0000","last_became_peered":"2023-10-09T05:58:07.414511+0000","last_unstale":"2023-10-09T06:00:14.297244+0000","last_undegraded":"2023-10-09T06:00:14.297244+0000","last_fullsized":"2023-10-09T06:00:14.297244+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:10.158875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":339,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.295507+0000","last_change":"2023-10-09T06:00:05.325284+0000","last_active":"2023-10-09T06:00:14.295507+0000","last_peered":"2023-10-09T06:00:14.295507+0000","last_clean":"2023-10-09T06:00:14.295507+0000","last_became_active":"2023-10-09T06:00:05.324588+0000","last_became_peered":"2023-10-09T06:00:05.324588+0000","last_unstale":"2023-10-09T06:00:14.295507+0000","last_undegraded":"2023-10-09T06:00:14.295507+0000","last_fullsized":"2023-10-09T06:00:14.295507+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:59.068162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"481'6746","reported_seq":9299,"reported_epoch":481,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:00:20.085521+0000","last_change":"2023-10-09T06:00:17.809360+0000","last_active":"2023-10-09T06:00:20.085521+0000","last_peered":"2023-10-09T06:00:20.085521+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T06:00:17.616889+0000","last_became_peered":"2023-10-09T06:00:17.616889+0000","last_unstale":"2023-10-09T06:00:20.085521+0000","last_undegraded":"2023-10-09T06:00:12.461200+0000","last_fullsized":"2023-10-09T06:00:07.517576+0000","mapping_epoch":477,"log_start":"368'736","ondisk_log_start":"368'736","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6010,"log_dups_size":0,"ondisk_log_size":6010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66224128,"num_objects":1011,"num_object_clones":0,"num_object_copies":3033,"num_objects_missing_on_primary":847,"num_objects_missing":847,"num_objects_degraded":3390,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4305,"num_write_kb":170080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":770,"num_bytes_recovered":50462720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[1,0],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,1","objects":-40},{"shards":"0,1,5","objects":1026}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":427,"reported_epoch":480,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:00:14.303100+0000","last_change":"2023-10-09T06:00:05.353559+0000","last_active":"2023-10-09T06:00:14.303100+0000","last_peered":"2023-10-09T06:00:14.303100+0000","last_clean":"2023-10-09T05:57:53.237962+0000","last_became_active":"2023-10-09T06:00:05.353245+0000","last_became_peered":"2023-10-09T06:00:05.353245+0000","last_unstale":"2023-10-09T06:00:14.303100+0000","last_undegraded":"2023-10-09T06:00:05.290221+0000","last_fullsized":"2023-10-09T06:00:14.303100+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:46.427146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":["0","2","5","6"],"object_location_counts":[{"shards":"0,2,5,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":22,"reported_epoch":480,"state":"active+clean","last_fresh":"2023-10-09T06:00:14.294648+0000","last_change":"2023-10-09T05:59:31.411398+0000","last_active":"2023-10-09T06:00:14.294648+0000","last_peered":"2023-10-09T06:00:14.294648+0000","last_clean":"2023-10-09T06:00:14.294648+0000","last_became_active":"2023-10-09T05:59:31.411060+0000","last_became_peered":"2023-10-09T05:59:31.411060+0000","last_unstale":"2023-10-09T06:00:14.294648+0000","last_undegraded":"2023-10-09T06:00:14.294648+0000","last_fullsized":"2023-10-09T06:00:14.294648+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:35.484072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":15826,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":2117140504,"num_objects":32306,"num_object_clones":0,"num_object_copies":96918,"num_objects_missing_on_primary":12753,"num_objects_missing":12753,"num_objects_degraded":54748,"num_objects_misplaced":13924,"num_objects_unfound":0,"num_objects_dirty":32306,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136036,"num_write_kb":5386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43586,"num_bytes_recovered":2469462016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8699719680,"data_stored":8699707464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":153966,"ondisk_log_size":153966,"up":78,"acting":61,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":52,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13211,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":98,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846895,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1992480,"kb_used_data":1082648,"kb_used_omap":14,"kb_used_meta":909809,"kb_avail":91752672,"statfs":{"total":95995035648,"available":93954736128,"internally_reserved":0,"allocated":1108631552,"data_stored":1107597762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":931645274},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,2,5,2,2,6,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.875,"15min":4.1150000000000002},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.20699999999999999},"max":{"1min":5.0960000000000001,"5min":117.756,"15min":212.64400000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.1299999999999999,"5min":2.0510000000000002,"15min":4.1609999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.245},"max":{"1min":4.6769999999999996,"5min":117.73,"15min":212.655},"last":0.28799999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":4.2720000000000002,"5min":2.5430000000000001,"15min":2.1699999999999999},"min":{"1min":0.33800000000000002,"5min":0.26900000000000002,"15min":0.24199999999999999},"max":{"1min":55.212000000000003,"5min":78.013000000000005,"15min":114.422},"last":4.2519999999999998},{"interface":"front","average":{"1min":4.0229999999999997,"5min":2.456,"15min":2.2480000000000002},"min":{"1min":0.36299999999999999,"5min":0.26800000000000002,"15min":0.255},"max":{"1min":55.430999999999997,"5min":78.174999999999997,"15min":111.708},"last":0.29799999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.155,"15min":2.444},"min":{"1min":0.30599999999999999,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":8.0559999999999992,"5min":12.047000000000001,"15min":205.863},"last":4.2809999999999997},{"interface":"front","average":{"1min":1.171,"5min":0.86799999999999999,"15min":2.2890000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.28299999999999997},"max":{"1min":5.1909999999999998,"5min":5.5010000000000003,"15min":205.88399999999999},"last":0.38900000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":2.2210000000000001,"15min":3.0840000000000001},"min":{"1min":0.34100000000000003,"5min":0.315,"15min":0.28399999999999997},"max":{"1min":5.2469999999999999,"5min":115.69799999999999,"15min":115.69799999999999},"last":0.37},{"interface":"front","average":{"1min":1.292,"5min":2.327,"15min":3.3090000000000002},"min":{"1min":0.40200000000000002,"5min":0.29699999999999999,"15min":0.29299999999999998},"max":{"1min":4.1879999999999997,"5min":115.75700000000001,"15min":115.75700000000001},"last":0.33400000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":0.71699999999999997,"15min":1.958},"min":{"1min":0.29499999999999998,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":4.8390000000000004,"5min":5.6130000000000004,"15min":159.50299999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.2909999999999999,"5min":0.89000000000000001,"15min":2.0030000000000001},"min":{"1min":0.28499999999999998,"5min":0.28199999999999997,"15min":0.27800000000000002},"max":{"1min":4.0899999999999999,"5min":4.9800000000000004,"15min":159.483},"last":0.497}]},{"osd":5,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":1.9159999999999999,"15min":1.782},"min":{"1min":0.34399999999999997,"5min":0.29399999999999998,"15min":0.27800000000000002},"max":{"1min":8.968,"5min":94.093999999999994,"15min":94.093999999999994},"last":5.2069999999999999},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.8779999999999999,"15min":1.8560000000000001},"min":{"1min":0.40999999999999998,"5min":0.312,"15min":0.308},"max":{"1min":8.1010000000000009,"5min":95.579999999999998,"15min":95.579999999999998},"last":4.1909999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":3.2109999999999999,"15min":1.663},"min":{"1min":0.40200000000000002,"5min":0.32900000000000001,"15min":0.29299999999999998},"max":{"1min":5.7590000000000003,"5min":209.322,"15min":209.322},"last":4.0119999999999996},{"interface":"front","average":{"1min":2.5249999999999999,"5min":3.3420000000000001,"15min":1.7569999999999999},"min":{"1min":0.35599999999999998,"5min":0.30599999999999999,"15min":0.26700000000000002},"max":{"1min":8.1289999999999996,"5min":209.19800000000001,"15min":209.19800000000001},"last":4.2190000000000003}]}]},{"osd":6,"up_from":0,"seq":180388626927,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":93,"seq":399431958887,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2552624,"kb_used_data":1654632,"kb_used_omap":7,"kb_used_meta":897976,"kb_avail":91192528,"statfs":{"total":95995035648,"available":93381148672,"internally_reserved":0,"allocated":1694343168,"data_stored":1692531882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":919527950},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:18 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":0.71799999999999997,"15min":2.569},"min":{"1min":0.22700000000000001,"5min":0.214,"15min":0.17499999999999999},"max":{"1min":6.1920000000000002,"5min":6.96,"15min":131.91300000000001},"last":0.308},{"interface":"front","average":{"1min":1.171,"5min":0.63800000000000001,"15min":2.7679999999999998},"min":{"1min":0.29099999999999998,"5min":0.246,"15min":0.223},"max":{"1min":5.6959999999999997,"5min":5.6959999999999997,"15min":132.191},"last":0.376}]},{"osd":2,"last update":"Mon Oct 9 05:59:43 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":0.81599999999999995,"15min":2.0339999999999998},"min":{"1min":0.23300000000000001,"5min":0.221,"15min":0.19800000000000001},"max":{"1min":8.8100000000000005,"5min":10.782,"15min":132.35400000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.286,"5min":0.85099999999999998,"15min":1.8440000000000001},"min":{"1min":0.29899999999999999,"5min":0.23699999999999999,"15min":0.20699999999999999},"max":{"1min":7.3929999999999998,"5min":7.3929999999999998,"15min":131.70599999999999},"last":0.39600000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:00:18 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.0680000000000001,"15min":2.536},"min":{"1min":0.251,"5min":0.251,"15min":0.20200000000000001},"max":{"1min":3.3490000000000002,"5min":22.236000000000001,"15min":135.10900000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":1.228,"5min":1.131,"15min":2.4900000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.248},"max":{"1min":6.1580000000000004,"5min":22.913,"15min":131.886},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:00:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":0.80500000000000005,"15min":1.772},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.6150000000000002,"5min":5.6150000000000002,"15min":131.636},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.1699999999999999,"5min":0.91700000000000004,"15min":1.6499999999999999},"min":{"1min":0.38100000000000001,"5min":0.36299999999999999,"15min":0.32700000000000001},"max":{"1min":6.0250000000000004,"5min":9.1010000000000009,"15min":131.75999999999999},"last":0.59799999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:00:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":0.98399999999999999,"15min":2.355},"min":{"1min":0.34699999999999998,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":4.1310000000000002,"5min":8.7300000000000004,"15min":132.291},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.222,"5min":1.28,"15min":2.3780000000000001},"min":{"1min":0.36799999999999999,"5min":0.33700000000000002,"15min":0.32200000000000001},"max":{"1min":5.665,"5min":22.268999999999998,"15min":131.738},"last":0.55400000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:00:18 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":2.5760000000000001,"15min":2.4169999999999998},"min":{"1min":0.40899999999999997,"5min":0.34200000000000003,"15min":0.33300000000000002},"max":{"1min":12.513999999999999,"5min":158.50399999999999,"15min":158.50399999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.77,"5min":2.669,"15min":2.3460000000000001},"min":{"1min":0.38,"5min":0.377,"15min":0.37},"max":{"1min":8.6319999999999997,"5min":158.577,"15min":158.577},"last":0.51600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:00:18 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.825,"15min":2.0329999999999999},"min":{"1min":0.40200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":6.1059999999999999,"5min":89.787000000000006,"15min":131.821},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.6559999999999999,"5min":1.9450000000000001,"15min":2.1920000000000002},"min":{"1min":0.44700000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.089,"5min":89.869,"15min":131.852},"last":0.63300000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122413,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1973360,"kb_used_data":1205224,"kb_used_omap":8,"kb_used_meta":768119,"kb_avail":91771792,"statfs":{"total":95995035648,"available":93974315008,"internally_reserved":0,"allocated":1234149376,"data_stored":1232280490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":786554632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:00:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":1.716,"15min":1.6899999999999999},"min":{"1min":0.19400000000000001,"5min":0.191,"15min":0.17999999999999999},"max":{"1min":4.7290000000000001,"5min":104.26600000000001,"15min":104.26600000000001},"last":4.7290000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":1.7330000000000001,"15min":1.8680000000000001},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.19600000000000001},"max":{"1min":4.5359999999999996,"5min":104.22,"15min":104.22},"last":4.5359999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:59:50 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.65400000000000003,"15min":1.984},"min":{"1min":0.24099999999999999,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":6.2679999999999998,"5min":6.2679999999999998,"15min":70.188999999999993},"last":0.84299999999999997},{"interface":"front","average":{"1min":1.1120000000000001,"5min":0.71499999999999997,"15min":1.6950000000000001},"min":{"1min":0.26100000000000001,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":9.8559999999999999,"5min":9.8559999999999999,"15min":71.144000000000005},"last":4.6879999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:00:21 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.54800000000000004,"15min":1.8620000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.89100000000000001,"5min":5.7300000000000004,"15min":62.698999999999998},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.71299999999999997,"15min":2.2570000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":0.96799999999999997,"5min":6.3070000000000004,"15min":72.632999999999996},"last":0.82299999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:59:50 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.78100000000000003,"15min":1.6279999999999999},"min":{"1min":0.34200000000000003,"5min":0.29099999999999998,"15min":0.28399999999999997},"max":{"1min":2.0270000000000001,"5min":9.8949999999999996,"15min":149.63999999999999},"last":0.434},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.82299999999999995,"15min":1.8180000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.312},"max":{"1min":4.5279999999999996,"5min":9.8409999999999993,"15min":149.65899999999999},"last":0.33900000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:59:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":4.1580000000000004,"15min":4.0220000000000002},"min":{"1min":0.378,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.385,"5min":210.77799999999999,"15min":246.761},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.4790000000000001,"5min":4.4160000000000004,"15min":4.3529999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":12.404999999999999,"5min":209.91,"15min":247.48400000000001},"last":5.3109999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.85399999999999998,"15min":1.258},"min":{"1min":0.42399999999999999,"5min":0.375,"15min":0.318},"max":{"1min":8.2899999999999991,"5min":8.2899999999999991,"15min":19.579000000000001},"last":0.372},{"interface":"front","average":{"1min":1.9410000000000001,"5min":1.0449999999999999,"15min":1.4179999999999999},"min":{"1min":0.41399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":12.457000000000001,"5min":12.457000000000001,"15min":19.710999999999999},"last":0.85999999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:00:21 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":4.4939999999999998,"15min":2.4169999999999998},"min":{"1min":0.39700000000000002,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":3.2370000000000001,"5min":146.536,"15min":146.536},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.74199999999999999,"5min":4.4939999999999998,"15min":2.5459999999999998},"min":{"1min":0.46300000000000002,"5min":0.40100000000000002,"15min":0.36299999999999999},"max":{"1min":3.222,"5min":146.49700000000001,"15min":146.49700000000001},"last":0.75700000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795361,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1535172,"kb_used_data":655292,"kb_used_omap":11,"kb_used_meta":879860,"kb_avail":92209980,"statfs":{"total":95995035648,"available":94423019520,"internally_reserved":0,"allocated":671019008,"data_stored":668894046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":900977237},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.66500000000000004,"15min":2.8530000000000002},"min":{"1min":0.24299999999999999,"5min":0.20799999999999999,"15min":0.17999999999999999},"max":{"1min":1.595,"5min":9.7070000000000007,"15min":162.285},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.91900000000000004,"15min":3.133},"min":{"1min":0.23899999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.6230000000000002,"5min":17.109999999999999,"15min":162.643},"last":0.28799999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.393,"15min":2.4039999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.193},"max":{"1min":2.6659999999999999,"5min":75.260000000000005,"15min":125.77500000000001},"last":0.375},{"interface":"front","average":{"1min":0.72599999999999998,"5min":1.5269999999999999,"15min":2.4780000000000002},"min":{"1min":0.255,"5min":0.22900000000000001,"15min":0.20999999999999999},"max":{"1min":2.3220000000000001,"5min":75.114999999999995,"15min":125.857},"last":0.307}]},{"osd":3,"last update":"Mon Oct 9 05:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":1.1859999999999999,"15min":2.4140000000000001},"min":{"1min":0.25700000000000001,"5min":0.246,"15min":0.21099999999999999},"max":{"1min":2.351,"5min":15.193,"15min":159.35499999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.079,"15min":2.3340000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.217},"max":{"1min":9.7680000000000007,"5min":9.7680000000000007,"15min":159.196},"last":0.52300000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":0.84499999999999997,"15min":1.3300000000000001},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.28699999999999998},"max":{"1min":2.0649999999999999,"5min":9.1539999999999999,"15min":68.052000000000007},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.77900000000000003,"15min":1.3129999999999999},"min":{"1min":0.317,"5min":0.313,"15min":0.29199999999999998},"max":{"1min":5.4370000000000003,"5min":9.5800000000000001,"15min":68.727000000000004},"last":0.501}]},{"osd":5,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.0609999999999999,"15min":2.7589999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.29099999999999998},"max":{"1min":3.3050000000000002,"5min":18.350000000000001,"15min":207.80500000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.1479999999999999,"15min":2.823},"min":{"1min":0.371,"5min":0.371,"15min":0.30499999999999999},"max":{"1min":4.5780000000000003,"5min":9.1259999999999994,"15min":207.54900000000001},"last":0.437}]},{"osd":6,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0289999999999999,"15min":1.252},"min":{"1min":0.40500000000000003,"5min":0.31,"15min":0.31},"max":{"1min":9.0370000000000008,"5min":9.9320000000000004,"15min":25.818999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.008,"5min":0.97299999999999998,"15min":1.4430000000000001},"min":{"1min":0.44600000000000001,"5min":0.41399999999999998,"15min":0.34100000000000003},"max":{"1min":4.0220000000000002,"5min":8.7100000000000009,"15min":24.481000000000002},"last":1.853}]},{"osd":7,"last update":"Mon Oct 9 05:59:37 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":2.2229999999999999,"15min":2.3210000000000002},"min":{"1min":0.42999999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":9.6359999999999992,"5min":122.827,"15min":162.09200000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.296,"5min":2.4039999999999999,"15min":2.4940000000000002},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.373},"max":{"1min":5.6619999999999999,"5min":123.56399999999999,"15min":162.35400000000001},"last":0.61799999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631835,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2137756,"kb_used_data":1158036,"kb_used_omap":4,"kb_used_meta":979707,"kb_avail":91607396,"statfs":{"total":95995035648,"available":93805973504,"internally_reserved":0,"allocated":1185828864,"data_stored":1183969547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1003220574},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":2.6120000000000001,"15min":4.3170000000000002},"min":{"1min":0.22700000000000001,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":12.728,"5min":142.37700000000001,"15min":312.56599999999997},"last":1.629},{"interface":"front","average":{"1min":1.1439999999999999,"5min":2.6749999999999998,"15min":4.2789999999999999},"min":{"1min":0.23499999999999999,"5min":0.216,"15min":0.20499999999999999},"max":{"1min":7.6360000000000001,"5min":142.75800000000001,"15min":311.02300000000002},"last":1.6719999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":2.423,"15min":4.0129999999999999},"min":{"1min":0.28000000000000003,"5min":0.24199999999999999,"15min":0.214},"max":{"1min":7.0220000000000002,"5min":142.32599999999999,"15min":150.29599999999999},"last":1.3100000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":2.5249999999999999,"15min":4.2169999999999996},"min":{"1min":0.27500000000000002,"5min":0.23799999999999999,"15min":0.216},"max":{"1min":3.8860000000000001,"5min":139.33199999999999,"15min":151.25700000000001},"last":0.46400000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":3.8029999999999999,"5min":1.4079999999999999,"15min":2.359},"min":{"1min":0.29199999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":50.579000000000001,"5min":50.579000000000001,"15min":150.006},"last":0.29599999999999999},{"interface":"front","average":{"1min":3.266,"5min":1.3240000000000001,"15min":2.2749999999999999},"min":{"1min":0.29099999999999998,"5min":0.255,"15min":0.20499999999999999},"max":{"1min":50.628999999999998,"5min":50.628999999999998,"15min":150.65100000000001},"last":1.589}]},{"osd":4,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.78200000000000003,"15min":2.274},"min":{"1min":0.36199999999999999,"5min":0.35499999999999998,"15min":0.28999999999999998},"max":{"1min":3.278,"5min":4.7510000000000003,"15min":149.84700000000001},"last":1.548},{"interface":"front","average":{"1min":1.167,"5min":0.747,"15min":2.613},"min":{"1min":0.39100000000000001,"5min":0.32500000000000001,"15min":0.28999999999999998},"max":{"1min":7.1390000000000002,"5min":7.1390000000000002,"15min":149.74700000000001},"last":0.39600000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.0189999999999999,"15min":2.1989999999999998},"min":{"1min":0.36499999999999999,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":4.3540000000000001,"5min":13.380000000000001,"15min":149.43000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.8819999999999999,"5min":0.96299999999999997,"15min":2.0600000000000001},"min":{"1min":0.433,"5min":0.33400000000000002,"15min":0.29399999999999998},"max":{"1min":12.778,"5min":12.778,"15min":149.69300000000001},"last":0.33400000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":0.83599999999999997,"15min":1.8480000000000001},"min":{"1min":0.53700000000000003,"5min":0.38200000000000001,"15min":0.34499999999999997},"max":{"1min":7.1719999999999997,"5min":7.1719999999999997,"15min":149.72399999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":0.89800000000000002,"15min":2.0049999999999999},"min":{"1min":0.45600000000000002,"5min":0.39400000000000002,"15min":0.33200000000000002},"max":{"1min":3.3279999999999998,"5min":5.7480000000000002,"15min":149.589},"last":0.51300000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:59:44 2023","interfaces":[{"interface":"back","average":{"1min":2.379,"5min":1.1519999999999999,"15min":1.9159999999999999},"min":{"1min":0.434,"5min":0.38300000000000001,"15min":0.35699999999999998},"max":{"1min":11.948,"5min":11.948,"15min":149.77199999999999},"last":1.647},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.1479999999999999,"15min":1.9059999999999999},"min":{"1min":0.41799999999999998,"5min":0.40000000000000002,"15min":0.374},"max":{"1min":11.722,"5min":11.722,"15min":149.80799999999999},"last":1.621}]}]},{"osd":4,"up_from":30,"seq":128849019383,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1470172,"kb_used_data":572820,"kb_used_omap":0,"kb_used_meta":897344,"kb_avail":92274980,"statfs":{"total":95995035648,"available":94489579520,"internally_reserved":0,"allocated":586567680,"data_stored":585537396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918880256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.76900000000000002,"15min":3.0800000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.4399999999999999,"5min":7.8609999999999998,"15min":132.35599999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":1.131,"5min":1.081,"15min":3.4620000000000002},"min":{"1min":0.29099999999999998,"5min":0.24299999999999999,"15min":0.23699999999999999},"max":{"1min":4.6040000000000001,"5min":12.462,"15min":132.501},"last":1.111}]},{"osd":1,"last update":"Mon Oct 9 05:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":2.7429999999999999,"15min":2.569},"min":{"1min":0.30599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.0419999999999998,"5min":96.033000000000001,"15min":96.033000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.3720000000000001,"5min":3.0030000000000001,"15min":2.5510000000000002},"min":{"1min":0.41399999999999998,"5min":0.255,"15min":0.255},"max":{"1min":4.5629999999999997,"5min":96.022000000000006,"15min":96.022000000000006},"last":1.3859999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:59:46 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.72599999999999998,"15min":1.611},"min":{"1min":0.29599999999999999,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":2.0670000000000002,"5min":5.1639999999999997,"15min":51.768000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.482,"5min":1.8440000000000001,"15min":1.859},"min":{"1min":0.32100000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.9269999999999996,"5min":84.123999999999995,"15min":84.123999999999995},"last":0.70399999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:00:19 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":1.0329999999999999,"15min":2.875},"min":{"1min":0.378,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":4.6589999999999998,"5min":11.055999999999999,"15min":123.59},"last":1.1719999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.93100000000000005,"15min":2.7549999999999999},"min":{"1min":0.36099999999999999,"5min":0.32900000000000001,"15min":0.30499999999999999},"max":{"1min":2.528,"5min":7.9379999999999997,"15min":123.566},"last":0.76600000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:00:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":1.224,"15min":1.04},"min":{"1min":0.35799999999999998,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":5.3040000000000003,"5min":15.597,"15min":15.597},"last":3.6930000000000001},{"interface":"front","average":{"1min":1.113,"5min":1.1830000000000001,"15min":0.98899999999999999},"min":{"1min":0.44400000000000001,"5min":0.34499999999999997,"15min":0.29799999999999999},"max":{"1min":5.3650000000000002,"5min":17.882000000000001,"15min":17.882000000000001},"last":3.516}]},{"osd":6,"last update":"Mon Oct 9 05:59:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":1.4930000000000001,"15min":1.625},"min":{"1min":0.40899999999999997,"5min":0.314,"15min":0.27600000000000002},"max":{"1min":6.0549999999999997,"5min":38.676000000000002,"15min":181.411},"last":0.79000000000000004},{"interface":"front","average":{"1min":1.772,"5min":1.504,"15min":1.722},"min":{"1min":0.40899999999999997,"5min":0.32200000000000001,"15min":0.29799999999999999},"max":{"1min":6.3440000000000003,"5min":38.438000000000002,"15min":180.63999999999999},"last":1.145}]},{"osd":7,"last update":"Mon Oct 9 05:59:36 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":1.2989999999999999,"15min":1.4419999999999999},"min":{"1min":0.35199999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.0810000000000004,"5min":17.832000000000001,"15min":96.131},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.25,"5min":1.0569999999999999,"15min":1.4039999999999999},"min":{"1min":0.35699999999999998,"5min":0.32900000000000001,"15min":0.27200000000000002},"max":{"1min":4.9950000000000001,"5min":17.742999999999999,"15min":96.465000000000003},"last":0.61199999999999999}]}]},{"osd":5,"up_from":0,"seq":154618823155,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1226113024,"data_stored":1226113024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1688408064,"data_stored":1688403992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":657653760,"data_stored":657653760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1175781376,"data_stored":1175781376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":580521984,"data_stored":580517912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1762856960,"data_stored":1762852888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":509804544,"data_stored":509804544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1098579968,"data_stored":1098579968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:00:26.091 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 61 2023-10-09T06:00:26.112 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:26.181 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:26.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:26.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:26.289 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:26.336 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:26.396 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:26.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:26.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:26.471 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:26.501 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:26.512 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:26 smithi191 ceph-mon[161669]: from='client.36885 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:00:26.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:26 smithi191 ceph-mon[161669]: pgmap v1440: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 54811/97002 objects degraded (56.505%); 13934/97002 objects misplaced (14.365%); 2.5 MiB/s, 49 objects/s recovering 2023-10-09T06:00:26.558 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:26 smithi008 ceph-mon[184418]: from='client.36885 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:00:26.559 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:26 smithi008 ceph-mon[184418]: pgmap v1440: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 54811/97002 objects degraded (56.505%); 13934/97002 objects misplaced (14.365%); 2.5 MiB/s, 49 objects/s recovering 2023-10-09T06:00:26.559 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:26 smithi008 ceph-mon[191227]: from='client.36885 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:00:26.559 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:26 smithi008 ceph-mon[191227]: pgmap v1440: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 54811/97002 objects degraded (56.505%); 13934/97002 objects misplaced (14.365%); 2.5 MiB/s, 49 objects/s recovering 2023-10-09T06:00:26.566 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:26.617 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:26.630 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:26.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:26.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:26.736 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:26.763 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:26.864 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:26.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:26.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:27.011 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:27.128 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:27.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:27.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:27.234 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:27.269 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:27.361 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:00:27.389 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/560195543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:27.389 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/560195543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:27.539 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:27.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/560195543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:27.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:27.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:27.646 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:27.685 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:28.180 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:00:28.180 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:00:28.199 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:28.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:28.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:28.306 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:28.335 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:28.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[184418]: pgmap v1441: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 163 op/s; 54505/94002 objects degraded (57.983%); 13467/94002 objects misplaced (14.326%); 4.1 MiB/s, 77 objects/s recovering 2023-10-09T06:00:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[184418]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[184418]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[184418]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[184418]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/560195543' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:00:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[184418]: osdmap e482: 8 total, 8 up, 6 in 2023-10-09T06:00:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[184418]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 54505/94002 objects degraded (57.983%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[191227]: pgmap v1441: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 163 op/s; 54505/94002 objects degraded (57.983%); 13467/94002 objects misplaced (14.326%); 4.1 MiB/s, 77 objects/s recovering 2023-10-09T06:00:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[191227]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[191227]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[191227]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[191227]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/560195543' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:00:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[191227]: osdmap e482: 8 total, 8 up, 6 in 2023-10-09T06:00:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[191227]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 54505/94002 objects degraded (57.983%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:28 smithi191 ceph-mon[161669]: pgmap v1441: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 163 op/s; 54505/94002 objects degraded (57.983%); 13467/94002 objects misplaced (14.326%); 4.1 MiB/s, 77 objects/s recovering 2023-10-09T06:00:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:28 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:28 smithi191 ceph-mon[161669]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:28 smithi191 ceph-mon[161669]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:28 smithi191 ceph-mon[161669]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:28 smithi191 ceph-mon[161669]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/560195543' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:00:28.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:28 smithi191 ceph-mon[161669]: osdmap e482: 8 total, 8 up, 6 in 2023-10-09T06:00:28.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:28 smithi191 ceph-mon[161669]: failed to encode map e482 with expected crc 2023-10-09T06:00:28.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 54505/94002 objects degraded (57.983%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:29.064 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:00:29.238 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:29.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:29.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:29.348 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:29.385 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:29.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:29.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:29.496 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:29.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:29.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:29.601 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:29.631 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:29.671 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:29.694 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:29.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:29.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:29.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:29.801 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:29.841 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:29.875 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:00:29.894 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:00:29.916 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:00:29.918 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:29.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:29.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:29.938 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:00:29.958 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:00:29.976 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:00:29.995 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:00:30.014 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:00:30.015 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.016 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.016 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.016 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.016 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.016 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.016 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.016 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.026 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:30.056 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:30.211 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:30.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:30.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:30.264 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T06:00:30.317 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:30.347 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:30.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:30 smithi191 ceph-mon[161669]: pgmap v1443: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 196 op/s; 54453/93354 objects degraded (58.330%); 13392/93354 objects misplaced (14.345%); 4.7 MiB/s, 87 objects/s recovering 2023-10-09T06:00:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/522001862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:30.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2267395043' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "61"}]: dispatch 2023-10-09T06:00:30.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:30 smithi008 ceph-mon[184418]: pgmap v1443: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 196 op/s; 54453/93354 objects degraded (58.330%); 13392/93354 objects misplaced (14.345%); 4.7 MiB/s, 87 objects/s recovering 2023-10-09T06:00:30.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/522001862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:30.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2267395043' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "61"}]: dispatch 2023-10-09T06:00:30.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:00:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:00:30] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.33.4" 2023-10-09T06:00:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:30 smithi008 ceph-mon[191227]: pgmap v1443: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 196 op/s; 54453/93354 objects degraded (58.330%); 13392/93354 objects misplaced (14.345%); 4.7 MiB/s, 87 objects/s recovering 2023-10-09T06:00:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/522001862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2267395043' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "61"}]: dispatch 2023-10-09T06:00:30.857 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:00:30.923 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:30.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:30.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:31.027 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:31.057 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:31.172 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:31.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:31.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:31.277 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:31.293 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:00:31.293 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:00:31.293 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T06:00:31.293 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T06:00:31.293 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:00:31.338 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:31.679 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T06:00:31.766 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:31.770 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:31.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:31.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:31.883 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:31.917 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:32.021 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:32.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:32.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:32.126 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:32.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:00:32.155 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:32.459 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:32.460 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:32 smithi008 ceph-mon[184418]: pgmap v1444: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 190 op/s; 54135/91557 objects degraded (59.127%); 13242/91557 objects misplaced (14.463%); 5.2 MiB/s, 86 objects/s recovering 2023-10-09T06:00:32.461 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:32 smithi008 ceph-mon[191227]: pgmap v1444: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 190 op/s; 54135/91557 objects degraded (59.127%); 13242/91557 objects misplaced (14.463%); 5.2 MiB/s, 86 objects/s recovering 2023-10-09T06:00:32.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:32.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:32.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:32 smithi191 ceph-mon[161669]: pgmap v1444: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 190 op/s; 54135/91557 objects degraded (59.127%); 13242/91557 objects misplaced (14.463%); 5.2 MiB/s, 86 objects/s recovering 2023-10-09T06:00:32.562 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:32.590 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:32.655 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:32.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:32.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:32.698 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:32.762 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:32.804 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:32.912 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:32.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:32.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:32.990 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:00:33.015 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:33.062 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:33.149 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:33.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:33.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:33.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:33.289 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:33.399 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:33.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:33.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:33.503 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:33.538 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:33.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 54135/91557 objects degraded (59.127%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:33.602 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:33.655 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:33.658 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 54135/91557 objects degraded (59.127%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:33.658 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 54135/91557 objects degraded (59.127%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:33.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:33.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:33.760 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:33.803 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:34.072 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:34.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:34.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:34.180 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:34.211 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:34.368 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:34 smithi008 ceph-mon[191227]: pgmap v1445: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 189 op/s; 54074/91302 objects degraded (59.225%); 12813/91302 objects misplaced (14.034%); 5.4 MiB/s, 90 objects/s recovering 2023-10-09T06:00:34.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2033441994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:34.369 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:34 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:34.376 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:00:34.456 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:34.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:34.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:34.560 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:00:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T06:00:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T06:00:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T06:00:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117314 9.19 9:9d16215d:::benchmark_data_smithi008_502820_object33141:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:00:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.240001+0000", 2023-10-09T06:00:34.560 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 157.314932419, 2023-10-09T06:00:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.143222604999998, 2023-10-09T06:00:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117314 2023-10-09T06:00:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.561 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240001+0000", 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240001+0000", 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240000+0000", 2023-10-09T06:00:34.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.96e-06 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240003+0000", 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0399999999999998e-06 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240012+0000", 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6589999999999998e-06 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240017+0000", 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000471003 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240488+0000", 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0127e-05 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T06:00:34.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240498+0000", 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001336481 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.241834+0000", 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6310000000000003e-06 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.241842+0000", 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4456201419999992 2023-10-09T06:00:34.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.687462+0000", 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073848840000000004 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.694847+0000", 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99199552499999999 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686843+0000", 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3572e-05 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686856+0000", 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.69965971100000002 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386516+0000", 2023-10-09T06:00:34.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2209999999999997e-06 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386524+0000", 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3956454790000006 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.782170+0000", 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003363689 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.785533+0000", 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0053417940000000004 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.790875+0000", 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.040818535000000003 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.831694+0000", 2023-10-09T06:00:34.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5428561090000001 2023-10-09T06:00:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.374550+0000", 2023-10-09T06:00:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00093305999999999999 2023-10-09T06:00:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.375483+0000", 2023-10-09T06:00:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001490901 2023-10-09T06:00:34.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.376974+0000", 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006249714 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.383223+0000", 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.143222604999998 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117319 9.19 9:991b5772:::benchmark_data_smithi008_502820_object33146:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 157.31491170499999, 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.166676882000001, 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117319 2023-10-09T06:00:34.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6680000000000001e-06 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240024+0000", 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2449999999999998e-06 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240029+0000", 2023-10-09T06:00:34.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4620000000000001e-06 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240032+0000", 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021777950000000002 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242210+0000", 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3260000000000002e-06 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242219+0000", 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4529842770000005 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.695204+0000", 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099633949999999999 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:34.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.705167+0000", 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98175812600000001 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686925+0000", 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0090000000000001e-06 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686932+0000", 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.69967429400000003 2023-10-09T06:00:34.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386607+0000", 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6420000000000001e-06 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386613+0000", 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4280742919999998 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.814688+0000", 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046527440000000003 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.819340+0000", 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068654730000000004 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.826206+0000", 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.025179699999999999 2023-10-09T06:00:34.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.851385+0000", 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5449409780000001 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.396326+0000", 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001055615 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.397382+0000", 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017229529999999999 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.399105+0000", 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075933620000000002 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.406698+0000", 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.166676882000001 2023-10-09T06:00:34.581 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117320 9.19 9:9ca20909:::benchmark_data_smithi008_502820_object33147:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 157.31489710299999, 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.155182235000002, 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.582 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117320 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.638e-06 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240038+0000", 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0300000000000001e-06 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240043+0000", 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8990000000000003e-06 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240047+0000", 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021981399999999999 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242245+0000", 2023-10-09T06:00:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2699999999999999e-06 2023-10-09T06:00:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242253+0000", 2023-10-09T06:00:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4632743759999993 2023-10-09T06:00:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.705527+0000", 2023-10-09T06:00:34.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075602960000000002 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.713087+0000", 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97380087699999995 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686888+0000", 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8900000000000001e-06 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686895+0000", 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.69966168100000004 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386557+0000", 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3550000000000005e-06 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386565+0000", 2023-10-09T06:00:34.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4098301660000008 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.796395+0000", 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048933850000000001 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.801289+0000", 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071789369999999998 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.808468+0000", 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.042643954999999997 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.851112+0000", 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.532899402 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.384011+0000", 2023-10-09T06:00:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010206099999999999 2023-10-09T06:00:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.385032+0000", 2023-10-09T06:00:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017808520000000001 2023-10-09T06:00:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.386812+0000", 2023-10-09T06:00:34.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0084058440000000009 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.395218+0000", 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.155182235000002 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117329 9.19 9:98a1b605:::benchmark_data_smithi008_502820_object33156:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-09T06:00:34.592 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:58.537538+0000", 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 156.01739476, 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 22.047981301, 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117329 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537538+0000", 2023-10-09T06:00:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537538+0000", 2023-10-09T06:00:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537537+0000", 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.357e-06 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537544+0000", 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5744e-05 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537560+0000", 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1219999999999995e-06 2023-10-09T06:00:34.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537568+0000", 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1875399160000004 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725108+0000", 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7539999999999995e-06 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725117+0000", 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66151549200000004 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386632+0000", 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.083e-06 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386638+0000", 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4453952789999995 2023-10-09T06:00:34.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.832033+0000", 2023-10-09T06:00:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0096413320000000007 2023-10-09T06:00:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.841675+0000", 2023-10-09T06:00:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5839331400000001 2023-10-09T06:00:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.425608+0000", 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020811639999999999 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.427689+0000", 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052414740000000003 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:00:34.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.432931+0000", 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.046699651000000002 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.479630+0000", 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.094573824000000001 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.574204+0000", 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001875704 2023-10-09T06:00:34.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.576080+0000", 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018718509999999999 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.577952+0000", 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.007568165 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.585520+0000", 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 22.047981301 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117332 9.19 9:9d36e61d:::benchmark_data_smithi008_502820_object33159:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:58.661367+0000", 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 155.893566413, 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 21.935008709000002, 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.602 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117332 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661367+0000", 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.603 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661367+0000", 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661365+0000", 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8889999999999997e-06 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661375+0000", 2023-10-09T06:00:34.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2055999999999999e-05 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661387+0000", 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9869999999999996e-06 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661393+0000", 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0637442499999992 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725137+0000", 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0648e-05 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725148+0000", 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66150936400000004 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386657+0000", 2023-10-09T06:00:34.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.976e-06 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386663+0000", 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4554108919999997 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.842074+0000", 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087053159999999994 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.850779+0000", 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6107857050000001 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.461565+0000", 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023301989999999998 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.463895+0000", 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052078059999999997 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.469103+0000", 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010784145 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.479887+0000", 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.106784905 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.586672+0000", 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011016520000000001 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.587774+0000", 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001720217 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.589494+0000", 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068816320000000004 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.596375+0000", 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 21.935008709000002 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.611 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117337 9.1 9:857ac274:::benchmark_data_smithi008_502820_object33164:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:02.529277+0000", 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 152.02565650099999, 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 18.012440933000001, 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117337 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529277+0000", 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529277+0000", 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529275+0000", 2023-10-09T06:00:34.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6810000000000001e-06 2023-10-09T06:00:34.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529282+0000", 2023-10-09T06:00:34.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5460000000000001e-05 2023-10-09T06:00:34.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529297+0000", 2023-10-09T06:00:34.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0799999999999996e-06 2023-10-09T06:00:34.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529302+0000", 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1958720979999997 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725174+0000", 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9480000000000003e-06 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725184+0000", 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66691750900000002 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392102+0000", 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8009999999999998e-06 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392112+0000", 2023-10-09T06:00:34.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.865323069 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.257435+0000", 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015459999999999999 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.257589+0000", 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028764399999999999 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.257877+0000", 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035365399999999999 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.261413+0000", 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27957103 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.540984+0000", 2023-10-09T06:00:34.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1287999999999998e-05 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541066+0000", 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0819999999999998e-05 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541117+0000", 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060101399999999998 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541718+0000", 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 18.012440933000001 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117340 9.9 9:91f28550:::benchmark_data_smithi008_502820_object33167:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:02.596319+0000", 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 151.95861390100001, 2023-10-09T06:00:34.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 17.955864099999999, 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117340 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596319+0000", 2023-10-09T06:00:34.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596319+0000", 2023-10-09T06:00:34.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:34.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596318+0000", 2023-10-09T06:00:34.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0000000000000002e-06 2023-10-09T06:00:34.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596324+0000", 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3848e-05 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596338+0000", 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3350000000000003e-06 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596344+0000", 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1288643570000003 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725209+0000", 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0413e-05 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725219+0000", 2023-10-09T06:00:34.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65901121200000001 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384230+0000", 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3641e-05 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384244+0000", 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4740669539999995 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.858311+0000", 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086265019999999994 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.866937+0000", 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4266018129999996 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.293539+0000", 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002220236 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.295759+0000", 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050211379999999996 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.300781+0000", 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033493818000000002 2023-10-09T06:00:34.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.334274+0000", 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20914828399999999 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.543423+0000", 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094928299999999998 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.544372+0000", 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013903489999999999 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.545762+0000", 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0064211219999999996 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.552183+0000", 2023-10-09T06:00:34.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 17.955864099999999 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117343 9.9 9:978d0f59:::benchmark_data_smithi008_502820_object33170:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.479396+0000", 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 149.07553712800001, 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.081940300999999, 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.630 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117343 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479396+0000", 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479396+0000", 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999967 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479393+0000", 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1915e-05 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479405+0000", 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4262e-05 2023-10-09T06:00:34.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479419+0000", 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3119999999999994e-06 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479427+0000", 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2458186229999999 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725246+0000", 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0760000000000003e-06 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725254+0000", 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65942001299999997 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384674+0000", 2023-10-09T06:00:34.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1280999999999999e-05 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384685+0000", 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4825987420000004 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.867284+0000", 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0098823669999999995 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.877166+0000", 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4447639959999998 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.321930+0000", 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020355619999999999 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.636 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.323966+0000", 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059662459999999997 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.329932+0000", 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022155676999999999 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.352088+0000", 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.201059509 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.553147+0000", 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00084139399999999997 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.553989+0000", 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001347577 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.555336+0000", 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060000390000000004 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.561336+0000", 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.081940300999999 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.639 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117347 9.1 9:83732981:::benchmark_data_smithi008_502820_object33174:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.499313+0000", 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 149.05562059600001, 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.043182195, 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117347 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499313+0000", 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499313+0000", 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499310+0000", 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6179999999999995e-06 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499319+0000", 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5282e-05 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499334+0000", 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1779999999999997e-06 2023-10-09T06:00:34.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499341+0000", 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.225918622 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725260+0000", 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4930000000000001e-06 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.643 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725268+0000", 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66689868200000002 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392166+0000", 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6649999999999992e-06 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392175+0000", 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.866980590000001 2023-10-09T06:00:34.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.259156+0000", 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000185317 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.259341+0000", 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00037832499999999999 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.259719+0000", 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035454420000000002 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.263265+0000", 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.277776933 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541042+0000", 2023-10-09T06:00:34.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00079637999999999996 2023-10-09T06:00:34.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541838+0000", 2023-10-09T06:00:34.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5203999999999994e-05 2023-10-09T06:00:34.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541903+0000", 2023-10-09T06:00:34.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000591623 2023-10-09T06:00:34.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.542495+0000", 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.043182195 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117351 9.11 9:8879fcf3:::benchmark_data_smithi008_502820_object33178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.521160+0000", 2023-10-09T06:00:34.648 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 149.03377322899999, 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.849076910000001, 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117351 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521160+0000", 2023-10-09T06:00:34.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521160+0000", 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521158+0000", 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.622e-06 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521166+0000", 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2873e-05 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521179+0000", 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7890000000000006e-06 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521189+0000", 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2040851159999999 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725274+0000", 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8170000000000001e-06 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725281+0000", 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66564084199999995 2023-10-09T06:00:34.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.390922+0000", 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0733999999999999e-05 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.390932+0000", 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5330637649999996 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.654 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.923996+0000", 2023-10-09T06:00:34.654 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013707300000000001 2023-10-09T06:00:34.654 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.654 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.654 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.924133+0000", 2023-10-09T06:00:34.654 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00038352900000000002 2023-10-09T06:00:34.654 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.654 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4,6", 2023-10-09T06:00:34.654 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.924517+0000", 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.87137725899999996 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.795894+0000", 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5739970530000003 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.369891+0000", 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6699000000000002e-05 2023-10-09T06:00:34.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.369928+0000", 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010414800000000001 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.370032+0000", 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5777000000000004e-05 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.370098+0000", 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013928500000000001 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.370237+0000", 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.849076910000001 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.657 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117352 9.9 9:93c88060:::benchmark_data_smithi008_502820_object33179:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.574636+0000", 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 148.98029714800001, 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.996202152, 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117352 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.658 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574636+0000", 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574636+0000", 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574634+0000", 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2150000000000001e-06 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574643+0000", 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6385e-05 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574659+0000", 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6510000000000001e-06 2023-10-09T06:00:34.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574667+0000", 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.150622016 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725289+0000", 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1060000000000002e-06 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725296+0000", 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65942802199999995 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384724+0000", 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1769999999999998e-06 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384732+0000", 2023-10-09T06:00:34.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4928461019999997 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.877578+0000", 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011095785 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.888674+0000", 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4637061429999996 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.352380+0000", 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020417460000000001 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.354422+0000", 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0058679479999999996 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.360290+0000", 2023-10-09T06:00:34.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.036684833 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.396974+0000", 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16525932500000001 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.562234+0000", 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00104692 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.665 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.563281+0000", 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014355430000000001 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.564716+0000", 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0061219309999999997 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.570838+0000", 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.996202152 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117356 9.19 9:9e99d7bd:::benchmark_data_smithi008_502820_object33183:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.607515+0000", 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 148.947418511, 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.999065100999999, 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.667 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117356 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607515+0000", 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607515+0000", 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607513+0000", 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5640000000000002e-06 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607520+0000", 2023-10-09T06:00:34.669 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1073000000000001e-05 2023-10-09T06:00:34.670 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.670 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.670 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.670 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607531+0000", 2023-10-09T06:00:34.670 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9660000000000001e-06 2023-10-09T06:00:34.670 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.670 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.670 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.670 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607537+0000", 2023-10-09T06:00:34.670 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1177633890000001 2023-10-09T06:00:34.670 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725300+0000", 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7010000000000001e-06 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725306+0000", 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66137529500000003 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.671 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386681+0000", 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0240000000000003e-06 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386687+0000", 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4650221170000002 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.851709+0000", 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099491639999999999 2023-10-09T06:00:34.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.861658+0000", 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6314088509999998 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.493067+0000", 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002267879 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.495335+0000", 2023-10-09T06:00:34.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0054393109999999996 2023-10-09T06:00:34.674 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.674 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.674 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:00:34.674 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.500774+0000", 2023-10-09T06:00:34.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024364858999999999 2023-10-09T06:00:34.674 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.674 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.674 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.674 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.525139+0000", 2023-10-09T06:00:34.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.072382734000000004 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.597522+0000", 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001197832 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.598720+0000", 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001605965 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.600326+0000", 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0062539149999999996 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.606580+0000", 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.999065100999999 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.676 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135245 9.d 9:b964d666:::benchmark_data_smithi008_502820_object51072:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.596664+0000", 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 30.958269287, 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789031075, 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135245 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.677 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596664+0000", 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596664+0000", 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596663+0000", 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.011e-06 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596668+0000", 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0095000000000001e-05 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596678+0000", 2023-10-09T06:00:34.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5299999999999998e-06 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596682+0000", 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.063468346999999994 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.660151+0000", 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033404179999999999 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.663491+0000", 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017397383999999998 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.680889+0000", 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004312265 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.685201+0000", 2023-10-09T06:00:34.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.61805938000000005 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.303260+0000", 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035192460000000002 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.306780+0000", 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0085941420000000008 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.315374+0000", 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.057904045000000001 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373278+0000", 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0834e-05 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373289+0000", 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91257296300000001 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285861+0000", 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8009999999999998e-06 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285868+0000", 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.830804881000001 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.116673+0000", 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050648639999999997 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.121738+0000", 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.26043395800000002 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.685 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.382172+0000", 2023-10-09T06:00:34.686 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035229559999999998 2023-10-09T06:00:34.686 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.686 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.686 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.686 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.385695+0000", 2023-10-09T06:00:34.686 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789031075 2023-10-09T06:00:34.686 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.686 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.686 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.686 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.686 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135269 9.d 9:b1952556:::benchmark_data_smithi008_502820_object51096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.688239+0000", 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 30.866694381999999, 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.735737110000001, 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135269 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.687 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688239+0000", 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688239+0000", 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688238+0000", 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5710000000000003e-06 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688244+0000", 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8319999999999995e-06 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688253+0000", 2023-10-09T06:00:34.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7010000000000001e-06 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688259+0000", 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024856844999999999 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.713116+0000", 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041163090000000003 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.690 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.717232+0000", 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.032543962000000003 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749776+0000", 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042646100000000003 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:34.691 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.754041+0000", 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.57617059599999998 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.330211+0000", 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003060962 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.333272+0000", 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059605719999999999 2023-10-09T06:00:34.692 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.339233+0000", 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034085450000000003 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373318+0000", 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4910000000000002e-06 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.693 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373325+0000", 2023-10-09T06:00:34.694 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91256626500000004 2023-10-09T06:00:34.694 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.694 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.694 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285891+0000", 2023-10-09T06:00:34.694 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2830000000000004e-06 2023-10-09T06:00:34.694 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.694 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.694 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285898+0000", 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.836523452 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.122422+0000", 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0067783660000000001 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.129200+0000", 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29146132499999999 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.420661+0000", 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033146180000000001 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.423976+0000", 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.735737110000001 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.696 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135271 9.d 9:ba573113:::benchmark_data_smithi008_502820_object51098:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.694068+0000", 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 30.860865435000001, 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.803375333, 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135271 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.697 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694068+0000", 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694068+0000", 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694065+0000", 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2420000000000006e-06 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694075+0000", 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5207e-05 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694090+0000", 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.721e-06 2023-10-09T06:00:34.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694099+0000", 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.060414994999999999 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.754514+0000", 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047620040000000002 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.759276+0000", 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01924886 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.778524+0000", 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045597650000000003 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.783084+0000", 2023-10-09T06:00:34.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.59039565999999999 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373480+0000", 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1539999999999999e-06 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373486+0000", 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91251702300000004 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286003+0000", 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.336e-06 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286009+0000", 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.906405661999999 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.703 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.192415+0000", 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00484973 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.197265+0000", 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0089266180000000007 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.206191+0000", 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.065920287999999994 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.705 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.705 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.272112+0000", 2023-10-09T06:00:34.705 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20899689099999999 2023-10-09T06:00:34.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.705 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.705 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.481109+0000", 2023-10-09T06:00:34.705 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001501549 2023-10-09T06:00:34.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.705 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.482610+0000", 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015254660000000001 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.484136+0000", 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026156030000000002 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.486751+0000", 2023-10-09T06:00:34.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010691813999999999 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.497443+0000", 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.803375333 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135277 9.d 9:bcf8be17:::benchmark_data_smithi008_502820_object51104:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:00:34.707 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.729022+0000", 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 30.825910713999999, 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.750575383999999, 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135277 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.708 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729022+0000", 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729022+0000", 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729020+0000", 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0536e-05 2023-10-09T06:00:34.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.710 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.710 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.710 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729031+0000", 2023-10-09T06:00:34.710 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2653999999999999e-05 2023-10-09T06:00:34.710 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.710 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.710 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.710 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729044+0000", 2023-10-09T06:00:34.710 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6559999999999992e-06 2023-10-09T06:00:34.710 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.710 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729051+0000", 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.031414678000000001 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.760466+0000", 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032615449999999998 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:00:34.711 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.763727+0000", 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.036828547000000003 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.800556+0000", 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032710500000000002 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.803827+0000", 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.56961666799999999 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373444+0000", 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1970000000000001e-06 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373450+0000", 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91252582299999996 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285976+0000", 2023-10-09T06:00:34.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5559999999999998e-06 2023-10-09T06:00:34.714 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.714 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.714 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.714 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285981+0000", 2023-10-09T06:00:34.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.882853229 2023-10-09T06:00:34.714 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.714 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.714 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.714 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.168835+0000", 2023-10-09T06:00:34.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048070370000000001 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.173642+0000", 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086679089999999997 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.182309+0000", 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.051578453000000003 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.715 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.233888+0000", 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22473416600000001 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.458622+0000", 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001297997 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.459920+0000", 2023-10-09T06:00:34.716 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014969269999999999 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.461417+0000", 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025559990000000002 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.463973+0000", 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015624812 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.479598+0000", 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.750575383999999 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135285 9.d 9:b8ae6d39:::benchmark_data_smithi008_502820_object51112:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 30.805094829000002, 2023-10-09T06:00:34.718 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789996110000001, 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135285 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:00:34.719 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.281e-06 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.720 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749841+0000", 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0190000000000001e-06 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749848+0000", 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0799999999999999e-06 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749852+0000", 2023-10-09T06:00:34.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033638386999999999 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.783491+0000", 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031708800000000001 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.786661+0000", 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034315245000000001 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.820977+0000", 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032606850000000001 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.824237+0000", 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.54932086199999997 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373558+0000", 2023-10-09T06:00:34.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1770000000000003e-06 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373564+0000", 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91249455899999998 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286059+0000", 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7699999999999998e-06 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286065+0000", 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.948433641999999 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.234498+0000", 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042428049999999997 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.238741+0000", 2023-10-09T06:00:34.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073975059999999999 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.246139+0000", 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.045528540999999999 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.291667+0000", 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22418721699999999 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.515854+0000", 2023-10-09T06:00:34.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001393544 2023-10-09T06:00:34.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.517248+0000", 2023-10-09T06:00:34.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001617402 2023-10-09T06:00:34.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.518865+0000", 2023-10-09T06:00:34.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0061180770000000004 2023-10-09T06:00:34.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.524983+0000", 2023-10-09T06:00:34.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014850993999999999 2023-10-09T06:00:34.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.539834+0000", 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789996110000001 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135292 9.d 9:b1d0c439:::benchmark_data_smithi008_502820_object51119:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.788529+0000", 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 30.766404059999999, 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.832817245999999, 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.729 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135292 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788529+0000", 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788529+0000", 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788528+0000", 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8979999999999998e-06 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788532+0000", 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3579999999999999e-06 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788539+0000", 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.58e-06 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788542+0000", 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.037230805999999998 2023-10-09T06:00:34.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.825773+0000", 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045452899999999996 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.830318+0000", 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.044094162999999999 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.874413+0000", 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033226699999999998 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.877735+0000", 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.496008956 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373744+0000", 2023-10-09T06:00:34.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2219999999999999e-06 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373750+0000", 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91544140500000004 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289192+0000", 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5060000000000001e-06 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289198+0000", 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.022931246000001 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.312130+0000", 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041899379999999998 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.316319+0000", 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0083299189999999999 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.324649+0000", 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0088983910000000003 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.333548+0000", 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27093540199999999 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.604483+0000", 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001811764 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.606295+0000", 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002186805 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.738 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.608482+0000", 2023-10-09T06:00:34.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024063980000000001 2023-10-09T06:00:34.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.610888+0000", 2023-10-09T06:00:34.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010458169999999999 2023-10-09T06:00:34.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.621346+0000", 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.832817245999999 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135349 9.d 9:b7ad6e1c:::benchmark_data_smithi008_502820_object51176:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:04.105455+0000", 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 30.449477816000002, 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.459172209, 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.740 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135349 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105455+0000", 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105455+0000", 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105453+0000", 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0417999999999999e-05 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105464+0000", 2023-10-09T06:00:34.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.025e-05 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105474+0000", 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0610000000000001e-06 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105480+0000", 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.055762855 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.161243+0000", 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031189049999999999 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.164362+0000", 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019388989999999998 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.183751+0000", 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003301151 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.187052+0000", 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18655844199999999 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373610+0000", 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1009999999999997e-06 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.746 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373617+0000", 2023-10-09T06:00:34.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91246933500000005 2023-10-09T06:00:34.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.746 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286086+0000", 2023-10-09T06:00:34.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2680000000000002e-06 2023-10-09T06:00:34.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286091+0000", 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.966898898 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.252990+0000", 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047099940000000003 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.257700+0000", 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0077358369999999997 2023-10-09T06:00:34.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.265436+0000", 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.026715319000000001 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.292151+0000", 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.249149184 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.541300+0000", 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014578600000000001 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.542758+0000", 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046799320000000004 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.547438+0000", 2023-10-09T06:00:34.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026429719999999999 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.550081+0000", 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014546408 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.564628+0000", 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.459172209 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.750 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135364 9.d 9:b4fd37ff:::benchmark_data_smithi008_502820_object51191:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:04.153115+0000", 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 30.401818289000001, 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.449789453999999, 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:00:34.751 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135364 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153115+0000", 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153115+0000", 2023-10-09T06:00:34.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153114+0000", 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1359999999999999e-06 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153118+0000", 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6540000000000005e-06 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153127+0000", 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9570000000000001e-06 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153132+0000", 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011677977000000001 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.164810+0000", 2023-10-09T06:00:34.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040417029999999998 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.168851+0000", 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034353937000000001 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.203205+0000", 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030759429999999998 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.755 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.206281+0000", 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16741861399999999 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373700+0000", 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9889999999999992e-06 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373708+0000", 2023-10-09T06:00:34.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91545159799999998 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289159+0000", 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7560000000000002e-06 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289166+0000", 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.003512383 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.292679+0000", 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059218559999999996 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.298600+0000", 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071744410000000002 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.305775+0000", 2023-10-09T06:00:34.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.026882515999999999 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.332657+0000", 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25068875899999998 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.583346+0000", 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016443880000000001 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.584991+0000", 2023-10-09T06:00:34.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014049830000000001 2023-10-09T06:00:34.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:34.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.586396+0000", 2023-10-09T06:00:34.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025583310000000001 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.588954+0000", 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01395045 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.602904+0000", 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.449789453999999 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.762 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.763 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.763 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:00:34.763 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:00:34.763 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:00:34.763 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:34.764 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:34 smithi008 ceph-mon[184418]: pgmap v1445: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 189 op/s; 54074/91302 objects degraded (59.225%); 12813/91302 objects misplaced (14.034%); 5.4 MiB/s, 90 objects/s recovering 2023-10-09T06:00:34.764 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2033441994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:34.764 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:34 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:34.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:34 smithi191 ceph-mon[161669]: pgmap v1445: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 189 op/s; 54074/91302 objects degraded (59.225%); 12813/91302 objects misplaced (14.034%); 5.4 MiB/s, 90 objects/s recovering 2023-10-09T06:00:34.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2033441994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:34.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:34 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:34.794 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:35.190 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:35.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:35.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:35.294 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:35.321 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:00:35.329 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:35.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[184418]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[184418]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[184418]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[184418]: osdmap e483: 8 total, 8 up, 6 in 2023-10-09T06:00:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[184418]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[184418]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[184418]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[184418]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[191227]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[191227]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[191227]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[191227]: osdmap e483: 8 total, 8 up, 6 in 2023-10-09T06:00:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[191227]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[191227]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[191227]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:35 smithi008 ceph-mon[191227]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.581 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:35.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:35.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:35.687 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:35.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:35 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:35.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:35 smithi191 ceph-mon[161669]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:35 smithi191 ceph-mon[161669]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:35 smithi191 ceph-mon[161669]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.723 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:35 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:35.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:35 smithi191 ceph-mon[161669]: osdmap e483: 8 total, 8 up, 6 in 2023-10-09T06:00:35.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:35 smithi191 ceph-mon[161669]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:35 smithi191 ceph-mon[161669]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:35 smithi191 ceph-mon[161669]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.724 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:35 smithi191 ceph-mon[161669]: failed to encode map e483 with expected crc 2023-10-09T06:00:35.727 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:35.796 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:35.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:35.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:35.903 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:35.945 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:35.998 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:36.024 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:36.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:36.062 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:00:36.085 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:00:36.105 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:00:36.126 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:00:36.138 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:36.148 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:00:36.167 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:00:36.168 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:36.196 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:00:36.218 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:00:36.219 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:00:36.220 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:00:36.220 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:00:36.220 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:00:36.220 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:00:36.220 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:00:36.220 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:00:36.220 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:00:36.289 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:36.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:36.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:36.393 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:36.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:36 smithi008 ceph-mon[184418]: pgmap v1447: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 54074/90504 objects degraded (59.748%); 12792/90504 objects misplaced (14.134%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:00:36.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/509112918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:36.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:36 smithi008 ceph-mon[191227]: pgmap v1447: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 54074/90504 objects degraded (59.748%); 12792/90504 objects misplaced (14.134%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:00:36.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/509112918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:36.467 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:36.526 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:36.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:36.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:36.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:36.660 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:36 smithi191 ceph-mon[161669]: pgmap v1447: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 54074/90504 objects degraded (59.748%); 12792/90504 objects misplaced (14.134%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:00:36.660 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/509112918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:36.663 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:36.680 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:00:36.680 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:00:36.680 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:00:36.680 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-10-09T06:00:36.680 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0 2023-10-09T06:00:36.728 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:36.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:36.836 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:36.872 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:37.656 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:00:37.865 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:37.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:37.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:37.971 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:38.035 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:38.162 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:38.347 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:38.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:38.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:38.451 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:38.486 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:38.629 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:38.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:38 smithi008 ceph-mon[184418]: pgmap v1448: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 53794/88743 objects degraded (60.618%); 12247/88743 objects misplaced (13.801%); 6.2 MiB/s, 102 objects/s recovering 2023-10-09T06:00:38.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 53794/88743 objects degraded (60.618%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:38.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:38 smithi008 ceph-mon[191227]: pgmap v1448: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 53794/88743 objects degraded (60.618%); 12247/88743 objects misplaced (13.801%); 6.2 MiB/s, 102 objects/s recovering 2023-10-09T06:00:38.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 53794/88743 objects degraded (60.618%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:38.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:38 smithi191 ceph-mon[161669]: pgmap v1448: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 53794/88743 objects degraded (60.618%); 12247/88743 objects misplaced (13.801%); 6.2 MiB/s, 102 objects/s recovering 2023-10-09T06:00:38.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 53794/88743 objects degraded (60.618%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:39.014 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:39.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:39.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:39.027 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:00:39.027 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:00:39.118 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:39.148 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:39.197 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:39.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:39.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:39.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:39.334 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:39.392 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:39.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:39.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:39.500 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:39.539 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:39.619 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:39.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:39.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:39.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:39.764 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:39.817 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:39.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:39.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:39.863 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:00:39.925 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:39.957 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:40.606 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:00:40.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:40 smithi008 ceph-mon[191227]: pgmap v1449: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 190 op/s; 53733/87630 objects degraded (61.318%); 12153/87630 objects misplaced (13.869%); 5.9 MiB/s, 97 objects/s recovering 2023-10-09T06:00:40.678 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:00:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:00:40] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.33.4" 2023-10-09T06:00:40.682 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:40 smithi008 ceph-mon[184418]: pgmap v1449: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 190 op/s; 53733/87630 objects degraded (61.318%); 12153/87630 objects misplaced (13.869%); 5.9 MiB/s, 97 objects/s recovering 2023-10-09T06:00:40.700 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:40.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:40.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:40 smithi191 ceph-mon[161669]: pgmap v1449: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 190 op/s; 53733/87630 objects degraded (61.318%); 12153/87630 objects misplaced (13.869%); 5.9 MiB/s, 97 objects/s recovering 2023-10-09T06:00:40.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:40.845 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:40.908 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:40.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:40.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:41.014 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:41.051 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:41.121 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:41.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:41.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:41.233 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:41.270 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:41.649 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:41.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:41.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:41.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:41.788 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:41.861 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:41.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:41.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:41.968 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:42.001 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:42.061 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:42.068 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:42.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:42.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:42.175 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:42.204 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:42.261 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:42.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:42.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:42.368 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:42.400 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.1 primary-affinity to 0 (802) 2023-10-09T06:00:42.413 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:42.472 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:42.477 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:42 smithi191 ceph-mon[161669]: pgmap v1450: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 53373/85419 objects degraded (62.484%); 11960/85419 objects misplaced (14.002%); 6.0 MiB/s, 101 objects/s recovering 2023-10-09T06:00:42.477 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/948978076' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T06:00:42.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:42.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:42.567 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:42 smithi008 ceph-mon[184418]: pgmap v1450: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 53373/85419 objects degraded (62.484%); 11960/85419 objects misplaced (14.002%); 6.0 MiB/s, 101 objects/s recovering 2023-10-09T06:00:42.567 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/948978076' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T06:00:42.568 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:42 smithi008 ceph-mon[191227]: pgmap v1450: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 53373/85419 objects degraded (62.484%); 11960/85419 objects misplaced (14.002%); 6.0 MiB/s, 101 objects/s recovering 2023-10-09T06:00:42.568 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/948978076' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T06:00:42.578 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:42.640 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:42.826 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:42.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:42.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:42.931 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:42.985 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:43.424 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:00:43.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[191227]: failed to encode map e484 with expected crc 2023-10-09T06:00:43.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/948978076' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T06:00:43.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[191227]: osdmap e484: 8 total, 8 up, 6 in 2023-10-09T06:00:43.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[191227]: failed to encode map e484 with expected crc 2023-10-09T06:00:43.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[191227]: failed to encode map e484 with expected crc 2023-10-09T06:00:43.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[191227]: pgmap v1452: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 53294/85080 objects degraded (62.640%); 11290/85080 objects misplaced (13.270%); 8.2 MiB/s, 137 objects/s recovering 2023-10-09T06:00:43.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1215160184' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:43.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:43.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 53373/85419 objects degraded (62.484%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[184418]: failed to encode map e484 with expected crc 2023-10-09T06:00:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/948978076' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T06:00:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[184418]: osdmap e484: 8 total, 8 up, 6 in 2023-10-09T06:00:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[184418]: failed to encode map e484 with expected crc 2023-10-09T06:00:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[184418]: failed to encode map e484 with expected crc 2023-10-09T06:00:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[184418]: pgmap v1452: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 53294/85080 objects degraded (62.640%); 11290/85080 objects misplaced (13.270%); 8.2 MiB/s, 137 objects/s recovering 2023-10-09T06:00:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1215160184' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:43.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 53373/85419 objects degraded (62.484%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:43.725 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:43.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:43.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:43 smithi191 ceph-mon[161669]: failed to encode map e484 with expected crc 2023-10-09T06:00:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:43 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/948978076' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T06:00:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:43 smithi191 ceph-mon[161669]: osdmap e484: 8 total, 8 up, 6 in 2023-10-09T06:00:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:43 smithi191 ceph-mon[161669]: failed to encode map e484 with expected crc 2023-10-09T06:00:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:43 smithi191 ceph-mon[161669]: failed to encode map e484 with expected crc 2023-10-09T06:00:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:43 smithi191 ceph-mon[161669]: pgmap v1452: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 53294/85080 objects degraded (62.640%); 11290/85080 objects misplaced (13.270%); 8.2 MiB/s, 137 objects/s recovering 2023-10-09T06:00:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:43 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1215160184' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:43.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:43 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:43.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 53373/85419 objects degraded (62.484%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:43.829 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:43.856 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[184418]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[184418]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:00:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[184418]: osdmap e485: 8 total, 8 up, 6 in 2023-10-09T06:00:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[184418]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[184418]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[191227]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[191227]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:00:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[191227]: osdmap e485: 8 total, 8 up, 6 in 2023-10-09T06:00:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[191227]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[191227]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:44.720 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:44.727 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:44.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:44.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:44.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:44 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:44 smithi191 ceph-mon[161669]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:44 smithi191 ceph-mon[161669]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:44 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:00:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:44 smithi191 ceph-mon[161669]: osdmap e485: 8 total, 8 up, 6 in 2023-10-09T06:00:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:44 smithi191 ceph-mon[161669]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:44 smithi191 ceph-mon[161669]: failed to encode map e485 with expected crc 2023-10-09T06:00:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:44.834 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:44.860 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:44.919 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:44.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:44.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:45.032 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:45.070 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:45.188 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:45.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:45.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:45.296 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:45.337 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:00:45.338 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:45.367 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:00:45.395 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:00:45.421 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:00:45.446 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:00:45.471 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:00:45.493 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:00:45.515 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:00:45.518 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:00:45.518 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:00:45.518 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:00:45.518 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:00:45.518 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:00:45.518 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:00:45.518 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:00:45.519 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:00:45.544 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:45 smithi008 ceph-mon[191227]: pgmap v1454: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 53294/84060 objects degraded (63.400%); 11268/84060 objects misplaced (13.405%); 5.4 MiB/s, 90 objects/s recovering 2023-10-09T06:00:45.544 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2474832165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:45.546 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:45 smithi008 ceph-mon[184418]: pgmap v1454: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 53294/84060 objects degraded (63.400%); 11268/84060 objects misplaced (13.405%); 5.4 MiB/s, 90 objects/s recovering 2023-10-09T06:00:45.546 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2474832165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:45 smithi191 ceph-mon[161669]: pgmap v1454: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 53294/84060 objects degraded (63.400%); 11268/84060 objects misplaced (13.405%); 5.4 MiB/s, 90 objects/s recovering 2023-10-09T06:00:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2474832165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:46.104 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:46.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:46.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:46.208 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:46.262 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:46.349 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:46.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:46.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:46.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:46.510 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:46.586 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:46.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:46.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:46.705 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:46.764 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:46.839 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:00:47.000 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:47.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:47.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:47.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:47.167 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:47.241 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:47.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:47.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:47.349 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:47.390 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:47.454 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:47.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:47.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:47.500 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:47.524 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:00:47.524 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:00:47.524 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T06:00:47.524 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T06:00:47.524 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:00:47.559 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:47.592 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:47.814 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:47 smithi008 ceph-mon[184418]: pgmap v1455: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 52876/81738 objects degraded (64.690%); 11072/81738 objects misplaced (13.546%); 7.1 MiB/s, 183 objects/s recovering 2023-10-09T06:00:47.818 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:47 smithi008 ceph-mon[191227]: pgmap v1455: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 52876/81738 objects degraded (64.690%); 11072/81738 objects misplaced (13.546%); 7.1 MiB/s, 183 objects/s recovering 2023-10-09T06:00:47.897 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:47.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:47.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:48.002 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:48.033 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:48.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:47 smithi191 ceph-mon[161669]: pgmap v1455: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 52876/81738 objects degraded (64.690%); 11072/81738 objects misplaced (13.546%); 7.1 MiB/s, 183 objects/s recovering 2023-10-09T06:00:48.953 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:00:49.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 52876/81738 objects degraded (64.690%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:49.085 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 52876/81738 objects degraded (64.690%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:49.086 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 52876/81738 objects degraded (64.690%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:49.104 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:00:49.291 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:49.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:49.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:49.395 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:49.424 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:49.480 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:49.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:49.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:49.522 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:49.584 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:49.620 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:49.703 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:49.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:49.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:49.752 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:00:49.752 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:00:49.752 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T06:00:49.752 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:00:49.753 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:00:49.753 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T06:00:49.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:49.856 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:49.859 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:49.914 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:49.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:49.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:50.026 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:50.093 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:50.189 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:50.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:50.287 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:00:50.297 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:50.314 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:00:50.344 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:50 smithi008 ceph-mon[184418]: pgmap v1456: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 211 op/s; 52784/80334 objects degraded (65.706%); 10941/80334 objects misplaced (13.619%); 5.8 MiB/s, 159 objects/s recovering 2023-10-09T06:00:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1407605813' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:50 smithi008 ceph-mon[191227]: pgmap v1456: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 211 op/s; 52784/80334 objects degraded (65.706%); 10941/80334 objects misplaced (13.619%); 5.8 MiB/s, 159 objects/s recovering 2023-10-09T06:00:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1407605813' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:50.528 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:50 smithi191 ceph-mon[161669]: pgmap v1456: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 211 op/s; 52784/80334 objects degraded (65.706%); 10941/80334 objects misplaced (13.619%); 5.8 MiB/s, 159 objects/s recovering 2023-10-09T06:00:50.528 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1407605813' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:50.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:00:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:00:50] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.33.4" 2023-10-09T06:00:51.354 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:51.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:51.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:51.458 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:51.497 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:51.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:51 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:51.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:51 smithi191 ceph-mon[161669]: failed to encode map e486 with expected crc 2023-10-09T06:00:51.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:51 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1407605813' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:51.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:51 smithi191 ceph-mon[161669]: osdmap e486: 8 total, 8 up, 6 in 2023-10-09T06:00:51.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:51 smithi191 ceph-mon[161669]: failed to encode map e486 with expected crc 2023-10-09T06:00:51.556 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:51.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:51.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:51.662 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:51.692 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:51 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:51.692 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:51 smithi008 ceph-mon[184418]: failed to encode map e486 with expected crc 2023-10-09T06:00:51.692 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:51 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1407605813' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:51.692 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:51 smithi008 ceph-mon[184418]: osdmap e486: 8 total, 8 up, 6 in 2023-10-09T06:00:51.692 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:51 smithi008 ceph-mon[184418]: failed to encode map e486 with expected crc 2023-10-09T06:00:51.692 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:51 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:51.692 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:51 smithi008 ceph-mon[191227]: failed to encode map e486 with expected crc 2023-10-09T06:00:51.692 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:51 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1407605813' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:51.692 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:51 smithi008 ceph-mon[191227]: osdmap e486: 8 total, 8 up, 6 in 2023-10-09T06:00:51.692 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:51 smithi008 ceph-mon[191227]: failed to encode map e486 with expected crc 2023-10-09T06:00:51.696 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:51.907 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:51.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:51.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:52.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:52.042 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:52.136 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:52.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:52.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:52.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:52.311 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:52.383 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:52.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:52.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:52.488 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:52.497 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:52 smithi008 ceph-mon[184418]: pgmap v1458: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 52448/78336 objects degraded (66.953%); 10740/78336 objects misplaced (13.710%); 4.8 MiB/s, 146 objects/s recovering 2023-10-09T06:00:52.498 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:52 smithi008 ceph-mon[191227]: pgmap v1458: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 52448/78336 objects degraded (66.953%); 10740/78336 objects misplaced (13.710%); 4.8 MiB/s, 146 objects/s recovering 2023-10-09T06:00:52.524 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:52.604 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:52.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:52.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:52.708 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:52.745 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:52.746 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:52 smithi191 ceph-mon[161669]: pgmap v1458: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 52448/78336 objects degraded (66.953%); 10740/78336 objects misplaced (13.710%); 4.8 MiB/s, 146 objects/s recovering 2023-10-09T06:00:52.964 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:52.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:52.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:53.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:53.098 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:53.167 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:53.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:53.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:53.270 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:00:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T06:00:53.270 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T06:00:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T06:00:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45727 9.5 9:af6b034d:::benchmark_data_smithi008_502820_object45726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T06:00:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:45.884586+0000", 2023-10-09T06:00:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 427.38236321400001, 2023-10-09T06:00:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.280584508, 2023-10-09T06:00:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:00:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.271 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45727 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884586+0000", 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884586+0000", 2023-10-09T06:00:53.272 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884585+0000", 2023-10-09T06:00:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884616+0000", 2023-10-09T06:00:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0423e-05 2023-10-09T06:00:53.273 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884625+0000", 2023-10-09T06:00:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0250000000000008e-06 2023-10-09T06:00:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884631+0000", 2023-10-09T06:00:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3840000000000002e-06 2023-10-09T06:00:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.274 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884669+0000", 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7698999999999999e-05 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884795+0000", 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012634500000000001 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T06:00:53.275 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884920+0000", 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012516900000000001 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.061847+0000", 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17692612699999999 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.061860+0000", 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3361e-05 2023-10-09T06:00:53.276 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933452+0000", 2023-10-09T06:00:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.87159255700000005 2023-10-09T06:00:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933460+0000", 2023-10-09T06:00:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5549999999999997e-06 2023-10-09T06:00:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.120071+0000", 2023-10-09T06:00:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1866107539999999 2023-10-09T06:00:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.120662+0000", 2023-10-09T06:00:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00059118300000000001 2023-10-09T06:00:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165062+0000", 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.044399986000000002 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165171+0000", 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010856700000000001 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.279 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45730 9.5 9:a76420fe:::benchmark_data_smithi008_502820_object45729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T06:00:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:45.884645+0000", 2023-10-09T06:00:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 427.38230377500003, 2023-10-09T06:00:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2944595059999999, 2023-10-09T06:00:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:00:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45730 2023-10-09T06:00:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.280 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884645+0000", 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884645+0000", 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884645+0000", 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884669+0000", 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4326000000000001e-05 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884678+0000", 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8829999999999995e-06 2023-10-09T06:00:53.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884684+0000", 2023-10-09T06:00:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.806e-06 2023-10-09T06:00:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055897+0000", 2023-10-09T06:00:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17121303700000001 2023-10-09T06:00:53.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.056018+0000", 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000120831 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.056156+0000", 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013870000000000001 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066733+0000", 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010576189999999999 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.066739+0000", 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4500000000000001e-06 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933477+0000", 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86673776000000002 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933485+0000", 2023-10-09T06:00:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2779999999999993e-06 2023-10-09T06:00:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165241+0000", 2023-10-09T06:00:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.231755605 2023-10-09T06:00:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:53.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.165788+0000", 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00054761800000000004 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.178974+0000", 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013186164 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.179105+0000", 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000130462 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.287 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:45731 9.9 9:9d7980f6:::benchmark_data_smithi008_502820_object45730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e371)", 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:53:45.884828+0000", 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 427.38212167400002, 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.325062725, 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.288 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 45731 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884828+0000", 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884828+0000", 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884827+0000", 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884851+0000", 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4647999999999998e-05 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.290 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884863+0000", 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1389000000000001e-05 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:45.884870+0000", 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1250000000000004e-06 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055752+0000", 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17088223499999999 2023-10-09T06:00:53.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.055870+0000", 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011758500000000001 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 5,7", 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.056006+0000", 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000136243 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.292 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064121+0000", 2023-10-09T06:00:53.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081152840000000004 2023-10-09T06:00:53.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.064134+0000", 2023-10-09T06:00:53.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2826999999999999e-05 2023-10-09T06:00:53.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933097+0000", 2023-10-09T06:00:53.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86896320299999996 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:46.933109+0000", 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1987999999999999e-05 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.184026+0000", 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2509170430000001 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.184835+0000", 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00080872799999999999 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.209763+0000", 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024928049000000001 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:53:48.209890+0000", 2023-10-09T06:00:53.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000127267 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77614 9.15 9:ad1d5133:::benchmark_data_smithi008_502820_object77613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:36.822615+0000", 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 376.44433460300002, 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3942615420000002, 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:00:53.296 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77614 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822615+0000", 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.298 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822615+0000", 2023-10-09T06:00:53.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.298 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.298 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.298 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822614+0000", 2023-10-09T06:00:53.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:53.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.298 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.298 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822641+0000", 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6928e-05 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822648+0000", 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7700000000000004e-06 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.822654+0000", 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6119999999999998e-06 2023-10-09T06:00:53.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.879602+0000", 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.056948375000000002 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.879611+0000", 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5470000000000002e-06 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.300 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:38.167351+0000", 2023-10-09T06:00:53.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2877391039999999 2023-10-09T06:00:53.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.301 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.301 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:38.167362+0000", 2023-10-09T06:00:53.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1535999999999999e-05 2023-10-09T06:00:53.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.301 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.301 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:39.204324+0000", 2023-10-09T06:00:53.302 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0369618030000001 2023-10-09T06:00:53.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T06:00:53.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:39.204880+0000", 2023-10-09T06:00:53.302 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00055560899999999999 2023-10-09T06:00:53.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:39.216793+0000", 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011913464 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:39.216876+0000", 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3181000000000002e-05 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.303 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93502 9.e 9:7c9e1516:::benchmark_data_smithi008_502820_object9329:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e392)", 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.257612+0000", 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 354.009336948, 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5408023110000002, 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93502 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257612+0000", 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257612+0000", 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257612+0000", 2023-10-09T06:00:53.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257615+0000", 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8990000000000001e-06 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257622+0000", 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.08e-06 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257627+0000", 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3120000000000001e-06 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257937+0000", 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00030955700000000002 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257946+0000", 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2639999999999997e-06 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.343810+0000", 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.085863609 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.347315+0000", 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035057539999999998 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.372730+0000", 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.025414923999999998 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.376888+0000", 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041572459999999999 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.763791+0000", 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.38690379400000002 2023-10-09T06:00:53.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.766585+0000", 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027940199999999999 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.771743+0000", 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051580180000000003 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.788797+0000", 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017053295 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.789756+0000", 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095954999999999999 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.790067+0000", 2023-10-09T06:00:53.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000310468 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.791632+0000", 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015649539999999999 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.798415+0000", 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0067829220000000003 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.312 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93503 9.e 9:722ffbd5:::benchmark_data_smithi008_502820_object9330:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e392)", 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.257654+0000", 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 354.00929563900002, 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5884587200000002, 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.313 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93503 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257654+0000", 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257654+0000", 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257653+0000", 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257656+0000", 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1779999999999999e-06 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257663+0000", 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9489999999999999e-06 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257669+0000", 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8549999999999998e-06 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.258141+0000", 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00047206199999999998 2023-10-09T06:00:53.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.258149+0000", 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1899999999999995e-06 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.394518+0000", 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1363689040000002 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.397990+0000", 2023-10-09T06:00:53.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034717139999999999 2023-10-09T06:00:53.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.443090+0000", 2023-10-09T06:00:53.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.045099601000000003 2023-10-09T06:00:53.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.446520+0000", 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034299270000000001 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.798754+0000", 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35223448899999998 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.801843+0000", 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030893740000000002 2023-10-09T06:00:53.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.807614+0000", 2023-10-09T06:00:53.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005771132 2023-10-09T06:00:53.320 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.836687+0000", 2023-10-09T06:00:53.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.029072210000000001 2023-10-09T06:00:53.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.321 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.321 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.836991+0000", 2023-10-09T06:00:53.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00030478900000000003 2023-10-09T06:00:53.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.321 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.321 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.837946+0000", 2023-10-09T06:00:53.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095470500000000003 2023-10-09T06:00:53.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.322 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.322 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.322 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.839570+0000", 2023-10-09T06:00:53.322 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016237630000000001 2023-10-09T06:00:53.322 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.322 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.322 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.322 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.846112+0000", 2023-10-09T06:00:53.322 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006542338 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93531 9.e 9:7b068cda:::benchmark_data_smithi008_502820_object9358:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e392)", 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.257964+0000", 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 354.00898555100002, 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6005753459999998, 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.323 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93531 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257964+0000", 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257964+0000", 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257963+0000", 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257967+0000", 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8739999999999997e-06 2023-10-09T06:00:53.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257976+0000", 2023-10-09T06:00:53.326 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5760000000000006e-06 2023-10-09T06:00:53.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.257982+0000", 2023-10-09T06:00:53.326 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5239999999999997e-06 2023-10-09T06:00:53.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.327 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.258187+0000", 2023-10-09T06:00:53.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000204546 2023-10-09T06:00:53.327 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.327 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.327 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.258195+0000", 2023-10-09T06:00:53.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0560000000000005e-06 2023-10-09T06:00:53.327 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.327 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.398339+0000", 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1401438540000002 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.401890+0000", 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035510469999999999 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.460275+0000", 2023-10-09T06:00:53.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.058385640000000003 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.463759+0000", 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034834689999999999 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.822402+0000", 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.358643673 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.825300+0000", 2023-10-09T06:00:53.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002897929 2023-10-09T06:00:53.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.830267+0000", 2023-10-09T06:00:53.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049663010000000002 2023-10-09T06:00:53.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.331 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.847190+0000", 2023-10-09T06:00:53.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016923032000000001 2023-10-09T06:00:53.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.331 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.847522+0000", 2023-10-09T06:00:53.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000331952 2023-10-09T06:00:53.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.331 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.848622+0000", 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001100125 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.850417+0000", 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017952689999999999 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.858539+0000", 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081220330000000007 2023-10-09T06:00:53.332 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93555 9.e 9:73c791c6:::benchmark_data_smithi008_502820_object9382:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.314621+0000", 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 353.952327959, 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6041918630000001, 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.333 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.334 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.334 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93555 2023-10-09T06:00:53.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.334 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.334 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314621+0000", 2023-10-09T06:00:53.334 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314621+0000", 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314620+0000", 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314628+0000", 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.038e-06 2023-10-09T06:00:53.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314651+0000", 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2671e-05 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314662+0000", 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0829000000000001e-05 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.336 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314731+0000", 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9561999999999996e-05 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.314744+0000", 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3275000000000001e-05 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.402420+0000", 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.08767521 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.405867+0000", 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034470909999999998 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.478021+0000", 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.072154274000000004 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.338 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.481605+0000", 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035844610000000002 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.867423+0000", 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.385817405 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.870665+0000", 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032420019999999999 2023-10-09T06:00:53.339 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.875924+0000", 2023-10-09T06:00:53.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005258764 2023-10-09T06:00:53.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.907546+0000", 2023-10-09T06:00:53.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.031622665000000001 2023-10-09T06:00:53.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.908496+0000", 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00094938399999999995 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.911083+0000", 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002586933 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.912705+0000", 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001622823 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.918813+0000", 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061077049999999997 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.342 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93563 9.e 9:7ced5d1d:::benchmark_data_smithi008_502820_object9390:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T06:00:53.343 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.328334+0000", 2023-10-09T06:00:53.343 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 353.938615427, 2023-10-09T06:00:53.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5994731469999999, 2023-10-09T06:00:53.343 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.343 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.343 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.343 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.343 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.343 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93563 2023-10-09T06:00:53.343 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.343 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.344 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.344 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328334+0000", 2023-10-09T06:00:53.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.344 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.344 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328334+0000", 2023-10-09T06:00:53.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.345 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328332+0000", 2023-10-09T06:00:53.345 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:53.345 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.345 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.345 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328339+0000", 2023-10-09T06:00:53.345 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9129999999999997e-06 2023-10-09T06:00:53.345 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.345 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328355+0000", 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5471000000000001e-05 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328362+0000", 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0219999999999999e-06 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328425+0000", 2023-10-09T06:00:53.346 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2731999999999995e-05 2023-10-09T06:00:53.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.347 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.347 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.347 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.328435+0000", 2023-10-09T06:00:53.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0389e-05 2023-10-09T06:00:53.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.347 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.347 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.347 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.483156+0000", 2023-10-09T06:00:53.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1547212980000001 2023-10-09T06:00:53.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.486776+0000", 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036195910000000001 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.511225+0000", 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024448785000000001 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.515218+0000", 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039933829999999997 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.890925+0000", 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37570650999999999 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.894485+0000", 2023-10-09T06:00:53.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035602559999999999 2023-10-09T06:00:53.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.900641+0000", 2023-10-09T06:00:53.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006156089 2023-10-09T06:00:53.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.350 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.909685+0000", 2023-10-09T06:00:53.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0090444210000000004 2023-10-09T06:00:53.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.910760+0000", 2023-10-09T06:00:53.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001074965 2023-10-09T06:00:53.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.919115+0000", 2023-10-09T06:00:53.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0083542170000000006 2023-10-09T06:00:53.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.352 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.920728+0000", 2023-10-09T06:00:53.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00161359 2023-10-09T06:00:53.352 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.352 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.927807+0000", 2023-10-09T06:00:53.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070788229999999997 2023-10-09T06:00:53.352 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.352 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93576 9.e 9:775af324:::benchmark_data_smithi008_502820_object9403:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.344876+0000", 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 353.922073001, 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.712756782, 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.353 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93576 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344876+0000", 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344876+0000", 2023-10-09T06:00:53.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344874+0000", 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344882+0000", 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4549999999999998e-06 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344894+0000", 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1877e-05 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344901+0000", 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4229999999999999e-06 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344960+0000", 2023-10-09T06:00:53.356 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8801000000000002e-05 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.344970+0000", 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9490000000000008e-06 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.487235+0000", 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1422650170000002 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.357 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.358 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.492031+0000", 2023-10-09T06:00:53.358 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0047961879999999998 2023-10-09T06:00:53.358 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.358 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.358 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.358 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.529617+0000", 2023-10-09T06:00:53.358 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037586040000000001 2023-10-09T06:00:53.358 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.358 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.358 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.533059+0000", 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034415890000000001 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.012907+0000", 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47984856799999998 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.016476+0000", 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035688650000000001 2023-10-09T06:00:53.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.021478+0000", 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050017769999999998 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.047526+0000", 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.026047573000000001 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.048644+0000", 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001118703 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.049061+0000", 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000416425 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.050924+0000", 2023-10-09T06:00:53.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001863504 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.057633+0000", 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0067087539999999999 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93584 9.e 9:788be11e:::benchmark_data_smithi008_502820_object9411:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T06:00:53.362 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.353872+0000", 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 353.91307730199998, 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7661968130000001, 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93584 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.363 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353872+0000", 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353872+0000", 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353871+0000", 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:53.364 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353877+0000", 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3980000000000004e-06 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353890+0000", 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2541000000000001e-05 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.365 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353898+0000", 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4239999999999993e-06 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353966+0000", 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7978999999999994e-05 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.353979+0000", 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.293e-05 2023-10-09T06:00:53.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.493408+0000", 2023-10-09T06:00:53.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.139429239 2023-10-09T06:00:53.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.497116+0000", 2023-10-09T06:00:53.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037075620000000002 2023-10-09T06:00:53.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.546542+0000", 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.049425963000000003 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.549991+0000", 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034494270000000001 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.068138+0000", 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.51814739300000001 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.071757+0000", 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003618095 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.078191+0000", 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064343730000000002 2023-10-09T06:00:53.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.094886+0000", 2023-10-09T06:00:53.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016694831 2023-10-09T06:00:53.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.111048+0000", 2023-10-09T06:00:53.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016162398000000001 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.112084+0000", 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010355239999999999 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.113683+0000", 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001599222 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.120069+0000", 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0063858350000000003 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93589 9.e 9:787a8e5e:::benchmark_data_smithi008_502820_object9416:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.358349+0000", 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 353.90860048899998, 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.772108904, 2023-10-09T06:00:53.372 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.373 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.373 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.373 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.373 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.373 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93589 2023-10-09T06:00:53.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.373 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358349+0000", 2023-10-09T06:00:53.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.374 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358349+0000", 2023-10-09T06:00:53.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.374 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358348+0000", 2023-10-09T06:00:53.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:00:53.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358351+0000", 2023-10-09T06:00:53.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.044e-06 2023-10-09T06:00:53.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358361+0000", 2023-10-09T06:00:53.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.516e-06 2023-10-09T06:00:53.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.376 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.376 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358367+0000", 2023-10-09T06:00:53.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8710000000000002e-06 2023-10-09T06:00:53.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358431+0000", 2023-10-09T06:00:53.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4703999999999996e-05 2023-10-09T06:00:53.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.358447+0000", 2023-10-09T06:00:53.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5749000000000001e-05 2023-10-09T06:00:53.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.550328+0000", 2023-10-09T06:00:53.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1918804390000002 2023-10-09T06:00:53.378 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.378 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.378 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.554496+0000", 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041680650000000003 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.576252+0000", 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021756074 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.579807+0000", 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003555599 2023-10-09T06:00:53.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.095351+0000", 2023-10-09T06:00:53.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.51554361800000004 2023-10-09T06:00:53.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.099028+0000", 2023-10-09T06:00:53.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036770129999999998 2023-10-09T06:00:53.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.104732+0000", 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0057035779999999999 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.120380+0000", 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015648539 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.121336+0000", 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095552600000000005 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.122385+0000", 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010494969999999999 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.124126+0000", 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017405159999999999 2023-10-09T06:00:53.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.130458+0000", 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0063319300000000004 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93594 9.e 9:7f17079c:::benchmark_data_smithi008_502820_object9421:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.374295+0000", 2023-10-09T06:00:53.383 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 353.89265386099999, 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8370609930000001, 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93594 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374295+0000", 2023-10-09T06:00:53.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374295+0000", 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374293+0000", 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.385 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374302+0000", 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8829999999999995e-06 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374317+0000", 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4803999999999999e-05 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.386 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374326+0000", 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5480000000000007e-06 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374387+0000", 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0439000000000002e-05 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374401+0000", 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4321000000000001e-05 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.555902+0000", 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1815005030000001 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.559403+0000", 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035008999999999999 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.596637+0000", 2023-10-09T06:00:53.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037234872000000002 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.600233+0000", 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035955890000000002 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.148074+0000", 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.54784091499999998 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.151004+0000", 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029301489999999999 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.156003+0000", 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049993269999999996 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.180475+0000", 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024471809000000001 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.200700+0000", 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020225205 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.202560+0000", 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018592120000000001 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.204613+0000", 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020536410000000001 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.211356+0000", 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0067430190000000003 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.392 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93595 9.e 9:768fa25c:::benchmark_data_smithi008_502820_object9422:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T06:00:53.393 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.374670+0000", 2023-10-09T06:00:53.393 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 353.89227903699998, 2023-10-09T06:00:53.393 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9132208020000001, 2023-10-09T06:00:53.393 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.393 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.393 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.393 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.393 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.393 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93595 2023-10-09T06:00:53.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.393 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374670+0000", 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374670+0000", 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374669+0000", 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374675+0000", 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2899999999999999e-06 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374687+0000", 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1705999999999999e-05 2023-10-09T06:00:53.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374695+0000", 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7400000000000004e-06 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374776+0000", 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0828000000000006e-05 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.374784+0000", 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8419999999999994e-06 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.601628+0000", 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.226844104 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.605105+0000", 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003476252 2023-10-09T06:00:53.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.629729+0000", 2023-10-09T06:00:53.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024624511000000002 2023-10-09T06:00:53.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.633203+0000", 2023-10-09T06:00:53.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034742140000000002 2023-10-09T06:00:53.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.399 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.399 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.228786+0000", 2023-10-09T06:00:53.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.595582308 2023-10-09T06:00:53.399 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.399 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.399 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.399 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.231835+0000", 2023-10-09T06:00:53.399 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030496080000000001 2023-10-09T06:00:53.399 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.399 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.399 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.236845+0000", 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050095119999999998 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.273038+0000", 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.036192884000000002 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.278799+0000", 2023-10-09T06:00:53.400 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0057609829999999999 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.279816+0000", 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001016889 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.281482+0000", 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001666257 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.401 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.287891+0000", 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0064091240000000004 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93610 9.e 9:740b9f1d:::benchmark_data_smithi008_502820_object9437:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.406485+0000", 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 353.860464294, 2023-10-09T06:00:53.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9398336600000001, 2023-10-09T06:00:53.403 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.403 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.403 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.403 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.403 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.403 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93610 2023-10-09T06:00:53.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.403 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406485+0000", 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406485+0000", 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406484+0000", 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:00:53.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406489+0000", 2023-10-09T06:00:53.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.899e-06 2023-10-09T06:00:53.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406503+0000", 2023-10-09T06:00:53.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3641e-05 2023-10-09T06:00:53.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406510+0000", 2023-10-09T06:00:53.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9840000000000004e-06 2023-10-09T06:00:53.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406560+0000", 2023-10-09T06:00:53.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0145999999999997e-05 2023-10-09T06:00:53.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.406574+0000", 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3728e-05 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.605471+0000", 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1988978069999998 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.609030+0000", 2023-10-09T06:00:53.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035584269999999999 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.646970+0000", 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037940146000000001 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.650410+0000", 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034402059999999999 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.408 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.298075+0000", 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.64766473599999996 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.301969+0000", 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038939159999999999 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.409 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.307872+0000", 2023-10-09T06:00:53.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0059028589999999999 2023-10-09T06:00:53.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.335322+0000", 2023-10-09T06:00:53.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.027449871000000001 2023-10-09T06:00:53.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.336234+0000", 2023-10-09T06:00:53.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00091258099999999998 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.337575+0000", 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00134044 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.339121+0000", 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015462939999999999 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.346319+0000", 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0071977279999999996 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93677 9.e 9:78283dde:::benchmark_data_smithi008_502820_object9504:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.566695+0000", 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 353.70025428299999, 2023-10-09T06:00:53.412 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7929214409999998, 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93677 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566695+0000", 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566695+0000", 2023-10-09T06:00:53.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566693+0000", 2023-10-09T06:00:53.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:53.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566701+0000", 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1240000000000005e-06 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566720+0000", 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8777e-05 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566729+0000", 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0990000000000005e-06 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566809+0000", 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9973000000000002e-05 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.566824+0000", 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5009e-05 2023-10-09T06:00:53.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.610494+0000", 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0436703280000001 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.615455+0000", 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049606219999999996 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.663800+0000", 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.048344564 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.667270+0000", 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034707660000000001 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.321874+0000", 2023-10-09T06:00:53.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.65460370099999998 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.324634+0000", 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027600049999999998 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.329479+0000", 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048448149999999997 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.420 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.420 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.337263+0000", 2023-10-09T06:00:53.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007783852 2023-10-09T06:00:53.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.420 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.420 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.347338+0000", 2023-10-09T06:00:53.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010074797 2023-10-09T06:00:53.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.420 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.351320+0000", 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039819499999999997 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.352893+0000", 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015735219999999999 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.359616+0000", 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0067233279999999998 2023-10-09T06:00:53.421 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.422 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.422 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.422 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:93714 9.e 9:772117e4:::benchmark_data_smithi008_502820_object9541:head [delete] snapc 0=[] ondisk+write+known_if_redirected e392)", 2023-10-09T06:00:53.422 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:59.733175+0000", 2023-10-09T06:00:53.422 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 353.53377390700001, 2023-10-09T06:00:53.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.706776166, 2023-10-09T06:00:53.422 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.422 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.422 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 93714 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733175+0000", 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.423 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733175+0000", 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733174+0000", 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733183+0000", 2023-10-09T06:00:53.424 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1949999999999992e-06 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733202+0000", 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9624e-05 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733212+0000", 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7999999999999993e-06 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.425 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733269+0000", 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6560000000000001e-05 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:59.733279+0000", 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0774000000000001e-05 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.426 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.667590+0000", 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.934310237 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.671086+0000", 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034961910000000001 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.689198+0000", 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018112329 2023-10-09T06:00:53.427 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:01.692629+0000", 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034307169999999998 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.408630+0000", 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.71600111300000002 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.428 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.411913+0000", 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032832550000000001 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.417307+0000", 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0053933360000000003 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.429677+0000", 2023-10-09T06:00:53.429 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012370188000000001 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.430738+0000", 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001061541 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.431132+0000", 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00039332199999999999 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.430 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.431 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.431 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.433108+0000", 2023-10-09T06:00:53.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019765020000000002 2023-10-09T06:00:53.431 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.431 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.431 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.431 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:55:02.439951+0000", 2023-10-09T06:00:53.431 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0068432559999999998 2023-10-09T06:00:53.431 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.431 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.431 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:109057 9.e 9:7bb7eef4:::benchmark_data_smithi008_502820_object24884:head [delete] snapc 0=[] ondisk+write+known_if_redirected e433)", 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:07.182182+0000", 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 226.084767156, 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8350503200000001, 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.432 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 109057 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.182182+0000", 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.182182+0000", 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.433 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.182180+0000", 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.182188+0000", 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1380000000000001e-06 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.434 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.182200+0000", 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2860000000000001e-05 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.182208+0000", 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2010000000000002e-06 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.291139+0000", 2023-10-09T06:00:53.435 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.108931338 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.304280+0000", 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013141342 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.369788+0000", 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.065508155999999998 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.436 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.437 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.377944+0000", 2023-10-09T06:00:53.437 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0081553690000000009 2023-10-09T06:00:53.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.437 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:08.904483+0000", 2023-10-09T06:00:53.437 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.526539168 2023-10-09T06:00:53.437 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.437 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.437 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:08.909570+0000", 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050870350000000002 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:08.928577+0000", 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019006578999999999 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:08.967891+0000", 2023-10-09T06:00:53.438 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.039314132000000002 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:08.998858+0000", 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030967410000000001 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.002157+0000", 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032987960000000001 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.439 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.003854+0000", 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016972720000000001 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.006513+0000", 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026589830000000002 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.440 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.017232+0000", 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.01071921 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:109060 9.e 9:7b9f909b:::benchmark_data_smithi008_502820_object24887:head [delete] snapc 0=[] ondisk+write+known_if_redirected e433)", 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:07.223491+0000", 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 226.04345797600001, 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.983487276, 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.441 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.442 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.442 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.442 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.442 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 109060 2023-10-09T06:00:53.442 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.442 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.442 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.442 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.442 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.223491+0000", 2023-10-09T06:00:53.442 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.442 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.223491+0000", 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.223489+0000", 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.443 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.223497+0000", 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3389999999999993e-06 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.223512+0000", 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4949999999999999e-05 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.223520+0000", 2023-10-09T06:00:53.444 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2860000000000002e-06 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.305574+0000", 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.082054345000000001 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.316516+0000", 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010942174000000001 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.445 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.414613+0000", 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.098096502000000002 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.420887+0000", 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0062745919999999998 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.143924+0000", 2023-10-09T06:00:53.446 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.723036901 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.148344+0000", 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004420261 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.156220+0000", 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078757600000000007 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.447 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.190517+0000", 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.034296259000000003 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.190997+0000", 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00048024800000000001 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.193431+0000", 2023-10-09T06:00:53.448 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0024338239999999998 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.194865+0000", 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014345670000000001 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.197249+0000", 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023836740000000001 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.449 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.450 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.450 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.206979+0000", 2023-10-09T06:00:53.450 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0097297059999999994 2023-10-09T06:00:53.450 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.450 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.450 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.450 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.450 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.450 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:109066 9.e 9:75b4bf9a:::benchmark_data_smithi008_502820_object24893:head [delete] snapc 0=[] ondisk+write+known_if_redirected e433)", 2023-10-09T06:00:53.450 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:07.497729+0000", 2023-10-09T06:00:53.450 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 225.769219924, 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.792146577, 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 109066 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:00:53.451 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.497729+0000", 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.497729+0000", 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.497728+0000", 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.452 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.497736+0000", 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5089999999999993e-06 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.497751+0000", 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5218999999999999e-05 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:00:53.453 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.497762+0000", 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0308e-05 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.639290+0000", 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.141528667 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.649768+0000", 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010477887999999999 2023-10-09T06:00:53.454 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.716513+0000", 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.066745105999999998 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:07.724202+0000", 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0076884950000000001 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.455 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.225099+0000", 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.50089695 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.229434+0000", 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0043354270000000002 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.237339+0000", 2023-10-09T06:00:53.456 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0079044760000000006 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.253877+0000", 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016537930999999999 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.254403+0000", 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052625000000000003 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.457 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.255956+0000", 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015531220000000001 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.275616+0000", 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019659787000000001 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.278424+0000", 2023-10-09T06:00:53.458 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028079580000000002 2023-10-09T06:00:53.459 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:00:53.459 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:00:53.459 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:00:53.459 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:09.289876+0000", 2023-10-09T06:00:53.459 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011452051 2023-10-09T06:00:53.459 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.459 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.459 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.459 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:00:53.459 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:00:53.459 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:00:53.460 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:53.461 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:53 smithi191 ceph-mon[161669]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.461 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:53 smithi191 ceph-mon[161669]: osdmap e487: 8 total, 8 up, 6 in 2023-10-09T06:00:53.461 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:53 smithi191 ceph-mon[161669]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.461 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:53 smithi191 ceph-mon[161669]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.461 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:53 smithi191 ceph-mon[161669]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.461 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:53 smithi191 ceph-mon[161669]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.461 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:53 smithi191 ceph-mon[161669]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.461 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:53 smithi191 ceph-mon[161669]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.461 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:53 smithi191 ceph-mon[161669]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.461 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 52448/78336 objects degraded (66.953%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:53.462 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:00:53.462 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:00:53.462 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:00:53.462 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:00:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[184418]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[184418]: osdmap e487: 8 total, 8 up, 6 in 2023-10-09T06:00:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[184418]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[184418]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[184418]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[184418]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[184418]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[184418]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[184418]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 52448/78336 objects degraded (66.953%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[191227]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[191227]: osdmap e487: 8 total, 8 up, 6 in 2023-10-09T06:00:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[191227]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[191227]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[191227]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[191227]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[191227]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[191227]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[191227]: failed to encode map e487 with expected crc 2023-10-09T06:00:53.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 52448/78336 objects degraded (66.953%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:53.475 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:00:53.502 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:00:53.502 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:53.528 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:00:53.554 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:00:53.555 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:00:53.555 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:00:53.556 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:00:53.556 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:00:53.556 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:00:53.556 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:00:53.556 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:00:53.556 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:00:53.560 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:53.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:53.668 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:53.687 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:00:53.687 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:00:53.687 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:00:53.708 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:53.777 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:53.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:53.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:53.882 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:53.918 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:53.968 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:53.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:53.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:54.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:54.102 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:54.174 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:54.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:54.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:54.285 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:54.316 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:54.483 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:54.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:54.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:54.584 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:00:54.595 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:54.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[191227]: pgmap v1460: 99 pgs: 1 peering, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 52379/78003 objects degraded (67.150%); 10671/78003 objects misplaced (13.680%); 5.3 MiB/s, 288 objects/s recovering 2023-10-09T06:00:54.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1125791975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:54.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[191227]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[191227]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[191227]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[191227]: osdmap e488: 8 total, 8 up, 6 in 2023-10-09T06:00:54.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[191227]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.611 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[191227]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[184418]: pgmap v1460: 99 pgs: 1 peering, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 52379/78003 objects degraded (67.150%); 10671/78003 objects misplaced (13.680%); 5.3 MiB/s, 288 objects/s recovering 2023-10-09T06:00:54.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1125791975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:54.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[184418]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[184418]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[184418]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[184418]: osdmap e488: 8 total, 8 up, 6 in 2023-10-09T06:00:54.611 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[184418]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.612 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:54 smithi008 ceph-mon[184418]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.638 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:54.770 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:00:54.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:54 smithi191 ceph-mon[161669]: pgmap v1460: 99 pgs: 1 peering, 10 active+recovery_wait+degraded, 1 active+recovering, 4 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 52379/78003 objects degraded (67.150%); 10671/78003 objects misplaced (13.680%); 5.3 MiB/s, 288 objects/s recovering 2023-10-09T06:00:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:54 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1125791975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:54 smithi191 ceph-mon[161669]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:54 smithi191 ceph-mon[161669]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:54 smithi191 ceph-mon[161669]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:54 smithi191 ceph-mon[161669]: osdmap e488: 8 total, 8 up, 6 in 2023-10-09T06:00:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:54 smithi191 ceph-mon[161669]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:54 smithi191 ceph-mon[161669]: failed to encode map e488 with expected crc 2023-10-09T06:00:54.969 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:54.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:55.073 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:55.114 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:55.187 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:55.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:55.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:55.288 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:55.296 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:55.326 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:55.336 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:55.380 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:55.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:55.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:55.488 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:55.529 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:55.582 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:55.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:55.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:55.688 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:55.721 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:55.931 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:55.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:55.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:56.034 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:56.066 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:56.542 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:56 smithi008 ceph-mon[184418]: pgmap v1462: 99 pgs: 1 peering, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49228/77049 objects degraded (63.892%); 11455/77049 objects misplaced (14.867%); 3.2 MiB/s, 241 objects/s recovering 2023-10-09T06:00:56.542 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:56 smithi008 ceph-mon[191227]: pgmap v1462: 99 pgs: 1 peering, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49228/77049 objects degraded (63.892%); 11455/77049 objects misplaced (14.867%); 3.2 MiB/s, 241 objects/s recovering 2023-10-09T06:00:56.588 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:56.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:56.694 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:56.720 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:56 smithi191 ceph-mon[161669]: pgmap v1462: 99 pgs: 1 peering, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49228/77049 objects degraded (63.892%); 11455/77049 objects misplaced (14.867%); 3.2 MiB/s, 241 objects/s recovering 2023-10-09T06:00:56.722 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:56.776 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:56.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:56.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:56.886 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:56.929 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:57.069 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:00:57.549 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:57.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:57.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:57.653 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:57.708 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:57.908 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:57.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:57.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:58.012 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:58.017 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:58.055 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:58.147 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:58.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:58.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:58.256 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:58.299 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:58.410 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:00:58.410 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:00:58.479 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:58.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:58.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:58.597 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:58.626 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:58 smithi008 ceph-mon[184418]: pgmap v1463: 99 pgs: 1 peering, 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 200 op/s; 48867/75030 objects degraded (65.130%); 11293/75030 objects misplaced (15.051%); 3.8 MiB/s, 237 objects/s recovering 2023-10-09T06:00:58.626 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 48867/75030 objects degraded (65.130%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:58.626 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:58 smithi008 ceph-mon[191227]: pgmap v1463: 99 pgs: 1 peering, 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 200 op/s; 48867/75030 objects degraded (65.130%); 11293/75030 objects misplaced (15.051%); 3.8 MiB/s, 237 objects/s recovering 2023-10-09T06:00:58.627 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 48867/75030 objects degraded (65.130%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:58.629 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:58.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:58 smithi191 ceph-mon[161669]: pgmap v1463: 99 pgs: 1 peering, 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 63 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 200 op/s; 48867/75030 objects degraded (65.130%); 11293/75030 objects misplaced (15.051%); 3.8 MiB/s, 237 objects/s recovering 2023-10-09T06:00:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 48867/75030 objects degraded (65.130%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:00:59.288 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:59.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:59.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:59.361 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:00:59.393 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:59.447 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:59.479 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:00:59.479 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:00:59.479 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:00:59.479 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.182013 2023-10-09T06:00:59.480 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0.18201301825471816 2023-10-09T06:00:59.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2399084174' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:59.536 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:00:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:59.536 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2399084174' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:59.536 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:00:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:59.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2399084174' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:59.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:00:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:00.029 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:01:00.226 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:00.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:00.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:00.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:00.364 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:00.418 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:00.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:00.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:00.525 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:00.555 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:00 smithi008 ceph-mon[184418]: pgmap v1464: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 206 op/s; 48840/73368 objects degraded (66.569%); 11111/73368 objects misplaced (15.144%); 3.2 MiB/s, 200 objects/s recovering 2023-10-09T06:01:00.555 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:00 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:00.555 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:00 smithi008 ceph-mon[184418]: failed to encode map e489 with expected crc 2023-10-09T06:01:00.555 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2399084174' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:01:00.555 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:00 smithi008 ceph-mon[184418]: osdmap e489: 8 total, 8 up, 6 in 2023-10-09T06:01:00.555 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:00 smithi008 ceph-mon[191227]: pgmap v1464: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 206 op/s; 48840/73368 objects degraded (66.569%); 11111/73368 objects misplaced (15.144%); 3.2 MiB/s, 200 objects/s recovering 2023-10-09T06:01:00.556 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:00 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:00.556 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:00 smithi008 ceph-mon[191227]: failed to encode map e489 with expected crc 2023-10-09T06:01:00.556 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2399084174' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:01:00.556 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:00 smithi008 ceph-mon[191227]: osdmap e489: 8 total, 8 up, 6 in 2023-10-09T06:01:00.556 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:01:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:01:00] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.33.4" 2023-10-09T06:01:00.556 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:00 smithi191 ceph-mon[161669]: pgmap v1464: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 206 op/s; 48840/73368 objects degraded (66.569%); 11111/73368 objects misplaced (15.144%); 3.2 MiB/s, 200 objects/s recovering 2023-10-09T06:01:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:00 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:00 smithi191 ceph-mon[161669]: failed to encode map e489 with expected crc 2023-10-09T06:01:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2399084174' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:01:00.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:00 smithi191 ceph-mon[161669]: osdmap e489: 8 total, 8 up, 6 in 2023-10-09T06:01:01.037 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:01.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:01.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:01.141 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:01.173 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:01.244 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:01.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:01.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:01.351 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:01.386 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:01.411 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:01.411 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:01.460 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:01.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:01.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:01.566 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:01.598 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:01.655 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:01.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:01.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:01.761 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:01.772 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:01:01.802 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:01:01.806 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:01.828 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:01:01.866 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:01:01.897 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:01:01.921 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:01:01.952 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:01:01.990 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:01:01.992 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:01:01.992 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:01:01.992 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:01:01.992 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:01:01.992 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:01:01.992 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:01:01.993 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:01:01.993 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:01:02.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:02 smithi008 ceph-mon[184418]: pgmap v1466: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 48386/70836 objects degraded (68.307%); 10875/70836 objects misplaced (15.352%); 5.1 MiB/s, 104 objects/s recovering 2023-10-09T06:01:02.794 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:02 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/253346811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:02.794 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:02 smithi008 ceph-mon[191227]: pgmap v1466: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 48386/70836 objects degraded (68.307%); 10875/70836 objects misplaced (15.352%); 5.1 MiB/s, 104 objects/s recovering 2023-10-09T06:01:02.794 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:02 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/253346811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:02.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:02 smithi191 ceph-mon[161669]: pgmap v1466: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 48386/70836 objects degraded (68.307%); 10875/70836 objects misplaced (15.352%); 5.1 MiB/s, 104 objects/s recovering 2023-10-09T06:01:02.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:02 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/253346811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:03.121 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:01:03.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 48386/70836 objects degraded (68.307%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:03.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 48386/70836 objects degraded (68.307%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:03.760 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:03.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:03.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:03.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 48386/70836 objects degraded (68.307%), 30 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:03.865 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:03.899 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:03.973 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:03.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:03.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:04.083 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:04.115 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:04.177 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:04.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:04.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:04.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:04.315 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:04.364 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:04.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:04.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:04.457 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:04.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:04 smithi008 ceph-mon[184418]: pgmap v1467: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 48308/70431 objects degraded (68.589%); 10797/70431 objects misplaced (15.330%); 4.9 MiB/s, 97 objects/s recovering 2023-10-09T06:01:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:04 smithi008 ceph-mon[191227]: pgmap v1467: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 48308/70431 objects degraded (68.589%); 10797/70431 objects misplaced (15.330%); 4.9 MiB/s, 97 objects/s recovering 2023-10-09T06:01:04.470 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:04.526 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:04.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:04 smithi191 ceph-mon[161669]: pgmap v1467: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 48308/70431 objects degraded (68.589%); 10797/70431 objects misplaced (15.330%); 4.9 MiB/s, 97 objects/s recovering 2023-10-09T06:01:04.693 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:04.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:04.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:04.797 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:04.834 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:05.050 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:05.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:05.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:05.155 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:05.189 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:05.250 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:05.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:05.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:05.312 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:01:05.312 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:01:05.313 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T06:01:05.313 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T06:01:05.313 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:01:05.358 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:05.396 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.1 primary-affinity to 0.182013 (8119282) 2023-10-09T06:01:05.433 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:05.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:05 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1424776078' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.18201301825471816}]: dispatch 2023-10-09T06:01:05.653 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:05 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1424776078' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.18201301825471816}]: dispatch 2023-10-09T06:01:05.654 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:05 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1424776078' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.18201301825471816}]: dispatch 2023-10-09T06:01:06.166 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:01:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[184418]: pgmap v1468: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.4 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 270 op/s; 48359/68919 objects degraded (70.168%); 10702/68919 objects misplaced (15.528%); 4.6 MiB/s, 92 objects/s recovering 2023-10-09T06:01:06.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[184418]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[184418]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1424776078' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.18201301825471816}]': finished 2023-10-09T06:01:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[184418]: osdmap e490: 8 total, 8 up, 6 in 2023-10-09T06:01:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[184418]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[184418]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[184418]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[191227]: pgmap v1468: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.4 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 270 op/s; 48359/68919 objects degraded (70.168%); 10702/68919 objects misplaced (15.528%); 4.6 MiB/s, 92 objects/s recovering 2023-10-09T06:01:06.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[191227]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[191227]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1424776078' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.18201301825471816}]': finished 2023-10-09T06:01:06.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[191227]: osdmap e490: 8 total, 8 up, 6 in 2023-10-09T06:01:06.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[191227]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[191227]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[191227]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:06 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:06 smithi191 ceph-mon[161669]: pgmap v1468: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.4 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 270 op/s; 48359/68919 objects degraded (70.168%); 10702/68919 objects misplaced (15.528%); 4.6 MiB/s, 92 objects/s recovering 2023-10-09T06:01:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:06 smithi191 ceph-mon[161669]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:06 smithi191 ceph-mon[161669]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1424776078' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.18201301825471816}]': finished 2023-10-09T06:01:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:06 smithi191 ceph-mon[161669]: osdmap e490: 8 total, 8 up, 6 in 2023-10-09T06:01:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:06 smithi191 ceph-mon[161669]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:06 smithi191 ceph-mon[161669]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:06 smithi191 ceph-mon[161669]: failed to encode map e490 with expected crc 2023-10-09T06:01:06.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:06 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:07.467 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:01:07.526 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:07.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:07.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:07.633 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:07.673 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[184418]: osdmap e491: 8 total, 8 up, 6 in 2023-10-09T06:01:07.673 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[184418]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.673 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[184418]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.673 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[184418]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.673 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[184418]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.673 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[184418]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.673 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[191227]: osdmap e491: 8 total, 8 up, 6 in 2023-10-09T06:01:07.674 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[191227]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.674 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[191227]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.674 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[191227]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.674 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[191227]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.674 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:07 smithi008 ceph-mon[191227]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.678 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:07.758 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:07.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:07.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:07.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:07 smithi191 ceph-mon[161669]: osdmap e491: 8 total, 8 up, 6 in 2023-10-09T06:01:07.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:07 smithi191 ceph-mon[161669]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:07 smithi191 ceph-mon[161669]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:07 smithi191 ceph-mon[161669]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:07 smithi191 ceph-mon[161669]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:07 smithi191 ceph-mon[161669]: failed to encode map e491 with expected crc 2023-10-09T06:01:07.835 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:07.862 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:07.909 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:08.002 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:08.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:08.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:08.124 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:08.173 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:08.312 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:08.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:08.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:08.474 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:08.483 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:08.653 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:08.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:08.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:08.666 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:08 smithi191 ceph-mon[161669]: pgmap v1471: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 324 op/s; 47899/66183 objects degraded (72.374%); 10465/66183 objects misplaced (15.812%); 6.1 MiB/s, 117 objects/s recovering 2023-10-09T06:01:08.666 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 47899/66183 objects degraded (72.374%), 30 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:08.690 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:08 smithi008 ceph-mon[184418]: pgmap v1471: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 324 op/s; 47899/66183 objects degraded (72.374%); 10465/66183 objects misplaced (15.812%); 6.1 MiB/s, 117 objects/s recovering 2023-10-09T06:01:08.690 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 47899/66183 objects degraded (72.374%), 30 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:08.692 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:08 smithi008 ceph-mon[191227]: pgmap v1471: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 324 op/s; 47899/66183 objects degraded (72.374%); 10465/66183 objects misplaced (15.812%); 6.1 MiB/s, 117 objects/s recovering 2023-10-09T06:01:08.692 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 47899/66183 objects degraded (72.374%), 30 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:08.766 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:08.801 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:08.857 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:08.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:08.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:08.963 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:08.989 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:09.049 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:09.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:09.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:09.153 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:09.190 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:09.415 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:01:09.452 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:09 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/839969854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:09.452 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:09 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:09.453 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:09 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/839969854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:09.453 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:09 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:09.710 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:09.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:09.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:09.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:09 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/839969854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:09 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:09.816 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:09.862 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:10.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: pgmap v1472: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 47881/64401 objects degraded (74.348%); 10285/64401 objects misplaced (15.970%); 3.7 MiB/s, 78 objects/s recovering 2023-10-09T06:01:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:01:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: osdmap e492: 8 total, 8 up, 6 in 2023-10-09T06:01:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:10.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:10.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: pgmap v1472: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 47881/64401 objects degraded (74.348%); 10285/64401 objects misplaced (15.970%); 3.7 MiB/s, 78 objects/s recovering 2023-10-09T06:01:10.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:01:10.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:10.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: osdmap e492: 8 total, 8 up, 6 in 2023-10-09T06:01:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:10.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:10 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:10.718 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:01:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:01:10] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.33.4" 2023-10-09T06:01:10.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: pgmap v1472: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 47881/64401 objects degraded (74.348%); 10285/64401 objects misplaced (15.970%); 3.7 MiB/s, 78 objects/s recovering 2023-10-09T06:01:10.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:01:10.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:10.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: osdmap e492: 8 total, 8 up, 6 in 2023-10-09T06:01:10.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: failed to encode map e492 with expected crc 2023-10-09T06:01:10.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:10.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:10 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:10.896 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:10.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:10.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:11.003 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:11.040 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:11.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:11 smithi008 ceph-mon[184418]: pgmap v1474: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 47447/61851 objects degraded (76.712%); 10067/61851 objects misplaced (16.276%); 7.2 MiB/s, 139 objects/s recovering 2023-10-09T06:01:11.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:11 smithi008 ceph-mon[191227]: pgmap v1474: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 47447/61851 objects degraded (76.712%); 10067/61851 objects misplaced (16.276%); 7.2 MiB/s, 139 objects/s recovering 2023-10-09T06:01:11.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:11 smithi191 ceph-mon[161669]: pgmap v1474: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 47447/61851 objects degraded (76.712%); 10067/61851 objects misplaced (16.276%); 7.2 MiB/s, 139 objects/s recovering 2023-10-09T06:01:11.833 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:11.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:11.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:11.974 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:12.053 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:12.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:12.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:12.162 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:12.187 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:01:12.227 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:01:12.250 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:12.269 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:01:12.303 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:01:12.356 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:01:12.377 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:01:12.377 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:01:12.377 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T06:01:12.377 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T06:01:12.378 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:12.389 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:01:12.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:12.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:12.437 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:01:12.476 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:01:12.477 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:01:12.477 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:01:12.477 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:01:12.477 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:01:12.477 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:01:12.477 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:01:12.477 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:01:12.477 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:01:12.496 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:12.528 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/300379391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:12.529 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/300379391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:12.557 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:12.677 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:01:12.678 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:01:12.700 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:12.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:12.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/300379391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:12.818 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:12.849 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:12.870 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:01:13.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:13 smithi008 ceph-mon[184418]: pgmap v1475: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 254 op/s; 47347/61398 objects degraded (77.115%); 9967/61398 objects misplaced (16.233%); 4.7 MiB/s, 89 objects/s recovering 2023-10-09T06:01:13.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 47447/61851 objects degraded (76.712%), 30 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:13 smithi008 ceph-mon[191227]: pgmap v1475: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 254 op/s; 47347/61398 objects degraded (77.115%); 9967/61398 objects misplaced (16.233%); 4.7 MiB/s, 89 objects/s recovering 2023-10-09T06:01:13.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 47447/61851 objects degraded (76.712%), 30 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:13 smithi191 ceph-mon[161669]: pgmap v1475: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.3 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 254 op/s; 47347/61398 objects degraded (77.115%); 9967/61398 objects misplaced (16.233%); 4.7 MiB/s, 89 objects/s recovering 2023-10-09T06:01:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 47447/61851 objects degraded (76.712%), 30 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:14.170 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:01:14.189 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:14.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:14.303 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:14.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:14.351 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:14.418 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:14.421 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:14.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:14.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:14.533 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:14.578 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:14.664 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:14.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:14.772 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:14.828 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:14.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:14.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:14.933 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:14.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:14.976 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:14.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:14.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:15.087 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:15.118 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:15.227 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:15.238 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:15.238 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:15.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:15.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:15.340 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:15.371 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:15.435 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:15.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:15.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:15.545 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:15.581 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:15.640 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:15.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:15.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:15.747 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:15.791 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:16.111 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:01:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:15 smithi008 ceph-mon[184418]: pgmap v1476: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 47392/59865 objects degraded (79.165%); 9864/59865 objects misplaced (16.477%); 3.5 MiB/s, 77 objects/s recovering 2023-10-09T06:01:16.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:15 smithi008 ceph-mon[191227]: pgmap v1476: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 47392/59865 objects degraded (79.165%); 9864/59865 objects misplaced (16.477%); 3.5 MiB/s, 77 objects/s recovering 2023-10-09T06:01:16.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:15 smithi191 ceph-mon[161669]: pgmap v1476: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 47392/59865 objects degraded (79.165%); 9864/59865 objects misplaced (16.477%); 3.5 MiB/s, 77 objects/s recovering 2023-10-09T06:01:16.911 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:16.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:16.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:17.015 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:17.059 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:17.102 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:17.302 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:17.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:17.412 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:17.467 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:17.619 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:17.779 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:17.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:17.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:17.883 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:17.964 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:18.197 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:18.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:18.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:18.204 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:17 smithi008 ceph-mon[184418]: pgmap v1477: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 46925/56985 objects degraded (82.346%); 9609/56985 objects misplaced (16.862%); 5.0 MiB/s, 103 objects/s recovering 2023-10-09T06:01:18.207 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:17 smithi008 ceph-mon[191227]: pgmap v1477: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 46925/56985 objects degraded (82.346%); 9609/56985 objects misplaced (16.862%); 5.0 MiB/s, 103 objects/s recovering 2023-10-09T06:01:18.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:17 smithi191 ceph-mon[161669]: pgmap v1477: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 46925/56985 objects degraded (82.346%); 9609/56985 objects misplaced (16.862%); 5.0 MiB/s, 103 objects/s recovering 2023-10-09T06:01:18.305 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:18.356 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:18.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:18 smithi008 conmon[184394]: 2023-10-09T06:01:18.212+0000 7f8a66521700 -1 mon.a@0(leader).osd e492 definitely_dead 0 2023-10-09T06:01:18.982 INFO:teuthology.orchestra.run.smithi008.stderr:marked in osd.6. 2023-10-09T06:01:18.982 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:01:19.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3817810962' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:01:19.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:01:19.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 46925/56985 objects degraded (82.346%), 30 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4269914819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:19.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:19.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3817810962' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:01:19.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:01:19.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 46925/56985 objects degraded (82.346%), 30 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:19.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4269914819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:19.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3817810962' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:01:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:01:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 46925/56985 objects degraded (82.346%), 30 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4269914819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:19.945 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:19.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:19.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:20.050 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:20.120 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:20.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[184418]: pgmap v1478: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 312 op/s; 46889/55020 objects degraded (85.222%); 9424/55020 objects misplaced (17.128%); 5.1 MiB/s, 122 objects/s recovering 2023-10-09T06:01:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[184418]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[184418]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:01:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:01:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[184418]: osdmap e493: 8 total, 8 up, 7 in 2023-10-09T06:01:20.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[184418]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[184418]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[184418]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[184418]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[191227]: pgmap v1478: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 312 op/s; 46889/55020 objects degraded (85.222%); 9424/55020 objects misplaced (17.128%); 5.1 MiB/s, 122 objects/s recovering 2023-10-09T06:01:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:20.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[191227]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[191227]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:01:20.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:01:20.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[191227]: osdmap e493: 8 total, 8 up, 7 in 2023-10-09T06:01:20.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[191227]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[191227]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[191227]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:19 smithi008 ceph-mon[191227]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.293 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:20.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:20.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:20.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:19 smithi191 ceph-mon[161669]: pgmap v1478: 99 pgs: 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 1.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 312 op/s; 46889/55020 objects degraded (85.222%); 9424/55020 objects misplaced (17.128%); 5.1 MiB/s, 122 objects/s recovering 2023-10-09T06:01:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:19 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:19 smithi191 ceph-mon[161669]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:19 smithi191 ceph-mon[161669]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:01:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:01:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:19 smithi191 ceph-mon[161669]: osdmap e493: 8 total, 8 up, 7 in 2023-10-09T06:01:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:19 smithi191 ceph-mon[161669]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:19 smithi191 ceph-mon[161669]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:19 smithi191 ceph-mon[161669]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:19 smithi191 ceph-mon[161669]: failed to encode map e493 with expected crc 2023-10-09T06:01:20.398 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:20.423 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:20.470 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:20.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:20.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:20.579 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:20.606 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:01:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:01:20] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.33.4" 2023-10-09T06:01:20.609 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:21.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[184418]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[184418]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[184418]: osdmap e494: 8 total, 8 up, 7 in 2023-10-09T06:01:21.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[184418]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[184418]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[184418]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 492'9508 2023-10-09T06:01:21.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[191227]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[191227]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[191227]: osdmap e494: 8 total, 8 up, 7 in 2023-10-09T06:01:21.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[191227]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[191227]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.221 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:20 smithi008 ceph-mon[191227]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 492'9508 2023-10-09T06:01:21.247 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:21.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:21.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:20 smithi191 ceph-mon[161669]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:20 smithi191 ceph-mon[161669]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:20 smithi191 ceph-mon[161669]: osdmap e494: 8 total, 8 up, 7 in 2023-10-09T06:01:21.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:20 smithi191 ceph-mon[161669]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:20 smithi191 ceph-mon[161669]: failed to encode map e494 with expected crc 2023-10-09T06:01:21.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:20 smithi191 ceph-mon[161669]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 492'9508 2023-10-09T06:01:21.354 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:21.413 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:21.489 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:21.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:21.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:21.599 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:21.640 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:22.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[184418]: pgmap v1481: 99 pgs: 2 remapped+peering, 4 peering, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 60 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 39381/54432 objects degraded (72.349%); 9424/54432 objects misplaced (17.313%); 3.7 MiB/s, 104 objects/s recovering 2023-10-09T06:01:22.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:01:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[184418]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[184418]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[184418]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[184418]: osdmap e495: 8 total, 8 up, 7 in 2023-10-09T06:01:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[184418]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[184418]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[191227]: pgmap v1481: 99 pgs: 2 remapped+peering, 4 peering, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 60 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 39381/54432 objects degraded (72.349%); 9424/54432 objects misplaced (17.313%); 3.7 MiB/s, 104 objects/s recovering 2023-10-09T06:01:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:01:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[191227]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[191227]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[191227]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[191227]: osdmap e495: 8 total, 8 up, 7 in 2023-10-09T06:01:22.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[191227]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:21 smithi008 ceph-mon[191227]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.295 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:01:22.295 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:01:22.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:21 smithi191 ceph-mon[161669]: pgmap v1481: 99 pgs: 2 remapped+peering, 4 peering, 10 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 60 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 39381/54432 objects degraded (72.349%); 9424/54432 objects misplaced (17.313%); 3.7 MiB/s, 104 objects/s recovering 2023-10-09T06:01:22.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:21 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:01:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:21 smithi191 ceph-mon[161669]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:21 smithi191 ceph-mon[161669]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:21 smithi191 ceph-mon[161669]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:21 smithi191 ceph-mon[161669]: osdmap e495: 8 total, 8 up, 7 in 2023-10-09T06:01:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:21 smithi191 ceph-mon[161669]: failed to encode map e495 with expected crc 2023-10-09T06:01:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:21 smithi191 ceph-mon[161669]: failed to encode map e495 with expected crc 2023-10-09T06:01:23.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:22 smithi008 ceph-mon[184418]: failed to encode map e495 with expected crc 2023-10-09T06:01:23.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:22 smithi008 ceph-mon[184418]: 9.d starting backfill to osd.6 from (369'1961,446'7325] MIN to 492'9382 2023-10-09T06:01:23.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:22 smithi008 ceph-mon[191227]: failed to encode map e495 with expected crc 2023-10-09T06:01:23.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:22 smithi008 ceph-mon[191227]: 9.d starting backfill to osd.6 from (369'1961,446'7325] MIN to 492'9382 2023-10-09T06:01:23.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:22 smithi191 ceph-mon[161669]: failed to encode map e495 with expected crc 2023-10-09T06:01:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:22 smithi191 ceph-mon[161669]: 9.d starting backfill to osd.6 from (369'1961,446'7325] MIN to 492'9382 2023-10-09T06:01:23.509 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:23.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:23.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:23.592 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:01:23.617 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:23.702 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:23.917 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T06:01:24.080 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:24.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:24.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:24.179 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:23 smithi008 ceph-mon[184418]: pgmap v1483: 99 pgs: 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 activating, 2 remapped+peering, 6 peering, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 44518/53013 objects degraded (83.976%); 4350/53013 objects misplaced (8.206%); 3.2 MiB/s, 99 objects/s recovering 2023-10-09T06:01:24.179 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 44518/53013 objects degraded (83.976%), 21 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:24.180 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:23 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:24.180 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:23 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:24.180 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:23 smithi008 ceph-mon[191227]: pgmap v1483: 99 pgs: 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 activating, 2 remapped+peering, 6 peering, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 44518/53013 objects degraded (83.976%); 4350/53013 objects misplaced (8.206%); 3.2 MiB/s, 99 objects/s recovering 2023-10-09T06:01:24.180 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 44518/53013 objects degraded (83.976%), 21 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:24.180 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:23 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:24.180 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:23 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:24.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:24.199 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:01:24.212 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:01:24.218 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:24.237 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:01:24.280 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:01:24.292 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:24.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:23 smithi191 ceph-mon[161669]: pgmap v1483: 99 pgs: 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 activating, 2 remapped+peering, 6 peering, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 44518/53013 objects degraded (83.976%); 4350/53013 objects misplaced (8.206%); 3.2 MiB/s, 99 objects/s recovering 2023-10-09T06:01:24.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 44518/53013 objects degraded (83.976%), 21 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:24.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:23 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:24.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:23 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:24.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:24.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:24.328 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:01:24.378 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:01:24.396 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:24.404 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:24.422 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:01:24.467 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:24.469 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:01:24.512 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:01:24.529 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:01:24.529 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:01:24.529 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:01:24.529 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:01:24.530 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:01:24.530 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:01:24.530 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:01:24.530 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:01:24.556 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:24.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:24.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:24.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:24.721 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:24.817 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:24.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:24.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:24.928 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:24.998 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:25.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:24 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2175148106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:25.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:24 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:25.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:24 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:25.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:24 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:25.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:24 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2175148106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:25.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:24 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:25.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:24 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:25.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:24 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:25.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:24 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2175148106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:25.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:24 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:25.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:24 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:25.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:24 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:25.940 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:25.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:25.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:26.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:26.088 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:26.140 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:25 smithi008 ceph-mon[191227]: pgmap v1484: 99 pgs: 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 activating, 2 remapped+peering, 3 peering, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 44523/52593 objects degraded (84.656%); 4324/52593 objects misplaced (8.222%); 2.2 MiB/s, 99 objects/s recovering 2023-10-09T06:01:26.141 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:25 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:01:26.141 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:25 smithi008 ceph-mon[184418]: pgmap v1484: 99 pgs: 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 activating, 2 remapped+peering, 3 peering, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 44523/52593 objects degraded (84.656%); 4324/52593 objects misplaced (8.222%); 2.2 MiB/s, 99 objects/s recovering 2023-10-09T06:01:26.141 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:25 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:01:26.147 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:26.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:26.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:26.252 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:26.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:25 smithi191 ceph-mon[161669]: pgmap v1484: 99 pgs: 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 activating, 2 remapped+peering, 3 peering, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean; 1.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 44523/52593 objects degraded (84.656%); 4324/52593 objects misplaced (8.222%); 2.2 MiB/s, 99 objects/s recovering 2023-10-09T06:01:26.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:25 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:01:26.301 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:26.384 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:26.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:26.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:26.490 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:26.533 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:26.602 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:26.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:26.707 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:26.740 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:26.785 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:26.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:26.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:26.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:26.933 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:27.008 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:27.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:27.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:27.116 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:27.157 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:27.320 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:27.345 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:27.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:27.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:27.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:27.495 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:27.564 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:27.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:27.675 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:27.714 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:27.787 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:27.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:27.904 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:27.961 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:28.029 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:01:28.036 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:28.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:28.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:28.144 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:28.173 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:27 smithi191 ceph-mon[161669]: pgmap v1485: 99 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 50420/50331 objects degraded (100.177%); 4096/50331 objects misplaced (8.138%); 4.3 MiB/s, 131 objects/s recovering 2023-10-09T06:01:28.175 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:28.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:27 smithi008 ceph-mon[184418]: pgmap v1485: 99 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 50420/50331 objects degraded (100.177%); 4096/50331 objects misplaced (8.138%); 4.3 MiB/s, 131 objects/s recovering 2023-10-09T06:01:28.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:27 smithi008 ceph-mon[191227]: pgmap v1485: 99 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 200 op/s; 50420/50331 objects degraded (100.177%); 4096/50331 objects misplaced (8.138%); 4.3 MiB/s, 131 objects/s recovering 2023-10-09T06:01:28.249 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:28.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:28.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:28.359 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:28.431 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:28.918 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:01:28.918 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:01:28.919 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-10-09T06:01:28.919 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:01:29.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 50420/50331 objects degraded (100.177%), 25 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:29.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1465007276' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:29.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:28 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:29.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:28 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:29.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 50420/50331 objects degraded (100.177%), 25 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:29.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1465007276' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:29.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:28 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:29.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:28 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:29.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 50420/50331 objects degraded (100.177%), 25 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:29.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1465007276' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:29.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:28 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:01:29.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:28 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:29.337 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:01:29.648 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:29.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:29.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:29.757 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:29.797 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:29 smithi191 ceph-mon[161669]: pgmap v1486: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1003 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 50355/48000 objects degraded (104.906%); 3987/48000 objects misplaced (8.306%); 4.6 MiB/s, 187 objects/s recovering 2023-10-09T06:01:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:29 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1465007276' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:29 smithi191 ceph-mon[161669]: osdmap e496: 8 total, 8 up, 7 in 2023-10-09T06:01:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:29 smithi191 ceph-mon[161669]: failed to encode map e496 with expected crc 2023-10-09T06:01:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:29 smithi191 ceph-mon[161669]: failed to encode map e496 with expected crc 2023-10-09T06:01:30.401 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[184418]: pgmap v1486: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1003 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 50355/48000 objects degraded (104.906%); 3987/48000 objects misplaced (8.306%); 4.6 MiB/s, 187 objects/s recovering 2023-10-09T06:01:30.401 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:30.401 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1465007276' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:30.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[184418]: osdmap e496: 8 total, 8 up, 7 in 2023-10-09T06:01:30.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[184418]: failed to encode map e496 with expected crc 2023-10-09T06:01:30.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[184418]: failed to encode map e496 with expected crc 2023-10-09T06:01:30.404 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[191227]: pgmap v1486: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1003 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 50355/48000 objects degraded (104.906%); 3987/48000 objects misplaced (8.306%); 4.6 MiB/s, 187 objects/s recovering 2023-10-09T06:01:30.404 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:30.404 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1465007276' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:30.404 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[191227]: osdmap e496: 8 total, 8 up, 7 in 2023-10-09T06:01:30.404 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[191227]: failed to encode map e496 with expected crc 2023-10-09T06:01:30.404 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:29 smithi008 ceph-mon[191227]: failed to encode map e496 with expected crc 2023-10-09T06:01:30.421 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:30.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:30.526 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:30.551 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:30.583 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:30.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:01:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:01:30] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.33.4" 2023-10-09T06:01:30.882 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:30.971 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:30.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:30.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:31.029 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:01:31.029 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:01:31.029 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T06:01:31.030 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T06:01:31.030 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:01:31.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:31.129 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:31.228 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:31.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:31.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:31.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:31 smithi191 ceph-mon[161669]: failed to encode map e496 with expected crc 2023-10-09T06:01:31.308 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:31 smithi008 ceph-mon[184418]: failed to encode map e496 with expected crc 2023-10-09T06:01:31.308 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:31 smithi008 ceph-mon[191227]: failed to encode map e496 with expected crc 2023-10-09T06:01:31.341 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:31.387 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:31.446 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:31.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:31.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:31.555 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:31.584 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:31.639 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:31.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:31.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:31.747 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:31.784 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:32.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:32 smithi191 ceph-mon[161669]: pgmap v1488: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 975 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 50074/46668 objects degraded (107.298%); 3913/46668 objects misplaced (8.385%); 4.5 MiB/s, 166 objects/s recovering 2023-10-09T06:01:32.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:32 smithi008 ceph-mon[184418]: pgmap v1488: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 975 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 50074/46668 objects degraded (107.298%); 3913/46668 objects misplaced (8.385%); 4.5 MiB/s, 166 objects/s recovering 2023-10-09T06:01:32.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:32 smithi008 ceph-mon[191227]: pgmap v1488: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 975 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 50074/46668 objects degraded (107.298%); 3913/46668 objects misplaced (8.385%); 4.5 MiB/s, 166 objects/s recovering 2023-10-09T06:01:32.438 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:01:32.629 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:32.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:32.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:32.735 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:32.787 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:33.263 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:01:33.640 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:33.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:33.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:33.748 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:33.807 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:33.919 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:33.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:34.028 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:34.070 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:34.133 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:34.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:34.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:34.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:34 smithi008 ceph-mon[184418]: pgmap v1489: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 927 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 49654/44355 objects degraded (111.947%); 3801/44355 objects misplaced (8.569%); 6.1 MiB/s, 193 objects/s recovering 2023-10-09T06:01:34.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:34 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 49654/44355 objects degraded (111.947%), 25 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:34 smithi008 ceph-mon[191227]: pgmap v1489: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 927 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 49654/44355 objects degraded (111.947%); 3801/44355 objects misplaced (8.569%); 6.1 MiB/s, 193 objects/s recovering 2023-10-09T06:01:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:34 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 49654/44355 objects degraded (111.947%), 25 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:34.239 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:34.268 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:34 smithi191 ceph-mon[161669]: pgmap v1489: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 927 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 49654/44355 objects degraded (111.947%); 3801/44355 objects misplaced (8.569%); 6.1 MiB/s, 193 objects/s recovering 2023-10-09T06:01:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:34 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 49654/44355 objects degraded (111.947%), 25 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:34.338 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:34.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:34.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:34.486 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:34.547 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:34.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:34.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:34.657 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:34.681 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:01:34.682 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T06:01:34.683 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:34.725 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:34.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:34.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:34.832 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:34.874 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:35.753 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:35.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:35.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:35.819 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:35.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:35.908 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:35.980 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:35.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:35.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:36.024 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:36 smithi008 ceph-mon[184418]: from='client.36987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:01:36.024 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:36 smithi008 ceph-mon[184418]: pgmap v1490: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 893 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 328 op/s; 49660/42726 objects degraded (116.229%); 3250/42726 objects misplaced (7.607%); 9.2 MiB/s, 232 objects/s recovering 2023-10-09T06:01:36.089 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:36.093 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1489,"stamp":"2023-10-09T06:01:32.738767+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":971839435,"num_objects":14785,"num_object_clones":0,"num_object_copies":44355,"num_objects_missing_on_primary":4861,"num_objects_missing":4861,"num_objects_degraded":49654,"num_objects_misplaced":3801,"num_objects_unfound":0,"num_objects_dirty":14785,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":154691,"num_write_kb":5402022,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53007,"num_bytes_recovered":2858369573,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":152683,"ondisk_log_size":152683,"up":297,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":409,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10498120,"kb_used_data":4169104,"kb_used_omap":45,"kb_used_meta":6328914,"kb_avail":645717944,"statfs":{"total":671965249536,"available":661215174656,"internally_reserved":0,"allocated":4269162496,"data_stored":4257817821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":6480808227},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,3,2,0,2,1,1,6,5],"upper_bound":1024},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-189136896,"num_objects":-2886,"num_object_clones":0,"num_object_copies":-8658,"num_objects_missing_on_primary":-2901,"num_objects_missing":-2901,"num_objects_degraded":5136,"num_objects_misplaced":-549,"num_objects_unfound":0,"num_objects_dirty":-2886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2886,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1935,"num_bytes_recovered":64421888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003713"},"pg_stats":[{"pgid":"9.10","version":"496'7141","reported_seq":10560,"reported_epoch":496,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:30.522637+0000","last_change":"2023-10-09T06:01:21.220649+0000","last_active":"2023-10-09T06:01:30.522637+0000","last_peered":"2023-10-09T06:01:30.522637+0000","last_clean":"2023-10-09T05:56:48.221888+0000","last_became_active":"2023-10-09T06:01:21.168993+0000","last_became_peered":"2023-10-09T06:01:21.168993+0000","last_unstale":"2023-10-09T06:01:30.522637+0000","last_undegraded":"2023-10-09T06:01:21.011813+0000","last_fullsized":"2023-10-09T06:01:20.968896+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7141,"log_dups_size":0,"ondisk_log_size":7141,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28639232,"num_objects":437,"num_object_clones":0,"num_object_copies":1311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3039,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4751,"num_write_kb":166016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1102,"num_bytes_recovered":72220672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,0],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"0,2","objects":-2},{"shards":"1,2,6","objects":439}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"59'1","reported_seq":118,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.969574+0000","last_change":"2023-10-09T05:54:59.256087+0000","last_active":"2023-10-09T06:01:20.969574+0000","last_peered":"2023-10-09T06:01:20.969574+0000","last_clean":"2023-10-09T06:01:20.969574+0000","last_became_active":"2023-10-09T05:54:59.255773+0000","last_became_peered":"2023-10-09T05:54:59.255773+0000","last_unstale":"2023-10-09T06:01:20.969574+0000","last_undegraded":"2023-10-09T06:01:20.969574+0000","last_fullsized":"2023-10-09T06:01:20.969574+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:19.545754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":93,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.966075+0000","last_change":"2023-10-09T05:56:00.247579+0000","last_active":"2023-10-09T06:01:20.966075+0000","last_peered":"2023-10-09T06:01:20.966075+0000","last_clean":"2023-10-09T06:01:20.966075+0000","last_became_active":"2023-10-09T05:56:00.238151+0000","last_became_peered":"2023-10-09T05:56:00.238151+0000","last_unstale":"2023-10-09T06:01:20.966075+0000","last_undegraded":"2023-10-09T06:01:20.966075+0000","last_fullsized":"2023-10-09T06:01:20.966075+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:05.692190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"495'7160","reported_seq":13048,"reported_epoch":495,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:01:28.440045+0000","last_change":"2023-10-09T05:56:01.319470+0000","last_active":"2023-10-09T06:01:28.440045+0000","last_peered":"2023-10-09T06:01:28.440045+0000","last_clean":"2023-10-09T05:55:24.768851+0000","last_became_active":"2023-10-09T05:56:01.311996+0000","last_became_peered":"2023-10-09T05:56:01.311996+0000","last_unstale":"2023-10-09T06:01:28.440045+0000","last_undegraded":"2023-10-09T06:01:28.440045+0000","last_fullsized":"2023-10-09T06:01:28.440045+0000","mapping_epoch":416,"log_start":"369'1544","ondisk_log_start":"369'1544","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5616,"log_dups_size":0,"ondisk_log_size":5616,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33751040,"num_objects":515,"num_object_clones":0,"num_object_copies":1545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":515,"num_objects_unfound":0,"num_objects_dirty":515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4747,"num_write_kb":168384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1422,"num_bytes_recovered":93192192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":515}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":198,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:01:19.978622+0000","last_change":"2023-10-09T05:56:00.243822+0000","last_active":"2023-10-09T06:01:19.978622+0000","last_peered":"2023-10-09T06:01:19.978622+0000","last_clean":"2023-10-09T06:01:19.978622+0000","last_became_active":"2023-10-09T05:56:00.243483+0000","last_became_peered":"2023-10-09T05:56:00.243483+0000","last_unstale":"2023-10-09T06:01:19.978622+0000","last_undegraded":"2023-10-09T06:01:19.978622+0000","last_fullsized":"2023-10-09T06:01:19.978622+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:39.811952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":505,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.092790+0000","last_change":"2023-10-09T06:00:05.354195+0000","last_active":"2023-10-09T06:01:21.092790+0000","last_peered":"2023-10-09T06:01:21.092790+0000","last_clean":"2023-10-09T06:01:21.092790+0000","last_became_active":"2023-10-09T06:00:05.353827+0000","last_became_peered":"2023-10-09T06:00:05.353827+0000","last_unstale":"2023-10-09T06:01:21.092790+0000","last_undegraded":"2023-10-09T06:01:21.092790+0000","last_fullsized":"2023-10-09T06:01:21.092790+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:44.449493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.12","version":"496'7265","reported_seq":12836,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:01:30.461134+0000","last_change":"2023-10-09T05:58:12.409601+0000","last_active":"2023-10-09T06:01:30.461134+0000","last_peered":"2023-10-09T06:01:30.461134+0000","last_clean":"2023-10-09T06:01:30.461134+0000","last_became_active":"2023-10-09T05:58:12.403280+0000","last_became_peered":"2023-10-09T05:58:12.403280+0000","last_unstale":"2023-10-09T06:01:30.461134+0000","last_undegraded":"2023-10-09T06:01:30.461134+0000","last_fullsized":"2023-10-09T06:01:30.461134+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7265,"log_dups_size":0,"ondisk_log_size":7265,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30375936,"num_objects":464,"num_object_clones":0,"num_object_copies":1392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4844,"num_write_kb":169824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2689,"num_bytes_recovered":106627072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":440,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.089906+0000","last_change":"2023-10-09T06:00:05.620750+0000","last_active":"2023-10-09T06:01:21.089906+0000","last_peered":"2023-10-09T06:01:21.089906+0000","last_clean":"2023-10-09T06:01:21.089906+0000","last_became_active":"2023-10-09T06:00:05.620298+0000","last_became_peered":"2023-10-09T06:00:05.620298+0000","last_unstale":"2023-10-09T06:01:21.089906+0000","last_undegraded":"2023-10-09T06:01:21.089906+0000","last_fullsized":"2023-10-09T06:01:21.089906+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:29.560982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":40,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.086226+0000","last_change":"2023-10-09T06:01:19.996464+0000","last_active":"2023-10-09T06:01:21.086226+0000","last_peered":"2023-10-09T06:01:21.086226+0000","last_clean":"2023-10-09T06:01:21.086226+0000","last_became_active":"2023-10-09T06:01:19.995823+0000","last_became_peered":"2023-10-09T06:01:19.995823+0000","last_unstale":"2023-10-09T06:01:21.086226+0000","last_undegraded":"2023-10-09T06:01:21.086226+0000","last_fullsized":"2023-10-09T06:01:21.086226+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:17:13.034516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":435,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.086418+0000","last_change":"2023-10-09T06:01:19.980073+0000","last_active":"2023-10-09T06:01:21.086418+0000","last_peered":"2023-10-09T06:01:21.086418+0000","last_clean":"2023-10-09T06:01:21.086418+0000","last_became_active":"2023-10-09T06:01:19.979716+0000","last_became_peered":"2023-10-09T06:01:19.979716+0000","last_unstale":"2023-10-09T06:01:21.086418+0000","last_undegraded":"2023-10-09T06:01:21.086418+0000","last_fullsized":"2023-10-09T06:01:21.086418+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:56.881360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":111,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.965599+0000","last_change":"2023-10-09T06:01:19.995701+0000","last_active":"2023-10-09T06:01:20.965599+0000","last_peered":"2023-10-09T06:01:20.965599+0000","last_clean":"2023-10-09T06:01:20.965599+0000","last_became_active":"2023-10-09T06:01:19.995221+0000","last_became_peered":"2023-10-09T06:01:19.995221+0000","last_unstale":"2023-10-09T06:01:20.965599+0000","last_undegraded":"2023-10-09T06:01:20.965599+0000","last_fullsized":"2023-10-09T06:01:20.965599+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:28.118628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":153,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.988012+0000","last_change":"2023-10-09T06:00:05.346651+0000","last_active":"2023-10-09T06:01:20.988012+0000","last_peered":"2023-10-09T06:01:20.988012+0000","last_clean":"2023-10-09T06:01:20.988012+0000","last_became_active":"2023-10-09T06:00:05.346292+0000","last_became_peered":"2023-10-09T06:00:05.346292+0000","last_unstale":"2023-10-09T06:01:20.988012+0000","last_undegraded":"2023-10-09T06:01:20.988012+0000","last_fullsized":"2023-10-09T06:01:20.988012+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":147,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:22.567619+0000","last_change":"2023-10-09T06:01:22.567256+0000","last_active":"2023-10-09T06:01:22.567619+0000","last_peered":"2023-10-09T06:01:22.567619+0000","last_clean":"2023-10-09T06:01:22.567619+0000","last_became_active":"2023-10-09T06:01:20.846276+0000","last_became_peered":"2023-10-09T06:01:20.846276+0000","last_unstale":"2023-10-09T06:01:22.567619+0000","last_undegraded":"2023-10-09T06:01:22.567619+0000","last_fullsized":"2023-10-09T06:01:22.567619+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":128,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.980983+0000","last_change":"2023-10-09T05:56:00.348771+0000","last_active":"2023-10-09T06:01:20.980983+0000","last_peered":"2023-10-09T06:01:20.980983+0000","last_clean":"2023-10-09T06:01:20.980983+0000","last_became_active":"2023-10-09T05:56:00.224347+0000","last_became_peered":"2023-10-09T05:56:00.224347+0000","last_unstale":"2023-10-09T06:01:20.980983+0000","last_undegraded":"2023-10-09T06:01:20.980983+0000","last_fullsized":"2023-10-09T06:01:20.980983+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":285,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.970880+0000","last_change":"2023-10-09T06:01:19.992812+0000","last_active":"2023-10-09T06:01:20.970880+0000","last_peered":"2023-10-09T06:01:20.970880+0000","last_clean":"2023-10-09T06:01:20.970880+0000","last_became_active":"2023-10-09T06:01:19.992374+0000","last_became_peered":"2023-10-09T06:01:19.992374+0000","last_unstale":"2023-10-09T06:01:20.970880+0000","last_undegraded":"2023-10-09T06:01:20.970880+0000","last_fullsized":"2023-10-09T06:01:20.970880+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:43.064961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":130,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.980606+0000","last_change":"2023-10-09T06:01:19.984380+0000","last_active":"2023-10-09T06:01:20.980606+0000","last_peered":"2023-10-09T06:01:20.980606+0000","last_clean":"2023-10-09T06:01:20.980606+0000","last_became_active":"2023-10-09T06:01:19.983167+0000","last_became_peered":"2023-10-09T06:01:19.983167+0000","last_unstale":"2023-10-09T06:01:20.980606+0000","last_undegraded":"2023-10-09T06:01:20.980606+0000","last_fullsized":"2023-10-09T06:01:20.980606+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":490,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.968892+0000","last_change":"2023-10-09T06:01:19.998578+0000","last_active":"2023-10-09T06:01:20.968892+0000","last_peered":"2023-10-09T06:01:20.968892+0000","last_clean":"2023-10-09T06:01:20.968892+0000","last_became_active":"2023-10-09T06:01:19.993114+0000","last_became_peered":"2023-10-09T06:01:19.993114+0000","last_unstale":"2023-10-09T06:01:20.968892+0000","last_undegraded":"2023-10-09T06:01:20.968892+0000","last_fullsized":"2023-10-09T06:01:20.968892+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:12.898507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.19","version":"496'7196","reported_seq":17446,"reported_epoch":496,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:32.446877+0000","last_change":"2023-10-09T06:01:23.298139+0000","last_active":"2023-10-09T06:01:32.446877+0000","last_peered":"2023-10-09T06:01:32.446877+0000","last_clean":"2023-10-09T06:01:18.849705+0000","last_became_active":"2023-10-09T06:01:23.173316+0000","last_became_peered":"2023-10-09T06:01:23.173316+0000","last_unstale":"2023-10-09T06:01:32.446877+0000","last_undegraded":"2023-10-09T06:01:21.114364+0000","last_fullsized":"2023-10-09T06:01:20.967987+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7196,"log_dups_size":0,"ondisk_log_size":7196,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29982720,"num_objects":458,"num_object_clones":0,"num_object_copies":1374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2628,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4790,"num_write_kb":167904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3673,"num_bytes_recovered":136937472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7],"avail_no_missing":["0","3","4","5","7"],"object_location_counts":[{"shards":"0,4,7","objects":462},{"shards":"0,7","objects":-114}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":110,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.978981+0000","last_change":"2023-10-09T05:55:27.092744+0000","last_active":"2023-10-09T06:01:20.978981+0000","last_peered":"2023-10-09T06:01:20.978981+0000","last_clean":"2023-10-09T06:01:20.978981+0000","last_became_active":"2023-10-09T05:55:26.271096+0000","last_became_peered":"2023-10-09T05:55:26.271096+0000","last_unstale":"2023-10-09T06:01:20.978981+0000","last_undegraded":"2023-10-09T06:01:20.978981+0000","last_fullsized":"2023-10-09T06:01:20.978981+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":131,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.980519+0000","last_change":"2023-10-09T05:56:49.266384+0000","last_active":"2023-10-09T06:01:20.980519+0000","last_peered":"2023-10-09T06:01:20.980519+0000","last_clean":"2023-10-09T06:01:20.980519+0000","last_became_active":"2023-10-09T05:56:49.266051+0000","last_became_peered":"2023-10-09T05:56:49.266051+0000","last_unstale":"2023-10-09T06:01:20.980519+0000","last_undegraded":"2023-10-09T06:01:20.980519+0000","last_fullsized":"2023-10-09T06:01:20.980519+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"495'7193","reported_seq":18051,"reported_epoch":495,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:28.107586+0000","last_change":"2023-10-09T06:00:06.395295+0000","last_active":"2023-10-09T06:01:28.107586+0000","last_peered":"2023-10-09T06:01:28.107586+0000","last_clean":"2023-10-09T06:00:03.898278+0000","last_became_active":"2023-10-09T06:00:06.375058+0000","last_became_peered":"2023-10-09T06:00:06.375058+0000","last_unstale":"2023-10-09T06:01:28.107586+0000","last_undegraded":"2023-10-09T06:00:06.297529+0000","last_fullsized":"2023-10-09T06:00:06.282061+0000","mapping_epoch":476,"log_start":"369'1694","ondisk_log_start":"369'1694","created":385,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5499,"log_dups_size":0,"ondisk_log_size":5499,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29097984,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":993,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4780,"num_write_kb":167168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2787,"num_bytes_recovered":134348800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[1,7],"avail_no_missing":["1","2","3","5","7"],"object_location_counts":[{"shards":"1,7","objects":444}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":478,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.088652+0000","last_change":"2023-10-09T06:00:05.355525+0000","last_active":"2023-10-09T06:01:21.088652+0000","last_peered":"2023-10-09T06:01:21.088652+0000","last_clean":"2023-10-09T06:01:21.088652+0000","last_became_active":"2023-10-09T06:00:05.354517+0000","last_became_peered":"2023-10-09T06:00:05.354517+0000","last_unstale":"2023-10-09T06:01:21.088652+0000","last_undegraded":"2023-10-09T06:01:21.088652+0000","last_fullsized":"2023-10-09T06:01:21.088652+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:44.826046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":199,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.981070+0000","last_change":"2023-10-09T06:01:20.027680+0000","last_active":"2023-10-09T06:01:20.981070+0000","last_peered":"2023-10-09T06:01:20.981070+0000","last_clean":"2023-10-09T06:01:20.981070+0000","last_became_active":"2023-10-09T06:01:20.000504+0000","last_became_peered":"2023-10-09T06:01:20.000504+0000","last_unstale":"2023-10-09T06:01:20.981070+0000","last_undegraded":"2023-10-09T06:01:20.981070+0000","last_fullsized":"2023-10-09T06:01:20.981070+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":129,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:22.567378+0000","last_change":"2023-10-09T06:01:22.566883+0000","last_active":"2023-10-09T06:01:22.567378+0000","last_peered":"2023-10-09T06:01:22.567378+0000","last_clean":"2023-10-09T06:01:22.567378+0000","last_became_active":"2023-10-09T06:01:20.845907+0000","last_became_peered":"2023-10-09T06:01:20.845907+0000","last_unstale":"2023-10-09T06:01:22.567378+0000","last_undegraded":"2023-10-09T06:01:22.567378+0000","last_fullsized":"2023-10-09T06:01:22.567378+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":475,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.092879+0000","last_change":"2023-10-09T06:00:05.322793+0000","last_active":"2023-10-09T06:01:21.092879+0000","last_peered":"2023-10-09T06:01:21.092879+0000","last_clean":"2023-10-09T06:01:21.092879+0000","last_became_active":"2023-10-09T06:00:05.322462+0000","last_became_peered":"2023-10-09T06:00:05.322462+0000","last_unstale":"2023-10-09T06:01:21.092879+0000","last_undegraded":"2023-10-09T06:01:21.092879+0000","last_fullsized":"2023-10-09T06:01:21.092879+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:11.349003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":274,"reported_epoch":495,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:01:21.089687+0000","last_change":"2023-10-09T06:01:19.993878+0000","last_active":"2023-10-09T06:01:21.089687+0000","last_peered":"2023-10-09T06:01:21.089687+0000","last_clean":"2023-10-09T06:01:09.405747+0000","last_became_active":"2023-10-09T06:01:19.993580+0000","last_became_peered":"2023-10-09T06:01:19.993580+0000","last_unstale":"2023-10-09T06:01:21.089687+0000","last_undegraded":"2023-10-09T06:01:19.971089+0000","last_fullsized":"2023-10-09T06:01:21.089687+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:39.307684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":130,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.980998+0000","last_change":"2023-10-09T05:56:49.263088+0000","last_active":"2023-10-09T06:01:20.980998+0000","last_peered":"2023-10-09T06:01:20.980998+0000","last_clean":"2023-10-09T06:01:20.980998+0000","last_became_active":"2023-10-09T05:56:49.262781+0000","last_became_peered":"2023-10-09T05:56:49.262781+0000","last_unstale":"2023-10-09T06:01:20.980998+0000","last_undegraded":"2023-10-09T06:01:20.980998+0000","last_fullsized":"2023-10-09T06:01:20.980998+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"495'7215","reported_seq":9691,"reported_epoch":495,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:28.389923+0000","last_change":"2023-10-09T06:00:17.809415+0000","last_active":"2023-10-09T06:01:28.389923+0000","last_peered":"2023-10-09T06:01:28.389923+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T06:00:17.617011+0000","last_became_peered":"2023-10-09T06:00:17.617011+0000","last_unstale":"2023-10-09T06:01:28.389923+0000","last_undegraded":"2023-10-09T06:00:12.226744+0000","last_fullsized":"2023-10-09T06:00:07.517595+0000","mapping_epoch":477,"log_start":"369'1640","ondisk_log_start":"369'1640","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32604160,"num_objects":497,"num_object_clones":0,"num_object_copies":1491,"num_objects_missing_on_primary":888,"num_objects_missing":888,"num_objects_degraded":3338,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4773,"num_write_kb":168672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":774,"num_bytes_recovered":50724864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[1,0],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,1","objects":-467},{"shards":"0,1,5","objects":482}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":137,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.981091+0000","last_change":"2023-10-09T06:00:05.328315+0000","last_active":"2023-10-09T06:01:20.981091+0000","last_peered":"2023-10-09T06:01:20.981091+0000","last_clean":"2023-10-09T06:01:20.981091+0000","last_became_active":"2023-10-09T06:00:05.327973+0000","last_became_peered":"2023-10-09T06:00:05.327973+0000","last_unstale":"2023-10-09T06:01:20.981091+0000","last_undegraded":"2023-10-09T06:01:20.981091+0000","last_fullsized":"2023-10-09T06:01:20.981091+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":168,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.976928+0000","last_change":"2023-10-09T06:01:19.989058+0000","last_active":"2023-10-09T06:01:20.976928+0000","last_peered":"2023-10-09T06:01:20.976928+0000","last_clean":"2023-10-09T06:01:20.976928+0000","last_became_active":"2023-10-09T06:01:19.988642+0000","last_became_peered":"2023-10-09T06:01:19.988642+0000","last_unstale":"2023-10-09T06:01:20.976928+0000","last_undegraded":"2023-10-09T06:01:20.976928+0000","last_fullsized":"2023-10-09T06:01:20.976928+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"495'7233","reported_seq":7408,"reported_epoch":495,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:01:28.418333+0000","last_change":"2023-10-09T05:56:48.413650+0000","last_active":"2023-10-09T06:01:28.418333+0000","last_peered":"2023-10-09T06:01:28.418333+0000","last_clean":"2023-10-09T05:55:48.934818+0000","last_became_active":"2023-10-09T05:56:01.246051+0000","last_became_peered":"2023-10-09T05:56:01.246051+0000","last_unstale":"2023-10-09T06:01:28.418333+0000","last_undegraded":"2023-10-09T05:56:48.413475+0000","last_fullsized":"2023-10-09T06:01:28.418333+0000","mapping_epoch":416,"log_start":"369'1685","ondisk_log_start":"369'1685","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31391744,"num_objects":479,"num_object_clones":0,"num_object_copies":1437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4803,"num_write_kb":169024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":479}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":108,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.987925+0000","last_change":"2023-10-09T05:55:26.358519+0000","last_active":"2023-10-09T06:01:20.987925+0000","last_peered":"2023-10-09T06:01:20.987925+0000","last_clean":"2023-10-09T06:01:20.987925+0000","last_became_active":"2023-10-09T05:55:26.260138+0000","last_became_peered":"2023-10-09T05:55:26.260138+0000","last_unstale":"2023-10-09T06:01:20.987925+0000","last_undegraded":"2023-10-09T06:01:20.987925+0000","last_fullsized":"2023-10-09T06:01:20.987925+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":494,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.975007+0000","last_change":"2023-10-09T06:01:19.995107+0000","last_active":"2023-10-09T06:01:20.975007+0000","last_peered":"2023-10-09T06:01:20.975007+0000","last_clean":"2023-10-09T06:01:20.975007+0000","last_became_active":"2023-10-09T06:01:19.994466+0000","last_became_peered":"2023-10-09T06:01:19.994466+0000","last_unstale":"2023-10-09T06:01:20.975007+0000","last_undegraded":"2023-10-09T06:01:20.975007+0000","last_fullsized":"2023-10-09T06:01:20.975007+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"496'7155","reported_seq":11071,"reported_epoch":496,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:30.420897+0000","last_change":"2023-10-09T05:58:02.333623+0000","last_active":"2023-10-09T06:01:30.420897+0000","last_peered":"2023-10-09T06:01:30.420897+0000","last_clean":"2023-10-09T05:53:45.891460+0000","last_became_active":"2023-10-09T05:58:02.118107+0000","last_became_peered":"2023-10-09T05:58:02.118107+0000","last_unstale":"2023-10-09T06:01:30.420897+0000","last_undegraded":"2023-10-09T05:58:01.208593+0000","last_fullsized":"2023-10-09T05:57:58.248025+0000","mapping_epoch":444,"log_start":"368'850","ondisk_log_start":"368'850","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6305,"log_dups_size":0,"ondisk_log_size":6305,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32440320,"num_objects":495,"num_object_clones":0,"num_object_copies":1485,"num_objects_missing_on_primary":872,"num_objects_missing":872,"num_objects_degraded":3455,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4731,"num_write_kb":167232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1619,"num_bytes_recovered":103809024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"2,3,5","objects":410},{"shards":"2,7","objects":-556}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":130,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.981160+0000","last_change":"2023-10-09T06:01:19.985414+0000","last_active":"2023-10-09T06:01:20.981160+0000","last_peered":"2023-10-09T06:01:20.981160+0000","last_clean":"2023-10-09T06:01:20.981160+0000","last_became_active":"2023-10-09T06:01:19.984885+0000","last_became_peered":"2023-10-09T06:01:19.984885+0000","last_unstale":"2023-10-09T06:01:20.981160+0000","last_undegraded":"2023-10-09T06:01:20.981160+0000","last_fullsized":"2023-10-09T06:01:20.981160+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":140,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.978989+0000","last_change":"2023-10-09T06:01:19.993334+0000","last_active":"2023-10-09T06:01:20.978989+0000","last_peered":"2023-10-09T06:01:20.978989+0000","last_clean":"2023-10-09T06:01:20.978989+0000","last_became_active":"2023-10-09T06:01:19.992999+0000","last_became_peered":"2023-10-09T06:01:19.992999+0000","last_unstale":"2023-10-09T06:01:20.978989+0000","last_undegraded":"2023-10-09T06:01:20.978989+0000","last_fullsized":"2023-10-09T06:01:20.978989+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"496'7109","reported_seq":7299,"reported_epoch":496,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:30.296118+0000","last_change":"2023-10-09T06:01:20.199017+0000","last_active":"2023-10-09T06:01:30.296118+0000","last_peered":"2023-10-09T06:01:30.296118+0000","last_clean":"2023-10-09T05:57:55.996837+0000","last_became_active":"2023-10-09T06:01:20.180240+0000","last_became_peered":"2023-10-09T06:01:20.180240+0000","last_unstale":"2023-10-09T06:01:30.296118+0000","last_undegraded":"2023-10-09T06:01:20.012295+0000","last_fullsized":"2023-10-09T06:01:20.004876+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7109,"log_dups_size":0,"ondisk_log_size":7109,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30801920,"num_objects":470,"num_object_clones":0,"num_object_copies":1410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2570,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4710,"num_write_kb":165760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","1","5"],"object_location_counts":[{"shards":"1,2","objects":-72},{"shards":"1,2,5","objects":450}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":419,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.966408+0000","last_change":"2023-10-09T06:01:19.987220+0000","last_active":"2023-10-09T06:01:20.966408+0000","last_peered":"2023-10-09T06:01:20.966408+0000","last_clean":"2023-10-09T06:01:20.966408+0000","last_became_active":"2023-10-09T06:01:19.986675+0000","last_became_peered":"2023-10-09T06:01:19.986675+0000","last_unstale":"2023-10-09T06:01:20.966408+0000","last_undegraded":"2023-10-09T06:01:20.966408+0000","last_fullsized":"2023-10-09T06:01:20.966408+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:06.685801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":427,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.098317+0000","last_change":"2023-10-09T06:00:05.352723+0000","last_active":"2023-10-09T06:01:21.098317+0000","last_peered":"2023-10-09T06:01:21.098317+0000","last_clean":"2023-10-09T06:01:21.098317+0000","last_became_active":"2023-10-09T06:00:05.352008+0000","last_became_peered":"2023-10-09T06:00:05.352008+0000","last_unstale":"2023-10-09T06:01:21.098317+0000","last_undegraded":"2023-10-09T06:01:21.098317+0000","last_fullsized":"2023-10-09T06:01:21.098317+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:25.302918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"496'7329","reported_seq":18184,"reported_epoch":496,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:30.830772+0000","last_change":"2023-10-09T06:01:22.568077+0000","last_active":"2023-10-09T06:01:30.830772+0000","last_peered":"2023-10-09T06:01:30.830772+0000","last_clean":"2023-10-09T05:58:03.790253+0000","last_became_active":"2023-10-09T06:01:20.846693+0000","last_became_peered":"2023-10-09T06:01:20.846693+0000","last_unstale":"2023-10-09T06:01:30.830772+0000","last_undegraded":"2023-10-09T06:01:19.986399+0000","last_fullsized":"2023-10-09T06:01:19.964834+0000","mapping_epoch":493,"log_start":"368'1350","ondisk_log_start":"368'1350","created":364,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5979,"log_dups_size":0,"ondisk_log_size":5979,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26705920,"num_objects":407,"num_object_clones":0,"num_object_copies":1221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4875,"num_write_kb":169056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4116,"num_bytes_recovered":234323968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":407}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":206,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.098429+0000","last_change":"2023-10-09T06:00:05.340070+0000","last_active":"2023-10-09T06:01:21.098429+0000","last_peered":"2023-10-09T06:01:21.098429+0000","last_clean":"2023-10-09T06:01:21.098429+0000","last_became_active":"2023-10-09T06:00:05.329639+0000","last_became_peered":"2023-10-09T06:00:05.329639+0000","last_unstale":"2023-10-09T06:01:21.098429+0000","last_undegraded":"2023-10-09T06:01:21.098429+0000","last_fullsized":"2023-10-09T06:01:21.098429+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:36.059049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":494,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.974788+0000","last_change":"2023-10-09T06:01:19.997595+0000","last_active":"2023-10-09T06:01:20.974788+0000","last_peered":"2023-10-09T06:01:20.974788+0000","last_clean":"2023-10-09T06:01:20.974788+0000","last_became_active":"2023-10-09T06:01:19.995775+0000","last_became_peered":"2023-10-09T06:01:19.995775+0000","last_unstale":"2023-10-09T06:01:20.974788+0000","last_undegraded":"2023-10-09T06:01:20.974788+0000","last_fullsized":"2023-10-09T06:01:20.974788+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"496'7125","reported_seq":10598,"reported_epoch":496,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:30.547005+0000","last_change":"2023-10-09T06:01:21.227574+0000","last_active":"2023-10-09T06:01:30.547005+0000","last_peered":"2023-10-09T06:01:30.547005+0000","last_clean":"2023-10-09T05:56:47.068306+0000","last_became_active":"2023-10-09T06:01:21.170441+0000","last_became_peered":"2023-10-09T06:01:21.170441+0000","last_unstale":"2023-10-09T06:01:30.547005+0000","last_undegraded":"2023-10-09T06:01:21.012492+0000","last_fullsized":"2023-10-09T06:01:20.968631+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7125,"log_dups_size":0,"ondisk_log_size":7125,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30474240,"num_objects":465,"num_object_clones":0,"num_object_copies":1395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4735,"num_write_kb":166400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1126,"num_bytes_recovered":73793536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,0],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"0,2","objects":2},{"shards":"1,2,6","objects":463}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.971995+0000","last_change":"2023-10-09T06:01:19.981078+0000","last_active":"2023-10-09T06:01:20.971995+0000","last_peered":"2023-10-09T06:01:20.971995+0000","last_clean":"2023-10-09T06:01:20.971995+0000","last_became_active":"2023-10-09T06:01:19.980723+0000","last_became_peered":"2023-10-09T06:01:19.980723+0000","last_unstale":"2023-10-09T06:01:20.971995+0000","last_undegraded":"2023-10-09T06:01:20.971995+0000","last_fullsized":"2023-10-09T06:01:20.971995+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":427,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.968471+0000","last_change":"2023-10-09T05:54:05.161959+0000","last_active":"2023-10-09T06:01:20.968471+0000","last_peered":"2023-10-09T06:01:20.968471+0000","last_clean":"2023-10-09T06:01:20.968471+0000","last_became_active":"2023-10-09T05:54:05.161671+0000","last_became_peered":"2023-10-09T05:54:05.161671+0000","last_unstale":"2023-10-09T06:01:20.968471+0000","last_undegraded":"2023-10-09T06:01:20.968471+0000","last_fullsized":"2023-10-09T06:01:20.968471+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:47.282176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"495'7176","reported_seq":13149,"reported_epoch":495,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:28.443847+0000","last_change":"2023-10-09T05:56:01.447837+0000","last_active":"2023-10-09T06:01:28.443847+0000","last_peered":"2023-10-09T06:01:28.443847+0000","last_clean":"2023-10-09T05:55:25.103026+0000","last_became_active":"2023-10-09T05:56:01.440057+0000","last_became_peered":"2023-10-09T05:56:01.440057+0000","last_unstale":"2023-10-09T06:01:28.443847+0000","last_undegraded":"2023-10-09T05:56:01.309926+0000","last_fullsized":"2023-10-09T05:56:01.207038+0000","mapping_epoch":416,"log_start":"369'1675","ondisk_log_start":"369'1675","created":364,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5501,"log_dups_size":0,"ondisk_log_size":5501,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29425664,"num_objects":449,"num_object_clones":0,"num_object_copies":1347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4763,"num_write_kb":166784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1468,"num_bytes_recovered":96206848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[1,3],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1,3","objects":-1626},{"shards":"1,3,5","objects":478}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":499,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.968628+0000","last_change":"2023-10-09T05:55:26.256379+0000","last_active":"2023-10-09T06:01:20.968628+0000","last_peered":"2023-10-09T06:01:20.968628+0000","last_clean":"2023-10-09T06:01:20.968628+0000","last_became_active":"2023-10-09T05:55:26.256062+0000","last_became_peered":"2023-10-09T05:55:26.256062+0000","last_unstale":"2023-10-09T06:01:20.968628+0000","last_undegraded":"2023-10-09T06:01:20.968628+0000","last_fullsized":"2023-10-09T06:01:20.968628+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:00.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":460,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.975988+0000","last_change":"2023-10-09T06:01:20.003486+0000","last_active":"2023-10-09T06:01:20.975988+0000","last_peered":"2023-10-09T06:01:20.975988+0000","last_clean":"2023-10-09T06:01:20.975988+0000","last_became_active":"2023-10-09T06:01:20.003139+0000","last_became_peered":"2023-10-09T06:01:20.003139+0000","last_unstale":"2023-10-09T06:01:20.975988+0000","last_undegraded":"2023-10-09T06:01:20.975988+0000","last_fullsized":"2023-10-09T06:01:20.975988+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"496'7223","reported_seq":12738,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:01:30.378863+0000","last_change":"2023-10-09T05:58:52.969080+0000","last_active":"2023-10-09T06:01:30.378863+0000","last_peered":"2023-10-09T06:01:30.378863+0000","last_clean":"2023-10-09T06:01:30.378863+0000","last_became_active":"2023-10-09T05:58:52.906218+0000","last_became_peered":"2023-10-09T05:58:52.906218+0000","last_unstale":"2023-10-09T06:01:30.378863+0000","last_undegraded":"2023-10-09T06:01:30.378863+0000","last_fullsized":"2023-10-09T06:01:30.378863+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":364,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7223,"log_dups_size":0,"ondisk_log_size":7223,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28540928,"num_objects":435,"num_object_clones":0,"num_object_copies":1305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4801,"num_write_kb":167584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2659,"num_bytes_recovered":101056512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":130,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.977271+0000","last_change":"2023-10-09T05:56:49.287906+0000","last_active":"2023-10-09T06:01:20.977271+0000","last_peered":"2023-10-09T06:01:20.977271+0000","last_clean":"2023-10-09T06:01:20.977271+0000","last_became_active":"2023-10-09T05:56:49.275081+0000","last_became_peered":"2023-10-09T05:56:49.275081+0000","last_unstale":"2023-10-09T06:01:20.977271+0000","last_undegraded":"2023-10-09T06:01:20.977271+0000","last_fullsized":"2023-10-09T06:01:20.977271+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":183,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:01:19.982466+0000","last_change":"2023-10-09T05:56:00.225724+0000","last_active":"2023-10-09T06:01:19.982466+0000","last_peered":"2023-10-09T06:01:19.982466+0000","last_clean":"2023-10-09T06:01:19.982466+0000","last_became_active":"2023-10-09T05:56:00.225335+0000","last_became_peered":"2023-10-09T05:56:00.225335+0000","last_unstale":"2023-10-09T06:01:19.982466+0000","last_undegraded":"2023-10-09T06:01:19.982466+0000","last_fullsized":"2023-10-09T06:01:19.982466+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:41.405222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"495'9713","reported_seq":9876,"reported_epoch":495,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:28.448337+0000","last_change":"2023-10-09T05:57:58.501288+0000","last_active":"2023-10-09T06:01:28.448337+0000","last_peered":"2023-10-09T06:01:28.448337+0000","last_clean":"2023-10-09T05:55:49.154347+0000","last_became_active":"2023-10-09T05:57:58.384024+0000","last_became_peered":"2023-10-09T05:57:58.384024+0000","last_unstale":"2023-10-09T06:01:28.448337+0000","last_undegraded":"2023-10-09T05:57:58.373205+0000","last_fullsized":"2023-10-09T05:57:58.246583+0000","mapping_epoch":444,"log_start":"390'5702","ondisk_log_start":"390'5702","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4011,"log_dups_size":0,"ondisk_log_size":4011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:58.015603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63766528,"num_objects":973,"num_object_clones":0,"num_object_copies":2919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5436,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9713,"num_write_kb":341952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[1,3],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,3","objects":-2268},{"shards":"1,3,5","objects":973}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":923,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.090402+0000","last_change":"2023-10-09T06:00:05.523981+0000","last_active":"2023-10-09T06:01:21.090402+0000","last_peered":"2023-10-09T06:01:21.090402+0000","last_clean":"2023-10-09T06:01:21.090402+0000","last_became_active":"2023-10-09T06:00:05.338627+0000","last_became_peered":"2023-10-09T06:00:05.338627+0000","last_unstale":"2023-10-09T06:01:21.090402+0000","last_undegraded":"2023-10-09T06:01:21.090402+0000","last_fullsized":"2023-10-09T06:01:21.090402+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:25.623236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":489,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.968545+0000","last_change":"2023-10-09T06:01:20.018047+0000","last_active":"2023-10-09T06:01:20.968545+0000","last_peered":"2023-10-09T06:01:20.968545+0000","last_clean":"2023-10-09T06:01:20.968545+0000","last_became_active":"2023-10-09T06:01:20.017698+0000","last_became_peered":"2023-10-09T06:01:20.017698+0000","last_unstale":"2023-10-09T06:01:20.968545+0000","last_undegraded":"2023-10-09T06:01:20.968545+0000","last_fullsized":"2023-10-09T06:01:20.968545+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:44.970159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":31,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.968278+0000","last_change":"2023-10-09T06:00:05.355057+0000","last_active":"2023-10-09T06:01:20.968278+0000","last_peered":"2023-10-09T06:01:20.968278+0000","last_clean":"2023-10-09T06:01:20.968278+0000","last_became_active":"2023-10-09T06:00:05.352475+0000","last_became_peered":"2023-10-09T06:00:05.352475+0000","last_unstale":"2023-10-09T06:01:20.968278+0000","last_undegraded":"2023-10-09T06:01:20.968278+0000","last_fullsized":"2023-10-09T06:01:20.968278+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:50.255773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.9","version":"496'7164","reported_seq":11822,"reported_epoch":496,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:30.766055+0000","last_change":"2023-10-09T06:01:22.210292+0000","last_active":"2023-10-09T06:01:30.766055+0000","last_peered":"2023-10-09T06:01:30.766055+0000","last_clean":"2023-10-09T05:53:46.057808+0000","last_became_active":"2023-10-09T06:01:20.874735+0000","last_became_peered":"2023-10-09T06:01:20.874735+0000","last_unstale":"2023-10-09T06:01:30.766055+0000","last_undegraded":"2023-10-09T06:01:20.812993+0000","last_fullsized":"2023-10-09T06:01:19.979718+0000","mapping_epoch":493,"log_start":"368'1081","ondisk_log_start":"368'1081","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6083,"log_dups_size":0,"ondisk_log_size":6083,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30113792,"num_objects":459,"num_object_clones":0,"num_object_copies":1377,"num_objects_missing_on_primary":904,"num_objects_missing":904,"num_objects_degraded":4440,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4757,"num_write_kb":166944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1573,"num_bytes_recovered":103120896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[7,0],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,3,5","objects":455},{"shards":"0,7","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":792,"reported_epoch":494,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:01:19.996341+0000","last_change":"2023-10-09T06:01:19.996341+0000","last_active":"2023-10-09T06:01:19.996341+0000","last_peered":"2023-10-09T06:01:19.996341+0000","last_clean":"2023-10-09T05:48:52.714098+0000","last_became_active":"2023-10-09T06:01:19.995418+0000","last_became_peered":"2023-10-09T06:01:19.995418+0000","last_unstale":"2023-10-09T06:01:19.996341+0000","last_undegraded":"2023-10-09T06:01:19.977315+0000","last_fullsized":"2023-10-09T06:01:19.996341+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:29:36.218008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":127,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.098667+0000","last_change":"2023-10-09T06:01:19.980238+0000","last_active":"2023-10-09T06:01:21.098667+0000","last_peered":"2023-10-09T06:01:21.098667+0000","last_clean":"2023-10-09T06:01:21.098667+0000","last_became_active":"2023-10-09T06:01:19.979877+0000","last_became_peered":"2023-10-09T06:01:19.979877+0000","last_unstale":"2023-10-09T06:01:21.098667+0000","last_undegraded":"2023-10-09T06:01:21.098667+0000","last_fullsized":"2023-10-09T06:01:21.098667+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:30.931312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":351,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:01:29.323539+0000","last_change":"2023-10-09T06:01:19.992782+0000","last_active":"2023-10-09T06:01:29.323539+0000","last_peered":"2023-10-09T06:01:29.323539+0000","last_clean":"2023-10-09T06:01:29.323539+0000","last_became_active":"2023-10-09T06:01:19.992426+0000","last_became_peered":"2023-10-09T06:01:19.992426+0000","last_unstale":"2023-10-09T06:01:29.323539+0000","last_undegraded":"2023-10-09T06:01:29.323539+0000","last_fullsized":"2023-10-09T06:01:29.323539+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:05.460642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"496'9701","reported_seq":20525,"reported_epoch":496,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:01:30.750415+0000","last_change":"2023-10-09T06:01:20.125933+0000","last_active":"2023-10-09T06:01:30.750415+0000","last_peered":"2023-10-09T06:01:30.750415+0000","last_clean":"2023-10-09T05:59:30.366736+0000","last_became_active":"2023-10-09T06:01:20.081322+0000","last_became_peered":"2023-10-09T06:01:20.081322+0000","last_unstale":"2023-10-09T06:01:30.750415+0000","last_undegraded":"2023-10-09T06:01:30.750415+0000","last_fullsized":"2023-10-09T06:01:30.750415+0000","mapping_epoch":493,"log_start":"389'5376","ondisk_log_start":"389'5376","created":364,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4325,"log_dups_size":0,"ondisk_log_size":4325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59310080,"num_objects":905,"num_object_clones":0,"num_object_copies":2715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":905,"num_objects_unfound":0,"num_objects_dirty":905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9701,"num_write_kb":339392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5309,"num_bytes_recovered":309723136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,3],"avail_no_missing":["7","1","3","5"],"object_location_counts":[{"shards":"1,3,7","objects":905}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":853,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.968719+0000","last_change":"2023-10-09T06:00:05.324594+0000","last_active":"2023-10-09T06:01:20.968719+0000","last_peered":"2023-10-09T06:01:20.968719+0000","last_clean":"2023-10-09T06:01:20.968719+0000","last_became_active":"2023-10-09T06:00:05.323845+0000","last_became_peered":"2023-10-09T06:00:05.323845+0000","last_unstale":"2023-10-09T06:01:20.968719+0000","last_undegraded":"2023-10-09T06:01:20.968719+0000","last_fullsized":"2023-10-09T06:01:20.968719+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:02.539703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":308,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.013031+0000","last_change":"2023-10-09T06:01:19.994662+0000","last_active":"2023-10-09T06:01:21.013031+0000","last_peered":"2023-10-09T06:01:21.013031+0000","last_clean":"2023-10-09T06:01:21.013031+0000","last_became_active":"2023-10-09T06:01:19.994264+0000","last_became_peered":"2023-10-09T06:01:19.994264+0000","last_unstale":"2023-10-09T06:01:21.013031+0000","last_undegraded":"2023-10-09T06:01:21.013031+0000","last_fullsized":"2023-10-09T06:01:21.013031+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:43.503821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":234,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.086329+0000","last_change":"2023-10-09T06:00:05.328910+0000","last_active":"2023-10-09T06:01:21.086329+0000","last_peered":"2023-10-09T06:01:21.086329+0000","last_clean":"2023-10-09T06:01:21.086329+0000","last_became_active":"2023-10-09T06:00:05.328512+0000","last_became_peered":"2023-10-09T06:00:05.328512+0000","last_unstale":"2023-10-09T06:01:21.086329+0000","last_undegraded":"2023-10-09T06:01:21.086329+0000","last_fullsized":"2023-10-09T06:01:21.086329+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:27.757193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"495'7159","reported_seq":17833,"reported_epoch":495,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:28.009415+0000","last_change":"2023-10-09T06:00:06.413710+0000","last_active":"2023-10-09T06:01:28.009415+0000","last_peered":"2023-10-09T06:01:28.009415+0000","last_clean":"2023-10-09T06:00:04.116692+0000","last_became_active":"2023-10-09T06:00:06.395225+0000","last_became_peered":"2023-10-09T06:00:06.395225+0000","last_unstale":"2023-10-09T06:01:28.009415+0000","last_undegraded":"2023-10-09T06:00:06.304668+0000","last_fullsized":"2023-10-09T06:00:06.297952+0000","mapping_epoch":476,"log_start":"369'1546","ondisk_log_start":"369'1546","created":364,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5613,"log_dups_size":0,"ondisk_log_size":5613,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31981568,"num_objects":488,"num_object_clones":0,"num_object_copies":1464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":991,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4746,"num_write_kb":167488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2749,"num_bytes_recovered":132644864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[1,7],"avail_no_missing":["1","2","5","7"],"object_location_counts":[{"shards":"1,7","objects":488}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":710,"reported_epoch":495,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:20.969022+0000","last_change":"2023-10-09T05:58:07.894313+0000","last_active":"2023-10-09T06:01:20.969022+0000","last_peered":"2023-10-09T06:01:20.969022+0000","last_clean":"2023-10-09T05:55:22.245899+0000","last_became_active":"2023-10-09T05:58:05.515736+0000","last_became_peered":"2023-10-09T05:58:05.515736+0000","last_unstale":"2023-10-09T06:01:20.969022+0000","last_undegraded":"2023-10-09T05:57:58.245342+0000","last_fullsized":"2023-10-09T05:57:58.244553+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:13.491224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":887,"num_read_kb":802,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":660,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.977078+0000","last_change":"2023-10-09T06:01:19.975320+0000","last_active":"2023-10-09T06:01:20.977078+0000","last_peered":"2023-10-09T06:01:20.977078+0000","last_clean":"2023-10-09T06:01:20.977078+0000","last_became_active":"2023-10-09T06:01:19.974866+0000","last_became_peered":"2023-10-09T06:01:19.974866+0000","last_unstale":"2023-10-09T06:01:20.977078+0000","last_undegraded":"2023-10-09T06:01:20.977078+0000","last_fullsized":"2023-10-09T06:01:20.977078+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":375,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.965532+0000","last_change":"2023-10-09T06:01:19.987073+0000","last_active":"2023-10-09T06:01:20.965532+0000","last_peered":"2023-10-09T06:01:20.965532+0000","last_clean":"2023-10-09T06:01:20.965532+0000","last_became_active":"2023-10-09T06:01:19.986758+0000","last_became_peered":"2023-10-09T06:01:19.986758+0000","last_unstale":"2023-10-09T06:01:20.965532+0000","last_undegraded":"2023-10-09T06:01:20.965532+0000","last_fullsized":"2023-10-09T06:01:20.965532+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:53.291650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":126,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.986745+0000","last_change":"2023-10-09T06:01:20.982313+0000","last_active":"2023-10-09T06:01:20.986745+0000","last_peered":"2023-10-09T06:01:20.986745+0000","last_clean":"2023-10-09T06:01:20.986745+0000","last_became_active":"2023-10-09T06:01:20.981964+0000","last_became_peered":"2023-10-09T06:01:20.981964+0000","last_unstale":"2023-10-09T06:01:20.986745+0000","last_undegraded":"2023-10-09T06:01:20.986745+0000","last_fullsized":"2023-10-09T06:01:20.986745+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"496'9758","reported_seq":12316,"reported_epoch":496,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:01:32.478660+0000","last_change":"2023-10-09T06:01:21.287092+0000","last_active":"2023-10-09T06:01:32.478660+0000","last_peered":"2023-10-09T06:01:32.478660+0000","last_clean":"2023-10-09T05:56:10.073783+0000","last_became_active":"2023-10-09T06:01:21.113355+0000","last_became_peered":"2023-10-09T06:01:21.113355+0000","last_unstale":"2023-10-09T06:01:32.478660+0000","last_undegraded":"2023-10-09T06:01:21.096880+0000","last_fullsized":"2023-10-09T06:01:32.478660+0000","mapping_epoch":494,"log_start":"380'4541","ondisk_log_start":"380'4541","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5217,"log_dups_size":0,"ondisk_log_size":5217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:28.933549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55967744,"num_objects":854,"num_object_clones":0,"num_object_copies":2562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4115,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9758,"num_write_kb":339584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1191,"num_bytes_recovered":78053376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":["0","1","3","5","6"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"0,5,6","objects":853}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":1023,"reported_epoch":495,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:01:20.976289+0000","last_change":"2023-10-09T06:00:06.308995+0000","last_active":"2023-10-09T06:01:20.976289+0000","last_peered":"2023-10-09T06:01:20.976289+0000","last_clean":"2023-10-09T05:56:02.257755+0000","last_became_active":"2023-10-09T06:00:06.308176+0000","last_became_peered":"2023-10-09T06:00:06.308176+0000","last_unstale":"2023-10-09T06:01:20.976289+0000","last_undegraded":"2023-10-09T06:00:06.282859+0000","last_fullsized":"2023-10-09T06:01:20.976289+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":464,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.115408+0000","last_change":"2023-10-09T06:00:05.314057+0000","last_active":"2023-10-09T06:01:21.115408+0000","last_peered":"2023-10-09T06:01:21.115408+0000","last_clean":"2023-10-09T06:01:21.115408+0000","last_became_active":"2023-10-09T06:00:05.313743+0000","last_became_peered":"2023-10-09T06:00:05.313743+0000","last_unstale":"2023-10-09T06:01:21.115408+0000","last_undegraded":"2023-10-09T06:01:21.115408+0000","last_fullsized":"2023-10-09T06:01:21.115408+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:29.568391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":442,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.092574+0000","last_change":"2023-10-09T06:00:05.590816+0000","last_active":"2023-10-09T06:01:21.092574+0000","last_peered":"2023-10-09T06:01:21.092574+0000","last_clean":"2023-10-09T06:01:21.092574+0000","last_became_active":"2023-10-09T06:00:05.590227+0000","last_became_peered":"2023-10-09T06:00:05.590227+0000","last_unstale":"2023-10-09T06:01:21.092574+0000","last_undegraded":"2023-10-09T06:01:21.092574+0000","last_fullsized":"2023-10-09T06:01:21.092574+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:51.060693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"496'9510","reported_seq":16340,"reported_epoch":496,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:30.802260+0000","last_change":"2023-10-09T06:00:13.414491+0000","last_active":"2023-10-09T06:01:30.802260+0000","last_peered":"2023-10-09T06:01:30.802260+0000","last_clean":"2023-10-09T05:53:45.896176+0000","last_became_active":"2023-10-09T06:00:12.549930+0000","last_became_peered":"2023-10-09T06:00:12.549930+0000","last_unstale":"2023-10-09T06:01:30.802260+0000","last_undegraded":"2023-10-09T06:00:11.004109+0000","last_fullsized":"2023-10-09T06:00:06.282679+0000","mapping_epoch":476,"log_start":"389'5238","ondisk_log_start":"389'5238","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4272,"log_dups_size":0,"ondisk_log_size":4272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61210624,"num_objects":934,"num_object_clones":0,"num_object_copies":2802,"num_objects_missing_on_primary":2142,"num_objects_missing":2142,"num_objects_degraded":3105,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9510,"num_write_kb":334208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3313,"num_bytes_recovered":216858624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"0,1,4","objects":922},{"shards":"0,1,4,7","objects":12}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":671,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.981539+0000","last_change":"2023-10-09T06:00:17.336621+0000","last_active":"2023-10-09T06:01:20.981539+0000","last_peered":"2023-10-09T06:01:20.981539+0000","last_clean":"2023-10-09T06:01:20.981539+0000","last_became_active":"2023-10-09T06:00:14.622413+0000","last_became_peered":"2023-10-09T06:00:14.622413+0000","last_unstale":"2023-10-09T06:01:20.981539+0000","last_undegraded":"2023-10-09T06:01:20.981539+0000","last_fullsized":"2023-10-09T06:01:20.981539+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":326,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.968909+0000","last_change":"2023-10-09T06:00:05.324944+0000","last_active":"2023-10-09T06:01:20.968909+0000","last_peered":"2023-10-09T06:01:20.968909+0000","last_clean":"2023-10-09T06:01:20.968909+0000","last_became_active":"2023-10-09T06:00:05.324533+0000","last_became_peered":"2023-10-09T06:00:05.324533+0000","last_unstale":"2023-10-09T06:01:20.968909+0000","last_undegraded":"2023-10-09T06:01:20.968909+0000","last_fullsized":"2023-10-09T06:01:20.968909+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:33:33.376129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":446,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.098592+0000","last_change":"2023-10-09T06:00:05.352906+0000","last_active":"2023-10-09T06:01:21.098592+0000","last_peered":"2023-10-09T06:01:21.098592+0000","last_clean":"2023-10-09T06:01:21.098592+0000","last_became_active":"2023-10-09T06:00:05.352436+0000","last_became_peered":"2023-10-09T06:00:05.352436+0000","last_unstale":"2023-10-09T06:01:21.098592+0000","last_undegraded":"2023-10-09T06:01:21.098592+0000","last_fullsized":"2023-10-09T06:01:21.098592+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:20.223563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"496'9563","reported_seq":15376,"reported_epoch":496,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:29.787247+0000","last_change":"2023-10-09T06:01:21.327657+0000","last_active":"2023-10-09T06:01:29.787247+0000","last_peered":"2023-10-09T06:01:29.787247+0000","last_clean":"2023-10-09T05:53:45.883404+0000","last_became_active":"2023-10-09T06:01:21.281536+0000","last_became_peered":"2023-10-09T06:01:21.281536+0000","last_unstale":"2023-10-09T06:01:29.787247+0000","last_undegraded":"2023-10-09T06:01:21.121726+0000","last_fullsized":"2023-10-09T06:01:20.970867+0000","mapping_epoch":494,"log_start":"389'5096","ondisk_log_start":"389'5096","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4467,"log_dups_size":0,"ondisk_log_size":4467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59375640,"num_objects":907,"num_object_clones":0,"num_object_copies":2721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2743,"num_objects_misplaced":1884,"num_objects_unfound":0,"num_objects_dirty":907,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9563,"num_write_kb":334977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2800,"num_bytes_recovered":183369728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[5,7],"avail_no_missing":["5","0","1","2"],"object_location_counts":[{"shards":"1,4,5","objects":906},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":162,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.981196+0000","last_change":"2023-10-09T06:00:05.329139+0000","last_active":"2023-10-09T06:01:20.981196+0000","last_peered":"2023-10-09T06:01:20.981196+0000","last_clean":"2023-10-09T06:01:20.981196+0000","last_became_active":"2023-10-09T06:00:05.328811+0000","last_became_peered":"2023-10-09T06:00:05.328811+0000","last_unstale":"2023-10-09T06:01:20.981196+0000","last_undegraded":"2023-10-09T06:01:20.981196+0000","last_fullsized":"2023-10-09T06:01:20.981196+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":478,"reported_epoch":495,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:01:20.980512+0000","last_change":"2023-10-09T06:01:20.980512+0000","last_active":"2023-10-09T06:01:20.980512+0000","last_peered":"2023-10-09T06:01:20.980512+0000","last_clean":"2023-10-09T05:58:01.663036+0000","last_became_active":"2023-10-09T06:01:20.980166+0000","last_became_peered":"2023-10-09T06:01:20.980166+0000","last_unstale":"2023-10-09T06:01:20.980512+0000","last_undegraded":"2023-10-09T06:01:20.972658+0000","last_fullsized":"2023-10-09T06:01:20.980512+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":166,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.976762+0000","last_change":"2023-10-09T06:00:05.328236+0000","last_active":"2023-10-09T06:01:20.976762+0000","last_peered":"2023-10-09T06:01:20.976762+0000","last_clean":"2023-10-09T06:01:20.976762+0000","last_became_active":"2023-10-09T06:00:05.327876+0000","last_became_peered":"2023-10-09T06:00:05.327876+0000","last_unstale":"2023-10-09T06:01:20.976762+0000","last_undegraded":"2023-10-09T06:01:20.976762+0000","last_fullsized":"2023-10-09T06:01:20.976762+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"496'9646","reported_seq":17726,"reported_epoch":496,"state":"active+recovering","last_fresh":"2023-10-09T06:01:30.660065+0000","last_change":"2023-10-09T05:56:49.370077+0000","last_active":"2023-10-09T06:01:30.660065+0000","last_peered":"2023-10-09T06:01:30.660065+0000","last_clean":"2023-10-09T05:53:45.907519+0000","last_became_active":"2023-10-09T05:56:03.326563+0000","last_became_peered":"2023-10-09T05:56:03.326563+0000","last_unstale":"2023-10-09T06:01:30.660065+0000","last_undegraded":"2023-10-09T06:01:30.660065+0000","last_fullsized":"2023-10-09T06:01:30.660065+0000","mapping_epoch":416,"log_start":"389'5371","ondisk_log_start":"389'5371","created":364,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":4275,"log_dups_size":0,"ondisk_log_size":4275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61865984,"num_objects":944,"num_object_clones":0,"num_object_copies":2832,"num_objects_missing_on_primary":55,"num_objects_missing":55,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9646,"num_write_kb":338880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3947,"num_bytes_recovered":200015872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":["4"],"object_location_counts":[{"shards":"3,4,7","objects":944}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":684,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.986741+0000","last_change":"2023-10-09T06:01:20.985388+0000","last_active":"2023-10-09T06:01:20.986741+0000","last_peered":"2023-10-09T06:01:20.986741+0000","last_clean":"2023-10-09T06:01:20.986741+0000","last_became_active":"2023-10-09T06:01:20.984892+0000","last_became_peered":"2023-10-09T06:01:20.984892+0000","last_unstale":"2023-10-09T06:01:20.986741+0000","last_undegraded":"2023-10-09T06:01:20.986741+0000","last_fullsized":"2023-10-09T06:01:20.986741+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":437,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.092686+0000","last_change":"2023-10-09T06:00:05.352581+0000","last_active":"2023-10-09T06:01:21.092686+0000","last_peered":"2023-10-09T06:01:21.092686+0000","last_clean":"2023-10-09T06:01:21.092686+0000","last_became_active":"2023-10-09T06:00:05.351836+0000","last_became_peered":"2023-10-09T06:00:05.351836+0000","last_unstale":"2023-10-09T06:01:21.092686+0000","last_undegraded":"2023-10-09T06:01:21.092686+0000","last_fullsized":"2023-10-09T06:01:21.092686+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:31.576840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":40,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.086941+0000","last_change":"2023-10-09T06:01:19.981055+0000","last_active":"2023-10-09T06:01:21.086941+0000","last_peered":"2023-10-09T06:01:21.086941+0000","last_clean":"2023-10-09T06:01:21.086941+0000","last_became_active":"2023-10-09T06:01:19.980717+0000","last_became_peered":"2023-10-09T06:01:19.980717+0000","last_unstale":"2023-10-09T06:01:21.086941+0000","last_undegraded":"2023-10-09T06:01:21.086941+0000","last_fullsized":"2023-10-09T06:01:21.086941+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:55.973927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.11","version":"496'7272","reported_seq":16417,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:01:30.543661+0000","last_change":"2023-10-09T06:01:22.567999+0000","last_active":"2023-10-09T06:01:30.543661+0000","last_peered":"2023-10-09T06:01:30.543661+0000","last_clean":"2023-10-09T06:01:30.543661+0000","last_became_active":"2023-10-09T06:01:20.846382+0000","last_became_peered":"2023-10-09T06:01:20.846382+0000","last_unstale":"2023-10-09T06:01:30.543661+0000","last_undegraded":"2023-10-09T06:01:30.543661+0000","last_fullsized":"2023-10-09T06:01:30.543661+0000","mapping_epoch":493,"log_start":"368'1132","ondisk_log_start":"368'1132","created":385,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":6140,"log_dups_size":0,"ondisk_log_size":6140,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32276480,"num_objects":493,"num_object_clones":0,"num_object_copies":1479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4819,"num_write_kb":169952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3255,"num_bytes_recovered":213286912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":447,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.098517+0000","last_change":"2023-10-09T06:00:05.416922+0000","last_active":"2023-10-09T06:01:21.098517+0000","last_peered":"2023-10-09T06:01:21.098517+0000","last_clean":"2023-10-09T06:01:21.098517+0000","last_became_active":"2023-10-09T06:00:05.416444+0000","last_became_peered":"2023-10-09T06:00:05.416444+0000","last_unstale":"2023-10-09T06:01:21.098517+0000","last_undegraded":"2023-10-09T06:01:21.098517+0000","last_fullsized":"2023-10-09T06:01:21.098517+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:26.775246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":189,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.968848+0000","last_change":"2023-10-09T05:56:00.226695+0000","last_active":"2023-10-09T06:01:20.968848+0000","last_peered":"2023-10-09T06:01:20.968848+0000","last_clean":"2023-10-09T06:01:20.968848+0000","last_became_active":"2023-10-09T05:56:00.226359+0000","last_became_peered":"2023-10-09T05:56:00.226359+0000","last_unstale":"2023-10-09T06:01:20.968848+0000","last_undegraded":"2023-10-09T06:01:20.968848+0000","last_fullsized":"2023-10-09T06:01:20.968848+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:20.323704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"496'7178","reported_seq":7366,"reported_epoch":496,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:30.347948+0000","last_change":"2023-10-09T06:01:20.198287+0000","last_active":"2023-10-09T06:01:30.347948+0000","last_peered":"2023-10-09T06:01:30.347948+0000","last_clean":"2023-10-09T05:57:55.523800+0000","last_became_active":"2023-10-09T06:01:20.176060+0000","last_became_peered":"2023-10-09T06:01:20.176060+0000","last_unstale":"2023-10-09T06:01:30.347948+0000","last_undegraded":"2023-10-09T06:01:20.004156+0000","last_fullsized":"2023-10-09T06:01:19.973786+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":385,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":7178,"log_dups_size":0,"ondisk_log_size":7178,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28114944,"num_objects":429,"num_object_clones":0,"num_object_copies":1287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2624,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4779,"num_write_kb":166656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","1","5"],"object_location_counts":[{"shards":"1,2","objects":-120},{"shards":"1,2,5","objects":449}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":41,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.974943+0000","last_change":"2023-10-09T06:01:19.995155+0000","last_active":"2023-10-09T06:01:20.974943+0000","last_peered":"2023-10-09T06:01:20.974943+0000","last_clean":"2023-10-09T06:01:20.974943+0000","last_became_active":"2023-10-09T06:01:19.994697+0000","last_became_peered":"2023-10-09T06:01:19.994697+0000","last_unstale":"2023-10-09T06:01:20.974943+0000","last_undegraded":"2023-10-09T06:01:20.974943+0000","last_fullsized":"2023-10-09T06:01:20.974943+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":78,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.968726+0000","last_change":"2023-10-09T05:56:49.252225+0000","last_active":"2023-10-09T06:01:20.968726+0000","last_peered":"2023-10-09T06:01:20.968726+0000","last_clean":"2023-10-09T06:01:20.968726+0000","last_became_active":"2023-10-09T05:56:49.251880+0000","last_became_peered":"2023-10-09T05:56:49.251880+0000","last_unstale":"2023-10-09T06:01:20.968726+0000","last_undegraded":"2023-10-09T06:01:20.968726+0000","last_fullsized":"2023-10-09T06:01:20.968726+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:34.079170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.17","version":"496'7301","reported_seq":13924,"reported_epoch":496,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:30.685496+0000","last_change":"2023-10-09T05:58:02.382245+0000","last_active":"2023-10-09T06:01:30.685496+0000","last_peered":"2023-10-09T06:01:30.685496+0000","last_clean":"2023-10-09T05:57:55.399317+0000","last_became_active":"2023-10-09T05:58:02.319859+0000","last_became_peered":"2023-10-09T05:58:02.319859+0000","last_unstale":"2023-10-09T06:01:30.685496+0000","last_undegraded":"2023-10-09T05:58:01.234664+0000","last_fullsized":"2023-10-09T05:58:01.215806+0000","mapping_epoch":444,"log_start":"368'1436","ondisk_log_start":"368'1436","created":385,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5865,"log_dups_size":0,"ondisk_log_size":5865,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24838144,"num_objects":379,"num_object_clones":0,"num_object_copies":1137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2713,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4877,"num_write_kb":168192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3182,"num_bytes_recovered":157876224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["7","2","3","5"],"object_location_counts":[{"shards":"2,5,7","objects":464},{"shards":"2,7","objects":-1230}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":41,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.975424+0000","last_change":"2023-10-09T06:01:19.994887+0000","last_active":"2023-10-09T06:01:20.975424+0000","last_peered":"2023-10-09T06:01:20.975424+0000","last_clean":"2023-10-09T06:01:20.975424+0000","last_became_active":"2023-10-09T06:01:19.994497+0000","last_became_peered":"2023-10-09T06:01:19.994497+0000","last_unstale":"2023-10-09T06:01:20.975424+0000","last_undegraded":"2023-10-09T06:01:20.975424+0000","last_fullsized":"2023-10-09T06:01:20.975424+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":242,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.968379+0000","last_change":"2023-10-09T06:00:05.340542+0000","last_active":"2023-10-09T06:01:20.968379+0000","last_peered":"2023-10-09T06:01:20.968379+0000","last_clean":"2023-10-09T06:01:20.968379+0000","last_became_active":"2023-10-09T06:00:05.340213+0000","last_became_peered":"2023-10-09T06:00:05.340213+0000","last_unstale":"2023-10-09T06:01:20.968379+0000","last_undegraded":"2023-10-09T06:01:20.968379+0000","last_fullsized":"2023-10-09T06:01:20.968379+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:34.412535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.14","version":"495'7209","reported_seq":7382,"reported_epoch":495,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:01:28.492242+0000","last_change":"2023-10-09T05:56:01.281023+0000","last_active":"2023-10-09T06:01:28.492242+0000","last_peered":"2023-10-09T06:01:28.492242+0000","last_clean":"2023-10-09T05:55:48.964110+0000","last_became_active":"2023-10-09T05:56:01.245699+0000","last_became_peered":"2023-10-09T05:56:01.245699+0000","last_unstale":"2023-10-09T06:01:28.492242+0000","last_undegraded":"2023-10-09T05:56:01.216362+0000","last_fullsized":"2023-10-09T06:01:28.492242+0000","mapping_epoch":416,"log_start":"369'1686","ondisk_log_start":"369'1686","created":385,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5523,"log_dups_size":0,"ondisk_log_size":5523,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32440320,"num_objects":495,"num_object_clones":0,"num_object_copies":1485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":40,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4779,"num_write_kb":168768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":495}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":255,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.969394+0000","last_change":"2023-10-09T06:01:20.198041+0000","last_active":"2023-10-09T06:01:20.969394+0000","last_peered":"2023-10-09T06:01:20.969394+0000","last_clean":"2023-10-09T06:01:20.969394+0000","last_became_active":"2023-10-09T06:01:20.178975+0000","last_became_peered":"2023-10-09T06:01:20.178975+0000","last_unstale":"2023-10-09T06:01:20.969394+0000","last_undegraded":"2023-10-09T06:01:20.969394+0000","last_fullsized":"2023-10-09T06:01:20.969394+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:40.386988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":355,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.968806+0000","last_change":"2023-10-09T06:00:05.325284+0000","last_active":"2023-10-09T06:01:20.968806+0000","last_peered":"2023-10-09T06:01:20.968806+0000","last_clean":"2023-10-09T06:01:20.968806+0000","last_became_active":"2023-10-09T06:00:05.324588+0000","last_became_peered":"2023-10-09T06:00:05.324588+0000","last_unstale":"2023-10-09T06:01:20.968806+0000","last_undegraded":"2023-10-09T06:01:20.968806+0000","last_fullsized":"2023-10-09T06:01:20.968806+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:59.068162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"495'7275","reported_seq":13225,"reported_epoch":495,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:01:28.518795+0000","last_change":"2023-10-09T06:00:51.635672+0000","last_active":"2023-10-09T06:01:28.518795+0000","last_peered":"2023-10-09T06:01:28.518795+0000","last_clean":"2023-10-09T05:53:45.884391+0000","last_became_active":"2023-10-09T06:00:17.616889+0000","last_became_peered":"2023-10-09T06:00:17.616889+0000","last_unstale":"2023-10-09T06:01:28.518795+0000","last_undegraded":"2023-10-09T06:00:12.461200+0000","last_fullsized":"2023-10-09T06:00:07.517576+0000","mapping_epoch":477,"log_start":"369'1813","ondisk_log_start":"369'1813","created":385,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:53:07.200059+0000","last_clean_scrub_stamp":"2023-10-09T05:53:07.200059+0000","objects_scrubbed":0,"log_size":5462,"log_dups_size":0,"ondisk_log_size":5462,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31555584,"num_objects":482,"num_object_clones":0,"num_object_copies":1446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":554,"num_objects_misplaced":497,"num_objects_unfound":0,"num_objects_dirty":482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4834,"num_write_kb":170080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2102,"num_bytes_recovered":50462720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[1,0],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"0,1","objects":-569},{"shards":"0,1,5","objects":497}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":451,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:21.086497+0000","last_change":"2023-10-09T06:01:19.996238+0000","last_active":"2023-10-09T06:01:21.086497+0000","last_peered":"2023-10-09T06:01:21.086497+0000","last_clean":"2023-10-09T06:01:21.086497+0000","last_became_active":"2023-10-09T06:01:19.995257+0000","last_became_peered":"2023-10-09T06:01:19.995257+0000","last_unstale":"2023-10-09T06:01:21.086497+0000","last_undegraded":"2023-10-09T06:01:21.086497+0000","last_fullsized":"2023-10-09T06:01:21.086497+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:46:23.092370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":473,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:01:20.975054+0000","last_change":"2023-10-09T06:01:19.994582+0000","last_active":"2023-10-09T06:01:20.975054+0000","last_peered":"2023-10-09T06:01:20.975054+0000","last_clean":"2023-10-09T06:01:20.975054+0000","last_became_active":"2023-10-09T06:01:19.994185+0000","last_became_peered":"2023-10-09T06:01:19.994185+0000","last_unstale":"2023-10-09T06:01:20.975054+0000","last_undegraded":"2023-10-09T06:01:20.975054+0000","last_fullsized":"2023-10-09T06:01:20.975054+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":15826,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":967049240,"num_objects":14757,"num_object_clones":0,"num_object_copies":44271,"num_objects_missing_on_primary":4861,"num_objects_missing":4861,"num_objects_degraded":49614,"num_objects_misplaced":3801,"num_objects_unfound":0,"num_objects_dirty":14757,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153585,"num_write_kb":5386881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52859,"num_bytes_recovered":2848784384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5802110976,"data_stored":5802098760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":151513,"ondisk_log_size":151513,"up":78,"acting":61,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":36,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13211,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":98,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846909,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1512980,"kb_used_data":594956,"kb_used_omap":14,"kb_used_meta":918001,"kb_avail":92232172,"statfs":{"total":95995035648,"available":94445744128,"internally_reserved":0,"allocated":609234944,"data_stored":608144500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":940033882},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,0,0,2,1,1,6,5],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.629,"15min":4.0780000000000003},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.20699999999999999},"max":{"1min":3.774,"5min":5.0960000000000001,"15min":212.64400000000001},"last":9.9480000000000004},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.875,"15min":4.173},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.25900000000000001},"max":{"1min":3.7530000000000001,"5min":5.0549999999999997,"15min":212.655},"last":2.5299999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:00:31 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.6899999999999999,"15min":2.149},"min":{"1min":0.252,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":4.2519999999999998,"5min":55.212000000000003,"15min":114.422},"last":4.258},{"interface":"front","average":{"1min":0.69299999999999995,"5min":1.5880000000000001,"15min":2.2130000000000001},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":3.71,"5min":55.430999999999997,"15min":111.708},"last":6.2629999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:00:31 2023","interfaces":[{"interface":"back","average":{"1min":5.6950000000000003,"5min":1.9339999999999999,"15min":2.7400000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28499999999999998},"max":{"1min":89.778000000000006,"5min":89.778000000000006,"15min":205.863},"last":4.3029999999999999},{"interface":"front","average":{"1min":5.9210000000000003,"5min":1.827,"15min":2.6019999999999999},"min":{"1min":0.35599999999999998,"5min":0.308,"15min":0.28299999999999997},"max":{"1min":89.912999999999997,"5min":89.912999999999997,"15min":205.88399999999999},"last":4.3140000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:00:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":0.92000000000000004,"15min":2.8250000000000002},"min":{"1min":0.37,"5min":0.34000000000000002,"15min":0.28399999999999997},"max":{"1min":6.4130000000000003,"5min":9.6820000000000004,"15min":115.69799999999999},"last":6.9619999999999997},{"interface":"front","average":{"1min":1.0609999999999999,"5min":0.98499999999999999,"15min":3.081},"min":{"1min":0.29799999999999999,"5min":0.29699999999999999,"15min":0.29299999999999998},"max":{"1min":3.8340000000000001,"5min":5.0890000000000004,"15min":115.75700000000001},"last":2.5649999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:00:31 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.69499999999999995,"15min":1.9359999999999999},"min":{"1min":0.30499999999999999,"5min":0.27800000000000002,"15min":0.26000000000000001},"max":{"1min":3.8029999999999999,"5min":4.8390000000000004,"15min":159.50299999999999},"last":4.524},{"interface":"front","average":{"1min":1.3919999999999999,"5min":1.032,"15min":2.0510000000000002},"min":{"1min":0.32300000000000001,"5min":0.28299999999999997,"15min":0.27800000000000002},"max":{"1min":4.875,"5min":4.9800000000000004,"15min":159.483},"last":2.609}]},{"osd":5,"last update":"Mon Oct 9 06:00:31 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.1419999999999999,"15min":1.8160000000000001},"min":{"1min":0.39400000000000002,"5min":0.29399999999999998,"15min":0.27800000000000002},"max":{"1min":5.2069999999999999,"5min":8.968,"15min":94.093999999999994},"last":2.5870000000000002},{"interface":"front","average":{"1min":0.92900000000000005,"5min":0.98699999999999999,"15min":1.8620000000000001},"min":{"1min":0.39400000000000002,"5min":0.35999999999999999,"15min":0.308},"max":{"1min":4.1909999999999998,"5min":8.1010000000000009,"15min":95.579999999999998},"last":4.1879999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:00:31 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":3.415,"15min":1.7210000000000001},"min":{"1min":0.439,"5min":0.34200000000000003,"15min":0.29299999999999998},"max":{"1min":7.0999999999999996,"5min":209.322,"15min":209.322},"last":4.5359999999999996},{"interface":"front","average":{"1min":1.1819999999999999,"5min":3.4369999999999998,"15min":1.7669999999999999},"min":{"1min":0.40600000000000003,"5min":0.308,"15min":0.26700000000000002},"max":{"1min":4.2190000000000003,"5min":209.19800000000001,"15min":209.19800000000001},"last":4.2300000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626941,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1293376,"kb_used_data":389240,"kb_used_omap":0,"kb_used_meta":904128,"kb_avail":92451776,"statfs":{"total":95995035648,"available":94670618624,"internally_reserved":0,"allocated":398581760,"data_stored":397503560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925827072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:03 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.61199999999999999,"15min":1.5700000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20999999999999999},"max":{"1min":2.9780000000000002,"5min":5.5129999999999999,"15min":50.427},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.133,"5min":0.86299999999999999,"15min":1.7030000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.22700000000000001},"max":{"1min":8.1430000000000007,"5min":8.1430000000000007,"15min":50.634999999999998},"last":0.249}]},{"osd":1,"last update":"Mon Oct 9 06:00:29 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":2.3490000000000002,"15min":2.7519999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":19.596,"5min":129.822,"15min":165.66399999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.79300000000000004,"5min":2.0550000000000002,"15min":2.71},"min":{"1min":0.35599999999999998,"5min":0.246,"15min":0.246},"max":{"1min":5.5499999999999998,"5min":129.83600000000001,"15min":165.649},"last":0.27300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:01:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":0.874,"15min":1.337},"min":{"1min":0.33900000000000002,"5min":0.224,"15min":0.224},"max":{"1min":5.5640000000000001,"5min":6.2320000000000002,"15min":91.049999999999997},"last":0.308},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.1879999999999999,"15min":1.498},"min":{"1min":0.35899999999999999,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":6.3330000000000002,"5min":13.996,"15min":91.105000000000004},"last":0.56100000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:00:29 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.0700000000000001,"15min":2.9980000000000002},"min":{"1min":0.316,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":13.904,"5min":13.904,"15min":152.74700000000001},"last":2.2000000000000002},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.0669999999999999,"15min":2.9550000000000001},"min":{"1min":0.42999999999999999,"5min":0.35399999999999998,"15min":0.313},"max":{"1min":12.284000000000001,"5min":12.284000000000001,"15min":152.78100000000001},"last":0.39800000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:00:52 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.65500000000000003,"15min":1.8220000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.248},"max":{"1min":2.9489999999999998,"5min":11.009,"15min":257.375},"last":0.37},{"interface":"front","average":{"1min":1.3560000000000001,"5min":0.90100000000000002,"15min":2.0209999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.246},"max":{"1min":8.2349999999999994,"5min":8.2349999999999994,"15min":257.61200000000002},"last":0.66500000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:00:52 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.72799999999999998,"15min":2.3119999999999998},"min":{"1min":0.39000000000000001,"5min":0.31900000000000001,"15min":0.27800000000000002},"max":{"1min":3.04,"5min":8.1280000000000001,"15min":256.05700000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.494,"5min":1.0349999999999999,"15min":2.5350000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.28699999999999998},"max":{"1min":8.2040000000000006,"5min":8.2040000000000006,"15min":256.08499999999998},"last":0.46700000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:00:29 2023","interfaces":[{"interface":"back","average":{"1min":6.6180000000000003,"5min":2.0099999999999998,"15min":2.056},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.31},"max":{"1min":101.968,"5min":101.968,"15min":204.15299999999999},"last":0.65200000000000002},{"interface":"front","average":{"1min":6.9889999999999999,"5min":2.1720000000000002,"15min":2.2749999999999999},"min":{"1min":0.35699999999999998,"5min":0.34200000000000003,"15min":0.28199999999999997},"max":{"1min":102.001,"5min":102.001,"15min":203.85499999999999},"last":0.61399999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958901,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2037900,"kb_used_data":1109572,"kb_used_omap":7,"kb_used_meta":928312,"kb_avail":91707252,"statfs":{"total":95995035648,"available":93908226048,"internally_reserved":0,"allocated":1136201728,"data_stored":1134289087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":950592014},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:19 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.82299999999999995,"15min":2.4340000000000002},"min":{"1min":0.23300000000000001,"5min":0.214,"15min":0.17499999999999999},"max":{"1min":4.0339999999999998,"5min":6.96,"15min":131.91300000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.70099999999999996,"15min":2.6110000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.223},"max":{"1min":3.9350000000000001,"5min":5.6959999999999997,"15min":132.191},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.75900000000000001,"15min":2.032},"min":{"1min":0.252,"5min":0.221,"15min":0.221},"max":{"1min":3.8570000000000002,"5min":8.8100000000000005,"15min":132.35400000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.9590000000000001,"5min":0.96699999999999997,"15min":1.9339999999999999},"min":{"1min":0.26800000000000002,"5min":0.23699999999999999,"15min":0.20699999999999999},"max":{"1min":13.901999999999999,"5min":13.901999999999999,"15min":131.70599999999999},"last":0.48499999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:01:19 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.2190000000000001,"15min":2.4780000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.20200000000000001},"max":{"1min":4.4219999999999997,"5min":22.236000000000001,"15min":135.10900000000001},"last":0.437},{"interface":"front","average":{"1min":0.76100000000000001,"5min":1.198,"15min":2.4199999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.248},"max":{"1min":2.9009999999999998,"5min":22.913,"15min":131.886},"last":1.3420000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:01:19 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.874,"15min":1.7789999999999999},"min":{"1min":0.27800000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.1470000000000002,"5min":6.1470000000000002,"15min":131.636},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.012,"15min":1.6779999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.4969999999999999,"5min":9.1010000000000009,"15min":131.75999999999999},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:01:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.105,"15min":2.3599999999999999},"min":{"1min":0.32700000000000001,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":4.3520000000000003,"5min":8.7300000000000004,"15min":132.291},"last":1.0489999999999999},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.429,"15min":2.3969999999999998},"min":{"1min":0.373,"5min":0.33700000000000002,"15min":0.32200000000000001},"max":{"1min":4.024,"5min":22.268999999999998,"15min":131.738},"last":0.46300000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:01:19 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":1.3240000000000001,"15min":2.4500000000000002},"min":{"1min":0.375,"5min":0.34200000000000003,"15min":0.33300000000000002},"max":{"1min":7.5919999999999996,"5min":12.513999999999999,"15min":158.50399999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.242,"15min":2.3570000000000002},"min":{"1min":0.38900000000000001,"5min":0.377,"15min":0.37},"max":{"1min":4.7960000000000003,"5min":9.8559999999999999,"15min":158.577},"last":0.53200000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:01:19 2023","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":1.986,"15min":2.0659999999999998},"min":{"1min":0.44700000000000001,"5min":0.31,"15min":0.31},"max":{"1min":4.7560000000000002,"5min":89.787000000000006,"15min":131.821},"last":1.831},{"interface":"front","average":{"1min":1.895,"5min":2.2000000000000002,"15min":2.2549999999999999},"min":{"1min":0.45100000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.2779999999999996,"5min":89.869,"15min":131.852},"last":0.55200000000000005}]}]},{"osd":0,"up_from":88,"seq":377957122427,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1519892,"kb_used_data":724556,"kb_used_omap":8,"kb_used_meta":795319,"kb_avail":92225260,"statfs":{"total":95995035648,"available":94438666240,"internally_reserved":0,"allocated":741945344,"data_stored":739975036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":814407432},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:01:22 2023","interfaces":[{"interface":"back","average":{"1min":15.24,"5min":4.6619999999999999,"15min":2.677},"min":{"1min":0.20300000000000001,"5min":0.191,"15min":0.186},"max":{"1min":163.572,"5min":163.572,"15min":163.572},"last":0.26000000000000001},{"interface":"front","average":{"1min":15.175000000000001,"5min":4.6790000000000003,"15min":2.8300000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.19600000000000001},"max":{"1min":162.48699999999999,"5min":162.48699999999999,"15min":162.48699999999999},"last":0.33000000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.58899999999999997,"15min":1.9770000000000001},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20000000000000001},"max":{"1min":1.4019999999999999,"5min":6.2679999999999998,"15min":70.188999999999993},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.71399999999999997,"15min":1.706},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":4.6879999999999997,"5min":9.8559999999999999,"15min":71.144000000000005},"last":0.48999999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:01:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.54400000000000004,"15min":1.696},"min":{"1min":0.26500000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.5329999999999999,"5min":3.1120000000000001,"15min":62.698999999999998},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.73399999999999999,"15min":2.0939999999999999},"min":{"1min":0.30399999999999999,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":5.0279999999999996,"5min":5.8810000000000002,"15min":72.632999999999996},"last":0.34599999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.71699999999999997,"15min":1.6419999999999999},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.28399999999999997},"max":{"1min":4.319,"5min":9.8949999999999996,"15min":149.63999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.85699999999999998,"15min":1.833},"min":{"1min":0.33900000000000002,"5min":0.32300000000000001,"15min":0.312},"max":{"1min":4.3300000000000001,"5min":9.8409999999999993,"15min":149.65899999999999},"last":0.61399999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":4.2000000000000002,"15min":4.0099999999999998},"min":{"1min":0.34999999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.6790000000000003,"5min":210.77799999999999,"15min":246.761},"last":4.508},{"interface":"front","average":{"1min":1.3360000000000001,"5min":4.5369999999999999,"15min":4.3890000000000002},"min":{"1min":0.38500000000000001,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":10.891,"5min":209.91,"15min":247.48400000000001},"last":0.44700000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.84999999999999998,"15min":1.2270000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.4449999999999998,"5min":8.2899999999999991,"15min":19.579000000000001},"last":0.504},{"interface":"front","average":{"1min":0.67000000000000004,"5min":1.03,"15min":1.383},"min":{"1min":0.33800000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.5580000000000001,"5min":12.457000000000001,"15min":19.710999999999999},"last":0.40999999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:01:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9379999999999999,"5min":2.8570000000000002,"15min":2.508},"min":{"1min":0.39300000000000002,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":11.430999999999999,"5min":124.90900000000001,"15min":146.536},"last":4.7199999999999998},{"interface":"front","average":{"1min":1.6910000000000001,"5min":2.8599999999999999,"15min":2.6139999999999999},"min":{"1min":0.45600000000000002,"5min":0.40100000000000002,"15min":0.36299999999999999},"max":{"1min":11.446,"5min":125.041,"15min":146.49700000000001},"last":4.6909999999999998}]}]},{"osd":2,"up_from":98,"seq":420906795375,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1252848,"kb_used_data":372968,"kb_used_omap":11,"kb_used_meta":879860,"kb_avail":92492304,"statfs":{"total":95995035648,"available":94712119296,"internally_reserved":0,"allocated":381919232,"data_stored":379685202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":900977237},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:07 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.61399999999999999,"15min":2.02},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19500000000000001},"max":{"1min":1.923,"5min":9.7070000000000007,"15min":140.71100000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.443,"5min":0.504,"15min":2.2690000000000001},"min":{"1min":0.23599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.0009999999999999,"5min":2.8340000000000001,"15min":142.542},"last":4.8280000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:01:07 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":1.3069999999999999,"15min":2.4209999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":4.5519999999999996,"5min":75.260000000000005,"15min":125.77500000000001},"last":4.8810000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":1.3779999999999999,"15min":2.48},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":2.4860000000000002,"5min":75.114999999999995,"15min":125.857},"last":0.86799999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:00:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.80400000000000005,"15min":1.583},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.21099999999999999},"max":{"1min":2.617,"5min":15.193,"15min":29.606999999999999},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.81399999999999995,"15min":1.468},"min":{"1min":0.252,"5min":0.252,"15min":0.217},"max":{"1min":0.92100000000000004,"5min":9.7680000000000007,"15min":29.920999999999999},"last":0.64700000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:01:07 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.72699999999999998,"15min":1.349},"min":{"1min":0.34100000000000003,"5min":0.32200000000000001,"15min":0.28699999999999998},"max":{"1min":2.5600000000000001,"5min":4.7190000000000003,"15min":68.052000000000007},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.77000000000000002,"15min":1.3180000000000001},"min":{"1min":0.32900000000000001,"5min":0.313,"15min":0.29199999999999998},"max":{"1min":1.534,"5min":9.5800000000000001,"15min":68.727000000000004},"last":0.624}]},{"osd":5,"last update":"Mon Oct 9 06:01:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.88300000000000001,"15min":2.7400000000000002},"min":{"1min":0.376,"5min":0.34599999999999997,"15min":0.29099999999999998},"max":{"1min":2.024,"5min":18.350000000000001,"15min":207.80500000000001},"last":4.9359999999999999},{"interface":"front","average":{"1min":1.6759999999999999,"5min":1.1819999999999999,"15min":2.8879999999999999},"min":{"1min":0.39400000000000002,"5min":0.371,"15min":0.30499999999999999},"max":{"1min":9.4239999999999995,"5min":9.4239999999999995,"15min":207.54900000000001},"last":0.89300000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:01:07 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":1.0329999999999999,"15min":1.1990000000000001},"min":{"1min":0.39300000000000002,"5min":0.31,"15min":0.31},"max":{"1min":1.456,"5min":9.9320000000000004,"15min":25.818999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.80300000000000005,"15min":1.3779999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.34100000000000003},"max":{"1min":1.853,"5min":4.0220000000000002,"15min":24.481000000000002},"last":4.9100000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:00:40 2023","interfaces":[{"interface":"back","average":{"1min":3.4540000000000002,"5min":2.75,"15min":2.2930000000000001},"min":{"1min":0.42699999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":51.749000000000002,"5min":122.827,"15min":162.09200000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":3.7919999999999998,"5min":2.738,"15min":2.468},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.378},"max":{"1min":51.801000000000002,"5min":123.56399999999999,"15min":162.35400000000001},"last":4.96}]}]},{"osd":3,"up_from":103,"seq":442381631850,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1667540,"kb_used_data":673868,"kb_used_omap":4,"kb_used_meta":993659,"kb_avail":92077612,"statfs":{"total":95995035648,"available":94287474688,"internally_reserved":0,"allocated":690040832,"data_stored":688068718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1017507422},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:21 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.64000000000000001,"15min":3.6659999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.188},"max":{"1min":1.671,"5min":12.728,"15min":312.56599999999997},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.71099999999999997,"15min":3.6459999999999999},"min":{"1min":0.23100000000000001,"5min":0.20899999999999999,"15min":0.20499999999999999},"max":{"1min":1.903,"5min":7.6360000000000001,"15min":311.02300000000002},"last":0.53100000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:01:21 2023","interfaces":[{"interface":"back","average":{"1min":16.75,"5min":3.8399999999999999,"15min":4.5990000000000002},"min":{"1min":0.27400000000000002,"5min":0.24199999999999999,"15min":0.214},"max":{"1min":137.00299999999999,"5min":137.00299999999999,"15min":150.29599999999999},"last":0.24299999999999999},{"interface":"front","average":{"1min":16.812999999999999,"5min":3.8690000000000002,"15min":4.7599999999999998},"min":{"1min":0.28499999999999998,"5min":0.23799999999999999,"15min":0.216},"max":{"1min":136.947,"5min":136.947,"15min":151.25700000000001},"last":0.34100000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:01:21 2023","interfaces":[{"interface":"back","average":{"1min":5.3070000000000004,"5min":2.1070000000000002,"15min":2.161},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.23899999999999999},"max":{"1min":87.542000000000002,"5min":87.542000000000002,"15min":87.542000000000002},"last":2.5670000000000002},{"interface":"front","average":{"1min":5.4000000000000004,"5min":2.21,"15min":2.1280000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.20499999999999999},"max":{"1min":87.727999999999994,"5min":87.727999999999994,"15min":87.727999999999994},"last":1.4390000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:01:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.68200000000000005,"15min":1.7809999999999999},"min":{"1min":0.33700000000000002,"5min":0.33400000000000002,"15min":0.28999999999999998},"max":{"1min":1.6299999999999999,"5min":3.278,"15min":126.566},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.745,"15min":2.1699999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":1.671,"5min":7.1390000000000002,"15min":126.651},"last":0.55200000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:01:21 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.68999999999999995,"15min":1.712},"min":{"1min":0.36099999999999999,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":3.173,"5min":4.3540000000000001,"15min":71.203999999999994},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.745,"5min":0.88100000000000001,"15min":1.569},"min":{"1min":0.39400000000000002,"5min":0.33400000000000002,"15min":0.29399999999999998},"max":{"1min":3.1539999999999999,"5min":12.778,"15min":70.397000000000006},"last":0.58999999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:01:21 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.79600000000000004,"15min":1.3560000000000001},"min":{"1min":0.44600000000000001,"5min":0.36699999999999999,"15min":0.34499999999999997},"max":{"1min":2.0259999999999998,"5min":7.1719999999999997,"15min":70.474000000000004},"last":1.0189999999999999},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.75700000000000001,"15min":1.5109999999999999},"min":{"1min":0.40600000000000003,"5min":0.36599999999999999,"15min":0.33200000000000002},"max":{"1min":1.038,"5min":5.1459999999999999,"15min":71.251999999999995},"last":0.438}]},{"osd":7,"last update":"Mon Oct 9 06:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.222,"15min":1.5289999999999999},"min":{"1min":0.40000000000000002,"5min":0.38300000000000001,"15min":0.35699999999999998},"max":{"1min":9.1720000000000006,"5min":11.948,"15min":69.277000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.3160000000000001,"15min":1.556},"min":{"1min":0.49299999999999999,"5min":0.40000000000000002,"15min":0.374},"max":{"1min":9.2490000000000006,"5min":11.722,"15min":69.299000000000007},"last":0.48399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019397,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1213584,"kb_used_data":303944,"kb_used_omap":0,"kb_used_meta":909632,"kb_avail":92531568,"statfs":{"total":95995035648,"available":94752325632,"internally_reserved":0,"allocated":311238656,"data_stored":310151718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":931463168},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.65400000000000003,"15min":3.012},"min":{"1min":0.26200000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.3359999999999999,"5min":6.0019999999999998,"15min":132.35599999999999},"last":50.106999999999999},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.1100000000000001,"15min":3.3929999999999998},"min":{"1min":0.30299999999999999,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.827,"5min":12.462,"15min":132.501},"last":49.334000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.85699999999999998,"15min":2.5579999999999998},"min":{"1min":0.28999999999999998,"5min":0.27400000000000002,"15min":0.26600000000000001},"max":{"1min":3.327,"5min":11.215,"15min":96.033000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.77800000000000002,"5min":1.0760000000000001,"15min":2.5470000000000002},"min":{"1min":0.32600000000000001,"5min":0.29099999999999998,"15min":0.255},"max":{"1min":3.7650000000000001,"5min":12.516,"15min":96.022000000000006},"last":0.46100000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:00:46 2023","interfaces":[{"interface":"back","average":{"1min":8.9030000000000005,"5min":2.2890000000000001,"15min":2.1059999999999999},"min":{"1min":0.313,"5min":0.29599999999999999,"15min":0.28599999999999998},"max":{"1min":143.71000000000001,"5min":143.71000000000001,"15min":143.71000000000001},"last":0.372},{"interface":"front","average":{"1min":8.7309999999999999,"5min":3.3119999999999998,"15min":2.347},"min":{"1min":0.40100000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":143.768,"5min":143.768,"15min":143.768},"last":0.34499999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:01:23 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.145,"15min":2.8929999999999998},"min":{"1min":0.36899999999999999,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":4.8890000000000002,"5min":11.055999999999999,"15min":123.59},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.81699999999999995,"15min":2.7839999999999998},"min":{"1min":0.36199999999999999,"5min":0.32900000000000001,"15min":0.30499999999999999},"max":{"1min":2.9670000000000001,"5min":5.9729999999999999,"15min":123.566},"last":0.44900000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:01:23 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.2809999999999999,"15min":1.0669999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.25900000000000001},"max":{"1min":3.6930000000000001,"5min":15.597,"15min":15.597},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.018,"5min":1.1679999999999999,"15min":1.022},"min":{"1min":0.39400000000000002,"5min":0.34599999999999997,"15min":0.29799999999999999},"max":{"1min":3.516,"5min":17.882000000000001,"15min":17.882000000000001},"last":0.47899999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.526,"15min":1.6319999999999999},"min":{"1min":0.36699999999999999,"5min":0.314,"15min":0.27600000000000002},"max":{"1min":2.7120000000000002,"5min":38.676000000000002,"15min":181.411},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.5549999999999999,"5min":1.589,"15min":1.7170000000000001},"min":{"1min":0.39000000000000001,"5min":0.32200000000000001,"15min":0.29799999999999999},"max":{"1min":7.3949999999999996,"5min":38.438000000000002,"15min":180.63999999999999},"last":0.432}]},{"osd":7,"last update":"Mon Oct 9 06:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":1.3380000000000001,"15min":1.46},"min":{"1min":0.40100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.0089999999999999,"5min":17.832000000000001,"15min":96.131},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.89400000000000002,"5min":1.105,"15min":1.395},"min":{"1min":0.35899999999999999,"5min":0.32900000000000001,"15min":0.27200000000000002},"max":{"1min":3.3940000000000001,"5min":17.742999999999999,"15min":96.465000000000003},"last":0.442}]}]},{"osd":5,"up_from":0,"seq":154618823169,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":732561408,"data_stored":732561408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1125453824,"data_stored":1125449752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":371589120,"data_stored":371589120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":677642240,"data_stored":677642240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":302714880,"data_stored":302710808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1604653056,"data_stored":1604648984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":386465792,"data_stored":386465792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":601030656,"data_stored":601030656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:01:36.099 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 26 2023-10-09T06:01:36.115 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:36.138 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:36.288 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:36 smithi008 ceph-mon[191227]: from='client.36987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:01:36.288 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:36 smithi008 ceph-mon[191227]: pgmap v1490: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 893 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 328 op/s; 49660/42726 objects degraded (116.229%); 3250/42726 objects misplaced (7.607%); 9.2 MiB/s, 232 objects/s recovering 2023-10-09T06:01:36.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:36 smithi191 ceph-mon[161669]: from='client.36987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:01:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:36 smithi191 ceph-mon[161669]: pgmap v1490: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 893 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 328 op/s; 49660/42726 objects degraded (116.229%); 3250/42726 objects misplaced (7.607%); 9.2 MiB/s, 232 objects/s recovering 2023-10-09T06:01:36.428 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:36.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:36.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:36.533 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:36.564 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:36.622 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:36.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:36.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:36.729 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:36.778 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:36.863 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:01:36.882 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:01:36.897 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:36.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:36.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:36.906 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:01:36.934 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:01:36.958 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:01:36.982 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:01:37.004 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:37.010 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:01:37.042 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:37.047 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:01:37.060 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:01:37.060 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:01:37.061 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:01:37.061 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:01:37.061 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:01:37.061 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:01:37.061 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:01:37.061 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:01:37.092 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:37 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3349689708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:37.092 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:37 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3349689708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:37.105 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:37.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:37.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:37.240 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:37.288 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:37.292 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:37 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3349689708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:37.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:37.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:37.394 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:37.429 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:38.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:38 smithi191 ceph-mon[161669]: pgmap v1491: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 825 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 362 op/s; 48989/39453 objects degraded (124.171%); 3122/39453 objects misplaced (7.913%); 9.9 MiB/s, 238 objects/s recovering 2023-10-09T06:01:38.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:38 smithi191 ceph-mon[161669]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:38 smithi191 ceph-mon[161669]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:38 smithi191 ceph-mon[161669]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:38 smithi191 ceph-mon[161669]: osdmap e497: 8 total, 8 up, 7 in 2023-10-09T06:01:38.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:38 smithi191 ceph-mon[161669]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:38 smithi191 ceph-mon[161669]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.306 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[184418]: pgmap v1491: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 825 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 362 op/s; 48989/39453 objects degraded (124.171%); 3122/39453 objects misplaced (7.913%); 9.9 MiB/s, 238 objects/s recovering 2023-10-09T06:01:38.306 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[184418]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.306 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[184418]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.307 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[184418]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.307 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[184418]: osdmap e497: 8 total, 8 up, 7 in 2023-10-09T06:01:38.307 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[184418]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.307 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[184418]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.309 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[191227]: pgmap v1491: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 825 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 362 op/s; 48989/39453 objects degraded (124.171%); 3122/39453 objects misplaced (7.913%); 9.9 MiB/s, 238 objects/s recovering 2023-10-09T06:01:38.309 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[191227]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.309 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[191227]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.309 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[191227]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.310 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[191227]: osdmap e497: 8 total, 8 up, 7 in 2023-10-09T06:01:38.310 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[191227]: failed to encode map e497 with expected crc 2023-10-09T06:01:38.310 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:38 smithi008 ceph-mon[191227]: failed to encode map e497 with expected crc 2023-10-09T06:01:39.057 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:39.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:39.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:39.138 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:39.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:39.200 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:39.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[184418]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[184418]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[184418]: osdmap e498: 8 total, 8 up, 7 in 2023-10-09T06:01:39.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[184418]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[184418]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[184418]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[184418]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 48989/39453 objects degraded (124.171%), 25 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[191227]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[191227]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[191227]: osdmap e498: 8 total, 8 up, 7 in 2023-10-09T06:01:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[191227]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[191227]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[191227]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[191227]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:39 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 48989/39453 objects degraded (124.171%), 25 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:39.268 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:39.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:39.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:39 smithi191 ceph-mon[161669]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:39 smithi191 ceph-mon[161669]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:39 smithi191 ceph-mon[161669]: osdmap e498: 8 total, 8 up, 7 in 2023-10-09T06:01:39.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:39 smithi191 ceph-mon[161669]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:39 smithi191 ceph-mon[161669]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:39 smithi191 ceph-mon[161669]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:39 smithi191 ceph-mon[161669]: failed to encode map e498 with expected crc 2023-10-09T06:01:39.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:39 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 48989/39453 objects degraded (124.171%), 25 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:39.379 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:39.434 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:39.625 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:39.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:39.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:39.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:39.762 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:39.816 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:39.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:39.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:39.925 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:39.964 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:40.024 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:40.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:40.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:40.086 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:01:40.086 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:01:40.132 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:40.172 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:40.244 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:40.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:40.331 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:40 smithi008 ceph-mon[191227]: pgmap v1494: 99 pgs: 1 peering, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 780 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 45564/37308 objects degraded (122.129%); 3446/37308 objects misplaced (9.237%); 11 MiB/s, 281 objects/s recovering 2023-10-09T06:01:40.331 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:40 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1819111166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:40.331 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:40 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:40.344 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:01:40.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:40.407 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:40.461 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:40.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:40.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:40.571 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:40.602 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:40 smithi191 ceph-mon[161669]: pgmap v1494: 99 pgs: 1 peering, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 780 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 45564/37308 objects degraded (122.129%); 3446/37308 objects misplaced (9.237%); 11 MiB/s, 281 objects/s recovering 2023-10-09T06:01:40.602 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:40 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1819111166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:40.603 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:40 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:40.604 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:40.658 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:40.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:40.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:40.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:40 smithi008 ceph-mon[184418]: pgmap v1494: 99 pgs: 1 peering, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 780 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 45564/37308 objects degraded (122.129%); 3446/37308 objects misplaced (9.237%); 11 MiB/s, 281 objects/s recovering 2023-10-09T06:01:40.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:40 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1819111166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:40.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:40 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:40.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:01:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:01:40] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.33.4" 2023-10-09T06:01:40.763 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:40.821 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:41.076 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:01:41.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[184418]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[184418]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:01:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[184418]: osdmap e499: 8 total, 8 up, 7 in 2023-10-09T06:01:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[184418]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[184418]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[184418]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[184418]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[191227]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[191227]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:01:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[191227]: osdmap e499: 8 total, 8 up, 7 in 2023-10-09T06:01:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[191227]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[191227]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[191227]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:41 smithi008 ceph-mon[191227]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:41 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:41 smithi191 ceph-mon[161669]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:41 smithi191 ceph-mon[161669]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:41 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:01:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:41 smithi191 ceph-mon[161669]: osdmap e499: 8 total, 8 up, 7 in 2023-10-09T06:01:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:41 smithi191 ceph-mon[161669]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:41 smithi191 ceph-mon[161669]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:41 smithi191 ceph-mon[161669]: failed to encode map e499 with expected crc 2023-10-09T06:01:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:41 smithi191 ceph-mon[161669]: failed to encode map e499 with expected crc 2023-10-09T06:01:42.527 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:42.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:42.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:42.628 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:42 smithi008 ceph-mon[184418]: pgmap v1496: 99 pgs: 1 peering, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 754 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 45268/36051 objects degraded (125.567%); 3446/36051 objects misplaced (9.559%); 7.9 MiB/s, 235 objects/s recovering 2023-10-09T06:01:42.629 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:42 smithi008 ceph-mon[191227]: pgmap v1496: 99 pgs: 1 peering, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 754 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 45268/36051 objects degraded (125.567%); 3446/36051 objects misplaced (9.559%); 7.9 MiB/s, 235 objects/s recovering 2023-10-09T06:01:42.632 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:42.666 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:42.667 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:42 smithi191 ceph-mon[161669]: pgmap v1496: 99 pgs: 1 peering, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 754 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 45268/36051 objects degraded (125.567%); 3446/36051 objects misplaced (9.559%); 7.9 MiB/s, 235 objects/s recovering 2023-10-09T06:01:42.701 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:01:42.723 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:42.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:42.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:42.830 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:42.862 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:42.950 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:42.995 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:42.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:42.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:43.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:43.138 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:43.188 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:43.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:43.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:43.298 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:43.326 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:43.383 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:43.387 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 45268/36051 objects degraded (125.567%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:43.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:43.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:43.489 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:43.538 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 45268/36051 objects degraded (125.567%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:43.539 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 45268/36051 objects degraded (125.567%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:43.544 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:43.868 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:43.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:43.972 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:44.000 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:44.045 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:44.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:44.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:44.149 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:44.184 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:44.254 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:44.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:44.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:44.366 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:44.368 INFO:teuthology.orchestra.run.smithi008.stderr:set pool 9 pgp_num to 26 2023-10-09T06:01:44.397 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:44.453 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:44.459 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:44 smithi191 ceph-mon[161669]: pgmap v1497: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 701 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 329 op/s; 44810/33519 objects degraded (133.685%); 3360/33519 objects misplaced (10.024%); 6.7 MiB/s, 214 objects/s recovering 2023-10-09T06:01:44.459 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3473996624' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:01:44.459 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:44.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:44.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:44.561 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:44.569 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:01:44.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:44 smithi008 ceph-mon[184418]: pgmap v1497: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 701 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 329 op/s; 44810/33519 objects degraded (133.685%); 3360/33519 objects misplaced (10.024%); 6.7 MiB/s, 214 objects/s recovering 2023-10-09T06:01:44.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3473996624' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:01:44.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:44.608 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:44 smithi008 ceph-mon[191227]: pgmap v1497: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 701 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 329 op/s; 44810/33519 objects degraded (133.685%); 3360/33519 objects misplaced (10.024%); 6.7 MiB/s, 214 objects/s recovering 2023-10-09T06:01:44.608 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3473996624' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:01:44.608 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:44.609 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:44.688 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:44.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:44.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:44.797 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:44.827 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:45.123 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:45.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:45.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:45.197 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:45.227 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:45.286 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:45.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:45 smithi008 ceph-mon[191227]: failed to encode map e500 with expected crc 2023-10-09T06:01:45.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:45 smithi008 ceph-mon[191227]: failed to encode map e500 with expected crc 2023-10-09T06:01:45.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:45 smithi008 ceph-mon[191227]: failed to encode map e500 with expected crc 2023-10-09T06:01:45.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3473996624' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T06:01:45.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:45 smithi008 ceph-mon[191227]: osdmap e500: 8 total, 8 up, 7 in 2023-10-09T06:01:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:45 smithi008 ceph-mon[184418]: failed to encode map e500 with expected crc 2023-10-09T06:01:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:45 smithi008 ceph-mon[184418]: failed to encode map e500 with expected crc 2023-10-09T06:01:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:45 smithi008 ceph-mon[184418]: failed to encode map e500 with expected crc 2023-10-09T06:01:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3473996624' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T06:01:45.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:45 smithi008 ceph-mon[184418]: osdmap e500: 8 total, 8 up, 7 in 2023-10-09T06:01:45.523 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:45.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:45.628 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:45.666 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:45.710 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:45.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:45 smithi191 ceph-mon[161669]: failed to encode map e500 with expected crc 2023-10-09T06:01:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:45 smithi191 ceph-mon[161669]: failed to encode map e500 with expected crc 2023-10-09T06:01:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:45 smithi191 ceph-mon[161669]: failed to encode map e500 with expected crc 2023-10-09T06:01:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3473996624' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T06:01:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:45 smithi191 ceph-mon[161669]: osdmap e500: 8 total, 8 up, 7 in 2023-10-09T06:01:45.879 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:01:45.903 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:01:45.932 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:01:45.957 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:45.960 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:01:45.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:45.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:45.988 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:01:46.011 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:01:46.037 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:01:46.056 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:01:46.057 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.057 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.058 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.058 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.058 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.058 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.058 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.058 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.064 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:46.089 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:46.130 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:46.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:46.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:46.235 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:46.272 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:46.328 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:46.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:46 smithi191 ceph-mon[161669]: pgmap v1499: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 668 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 44869/31926 objects degraded (140.541%); 3247/31926 objects misplaced (10.170%); 4.9 MiB/s, 135 objects/s recovering 2023-10-09T06:01:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2451524516' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:46.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:46.461 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:46.510 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:46.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:46.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:46.617 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:46.648 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:46 smithi008 ceph-mon[184418]: pgmap v1499: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 668 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 44869/31926 objects degraded (140.541%); 3247/31926 objects misplaced (10.170%); 4.9 MiB/s, 135 objects/s recovering 2023-10-09T06:01:46.648 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2451524516' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:46.649 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:46 smithi008 ceph-mon[191227]: pgmap v1499: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 668 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 44869/31926 objects degraded (140.541%); 3247/31926 objects misplaced (10.170%); 4.9 MiB/s, 135 objects/s recovering 2023-10-09T06:01:46.649 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2451524516' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:46.654 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:46.910 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:46.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:46.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:47.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:47.045 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:47.105 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:47.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:47.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:47.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:47.242 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:47.297 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:47.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:47.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:47.405 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:47.433 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:47.504 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:47.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:47.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:47.615 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:47.644 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:47.697 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:47.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:47.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:47.814 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:47.846 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:47.900 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:47.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:47.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:48.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:48.030 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:48.109 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:48.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:48.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:48.221 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:48.276 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:48.609 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:48.615 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:48 smithi008 ceph-mon[191227]: pgmap v1500: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 605 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 350 op/s; 44240/28890 objects degraded (153.133%); 3135/28890 objects misplaced (10.852%); 7.5 MiB/s, 160 objects/s recovering 2023-10-09T06:01:48.615 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 44240/28890 objects degraded (153.133%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:48.615 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:48 smithi008 ceph-mon[184418]: pgmap v1500: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 605 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 350 op/s; 44240/28890 objects degraded (153.133%); 3135/28890 objects misplaced (10.852%); 7.5 MiB/s, 160 objects/s recovering 2023-10-09T06:01:48.616 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 44240/28890 objects degraded (153.133%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:48.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:48.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:48.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:48.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:48 smithi191 ceph-mon[161669]: pgmap v1500: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 605 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 350 op/s; 44240/28890 objects degraded (153.133%); 3135/28890 objects misplaced (10.852%); 7.5 MiB/s, 160 objects/s recovering 2023-10-09T06:01:48.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 44240/28890 objects degraded (153.133%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:48.743 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:48.752 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:01:48.752 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:01:48.752 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T06:01:48.752 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:01:48.792 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:48.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:48.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:48.897 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:48.943 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:49.173 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:49.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:49.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:49.280 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:49.333 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:49.423 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:49.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:49.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:49.527 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:49.564 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:49.888 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:49.890 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:01:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:49.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:49.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:50.027 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:50.101 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:50.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:50.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:50.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:50.234 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:01:50.234 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:01:50.235 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T06:01:50.235 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:01:50.235 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:01:50.235 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T06:01:50.240 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:50.340 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:50.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:50.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:50.424 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:01:50.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:50.504 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:01:50.532 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:50 smithi191 ceph-mon[161669]: pgmap v1501: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 565 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 44225/26985 objects degraded (163.887%); 3019/26985 objects misplaced (11.188%); 7.0 MiB/s, 182 objects/s recovering 2023-10-09T06:01:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2552562119' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:50.610 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:50.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:50.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:50 smithi008 ceph-mon[184418]: pgmap v1501: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 565 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 44225/26985 objects degraded (163.887%); 3019/26985 objects misplaced (11.188%); 7.0 MiB/s, 182 objects/s recovering 2023-10-09T06:01:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2552562119' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:50.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:01:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:01:50] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.33.4" 2023-10-09T06:01:50.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:50 smithi008 ceph-mon[191227]: pgmap v1501: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 565 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 44225/26985 objects degraded (163.887%); 3019/26985 objects misplaced (11.188%); 7.0 MiB/s, 182 objects/s recovering 2023-10-09T06:01:50.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2552562119' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:50.717 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:50.770 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:50.825 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:50.855 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:50.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:50.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:50.959 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:50.986 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:51.081 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:51.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:51.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:51.185 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:51.245 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:51.490 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:51.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:01:51.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[191227]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[191227]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2552562119' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:51.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[191227]: osdmap e501: 8 total, 8 up, 7 in 2023-10-09T06:01:51.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[191227]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[191227]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[191227]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.500 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[191227]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.508 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:01:51.508 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[184418]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.508 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[184418]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.508 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2552562119' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:51.508 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[184418]: osdmap e501: 8 total, 8 up, 7 in 2023-10-09T06:01:51.508 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[184418]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.508 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[184418]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.509 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[184418]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.509 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:51 smithi008 ceph-mon[184418]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.567 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:51.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:51.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:51.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:51.682 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:51.721 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:51 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:01:51.721 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:51 smithi191 ceph-mon[161669]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.721 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:51 smithi191 ceph-mon[161669]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.721 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:51 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2552562119' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:51.721 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:51 smithi191 ceph-mon[161669]: osdmap e501: 8 total, 8 up, 7 in 2023-10-09T06:01:51.721 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:51 smithi191 ceph-mon[161669]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.721 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:51 smithi191 ceph-mon[161669]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.721 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:51 smithi191 ceph-mon[161669]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.721 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:51 smithi191 ceph-mon[161669]: failed to encode map e501 with expected crc 2023-10-09T06:01:51.727 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:51.789 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:51.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:51.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:51.893 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:52.077 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:52.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:52.181 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:52.233 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:52.459 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:52.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:52.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:52.567 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:52.608 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:52 smithi008 ceph-mon[184418]: pgmap v1503: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 538 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 43895/25683 objects degraded (170.911%); 3019/25683 objects misplaced (11.755%); 5.7 MiB/s, 161 objects/s recovering 2023-10-09T06:01:52.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:52 smithi008 ceph-mon[191227]: pgmap v1503: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 538 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 43895/25683 objects degraded (170.911%); 3019/25683 objects misplaced (11.755%); 5.7 MiB/s, 161 objects/s recovering 2023-10-09T06:01:52.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:52 smithi191 ceph-mon[161669]: pgmap v1503: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 538 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 43895/25683 objects degraded (170.911%); 3019/25683 objects misplaced (11.755%); 5.7 MiB/s, 161 objects/s recovering 2023-10-09T06:01:53.341 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:53.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:53.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:53.447 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:53.479 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:53.538 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:01:53.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 43895/25683 objects degraded (170.911%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:53.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 43895/25683 objects degraded (170.911%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:53.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 43895/25683 objects degraded (170.911%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:54.081 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:54.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:54.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:54.185 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:54.221 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:54.332 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:54.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:54.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:54.407 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:54 smithi008 ceph-mon[191227]: pgmap v1504: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 485 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 43468/23130 objects degraded (187.929%); 2903/23130 objects misplaced (12.551%); 7.9 MiB/s, 162 objects/s recovering 2023-10-09T06:01:54.436 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:54.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:54 smithi191 ceph-mon[161669]: pgmap v1504: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 485 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 43468/23130 objects degraded (187.929%); 2903/23130 objects misplaced (12.551%); 7.9 MiB/s, 162 objects/s recovering 2023-10-09T06:01:54.455 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:54.486 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:54.666 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:54.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:54.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:54.673 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:54 smithi008 ceph-mon[184418]: pgmap v1504: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 485 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 349 op/s; 43468/23130 objects degraded (187.929%); 2903/23130 objects misplaced (12.551%); 7.9 MiB/s, 162 objects/s recovering 2023-10-09T06:01:54.674 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:01:54.674 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "" 2023-10-09T06:01:54.674 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:01:54.774 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:54.836 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:54.851 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:01:54.851 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:01:54.851 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T06:01:54.851 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T06:01:54.851 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:01:54.956 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:01:54.993 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:01:55.016 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:01:55.041 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:01:55.066 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:01:55.089 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:01:55.109 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:01:55.148 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:01:55.150 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:01:55.150 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:01:55.150 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:01:55.150 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:01:55.150 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:01:55.150 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:01:55.150 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:01:55.151 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:01:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:55 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/463552678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:55.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:55 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/463552678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:55.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:55 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/463552678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:56 smithi008 ceph-mon[184418]: pgmap v1505: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 452 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 345 op/s; 43528/21558 objects degraded (201.911%); 2782/21558 objects misplaced (12.905%); 6.6 MiB/s, 169 objects/s recovering 2023-10-09T06:01:56.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:56 smithi008 ceph-mon[191227]: pgmap v1505: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 452 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 345 op/s; 43528/21558 objects degraded (201.911%); 2782/21558 objects misplaced (12.905%); 6.6 MiB/s, 169 objects/s recovering 2023-10-09T06:01:56.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:56 smithi191 ceph-mon[161669]: pgmap v1505: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 452 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 345 op/s; 43528/21558 objects degraded (201.911%); 2782/21558 objects misplaced (12.905%); 6.6 MiB/s, 169 objects/s recovering 2023-10-09T06:01:57.488 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:57.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:57.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:57.593 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:57.630 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:57.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:57 smithi008 ceph-mon[184418]: pgmap v1506: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 388 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 346 op/s; 42903/18486 objects degraded (232.084%); 2669/18486 objects misplaced (14.438%); 6.6 MiB/s, 169 objects/s recovering 2023-10-09T06:01:57.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:57 smithi008 ceph-mon[191227]: pgmap v1506: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 388 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 346 op/s; 42903/18486 objects degraded (232.084%); 2669/18486 objects misplaced (14.438%); 6.6 MiB/s, 169 objects/s recovering 2023-10-09T06:01:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:57 smithi191 ceph-mon[161669]: pgmap v1506: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 388 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 346 op/s; 42903/18486 objects degraded (232.084%); 2669/18486 objects misplaced (14.438%); 6.6 MiB/s, 169 objects/s recovering 2023-10-09T06:01:58.263 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:01:58.413 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:58.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:58.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:58.503 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 42903/18486 objects degraded (232.084%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:58.504 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 42903/18486 objects degraded (232.084%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:58.522 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:58.568 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:58.700 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:58.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:58.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 42903/18486 objects degraded (232.084%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:01:58.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:58.852 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:58.923 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:58.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:58.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:59.031 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:59.096 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:59.113 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:59.190 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:59.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:59.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:59.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:59.332 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:59.408 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:59.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:59.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:59.519 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:59.550 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:59.619 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:59.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:59.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:59.724 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:59.763 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:59.831 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:59.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:59.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:59.937 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:59.949 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:01:59.978 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:00.048 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:00.051 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:59 smithi008 ceph-mon[184418]: pgmap v1507: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 341 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 42837/16245 objects degraded (263.693%); 2415/16245 objects misplaced (14.866%); 7.7 MiB/s, 221 objects/s recovering 2023-10-09T06:02:00.051 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:00.051 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:00.051 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:01:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4069543147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:00.052 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:59 smithi008 ceph-mon[191227]: pgmap v1507: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 341 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 42837/16245 objects degraded (263.693%); 2415/16245 objects misplaced (14.866%); 7.7 MiB/s, 221 objects/s recovering 2023-10-09T06:02:00.052 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:00.052 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:00.052 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:01:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4069543147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:00.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:00.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:00.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:00.182 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:00.234 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:00.240 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:59 smithi191 ceph-mon[161669]: pgmap v1507: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 341 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 42837/16245 objects degraded (263.693%); 2415/16245 objects misplaced (14.866%); 7.7 MiB/s, 221 objects/s recovering 2023-10-09T06:02:00.240 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:00.240 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:00.240 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:01:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4069543147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:00.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:00.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:00.261 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:02:00.342 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:00.389 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:00.464 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:02:00.465 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:02:00.465 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:02:00.465 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.560994 2023-10-09T06:02:00.465 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0.5609944599663401 2023-10-09T06:02:00.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:02:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:02:00] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.33.4" 2023-10-09T06:02:01.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[191227]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4069543147' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:02:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[191227]: osdmap e502: 8 total, 8 up, 7 in 2023-10-09T06:02:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[191227]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[191227]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[191227]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:01.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[184418]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4069543147' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:02:01.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[184418]: osdmap e502: 8 total, 8 up, 7 in 2023-10-09T06:02:01.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[184418]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[184418]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:00 smithi008 ceph-mon[184418]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:00 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:00 smithi191 ceph-mon[161669]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4069543147' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:02:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:00 smithi191 ceph-mon[161669]: osdmap e502: 8 total, 8 up, 7 in 2023-10-09T06:02:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:00 smithi191 ceph-mon[161669]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:00 smithi191 ceph-mon[161669]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:00 smithi191 ceph-mon[161669]: failed to encode map e502 with expected crc 2023-10-09T06:02:01.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:01.694 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:01.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:01.795 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:01.830 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:01.893 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:01.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:01.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:02.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:02.036 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:02.112 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:02.120 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:01 smithi191 ceph-mon[161669]: pgmap v1509: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 314 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 42542/14943 objects degraded (284.695%); 2415/14943 objects misplaced (16.161%); 7.6 MiB/s, 219 objects/s recovering 2023-10-09T06:02:02.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:02.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:02.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:01 smithi008 ceph-mon[184418]: pgmap v1509: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 314 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 42542/14943 objects degraded (284.695%); 2415/14943 objects misplaced (16.161%); 7.6 MiB/s, 219 objects/s recovering 2023-10-09T06:02:02.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:01 smithi008 ceph-mon[191227]: pgmap v1509: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 314 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 357 op/s; 42542/14943 objects degraded (284.695%); 2415/14943 objects misplaced (16.161%); 7.6 MiB/s, 219 objects/s recovering 2023-10-09T06:02:02.221 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:02.263 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:02.323 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:02.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:02.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:02.435 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:02.479 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:02.598 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:02.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:02.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:02.611 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:02.660 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:02.703 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:02.732 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:02.738 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:02:02.738 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:02:02.885 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:02.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:02.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:02.991 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:03.022 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:03.191 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:03.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:03.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:03.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:03.320 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:03.365 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:03.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:03.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:03.470 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:03.503 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:04.179 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:02:04.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[191227]: pgmap v1510: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 258 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 363 op/s; 42082/12234 objects degraded (343.976%); 2310/12234 objects misplaced (18.882%); 7.7 MiB/s, 220 objects/s recovering 2023-10-09T06:02:04.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[191227]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[191227]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[191227]: osdmap e503: 8 total, 8 up, 7 in 2023-10-09T06:02:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[191227]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[191227]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 42082/12234 objects degraded (343.976%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:04.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[184418]: pgmap v1510: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 258 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 363 op/s; 42082/12234 objects degraded (343.976%); 2310/12234 objects misplaced (18.882%); 7.7 MiB/s, 220 objects/s recovering 2023-10-09T06:02:04.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[184418]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[184418]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[184418]: osdmap e503: 8 total, 8 up, 7 in 2023-10-09T06:02:04.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[184418]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[184418]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 42082/12234 objects degraded (343.976%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:04.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:03 smithi191 ceph-mon[161669]: pgmap v1510: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 258 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 363 op/s; 42082/12234 objects degraded (343.976%); 2310/12234 objects misplaced (18.882%); 7.7 MiB/s, 220 objects/s recovering 2023-10-09T06:02:04.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:03 smithi191 ceph-mon[161669]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:03 smithi191 ceph-mon[161669]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:03 smithi191 ceph-mon[161669]: osdmap e503: 8 total, 8 up, 7 in 2023-10-09T06:02:04.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:03 smithi191 ceph-mon[161669]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:03 smithi191 ceph-mon[161669]: failed to encode map e503 with expected crc 2023-10-09T06:02:04.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 42082/12234 objects degraded (343.976%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:05.002 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:05.003 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:02:05.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:05.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:05.108 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:05.142 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:05.206 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:05.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:05.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:04 smithi008 ceph-mon[184418]: failed to encode map e504 with expected crc 2023-10-09T06:02:05.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:04 smithi008 ceph-mon[184418]: osdmap e504: 8 total, 8 up, 7 in 2023-10-09T06:02:05.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:04 smithi008 ceph-mon[184418]: failed to encode map e504 with expected crc 2023-10-09T06:02:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:04 smithi008 ceph-mon[191227]: failed to encode map e504 with expected crc 2023-10-09T06:02:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:04 smithi008 ceph-mon[191227]: osdmap e504: 8 total, 8 up, 7 in 2023-10-09T06:02:05.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:04 smithi008 ceph-mon[191227]: failed to encode map e504 with expected crc 2023-10-09T06:02:05.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:04 smithi191 ceph-mon[161669]: failed to encode map e504 with expected crc 2023-10-09T06:02:05.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:04 smithi191 ceph-mon[161669]: osdmap e504: 8 total, 8 up, 7 in 2023-10-09T06:02:05.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:04 smithi191 ceph-mon[161669]: failed to encode map e504 with expected crc 2023-10-09T06:02:05.311 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:05.348 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:05.414 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:05.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:05.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:05.439 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:05.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:05.550 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:05.603 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:05.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:05.710 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:05.759 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:05.814 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:05.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:05.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:05.918 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:05.964 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:06.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:05 smithi008 ceph-mon[184418]: pgmap v1513: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 230 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 42082/10926 objects degraded (385.155%); 2254/10926 objects misplaced (20.630%); 5.7 MiB/s, 92 objects/s recovering 2023-10-09T06:02:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:05 smithi008 ceph-mon[191227]: pgmap v1513: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 230 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 42082/10926 objects degraded (385.155%); 2254/10926 objects misplaced (20.630%); 5.7 MiB/s, 92 objects/s recovering 2023-10-09T06:02:06.276 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:06.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:06.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:06.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:05 smithi191 ceph-mon[161669]: pgmap v1513: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 230 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 42082/10926 objects degraded (385.155%); 2254/10926 objects misplaced (20.630%); 5.7 MiB/s, 92 objects/s recovering 2023-10-09T06:02:06.383 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:06.433 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:06.528 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:06.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:06.546 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:06.637 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:06.678 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:06.756 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:06.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:06.861 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:06.914 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:07.376 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:07.379 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:02:07.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:07.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:07.410 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:02:07.438 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:02:07.463 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:02:07.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:07.494 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:02:07.518 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:07.525 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:02:07.549 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:02:07.578 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:02:07.582 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:02:07.582 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:02:07.583 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:02:07.583 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:02:07.583 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:02:07.583 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:02:07.583 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:02:07.583 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:02:07.736 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:07.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:07.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:07.842 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:07.885 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:07.996 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.0 primary-affinity to 0.560994 (8367652) 2023-10-09T06:02:08.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:07 smithi008 ceph-mon[184418]: pgmap v1514: 99 pgs: 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 172 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 382 op/s; 39378/8142 objects degraded (483.640%); 2273/8142 objects misplaced (27.917%); 7.8 MiB/s, 127 objects/s recovering 2023-10-09T06:02:08.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:07 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/226466305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:08.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:07 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1690950231' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5609944599663401}]: dispatch 2023-10-09T06:02:08.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:07 smithi008 ceph-mon[191227]: pgmap v1514: 99 pgs: 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 172 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 382 op/s; 39378/8142 objects degraded (483.640%); 2273/8142 objects misplaced (27.917%); 7.8 MiB/s, 127 objects/s recovering 2023-10-09T06:02:08.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:07 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/226466305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:08.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:07 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1690950231' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5609944599663401}]: dispatch 2023-10-09T06:02:08.231 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:02:08.231 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:02:08.232 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T06:02:08.232 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:02:08.255 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:07 smithi191 ceph-mon[161669]: pgmap v1514: 99 pgs: 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 172 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 382 op/s; 39378/8142 objects degraded (483.640%); 2273/8142 objects misplaced (27.917%); 7.8 MiB/s, 127 objects/s recovering 2023-10-09T06:02:08.255 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:07 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/226466305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:08.255 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:07 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1690950231' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5609944599663401}]: dispatch 2023-10-09T06:02:08.807 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:08.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:08.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:08.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:08.944 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:09.092 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:09.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:09.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:09.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:02:09.196 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:09.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1690950231' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5609944599663401}]': finished 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: osdmap e505: 8 total, 8 up, 7 in 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 39378/8142 objects degraded (483.640%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: osdmap e506: 8 total, 8 up, 7 in 2023-10-09T06:02:09.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[184418]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1690950231' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5609944599663401}]': finished 2023-10-09T06:02:09.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: osdmap e505: 8 total, 8 up, 7 in 2023-10-09T06:02:09.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 39378/8142 objects degraded (483.640%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: osdmap e506: 8 total, 8 up, 7 in 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:08 smithi008 ceph-mon[191227]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.224 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1690950231' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5609944599663401}]': finished 2023-10-09T06:02:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: osdmap e505: 8 total, 8 up, 7 in 2023-10-09T06:02:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e505 with expected crc 2023-10-09T06:02:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 39378/8142 objects degraded (483.640%), 24 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: osdmap e506: 8 total, 8 up, 7 in 2023-10-09T06:02:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:08 smithi191 ceph-mon[161669]: failed to encode map e506 with expected crc 2023-10-09T06:02:09.612 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:09 smithi191 ceph-mon[161669]: failed to encode map e506 with expected crc 2023-10-09T06:02:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:09 smithi191 ceph-mon[161669]: pgmap v1517: 99 pgs: 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 135 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 338 op/s; 37825/6363 objects degraded (594.452%); 2222/6363 objects misplaced (34.921%); 6.2 MiB/s, 336 objects/s recovering 2023-10-09T06:02:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:09 smithi191 ceph-mon[161669]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:09 smithi191 ceph-mon[161669]: osdmap e507: 8 total, 8 up, 7 in 2023-10-09T06:02:10.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:09 smithi191 ceph-mon[161669]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:09 smithi191 ceph-mon[161669]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:09 smithi191 ceph-mon[161669]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[184418]: failed to encode map e506 with expected crc 2023-10-09T06:02:10.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[184418]: pgmap v1517: 99 pgs: 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 135 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 338 op/s; 37825/6363 objects degraded (594.452%); 2222/6363 objects misplaced (34.921%); 6.2 MiB/s, 336 objects/s recovering 2023-10-09T06:02:10.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[184418]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[184418]: osdmap e507: 8 total, 8 up, 7 in 2023-10-09T06:02:10.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[184418]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[184418]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[184418]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[191227]: failed to encode map e506 with expected crc 2023-10-09T06:02:10.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[191227]: pgmap v1517: 99 pgs: 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 135 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 338 op/s; 37825/6363 objects degraded (594.452%); 2222/6363 objects misplaced (34.921%); 6.2 MiB/s, 336 objects/s recovering 2023-10-09T06:02:10.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[191227]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[191227]: osdmap e507: 8 total, 8 up, 7 in 2023-10-09T06:02:10.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[191227]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[191227]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:09 smithi008 ceph-mon[191227]: failed to encode map e507 with expected crc 2023-10-09T06:02:10.482 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:10.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:10.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:10.587 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:10.629 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:10.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:02:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:02:10] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.33.4" 2023-10-09T06:02:11.239 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:10 smithi191 ceph-mon[161669]: failed to encode map e507 with expected crc 2023-10-09T06:02:11.239 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:10 smithi191 ceph-mon[161669]: failed to encode map e507 with expected crc 2023-10-09T06:02:11.239 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:10 smithi191 ceph-mon[161669]: failed to encode map e507 with expected crc 2023-10-09T06:02:11.420 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:11.422 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:10 smithi008 ceph-mon[184418]: failed to encode map e507 with expected crc 2023-10-09T06:02:11.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:10 smithi008 ceph-mon[184418]: failed to encode map e507 with expected crc 2023-10-09T06:02:11.423 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:10 smithi008 ceph-mon[184418]: failed to encode map e507 with expected crc 2023-10-09T06:02:11.423 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:10 smithi008 ceph-mon[191227]: failed to encode map e507 with expected crc 2023-10-09T06:02:11.423 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:10 smithi008 ceph-mon[191227]: failed to encode map e507 with expected crc 2023-10-09T06:02:11.423 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:10 smithi008 ceph-mon[191227]: failed to encode map e507 with expected crc 2023-10-09T06:02:11.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:11.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:11.525 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:11.541 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:02:11.541 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:02:11.541 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T06:02:11.541 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T06:02:11.541 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:02:11.561 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:11.633 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:11.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:11.738 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:11.788 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:11.973 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:11.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:11.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:12.078 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:12.128 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:12.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:11 smithi191 ceph-mon[161669]: pgmap v1519: 99 pgs: 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 112 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 37539/5265 objects degraded (712.991%); 2222/5265 objects misplaced (42.203%); 8.0 MiB/s, 357 objects/s recovering 2023-10-09T06:02:12.348 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:02:12.351 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:12.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:12.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:12.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:12.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:11 smithi008 ceph-mon[184418]: pgmap v1519: 99 pgs: 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 112 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 37539/5265 objects degraded (712.991%); 2222/5265 objects misplaced (42.203%); 8.0 MiB/s, 357 objects/s recovering 2023-10-09T06:02:12.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:11 smithi008 ceph-mon[191227]: pgmap v1519: 99 pgs: 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 112 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 37539/5265 objects degraded (712.991%); 2222/5265 objects misplaced (42.203%); 8.0 MiB/s, 357 objects/s recovering 2023-10-09T06:02:12.507 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:12.591 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:12.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:12.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:12.702 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:12.742 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:12.804 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:12.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:12.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:12.872 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:12.907 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:12.940 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:13.001 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:13.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:13.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:13.005 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:02:13.106 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:13.141 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:13.235 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:02:13.244 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:13.246 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1147843247' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:13.247 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1147843247' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:13.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:13.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:13.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1147843247' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:13.348 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:13.376 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:13.427 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:13.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:13.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:13.531 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:13.567 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:13.617 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:13.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:13.720 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:13.749 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:13.857 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Removed 84170 objects 2023-10-09T06:02:13.857 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Clean up completed and total clean up time :446.967 2023-10-09T06:02:14.154 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T06:02:14.154 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T06:02:14.180 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: pgmap v1520: 99 pgs: 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 71 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 277 op/s; 37254/3297 objects degraded (1129.936%); 2058/3297 objects misplaced (62.420%); 6.7 MiB/s, 335 objects/s recovering 2023-10-09T06:02:14.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:14.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1147843247' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:14.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: osdmap e508: 8 total, 8 up, 7 in 2023-10-09T06:02:14.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 37254/3297 objects degraded (1129.936%), 23 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:14.181 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:14.182 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:14.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: pgmap v1520: 99 pgs: 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 71 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 277 op/s; 37254/3297 objects degraded (1129.936%); 2058/3297 objects misplaced (62.420%); 6.7 MiB/s, 335 objects/s recovering 2023-10-09T06:02:14.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:14.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1147843247' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:14.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: osdmap e508: 8 total, 8 up, 7 in 2023-10-09T06:02:14.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.182 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 37254/3297 objects degraded (1129.936%), 23 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:14.183 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:14.183 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:13 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: pgmap v1520: 99 pgs: 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 71 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 277 op/s; 37254/3297 objects degraded (1129.936%); 2058/3297 objects misplaced (62.420%); 6.7 MiB/s, 335 objects/s recovering 2023-10-09T06:02:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1147843247' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: osdmap e508: 8 total, 8 up, 7 in 2023-10-09T06:02:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: failed to encode map e508 with expected crc 2023-10-09T06:02:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 37254/3297 objects degraded (1129.936%), 23 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:14.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:13 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:14.669 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:14.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:14.675 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:02:14.675 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T06:02:14.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T06:02:14.675 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T06:02:14.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.675 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:45782 9.e 9:7b605443:::benchmark_data_smithi008_502820_object45781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e372)", 2023-10-09T06:02:14.675 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:53:46.078272+0000", 2023-10-09T06:02:14.675 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 508.59484081400001, 2023-10-09T06:02:14.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0922387410000001, 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 45782 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078272+0000", 2023-10-09T06:02:14.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078272+0000", 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078270+0000", 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8878e-05 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078299+0000", 2023-10-09T06:02:14.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2486999999999999e-05 2023-10-09T06:02:14.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078311+0000", 2023-10-09T06:02:14.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0040000000000002e-06 2023-10-09T06:02:14.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078318+0000", 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6557000000000002e-05 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078365+0000", 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0165e-05 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.078375+0000", 2023-10-09T06:02:14.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85827073899999995 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.936646+0000", 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1354e-05 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:46.936657+0000", 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2267288569999999 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.163386+0000", 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091583599999999995 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.164302+0000", 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000143702 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.164446+0000", 2023-10-09T06:02:14.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003236381 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.167682+0000", 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001185424 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.168867+0000", 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00053191999999999996 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.169399+0000", 2023-10-09T06:02:14.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4771000000000002e-05 2023-10-09T06:02:14.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:14.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.169444+0000", 2023-10-09T06:02:14.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010662359999999999 2023-10-09T06:02:14.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.684 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:48.170510+0000", 2023-10-09T06:02:14.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0922387410000001 2023-10-09T06:02:14.684 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.684 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.684 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.684 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:77617 9.1 9:84bc2af6:::benchmark_data_smithi008_502820_object77616:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T06:02:14.684 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.819004+0000", 2023-10-09T06:02:14.684 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 457.85410834100003, 2023-10-09T06:02:14.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92076191299999999, 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 77617 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819004+0000", 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.686 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.686 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.686 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.686 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819004+0000", 2023-10-09T06:02:14.686 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:02:14.686 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.686 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.686 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.686 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819002+0000", 2023-10-09T06:02:14.686 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6029999999999997e-05 2023-10-09T06:02:14.686 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819058+0000", 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4360999999999999e-05 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819082+0000", 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7680000000000003e-06 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.687 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819092+0000", 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.045903408999999999 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.864995+0000", 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5949999999999999e-06 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.865004+0000", 2023-10-09T06:02:14.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.87466948200000005 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.739673+0000", 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2732000000000006e-05 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.739766+0000", 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92076191299999999 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.689 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.690 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.690 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:77618 9.11 9:8a9bea20:::benchmark_data_smithi008_502820_object77617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T06:02:14.690 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.819101+0000", 2023-10-09T06:02:14.690 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 457.85401085799998, 2023-10-09T06:02:14.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.928982839, 2023-10-09T06:02:14.690 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.690 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:02:14.690 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.690 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.690 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 77618 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819101+0000", 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819101+0000", 2023-10-09T06:02:14.691 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819101+0000", 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2029e-05 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819143+0000", 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6980000000000001e-06 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.692 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.693 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.693 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819150+0000", 2023-10-09T06:02:14.693 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9419999999999997e-06 2023-10-09T06:02:14.693 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.693 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.693 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.693 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.819154+0000", 2023-10-09T06:02:14.693 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.045857321999999999 2023-10-09T06:02:14.693 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.693 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.693 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.865011+0000", 2023-10-09T06:02:14.694 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5409999999999997e-06 2023-10-09T06:02:14.694 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.694 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.694 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.865017+0000", 2023-10-09T06:02:14.694 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88297774600000001 2023-10-09T06:02:14.694 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.694 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.694 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.747995+0000", 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9123999999999996e-05 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:37.748084+0000", 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.928982839 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.695 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:90125 9.c 9:3fead019:::benchmark_data_smithi008_502820_object5952:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:54.013383+0000", 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 440.65972969699999, 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.394155001, 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 90125 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.696 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.697 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013383+0000", 2023-10-09T06:02:14.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.697 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.697 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013383+0000", 2023-10-09T06:02:14.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:14.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.697 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013382+0000", 2023-10-09T06:02:14.698 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.937e-06 2023-10-09T06:02:14.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013385+0000", 2023-10-09T06:02:14.698 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1168e-05 2023-10-09T06:02:14.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013396+0000", 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.844e-06 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.013400+0000", 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003200779 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.016601+0000", 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000143905 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.699 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.700 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.016745+0000", 2023-10-09T06:02:14.700 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000168927 2023-10-09T06:02:14.700 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:02:14.700 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.016914+0000", 2023-10-09T06:02:14.700 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031704670000000002 2023-10-09T06:02:14.700 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.020084+0000", 2023-10-09T06:02:14.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002970664 2023-10-09T06:02:14.701 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.701 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.701 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.023055+0000", 2023-10-09T06:02:14.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.38235074600000002 2023-10-09T06:02:14.701 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.701 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.701 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.405406+0000", 2023-10-09T06:02:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4887000000000004e-05 2023-10-09T06:02:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.405470+0000", 2023-10-09T06:02:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020671679999999999 2023-10-09T06:02:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:54.407538+0000", 2023-10-09T06:02:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.394155001 2023-10-09T06:02:14.703 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.703 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.703 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.703 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.703 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.703 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93474 9.e 9:77e94385:::benchmark_data_smithi008_502820_object9301:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:02:14.703 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.096743+0000", 2023-10-09T06:02:14.703 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.57636972799997, 2023-10-09T06:02:14.703 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.154875152, 2023-10-09T06:02:14.703 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.703 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93474 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096743+0000", 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.705 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096743+0000", 2023-10-09T06:02:14.705 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:14.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.705 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.705 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096741+0000", 2023-10-09T06:02:14.705 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9869999999999999e-06 2023-10-09T06:02:14.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.705 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.705 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096749+0000", 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1752e-05 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096761+0000", 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.708e-06 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.096768+0000", 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000324956 2023-10-09T06:02:14.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.097092+0000", 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016051600000000001 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.097253+0000", 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000160118 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:02:14.708 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.097413+0000", 2023-10-09T06:02:14.708 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035366690000000001 2023-10-09T06:02:14.708 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.708 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.708 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.708 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100950+0000", 2023-10-09T06:02:14.708 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021417300000000001 2023-10-09T06:02:14.708 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.708 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.708 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.708 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.101164+0000", 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14614434300000001 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247308+0000", 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1691e-05 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247320+0000", 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0042787399999999 2023-10-09T06:02:14.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.710 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.710 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.710 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251599+0000", 2023-10-09T06:02:14.710 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9018e-05 2023-10-09T06:02:14.710 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.710 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.710 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.710 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251618+0000", 2023-10-09T06:02:14.710 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.154875152 2023-10-09T06:02:14.710 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.710 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93475 9.c 9:3e40594a:::benchmark_data_smithi008_502820_object9302:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.100682+0000", 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.57243074100001, 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2206704390000001, 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.711 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93475 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100682+0000", 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100682+0000", 2023-10-09T06:02:14.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100680+0000", 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5109999999999997e-06 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100688+0000", 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.63e-05 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100704+0000", 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6109999999999999e-06 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100711+0000", 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6734000000000003e-05 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.714 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100767+0000", 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015612699999999999 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.100923+0000", 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016590299999999999 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.101089+0000", 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030346919999999999 2023-10-09T06:02:14.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.104124+0000", 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102549 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.104227+0000", 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.143401417 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.716 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247628+0000", 2023-10-09T06:02:14.717 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9989999999999997e-06 2023-10-09T06:02:14.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.717 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247637+0000", 2023-10-09T06:02:14.717 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0299526569999999 2023-10-09T06:02:14.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.717 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.277590+0000", 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9399000000000002e-05 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.277679+0000", 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.043604728000000002 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.321284+0000", 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8312000000000006e-05 2023-10-09T06:02:14.718 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.719 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.719 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.719 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.321352+0000", 2023-10-09T06:02:14.719 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2206704390000001 2023-10-09T06:02:14.719 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.719 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.719 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.719 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.719 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.719 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93483 9.e 9:7fc9cbe3:::benchmark_data_smithi008_502820_object9310:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:02:14.719 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.108830+0000", 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.56428220200002, 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1428273499999999, 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93483 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.720 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108830+0000", 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108830+0000", 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108829+0000", 2023-10-09T06:02:14.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7689999999999999e-06 2023-10-09T06:02:14.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108835+0000", 2023-10-09T06:02:14.722 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2876e-05 2023-10-09T06:02:14.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108848+0000", 2023-10-09T06:02:14.722 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8799000000000001e-05 2023-10-09T06:02:14.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108867+0000", 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6572e-05 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.108924+0000", 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000126241 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.109050+0000", 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012856799999999999 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.109178+0000", 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003197222 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.112376+0000", 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001610319 2023-10-09T06:02:14.724 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.113986+0000", 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13337320899999999 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247359+0000", 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8019999999999992e-06 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247367+0000", 2023-10-09T06:02:14.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0042815700000001 2023-10-09T06:02:14.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251648+0000", 2023-10-09T06:02:14.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0310000000000004e-06 2023-10-09T06:02:14.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251657+0000", 2023-10-09T06:02:14.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1428273499999999 2023-10-09T06:02:14.726 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93490 9.e 9:7c8d93dd:::benchmark_data_smithi008_502820_object9317:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.117443+0000", 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.55566906600001, 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.134247681, 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.727 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93490 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117443+0000", 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117443+0000", 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117442+0000", 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7300000000000002e-06 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117448+0000", 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0936e-05 2023-10-09T06:02:14.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117459+0000", 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3310000000000004e-06 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117467+0000", 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7256000000000003e-05 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117514+0000", 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019338100000000001 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117708+0000", 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016922399999999999 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.117877+0000", 2023-10-09T06:02:14.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035029470000000002 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.121380+0000", 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0540999999999999e-05 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.121400+0000", 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12599297000000001 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247393+0000", 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1810000000000004e-06 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247400+0000", 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.004281518 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251682+0000", 2023-10-09T06:02:14.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0119999999999994e-06 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251691+0000", 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.134247681 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93496 9.c 9:33b252c8:::benchmark_data_smithi008_502820_object9323:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-09T06:02:14.734 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.123753+0000", 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.54935958999999, 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.192581734, 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93496 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.735 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123753+0000", 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123753+0000", 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123751+0000", 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2879999999999998e-06 2023-10-09T06:02:14.736 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123758+0000", 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1221e-05 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123769+0000", 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7299999999999996e-06 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.737 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123774+0000", 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2144000000000001e-05 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123826+0000", 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000124738 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.123951+0000", 2023-10-09T06:02:14.738 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.123715246 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247666+0000", 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3069999999999997e-06 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.247673+0000", 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0300421200000001 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.739 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.277715+0000", 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7616999999999998e-05 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.277803+0000", 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0111000000000005e-05 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:02:14.740 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.277893+0000", 2023-10-09T06:02:14.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033372605 2023-10-09T06:02:14.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.311266+0000", 2023-10-09T06:02:14.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002796991 2023-10-09T06:02:14.741 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.741 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.741 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.741 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314063+0000", 2023-10-09T06:02:14.741 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017246500000000001 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314235+0000", 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8576999999999999e-05 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314284+0000", 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020508919999999999 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.742 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.316334+0000", 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.192581734 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93502 9.e 9.68a8793e (undecoded) ondisk+write+known_if_redirected e391)", 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.383448+0000", 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.28966399199999, 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86827123699999997, 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93502 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383448+0000", 2023-10-09T06:02:14.744 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383448+0000", 2023-10-09T06:02:14.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:14.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.745 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.745 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.745 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383447+0000", 2023-10-09T06:02:14.745 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4139999999999997e-06 2023-10-09T06:02:14.745 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383454+0000", 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4278e-05 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383468+0000", 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5860000000000001e-06 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.746 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383475+0000", 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3341999999999999e-05 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383528+0000", 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.009e-05 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383539+0000", 2023-10-09T06:02:14.747 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86817271100000004 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251711+0000", 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3489999999999992e-06 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251720+0000", 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86827123699999997 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.748 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.749 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.749 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.749 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.749 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93503 9.e 9.abdff44e (undecoded) ondisk+write+known_if_redirected e391)", 2023-10-09T06:02:14.749 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.383488+0000", 2023-10-09T06:02:14.749 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.28962422199999, 2023-10-09T06:02:14.749 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86825971800000001, 2023-10-09T06:02:14.749 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.749 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:02:14.749 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.749 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.750 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.750 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93503 2023-10-09T06:02:14.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.750 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.750 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383488+0000", 2023-10-09T06:02:14.750 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.750 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.750 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.750 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383488+0000", 2023-10-09T06:02:14.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:14.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383488+0000", 2023-10-09T06:02:14.751 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0340000000000001e-06 2023-10-09T06:02:14.751 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.751 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.751 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.751 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383491+0000", 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3049999999999998e-06 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383497+0000", 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.172e-06 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383501+0000", 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00046004500000000002 2023-10-09T06:02:14.752 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383961+0000", 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3599999999999998e-06 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.383969+0000", 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86777167399999999 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.753 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251740+0000", 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6320000000000008e-06 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251748+0000", 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86825971800000001 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.754 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.755 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93512 9.c 9:347bba79:::benchmark_data_smithi008_502820_object9339:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-09T06:02:14.755 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.396258+0000", 2023-10-09T06:02:14.755 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.27685458000002, 2023-10-09T06:02:14.755 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92175368199999996, 2023-10-09T06:02:14.755 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.755 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:14.755 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.755 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.755 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.755 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93512 2023-10-09T06:02:14.755 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396258+0000", 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396258+0000", 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.756 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396256+0000", 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1110000000000002e-06 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396264+0000", 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2522e-05 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.757 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396276+0000", 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0119999999999996e-06 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396281+0000", 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5559000000000002e-05 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396327+0000", 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3000000000000007e-06 2023-10-09T06:02:14.758 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.396336+0000", 2023-10-09T06:02:14.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.88407117199999996 2023-10-09T06:02:14.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.759 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.759 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.280407+0000", 2023-10-09T06:02:14.759 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012818199999999999 2023-10-09T06:02:14.759 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.759 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.280535+0000", 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010833800000000001 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.280644+0000", 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033468772000000001 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.760 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314112+0000", 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0919999999999999e-05 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314173+0000", 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002239053 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.316412+0000", 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9391999999999997e-05 2023-10-09T06:02:14.761 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:14.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.316462+0000", 2023-10-09T06:02:14.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00154963 2023-10-09T06:02:14.762 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.762 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.762 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.762 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.318011+0000", 2023-10-09T06:02:14.762 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92175368199999996 2023-10-09T06:02:14.762 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.762 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.763 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.763 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.763 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.763 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93526 9.c 9:38adb9cd:::benchmark_data_smithi008_502820_object9353:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-09T06:02:14.763 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.420062+0000", 2023-10-09T06:02:14.763 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.25305066700002, 2023-10-09T06:02:14.763 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89930828500000004, 2023-10-09T06:02:14.763 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.763 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:14.763 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.764 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.764 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.764 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93526 2023-10-09T06:02:14.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.764 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.764 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.764 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420062+0000", 2023-10-09T06:02:14.764 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.764 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.764 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420062+0000", 2023-10-09T06:02:14.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:14.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.765 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420060+0000", 2023-10-09T06:02:14.765 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.994e-06 2023-10-09T06:02:14.765 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.765 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.765 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420065+0000", 2023-10-09T06:02:14.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0139000000000001e-05 2023-10-09T06:02:14.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420075+0000", 2023-10-09T06:02:14.766 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8459999999999999e-06 2023-10-09T06:02:14.766 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.766 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.766 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.766 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.420080+0000", 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073188280000000003 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.427399+0000", 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4809999999999995e-06 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.427409+0000", 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85476691800000004 2023-10-09T06:02:14.767 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.282176+0000", 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012473900000000001 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.282300+0000", 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000178133 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.768 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.282478+0000", 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.031680262000000001 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314159+0000", 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3935e-05 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.314183+0000", 2023-10-09T06:02:14.769 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038877590000000002 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.318070+0000", 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3279000000000002e-05 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.318134+0000", 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00123628 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.770 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.319370+0000", 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89930828500000004 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:93531 9.e 9.5b3160de (undecoded) ondisk+write+known_if_redirected e391)", 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:58.434836+0000", 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.23827672099998, 2023-10-09T06:02:14.771 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81693827500000005, 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 93531 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434836+0000", 2023-10-09T06:02:14.772 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434836+0000", 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434834+0000", 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8570000000000001e-06 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.773 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434843+0000", 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6622000000000001e-05 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434859+0000", 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1670000000000004e-06 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.774 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434866+0000", 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7221000000000001e-05 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434913+0000", 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2180000000000005e-06 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:58.434922+0000", 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81684441699999999 2023-10-09T06:02:14.775 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251766+0000", 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5000000000000002e-06 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:59.251774+0000", 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.81693827500000005 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.776 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:128121 9.e e471/470 9:7c9069ca:::benchmark_data_smithi008_502820_object43948:head v 471'7986, mlcod=0'0)", 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:59:32.599934+0000", 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 162.07317830100001, 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.041291387999999998, 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.777 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599934+0000", 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599934+0000", 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599933+0000", 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7060000000000004e-06 2023-10-09T06:02:14.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599941+0000", 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4945e-05 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599956+0000", 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8809999999999998e-06 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.599962+0000", 2023-10-09T06:02:14.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7352000000000002e-05 2023-10-09T06:02:14.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.600020+0000", 2023-10-09T06:02:14.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6948999999999999e-05 2023-10-09T06:02:14.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.600047+0000", 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.041137582999999998 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.641184+0000", 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1075999999999998e-05 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:59:32.641225+0000", 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.041291387999999998 2023-10-09T06:02:14.781 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135465 9.19 9:9f806870:::benchmark_data_smithi008_502820_object51292:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:17.584034+0000", 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 117.089078775, 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.053033229000000001, 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.782 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135465 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.584034+0000", 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.584034+0000", 2023-10-09T06:02:14.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:14.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.784 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.784 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.584032+0000", 2023-10-09T06:02:14.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8679999999999999e-06 2023-10-09T06:02:14.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.784 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.784 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.584040+0000", 2023-10-09T06:02:14.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2580000000000001e-05 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.584062+0000", 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7909999999999999e-06 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.584069+0000", 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010139557 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.594209+0000", 2023-10-09T06:02:14.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002682593 2023-10-09T06:02:14.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.596891+0000", 2023-10-09T06:02:14.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048169950000000001 2023-10-09T06:02:14.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.601708+0000", 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013086867 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.614795+0000", 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028284300000000002 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.615078+0000", 2023-10-09T06:02:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015632956999999999 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.630711+0000", 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001202648 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.631914+0000", 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0051532710000000001 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.637067+0000", 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.053033229000000001 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135637 9.19 9:9a1ed32e:::benchmark_data_smithi008_502820_object51464:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:18.218191+0000", 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 116.45492175699999, 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.053393322999999999, 2023-10-09T06:02:14.789 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.790 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:14.790 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.790 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.790 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.790 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135637 2023-10-09T06:02:14.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.790 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.218191+0000", 2023-10-09T06:02:14.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.218191+0000", 2023-10-09T06:02:14.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:14.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.218189+0000", 2023-10-09T06:02:14.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8370000000000003e-06 2023-10-09T06:02:14.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.218198+0000", 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7189000000000001e-05 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.218215+0000", 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3359999999999997e-06 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.218222+0000", 2023-10-09T06:02:14.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023679385000000001 2023-10-09T06:02:14.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.241902+0000", 2023-10-09T06:02:14.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002637099 2023-10-09T06:02:14.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.244539+0000", 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037200829999999999 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.248259+0000", 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015803897000000001 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.264063+0000", 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00078032799999999995 2023-10-09T06:02:14.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.264843+0000", 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000239779 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.265083+0000", 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012074099999999999 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:14.795 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.266290+0000", 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052937560000000002 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:18.271584+0000", 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.053393322999999999 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.796 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135882 9.19 9:9829714a:::benchmark_data_smithi008_502820_object51709:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:19.445250+0000", 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 115.227862171, 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.042005708000000003, 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135882 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.445250+0000", 2023-10-09T06:02:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.445250+0000", 2023-10-09T06:02:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:02:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.445248+0000", 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1722999999999999e-05 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.445260+0000", 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8626000000000001e-05 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.445278+0000", 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6850000000000007e-06 2023-10-09T06:02:14.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.445287+0000", 2023-10-09T06:02:14.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004252736 2023-10-09T06:02:14.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.449540+0000", 2023-10-09T06:02:14.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022961769999999999 2023-10-09T06:02:14.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.451836+0000", 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004057115 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.455893+0000", 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0096318429999999993 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.465525+0000", 2023-10-09T06:02:14.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00072013999999999997 2023-10-09T06:02:14.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.466245+0000", 2023-10-09T06:02:14.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015260235 2023-10-09T06:02:14.802 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.802 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.802 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.802 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.481505+0000", 2023-10-09T06:02:14.802 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00092144099999999995 2023-10-09T06:02:14.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.803 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:14.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.482427+0000", 2023-10-09T06:02:14.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048292559999999997 2023-10-09T06:02:14.803 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.803 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.803 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.803 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.487256+0000", 2023-10-09T06:02:14.803 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.042005708000000003 2023-10-09T06:02:14.803 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135924 9.19 9:9d5c6567:::benchmark_data_smithi008_502820_object51751:head [delete] snapc 0=[] ondisk+write+known_if_redirected e481)", 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:19.597462+0000", 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 115.075650111, 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.045627875999999998, 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.804 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135924 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.597462+0000", 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.805 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.597462+0000", 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.597460+0000", 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.169e-05 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.597472+0000", 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7853999999999999e-05 2023-10-09T06:02:14.806 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.597489+0000", 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8370000000000003e-06 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.597498+0000", 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026866910000000002 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.807 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.600185+0000", 2023-10-09T06:02:14.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025690790000000002 2023-10-09T06:02:14.809 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.809 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.809 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.809 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.602754+0000", 2023-10-09T06:02:14.809 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045478619999999997 2023-10-09T06:02:14.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:02:14.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.607302+0000", 2023-10-09T06:02:14.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0069385319999999999 2023-10-09T06:02:14.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.810 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.810 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.810 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.614240+0000", 2023-10-09T06:02:14.810 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010052238 2023-10-09T06:02:14.810 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.811 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.811 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.624293+0000", 2023-10-09T06:02:14.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012318301 2023-10-09T06:02:14.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.811 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.811 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.636611+0000", 2023-10-09T06:02:14.811 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00118852 2023-10-09T06:02:14.811 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.811 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.637799+0000", 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052906730000000001 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:19.643090+0000", 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.045627875999999998 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.812 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:136347 9.19 9:9b636a4c:::benchmark_data_smithi008_502820_object52174:head [delete] snapc 0=[] ondisk+write+known_if_redirected e481)", 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:21.732844+0000", 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 112.94026872400001, 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.042053720000000003, 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 136347 2023-10-09T06:02:14.813 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.732844+0000", 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.732844+0000", 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.814 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.732842+0000", 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7649999999999993e-06 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.732850+0000", 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0237e-05 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:14.815 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.732860+0000", 2023-10-09T06:02:14.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9280000000000001e-06 2023-10-09T06:02:14.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:14.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.732865+0000", 2023-10-09T06:02:14.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017265038999999999 2023-10-09T06:02:14.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:14.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.750130+0000", 2023-10-09T06:02:14.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022564540000000002 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.752387+0000", 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003510885 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.755897+0000", 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013081104 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.768979+0000", 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000683495 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.769662+0000", 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00054090599999999996 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.770203+0000", 2023-10-09T06:02:14.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000865951 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.771069+0000", 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003828442 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:21.774897+0000", 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.042053720000000003 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.819 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.820 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:14.820 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:14.820 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:02:14.820 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:14.891 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:14.930 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:14.954 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:14.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:14.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:14.986 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:15.060 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:15.088 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:15.187 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:15.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:15.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:15.291 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:15.321 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:15.378 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:15.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:15.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:15.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:02:15.484 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:15.514 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:15.795 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:15.814 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:15.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:15.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:15.918 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:15.946 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:16.047 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:16.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:16.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:16.152 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:16.181 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:15 smithi008 ceph-mon[184418]: pgmap v1522: 99 pgs: 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 57 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 36532/2616 objects degraded (1396.483%); 2013/2616 objects misplaced (76.950%); 5.3 MiB/s, 283 objects/s recovering 2023-10-09T06:02:16.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:15 smithi008 ceph-mon[191227]: pgmap v1522: 99 pgs: 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 57 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 36532/2616 objects degraded (1396.483%); 2013/2616 objects misplaced (76.950%); 5.3 MiB/s, 283 objects/s recovering 2023-10-09T06:02:16.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:15 smithi191 ceph-mon[161669]: pgmap v1522: 99 pgs: 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 57 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 36532/2616 objects degraded (1396.483%); 2013/2616 objects misplaced (76.950%); 5.3 MiB/s, 283 objects/s recovering 2023-10-09T06:02:16.380 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:16.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:16.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:16.483 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:16.512 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:16.560 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:16.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:16.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:16.654 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:02:16.655 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:02:16.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:16.691 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:16.803 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:16.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:16.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:16.823 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:02:16.907 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:16.935 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:17.273 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:02:17.273 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:02:17.273 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T06:02:17.273 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:02:17.793 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:17.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:17.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:17.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:17.927 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:18.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:02:18.036 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:18.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:18.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:18.142 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:18.172 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:18.213 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:18.221 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:18.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:18.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:18 smithi191 ceph-mon[161669]: pgmap v1523: 99 pgs: 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 19 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 258 op/s; 36147/783 objects degraded (4616.475%); 1913/783 objects misplaced (244.317%); 6.0 MiB/s, 199 objects/s recovering 2023-10-09T06:02:18.325 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:18.329 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:18 smithi008 ceph-mon[184418]: pgmap v1523: 99 pgs: 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 19 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 258 op/s; 36147/783 objects degraded (4616.475%); 1913/783 objects misplaced (244.317%); 6.0 MiB/s, 199 objects/s recovering 2023-10-09T06:02:18.330 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:18 smithi008 ceph-mon[191227]: pgmap v1523: 99 pgs: 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 19 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 258 op/s; 36147/783 objects degraded (4616.475%); 1913/783 objects misplaced (244.317%); 6.0 MiB/s, 199 objects/s recovering 2023-10-09T06:02:18.357 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:18.431 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:18.496 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:02:18.516 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:02:18.537 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:02:18.558 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:02:18.579 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:02:18.605 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:02:18.613 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:18.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:18.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:18.624 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:02:18.643 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:02:18.645 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:02:18.645 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:02:18.646 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:02:18.646 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:02:18.646 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:02:18.646 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:02:18.646 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:02:18.646 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:02:18.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:18.747 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:18.846 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:18.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:18.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:18.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:18.983 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:19.009 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_4' removed 2023-10-09T06:02:19.064 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:19.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:19.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:19.168 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:19.203 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:19.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: osdmap e509: 8 total, 8 up, 7 in 2023-10-09T06:02:19.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 36147/783 objects degraded (4616.475%), 23 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:19.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4192334589' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:19.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/346574579' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:02:19.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:02:19.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:02:19.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:19 smithi191 ceph-mon[161669]: osdmap e510: 8 total, 8 up, 7 in 2023-10-09T06:02:19.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: osdmap e509: 8 total, 8 up, 7 in 2023-10-09T06:02:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 36147/783 objects degraded (4616.475%), 23 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4192334589' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/346574579' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:02:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:02:19.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:02:19.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[184418]: osdmap e510: 8 total, 8 up, 7 in 2023-10-09T06:02:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: osdmap e509: 8 total, 8 up, 7 in 2023-10-09T06:02:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: failed to encode map e509 with expected crc 2023-10-09T06:02:19.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 36147/783 objects degraded (4616.475%), 23 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:02:19.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4192334589' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:19.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/346574579' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:02:19.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:02:19.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:02:19.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:19 smithi008 ceph-mon[191227]: osdmap e510: 8 total, 8 up, 7 in 2023-10-09T06:02:20.243 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:02:20.243 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:02:20.280 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:20 smithi191 ceph-mon[161669]: pgmap v1525: 99 pgs: 8 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 18 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 36088/738 objects degraded (4889.973%); 1913/738 objects misplaced (259.214%); 4.6 MiB/s, 183 objects/s recovering 2023-10-09T06:02:20.280 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:20 smithi191 ceph-mon[161669]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:20 smithi191 ceph-mon[161669]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:20 smithi191 ceph-mon[161669]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:20 smithi191 ceph-mon[161669]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:20 smithi191 ceph-mon[161669]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:20 smithi191 ceph-mon[161669]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.308 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:20.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[184418]: pgmap v1525: 99 pgs: 8 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 18 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 36088/738 objects degraded (4889.973%); 1913/738 objects misplaced (259.214%); 4.6 MiB/s, 183 objects/s recovering 2023-10-09T06:02:20.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[184418]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[184418]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.310 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[184418]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.311 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[184418]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.311 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[184418]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.311 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[184418]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[191227]: pgmap v1525: 99 pgs: 8 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 18 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 36088/738 objects degraded (4889.973%); 1913/738 objects misplaced (259.214%); 4.6 MiB/s, 183 objects/s recovering 2023-10-09T06:02:20.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[191227]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[191227]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[191227]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[191227]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.311 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[191227]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.312 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:20 smithi008 ceph-mon[191227]: failed to encode map e510 with expected crc 2023-10-09T06:02:20.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:20.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:20.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:20.442 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:20.488 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:20.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:20.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:20.507 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:02:20.508 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:02:20.508 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T06:02:20.508 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T06:02:20.592 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:20.617 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:02:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:02:20] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.33.4" 2023-10-09T06:02:20.621 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:20.670 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:20.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:20.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:20.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:20.801 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:20.915 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:20.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:20.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:20.970 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:02:21.019 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:21.047 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:21.090 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:21.091 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:21.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:21.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:21.198 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:21.228 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:21.324 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:21.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:21.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:21.328 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:21.429 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:21.457 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:21.555 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:21.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:21.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:21.660 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:21.688 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:21.883 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:21.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:21.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:21.986 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:22.016 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:22.038 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:02:22.293 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:22 smithi008 ceph-mon[184418]: pgmap v1527: 73 pgs: 1 active+recovering, 2 active+recovery_wait+degraded, 70 active+clean; 4.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 18/84 objects degraded (21.429%) 2023-10-09T06:02:22.293 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:22 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/563882789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:02:22.293 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:22 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:02:22.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:22 smithi008 ceph-mon[191227]: pgmap v1527: 73 pgs: 1 active+recovering, 2 active+recovery_wait+degraded, 70 active+clean; 4.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 18/84 objects degraded (21.429%) 2023-10-09T06:02:22.294 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:22 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/563882789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:02:22.294 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:22 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:02:22.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:22 smithi191 ceph-mon[161669]: pgmap v1527: 73 pgs: 1 active+recovering, 2 active+recovery_wait+degraded, 70 active+clean; 4.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 18/84 objects degraded (21.429%) 2023-10-09T06:02:22.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:22 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/563882789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:02:22.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:22 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:02:22.317 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:22.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:22.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:22.421 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:22.449 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:22.751 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:02:22.752 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:02:22.807 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:02:22.811 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:22.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:22.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:22.917 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:22.945 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:23.046 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:23.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:23.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:23.090 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:23.090 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[184418]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.090 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[184418]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.090 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:02:23.090 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[184418]: osdmap e511: 8 total, 8 up, 7 in 2023-10-09T06:02:23.090 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[184418]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.090 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[184418]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.091 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:23.091 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[191227]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.091 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[191227]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.091 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:02:23.091 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[191227]: osdmap e511: 8 total, 8 up, 7 in 2023-10-09T06:02:23.091 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[191227]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.091 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:23 smithi008 ceph-mon[191227]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.150 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:23.181 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:23 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:23 smithi191 ceph-mon[161669]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:23 smithi191 ceph-mon[161669]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:23 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:02:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:23 smithi191 ceph-mon[161669]: osdmap e511: 8 total, 8 up, 7 in 2023-10-09T06:02:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:23 smithi191 ceph-mon[161669]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:23 smithi191 ceph-mon[161669]: failed to encode map e511 with expected crc 2023-10-09T06:02:23.304 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:23.358 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:23.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:23.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:23.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:02:23.461 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:23.491 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:23.548 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:23.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:23.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:23.656 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:23.684 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:23.729 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:23.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:23.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:23.760 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:23.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:23.866 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:23.958 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:23.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:23.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:24.062 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:24.088 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:24.183 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:24.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:24.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:24 smithi008 ceph-mon[184418]: pgmap v1529: 73 pgs: 1 active+recovering, 1 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 8/84 objects degraded (9.524%) 2023-10-09T06:02:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:24 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 8/84 objects degraded (9.524%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:02:24.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:24 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1357070702' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:02:24.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:24 smithi008 ceph-mon[191227]: pgmap v1529: 73 pgs: 1 active+recovering, 1 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 8/84 objects degraded (9.524%) 2023-10-09T06:02:24.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:24 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 8/84 objects degraded (9.524%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:02:24.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:24 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1357070702' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:02:24.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:24.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:24 smithi191 ceph-mon[161669]: pgmap v1529: 73 pgs: 1 active+recovering, 1 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 8/84 objects degraded (9.524%) 2023-10-09T06:02:24.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:24 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 8/84 objects degraded (9.524%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:02:24.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:24 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1357070702' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:02:24.315 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:24.335 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_5' created 2023-10-09T06:02:24.363 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:24.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:24.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:24.467 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:24.494 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:24.738 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:24.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:24.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:24.842 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:24.871 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:24.873 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:02:24.971 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:24.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:24.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:25.075 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:25.104 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:25.190 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T06:02:25.202 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:25.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:25.296 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:02:25.296 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:02:25.306 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:25.336 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:25 smithi191 ceph-mon[161669]: failed to encode map e512 with expected crc 2023-10-09T06:02:25.336 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:25 smithi191 ceph-mon[161669]: failed to encode map e512 with expected crc 2023-10-09T06:02:25.336 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:25 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1357070702' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:02:25.337 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:25 smithi191 ceph-mon[161669]: osdmap e512: 8 total, 8 up, 7 in 2023-10-09T06:02:25.337 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:25 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:25.343 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:25.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:25 smithi008 ceph-mon[184418]: failed to encode map e512 with expected crc 2023-10-09T06:02:25.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:25 smithi008 ceph-mon[184418]: failed to encode map e512 with expected crc 2023-10-09T06:02:25.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:25 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1357070702' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:02:25.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:25 smithi008 ceph-mon[184418]: osdmap e512: 8 total, 8 up, 7 in 2023-10-09T06:02:25.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:25 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:25.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:25 smithi008 ceph-mon[191227]: failed to encode map e512 with expected crc 2023-10-09T06:02:25.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:25 smithi008 ceph-mon[191227]: failed to encode map e512 with expected crc 2023-10-09T06:02:25.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:25 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1357070702' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:02:25.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:25 smithi008 ceph-mon[191227]: osdmap e512: 8 total, 8 up, 7 in 2023-10-09T06:02:25.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:25 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:25.532 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:02:25.561 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:02:25.586 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:02:25.608 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:25.614 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:02:25.640 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:02:25.663 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:02:25.686 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:02:25.710 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:02:25.714 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:02:25.714 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:02:25.715 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:02:25.715 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:02:25.715 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:02:25.715 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:02:25.715 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:02:25.715 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:02:25.728 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:26.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:02:26.050 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:26.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:26.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:26.157 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:26.186 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:26.510 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:26.668 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[184418]: pgmap v1531: 89 pgs: 16 unknown, 1 active+recovering, 72 active+clean; 4.6 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:02:26.668 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[184418]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/84 objects degraded (9.524%), 1 pg degraded) 2023-10-09T06:02:26.668 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[184418]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.669 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[184418]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.669 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[184418]: osdmap e513: 8 total, 8 up, 7 in 2023-10-09T06:02:26.669 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[184418]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.669 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[184418]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.669 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/527455767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:26.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[191227]: pgmap v1531: 89 pgs: 16 unknown, 1 active+recovering, 72 active+clean; 4.6 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:02:26.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[191227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/84 objects degraded (9.524%), 1 pg degraded) 2023-10-09T06:02:26.669 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[191227]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[191227]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[191227]: osdmap e513: 8 total, 8 up, 7 in 2023-10-09T06:02:26.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[191227]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[191227]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.670 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/527455767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:26.728 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:26 smithi191 ceph-mon[161669]: pgmap v1531: 89 pgs: 16 unknown, 1 active+recovering, 72 active+clean; 4.6 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:02:26.728 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:26 smithi191 ceph-mon[161669]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/84 objects degraded (9.524%), 1 pg degraded) 2023-10-09T06:02:26.728 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:26 smithi191 ceph-mon[161669]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.728 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:26 smithi191 ceph-mon[161669]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.728 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:26 smithi191 ceph-mon[161669]: osdmap e513: 8 total, 8 up, 7 in 2023-10-09T06:02:26.728 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:26 smithi191 ceph-mon[161669]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.728 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:26 smithi191 ceph-mon[161669]: failed to encode map e513 with expected crc 2023-10-09T06:02:26.728 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/527455767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:27.088 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:27.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:27.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:27.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:27.220 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:27.366 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:27.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:27.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:27.470 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:27.500 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:27.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:27 smithi191 ceph-mon[161669]: failed to encode map e514 with expected crc 2023-10-09T06:02:27.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:27 smithi191 ceph-mon[161669]: failed to encode map e514 with expected crc 2023-10-09T06:02:27.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:27 smithi191 ceph-mon[161669]: osdmap e514: 8 total, 8 up, 7 in 2023-10-09T06:02:27.609 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:27 smithi008 ceph-mon[184418]: failed to encode map e514 with expected crc 2023-10-09T06:02:27.609 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:27 smithi008 ceph-mon[184418]: failed to encode map e514 with expected crc 2023-10-09T06:02:27.609 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:27 smithi008 ceph-mon[184418]: osdmap e514: 8 total, 8 up, 7 in 2023-10-09T06:02:27.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:27 smithi008 ceph-mon[191227]: failed to encode map e514 with expected crc 2023-10-09T06:02:27.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:27 smithi008 ceph-mon[191227]: failed to encode map e514 with expected crc 2023-10-09T06:02:27.610 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:27 smithi008 ceph-mon[191227]: osdmap e514: 8 total, 8 up, 7 in 2023-10-09T06:02:28.202 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:28.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:28.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:28.228 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:02:28.228 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:02:28.228 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T06:02:28.229 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T06:02:28.229 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:02:28.306 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:28.335 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:28.440 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:28.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:28.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:28.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:28 smithi008 ceph-mon[184418]: pgmap v1534: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 4.6 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 4 B/s, 1 objects/s recovering 2023-10-09T06:02:28.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:28 smithi008 ceph-mon[184418]: failed to encode map e514 with expected crc 2023-10-09T06:02:28.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:28 smithi008 ceph-mon[184418]: failed to encode map e514 with expected crc 2023-10-09T06:02:28.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:28 smithi008 ceph-mon[191227]: pgmap v1534: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 4.6 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 4 B/s, 1 objects/s recovering 2023-10-09T06:02:28.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:28 smithi008 ceph-mon[191227]: failed to encode map e514 with expected crc 2023-10-09T06:02:28.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:28 smithi008 ceph-mon[191227]: failed to encode map e514 with expected crc 2023-10-09T06:02:28.545 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:28.579 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:28.603 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:28 smithi191 ceph-mon[161669]: pgmap v1534: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 4.6 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 4 B/s, 1 objects/s recovering 2023-10-09T06:02:28.603 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:28 smithi191 ceph-mon[161669]: failed to encode map e514 with expected crc 2023-10-09T06:02:28.603 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:28 smithi191 ceph-mon[161669]: failed to encode map e514 with expected crc 2023-10-09T06:02:28.621 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:28.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:28.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:28.725 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:28.751 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:28.826 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:02:29.011 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:29.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:29.081 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:02:29.115 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:29.151 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:29.250 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:29.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:29.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:29.354 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:29.387 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:29.438 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:29.586 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:29.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:29.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:29.648 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:29.691 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:29.720 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:29.764 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:29.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:29.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:29.867 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:29.896 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:29.942 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:02:29.943 INFO:teuthology.orchestra.run.smithi008.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T06:02:29.972 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:29 smithi191 ceph-mon[161669]: pgmap v1535: 89 pgs: 1 creating+peering, 88 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 3 B/s, 1 objects/s recovering 2023-10-09T06:02:29.972 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:29.972 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:29.972 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1697794728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:29.973 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4248687464' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:02:29.973 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:29 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:02:29.994 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:29.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:29.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:30.021 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[184418]: pgmap v1535: 89 pgs: 1 creating+peering, 88 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 3 B/s, 1 objects/s recovering 2023-10-09T06:02:30.021 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:30.021 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:30.022 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1697794728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:30.022 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4248687464' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:02:30.022 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:02:30.022 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[191227]: pgmap v1535: 89 pgs: 1 creating+peering, 88 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 3 B/s, 1 objects/s recovering 2023-10-09T06:02:30.022 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:30.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:30.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1697794728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:30.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4248687464' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:02:30.023 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:29 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:02:30.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:30.128 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:30.275 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:30.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:30.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:30.379 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:30.409 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:30.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:02:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:02:30] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.33.4" 2023-10-09T06:02:30.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:30 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:02:30.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:30 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T06:02:30.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:30 smithi191 ceph-mon[161669]: failed to encode map e515 with expected crc 2023-10-09T06:02:30.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1697794728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:30.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:30 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:02:30.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:30 smithi191 ceph-mon[161669]: osdmap e515: 8 total, 8 up, 7 in 2023-10-09T06:02:30.975 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:30 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:30.976 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:30 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:30.976 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:30 smithi191 ceph-mon[161669]: failed to encode map e515 with expected crc 2023-10-09T06:02:31.164 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:02:31.165 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T06:02:31.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T06:02:31.165 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T06:02:31.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.165 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:16705 9.0 e368/364 9:0a7d3e3b:::benchmark_data_smithi008_502820_object16704:head v 368'1048, mlcod=368'1046)", 2023-10-09T06:02:31.165 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:53:25.240076+0000", 2023-10-09T06:02:31.165 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 545.92295374699995, 2023-10-09T06:02:31.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28665484899999999, 2023-10-09T06:02:31.165 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.165 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.165 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240076+0000", 2023-10-09T06:02:31.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240076+0000", 2023-10-09T06:02:31.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:31.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240074+0000", 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7774000000000005e-05 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240142+0000", 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3613e-05 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240156+0000", 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5190000000000003e-06 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240163+0000", 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3522999999999995e-05 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240227+0000", 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0446e-05 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240247+0000", 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023687859999999999 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.242616+0000", 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.284114539 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.526731+0000", 2023-10-09T06:02:31.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28665484899999999 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:16709 9.1 e368/364 9:82a6c9a8:::benchmark_data_smithi008_502820_object16708:head v 368'1053, mlcod=368'1051)", 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:53:25.240758+0000", 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 545.92227185299998, 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28606809500000002, 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.170 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240758+0000", 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240758+0000", 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240757+0000", 2023-10-09T06:02:31.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4219e-05 2023-10-09T06:02:31.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240781+0000", 2023-10-09T06:02:31.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2165e-05 2023-10-09T06:02:31.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240793+0000", 2023-10-09T06:02:31.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2404e-05 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240806+0000", 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7112000000000004e-05 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240873+0000", 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1267000000000002e-05 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.240894+0000", 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00088232899999999995 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.241776+0000", 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28504944799999998 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.526826+0000", 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28606809500000002 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:16708 9.0 e368/364 9:07ac6ad8:::benchmark_data_smithi008_502820_object16707:head v 368'1049, mlcod=368'1046)", 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:53:25.242846+0000", 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 545.92018374300005, 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28675348299999998, 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.175 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.242846+0000", 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.242846+0000", 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.242845+0000", 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4651000000000002e-05 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.242909+0000", 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28378102999999999 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.526690+0000", 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0631999999999999e-05 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.526701+0000", 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9734999999999994e-05 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.526791+0000", 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0696999999999999e-05 2023-10-09T06:02:31.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.526811+0000", 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027355729999999998 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.529547+0000", 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2176000000000003e-05 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.529599+0000", 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28675348299999998 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:16727 9.b e368/364 9:d26cfa07:::benchmark_data_smithi008_502820_object16726:head v 368'1038, mlcod=368'1036)", 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:53:25.249214+0000", 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 545.91381496999998, 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27945837200000001, 2023-10-09T06:02:31.180 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.249214+0000", 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.249214+0000", 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:31.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.249213+0000", 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9053e-05 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.249252+0000", 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27744542599999999 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.526698+0000", 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0835999999999999e-05 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.526709+0000", 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9388999999999997e-05 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.526768+0000", 2023-10-09T06:02:31.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1770999999999999e-05 2023-10-09T06:02:31.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.526790+0000", 2023-10-09T06:02:31.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018404599999999999 2023-10-09T06:02:31.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.528630+0000", 2023-10-09T06:02:31.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2700000000000001e-05 2023-10-09T06:02:31.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:53:25.528673+0000", 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27945837200000001 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:77617 9.1 e384/382 9:84bc2af6:::benchmark_data_smithi008_502820_object77616:head v 384'4906, mlcod=384'4904)", 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.706670+0000", 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 474.45635970900003, 2023-10-09T06:02:31.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.029753736999999999, 2023-10-09T06:02:31.186 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.186 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.186 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.706670+0000", 2023-10-09T06:02:31.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.706670+0000", 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.706668+0000", 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7237000000000002e-05 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.706715+0000", 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.419e-05 2023-10-09T06:02:31.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.706739+0000", 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0159e-05 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.706749+0000", 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2736000000000002e-05 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.706832+0000", 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8983999999999999e-05 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.706861+0000", 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.029509489 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.736370+0000", 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3056999999999997e-05 2023-10-09T06:02:31.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.736423+0000", 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.029753736999999999 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:77618 9.1 e384/382 9:8a9bea20:::benchmark_data_smithi008_502820_object77617:head v 384'4907, mlcod=384'4904)", 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:36.709726+0000", 2023-10-09T06:02:31.190 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 474.45330381100001, 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.111253955, 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.709726+0000", 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.709726+0000", 2023-10-09T06:02:31.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:31.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.709724+0000", 2023-10-09T06:02:31.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5880000000000002e-05 2023-10-09T06:02:31.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.709760+0000", 2023-10-09T06:02:31.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1956999999999999e-05 2023-10-09T06:02:31.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.709772+0000", 2023-10-09T06:02:31.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1990000000000006e-06 2023-10-09T06:02:31.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.709780+0000", 2023-10-09T06:02:31.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019103689 2023-10-09T06:02:31.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.728884+0000", 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8044000000000001e-05 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.728912+0000", 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.092067939000000001 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:36.820980+0000", 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.111253955 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:81587 9.11 e389/385 9:895c7a80:::benchmark_data_smithi008_502820_object81586:head v 389'5036, mlcod=389'5034)", 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:43.561473+0000", 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 467.60155631600003, 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.061515009000000002, 2023-10-09T06:02:31.195 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.561473+0000", 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.561473+0000", 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:31.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.561472+0000", 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9019000000000001e-05 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.561501+0000", 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6682999999999999e-05 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.561518+0000", 2023-10-09T06:02:31.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3030000000000002e-06 2023-10-09T06:02:31.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.561525+0000", 2023-10-09T06:02:31.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2442999999999998e-05 2023-10-09T06:02:31.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.561587+0000", 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2682999999999999e-05 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.561610+0000", 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001551352 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.563161+0000", 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.059826807000000003 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.622988+0000", 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.061515009000000002 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:81596 9.10 e389/385 9:08dc9b07:::benchmark_data_smithi008_502820_object81595:head v 389'4893, mlcod=389'4891)", 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:43.577746+0000", 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 467.58528306900001, 2023-10-09T06:02:31.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.047136867999999998, 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.577746+0000", 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.577746+0000", 2023-10-09T06:02:31.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:02:31.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.577745+0000", 2023-10-09T06:02:31.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5331000000000003e-05 2023-10-09T06:02:31.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.577801+0000", 2023-10-09T06:02:31.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.045163937000000001 2023-10-09T06:02:31.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.622965+0000", 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0621e-05 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.622975+0000", 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000113057 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.623088+0000", 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3883000000000001e-05 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.623102+0000", 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017422449999999999 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.624845+0000", 2023-10-09T06:02:31.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8695000000000001e-05 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.624883+0000", 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.047136867999999998 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:81602 9.11 e389/385 9:89d3e0e7:::benchmark_data_smithi008_502820_object81601:head v 389'5037, mlcod=389'5034)", 2023-10-09T06:02:31.205 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:43.585630+0000", 2023-10-09T06:02:31.206 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 467.57739913500001, 2023-10-09T06:02:31.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.040291449, 2023-10-09T06:02:31.206 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.206 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.206 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.585630+0000", 2023-10-09T06:02:31.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.585630+0000", 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.585629+0000", 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9085999999999999e-05 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.585658+0000", 2023-10-09T06:02:31.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.037318697999999997 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.622977+0000", 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6839999999999999e-06 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.622983+0000", 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016529050000000001 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.624636+0000", 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8709e-05 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.624655+0000", 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001238261 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.625893+0000", 2023-10-09T06:02:31.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8362000000000001e-05 2023-10-09T06:02:31.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.625922+0000", 2023-10-09T06:02:31.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.040291449 2023-10-09T06:02:31.210 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.210 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.210 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.210 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:81611 9.b e389/364 9:d65c9543:::benchmark_data_smithi008_502820_object81610:head v 389'5149, mlcod=389'5147)", 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:54:43.593036+0000", 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 467.56999307400002, 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.032590622999999999, 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.593036+0000", 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.593036+0000", 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.593035+0000", 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9262999999999998e-05 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.593094+0000", 2023-10-09T06:02:31.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.029862639 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.622956+0000", 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4351e-05 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.622971+0000", 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5827e-05 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.623017+0000", 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1781999999999999e-05 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.623038+0000", 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025550490000000002 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.625594+0000", 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3461999999999999e-05 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:54:43.625627+0000", 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.032590622999999999 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.215 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:99279 9.11 e406/402 9:8e9b682f:::benchmark_data_smithi008_502820_object15106:head v 406'5581, mlcod=400'5579)", 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:55:39.404506+0000", 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 411.75852317800002, 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.092127698999999993, 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.404506+0000", 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.404506+0000", 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.404506+0000", 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1279999999999996e-06 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.404512+0000", 2023-10-09T06:02:31.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.090219032000000005 2023-10-09T06:02:31.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.494731+0000", 2023-10-09T06:02:31.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8028000000000001e-05 2023-10-09T06:02:31.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.494749+0000", 2023-10-09T06:02:31.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1785000000000005e-05 2023-10-09T06:02:31.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.494821+0000", 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0685999999999997e-05 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.494852+0000", 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017422570000000001 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.496594+0000", 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0144000000000001e-05 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:55:39.496634+0000", 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.092127698999999993 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.220 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:119075 9.b e456/449 9:ded4ed15:::benchmark_data_smithi008_502820_object34902:head v 456'7511, mlcod=369'2101)", 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:35.355282+0000", 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 235.80774746099999, 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022608672999999999, 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:35.355282+0000", 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:35.355282+0000", 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:35.355282+0000", 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.636e-06 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:35.355290+0000", 2023-10-09T06:02:31.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1169e-05 2023-10-09T06:02:31.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:35.355311+0000", 2023-10-09T06:02:31.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1889999999999996e-06 2023-10-09T06:02:31.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:35.355321+0000", 2023-10-09T06:02:31.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7661000000000002e-05 2023-10-09T06:02:31.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:35.355378+0000", 2023-10-09T06:02:31.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9326e-05 2023-10-09T06:02:31.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:35.355408+0000", 2023-10-09T06:02:31.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022446509999999999 2023-10-09T06:02:31.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:35.377854+0000", 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6600000000000002e-05 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:35.377891+0000", 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022608672999999999 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.225 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:161838 9.10 e501/494 9:0a368041:::benchmark_data_smithi008_502820_object77665:head v 501'7379, mlcod=386'4782)", 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:01:53.592764+0000", 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 37.570265448000001, 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.066155467999999995, 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.592764+0000", 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.592764+0000", 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.592762+0000", 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1916000000000001e-05 2023-10-09T06:02:31.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.592774+0000", 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4184e-05 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.592789+0000", 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1299999999999998e-06 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.592795+0000", 2023-10-09T06:02:31.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1918000000000001e-05 2023-10-09T06:02:31.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.592837+0000", 2023-10-09T06:02:31.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9762000000000002e-05 2023-10-09T06:02:31.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.592856+0000", 2023-10-09T06:02:31.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.066022058999999994 2023-10-09T06:02:31.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.658878+0000", 2023-10-09T06:02:31.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0986999999999999e-05 2023-10-09T06:02:31.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.658919+0000", 2023-10-09T06:02:31.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.066155467999999995 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:161840 9.11 e501/493 9:89a5ba47:::benchmark_data_smithi008_502820_object77667:head v 501'7540, mlcod=501'7538)", 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:01:53.593591+0000", 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 37.569438839999997, 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.070789369000000005, 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.231 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.593591+0000", 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.593591+0000", 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.593590+0000", 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3070000000000003e-06 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.593598+0000", 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0122999999999999e-05 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.593608+0000", 2023-10-09T06:02:31.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3850000000000004e-06 2023-10-09T06:02:31.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.593613+0000", 2023-10-09T06:02:31.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1363999999999999e-05 2023-10-09T06:02:31.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.593654+0000", 2023-10-09T06:02:31.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0064999999999998e-05 2023-10-09T06:02:31.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.593674+0000", 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.070675752999999994 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.664350+0000", 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0239e-05 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.664380+0000", 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.070789369000000005 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:161842 9.19 e501/494 9:9ebef787:::benchmark_data_smithi008_502820_object77669:head v 501'7442, mlcod=380'4109)", 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:01:53.594508+0000", 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 37.568521584999999, 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.071248230999999995, 2023-10-09T06:02:31.236 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.594508+0000", 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.594508+0000", 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:02:31.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.594507+0000", 2023-10-09T06:02:31.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1141000000000001e-05 2023-10-09T06:02:31.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.594518+0000", 2023-10-09T06:02:31.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.064348021000000005 2023-10-09T06:02:31.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.658866+0000", 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3851e-05 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.658880+0000", 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00183706 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.660717+0000", 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2436e-05 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.660740+0000", 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0049975979999999998 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.665737+0000", 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8892000000000001e-05 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.665756+0000", 2023-10-09T06:02:31.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.071248230999999995 2023-10-09T06:02:31.241 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.241 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.241 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.241 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:161843 9.11 e501/493 9:8d283826:::benchmark_data_smithi008_502820_object77670:head v 501'7541, mlcod=501'7538)", 2023-10-09T06:02:31.241 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:01:53.599014+0000", 2023-10-09T06:02:31.241 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 37.564015621999999, 2023-10-09T06:02:31.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.066707379999999997, 2023-10-09T06:02:31.242 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.242 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.242 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.599014+0000", 2023-10-09T06:02:31.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.599014+0000", 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.599013+0000", 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5360000000000004e-06 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.599020+0000", 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.059846360000000001 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.658867+0000", 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2673e-05 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.658880+0000", 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021015819999999998 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.660981+0000", 2023-10-09T06:02:31.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3645999999999999e-05 2023-10-09T06:02:31.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.660995+0000", 2023-10-09T06:02:31.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047034499999999996 2023-10-09T06:02:31.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.665698+0000", 2023-10-09T06:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2972999999999998e-05 2023-10-09T06:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:01:53.665721+0000", 2023-10-09T06:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.066707379999999997 2023-10-09T06:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.247 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:167079 9.1 e507/493 9:84fcb84b:::benchmark_data_smithi008_502820_object82906:head v 507'7727, mlcod=444'6132)", 2023-10-09T06:02:31.247 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:02:10.011592+0000", 2023-10-09T06:02:31.247 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 21.151437575999999, 2023-10-09T06:02:31.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028159052, 2023-10-09T06:02:31.247 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.247 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.247 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011592+0000", 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011592+0000", 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011591+0000", 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4529999999999996e-06 2023-10-09T06:02:31.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011599+0000", 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.418e-05 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011613+0000", 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1469999999999998e-06 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011619+0000", 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0098382150000000008 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.021457+0000", 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2436999999999999e-05 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.021480+0000", 2023-10-09T06:02:31.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018242286999999999 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.039722+0000", 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8954000000000001e-05 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.039751+0000", 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028159052 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.251 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:167102 9.11 e507/493 9:8f27971d:::benchmark_data_smithi008_502820_object82929:head v 507'7707, mlcod=504'7705)", 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:02:10.011644+0000", 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 21.151385854000001, 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034782459000000002, 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011644+0000", 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011644+0000", 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011643+0000", 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4240000000000003e-06 2023-10-09T06:02:31.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011648+0000", 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3470000000000004e-06 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011654+0000", 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1670000000000001e-06 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.011658+0000", 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012455601 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.024114+0000", 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4542999999999999e-05 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.024138+0000", 2023-10-09T06:02:31.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022252574000000001 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.046391+0000", 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5203999999999997e-05 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.046426+0000", 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034782459000000002 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.256 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:166960 9.19 e507/506 9:9ace92ad:::benchmark_data_smithi008_502820_object82787:head v 507'7601, mlcod=0'0)", 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:02:10.020925+0000", 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 21.142104069999998, 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028612162999999999, 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.020925+0000", 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.020925+0000", 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.020924+0000", 2023-10-09T06:02:31.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2110000000000004e-06 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.020933+0000", 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0964999999999999e-05 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.020944+0000", 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5939999999999999e-06 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.020951+0000", 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037903030000000001 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.024741+0000", 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3394e-05 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.024765+0000", 2023-10-09T06:02:31.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024729131000000001 2023-10-09T06:02:31.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.049494+0000", 2023-10-09T06:02:31.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3655000000000001e-05 2023-10-09T06:02:31.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.049538+0000", 2023-10-09T06:02:31.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028612162999999999 2023-10-09T06:02:31.262 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.262 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.262 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.262 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36258.0:166994 9.19 e507/506 9:9fcc3a2e:::benchmark_data_smithi008_502820_object82821:head v 507'7604, mlcod=0'0)", 2023-10-09T06:02:31.262 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:02:10.032664+0000", 2023-10-09T06:02:31.262 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 21.130365335, 2023-10-09T06:02:31.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024633021000000001, 2023-10-09T06:02:31.262 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:02:31.262 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.032664+0000", 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.032664+0000", 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.032663+0000", 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2130000000000008e-06 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.032671+0000", 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4618e-05 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:02:31.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.032686+0000", 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6019999999999999e-05 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.032702+0000", 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070161370000000004 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.039718+0000", 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3719e-05 2023-10-09T06:02:31.265 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:02:31.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.039752+0000", 2023-10-09T06:02:31.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017508111999999999 2023-10-09T06:02:31.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.266 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:02:31.266 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.057260+0000", 2023-10-09T06:02:31.266 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7086999999999999e-05 2023-10-09T06:02:31.266 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:02:31.266 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:02:31.267 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:02:31.267 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:02:10.057297+0000", 2023-10-09T06:02:31.267 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024633021000000001 2023-10-09T06:02:31.267 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.267 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.267 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.267 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:02:31.267 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:02:31.267 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:02:31.268 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:02:31.268 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T06:02:31.268 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[191227]: failed to encode map e515 with expected crc 2023-10-09T06:02:31.268 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1697794728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:31.268 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:02:31.268 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[191227]: osdmap e515: 8 total, 8 up, 7 in 2023-10-09T06:02:31.268 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:31.268 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:31.269 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[191227]: failed to encode map e515 with expected crc 2023-10-09T06:02:31.269 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:02:31.269 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T06:02:31.269 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[184418]: failed to encode map e515 with expected crc 2023-10-09T06:02:31.269 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1697794728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:31.269 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:02:31.269 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[184418]: osdmap e515: 8 total, 8 up, 7 in 2023-10-09T06:02:31.269 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:31.269 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:31.269 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:30 smithi008 ceph-mon[184418]: failed to encode map e515 with expected crc 2023-10-09T06:02:31.464 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:31.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:31.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:31.568 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:31.600 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:31.848 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:31.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:31.927 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:02:31.951 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:31.985 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:32.094 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:32.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:32.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:32.201 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:32.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:31 smithi008 ceph-mon[184418]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:31 smithi008 ceph-mon[184418]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:31 smithi008 ceph-mon[184418]: pgmap v1537: 89 pgs: 89 active+clean; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:02:32.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:31 smithi008 ceph-mon[184418]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:31 smithi008 ceph-mon[184418]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:31 smithi008 ceph-mon[191227]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:31 smithi008 ceph-mon[191227]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:31 smithi008 ceph-mon[191227]: pgmap v1537: 89 pgs: 89 active+clean; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:02:32.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:31 smithi008 ceph-mon[191227]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:31 smithi008 ceph-mon[191227]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.231 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:32.274 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:32.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:31 smithi191 ceph-mon[161669]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:31 smithi191 ceph-mon[161669]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:31 smithi191 ceph-mon[161669]: pgmap v1537: 89 pgs: 89 active+clean; 4.6 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:02:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:31 smithi191 ceph-mon[161669]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:31 smithi191 ceph-mon[161669]: failed to encode map e515 with expected crc 2023-10-09T06:02:32.385 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:32.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:32.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:32.490 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:32.521 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:32.574 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:02:32.593 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:02:32.612 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:02:32.631 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:02:32.649 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:02:32.670 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:02:32.679 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:32.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:32.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:32.690 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:02:32.708 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:02:32.709 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:02:32.709 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:02:32.710 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:02:32.710 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:02:32.710 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:02:32.710 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:02:32.710 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:02:32.710 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:02:32.784 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:32.814 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:32.968 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:32.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:32.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:33.071 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:33.082 DEBUG:teuthology.orchestra.run.smithi008:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T06:02:33.083 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T06:02:33.083 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:02:33.096 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:32 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2119273709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:33.097 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:33.097 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:33.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:32 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2119273709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:33.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:33.097 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:33.099 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:02:33.099 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:02:33.100 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:33.150 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:hints = 1 2023-10-09T06:02:33.150 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:02:33.150 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object prefix: benchmark_data_smithi008_634586 2023-10-09T06:02:33.151 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:02:33.151 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:02:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:32 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2119273709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:33.416 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:33.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:33.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:33.521 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:33.585 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:33.698 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:33.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:33.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:33.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:33.836 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:33.851 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:33.884 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:02:33.885 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:02:33.923 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:33.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:34.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:34.044 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:02:34.068 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:34.127 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:34.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:34.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:34.151 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 1 16 2592 2576 160.991 161 0.00591589 0.0061905 2023-10-09T06:02:34.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:34.256 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:34.296 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:34.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:34.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:34.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:34 smithi008 ceph-mon[184418]: pgmap v1538: 89 pgs: 89 active+clean; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:02:34.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:34 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:34.362 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:34 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:34.362 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:34 smithi008 ceph-mon[191227]: pgmap v1538: 89 pgs: 89 active+clean; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:02:34.362 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:34 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:34.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:34 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:34.400 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:34.426 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:34.468 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:34.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:34.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:34 smithi191 ceph-mon[161669]: pgmap v1538: 89 pgs: 89 active+clean; 4.6 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:02:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:34 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:34.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:34 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:34.572 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:34.629 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:34.732 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:34.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:34.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:34.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:34.852 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:02:34.886 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:34.988 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:34.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:34.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:35.096 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:35.151 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 2 16 4783 4767 148.943 136.938 0.00195565 0.00670238 2023-10-09T06:02:35.154 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:35.255 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:35.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:35.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:35.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:35.405 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:35.445 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:35.451 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:35.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:35.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:35.530 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:02:35.561 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:35.586 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:35.687 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:35.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:35.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:35.792 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:35.830 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:36.006 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:36.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:36.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:36.152 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 3 16 7095 7079 147.43 144.5 0.00710653 0.00677485 2023-10-09T06:02:36.156 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:36.232 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:36.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:36.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:36.336 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:36.375 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:36.551 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:36.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:36.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:36.587 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:36.587 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:36 smithi008 ceph-mon[184418]: pgmap v1539: 89 pgs: 89 active+clean; 8.8 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 509 KiB/s wr, 7 op/s 2023-10-09T06:02:36.588 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:36 smithi008 ceph-mon[191227]: pgmap v1539: 89 pgs: 89 active+clean; 8.8 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 509 KiB/s wr, 7 op/s 2023-10-09T06:02:36.654 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:36.737 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:36.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:36 smithi191 ceph-mon[161669]: pgmap v1539: 89 pgs: 89 active+clean; 8.8 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 509 KiB/s wr, 7 op/s 2023-10-09T06:02:36.818 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:36.947 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:36.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:36.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:37.055 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:37.102 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:37.152 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 4 16 8981 8965 140.04 117.875 0.00487065 0.00713047 2023-10-09T06:02:37.187 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:37.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:37.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:37.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:37.339 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:37.414 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:37.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:37.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:37.520 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:37.588 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:37.678 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:02:37.679 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:02:37.996 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:37.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:38.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:38.101 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:38.133 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:02:38.141 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T06:02:38.152 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 5 16 11463 11447 143.054 155.125 0.00723634 0.00698404 2023-10-09T06:02:38.156 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:38.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:38 smithi008 ceph-mon[184418]: pgmap v1540: 89 pgs: 89 active+clean; 242 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 24 MiB/s wr, 380 op/s 2023-10-09T06:02:38.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:38 smithi008 ceph-mon[191227]: pgmap v1540: 89 pgs: 89 active+clean; 242 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 24 MiB/s wr, 380 op/s 2023-10-09T06:02:38.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:38 smithi191 ceph-mon[161669]: pgmap v1540: 89 pgs: 89 active+clean; 242 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 24 MiB/s wr, 380 op/s 2023-10-09T06:02:38.918 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:02:39.152 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 6 16 13559 13543 141.043 131 0.00604116 0.00708349 2023-10-09T06:02:39.521 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:39.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:39 smithi008 ceph-mon[184418]: from='client.47042 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:02:39.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:39 smithi008 ceph-mon[191227]: from='client.47042 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:02:39.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:39 smithi191 ceph-mon[161669]: from='client.47042 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:02:40.152 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 7 16 16252 16236 144.937 168.312 0.00662989 0.00689258 2023-10-09T06:02:40.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:40 smithi008 ceph-mon[184418]: pgmap v1541: 89 pgs: 89 active+clean; 443 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T06:02:40.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:40 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:40.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:40 smithi008 ceph-mon[191227]: pgmap v1541: 89 pgs: 89 active+clean; 443 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T06:02:40.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:40 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:40 smithi191 ceph-mon[161669]: pgmap v1541: 89 pgs: 89 active+clean; 443 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T06:02:40.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:40 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:40.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:02:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:02:40] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.33.4" 2023-10-09T06:02:41.001 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:41.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:41.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:41.107 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:41.135 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:41.152 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 8 16 17930 17914 139.928 104.875 0.00166421 0.00703255 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:41.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:41.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:41.379 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:41.409 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:41.715 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1540,"stamp":"2023-10-09T06:02:36.761223+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":254154675,"num_objects":3833,"num_object_clones":0,"num_object_copies":11499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3833,"num_whiteouts":0,"num_read":6288,"num_read_kb":13711,"num_write":4911,"num_write_kb":258661,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":188,"num_bytes_recovered":9585208,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4975,"ondisk_log_size":4975,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":290,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8183632,"kb_used_data":1782488,"kb_used_omap":45,"kb_used_meta":6401042,"kb_avail":648032432,"statfs":{"total":671965249536,"available":663585210368,"internally_reserved":0,"allocated":1825267712,"data_stored":1813267672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46813,"internal_metadata":6554667299},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":249364480,"num_objects":3805,"num_object_clones":0,"num_object_copies":11415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3805,"num_write_kb":243520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002690"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":138,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:30.573886+0000","last_change":"2023-10-09T05:54:59.256087+0000","last_active":"2023-10-09T06:02:30.573886+0000","last_peered":"2023-10-09T06:02:30.573886+0000","last_clean":"2023-10-09T06:02:30.573886+0000","last_became_active":"2023-10-09T05:54:59.255773+0000","last_became_peered":"2023-10-09T05:54:59.255773+0000","last_unstale":"2023-10-09T06:02:30.573886+0000","last_undegraded":"2023-10-09T06:02:30.573886+0000","last_fullsized":"2023-10-09T06:02:30.573886+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:19.545754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":112,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.621311+0000","last_change":"2023-10-09T05:56:00.247579+0000","last_active":"2023-10-09T06:02:27.621311+0000","last_peered":"2023-10-09T06:02:27.621311+0000","last_clean":"2023-10-09T06:02:27.621311+0000","last_became_active":"2023-10-09T05:56:00.238151+0000","last_became_peered":"2023-10-09T05:56:00.238151+0000","last_unstale":"2023-10-09T06:02:27.621311+0000","last_undegraded":"2023-10-09T06:02:27.621311+0000","last_fullsized":"2023-10-09T06:02:27.621311+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:05.692190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":219,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.312359+0000","last_change":"2023-10-09T05:56:00.243822+0000","last_active":"2023-10-09T06:02:27.312359+0000","last_peered":"2023-10-09T06:02:27.312359+0000","last_clean":"2023-10-09T06:02:27.312359+0000","last_became_active":"2023-10-09T05:56:00.243483+0000","last_became_peered":"2023-10-09T05:56:00.243483+0000","last_unstale":"2023-10-09T06:02:27.312359+0000","last_undegraded":"2023-10-09T06:02:27.312359+0000","last_fullsized":"2023-10-09T06:02:27.312359+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:39.811952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":564,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.337484+0000","last_change":"2023-10-09T06:02:08.345185+0000","last_active":"2023-10-09T06:02:26.337484+0000","last_peered":"2023-10-09T06:02:26.337484+0000","last_clean":"2023-10-09T06:02:26.337484+0000","last_became_active":"2023-10-09T06:02:08.344803+0000","last_became_peered":"2023-10-09T06:02:08.344803+0000","last_unstale":"2023-10-09T06:02:26.337484+0000","last_undegraded":"2023-10-09T06:02:26.337484+0000","last_fullsized":"2023-10-09T06:02:26.337484+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":460,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:29.936695+0000","last_change":"2023-10-09T06:00:05.620750+0000","last_active":"2023-10-09T06:02:29.936695+0000","last_peered":"2023-10-09T06:02:29.936695+0000","last_clean":"2023-10-09T06:02:29.936695+0000","last_became_active":"2023-10-09T06:00:05.620298+0000","last_became_peered":"2023-10-09T06:00:05.620298+0000","last_unstale":"2023-10-09T06:02:29.936695+0000","last_undegraded":"2023-10-09T06:02:29.936695+0000","last_fullsized":"2023-10-09T06:02:29.936695+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:29.560982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.409953+0000","last_change":"2023-10-09T06:01:19.996464+0000","last_active":"2023-10-09T06:02:27.409953+0000","last_peered":"2023-10-09T06:02:27.409953+0000","last_clean":"2023-10-09T06:02:27.409953+0000","last_became_active":"2023-10-09T06:01:19.995823+0000","last_became_peered":"2023-10-09T06:01:19.995823+0000","last_unstale":"2023-10-09T06:02:27.409953+0000","last_undegraded":"2023-10-09T06:02:27.409953+0000","last_fullsized":"2023-10-09T06:02:27.409953+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:17:13.034516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":455,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:29.937769+0000","last_change":"2023-10-09T06:01:19.980073+0000","last_active":"2023-10-09T06:02:29.937769+0000","last_peered":"2023-10-09T06:02:29.937769+0000","last_clean":"2023-10-09T06:02:29.937769+0000","last_became_active":"2023-10-09T06:01:19.979716+0000","last_became_peered":"2023-10-09T06:01:19.979716+0000","last_unstale":"2023-10-09T06:02:29.937769+0000","last_undegraded":"2023-10-09T06:02:29.937769+0000","last_fullsized":"2023-10-09T06:02:29.937769+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:56.881360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":130,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.620857+0000","last_change":"2023-10-09T06:01:19.995701+0000","last_active":"2023-10-09T06:02:27.620857+0000","last_peered":"2023-10-09T06:02:27.620857+0000","last_clean":"2023-10-09T06:02:27.620857+0000","last_became_active":"2023-10-09T06:01:19.995221+0000","last_became_peered":"2023-10-09T06:01:19.995221+0000","last_unstale":"2023-10-09T06:02:27.620857+0000","last_undegraded":"2023-10-09T06:02:27.620857+0000","last_fullsized":"2023-10-09T06:02:27.620857+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:28.118628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":172,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.337805+0000","last_change":"2023-10-09T06:00:05.346651+0000","last_active":"2023-10-09T06:02:26.337805+0000","last_peered":"2023-10-09T06:02:26.337805+0000","last_clean":"2023-10-09T06:02:26.337805+0000","last_became_active":"2023-10-09T06:00:05.346292+0000","last_became_peered":"2023-10-09T06:00:05.346292+0000","last_unstale":"2023-10-09T06:02:26.337805+0000","last_undegraded":"2023-10-09T06:02:26.337805+0000","last_fullsized":"2023-10-09T06:02:26.337805+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":166,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.340005+0000","last_change":"2023-10-09T06:01:22.567256+0000","last_active":"2023-10-09T06:02:26.340005+0000","last_peered":"2023-10-09T06:02:26.340005+0000","last_clean":"2023-10-09T06:02:26.340005+0000","last_became_active":"2023-10-09T06:01:20.846276+0000","last_became_peered":"2023-10-09T06:01:20.846276+0000","last_unstale":"2023-10-09T06:02:26.340005+0000","last_undegraded":"2023-10-09T06:02:26.340005+0000","last_fullsized":"2023-10-09T06:02:26.340005+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":147,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.339005+0000","last_change":"2023-10-09T05:56:00.348771+0000","last_active":"2023-10-09T06:02:26.339005+0000","last_peered":"2023-10-09T06:02:26.339005+0000","last_clean":"2023-10-09T06:02:26.339005+0000","last_became_active":"2023-10-09T05:56:00.224347+0000","last_became_peered":"2023-10-09T05:56:00.224347+0000","last_unstale":"2023-10-09T06:02:26.339005+0000","last_undegraded":"2023-10-09T06:02:26.339005+0000","last_fullsized":"2023-10-09T06:02:26.339005+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":304,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.335680+0000","last_change":"2023-10-09T06:01:19.992812+0000","last_active":"2023-10-09T06:02:26.335680+0000","last_peered":"2023-10-09T06:02:26.335680+0000","last_clean":"2023-10-09T06:02:26.335680+0000","last_became_active":"2023-10-09T06:01:19.992374+0000","last_became_peered":"2023-10-09T06:01:19.992374+0000","last_unstale":"2023-10-09T06:02:26.335680+0000","last_undegraded":"2023-10-09T06:02:26.335680+0000","last_fullsized":"2023-10-09T06:02:26.335680+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:43.064961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":149,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.337193+0000","last_change":"2023-10-09T06:01:19.984380+0000","last_active":"2023-10-09T06:02:26.337193+0000","last_peered":"2023-10-09T06:02:26.337193+0000","last_clean":"2023-10-09T06:02:26.337193+0000","last_became_active":"2023-10-09T06:01:19.983167+0000","last_became_peered":"2023-10-09T06:01:19.983167+0000","last_unstale":"2023-10-09T06:02:26.337193+0000","last_undegraded":"2023-10-09T06:02:26.337193+0000","last_fullsized":"2023-10-09T06:02:26.337193+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":510,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.336382+0000","last_change":"2023-10-09T06:01:19.998578+0000","last_active":"2023-10-09T06:02:26.336382+0000","last_peered":"2023-10-09T06:02:26.336382+0000","last_clean":"2023-10-09T06:02:26.336382+0000","last_became_active":"2023-10-09T06:01:19.993114+0000","last_became_peered":"2023-10-09T06:01:19.993114+0000","last_unstale":"2023-10-09T06:02:26.336382+0000","last_undegraded":"2023-10-09T06:02:26.336382+0000","last_fullsized":"2023-10-09T06:02:26.336382+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:12.898507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":129,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.337060+0000","last_change":"2023-10-09T05:55:27.092744+0000","last_active":"2023-10-09T06:02:26.337060+0000","last_peered":"2023-10-09T06:02:26.337060+0000","last_clean":"2023-10-09T06:02:26.337060+0000","last_became_active":"2023-10-09T05:55:26.271096+0000","last_became_peered":"2023-10-09T05:55:26.271096+0000","last_unstale":"2023-10-09T06:02:26.337060+0000","last_undegraded":"2023-10-09T06:02:26.337060+0000","last_fullsized":"2023-10-09T06:02:26.337060+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":150,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.337112+0000","last_change":"2023-10-09T05:56:49.266384+0000","last_active":"2023-10-09T06:02:26.337112+0000","last_peered":"2023-10-09T06:02:26.337112+0000","last_clean":"2023-10-09T06:02:26.337112+0000","last_became_active":"2023-10-09T05:56:49.266051+0000","last_became_peered":"2023-10-09T05:56:49.266051+0000","last_unstale":"2023-10-09T06:02:26.337112+0000","last_undegraded":"2023-10-09T06:02:26.337112+0000","last_fullsized":"2023-10-09T06:02:26.337112+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":538,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.338011+0000","last_change":"2023-10-09T06:02:08.342155+0000","last_active":"2023-10-09T06:02:26.338011+0000","last_peered":"2023-10-09T06:02:26.338011+0000","last_clean":"2023-10-09T06:02:26.338011+0000","last_became_active":"2023-10-09T06:02:08.341867+0000","last_became_peered":"2023-10-09T06:02:08.341867+0000","last_unstale":"2023-10-09T06:02:26.338011+0000","last_undegraded":"2023-10-09T06:02:26.338011+0000","last_fullsized":"2023-10-09T06:02:26.338011+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":218,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.339111+0000","last_change":"2023-10-09T06:01:20.027680+0000","last_active":"2023-10-09T06:02:26.339111+0000","last_peered":"2023-10-09T06:02:26.339111+0000","last_clean":"2023-10-09T06:02:26.339111+0000","last_became_active":"2023-10-09T06:01:20.000504+0000","last_became_peered":"2023-10-09T06:01:20.000504+0000","last_unstale":"2023-10-09T06:02:26.339111+0000","last_undegraded":"2023-10-09T06:02:26.339111+0000","last_fullsized":"2023-10-09T06:02:26.339111+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":148,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.339922+0000","last_change":"2023-10-09T06:01:22.566883+0000","last_active":"2023-10-09T06:02:26.339922+0000","last_peered":"2023-10-09T06:02:26.339922+0000","last_clean":"2023-10-09T06:02:26.339922+0000","last_became_active":"2023-10-09T06:01:20.845907+0000","last_became_peered":"2023-10-09T06:01:20.845907+0000","last_unstale":"2023-10-09T06:02:26.339922+0000","last_undegraded":"2023-10-09T06:02:26.339922+0000","last_fullsized":"2023-10-09T06:02:26.339922+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":517,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.337588+0000","last_change":"2023-10-09T06:02:08.344908+0000","last_active":"2023-10-09T06:02:26.337588+0000","last_peered":"2023-10-09T06:02:26.337588+0000","last_clean":"2023-10-09T06:02:26.337588+0000","last_became_active":"2023-10-09T06:02:08.344609+0000","last_became_peered":"2023-10-09T06:02:08.344609+0000","last_unstale":"2023-10-09T06:02:26.337588+0000","last_undegraded":"2023-10-09T06:02:26.337588+0000","last_fullsized":"2023-10-09T06:02:26.337588+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":306,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.506377+0000","last_change":"2023-10-09T06:02:17.089713+0000","last_active":"2023-10-09T06:02:27.506377+0000","last_peered":"2023-10-09T06:02:27.506377+0000","last_clean":"2023-10-09T06:02:27.506377+0000","last_became_active":"2023-10-09T06:01:19.993580+0000","last_became_peered":"2023-10-09T06:01:19.993580+0000","last_unstale":"2023-10-09T06:02:27.506377+0000","last_undegraded":"2023-10-09T06:02:27.506377+0000","last_fullsized":"2023-10-09T06:02:27.506377+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:39.307684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":149,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.338321+0000","last_change":"2023-10-09T05:56:49.263088+0000","last_active":"2023-10-09T06:02:26.338321+0000","last_peered":"2023-10-09T06:02:26.338321+0000","last_clean":"2023-10-09T06:02:26.338321+0000","last_became_active":"2023-10-09T05:56:49.262781+0000","last_became_peered":"2023-10-09T05:56:49.262781+0000","last_unstale":"2023-10-09T06:02:26.338321+0000","last_undegraded":"2023-10-09T06:02:26.338321+0000","last_fullsized":"2023-10-09T06:02:26.338321+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":12,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.506463+0000","last_change":"2023-10-09T06:02:25.343221+0000","last_active":"2023-10-09T06:02:27.506463+0000","last_peered":"2023-10-09T06:02:27.506463+0000","last_clean":"2023-10-09T06:02:27.506463+0000","last_became_active":"2023-10-09T06:02:25.342973+0000","last_became_peered":"2023-10-09T06:02:25.342973+0000","last_unstale":"2023-10-09T06:02:27.506463+0000","last_undegraded":"2023-10-09T06:02:27.506463+0000","last_fullsized":"2023-10-09T06:02:27.506463+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:44.755600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":156,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.338442+0000","last_change":"2023-10-09T06:00:05.328315+0000","last_active":"2023-10-09T06:02:26.338442+0000","last_peered":"2023-10-09T06:02:26.338442+0000","last_clean":"2023-10-09T06:02:26.338442+0000","last_became_active":"2023-10-09T06:00:05.327973+0000","last_became_peered":"2023-10-09T06:00:05.327973+0000","last_unstale":"2023-10-09T06:02:26.338442+0000","last_undegraded":"2023-10-09T06:02:26.338442+0000","last_fullsized":"2023-10-09T06:02:26.338442+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":187,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.336870+0000","last_change":"2023-10-09T06:01:19.989058+0000","last_active":"2023-10-09T06:02:26.336870+0000","last_peered":"2023-10-09T06:02:26.336870+0000","last_clean":"2023-10-09T06:02:26.336870+0000","last_became_active":"2023-10-09T06:01:19.988642+0000","last_became_peered":"2023-10-09T06:01:19.988642+0000","last_unstale":"2023-10-09T06:02:26.336870+0000","last_undegraded":"2023-10-09T06:02:26.336870+0000","last_fullsized":"2023-10-09T06:02:26.336870+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"515'357","reported_seq":369,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:35.387160+0000","last_change":"2023-10-09T06:02:25.351087+0000","last_active":"2023-10-09T06:02:35.387160+0000","last_peered":"2023-10-09T06:02:35.387160+0000","last_clean":"2023-10-09T06:02:35.387160+0000","last_became_active":"2023-10-09T06:02:25.350777+0000","last_became_peered":"2023-10-09T06:02:25.350777+0000","last_unstale":"2023-10-09T06:02:35.387160+0000","last_undegraded":"2023-10-09T06:02:35.387160+0000","last_fullsized":"2023-10-09T06:02:35.387160+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":357,"log_dups_size":0,"ondisk_log_size":357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:44.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23396352,"num_objects":357,"num_object_clones":0,"num_object_copies":1071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":357,"num_write_kb":22848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":127,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.337696+0000","last_change":"2023-10-09T05:55:26.358519+0000","last_active":"2023-10-09T06:02:26.337696+0000","last_peered":"2023-10-09T06:02:26.337696+0000","last_clean":"2023-10-09T06:02:26.337696+0000","last_became_active":"2023-10-09T05:55:26.260138+0000","last_became_peered":"2023-10-09T05:55:26.260138+0000","last_unstale":"2023-10-09T06:02:26.337696+0000","last_undegraded":"2023-10-09T06:02:26.337696+0000","last_fullsized":"2023-10-09T06:02:26.337696+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":513,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.574135+0000","last_change":"2023-10-09T06:01:19.995107+0000","last_active":"2023-10-09T06:02:27.574135+0000","last_peered":"2023-10-09T06:02:27.574135+0000","last_clean":"2023-10-09T06:02:27.574135+0000","last_became_active":"2023-10-09T06:01:19.994466+0000","last_became_peered":"2023-10-09T06:01:19.994466+0000","last_unstale":"2023-10-09T06:02:27.574135+0000","last_undegraded":"2023-10-09T06:02:27.574135+0000","last_fullsized":"2023-10-09T06:02:27.574135+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":12,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.506444+0000","last_change":"2023-10-09T06:02:25.342837+0000","last_active":"2023-10-09T06:02:27.506444+0000","last_peered":"2023-10-09T06:02:27.506444+0000","last_clean":"2023-10-09T06:02:27.506444+0000","last_became_active":"2023-10-09T06:02:25.342530+0000","last_became_peered":"2023-10-09T06:02:25.342530+0000","last_unstale":"2023-10-09T06:02:27.506444+0000","last_undegraded":"2023-10-09T06:02:27.506444+0000","last_fullsized":"2023-10-09T06:02:27.506444+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:30.609714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":149,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.339221+0000","last_change":"2023-10-09T06:01:19.985414+0000","last_active":"2023-10-09T06:02:26.339221+0000","last_peered":"2023-10-09T06:02:26.339221+0000","last_clean":"2023-10-09T06:02:26.339221+0000","last_became_active":"2023-10-09T06:01:19.984885+0000","last_became_peered":"2023-10-09T06:01:19.984885+0000","last_unstale":"2023-10-09T06:02:26.339221+0000","last_undegraded":"2023-10-09T06:02:26.339221+0000","last_fullsized":"2023-10-09T06:02:26.339221+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":159,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.338109+0000","last_change":"2023-10-09T06:01:19.993334+0000","last_active":"2023-10-09T06:02:26.338109+0000","last_peered":"2023-10-09T06:02:26.338109+0000","last_clean":"2023-10-09T06:02:26.338109+0000","last_became_active":"2023-10-09T06:01:19.992999+0000","last_became_peered":"2023-10-09T06:01:19.992999+0000","last_unstale":"2023-10-09T06:02:26.338109+0000","last_undegraded":"2023-10-09T06:02:26.338109+0000","last_fullsized":"2023-10-09T06:02:26.338109+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"515'310","reported_seq":322,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:35.365614+0000","last_change":"2023-10-09T06:02:25.346804+0000","last_active":"2023-10-09T06:02:35.365614+0000","last_peered":"2023-10-09T06:02:35.365614+0000","last_clean":"2023-10-09T06:02:35.365614+0000","last_became_active":"2023-10-09T06:02:25.346477+0000","last_became_peered":"2023-10-09T06:02:25.346477+0000","last_unstale":"2023-10-09T06:02:35.365614+0000","last_undegraded":"2023-10-09T06:02:35.365614+0000","last_fullsized":"2023-10-09T06:02:35.365614+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":310,"log_dups_size":0,"ondisk_log_size":310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:43.482594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20316160,"num_objects":310,"num_object_clones":0,"num_object_copies":930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":310,"num_write_kb":19840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":438,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.621158+0000","last_change":"2023-10-09T06:01:19.987220+0000","last_active":"2023-10-09T06:02:27.621158+0000","last_peered":"2023-10-09T06:02:27.621158+0000","last_clean":"2023-10-09T06:02:27.621158+0000","last_became_active":"2023-10-09T06:01:19.986675+0000","last_became_peered":"2023-10-09T06:01:19.986675+0000","last_unstale":"2023-10-09T06:02:27.621158+0000","last_undegraded":"2023-10-09T06:02:27.621158+0000","last_fullsized":"2023-10-09T06:02:27.621158+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:06.685801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":447,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.409672+0000","last_change":"2023-10-09T06:00:05.352723+0000","last_active":"2023-10-09T06:02:27.409672+0000","last_peered":"2023-10-09T06:02:27.409672+0000","last_clean":"2023-10-09T06:02:27.409672+0000","last_became_active":"2023-10-09T06:00:05.352008+0000","last_became_peered":"2023-10-09T06:00:05.352008+0000","last_unstale":"2023-10-09T06:02:27.409672+0000","last_undegraded":"2023-10-09T06:02:27.409672+0000","last_fullsized":"2023-10-09T06:02:27.409672+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:25.302918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"515'411","reported_seq":422,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:36.028893+0000","last_change":"2023-10-09T06:02:25.339641+0000","last_active":"2023-10-09T06:02:36.028893+0000","last_peered":"2023-10-09T06:02:36.028893+0000","last_clean":"2023-10-09T06:02:36.028893+0000","last_became_active":"2023-10-09T06:02:25.339307+0000","last_became_peered":"2023-10-09T06:02:25.339307+0000","last_unstale":"2023-10-09T06:02:36.028893+0000","last_undegraded":"2023-10-09T06:02:36.028893+0000","last_fullsized":"2023-10-09T06:02:36.028893+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":411,"log_dups_size":0,"ondisk_log_size":411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26935296,"num_objects":411,"num_object_clones":0,"num_object_copies":1233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":411,"num_write_kb":26304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":351,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.621040+0000","last_change":"2023-10-09T06:02:08.331401+0000","last_active":"2023-10-09T06:02:27.621040+0000","last_peered":"2023-10-09T06:02:27.621040+0000","last_clean":"2023-10-09T06:02:27.621040+0000","last_became_active":"2023-10-09T06:02:08.331003+0000","last_became_peered":"2023-10-09T06:02:08.331003+0000","last_unstale":"2023-10-09T06:02:27.621040+0000","last_undegraded":"2023-10-09T06:02:27.621040+0000","last_fullsized":"2023-10-09T06:02:27.621040+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:42.509695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":513,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.574259+0000","last_change":"2023-10-09T06:01:19.997595+0000","last_active":"2023-10-09T06:02:27.574259+0000","last_peered":"2023-10-09T06:02:27.574259+0000","last_clean":"2023-10-09T06:02:27.574259+0000","last_became_active":"2023-10-09T06:01:19.995775+0000","last_became_peered":"2023-10-09T06:01:19.995775+0000","last_unstale":"2023-10-09T06:02:27.574259+0000","last_undegraded":"2023-10-09T06:02:27.574259+0000","last_fullsized":"2023-10-09T06:02:27.574259+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"515'335","reported_seq":347,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:35.398956+0000","last_change":"2023-10-09T06:02:25.344811+0000","last_active":"2023-10-09T06:02:35.398956+0000","last_peered":"2023-10-09T06:02:35.398956+0000","last_clean":"2023-10-09T06:02:35.398956+0000","last_became_active":"2023-10-09T06:02:25.344522+0000","last_became_peered":"2023-10-09T06:02:25.344522+0000","last_unstale":"2023-10-09T06:02:35.398956+0000","last_undegraded":"2023-10-09T06:02:35.398956+0000","last_fullsized":"2023-10-09T06:02:35.398956+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":335,"log_dups_size":0,"ondisk_log_size":335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.655360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21954560,"num_objects":335,"num_object_clones":0,"num_object_copies":1005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":335,"num_write_kb":21440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":31,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.622517+0000","last_change":"2023-10-09T06:01:19.981078+0000","last_active":"2023-10-09T06:02:27.622517+0000","last_peered":"2023-10-09T06:02:27.622517+0000","last_clean":"2023-10-09T06:02:27.622517+0000","last_became_active":"2023-10-09T06:01:19.980723+0000","last_became_peered":"2023-10-09T06:01:19.980723+0000","last_unstale":"2023-10-09T06:02:27.622517+0000","last_undegraded":"2023-10-09T06:02:27.622517+0000","last_fullsized":"2023-10-09T06:02:27.622517+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":446,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.335188+0000","last_change":"2023-10-09T05:54:05.161959+0000","last_active":"2023-10-09T06:02:26.335188+0000","last_peered":"2023-10-09T06:02:26.335188+0000","last_clean":"2023-10-09T06:02:26.335188+0000","last_became_active":"2023-10-09T05:54:05.161671+0000","last_became_peered":"2023-10-09T05:54:05.161671+0000","last_unstale":"2023-10-09T06:02:26.335188+0000","last_undegraded":"2023-10-09T06:02:26.335188+0000","last_fullsized":"2023-10-09T06:02:26.335188+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:47.282176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"515'201","reported_seq":212,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:34.773499+0000","last_change":"2023-10-09T06:02:25.337899+0000","last_active":"2023-10-09T06:02:34.773499+0000","last_peered":"2023-10-09T06:02:34.773499+0000","last_clean":"2023-10-09T06:02:34.773499+0000","last_became_active":"2023-10-09T06:02:25.337595+0000","last_became_peered":"2023-10-09T06:02:25.337595+0000","last_unstale":"2023-10-09T06:02:34.773499+0000","last_undegraded":"2023-10-09T06:02:34.773499+0000","last_fullsized":"2023-10-09T06:02:34.773499+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":201,"log_dups_size":0,"ondisk_log_size":201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13172736,"num_objects":201,"num_object_clones":0,"num_object_copies":603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":201,"num_write_kb":12864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":519,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.334923+0000","last_change":"2023-10-09T05:55:26.256379+0000","last_active":"2023-10-09T06:02:26.334923+0000","last_peered":"2023-10-09T06:02:26.334923+0000","last_clean":"2023-10-09T06:02:26.334923+0000","last_became_active":"2023-10-09T05:55:26.256062+0000","last_became_peered":"2023-10-09T05:55:26.256062+0000","last_unstale":"2023-10-09T06:02:26.334923+0000","last_undegraded":"2023-10-09T06:02:26.334923+0000","last_fullsized":"2023-10-09T06:02:26.334923+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:00.015437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":479,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.574499+0000","last_change":"2023-10-09T06:01:20.003486+0000","last_active":"2023-10-09T06:02:27.574499+0000","last_peered":"2023-10-09T06:02:27.574499+0000","last_clean":"2023-10-09T06:02:27.574499+0000","last_became_active":"2023-10-09T06:01:20.003139+0000","last_became_peered":"2023-10-09T06:01:20.003139+0000","last_unstale":"2023-10-09T06:02:27.574499+0000","last_undegraded":"2023-10-09T06:02:27.574499+0000","last_fullsized":"2023-10-09T06:02:27.574499+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"515'373","reported_seq":384,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:35.859171+0000","last_change":"2023-10-09T06:02:25.342740+0000","last_active":"2023-10-09T06:02:35.859171+0000","last_peered":"2023-10-09T06:02:35.859171+0000","last_clean":"2023-10-09T06:02:35.859171+0000","last_became_active":"2023-10-09T06:02:25.342409+0000","last_became_peered":"2023-10-09T06:02:25.342409+0000","last_unstale":"2023-10-09T06:02:35.859171+0000","last_undegraded":"2023-10-09T06:02:35.859171+0000","last_fullsized":"2023-10-09T06:02:35.859171+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":373,"log_dups_size":0,"ondisk_log_size":373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24444928,"num_objects":373,"num_object_clones":0,"num_object_copies":1119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":373,"num_write_kb":23872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":149,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.336773+0000","last_change":"2023-10-09T05:56:49.287906+0000","last_active":"2023-10-09T06:02:26.336773+0000","last_peered":"2023-10-09T06:02:26.336773+0000","last_clean":"2023-10-09T06:02:26.336773+0000","last_became_active":"2023-10-09T05:56:49.275081+0000","last_became_peered":"2023-10-09T05:56:49.275081+0000","last_unstale":"2023-10-09T06:02:26.336773+0000","last_undegraded":"2023-10-09T06:02:26.336773+0000","last_fullsized":"2023-10-09T06:02:26.336773+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":204,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.312776+0000","last_change":"2023-10-09T05:56:00.225724+0000","last_active":"2023-10-09T06:02:27.312776+0000","last_peered":"2023-10-09T06:02:27.312776+0000","last_clean":"2023-10-09T06:02:27.312776+0000","last_became_active":"2023-10-09T05:56:00.225335+0000","last_became_peered":"2023-10-09T05:56:00.225335+0000","last_unstale":"2023-10-09T06:02:27.312776+0000","last_undegraded":"2023-10-09T06:02:27.312776+0000","last_fullsized":"2023-10-09T06:02:27.312776+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:41.405222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"515'390","reported_seq":400,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:35.877696+0000","last_change":"2023-10-09T06:02:25.343092+0000","last_active":"2023-10-09T06:02:35.877696+0000","last_peered":"2023-10-09T06:02:35.877696+0000","last_clean":"2023-10-09T06:02:35.877696+0000","last_became_active":"2023-10-09T06:02:25.342832+0000","last_became_peered":"2023-10-09T06:02:25.342832+0000","last_unstale":"2023-10-09T06:02:35.877696+0000","last_undegraded":"2023-10-09T06:02:35.877696+0000","last_fullsized":"2023-10-09T06:02:35.877696+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":390,"log_dups_size":0,"ondisk_log_size":390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25559040,"num_objects":390,"num_object_clones":0,"num_object_copies":1170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":390,"num_write_kb":24960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":922,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.337031+0000","last_change":"2023-10-09T06:02:08.341435+0000","last_active":"2023-10-09T06:02:26.337031+0000","last_peered":"2023-10-09T06:02:26.337031+0000","last_clean":"2023-10-09T06:02:26.337031+0000","last_became_active":"2023-10-09T06:02:08.340863+0000","last_became_peered":"2023-10-09T06:02:08.340863+0000","last_unstale":"2023-10-09T06:02:26.337031+0000","last_undegraded":"2023-10-09T06:02:26.337031+0000","last_fullsized":"2023-10-09T06:02:26.337031+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":509,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.335280+0000","last_change":"2023-10-09T06:01:20.018047+0000","last_active":"2023-10-09T06:02:26.335280+0000","last_peered":"2023-10-09T06:02:26.335280+0000","last_clean":"2023-10-09T06:02:26.335280+0000","last_became_active":"2023-10-09T06:01:20.017698+0000","last_became_peered":"2023-10-09T06:01:20.017698+0000","last_unstale":"2023-10-09T06:02:26.335280+0000","last_undegraded":"2023-10-09T06:02:26.335280+0000","last_fullsized":"2023-10-09T06:02:26.335280+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:44.970159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":51,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.409870+0000","last_change":"2023-10-09T06:00:05.355057+0000","last_active":"2023-10-09T06:02:27.409870+0000","last_peered":"2023-10-09T06:02:27.409870+0000","last_clean":"2023-10-09T06:02:27.409870+0000","last_became_active":"2023-10-09T06:00:05.352475+0000","last_became_peered":"2023-10-09T06:00:05.352475+0000","last_unstale":"2023-10-09T06:02:27.409870+0000","last_undegraded":"2023-10-09T06:02:27.409870+0000","last_fullsized":"2023-10-09T06:02:27.409870+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:50.255773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":12,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.506422+0000","last_change":"2023-10-09T06:02:25.339635+0000","last_active":"2023-10-09T06:02:27.506422+0000","last_peered":"2023-10-09T06:02:27.506422+0000","last_clean":"2023-10-09T06:02:27.506422+0000","last_became_active":"2023-10-09T06:02:25.339061+0000","last_became_peered":"2023-10-09T06:02:25.339061+0000","last_unstale":"2023-10-09T06:02:27.506422+0000","last_undegraded":"2023-10-09T06:02:27.506422+0000","last_fullsized":"2023-10-09T06:02:27.506422+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:56.931910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":832,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.312379+0000","last_change":"2023-10-09T06:02:20.246037+0000","last_active":"2023-10-09T06:02:27.312379+0000","last_peered":"2023-10-09T06:02:27.312379+0000","last_clean":"2023-10-09T06:02:27.312379+0000","last_became_active":"2023-10-09T06:01:19.995418+0000","last_became_peered":"2023-10-09T06:01:19.995418+0000","last_unstale":"2023-10-09T06:02:27.312379+0000","last_undegraded":"2023-10-09T06:02:27.312379+0000","last_fullsized":"2023-10-09T06:02:27.312379+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:29:36.218008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":444,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.574286+0000","last_change":"2023-10-09T06:02:08.336925+0000","last_active":"2023-10-09T06:02:27.574286+0000","last_peered":"2023-10-09T06:02:27.574286+0000","last_clean":"2023-10-09T06:02:27.574286+0000","last_became_active":"2023-10-09T06:02:08.336466+0000","last_became_peered":"2023-10-09T06:02:08.336466+0000","last_unstale":"2023-10-09T06:02:27.574286+0000","last_undegraded":"2023-10-09T06:02:27.574286+0000","last_fullsized":"2023-10-09T06:02:27.574286+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":370,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.335131+0000","last_change":"2023-10-09T06:01:19.992782+0000","last_active":"2023-10-09T06:02:26.335131+0000","last_peered":"2023-10-09T06:02:26.335131+0000","last_clean":"2023-10-09T06:02:26.335131+0000","last_became_active":"2023-10-09T06:01:19.992426+0000","last_became_peered":"2023-10-09T06:01:19.992426+0000","last_unstale":"2023-10-09T06:02:26.335131+0000","last_undegraded":"2023-10-09T06:02:26.335131+0000","last_fullsized":"2023-10-09T06:02:26.335131+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:05.460642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"515'364","reported_seq":376,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:35.403842+0000","last_change":"2023-10-09T06:02:25.344681+0000","last_active":"2023-10-09T06:02:35.403842+0000","last_peered":"2023-10-09T06:02:35.403842+0000","last_clean":"2023-10-09T06:02:35.403842+0000","last_became_active":"2023-10-09T06:02:25.344470+0000","last_became_peered":"2023-10-09T06:02:25.344470+0000","last_unstale":"2023-10-09T06:02:35.403842+0000","last_undegraded":"2023-10-09T06:02:35.403842+0000","last_fullsized":"2023-10-09T06:02:35.403842+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":364,"log_dups_size":0,"ondisk_log_size":364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:24.576548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23855104,"num_objects":364,"num_object_clones":0,"num_object_copies":1092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":364,"num_write_kb":23296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":873,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.335123+0000","last_change":"2023-10-09T06:00:05.324594+0000","last_active":"2023-10-09T06:02:26.335123+0000","last_peered":"2023-10-09T06:02:26.335123+0000","last_clean":"2023-10-09T06:02:26.335123+0000","last_became_active":"2023-10-09T06:00:05.323845+0000","last_became_peered":"2023-10-09T06:00:05.323845+0000","last_unstale":"2023-10-09T06:02:26.335123+0000","last_undegraded":"2023-10-09T06:02:26.335123+0000","last_fullsized":"2023-10-09T06:02:26.335123+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:02.539703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":328,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:30.573848+0000","last_change":"2023-10-09T06:01:19.994662+0000","last_active":"2023-10-09T06:02:30.573848+0000","last_peered":"2023-10-09T06:02:30.573848+0000","last_clean":"2023-10-09T06:02:30.573848+0000","last_became_active":"2023-10-09T06:01:19.994264+0000","last_became_peered":"2023-10-09T06:01:19.994264+0000","last_unstale":"2023-10-09T06:02:30.573848+0000","last_undegraded":"2023-10-09T06:02:30.573848+0000","last_fullsized":"2023-10-09T06:02:30.573848+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:43.503821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":303,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.338227+0000","last_change":"2023-10-09T06:02:08.345141+0000","last_active":"2023-10-09T06:02:26.338227+0000","last_peered":"2023-10-09T06:02:26.338227+0000","last_clean":"2023-10-09T06:02:26.338227+0000","last_became_active":"2023-10-09T06:02:08.344568+0000","last_became_peered":"2023-10-09T06:02:08.344568+0000","last_unstale":"2023-10-09T06:02:26.338227+0000","last_undegraded":"2023-10-09T06:02:26.338227+0000","last_fullsized":"2023-10-09T06:02:26.338227+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.b","version":"515'67","reported_seq":79,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:33.582078+0000","last_change":"2023-10-09T06:02:25.344895+0000","last_active":"2023-10-09T06:02:33.582078+0000","last_peered":"2023-10-09T06:02:33.582078+0000","last_clean":"2023-10-09T06:02:33.582078+0000","last_became_active":"2023-10-09T06:02:25.344582+0000","last_became_peered":"2023-10-09T06:02:25.344582+0000","last_unstale":"2023-10-09T06:02:33.582078+0000","last_undegraded":"2023-10-09T06:02:33.582078+0000","last_fullsized":"2023-10-09T06:02:33.582078+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:25.602904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4390912,"num_objects":67,"num_object_clones":0,"num_object_copies":201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":67,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":67,"num_write_kb":4288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":757,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.336427+0000","last_change":"2023-10-09T06:02:19.008664+0000","last_active":"2023-10-09T06:02:26.336427+0000","last_peered":"2023-10-09T06:02:26.336427+0000","last_clean":"2023-10-09T06:02:26.336427+0000","last_became_active":"2023-10-09T06:02:19.008087+0000","last_became_peered":"2023-10-09T06:02:19.008087+0000","last_unstale":"2023-10-09T06:02:26.336427+0000","last_undegraded":"2023-10-09T06:02:26.336427+0000","last_fullsized":"2023-10-09T06:02:26.336427+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:09.298922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":887,"num_read_kb":802,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":679,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.336886+0000","last_change":"2023-10-09T06:01:19.975320+0000","last_active":"2023-10-09T06:02:26.336886+0000","last_peered":"2023-10-09T06:02:26.336886+0000","last_clean":"2023-10-09T06:02:26.336886+0000","last_became_active":"2023-10-09T06:01:19.974866+0000","last_became_peered":"2023-10-09T06:01:19.974866+0000","last_unstale":"2023-10-09T06:02:26.336886+0000","last_undegraded":"2023-10-09T06:02:26.336886+0000","last_fullsized":"2023-10-09T06:02:26.336886+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":395,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.621024+0000","last_change":"2023-10-09T06:01:19.987073+0000","last_active":"2023-10-09T06:02:27.621024+0000","last_peered":"2023-10-09T06:02:27.621024+0000","last_clean":"2023-10-09T06:02:27.621024+0000","last_became_active":"2023-10-09T06:01:19.986758+0000","last_became_peered":"2023-10-09T06:01:19.986758+0000","last_unstale":"2023-10-09T06:02:27.621024+0000","last_undegraded":"2023-10-09T06:02:27.621024+0000","last_fullsized":"2023-10-09T06:02:27.621024+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:53.291650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":145,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.338886+0000","last_change":"2023-10-09T06:01:20.982313+0000","last_active":"2023-10-09T06:02:26.338886+0000","last_peered":"2023-10-09T06:02:26.338886+0000","last_clean":"2023-10-09T06:02:26.338886+0000","last_became_active":"2023-10-09T06:01:20.981964+0000","last_became_peered":"2023-10-09T06:01:20.981964+0000","last_unstale":"2023-10-09T06:02:26.338886+0000","last_undegraded":"2023-10-09T06:02:26.338886+0000","last_fullsized":"2023-10-09T06:02:26.338886+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"515'340","reported_seq":352,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:35.373208+0000","last_change":"2023-10-09T06:02:25.345595+0000","last_active":"2023-10-09T06:02:35.373208+0000","last_peered":"2023-10-09T06:02:35.373208+0000","last_clean":"2023-10-09T06:02:35.373208+0000","last_became_active":"2023-10-09T06:02:25.345371+0000","last_became_peered":"2023-10-09T06:02:25.345371+0000","last_unstale":"2023-10-09T06:02:35.373208+0000","last_undegraded":"2023-10-09T06:02:35.373208+0000","last_fullsized":"2023-10-09T06:02:35.373208+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":340,"log_dups_size":0,"ondisk_log_size":340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:16.252828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22282240,"num_objects":340,"num_object_clones":0,"num_object_copies":1020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":340,"num_write_kb":21760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":1066,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.336983+0000","last_change":"2023-10-09T06:02:20.238945+0000","last_active":"2023-10-09T06:02:26.336983+0000","last_peered":"2023-10-09T06:02:26.336983+0000","last_clean":"2023-10-09T06:02:26.336983+0000","last_became_active":"2023-10-09T06:00:06.308176+0000","last_became_peered":"2023-10-09T06:00:06.308176+0000","last_unstale":"2023-10-09T06:02:26.336983+0000","last_undegraded":"2023-10-09T06:02:26.336983+0000","last_fullsized":"2023-10-09T06:02:26.336983+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":484,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.409537+0000","last_change":"2023-10-09T06:00:05.314057+0000","last_active":"2023-10-09T06:02:27.409537+0000","last_peered":"2023-10-09T06:02:27.409537+0000","last_clean":"2023-10-09T06:02:27.409537+0000","last_became_active":"2023-10-09T06:00:05.313743+0000","last_became_peered":"2023-10-09T06:00:05.313743+0000","last_unstale":"2023-10-09T06:02:27.409537+0000","last_undegraded":"2023-10-09T06:02:27.409537+0000","last_fullsized":"2023-10-09T06:02:27.409537+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:29.568391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":462,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.410145+0000","last_change":"2023-10-09T06:00:05.590816+0000","last_active":"2023-10-09T06:02:27.410145+0000","last_peered":"2023-10-09T06:02:27.410145+0000","last_clean":"2023-10-09T06:02:27.410145+0000","last_became_active":"2023-10-09T06:00:05.590227+0000","last_became_peered":"2023-10-09T06:00:05.590227+0000","last_unstale":"2023-10-09T06:02:27.410145+0000","last_undegraded":"2023-10-09T06:02:27.410145+0000","last_fullsized":"2023-10-09T06:02:27.410145+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:51.060693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"0'0","reported_seq":12,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.506454+0000","last_change":"2023-10-09T06:02:25.341999+0000","last_active":"2023-10-09T06:02:27.506454+0000","last_peered":"2023-10-09T06:02:27.506454+0000","last_clean":"2023-10-09T06:02:27.506454+0000","last_became_active":"2023-10-09T06:02:25.341823+0000","last_became_peered":"2023-10-09T06:02:25.341823+0000","last_unstale":"2023-10-09T06:02:27.506454+0000","last_undegraded":"2023-10-09T06:02:27.506454+0000","last_fullsized":"2023-10-09T06:02:27.506454+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:21.458234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":690,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.339416+0000","last_change":"2023-10-09T06:00:17.336621+0000","last_active":"2023-10-09T06:02:26.339416+0000","last_peered":"2023-10-09T06:02:26.339416+0000","last_clean":"2023-10-09T06:02:26.339416+0000","last_became_active":"2023-10-09T06:00:14.622413+0000","last_became_peered":"2023-10-09T06:00:14.622413+0000","last_unstale":"2023-10-09T06:02:26.339416+0000","last_undegraded":"2023-10-09T06:02:26.339416+0000","last_fullsized":"2023-10-09T06:02:26.339416+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":345,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.335315+0000","last_change":"2023-10-09T06:00:05.324944+0000","last_active":"2023-10-09T06:02:26.335315+0000","last_peered":"2023-10-09T06:02:26.335315+0000","last_clean":"2023-10-09T06:02:26.335315+0000","last_became_active":"2023-10-09T06:00:05.324533+0000","last_became_peered":"2023-10-09T06:00:05.324533+0000","last_unstale":"2023-10-09T06:02:26.335315+0000","last_undegraded":"2023-10-09T06:02:26.335315+0000","last_fullsized":"2023-10-09T06:02:26.335315+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:33:33.376129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":453,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.337293+0000","last_change":"2023-10-09T06:02:08.340506+0000","last_active":"2023-10-09T06:02:26.337293+0000","last_peered":"2023-10-09T06:02:26.337293+0000","last_clean":"2023-10-09T06:02:26.337293+0000","last_became_active":"2023-10-09T06:02:08.340152+0000","last_became_peered":"2023-10-09T06:02:08.340152+0000","last_unstale":"2023-10-09T06:02:26.337293+0000","last_undegraded":"2023-10-09T06:02:26.337293+0000","last_fullsized":"2023-10-09T06:02:26.337293+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"515'326","reported_seq":338,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:35.398268+0000","last_change":"2023-10-09T06:02:25.345875+0000","last_active":"2023-10-09T06:02:35.398268+0000","last_peered":"2023-10-09T06:02:35.398268+0000","last_clean":"2023-10-09T06:02:35.398268+0000","last_became_active":"2023-10-09T06:02:25.345552+0000","last_became_peered":"2023-10-09T06:02:25.345552+0000","last_unstale":"2023-10-09T06:02:35.398268+0000","last_undegraded":"2023-10-09T06:02:35.398268+0000","last_fullsized":"2023-10-09T06:02:35.398268+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":326,"log_dups_size":0,"ondisk_log_size":326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:32.546131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21364736,"num_objects":326,"num_object_clones":0,"num_object_copies":978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":326,"num_write_kb":20864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":181,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.338671+0000","last_change":"2023-10-09T06:00:05.329139+0000","last_active":"2023-10-09T06:02:26.338671+0000","last_peered":"2023-10-09T06:02:26.338671+0000","last_clean":"2023-10-09T06:02:26.338671+0000","last_became_active":"2023-10-09T06:00:05.328811+0000","last_became_peered":"2023-10-09T06:00:05.328811+0000","last_unstale":"2023-10-09T06:02:26.338671+0000","last_undegraded":"2023-10-09T06:02:26.338671+0000","last_fullsized":"2023-10-09T06:02:26.338671+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":521,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.622814+0000","last_change":"2023-10-09T06:02:19.808362+0000","last_active":"2023-10-09T06:02:27.622814+0000","last_peered":"2023-10-09T06:02:27.622814+0000","last_clean":"2023-10-09T06:02:27.622814+0000","last_became_active":"2023-10-09T06:01:20.980166+0000","last_became_peered":"2023-10-09T06:01:20.980166+0000","last_unstale":"2023-10-09T06:02:27.622814+0000","last_undegraded":"2023-10-09T06:02:27.622814+0000","last_fullsized":"2023-10-09T06:02:27.622814+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":185,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.336787+0000","last_change":"2023-10-09T06:00:05.328236+0000","last_active":"2023-10-09T06:02:26.336787+0000","last_peered":"2023-10-09T06:02:26.336787+0000","last_clean":"2023-10-09T06:02:26.336787+0000","last_became_active":"2023-10-09T06:00:05.327876+0000","last_became_peered":"2023-10-09T06:00:05.327876+0000","last_unstale":"2023-10-09T06:02:26.336787+0000","last_undegraded":"2023-10-09T06:02:26.336787+0000","last_fullsized":"2023-10-09T06:02:26.336787+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"515'331","reported_seq":343,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:35.381021+0000","last_change":"2023-10-09T06:02:25.346947+0000","last_active":"2023-10-09T06:02:35.381021+0000","last_peered":"2023-10-09T06:02:35.381021+0000","last_clean":"2023-10-09T06:02:35.381021+0000","last_became_active":"2023-10-09T06:02:25.346599+0000","last_became_peered":"2023-10-09T06:02:25.346599+0000","last_unstale":"2023-10-09T06:02:35.381021+0000","last_undegraded":"2023-10-09T06:02:35.381021+0000","last_fullsized":"2023-10-09T06:02:35.381021+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":331,"log_dups_size":0,"ondisk_log_size":331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:53.556273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21692416,"num_objects":331,"num_object_clones":0,"num_object_copies":993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":331,"num_write_kb":21184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":703,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.338551+0000","last_change":"2023-10-09T06:01:20.985388+0000","last_active":"2023-10-09T06:02:26.338551+0000","last_peered":"2023-10-09T06:02:26.338551+0000","last_clean":"2023-10-09T06:02:26.338551+0000","last_became_active":"2023-10-09T06:01:20.984892+0000","last_became_peered":"2023-10-09T06:01:20.984892+0000","last_unstale":"2023-10-09T06:02:26.338551+0000","last_undegraded":"2023-10-09T06:02:26.338551+0000","last_fullsized":"2023-10-09T06:02:26.338551+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":457,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.410300+0000","last_change":"2023-10-09T06:00:05.352581+0000","last_active":"2023-10-09T06:02:27.410300+0000","last_peered":"2023-10-09T06:02:27.410300+0000","last_clean":"2023-10-09T06:02:27.410300+0000","last_became_active":"2023-10-09T06:00:05.351836+0000","last_became_peered":"2023-10-09T06:00:05.351836+0000","last_unstale":"2023-10-09T06:02:27.410300+0000","last_undegraded":"2023-10-09T06:02:27.410300+0000","last_fullsized":"2023-10-09T06:02:27.410300+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:31.576840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.409340+0000","last_change":"2023-10-09T06:01:19.981055+0000","last_active":"2023-10-09T06:02:27.409340+0000","last_peered":"2023-10-09T06:02:27.409340+0000","last_clean":"2023-10-09T06:02:27.409340+0000","last_became_active":"2023-10-09T06:01:19.980717+0000","last_became_peered":"2023-10-09T06:01:19.980717+0000","last_unstale":"2023-10-09T06:02:27.409340+0000","last_undegraded":"2023-10-09T06:02:27.409340+0000","last_fullsized":"2023-10-09T06:02:27.409340+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:55.973927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":466,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.409769+0000","last_change":"2023-10-09T06:00:05.416922+0000","last_active":"2023-10-09T06:02:27.409769+0000","last_peered":"2023-10-09T06:02:27.409769+0000","last_clean":"2023-10-09T06:02:27.409769+0000","last_became_active":"2023-10-09T06:00:05.416444+0000","last_became_peered":"2023-10-09T06:00:05.416444+0000","last_unstale":"2023-10-09T06:02:27.409769+0000","last_undegraded":"2023-10-09T06:02:27.409769+0000","last_fullsized":"2023-10-09T06:02:27.409769+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:26.775246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":208,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.335225+0000","last_change":"2023-10-09T05:56:00.226695+0000","last_active":"2023-10-09T06:02:26.335225+0000","last_peered":"2023-10-09T06:02:26.335225+0000","last_clean":"2023-10-09T06:02:26.335225+0000","last_became_active":"2023-10-09T05:56:00.226359+0000","last_became_peered":"2023-10-09T05:56:00.226359+0000","last_unstale":"2023-10-09T06:02:26.335225+0000","last_undegraded":"2023-10-09T06:02:26.335225+0000","last_fullsized":"2023-10-09T06:02:26.335225+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:20.323704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":60,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.574352+0000","last_change":"2023-10-09T06:01:19.995155+0000","last_active":"2023-10-09T06:02:27.574352+0000","last_peered":"2023-10-09T06:02:27.574352+0000","last_clean":"2023-10-09T06:02:27.574352+0000","last_became_active":"2023-10-09T06:01:19.994697+0000","last_became_peered":"2023-10-09T06:01:19.994697+0000","last_unstale":"2023-10-09T06:02:27.574352+0000","last_undegraded":"2023-10-09T06:02:27.574352+0000","last_fullsized":"2023-10-09T06:02:27.574352+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":98,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.336341+0000","last_change":"2023-10-09T05:56:49.252225+0000","last_active":"2023-10-09T06:02:26.336341+0000","last_peered":"2023-10-09T06:02:26.336341+0000","last_clean":"2023-10-09T06:02:26.336341+0000","last_became_active":"2023-10-09T05:56:49.251880+0000","last_became_peered":"2023-10-09T05:56:49.251880+0000","last_unstale":"2023-10-09T06:02:26.336341+0000","last_undegraded":"2023-10-09T06:02:26.336341+0000","last_fullsized":"2023-10-09T06:02:26.336341+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:34.079170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":60,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.574478+0000","last_change":"2023-10-09T06:01:19.994887+0000","last_active":"2023-10-09T06:02:27.574478+0000","last_peered":"2023-10-09T06:02:27.574478+0000","last_clean":"2023-10-09T06:02:27.574478+0000","last_became_active":"2023-10-09T06:01:19.994497+0000","last_became_peered":"2023-10-09T06:01:19.994497+0000","last_unstale":"2023-10-09T06:02:27.574478+0000","last_undegraded":"2023-10-09T06:02:27.574478+0000","last_fullsized":"2023-10-09T06:02:27.574478+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":233,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.574083+0000","last_change":"2023-10-09T06:02:08.333016+0000","last_active":"2023-10-09T06:02:27.574083+0000","last_peered":"2023-10-09T06:02:27.574083+0000","last_clean":"2023-10-09T06:02:27.574083+0000","last_became_active":"2023-10-09T06:02:08.332684+0000","last_became_peered":"2023-10-09T06:02:08.332684+0000","last_unstale":"2023-10-09T06:02:27.574083+0000","last_undegraded":"2023-10-09T06:02:27.574083+0000","last_fullsized":"2023-10-09T06:02:27.574083+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":275,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T06:02:30.587027+0000","last_change":"2023-10-09T06:01:20.198041+0000","last_active":"2023-10-09T06:02:30.587027+0000","last_peered":"2023-10-09T06:02:30.587027+0000","last_clean":"2023-10-09T06:02:30.587027+0000","last_became_active":"2023-10-09T06:01:20.178975+0000","last_became_peered":"2023-10-09T06:01:20.178975+0000","last_unstale":"2023-10-09T06:02:30.587027+0000","last_undegraded":"2023-10-09T06:02:30.587027+0000","last_fullsized":"2023-10-09T06:02:30.587027+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:40.386988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":374,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.335215+0000","last_change":"2023-10-09T06:00:05.325284+0000","last_active":"2023-10-09T06:02:26.335215+0000","last_peered":"2023-10-09T06:02:26.335215+0000","last_clean":"2023-10-09T06:02:26.335215+0000","last_became_active":"2023-10-09T06:00:05.324588+0000","last_became_peered":"2023-10-09T06:00:05.324588+0000","last_unstale":"2023-10-09T06:02:26.335215+0000","last_undegraded":"2023-10-09T06:02:26.335215+0000","last_fullsized":"2023-10-09T06:02:26.335215+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:59.068162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":471,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.409888+0000","last_change":"2023-10-09T06:01:19.996238+0000","last_active":"2023-10-09T06:02:27.409888+0000","last_peered":"2023-10-09T06:02:27.409888+0000","last_clean":"2023-10-09T06:02:27.409888+0000","last_became_active":"2023-10-09T06:01:19.995257+0000","last_became_peered":"2023-10-09T06:01:19.995257+0000","last_unstale":"2023-10-09T06:02:27.409888+0000","last_undegraded":"2023-10-09T06:02:27.409888+0000","last_fullsized":"2023-10-09T06:02:27.409888+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:46:23.092370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":492,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T06:02:27.573874+0000","last_change":"2023-10-09T06:01:19.994582+0000","last_active":"2023-10-09T06:02:27.573874+0000","last_peered":"2023-10-09T06:02:27.573874+0000","last_clean":"2023-10-09T06:02:27.573874+0000","last_became_active":"2023-10-09T06:01:19.994185+0000","last_became_peered":"2023-10-09T06:01:19.994185+0000","last_unstale":"2023-10-09T06:02:27.573874+0000","last_undegraded":"2023-10-09T06:02:27.573874+0000","last_fullsized":"2023-10-09T06:02:27.573874+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":249364480,"num_objects":3805,"num_object_clones":0,"num_object_copies":11415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3805,"num_write_kb":243520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":787873792,"data_stored":787873792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3805,"ondisk_log_size":3805,"up":48,"acting":48,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":15826,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38956,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13211,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":134,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846922,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1074532,"kb_used_data":160604,"kb_used_omap":14,"kb_used_meta":913905,"kb_avail":92670620,"statfs":{"total":95995035648,"available":94894714880,"internally_reserved":0,"allocated":164458496,"data_stored":163309724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":935839578},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":0.76700000000000002,"15min":3.4060000000000001},"min":{"1min":0.247,"5min":0.23200000000000001,"15min":0.224},"max":{"1min":9.9480000000000004,"5min":9.9480000000000004,"15min":212.64400000000001},"last":176.89500000000001},{"interface":"front","average":{"1min":1.732,"5min":1.103,"15min":3.528},"min":{"1min":0.29599999999999999,"5min":0.26800000000000002,"15min":0.25900000000000001},"max":{"1min":7.5449999999999999,"5min":7.5449999999999999,"15min":212.655},"last":176.947}]},{"osd":1,"last update":"Mon Oct 9 06:01:36 2023","interfaces":[{"interface":"back","average":{"1min":5.3769999999999998,"5min":2.3719999999999999,"15min":2.4809999999999999},"min":{"1min":0.28100000000000003,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":81.271000000000001,"5min":81.271000000000001,"15min":114.422},"last":176.90600000000001},{"interface":"front","average":{"1min":5.1660000000000004,"5min":2.2109999999999999,"15min":2.5099999999999998},"min":{"1min":0.29199999999999998,"5min":0.28199999999999997,"15min":0.25700000000000001},"max":{"1min":82.549999999999997,"5min":82.549999999999997,"15min":111.708},"last":176.86799999999999}]},{"osd":2,"last update":"Mon Oct 9 06:01:36 2023","interfaces":[{"interface":"back","average":{"1min":5.8040000000000003,"5min":2.8860000000000001,"15min":3.0880000000000001},"min":{"1min":0.34999999999999998,"5min":0.29099999999999998,"15min":0.28499999999999998},"max":{"1min":83.263999999999996,"5min":89.778000000000006,"15min":205.863},"last":176.94},{"interface":"front","average":{"1min":4.7430000000000003,"5min":2.6280000000000001,"15min":2.8719999999999999},"min":{"1min":0.33900000000000002,"5min":0.308,"15min":0.30499999999999999},"max":{"1min":74.543999999999997,"5min":89.912999999999997,"15min":205.88399999999999},"last":176.81200000000001}]},{"osd":3,"last update":"Mon Oct 9 06:01:36 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":0.92400000000000004,"15min":2.73},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":6.9619999999999997,"5min":6.9619999999999997,"15min":115.69799999999999},"last":176.958},{"interface":"front","average":{"1min":1.764,"5min":1.1930000000000001,"15min":3.028},"min":{"1min":0.33100000000000002,"5min":0.29699999999999999,"15min":0.29299999999999998},"max":{"1min":7.6210000000000004,"5min":7.6210000000000004,"15min":115.75700000000001},"last":176.91800000000001}]},{"osd":4,"last update":"Mon Oct 9 06:01:36 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":0.93899999999999995,"15min":2.0230000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26000000000000001},"max":{"1min":11.144,"5min":11.144,"15min":159.50299999999999},"last":178.995},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.2629999999999999,"15min":2.1349999999999998},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":8.6140000000000008,"5min":8.6140000000000008,"15min":159.483},"last":178.899}]},{"osd":5,"last update":"Mon Oct 9 06:01:36 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":1.3939999999999999,"15min":1.887},"min":{"1min":0.377,"5min":0.29699999999999999,"15min":0.27800000000000002},"max":{"1min":7.5839999999999996,"5min":8.968,"15min":94.093999999999994},"last":176.82400000000001},{"interface":"front","average":{"1min":1.764,"5min":1.1719999999999999,"15min":1.946},"min":{"1min":0.374,"5min":0.35999999999999999,"15min":0.308},"max":{"1min":8.4849999999999994,"5min":8.4849999999999994,"15min":95.579999999999998},"last":176.79499999999999}]},{"osd":6,"last update":"Mon Oct 9 06:01:36 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":1.456,"15min":1.8080000000000001},"min":{"1min":0.35099999999999998,"5min":0.34200000000000003,"15min":0.29299999999999998},"max":{"1min":7.6029999999999998,"5min":7.6029999999999998,"15min":209.322},"last":177.01499999999999},{"interface":"front","average":{"1min":2.2120000000000002,"5min":1.53,"15min":1.8799999999999999},"min":{"1min":0.41299999999999998,"5min":0.308,"15min":0.26700000000000002},"max":{"1min":8.0600000000000005,"5min":8.1289999999999996,"15min":209.19800000000001},"last":177.00200000000001}]}]},{"osd":6,"up_from":42,"seq":180388626954,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1111452,"kb_used_data":207316,"kb_used_omap":0,"kb_used_meta":904128,"kb_avail":92633700,"statfs":{"total":95995035648,"available":94856908800,"internally_reserved":0,"allocated":212291584,"data_stored":211151166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925827072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:06 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.628,"15min":1.5329999999999999},"min":{"1min":0.23100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.0720000000000001,"5min":5.5129999999999999,"15min":50.427},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.89500000000000002,"15min":1.635},"min":{"1min":0.249,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":3.6259999999999999,"5min":8.1430000000000007,"15min":50.634999999999998},"last":0.53600000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:02:33 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":1.391,"15min":2.843},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":3.7519999999999998,"5min":19.596,"15min":165.66399999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.89200000000000002,"15min":2.7450000000000001},"min":{"1min":0.28100000000000003,"5min":0.246,"15min":0.246},"max":{"1min":2.528,"5min":6.3689999999999998,"15min":165.649},"last":0.46600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:02:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":0.98099999999999998,"15min":1.371},"min":{"1min":0.308,"5min":0.224,"15min":0.224},"max":{"1min":3.0270000000000001,"5min":6.2320000000000002,"15min":91.049999999999997},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.478,"5min":1.333,"15min":1.5349999999999999},"min":{"1min":0.39100000000000001,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":3.6840000000000002,"5min":13.996,"15min":91.105000000000004},"last":0.48599999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:02:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2589999999999999,"5min":1.411,"15min":3.093},"min":{"1min":0.36099999999999999,"5min":0.316,"15min":0.26500000000000001},"max":{"1min":3.7890000000000001,"5min":13.904,"15min":152.74700000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.2609999999999999,"15min":2.9809999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.313},"max":{"1min":2.5550000000000002,"5min":12.284000000000001,"15min":152.78100000000001},"last":2.5550000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:01:52 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.76100000000000001,"15min":1.853},"min":{"1min":0.33600000000000002,"5min":0.25800000000000001,"15min":0.248},"max":{"1min":3.0179999999999998,"5min":11.009,"15min":257.375},"last":0.312},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.91000000000000003,"15min":2.028},"min":{"1min":0.29199999999999998,"5min":0.26700000000000002,"15min":0.246},"max":{"1min":5.5810000000000004,"5min":8.2349999999999994,"15min":257.61200000000002},"last":0.56299999999999994}]},{"osd":5,"last update":"Mon Oct 9 06:01:52 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":0.81399999999999995,"15min":2.3410000000000002},"min":{"1min":0.39400000000000002,"5min":0.31900000000000001,"15min":0.27800000000000002},"max":{"1min":2.9729999999999999,"5min":8.1280000000000001,"15min":256.05700000000002},"last":0.439},{"interface":"front","average":{"1min":1.3140000000000001,"5min":1.101,"15min":2.5659999999999998},"min":{"1min":0.40899999999999997,"5min":0.35099999999999998,"15min":0.28699999999999998},"max":{"1min":6.0490000000000004,"5min":8.2040000000000006,"15min":256.08499999999998},"last":0.5}]},{"osd":7,"last update":"Mon Oct 9 06:02:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":2.1339999999999999,"15min":2.1360000000000001},"min":{"1min":0.38700000000000001,"5min":0.33900000000000002,"15min":0.31},"max":{"1min":6.6420000000000003,"5min":101.968,"15min":204.15299999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.96699999999999997,"5min":2.3380000000000001,"15min":2.3180000000000001},"min":{"1min":0.44700000000000001,"5min":0.312,"15min":0.28199999999999997},"max":{"1min":2.4950000000000001,"5min":102.001,"15min":203.85499999999999},"last":0.44700000000000001}]}]},{"osd":1,"up_from":93,"seq":399431958914,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1461740,"kb_used_data":519460,"kb_used_omap":7,"kb_used_meta":942264,"kb_avail":92283412,"statfs":{"total":95995035648,"available":94498213888,"internally_reserved":0,"allocated":531927040,"data_stored":529888445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":964878862},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:20 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.93300000000000005,"15min":2.2440000000000002},"min":{"1min":0.253,"5min":0.214,"15min":0.17499999999999999},"max":{"1min":10.449,"5min":10.449,"15min":131.91300000000001},"last":0.441},{"interface":"front","average":{"1min":0.81599999999999995,"5min":0.78300000000000003,"15min":2.3730000000000002},"min":{"1min":0.27100000000000002,"5min":0.23999999999999999,"15min":0.223},"max":{"1min":5.5609999999999999,"5min":5.6959999999999997,"15min":132.191},"last":0.48999999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:01:46 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.83399999999999996,"15min":1.956},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.4729999999999999,"5min":8.8100000000000005,"15min":132.35400000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.306,"5min":1.149,"15min":1.8859999999999999},"min":{"1min":0.29899999999999999,"5min":0.23699999999999999,"15min":0.20699999999999999},"max":{"1min":10.478,"5min":13.901999999999999,"15min":131.70599999999999},"last":0.50900000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:02:20 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.98999999999999999,"15min":2.4039999999999999},"min":{"1min":0.26800000000000002,"5min":0.23799999999999999,"15min":0.20200000000000001},"max":{"1min":1.873,"5min":8.843,"15min":135.10900000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.0920000000000001,"5min":0.96799999999999997,"15min":2.3450000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.407999999999999,"5min":10.407999999999999,"15min":131.886},"last":0.56100000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:02:20 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.97399999999999998,"15min":1.7649999999999999},"min":{"1min":0.29099999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.8780000000000001,"5min":6.1470000000000002,"15min":131.636},"last":0.63},{"interface":"front","average":{"1min":1.046,"5min":1.125,"15min":1.696},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.32,"5min":10.32,"15min":131.75999999999999},"last":0.73899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.177,"15min":2.3479999999999999},"min":{"1min":0.33500000000000002,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":4.0369999999999999,"5min":8.7300000000000004,"15min":132.291},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.032,"5min":1.24,"15min":2.3809999999999998},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.2799999999999998,"5min":9.7919999999999998,"15min":131.738},"last":0.78800000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:02:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.365,"15min":2.4220000000000002},"min":{"1min":0.41899999999999998,"5min":0.34200000000000003,"15min":0.33300000000000002},"max":{"1min":1.903,"5min":12.513999999999999,"15min":158.50399999999999},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.294,"15min":2.3090000000000002},"min":{"1min":0.41099999999999998,"5min":0.38,"15min":0.37},"max":{"1min":3.4470000000000001,"5min":9.8559999999999999,"15min":158.577},"last":0.72399999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":2.1219999999999999,"15min":2.0950000000000002},"min":{"1min":0.41299999999999998,"5min":0.40000000000000002,"15min":0.31},"max":{"1min":10.263999999999999,"5min":89.787000000000006,"15min":131.821},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.3300000000000001,"5min":2.3100000000000001,"15min":2.282},"min":{"1min":0.378,"5min":0.378,"15min":0.35199999999999998},"max":{"1min":10.365,"5min":89.869,"15min":131.852},"last":0.70499999999999996}]}]},{"osd":0,"up_from":88,"seq":377957122439,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1069228,"kb_used_data":255844,"kb_used_omap":8,"kb_used_meta":813367,"kb_avail":92675924,"statfs":{"total":95995035648,"available":94900146176,"internally_reserved":0,"allocated":261984256,"data_stored":259887815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":832888584},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:02:23 2023","interfaces":[{"interface":"back","average":{"1min":5.5700000000000003,"5min":5.6509999999999998,"15min":2.992},"min":{"1min":0.20899999999999999,"5min":0.19400000000000001,"15min":0.186},"max":{"1min":113.619,"5min":163.572,"15min":163.572},"last":0.37},{"interface":"front","average":{"1min":6.0410000000000004,"5min":5.7610000000000001,"15min":3.161},"min":{"1min":0.25600000000000001,"5min":0.24399999999999999,"15min":0.19600000000000001},"max":{"1min":118.233,"5min":162.48699999999999,"15min":162.48699999999999},"last":0.40400000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:01:59 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":0.65700000000000003,"15min":2.0019999999999998},"min":{"1min":0.23899999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.0570000000000004,"5min":6.2679999999999998,"15min":70.188999999999993},"last":2.2879999999999998},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.70999999999999996,"15min":1.704},"min":{"1min":0.223,"5min":0.221,"15min":0.221},"max":{"1min":4.5899999999999999,"5min":9.8559999999999999,"15min":71.144000000000005},"last":0.69299999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:02:23 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.55000000000000004,"15min":1.619},"min":{"1min":0.29399999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.5,"5min":3.5,"15min":62.698999999999998},"last":0.499},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.75900000000000001,"15min":2.0169999999999999},"min":{"1min":0.32300000000000001,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":4.0990000000000002,"5min":5.8810000000000002,"15min":72.632999999999996},"last":1.351}]},{"osd":4,"last update":"Mon Oct 9 06:01:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.70699999999999996,"15min":1.6799999999999999},"min":{"1min":0.34499999999999997,"5min":0.29099999999999998,"15min":0.28399999999999997},"max":{"1min":4.6840000000000002,"5min":4.6840000000000002,"15min":149.63999999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.4570000000000001,"5min":0.91600000000000004,"15min":1.8899999999999999},"min":{"1min":0.38500000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.4180000000000001,"5min":6.4180000000000001,"15min":149.65899999999999},"last":0.57999999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:01:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":2.6890000000000001,"15min":4.0579999999999998},"min":{"1min":0.38900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.508,"5min":210.77799999999999,"15min":246.761},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.1240000000000001,"5min":3.0019999999999998,"15min":4.4029999999999996},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":4.9809999999999999,"5min":209.91,"15min":247.48400000000001},"last":1.4299999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:01:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.91100000000000003,"15min":1.26},"min":{"1min":0.35499999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.7510000000000003,"5min":8.2899999999999991,"15min":19.579000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.399,"5min":1.077,"15min":1.401},"min":{"1min":0.38600000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.2050000000000001,"5min":12.457000000000001,"15min":19.710999999999999},"last":1.4730000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:02:23 2023","interfaces":[{"interface":"back","average":{"1min":1.46,"5min":2.9630000000000001,"15min":2.4569999999999999},"min":{"1min":0.40500000000000003,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":14.746,"5min":124.90900000000001,"15min":146.536},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.7470000000000001,"5min":3.028,"15min":2.5289999999999999},"min":{"1min":0.46500000000000002,"5min":0.40100000000000002,"15min":0.36299999999999999},"max":{"1min":14.787000000000001,"5min":125.041,"15min":146.49700000000001},"last":0.71499999999999997}]}]},{"osd":2,"up_from":98,"seq":420906795388,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1069440,"kb_used_data":175608,"kb_used_omap":11,"kb_used_meta":893812,"kb_avail":92675712,"statfs":{"total":95995035648,"available":94899929088,"internally_reserved":0,"allocated":179822592,"data_stored":177470321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":915264085},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.69199999999999995,"15min":1.798},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":6.359,"5min":9.7070000000000007,"15min":140.71100000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.004,"5min":0.63,"15min":2.056},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":6.4100000000000001,"5min":6.4100000000000001,"15min":142.542},"last":2.0800000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:02:07 2023","interfaces":[{"interface":"back","average":{"1min":3.3940000000000001,"5min":1.9119999999999999,"15min":2.6080000000000001},"min":{"1min":0.23300000000000001,"5min":0.19,"15min":0.19},"max":{"1min":43.335999999999999,"5min":75.260000000000005,"15min":125.77500000000001},"last":2.044},{"interface":"front","average":{"1min":3.4489999999999998,"5min":1.9399999999999999,"15min":2.6749999999999998},"min":{"1min":0.219,"5min":0.214,"15min":0.214},"max":{"1min":43.515000000000001,"5min":75.114999999999995,"15min":125.857},"last":5.5640000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:01:44 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.75600000000000001,"15min":1.6120000000000001},"min":{"1min":0.27200000000000002,"5min":0.23300000000000001,"15min":0.21099999999999999},"max":{"1min":3.2559999999999998,"5min":4.6989999999999998,"15min":29.606999999999999},"last":3.0179999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.83799999999999997,"15min":1.4630000000000001},"min":{"1min":0.26300000000000001,"5min":0.252,"15min":0.217},"max":{"1min":1.5580000000000001,"5min":9.7680000000000007,"15min":29.920999999999999},"last":0.66100000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.76900000000000002,"15min":1.3660000000000001},"min":{"1min":0.32300000000000001,"5min":0.32200000000000001,"15min":0.28699999999999998},"max":{"1min":3.2879999999999998,"5min":3.2879999999999998,"15min":68.052000000000007},"last":2.98},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.78900000000000003,"15min":1.3080000000000001},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":1.413,"5min":9.5800000000000001,"15min":68.727000000000004},"last":0.747}]},{"osd":5,"last update":"Mon Oct 9 06:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.78100000000000003,"15min":2.7530000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.29099999999999998},"max":{"1min":4.9359999999999999,"5min":4.9359999999999999,"15min":207.80500000000001},"last":3.3730000000000002},{"interface":"front","average":{"1min":1.353,"5min":1.236,"15min":2.923},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.30499999999999999},"max":{"1min":4.5579999999999998,"5min":9.4239999999999995,"15min":207.54900000000001},"last":2.496}]},{"osd":6,"last update":"Mon Oct 9 06:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.1040000000000001,"15min":1.208},"min":{"1min":0.32600000000000001,"5min":0.31,"15min":0.31},"max":{"1min":3.4870000000000001,"5min":9.9320000000000004,"15min":25.818999999999999},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.873,"15min":1.387},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.9100000000000001,"5min":4.9100000000000001,"15min":24.481000000000002},"last":2.9399999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:01:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":2.8399999999999999,"15min":2.3109999999999999},"min":{"1min":0.433,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.9589999999999996,"5min":122.827,"15min":162.09200000000001},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.85099999999999998,"5min":2.7829999999999999,"15min":2.4729999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.378},"max":{"1min":4.96,"5min":123.56399999999999,"15min":162.35400000000001},"last":3.4279999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631863,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1241564,"kb_used_data":209428,"kb_used_omap":4,"kb_used_meta":1032123,"kb_avail":92503588,"statfs":{"total":95995035648,"available":94723674112,"internally_reserved":0,"allocated":214454272,"data_stored":212371923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1056894558},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.63100000000000001,"15min":3.6400000000000001},"min":{"1min":0.246,"5min":0.20399999999999999,"15min":0.188},"max":{"1min":0.95099999999999996,"5min":12.728,"15min":312.56599999999997},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.70899999999999996,"15min":3.6320000000000001},"min":{"1min":0.24099999999999999,"5min":0.20899999999999999,"15min":0.20499999999999999},"max":{"1min":2.9409999999999998,"5min":7.6360000000000001,"15min":311.02300000000002},"last":0.27000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":3.8260000000000001,"15min":4.585},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.214},"max":{"1min":3.9169999999999998,"5min":137.00299999999999,"15min":150.29599999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.47899999999999998,"5min":3.8140000000000001,"15min":4.7080000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.216},"max":{"1min":1.2130000000000001,"5min":136.947,"15min":151.25700000000001},"last":0.30499999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":2.1640000000000001,"15min":2.1560000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23899999999999999},"max":{"1min":2.5670000000000002,"5min":87.542000000000002,"15min":87.542000000000002},"last":0.432},{"interface":"front","average":{"1min":0.72799999999999998,"5min":2.2160000000000002,"15min":2.1120000000000001},"min":{"1min":0.253,"5min":0.248,"15min":0.20499999999999999},"max":{"1min":2.9740000000000002,"5min":87.727999999999994,"15min":87.727999999999994},"last":0.32000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.70799999999999996,"15min":1.7549999999999999},"min":{"1min":0.33800000000000002,"5min":0.33400000000000002,"15min":0.28999999999999998},"max":{"1min":3.0390000000000001,"5min":3.278,"15min":126.566},"last":0.495},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.77200000000000002,"15min":2.1520000000000001},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":2.5,"5min":7.1390000000000002,"15min":126.651},"last":0.65300000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.72099999999999997,"15min":1.6659999999999999},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":4.8579999999999997,"5min":4.8579999999999997,"15min":71.203999999999994},"last":0.63},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.92200000000000004,"15min":1.55},"min":{"1min":0.38100000000000001,"5min":0.33400000000000002,"15min":0.29399999999999998},"max":{"1min":5.4610000000000003,"5min":12.778,"15min":70.397000000000006},"last":0.72999999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.84199999999999997,"15min":1.349},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.7200000000000002,"5min":7.1719999999999997,"15min":70.474000000000004},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.3240000000000001,"5min":0.92100000000000004,"15min":1.516},"min":{"1min":0.438,"5min":0.36599999999999999,"15min":0.33200000000000002},"max":{"1min":5.1970000000000001,"5min":5.1970000000000001,"15min":71.251999999999995},"last":0.66900000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:01:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4249999999999998,"5min":1.5329999999999999,"15min":1.605},"min":{"1min":0.41099999999999998,"5min":0.38300000000000001,"15min":0.35699999999999998},"max":{"1min":12.218,"5min":12.218,"15min":69.277000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.468,"15min":1.579},"min":{"1min":0.438,"5min":0.40000000000000002,"15min":0.374},"max":{"1min":7.0579999999999998,"5min":11.722,"15min":69.299000000000007},"last":0.64100000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019410,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1155676,"kb_used_data":254228,"kb_used_omap":0,"kb_used_meta":901440,"kb_avail":92589476,"statfs":{"total":95995035648,"available":94811623424,"internally_reserved":0,"allocated":260329472,"data_stored":259188278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923074560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:07 2023","interfaces":[{"interface":"back","average":{"1min":3.5720000000000001,"5min":1.2869999999999999,"15min":3.1619999999999999},"min":{"1min":0.22800000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":50.106999999999999,"5min":50.106999999999999,"15min":132.35599999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":3.7879999999999998,"5min":1.621,"15min":3.556},"min":{"1min":0.28199999999999997,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":49.334000000000003,"5min":49.334000000000003,"15min":132.501},"last":0.61699999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:01:41 2023","interfaces":[{"interface":"back","average":{"1min":4.0499999999999998,"5min":1.335,"15min":2.778},"min":{"1min":0.32800000000000001,"5min":0.28999999999999998,"15min":0.26600000000000001},"max":{"1min":55.799999999999997,"5min":55.799999999999997,"15min":96.033000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":4.202,"5min":1.6839999999999999,"15min":2.7770000000000001},"min":{"1min":0.29699999999999999,"5min":0.29099999999999998,"15min":0.255},"max":{"1min":55.784999999999997,"5min":55.784999999999997,"15min":96.022000000000006},"last":0.58999999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:01:49 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":2.3140000000000001,"15min":2.1219999999999999},"min":{"1min":0.29899999999999999,"5min":0.29599999999999999,"15min":0.28599999999999998},"max":{"1min":3.77,"5min":143.71000000000001,"15min":143.71000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.2669999999999999,"5min":2.605,"15min":2.3940000000000001},"min":{"1min":0.314,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.2539999999999996,"5min":143.768,"15min":143.768},"last":0.59899999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:02:23 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.179,"15min":2.452},"min":{"1min":0.33700000000000002,"5min":0.33400000000000002,"15min":0.28599999999999998},"max":{"1min":3.0329999999999999,"5min":11.055999999999999,"15min":100.009},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.85399999999999998,"15min":2.3330000000000002},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.30499999999999999},"max":{"1min":2.2610000000000001,"5min":5.9729999999999999,"15min":100.254},"last":0.5}]},{"osd":5,"last update":"Mon Oct 9 06:02:23 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.3640000000000001,"15min":1.079},"min":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.25900000000000001},"max":{"1min":3.423,"5min":15.597,"15min":15.597},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.80400000000000005,"5min":1.2270000000000001,"15min":1.0349999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.4079999999999999,"5min":17.882000000000001,"15min":17.882000000000001},"last":0.434}]},{"osd":6,"last update":"Mon Oct 9 06:01:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.0880000000000001,"15min":1.653},"min":{"1min":0.36399999999999999,"5min":0.314,"15min":0.27600000000000002},"max":{"1min":3.294,"5min":6.9859999999999998,"15min":181.411},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.002,"5min":1.304,"15min":1.728},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.29799999999999999},"max":{"1min":2.3769999999999998,"5min":11,"15min":180.63999999999999},"last":0.38300000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:01:41 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.264,"15min":1.4890000000000001},"min":{"1min":0.379,"5min":0.33100000000000002,"15min":0.29799999999999999},"max":{"1min":2.4630000000000001,"5min":17.832000000000001,"15min":96.131},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.073,"5min":1.1599999999999999,"15min":1.423},"min":{"1min":0.38100000000000001,"5min":0.35699999999999998,"15min":0.27200000000000002},"max":{"1min":5.6390000000000002,"5min":17.742999999999999,"15min":96.465000000000003},"last":0.40699999999999997}]}]},{"osd":5,"up_from":0,"seq":154618823182,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":13303808,"data_stored":13303808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":177930240,"data_stored":177930240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":74776576,"data_stored":74776576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":304873472,"data_stored":304873472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":114819072,"data_stored":114819072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":102170624,"data_stored":102170624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:02:41.722 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:02:41.722 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 71 2023-10-09T06:02:41.815 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:41.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:41.924 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:41.935 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:02:41.935 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:02:42.043 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:42.153 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 9 15 19700 19685 136.678 110.688 0.00818492 0.00731111 2023-10-09T06:02:42.355 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:42.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:42.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:42.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:42.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:42 smithi008 ceph-mon[184418]: pgmap v1542: 89 pgs: 89 active+clean; 815 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:02:42.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:42 smithi008 ceph-mon[191227]: pgmap v1542: 89 pgs: 89 active+clean; 815 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:02:42.490 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:42.533 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:42.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:42.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:42.636 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:42.700 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:42.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:42 smithi191 ceph-mon[161669]: pgmap v1542: 89 pgs: 89 active+clean; 815 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:02:42.891 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:02:43.153 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 10 16 21223 21207 132.52 95.125 0.00420946 0.00754016 2023-10-09T06:02:44.153 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 11 16 23030 23014 130.739 112.938 0.0016574 0.00764205 2023-10-09T06:02:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:44 smithi008 ceph-mon[184418]: pgmap v1543: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:02:44.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:44.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:44 smithi008 ceph-mon[191227]: pgmap v1543: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:02:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:44.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:44.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:44 smithi191 ceph-mon[161669]: pgmap v1543: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:02:44.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:44.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:44.911 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:44.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:45.019 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:45.099 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:45.155 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 12 16 24349 24333 126.688 82.4375 0.0262049 0.00788159 2023-10-09T06:02:45.176 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:45.188 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:45.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:45.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:45.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:45.332 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:45.396 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:45.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:45.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:45.502 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:45.575 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:45.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 2]}]: dispatch 2023-10-09T06:02:45.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 2]}]: dispatch 2023-10-09T06:02:45.797 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:02:45.797 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:45.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 2]}]: dispatch 2023-10-09T06:02:45.825 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:45.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:45.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:45.852 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:02:45.911 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:02:45.933 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:45.953 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:02:45.967 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:45.990 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:02:46.025 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:02:46.034 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:46.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:46.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:46.074 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:02:46.108 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:02:46.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:02:46.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:02:46.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:02:46.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:02:46.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:02:46.117 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:02:46.118 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:02:46.118 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:02:46.140 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:46.155 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 13 16 25972 25956 124.745 101.438 0.0078602 0.00801095 2023-10-09T06:02:46.173 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:46.241 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:46.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:46.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:46.351 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:46.387 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[191227]: pgmap v1544: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T06:02:46.395 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:02:46.430 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:46 smithi191 ceph-mon[161669]: pgmap v1544: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T06:02:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:46 smithi191 ceph-mon[161669]: failed to encode map e516 with expected crc 2023-10-09T06:02:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 2]}]': finished 2023-10-09T06:02:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:46 smithi191 ceph-mon[161669]: osdmap e516: 8 total, 8 up, 7 in 2023-10-09T06:02:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:46 smithi191 ceph-mon[161669]: failed to encode map e516 with expected crc 2023-10-09T06:02:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:46 smithi191 ceph-mon[161669]: failed to encode map e516 with expected crc 2023-10-09T06:02:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/215295025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2286605194' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:46.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[191227]: failed to encode map e516 with expected crc 2023-10-09T06:02:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 2]}]': finished 2023-10-09T06:02:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[191227]: osdmap e516: 8 total, 8 up, 7 in 2023-10-09T06:02:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[191227]: failed to encode map e516 with expected crc 2023-10-09T06:02:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[191227]: failed to encode map e516 with expected crc 2023-10-09T06:02:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/215295025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2286605194' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[184418]: pgmap v1544: 89 pgs: 89 active+clean; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T06:02:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[184418]: failed to encode map e516 with expected crc 2023-10-09T06:02:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 2]}]': finished 2023-10-09T06:02:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[184418]: osdmap e516: 8 total, 8 up, 7 in 2023-10-09T06:02:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[184418]: failed to encode map e516 with expected crc 2023-10-09T06:02:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[184418]: failed to encode map e516 with expected crc 2023-10-09T06:02:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/215295025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:46.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2286605194' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:47.155 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 14 16 27840 27824 124.173 116.75 0.00678898 0.00804807 2023-10-09T06:02:47.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:47.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[184418]: failed to encode map e517 with expected crc 2023-10-09T06:02:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[184418]: failed to encode map e517 with expected crc 2023-10-09T06:02:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/215295025' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:02:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[184418]: osdmap e517: 8 total, 8 up, 7 in 2023-10-09T06:02:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[184418]: failed to encode map e517 with expected crc 2023-10-09T06:02:47.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[184418]: failed to encode map e517 with expected crc 2023-10-09T06:02:47.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:47.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[191227]: failed to encode map e517 with expected crc 2023-10-09T06:02:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[191227]: failed to encode map e517 with expected crc 2023-10-09T06:02:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/215295025' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:02:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[191227]: osdmap e517: 8 total, 8 up, 7 in 2023-10-09T06:02:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[191227]: failed to encode map e517 with expected crc 2023-10-09T06:02:47.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:47 smithi008 ceph-mon[191227]: failed to encode map e517 with expected crc 2023-10-09T06:02:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:47 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:47 smithi191 ceph-mon[161669]: failed to encode map e517 with expected crc 2023-10-09T06:02:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:47 smithi191 ceph-mon[161669]: failed to encode map e517 with expected crc 2023-10-09T06:02:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/215295025' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:02:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:47 smithi191 ceph-mon[161669]: osdmap e517: 8 total, 8 up, 7 in 2023-10-09T06:02:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:47 smithi191 ceph-mon[161669]: failed to encode map e517 with expected crc 2023-10-09T06:02:47.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:47 smithi191 ceph-mon[161669]: failed to encode map e517 with expected crc 2023-10-09T06:02:48.156 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 15 16 29843 29827 124.24 125.188 0.00303253 0.00804414 2023-10-09T06:02:48.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:48 smithi008 ceph-mon[184418]: pgmap v1547: 89 pgs: 1 peering, 88 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T06:02:48.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:48 smithi008 ceph-mon[191227]: pgmap v1547: 89 pgs: 1 peering, 88 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T06:02:48.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:48 smithi191 ceph-mon[161669]: pgmap v1547: 89 pgs: 1 peering, 88 active+clean; 1.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T06:02:49.156 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 16 16 31639 31623 123.49 112.25 0.0101934 0.00809104 2023-10-09T06:02:50.156 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 17 16 33161 33145 121.822 95.125 0.00630138 0.0082039 2023-10-09T06:02:50.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:50 smithi008 ceph-mon[184418]: pgmap v1548: 89 pgs: 1 peering, 88 active+clean; 1.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 103 MiB/s wr, 1.66k op/s 2023-10-09T06:02:50.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:50 smithi008 ceph-mon[191227]: pgmap v1548: 89 pgs: 1 peering, 88 active+clean; 1.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 103 MiB/s wr, 1.66k op/s 2023-10-09T06:02:50.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:02:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:02:50] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.33.4" 2023-10-09T06:02:50.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:50 smithi191 ceph-mon[161669]: pgmap v1548: 89 pgs: 1 peering, 88 active+clean; 1.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 103 MiB/s wr, 1.66k op/s 2023-10-09T06:02:51.156 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 18 16 34600 34584 120.05 89.9375 0.00248449 0.0083049 2023-10-09T06:02:51.505 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:51.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:51.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:51.621 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:51.693 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:52.161 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 19 16 36167 36151 118.853 97.9375 0.00535201 0.00840817 2023-10-09T06:02:52.209 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:52.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:52.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:52.319 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:52.383 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:52.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[184418]: pgmap v1549: 89 pgs: 1 peering, 88 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T06:02:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:52.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[191227]: pgmap v1549: 89 pgs: 1 peering, 88 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T06:02:52.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:52.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:52.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:52.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:52.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:52.698 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:52.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:52.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:52 smithi191 ceph-mon[161669]: pgmap v1549: 89 pgs: 1 peering, 88 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T06:02:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:52.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:02:52.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:52.831 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:02:52.831 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T06:02:52.831 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.831 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:37091 10.4 10:2df8be5e:::benchmark_data_smithi008_634586_object37090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e517)", 2023-10-09T06:02:52.831 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:02:52.826229+0000", 2023-10-09T06:02:52.831 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0045483839999999999, 2023-10-09T06:02:52.831 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045822679999999996, 2023-10-09T06:02:52.831 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:02:52.832 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:02:52.832 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:02:52.832 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:02:52.832 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:02:52.832 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 37091 2023-10-09T06:02:52.832 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.832 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:02:52.832 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.832 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:02:52.832 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.826229+0000", 2023-10-09T06:02:52.832 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:02:52.833 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.833 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.833 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:02:52.833 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.826229+0000", 2023-10-09T06:02:52.833 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:02:52.833 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.833 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.833 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:02:52.834 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.826228+0000", 2023-10-09T06:02:52.834 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:52.834 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.834 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.834 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:02:52.834 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.826252+0000", 2023-10-09T06:02:52.834 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.425e-05 2023-10-09T06:02:52.834 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.834 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.834 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:02:52.835 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.826264+0000", 2023-10-09T06:02:52.835 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2208000000000001e-05 2023-10-09T06:02:52.835 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.835 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.835 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:02:52.835 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.826272+0000", 2023-10-09T06:02:52.835 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1999999999999994e-06 2023-10-09T06:02:52.835 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.835 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.835 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:02:52.836 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.826320+0000", 2023-10-09T06:02:52.836 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7978e-05 2023-10-09T06:02:52.836 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.836 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.836 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:02:52.836 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.826511+0000", 2023-10-09T06:02:52.836 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000190333 2023-10-09T06:02:52.836 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.836 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.836 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:02:52.837 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.826685+0000", 2023-10-09T06:02:52.837 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017462099999999999 2023-10-09T06:02:52.837 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.837 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.837 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:02:52.837 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828262+0000", 2023-10-09T06:02:52.837 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001576668 2023-10-09T06:02:52.837 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.837 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.837 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:52.838 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.830246+0000", 2023-10-09T06:02:52.838 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019842660000000002 2023-10-09T06:02:52.838 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:02:52.838 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:02:52.838 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:02:52.838 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.838 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.838 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.47036.0:37087 10.2 e517/512 10:43ebfcf1:::benchmark_data_smithi008_634586_object37086:head v 517'2333, mlcod=517'2330)", 2023-10-09T06:02:52.839 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:02:52.828365+0000", 2023-10-09T06:02:52.839 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.002412759, 2023-10-09T06:02:52.839 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025431609999999999, 2023-10-09T06:02:52.839 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:02:52.839 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T06:02:52.839 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:02:52.839 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.839 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:02:52.839 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828365+0000", 2023-10-09T06:02:52.840 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:02:52.840 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.840 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.840 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:02:52.840 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828365+0000", 2023-10-09T06:02:52.840 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:02:52.840 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.840 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.840 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:02:52.841 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828364+0000", 2023-10-09T06:02:52.841 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:52.841 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.841 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.841 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:02:52.841 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828398+0000", 2023-10-09T06:02:52.841 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4243999999999999e-05 2023-10-09T06:02:52.841 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.841 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.842 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:02:52.842 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828416+0000", 2023-10-09T06:02:52.842 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7178999999999999e-05 2023-10-09T06:02:52.842 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.842 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.842 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:02:52.842 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828423+0000", 2023-10-09T06:02:52.842 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.593e-06 2023-10-09T06:02:52.843 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.843 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.843 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:02:52.843 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828612+0000", 2023-10-09T06:02:52.843 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018895399999999999 2023-10-09T06:02:52.843 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.843 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.843 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:02:52.843 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828633+0000", 2023-10-09T06:02:52.844 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0891999999999999e-05 2023-10-09T06:02:52.844 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:02:52.844 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:02:52.844 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:02:52.844 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.844 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.844 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:37095 10.4 10:29e07c59:::benchmark_data_smithi008_634586_object37094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e517)", 2023-10-09T06:02:52.844 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:02:52.828456+0000", 2023-10-09T06:02:52.844 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0023216370000000001, 2023-10-09T06:02:52.845 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025206450000000002, 2023-10-09T06:02:52.845 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:02:52.845 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:02:52.845 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:02:52.845 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:02:52.845 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:02:52.845 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 37095 2023-10-09T06:02:52.845 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.846 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:02:52.846 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.846 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:02:52.846 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828456+0000", 2023-10-09T06:02:52.846 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:02:52.846 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.846 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.846 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:02:52.846 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828456+0000", 2023-10-09T06:02:52.847 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:02:52.847 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.847 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.847 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:02:52.847 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828455+0000", 2023-10-09T06:02:52.847 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:52.847 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.847 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.847 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:02:52.847 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828477+0000", 2023-10-09T06:02:52.848 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2376999999999999e-05 2023-10-09T06:02:52.848 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.848 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.848 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:02:52.848 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828487+0000", 2023-10-09T06:02:52.848 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0378e-05 2023-10-09T06:02:52.848 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.848 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.848 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:02:52.848 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828494+0000", 2023-10-09T06:02:52.848 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9700000000000002e-06 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828549+0000", 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4982999999999999e-05 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828693+0000", 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014359400000000001 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828827+0000", 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000133981 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:37097 10.a 10:53325898:::benchmark_data_smithi008_634586_object37096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e517)", 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:02:52.828509+0000", 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0022686440000000002, 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025476919999999998, 2023-10-09T06:02:52.850 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:02:52.851 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:02:52.851 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:02:52.851 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:02:52.851 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:02:52.851 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 37097 2023-10-09T06:02:52.851 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.851 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:02:52.851 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.851 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:02:52.851 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828509+0000", 2023-10-09T06:02:52.851 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828509+0000", 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828508+0000", 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.852 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828529+0000", 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0324999999999999e-05 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828536+0000", 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6520000000000006e-06 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:02:52.853 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.828542+0000", 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5999999999999997e-06 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.830045+0000", 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001502665 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.830175+0000", 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000129944 2023-10-09T06:02:52.854 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:02:52.855 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:02:52.855 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:02:52.855 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:52.830315+0000", 2023-10-09T06:02:52.855 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000139951 2023-10-09T06:02:52.855 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:02:52.855 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:02:52.855 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:02:52.855 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:02:52.855 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T06:02:52.855 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 4 2023-10-09T06:02:52.855 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:02:52.856 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:52.927 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:52.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:52.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:52.975 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T06:02:53.035 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:53.096 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:53.161 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T06:02:53.162286+0000 min lat: 0.00115919 max lat: 0.315704 avg lat: 0.0085087 2023-10-09T06:02:53.161 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:02:53.162 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 20 16 37619 37603 117.448 90.75 0.00640371 0.0085087 2023-10-09T06:02:53.163 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:53.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:53.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:53.274 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:53.345 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:53.426 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:53 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2553814752' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T06:02:53.427 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:53 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T06:02:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:53 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2553814752' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T06:02:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:53 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T06:02:53.634 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:02:53.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:53 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2553814752' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T06:02:53.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:53 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T06:02:53.759 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:02:54.167 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 21 16 39212 39196 116.567 99.5625 0.00871676 0.00857207 2023-10-09T06:02:54.367 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:54.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:54.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:54.472 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:54.657 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:54 smithi008 ceph-mon[191227]: pgmap v1550: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T06:02:54.660 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:54 smithi008 ceph-mon[184418]: pgmap v1550: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T06:02:54.663 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:54.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:54 smithi191 ceph-mon[161669]: pgmap v1550: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T06:02:55.042 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:55.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:55.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:55.145 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:55.166 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 22 16 40343 40327 114.483 70.6875 0.0024273 0.00870564 2023-10-09T06:02:55.172 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:55.216 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:55.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:55.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:55.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:55.355 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:55.459 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:55.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:55.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:55.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:55.606 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:55.672 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:02:55.681 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:55.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:55.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:55.751 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T06:02:55.793 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:55.830 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:55.873 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:55.894 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:56.022 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:56.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:56.130 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:56.165 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:56.167 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 23 16 41660 41644 113.084 82.3125 0.0105064 0.00883795 2023-10-09T06:02:56.243 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:56.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:56.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:56.351 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:56.391 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:56.436 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:56 smithi191 ceph-mon[161669]: pgmap v1551: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T06:02:56.448 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:56.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:56.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:56.554 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:56.592 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:56.664 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:56.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:56.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:56 smithi008 ceph-mon[184418]: pgmap v1551: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T06:02:56.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:56 smithi008 ceph-mon[191227]: pgmap v1551: 89 pgs: 89 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-09T06:02:56.770 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:56.854 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:57.167 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 24 16 43099 43083 112.12 89.9375 0.00556056 0.0089138 2023-10-09T06:02:57.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:57 smithi008 ceph-mon[184418]: pgmap v1552: 89 pgs: 89 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:02:57.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:57 smithi008 ceph-mon[191227]: pgmap v1552: 89 pgs: 89 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:02:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:57 smithi191 ceph-mon[161669]: pgmap v1552: 89 pgs: 89 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:02:58.170 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 25 16 44258 44242 110.528 72.4375 0.00967692 0.00904149 2023-10-09T06:02:58.950 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:58.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:58.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:59.003 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:59.057 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:59.148 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:59.168 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 26 16 45406 45390 109.037 71.75 0.00670999 0.00916693 2023-10-09T06:02:59.338 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:59.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:59.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:59.447 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:59.487 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:59.562 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:59.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:59.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:59.669 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:59.739 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:59.975 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:02:59.989 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:00.001 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:59 smithi008 ceph-mon[184418]: pgmap v1553: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:03:00.001 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:00.001 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:00.001 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:00.001 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:02:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/894577757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:00.001 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:59 smithi008 ceph-mon[191227]: pgmap v1553: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:03:00.002 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:00.002 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:00.002 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:00.002 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:02:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/894577757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:00.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:00.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:00.018 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:03:00.056 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:03:00.101 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:03:00.105 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:00.137 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:03:00.164 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:00.170 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 27 16 46736 46720 108.071 83.125 0.0129987 0.00924743 2023-10-09T06:03:00.175 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:03:00.220 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:03:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:59 smithi191 ceph-mon[161669]: pgmap v1553: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:03:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:02:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/894577757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:00.320 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:00.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:00.427 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:00.495 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:00.716 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:03:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:03:00] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.33.4" 2023-10-09T06:03:00.752 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:03:00.753 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:03:00.753 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T06:03:00.753 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-10-09T06:03:00.965 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:03:01.170 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 28 16 48055 48039 107.155 82.4375 0.0059643 0.00932702 2023-10-09T06:03:01.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/336895245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:01.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/336895245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:01.242 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:01.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:01.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/336895245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:01.347 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:01.376 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:01.450 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:01.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:01.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:01.559 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:01.616 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:02.162 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:02.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:02.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:02.171 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 29 16 49433 49417 106.428 86.125 0.00556223 0.00939091 2023-10-09T06:03:02.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[184418]: pgmap v1554: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T06:03:02.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:02.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[184418]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/336895245' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:02.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[184418]: osdmap e518: 8 total, 8 up, 7 in 2023-10-09T06:03:02.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[184418]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[184418]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[184418]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[191227]: pgmap v1554: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T06:03:02.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:02.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[191227]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/336895245' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:02.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[191227]: osdmap e518: 8 total, 8 up, 7 in 2023-10-09T06:03:02.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[191227]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[191227]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:01 smithi008 ceph-mon[191227]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.267 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:01 smithi191 ceph-mon[161669]: pgmap v1554: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T06:03:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:01 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:01 smithi191 ceph-mon[161669]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:01 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/336895245' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:01 smithi191 ceph-mon[161669]: osdmap e518: 8 total, 8 up, 7 in 2023-10-09T06:03:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:01 smithi191 ceph-mon[161669]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:01 smithi191 ceph-mon[161669]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:01 smithi191 ceph-mon[161669]: failed to encode map e518 with expected crc 2023-10-09T06:03:02.309 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:02.385 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:02.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:02.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:02.494 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:02.535 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:02.609 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:02.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:02.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:02.719 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:02.778 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:03.171 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 30 16 50788 50772 105.704 84.6875 0.00780119 0.00945386 2023-10-09T06:03:03.303 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:03.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:03.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:03.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:03.452 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:03.516 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:03.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:03.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:03.622 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:03.658 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:03.732 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:03.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:03.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:03.758 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:03.842 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:03.899 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:03:03.899 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:03:03.912 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:04.109 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:04.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:04.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:04.171 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 31 16 52073 52057 104.885 80.3125 0.00704394 0.00952882 2023-10-09T06:03:04.216 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:04.287 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:04 smithi008 ceph-mon[191227]: pgmap v1556: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:03:04.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:04 smithi008 ceph-mon[184418]: pgmap v1556: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:03:04.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:04 smithi191 ceph-mon[161669]: pgmap v1556: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:03:05.171 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 32 16 53162 53146 103.735 68.0625 0.0105184 0.00963499 2023-10-09T06:03:05.222 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:03:05.375 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:05.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:05.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:05.413 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:03:05.475 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:03:05.481 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:05.642 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:05.886 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:05.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:05.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:05.993 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:06.051 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:06.171 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 33 16 54052 54036 102.277 55.625 0.0123024 0.00973263 2023-10-09T06:03:06.570 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:06.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:06.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:06.675 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:06.705 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:06 smithi191 ceph-mon[161669]: pgmap v1557: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:03:06.709 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:06.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:06 smithi008 ceph-mon[184418]: pgmap v1557: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:03:06.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:06 smithi008 ceph-mon[191227]: pgmap v1557: 89 pgs: 89 active+clean; 2.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:03:06.773 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:06.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:06.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:06.869 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:06.878 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:06.927 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:06.966 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:07.073 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:07.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:07.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:07.171 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 34 16 55121 55105 101.234 66.8125 0.0192504 0.00987258 2023-10-09T06:03:07.185 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:07.223 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:07.292 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:07.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:07.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:07.327 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:07.404 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:07.436 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:07.494 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:07.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:07.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:07.599 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:07.639 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:07.702 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:07.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:07.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:07.810 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:07.884 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:08.172 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 35 16 56288 56272 100.424 72.9375 0.00870685 0.00995226 2023-10-09T06:03:08.360 INFO:teuthology.orchestra.run.smithi008.stdout:moved 28 / 267 (10.4869%) 2023-10-09T06:03:08.360 INFO:teuthology.orchestra.run.smithi008.stdout:avg 38.1429 2023-10-09T06:03:08.361 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 5.89015 -> 4.08581 (expected baseline 5.71786) 2023-10-09T06:03:08.361 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.1 with 29 -> 35 pgs (0.7603 -> 0.917603 * mean) 2023-10-09T06:03:08.361 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.4 with 47 -> 38 pgs (1.23221 -> 0.996255 * mean) 2023-10-09T06:03:08.361 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:03:08.361 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T06:03:08.361 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 1 2023-10-09T06:03:08.361 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T06:03:08.361 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0258 2023-10-09T06:03:08.361 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0284 2023-10-09T06:03:08.361 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 weight 1.0000 -> 0.7412 2023-10-09T06:03:08.361 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 0.6602 -> 0.9360 2023-10-09T06:03:08.362 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0 weight 1.0000 -> 0.7855 2023-10-09T06:03:08.362 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 weight 0.5961 -> 0.7855 2023-10-09T06:03:08.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:08 smithi008 ceph-mon[184418]: pgmap v1558: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:03:08.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:08 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'51479','1':'51478','3':'61344','4':'48573'}"}]: dispatch 2023-10-09T06:03:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:08 smithi008 ceph-mon[191227]: pgmap v1558: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:03:08.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:08 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'51479','1':'51478','3':'61344','4':'48573'}"}]: dispatch 2023-10-09T06:03:08.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:08 smithi191 ceph-mon[161669]: pgmap v1558: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:03:08.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:08 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'51479','1':'51478','3':'61344','4':'48573'}"}]: dispatch 2023-10-09T06:03:09.172 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 36 16 56608 56592 98.1915 20 0.00221096 0.00995878 2023-10-09T06:03:09.604 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:09.606 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[184418]: from='client.56920 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:09.606 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'51479','1':'51478','3':'61344','4':'48573'}"}]': finished 2023-10-09T06:03:09.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[184418]: osdmap e519: 8 total, 8 up, 7 in 2023-10-09T06:03:09.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[184418]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[184418]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[184418]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.607 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[184418]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.608 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[184418]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.608 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[184418]: osdmap e520: 8 total, 8 up, 7 in 2023-10-09T06:03:09.608 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[191227]: from='client.56920 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:09.608 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'51479','1':'51478','3':'61344','4':'48573'}"}]': finished 2023-10-09T06:03:09.608 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[191227]: osdmap e519: 8 total, 8 up, 7 in 2023-10-09T06:03:09.609 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[191227]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.609 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[191227]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.609 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[191227]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.609 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[191227]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.609 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[191227]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.609 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:09 smithi008 ceph-mon[191227]: osdmap e520: 8 total, 8 up, 7 in 2023-10-09T06:03:09.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:09.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:09.711 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:09.741 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:09 smithi191 ceph-mon[161669]: from='client.56920 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:09 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'51479','1':'51478','3':'61344','4':'48573'}"}]': finished 2023-10-09T06:03:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:09 smithi191 ceph-mon[161669]: osdmap e519: 8 total, 8 up, 7 in 2023-10-09T06:03:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:09 smithi191 ceph-mon[161669]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:09 smithi191 ceph-mon[161669]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:09 smithi191 ceph-mon[161669]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:09 smithi191 ceph-mon[161669]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:09 smithi191 ceph-mon[161669]: failed to encode map e519 with expected crc 2023-10-09T06:03:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:09 smithi191 ceph-mon[161669]: osdmap e520: 8 total, 8 up, 7 in 2023-10-09T06:03:10.172 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 37 16 56608 56592 95.539 0 - 0.00995878 2023-10-09T06:03:10.572 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:03:10.572 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-10-09T06:03:10.613 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:10.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:10.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:10.652 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[184418]: pgmap v1560: 89 pgs: 1 peering, 2 unknown, 86 active+clean; 3.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 46 MiB/s wr, 717 op/s 2023-10-09T06:03:10.652 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[184418]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.652 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[184418]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.652 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[184418]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.652 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[184418]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.652 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[184418]: osdmap e521: 8 total, 8 up, 7 in 2023-10-09T06:03:10.652 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:03:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:03:10] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.33.4" 2023-10-09T06:03:10.659 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[191227]: pgmap v1560: 89 pgs: 1 peering, 2 unknown, 86 active+clean; 3.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 46 MiB/s wr, 717 op/s 2023-10-09T06:03:10.659 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[191227]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.659 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[191227]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.659 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[191227]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.659 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[191227]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.659 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:10 smithi008 ceph-mon[191227]: osdmap e521: 8 total, 8 up, 7 in 2023-10-09T06:03:10.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:10.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:10 smithi191 ceph-mon[161669]: pgmap v1560: 89 pgs: 1 peering, 2 unknown, 86 active+clean; 3.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 46 MiB/s wr, 717 op/s 2023-10-09T06:03:10.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:10 smithi191 ceph-mon[161669]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:10 smithi191 ceph-mon[161669]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:10 smithi191 ceph-mon[161669]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:10 smithi191 ceph-mon[161669]: failed to encode map e520 with expected crc 2023-10-09T06:03:10.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:10 smithi191 ceph-mon[161669]: osdmap e521: 8 total, 8 up, 7 in 2023-10-09T06:03:10.769 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:10.819 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:10.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:10.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:10.926 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:11.004 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:11.173 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 38 16 56981 56965 93.639 11.6562 0.018088 0.0106748 2023-10-09T06:03:11.404 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:11 smithi008 ceph-mon[191227]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.405 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:11 smithi008 ceph-mon[191227]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.405 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:11 smithi008 ceph-mon[191227]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.405 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:11 smithi008 ceph-mon[191227]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.405 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:11 smithi008 ceph-mon[191227]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.410 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:11.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:11.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:11.523 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:11.552 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:11.612 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:11.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:11.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:11 smithi008 ceph-mon[184418]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:11 smithi008 ceph-mon[184418]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:11 smithi008 ceph-mon[184418]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:11 smithi008 ceph-mon[184418]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:11 smithi008 ceph-mon[184418]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:11.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:11 smithi191 ceph-mon[161669]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:11 smithi191 ceph-mon[161669]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.758 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:11 smithi191 ceph-mon[161669]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.758 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:11 smithi191 ceph-mon[161669]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.758 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:11 smithi191 ceph-mon[161669]: failed to encode map e521 with expected crc 2023-10-09T06:03:11.761 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:11.815 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:11.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:11.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:11.920 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:12.017 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:12.175 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 39 16 57741 57725 92.4503 47.5 0.0230475 0.0108085 2023-10-09T06:03:12.194 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:03:12.194 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:03:12.335 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:12.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:12.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:12.452 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:12.483 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:12.538 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:12.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:12.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:12.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:12.677 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:12 smithi191 ceph-mon[161669]: pgmap v1563: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 2 unknown, 84 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 670 op/s; 7043/157050 objects degraded (4.485%); 92 B/s, 0 objects/s recovering 2023-10-09T06:03:12.677 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:12 smithi191 ceph-mon[161669]: Health check failed: Degraded data redundancy: 7043/157050 objects degraded (4.485%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:12.678 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:12 smithi191 ceph-mon[161669]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.678 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:12 smithi191 ceph-mon[161669]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.678 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:12 smithi191 ceph-mon[161669]: osdmap e522: 8 total, 8 up, 7 in 2023-10-09T06:03:12.678 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:12 smithi191 ceph-mon[161669]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.678 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:12 smithi191 ceph-mon[161669]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.678 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:12 smithi191 ceph-mon[161669]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.681 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:12.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[184418]: pgmap v1563: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 2 unknown, 84 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 670 op/s; 7043/157050 objects degraded (4.485%); 92 B/s, 0 objects/s recovering 2023-10-09T06:03:12.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[184418]: Health check failed: Degraded data redundancy: 7043/157050 objects degraded (4.485%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:12.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[184418]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[184418]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[184418]: osdmap e522: 8 total, 8 up, 7 in 2023-10-09T06:03:12.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[184418]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[184418]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[184418]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[191227]: pgmap v1563: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 2 unknown, 84 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 670 op/s; 7043/157050 objects degraded (4.485%); 92 B/s, 0 objects/s recovering 2023-10-09T06:03:12.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[191227]: Health check failed: Degraded data redundancy: 7043/157050 objects degraded (4.485%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:12.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[191227]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[191227]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[191227]: osdmap e522: 8 total, 8 up, 7 in 2023-10-09T06:03:12.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[191227]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[191227]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:12 smithi008 ceph-mon[191227]: failed to encode map e522 with expected crc 2023-10-09T06:03:12.743 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:12.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:12.849 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:12.931 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:13.034 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:03:13.075 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:03:13.122 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:03:13.125 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:13.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:13.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:13.162 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:03:13.175 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T06:03:13.176291+0000 min lat: 0.00115919 max lat: 2.23699 avg lat: 0.0109389 2023-10-09T06:03:13.175 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:03:13.175 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 40 16 58530 58514 91.3723 49.3125 0.0241249 0.0109389 2023-10-09T06:03:13.202 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:03:13.232 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:03:13.233 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:13.263 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:13.264 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:03:13.298 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:03:13.310 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:03:13.310 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:03:13.310 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:03:13.310 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:03:13.310 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:03:13.311 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:03:13.311 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:03:13.311 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:03:13.319 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:13.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:13.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:13.425 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:13.485 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:03:13.499 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:13.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:13 smithi191 ceph-mon[161669]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:13 smithi191 ceph-mon[161669]: osdmap e523: 8 total, 8 up, 7 in 2023-10-09T06:03:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:13 smithi191 ceph-mon[161669]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:13 smithi191 ceph-mon[161669]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:13 smithi191 ceph-mon[161669]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1183578728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:03:13.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1541908322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:13.550 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:13.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:13.654 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:13.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[184418]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[184418]: osdmap e523: 8 total, 8 up, 7 in 2023-10-09T06:03:13.710 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[184418]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.711 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[184418]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.711 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[184418]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.711 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1183578728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:03:13.711 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1541908322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:13.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[191227]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[191227]: osdmap e523: 8 total, 8 up, 7 in 2023-10-09T06:03:13.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[191227]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.711 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[191227]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.713 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[191227]: failed to encode map e523 with expected crc 2023-10-09T06:03:13.713 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1183578728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:03:13.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1541908322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:13.721 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:14.182 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 41 16 59285 59269 90.2896 47.1875 0.0324086 0.0110703 2023-10-09T06:03:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[191227]: pgmap v1566: 89 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped, 82 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 217 KiB/s rd, 91 MiB/s wr, 1.66k op/s; 13987/170280 objects degraded (8.214%); 143 B/s, 6 objects/s recovering 2023-10-09T06:03:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[191227]: failed to encode map e524 with expected crc 2023-10-09T06:03:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[191227]: failed to encode map e524 with expected crc 2023-10-09T06:03:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[191227]: failed to encode map e524 with expected crc 2023-10-09T06:03:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1183578728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:03:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[191227]: osdmap e524: 8 total, 8 up, 7 in 2023-10-09T06:03:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:14.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:14.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[184418]: pgmap v1566: 89 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped, 82 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 217 KiB/s rd, 91 MiB/s wr, 1.66k op/s; 13987/170280 objects degraded (8.214%); 143 B/s, 6 objects/s recovering 2023-10-09T06:03:14.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:14.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[184418]: failed to encode map e524 with expected crc 2023-10-09T06:03:14.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[184418]: failed to encode map e524 with expected crc 2023-10-09T06:03:14.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[184418]: failed to encode map e524 with expected crc 2023-10-09T06:03:14.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1183578728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:03:14.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[184418]: osdmap e524: 8 total, 8 up, 7 in 2023-10-09T06:03:14.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:14.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:14 smithi191 ceph-mon[161669]: pgmap v1566: 89 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped, 82 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 217 KiB/s rd, 91 MiB/s wr, 1.66k op/s; 13987/170280 objects degraded (8.214%); 143 B/s, 6 objects/s recovering 2023-10-09T06:03:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:14 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:14 smithi191 ceph-mon[161669]: failed to encode map e524 with expected crc 2023-10-09T06:03:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:14 smithi191 ceph-mon[161669]: failed to encode map e524 with expected crc 2023-10-09T06:03:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:14 smithi191 ceph-mon[161669]: failed to encode map e524 with expected crc 2023-10-09T06:03:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:14 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1183578728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:03:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:14 smithi191 ceph-mon[161669]: osdmap e524: 8 total, 8 up, 7 in 2023-10-09T06:03:14.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:14.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:15.102 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:15.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:15.178 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 42 16 60105 60089 89.3605 51.25 0.0198396 0.0111827 2023-10-09T06:03:15.208 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:15.288 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:15.446 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:15 smithi008 ceph-mon[191227]: pgmap v1568: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 196 KiB/s rd, 89 MiB/s wr, 1.60k op/s; 13987/171669 objects degraded (8.148%); 3354/171669 objects misplaced (1.954%); 2.1 MiB/s, 39 objects/s recovering 2023-10-09T06:03:15.694 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:15.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:15 smithi008 ceph-mon[184418]: pgmap v1568: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 196 KiB/s rd, 89 MiB/s wr, 1.60k op/s; 13987/171669 objects degraded (8.148%); 3354/171669 objects misplaced (1.954%); 2.1 MiB/s, 39 objects/s recovering 2023-10-09T06:03:15.739 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:03:15.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:15 smithi191 ceph-mon[161669]: pgmap v1568: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 196 KiB/s rd, 89 MiB/s wr, 1.60k op/s; 13987/171669 objects degraded (8.148%); 3354/171669 objects misplaced (1.954%); 2.1 MiB/s, 39 objects/s recovering 2023-10-09T06:03:15.909 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:15.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:15.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:16.016 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:16.125 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:16.178 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 43 16 60777 60761 88.2596 42 0.0184923 0.0113242 2023-10-09T06:03:16.318 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:16.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:16.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:16.429 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:16.487 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:16.716 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:16.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:16.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:16.824 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:16.857 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:16.931 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:16.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:16.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:16.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:16 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13987/179433 objects degraded (7.795%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:16.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:16 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13987/179433 objects degraded (7.795%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:17.036 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:17.062 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:17.105 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:17.110 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:16 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13987/179433 objects degraded (7.795%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:17.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:17.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:17.178 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 44 16 61443 61427 87.2002 41.625 0.0179679 0.0114134 2023-10-09T06:03:17.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:17.237 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:17.278 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:17.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:17.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:17.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:17.420 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:17.478 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:17.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:17.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:17.585 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:17.620 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:17.688 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:17.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:17.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:17.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:17.828 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:17.885 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:17.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:17.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:17.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:18.002 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:18.018 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:17 smithi191 ceph-mon[161669]: pgmap v1569: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 145 KiB/s rd, 78 MiB/s wr, 1.38k op/s; 13987/179433 objects degraded (7.795%); 3354/179433 objects misplaced (1.869%); 1.5 MiB/s, 28 objects/s recovering 2023-10-09T06:03:18.023 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:18.048 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:03:18.083 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:18.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:18.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:18.107 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:03:18.131 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:17 smithi008 ceph-mon[184418]: pgmap v1569: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 145 KiB/s rd, 78 MiB/s wr, 1.38k op/s; 13987/179433 objects degraded (7.795%); 3354/179433 objects misplaced (1.869%); 1.5 MiB/s, 28 objects/s recovering 2023-10-09T06:03:18.131 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:17 smithi008 ceph-mon[191227]: pgmap v1569: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 145 KiB/s rd, 78 MiB/s wr, 1.38k op/s; 13987/179433 objects degraded (7.795%); 3354/179433 objects misplaced (1.869%); 1.5 MiB/s, 28 objects/s recovering 2023-10-09T06:03:18.179 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 45 16 62146 62130 86.2383 43.9375 0.0114383 0.0115883 2023-10-09T06:03:18.190 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:18.353 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:18.579 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:18.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:18.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:18.693 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:18.850 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:19.115 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:19.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:19.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:19.179 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 46 16 62793 62777 85.2431 40.4375 0.00977958 0.0117256 2023-10-09T06:03:19.206 INFO:teuthology.orchestra.run.smithi008.stdout:moved 12 / 267 (4.49438%) 2023-10-09T06:03:19.206 INFO:teuthology.orchestra.run.smithi008.stdout:avg 38.1429 2023-10-09T06:03:19.206 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 4.08581 -> 4.76381 (expected baseline 5.71786) 2023-10-09T06:03:19.206 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.2 with 33 -> 31 pgs (0.865169 -> 0.812734 * mean) 2023-10-09T06:03:19.206 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.3 with 45 -> 42 pgs (1.17978 -> 1.10112 * mean) 2023-10-09T06:03:19.206 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:03:19.206 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T06:03:19.206 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 3 2023-10-09T06:03:19.206 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T06:03:19.206 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0281 2023-10-09T06:03:19.207 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.2806 2023-10-09T06:03:19.207 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 0.9360 -> 1.0000 2023-10-09T06:03:19.207 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 weight 0.7855 -> 1.0000 2023-10-09T06:03:19.222 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:19.245 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T06:03:19.307 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:19.374 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:19.376 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:03:19 smithi008 conmon[177229]: 2023-10-09T06:03:19.203+0000 7f020285b700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T06:03:19.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:19.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:19.478 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:19.553 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:20.180 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 47 16 63364 63348 84.1873 35.6875 0.0104802 0.011873 2023-10-09T06:03:20.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:20 smithi008 ceph-mon[191227]: pgmap v1570: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 13987/182832 objects degraded (7.650%); 3134/182832 objects misplaced (1.714%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:03:20.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:20 smithi008 ceph-mon[191227]: from='client.37125 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:20.716 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:03:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:03:20] "GET /metrics HTTP/1.1" 200 36634 "" "Prometheus/2.33.4" 2023-10-09T06:03:20.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:20 smithi008 ceph-mon[184418]: pgmap v1570: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 13987/182832 objects degraded (7.650%); 3134/182832 objects misplaced (1.714%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:03:20.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:20 smithi008 ceph-mon[184418]: from='client.37125 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:20.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:20 smithi191 ceph-mon[161669]: pgmap v1570: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 13987/182832 objects degraded (7.650%); 3134/182832 objects misplaced (1.714%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:03:20.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:20 smithi191 ceph-mon[161669]: from='client.37125 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:21.157 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:21.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:21.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:21.180 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 48 16 64074 64058 83.3582 44.375 0.0230647 0.0119892 2023-10-09T06:03:21.201 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:03:21.202 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T06:03:21.275 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:21.306 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:21.356 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:21.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:21.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:21.403 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:21.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:21.489 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:21.540 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:21.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:21.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:21.647 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:21.683 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:21.743 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:21.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:21.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:21.851 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:21.880 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:21.918 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:21.934 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:21.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:21.981 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:03:21.981 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:03:21.981 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T06:03:21.981 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:03:22.043 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:22.109 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:22.180 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 49 16 64717 64701 82.4776 40.1875 0.00665403 0.0121183 2023-10-09T06:03:22.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:22 smithi008 ceph-mon[184418]: pgmap v1571: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 13987/187797 objects degraded (7.448%); 3134/187797 objects misplaced (1.669%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:03:22.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:22 smithi008 ceph-mon[191227]: pgmap v1571: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 13987/187797 objects degraded (7.448%); 3134/187797 objects misplaced (1.669%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:03:22.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:22 smithi191 ceph-mon[161669]: pgmap v1571: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 13987/187797 objects degraded (7.448%); 3134/187797 objects misplaced (1.669%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:03:23.180 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 50 16 65409 65393 81.6933 43.25 0.0160429 0.012235 2023-10-09T06:03:23.712 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:23.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13987/187797 objects degraded (7.448%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:23.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13987/187797 objects degraded (7.448%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:23.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:23.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:23.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13987/187797 objects degraded (7.448%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:23.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:23.844 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:23.885 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:23.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:23.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:23.992 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:24.077 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:24.180 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 51 16 66080 66064 80.9141 41.9375 0.026605 0.012354 2023-10-09T06:03:24.397 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:24 smithi008 ceph-mon[184418]: pgmap v1572: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 44 MiB/s wr, 705 op/s; 13987/191463 objects degraded (7.305%); 3134/191463 objects misplaced (1.637%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:03:24.399 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:24 smithi008 ceph-mon[191227]: pgmap v1572: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 44 MiB/s wr, 705 op/s; 13987/191463 objects degraded (7.305%); 3134/191463 objects misplaced (1.637%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:03:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:24 smithi191 ceph-mon[161669]: pgmap v1572: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 44 MiB/s wr, 705 op/s; 13987/191463 objects degraded (7.305%); 3134/191463 objects misplaced (1.637%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:03:25.180 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 52 16 66900 66884 80.3438 51.25 0.00901448 0.0124421 2023-10-09T06:03:25.293 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:03:25.354 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:25.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:25.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:25.468 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:25.497 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:25.550 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:25.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:25.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:25.655 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:25.686 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:25.710 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:25.734 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:25.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:25.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:25.842 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:25.872 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:25.921 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:25.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:25.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:26.028 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:26.062 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:26.120 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:26.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:26.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:26.180 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 53 16 67564 67548 79.6112 41.5 0.00960769 0.0125556 2023-10-09T06:03:26.225 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:26.302 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:26.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:26 smithi008 ceph-mon[184418]: pgmap v1573: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 13987/192687 objects degraded (7.259%); 2885/192687 objects misplaced (1.497%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:03:26.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:26 smithi008 ceph-mon[191227]: pgmap v1573: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 13987/192687 objects degraded (7.259%); 2885/192687 objects misplaced (1.497%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:03:26.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:26 smithi191 ceph-mon[161669]: pgmap v1573: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 13987/192687 objects degraded (7.259%); 2885/192687 objects misplaced (1.497%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:03:26.911 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:26.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:27.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:27.061 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:27.122 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:27.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:27.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:27.183 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 54 16 68385 68369 79.0832 51.3125 0.0260019 0.0126386 2023-10-09T06:03:27.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:27.272 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:27.324 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:27.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:27.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:27.461 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:27.508 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:27.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:27.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:27.616 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:27.653 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:27.713 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:27.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:27.819 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:27.918 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:27.938 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:03:27.995 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:03:28.049 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:03:28.087 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:03:28.122 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:03:28.142 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:28.161 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:03:28.183 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 55 16 69104 69088 78.4626 44.9375 0.0154323 0.0127385 2023-10-09T06:03:28.203 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:03:28.419 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:03:28.419 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:03:28.419 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:03:28.419 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:03:28.419 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:03:28.419 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:03:28.419 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:03:28.420 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:03:28.420 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:03:28.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:28 smithi008 ceph-mon[184418]: pgmap v1574: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 783 op/s; 13987/199881 objects degraded (6.998%); 2885/199881 objects misplaced (1.443%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T06:03:28.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2289812606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:28.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13987/199881 objects degraded (6.998%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:28.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:28 smithi008 ceph-mon[191227]: pgmap v1574: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 783 op/s; 13987/199881 objects degraded (6.998%); 2885/199881 objects misplaced (1.443%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T06:03:28.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2289812606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:28.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13987/199881 objects degraded (6.998%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:28 smithi191 ceph-mon[161669]: pgmap v1574: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 783 op/s; 13987/199881 objects degraded (6.998%); 2885/199881 objects misplaced (1.443%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T06:03:28.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2289812606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:28.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13987/199881 objects degraded (6.998%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:28.941 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:28.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:28.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:29.055 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:29.148 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:29.183 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 56 16 69721 69705 77.7504 38.5625 0.00383955 0.012854 2023-10-09T06:03:29.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:29.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:29.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:30.184 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 57 16 70224 70208 76.9382 31.4375 0.0100853 0.0129842 2023-10-09T06:03:30.564 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:30.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:30.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:30.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:30.688 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:30 smithi008 ceph-mon[184418]: pgmap v1575: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s; 13987/203307 objects degraded (6.880%); 2637/203307 objects misplaced (1.297%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:03:30.688 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:03:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:03:30] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.33.4" 2023-10-09T06:03:30.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:30 smithi008 ceph-mon[191227]: pgmap v1575: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s; 13987/203307 objects degraded (6.880%); 2637/203307 objects misplaced (1.297%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:03:30.705 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:30 smithi191 ceph-mon[161669]: pgmap v1575: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s; 13987/203307 objects degraded (6.880%); 2637/203307 objects misplaced (1.297%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:03:30.711 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:30.766 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:30.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:30.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:30.872 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:30.901 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:30.954 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:30.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:30.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:31.063 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:31.168 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:31.184 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 58 16 70964 70948 76.4092 46.25 0.00571519 0.0130798 2023-10-09T06:03:31.563 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:31.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:31.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:31.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:31.719 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:31.780 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:31.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:31.886 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:31.985 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:32.113 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:32.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:32.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:32.184 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 59 16 71720 71704 75.9152 47.25 0.0138937 0.0131663 2023-10-09T06:03:32.222 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:32.254 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:32.306 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:32.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:32.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:32.412 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:32.474 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:03:32.506 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:32.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:32 smithi191 ceph-mon[161669]: pgmap v1576: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 13987/208458 objects degraded (6.710%); 2637/208458 objects misplaced (1.265%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:03:32.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:32 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3042566648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:32.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:32 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:32 smithi008 ceph-mon[184418]: pgmap v1576: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 13987/208458 objects degraded (6.710%); 2637/208458 objects misplaced (1.265%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:03:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:32 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3042566648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:32 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:32.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:32 smithi008 ceph-mon[191227]: pgmap v1576: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 13987/208458 objects degraded (6.710%); 2637/208458 objects misplaced (1.265%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:03:32.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:32 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3042566648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:32.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:32 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:32.805 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:32.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:32.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:32.897 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:03:32.932 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:32.963 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:33.013 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:33.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:33.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:33.121 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:33.185 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T06:03:33.185174+0000 min lat: 0.00115919 max lat: 2.23699 avg lat: 0.0132552 2023-10-09T06:03:33.186 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:03:33.186 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 60 16 72456 72440 75.4164 46 0.0123144 0.0132552 2023-10-09T06:03:33.241 INFO:teuthology.orchestra.run.smithi008.stdout:moved 12 / 267 (4.49438%) 2023-10-09T06:03:33.241 INFO:teuthology.orchestra.run.smithi008.stdout:avg 38.1429 2023-10-09T06:03:33.241 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 4.08581 -> 4.76381 (expected baseline 5.71786) 2023-10-09T06:03:33.241 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.2 with 33 -> 31 pgs (0.865169 -> 0.812734 * mean) 2023-10-09T06:03:33.241 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.3 with 45 -> 42 pgs (1.17978 -> 1.10112 * mean) 2023-10-09T06:03:33.241 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:03:33.241 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T06:03:33.241 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 1 2023-10-09T06:03:33.242 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T06:03:33.242 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0310 2023-10-09T06:03:33.242 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.3100 2023-10-09T06:03:33.242 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 weight 0.7855 -> 1.0000 2023-10-09T06:03:33.242 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 0.9360 -> 1.0000 2023-10-09T06:03:33.248 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T06:03:33.274 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:33.433 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:03:33 smithi008 conmon[177229]: 2023-10-09T06:03:33.228+0000 7f020285b700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T06:03:33.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:03:33.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:33.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[184418]: osdmap e525: 8 total, 8 up, 7 in 2023-10-09T06:03:33.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[184418]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[184418]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[184418]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[184418]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.434 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13987/208458 objects degraded (6.710%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:33.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:03:33.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:33.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[191227]: osdmap e525: 8 total, 8 up, 7 in 2023-10-09T06:03:33.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[191227]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[191227]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[191227]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[191227]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.435 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13987/208458 objects degraded (6.710%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:33.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:33 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:03:33.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:33 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:33.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:33 smithi191 ceph-mon[161669]: osdmap e525: 8 total, 8 up, 7 in 2023-10-09T06:03:33.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:33 smithi191 ceph-mon[161669]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:33 smithi191 ceph-mon[161669]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:33 smithi191 ceph-mon[161669]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:33 smithi191 ceph-mon[161669]: failed to encode map e525 with expected crc 2023-10-09T06:03:33.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13987/208458 objects degraded (6.710%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:34.184 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 61 16 73137 73121 74.878 42.5625 0.0221828 0.0133493 2023-10-09T06:03:34.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:34 smithi008 ceph-mon[184418]: pgmap v1578: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 694 op/s; 13987/212325 objects degraded (6.588%); 2637/212325 objects misplaced (1.242%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:03:34.566 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:34 smithi008 ceph-mon[184418]: from='client.56941 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:34.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:34 smithi008 ceph-mon[191227]: pgmap v1578: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 694 op/s; 13987/212325 objects degraded (6.588%); 2637/212325 objects misplaced (1.242%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:03:34.567 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:34 smithi008 ceph-mon[191227]: from='client.56941 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:34.579 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:34.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:34.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:34.687 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:34.776 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:34.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:34 smithi191 ceph-mon[161669]: pgmap v1578: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 694 op/s; 13987/212325 objects degraded (6.588%); 2637/212325 objects misplaced (1.242%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:03:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:34 smithi191 ceph-mon[161669]: from='client.56941 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:34.923 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:34.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:34.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:35.033 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:35.099 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:35.185 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 62 16 73919 73903 74.4574 48.875 0.00997192 0.0134249 2023-10-09T06:03:35.459 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:35.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:35.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:35.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:35.603 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:35.663 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:35.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:35.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:35.714 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:03:35.715 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-10-09T06:03:35.775 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:35.855 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:35.996 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:36.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:36.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:36.112 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:36.125 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:36.141 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:36.185 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 63 16 74629 74613 73.9801 44.375 0.0148846 0.0135094 2023-10-09T06:03:36.191 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:36.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:36.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:36.195 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:03:36.195 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:03:36.296 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:36.325 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:36.385 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:36.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:36.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:36.491 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:36.564 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:36 smithi008 ceph-mon[184418]: pgmap v1579: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 694 op/s; 13987/213528 objects degraded (6.550%); 2353/213528 objects misplaced (1.102%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:03:36.565 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:36 smithi008 ceph-mon[191227]: pgmap v1579: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 694 op/s; 13987/213528 objects degraded (6.550%); 2353/213528 objects misplaced (1.102%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:03:36.567 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:03:36.582 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:36.770 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:36.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:36.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:36.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:36 smithi191 ceph-mon[161669]: pgmap v1579: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 694 op/s; 13987/213528 objects degraded (6.550%); 2353/213528 objects misplaced (1.102%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:03:36.878 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:36.904 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:36.947 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:36.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:36.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:37.052 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:37.078 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:37.121 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:37.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:37.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:37.186 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 64 16 75114 75098 73.2975 30.3125 0.0128155 0.0136371 2023-10-09T06:03:37.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:37.262 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:37.336 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:37.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:37.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:37.441 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:37.477 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:37.536 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:37.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:37.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:37.643 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:37.682 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:37.739 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:37.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:37.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:37.847 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:37.857 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:03:37.961 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:38.170 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:38.186 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 65 16 75840 75824 72.8681 45.375 0.019092 0.0137165 2023-10-09T06:03:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:38.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:38.247 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:38.289 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:38.381 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:38.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:38 smithi008 ceph-mon[184418]: pgmap v1580: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s; 13987/221217 objects degraded (6.323%); 2353/221217 objects misplaced (1.064%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:03:38.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13987/221217 objects degraded (6.323%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:38.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:38 smithi008 ceph-mon[191227]: pgmap v1580: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s; 13987/221217 objects degraded (6.323%); 2353/221217 objects misplaced (1.064%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:03:38.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13987/221217 objects degraded (6.323%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:38.565 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:38.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:38.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:38.672 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:38.764 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:38.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:38 smithi191 ceph-mon[161669]: pgmap v1580: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s; 13987/221217 objects degraded (6.323%); 2353/221217 objects misplaced (1.064%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:03:38.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13987/221217 objects degraded (6.323%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:39.191 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 66 16 76389 76373 72.2841 34.3125 0.00291351 0.0137797 2023-10-09T06:03:39.583 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:39.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:39.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:39.690 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:39.776 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:39 smithi008 ceph-mon[184418]: pgmap v1581: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s; 13987/224388 objects degraded (6.233%); 2024/224388 objects misplaced (0.902%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:03:39.778 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:39 smithi008 ceph-mon[191227]: pgmap v1581: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s; 13987/224388 objects degraded (6.233%); 2024/224388 objects misplaced (0.902%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:03:39.795 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:39.797 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:39.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:39 smithi191 ceph-mon[161669]: pgmap v1581: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s; 13987/224388 objects degraded (6.233%); 2024/224388 objects misplaced (0.902%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:03:39.950 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:39.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:39.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:40.062 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:40.095 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:40.162 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:40.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:40.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:40.186 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 67 16 77040 77024 71.8126 40.6875 0.0102763 0.0139201 2023-10-09T06:03:40.273 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:40.305 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:40.371 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:40.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:40.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:40.478 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:40.574 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:40.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:03:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:03:40] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.33.4" 2023-10-09T06:03:41.186 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 68 16 77670 77654 71.3357 39.375 0.00823072 0.0140121 2023-10-09T06:03:41.769 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:41.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:41.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:41.853 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:41 smithi008 ceph-mon[184418]: pgmap v1582: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s; 13987/229095 objects degraded (6.105%); 2024/229095 objects misplaced (0.883%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:03:41.899 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:41.933 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:41.986 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:41.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:41.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:42.072 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:42.092 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:42.181 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:41 smithi008 ceph-mon[191227]: pgmap v1582: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s; 13987/229095 objects degraded (6.105%); 2024/229095 objects misplaced (0.883%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:03:42.191 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 69 16 78349 78333 70.9128 42.4375 0.0181603 0.0140946 2023-10-09T06:03:42.193 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:42.217 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:03:42.265 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:03:42.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:41 smithi191 ceph-mon[161669]: pgmap v1582: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s; 13987/229095 objects degraded (6.105%); 2024/229095 objects misplaced (0.883%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:03:42.317 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:03:42.377 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:03:42.385 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:42.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:42.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:42.429 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:03:42.470 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:03:42.495 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:42.505 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:03:42.545 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:03:42.557 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:03:42.557 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:03:42.557 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:03:42.557 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:03:42.557 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:03:42.557 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:03:42.557 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:03:42.557 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:03:42.597 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:42.695 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:42.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:42.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:42.810 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:42.855 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:42.912 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:42.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:42.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:43.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:43.047 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:43.088 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:43.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:43.191 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 70 16 78952 78936 70.4383 37.6875 0.012577 0.0141837 2023-10-09T06:03:43.194 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:43.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3746853558' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:43.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3746853558' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:43.228 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3746853558' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:43.230 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:43.305 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:43.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:43.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:43.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:43.442 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:43.502 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:43.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:43.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:43.611 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:43.690 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:43 smithi008 ceph-mon[184418]: pgmap v1583: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s; 13987/232536 objects degraded (6.015%); 2024/232536 objects misplaced (0.870%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:03:43.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13987/232536 objects degraded (6.015%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:43.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:43 smithi008 ceph-mon[191227]: pgmap v1583: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s; 13987/232536 objects degraded (6.015%); 2024/232536 objects misplaced (0.870%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:03:43.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13987/232536 objects degraded (6.015%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:44.055 INFO:teuthology.orchestra.run.smithi008.stdout:moved 2 / 267 (0.749064%) 2023-10-09T06:03:44.055 INFO:teuthology.orchestra.run.smithi008.stdout:avg 38.1429 2023-10-09T06:03:44.056 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 4.08581 -> 4.91146 (expected baseline 5.71786) 2023-10-09T06:03:44.056 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.2 with 33 -> 32 pgs (0.865169 -> 0.838951 * mean) 2023-10-09T06:03:44.056 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.3 with 45 -> 47 pgs (1.17978 -> 1.23221 * mean) 2023-10-09T06:03:44.056 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:03:44.056 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T06:03:44.056 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 0.05 2023-10-09T06:03:44.056 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T06:03:44.056 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0331 2023-10-09T06:03:44.056 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.3312 2023-10-09T06:03:44.056 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 weight 0.7855 -> 0.8355 2023-10-09T06:03:44.056 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 0.9360 -> 0.9860 2023-10-09T06:03:44.058 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T06:03:44.195 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 71 16 79553 79537 69.9754 37.5625 0.0189138 0.0142864 2023-10-09T06:03:44.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:43 smithi191 ceph-mon[161669]: pgmap v1583: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s; 13987/232536 objects degraded (6.015%); 2024/232536 objects misplaced (0.870%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:03:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13987/232536 objects degraded (6.015%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:44.464 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:03:44 smithi008 conmon[177229]: 2023-10-09T06:03:44.054+0000 7f020285b700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T06:03:44.812 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:44.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:44.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:44.936 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:44.973 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:45.048 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:45.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:45.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:45.156 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:45.184 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:45.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:45.185 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:44 smithi008 ceph-mon[184418]: from='client.37143 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:45.185 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:45.185 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:45.185 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:44 smithi008 ceph-mon[191227]: from='client.37143 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:45.188 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:45.191 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 72 16 80254 80238 69.6122 43.8125 0.0063628 0.0143424 2023-10-09T06:03:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:44 smithi191 ceph-mon[161669]: from='client.37143 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:45.853 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:45.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:45.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:45.909 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T06:03:45.909 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T06:03:45.961 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:45.989 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:46.040 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:46.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:46.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:46.149 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:46.191 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 73 16 80945 80929 69.2503 43.1875 0.0188308 0.0144333 2023-10-09T06:03:46.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:45 smithi008 ceph-mon[184418]: pgmap v1584: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 37 MiB/s wr, 592 op/s; 13987/233688 objects degraded (5.985%); 1747/233688 objects misplaced (0.748%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:03:46.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:45 smithi008 ceph-mon[191227]: pgmap v1584: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 37 MiB/s wr, 592 op/s; 13987/233688 objects degraded (5.985%); 1747/233688 objects misplaced (0.748%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:03:46.284 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:46.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:45 smithi191 ceph-mon[161669]: pgmap v1584: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 37 MiB/s wr, 592 op/s; 13987/233688 objects degraded (5.985%); 1747/233688 objects misplaced (0.748%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:03:46.418 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:46.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:46.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:46.529 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:46.558 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:46.618 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:46.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:46.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:46.725 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:46.815 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:47.035 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:03:47.076 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1554764897' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:47.077 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1554764897' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:47.191 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 74 16 81646 81630 68.9067 43.8125 0.0160404 0.0145067 2023-10-09T06:03:47.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1554764897' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:47.467 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:03:47.518 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:47.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:47.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:47.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:47.670 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:47.715 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:47.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:47.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:47.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:47.846 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:47.886 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:47.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:47.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:47.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:48.032 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:48.089 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:48.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:48.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:48.191 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 75 16 82219 82203 68.4656 35.8125 0.0138808 0.0145984 2023-10-09T06:03:48.196 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:48.273 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[184418]: pgmap v1585: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 13987/240345 objects degraded (5.820%); 1747/240345 objects misplaced (0.727%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:03:48.273 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:48.274 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[184418]: failed to encode map e526 with expected crc 2023-10-09T06:03:48.274 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1554764897' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:03:48.274 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[184418]: osdmap e526: 8 total, 8 up, 7 in 2023-10-09T06:03:48.274 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[184418]: failed to encode map e526 with expected crc 2023-10-09T06:03:48.278 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[191227]: pgmap v1585: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 13987/240345 objects degraded (5.820%); 1747/240345 objects misplaced (0.727%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:03:48.278 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:48.278 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[191227]: failed to encode map e526 with expected crc 2023-10-09T06:03:48.278 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1554764897' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:03:48.279 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[191227]: osdmap e526: 8 total, 8 up, 7 in 2023-10-09T06:03:48.279 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:47 smithi008 ceph-mon[191227]: failed to encode map e526 with expected crc 2023-10-09T06:03:48.285 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:48.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:47 smithi191 ceph-mon[161669]: pgmap v1585: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 13987/240345 objects degraded (5.820%); 1747/240345 objects misplaced (0.727%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:03:48.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:47 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:47 smithi191 ceph-mon[161669]: failed to encode map e526 with expected crc 2023-10-09T06:03:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1554764897' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:03:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:47 smithi191 ceph-mon[161669]: osdmap e526: 8 total, 8 up, 7 in 2023-10-09T06:03:48.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:47 smithi191 ceph-mon[161669]: failed to encode map e526 with expected crc 2023-10-09T06:03:48.541 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:48.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:48.645 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:48.721 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:48.806 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:48.854 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:48.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:48.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:48.894 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:48.974 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:49.004 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:49.030 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:03:49.031 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T06:03:49.031 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T06:03:49.031 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T06:03:49.031 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.031 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:56930 9.4 9:208f0955:::benchmark_data_smithi008_502820_object56929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T06:03:49.031 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:04.180242+0000", 2023-10-09T06:03:49.031 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 584.841187676, 2023-10-09T06:03:49.031 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99321452499999996, 2023-10-09T06:03:49.031 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.031 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.031 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.032 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.032 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.032 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56930 2023-10-09T06:03:49.032 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.032 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.032 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.032 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180242+0000", 2023-10-09T06:03:49.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.032 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.032 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.033 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180242+0000", 2023-10-09T06:03:49.033 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.033 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180242+0000", 2023-10-09T06:03:49.033 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180263+0000", 2023-10-09T06:03:49.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1599000000000001e-05 2023-10-09T06:03:49.034 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.034 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.034 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180270+0000", 2023-10-09T06:03:49.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8889999999999996e-06 2023-10-09T06:03:49.034 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.034 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.034 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.180276+0000", 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5199999999999997e-06 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.181919+0000", 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016433559999999999 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.181931+0000", 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1698999999999999e-05 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.166507+0000", 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.98457661200000002 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.167509+0000", 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010012490000000001 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.167615+0000", 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000106848 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173305+0000", 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0056890309999999998 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173358+0000", 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3087999999999997e-05 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173383+0000", 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5792e-05 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173420+0000", 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7038e-05 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173457+0000", 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6229000000000002e-05 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:56935 9.4 9:2e5431a0:::benchmark_data_smithi008_502820_object56934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:04.187552+0000", 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 584.83387792899998, 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99016123, 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56935 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187552+0000", 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187552+0000", 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187551+0000", 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187573+0000", 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2022e-05 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187583+0000", 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0207000000000001e-05 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.187590+0000", 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2949999999999996e-06 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.189845+0000", 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002255047 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.189856+0000", 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1161999999999999e-05 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.169499+0000", 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97964239200000003 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.169675+0000", 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017621499999999999 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.169851+0000", 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017612199999999999 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173502+0000", 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036512459999999999 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173540+0000", 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7245e-05 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177611+0000", 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040713750000000003 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177670+0000", 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9184000000000001e-05 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177713+0000", 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2997000000000002e-05 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:56945 9.4 9:25604571:::benchmark_data_smithi008_502820_object56944:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:04.198697+0000", 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 584.82273249699995, 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97919550899999996, 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56945 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198697+0000", 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198697+0000", 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198697+0000", 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198720+0000", 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3595e-05 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198728+0000", 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9619999999999993e-06 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198738+0000", 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7280000000000007e-06 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198805+0000", 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7163999999999996e-05 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.198816+0000", 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1018e-05 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.171495+0000", 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97267913900000003 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.171653+0000", 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000157152 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.171818+0000", 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016535700000000001 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173547+0000", 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017288850000000001 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173574+0000", 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.756e-05 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177808+0000", 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0042333329999999997 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177854+0000", 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6777000000000003e-05 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177893+0000", 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8414000000000002e-05 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:56948 9.4 9:2ebab9cb:::benchmark_data_smithi008_502820_object56947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:04.209232+0000", 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 584.81219806800004, 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97360594499999997, 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56948 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.209232+0000", 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.209232+0000", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.209230+0000", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.209264+0000", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3652999999999998e-05 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.211202+0000", 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019386049999999999 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.211216+0000", 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3991e-05 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.211358+0000", 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000142161 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.211371+0000", 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2241e-05 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173589+0000", 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.96221823699999998 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173722+0000", 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000133311 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.173878+0000", 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015562099999999999 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.177911+0000", 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040336369999999996 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182682+0000", 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004770364 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182729+0000", 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6730000000000002e-05 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182791+0000", 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2329999999999994e-05 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182838+0000", 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6848999999999998e-05 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:56965 9.4 9:22816604:::benchmark_data_smithi008_502820_object56964:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:04.236294+0000", 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 584.78513540300003, 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.94675013200000002, 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 56965 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236294+0000", 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236294+0000", 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236293+0000", 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236321+0000", 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8558999999999999e-05 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236338+0000", 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7093000000000001e-05 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236348+0000", 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9380000000000004e-06 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236420+0000", 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1601999999999996e-05 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:04.236434+0000", 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3636e-05 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.175615+0000", 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93918106800000001 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.175753+0000", 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000138543 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.175954+0000", 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020097299999999999 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.180332+0000", 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0043779500000000002 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182893+0000", 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025605839999999999 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.182958+0000", 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5178000000000005e-05 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.183006+0000", 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7645000000000002e-05 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:05.183045+0000", 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8965999999999999e-05 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77620 9.14 9:2ff0425c:::benchmark_data_smithi008_502820_object77619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:36.817602+0000", 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 552.20382759400002, 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.935633622, 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77620 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817602+0000", 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817602+0000", 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817600+0000", 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817634+0000", 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4471000000000003e-05 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817660+0000", 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5888e-05 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817673+0000", 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2499e-05 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.912688+0000", 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.095015316000000002 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.912700+0000", 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.241e-05 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.749377+0000", 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83667673600000003 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for degraded object", 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.749824+0000", 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00044703499999999998 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.753124+0000", 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032999209999999999 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.753236+0000", 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000111821 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117310 9.f 9:fadc0f2a:::benchmark_data_smithi008_502820_object33137:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:57.240003+0000", 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 351.78142661599998, 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.29976151, 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117310 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240003+0000", 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240003+0000", 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240001+0000", 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240012+0000", 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0919e-05 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3413000000000001e-05 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240048+0000", 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2472e-05 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240128+0000", 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9445999999999998e-05 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op must wait for map", 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240136+0000", 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4519999999999997e-06 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.256039+0000", 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015902785999999999 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.256052+0000", 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3036e-05 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.412425+0000", 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1563727260000001 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.415543+0000", 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031188029999999999 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.416007+0000", 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00046389599999999999 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.505488+0000", 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.089480572999999994 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.537229+0000", 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.031741577 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.537272+0000", 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3052e-05 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.537346+0000", 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3397000000000002e-05 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.539765+0000", 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00241888 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117318 9.f 9:f9d15496:::benchmark_data_smithi008_502820_object33145:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:57.240063+0000", 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 351.781366601, 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3088910030000001, 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117318 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240063+0000", 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240063+0000", 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240063+0000", 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240066+0000", 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2619999999999999e-06 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240073+0000", 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7610000000000002e-06 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240078+0000", 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6710000000000004e-06 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240196+0000", 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011796899999999999 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting_for_map not empty", 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240203+0000", 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4620000000000001e-06 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.256459+0000", 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016256135000000001 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.256468+0000", 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2180000000000005e-06 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.539808+0000", 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.283339472 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.539927+0000", 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011948 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.540039+0000", 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000112358 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.542135+0000", 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020953310000000002 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.543074+0000", 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00093911000000000005 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.546321+0000", 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032469949999999999 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.546365+0000", 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3677000000000002e-05 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.548954+0000", 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025896270000000002 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135296 9.15 9:ae091abe:::benchmark_data_smithi008_502820_object51123:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e477)", 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:06.289091+0000", 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 222.732339295, 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 11.558454653, 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135296 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289091+0000", 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289091+0000", 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289088+0000", 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289100+0000", 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.208e-05 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289120+0000", 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9913999999999999e-05 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289135+0000", 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4450000000000001e-05 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289222+0000", 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7608999999999994e-05 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289238+0000", 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6174000000000001e-05 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.794145+0000", 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 11.504906586000001 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.796652+0000", 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025069179999999999 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.801845+0000", 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051928349999999998 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.814352+0000", 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012507364999999999 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.816332+0000", 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019798580000000001 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.841671+0000", 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.025339265 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.842816+0000", 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011452649999999999 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.847545+0000", 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0047286979999999999 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135428 9.5 9:a3b356e8:::benchmark_data_smithi008_502820_object51255:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.442799+0000", 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 211.57863069499999, 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.46811048300000002, 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135428 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442799+0000", 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442799+0000", 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442798+0000", 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442805+0000", 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8600000000000004e-06 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442821+0000", 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6042e-05 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442829+0000", 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1389999999999995e-06 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.443441+0000", 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000612033 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.443654+0000", 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000213434 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.832185+0000", 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.38853078499999999 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.834228+0000", 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020425539999999998 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.837657+0000", 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034290000000000002 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.848361+0000", 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010704959999999999 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.866209+0000", 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017847569000000001 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.905664+0000", 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.039454522999999998 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.906752+0000", 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001088807 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.910910+0000", 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041572320000000003 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135438 9.15 9:ac71c127:::benchmark_data_smithi008_502820_object51265:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.467551+0000", 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 211.55387876899999, 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.424989799, 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135438 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467551+0000", 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467551+0000", 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467550+0000", 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467555+0000", 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3240000000000002e-06 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467566+0000", 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0946e-05 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467572+0000", 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6130000000000003e-06 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.468100+0000", 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052845400000000003 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.468325+0000", 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00022488200000000001 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.816574+0000", 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.34824860000000002 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.819442+0000", 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002867882 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.824069+0000", 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0046273820000000002 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.829908+0000", 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058389480000000001 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.847904+0000", 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017996504 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.887093+0000", 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.039188134999999999 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.888018+0000", 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00092505699999999996 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.892541+0000", 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045231919999999997 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135455 9.5 9:a744df51:::benchmark_data_smithi008_502820_object51282:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.555805+0000", 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 211.46562493100001, 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.36574448700000001, 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135455 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555805+0000", 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555805+0000", 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555804+0000", 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555809+0000", 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7189999999999998e-06 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555822+0000", 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2437e-05 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555828+0000", 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9009999999999999e-06 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.556356+0000", 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052816600000000005 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.556533+0000", 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017680899999999999 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.848754+0000", 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29222136599999998 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.850864+0000", 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021101840000000002 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.854254+0000", 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003389909 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.866794+0000", 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012540112000000001 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.886075+0000", 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019280761 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.916939+0000", 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030863607000000001 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.917782+0000", 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00084381300000000001 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.921549+0000", 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003766892 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135464 9.5 9:a4c9db9f:::benchmark_data_smithi008_502820_object51291:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.581489+0000", 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 211.43994130900001, 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35698092199999998, 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135464 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581489+0000", 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581489+0000", 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581487+0000", 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581494+0000", 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8299999999999998e-06 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581506+0000", 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2165e-05 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581513+0000", 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5320000000000003e-06 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.582012+0000", 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00049926799999999998 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.582190+0000", 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017807399999999999 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.857269+0000", 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27507849099999998 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.859832+0000", 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025629759999999998 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.863254+0000", 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034221020000000002 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.867308+0000", 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040540230000000003 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.886247+0000", 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018939448000000001 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.933550+0000", 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.047303167 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.934616+0000", 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001065489 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.938469+0000", 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038536320000000001 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135469 9.5 9:a4af4507:::benchmark_data_smithi008_502820_object51296:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.593956+0000", 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 211.427473459, 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35374935200000002, 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135469 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593956+0000", 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593956+0000", 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593955+0000", 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593961+0000", 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1500000000000004e-06 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593973+0000", 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2228e-05 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593979+0000", 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0220000000000001e-06 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.594482+0000", 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00050292100000000003 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.594661+0000", 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017859899999999999 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.867482+0000", 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27282152999999998 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.869478+0000", 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019958799999999998 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.872688+0000", 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032103909999999999 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.886833+0000", 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014144818 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.893123+0000", 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0062893610000000003 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.939942+0000", 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.046819500999999999 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.941017+0000", 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001075211 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.947706+0000", 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066884229999999998 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135471 9.5 9:a1af57c1:::benchmark_data_smithi008_502820_object51298:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.615863+0000", 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 211.40556726700001, 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33742538799999999, 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135471 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615863+0000", 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615863+0000", 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615861+0000", 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615868+0000", 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7599999999999997e-06 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615880+0000", 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2174999999999999e-05 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615888+0000", 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.525e-06 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.876726+0000", 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26083806399999998 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.879008+0000", 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022821320000000001 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.882350+0000", 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033419690000000002 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.887338+0000", 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049886599999999998 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.893331+0000", 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0059928400000000001 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.948636+0000", 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.055305193000000002 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.949599+0000", 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00096221399999999997 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.953288+0000", 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036893070000000002 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:153220 9.15 9:ac0b5463:::benchmark_data_smithi008_502820_object69047:head [delete] snapc 0=[] ondisk+write+known_if_redirected e495)", 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:01:28.845935+0000", 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 140.175494652, 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33700997300000002, 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 153220 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.845935+0000", 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.845935+0000", 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.845934+0000", 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.845940+0000", 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4190000000000001e-06 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.845948+0000", 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5799999999999992e-06 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.846112+0000", 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016346700000000001 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.846185+0000", 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3405999999999995e-05 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.846286+0000", 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010111300000000001 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.846679+0000", 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00039266000000000003 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.848724+0000", 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020446890000000001 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.849533+0000", 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00080934699999999997 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:29.182412+0000", 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33287920199999999 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:29.182471+0000", 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8455000000000002e-05 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:29.182945+0000", 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000474625 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:166957 9.8 9.f7ad5128 (undecoded) ondisk+write+known_if_redirected e504)", 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:02:07.989499+0000", 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 101.031930328, 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33934386999999999, 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 166957 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989499+0000", 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989499+0000", 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989497+0000", 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989508+0000", 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0900999999999999e-05 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989533+0000", 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4547999999999999e-05 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989546+0000", 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3089e-05 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.994259+0000", 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0047132529999999997 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.994272+0000", 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3169999999999999e-05 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:08.328828+0000", 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33455541799999999 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:08.328843+0000", 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5511000000000001e-05 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:63139 10.0 10:04e05b18:::benchmark_data_smithi008_634586_object63138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:19.662902+0000", 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 29.35852766, 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.32210230099999998, 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 63139 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662902+0000", 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662902+0000", 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662902+0000", 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.178 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662925+0000", 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3309e-05 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662932+0000", 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8290000000000002e-06 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662936+0000", 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2729999999999996e-06 2023-10-09T06:03:49.179 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.180 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.180 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.969240+0000", 2023-10-09T06:03:49.180 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.306303885 2023-10-09T06:03:49.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.180 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.180 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.180 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.969387+0000", 2023-10-09T06:03:49.180 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000147045 2023-10-09T06:03:49.180 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.969516+0000", 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012888400000000001 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.977219+0000", 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0077028369999999997 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.181 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.982574+0000", 2023-10-09T06:03:49.182 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005355355 2023-10-09T06:03:49.182 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.182 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.182 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.182 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.982632+0000", 2023-10-09T06:03:49.182 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8121000000000001e-05 2023-10-09T06:03:49.182 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.182 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.182 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.182 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.982689+0000", 2023-10-09T06:03:49.183 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6557000000000001e-05 2023-10-09T06:03:49.183 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.183 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.183 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.985004+0000", 2023-10-09T06:03:49.183 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002315774 2023-10-09T06:03:49.183 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.183 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.183 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.183 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.183 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:63140 10.0 10:083e2f66:::benchmark_data_smithi008_634586_object63139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:19.662941+0000", 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 29.358488976, 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.32458843900000001, 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 63140 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.184 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662941+0000", 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662941+0000", 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.185 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662941+0000", 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662957+0000", 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6189999999999999e-05 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662961+0000", 2023-10-09T06:03:49.186 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3409999999999996e-06 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662964+0000", 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2499999999999998e-06 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.971739+0000", 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30877492499999998 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.187 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.971940+0000", 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000200454 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.972155+0000", 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00021504299999999999 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.188 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.977263+0000", 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051080709999999996 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.985094+0000", 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078314360000000006 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.985107+0000", 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3104e-05 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.189 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.985194+0000", 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7143000000000002e-05 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.987529+0000", 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023347950000000002 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.190 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "MOSDPGPushReply(10.0 526/520 [PushReplyOp(10:0a24d2ca:::benchmark_data_smithi008_634586_object32593:head)])", 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:47.693776+0000", 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 1.327653846, 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23455668599999999, 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.693776+0000", 2023-10-09T06:03:49.191 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.693776+0000", 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.693775+0000", 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.192 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.693783+0000", 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2670000000000006e-06 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.693793+0000", 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9520000000000006e-06 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.693800+0000", 2023-10-09T06:03:49.193 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1179999999999997e-06 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.693834+0000", 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4802000000000003e-05 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.928333+0000", 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23449831500000001 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.194 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.195 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:03:49.195 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:03:49.195 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:03:49.195 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 76 16 82864 82848 68.0952 40.3125 0.0262328 0.0146787 2023-10-09T06:03:49.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:49.196 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:49.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:49.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:49 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13987/240345 objects degraded (5.820%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:49.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:49 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13987/240345 objects degraded (5.820%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:49.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:49.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:49 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13987/240345 objects degraded (5.820%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:49.335 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:49.391 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:49.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:49.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:49.497 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:49.544 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:49.613 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:49.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:49.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:49.729 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:49.814 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:50.191 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 77 16 83527 83511 67.7491 41.4375 0.0040696 0.0147368 2023-10-09T06:03:50.483 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:03:50.516 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:50.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:50.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:50.570 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:50 smithi008 ceph-mon[184418]: pgmap v1587: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 37 MiB/s wr, 593 op/s; 13987/242190 objects degraded (5.775%); 1747/242190 objects misplaced (0.721%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:03:50.571 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:03:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:03:50] "GET /metrics HTTP/1.1" 200 36647 "" "Prometheus/2.33.4" 2023-10-09T06:03:50.571 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:50 smithi008 ceph-mon[191227]: pgmap v1587: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 37 MiB/s wr, 593 op/s; 13987/242190 objects degraded (5.775%); 1747/242190 objects misplaced (0.721%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:03:50.628 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:50.797 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:03:50.797 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:50.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:50 smithi191 ceph-mon[161669]: pgmap v1587: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 37 MiB/s wr, 593 op/s; 13987/242190 objects degraded (5.775%); 1747/242190 objects misplaced (0.721%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:03:51.194 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 78 16 84155 84139 67.3839 39.25 0.0302986 0.0148347 2023-10-09T06:03:51.516 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:51.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:51.624 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:51.658 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:51.721 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:51.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:51.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:51.832 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:51.913 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:52.193 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 79 16 84848 84832 67.0781 43.3125 0.0253669 0.0149017 2023-10-09T06:03:52.470 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:52.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:52.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:52.577 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:52.607 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:52 smithi008 ceph-mon[184418]: pgmap v1588: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s; 13987/248343 objects degraded (5.632%); 1418/248343 objects misplaced (0.571%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:03:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:52 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:52.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:52 smithi008 ceph-mon[191227]: pgmap v1588: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s; 13987/248343 objects degraded (5.632%); 1418/248343 objects misplaced (0.571%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:03:52.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:52 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:52.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:52 smithi191 ceph-mon[161669]: pgmap v1588: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s; 13987/248343 objects degraded (5.632%); 1418/248343 objects misplaced (0.571%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:03:52.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:52 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:52.971 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:52.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:52.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:53.078 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:53.107 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:53.175 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:53.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:53.193 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T06:03:53.194158+0000 min lat: 0.00115919 max lat: 2.23699 avg lat: 0.0149859 2023-10-09T06:03:53.193 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:03:53.193 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 80 16 85427 85411 66.6921 36.1875 0.0094203 0.0149859 2023-10-09T06:03:53.261 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:53.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:53.320 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:53.372 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:53.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:53.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:53.382 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:53.479 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:53.581 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:53.691 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13987/248343 objects degraded (5.632%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:53.693 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13987/248343 objects degraded (5.632%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13987/248343 objects degraded (5.632%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:53.825 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:53.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:53.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:53.938 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:53.966 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:54.194 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 81 16 86036 86020 66.338 38.0625 0.0107368 0.0150669 2023-10-09T06:03:54.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:54 smithi008 ceph-mon[184418]: pgmap v1589: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 644 op/s; 13987/251886 objects degraded (5.553%); 1418/251886 objects misplaced (0.563%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:03:54.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:54 smithi008 ceph-mon[191227]: pgmap v1589: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 644 op/s; 13987/251886 objects degraded (5.553%); 1418/251886 objects misplaced (0.563%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:03:54.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:54 smithi191 ceph-mon[161669]: pgmap v1589: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 644 op/s; 13987/251886 objects degraded (5.553%); 1418/251886 objects misplaced (0.563%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:03:55.199 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 82 16 86692 86676 66.0258 41 0.022573 0.0151398 2023-10-09T06:03:55.463 INFO:teuthology.orchestra.run.smithi008.stdout:moved 27 / 267 (10.1124%) 2023-10-09T06:03:55.464 INFO:teuthology.orchestra.run.smithi008.stdout:avg 38.1429 2023-10-09T06:03:55.464 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 4.08581 -> 8.49249 (expected baseline 5.71786) 2023-10-09T06:03:55.464 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.2 with 33 -> 33 pgs (0.865169 -> 0.865169 * mean) 2023-10-09T06:03:55.464 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.3 with 45 -> 45 pgs (1.17978 -> 1.17978 * mean) 2023-10-09T06:03:55.464 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:03:55.464 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T06:03:55.464 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 3 2023-10-09T06:03:55.464 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T06:03:55.465 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0351 2023-10-09T06:03:55.465 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0386 2023-10-09T06:03:55.465 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0 weight 0.7855 -> 0.5845 2023-10-09T06:03:55.465 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 weight 0.7412 -> 0.5592 2023-10-09T06:03:55.465 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1 weight 0.7855 -> 1.0000 2023-10-09T06:03:55.465 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3 weight 0.9360 -> 1.0000 2023-10-09T06:03:55.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:55 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'38308','1':'65536','3':'65536','4':'36648'}"}]: dispatch 2023-10-09T06:03:55.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:55 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'38308','1':'65536','3':'65536','4':'36648'}"}]: dispatch 2023-10-09T06:03:55.762 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:03:55.762 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:03:55.763 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:03:55.763 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:03:55.763 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:03:55.775 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:03:55.793 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:55.796 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:03:55.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:55.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:55.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:55 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'38308','1':'65536','3':'65536','4':'36648'}"}]: dispatch 2023-10-09T06:03:55.818 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:03:55.820 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.820 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.820 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.820 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.821 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.821 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.821 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.821 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.897 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:55.927 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:55.971 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:55.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:55.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:56.075 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:56.107 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:56.198 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 83 16 86859 86843 65.3563 10.4375 0.00396935 0.015169 2023-10-09T06:03:56.592 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:56.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:56.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:56.698 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[184418]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 646 op/s; 13987/253092 objects degraded (5.526%); 1035/253092 objects misplaced (0.409%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T06:03:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[184418]: from='client.47111 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[184418]: failed to encode map e527 with expected crc 2023-10-09T06:03:56.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[184418]: failed to encode map e527 with expected crc 2023-10-09T06:03:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[184418]: failed to encode map e527 with expected crc 2023-10-09T06:03:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'38308','1':'65536','3':'65536','4':'36648'}"}]': finished 2023-10-09T06:03:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[184418]: osdmap e527: 8 total, 8 up, 7 in 2023-10-09T06:03:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3204056738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:56.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[191227]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 646 op/s; 13987/253092 objects degraded (5.526%); 1035/253092 objects misplaced (0.409%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T06:03:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[191227]: from='client.47111 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[191227]: failed to encode map e527 with expected crc 2023-10-09T06:03:56.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[191227]: failed to encode map e527 with expected crc 2023-10-09T06:03:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[191227]: failed to encode map e527 with expected crc 2023-10-09T06:03:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'38308','1':'65536','3':'65536','4':'36648'}"}]': finished 2023-10-09T06:03:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[191227]: osdmap e527: 8 total, 8 up, 7 in 2023-10-09T06:03:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3204056738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:56.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:56.721 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:56 smithi191 ceph-mon[161669]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 40 MiB/s wr, 646 op/s; 13987/253092 objects degraded (5.526%); 1035/253092 objects misplaced (0.409%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T06:03:56.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:56 smithi191 ceph-mon[161669]: from='client.47111 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:56.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:56 smithi191 ceph-mon[161669]: failed to encode map e527 with expected crc 2023-10-09T06:03:56.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:56 smithi191 ceph-mon[161669]: failed to encode map e527 with expected crc 2023-10-09T06:03:56.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:56 smithi191 ceph-mon[161669]: failed to encode map e527 with expected crc 2023-10-09T06:03:56.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'38308','1':'65536','3':'65536','4':'36648'}"}]': finished 2023-10-09T06:03:56.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:56 smithi191 ceph-mon[161669]: osdmap e527: 8 total, 8 up, 7 in 2023-10-09T06:03:56.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:56 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3204056738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:56.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:56.722 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:03:56.724 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:56.771 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:56.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:56.876 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:56.907 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:57.198 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 84 16 86859 86843 64.5787 0 - 0.015169 2023-10-09T06:03:57.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:57 smithi191 ceph-mon[161669]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:57 smithi191 ceph-mon[161669]: osdmap e528: 8 total, 8 up, 7 in 2023-10-09T06:03:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:57 smithi191 ceph-mon[161669]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:57 smithi191 ceph-mon[161669]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:57 smithi191 ceph-mon[161669]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:57 smithi191 ceph-mon[161669]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:57 smithi191 ceph-mon[161669]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[184418]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[184418]: osdmap e528: 8 total, 8 up, 7 in 2023-10-09T06:03:57.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[184418]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[184418]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[184418]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[184418]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[184418]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[191227]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[191227]: osdmap e528: 8 total, 8 up, 7 in 2023-10-09T06:03:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[191227]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[191227]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[191227]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[191227]: failed to encode map e528 with expected crc 2023-10-09T06:03:57.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:57 smithi008 ceph-mon[191227]: failed to encode map e528 with expected crc 2023-10-09T06:03:58.084 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:58.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:58.188 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:58.198 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 85 16 87035 87019 63.9486 5.5 0.0125434 0.015553 2023-10-09T06:03:58.250 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:58.324 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:58.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:58.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:58.428 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:58.465 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:58.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:58 smithi008 ceph-mon[184418]: pgmap v1593: 89 pgs: 21 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 13987/259284 objects degraded (5.394%); 1035/259284 objects misplaced (0.399%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T06:03:58.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:58 smithi008 ceph-mon[184418]: failed to encode map e529 with expected crc 2023-10-09T06:03:58.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:58 smithi008 ceph-mon[184418]: osdmap e529: 8 total, 8 up, 7 in 2023-10-09T06:03:58.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:58 smithi008 ceph-mon[184418]: failed to encode map e529 with expected crc 2023-10-09T06:03:58.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13987/259284 objects degraded (5.394%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:58.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:58 smithi008 ceph-mon[191227]: pgmap v1593: 89 pgs: 21 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 13987/259284 objects degraded (5.394%); 1035/259284 objects misplaced (0.399%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T06:03:58.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:58 smithi008 ceph-mon[191227]: failed to encode map e529 with expected crc 2023-10-09T06:03:58.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:58 smithi008 ceph-mon[191227]: osdmap e529: 8 total, 8 up, 7 in 2023-10-09T06:03:58.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:58 smithi008 ceph-mon[191227]: failed to encode map e529 with expected crc 2023-10-09T06:03:58.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13987/259284 objects degraded (5.394%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:58.741 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:58.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:58.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:58.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:58 smithi191 ceph-mon[161669]: pgmap v1593: 89 pgs: 21 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 13987/259284 objects degraded (5.394%); 1035/259284 objects misplaced (0.399%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T06:03:58.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:58 smithi191 ceph-mon[161669]: failed to encode map e529 with expected crc 2023-10-09T06:03:58.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:58 smithi191 ceph-mon[161669]: osdmap e529: 8 total, 8 up, 7 in 2023-10-09T06:03:58.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:58 smithi191 ceph-mon[161669]: failed to encode map e529 with expected crc 2023-10-09T06:03:58.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13987/259284 objects degraded (5.394%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:03:58.846 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:58.877 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:58.927 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:58.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:59.035 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:59.075 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:59.128 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:59.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:59.199 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 86 16 87162 87146 63.2976 7.9375 0.00372289 0.0155456 2023-10-09T06:03:59.232 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:59.260 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:59.308 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:59.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:59.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:59.444 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:59.494 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:59.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:59.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:59.601 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:59.632 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:00.199 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 87 16 87162 87146 62.5704 0 - 0.0155456 2023-10-09T06:04:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:59 smithi008 ceph-mon[184418]: pgmap v1595: 89 pgs: 21 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 28 MiB/s wr, 445 op/s; 10467/259902 objects degraded (4.027%); 4468/259902 objects misplaced (1.719%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T06:04:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:03:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:00.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:59 smithi008 ceph-mon[191227]: pgmap v1595: 89 pgs: 21 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 28 MiB/s wr, 445 op/s; 10467/259902 objects degraded (4.027%); 4468/259902 objects misplaced (1.719%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T06:04:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:03:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:00.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:59 smithi191 ceph-mon[161669]: pgmap v1595: 89 pgs: 21 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 28 MiB/s wr, 445 op/s; 10467/259902 objects degraded (4.027%); 4468/259902 objects misplaced (1.719%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T06:04:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:03:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:00.334 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:00.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:00.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:00.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:00.467 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:00.522 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:00.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:00.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:00.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:00.654 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:00.697 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:00.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:00.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:00.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:04:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:04:00] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.33.4" 2023-10-09T06:04:00.802 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:00.852 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:00.910 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:04:01.110 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:01.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:01.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:01.199 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 88 16 87162 87146 61.8597 0 - 0.0155456 2023-10-09T06:04:01.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:01.241 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:01.286 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:01.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:01.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:01.392 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:01.429 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:01.481 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:04:01.481 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:04:01.492 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:01.504 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:04:01.504 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:01.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:01.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:01 smithi191 ceph-mon[161669]: failed to encode map e530 with expected crc 2023-10-09T06:04:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:01 smithi191 ceph-mon[161669]: failed to encode map e530 with expected crc 2023-10-09T06:04:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:01 smithi191 ceph-mon[161669]: osdmap e530: 8 total, 8 up, 7 in 2023-10-09T06:04:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:01 smithi191 ceph-mon[161669]: failed to encode map e530 with expected crc 2023-10-09T06:04:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:01 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1872802090' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:01.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:01 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:01.612 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:01.642 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:01.688 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[184418]: failed to encode map e530 with expected crc 2023-10-09T06:04:01.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[184418]: failed to encode map e530 with expected crc 2023-10-09T06:04:01.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[184418]: osdmap e530: 8 total, 8 up, 7 in 2023-10-09T06:04:01.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[184418]: failed to encode map e530 with expected crc 2023-10-09T06:04:01.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1872802090' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:01.689 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:01.689 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[191227]: failed to encode map e530 with expected crc 2023-10-09T06:04:01.689 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[191227]: failed to encode map e530 with expected crc 2023-10-09T06:04:01.689 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[191227]: osdmap e530: 8 total, 8 up, 7 in 2023-10-09T06:04:01.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[191227]: failed to encode map e530 with expected crc 2023-10-09T06:04:01.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1872802090' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:01.690 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:01 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:02.199 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 89 16 87162 87146 61.165 0 - 0.0155456 2023-10-09T06:04:02.217 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:04:02.217 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:04:02.218 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:04:02.218 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T06:04:02.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T06:04:02.524 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:02 smithi191 ceph-mon[161669]: pgmap v1597: 89 pgs: 15 peering, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 30 MiB/s wr, 487 op/s; 15871/260910 objects degraded (6.083%); 3646/260910 objects misplaced (1.397%); 11 MiB/s, 172 objects/s recovering 2023-10-09T06:04:02.524 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:02 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:02.524 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:02 smithi191 ceph-mon[161669]: failed to encode map e531 with expected crc 2023-10-09T06:04:02.524 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:02 smithi191 ceph-mon[161669]: failed to encode map e531 with expected crc 2023-10-09T06:04:02.525 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:02 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:02.525 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:02 smithi191 ceph-mon[161669]: osdmap e531: 8 total, 8 up, 7 in 2023-10-09T06:04:02.525 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:02 smithi191 systemd[1]: Stopping Ceph osd.6 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:04:02.799 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:02 smithi191 conmon[119259]: debug 2023-10-09T06:04:02.522+0000 7f4756b34700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:04:02.800 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:02 smithi191 conmon[119259]: debug 2023-10-09T06:04:02.522+0000 7f4756b34700 -1 osd.6 531 *** Got signal Terminated *** 2023-10-09T06:04:02.800 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:02 smithi191 conmon[119259]: debug 2023-10-09T06:04:02.522+0000 7f4756b34700 -1 osd.6 531 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:04:02.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[191227]: pgmap v1597: 89 pgs: 15 peering, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 30 MiB/s wr, 487 op/s; 15871/260910 objects degraded (6.083%); 3646/260910 objects misplaced (1.397%); 11 MiB/s, 172 objects/s recovering 2023-10-09T06:04:02.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:02.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[191227]: failed to encode map e531 with expected crc 2023-10-09T06:04:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[191227]: failed to encode map e531 with expected crc 2023-10-09T06:04:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[191227]: osdmap e531: 8 total, 8 up, 7 in 2023-10-09T06:04:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[184418]: pgmap v1597: 89 pgs: 15 peering, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 30 MiB/s wr, 487 op/s; 15871/260910 objects degraded (6.083%); 3646/260910 objects misplaced (1.397%); 11 MiB/s, 172 objects/s recovering 2023-10-09T06:04:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[184418]: failed to encode map e531 with expected crc 2023-10-09T06:04:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[184418]: failed to encode map e531 with expected crc 2023-10-09T06:04:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:02.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:02 smithi008 ceph-mon[184418]: osdmap e531: 8 total, 8 up, 7 in 2023-10-09T06:04:03.199 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 90 16 87162 87146 60.4857 0 - 0.0155456 2023-10-09T06:04:03.601 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:03.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:03.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:03.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:03.736 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:03.789 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:04.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:03 smithi008 ceph-mon[184418]: osd.6 marked itself down and dead 2023-10-09T06:04:04.901 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:03 smithi008 ceph-mon[184418]: pgmap v1599: 89 pgs: 1 creating+peering, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 6.6 MiB/s wr, 105 op/s; 17779/261183 objects degraded (6.807%); 6303/261183 objects misplaced (2.413%); 15 MiB/s, 1 keys/s, 236 objects/s recovering 2023-10-09T06:04:04.901 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:03 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15871/260910 objects degraded (6.083%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:04:04.901 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:03 smithi191 ceph-mon[161669]: osd.6 marked itself down and dead 2023-10-09T06:04:04.901 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:03 smithi191 ceph-mon[161669]: pgmap v1599: 89 pgs: 1 creating+peering, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 6.6 MiB/s wr, 105 op/s; 17779/261183 objects degraded (6.807%); 6303/261183 objects misplaced (2.413%); 15 MiB/s, 1 keys/s, 236 objects/s recovering 2023-10-09T06:04:04.901 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:03 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15871/260910 objects degraded (6.083%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:04:04.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:03 smithi008 ceph-mon[191227]: osd.6 marked itself down and dead 2023-10-09T06:04:04.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:03 smithi008 ceph-mon[191227]: pgmap v1599: 89 pgs: 1 creating+peering, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 6.6 MiB/s wr, 105 op/s; 17779/261183 objects degraded (6.807%); 6303/261183 objects misplaced (2.413%); 15 MiB/s, 1 keys/s, 236 objects/s recovering 2023-10-09T06:04:04.902 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:03 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15871/260910 objects degraded (6.083%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:04:04.902 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 91 16 87162 87146 59.8213 0 - 0.0155456 2023-10-09T06:04:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:04.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:04.903 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:04 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:04:04.904 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:04 smithi191 ceph-mon[161669]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:04:04.904 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:04 smithi191 ceph-mon[161669]: failed to encode map e532 with expected crc 2023-10-09T06:04:04.904 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:04 smithi191 ceph-mon[161669]: failed to encode map e532 with expected crc 2023-10-09T06:04:04.904 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:04 smithi191 ceph-mon[161669]: osdmap e532: 8 total, 7 up, 7 in 2023-10-09T06:04:04.904 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:04:04.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:04 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:04:04.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:04 smithi008 ceph-mon[184418]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:04:04.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:04 smithi008 ceph-mon[184418]: failed to encode map e532 with expected crc 2023-10-09T06:04:04.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:04 smithi008 ceph-mon[184418]: failed to encode map e532 with expected crc 2023-10-09T06:04:04.906 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:04 smithi008 ceph-mon[184418]: osdmap e532: 8 total, 7 up, 7 in 2023-10-09T06:04:04.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:04 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:04:04.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:04 smithi008 ceph-mon[191227]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:04:04.906 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:04 smithi008 ceph-mon[191227]: failed to encode map e532 with expected crc 2023-10-09T06:04:04.999 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:04 smithi008 ceph-mon[191227]: failed to encode map e532 with expected crc 2023-10-09T06:04:04.999 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:04 smithi008 ceph-mon[191227]: osdmap e532: 8 total, 7 up, 7 in 2023-10-09T06:04:05.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:05.084 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:05.240 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 92 16 87162 87146 59.1714 0 - 0.0155456 2023-10-09T06:04:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:05.338 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:05.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:05.440 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:05.502 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:05 smithi191 ceph-mon[161669]: failed to encode map e533 with expected crc 2023-10-09T06:04:05.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:05 smithi191 ceph-mon[161669]: failed to encode map e533 with expected crc 2023-10-09T06:04:05.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:05 smithi191 ceph-mon[161669]: osdmap e533: 8 total, 7 up, 7 in 2023-10-09T06:04:05.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:05 smithi191 ceph-mon[161669]: failed to encode map e533 with expected crc 2023-10-09T06:04:05.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:05 smithi191 ceph-mon[161669]: pgmap v1602: 89 pgs: 4 peering, 1 stale+active+clean, 1 creating+peering, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 3.7 MiB/s wr, 59 op/s; 17779/261183 objects degraded (6.807%); 6303/261183 objects misplaced (2.413%); 7.2 MiB/s, 2 keys/s, 117 objects/s recovering 2023-10-09T06:04:05.562 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:05.633 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:05.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:05.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:05.762 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:05.790 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:05.843 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:05.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:05.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:05.947 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:05.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:05 smithi008 ceph-mon[184418]: failed to encode map e533 with expected crc 2023-10-09T06:04:05.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:05 smithi008 ceph-mon[184418]: failed to encode map e533 with expected crc 2023-10-09T06:04:05.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:05 smithi008 ceph-mon[184418]: osdmap e533: 8 total, 7 up, 7 in 2023-10-09T06:04:05.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:05 smithi008 ceph-mon[184418]: failed to encode map e533 with expected crc 2023-10-09T06:04:05.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:05 smithi008 ceph-mon[184418]: pgmap v1602: 89 pgs: 4 peering, 1 stale+active+clean, 1 creating+peering, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 3.7 MiB/s wr, 59 op/s; 17779/261183 objects degraded (6.807%); 6303/261183 objects misplaced (2.413%); 7.2 MiB/s, 2 keys/s, 117 objects/s recovering 2023-10-09T06:04:05.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:05 smithi008 ceph-mon[191227]: failed to encode map e533 with expected crc 2023-10-09T06:04:05.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:05 smithi008 ceph-mon[191227]: failed to encode map e533 with expected crc 2023-10-09T06:04:05.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:05 smithi008 ceph-mon[191227]: osdmap e533: 8 total, 7 up, 7 in 2023-10-09T06:04:05.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:05 smithi008 ceph-mon[191227]: failed to encode map e533 with expected crc 2023-10-09T06:04:05.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:05 smithi008 ceph-mon[191227]: pgmap v1602: 89 pgs: 4 peering, 1 stale+active+clean, 1 creating+peering, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 3.7 MiB/s wr, 59 op/s; 17779/261183 objects degraded (6.807%); 6303/261183 objects misplaced (2.413%); 7.2 MiB/s, 2 keys/s, 117 objects/s recovering 2023-10-09T06:04:05.980 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:06.199 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 93 16 87162 87146 58.5354 0 - 0.0155456 2023-10-09T06:04:06.646 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:06.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:06.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:06.750 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:06.772 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:06.795 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:06 smithi008 ceph-mon[184418]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.795 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:06 smithi008 ceph-mon[184418]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.795 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:06 smithi008 ceph-mon[184418]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.795 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:06 smithi008 ceph-mon[184418]: osdmap e534: 8 total, 7 up, 7 in 2023-10-09T06:04:06.795 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:06 smithi008 ceph-mon[184418]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.796 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:06 smithi008 ceph-mon[191227]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.796 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:06 smithi008 ceph-mon[191227]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.796 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:06 smithi008 ceph-mon[191227]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.796 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:06 smithi008 ceph-mon[191227]: osdmap e534: 8 total, 7 up, 7 in 2023-10-09T06:04:06.796 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:06 smithi008 ceph-mon[191227]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:06 smithi191 ceph-mon[161669]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:06 smithi191 ceph-mon[161669]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:06 smithi191 ceph-mon[161669]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:06 smithi191 ceph-mon[161669]: osdmap e534: 8 total, 7 up, 7 in 2023-10-09T06:04:06.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:06 smithi191 ceph-mon[161669]: failed to encode map e534 with expected crc 2023-10-09T06:04:06.801 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:06.906 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:04:07.199 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 94 16 87162 87146 57.9129 0 - 0.0155456 2023-10-09T06:04:07.455 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:07.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:07.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:07.562 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:07.594 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:07.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:07 smithi008 ceph-mon[184418]: pgmap v1604: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 4 peering, 1 stale+active+clean, 14 active+undersized, 2 active+recovering+undersized+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 51 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 518 KiB/s wr, 8 op/s; 34576/261183 objects degraded (13.238%); 16846/261183 objects misplaced (6.450%); 8.6 MiB/s, 136 objects/s recovering 2023-10-09T06:04:07.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:07 smithi008 ceph-mon[191227]: pgmap v1604: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 4 peering, 1 stale+active+clean, 14 active+undersized, 2 active+recovering+undersized+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 51 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 518 KiB/s wr, 8 op/s; 34576/261183 objects degraded (13.238%); 16846/261183 objects misplaced (6.450%); 8.6 MiB/s, 136 objects/s recovering 2023-10-09T06:04:07.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:07 smithi191 ceph-mon[161669]: pgmap v1604: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 4 peering, 1 stale+active+clean, 14 active+undersized, 2 active+recovering+undersized+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 51 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 518 KiB/s wr, 8 op/s; 34576/261183 objects degraded (13.238%); 16846/261183 objects misplaced (6.450%); 8.6 MiB/s, 136 objects/s recovering 2023-10-09T06:04:07.853 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:07.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:07.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:07.957 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:07.987 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:08.041 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:08.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:08.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:08.145 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:08.178 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:08.199 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 95 16 87162 87146 57.3036 0 - 0.0155456 2023-10-09T06:04:08.299 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:07 smithi191 bash[390161]: 0039e7e7a1999d1983988616daa1dbfd7afd74883a8be270b52b2245c745515b 2023-10-09T06:04:08.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:08 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:04:08.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:08 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 34576/261183 objects degraded (13.238%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:04:08.892 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:08 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:04:08.892 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:08 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 34576/261183 objects degraded (13.238%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:04:08.893 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:08 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:04:08.893 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:08 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 34576/261183 objects degraded (13.238%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:04:08.935 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:08.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:08.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:09.040 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:09.082 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:09.098 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:04:09.122 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:04:09.149 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:04:09.172 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:04:09.199 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 96 16 87162 87146 56.7069 0 - 0.0155456 2023-10-09T06:04:09.390 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:09.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:09.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:09.463 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:09.493 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:09.523 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:09.569 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:09.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:09.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:09.700 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:09 smithi008 ceph-mon[184418]: pgmap v1605: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 4 peering, 1 stale+active+clean, 17 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 37002/261183 objects degraded (14.167%); 13861/261183 objects misplaced (5.307%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T06:04:09.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:09 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1529190019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:09 smithi008 ceph-mon[191227]: pgmap v1605: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 4 peering, 1 stale+active+clean, 17 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 37002/261183 objects degraded (14.167%); 13861/261183 objects misplaced (5.307%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T06:04:09.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:09 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1529190019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:09.744 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:09.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:09.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:09.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:09 smithi191 ceph-mon[161669]: pgmap v1605: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 4 peering, 1 stale+active+clean, 17 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 37002/261183 objects degraded (14.167%); 13861/261183 objects misplaced (5.307%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T06:04:09.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:09 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1529190019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:09.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:09.874 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:09.917 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:09.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:09.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:10.021 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:10.053 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:10.199 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 97 16 87162 87146 56.1226 0 - 0.0155456 2023-10-09T06:04:10.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:04:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:04:10] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.33.4" 2023-10-09T06:04:11.200 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 98 16 87162 87146 55.5501 0 - 0.0155456 2023-10-09T06:04:12.200 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 99 16 87162 87146 54.9892 0 - 0.0155456 2023-10-09T06:04:12.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:11 smithi008 ceph-mon[184418]: pgmap v1606: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 1 stale+active+clean, 19 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 42330/261183 objects degraded (16.207%); 13861/261183 objects misplaced (5.307%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T06:04:12.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:11 smithi008 ceph-mon[191227]: pgmap v1606: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 1 stale+active+clean, 19 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 42330/261183 objects degraded (16.207%); 13861/261183 objects misplaced (5.307%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T06:04:12.260 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:11 smithi191 ceph-mon[161669]: pgmap v1606: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 1 stale+active+clean, 19 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 42330/261183 objects degraded (16.207%); 13861/261183 objects misplaced (5.307%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T06:04:12.279 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T06:04:12.315 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:12.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:12.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:12.323 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:04:12.323 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T06:04:12.420 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:12.460 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:12.533 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:12.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:12.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:12.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:12.669 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:12.721 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:12.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:12.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:12.829 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:12.867 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:13.131 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:13.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:13.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:13.200 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:2023-10-09T06:04:13.200840+0000 min lat: 0.00115919 max lat: 2.47497 avg lat: 0.0155456 2023-10-09T06:04:13.200 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:04:13.200 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 100 16 87162 87146 54.4396 0 - 0.0155456 2023-10-09T06:04:13.251 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:13.290 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:13.302 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:04:13.302 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:04:13.302 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T06:04:13.302 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T06:04:13.302 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:04:13.864 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:13 smithi008 ceph-mon[191227]: pgmap v1607: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 1 stale+active+clean, 19 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 41699/261183 objects degraded (15.965%); 13861/261183 objects misplaced (5.307%); 13 MiB/s, 214 objects/s recovering 2023-10-09T06:04:13.865 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 41699/261183 objects degraded (15.965%), 16 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:13.948 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:13.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:13.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:14.052 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:14.086 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:14.200 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 101 16 87162 87146 53.9008 0 - 0.0155456 2023-10-09T06:04:14.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:13 smithi008 ceph-mon[184418]: pgmap v1607: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 1 stale+active+clean, 19 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 41699/261183 objects degraded (15.965%); 13861/261183 objects misplaced (5.307%); 13 MiB/s, 214 objects/s recovering 2023-10-09T06:04:14.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 41699/261183 objects degraded (15.965%), 16 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:14.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:13 smithi191 ceph-mon[161669]: pgmap v1607: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 1 stale+active+clean, 19 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 41699/261183 objects degraded (15.965%); 13861/261183 objects misplaced (5.307%); 13 MiB/s, 214 objects/s recovering 2023-10-09T06:04:14.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 41699/261183 objects degraded (15.965%), 16 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:15.200 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 102 16 87162 87146 53.3726 0 - 0.0155456 2023-10-09T06:04:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:15.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:15.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:15.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:15.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:16.200 INFO:tasks.radosbench.radosbench.0.smithi008.stdout: 103 16 87162 87146 52.8546 0 - 0.0155456 2023-10-09T06:04:16.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:15 smithi008 ceph-mon[184418]: pgmap v1608: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 1 stale+active+clean, 19 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 41699/261183 objects degraded (15.965%); 13861/261183 objects misplaced (5.307%); 11 MiB/s, 177 objects/s recovering 2023-10-09T06:04:16.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:15 smithi008 ceph-mon[191227]: pgmap v1608: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 1 stale+active+clean, 19 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 41699/261183 objects degraded (15.965%); 13861/261183 objects misplaced (5.307%); 11 MiB/s, 177 objects/s recovering 2023-10-09T06:04:16.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:15 smithi191 ceph-mon[161669]: pgmap v1608: 89 pgs: 1 activating+remapped, 3 active+clean+remapped, 1 stale+active+clean, 19 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 48 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 41699/261183 objects degraded (15.965%); 13861/261183 objects misplaced (5.307%); 11 MiB/s, 177 objects/s recovering 2023-10-09T06:04:16.460 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:04:16.487 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:04:16.507 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:04:16.508 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:04:16.508 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:04:16.508 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:04:16.508 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:04:16.508 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:04:16.508 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:04:16.508 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-09T06:04:16.508 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:04:16.970 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:16.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:16.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:17.076 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:17.106 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:17.150 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:17.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:17.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:17.200 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total time run: 103.517 2023-10-09T06:04:17.200 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Total writes made: 87162 2023-10-09T06:04:17.201 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Write size: 65536 2023-10-09T06:04:17.201 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Object size: 65536 2023-10-09T06:04:17.201 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Bandwidth (MB/sec): 52.6253 2023-10-09T06:04:17.201 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Bandwidth: 41.346 2023-10-09T06:04:17.201 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max bandwidth (MB/sec): 168.312 2023-10-09T06:04:17.201 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:04:17.201 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average IOPS: 842 2023-10-09T06:04:17.201 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev IOPS: 661.541 2023-10-09T06:04:17.201 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max IOPS: 2693 2023-10-09T06:04:17.201 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min IOPS: 0 2023-10-09T06:04:17.201 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Average Latency(s): 0.0189908 2023-10-09T06:04:17.202 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Stddev Latency(s): 0.258695 2023-10-09T06:04:17.202 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Max latency(s): 21.0712 2023-10-09T06:04:17.202 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Min latency(s): 0.00115919 2023-10-09T06:04:17.211 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:04:17.257 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:17.285 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:17.323 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-10-09T06:04:17.324 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:04:17.324 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2023-10-09T06:04:17.324 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2023-10-09T06:04:17.324 DEBUG:teuthology.orchestra.run.smithi191:osd.6> sudo journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T06:04:17.326 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl start ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T06:04:17.335 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:17.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:17.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:17.360 INFO:journalctl@ceph.osd.6.smithi191.stdout:-- Logs begin at Mon 2023-10-09 04:57:51 UTC. -- 2023-10-09T06:04:17.442 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:17.484 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:17.540 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:17.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:17.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:17.647 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:17.761 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:17.799 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:17 smithi191 systemd[1]: Starting Ceph osd.6 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:04:18.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:17 smithi008 ceph-mon[184418]: pgmap v1609: 89 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 3 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 645 KiB/s wr, 10 op/s; 50484/261525 objects degraded (19.304%); 16110/261525 objects misplaced (6.160%); 12 MiB/s, 196 objects/s recovering 2023-10-09T06:04:18.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:17 smithi008 ceph-mon[191227]: pgmap v1609: 89 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 3 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 645 KiB/s wr, 10 op/s; 50484/261525 objects degraded (19.304%); 16110/261525 objects misplaced (6.160%); 12 MiB/s, 196 objects/s recovering 2023-10-09T06:04:18.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:17 smithi191 ceph-mon[161669]: pgmap v1609: 89 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 3 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 645 KiB/s wr, 10 op/s; 50484/261525 objects degraded (19.304%); 16110/261525 objects misplaced (6.160%); 12 MiB/s, 196 objects/s recovering 2023-10-09T06:04:18.658 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:18.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:18.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:18.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:18.800 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:18 smithi191 bash[391497]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-10-09T06:04:18.800 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:18 smithi191 bash[391497]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2023-10-09T06:04:18.800 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:18 smithi191 bash[391497]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2023-10-09T06:04:18.800 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:18 smithi191 bash[391497]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T06:04:18.801 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:18 smithi191 bash[391497]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2023-10-09T06:04:18.801 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:18 smithi191 bash[391497]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-10-09T06:04:18.801 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:18 smithi191 bash[391497]: --> ceph-volume raw activate successful for osd ID: 6 2023-10-09T06:04:18.806 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:18.862 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:18.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:18.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:18.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:18.997 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:19.043 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:04:19.043 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:19.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:19.146 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:19.285 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:19 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:19.285 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:19 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:19.285 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:19 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 50484/261525 objects degraded (19.304%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:19.293 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:19.297 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:19 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:19.298 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:19 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:19.298 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:19 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 50484/261525 objects degraded (19.304%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:19.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:19 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:19 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:19.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:19 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 50484/261525 objects degraded (19.304%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:19.439 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:04:19.440 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:04:19.548 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:19.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:19.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:19.664 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:19.758 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:19.922 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:19.923 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:04:19.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:19.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:20.033 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:20.079 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:20.155 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:20.300 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:19 smithi191 conmon[392089]: debug 2023-10-09T06:04:19.904+0000 7fe0e2f9a700 1 -- 172.21.15.191:0/2640183852 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557bedd3a680 con 0x557bedd01400 2023-10-09T06:04:20.300 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:19 smithi191 bash[392026]: 4feaa20b7a1a2c1f899a3d8bc59ad3a6f7dc9e42046714f508e5786ae09e6adc 2023-10-09T06:04:20.300 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:19 smithi191 systemd[1]: Started Ceph osd.6 for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T06:04:20.319 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:04:20.425 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:20.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:20.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:20.534 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:20.571 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:20.621 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:20.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:20.693 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:04:20.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[184418]: pgmap v1610: 89 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 3 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 608 KiB/s wr, 12 op/s; 49760/261429 objects degraded (19.034%); 16110/261429 objects misplaced (6.162%); 14 MiB/s, 222 objects/s recovering 2023-10-09T06:04:20.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:20.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:20.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:20.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1939026309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:20.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:20.718 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:04:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:04:20] "GET /metrics HTTP/1.1" 200 36649 "" "Prometheus/2.33.4" 2023-10-09T06:04:20.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[191227]: pgmap v1610: 89 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 3 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 608 KiB/s wr, 12 op/s; 49760/261429 objects degraded (19.034%); 16110/261429 objects misplaced (6.162%); 14 MiB/s, 222 objects/s recovering 2023-10-09T06:04:20.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:20.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:20.723 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:20.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1939026309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:20.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:20 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:20.725 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:20.731 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:20 smithi191 conmon[392089]: debug 2023-10-09T06:04:20.426+0000 7fe0eaa15200 -1 Falling back to public interface 2023-10-09T06:04:20.731 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:20 smithi191 ceph-mon[161669]: pgmap v1610: 89 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 3 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 608 KiB/s wr, 12 op/s; 49760/261429 objects degraded (19.034%); 16110/261429 objects misplaced (6.162%); 14 MiB/s, 222 objects/s recovering 2023-10-09T06:04:20.731 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:20.732 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:20.732 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:20.732 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:20 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1939026309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:20.732 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:20 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:20.849 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:21.405 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:21.405 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[184418]: failed to encode map e535 with expected crc 2023-10-09T06:04:21.406 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[184418]: failed to encode map e535 with expected crc 2023-10-09T06:04:21.406 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:04:21.406 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[184418]: osdmap e535: 8 total, 7 up, 7 in 2023-10-09T06:04:21.406 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[184418]: failed to encode map e535 with expected crc 2023-10-09T06:04:21.406 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:21.406 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[191227]: failed to encode map e535 with expected crc 2023-10-09T06:04:21.407 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[191227]: failed to encode map e535 with expected crc 2023-10-09T06:04:21.407 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:04:21.407 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[191227]: osdmap e535: 8 total, 7 up, 7 in 2023-10-09T06:04:21.407 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:21 smithi008 ceph-mon[191227]: failed to encode map e535 with expected crc 2023-10-09T06:04:21.441 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:04:21.700 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:21.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:21 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:21 smithi191 ceph-mon[161669]: failed to encode map e535 with expected crc 2023-10-09T06:04:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:21 smithi191 ceph-mon[161669]: failed to encode map e535 with expected crc 2023-10-09T06:04:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:21 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:04:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:21 smithi191 ceph-mon[161669]: osdmap e535: 8 total, 7 up, 7 in 2023-10-09T06:04:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:21 smithi191 ceph-mon[161669]: failed to encode map e535 with expected crc 2023-10-09T06:04:21.811 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:21.918 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:22.082 INFO:teuthology.orchestra.run.smithi191.stderr:no valid command found; 10 closest matches: 2023-10-09T06:04:22.082 INFO:teuthology.orchestra.run.smithi191.stderr:0 2023-10-09T06:04:22.082 INFO:teuthology.orchestra.run.smithi191.stderr:1 2023-10-09T06:04:22.082 INFO:teuthology.orchestra.run.smithi191.stderr:2 2023-10-09T06:04:22.082 INFO:teuthology.orchestra.run.smithi191.stderr:abort 2023-10-09T06:04:22.082 INFO:teuthology.orchestra.run.smithi191.stderr:assert 2023-10-09T06:04:22.083 INFO:teuthology.orchestra.run.smithi191.stderr:config diff 2023-10-09T06:04:22.083 INFO:teuthology.orchestra.run.smithi191.stderr:config diff get 2023-10-09T06:04:22.083 INFO:teuthology.orchestra.run.smithi191.stderr:config get 2023-10-09T06:04:22.083 INFO:teuthology.orchestra.run.smithi191.stderr:config help [] 2023-10-09T06:04:22.083 INFO:teuthology.orchestra.run.smithi191.stderr:config set ... 2023-10-09T06:04:22.083 INFO:teuthology.orchestra.run.smithi191.stderr:admin_socket: invalid command 2023-10-09T06:04:22.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:22 smithi191 ceph-mon[161669]: pgmap v1612: 89 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 3 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 729 KiB/s wr, 18 op/s; 49717/261300 objects degraded (19.027%); 16110/261300 objects misplaced (6.165%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:04:22.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:22 smithi008 ceph-mon[184418]: pgmap v1612: 89 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 3 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 729 KiB/s wr, 18 op/s; 49717/261300 objects degraded (19.027%); 16110/261300 objects misplaced (6.165%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:04:22.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:22 smithi008 ceph-mon[191227]: pgmap v1612: 89 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 3 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 729 KiB/s wr, 18 op/s; 49717/261300 objects degraded (19.027%); 16110/261300 objects misplaced (6.165%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:04:22.798 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:04:22.799 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2023-10-09T06:04:23.019 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:23.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:23.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:23.130 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:23.167 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:23.223 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:23.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:23.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:23.329 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:23.473 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:23.550 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:23.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 49717/261300 objects degraded (19.027%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:23.727 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 49717/261300 objects degraded (19.027%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:23.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 49717/261300 objects degraded (19.027%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:24.123 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:24.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:24.233 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:24.275 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:04:24.275 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T06:04:24.275 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T06:04:24.275 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T06:04:24.276 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.276 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77599 9.0 9:0003d9cd:::benchmark_data_smithi008_502820_object77598:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e385)", 2023-10-09T06:04:24.276 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:36.817794+0000", 2023-10-09T06:04:24.276 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 587.45485604999999, 2023-10-09T06:04:24.276 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.88810962599999999, 2023-10-09T06:04:24.276 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.276 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:04:24.276 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.276 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:04:24.276 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:04:24.276 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77599 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817794+0000", 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817794+0000", 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.277 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817793+0000", 2023-10-09T06:04:24.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:04:24.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.278 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817819+0000", 2023-10-09T06:04:24.278 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6563999999999998e-05 2023-10-09T06:04:24.278 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.278 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.278 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817840+0000", 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1183999999999999e-05 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817850+0000", 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5040000000000008e-06 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817907+0000", 2023-10-09T06:04:24.279 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6700000000000003e-05 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.817919+0000", 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2755000000000001e-05 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.705815+0000", 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.88789530699999997 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.280 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.281 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.281 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.705904+0000", 2023-10-09T06:04:24.281 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9301000000000004e-05 2023-10-09T06:04:24.281 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.281 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.281 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.281 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.281 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.281 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77622 9.2 9:43e6566f:::benchmark_data_smithi008_502820_object77621:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e385)", 2023-10-09T06:04:24.281 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:36.872539+0000", 2023-10-09T06:04:24.281 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 587.40011103300003, 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86671261899999996, 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77622 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.282 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872539+0000", 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872539+0000", 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872537+0000", 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:04:24.283 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872566+0000", 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8402999999999999e-05 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872580+0000", 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4102e-05 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.284 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872590+0000", 2023-10-09T06:04:24.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8509999999999994e-06 2023-10-09T06:04:24.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.285 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.285 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872642+0000", 2023-10-09T06:04:24.285 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2284999999999998e-05 2023-10-09T06:04:24.285 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.285 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:36.872653+0000", 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0428000000000001e-05 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.703374+0000", 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83072101700000001 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.286 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.703547+0000", 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000173058 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.703705+0000", 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000158813 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.714288+0000", 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010582225000000001 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.287 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.725463+0000", 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011175243 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.739134+0000", 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013670748 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.288 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.739206+0000", 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2875999999999998e-05 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.739252+0000", 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5386000000000002e-05 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.289 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:77624 9.10 9:0cfb0a88:::benchmark_data_smithi008_502820_object77623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e385)", 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:54:37.004849+0000", 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 587.26780122000002, 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.76289203999999999, 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 77624 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.290 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004849+0000", 2023-10-09T06:04:24.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.291 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.291 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.291 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004849+0000", 2023-10-09T06:04:24.291 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.291 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.292 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.292 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004847+0000", 2023-10-09T06:04:24.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:04:24.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004881+0000", 2023-10-09T06:04:24.293 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3456e-05 2023-10-09T06:04:24.293 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.293 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.293 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.293 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004897+0000", 2023-10-09T06:04:24.294 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.594e-05 2023-10-09T06:04:24.294 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.294 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004908+0000", 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1667e-05 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.004998+0000", 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9231e-05 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.005010+0000", 2023-10-09T06:04:24.295 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.225e-05 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.747777+0000", 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.742767222 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.747948+0000", 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017099300000000001 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.296 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.752637+0000", 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0046887279999999996 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.763870+0000", 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011232956000000001 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.297 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.764548+0000", 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00067862 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.767630+0000", 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030816730000000001 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.767696+0000", 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5646000000000001e-05 2023-10-09T06:04:24.298 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:54:37.767741+0000", 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5454000000000002e-05 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:102993 9.b 9.bd9d978b (undecoded) ondisk+write+known_if_redirected e419)", 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:56:10.583702+0000", 2023-10-09T06:04:24.299 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 493.68894864999999, 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.74320509000000001, 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 102993 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.300 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583702+0000", 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583702+0000", 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583699+0000", 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999971 2023-10-09T06:04:24.301 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583715+0000", 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6668999999999999e-05 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583750+0000", 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4641000000000003e-05 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.302 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583765+0000", 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5095e-05 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583831+0000", 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5693999999999998e-05 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:10.583842+0000", 2023-10-09T06:04:24.303 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1454000000000001e-05 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:11.326897+0000", 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.74305512600000001 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:56:11.326907+0000", 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2520000000000005e-06 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.304 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117296 9.10 9:0bb04225:::benchmark_data_smithi008_502820_object33123:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:56.263811+0000", 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 388.00883942500002, 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.170647588, 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:04:24.305 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117296 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263811+0000", 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263811+0000", 2023-10-09T06:04:24.306 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263809+0000", 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263819+0000", 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0018e-05 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.307 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263842+0000", 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3445999999999999e-05 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263852+0000", 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9650000000000003e-06 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.308 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263925+0000", 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2465999999999994e-05 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263942+0000", 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7008e-05 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.429345+0000", 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1654032759999999 2023-10-09T06:04:24.309 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.429511+0000", 2023-10-09T06:04:24.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016623000000000001 2023-10-09T06:04:24.310 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.310 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:04:24.310 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.429618+0000", 2023-10-09T06:04:24.310 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000106716 2023-10-09T06:04:24.310 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.310 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.432625+0000", 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030070420000000001 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.432632+0000", 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3440000000000003e-06 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.311 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.433415+0000", 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00078227500000000003 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.433475+0000", 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0384000000000003e-05 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.434458+0000", 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00098339400000000006 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.312 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117298 9.6 9:64c5977e:::benchmark_data_smithi008_502820_object33125:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:56.293379+0000", 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 387.97927142399999, 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4447610179999999, 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:04:24.313 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117298 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293379+0000", 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293379+0000", 2023-10-09T06:04:24.314 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293377+0000", 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293386+0000", 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2519999999999999e-06 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.315 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293406+0000", 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0307000000000001e-05 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293417+0000", 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1223000000000001e-05 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.316 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293499+0000", 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1650999999999995e-05 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293513+0000", 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3808999999999999e-05 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.244709+0000", 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.95119675999999997 2023-10-09T06:04:24.317 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.244723+0000", 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3230999999999999e-05 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.722284+0000", 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4775617670000001 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.318 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.722454+0000", 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016943700000000001 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.722557+0000", 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000102985 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.725888+0000", 2023-10-09T06:04:24.319 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003330937 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.725901+0000", 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.288e-05 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.736945+0000", 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011044221999999999 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.320 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.737001+0000", 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6258000000000002e-05 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.738140+0000", 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011387680000000001 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.321 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.322 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117300 9.16 9:6b0df22d:::benchmark_data_smithi008_502820_object33127:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-10-09T06:04:24.322 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:56.313799+0000", 2023-10-09T06:04:24.322 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 387.958850928, 2023-10-09T06:04:24.322 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4264962080000001, 2023-10-09T06:04:24.322 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.322 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.322 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.322 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:04:24.322 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:04:24.322 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117300 2023-10-09T06:04:24.322 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313799+0000", 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313799+0000", 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.323 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313798+0000", 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313809+0000", 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1031e-05 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313823+0000", 2023-10-09T06:04:24.324 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3562e-05 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313830+0000", 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7360000000000001e-06 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313887+0000", 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6759999999999999e-05 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.325 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313899+0000", 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1528999999999999e-05 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.244751+0000", 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93085188200000002 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.326 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.244760+0000", 2023-10-09T06:04:24.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7780000000000002e-06 2023-10-09T06:04:24.327 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.327 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.327 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.734776+0000", 2023-10-09T06:04:24.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.490015206 2023-10-09T06:04:24.327 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.327 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.327 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.327 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.734873+0000", 2023-10-09T06:04:24.327 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7702999999999996e-05 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.734974+0000", 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010043999999999999 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.738071+0000", 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030975210000000002 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.328 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.738084+0000", 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2621e-05 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.739642+0000", 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001558282 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.329 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.739707+0000", 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4712e-05 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.740296+0000", 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00058872799999999995 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.330 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117310 9.f 9.54f03b5f (undecoded) ondisk+write+known_if_redirected e443)", 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:56.499315+0000", 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 387.773334863, 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.74700570899999996, 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117310 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.331 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.499315+0000", 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.499315+0000", 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.499314+0000", 2023-10-09T06:04:24.332 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.499322+0000", 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1650000000000006e-06 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.499341+0000", 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8859e-05 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.333 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.499349+0000", 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9289999999999998e-06 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.499396+0000", 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7250000000000003e-05 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.499405+0000", 2023-10-09T06:04:24.334 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3510000000000008e-06 2023-10-09T06:04:24.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.246308+0000", 2023-10-09T06:04:24.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.74690213999999999 2023-10-09T06:04:24.335 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.335 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.335 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.335 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.246321+0000", 2023-10-09T06:04:24.335 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3517e-05 2023-10-09T06:04:24.335 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117313 9.0 9:0696ae4d:::benchmark_data_smithi008_502820_object33140:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:56.675472+0000", 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 387.59717846799998, 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.76257717700000005, 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.336 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117313 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.675472+0000", 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.675472+0000", 2023-10-09T06:04:24.337 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.338 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.338 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.338 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.675470+0000", 2023-10-09T06:04:24.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:04:24.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.338 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.338 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.338 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.675481+0000", 2023-10-09T06:04:24.338 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1446e-05 2023-10-09T06:04:24.338 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.675500+0000", 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8762000000000001e-05 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.675509+0000", 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.302e-06 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.339 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.675575+0000", 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5623000000000005e-05 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.675590+0000", 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5037e-05 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.432627+0000", 2023-10-09T06:04:24.340 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.75703654600000003 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.432746+0000", 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011945 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.432854+0000", 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000107588 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.341 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.434933+0000", 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020797099999999998 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.435261+0000", 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00032788000000000001 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.342 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.436852+0000", 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015902889999999999 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.436915+0000", 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3329000000000003e-05 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.438049+0000", 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001134136 2023-10-09T06:04:24.343 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117318 9.f 9.692a8b9f (undecoded) ondisk+write+known_if_redirected e443)", 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:56.790266+0000", 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 387.482384696, 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.45608858000000002, 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.344 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117318 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.790266+0000", 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.345 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.790266+0000", 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.790264+0000", 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.790272+0000", 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2879999999999993e-06 2023-10-09T06:04:24.346 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.790291+0000", 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8422000000000001e-05 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.790299+0000", 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0180000000000001e-06 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.347 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.790348+0000", 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.8649000000000001e-05 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.790357+0000", 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1800000000000002e-06 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.246345+0000", 2023-10-09T06:04:24.348 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.455988059 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.246354+0000", 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3700000000000001e-06 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:61401 10.7 10:ef013cfa:::benchmark_data_smithi008_634586_object61400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:16.952393+0000", 2023-10-09T06:04:24.349 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 67.320256837000002, 2023-10-09T06:04:24.350 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37074901799999999, 2023-10-09T06:04:24.350 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.350 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.350 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.350 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:04:24.350 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:04:24.350 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 61401 2023-10-09T06:04:24.350 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.350 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.350 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.350 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.952393+0000", 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.952393+0000", 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.952393+0000", 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:24.351 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.952414+0000", 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0687999999999999e-05 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.952420+0000", 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2319999999999998e-06 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.352 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.952426+0000", 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7780000000000004e-06 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.957768+0000", 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005341864 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.957934+0000", 2023-10-09T06:04:24.353 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016685800000000001 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.958115+0000", 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018020699999999999 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.962917+0000", 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0048020820000000001 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.354 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.967480+0000", 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045630460000000003 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.320370+0000", 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35288984699999998 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.320421+0000", 2023-10-09T06:04:24.355 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1628000000000001e-05 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323142+0000", 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027211879999999998 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:61415 10.7 10:eabc83ed:::benchmark_data_smithi008_634586_object61414:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:16.970526+0000", 2023-10-09T06:04:24.356 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 67.302124367000005, 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35575954199999998, 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 61415 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.357 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.970526+0000", 2023-10-09T06:04:24.358 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.358 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.358 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.358 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.358 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.970526+0000", 2023-10-09T06:04:24.358 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.358 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.358 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.358 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.358 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.970524+0000", 2023-10-09T06:04:24.358 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.970552+0000", 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7829000000000001e-05 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.970563+0000", 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1049e-05 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.359 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.970570+0000", 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.34e-06 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.975348+0000", 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.004777842 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.360 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.975559+0000", 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00021036199999999999 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.975761+0000", 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00020204000000000001 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.978894+0000", 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003133514 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.361 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.979614+0000", 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00072026700000000002 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323372+0000", 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.343757278 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.362 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323421+0000", 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9799000000000002e-05 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.326285+0000", 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002864021 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.363 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:61425 10.e 10:75bb83f6:::benchmark_data_smithi008_634586_object61424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:16.980426+0000", 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 67.292224270000006, 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.34291443700000002, 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 61425 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.364 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980426+0000", 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980426+0000", 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980424+0000", 2023-10-09T06:04:24.365 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980460+0000", 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5611999999999997e-05 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980471+0000", 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1185e-05 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.366 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980479+0000", 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8550000000000002e-06 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980548+0000", 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8919999999999997e-05 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.367 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980706+0000", 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015841100000000001 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980849+0000", 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014233200000000001 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.322751+0000", 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.34190267699999999 2023-10-09T06:04:24.368 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.322781+0000", 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0025000000000001e-05 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323257+0000", 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00047581099999999997 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.369 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323301+0000", 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3931999999999999e-05 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323340+0000", 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9267000000000002e-05 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.370 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:61427 10.e 10:7cb0a76c:::benchmark_data_smithi008_634586_object61426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:16.980546+0000", 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 67.292104132000006, 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.34287847100000002, 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 61427 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.371 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980546+0000", 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980546+0000", 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.372 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980546+0000", 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980579+0000", 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3689999999999998e-05 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980587+0000", 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8290000000000008e-06 2023-10-09T06:04:24.373 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.374 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980593+0000", 2023-10-09T06:04:24.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.976e-06 2023-10-09T06:04:24.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.374 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.374 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.983087+0000", 2023-10-09T06:04:24.374 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0024936820000000001 2023-10-09T06:04:24.374 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.374 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.983230+0000", 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014284899999999999 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.317689+0000", 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33445927800000003 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.322810+0000", 2023-10-09T06:04:24.375 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051208649999999996 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323240+0000", 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00043040199999999998 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323349+0000", 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000108931 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.376 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323387+0000", 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8142000000000002e-05 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323425+0000", 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.7327999999999999e-05 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.377 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.378 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.378 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:61432 10.e 10:7e08abcd:::benchmark_data_smithi008_634586_object61431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:04:24.378 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:16.980697+0000", 2023-10-09T06:04:24.378 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 67.291952918999996, 2023-10-09T06:04:24.378 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.34489857600000001, 2023-10-09T06:04:24.378 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.378 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.378 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.378 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:04:24.378 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:04:24.378 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 61432 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980697+0000", 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980697+0000", 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.379 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980697+0000", 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980717+0000", 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0446999999999999e-05 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.380 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980724+0000", 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5719999999999999e-06 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980729+0000", 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2809999999999999e-06 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.320200+0000", 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33947122699999999 2023-10-09T06:04:24.381 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.320369+0000", 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000168665 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.320525+0000", 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015546099999999999 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.382 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323464+0000", 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029395739999999999 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323471+0000", 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0870000000000001e-06 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.325478+0000", 2023-10-09T06:04:24.383 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020071429999999999 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.325542+0000", 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3448000000000006e-05 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.325596+0000", 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4119000000000001e-05 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.384 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:61433 10.7 10:eaf11762:::benchmark_data_smithi008_634586_object61432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:16.980735+0000", 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 67.291915699, 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.362992596, 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:04:24.385 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 61433 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980735+0000", 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980735+0000", 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.386 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980734+0000", 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980753+0000", 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9315999999999999e-05 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.387 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980760+0000", 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3609999999999997e-06 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:16.980765+0000", 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2379999999999997e-06 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.317527+0000", 2023-10-09T06:04:24.388 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33676147000000001 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.317722+0000", 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000195327 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.317885+0000", 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016320199999999999 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.389 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.323328+0000", 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0054426370000000002 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.326348+0000", 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003020628 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.390 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.341327+0000", 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014978673 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.341381+0000", 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3888999999999999e-05 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:17.343727+0000", 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00234629 2023-10-09T06:04:24.391 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:81976 10.7 10:e03922ef:::benchmark_data_smithi008_634586_object81975:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e526)", 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:47.631525+0000", 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 36.641125559000002, 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27440867400000002, 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.392 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 81976 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.631525+0000", 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.393 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.631525+0000", 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.631524+0000", 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.631546+0000", 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1868000000000001e-05 2023-10-09T06:04:24.394 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.631552+0000", 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8050000000000003e-06 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.631557+0000", 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9520000000000002e-06 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.395 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.638502+0000", 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0069448069999999999 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.638683+0000", 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018138900000000001 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.638856+0000", 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000172583 2023-10-09T06:04:24.396 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.644962+0000", 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061059779999999998 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.655412+0000", 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010450038 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.397 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.672956+0000", 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017544476 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.673013+0000", 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6499000000000002e-05 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.905933+0000", 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.23292079900000001 2023-10-09T06:04:24.398 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.399 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.399 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.399 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.399 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.399 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:81984 10.f 10:fdb1f540:::benchmark_data_smithi008_634586_object81983:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e526)", 2023-10-09T06:04:24.399 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:47.648766+0000", 2023-10-09T06:04:24.401 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 36.623884097000001, 2023-10-09T06:04:24.401 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.25892842100000002, 2023-10-09T06:04:24.401 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.401 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.401 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 81984 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.648766+0000", 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.402 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.648766+0000", 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.648765+0000", 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.648791+0000", 2023-10-09T06:04:24.403 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.671e-05 2023-10-09T06:04:24.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.648804+0000", 2023-10-09T06:04:24.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2235000000000001e-05 2023-10-09T06:04:24.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.404 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.404 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.404 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.648811+0000", 2023-10-09T06:04:24.404 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3690000000000001e-06 2023-10-09T06:04:24.404 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.655435+0000", 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066243070000000003 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.655611+0000", 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017617699999999999 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:24.405 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.655767+0000", 2023-10-09T06:04:24.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015580800000000001 2023-10-09T06:04:24.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.663233+0000", 2023-10-09T06:04:24.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0074656310000000004 2023-10-09T06:04:24.406 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.406 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.406 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.406 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.663637+0000", 2023-10-09T06:04:24.406 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00040454800000000002 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.907607+0000", 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.243969928 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.907656+0000", 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9095999999999999e-05 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.407 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.907695+0000", 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8127000000000003e-05 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:81986 10.f 10:f4c82009:::benchmark_data_smithi008_634586_object81985:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e526)", 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:47.651308+0000", 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 36.621342054000003, 2023-10-09T06:04:24.408 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26373907299999999, 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 81986 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.651308+0000", 2023-10-09T06:04:24.409 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.651308+0000", 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.651306+0000", 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.410 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.651335+0000", 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8234000000000001e-05 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.651349+0000", 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4078e-05 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.411 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.651357+0000", 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6710000000000005e-06 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.658877+0000", 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0075191679999999997 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.659031+0000", 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015427999999999999 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.412 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.659186+0000", 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000154752 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.667197+0000", 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0080110700000000003 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.413 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.907737+0000", 2023-10-09T06:04:24.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24054076999999999 2023-10-09T06:04:24.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.914965+0000", 2023-10-09T06:04:24.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.007227687 2023-10-09T06:04:24.414 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.414 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.414 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.414 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.915007+0000", 2023-10-09T06:04:24.414 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1427000000000002e-05 2023-10-09T06:04:24.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.415 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.415 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.915047+0000", 2023-10-09T06:04:24.415 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0809000000000002e-05 2023-10-09T06:04:24.415 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.415 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.415 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.415 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.415 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.415 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:81994 10.7 10:ee42a55d:::benchmark_data_smithi008_634586_object81993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e526)", 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:47.656507+0000", 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 36.616142855, 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26247593200000002, 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 81994 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:04:24.416 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.656507+0000", 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.656507+0000", 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:04:24.417 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.656507+0000", 2023-10-09T06:04:24.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:24.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:04:24.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.656528+0000", 2023-10-09T06:04:24.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.1437000000000002e-05 2023-10-09T06:04:24.418 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.418 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.418 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:04:24.418 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.656535+0000", 2023-10-09T06:04:24.418 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5309999999999998e-06 2023-10-09T06:04:24.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:04:24.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.656540+0000", 2023-10-09T06:04:24.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3190000000000002e-06 2023-10-09T06:04:24.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.419 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.419 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:04:24.419 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.660227+0000", 2023-10-09T06:04:24.419 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036867480000000001 2023-10-09T06:04:24.419 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.660382+0000", 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015520399999999999 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.660542+0000", 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015960799999999999 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.420 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.666202+0000", 2023-10-09T06:04:24.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0056601259999999997 2023-10-09T06:04:24.421 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.421 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.421 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:24.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.907573+0000", 2023-10-09T06:04:24.421 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.24137081399999999 2023-10-09T06:04:24.421 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.421 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.421 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:04:24.421 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.916611+0000", 2023-10-09T06:04:24.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0090385299999999995 2023-10-09T06:04:24.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.422 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:04:24.422 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.916677+0000", 2023-10-09T06:04:24.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5229999999999997e-05 2023-10-09T06:04:24.422 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:04:24.422 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:04:24.422 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:04:24.422 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:47.918983+0000", 2023-10-09T06:04:24.422 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002306808 2023-10-09T06:04:24.423 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.423 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.423 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.423 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:04:24.423 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:04:24.423 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:04:24.424 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:24.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:24 smithi191 ceph-mon[161669]: pgmap v1613: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 825 KiB/s wr, 183 op/s; 48911/256458 objects degraded (19.072%); 15829/256458 objects misplaced (6.172%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:04:24.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:24 smithi008 ceph-mon[191227]: pgmap v1613: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 825 KiB/s wr, 183 op/s; 48911/256458 objects degraded (19.072%); 15829/256458 objects misplaced (6.172%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:04:24.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:24 smithi008 ceph-mon[184418]: pgmap v1613: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 825 KiB/s wr, 183 op/s; 48911/256458 objects degraded (19.072%); 15829/256458 objects misplaced (6.172%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:04:25.132 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:25.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:25.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:25.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:25.269 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:25.323 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:25.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:25.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:25.432 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:25.468 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:25.474 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:04:25.519 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:25.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:25.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:25.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:25.659 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:25.722 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:25.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:25.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:25 smithi191 ceph-mon[161669]: pgmap v1614: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 825 KiB/s wr, 197 op/s; 48767/256026 objects degraded (19.048%); 15829/256026 objects misplaced (6.183%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:04:25.829 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:25.904 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:25 smithi008 ceph-mon[191227]: pgmap v1614: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 825 KiB/s wr, 197 op/s; 48767/256026 objects degraded (19.048%); 15829/256026 objects misplaced (6.183%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:04:25.908 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:25.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:25 smithi008 ceph-mon[184418]: pgmap v1614: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 825 KiB/s wr, 197 op/s; 48767/256026 objects degraded (19.048%); 15829/256026 objects misplaced (6.183%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:04:26.527 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:26.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:26.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:26.643 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:26.692 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:04:26.778 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:27.079 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:27.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:27.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:27.196 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:27.229 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:27.279 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:27.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:27.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:27.303 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:27.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:27.415 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:27.469 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:27.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:27.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:27.496 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:04:27.538 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:04:27.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:27.601 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:04:27.610 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:27.659 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:04:27.664 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:27.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:27.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:27.712 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:04:27.756 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:04:27.770 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:27.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:04:27.822 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:04:27.823 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:04:27.823 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:04:27.823 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:04:27.823 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:04:27.823 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:04:27.823 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:04:27.823 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-09T06:04:27.824 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:04:27.901 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:27 smithi191 ceph-mon[161669]: pgmap v1615: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.1 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 96 KiB/s wr, 380 op/s; 47997/250191 objects degraded (19.184%); 15372/250191 objects misplaced (6.144%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:04:27.902 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3405074342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:27.903 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:27 smithi008 ceph-mon[184418]: pgmap v1615: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.1 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 96 KiB/s wr, 380 op/s; 47997/250191 objects degraded (19.184%); 15372/250191 objects misplaced (6.144%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:04:27.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3405074342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:27.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:27 smithi008 ceph-mon[191227]: pgmap v1615: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.1 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 96 KiB/s wr, 380 op/s; 47997/250191 objects degraded (19.184%); 15372/250191 objects misplaced (6.144%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:04:27.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3405074342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:28.233 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:04:29.027 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:29.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:29.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:29.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 47997/250191 objects degraded (19.184%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:29.137 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:29.173 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 47997/250191 objects degraded (19.184%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:29.173 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 47997/250191 objects degraded (19.184%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:29.177 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:29.224 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:29.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:29.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:29.328 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:29.377 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:29.957 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:29 smithi191 ceph-mon[161669]: pgmap v1616: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 96 KiB/s wr, 405 op/s; 47702/249333 objects degraded (19.132%); 15372/249333 objects misplaced (6.165%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T06:04:29.957 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:30.079 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:30.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:30.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:30.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:30.195 INFO:teuthology.orchestra.run.smithi191.stderr:no valid command found; 10 closest matches: 2023-10-09T06:04:30.195 INFO:teuthology.orchestra.run.smithi191.stderr:0 2023-10-09T06:04:30.195 INFO:teuthology.orchestra.run.smithi191.stderr:1 2023-10-09T06:04:30.195 INFO:teuthology.orchestra.run.smithi191.stderr:2 2023-10-09T06:04:30.195 INFO:teuthology.orchestra.run.smithi191.stderr:abort 2023-10-09T06:04:30.195 INFO:teuthology.orchestra.run.smithi191.stderr:assert 2023-10-09T06:04:30.195 INFO:teuthology.orchestra.run.smithi191.stderr:bluefs debug_inject_read_zeros 2023-10-09T06:04:30.196 INFO:teuthology.orchestra.run.smithi191.stderr:bluefs files list 2023-10-09T06:04:30.196 INFO:teuthology.orchestra.run.smithi191.stderr:bluefs stats 2023-10-09T06:04:30.196 INFO:teuthology.orchestra.run.smithi191.stderr:bluestore allocator dump block 2023-10-09T06:04:30.196 INFO:teuthology.orchestra.run.smithi191.stderr:bluestore allocator fragmentation block 2023-10-09T06:04:30.196 INFO:teuthology.orchestra.run.smithi191.stderr:admin_socket: invalid command 2023-10-09T06:04:30.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:29 smithi008 ceph-mon[191227]: pgmap v1616: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 96 KiB/s wr, 405 op/s; 47702/249333 objects degraded (19.132%); 15372/249333 objects misplaced (6.165%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T06:04:30.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:29 smithi008 ceph-mon[184418]: pgmap v1616: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 96 KiB/s wr, 405 op/s; 47702/249333 objects degraded (19.132%); 15372/249333 objects misplaced (6.165%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T06:04:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:30.222 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:30.380 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:30.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:30.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:30.467 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:04:30.467 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:04:30.489 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:30.498 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:30.642 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:30.896 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:30.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:30.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:30.905 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:04:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:04:30] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.33.4" 2023-10-09T06:04:30.951 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:30 smithi191 conmon[392089]: debug 2023-10-09T06:04:30.476+0000 7fe0eaa15200 -1 osd.6 531 log_to_monitors {default=true} 2023-10-09T06:04:30.952 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:30 smithi191 ceph-mon[161669]: from='osd.6 [v2:172.21.15.191:6816/2409825908,v1:172.21.15.191:6817/2409825908]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T06:04:30.952 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:30 smithi191 ceph-mon[161669]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T06:04:31.005 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:31.250 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:30 smithi008 ceph-mon[184418]: from='osd.6 [v2:172.21.15.191:6816/2409825908,v1:172.21.15.191:6817/2409825908]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T06:04:31.250 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:30 smithi008 ceph-mon[184418]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T06:04:31.250 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:30 smithi008 ceph-mon[191227]: from='osd.6 [v2:172.21.15.191:6816/2409825908,v1:172.21.15.191:6817/2409825908]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T06:04:31.250 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:30 smithi008 ceph-mon[191227]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T06:04:31.251 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:04:31.251 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2023-10-09T06:04:31.299 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:04:30 smithi191 conmon[392089]: debug 2023-10-09T06:04:30.949+0000 7fe0dbf0f700 -1 osd.6 531 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:04:31.389 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:31.701 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:31.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:31.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:31.841 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:31.891 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:31.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:31.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:31.999 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:32.032 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[184418]: pgmap v1617: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 100 B/s rd, 94 KiB/s wr, 407 op/s; 47569/248934 objects degraded (19.109%); 15372/248934 objects misplaced (6.175%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T06:04:32.032 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[184418]: failed to encode map e536 with expected crc 2023-10-09T06:04:32.032 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[184418]: failed to encode map e536 with expected crc 2023-10-09T06:04:32.032 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[184418]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T06:04:32.033 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[184418]: osdmap e536: 8 total, 7 up, 7 in 2023-10-09T06:04:32.033 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[184418]: from='osd.6 [v2:172.21.15.191:6816/2409825908,v1:172.21.15.191:6817/2409825908]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T06:04:32.033 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[184418]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T06:04:32.035 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[191227]: pgmap v1617: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 100 B/s rd, 94 KiB/s wr, 407 op/s; 47569/248934 objects degraded (19.109%); 15372/248934 objects misplaced (6.175%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T06:04:32.035 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[191227]: failed to encode map e536 with expected crc 2023-10-09T06:04:32.035 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[191227]: failed to encode map e536 with expected crc 2023-10-09T06:04:32.035 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[191227]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T06:04:32.035 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[191227]: osdmap e536: 8 total, 7 up, 7 in 2023-10-09T06:04:32.035 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[191227]: from='osd.6 [v2:172.21.15.191:6816/2409825908,v1:172.21.15.191:6817/2409825908]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T06:04:32.036 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:31 smithi008 ceph-mon[191227]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T06:04:32.037 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:32.089 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:32.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:32.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:32.195 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:32.220 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:31 smithi191 ceph-mon[161669]: pgmap v1617: 89 pgs: 4 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 100 B/s rd, 94 KiB/s wr, 407 op/s; 47569/248934 objects degraded (19.109%); 15372/248934 objects misplaced (6.175%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T06:04:32.220 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:31 smithi191 ceph-mon[161669]: failed to encode map e536 with expected crc 2023-10-09T06:04:32.221 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:31 smithi191 ceph-mon[161669]: failed to encode map e536 with expected crc 2023-10-09T06:04:32.221 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:31 smithi191 ceph-mon[161669]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T06:04:32.221 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:31 smithi191 ceph-mon[161669]: osdmap e536: 8 total, 7 up, 7 in 2023-10-09T06:04:32.221 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:31 smithi191 ceph-mon[161669]: from='osd.6 [v2:172.21.15.191:6816/2409825908,v1:172.21.15.191:6817/2409825908]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T06:04:32.221 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:31 smithi191 ceph-mon[161669]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2023-10-09T06:04:32.222 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:32.265 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:32.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:32.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:32.372 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:32.402 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:32.455 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:32.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:32.468 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:04:32.561 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:32.593 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:32.648 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:32.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:32.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:32.752 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:32.784 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:32.949 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[184418]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:04:32.949 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[184418]: failed to encode map e537 with expected crc 2023-10-09T06:04:32.949 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[184418]: osd.6 [v2:172.21.15.191:6816/2409825908,v1:172.21.15.191:6817/2409825908] boot 2023-10-09T06:04:32.949 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[184418]: osdmap e537: 8 total, 8 up, 7 in 2023-10-09T06:04:32.949 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:04:32.950 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[184418]: failed to encode map e537 with expected crc 2023-10-09T06:04:32.950 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3983203859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:32.951 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[191227]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:04:32.951 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[191227]: failed to encode map e537 with expected crc 2023-10-09T06:04:32.951 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[191227]: osd.6 [v2:172.21.15.191:6816/2409825908,v1:172.21.15.191:6817/2409825908] boot 2023-10-09T06:04:32.951 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[191227]: osdmap e537: 8 total, 8 up, 7 in 2023-10-09T06:04:32.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:04:32.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[191227]: failed to encode map e537 with expected crc 2023-10-09T06:04:32.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3983203859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:32.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:32.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:04:32.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:32.952 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[191227]: osdmap e538: 8 total, 8 up, 7 in 2023-10-09T06:04:32.962 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:04:33.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:33.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:04:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:33.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:32 smithi008 ceph-mon[184418]: osdmap e538: 8 total, 8 up, 7 in 2023-10-09T06:04:33.225 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:33.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:33.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:33.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:32 smithi191 ceph-mon[161669]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:04:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:32 smithi191 ceph-mon[161669]: failed to encode map e537 with expected crc 2023-10-09T06:04:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:32 smithi191 ceph-mon[161669]: osd.6 [v2:172.21.15.191:6816/2409825908,v1:172.21.15.191:6817/2409825908] boot 2023-10-09T06:04:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:32 smithi191 ceph-mon[161669]: osdmap e537: 8 total, 8 up, 7 in 2023-10-09T06:04:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:04:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:32 smithi191 ceph-mon[161669]: failed to encode map e537 with expected crc 2023-10-09T06:04:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:32 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3983203859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:32 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:32 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:04:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:32 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:33.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:32 smithi191 ceph-mon[161669]: osdmap e538: 8 total, 8 up, 7 in 2023-10-09T06:04:33.330 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:33.361 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:34.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: failed to encode map e532 with expected crc 2023-10-09T06:04:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: failed to encode map e537 with expected crc 2023-10-09T06:04:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: pgmap v1620: 89 pgs: 3 peering, 4 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 4.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 46407/242838 objects degraded (19.110%); 15041/242838 objects misplaced (6.194%); 12 MiB/s, 194 objects/s recovering 2023-10-09T06:04:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 46407/242838 objects degraded (19.110%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: failed to encode map e539 with expected crc 2023-10-09T06:04:34.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: failed to encode map e539 with expected crc 2023-10-09T06:04:34.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: osdmap e539: 8 total, 8 up, 7 in 2023-10-09T06:04:34.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[184418]: failed to encode map e539 with expected crc 2023-10-09T06:04:34.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: failed to encode map e532 with expected crc 2023-10-09T06:04:34.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: failed to encode map e537 with expected crc 2023-10-09T06:04:34.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: pgmap v1620: 89 pgs: 3 peering, 4 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 4.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 46407/242838 objects degraded (19.110%); 15041/242838 objects misplaced (6.194%); 12 MiB/s, 194 objects/s recovering 2023-10-09T06:04:34.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 46407/242838 objects degraded (19.110%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:34.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: failed to encode map e539 with expected crc 2023-10-09T06:04:34.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: failed to encode map e539 with expected crc 2023-10-09T06:04:34.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: osdmap e539: 8 total, 8 up, 7 in 2023-10-09T06:04:34.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:33 smithi008 ceph-mon[191227]: failed to encode map e539 with expected crc 2023-10-09T06:04:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: failed to encode map e532 with expected crc 2023-10-09T06:04:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: failed to encode map e537 with expected crc 2023-10-09T06:04:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: pgmap v1620: 89 pgs: 3 peering, 4 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 10 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 41 active+clean; 4.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 46407/242838 objects degraded (19.110%); 15041/242838 objects misplaced (6.194%); 12 MiB/s, 194 objects/s recovering 2023-10-09T06:04:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: failed to encode map e538 with expected crc 2023-10-09T06:04:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 46407/242838 objects degraded (19.110%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:04:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: failed to encode map e539 with expected crc 2023-10-09T06:04:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: failed to encode map e539 with expected crc 2023-10-09T06:04:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: osdmap e539: 8 total, 8 up, 7 in 2023-10-09T06:04:34.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:33 smithi191 ceph-mon[161669]: failed to encode map e539 with expected crc 2023-10-09T06:04:34.437 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:34.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:34.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:34.561 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:34.585 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:34.657 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:34.787 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:34.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:34.895 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:34.972 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:35.070 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[184418]: failed to encode map e539 with expected crc 2023-10-09T06:04:35.070 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[184418]: failed to encode map e539 with expected crc 2023-10-09T06:04:35.070 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[184418]: failed to encode map e540 with expected crc 2023-10-09T06:04:35.070 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[184418]: osdmap e540: 8 total, 8 up, 7 in 2023-10-09T06:04:35.070 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[184418]: failed to encode map e540 with expected crc 2023-10-09T06:04:35.070 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[184418]: failed to encode map e540 with expected crc 2023-10-09T06:04:35.072 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[191227]: failed to encode map e539 with expected crc 2023-10-09T06:04:35.072 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[191227]: failed to encode map e539 with expected crc 2023-10-09T06:04:35.072 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[191227]: failed to encode map e540 with expected crc 2023-10-09T06:04:35.072 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[191227]: osdmap e540: 8 total, 8 up, 7 in 2023-10-09T06:04:35.072 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[191227]: failed to encode map e540 with expected crc 2023-10-09T06:04:35.073 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:34 smithi008 ceph-mon[191227]: failed to encode map e540 with expected crc 2023-10-09T06:04:35.146 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:35.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:35.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:35.187 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:04:35.188 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:04:35.256 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:35.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:34 smithi191 ceph-mon[161669]: failed to encode map e539 with expected crc 2023-10-09T06:04:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:34 smithi191 ceph-mon[161669]: failed to encode map e539 with expected crc 2023-10-09T06:04:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:34 smithi191 ceph-mon[161669]: failed to encode map e540 with expected crc 2023-10-09T06:04:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:34 smithi191 ceph-mon[161669]: osdmap e540: 8 total, 8 up, 7 in 2023-10-09T06:04:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:34 smithi191 ceph-mon[161669]: failed to encode map e540 with expected crc 2023-10-09T06:04:35.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:34 smithi191 ceph-mon[161669]: failed to encode map e540 with expected crc 2023-10-09T06:04:35.361 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:35.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[191227]: failed to encode map e540 with expected crc 2023-10-09T06:04:35.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[191227]: failed to encode map e540 with expected crc 2023-10-09T06:04:35.971 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[191227]: pgmap v1624: 89 pgs: 1 creating+peering, 3 peering, 4 active+clean+remapped, 19 active+undersized, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 44 active+clean; 4.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 436 op/s; 41804/242556 objects degraded (17.235%); 14953/242556 objects misplaced (6.165%); 11 MiB/s, 170 objects/s recovering 2023-10-09T06:04:36.252 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:04:36.278 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:35 smithi191 ceph-mon[161669]: failed to encode map e540 with expected crc 2023-10-09T06:04:36.278 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:35 smithi191 ceph-mon[161669]: failed to encode map e540 with expected crc 2023-10-09T06:04:36.278 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:35 smithi191 ceph-mon[161669]: pgmap v1624: 89 pgs: 1 creating+peering, 3 peering, 4 active+clean+remapped, 19 active+undersized, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 44 active+clean; 4.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 436 op/s; 41804/242556 objects degraded (17.235%); 14953/242556 objects misplaced (6.165%); 11 MiB/s, 170 objects/s recovering 2023-10-09T06:04:36.278 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:35 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:04:36.279 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:35 smithi191 ceph-mon[161669]: failed to encode map e541 with expected crc 2023-10-09T06:04:36.279 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:35 smithi191 ceph-mon[161669]: failed to encode map e541 with expected crc 2023-10-09T06:04:36.279 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:35 smithi191 ceph-mon[161669]: osdmap e541: 8 total, 8 up, 7 in 2023-10-09T06:04:36.459 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:04:36.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[184418]: failed to encode map e540 with expected crc 2023-10-09T06:04:36.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[184418]: failed to encode map e540 with expected crc 2023-10-09T06:04:36.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[184418]: pgmap v1624: 89 pgs: 1 creating+peering, 3 peering, 4 active+clean+remapped, 19 active+undersized, 8 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 44 active+clean; 4.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 436 op/s; 41804/242556 objects degraded (17.235%); 14953/242556 objects misplaced (6.165%); 11 MiB/s, 170 objects/s recovering 2023-10-09T06:04:36.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:04:36.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[184418]: failed to encode map e541 with expected crc 2023-10-09T06:04:36.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[184418]: failed to encode map e541 with expected crc 2023-10-09T06:04:36.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[184418]: osdmap e541: 8 total, 8 up, 7 in 2023-10-09T06:04:36.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:04:36.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[191227]: failed to encode map e541 with expected crc 2023-10-09T06:04:36.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[191227]: failed to encode map e541 with expected crc 2023-10-09T06:04:36.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:35 smithi008 ceph-mon[191227]: osdmap e541: 8 total, 8 up, 7 in 2023-10-09T06:04:36.665 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:04:36.690 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:36.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:36.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:36.817 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:36.889 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:37.205 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:37 smithi191 ceph-mon[161669]: failed to encode map e540 with expected crc 2023-10-09T06:04:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:37 smithi008 ceph-mon[184418]: failed to encode map e540 with expected crc 2023-10-09T06:04:37.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:37 smithi008 ceph-mon[191227]: failed to encode map e540 with expected crc 2023-10-09T06:04:37.328 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:37.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:37.446 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:37.499 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:37.682 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:04:37.765 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:37.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:37.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:37.871 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:37.908 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:37.954 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:37.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:37.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:37.966 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:38.057 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:38.152 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:38.271 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:38 smithi008 ceph-mon[184418]: pgmap v1626: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 creating+peering, 3 peering, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 24861/238590 objects degraded (10.420%); 2979/238590 objects misplaced (1.249%); 4.9 MiB/s, 82 objects/s recovering 2023-10-09T06:04:38.272 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:38 smithi008 ceph-mon[191227]: pgmap v1626: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 creating+peering, 3 peering, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 24861/238590 objects degraded (10.420%); 2979/238590 objects misplaced (1.249%); 4.9 MiB/s, 82 objects/s recovering 2023-10-09T06:04:38.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:38 smithi191 ceph-mon[161669]: pgmap v1626: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 creating+peering, 3 peering, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 24861/238590 objects degraded (10.420%); 2979/238590 objects misplaced (1.249%); 4.9 MiB/s, 82 objects/s recovering 2023-10-09T06:04:38.495 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:38.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:38.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:38.608 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:38.643 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:38.673 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:04:38.714 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:38.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:38.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:38.731 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:04:38.779 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:04:38.816 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:04:38.821 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:38.864 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:38.878 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:04:38.909 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:04:38.910 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T06:04:38.910 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:04:38.910 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.47036.0:96371 10.d e541/538 10:bbf88de7:::benchmark_data_smithi008_634586_object9206:head v 541'6200, mlcod=515'614)", 2023-10-09T06:04:38.910 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:04:38.907515+0000", 2023-10-09T06:04:38.910 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0016404239999999999, 2023-10-09T06:04:38.910 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001673136, 2023-10-09T06:04:38.910 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:04:38.910 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T06:04:38.910 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:04:38.910 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:38.907515+0000", 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:38.907515+0000", 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:04:38.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:38.907513+0000", 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0738e-05 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:38.907524+0000", 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5998000000000002e-05 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:38.907540+0000", 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3200000000000002e-06 2023-10-09T06:04:38.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:38.907547+0000", 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2500000000000001e-05 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:38.907610+0000", 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7319000000000001e-05 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:04:38.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:38.907637+0000", 2023-10-09T06:04:38.914 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012232700000000001 2023-10-09T06:04:38.914 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:04:38.914 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:04:38.914 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:04:38.914 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:04:38.914 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T06:04:38.914 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 1 2023-10-09T06:04:38.914 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:04:38.922 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:04:38.934 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:38.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:38.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:38.956 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:04:39.001 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:04:39.005 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:04:39.005 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:04:39.005 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:04:39.005 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:04:39.005 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:04:39.005 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:04:39.005 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:04:39.005 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:04:39.041 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:39.137 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:39.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:39 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24861/238590 objects degraded (10.420%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:39.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1442699788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:39.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:39 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24861/238590 objects degraded (10.420%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:39.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1442699788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:39.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:39 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24861/238590 objects degraded (10.420%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:39.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1442699788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:39.945 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:39.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:39.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:40.056 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:40.131 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:40.699 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:40 smithi008 ceph-mon[191227]: pgmap v1627: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 creating+peering, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 258 op/s; 24861/238263 objects degraded (10.434%); 2979/238263 objects misplaced (1.250%); 3.3 MiB/s, 55 objects/s recovering 2023-10-09T06:04:40.700 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:04:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:04:40] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.33.4" 2023-10-09T06:04:40.700 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:40 smithi008 ceph-mon[184418]: pgmap v1627: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 creating+peering, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 258 op/s; 24861/238263 objects degraded (10.434%); 2979/238263 objects misplaced (1.250%); 3.3 MiB/s, 55 objects/s recovering 2023-10-09T06:04:40.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:40 smithi191 ceph-mon[161669]: pgmap v1627: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 creating+peering, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 258 op/s; 24861/238263 objects degraded (10.434%); 2979/238263 objects misplaced (1.250%); 3.3 MiB/s, 55 objects/s recovering 2023-10-09T06:04:40.894 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:40.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:40.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:40.958 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:41.008 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:41.089 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:41.349 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:41.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:41.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:41.454 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:41.486 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:41.522 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:41 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:04:41.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:41 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:04:41.546 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:41.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:41.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:41.656 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:41.684 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:41.737 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:41.742 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:41 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:04:41.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:41.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:41.844 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:41.922 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:42.064 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:42.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:42.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:42.172 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:42.239 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:42.335 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:42.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:42.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:42.438 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:42.469 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:42.491 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:42 smithi008 ceph-mon[184418]: pgmap v1628: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 286 op/s; 24861/236442 objects degraded (10.515%); 2979/236442 objects misplaced (1.260%); 4.6 MiB/s, 76 objects/s recovering 2023-10-09T06:04:42.491 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4100087011' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:42.491 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:42 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:42.492 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:42 smithi008 ceph-mon[191227]: pgmap v1628: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 286 op/s; 24861/236442 objects degraded (10.515%); 2979/236442 objects misplaced (1.260%); 4.6 MiB/s, 76 objects/s recovering 2023-10-09T06:04:42.492 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4100087011' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:42.492 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:42 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:42.528 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:04:42.605 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:42.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:42.711 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:42.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:42 smithi191 ceph-mon[161669]: pgmap v1628: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 286 op/s; 24861/236442 objects degraded (10.515%); 2979/236442 objects misplaced (1.260%); 4.6 MiB/s, 76 objects/s recovering 2023-10-09T06:04:42.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4100087011' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:42.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:42 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:42.741 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:42.804 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:42.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:42.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:42.910 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:42.945 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:42.988 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:04:43.004 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:43.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:43.109 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:43.187 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:43.337 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:43.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:43.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:43.447 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:43.473 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:43.515 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:43.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:43.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:43.621 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:43.650 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:43.706 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:43.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:43.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[184418]: failed to encode map e542 with expected crc 2023-10-09T06:04:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:04:43.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[184418]: osdmap e542: 8 total, 8 up, 7 in 2023-10-09T06:04:43.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[184418]: failed to encode map e542 with expected crc 2023-10-09T06:04:43.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24861/236442 objects degraded (10.515%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:43.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:43.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[191227]: failed to encode map e542 with expected crc 2023-10-09T06:04:43.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:04:43.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[191227]: osdmap e542: 8 total, 8 up, 7 in 2023-10-09T06:04:43.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[191227]: failed to encode map e542 with expected crc 2023-10-09T06:04:43.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24861/236442 objects degraded (10.515%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:43 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:43 smithi191 ceph-mon[161669]: failed to encode map e542 with expected crc 2023-10-09T06:04:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:43 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:04:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:43 smithi191 ceph-mon[161669]: osdmap e542: 8 total, 8 up, 7 in 2023-10-09T06:04:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:43 smithi191 ceph-mon[161669]: failed to encode map e542 with expected crc 2023-10-09T06:04:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24861/236442 objects degraded (10.515%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:43.813 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:43.901 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:44.232 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:44.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:44.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:44.340 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:44.412 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:44.456 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:44.506 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:44 smithi008 ceph-mon[184418]: pgmap v1630: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 24577/231552 objects degraded (10.614%); 2813/231552 objects misplaced (1.215%); 9.0 MiB/s, 145 objects/s recovering 2023-10-09T06:04:44.506 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:44.506 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:44.583 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:44.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:44.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:44.594 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:04:44.594 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:04:44.594 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T06:04:44.594 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T06:04:44.594 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:04:44.693 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:44.700 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-09T06:04:44.700 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:04:44.700 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T06:04:44.700 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:04:44.765 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:44 smithi008 ceph-mon[191227]: pgmap v1630: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 24577/231552 objects degraded (10.614%); 2813/231552 objects misplaced (1.215%); 9.0 MiB/s, 145 objects/s recovering 2023-10-09T06:04:44.766 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:44.766 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:44.799 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:44.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:44 smithi191 ceph-mon[161669]: pgmap v1630: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 24577/231552 objects degraded (10.614%); 2813/231552 objects misplaced (1.215%); 9.0 MiB/s, 145 objects/s recovering 2023-10-09T06:04:44.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:04:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:45.024 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:45.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:45.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:45.134 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:45.186 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:45.282 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:45.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:45.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:45.473 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:45.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:45.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:45.584 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:45.618 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:45.674 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:45.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:45.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:45.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:45.815 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:45.877 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:45.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:45.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:45.987 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:46.095 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:04:46.105 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:46.110 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:04:46.255 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:46.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:46.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:46.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:46.398 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:46.443 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:46.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:46.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:46.548 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:46.649 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:46.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:46 smithi191 ceph-mon[161669]: pgmap v1631: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 24577/229176 objects degraded (10.724%); 2813/229176 objects misplaced (1.227%); 7.6 MiB/s, 123 objects/s recovering 2023-10-09T06:04:46.953 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:46 smithi008 ceph-mon[184418]: pgmap v1631: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 24577/229176 objects degraded (10.724%); 2813/229176 objects misplaced (1.227%); 7.6 MiB/s, 123 objects/s recovering 2023-10-09T06:04:46.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:46 smithi008 ceph-mon[191227]: pgmap v1631: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 24577/229176 objects degraded (10.724%); 2813/229176 objects misplaced (1.227%); 7.6 MiB/s, 123 objects/s recovering 2023-10-09T06:04:46.962 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:46.997 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:47.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:47.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:47.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:47.168 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:47.193 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:47.223 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:47.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:47.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:47.329 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:47.429 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:47.795 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:47.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:47.905 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:47.934 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:47.988 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:47.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:47.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:48.006 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:48.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:48.109 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:04:48.130 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:48.147 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:04:48.187 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:48.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:48.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:48.196 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:04:48.235 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:04:48.275 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:04:48.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:48.324 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:48.325 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:04:48.375 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:04:48.390 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:48.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:48.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:48.420 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:04:48.423 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:04:48.423 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:04:48.423 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:04:48.423 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:04:48.423 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:04:48.424 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:04:48.424 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:04:48.424 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:04:48.496 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:48.583 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:48.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:48 smithi008 ceph-mon[184418]: pgmap v1632: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 24260/224556 objects degraded (10.804%); 2669/224556 objects misplaced (1.189%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:04:48.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:48 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3519862500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:48.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24260/224556 objects degraded (10.804%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:48.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:48 smithi008 ceph-mon[191227]: pgmap v1632: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 24260/224556 objects degraded (10.804%); 2669/224556 objects misplaced (1.189%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:04:48.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:48 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3519862500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:48.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24260/224556 objects degraded (10.804%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:48.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:48 smithi191 ceph-mon[161669]: pgmap v1632: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 24260/224556 objects degraded (10.804%); 2669/224556 objects misplaced (1.189%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:04:48.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:48 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3519862500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:48.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24260/224556 objects degraded (10.804%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:49.484 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:49.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:49.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:49.590 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:49.623 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:49.680 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:49.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:49.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:49.787 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:49.866 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:49 smithi008 ceph-mon[184418]: pgmap v1633: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 24260/224055 objects degraded (10.828%); 2669/224055 objects misplaced (1.191%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:04:49.867 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:49 smithi008 ceph-mon[191227]: pgmap v1633: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 24260/224055 objects degraded (10.828%); 2669/224055 objects misplaced (1.191%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:04:49.867 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:50.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:49 smithi191 ceph-mon[161669]: pgmap v1633: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 24260/224055 objects degraded (10.828%); 2669/224055 objects misplaced (1.191%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:04:50.966 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:04:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:04:50] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.33.4" 2023-10-09T06:04:51.106 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:51.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:51.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:51.228 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:51.338 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:51.548 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:51.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:51.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:51.668 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:51.698 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:51.758 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:51.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:51.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:51.865 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:51.895 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:51.950 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:51.956 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:51 smithi191 ceph-mon[161669]: pgmap v1634: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 24260/221742 objects degraded (10.941%); 2669/221742 objects misplaced (1.204%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T06:04:51.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:51.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:52.058 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:52.166 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:51 smithi008 ceph-mon[184418]: pgmap v1634: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 24260/221742 objects degraded (10.941%); 2669/221742 objects misplaced (1.204%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T06:04:52.166 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:51 smithi008 ceph-mon[191227]: pgmap v1634: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 24260/221742 objects degraded (10.941%); 2669/221742 objects misplaced (1.204%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T06:04:52.180 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:52.442 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:04:52.458 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T06:04:52.939 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:52 smithi008 ceph-mon[191227]: from='client.47165 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:04:52.946 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:52.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:52.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:53.052 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:53.083 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:53.132 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:53.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:53.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:52 smithi008 ceph-mon[184418]: from='client.47165 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:04:53.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:53.267 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:52 smithi191 ceph-mon[161669]: from='client.47165 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:04:53.269 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:53.326 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:53.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:53.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:53.335 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:04:53.435 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:53.576 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:54.189 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:54.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:54.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:54.198 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:53 smithi008 ceph-mon[184418]: pgmap v1635: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 23948/216678 objects degraded (11.052%); 2521/216678 objects misplaced (1.163%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T06:04:54.198 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23948/216678 objects degraded (11.052%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:54.199 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:53 smithi008 ceph-mon[191227]: pgmap v1635: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 23948/216678 objects degraded (11.052%); 2521/216678 objects misplaced (1.163%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T06:04:54.199 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23948/216678 objects degraded (11.052%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:54.299 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:53 smithi191 ceph-mon[161669]: pgmap v1635: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 23948/216678 objects degraded (11.052%); 2521/216678 objects misplaced (1.163%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T06:04:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23948/216678 objects degraded (11.052%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:54.355 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1634,"stamp":"2023-10-09T06:04:50.844877+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4846917579,"num_objects":73914,"num_object_clones":0,"num_object_copies":221742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24260,"num_objects_misplaced":2669,"num_objects_unfound":0,"num_objects_dirty":73914,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":101546,"num_write_kb":5593510,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10742,"num_bytes_recovered":698112344,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100910,"ondisk_log_size":100910,"up":267,"acting":257,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":21435028,"kb_used_data":15193692,"kb_used_omap":59,"kb_used_meta":6241092,"kb_avail":634781036,"statfs":{"total":671965249536,"available":650015780864,"internally_reserved":0,"allocated":15558340608,"data_stored":15545923023,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":60463,"internal_metadata":6390879185},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2,5,3,9,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-321126400,"num_objects":-4900,"num_object_clones":0,"num_object_copies":-14700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-601,"num_objects_misplaced":-310,"num_objects_unfound":0,"num_objects_dirty":-4900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4900,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":911,"num_bytes_recovered":59703296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.012645"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":165,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.416384+0000","last_change":"2023-10-09T05:54:59.256087+0000","last_active":"2023-10-09T06:04:35.416384+0000","last_peered":"2023-10-09T06:04:35.416384+0000","last_clean":"2023-10-09T06:04:35.416384+0000","last_became_active":"2023-10-09T05:54:59.255773+0000","last_became_peered":"2023-10-09T05:54:59.255773+0000","last_unstale":"2023-10-09T06:04:35.416384+0000","last_undegraded":"2023-10-09T06:04:35.416384+0000","last_fullsized":"2023-10-09T06:04:35.416384+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:19.545754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":140,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.411448+0000","last_change":"2023-10-09T05:56:00.247579+0000","last_active":"2023-10-09T06:04:35.411448+0000","last_peered":"2023-10-09T06:04:35.411448+0000","last_clean":"2023-10-09T06:04:35.411448+0000","last_became_active":"2023-10-09T05:56:00.238151+0000","last_became_peered":"2023-10-09T05:56:00.238151+0000","last_unstale":"2023-10-09T06:04:35.411448+0000","last_undegraded":"2023-10-09T06:04:35.411448+0000","last_fullsized":"2023-10-09T06:04:35.411448+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:05.692190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":271,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:04:42.497227+0000","last_change":"2023-10-09T06:04:32.949398+0000","last_active":"2023-10-09T06:04:42.497227+0000","last_peered":"2023-10-09T06:04:42.497227+0000","last_clean":"2023-10-09T06:04:42.497227+0000","last_became_active":"2023-10-09T06:04:32.949063+0000","last_became_peered":"2023-10-09T06:04:32.949063+0000","last_unstale":"2023-10-09T06:04:42.497227+0000","last_undegraded":"2023-10-09T06:04:42.497227+0000","last_fullsized":"2023-10-09T06:04:42.497227+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:12.407318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":609,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.405243+0000","last_change":"2023-10-09T06:03:56.489473+0000","last_active":"2023-10-09T06:04:35.405243+0000","last_peered":"2023-10-09T06:04:35.405243+0000","last_clean":"2023-10-09T06:04:35.405243+0000","last_became_active":"2023-10-09T06:03:56.489102+0000","last_became_peered":"2023-10-09T06:03:56.489102+0000","last_unstale":"2023-10-09T06:04:35.405243+0000","last_undegraded":"2023-10-09T06:04:35.405243+0000","last_fullsized":"2023-10-09T06:04:35.405243+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":453,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.410741+0000","last_change":"2023-10-09T06:03:56.479324+0000","last_active":"2023-10-09T06:04:35.410741+0000","last_peered":"2023-10-09T06:04:35.410741+0000","last_clean":"2023-10-09T06:04:35.410741+0000","last_became_active":"2023-10-09T06:03:56.479014+0000","last_became_peered":"2023-10-09T06:03:56.479014+0000","last_unstale":"2023-10-09T06:04:35.410741+0000","last_undegraded":"2023-10-09T06:04:35.410741+0000","last_fullsized":"2023-10-09T06:04:35.410741+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:07.053268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":103,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.421814+0000","last_change":"2023-10-09T06:04:33.125479+0000","last_active":"2023-10-09T06:04:35.421814+0000","last_peered":"2023-10-09T06:04:35.421814+0000","last_clean":"2023-10-09T06:04:35.421814+0000","last_became_active":"2023-10-09T06:04:33.125135+0000","last_became_peered":"2023-10-09T06:04:33.125135+0000","last_unstale":"2023-10-09T06:04:35.421814+0000","last_undegraded":"2023-10-09T06:04:35.421814+0000","last_fullsized":"2023-10-09T06:04:35.421814+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:33.288989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":497,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.421915+0000","last_change":"2023-10-09T06:04:33.126133+0000","last_active":"2023-10-09T06:04:35.421915+0000","last_peered":"2023-10-09T06:04:35.421915+0000","last_clean":"2023-10-09T06:04:35.421915+0000","last_became_active":"2023-10-09T06:04:33.125857+0000","last_became_peered":"2023-10-09T06:04:33.125857+0000","last_unstale":"2023-10-09T06:04:35.421915+0000","last_undegraded":"2023-10-09T06:04:35.421915+0000","last_fullsized":"2023-10-09T06:04:35.421915+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:04.644424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":173,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.410446+0000","last_change":"2023-10-09T06:04:32.950240+0000","last_active":"2023-10-09T06:04:35.410446+0000","last_peered":"2023-10-09T06:04:35.410446+0000","last_clean":"2023-10-09T06:04:35.410446+0000","last_became_active":"2023-10-09T06:04:32.949852+0000","last_became_peered":"2023-10-09T06:04:32.949852+0000","last_unstale":"2023-10-09T06:04:35.410446+0000","last_undegraded":"2023-10-09T06:04:35.410446+0000","last_fullsized":"2023-10-09T06:04:35.410446+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:04.138187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":208,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.405509+0000","last_change":"2023-10-09T06:03:09.369870+0000","last_active":"2023-10-09T06:04:35.405509+0000","last_peered":"2023-10-09T06:04:35.405509+0000","last_clean":"2023-10-09T06:04:35.405509+0000","last_became_active":"2023-10-09T06:03:09.369216+0000","last_became_peered":"2023-10-09T06:03:09.369216+0000","last_unstale":"2023-10-09T06:04:35.405509+0000","last_undegraded":"2023-10-09T06:04:35.405509+0000","last_fullsized":"2023-10-09T06:04:35.405509+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":208,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404481+0000","last_change":"2023-10-09T06:04:32.956316+0000","last_active":"2023-10-09T06:04:35.404481+0000","last_peered":"2023-10-09T06:04:35.404481+0000","last_clean":"2023-10-09T06:04:35.404481+0000","last_became_active":"2023-10-09T06:04:32.955969+0000","last_became_peered":"2023-10-09T06:04:32.955969+0000","last_unstale":"2023-10-09T06:04:35.404481+0000","last_undegraded":"2023-10-09T06:04:35.404481+0000","last_fullsized":"2023-10-09T06:04:35.404481+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":174,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404847+0000","last_change":"2023-10-09T05:56:00.348771+0000","last_active":"2023-10-09T06:04:35.404847+0000","last_peered":"2023-10-09T06:04:35.404847+0000","last_clean":"2023-10-09T06:04:35.404847+0000","last_became_active":"2023-10-09T05:56:00.224347+0000","last_became_peered":"2023-10-09T05:56:00.224347+0000","last_unstale":"2023-10-09T06:04:35.404847+0000","last_undegraded":"2023-10-09T06:04:35.404847+0000","last_fullsized":"2023-10-09T06:04:35.404847+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":356,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.416461+0000","last_change":"2023-10-09T06:04:32.957724+0000","last_active":"2023-10-09T06:04:35.416461+0000","last_peered":"2023-10-09T06:04:35.416461+0000","last_clean":"2023-10-09T06:04:35.416461+0000","last_became_active":"2023-10-09T06:04:32.957429+0000","last_became_peered":"2023-10-09T06:04:32.957429+0000","last_unstale":"2023-10-09T06:04:35.416461+0000","last_undegraded":"2023-10-09T06:04:35.416461+0000","last_fullsized":"2023-10-09T06:04:35.416461+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:32.175240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":215,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404837+0000","last_change":"2023-10-09T06:04:33.403369+0000","last_active":"2023-10-09T06:04:35.404837+0000","last_peered":"2023-10-09T06:04:35.404837+0000","last_clean":"2023-10-09T06:04:35.404837+0000","last_became_active":"2023-10-09T06:04:33.402983+0000","last_became_peered":"2023-10-09T06:04:33.402983+0000","last_unstale":"2023-10-09T06:04:35.404837+0000","last_undegraded":"2023-10-09T06:04:35.404837+0000","last_fullsized":"2023-10-09T06:04:35.404837+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":553,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.413521+0000","last_change":"2023-10-09T06:04:32.954270+0000","last_active":"2023-10-09T06:04:35.413521+0000","last_peered":"2023-10-09T06:04:35.413521+0000","last_clean":"2023-10-09T06:04:35.413521+0000","last_became_active":"2023-10-09T06:04:32.953935+0000","last_became_peered":"2023-10-09T06:04:32.953935+0000","last_unstale":"2023-10-09T06:04:35.413521+0000","last_undegraded":"2023-10-09T06:04:35.413521+0000","last_fullsized":"2023-10-09T06:04:35.413521+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:37.569469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":191,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404397+0000","last_change":"2023-10-09T06:04:33.411665+0000","last_active":"2023-10-09T06:04:35.404397+0000","last_peered":"2023-10-09T06:04:35.404397+0000","last_clean":"2023-10-09T06:04:35.404397+0000","last_became_active":"2023-10-09T06:04:33.411254+0000","last_became_peered":"2023-10-09T06:04:33.411254+0000","last_unstale":"2023-10-09T06:04:35.404397+0000","last_undegraded":"2023-10-09T06:04:35.404397+0000","last_fullsized":"2023-10-09T06:04:35.404397+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":186,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404760+0000","last_change":"2023-10-09T06:03:56.484013+0000","last_active":"2023-10-09T06:04:35.404760+0000","last_peered":"2023-10-09T06:04:35.404760+0000","last_clean":"2023-10-09T06:04:35.404760+0000","last_became_active":"2023-10-09T06:03:56.483684+0000","last_became_peered":"2023-10-09T06:03:56.483684+0000","last_unstale":"2023-10-09T06:04:35.404760+0000","last_undegraded":"2023-10-09T06:04:35.404760+0000","last_fullsized":"2023-10-09T06:04:35.404760+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":583,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404286+0000","last_change":"2023-10-09T06:03:56.483873+0000","last_active":"2023-10-09T06:04:35.404286+0000","last_peered":"2023-10-09T06:04:35.404286+0000","last_clean":"2023-10-09T06:04:35.404286+0000","last_became_active":"2023-10-09T06:03:56.483250+0000","last_became_peered":"2023-10-09T06:03:56.483250+0000","last_unstale":"2023-10-09T06:04:35.404286+0000","last_undegraded":"2023-10-09T06:04:35.404286+0000","last_fullsized":"2023-10-09T06:04:35.404286+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":260,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.405618+0000","last_change":"2023-10-09T06:04:32.952030+0000","last_active":"2023-10-09T06:04:35.405618+0000","last_peered":"2023-10-09T06:04:35.405618+0000","last_clean":"2023-10-09T06:04:35.405618+0000","last_became_active":"2023-10-09T06:04:32.951603+0000","last_became_peered":"2023-10-09T06:04:32.951603+0000","last_unstale":"2023-10-09T06:04:35.405618+0000","last_undegraded":"2023-10-09T06:04:35.405618+0000","last_fullsized":"2023-10-09T06:04:35.405618+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":190,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404410+0000","last_change":"2023-10-09T06:04:32.956628+0000","last_active":"2023-10-09T06:04:35.404410+0000","last_peered":"2023-10-09T06:04:35.404410+0000","last_clean":"2023-10-09T06:04:35.404410+0000","last_became_active":"2023-10-09T06:04:32.956260+0000","last_became_peered":"2023-10-09T06:04:32.956260+0000","last_unstale":"2023-10-09T06:04:35.404410+0000","last_undegraded":"2023-10-09T06:04:35.404410+0000","last_fullsized":"2023-10-09T06:04:35.404410+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":553,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.405325+0000","last_change":"2023-10-09T06:03:56.483827+0000","last_active":"2023-10-09T06:04:35.405325+0000","last_peered":"2023-10-09T06:04:35.405325+0000","last_clean":"2023-10-09T06:04:35.405325+0000","last_became_active":"2023-10-09T06:03:56.483584+0000","last_became_peered":"2023-10-09T06:03:56.483584+0000","last_unstale":"2023-10-09T06:04:35.405325+0000","last_undegraded":"2023-10-09T06:04:35.405325+0000","last_fullsized":"2023-10-09T06:04:35.405325+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":58,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.410535+0000","last_change":"2023-10-09T06:04:32.957950+0000","last_active":"2023-10-09T06:04:35.410535+0000","last_peered":"2023-10-09T06:04:35.410535+0000","last_clean":"2023-10-09T06:04:35.410535+0000","last_became_active":"2023-10-09T06:04:32.957293+0000","last_became_peered":"2023-10-09T06:04:32.957293+0000","last_unstale":"2023-10-09T06:04:35.410535+0000","last_undegraded":"2023-10-09T06:04:35.410535+0000","last_fullsized":"2023-10-09T06:04:35.410535+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:18.950826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":185,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.406076+0000","last_change":"2023-10-09T06:03:09.369599+0000","last_active":"2023-10-09T06:04:35.406076+0000","last_peered":"2023-10-09T06:04:35.406076+0000","last_clean":"2023-10-09T06:04:35.406076+0000","last_became_active":"2023-10-09T06:03:09.369274+0000","last_became_peered":"2023-10-09T06:03:09.369274+0000","last_unstale":"2023-10-09T06:04:35.406076+0000","last_undegraded":"2023-10-09T06:04:35.406076+0000","last_fullsized":"2023-10-09T06:04:35.406076+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"542'6264","reported_seq":6304,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:04:47.430328+0000","last_change":"2023-10-09T06:02:25.343221+0000","last_active":"2023-10-09T06:04:47.430328+0000","last_peered":"2023-10-09T06:04:47.430328+0000","last_clean":"2023-10-09T06:04:47.430328+0000","last_became_active":"2023-10-09T06:02:25.342973+0000","last_became_peered":"2023-10-09T06:02:25.342973+0000","last_unstale":"2023-10-09T06:04:47.430328+0000","last_undegraded":"2023-10-09T06:04:47.430328+0000","last_fullsized":"2023-10-09T06:04:47.430328+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6264,"log_dups_size":0,"ondisk_log_size":6264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:44.755600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299761664,"num_objects":4574,"num_object_clones":0,"num_object_copies":13722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6264,"num_write_kb":346816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":209,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.406163+0000","last_change":"2023-10-09T06:03:56.617740+0000","last_active":"2023-10-09T06:04:35.406163+0000","last_peered":"2023-10-09T06:04:35.406163+0000","last_clean":"2023-10-09T06:04:35.406163+0000","last_became_active":"2023-10-09T06:03:56.481801+0000","last_became_peered":"2023-10-09T06:03:56.481801+0000","last_unstale":"2023-10-09T06:04:35.406163+0000","last_undegraded":"2023-10-09T06:04:35.406163+0000","last_fullsized":"2023-10-09T06:04:35.406163+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":229,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404556+0000","last_change":"2023-10-09T06:04:32.969529+0000","last_active":"2023-10-09T06:04:35.404556+0000","last_peered":"2023-10-09T06:04:35.404556+0000","last_clean":"2023-10-09T06:04:35.404556+0000","last_became_active":"2023-10-09T06:04:32.969220+0000","last_became_peered":"2023-10-09T06:04:32.969220+0000","last_unstale":"2023-10-09T06:04:35.404556+0000","last_undegraded":"2023-10-09T06:04:35.404556+0000","last_fullsized":"2023-10-09T06:04:35.404556+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"542'6369","reported_seq":8043,"reported_epoch":542,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:04:45.961418+0000","last_change":"2023-10-09T06:04:33.595579+0000","last_active":"2023-10-09T06:04:45.961418+0000","last_peered":"2023-10-09T06:04:45.961418+0000","last_clean":"2023-10-09T06:03:08.351210+0000","last_became_active":"2023-10-09T06:04:33.580659+0000","last_became_peered":"2023-10-09T06:04:33.580659+0000","last_unstale":"2023-10-09T06:04:45.961418+0000","last_undegraded":"2023-10-09T06:04:33.499150+0000","last_fullsized":"2023-10-09T06:04:33.389578+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6369,"log_dups_size":0,"ondisk_log_size":6369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:17:23.213426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312016896,"num_objects":4761,"num_object_clones":0,"num_object_copies":14283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3176,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6369,"num_write_kb":356160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":804,"num_bytes_recovered":52690944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,3],"avail_no_missing":["2"],"object_location_counts":[{"shards":"2,3","objects":2021},{"shards":"2,4,6","objects":2740}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":167,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.405420+0000","last_change":"2023-10-09T06:03:09.468231+0000","last_active":"2023-10-09T06:04:35.405420+0000","last_peered":"2023-10-09T06:04:35.405420+0000","last_clean":"2023-10-09T06:04:35.405420+0000","last_became_active":"2023-10-09T06:03:09.371820+0000","last_became_peered":"2023-10-09T06:03:09.371820+0000","last_unstale":"2023-10-09T06:04:35.405420+0000","last_undegraded":"2023-10-09T06:04:35.405420+0000","last_fullsized":"2023-10-09T06:04:35.405420+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":549,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.400512+0000","last_change":"2023-10-09T06:03:09.372301+0000","last_active":"2023-10-09T06:04:35.400512+0000","last_peered":"2023-10-09T06:04:35.400512+0000","last_clean":"2023-10-09T06:04:35.400512+0000","last_became_active":"2023-10-09T06:03:09.371908+0000","last_became_peered":"2023-10-09T06:03:09.371908+0000","last_unstale":"2023-10-09T06:04:35.400512+0000","last_undegraded":"2023-10-09T06:04:35.400512+0000","last_fullsized":"2023-10-09T06:04:35.400512+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"542'6114","reported_seq":6581,"reported_epoch":542,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:04:46.362762+0000","last_change":"2023-10-09T06:04:05.675141+0000","last_active":"2023-10-09T06:04:46.362762+0000","last_peered":"2023-10-09T06:04:46.362762+0000","last_clean":"2023-10-09T06:03:08.351342+0000","last_became_active":"2023-10-09T06:03:10.569384+0000","last_became_peered":"2023-10-09T06:03:10.569384+0000","last_unstale":"2023-10-09T06:04:46.362762+0000","last_undegraded":"2023-10-09T06:04:05.673432+0000","last_fullsized":"2023-10-09T06:03:10.361615+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6114,"log_dups_size":0,"ondisk_log_size":6114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295567360,"num_objects":4510,"num_object_clones":0,"num_object_copies":13530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3215,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6114,"num_write_kb":339968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":209,"num_bytes_recovered":13697024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":2622},{"shards":"3,4","objects":1295}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":191,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.405704+0000","last_change":"2023-10-09T06:04:32.951565+0000","last_active":"2023-10-09T06:04:35.405704+0000","last_peered":"2023-10-09T06:04:35.405704+0000","last_clean":"2023-10-09T06:04:35.405704+0000","last_became_active":"2023-10-09T06:04:32.951221+0000","last_became_peered":"2023-10-09T06:04:32.951221+0000","last_unstale":"2023-10-09T06:04:35.405704+0000","last_undegraded":"2023-10-09T06:04:35.405704+0000","last_fullsized":"2023-10-09T06:04:35.405704+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":201,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404367+0000","last_change":"2023-10-09T06:04:32.959286+0000","last_active":"2023-10-09T06:04:35.404367+0000","last_peered":"2023-10-09T06:04:35.404367+0000","last_clean":"2023-10-09T06:04:35.404367+0000","last_became_active":"2023-10-09T06:04:32.958966+0000","last_became_peered":"2023-10-09T06:04:32.958966+0000","last_unstale":"2023-10-09T06:04:35.404367+0000","last_undegraded":"2023-10-09T06:04:35.404367+0000","last_fullsized":"2023-10-09T06:04:35.404367+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"542'6366","reported_seq":6406,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:04:45.964272+0000","last_change":"2023-10-09T06:02:25.346804+0000","last_active":"2023-10-09T06:04:45.964272+0000","last_peered":"2023-10-09T06:04:45.964272+0000","last_clean":"2023-10-09T06:04:45.964272+0000","last_became_active":"2023-10-09T06:02:25.346477+0000","last_became_peered":"2023-10-09T06:02:25.346477+0000","last_unstale":"2023-10-09T06:04:45.964272+0000","last_undegraded":"2023-10-09T06:04:45.964272+0000","last_fullsized":"2023-10-09T06:04:45.964272+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6366,"log_dups_size":0,"ondisk_log_size":6366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:43.482594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310640640,"num_objects":4740,"num_object_clones":0,"num_object_copies":14220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6366,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":481,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.410214+0000","last_change":"2023-10-09T06:04:32.950172+0000","last_active":"2023-10-09T06:04:35.410214+0000","last_peered":"2023-10-09T06:04:35.410214+0000","last_clean":"2023-10-09T06:04:35.410214+0000","last_became_active":"2023-10-09T06:04:32.949900+0000","last_became_peered":"2023-10-09T06:04:32.949900+0000","last_unstale":"2023-10-09T06:04:35.410214+0000","last_undegraded":"2023-10-09T06:04:35.410214+0000","last_fullsized":"2023-10-09T06:04:35.410214+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:58.660448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":475,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:37.624865+0000","last_change":"2023-10-09T06:00:05.352723+0000","last_active":"2023-10-09T06:04:37.624865+0000","last_peered":"2023-10-09T06:04:37.624865+0000","last_clean":"2023-10-09T06:04:37.624865+0000","last_became_active":"2023-10-09T06:00:05.352008+0000","last_became_peered":"2023-10-09T06:00:05.352008+0000","last_unstale":"2023-10-09T06:04:37.624865+0000","last_undegraded":"2023-10-09T06:04:37.624865+0000","last_fullsized":"2023-10-09T06:04:37.624865+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:25.302918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"542'6297","reported_seq":12914,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:04:48.781414+0000","last_change":"2023-10-09T06:04:34.400706+0000","last_active":"2023-10-09T06:04:48.781414+0000","last_peered":"2023-10-09T06:04:48.781414+0000","last_clean":"2023-10-09T06:04:48.781414+0000","last_became_active":"2023-10-09T06:04:34.391159+0000","last_became_peered":"2023-10-09T06:04:34.391159+0000","last_unstale":"2023-10-09T06:04:48.781414+0000","last_undegraded":"2023-10-09T06:04:48.781414+0000","last_fullsized":"2023-10-09T06:04:48.781414+0000","mapping_epoch":539,"log_start":"515'400","ondisk_log_start":"515'400","created":512,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":5897,"log_dups_size":0,"ondisk_log_size":5897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:01.426153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295501824,"num_objects":4509,"num_object_clones":0,"num_object_copies":13527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6297,"num_write_kb":345792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3520,"num_bytes_recovered":230686720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":392,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.417088+0000","last_change":"2023-10-09T06:03:56.486406+0000","last_active":"2023-10-09T06:04:35.417088+0000","last_peered":"2023-10-09T06:04:35.417088+0000","last_clean":"2023-10-09T06:04:35.417088+0000","last_became_active":"2023-10-09T06:03:56.477719+0000","last_became_peered":"2023-10-09T06:03:56.477719+0000","last_unstale":"2023-10-09T06:04:35.417088+0000","last_undegraded":"2023-10-09T06:04:35.417088+0000","last_fullsized":"2023-10-09T06:04:35.417088+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:04.677042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":555,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.403175+0000","last_change":"2023-10-09T06:04:32.947237+0000","last_active":"2023-10-09T06:04:35.403175+0000","last_peered":"2023-10-09T06:04:35.403175+0000","last_clean":"2023-10-09T06:04:35.403175+0000","last_became_active":"2023-10-09T06:04:32.946963+0000","last_became_peered":"2023-10-09T06:04:32.946963+0000","last_unstale":"2023-10-09T06:04:35.403175+0000","last_undegraded":"2023-10-09T06:04:35.403175+0000","last_fullsized":"2023-10-09T06:04:35.403175+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"542'6191","reported_seq":6427,"reported_epoch":542,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:04:45.945307+0000","last_change":"2023-10-09T06:04:33.600723+0000","last_active":"2023-10-09T06:04:45.945307+0000","last_peered":"2023-10-09T06:04:45.945307+0000","last_clean":"2023-10-09T06:03:55.466728+0000","last_became_active":"2023-10-09T06:03:57.870233+0000","last_became_peered":"2023-10-09T06:03:57.870233+0000","last_unstale":"2023-10-09T06:04:45.945307+0000","last_undegraded":"2023-10-09T06:03:57.627426+0000","last_fullsized":"2023-10-09T06:03:57.471432+0000","mapping_epoch":528,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6191,"log_dups_size":0,"ondisk_log_size":6191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:35.206036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307036160,"num_objects":4685,"num_object_clones":0,"num_object_copies":14055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5321,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6191,"num_write_kb":348032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":6094848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","4"],"object_location_counts":[{"shards":"0,2","objects":-636},{"shards":"0,2,4","objects":4660}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.407808+0000","last_change":"2023-10-09T06:04:32.950583+0000","last_active":"2023-10-09T06:04:35.407808+0000","last_peered":"2023-10-09T06:04:35.407808+0000","last_clean":"2023-10-09T06:04:35.407808+0000","last_became_active":"2023-10-09T06:04:32.950108+0000","last_became_peered":"2023-10-09T06:04:32.950108+0000","last_unstale":"2023-10-09T06:04:35.407808+0000","last_undegraded":"2023-10-09T06:04:35.407808+0000","last_fullsized":"2023-10-09T06:04:35.407808+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":498,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.412433+0000","last_change":"2023-10-09T06:04:32.958614+0000","last_active":"2023-10-09T06:04:35.412433+0000","last_peered":"2023-10-09T06:04:35.412433+0000","last_clean":"2023-10-09T06:04:35.412433+0000","last_became_active":"2023-10-09T06:04:32.958313+0000","last_became_peered":"2023-10-09T06:04:32.958313+0000","last_unstale":"2023-10-09T06:04:35.412433+0000","last_undegraded":"2023-10-09T06:04:35.412433+0000","last_fullsized":"2023-10-09T06:04:35.412433+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:07.609009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"542'6313","reported_seq":12831,"reported_epoch":542,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:04:49.605036+0000","last_change":"2023-10-09T06:04:33.570693+0000","last_active":"2023-10-09T06:04:49.605036+0000","last_peered":"2023-10-09T06:04:49.605036+0000","last_clean":"2023-10-09T06:04:01.511232+0000","last_became_active":"2023-10-09T06:04:33.557575+0000","last_became_peered":"2023-10-09T06:04:33.557575+0000","last_unstale":"2023-10-09T06:04:49.605036+0000","last_undegraded":"2023-10-09T06:04:33.391057+0000","last_fullsized":"2023-10-09T06:04:33.385878+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297467904,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":395,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6313,"num_write_kb":347264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3503,"num_bytes_recovered":229441536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":4539}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":556,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.410845+0000","last_change":"2023-10-09T06:03:56.487256+0000","last_active":"2023-10-09T06:04:35.410845+0000","last_peered":"2023-10-09T06:04:35.410845+0000","last_clean":"2023-10-09T06:04:35.410845+0000","last_became_active":"2023-10-09T06:03:56.486975+0000","last_became_peered":"2023-10-09T06:03:56.486975+0000","last_unstale":"2023-10-09T06:04:35.410845+0000","last_undegraded":"2023-10-09T06:04:35.410845+0000","last_fullsized":"2023-10-09T06:04:35.410845+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:53.615053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":530,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.401387+0000","last_change":"2023-10-09T06:04:32.949447+0000","last_active":"2023-10-09T06:04:35.401387+0000","last_peered":"2023-10-09T06:04:35.401387+0000","last_clean":"2023-10-09T06:04:35.401387+0000","last_became_active":"2023-10-09T06:04:32.949098+0000","last_became_peered":"2023-10-09T06:04:32.949098+0000","last_unstale":"2023-10-09T06:04:35.401387+0000","last_undegraded":"2023-10-09T06:04:35.401387+0000","last_fullsized":"2023-10-09T06:04:35.401387+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"542'6214","reported_seq":6257,"reported_epoch":542,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:04:49.597873+0000","last_change":"2023-10-09T06:03:57.683924+0000","last_active":"2023-10-09T06:04:49.597873+0000","last_peered":"2023-10-09T06:04:49.597873+0000","last_clean":"2023-10-09T06:03:55.449194+0000","last_became_active":"2023-10-09T06:03:57.667412+0000","last_became_peered":"2023-10-09T06:03:57.667412+0000","last_unstale":"2023-10-09T06:04:49.597873+0000","last_undegraded":"2023-10-09T06:03:57.501788+0000","last_fullsized":"2023-10-09T06:03:57.472588+0000","mapping_epoch":528,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6214,"log_dups_size":0,"ondisk_log_size":6214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291504128,"num_objects":4448,"num_object_clones":0,"num_object_copies":13344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5316,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6214,"num_write_kb":341184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":4433},{"shards":"0,7","objects":-868}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":185,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404395+0000","last_change":"2023-10-09T06:03:09.372084+0000","last_active":"2023-10-09T06:04:35.404395+0000","last_peered":"2023-10-09T06:04:35.404395+0000","last_clean":"2023-10-09T06:04:35.404395+0000","last_became_active":"2023-10-09T06:03:09.371713+0000","last_became_peered":"2023-10-09T06:03:09.371713+0000","last_unstale":"2023-10-09T06:04:35.404395+0000","last_undegraded":"2023-10-09T06:04:35.404395+0000","last_fullsized":"2023-10-09T06:04:35.404395+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":232,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.419083+0000","last_change":"2023-10-09T05:56:00.225724+0000","last_active":"2023-10-09T06:04:35.419083+0000","last_peered":"2023-10-09T06:04:35.419083+0000","last_clean":"2023-10-09T06:04:35.419083+0000","last_became_active":"2023-10-09T05:56:00.225335+0000","last_became_peered":"2023-10-09T05:56:00.225335+0000","last_unstale":"2023-10-09T06:04:35.419083+0000","last_undegraded":"2023-10-09T06:04:35.419083+0000","last_fullsized":"2023-10-09T06:04:35.419083+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:41.405222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"542'6330","reported_seq":6368,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:04:49.586681+0000","last_change":"2023-10-09T06:02:25.343092+0000","last_active":"2023-10-09T06:04:49.586681+0000","last_peered":"2023-10-09T06:04:49.586681+0000","last_clean":"2023-10-09T06:04:49.586681+0000","last_became_active":"2023-10-09T06:02:25.342832+0000","last_became_peered":"2023-10-09T06:02:25.342832+0000","last_unstale":"2023-10-09T06:04:49.586681+0000","last_undegraded":"2023-10-09T06:04:49.586681+0000","last_fullsized":"2023-10-09T06:04:49.586681+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6330,"log_dups_size":0,"ondisk_log_size":6330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300548096,"num_objects":4586,"num_object_clones":0,"num_object_copies":13758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6330,"num_write_kb":349312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":949,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404689+0000","last_change":"2023-10-09T06:02:08.341435+0000","last_active":"2023-10-09T06:04:35.404689+0000","last_peered":"2023-10-09T06:04:35.404689+0000","last_clean":"2023-10-09T06:04:35.404689+0000","last_became_active":"2023-10-09T06:02:08.340863+0000","last_became_peered":"2023-10-09T06:02:08.340863+0000","last_unstale":"2023-10-09T06:04:35.404689+0000","last_undegraded":"2023-10-09T06:04:35.404689+0000","last_fullsized":"2023-10-09T06:04:35.404689+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":552,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.412534+0000","last_change":"2023-10-09T06:04:32.944862+0000","last_active":"2023-10-09T06:04:35.412534+0000","last_peered":"2023-10-09T06:04:35.412534+0000","last_clean":"2023-10-09T06:04:35.412534+0000","last_became_active":"2023-10-09T06:04:32.944555+0000","last_became_peered":"2023-10-09T06:04:32.944555+0000","last_unstale":"2023-10-09T06:04:35.412534+0000","last_undegraded":"2023-10-09T06:04:35.412534+0000","last_fullsized":"2023-10-09T06:04:35.412534+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:30.196788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":97,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.434155+0000","last_change":"2023-10-09T06:03:56.478180+0000","last_active":"2023-10-09T06:04:35.434155+0000","last_peered":"2023-10-09T06:04:35.434155+0000","last_clean":"2023-10-09T06:04:35.434155+0000","last_became_active":"2023-10-09T06:03:56.477820+0000","last_became_peered":"2023-10-09T06:03:56.477820+0000","last_unstale":"2023-10-09T06:04:35.434155+0000","last_undegraded":"2023-10-09T06:04:35.434155+0000","last_fullsized":"2023-10-09T06:04:35.434155+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:23.156512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.a","version":"542'6214","reported_seq":10221,"reported_epoch":542,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:04:45.957793+0000","last_change":"2023-10-09T06:04:33.626640+0000","last_active":"2023-10-09T06:04:45.957793+0000","last_peered":"2023-10-09T06:04:45.957793+0000","last_clean":"2023-10-09T06:03:08.313089+0000","last_became_active":"2023-10-09T06:04:33.572850+0000","last_became_peered":"2023-10-09T06:04:33.572850+0000","last_unstale":"2023-10-09T06:04:45.957793+0000","last_undegraded":"2023-10-09T06:04:45.957793+0000","last_fullsized":"2023-10-09T06:04:33.383221+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6214,"log_dups_size":0,"ondisk_log_size":6214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:32.240245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303693824,"num_objects":4634,"num_object_clones":0,"num_object_copies":13902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2669,"num_objects_unfound":0,"num_objects_dirty":4634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6214,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1970,"num_bytes_recovered":129040384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,3],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,2,6","objects":2669},{"shards":"2,3","objects":1965}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":875,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.420444+0000","last_change":"2023-10-09T06:04:32.958038+0000","last_active":"2023-10-09T06:04:35.420444+0000","last_peered":"2023-10-09T06:04:35.420444+0000","last_clean":"2023-10-09T06:04:35.420444+0000","last_became_active":"2023-10-09T06:04:32.957507+0000","last_became_peered":"2023-10-09T06:04:32.957507+0000","last_unstale":"2023-10-09T06:04:35.420444+0000","last_undegraded":"2023-10-09T06:04:35.420444+0000","last_fullsized":"2023-10-09T06:04:35.420444+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:33.323626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":486,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.400724+0000","last_change":"2023-10-09T06:04:32.951743+0000","last_active":"2023-10-09T06:04:35.400724+0000","last_peered":"2023-10-09T06:04:35.400724+0000","last_clean":"2023-10-09T06:04:35.400724+0000","last_became_active":"2023-10-09T06:04:32.951415+0000","last_became_peered":"2023-10-09T06:04:32.951415+0000","last_unstale":"2023-10-09T06:04:35.400724+0000","last_undegraded":"2023-10-09T06:04:35.400724+0000","last_fullsized":"2023-10-09T06:04:35.400724+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":407,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.411242+0000","last_change":"2023-10-09T06:03:09.372497+0000","last_active":"2023-10-09T06:04:35.411242+0000","last_peered":"2023-10-09T06:04:35.411242+0000","last_clean":"2023-10-09T06:04:35.411242+0000","last_became_active":"2023-10-09T06:03:09.372018+0000","last_became_peered":"2023-10-09T06:03:09.372018+0000","last_unstale":"2023-10-09T06:04:35.411242+0000","last_undegraded":"2023-10-09T06:04:35.411242+0000","last_fullsized":"2023-10-09T06:04:35.411242+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:08.035705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"542'6348","reported_seq":6388,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:04:45.914705+0000","last_change":"2023-10-09T06:02:25.344681+0000","last_active":"2023-10-09T06:04:45.914705+0000","last_peered":"2023-10-09T06:04:45.914705+0000","last_clean":"2023-10-09T06:04:45.914705+0000","last_became_active":"2023-10-09T06:02:25.344470+0000","last_became_peered":"2023-10-09T06:02:25.344470+0000","last_unstale":"2023-10-09T06:04:45.914705+0000","last_undegraded":"2023-10-09T06:04:45.914705+0000","last_fullsized":"2023-10-09T06:04:45.914705+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6348,"log_dups_size":0,"ondisk_log_size":6348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:24.576548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310247424,"num_objects":4734,"num_object_clones":0,"num_object_copies":14202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6348,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":901,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.411007+0000","last_change":"2023-10-09T06:00:05.324594+0000","last_active":"2023-10-09T06:04:35.411007+0000","last_peered":"2023-10-09T06:04:35.411007+0000","last_clean":"2023-10-09T06:04:35.411007+0000","last_became_active":"2023-10-09T06:00:05.323845+0000","last_became_peered":"2023-10-09T06:00:05.323845+0000","last_unstale":"2023-10-09T06:04:35.411007+0000","last_undegraded":"2023-10-09T06:04:35.411007+0000","last_fullsized":"2023-10-09T06:04:35.411007+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:02.539703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":371,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.410815+0000","last_change":"2023-10-09T06:04:32.958174+0000","last_active":"2023-10-09T06:04:35.410815+0000","last_peered":"2023-10-09T06:04:35.410815+0000","last_clean":"2023-10-09T06:04:35.410815+0000","last_became_active":"2023-10-09T06:04:32.957663+0000","last_became_peered":"2023-10-09T06:04:32.957663+0000","last_unstale":"2023-10-09T06:04:35.410815+0000","last_undegraded":"2023-10-09T06:04:35.410815+0000","last_fullsized":"2023-10-09T06:04:35.410815+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:45:45.059675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":25,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.418856+0000","last_change":"2023-10-09T06:03:56.486017+0000","last_active":"2023-10-09T06:04:35.418856+0000","last_peered":"2023-10-09T06:04:35.418856+0000","last_clean":"2023-10-09T06:04:35.418856+0000","last_became_active":"2023-10-09T06:03:56.485734+0000","last_became_peered":"2023-10-09T06:03:56.485734+0000","last_unstale":"2023-10-09T06:04:35.418856+0000","last_undegraded":"2023-10-09T06:04:35.418856+0000","last_fullsized":"2023-10-09T06:04:35.418856+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:45.383409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"542'6212","reported_seq":6267,"reported_epoch":542,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:04:48.774601+0000","last_change":"2023-10-09T06:04:33.554155+0000","last_active":"2023-10-09T06:04:48.774601+0000","last_peered":"2023-10-09T06:04:48.774601+0000","last_clean":"2023-10-09T06:04:01.498991+0000","last_became_active":"2023-10-09T06:04:33.545041+0000","last_became_peered":"2023-10-09T06:04:33.545041+0000","last_unstale":"2023-10-09T06:04:48.774601+0000","last_undegraded":"2023-10-09T06:04:33.390397+0000","last_fullsized":"2023-10-09T06:04:33.380873+0000","mapping_epoch":538,"log_start":"515'300","ondisk_log_start":"515'300","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":5912,"log_dups_size":0,"ondisk_log_size":5912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:49.702473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291110912,"num_objects":4442,"num_object_clones":0,"num_object_copies":13326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":405,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6212,"num_write_kb":340928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":4442}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":785,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.413603+0000","last_change":"2023-10-09T06:02:19.008664+0000","last_active":"2023-10-09T06:04:35.413603+0000","last_peered":"2023-10-09T06:04:35.413603+0000","last_clean":"2023-10-09T06:04:35.413603+0000","last_became_active":"2023-10-09T06:02:19.008087+0000","last_became_peered":"2023-10-09T06:02:19.008087+0000","last_unstale":"2023-10-09T06:04:35.413603+0000","last_undegraded":"2023-10-09T06:04:35.413603+0000","last_fullsized":"2023-10-09T06:04:35.413603+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:09.298922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":887,"num_read_kb":802,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":732,"reported_epoch":541,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:04:35.404629+0000","last_change":"2023-10-09T06:04:33.393034+0000","last_active":"2023-10-09T06:04:35.404629+0000","last_peered":"2023-10-09T06:04:35.404629+0000","last_clean":"2023-10-09T06:03:47.002857+0000","last_became_active":"2023-10-09T06:04:33.392496+0000","last_became_peered":"2023-10-09T06:04:33.392496+0000","last_unstale":"2023-10-09T06:04:35.404629+0000","last_undegraded":"2023-10-09T06:04:33.385511+0000","last_fullsized":"2023-10-09T06:04:35.404629+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":["7","0","6"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":438,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.411433+0000","last_change":"2023-10-09T06:04:32.942556+0000","last_active":"2023-10-09T06:04:35.411433+0000","last_peered":"2023-10-09T06:04:35.411433+0000","last_clean":"2023-10-09T06:04:35.411433+0000","last_became_active":"2023-10-09T06:04:32.942269+0000","last_became_peered":"2023-10-09T06:04:32.942269+0000","last_unstale":"2023-10-09T06:04:35.411433+0000","last_undegraded":"2023-10-09T06:04:35.411433+0000","last_fullsized":"2023-10-09T06:04:35.411433+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:53.383296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":201,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404750+0000","last_change":"2023-10-09T06:04:33.392862+0000","last_active":"2023-10-09T06:04:35.404750+0000","last_peered":"2023-10-09T06:04:35.404750+0000","last_clean":"2023-10-09T06:04:35.404750+0000","last_became_active":"2023-10-09T06:04:33.392504+0000","last_became_peered":"2023-10-09T06:04:33.392504+0000","last_unstale":"2023-10-09T06:04:35.404750+0000","last_undegraded":"2023-10-09T06:04:35.404750+0000","last_fullsized":"2023-10-09T06:04:35.404750+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"542'6259","reported_seq":6299,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:04:45.918728+0000","last_change":"2023-10-09T06:02:25.345595+0000","last_active":"2023-10-09T06:04:45.918728+0000","last_peered":"2023-10-09T06:04:45.918728+0000","last_clean":"2023-10-09T06:04:45.918728+0000","last_became_active":"2023-10-09T06:02:25.345371+0000","last_became_peered":"2023-10-09T06:02:25.345371+0000","last_unstale":"2023-10-09T06:04:45.918728+0000","last_undegraded":"2023-10-09T06:04:45.918728+0000","last_fullsized":"2023-10-09T06:04:45.918728+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6259,"log_dups_size":0,"ondisk_log_size":6259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:16.252828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307298304,"num_objects":4689,"num_object_clones":0,"num_object_copies":14067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6259,"num_write_kb":350336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":1132,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404703+0000","last_change":"2023-10-09T06:03:12.481829+0000","last_active":"2023-10-09T06:04:35.404703+0000","last_peered":"2023-10-09T06:04:35.404703+0000","last_clean":"2023-10-09T06:04:35.404703+0000","last_became_active":"2023-10-09T06:03:12.479898+0000","last_became_peered":"2023-10-09T06:03:12.479898+0000","last_unstale":"2023-10-09T06:04:35.404703+0000","last_undegraded":"2023-10-09T06:04:35.404703+0000","last_fullsized":"2023-10-09T06:04:35.404703+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":521,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.422982+0000","last_change":"2023-10-09T06:03:09.367762+0000","last_active":"2023-10-09T06:04:35.422982+0000","last_peered":"2023-10-09T06:04:35.422982+0000","last_clean":"2023-10-09T06:04:35.422982+0000","last_became_active":"2023-10-09T06:03:09.367421+0000","last_became_peered":"2023-10-09T06:03:09.367421+0000","last_unstale":"2023-10-09T06:04:35.422982+0000","last_undegraded":"2023-10-09T06:04:35.422982+0000","last_fullsized":"2023-10-09T06:04:35.422982+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:57.088505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":446,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.412912+0000","last_change":"2023-10-09T06:03:09.371368+0000","last_active":"2023-10-09T06:04:35.412912+0000","last_peered":"2023-10-09T06:04:35.412912+0000","last_clean":"2023-10-09T06:04:35.412912+0000","last_became_active":"2023-10-09T06:03:09.370964+0000","last_became_peered":"2023-10-09T06:03:09.370964+0000","last_unstale":"2023-10-09T06:04:35.412912+0000","last_undegraded":"2023-10-09T06:04:35.412912+0000","last_fullsized":"2023-10-09T06:04:35.412912+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:17.239380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.d","version":"542'6434","reported_seq":7314,"reported_epoch":542,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:04:46.403150+0000","last_change":"2023-10-09T06:04:33.642051+0000","last_active":"2023-10-09T06:04:46.403150+0000","last_peered":"2023-10-09T06:04:46.403150+0000","last_clean":"2023-10-09T06:03:55.452461+0000","last_became_active":"2023-10-09T06:04:33.566461+0000","last_became_peered":"2023-10-09T06:04:33.566461+0000","last_unstale":"2023-10-09T06:04:46.403150+0000","last_undegraded":"2023-10-09T06:04:33.550801+0000","last_fullsized":"2023-10-09T06:04:33.384485+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6434,"log_dups_size":0,"ondisk_log_size":6434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310050840,"num_objects":4732,"num_object_clones":0,"num_object_copies":14196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5614,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4732,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6434,"num_write_kb":357249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":411,"num_bytes_recovered":26869760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[4,3],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,3,4","objects":17},{"shards":"0,4,6","objects":4715}],"blocked_by":[],"up_primary":6,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":764,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.426185+0000","last_change":"2023-10-09T06:04:35.425893+0000","last_active":"2023-10-09T06:04:35.426185+0000","last_peered":"2023-10-09T06:04:35.426185+0000","last_clean":"2023-10-09T06:04:35.426185+0000","last_became_active":"2023-10-09T06:04:35.423037+0000","last_became_peered":"2023-10-09T06:04:35.423037+0000","last_unstale":"2023-10-09T06:04:35.426185+0000","last_undegraded":"2023-10-09T06:04:35.426185+0000","last_fullsized":"2023-10-09T06:04:35.426185+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":402,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.411382+0000","last_change":"2023-10-09T06:04:33.399587+0000","last_active":"2023-10-09T06:04:35.411382+0000","last_peered":"2023-10-09T06:04:35.411382+0000","last_clean":"2023-10-09T06:04:35.411382+0000","last_became_active":"2023-10-09T06:04:33.396702+0000","last_became_peered":"2023-10-09T06:04:33.396702+0000","last_unstale":"2023-10-09T06:04:35.411382+0000","last_undegraded":"2023-10-09T06:04:35.411382+0000","last_fullsized":"2023-10-09T06:04:35.411382+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:32.926778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":480,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404897+0000","last_change":"2023-10-09T06:02:08.340506+0000","last_active":"2023-10-09T06:04:35.404897+0000","last_peered":"2023-10-09T06:04:35.404897+0000","last_clean":"2023-10-09T06:04:35.404897+0000","last_became_active":"2023-10-09T06:02:08.340152+0000","last_became_peered":"2023-10-09T06:02:08.340152+0000","last_unstale":"2023-10-09T06:04:35.404897+0000","last_undegraded":"2023-10-09T06:04:35.404897+0000","last_fullsized":"2023-10-09T06:04:35.404897+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"542'6184","reported_seq":6239,"reported_epoch":542,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:04:45.968077+0000","last_change":"2023-10-09T06:04:33.587539+0000","last_active":"2023-10-09T06:04:45.968077+0000","last_peered":"2023-10-09T06:04:45.968077+0000","last_clean":"2023-10-09T06:04:01.653228+0000","last_became_active":"2023-10-09T06:04:33.576931+0000","last_became_peered":"2023-10-09T06:04:33.576931+0000","last_unstale":"2023-10-09T06:04:45.968077+0000","last_undegraded":"2023-10-09T06:04:33.406904+0000","last_fullsized":"2023-10-09T06:04:33.389582+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6184,"log_dups_size":0,"ondisk_log_size":6184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:07.802984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300548096,"num_objects":4586,"num_object_clones":0,"num_object_copies":13758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":402,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6184,"num_write_kb":344640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":4586}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":208,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.406322+0000","last_change":"2023-10-09T06:00:05.329139+0000","last_active":"2023-10-09T06:04:35.406322+0000","last_peered":"2023-10-09T06:04:35.406322+0000","last_clean":"2023-10-09T06:04:35.406322+0000","last_became_active":"2023-10-09T06:00:05.328811+0000","last_became_peered":"2023-10-09T06:00:05.328811+0000","last_unstale":"2023-10-09T06:04:35.406322+0000","last_undegraded":"2023-10-09T06:04:35.406322+0000","last_fullsized":"2023-10-09T06:04:35.406322+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":582,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.418684+0000","last_change":"2023-10-09T06:04:32.959605+0000","last_active":"2023-10-09T06:04:35.418684+0000","last_peered":"2023-10-09T06:04:35.418684+0000","last_clean":"2023-10-09T06:04:35.418684+0000","last_became_active":"2023-10-09T06:04:32.959073+0000","last_became_peered":"2023-10-09T06:04:32.959073+0000","last_unstale":"2023-10-09T06:04:35.418684+0000","last_undegraded":"2023-10-09T06:04:35.418684+0000","last_fullsized":"2023-10-09T06:04:35.418684+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:00.843837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":221,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404483+0000","last_change":"2023-10-09T06:03:09.371385+0000","last_active":"2023-10-09T06:04:35.404483+0000","last_peered":"2023-10-09T06:04:35.404483+0000","last_clean":"2023-10-09T06:04:35.404483+0000","last_became_active":"2023-10-09T06:03:09.370954+0000","last_became_peered":"2023-10-09T06:03:09.370954+0000","last_unstale":"2023-10-09T06:04:35.404483+0000","last_undegraded":"2023-10-09T06:04:35.404483+0000","last_fullsized":"2023-10-09T06:04:35.404483+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"542'6331","reported_seq":6387,"reported_epoch":542,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:04:45.977316+0000","last_change":"2023-10-09T06:04:33.576854+0000","last_active":"2023-10-09T06:04:45.977316+0000","last_peered":"2023-10-09T06:04:45.977316+0000","last_clean":"2023-10-09T06:04:01.658266+0000","last_became_active":"2023-10-09T06:04:33.568529+0000","last_became_peered":"2023-10-09T06:04:33.568529+0000","last_unstale":"2023-10-09T06:04:45.977316+0000","last_undegraded":"2023-10-09T06:04:33.402607+0000","last_fullsized":"2023-10-09T06:04:33.386717+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":512,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:24.321830+0000","last_clean_scrub_stamp":"2023-10-09T06:02:24.321830+0000","objects_scrubbed":0,"log_size":6331,"log_dups_size":0,"ondisk_log_size":6331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:06.017872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309133312,"num_objects":4717,"num_object_clones":0,"num_object_copies":14151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":414,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6331,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":4717}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":745,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.406239+0000","last_change":"2023-10-09T06:04:32.961494+0000","last_active":"2023-10-09T06:04:35.406239+0000","last_peered":"2023-10-09T06:04:35.406239+0000","last_clean":"2023-10-09T06:04:35.406239+0000","last_became_active":"2023-10-09T06:04:32.961137+0000","last_became_peered":"2023-10-09T06:04:32.961137+0000","last_unstale":"2023-10-09T06:04:35.406239+0000","last_undegraded":"2023-10-09T06:04:35.406239+0000","last_fullsized":"2023-10-09T06:04:35.406239+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":28,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.405099+0000","last_change":"2023-10-09T06:03:56.573966+0000","last_active":"2023-10-09T06:04:35.405099+0000","last_peered":"2023-10-09T06:04:35.405099+0000","last_clean":"2023-10-09T06:04:35.405099+0000","last_became_active":"2023-10-09T06:03:56.481596+0000","last_became_peered":"2023-10-09T06:03:56.481596+0000","last_unstale":"2023-10-09T06:04:35.405099+0000","last_undegraded":"2023-10-09T06:04:35.405099+0000","last_fullsized":"2023-10-09T06:04:35.405099+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":112,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.422762+0000","last_change":"2023-10-09T06:04:32.957041+0000","last_active":"2023-10-09T06:04:35.422762+0000","last_peered":"2023-10-09T06:04:35.422762+0000","last_clean":"2023-10-09T06:04:35.422762+0000","last_became_active":"2023-10-09T06:04:32.956702+0000","last_became_peered":"2023-10-09T06:04:32.956702+0000","last_unstale":"2023-10-09T06:04:35.422762+0000","last_undegraded":"2023-10-09T06:04:35.422762+0000","last_fullsized":"2023-10-09T06:04:35.422762+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:31.030221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":506,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.404587+0000","last_change":"2023-10-09T06:03:56.482621+0000","last_active":"2023-10-09T06:04:35.404587+0000","last_peered":"2023-10-09T06:04:35.404587+0000","last_clean":"2023-10-09T06:04:35.404587+0000","last_became_active":"2023-10-09T06:03:56.482163+0000","last_became_peered":"2023-10-09T06:03:56.482163+0000","last_unstale":"2023-10-09T06:04:35.404587+0000","last_undegraded":"2023-10-09T06:04:35.404587+0000","last_fullsized":"2023-10-09T06:04:35.404587+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":236,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.411309+0000","last_change":"2023-10-09T05:56:00.226695+0000","last_active":"2023-10-09T06:04:35.411309+0000","last_peered":"2023-10-09T06:04:35.411309+0000","last_clean":"2023-10-09T06:04:35.411309+0000","last_became_active":"2023-10-09T05:56:00.226359+0000","last_became_peered":"2023-10-09T05:56:00.226359+0000","last_unstale":"2023-10-09T06:04:35.411309+0000","last_undegraded":"2023-10-09T06:04:35.411309+0000","last_fullsized":"2023-10-09T06:04:35.411309+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:20.323704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":102,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.403777+0000","last_change":"2023-10-09T06:04:32.957514+0000","last_active":"2023-10-09T06:04:35.403777+0000","last_peered":"2023-10-09T06:04:35.403777+0000","last_clean":"2023-10-09T06:04:35.403777+0000","last_became_active":"2023-10-09T06:04:32.957114+0000","last_became_peered":"2023-10-09T06:04:32.957114+0000","last_unstale":"2023-10-09T06:04:35.403777+0000","last_undegraded":"2023-10-09T06:04:35.403777+0000","last_fullsized":"2023-10-09T06:04:35.403777+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":25,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.419134+0000","last_change":"2023-10-09T06:03:56.485367+0000","last_active":"2023-10-09T06:04:35.419134+0000","last_peered":"2023-10-09T06:04:35.419134+0000","last_clean":"2023-10-09T06:04:35.419134+0000","last_became_active":"2023-10-09T06:03:56.484994+0000","last_became_peered":"2023-10-09T06:03:56.484994+0000","last_unstale":"2023-10-09T06:04:35.419134+0000","last_undegraded":"2023-10-09T06:04:35.419134+0000","last_fullsized":"2023-10-09T06:04:35.419134+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:20.358347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":102,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.416332+0000","last_change":"2023-10-09T06:04:32.957504+0000","last_active":"2023-10-09T06:04:35.416332+0000","last_peered":"2023-10-09T06:04:35.416332+0000","last_clean":"2023-10-09T06:04:35.416332+0000","last_became_active":"2023-10-09T06:04:32.957148+0000","last_became_peered":"2023-10-09T06:04:32.957148+0000","last_unstale":"2023-10-09T06:04:35.416332+0000","last_undegraded":"2023-10-09T06:04:35.416332+0000","last_fullsized":"2023-10-09T06:04:35.416332+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":33,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.413456+0000","last_change":"2023-10-09T06:03:09.371196+0000","last_active":"2023-10-09T06:04:35.413456+0000","last_peered":"2023-10-09T06:04:35.413456+0000","last_clean":"2023-10-09T06:04:35.413456+0000","last_became_active":"2023-10-09T06:03:09.369325+0000","last_became_peered":"2023-10-09T06:03:09.369325+0000","last_unstale":"2023-10-09T06:04:35.413456+0000","last_undegraded":"2023-10-09T06:04:35.413456+0000","last_fullsized":"2023-10-09T06:04:35.413456+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:20.350548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":318,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.975978+0000","last_change":"2023-10-09T06:04:32.947020+0000","last_active":"2023-10-09T06:04:35.975978+0000","last_peered":"2023-10-09T06:04:35.975978+0000","last_clean":"2023-10-09T06:04:35.975978+0000","last_became_active":"2023-10-09T06:04:32.946752+0000","last_became_peered":"2023-10-09T06:04:32.946752+0000","last_unstale":"2023-10-09T06:04:35.975978+0000","last_undegraded":"2023-10-09T06:04:35.975978+0000","last_fullsized":"2023-10-09T06:04:35.975978+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:51.230755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":402,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.411078+0000","last_change":"2023-10-09T06:00:05.325284+0000","last_active":"2023-10-09T06:04:35.411078+0000","last_peered":"2023-10-09T06:04:35.411078+0000","last_clean":"2023-10-09T06:04:35.411078+0000","last_became_active":"2023-10-09T06:00:05.324588+0000","last_became_peered":"2023-10-09T06:00:05.324588+0000","last_unstale":"2023-10-09T06:04:35.411078+0000","last_undegraded":"2023-10-09T06:04:35.411078+0000","last_fullsized":"2023-10-09T06:04:35.411078+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:59.068162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":472,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.417256+0000","last_change":"2023-10-09T06:04:32.946512+0000","last_active":"2023-10-09T06:04:35.417256+0000","last_peered":"2023-10-09T06:04:35.417256+0000","last_clean":"2023-10-09T06:04:35.417256+0000","last_became_active":"2023-10-09T06:04:32.946208+0000","last_became_peered":"2023-10-09T06:04:32.946208+0000","last_unstale":"2023-10-09T06:04:35.417256+0000","last_undegraded":"2023-10-09T06:04:35.417256+0000","last_fullsized":"2023-10-09T06:04:35.417256+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:21.139997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":534,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:04:35.400284+0000","last_change":"2023-10-09T06:04:32.949651+0000","last_active":"2023-10-09T06:04:35.400284+0000","last_peered":"2023-10-09T06:04:35.400284+0000","last_clean":"2023-10-09T06:04:35.400284+0000","last_became_active":"2023-10-09T06:04:32.949288+0000","last_became_peered":"2023-10-09T06:04:32.949288+0000","last_unstale":"2023-10-09T06:04:35.400284+0000","last_undegraded":"2023-10-09T06:04:35.400284+0000","last_fullsized":"2023-10-09T06:04:35.400284+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4842127384,"num_objects":73886,"num_object_clones":0,"num_object_copies":221658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24258,"num_objects_misplaced":2669,"num_objects_unfound":0,"num_objects_dirty":73886,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":100440,"num_write_kb":5578369,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10511,"num_bytes_recovered":688521216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15454973952,"data_stored":15454961736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99740,"ondisk_log_size":99740,"up":48,"acting":38,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":21746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":52606,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13211,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":158,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846949,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2390224,"kb_used_data":1476296,"kb_used_omap":14,"kb_used_meta":913905,"kb_avail":91354928,"statfs":{"total":95995035648,"available":93547446272,"internally_reserved":0,"allocated":1511727104,"data_stored":1510554231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":935839578},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:04:39 2023","interfaces":[{"interface":"back","average":{"1min":4.7960000000000003,"5min":3.847,"15min":4.1139999999999999},"min":{"1min":0.26900000000000002,"5min":0.24299999999999999,"15min":0.22800000000000001},"max":{"1min":54.595999999999997,"5min":176.89500000000001,"15min":212.64400000000001},"last":2.3260000000000001},{"interface":"front","average":{"1min":4.9379999999999997,"5min":4.0380000000000003,"15min":4.2350000000000003},"min":{"1min":0.30199999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":54.606999999999999,"5min":176.947,"15min":212.655},"last":2.3330000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:04:39 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":6.8769999999999998,"15min":4.2050000000000001},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":4.7069999999999999,"5min":176.90600000000001,"15min":176.90600000000001},"last":1.4219999999999999},{"interface":"front","average":{"1min":1.6379999999999999,"5min":6.851,"15min":4.2320000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25700000000000001},"max":{"1min":7.7530000000000001,"5min":176.86799999999999,"15min":176.86799999999999},"last":2.5329999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:04:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":9.798,"15min":5.3719999999999999},"min":{"1min":0.35299999999999998,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":22.477,"5min":302.01799999999997,"15min":302.01799999999997},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.8600000000000001,"5min":9.5120000000000005,"15min":5.1479999999999997},"min":{"1min":0.34300000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":15.679,"5min":304.988,"15min":304.988},"last":1.4690000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:04:39 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":3.9900000000000002,"15min":3.133},"min":{"1min":0.34399999999999997,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":10.627000000000001,"5min":176.958,"15min":176.958},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.3440000000000001,"5min":3.996,"15min":3.379},"min":{"1min":0.32900000000000001,"5min":0.29799999999999999,"15min":0.29299999999999998},"max":{"1min":10.448,"5min":176.91800000000001,"15min":176.91800000000001},"last":1.016}]},{"osd":4,"last update":"Mon Oct 9 06:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":3.0139999999999998,"15min":2.706},"min":{"1min":0.27300000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.47099999999999997,"5min":178.995,"15min":178.995},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.56100000000000005,"5min":3.1469999999999998,"15min":2.7639999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":2.0609999999999999,"5min":178.899,"15min":178.899},"last":0.29699999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":3.1899999999999999,"15min":2.524},"min":{"1min":0.29699999999999999,"5min":0.29399999999999998,"15min":0.27800000000000002},"max":{"1min":2.0979999999999999,"5min":176.82400000000001,"15min":176.82400000000001},"last":0.38},{"interface":"front","average":{"1min":0.79400000000000004,"5min":3.0779999999999998,"15min":2.581},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.242,"5min":176.79499999999999,"15min":176.79499999999999},"last":0.38900000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":3.4159999999999999,"15min":2.484},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29299999999999998},"max":{"1min":3.0960000000000001,"5min":177.01499999999999,"15min":209.322},"last":0.435},{"interface":"front","average":{"1min":0.59199999999999997,"5min":3.552,"15min":2.573},"min":{"1min":0.313,"5min":0.313,"15min":0.26700000000000002},"max":{"1min":2.3540000000000001,"5min":177.00200000000001,"15min":209.19800000000001},"last":0.42999999999999999}]}]},{"osd":6,"up_from":537,"seq":2306397437956,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2863372,"kb_used_data":2491780,"kb_used_omap":13,"kb_used_meta":371570,"kb_avail":90881780,"statfs":{"total":95995035648,"available":93062942720,"internally_reserved":0,"allocated":2551582720,"data_stored":2550414139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13650,"internal_metadata":380488366},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30599999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":5.0339999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.484}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":8.5649999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40200000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.434}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61799999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958941,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2290712,"kb_used_data":1283792,"kb_used_omap":7,"kb_used_meta":1006904,"kb_avail":91454440,"statfs":{"total":95995035648,"available":93649346560,"internally_reserved":0,"allocated":1314603008,"data_stored":1312482392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":1031070222},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:04:26 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.264,"15min":2.3799999999999999},"min":{"1min":0.25,"5min":0.22700000000000001,"15min":0.17499999999999999},"max":{"1min":3.9009999999999998,"5min":10.449,"15min":131.91300000000001},"last":17.5},{"interface":"front","average":{"1min":2.012,"5min":1.4610000000000001,"15min":2.5939999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.223},"max":{"1min":16.757000000000001,"5min":16.757000000000001,"15min":132.191},"last":2.718}]},{"osd":2,"last update":"Mon Oct 9 06:03:52 2023","interfaces":[{"interface":"back","average":{"1min":7.9370000000000003,"5min":2.4300000000000002,"15min":2.3500000000000001},"min":{"1min":0.28699999999999998,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":51.899000000000001,"5min":51.899000000000001,"15min":132.35400000000001},"last":3.661},{"interface":"front","average":{"1min":5.6050000000000004,"5min":2.27,"15min":2.1920000000000002},"min":{"1min":0.29399999999999998,"5min":0.26800000000000002,"15min":0.20699999999999999},"max":{"1min":49.584000000000003,"5min":49.584000000000003,"15min":131.70599999999999},"last":13.585000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:04:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":1.6100000000000001,"15min":2.5339999999999998},"min":{"1min":0.25900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":13.627000000000001,"5min":13.970000000000001,"15min":135.10900000000001},"last":4.274},{"interface":"front","average":{"1min":1.4530000000000001,"5min":1.321,"15min":2.4100000000000001},"min":{"1min":0.28299999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":9.8170000000000002,"5min":10.407999999999999,"15min":131.886},"last":13.734999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:04:26 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.3779999999999999,"15min":1.79},"min":{"1min":0.36199999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":13.647,"5min":13.647,"15min":131.636},"last":2.7589999999999999},{"interface":"front","average":{"1min":0.92300000000000004,"5min":1.1579999999999999,"15min":1.655},"min":{"1min":0.39000000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.379,"5min":10.32,"15min":131.75999999999999},"last":13.631}]},{"osd":5,"last update":"Mon Oct 9 06:04:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7689999999999999,"5min":1.4770000000000001,"15min":2.3079999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.28599999999999998},"max":{"1min":13.567,"5min":14.019,"15min":132.291},"last":3.367},{"interface":"front","average":{"1min":1.748,"5min":1.5089999999999999,"15min":2.403},"min":{"1min":0.39800000000000002,"5min":0.36799999999999999,"15min":0.33700000000000002},"max":{"1min":13.432,"5min":13.432,"15min":131.738},"last":2.8069999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":1.71,"15min":2.48},"min":{"1min":0.38800000000000001,"5min":0.34200000000000003,"15min":0.33300000000000002},"max":{"1min":14.057,"5min":14.057,"15min":158.50399999999999},"last":2.7829999999999999},{"interface":"front","average":{"1min":2.2450000000000001,"5min":1.5469999999999999,"15min":2.4140000000000001},"min":{"1min":0.45100000000000001,"5min":0.38,"15min":0.37},"max":{"1min":6.9340000000000002,"5min":9.8559999999999999,"15min":158.577},"last":13.685}]},{"osd":7,"last update":"Mon Oct 9 06:04:26 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.337,"15min":2.0099999999999998},"min":{"1min":0.439,"5min":0.40200000000000002,"15min":0.31},"max":{"1min":3.46,"5min":10.263999999999999,"15min":131.821},"last":13.712999999999999},{"interface":"front","average":{"1min":0.879,"5min":1.478,"15min":2.153},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.35199999999999998},"max":{"1min":3.4319999999999999,"5min":10.365,"15min":131.852},"last":13.657999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122466,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4134012,"kb_used_data":3190196,"kb_used_omap":8,"kb_used_meta":943799,"kb_avail":89611140,"statfs":{"total":95995035648,"available":91761807360,"internally_reserved":0,"allocated":3266760704,"data_stored":3264607207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":966450952},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":5.4619999999999997,"15min":3.294},"min":{"1min":0.23000000000000001,"5min":0.19400000000000001,"15min":0.186},"max":{"1min":6.468,"5min":163.572,"15min":163.572},"last":0.68200000000000005},{"interface":"front","average":{"1min":1.645,"5min":5.5739999999999998,"15min":3.456},"min":{"1min":0.247,"5min":0.19700000000000001,"15min":0.19600000000000001},"max":{"1min":5.3220000000000001,"5min":162.48699999999999,"15min":162.48699999999999},"last":7.1680000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:04:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":1.135,"15min":1.994},"min":{"1min":0.22900000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.6950000000000003,"5min":6.2679999999999998,"15min":70.188999999999993},"last":2.9609999999999999},{"interface":"front","average":{"1min":2.3530000000000002,"5min":1.1399999999999999,"15min":1.764},"min":{"1min":0.28399999999999997,"5min":0.221,"15min":0.221},"max":{"1min":13.667999999999999,"5min":13.667999999999999,"15min":71.144000000000005},"last":0.69799999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:04:25 2023","interfaces":[{"interface":"back","average":{"1min":2.032,"5min":1.153,"15min":1.7729999999999999},"min":{"1min":0.32600000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.4900000000000002,"5min":6.734,"15min":62.698999999999998},"last":7.7000000000000002},{"interface":"front","average":{"1min":1.996,"5min":1.2929999999999999,"15min":2.1549999999999998},"min":{"1min":0.35499999999999998,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":5.7060000000000004,"5min":8.9290000000000003,"15min":72.632999999999996},"last":2.8849999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:04:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":0.96599999999999997,"15min":1.6870000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":13.592000000000001,"5min":13.592000000000001,"15min":149.63999999999999},"last":0.621},{"interface":"front","average":{"1min":2.0230000000000001,"5min":1.25,"15min":1.905},"min":{"1min":0.40500000000000003,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.2910000000000004,"5min":6.4180000000000001,"15min":149.65899999999999},"last":3.0070000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:04:06 2023","interfaces":[{"interface":"back","average":{"1min":2.129,"5min":1.1779999999999999,"15min":4.0759999999999996},"min":{"1min":0.438,"5min":0.34999999999999998,"15min":0.27900000000000003},"max":{"1min":6.2560000000000002,"5min":6.2560000000000002,"15min":246.761},"last":3.0640000000000001},{"interface":"front","average":{"1min":1.163,"5min":1.258,"15min":4.3460000000000001},"min":{"1min":0.35299999999999998,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":4.8319999999999999,"5min":12.404999999999999,"15min":247.48400000000001},"last":2.8439999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:03:02 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":0.98099999999999998,"15min":1.2070000000000001},"min":{"1min":0.39700000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.3620000000000001,"5min":8.2899999999999991,"15min":15.789},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.304,"5min":1.1890000000000001,"15min":1.3540000000000001},"min":{"1min":0.48899999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.9660000000000002,"5min":12.457000000000001,"15min":19.710999999999999},"last":3.036}]},{"osd":7,"last update":"Mon Oct 9 06:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":2.8220000000000001,"15min":2.9780000000000002},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.8259999999999996,"5min":168.98599999999999,"15min":168.98599999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.349,"5min":2.9060000000000001,"15min":3.069},"min":{"1min":0.371,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.9980000000000002,"5min":169.03800000000001,"15min":169.03800000000001},"last":3.0899999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795414,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3427468,"kb_used_data":2416004,"kb_used_omap":11,"kb_used_meta":1011444,"kb_avail":90317684,"statfs":{"total":95995035648,"available":92485308416,"internally_reserved":0,"allocated":2473988096,"data_stored":2471527219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":1035719253},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:04:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":1.105,"15min":1.8069999999999999},"min":{"1min":0.24399999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.3220000000000001,"5min":10.141,"15min":140.71100000000001},"last":27.292000000000002},{"interface":"front","average":{"1min":3.141,"5min":1.2989999999999999,"15min":2.1160000000000001},"min":{"1min":0.26700000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":9.1270000000000007,"5min":11.205,"15min":142.542},"last":0.41399999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:04:09 2023","interfaces":[{"interface":"back","average":{"1min":3.9329999999999998,"5min":3.6080000000000001,"15min":3.375},"min":{"1min":0.25900000000000001,"5min":0.19,"15min":0.19},"max":{"1min":12.093,"5min":133.42500000000001,"15min":133.42500000000001},"last":0.378},{"interface":"front","average":{"1min":4.8250000000000002,"5min":3.7679999999999998,"15min":3.4950000000000001},"min":{"1min":0.32000000000000001,"5min":0.214,"15min":0.214},"max":{"1min":15.83,"5min":133.40799999999999,"15min":133.40799999999999},"last":1.8460000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:03:48 2023","interfaces":[{"interface":"back","average":{"1min":12.804,"5min":3.794,"15min":2.5910000000000002},"min":{"1min":0.25600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":142.26300000000001,"5min":142.26300000000001,"15min":142.26300000000001},"last":12.728},{"interface":"front","average":{"1min":10.567,"5min":3.21,"15min":2.2799999999999998},"min":{"1min":0.35499999999999998,"5min":0.252,"15min":0.217},"max":{"1min":141.477,"5min":141.477,"15min":141.477},"last":16.286000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:04:09 2023","interfaces":[{"interface":"back","average":{"1min":2.3570000000000002,"5min":1.175,"15min":1.4670000000000001},"min":{"1min":0.32600000000000001,"5min":0.32300000000000001,"15min":0.32200000000000001},"max":{"1min":10.438000000000001,"5min":10.438000000000001,"15min":68.052000000000007},"last":2.0790000000000002},{"interface":"front","average":{"1min":1.2470000000000001,"5min":0.81999999999999995,"15min":1.306},"min":{"1min":0.38500000000000001,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":4.5209999999999999,"5min":5.4370000000000003,"15min":68.727000000000004},"last":16.457000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:04:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4119999999999999,"5min":1.1619999999999999,"15min":2.8210000000000002},"min":{"1min":0.438,"5min":0.33000000000000002,"15min":0.29099999999999998},"max":{"1min":9.0359999999999996,"5min":9.0359999999999996,"15min":207.80500000000001},"last":0.66300000000000003},{"interface":"front","average":{"1min":2.4969999999999999,"5min":1.5529999999999999,"15min":3.0270000000000001},"min":{"1min":0.373,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.484,"5min":10.484,"15min":207.54900000000001},"last":2.109}]},{"osd":6,"last update":"Mon Oct 9 06:03:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1,"15min":1.208},"min":{"1min":0.35099999999999998,"5min":0.32600000000000001,"15min":0.31},"max":{"1min":2.0939999999999999,"5min":9.6739999999999995,"15min":25.818999999999999},"last":16.422000000000001},{"interface":"front","average":{"1min":1.163,"5min":0.95899999999999996,"15min":1.415},"min":{"1min":0.36699999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.6980000000000004,"5min":6.6980000000000004,"15min":24.481000000000002},"last":0.628}]},{"osd":7,"last update":"Mon Oct 9 06:03:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.7270000000000001,"15min":2.339},"min":{"1min":0.54100000000000004,"5min":0.38400000000000001,"15min":0.34399999999999997},"max":{"1min":4.5899999999999999,"5min":51.749000000000002,"15min":162.09200000000001},"last":16.382999999999999},{"interface":"front","average":{"1min":2.4849999999999999,"5min":1.879,"15min":2.5680000000000001},"min":{"1min":0.40000000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.9710000000000001,"5min":51.801000000000002,"15min":162.35400000000001},"last":0.69199999999999995}]}]},{"osd":3,"up_from":103,"seq":442381631889,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2255388,"kb_used_data":1159188,"kb_used_omap":4,"kb_used_meta":1096123,"kb_avail":91489764,"statfs":{"total":95995035648,"available":93685518336,"internally_reserved":0,"allocated":1187008512,"data_stored":1184825122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1122430558},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:04:27 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":3.29,"15min":4.1319999999999997},"min":{"1min":0.223,"5min":0.20399999999999999,"15min":0.19700000000000001},"max":{"1min":9.9109999999999996,"5min":135.501,"15min":312.56599999999997},"last":1.4059999999999999},{"interface":"front","average":{"1min":2.3340000000000001,"5min":3.4399999999999999,"15min":4.0999999999999996},"min":{"1min":0.22,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.239000000000001,"5min":135.99199999999999,"15min":311.02300000000002},"last":0.50600000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:04:27 2023","interfaces":[{"interface":"back","average":{"1min":10.670999999999999,"5min":5.9349999999999996,"15min":5.1779999999999999},"min":{"1min":0.28799999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":89.165000000000006,"5min":137.00299999999999,"15min":150.29599999999999},"last":2.3769999999999998},{"interface":"front","average":{"1min":10.951000000000001,"5min":5.9340000000000002,"15min":5.2720000000000002},"min":{"1min":0.245,"5min":0.24399999999999999,"15min":0.216},"max":{"1min":89.093999999999994,"5min":136.947,"15min":151.25700000000001},"last":0.49099999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:04:27 2023","interfaces":[{"interface":"back","average":{"1min":2.871,"5min":3.3340000000000001,"15min":2.649},"min":{"1min":0.27700000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.055,"5min":87.997,"15min":87.997},"last":5.484},{"interface":"front","average":{"1min":1.994,"5min":3.222,"15min":2.5430000000000001},"min":{"1min":0.25900000000000001,"5min":0.248,"15min":0.248},"max":{"1min":6.8380000000000001,"5min":90.850999999999999,"15min":90.850999999999999},"last":0.39900000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":0.90700000000000003,"15min":1.7649999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31},"max":{"1min":6.9080000000000004,"5min":6.9080000000000004,"15min":126.566},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.6339999999999999,"5min":0.92900000000000005,"15min":2.1280000000000001},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.28999999999999998},"max":{"1min":5.569,"5min":5.6749999999999998,"15min":126.651},"last":0.316}]},{"osd":5,"last update":"Mon Oct 9 06:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":0.93500000000000005,"15min":1.5700000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.307},"max":{"1min":7.1779999999999999,"5min":7.1779999999999999,"15min":71.203999999999994},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.653,"5min":0.95699999999999996,"15min":1.486},"min":{"1min":0.36699999999999999,"5min":0.33400000000000002,"15min":0.309},"max":{"1min":5.601,"5min":5.601,"15min":70.397000000000006},"last":0.38500000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.88500000000000001,"15min":1.3460000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.3420000000000001,"5min":7.1719999999999997,"15min":70.474000000000004},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.006,"5min":1.0129999999999999,"15min":1.516},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":2.6000000000000001,"5min":5.1970000000000001,"15min":71.251999999999995},"last":0.51500000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:03:50 2023","interfaces":[{"interface":"back","average":{"1min":2.036,"5min":1.716,"15min":1.6639999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":6.931,"5min":12.218,"15min":69.277000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.0350000000000001,"5min":1.649,"15min":1.6040000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.374},"max":{"1min":6.8879999999999999,"5min":11.722,"15min":69.299000000000007},"last":0.46000000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019434,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4073852,"kb_used_data":3176436,"kb_used_omap":0,"kb_used_meta":897344,"kb_avail":89671300,"statfs":{"total":95995035648,"available":91823411200,"internally_reserved":0,"allocated":3252670464,"data_stored":3251512713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918880256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:04:15 2023","interfaces":[{"interface":"back","average":{"1min":7.2839999999999998,"5min":2.75,"15min":3.4670000000000001},"min":{"1min":0.248,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":86.509,"5min":86.509,"15min":132.35599999999999},"last":0.246},{"interface":"front","average":{"1min":7.5229999999999997,"5min":3.0329999999999999,"15min":3.9060000000000001},"min":{"1min":0.318,"5min":0.246,"15min":0.23699999999999999},"max":{"1min":86.233999999999995,"5min":86.233999999999995,"15min":132.501},"last":4.2930000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:03:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.458,"15min":2.7719999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":4.7009999999999996,"5min":55.799999999999997,"15min":96.033000000000001},"last":4.4939999999999998},{"interface":"front","average":{"1min":1.3440000000000001,"5min":1.6639999999999999,"15min":2.7890000000000001},"min":{"1min":0.309,"5min":0.29699999999999999,"15min":0.255},"max":{"1min":4.843,"5min":55.784999999999997,"15min":96.022000000000006},"last":4.5049999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:03:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5549999999999997,"5min":3.1909999999999998,"15min":2.2690000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28599999999999998},"max":{"1min":35.003,"5min":143.71000000000001,"15min":143.71000000000001},"last":1.796},{"interface":"front","average":{"1min":6.2350000000000003,"5min":3.7599999999999998,"15min":2.6989999999999998},"min":{"1min":0.38800000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":36.729999999999997,"5min":143.768,"15min":143.768},"last":2.2599999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:04:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5800000000000001,"5min":1.3480000000000001,"15min":2.4740000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":11.125999999999999,"5min":11.125999999999999,"15min":100.009},"last":0.309},{"interface":"front","average":{"1min":1.381,"5min":1.0209999999999999,"15min":2.294},"min":{"1min":0.34999999999999998,"5min":0.34599999999999997,"15min":0.30499999999999999},"max":{"1min":9.5969999999999995,"5min":9.5969999999999995,"15min":100.254},"last":0.40000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:04:29 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.94999999999999996,"15min":0.94399999999999995},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.25900000000000001},"max":{"1min":4.516,"5min":5.3040000000000003,"15min":15.597},"last":2.1789999999999998},{"interface":"front","average":{"1min":1.0529999999999999,"5min":0.91800000000000004,"15min":0.92600000000000005},"min":{"1min":0.35699999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.9900000000000002,"5min":5.3650000000000002,"15min":17.882000000000001},"last":2.2229999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.036,"15min":1.6759999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.919,"5min":6.0549999999999997,"15min":181.411},"last":0.432},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.1990000000000001,"15min":1.7549999999999999},"min":{"1min":0.32900000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.6379999999999999,"5min":7.3949999999999996,"15min":180.63999999999999},"last":1.7569999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.0009999999999999,"15min":1.1739999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.29799999999999999},"max":{"1min":3.8860000000000001,"5min":6.0810000000000004,"15min":36.729999999999997},"last":3.194},{"interface":"front","average":{"1min":0.628,"5min":0.97899999999999998,"15min":1.1080000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.27200000000000002},"max":{"1min":1.512,"5min":6.8979999999999997,"15min":36.424999999999997},"last":2.5899999999999999}]}]},{"osd":5,"up_from":0,"seq":154618823209,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13650,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3246264320,"data_stored":3246260248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1307705344,"data_stored":1307705344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2460352512,"data_stored":2460352512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1180176384,"data_stored":1180172312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3233091584,"data_stored":3233087512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2535522304,"data_stored":2535522304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1491861504,"data_stored":1491861504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:04:54.363 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:04:54.363 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 81 2023-10-09T06:04:54.395 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:54.671 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:54.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:54.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:54.740 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:54.788 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:54.860 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:04:54.861 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:04:54.899 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:55.008 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:04:55.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:54 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/77781085' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:55.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:54 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:54 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/77781085' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:55.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:54 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:55.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:54 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/77781085' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:54 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:55.580 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:55.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:55.698 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:55.732 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:55.790 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:55.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:55.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:55.895 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:55.928 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:55.990 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:55.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:56.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:56.000 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:55 smithi191 ceph-mon[161669]: pgmap v1636: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 23948/214395 objects degraded (11.170%); 2521/214395 objects misplaced (1.176%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:04:56.000 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:55 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:56.000 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:55 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:56.000 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:55 smithi191 ceph-mon[161669]: osdmap e543: 8 total, 8 up, 7 in 2023-10-09T06:04:56.000 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:55 smithi191 ceph-mon[161669]: failed to encode map e543 with expected crc 2023-10-09T06:04:56.100 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:56.201 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:56.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:55 smithi008 ceph-mon[184418]: pgmap v1636: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 23948/214395 objects degraded (11.170%); 2521/214395 objects misplaced (1.176%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:04:56.466 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:55 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:56.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:55 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:56.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:55 smithi008 ceph-mon[184418]: osdmap e543: 8 total, 8 up, 7 in 2023-10-09T06:04:56.467 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:55 smithi008 ceph-mon[184418]: failed to encode map e543 with expected crc 2023-10-09T06:04:56.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:55 smithi008 ceph-mon[191227]: pgmap v1636: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 23948/214395 objects degraded (11.170%); 2521/214395 objects misplaced (1.176%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:04:56.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:55 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:56.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:55 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:56.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:55 smithi008 ceph-mon[191227]: osdmap e543: 8 total, 8 up, 7 in 2023-10-09T06:04:56.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:55 smithi008 ceph-mon[191227]: failed to encode map e543 with expected crc 2023-10-09T06:04:56.936 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:56.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:56.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:56.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:04:57.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:57.094 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:57.162 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:57.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:57.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:57.194 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:56 smithi008 ceph-mon[191227]: failed to encode map e543 with expected crc 2023-10-09T06:04:57.195 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:56 smithi008 ceph-mon[184418]: failed to encode map e543 with expected crc 2023-10-09T06:04:57.271 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:57.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:56 smithi191 ceph-mon[161669]: failed to encode map e543 with expected crc 2023-10-09T06:04:57.323 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:57.394 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:57.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:57.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:57.448 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:04:57.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:57.548 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:57.658 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:57.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:57.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:57.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:57.794 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:57.887 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:57.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:57.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:57.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:57.998 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:57 smithi191 ceph-mon[161669]: pgmap v1638: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 23612/209580 objects degraded (11.266%); 2370/209580 objects misplaced (1.131%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:04:58.026 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:58.090 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:58.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:58.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:58.197 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:58.229 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:58.290 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:58.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:58.298 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:58.306 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:57 smithi008 ceph-mon[184418]: pgmap v1638: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 23612/209580 objects degraded (11.266%); 2370/209580 objects misplaced (1.131%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:04:58.307 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:57 smithi008 ceph-mon[191227]: pgmap v1638: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 23612/209580 objects degraded (11.266%); 2370/209580 objects misplaced (1.131%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:04:58.394 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:58.505 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:58.771 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:58.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:58.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:58.876 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:58.983 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:59.029 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:04:59.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:59 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23612/209580 objects degraded (11.266%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:59.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:04:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:59.352 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:04:59.390 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:04:59.413 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:59.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:59.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:59.432 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:59 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23612/209580 objects degraded (11.266%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:59.432 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:04:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:59.433 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:59 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23612/209580 objects degraded (11.266%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:04:59.433 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:04:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:59.453 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:04:59.499 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:04:59.529 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:59.535 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:04:59.584 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:04:59.598 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:59.622 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:04:59.667 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:04:59.699 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:04:59.699 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:04:59.699 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:04:59.700 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:04:59.700 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:04:59.700 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:04:59.700 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:04:59.700 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:05:00.017 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:00 smithi008 ceph-mon[184418]: pgmap v1639: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23612/209112 objects degraded (11.292%); 2370/209112 objects misplaced (1.133%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:05:00.017 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1004111803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:00.212 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:05:00.212 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T06:05:00.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.213 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:106837 10.4 10:28be2b9a:::benchmark_data_smithi008_634586_object19672:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:00.213 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:00.182222+0000", 2023-10-09T06:05:00.213 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.029172419000000002, 2023-10-09T06:05:00.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.029194599000000002, 2023-10-09T06:05:00.213 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:00.213 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:05:00.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:00.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:00.213 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:00.214 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 106837 2023-10-09T06:05:00.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.214 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:00.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:00.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.182222+0000", 2023-10-09T06:05:00.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:00.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:00.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.182222+0000", 2023-10-09T06:05:00.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:00.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:00.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.182220+0000", 2023-10-09T06:05:00.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1389999999999998e-06 2023-10-09T06:05:00.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:00.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.182228+0000", 2023-10-09T06:05:00.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3942e-05 2023-10-09T06:05:00.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:00.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.182242+0000", 2023-10-09T06:05:00.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.677e-06 2023-10-09T06:05:00.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:00.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.182247+0000", 2023-10-09T06:05:00.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003798175 2023-10-09T06:05:00.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:00.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.186045+0000", 2023-10-09T06:05:00.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000141798 2023-10-09T06:05:00.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:00.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.186187+0000", 2023-10-09T06:05:00.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011242400000000001 2023-10-09T06:05:00.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:05:00.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.186300+0000", 2023-10-09T06:05:00.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030391810000000002 2023-10-09T06:05:00.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:00.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.189339+0000", 2023-10-09T06:05:00.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022803630000000001 2023-10-09T06:05:00.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:00.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.191619+0000", 2023-10-09T06:05:00.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0093968539999999996 2023-10-09T06:05:00.221 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:00.222 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:00.222 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:00.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.222 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:106842 10.4 10:24c08be3:::benchmark_data_smithi008_634586_object19677:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:00.222 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:00.192291+0000", 2023-10-09T06:05:00.222 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.019104185999999999, 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019185718000000001, 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 106842 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.192291+0000", 2023-10-09T06:05:00.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.192291+0000", 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.192289+0000", 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9370000000000004e-06 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.192297+0000", 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5554999999999999e-05 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.192312+0000", 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.99e-06 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.192319+0000", 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9848e-05 2023-10-09T06:05:00.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.192379+0000", 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000235703 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.192615+0000", 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00023988699999999999 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.192855+0000", 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041359370000000001 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.196991+0000", 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0091712900000000003 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.206162+0000", 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013871504999999999 2023-10-09T06:05:00.227 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.47036.0:106851 10.c e543/512 10:3cd77ae2:::benchmark_data_smithi008_634586_object19686:head v 543'6668, mlcod=543'6664)", 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:00.210641+0000", 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.000753759, 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00088666099999999996, 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:00.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.210641+0000", 2023-10-09T06:05:00.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:00.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:00.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.210641+0000", 2023-10-09T06:05:00.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:00.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:00.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.210640+0000", 2023-10-09T06:05:00.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6050000000000004e-06 2023-10-09T06:05:00.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:00.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.210649+0000", 2023-10-09T06:05:00.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5332000000000002e-05 2023-10-09T06:05:00.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:00.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.210665+0000", 2023-10-09T06:05:00.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7379999999999997e-06 2023-10-09T06:05:00.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:00.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.210671+0000", 2023-10-09T06:05:00.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7039000000000002e-05 2023-10-09T06:05:00.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.210708+0000", 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.628e-05 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.210725+0000", 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3720999999999993e-05 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:00.232 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:00.233 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-10-09T06:05:00.233 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 3 2023-10-09T06:05:00.233 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:05:00.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:00 smithi191 ceph-mon[161669]: pgmap v1639: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23612/209112 objects degraded (11.292%); 2370/209112 objects misplaced (1.133%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:05:00.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1004111803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:00.423 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:00.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:00.437 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:00 smithi008 ceph-mon[191227]: pgmap v1639: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23612/209112 objects degraded (11.292%); 2370/209112 objects misplaced (1.133%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:05:00.437 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1004111803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:00.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:00.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:00.582 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:00.716 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:05:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:05:00] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.33.4" 2023-10-09T06:05:00.792 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:00.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:00.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:00.891 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:05:00.897 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:00.968 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:01.012 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:01.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:01.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:01.118 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:01.203 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:01.328 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:05:01.333 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:01.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:01.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:01.435 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:01.444 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:01.483 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:01.589 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:01.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:01.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:01.693 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:01.728 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:01.892 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:01.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:01.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:01.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:02.027 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:02.072 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:02.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:02.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:02.176 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:02.205 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:02.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:02 smithi191 ceph-mon[161669]: pgmap v1640: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23612/206784 objects degraded (11.419%); 2370/206784 objects misplaced (1.146%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:05:02.464 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T06:05:02.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:02 smithi008 ceph-mon[184418]: pgmap v1640: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23612/206784 objects degraded (11.419%); 2370/206784 objects misplaced (1.146%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:05:02.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:02 smithi008 ceph-mon[191227]: pgmap v1640: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23612/206784 objects degraded (11.419%); 2370/206784 objects misplaced (1.146%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:05:02.491 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:02.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:02.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:02.599 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:02.654 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:02.701 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:02.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:02.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:02.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:02.838 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:02.885 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:02.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:02.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:02.931 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:05:02.931 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:05:02.932 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T06:05:02.932 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T06:05:02.932 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:05:02.988 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:03.017 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:03.162 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:03.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:03.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:03.266 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:03.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:03 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1814283857' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:05:03.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:03 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:05:03.309 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:05:03.317 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:03 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1814283857' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:05:03.317 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:03 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:05:03.317 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:03 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1814283857' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:05:03.317 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:03 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:05:03.324 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:03.721 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:05:03.941 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:03.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:03.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:04.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:04.084 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:04.153 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:04.155 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:05:04.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:04.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:04.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:04 smithi008 ceph-mon[191227]: pgmap v1641: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23346/201747 objects degraded (11.572%); 2199/201747 objects misplaced (1.090%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T06:05:04.219 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:04 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23346/201747 objects degraded (11.572%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:04.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:04 smithi008 ceph-mon[184418]: pgmap v1641: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23346/201747 objects degraded (11.572%); 2199/201747 objects misplaced (1.090%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T06:05:04.221 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:04 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23346/201747 objects degraded (11.572%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:04.275 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:04.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:04 smithi191 ceph-mon[161669]: pgmap v1641: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 23346/201747 objects degraded (11.572%); 2199/201747 objects misplaced (1.090%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T06:05:04.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:04 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23346/201747 objects degraded (11.572%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:04.303 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:04.326 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T06:05:04.423 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:04.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:04.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:04.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:04.566 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:04.697 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:04.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:04.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:04.712 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:04.802 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:04.917 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:05.108 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:05.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:05.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:05.218 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:05.293 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:05.713 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:05.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:05.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:05.821 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:05.878 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:05:05.878 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T06:05:05.878 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T06:05:05.878 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T06:05:05.878 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:63144 10.4 10:2f798f30:::benchmark_data_smithi008_634586_object63143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:19.670240+0000", 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 106.205230923, 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.321828318, 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 63144 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.879 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670240+0000", 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670240+0000", 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.880 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670238+0000", 2023-10-09T06:05:05.881 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8511000000000003e-05 2023-10-09T06:05:05.881 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.881 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.881 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.881 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670297+0000", 2023-10-09T06:05:05.881 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.396e-05 2023-10-09T06:05:05.881 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.881 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.881 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.881 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670311+0000", 2023-10-09T06:05:05.881 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4899999999999997e-06 2023-10-09T06:05:05.882 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.882 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.882 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.882 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670317+0000", 2023-10-09T06:05:05.882 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4267000000000002e-05 2023-10-09T06:05:05.882 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.882 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.882 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.882 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670362+0000", 2023-10-09T06:05:05.882 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000140066 2023-10-09T06:05:05.882 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670502+0000", 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000169261 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670671+0000", 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020061380000000002 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.883 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.672677+0000", 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041772579999999997 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.676854+0000", 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31327599900000003 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.990130+0000", 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7629000000000002e-05 2023-10-09T06:05:05.884 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.990168+0000", 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019004899999999999 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.992068+0000", 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.321828318 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.885 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:63154 10.4 10:2bf45174:::benchmark_data_smithi008_634586_object63153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:19.676995+0000", 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 106.19847580699999, 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31648402199999998, 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 63154 2023-10-09T06:05:05.886 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.676995+0000", 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.676995+0000", 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.887 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.676994+0000", 2023-10-09T06:05:05.888 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2188000000000003e-05 2023-10-09T06:05:05.888 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.888 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.888 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.888 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677036+0000", 2023-10-09T06:05:05.888 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2265e-05 2023-10-09T06:05:05.888 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677048+0000", 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.242e-06 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677054+0000", 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9261999999999998e-05 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.889 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677093+0000", 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013400700000000001 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677227+0000", 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014552799999999999 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677372+0000", 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001802884 2023-10-09T06:05:05.890 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.891 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.891 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.891 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.679175+0000", 2023-10-09T06:05:05.891 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010563809999999999 2023-10-09T06:05:05.891 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.891 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.891 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.891 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.680232+0000", 2023-10-09T06:05:05.891 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.311896966 2023-10-09T06:05:05.891 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.992129+0000", 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6134000000000002e-05 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.992185+0000", 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012944129999999999 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.993479+0000", 2023-10-09T06:05:05.892 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31648402199999998 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:63165 10.4 10:2b8ca226:::benchmark_data_smithi008_634586_object63164:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:19.690274+0000", 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 106.185196629, 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30505157300000002, 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.893 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.894 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.894 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 63165 2023-10-09T06:05:05.894 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.894 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.894 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.894 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.894 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690274+0000", 2023-10-09T06:05:05.894 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.894 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690274+0000", 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690273+0000", 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6658e-05 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690309+0000", 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5274e-05 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690325+0000", 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6409999999999996e-06 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690331+0000", 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2237000000000006e-05 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.896 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690413+0000", 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018728399999999999 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690601+0000", 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000158632 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690759+0000", 2023-10-09T06:05:05.897 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001824401 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.692584+0000", 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094702999999999996 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.693531+0000", 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.300027186 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.898 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.993558+0000", 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6448999999999999e-05 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.993634+0000", 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016915840000000001 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.995326+0000", 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30505157300000002 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:63172 10.4 10:2721e6a1:::benchmark_data_smithi008_634586_object63171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:19.698313+0000", 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 106.177158484, 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29883361000000003, 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.900 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.901 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 63172 2023-10-09T06:05:05.901 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.901 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.901 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.901 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.901 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698313+0000", 2023-10-09T06:05:05.901 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.901 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.901 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.901 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.901 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698313+0000", 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698311+0000", 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5070999999999997e-05 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698346+0000", 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.279e-05 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.902 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698359+0000", 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3580000000000002e-06 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698364+0000", 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0896999999999998e-05 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698405+0000", 2023-10-09T06:05:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000130159 2023-10-09T06:05:05.904 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.904 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.904 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.904 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698535+0000", 2023-10-09T06:05:05.904 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019234499999999999 2023-10-09T06:05:05.904 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.904 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.904 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:05:05.904 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698727+0000", 2023-10-09T06:05:05.904 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001770095 2023-10-09T06:05:05.904 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.905 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.905 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.905 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.700498+0000", 2023-10-09T06:05:05.905 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00108668 2023-10-09T06:05:05.905 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.905 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.906 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.906 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.701584+0000", 2023-10-09T06:05:05.906 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29380753199999998 2023-10-09T06:05:05.906 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.906 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.906 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.906 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.995392+0000", 2023-10-09T06:05:05.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.004e-05 2023-10-09T06:05:05.907 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.907 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.907 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.907 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.995462+0000", 2023-10-09T06:05:05.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016843330000000001 2023-10-09T06:05:05.907 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.997146+0000", 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29883361000000003 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:83465 10.4 10:236eb373:::benchmark_data_smithi008_634586_object83464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e526)", 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:49.960539+0000", 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 75.914932179000004, 2023-10-09T06:05:05.908 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27214083900000002, 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 83465 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960539+0000", 2023-10-09T06:05:05.909 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960539+0000", 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960537+0000", 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000148379 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.910 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960685+0000", 2023-10-09T06:05:05.911 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6019e-05 2023-10-09T06:05:05.911 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.911 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960701+0000", 2023-10-09T06:05:05.911 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.708e-06 2023-10-09T06:05:05.911 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.911 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.912 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960708+0000", 2023-10-09T06:05:05.912 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5919000000000004e-05 2023-10-09T06:05:05.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.912 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.912 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.912 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960774+0000", 2023-10-09T06:05:05.912 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000146866 2023-10-09T06:05:05.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.912 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.912 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.912 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960921+0000", 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020136000000000001 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.961122+0000", 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0062320800000000001 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.967354+0000", 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2861000000000001e-05 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.967407+0000", 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.26340967399999998 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.230817+0000", 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5763999999999997e-05 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.914 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.230873+0000", 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018070499999999999 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.232680+0000", 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27214083900000002 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:83467 10.4 10:28157da3:::benchmark_data_smithi008_634586_object83466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e526)", 2023-10-09T06:05:05.915 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:49.965092+0000", 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 75.910379231999997, 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.269243485, 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 83467 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.916 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965092+0000", 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965092+0000", 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965090+0000", 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7626000000000003e-05 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.917 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965188+0000", 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2163e-05 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965200+0000", 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3829999999999997e-06 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.918 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965207+0000", 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022278089999999999 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.967434+0000", 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017571899999999999 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.967610+0000", 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021979599999999999 2023-10-09T06:05:05.919 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.967830+0000", 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026064780000000002 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.970436+0000", 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033157719999999998 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.920 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.973752+0000", 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.258994418 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.232747+0000", 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4076999999999998e-05 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.232791+0000", 2023-10-09T06:05:05.921 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015446609999999999 2023-10-09T06:05:05.922 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.922 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.922 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.922 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.234335+0000", 2023-10-09T06:05:05.922 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.269243485 2023-10-09T06:05:05.922 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.922 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.922 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.922 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.922 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.923 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:96490 10.d 10:bd3938ce:::benchmark_data_smithi008_634586_object9325:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T06:05:05.923 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:04:39.104315+0000", 2023-10-09T06:05:05.923 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 26.771156148999999, 2023-10-09T06:05:05.923 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.256447588, 2023-10-09T06:05:05.923 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.923 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.923 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.923 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.923 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.923 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 96490 2023-10-09T06:05:05.923 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.924 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.924 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.924 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.924 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104315+0000", 2023-10-09T06:05:05.924 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.924 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.924 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.924 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.924 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104315+0000", 2023-10-09T06:05:05.924 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:05.924 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104314+0000", 2023-10-09T06:05:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8289999999999999e-06 2023-10-09T06:05:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104317+0000", 2023-10-09T06:05:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7299999999999994e-06 2023-10-09T06:05:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.925 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.926 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.926 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104326+0000", 2023-10-09T06:05:05.926 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8e-06 2023-10-09T06:05:05.926 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.926 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.926 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.926 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104330+0000", 2023-10-09T06:05:05.927 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002211845 2023-10-09T06:05:05.927 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.927 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.927 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.927 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.106542+0000", 2023-10-09T06:05:05.927 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00065125600000000001 2023-10-09T06:05:05.928 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.928 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.928 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:05.928 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.107193+0000", 2023-10-09T06:05:05.928 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.03810931 2023-10-09T06:05:05.928 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.928 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.929 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.929 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.145302+0000", 2023-10-09T06:05:05.929 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5633e-05 2023-10-09T06:05:05.929 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.929 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.929 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.930 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.145368+0000", 2023-10-09T06:05:05.930 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010242300000000001 2023-10-09T06:05:05.930 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.930 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.930 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:05.930 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.145470+0000", 2023-10-09T06:05:05.930 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014231242999999999 2023-10-09T06:05:05.931 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.931 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.931 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.931 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.159701+0000", 2023-10-09T06:05:05.931 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028590120000000002 2023-10-09T06:05:05.931 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.931 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.931 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.932 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.162560+0000", 2023-10-09T06:05:05.932 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.194264938 2023-10-09T06:05:05.932 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.932 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.932 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.932 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.356825+0000", 2023-10-09T06:05:05.932 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7241999999999998e-05 2023-10-09T06:05:05.933 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.933 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.933 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.933 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.356883+0000", 2023-10-09T06:05:05.933 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038798109999999999 2023-10-09T06:05:05.933 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.933 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.933 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.360762+0000", 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.256447588 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:96496 10.d 10:b1be06e8:::benchmark_data_smithi008_634586_object9331:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:04:39.104798+0000", 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 26.770673121000002, 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25924583099999998, 2023-10-09T06:05:05.934 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 96496 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104798+0000", 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.935 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.936 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.936 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.936 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104798+0000", 2023-10-09T06:05:05.936 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:05:05.936 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.937 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.937 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.937 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104797+0000", 2023-10-09T06:05:05.937 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4190000000000004e-06 2023-10-09T06:05:05.937 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.937 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.937 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.937 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104802+0000", 2023-10-09T06:05:05.937 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0399999999999998e-06 2023-10-09T06:05:05.937 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.938 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.938 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.938 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104811+0000", 2023-10-09T06:05:05.938 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3870000000000003e-06 2023-10-09T06:05:05.938 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.938 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.938 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.938 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104816+0000", 2023-10-09T06:05:05.938 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024788470000000002 2023-10-09T06:05:05.938 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.938 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.939 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.939 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.107295+0000", 2023-10-09T06:05:05.939 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00066429100000000001 2023-10-09T06:05:05.939 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.939 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.939 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:05.939 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.107959+0000", 2023-10-09T06:05:05.939 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.044045075000000003 2023-10-09T06:05:05.939 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.939 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.940 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.940 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.152004+0000", 2023-10-09T06:05:05.940 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2082999999999995e-05 2023-10-09T06:05:05.940 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.940 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.940 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.940 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.152066+0000", 2023-10-09T06:05:05.940 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4959999999999999e-05 2023-10-09T06:05:05.941 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.941 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.941 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:05.941 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.152161+0000", 2023-10-09T06:05:05.941 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010370736 2023-10-09T06:05:05.941 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.941 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.941 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.942 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.162532+0000", 2023-10-09T06:05:05.942 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5365999999999996e-05 2023-10-09T06:05:05.942 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.942 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.942 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.942 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.162607+0000", 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.198236308 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.360844+0000", 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5483999999999998e-05 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.360909+0000", 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031345639999999998 2023-10-09T06:05:05.943 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.364044+0000", 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25924583099999998 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:96499 10.d 10:b047fea9:::benchmark_data_smithi008_634586_object9334:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:04:39.114743+0000", 2023-10-09T06:05:05.944 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 26.760727541000001, 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25254888800000003, 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 96499 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114743+0000", 2023-10-09T06:05:05.945 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114743+0000", 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114742+0000", 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8010000000000004e-06 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.946 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114749+0000", 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2295e-05 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114762+0000", 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6520000000000006e-06 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114769+0000", 2023-10-09T06:05:05.947 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3637000000000002e-05 2023-10-09T06:05:05.948 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.948 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.948 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.948 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114823+0000", 2023-10-09T06:05:05.948 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000755129 2023-10-09T06:05:05.948 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.948 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.948 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:05.948 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.115578+0000", 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.044154153000000002 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.159732+0000", 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6231999999999998e-05 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.159809+0000", 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108804 2023-10-09T06:05:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.159917+0000", 2023-10-09T06:05:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004328068 2023-10-09T06:05:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.164245+0000", 2023-10-09T06:05:05.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5066999999999999e-05 2023-10-09T06:05:05.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.164271+0000", 2023-10-09T06:05:05.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19981747799999999 2023-10-09T06:05:05.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.364088+0000", 2023-10-09T06:05:05.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8321000000000001e-05 2023-10-09T06:05:05.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.953 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.364126+0000", 2023-10-09T06:05:05.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031660410000000001 2023-10-09T06:05:05.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.953 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.954 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.367292+0000", 2023-10-09T06:05:05.954 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25254888800000003 2023-10-09T06:05:05.954 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.954 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.954 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.954 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.955 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:107166 10.d 10:b89929d0:::benchmark_data_smithi008_634586_object20001:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:05.955 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:00.692047+0000", 2023-10-09T06:05:05.955 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 5.183424048, 2023-10-09T06:05:05.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14558733500000001, 2023-10-09T06:05:05.955 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.955 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.955 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.956 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.956 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.956 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 107166 2023-10-09T06:05:05.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.956 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.956 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692047+0000", 2023-10-09T06:05:05.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692047+0000", 2023-10-09T06:05:05.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:05.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.958 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692046+0000", 2023-10-09T06:05:05.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1569999999999997e-06 2023-10-09T06:05:05.958 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692052+0000", 2023-10-09T06:05:05.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1018e-05 2023-10-09T06:05:05.959 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.959 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692063+0000", 2023-10-09T06:05:05.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6460000000000004e-06 2023-10-09T06:05:05.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.960 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.960 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.960 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692069+0000", 2023-10-09T06:05:05.960 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.031788943 2023-10-09T06:05:05.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.961 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.723858+0000", 2023-10-09T06:05:05.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00056661999999999999 2023-10-09T06:05:05.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:05.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.724425+0000", 2023-10-09T06:05:05.962 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021166876000000001 2023-10-09T06:05:05.962 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.745592+0000", 2023-10-09T06:05:05.962 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3757000000000006e-05 2023-10-09T06:05:05.963 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.963 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.963 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.963 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.745666+0000", 2023-10-09T06:05:05.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011078699999999999 2023-10-09T06:05:05.963 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.963 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.964 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:05.964 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.745776+0000", 2023-10-09T06:05:05.964 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011310005999999999 2023-10-09T06:05:05.964 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.964 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.964 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.964 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.757086+0000", 2023-10-09T06:05:05.965 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014765399999999999 2023-10-09T06:05:05.965 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.965 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.965 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.965 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.758563+0000", 2023-10-09T06:05:05.965 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.076508077999999993 2023-10-09T06:05:05.965 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.965 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.966 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.966 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.835071+0000", 2023-10-09T06:05:05.966 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9296999999999997e-05 2023-10-09T06:05:05.966 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.966 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.966 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.966 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.835140+0000", 2023-10-09T06:05:05.967 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024940520000000001 2023-10-09T06:05:05.967 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.967 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.967 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.967 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.837634+0000", 2023-10-09T06:05:05.967 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14558733500000001 2023-10-09T06:05:05.967 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.968 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.968 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.968 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.968 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.968 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:107168 10.d 10:bd618b9b:::benchmark_data_smithi008_634586_object20003:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:05.968 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:00.692766+0000", 2023-10-09T06:05:05.968 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 5.1827048649999998, 2023-10-09T06:05:05.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14821151499999999, 2023-10-09T06:05:05.969 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.969 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.969 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.969 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.969 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.969 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 107168 2023-10-09T06:05:05.969 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.970 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.970 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.970 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692766+0000", 2023-10-09T06:05:05.970 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.970 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.971 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.971 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692766+0000", 2023-10-09T06:05:05.971 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:05:05.971 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.971 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.971 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.971 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692765+0000", 2023-10-09T06:05:05.972 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0990000000000004e-06 2023-10-09T06:05:05.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.972 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.972 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.972 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692771+0000", 2023-10-09T06:05:05.972 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.446e-05 2023-10-09T06:05:05.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.972 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.973 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.973 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692786+0000", 2023-10-09T06:05:05.973 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.336e-06 2023-10-09T06:05:05.973 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.973 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.973 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.973 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.692792+0000", 2023-10-09T06:05:05.973 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.031684459999999998 2023-10-09T06:05:05.974 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.974 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.974 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.974 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.724476+0000", 2023-10-09T06:05:05.974 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00066004099999999999 2023-10-09T06:05:05.974 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.975 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:05.975 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.725137+0000", 2023-10-09T06:05:05.975 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028213014000000002 2023-10-09T06:05:05.975 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.975 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.975 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.753350+0000", 2023-10-09T06:05:05.976 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2319999999999999e-05 2023-10-09T06:05:05.976 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.976 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.976 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.976 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.753422+0000", 2023-10-09T06:05:05.976 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106679 2023-10-09T06:05:05.976 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.977 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.977 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:05.977 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.753529+0000", 2023-10-09T06:05:05.977 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0036174530000000001 2023-10-09T06:05:05.977 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.977 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.977 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.978 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.757146+0000", 2023-10-09T06:05:05.978 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032948500000000002 2023-10-09T06:05:05.978 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.978 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.978 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.978 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.760441+0000", 2023-10-09T06:05:05.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.077272752 2023-10-09T06:05:05.979 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.979 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.837714+0000", 2023-10-09T06:05:05.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1266000000000006e-05 2023-10-09T06:05:05.979 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.980 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.980 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.837775+0000", 2023-10-09T06:05:05.980 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032028009999999999 2023-10-09T06:05:05.980 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.980 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.980 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.980 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.840978+0000", 2023-10-09T06:05:05.981 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14821151499999999 2023-10-09T06:05:05.981 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.981 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.981 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.981 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.981 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.981 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:107195 10.d 10:bf3e70f1:::benchmark_data_smithi008_634586_object20030:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:05.981 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:00.737038+0000", 2023-10-09T06:05:05.982 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 5.1384334020000004, 2023-10-09T06:05:05.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.131912538, 2023-10-09T06:05:05.982 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.982 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.982 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.982 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.983 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.983 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 107195 2023-10-09T06:05:05.983 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.983 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.983 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.983 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.983 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.737038+0000", 2023-10-09T06:05:05.983 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.984 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.984 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.984 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.984 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.737038+0000", 2023-10-09T06:05:05.984 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:05:05.984 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.984 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.985 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.985 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.737036+0000", 2023-10-09T06:05:05.985 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8239999999999996e-06 2023-10-09T06:05:05.985 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.985 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.985 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.985 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.737042+0000", 2023-10-09T06:05:05.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0322e-05 2023-10-09T06:05:05.986 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.986 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.986 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.737053+0000", 2023-10-09T06:05:05.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.87e-06 2023-10-09T06:05:05.986 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.987 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.987 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:05.987 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.737057+0000", 2023-10-09T06:05:05.987 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020128769000000001 2023-10-09T06:05:05.987 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.987 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.988 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.757186+0000", 2023-10-09T06:05:05.988 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00071024800000000002 2023-10-09T06:05:05.988 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.988 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:05.988 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.757896+0000", 2023-10-09T06:05:05.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.087522207000000005 2023-10-09T06:05:05.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.989 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.989 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:05.989 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.845419+0000", 2023-10-09T06:05:05.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0210000000000006e-05 2023-10-09T06:05:05.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.990 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:05.990 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.845499+0000", 2023-10-09T06:05:05.990 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000113649 2023-10-09T06:05:05.990 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.990 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:05.991 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.845613+0000", 2023-10-09T06:05:05.991 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019400034 2023-10-09T06:05:05.991 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.991 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.991 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:05.991 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.865013+0000", 2023-10-09T06:05:05.991 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4684999999999995e-05 2023-10-09T06:05:05.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.992 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.992 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.865077+0000", 2023-10-09T06:05:05.992 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6204000000000004e-05 2023-10-09T06:05:05.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:05.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.865153+0000", 2023-10-09T06:05:05.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8674999999999997e-05 2023-10-09T06:05:05.993 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.993 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:05.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.865202+0000", 2023-10-09T06:05:05.994 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037480230000000001 2023-10-09T06:05:05.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.994 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:05.994 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.868950+0000", 2023-10-09T06:05:05.994 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.131912538 2023-10-09T06:05:05.995 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.995 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:05.995 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:05.995 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.995 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.995 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:107204 10.d 10:bffaa473:::benchmark_data_smithi008_634586_object20039:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:05.995 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:00.743879+0000", 2023-10-09T06:05:05.996 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 5.1315925050000004, 2023-10-09T06:05:05.996 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13465060700000001, 2023-10-09T06:05:05.996 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:05.996 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:05.996 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:05.996 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:05.996 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:05.996 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 107204 2023-10-09T06:05:05.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.997 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:05.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:05.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.743879+0000", 2023-10-09T06:05:05.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:05.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:05.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.743879+0000", 2023-10-09T06:05:05.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:05.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:05.998 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.743877+0000", 2023-10-09T06:05:05.998 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1690000000000003e-06 2023-10-09T06:05:05.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:05.999 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.743884+0000", 2023-10-09T06:05:05.999 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3261999999999999e-05 2023-10-09T06:05:05.999 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:05.999 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:05.999 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:05.999 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.743897+0000", 2023-10-09T06:05:06.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7649999999999999e-06 2023-10-09T06:05:06.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.743903+0000", 2023-10-09T06:05:06.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014038128 2023-10-09T06:05:06.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.757941+0000", 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060791000000000003 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.758549+0000", 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.093878050000000005 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.852427+0000", 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0049000000000005e-05 2023-10-09T06:05:06.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:06.002 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.852517+0000", 2023-10-09T06:05:06.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000186968 2023-10-09T06:05:06.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.002 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.852704+0000", 2023-10-09T06:05:06.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012414378 2023-10-09T06:05:06.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.865119+0000", 2023-10-09T06:05:06.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9990000000000003e-06 2023-10-09T06:05:06.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:06.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.865129+0000", 2023-10-09T06:05:06.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099168180000000009 2023-10-09T06:05:06.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.875045+0000", 2023-10-09T06:05:06.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3354000000000004e-05 2023-10-09T06:05:06.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:06.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.875109+0000", 2023-10-09T06:05:06.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0034204159999999999 2023-10-09T06:05:06.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.006 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:06.006 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.878529+0000", 2023-10-09T06:05:06.006 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13465060700000001 2023-10-09T06:05:06.006 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.006 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:06.006 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.007 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.007 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.007 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:107205 10.d 10:b8c1e885:::benchmark_data_smithi008_634586_object20040:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:06.007 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:00.747669+0000", 2023-10-09T06:05:06.007 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 5.1278019969999997, 2023-10-09T06:05:06.007 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.141245115, 2023-10-09T06:05:06.007 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:06.007 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 107205 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.747669+0000", 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.747669+0000", 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.747668+0000", 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5889999999999999e-06 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.747674+0000", 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0899e-05 2023-10-09T06:05:06.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:06.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.747685+0000", 2023-10-09T06:05:06.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4469999999999997e-06 2023-10-09T06:05:06.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.747690+0000", 2023-10-09T06:05:06.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010920388 2023-10-09T06:05:06.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.758611+0000", 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00049980300000000005 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.759111+0000", 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.102331431 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.861442+0000", 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100189 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.861542+0000", 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012240800000000001 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.861665+0000", 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080427970000000008 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.869707+0000", 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040622829999999999 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.873770+0000", 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011540477 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.885310+0000", 2023-10-09T06:05:06.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8225000000000005e-05 2023-10-09T06:05:06.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:06.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.885378+0000", 2023-10-09T06:05:06.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035356540000000001 2023-10-09T06:05:06.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:06.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:00.888914+0000", 2023-10-09T06:05:06.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.141245115 2023-10-09T06:05:06.015 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.015 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:06.015 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:107850 10.4 10:2fc76099:::benchmark_data_smithi008_634586_object20685:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:02.040387+0000", 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 3.8350836319999999, 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12656953500000001, 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 107850 2023-10-09T06:05:06.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.040387+0000", 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.040387+0000", 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.040387+0000", 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2799999999999999e-06 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.040390+0000", 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.93e-06 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:06.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.040396+0000", 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4730000000000002e-06 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.040401+0000", 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.105066145 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.145467+0000", 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018330899999999999 2023-10-09T06:05:06.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.145650+0000", 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017270100000000001 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.145823+0000", 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01123556 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.020 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.157058+0000", 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003433045 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.160491+0000", 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004954094 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.165446+0000", 2023-10-09T06:05:06.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3475e-05 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.165489+0000", 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001467929 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.166957+0000", 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12656953500000001 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.022 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:107961 10.d 10:beb5c3d2:::benchmark_data_smithi008_634586_object20796:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:02.333187+0000", 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 3.5422836059999998, 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.11529943600000001, 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:06.023 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:06.025 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 107961 2023-10-09T06:05:06.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.025 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:06.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:06.025 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.333187+0000", 2023-10-09T06:05:06.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:06.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:06.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.333187+0000", 2023-10-09T06:05:06.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:05:06.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:06.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.333186+0000", 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3330000000000002e-06 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.333193+0000", 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6359999999999999e-05 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.333209+0000", 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0777e-05 2023-10-09T06:05:06.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.333220+0000", 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001971591 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.335191+0000", 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00066993700000000001 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.335861+0000", 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.050657122999999998 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.386518+0000", 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5974000000000001e-05 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.386604+0000", 2023-10-09T06:05:06.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120967 2023-10-09T06:05:06.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.386725+0000", 2023-10-09T06:05:06.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011257506 2023-10-09T06:05:06.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:06.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.397983+0000", 2023-10-09T06:05:06.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9029000000000005e-05 2023-10-09T06:05:06.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.398062+0000", 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.047848612999999998 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.445911+0000", 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0890000000000002e-05 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:06.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.445961+0000", 2023-10-09T06:05:06.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025254129999999998 2023-10-09T06:05:06.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:06.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.448487+0000", 2023-10-09T06:05:06.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.11529943600000001 2023-10-09T06:05:06.032 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.032 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:06.032 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:108082 10.d 10:be6cf74f:::benchmark_data_smithi008_634586_object20917:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:02.567231+0000", 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 3.3082404830000001, 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23090850800000001, 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 108082 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:06.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:06.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567231+0000", 2023-10-09T06:05:06.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:06.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:06.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567231+0000", 2023-10-09T06:05:06.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:06.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567229+0000", 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4339999999999998e-06 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567239+0000", 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5401999999999999e-05 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567254+0000", 2023-10-09T06:05:06.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1210000000000003e-06 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567260+0000", 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016035051000000002 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.583295+0000", 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00086684900000000005 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.584162+0000", 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.031746462000000003 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.615908+0000", 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0943000000000001e-05 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.615989+0000", 2023-10-09T06:05:06.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111888 2023-10-09T06:05:06.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.616101+0000", 2023-10-09T06:05:06.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006815644 2023-10-09T06:05:06.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.622917+0000", 2023-10-09T06:05:06.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002131675 2023-10-09T06:05:06.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:06.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.625049+0000", 2023-10-09T06:05:06.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17035199300000001 2023-10-09T06:05:06.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.795401+0000", 2023-10-09T06:05:06.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2364e-05 2023-10-09T06:05:06.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:06.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.795473+0000", 2023-10-09T06:05:06.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026661060000000001 2023-10-09T06:05:06.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:06.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.798139+0000", 2023-10-09T06:05:06.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23090850800000001 2023-10-09T06:05:06.041 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.041 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:06.041 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.041 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:108119 10.d 10:b9c774ff:::benchmark_data_smithi008_634586_object20954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:06.041 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:02.621972+0000", 2023-10-09T06:05:06.041 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 3.2534988990000002, 2023-10-09T06:05:06.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22555293000000001, 2023-10-09T06:05:06.041 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:06.041 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.042 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:06.042 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:06.042 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:06.042 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 108119 2023-10-09T06:05:06.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.042 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:06.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:06.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.621972+0000", 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.621972+0000", 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.621970+0000", 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1619999999999997e-06 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.621979+0000", 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5625999999999999e-05 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.621995+0000", 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3629999999999999e-06 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.622001+0000", 2023-10-09T06:05:06.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00095310800000000004 2023-10-09T06:05:06.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.622954+0000", 2023-10-09T06:05:06.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00054953100000000002 2023-10-09T06:05:06.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.623504+0000", 2023-10-09T06:05:06.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18212404900000001 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.805628+0000", 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000104003 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.805732+0000", 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122263 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.805854+0000", 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017202920999999999 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.823057+0000", 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021373919000000002 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.844431+0000", 2023-10-09T06:05:06.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2620000000000002e-05 2023-10-09T06:05:06.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.844473+0000", 2023-10-09T06:05:06.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6733000000000001e-05 2023-10-09T06:05:06.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:06.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.844520+0000", 2023-10-09T06:05:06.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030050229999999999 2023-10-09T06:05:06.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.847525+0000", 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22555293000000001 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:108122 10.d 10:b44ab0c2:::benchmark_data_smithi008_634586_object20957:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:02.627034+0000", 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 3.248437376, 2023-10-09T06:05:06.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22281990500000001, 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 108122 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627034+0000", 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627034+0000", 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627032+0000", 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8639999999999996e-06 2023-10-09T06:05:06.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627040+0000", 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5248e-05 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627055+0000", 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2979999999999997e-06 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627061+0000", 2023-10-09T06:05:06.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8594999999999998e-05 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627110+0000", 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00050693799999999996 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627617+0000", 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18666287200000001 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.814280+0000", 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011455600000000001 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.814394+0000", 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000195837 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.814590+0000", 2023-10-09T06:05:06.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021439581999999999 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.836030+0000", 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011599135 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.847629+0000", 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7178e-05 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.847686+0000", 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6995e-05 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.847733+0000", 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021206210000000001 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.849854+0000", 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22281990500000001 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.056 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:108126 10.d 10:be4545cf:::benchmark_data_smithi008_634586_object20961:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:02.633906+0000", 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 3.24156537, 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21816328500000001, 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:05:06.057 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 108126 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.633906+0000", 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.633906+0000", 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.633904+0000", 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3729999999999993e-06 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.633912+0000", 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4856e-05 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:06.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.633927+0000", 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7819999999999994e-06 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.633935+0000", 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2184e-05 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.633987+0000", 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00057134899999999997 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.634558+0000", 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18853076599999999 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.823089+0000", 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6428999999999996e-05 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:06.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.823165+0000", 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106146 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.823272+0000", 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021170099000000001 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.844442+0000", 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005505294 2023-10-09T06:05:06.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.849947+0000", 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7049999999999999e-05 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.849984+0000", 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0127000000000003e-05 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.850034+0000", 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020347619999999999 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.852069+0000", 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21816328500000001 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:06.064 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:06.065 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:05:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:06.066 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:06.066 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:06.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:06.087 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:05:06.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:06.192 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:06 smithi191 ceph-mon[161669]: pgmap v1642: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 23346/199569 objects degraded (11.698%); 2199/199569 objects misplaced (1.102%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T06:05:06.192 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1851206323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:06.192 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:06 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:06.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:06 smithi008 ceph-mon[191227]: pgmap v1642: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 23346/199569 objects degraded (11.698%); 2199/199569 objects misplaced (1.102%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T06:05:06.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1851206323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:06.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:06 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:06.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:06 smithi008 ceph-mon[184418]: pgmap v1642: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 23346/199569 objects degraded (11.698%); 2199/199569 objects misplaced (1.102%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T06:05:06.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1851206323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:06.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:06 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:06.244 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:06.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:06.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:06.347 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:06.450 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:06.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:05:06.759 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:06.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:06.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:06.871 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:06.910 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:06.996 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:07.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:07.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:07.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:07 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:07.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:07 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:05:07.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:07 smithi008 ceph-mon[184418]: osdmap e544: 8 total, 8 up, 7 in 2023-10-09T06:05:07.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:07 smithi008 ceph-mon[184418]: failed to encode map e544 with expected crc 2023-10-09T06:05:07.072 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:07 smithi008 ceph-mon[184418]: failed to encode map e544 with expected crc 2023-10-09T06:05:07.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:07.144 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:07.172 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:05:07.190 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:07.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:07.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:07.299 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:07 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:07 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:05:07.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:07 smithi191 ceph-mon[161669]: osdmap e544: 8 total, 8 up, 7 in 2023-10-09T06:05:07.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:07 smithi191 ceph-mon[161669]: failed to encode map e544 with expected crc 2023-10-09T06:05:07.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:07 smithi191 ceph-mon[161669]: failed to encode map e544 with expected crc 2023-10-09T06:05:07.338 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:07.414 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:07.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:07.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:07.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:07 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:07.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:07 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:05:07.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:07 smithi008 ceph-mon[191227]: osdmap e544: 8 total, 8 up, 7 in 2023-10-09T06:05:07.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:07 smithi008 ceph-mon[191227]: failed to encode map e544 with expected crc 2023-10-09T06:05:07.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:07 smithi008 ceph-mon[191227]: failed to encode map e544 with expected crc 2023-10-09T06:05:07.519 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:07.610 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:07.744 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:07.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:07.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:07.856 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:07.896 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:08.004 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:08.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:08.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:08.126 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:08.186 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:08 smithi008 ceph-mon[184418]: pgmap v1644: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 23028/195144 objects degraded (11.801%); 2039/195144 objects misplaced (1.045%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:05:08.187 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:08 smithi008 ceph-mon[191227]: pgmap v1644: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 23028/195144 objects degraded (11.801%); 2039/195144 objects misplaced (1.045%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:05:08.188 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:05:08.201 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:08.256 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:05:08.294 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:05:08.323 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:08.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:08.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:08.353 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:05:08.402 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:05:08.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:08.456 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:05:08.466 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:08 smithi191 ceph-mon[161669]: pgmap v1644: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 23028/195144 objects degraded (11.801%); 2039/195144 objects misplaced (1.045%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:05:08.481 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:08.485 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:05:08.529 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:05:08.569 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:05:08.569 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:05:08.569 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:05:08.569 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:05:08.570 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:05:08.570 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:05:08.570 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:05:08.570 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:05:08.585 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:08.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:08.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:08.690 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:08.783 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:08.815 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:05:08.815 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:05:08.921 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:08.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:08.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:09.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:09.059 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:09.105 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:09.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:09.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:09.209 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:09.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:09 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/774019815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:09.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:09 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23028/195144 objects degraded (11.801%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:09.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:09 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/774019815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:09.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:09 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23028/195144 objects degraded (11.801%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:09.245 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:09.294 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:09.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:09.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:09.328 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-09T06:05:09.328 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:05:09.328 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:05:09.328 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.466580 2023-10-09T06:05:09.328 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0.4665804668365776 2023-10-09T06:05:09.399 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:09.526 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:09.534 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:09 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/774019815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:09.535 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:09 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23028/195144 objects degraded (11.801%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:09.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:05:09.691 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:05:09.834 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:09.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:09.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:09.975 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:10.010 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:05:10.079 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:10.212 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:10.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:10.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:10.249 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:10.321 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:10.413 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:10.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:10 smithi191 ceph-mon[161669]: pgmap v1645: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 23028/194709 objects degraded (11.827%); 2039/194709 objects misplaced (1.047%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:05:10.674 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:10.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:10.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:10.687 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:10 smithi008 ceph-mon[184418]: pgmap v1645: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 23028/194709 objects degraded (11.827%); 2039/194709 objects misplaced (1.047%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:05:10.687 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:10 smithi008 ceph-mon[191227]: pgmap v1645: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 23028/194709 objects degraded (11.827%); 2039/194709 objects misplaced (1.047%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:05:10.687 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:10.689 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:05:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:05:10] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.33.4" 2023-10-09T06:05:10.788 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:10.818 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:11.193 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:11.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:11.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:11.311 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:11.404 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:11.667 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:05:11.667 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:05:11.893 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:05:11.973 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:11.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:11.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:12.077 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:12.220 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:12.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:05:12.468 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:12 smithi008 ceph-mon[184418]: pgmap v1646: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 23028/192276 objects degraded (11.977%); 2039/192276 objects misplaced (1.060%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:05:12.469 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:12 smithi008 ceph-mon[191227]: pgmap v1646: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 23028/192276 objects degraded (11.977%); 2039/192276 objects misplaced (1.060%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:05:12.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:12 smithi191 ceph-mon[161669]: pgmap v1646: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 23028/192276 objects degraded (11.977%); 2039/192276 objects misplaced (1.060%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:05:12.831 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:05:13.257 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:13.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:13.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:13.372 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:13.407 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:13.466 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:13.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:13.571 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:13.594 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 23028/192276 objects degraded (11.977%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:13.594 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 23028/192276 objects degraded (11.977%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:13.610 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 23028/192276 objects degraded (11.977%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:13.926 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:13.966 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:13.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:13.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:14.074 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:14.187 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:14.352 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:14.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:14.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:14.425 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:05:14.426 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:05:14.469 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:14.513 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:14.561 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.1 primary-affinity to 0.46658 (8305772) 2023-10-09T06:05:14.631 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:14.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:14.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:14.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:14 smithi008 ceph-mon[184418]: pgmap v1647: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 22752/187539 objects degraded (12.132%); 1848/187539 objects misplaced (0.985%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:05:14.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:14.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:14 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3103695829' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4665804668365776}]: dispatch 2023-10-09T06:05:14.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:14 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4665804668365776}]: dispatch 2023-10-09T06:05:14.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:14 smithi008 ceph-mon[191227]: pgmap v1647: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 22752/187539 objects degraded (12.132%); 1848/187539 objects misplaced (0.985%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:05:14.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:14.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:14 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3103695829' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4665804668365776}]: dispatch 2023-10-09T06:05:14.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:14 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4665804668365776}]: dispatch 2023-10-09T06:05:14.739 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:14.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:14 smithi191 ceph-mon[161669]: pgmap v1647: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 22752/187539 objects degraded (12.132%); 1848/187539 objects misplaced (0.985%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:05:14.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:14 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3103695829' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4665804668365776}]: dispatch 2023-10-09T06:05:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:14 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4665804668365776}]: dispatch 2023-10-09T06:05:14.827 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:15.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:05:15.249 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:15.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:15.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:15.368 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:15.451 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:15.534 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:15 smithi008 ceph-mon[184418]: failed to encode map e545 with expected crc 2023-10-09T06:05:15.534 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:15 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4665804668365776}]': finished 2023-10-09T06:05:15.534 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:15 smithi008 ceph-mon[184418]: osdmap e545: 8 total, 8 up, 7 in 2023-10-09T06:05:15.535 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:15 smithi008 ceph-mon[184418]: failed to encode map e545 with expected crc 2023-10-09T06:05:15.535 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1493791679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:15.535 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:15 smithi191 ceph-mon[161669]: failed to encode map e545 with expected crc 2023-10-09T06:05:15.535 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:15 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4665804668365776}]': finished 2023-10-09T06:05:15.535 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:15 smithi191 ceph-mon[161669]: osdmap e545: 8 total, 8 up, 7 in 2023-10-09T06:05:15.535 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:15 smithi191 ceph-mon[161669]: failed to encode map e545 with expected crc 2023-10-09T06:05:15.535 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1493791679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:15.551 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:05:15.603 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:15.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:15.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:15.670 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:05:15.715 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:15.796 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:15 smithi008 ceph-mon[191227]: failed to encode map e545 with expected crc 2023-10-09T06:05:15.796 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:15 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4665804668365776}]': finished 2023-10-09T06:05:15.797 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:15 smithi008 ceph-mon[191227]: osdmap e545: 8 total, 8 up, 7 in 2023-10-09T06:05:15.797 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:15 smithi008 ceph-mon[191227]: failed to encode map e545 with expected crc 2023-10-09T06:05:15.797 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1493791679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:15.800 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:16.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:16 smithi191 ceph-mon[161669]: pgmap v1649: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 22752/185361 objects degraded (12.274%); 1848/185361 objects misplaced (0.997%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T06:05:16.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:16 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:16.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:16 smithi191 ceph-mon[161669]: failed to encode map e546 with expected crc 2023-10-09T06:05:16.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:16 smithi191 ceph-mon[161669]: failed to encode map e546 with expected crc 2023-10-09T06:05:16.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:16 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1493791679' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:16.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:16 smithi191 ceph-mon[161669]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T06:05:16.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[184418]: pgmap v1649: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 22752/185361 objects degraded (12.274%); 1848/185361 objects misplaced (0.997%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T06:05:16.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:16.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[184418]: failed to encode map e546 with expected crc 2023-10-09T06:05:16.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[184418]: failed to encode map e546 with expected crc 2023-10-09T06:05:16.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1493791679' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:16.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[184418]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T06:05:16.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[191227]: pgmap v1649: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 22752/185361 objects degraded (12.274%); 1848/185361 objects misplaced (0.997%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T06:05:16.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:16.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[191227]: failed to encode map e546 with expected crc 2023-10-09T06:05:16.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[191227]: failed to encode map e546 with expected crc 2023-10-09T06:05:16.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1493791679' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:16.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:16 smithi008 ceph-mon[191227]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T06:05:17.356 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:05:17.356 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:05:17.387 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:17.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:17.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:17.497 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:17.575 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:17.671 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:17.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:17.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:17.776 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:17.820 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:18.039 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:18.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:18.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:18.103 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:05:18.146 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:18.254 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:18.302 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:05:18.351 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:05:18.428 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:05:18.456 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:05:18.458 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:18.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:18.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:18.493 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:05:18.538 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:05:18.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:18 smithi008 ceph-mon[191227]: pgmap v1651: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 22452/180891 objects degraded (12.412%); 1664/180891 objects misplaced (0.920%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:05:18.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2764218176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:18.558 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 22452/180891 objects degraded (12.412%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:18.558 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:05:18.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:18.573 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:05:18.598 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:18.607 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:05:18.640 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:05:18.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:05:18.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:05:18.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:05:18.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:05:18.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:05:18.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:05:18.641 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:05:18.720 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:18.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:18.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:18.824 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:18.827 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:18 smithi191 ceph-mon[161669]: pgmap v1651: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 22452/180891 objects degraded (12.412%); 1664/180891 objects misplaced (0.920%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:05:18.827 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2764218176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:18.827 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 22452/180891 objects degraded (12.412%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:18.913 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:18 smithi008 ceph-mon[184418]: pgmap v1651: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 22452/180891 objects degraded (12.412%); 1664/180891 objects misplaced (0.920%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:05:18.914 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2764218176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:18.914 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 22452/180891 objects degraded (12.412%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:18.938 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:19.088 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:19.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:19.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:19.190 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:19.269 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:19.367 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:19.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:19.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:19.472 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:19.547 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:19.571 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:05:19.659 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:19.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:19.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:19.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:19.819 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:19.907 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:19.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:20.011 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:20.142 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:20.246 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:05:20.246 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:05:20.246 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T06:05:20.246 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T06:05:20.246 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:05:20.327 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:20.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:20.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:20.451 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:20.489 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:20.637 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:20.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:20 smithi008 ceph-mon[184418]: pgmap v1652: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 22452/180456 objects degraded (12.442%); 1664/180456 objects misplaced (0.922%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:05:20.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:20.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:20 smithi008 ceph-mon[191227]: pgmap v1652: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 22452/180456 objects degraded (12.442%); 1664/180456 objects misplaced (0.922%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:05:20.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:20.716 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:05:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:05:20] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.33.4" 2023-10-09T06:05:20.922 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:20.927 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:20 smithi191 ceph-mon[161669]: pgmap v1652: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 22452/180456 objects degraded (12.442%); 1664/180456 objects misplaced (0.922%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:05:20.927 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:20.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:20.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:21.028 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:05:21.029 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:21.074 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:21.164 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:21.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:21.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:21.278 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:21.317 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:21.539 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:21.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:21.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:21.606 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:21 smithi191 ceph-mon[161669]: pgmap v1653: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 22452/178110 objects degraded (12.606%); 1664/178110 objects misplaced (0.934%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:05:21.645 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:21.675 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:05:21.700 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:21.793 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:21.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:21.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:21.897 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:21.949 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:21.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:21 smithi008 ceph-mon[191227]: pgmap v1653: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 22452/178110 objects degraded (12.606%); 1664/178110 objects misplaced (0.934%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:05:21.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:21 smithi008 ceph-mon[184418]: pgmap v1653: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 22452/178110 objects degraded (12.606%); 1664/178110 objects misplaced (0.934%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:05:22.125 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:22.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:22.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:22.173 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-09T06:05:22.174 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:05:22.174 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:05:22.174 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2023-10-09T06:05:22.174 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0 2023-10-09T06:05:22.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:22.264 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:22.379 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:05:22.476 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:22.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:22.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:22.580 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:22.610 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:23.222 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:23.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:23.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:23.326 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:23.370 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:23.436 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:23.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:23.492 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:23.540 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:23.579 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:23.776 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:23.879 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:05:23.879 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T06:05:23.879 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T06:05:23.879 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T06:05:23.879 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.879 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117314 9.19 9:9d16215d:::benchmark_data_smithi008_502820_object33141:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:05:23.879 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.240001+0000", 2023-10-09T06:05:23.879 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 446.635212131, 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.143222604999998, 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117314 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240001+0000", 2023-10-09T06:05:23.880 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.881 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.881 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.881 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.881 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240001+0000", 2023-10-09T06:05:23.881 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:23.881 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.881 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.881 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.881 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240000+0000", 2023-10-09T06:05:23.881 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.96e-06 2023-10-09T06:05:23.881 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240003+0000", 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0399999999999998e-06 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240012+0000", 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6589999999999998e-06 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.882 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.883 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240017+0000", 2023-10-09T06:05:23.883 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000471003 2023-10-09T06:05:23.883 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.883 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.883 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.883 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240488+0000", 2023-10-09T06:05:23.883 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0127e-05 2023-10-09T06:05:23.883 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.883 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.883 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T06:05:23.883 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240498+0000", 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001336481 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.241834+0000", 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6310000000000003e-06 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.241842+0000", 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4456201419999992 2023-10-09T06:05:23.884 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.687462+0000", 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073848840000000004 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.694847+0000", 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99199552499999999 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.885 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.886 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686843+0000", 2023-10-09T06:05:23.886 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3572e-05 2023-10-09T06:05:23.886 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.886 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.886 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.886 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686856+0000", 2023-10-09T06:05:23.886 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.69965971100000002 2023-10-09T06:05:23.886 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.886 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.886 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.886 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386516+0000", 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2209999999999997e-06 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386524+0000", 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3956454790000006 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.782170+0000", 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003363689 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.887 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.785533+0000", 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0053417940000000004 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.790875+0000", 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.040818535000000003 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.831694+0000", 2023-10-09T06:05:23.888 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5428561090000001 2023-10-09T06:05:23.889 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.889 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.889 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.889 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.374550+0000", 2023-10-09T06:05:23.889 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00093305999999999999 2023-10-09T06:05:23.889 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.889 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.889 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.889 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.375483+0000", 2023-10-09T06:05:23.889 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001490901 2023-10-09T06:05:23.889 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.376974+0000", 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006249714 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.383223+0000", 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.143222604999998 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.890 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117319 9.19 9:991b5772:::benchmark_data_smithi008_502820_object33146:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 446.63519141699999, 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.166676882000001, 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.891 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117319 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.892 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.893 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.893 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:05:23.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6680000000000001e-06 2023-10-09T06:05:23.893 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.893 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.893 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.893 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240024+0000", 2023-10-09T06:05:23.893 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2449999999999998e-06 2023-10-09T06:05:23.893 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.893 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.893 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.894 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240029+0000", 2023-10-09T06:05:23.894 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4620000000000001e-06 2023-10-09T06:05:23.894 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.894 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.894 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.894 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240032+0000", 2023-10-09T06:05:23.894 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021777950000000002 2023-10-09T06:05:23.894 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.894 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.894 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.894 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242210+0000", 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3260000000000002e-06 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242219+0000", 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4529842770000005 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.695204+0000", 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099633949999999999 2023-10-09T06:05:23.895 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.705167+0000", 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98175812600000001 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686925+0000", 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0090000000000001e-06 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.896 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686932+0000", 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.69967429400000003 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386607+0000", 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6420000000000001e-06 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386613+0000", 2023-10-09T06:05:23.897 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4280742919999998 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.814688+0000", 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046527440000000003 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.819340+0000", 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068654730000000004 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.898 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.826206+0000", 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.025179699999999999 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.851385+0000", 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5449409780000001 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.899 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.396326+0000", 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001055615 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.397382+0000", 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017229529999999999 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.399105+0000", 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075933620000000002 2023-10-09T06:05:23.900 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.406698+0000", 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.166676882000001 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117320 9.19 9:9ca20909:::benchmark_data_smithi008_502820_object33147:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:05:23.901 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 446.63517681500002, 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.155182235000002, 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117320 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.902 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.638e-06 2023-10-09T06:05:23.903 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240038+0000", 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0300000000000001e-06 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240043+0000", 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8990000000000003e-06 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.904 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240047+0000", 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021981399999999999 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242245+0000", 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2699999999999999e-06 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242253+0000", 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4632743759999993 2023-10-09T06:05:23.905 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.705527+0000", 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075602960000000002 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.713087+0000", 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97380087699999995 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.906 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686888+0000", 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8900000000000001e-06 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686895+0000", 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.69966168100000004 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386557+0000", 2023-10-09T06:05:23.907 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3550000000000005e-06 2023-10-09T06:05:23.908 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.908 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.908 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.908 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386565+0000", 2023-10-09T06:05:23.908 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4098301660000008 2023-10-09T06:05:23.908 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.908 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.908 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.908 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.796395+0000", 2023-10-09T06:05:23.908 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048933850000000001 2023-10-09T06:05:23.908 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.801289+0000", 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071789369999999998 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.808468+0000", 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.042643954999999997 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.909 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.851112+0000", 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.532899402 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.384011+0000", 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010206099999999999 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.385032+0000", 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017808520000000001 2023-10-09T06:05:23.910 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.386812+0000", 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0084058440000000009 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.395218+0000", 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.155182235000002 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.911 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.912 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.912 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.912 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117329 9.19 9:98a1b605:::benchmark_data_smithi008_502820_object33156:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-09T06:05:23.912 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:58.537538+0000", 2023-10-09T06:05:23.912 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 445.337674472, 2023-10-09T06:05:23.912 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 22.047981301, 2023-10-09T06:05:23.912 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.912 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.912 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.912 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.912 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117329 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537538+0000", 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537538+0000", 2023-10-09T06:05:23.913 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537537+0000", 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.357e-06 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537544+0000", 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5744e-05 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.914 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537560+0000", 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1219999999999995e-06 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537568+0000", 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1875399160000004 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.915 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725108+0000", 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7539999999999995e-06 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725117+0000", 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66151549200000004 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386632+0000", 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.083e-06 2023-10-09T06:05:23.916 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386638+0000", 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4453952789999995 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.832033+0000", 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0096413320000000007 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.917 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.841675+0000", 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5839331400000001 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.425608+0000", 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020811639999999999 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.427689+0000", 2023-10-09T06:05:23.918 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052414740000000003 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.432931+0000", 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.046699651000000002 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.479630+0000", 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.094573824000000001 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.919 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.574204+0000", 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001875704 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.576080+0000", 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018718509999999999 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.577952+0000", 2023-10-09T06:05:23.920 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.007568165 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.585520+0000", 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 22.047981301 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117332 9.19 9:9d36e61d:::benchmark_data_smithi008_502820_object33159:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:58.661367+0000", 2023-10-09T06:05:23.921 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 445.21384612499998, 2023-10-09T06:05:23.922 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 21.935008709000002, 2023-10-09T06:05:23.922 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.922 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.922 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.922 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.922 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.922 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117332 2023-10-09T06:05:23.922 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.922 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.922 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.922 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661367+0000", 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661367+0000", 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661365+0000", 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8889999999999997e-06 2023-10-09T06:05:23.923 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661375+0000", 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2055999999999999e-05 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661387+0000", 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9869999999999996e-06 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.924 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661393+0000", 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0637442499999992 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725137+0000", 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0648e-05 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725148+0000", 2023-10-09T06:05:23.925 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66150936400000004 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386657+0000", 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.976e-06 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386663+0000", 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4554108919999997 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.926 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.842074+0000", 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087053159999999994 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.850779+0000", 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6107857050000001 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.927 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.461565+0000", 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023301989999999998 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.463895+0000", 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052078059999999997 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.469103+0000", 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010784145 2023-10-09T06:05:23.928 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.479887+0000", 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.106784905 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.586672+0000", 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011016520000000001 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.929 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.587774+0000", 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001720217 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.589494+0000", 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068816320000000004 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.596375+0000", 2023-10-09T06:05:23.930 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 21.935008709000002 2023-10-09T06:05:23.931 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.931 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.931 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.931 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.931 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.931 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117337 9.1 9:857ac274:::benchmark_data_smithi008_502820_object33164:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:05:23.931 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:02.529277+0000", 2023-10-09T06:05:23.931 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 441.34593621300002, 2023-10-09T06:05:23.931 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 18.012440933000001, 2023-10-09T06:05:23.931 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.931 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117337 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529277+0000", 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.932 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529277+0000", 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529275+0000", 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6810000000000001e-06 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.933 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529282+0000", 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5460000000000001e-05 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529297+0000", 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0799999999999996e-06 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529302+0000", 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1958720979999997 2023-10-09T06:05:23.934 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.935 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.935 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.935 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725174+0000", 2023-10-09T06:05:23.935 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9480000000000003e-06 2023-10-09T06:05:23.935 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.935 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.935 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.935 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725184+0000", 2023-10-09T06:05:23.935 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66691750900000002 2023-10-09T06:05:23.935 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.935 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392102+0000", 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8009999999999998e-06 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392112+0000", 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.865323069 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.257435+0000", 2023-10-09T06:05:23.936 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015459999999999999 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.257589+0000", 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028764399999999999 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.257877+0000", 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035365399999999999 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.937 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.261413+0000", 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27957103 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.540984+0000", 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1287999999999998e-05 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.938 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541066+0000", 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0819999999999998e-05 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541117+0000", 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060101399999999998 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541718+0000", 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 18.012440933000001 2023-10-09T06:05:23.939 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117340 9.9 9:91f28550:::benchmark_data_smithi008_502820_object33167:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:02.596319+0000", 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 441.27889361299998, 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 17.955864099999999, 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.940 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117340 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596319+0000", 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596319+0000", 2023-10-09T06:05:23.941 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596318+0000", 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0000000000000002e-06 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596324+0000", 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3848e-05 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.942 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596338+0000", 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3350000000000003e-06 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596344+0000", 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1288643570000003 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.943 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725209+0000", 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0413e-05 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725219+0000", 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65901121200000001 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384230+0000", 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3641e-05 2023-10-09T06:05:23.944 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384244+0000", 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4740669539999995 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.858311+0000", 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086265019999999994 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:23.945 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.866937+0000", 2023-10-09T06:05:23.946 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4266018129999996 2023-10-09T06:05:23.946 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.946 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.946 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.946 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.293539+0000", 2023-10-09T06:05:23.946 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002220236 2023-10-09T06:05:23.946 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.946 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.946 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.946 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.295759+0000", 2023-10-09T06:05:23.946 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050211379999999996 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.300781+0000", 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033493818000000002 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.334274+0000", 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20914828399999999 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.947 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.543423+0000", 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094928299999999998 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.544372+0000", 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013903489999999999 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.545762+0000", 2023-10-09T06:05:23.948 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0064211219999999996 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.552183+0000", 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 17.955864099999999 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117343 9.9 9:978d0f59:::benchmark_data_smithi008_502820_object33170:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:05:23.949 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.479396+0000", 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 438.39581684000001, 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.081940300999999, 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117343 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.950 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479396+0000", 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479396+0000", 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999967 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479393+0000", 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1915e-05 2023-10-09T06:05:23.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479405+0000", 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4262e-05 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479419+0000", 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3119999999999994e-06 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479427+0000", 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2458186229999999 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725246+0000", 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0760000000000003e-06 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725254+0000", 2023-10-09T06:05:23.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65942001299999997 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384674+0000", 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1280999999999999e-05 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384685+0000", 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4825987420000004 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.955 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.867284+0000", 2023-10-09T06:05:23.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0098823669999999995 2023-10-09T06:05:23.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.955 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:23.955 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.877166+0000", 2023-10-09T06:05:23.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4447639959999998 2023-10-09T06:05:23.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.955 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.321930+0000", 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020355619999999999 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.323966+0000", 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059662459999999997 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.329932+0000", 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022155676999999999 2023-10-09T06:05:23.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.352088+0000", 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.201059509 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.553147+0000", 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00084139399999999997 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.553989+0000", 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001347577 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.555336+0000", 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060000390000000004 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.561336+0000", 2023-10-09T06:05:23.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.081940300999999 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117347 9.1 9:83732981:::benchmark_data_smithi008_502820_object33174:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.499313+0000", 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 438.37590030799998, 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.043182195, 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.959 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117347 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499313+0000", 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499313+0000", 2023-10-09T06:05:23.960 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:05:23.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.961 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499310+0000", 2023-10-09T06:05:23.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6179999999999995e-06 2023-10-09T06:05:23.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.961 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499319+0000", 2023-10-09T06:05:23.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5282e-05 2023-10-09T06:05:23.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499334+0000", 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1779999999999997e-06 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499341+0000", 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.225918622 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725260+0000", 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4930000000000001e-06 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725268+0000", 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66689868200000002 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392166+0000", 2023-10-09T06:05:23.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6649999999999992e-06 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392175+0000", 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.866980590000001 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.259156+0000", 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000185317 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.964 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.259341+0000", 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00037832499999999999 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.259719+0000", 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035454420000000002 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.965 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.263265+0000", 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.277776933 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541042+0000", 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00079637999999999996 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541838+0000", 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5203999999999994e-05 2023-10-09T06:05:23.966 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541903+0000", 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000591623 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.542495+0000", 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.043182195 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.967 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117351 9.11 9:8879fcf3:::benchmark_data_smithi008_502820_object33178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.521160+0000", 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 438.35405294100002, 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.849076910000001, 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.968 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117351 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521160+0000", 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521160+0000", 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:23.969 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521158+0000", 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.622e-06 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521166+0000", 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2873e-05 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.970 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.971 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521179+0000", 2023-10-09T06:05:23.971 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7890000000000006e-06 2023-10-09T06:05:23.971 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.971 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.971 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.971 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521189+0000", 2023-10-09T06:05:23.971 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2040851159999999 2023-10-09T06:05:23.971 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.971 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.971 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.971 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725274+0000", 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8170000000000001e-06 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725281+0000", 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66564084199999995 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.390922+0000", 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0733999999999999e-05 2023-10-09T06:05:23.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.390932+0000", 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5330637649999996 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.923996+0000", 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013707300000000001 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.973 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.974 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.924133+0000", 2023-10-09T06:05:23.974 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00038352900000000002 2023-10-09T06:05:23.974 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.974 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.974 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4,6", 2023-10-09T06:05:23.974 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.924517+0000", 2023-10-09T06:05:23.974 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.87137725899999996 2023-10-09T06:05:23.974 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.974 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.974 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.974 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.795894+0000", 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5739970530000003 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.369891+0000", 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6699000000000002e-05 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.369928+0000", 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010414800000000001 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.370032+0000", 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5777000000000004e-05 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.370098+0000", 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013928500000000001 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.976 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.370237+0000", 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.849076910000001 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117352 9.9 9:93c88060:::benchmark_data_smithi008_502820_object33179:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.574636+0000", 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 438.30057685999998, 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.996202152, 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.977 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.978 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.978 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.978 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117352 2023-10-09T06:05:23.978 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.978 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.978 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.978 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.978 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574636+0000", 2023-10-09T06:05:23.978 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.978 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.978 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574636+0000", 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574634+0000", 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2150000000000001e-06 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574643+0000", 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6385e-05 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574659+0000", 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6510000000000001e-06 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574667+0000", 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.150622016 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.980 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725289+0000", 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1060000000000002e-06 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725296+0000", 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65942802199999995 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.981 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384724+0000", 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1769999999999998e-06 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384732+0000", 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4928461019999997 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.877578+0000", 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011095785 2023-10-09T06:05:23.982 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.888674+0000", 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4637061429999996 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.352380+0000", 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020417460000000001 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.983 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.354422+0000", 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0058679479999999996 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.360290+0000", 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.036684833 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.396974+0000", 2023-10-09T06:05:23.984 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16525932500000001 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.562234+0000", 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00104692 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.563281+0000", 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014355430000000001 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.985 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.564716+0000", 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0061219309999999997 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.570838+0000", 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.996202152 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.986 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117356 9.19 9:9e99d7bd:::benchmark_data_smithi008_502820_object33183:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.607515+0000", 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 438.26769822300002, 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.999065100999999, 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117356 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.987 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607515+0000", 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607515+0000", 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.988 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607513+0000", 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5640000000000002e-06 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607520+0000", 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1073000000000001e-05 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607531+0000", 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9660000000000001e-06 2023-10-09T06:05:23.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.990 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.990 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607537+0000", 2023-10-09T06:05:23.990 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1177633890000001 2023-10-09T06:05:23.990 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.990 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.990 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725300+0000", 2023-10-09T06:05:23.990 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7010000000000001e-06 2023-10-09T06:05:23.990 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725306+0000", 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66137529500000003 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386681+0000", 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0240000000000003e-06 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386687+0000", 2023-10-09T06:05:23.991 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4650221170000002 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.851709+0000", 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099491639999999999 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.861658+0000", 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6314088509999998 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.493067+0000", 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002267879 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.495335+0000", 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0054393109999999996 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.500774+0000", 2023-10-09T06:05:23.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024364858999999999 2023-10-09T06:05:23.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.994 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:23.994 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.525139+0000", 2023-10-09T06:05:23.994 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.072382734000000004 2023-10-09T06:05:23.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.994 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.994 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.597522+0000", 2023-10-09T06:05:23.994 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001197832 2023-10-09T06:05:23.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.598720+0000", 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001605965 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.600326+0000", 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0062539149999999996 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.995 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.606580+0000", 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.999065100999999 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135245 9.d 9:b964d666:::benchmark_data_smithi008_502820_object51072:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.596664+0000", 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 320.27854899900001, 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789031075, 2023-10-09T06:05:23.996 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135245 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596664+0000", 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596664+0000", 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596663+0000", 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.011e-06 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:23.998 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596668+0000", 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0095000000000001e-05 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596678+0000", 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5299999999999998e-06 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596682+0000", 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.063468346999999994 2023-10-09T06:05:23.999 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.660151+0000", 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033404179999999999 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.663491+0000", 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017397383999999998 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.680889+0000", 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004312265 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.685201+0000", 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.61805938000000005 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.303260+0000", 2023-10-09T06:05:24.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035192460000000002 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.306780+0000", 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0085941420000000008 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.315374+0000", 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.057904045000000001 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373278+0000", 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0834e-05 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373289+0000", 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91257296300000001 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285861+0000", 2023-10-09T06:05:24.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8009999999999998e-06 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285868+0000", 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.830804881000001 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.116673+0000", 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050648639999999997 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.121738+0000", 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.26043395800000002 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.382172+0000", 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035229559999999998 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:24.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.385695+0000", 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789031075 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135269 9.d 9:b1952556:::benchmark_data_smithi008_502820_object51096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.688239+0000", 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 320.18697409399999, 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.735737110000001, 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:24.006 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135269 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688239+0000", 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.007 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688239+0000", 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688238+0000", 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5710000000000003e-06 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:24.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688244+0000", 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8319999999999995e-06 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688253+0000", 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7010000000000001e-06 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688259+0000", 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024856844999999999 2023-10-09T06:05:24.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.713116+0000", 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041163090000000003 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.717232+0000", 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.032543962000000003 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749776+0000", 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042646100000000003 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.754041+0000", 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.57617059599999998 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.330211+0000", 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003060962 2023-10-09T06:05:24.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.333272+0000", 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059605719999999999 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.339233+0000", 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034085450000000003 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373318+0000", 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4910000000000002e-06 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373325+0000", 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91256626500000004 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285891+0000", 2023-10-09T06:05:24.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2830000000000004e-06 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285898+0000", 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.836523452 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.122422+0000", 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0067783660000000001 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.129200+0000", 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29146132499999999 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.420661+0000", 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033146180000000001 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.423976+0000", 2023-10-09T06:05:24.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.735737110000001 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135271 9.d 9:ba573113:::benchmark_data_smithi008_502820_object51098:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.694068+0000", 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 320.181145147, 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.803375333, 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:24.016 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135271 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694068+0000", 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694068+0000", 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694065+0000", 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2420000000000006e-06 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694075+0000", 2023-10-09T06:05:24.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5207e-05 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694090+0000", 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.721e-06 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694099+0000", 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.060414994999999999 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.754514+0000", 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047620040000000002 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.759276+0000", 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01924886 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.020 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.778524+0000", 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045597650000000003 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.783084+0000", 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.59039565999999999 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373480+0000", 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1539999999999999e-06 2023-10-09T06:05:24.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373486+0000", 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91251702300000004 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286003+0000", 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.336e-06 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286009+0000", 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.906405661999999 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.192415+0000", 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00484973 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.197265+0000", 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0089266180000000007 2023-10-09T06:05:24.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.206191+0000", 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.065920287999999994 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.272112+0000", 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20899689099999999 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.481109+0000", 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001501549 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.482610+0000", 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015254660000000001 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.484136+0000", 2023-10-09T06:05:24.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026156030000000002 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.486751+0000", 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010691813999999999 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.497443+0000", 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.803375333 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.026 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135277 9.d 9:bcf8be17:::benchmark_data_smithi008_502820_object51104:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.729022+0000", 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 320.14619042599998, 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.750575383999999, 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:24.027 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135277 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729022+0000", 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729022+0000", 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:24.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729020+0000", 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0536e-05 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729031+0000", 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2653999999999999e-05 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:24.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729044+0000", 2023-10-09T06:05:24.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6559999999999992e-06 2023-10-09T06:05:24.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:24.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729051+0000", 2023-10-09T06:05:24.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.031414678000000001 2023-10-09T06:05:24.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.760466+0000", 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032615449999999998 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.763727+0000", 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.036828547000000003 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.800556+0000", 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032710500000000002 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.803827+0000", 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.56961666799999999 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373444+0000", 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1970000000000001e-06 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373450+0000", 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91252582299999996 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285976+0000", 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5559999999999998e-06 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285981+0000", 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.882853229 2023-10-09T06:05:24.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.168835+0000", 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048070370000000001 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.173642+0000", 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086679089999999997 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.182309+0000", 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.051578453000000003 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.233888+0000", 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22473416600000001 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.458622+0000", 2023-10-09T06:05:24.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001297997 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.459920+0000", 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014969269999999999 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.461417+0000", 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025559990000000002 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.463973+0000", 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015624812 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.479598+0000", 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.750575383999999 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135285 9.d 9:b8ae6d39:::benchmark_data_smithi008_502820_object51112:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 320.12537454099999, 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789996110000001, 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135285 2023-10-09T06:05:24.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.281e-06 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749841+0000", 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0190000000000001e-06 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749848+0000", 2023-10-09T06:05:24.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0799999999999999e-06 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749852+0000", 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033638386999999999 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.783491+0000", 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031708800000000001 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.786661+0000", 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034315245000000001 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.820977+0000", 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032606850000000001 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:05:24.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.824237+0000", 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.54932086199999997 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373558+0000", 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1770000000000003e-06 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373564+0000", 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91249455899999998 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286059+0000", 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7699999999999998e-06 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286065+0000", 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.948433641999999 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.234498+0000", 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042428049999999997 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.238741+0000", 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073975059999999999 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.246139+0000", 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.045528540999999999 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.291667+0000", 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22418721699999999 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.515854+0000", 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001393544 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.517248+0000", 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001617402 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.518865+0000", 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0061180770000000004 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.524983+0000", 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014850993999999999 2023-10-09T06:05:24.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.539834+0000", 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789996110000001 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135292 9.d 9:b1d0c439:::benchmark_data_smithi008_502820_object51119:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.788529+0000", 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 320.08668377200001, 2023-10-09T06:05:24.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.832817245999999, 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135292 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788529+0000", 2023-10-09T06:05:24.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788529+0000", 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788528+0000", 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8979999999999998e-06 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788532+0000", 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3579999999999999e-06 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788539+0000", 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.58e-06 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:24.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788542+0000", 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.037230805999999998 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.825773+0000", 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045452899999999996 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.830318+0000", 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.044094162999999999 2023-10-09T06:05:24.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.874413+0000", 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033226699999999998 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.877735+0000", 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.496008956 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373744+0000", 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2219999999999999e-06 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373750+0000", 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91544140500000004 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289192+0000", 2023-10-09T06:05:24.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5060000000000001e-06 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289198+0000", 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.022931246000001 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.312130+0000", 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041899379999999998 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.316319+0000", 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0083299189999999999 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.324649+0000", 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0088983910000000003 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.333548+0000", 2023-10-09T06:05:24.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27093540199999999 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.604483+0000", 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001811764 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.606295+0000", 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002186805 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.608482+0000", 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024063980000000001 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.610888+0000", 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010458169999999999 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:24.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.621346+0000", 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.832817245999999 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135349 9.d 9:b7ad6e1c:::benchmark_data_smithi008_502820_object51176:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:04.105455+0000", 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 319.76975752800001, 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.459172209, 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:24.059 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135349 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105455+0000", 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105455+0000", 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105453+0000", 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0417999999999999e-05 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105464+0000", 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.025e-05 2023-10-09T06:05:24.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105474+0000", 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0610000000000001e-06 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105480+0000", 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.055762855 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.161243+0000", 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031189049999999999 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.164362+0000", 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019388989999999998 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.183751+0000", 2023-10-09T06:05:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003301151 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.187052+0000", 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18655844199999999 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373610+0000", 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1009999999999997e-06 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373617+0000", 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91246933500000005 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286086+0000", 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2680000000000002e-06 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286091+0000", 2023-10-09T06:05:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.966898898 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.252990+0000", 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047099940000000003 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.257700+0000", 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0077358369999999997 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:05:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.265436+0000", 2023-10-09T06:05:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.026715319000000001 2023-10-09T06:05:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.292151+0000", 2023-10-09T06:05:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.249149184 2023-10-09T06:05:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.541300+0000", 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014578600000000001 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.542758+0000", 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046799320000000004 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.547438+0000", 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026429719999999999 2023-10-09T06:05:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.550081+0000", 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014546408 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.564628+0000", 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.459172209 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135364 9.d 9:b4fd37ff:::benchmark_data_smithi008_502820_object51191:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:05:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:04.153115+0000", 2023-10-09T06:05:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 319.72209800100001, 2023-10-09T06:05:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.449789453999999, 2023-10-09T06:05:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:05:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:05:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:05:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135364 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153115+0000", 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153115+0000", 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:05:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153114+0000", 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1359999999999999e-06 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153118+0000", 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6540000000000005e-06 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153127+0000", 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9570000000000001e-06 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153132+0000", 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011677977000000001 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.164810+0000", 2023-10-09T06:05:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040417029999999998 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.168851+0000", 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034353937000000001 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.203205+0000", 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030759429999999998 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.206281+0000", 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16741861399999999 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373700+0000", 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9889999999999992e-06 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373708+0000", 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91545159799999998 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289159+0000", 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7560000000000002e-06 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289166+0000", 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.003512383 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.292679+0000", 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059218559999999996 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.298600+0000", 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071744410000000002 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.305775+0000", 2023-10-09T06:05:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.026882515999999999 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.332657+0000", 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25068875899999998 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.583346+0000", 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016443880000000001 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.584991+0000", 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014049830000000001 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.586396+0000", 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025583310000000001 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.588954+0000", 2023-10-09T06:05:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01395045 2023-10-09T06:05:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:05:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:05:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:05:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.602904+0000", 2023-10-09T06:05:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.449789453999999 2023-10-09T06:05:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:05:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:05:24.080 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:05:24.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:24.082 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:24.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:24.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:24.199 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:23 smithi008 ceph-mon[184418]: pgmap v1654: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 22149/173199 objects degraded (12.788%); 1448/173199 objects misplaced (0.836%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T06:05:24.199 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 22149/173199 objects degraded (12.788%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:24.201 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:23 smithi008 ceph-mon[191227]: pgmap v1654: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 22149/173199 objects degraded (12.788%); 1448/173199 objects misplaced (0.836%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T06:05:24.201 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 22149/173199 objects degraded (12.788%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:24.217 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:23 smithi191 ceph-mon[161669]: pgmap v1654: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 22149/173199 objects degraded (12.788%); 1448/173199 objects misplaced (0.836%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T06:05:24.217 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 22149/173199 objects degraded (12.788%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:24.221 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:24.689 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:24.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:24.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:24.793 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:24.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:05:24.873 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:25.016 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:05:25.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:24 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/707865983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:25.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:24 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:25.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:24 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/707865983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:25.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:24 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:25.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:24 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/707865983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:25.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:24 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:25.491 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:05:26.098 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:26.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:26.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:26.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:26.249 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:25 smithi191 ceph-mon[161669]: pgmap v1655: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 22149/171087 objects degraded (12.946%); 1448/171087 objects misplaced (0.846%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:05:26.249 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:25 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:26.249 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:25 smithi191 ceph-mon[161669]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.250 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:25 smithi191 ceph-mon[161669]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.250 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:25 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:05:26.250 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:25 smithi191 ceph-mon[161669]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T06:05:26.250 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:25 smithi191 ceph-mon[161669]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.250 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:25 smithi191 ceph-mon[161669]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.250 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:25 smithi191 ceph-mon[161669]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.253 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:26.376 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:26.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:26.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:26.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[184418]: pgmap v1655: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 22149/171087 objects degraded (12.946%); 1448/171087 objects misplaced (0.846%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:05:26.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[184418]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[184418]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:05:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[184418]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T06:05:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[184418]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[184418]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[184418]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[191227]: pgmap v1655: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 22149/171087 objects degraded (12.946%); 1448/171087 objects misplaced (0.846%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:05:26.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:26.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[191227]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[191227]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:05:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[191227]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T06:05:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[191227]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[191227]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:25 smithi008 ceph-mon[191227]: failed to encode map e547 with expected crc 2023-10-09T06:05:26.481 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:26.527 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:27.115 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:27.115 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:27 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:27.158 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:27.158 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:27 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:27.211 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:27.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:27.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:27.322 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:27.370 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:27.390 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:05:27.390 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:05:27.391 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T06:05:27.391 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:05:27.438 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:27.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:27.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:27.471 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:27.472 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:27 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:27.545 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:27.586 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:27.639 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:27.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:27.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:27.770 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:27.819 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:27.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:27.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:27.930 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:28.027 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:28.147 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:05:28.291 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:28 smithi008 ceph-mon[184418]: pgmap v1657: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 21867/166821 objects degraded (13.108%); 1263/166821 objects misplaced (0.757%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:05:28.293 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:28 smithi008 ceph-mon[191227]: pgmap v1657: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 21867/166821 objects degraded (13.108%); 1263/166821 objects misplaced (0.757%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:05:28.295 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:28.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:28.407 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:28.439 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:28 smithi191 ceph-mon[161669]: pgmap v1657: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 21867/166821 objects degraded (13.108%); 1263/166821 objects misplaced (0.757%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:05:28.441 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:28.497 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:28.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:28.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:28.593 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:05:28.603 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:28.673 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:28.784 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:05:28.842 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:05:28.853 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:28.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:28.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:28.902 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:05:28.939 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:05:28.973 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:28.996 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:05:29.041 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:05:29.076 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:29.082 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:05:29.120 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:05:29.144 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:05:29.144 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:05:29.144 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:05:29.145 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:05:29.145 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:05:29.145 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:05:29.145 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:05:29.145 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:05:29.220 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:29.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:29.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:29.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:29.368 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:29.427 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:29.436 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:29 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 21867/166821 objects degraded (13.108%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:29.436 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2462017713' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:05:29.436 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2928182969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:29.436 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:29.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:29.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:29.450 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.2 primary-affinity to 0 (802) 2023-10-09T06:05:29.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:29 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 21867/166821 objects degraded (13.108%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:29.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2462017713' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:05:29.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2928182969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:29.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:29.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:29 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 21867/166821 objects degraded (13.108%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:29.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2462017713' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:05:29.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2928182969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:29.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:29.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:29.567 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:29.619 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:05:29.684 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:29.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:29.790 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:29.819 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:29.918 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:29.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:29.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:30.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:30.051 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:30.081 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:05:30.081 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:05:30.096 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:30.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:30.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:30.200 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:30.227 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:30.323 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:30.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:30.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:30.426 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:30.459 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:30.508 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:30.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:30.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:30.612 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:30.640 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:30.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:05:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:05:30] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.33.4" 2023-10-09T06:05:30.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:30 smithi008 ceph-mon[184418]: pgmap v1658: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 21867/166407 objects degraded (13.141%); 1263/166407 objects misplaced (0.759%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:05:30.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:30 smithi008 ceph-mon[184418]: failed to encode map e548 with expected crc 2023-10-09T06:05:30.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2462017713' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T06:05:30.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:30 smithi008 ceph-mon[184418]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T06:05:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:30 smithi008 ceph-mon[191227]: pgmap v1658: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 21867/166407 objects degraded (13.141%); 1263/166407 objects misplaced (0.759%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:05:30.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:30 smithi008 ceph-mon[191227]: failed to encode map e548 with expected crc 2023-10-09T06:05:30.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2462017713' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T06:05:30.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:30 smithi008 ceph-mon[191227]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T06:05:30.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:30 smithi191 ceph-mon[161669]: pgmap v1658: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 21867/166407 objects degraded (13.141%); 1263/166407 objects misplaced (0.759%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:05:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:30 smithi191 ceph-mon[161669]: failed to encode map e548 with expected crc 2023-10-09T06:05:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2462017713' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T06:05:30.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:30 smithi191 ceph-mon[161669]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T06:05:30.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:05:31.199 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:05:31.293 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:31.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:31.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:31.399 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:31.429 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:31.523 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:31.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:31.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:31.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:31.660 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:31.680 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:31 smithi191 ceph-mon[161669]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.680 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:31 smithi191 ceph-mon[161669]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.680 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:31 smithi191 ceph-mon[161669]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.680 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:31 smithi191 ceph-mon[161669]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T06:05:31.680 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:31 smithi191 ceph-mon[161669]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.680 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:31 smithi191 ceph-mon[161669]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.680 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:31 smithi191 ceph-mon[161669]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.681 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:31 smithi191 ceph-mon[161669]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.681 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:31 smithi191 ceph-mon[161669]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.681 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:31.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[184418]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[184418]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[184418]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[184418]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T06:05:31.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[184418]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[184418]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[184418]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[184418]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[184418]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[191227]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[191227]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[191227]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[191227]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T06:05:31.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[191227]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[191227]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[191227]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[191227]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:31 smithi008 ceph-mon[191227]: failed to encode map e549 with expected crc 2023-10-09T06:05:31.838 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:31.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:31.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:31.945 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:31.974 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:32.022 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:32.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:32.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:32.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:32.152 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:32.200 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:32.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:32.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:32.311 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:32.342 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:32.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: pgmap v1661: 89 pgs: 5 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 21867/163749 objects degraded (13.354%); 1263/163749 objects misplaced (0.771%); 4.9 MiB/s, 76 objects/s recovering 2023-10-09T06:05:32.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:05:32.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T06:05:32.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:32.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:32.441 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:32.442 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:32.442 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:32.442 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:32.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:32.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:32.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:32.588 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:32.649 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:32.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:32.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:32.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: pgmap v1661: 89 pgs: 5 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 21867/163749 objects degraded (13.354%); 1263/163749 objects misplaced (0.771%); 4.9 MiB/s, 76 objects/s recovering 2023-10-09T06:05:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:05:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T06:05:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:32.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:32.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:32.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: pgmap v1661: 89 pgs: 5 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 21867/163749 objects degraded (13.354%); 1263/163749 objects misplaced (0.771%); 4.9 MiB/s, 76 objects/s recovering 2023-10-09T06:05:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:05:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T06:05:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: failed to encode map e550 with expected crc 2023-10-09T06:05:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:32.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:05:32.754 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:32.783 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:32.964 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:05:32.964 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:05:33.008 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:33.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:33.112 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:33.149 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:33.209 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:33.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:33.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:33.314 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:33.345 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:33.404 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:33.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:33.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:33.551 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:33.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 21867/163749 objects degraded (13.354%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:33.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 21867/163749 objects degraded (13.354%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:33.786 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:33.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 21867/163749 objects degraded (13.354%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:33.864 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:05:33.874 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:33.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:33.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:33.981 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:34.010 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:34.063 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:34.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:34.168 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:34.228 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:34.459 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:34.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:34.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:34.468 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:05:34.494 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:34 smithi008 ceph-mon[184418]: pgmap v1663: 89 pgs: 1 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 17593/161247 objects degraded (10.911%); 1343/161247 objects misplaced (0.833%); 3.9 MiB/s, 59 objects/s recovering 2023-10-09T06:05:34.494 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3754490066' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:34.494 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:34 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:34.494 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:34 smithi008 ceph-mon[191227]: pgmap v1663: 89 pgs: 1 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 17593/161247 objects degraded (10.911%); 1343/161247 objects misplaced (0.833%); 3.9 MiB/s, 59 objects/s recovering 2023-10-09T06:05:34.495 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3754490066' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:34.495 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:34 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:34.566 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:34.616 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:34.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:34 smithi191 ceph-mon[161669]: pgmap v1663: 89 pgs: 1 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 17593/161247 objects degraded (10.911%); 1343/161247 objects misplaced (0.833%); 3.9 MiB/s, 59 objects/s recovering 2023-10-09T06:05:34.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3754490066' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:34.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:34 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:35.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[191227]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[191227]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T06:05:35.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[191227]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[191227]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[191227]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:35.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:35.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[184418]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[184418]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T06:05:35.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[184418]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[184418]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:35 smithi008 ceph-mon[184418]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.755 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:35.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:35.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:35.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:35 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:35.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:35 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:35.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:35 smithi191 ceph-mon[161669]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:35 smithi191 ceph-mon[161669]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T06:05:35.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:35 smithi191 ceph-mon[161669]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:35 smithi191 ceph-mon[161669]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:35 smithi191 ceph-mon[161669]: failed to encode map e551 with expected crc 2023-10-09T06:05:35.871 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:35.885 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:35.953 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:36.265 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-09T06:05:36.266 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:05:36.266 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:05:36.266 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.831032 2023-10-09T06:05:36.266 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0.831032440838854 2023-10-09T06:05:36.304 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:36.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:36.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:36.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:36.422 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:05:36.459 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:36.461 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:05:36.506 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:05:36.533 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:05:36.538 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:36.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:36.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:36.556 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:05:36.580 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:05:36.624 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:05:36.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:36.651 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:05:36.653 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:05:36.653 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:05:36.653 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:05:36.653 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:05:36.653 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:05:36.654 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:05:36.654 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:05:36.654 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:05:36.676 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:36 smithi008 ceph-mon[184418]: pgmap v1665: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 1 active+recovery_wait, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 14397/160761 objects degraded (8.956%); 4503/160761 objects misplaced (2.801%); 4.7 MiB/s, 72 objects/s recovering 2023-10-09T06:05:36.677 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:36 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T06:05:36.677 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/487033956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:36.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:36 smithi008 ceph-mon[191227]: pgmap v1665: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 1 active+recovery_wait, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 14397/160761 objects degraded (8.956%); 4503/160761 objects misplaced (2.801%); 4.7 MiB/s, 72 objects/s recovering 2023-10-09T06:05:36.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:36 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T06:05:36.677 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/487033956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:36.691 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:36.770 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:36.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:36.791 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:36 smithi191 ceph-mon[161669]: pgmap v1665: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 1 active+recovery_wait, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 14397/160761 objects degraded (8.956%); 4503/160761 objects misplaced (2.801%); 4.7 MiB/s, 72 objects/s recovering 2023-10-09T06:05:36.791 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:36 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T06:05:36.791 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/487033956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:36.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:36.928 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:36.973 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:36.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:36.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:37.081 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:37.107 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:37.176 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:37.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:37.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:37.288 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:37.342 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:37.418 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:37.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:37.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:37.537 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:37.578 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:37.646 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:37.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:37.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:37.748 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:05:37.762 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:37.797 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:37.886 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:37.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:37.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:37.994 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:38.044 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:38.325 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:38.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:38.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:38.431 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:38.467 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:38.541 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:38.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:38.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:38.569 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:38.648 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:38.667 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:38.695 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:05:38.695 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T06:05:38.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.695 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.47036.0:122763 10.b e551/538 10:d011a30e:::benchmark_data_smithi008_634586_object35598:head v 551'7494, mlcod=529'5327)", 2023-10-09T06:05:38.695 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:38.681601+0000", 2023-10-09T06:05:38.695 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0021279710000000002, 2023-10-09T06:05:38.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002159266, 2023-10-09T06:05:38.695 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:05:38.695 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.681601+0000", 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.681601+0000", 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.681600+0000", 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.681609+0000", 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.1710000000000008e-06 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.681624+0000", 2023-10-09T06:05:38.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4657999999999999e-05 2023-10-09T06:05:38.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:05:38.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.681632+0000", 2023-10-09T06:05:38.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1859999999999992e-06 2023-10-09T06:05:38.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:05:38.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.683660+0000", 2023-10-09T06:05:38.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020276550000000002 2023-10-09T06:05:38.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.683682+0000", 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2252999999999999e-05 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.47036.0:122752 10.b e551/538 10:d7d2c847:::benchmark_data_smithi008_634586_object35587:head v 551'7493, mlcod=529'5327)", 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:38.656765+0000", 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.026963981000000001, 2023-10-09T06:05:38.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.027066574999999999, 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.656765+0000", 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.656765+0000", 2023-10-09T06:05:38.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.656764+0000", 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.656772+0000", 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1250000000000004e-06 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.656782+0000", 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0518e-05 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.656788+0000", 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.3319999999999997e-06 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.660512+0000", 2023-10-09T06:05:38.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037235520000000002 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.660538+0000", 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5922999999999999e-05 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.47036.0:122754 10.3 e551/528 10:caa84256:::benchmark_data_smithi008_634586_object35589:head v 551'7686, mlcod=515'495)", 2023-10-09T06:05:38.703 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:38.659917+0000", 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.023812448, 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.023978151, 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.659917+0000", 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.659917+0000", 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.659915+0000", 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.659926+0000", 2023-10-09T06:05:38.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0971999999999999e-05 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.659951+0000", 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4712e-05 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.659962+0000", 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.048e-05 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.663746+0000", 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037839169999999999 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.663772+0000", 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6653e-05 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.47036.0:122761 10.3 e551/528 10:c516da61:::benchmark_data_smithi008_634586_object35596:head v 551'7687, mlcod=515'495)", 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:38.671290+0000", 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.012439086, 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012665635, 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.671290+0000", 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.671290+0000", 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.671289+0000", 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:05:38.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.671301+0000", 2023-10-09T06:05:38.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2112e-05 2023-10-09T06:05:38.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:05:38.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.671324+0000", 2023-10-09T06:05:38.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2770000000000001e-05 2023-10-09T06:05:38.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:05:38.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.671334+0000", 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0618999999999999e-05 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.679268+0000", 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0079338470000000008 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.679294+0000", 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6179000000000001e-05 2023-10-09T06:05:38.711 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.47036.0:122757 10.2 e551/539 10:4b8a49a8:::benchmark_data_smithi008_634586_object35592:head v 551'7654, mlcod=551'7652)", 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:38.678258+0000", 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0054708049999999996, 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0057611160000000002, 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "started", 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:05:38.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.678258+0000", 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.678258+0000", 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:05:38.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.678258+0000", 2023-10-09T06:05:38.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:05:38.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:05:38.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.678265+0000", 2023-10-09T06:05:38.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.712e-06 2023-10-09T06:05:38.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:05:38.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.678288+0000", 2023-10-09T06:05:38.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2887e-05 2023-10-09T06:05:38.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:05:38.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.678298+0000", 2023-10-09T06:05:38.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0132999999999999e-05 2023-10-09T06:05:38.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:05:38.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.683662+0000", 2023-10-09T06:05:38.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00536382 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:38.683682+0000", 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9984999999999999e-05 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 5 2023-10-09T06:05:38.716 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:05:38.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:38 smithi008 ceph-mon[184418]: pgmap v1666: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 230 op/s; 18225/159291 objects degraded (11.441%); 4287/159291 objects misplaced (2.691%); 5.7 MiB/s, 104 objects/s recovering 2023-10-09T06:05:38.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 18225/159291 objects degraded (11.441%), 10 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:38.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:38 smithi008 ceph-mon[191227]: pgmap v1666: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 230 op/s; 18225/159291 objects degraded (11.441%); 4287/159291 objects misplaced (2.691%); 5.7 MiB/s, 104 objects/s recovering 2023-10-09T06:05:38.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 18225/159291 objects degraded (11.441%), 10 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:38.752 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:38.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:38 smithi191 ceph-mon[161669]: pgmap v1666: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 230 op/s; 18225/159291 objects degraded (11.441%); 4287/159291 objects misplaced (2.691%); 5.7 MiB/s, 104 objects/s recovering 2023-10-09T06:05:38.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 18225/159291 objects degraded (11.441%), 10 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:39.221 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:39.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:39.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:39.328 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:39.379 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:39.646 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:39.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:39.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:39.751 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:39.816 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:39.924 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:39.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:39.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:40.008 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:05:40.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:40.075 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:40.127 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:05:40.149 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:40.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:40.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:40.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:40.303 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:40.407 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:40.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:40.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:40.483 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:40 smithi008 ceph-mon[184418]: pgmap v1667: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 18225/158688 objects degraded (11.485%); 4287/158688 objects misplaced (2.702%); 4.6 MiB/s, 84 objects/s recovering 2023-10-09T06:05:40.517 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:40.572 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:40 smithi191 ceph-mon[161669]: pgmap v1667: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 18225/158688 objects degraded (11.485%); 4287/158688 objects misplaced (2.702%); 4.6 MiB/s, 84 objects/s recovering 2023-10-09T06:05:40.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:05:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:05:40] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.33.4" 2023-10-09T06:05:40.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:40 smithi008 ceph-mon[191227]: pgmap v1667: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 18225/158688 objects degraded (11.485%); 4287/158688 objects misplaced (2.702%); 4.6 MiB/s, 84 objects/s recovering 2023-10-09T06:05:41.773 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:41.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:41.892 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:41.952 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:42.046 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:42.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:42.153 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:42.233 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:42.342 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:42.370 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:42.474 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:42.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:42.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:42.584 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:42.629 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:42.714 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:42.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:42.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:42.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:42 smithi191 ceph-mon[161669]: pgmap v1668: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 18241/155793 objects degraded (11.708%); 4160/155793 objects misplaced (2.670%); 3.1 MiB/s, 59 objects/s recovering 2023-10-09T06:05:42.821 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:42.845 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:42 smithi008 ceph-mon[191227]: pgmap v1668: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 18241/155793 objects degraded (11.708%); 4160/155793 objects misplaced (2.670%); 3.1 MiB/s, 59 objects/s recovering 2023-10-09T06:05:42.848 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:42 smithi008 ceph-mon[184418]: pgmap v1668: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 18241/155793 objects degraded (11.708%); 4160/155793 objects misplaced (2.670%); 3.1 MiB/s, 59 objects/s recovering 2023-10-09T06:05:42.862 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:42.940 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:42.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:42.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:43.053 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:43.113 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:43.221 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:43.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:43.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:43.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:43.366 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:43.460 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:43.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:43.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:43.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:43 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/450840240' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:43.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:43 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:43.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:43 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2262446486' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.831032440838854}]: dispatch 2023-10-09T06:05:43.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:43 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.831032440838854}]: dispatch 2023-10-09T06:05:43.503 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 18241/155793 objects degraded (11.708%), 10 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:43.578 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:43.607 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:05:43.609 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.2 primary-affinity to 0.831032 (8544622) 2023-10-09T06:05:43.614 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:43.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:43 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/450840240' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:43.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:43 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:43.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:43 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2262446486' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.831032440838854}]: dispatch 2023-10-09T06:05:43.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:43 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.831032440838854}]: dispatch 2023-10-09T06:05:43.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 18241/155793 objects degraded (11.708%), 10 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:43.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:43 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/450840240' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:43.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:43 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:43.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:43 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2262446486' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.831032440838854}]: dispatch 2023-10-09T06:05:43.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:43 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.831032440838854}]: dispatch 2023-10-09T06:05:43.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 18241/155793 objects degraded (11.708%), 10 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:44 smithi191 ceph-mon[161669]: pgmap v1669: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 17984/152085 objects degraded (11.825%); 4160/152085 objects misplaced (2.735%); 3.6 MiB/s, 78 objects/s recovering 2023-10-09T06:05:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:44 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:44 smithi191 ceph-mon[161669]: failed to encode map e552 with expected crc 2023-10-09T06:05:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:44 smithi191 ceph-mon[161669]: failed to encode map e552 with expected crc 2023-10-09T06:05:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:44 smithi191 ceph-mon[161669]: failed to encode map e552 with expected crc 2023-10-09T06:05:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:44 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:05:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:44 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.831032440838854}]': finished 2023-10-09T06:05:44.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:44 smithi191 ceph-mon[161669]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T06:05:44.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:44.844 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[184418]: pgmap v1669: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 17984/152085 objects degraded (11.825%); 4160/152085 objects misplaced (2.735%); 3.6 MiB/s, 78 objects/s recovering 2023-10-09T06:05:44.845 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:44.845 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[184418]: failed to encode map e552 with expected crc 2023-10-09T06:05:44.845 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[184418]: failed to encode map e552 with expected crc 2023-10-09T06:05:44.845 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[184418]: failed to encode map e552 with expected crc 2023-10-09T06:05:44.845 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:05:44.845 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.831032440838854}]': finished 2023-10-09T06:05:44.845 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[184418]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T06:05:44.845 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:44.846 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[191227]: pgmap v1669: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 17984/152085 objects degraded (11.825%); 4160/152085 objects misplaced (2.735%); 3.6 MiB/s, 78 objects/s recovering 2023-10-09T06:05:44.846 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:44.846 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[191227]: failed to encode map e552 with expected crc 2023-10-09T06:05:44.846 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[191227]: failed to encode map e552 with expected crc 2023-10-09T06:05:44.846 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[191227]: failed to encode map e552 with expected crc 2023-10-09T06:05:44.846 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:05:44.846 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.831032440838854}]': finished 2023-10-09T06:05:44.846 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[191227]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T06:05:44.847 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:45 smithi191 ceph-mon[161669]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:45 smithi191 ceph-mon[161669]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:45 smithi191 ceph-mon[161669]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T06:05:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:45 smithi191 ceph-mon[161669]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:45 smithi191 ceph-mon[161669]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:45 smithi191 ceph-mon[161669]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:45 smithi191 ceph-mon[161669]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.841 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[184418]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.841 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[184418]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.841 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[184418]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T06:05:45.841 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[184418]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.841 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[184418]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.842 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[184418]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.842 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[184418]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.842 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[191227]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.842 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[191227]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.842 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[191227]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T06:05:45.842 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[191227]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.842 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[191227]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.842 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[191227]: failed to encode map e553 with expected crc 2023-10-09T06:05:45.842 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:45 smithi008 ceph-mon[191227]: failed to encode map e553 with expected crc 2023-10-09T06:05:46.226 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:46.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:46.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:46.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:46.360 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:46.409 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:46.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:46.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:46.428 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:05:46.428 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:05:46.428 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T06:05:46.429 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T06:05:46.429 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:05:46.514 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:46.553 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:46.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:46 smithi008 ceph-mon[184418]: pgmap v1672: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 368 op/s; 18002/150453 objects degraded (11.965%); 4032/150453 objects misplaced (2.680%); 3.7 MiB/s, 72 objects/s recovering 2023-10-09T06:05:46.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:46 smithi008 ceph-mon[184418]: failed to encode map e554 with expected crc 2023-10-09T06:05:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:46 smithi008 ceph-mon[184418]: failed to encode map e554 with expected crc 2023-10-09T06:05:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:46 smithi008 ceph-mon[184418]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T06:05:46.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:46 smithi008 ceph-mon[184418]: failed to encode map e554 with expected crc 2023-10-09T06:05:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:46 smithi008 ceph-mon[191227]: pgmap v1672: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 368 op/s; 18002/150453 objects degraded (11.965%); 4032/150453 objects misplaced (2.680%); 3.7 MiB/s, 72 objects/s recovering 2023-10-09T06:05:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:46 smithi008 ceph-mon[191227]: failed to encode map e554 with expected crc 2023-10-09T06:05:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:46 smithi008 ceph-mon[191227]: failed to encode map e554 with expected crc 2023-10-09T06:05:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:46 smithi008 ceph-mon[191227]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T06:05:46.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:46 smithi008 ceph-mon[191227]: failed to encode map e554 with expected crc 2023-10-09T06:05:46.758 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:46.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:46.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:46.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:46 smithi191 ceph-mon[161669]: pgmap v1672: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 368 op/s; 18002/150453 objects degraded (11.965%); 4032/150453 objects misplaced (2.680%); 3.7 MiB/s, 72 objects/s recovering 2023-10-09T06:05:46.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:46 smithi191 ceph-mon[161669]: failed to encode map e554 with expected crc 2023-10-09T06:05:46.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:46 smithi191 ceph-mon[161669]: failed to encode map e554 with expected crc 2023-10-09T06:05:46.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:46 smithi191 ceph-mon[161669]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T06:05:46.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:46 smithi191 ceph-mon[161669]: failed to encode map e554 with expected crc 2023-10-09T06:05:46.849 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:05:46.862 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:46.894 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:46.946 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:46.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:46.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:47.053 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:47.084 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:47.240 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:05:47.244 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:05:47.263 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:05:47.282 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:05:47.299 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:05:47.318 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:05:47.337 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:05:47.354 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:05:47.355 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:05:47.356 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:05:47.356 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:05:47.356 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:05:47.356 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:05:47.356 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:05:47.356 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:05:47.356 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:05:47.667 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2895444650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:47.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2895444650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:47.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2895444650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:47.976 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:05:48.168 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:48.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:48.276 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:48.368 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:48.437 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:48.467 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:48.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:48.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:48.522 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:48 smithi008 ceph-mon[191227]: pgmap v1674: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 15124/148176 objects degraded (10.207%); 6025/148176 objects misplaced (4.066%); 5.9 MiB/s, 123 objects/s recovering 2023-10-09T06:05:48.523 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15124/148176 objects degraded (10.207%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:48.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:48.612 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:48.673 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:48.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:48.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:48.777 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:48 smithi008 ceph-mon[184418]: pgmap v1674: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 15124/148176 objects degraded (10.207%); 6025/148176 objects misplaced (4.066%); 5.9 MiB/s, 123 objects/s recovering 2023-10-09T06:05:48.777 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15124/148176 objects degraded (10.207%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:48.779 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:48.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:48 smithi191 ceph-mon[161669]: pgmap v1674: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 15124/148176 objects degraded (10.207%); 6025/148176 objects misplaced (4.066%); 5.9 MiB/s, 123 objects/s recovering 2023-10-09T06:05:48.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15124/148176 objects degraded (10.207%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:48.806 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:48.858 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:48.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:48.959 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:48.966 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:49.010 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:49.712 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:49.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:49.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:49.817 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:49.869 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:49.983 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:49.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:49.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:50.094 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:50.141 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:50.288 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:05:50.618 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:50.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:50.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:50.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:50.752 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:50.795 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:50.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:50 smithi191 ceph-mon[161669]: pgmap v1675: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 15124/148110 objects degraded (10.211%); 6025/148110 objects misplaced (4.068%); 3.2 MiB/s, 63 objects/s recovering 2023-10-09T06:05:50.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:50.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:50.866 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-09T06:05:50.866 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:05:50.866 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T06:05:50.866 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:05:50.866 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:05:50.867 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T06:05:50.893 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:50 smithi008 ceph-mon[184418]: pgmap v1675: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 15124/148110 objects degraded (10.211%); 6025/148110 objects misplaced (4.068%); 3.2 MiB/s, 63 objects/s recovering 2023-10-09T06:05:50.893 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:50 smithi008 ceph-mon[191227]: pgmap v1675: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 15124/148110 objects degraded (10.211%); 6025/148110 objects misplaced (4.068%); 3.2 MiB/s, 63 objects/s recovering 2023-10-09T06:05:50.894 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:05:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:05:50] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.33.4" 2023-10-09T06:05:50.901 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:50.936 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:51.337 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:51.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:51.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:51.443 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:51.505 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:51.574 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:05:51.609 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:51 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4042993158' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:51.609 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:51 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4042993158' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:51.711 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:51.842 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:51.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:51.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:51.949 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:52.009 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:52.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:51 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4042993158' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:52.429 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:52.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:52.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:52.536 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:52.567 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:52.635 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:52.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:52.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:52.652 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:05:52.652 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:05:52.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[184418]: pgmap v1676: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 206 op/s; 15124/147516 objects degraded (10.252%); 5910/147516 objects misplaced (4.006%); 3.6 MiB/s, 67 objects/s recovering 2023-10-09T06:05:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[184418]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[184418]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4042993158' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[184418]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T06:05:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[184418]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[184418]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[191227]: pgmap v1676: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 206 op/s; 15124/147516 objects degraded (10.252%); 5910/147516 objects misplaced (4.006%); 3.6 MiB/s, 67 objects/s recovering 2023-10-09T06:05:52.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:52.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[191227]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[191227]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4042993158' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:52.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[191227]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T06:05:52.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[191227]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:52 smithi008 ceph-mon[191227]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.744 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:52.776 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:52.830 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:52.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:52 smithi191 ceph-mon[161669]: pgmap v1676: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 206 op/s; 15124/147516 objects degraded (10.252%); 5910/147516 objects misplaced (4.006%); 3.6 MiB/s, 67 objects/s recovering 2023-10-09T06:05:52.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:52 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:52.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:52 smithi191 ceph-mon[161669]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:52 smithi191 ceph-mon[161669]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4042993158' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:52.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:52 smithi191 ceph-mon[161669]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T06:05:52.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:52 smithi191 ceph-mon[161669]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.838 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:52 smithi191 ceph-mon[161669]: failed to encode map e555 with expected crc 2023-10-09T06:05:52.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:52.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:52.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:52.974 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:53.041 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:53.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:53.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:53.150 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:53.193 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:53.268 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:53.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:53.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:53.382 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:53.435 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15124/147516 objects degraded (10.252%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:53.968 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15124/147516 objects degraded (10.252%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:54.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15124/147516 objects degraded (10.252%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:54.186 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:54.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:54.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:54.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:54.328 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:54.391 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:54.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:54.497 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:54.550 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:54.574 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:54 smithi191 ceph-mon[161669]: pgmap v1678: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 17073/144954 objects degraded (11.778%); 3864/144954 objects misplaced (2.666%); 3.3 MiB/s, 114 objects/s recovering 2023-10-09T06:05:54.599 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:54.615 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:54.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:54.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:54.632 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:05:54.726 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:54.793 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:54.910 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:54 smithi008 ceph-mon[184418]: pgmap v1678: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 17073/144954 objects degraded (11.778%); 3864/144954 objects misplaced (2.666%); 3.3 MiB/s, 114 objects/s recovering 2023-10-09T06:05:54.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:54 smithi008 ceph-mon[191227]: pgmap v1678: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 17073/144954 objects degraded (11.778%); 3864/144954 objects misplaced (2.666%); 3.3 MiB/s, 114 objects/s recovering 2023-10-09T06:05:55.113 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:55.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:55.226 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:55.293 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:55.673 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:05:55.927 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:05:55.944 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:05:55.954 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:55.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:55.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:55.975 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:05:56.009 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:05:56.040 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:05:56.066 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:56.070 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:05:56.102 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:56.103 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:05:56.141 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:05:56.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:05:56.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:05:56.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:05:56.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:05:56.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:05:56.162 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:05:56.163 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:05:56.163 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:05:56.184 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:56.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:56.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:56.292 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:56.375 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:56.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:56 smithi008 ceph-mon[184418]: pgmap v1679: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 17073/143274 objects degraded (11.916%); 3748/143274 objects misplaced (2.616%); 3.6 MiB/s, 110 objects/s recovering 2023-10-09T06:05:56.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:56 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1454818298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:56.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:56 smithi008 ceph-mon[191227]: pgmap v1679: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 17073/143274 objects degraded (11.916%); 3748/143274 objects misplaced (2.616%); 3.6 MiB/s, 110 objects/s recovering 2023-10-09T06:05:56.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:56 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1454818298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:57.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:56 smithi191 ceph-mon[161669]: pgmap v1679: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 17073/143274 objects degraded (11.916%); 3748/143274 objects misplaced (2.616%); 3.6 MiB/s, 110 objects/s recovering 2023-10-09T06:05:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:56 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1454818298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:57.141 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:57.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:57.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:57.254 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:57.322 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:57.479 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:57.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:57.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:57.590 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:57.599 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:57.653 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:57.655 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:57.755 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:05:57.755 INFO:teuthology.orchestra.run.smithi008.stdout: "success": "" 2023-10-09T06:05:57.755 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:05:57.820 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:57.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:57.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:57.933 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:57.985 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:58.327 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:58.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:58.437 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:58.474 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:58.568 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:58.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:58.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:58.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:58.713 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:58.769 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:58.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:58.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:58.779 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:58 smithi191 ceph-mon[161669]: pgmap v1680: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 16823/139710 objects degraded (12.041%); 3638/139710 objects misplaced (2.604%); 4.4 MiB/s, 113 objects/s recovering 2023-10-09T06:05:58.779 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16823/139710 objects degraded (12.041%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:58.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:58.918 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:58.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:58 smithi008 ceph-mon[184418]: pgmap v1680: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 16823/139710 objects degraded (12.041%); 3638/139710 objects misplaced (2.604%); 4.4 MiB/s, 113 objects/s recovering 2023-10-09T06:05:58.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16823/139710 objects degraded (12.041%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:58.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:58 smithi008 ceph-mon[191227]: pgmap v1680: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 16823/139710 objects degraded (12.041%); 3638/139710 objects misplaced (2.604%); 4.4 MiB/s, 113 objects/s recovering 2023-10-09T06:05:58.934 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16823/139710 objects degraded (12.041%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:05:58.991 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:58.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:58.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:59.100 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:59.164 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:05:59.174 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:00.108 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:00.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:00.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:59 smithi008 ceph-mon[184418]: pgmap v1681: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 16823/139341 objects degraded (12.073%); 3638/139341 objects misplaced (2.611%); 4.4 MiB/s, 113 objects/s recovering 2023-10-09T06:06:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:00.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:05:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:59 smithi008 ceph-mon[191227]: pgmap v1681: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 16823/139341 objects degraded (12.073%); 3638/139341 objects misplaced (2.611%); 4.4 MiB/s, 113 objects/s recovering 2023-10-09T06:06:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:00.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:05:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:00.221 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:00.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:59 smithi191 ceph-mon[161669]: pgmap v1681: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 16823/139341 objects degraded (12.073%); 3638/139341 objects misplaced (2.611%); 4.4 MiB/s, 113 objects/s recovering 2023-10-09T06:06:00.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:00.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:05:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:00.248 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:00.291 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:00.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:00.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:00.396 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:00.468 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:00.713 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:06:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:06:00] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.33.4" 2023-10-09T06:06:00.726 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:00.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:00.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:00.834 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:00.864 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:00.925 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:00.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:00.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:01.035 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:01.076 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:01.165 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:01.386 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:01.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:01.494 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:01.597 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:01.741 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:06:01.742 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T06:06:01.742 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.742 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:130293 10.b 10:d6804263:::benchmark_data_smithi008_634586_object43128:head [delete] snapc 0=[] ondisk+write+known_if_redirected e555)", 2023-10-09T06:06:01.742 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:01.722969+0000", 2023-10-09T06:06:01.742 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 0.0054999230000000003, 2023-10-09T06:06:01.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0055393309999999998, 2023-10-09T06:06:01.742 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:01.742 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:06:01.742 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:01.742 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:01.743 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:01.743 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 130293 2023-10-09T06:06:01.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:01.743 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:01.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:01.743 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.722969+0000", 2023-10-09T06:06:01.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:01.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:01.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:01.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.722969+0000", 2023-10-09T06:06:01.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:01.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:01.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:01.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.722968+0000", 2023-10-09T06:06:01.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:01.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:01.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:01.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.722976+0000", 2023-10-09T06:06:01.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8959999999999992e-06 2023-10-09T06:06:01.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:01.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:01.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.723001+0000", 2023-10-09T06:06:01.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4603000000000001e-05 2023-10-09T06:06:01.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:01.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:01.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.723014+0000", 2023-10-09T06:06:01.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2748999999999999e-05 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.723074+0000", 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9737000000000001e-05 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.723234+0000", 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016024999999999999 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:01.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.723383+0000", 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014886599999999999 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.726978+0000", 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003595151 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:01.747 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.727135+0000", 2023-10-09T06:06:01.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00015767999999999999 2023-10-09T06:06:01.748 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:01.748 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:01.748 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:01.748 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:01.748 INFO:teuthology.orchestra.run.smithi008.stdout: ], 2023-10-09T06:06:01.748 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 1 2023-10-09T06:06:01.748 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:06:01.749 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:01.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:01.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:01.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:01.889 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:01.953 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:01.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:01.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:01.987 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:06:02.060 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:02.094 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:02.151 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:02.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:02.156 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:01 smithi191 ceph-mon[161669]: pgmap v1682: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 332 op/s; 16823/137535 objects degraded (12.232%); 3524/137535 objects misplaced (2.562%); 4.4 MiB/s, 113 objects/s recovering 2023-10-09T06:06:02.156 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:01 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3425849644' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:02.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:01 smithi008 ceph-mon[184418]: pgmap v1682: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 332 op/s; 16823/137535 objects degraded (12.232%); 3524/137535 objects misplaced (2.562%); 4.4 MiB/s, 113 objects/s recovering 2023-10-09T06:06:02.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:01 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3425849644' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:02.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:01 smithi008 ceph-mon[191227]: pgmap v1682: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 332 op/s; 16823/137535 objects degraded (12.232%); 3524/137535 objects misplaced (2.562%); 4.4 MiB/s, 113 objects/s recovering 2023-10-09T06:06:02.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:01 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3425849644' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:02.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:02.288 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:02.343 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:02.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:02.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:02.451 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:02.484 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:03.015 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:03.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:03.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:03.120 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:03.157 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:03.205 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:03.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:03.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:03.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:02 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:02 smithi191 ceph-mon[161669]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:02 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3425849644' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:06:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:02 smithi191 ceph-mon[161669]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T06:06:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:02 smithi191 ceph-mon[161669]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:02 smithi191 ceph-mon[161669]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:02 smithi191 ceph-mon[161669]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.309 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:03.386 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:03.386 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[184418]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.386 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3425849644' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:06:03.386 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[184418]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T06:06:03.386 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[184418]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.387 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[184418]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.387 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[184418]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.387 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:03.387 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[191227]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.387 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3425849644' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:06:03.387 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[191227]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T06:06:03.387 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[191227]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.387 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[191227]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.387 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:02 smithi008 ceph-mon[191227]: failed to encode map e556 with expected crc 2023-10-09T06:06:03.389 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:03.709 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:03.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:03.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:03.825 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:03.866 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:04.126 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-09T06:06:04.126 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:06:04.127 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:06:04.127 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T06:06:04.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:04 smithi191 ceph-mon[161669]: pgmap v1684: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 16569/133296 objects degraded (12.430%); 3402/133296 objects misplaced (2.552%); 6.1 MiB/s, 96 objects/s recovering 2023-10-09T06:06:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:04 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16569/133296 objects degraded (12.430%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:04.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:04 smithi008 ceph-mon[184418]: pgmap v1684: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 16569/133296 objects degraded (12.430%); 3402/133296 objects misplaced (2.552%); 6.1 MiB/s, 96 objects/s recovering 2023-10-09T06:06:04.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:04 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16569/133296 objects degraded (12.430%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:04 smithi008 ceph-mon[191227]: pgmap v1684: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 16569/133296 objects degraded (12.430%); 3402/133296 objects misplaced (2.552%); 6.1 MiB/s, 96 objects/s recovering 2023-10-09T06:06:04.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:04 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16569/133296 objects degraded (12.430%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:04.485 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:04.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:04.525 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:06:04.590 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:04.670 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:04.768 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:04.792 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:04.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:04.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:04.899 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:04.937 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:05.007 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:05.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:05.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:05.115 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:05.204 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:05.342 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:06:05.348 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:05.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:05.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:05.376 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:06:05.416 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:06:05.457 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:06:05.465 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:05.498 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:06:05.543 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:06:05.556 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:05.584 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:06:05.633 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:06:05.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:06:05.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:06:05.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:06:05.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:06:05.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:06:05.662 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:06:05.663 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:06:05.663 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:06:05.747 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:06:05.759 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:05.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:05.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:05.871 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:05.925 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:06.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:06 smithi008 ceph-mon[184418]: pgmap v1685: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 16569/131616 objects degraded (12.589%); 3293/131616 objects misplaced (2.502%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:06:06.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/391791902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:06 smithi008 ceph-mon[191227]: pgmap v1685: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 16569/131616 objects degraded (12.589%); 3293/131616 objects misplaced (2.502%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:06:06.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/391791902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:06 smithi191 ceph-mon[161669]: pgmap v1685: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 16569/131616 objects degraded (12.589%); 3293/131616 objects misplaced (2.502%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:06:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/391791902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:07.058 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:07.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:07.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:07.099 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:07.168 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:07.251 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:07.480 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:07.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:07.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:07.590 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:07.685 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:07.792 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:07.807 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:07.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:07.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:07.920 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:08.032 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:08.257 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:08 smithi008 ceph-mon[191227]: pgmap v1686: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 384 op/s; 16332/128166 objects degraded (12.743%); 3169/128166 objects misplaced (2.473%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:06:08.259 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:08 smithi008 ceph-mon[184418]: pgmap v1686: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 384 op/s; 16332/128166 objects degraded (12.743%); 3169/128166 objects misplaced (2.473%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:06:08.259 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:06:08 smithi008 conmon[177229]: 2023-10-09T06:06:08.255+0000 7f020285b700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T06:06:08.260 INFO:teuthology.orchestra.run.smithi008.stdout:moved 14 / 267 (5.24345%) 2023-10-09T06:06:08.260 INFO:teuthology.orchestra.run.smithi008.stdout:avg 38.1429 2023-10-09T06:06:08.260 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 8.49249 -> 12.586 (expected baseline 5.71786) 2023-10-09T06:06:08.260 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.0 with 25 -> 17 pgs (0.655431 -> 0.445693 * mean) 2023-10-09T06:06:08.260 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.1 with 51 -> 55 pgs (1.33708 -> 1.44195 * mean) 2023-10-09T06:06:08.260 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:06:08.260 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T06:06:08.260 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 1 2023-10-09T06:06:08.260 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T06:06:08.260 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0250 2023-10-09T06:06:08.261 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0275 2023-10-09T06:06:08.261 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0 weight 0.5845 -> 0.4276 2023-10-09T06:06:08.261 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 weight 0.5592 -> 0.4118 2023-10-09T06:06:08.272 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T06:06:08.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:08 smithi191 ceph-mon[161669]: pgmap v1686: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 384 op/s; 16332/128166 objects degraded (12.743%); 3169/128166 objects misplaced (2.473%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:06:08.413 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:08.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:08.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:08.526 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:08.596 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:09.080 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:06:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:09 smithi191 ceph-mon[161669]: from='client.57106 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:09.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:09 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16332/128166 objects degraded (12.743%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:09.313 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:09 smithi008 ceph-mon[184418]: from='client.57106 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:09.314 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:09 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16332/128166 objects degraded (12.743%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:09.314 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:09 smithi008 ceph-mon[191227]: from='client.57106 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:09.314 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:09 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16332/128166 objects degraded (12.743%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:09.929 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:09.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:09.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:10.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:10.085 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:10.148 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:10.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:10.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:10.254 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:10.313 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:10.436 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:10 smithi191 ceph-mon[161669]: pgmap v1687: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 384 op/s; 16332/127800 objects degraded (12.779%); 3169/127800 objects misplaced (2.480%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:06:10.588 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:06:10.588 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-10-09T06:06:10.589 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:10 smithi008 ceph-mon[184418]: pgmap v1687: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 384 op/s; 16332/127800 objects degraded (12.779%); 3169/127800 objects misplaced (2.480%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:06:10.589 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:10 smithi008 ceph-mon[191227]: pgmap v1687: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 384 op/s; 16332/127800 objects degraded (12.779%); 3169/127800 objects misplaced (2.480%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:06:10.598 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:06:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:06:10] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.33.4" 2023-10-09T06:06:10.616 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:10.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:10.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:10.744 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:10.772 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:10.840 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:10.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:10.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:10.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:10.977 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:11.038 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:11.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:11.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:11.148 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:11.228 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:11.259 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:11.291 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:06:11.291 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:06:11.552 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:11.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:11.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:11.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:11.696 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:11.740 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:11.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:11.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:11.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:11.874 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:11.940 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:11.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:12.058 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:12.129 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:12.343 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:12.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:12.448 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:12.457 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:12 smithi008 ceph-mon[184418]: pgmap v1688: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 16332/126081 objects degraded (12.954%); 3054/126081 objects misplaced (2.422%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:06:12.556 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:06:12.562 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:12.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:12 smithi008 ceph-mon[191227]: pgmap v1688: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 16332/126081 objects degraded (12.954%); 3054/126081 objects misplaced (2.422%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:06:12.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:12 smithi191 ceph-mon[161669]: pgmap v1688: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 16332/126081 objects degraded (12.954%); 3054/126081 objects misplaced (2.422%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:06:12.814 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:12.856 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:12.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:12.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:12.962 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:13.001 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:13.054 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:13.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:13.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:13.159 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:13.188 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:13.492 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:06:13.509 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:13.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:13.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:13.623 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:13.692 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:13.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2862591983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:13.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:13 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 16332/126081 objects degraded (12.954%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:13.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2862591983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:13.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:13 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 16332/126081 objects degraded (12.954%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2862591983' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:13.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:13 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 16332/126081 objects degraded (12.954%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:14.519 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:14.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:14.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:14.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:14.667 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:14.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[184418]: pgmap v1689: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 16047/121380 objects degraded (13.220%); 2925/121380 objects misplaced (2.410%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:06:14.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:14.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[184418]: failed to encode map e557 with expected crc 2023-10-09T06:06:14.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[184418]: failed to encode map e557 with expected crc 2023-10-09T06:06:14.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2862591983' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:14.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[184418]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T06:06:14.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:14.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:14.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[191227]: pgmap v1689: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 16047/121380 objects degraded (13.220%); 2925/121380 objects misplaced (2.410%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:06:14.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:14.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[191227]: failed to encode map e557 with expected crc 2023-10-09T06:06:14.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[191227]: failed to encode map e557 with expected crc 2023-10-09T06:06:14.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2862591983' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:14.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[191227]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T06:06:14.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:14.719 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:14.734 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:14.739 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:14 smithi191 ceph-mon[161669]: pgmap v1689: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 16047/121380 objects degraded (13.220%); 2925/121380 objects misplaced (2.410%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:06:14.739 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:14 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:14.739 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:14 smithi191 ceph-mon[161669]: failed to encode map e557 with expected crc 2023-10-09T06:06:14.739 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:14 smithi191 ceph-mon[161669]: failed to encode map e557 with expected crc 2023-10-09T06:06:14.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:14 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2862591983' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:14.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:14 smithi191 ceph-mon[161669]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T06:06:14.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:14.740 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:14.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:14.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:14.842 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:14.874 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:14.924 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:14.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:14.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:15.032 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:15.068 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:15.132 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:15.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:15.240 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:15.271 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:15.324 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:15.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:15.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:15.389 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:06:15.430 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:15.490 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:15.536 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:15.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:15.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:15.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:15.664 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/87501544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:15.667 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:15.711 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:15.712 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:06:15.712 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:06:15.712 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:06:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:15.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:15.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/87501544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:15.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/87501544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:15.760 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:06:15.799 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:06:15.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:15.840 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:06:15.859 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:15.876 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:06:15.902 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:06:15.902 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:06:15.902 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:06:15.902 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:06:15.902 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:06:15.903 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:06:15.903 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:06:15.903 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:06:15.916 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:15.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:15.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:16.030 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:16.073 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:16.140 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:16.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:16.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:16.245 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:16.308 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:16.438 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:16.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:16.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:16.548 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:16.587 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:16.635 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:06:16.651 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:16.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:16.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:16.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:16 smithi008 ceph-mon[184418]: pgmap v1691: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 16047/119370 objects degraded (13.443%); 2814/119370 objects misplaced (2.357%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:06:16.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:16 smithi008 ceph-mon[191227]: pgmap v1691: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 16047/119370 objects degraded (13.443%); 2814/119370 objects misplaced (2.357%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:06:16.764 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:16.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:16 smithi191 ceph-mon[161669]: pgmap v1691: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 16047/119370 objects degraded (13.443%); 2814/119370 objects misplaced (2.357%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:06:16.802 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:16.859 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:16.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:16.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:16.934 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:16.966 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:17.058 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:17.482 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:17.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:17.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:17.597 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:17.745 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:17.959 INFO:teuthology.orchestra.run.smithi008.stdout:moved 3 / 267 (1.1236%) 2023-10-09T06:06:17.959 INFO:teuthology.orchestra.run.smithi008.stdout:avg 38.1429 2023-10-09T06:06:17.959 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 8.49249 -> 9.44803 (expected baseline 5.71786) 2023-10-09T06:06:17.959 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.0 with 25 -> 23 pgs (0.655431 -> 0.602996 * mean) 2023-10-09T06:06:17.959 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.1 with 51 -> 52 pgs (1.33708 -> 1.3633 * mean) 2023-10-09T06:06:17.959 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:06:17.960 INFO:teuthology.orchestra.run.smithi008.stdout:oload 110 2023-10-09T06:06:17.960 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 0.05 2023-10-09T06:06:17.960 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T06:06:17.960 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0240 2023-10-09T06:06:17.960 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.0263 2023-10-09T06:06:17.960 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0 weight 0.5845 -> 0.5345 2023-10-09T06:06:17.960 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4 weight 0.5592 -> 0.5092 2023-10-09T06:06:17.965 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:17.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:17.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:17.981 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T06:06:18.034 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:18.077 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:18.128 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:18.190 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:18.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:18.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:18.216 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:06:17 smithi008 conmon[177229]: 2023-10-09T06:06:17.949+0000 7f020285b700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T06:06:18.302 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:18.360 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:18.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:18 smithi008 ceph-mon[184418]: pgmap v1692: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 15794/116109 objects degraded (13.603%); 2814/116109 objects misplaced (2.424%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:06:18.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:18 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15794/116109 objects degraded (13.603%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:18.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:18 smithi008 ceph-mon[191227]: pgmap v1692: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 15794/116109 objects degraded (13.603%); 2814/116109 objects misplaced (2.424%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:06:18.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:18 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15794/116109 objects degraded (13.603%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:18.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:18 smithi191 ceph-mon[161669]: pgmap v1692: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 15794/116109 objects degraded (13.603%); 2814/116109 objects misplaced (2.424%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:06:18.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:18 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15794/116109 objects degraded (13.603%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:18.902 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:18.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:18.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:18.962 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:06:19.026 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:19.141 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:19.686 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:19.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:19.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:19.763 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:19 smithi008 ceph-mon[191227]: from='client.57121 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:19.763 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:19 smithi008 ceph-mon[184418]: from='client.57121 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:19.788 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:06:19.788 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-10-09T06:06:19.792 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:19.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:19 smithi191 ceph-mon[161669]: from='client.57121 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:19.953 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:20.251 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:20.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:20.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:20.358 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:20.395 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:20.445 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:20.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:20.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:20.536 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:06:20.537 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T06:06:20.537 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T06:06:20.537 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T06:06:20.537 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.537 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117310 9.f 9:fadc0f2a:::benchmark_data_smithi008_502820_object33137:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:06:20.537 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:57.240003+0000", 2023-10-09T06:06:20.537 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 503.29412224100003, 2023-10-09T06:06:20.537 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.29976151, 2023-10-09T06:06:20.537 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.538 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.538 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.538 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:20.538 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:20.538 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117310 2023-10-09T06:06:20.538 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.538 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.538 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.538 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.538 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240003+0000", 2023-10-09T06:06:20.538 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240003+0000", 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240001+0000", 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.539 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.540 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240012+0000", 2023-10-09T06:06:20.540 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0919e-05 2023-10-09T06:06:20.540 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.540 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.540 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.540 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:06:20.540 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3413000000000001e-05 2023-10-09T06:06:20.540 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.540 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.540 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.540 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240048+0000", 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2472e-05 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240128+0000", 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9445999999999998e-05 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op must wait for map", 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240136+0000", 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4519999999999997e-06 2023-10-09T06:06:20.541 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.256039+0000", 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015902785999999999 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.256052+0000", 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3036e-05 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.542 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.412425+0000", 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1563727260000001 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.415543+0000", 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031188029999999999 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.416007+0000", 2023-10-09T06:06:20.543 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00046389599999999999 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.505488+0000", 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.089480572999999994 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.537229+0000", 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.031741577 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.544 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.537272+0000", 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3052e-05 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.537346+0000", 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3397000000000002e-05 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.539765+0000", 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00241888 2023-10-09T06:06:20.545 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.546 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.546 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.546 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.546 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.546 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117318 9.f 9:f9d15496:::benchmark_data_smithi008_502820_object33145:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:06:20.546 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:57.240063+0000", 2023-10-09T06:06:20.546 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 503.29406222599999, 2023-10-09T06:06:20.546 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3088910030000001, 2023-10-09T06:06:20.546 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.546 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.546 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117318 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240063+0000", 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.547 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240063+0000", 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240063+0000", 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240066+0000", 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2619999999999999e-06 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.548 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240073+0000", 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7610000000000002e-06 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240078+0000", 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6710000000000004e-06 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240196+0000", 2023-10-09T06:06:20.549 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011796899999999999 2023-10-09T06:06:20.550 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.550 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.550 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting_for_map not empty", 2023-10-09T06:06:20.550 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.240203+0000", 2023-10-09T06:06:20.550 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4620000000000001e-06 2023-10-09T06:06:20.550 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.550 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.550 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.550 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.256459+0000", 2023-10-09T06:06:20.550 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016256135000000001 2023-10-09T06:06:20.550 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.256468+0000", 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2180000000000005e-06 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.539808+0000", 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.283339472 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.551 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.552 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.539927+0000", 2023-10-09T06:06:20.552 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011948 2023-10-09T06:06:20.552 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.552 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.552 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:06:20.552 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.540039+0000", 2023-10-09T06:06:20.552 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000112358 2023-10-09T06:06:20.552 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.552 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.552 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.552 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.542135+0000", 2023-10-09T06:06:20.553 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:20.554 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020953310000000002 2023-10-09T06:06:20.554 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.554 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.554 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.554 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.543074+0000", 2023-10-09T06:06:20.554 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00093911000000000005 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.546321+0000", 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032469949999999999 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.546365+0000", 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3677000000000002e-05 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.555 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.556 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.556 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.548954+0000", 2023-10-09T06:06:20.556 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025896270000000002 2023-10-09T06:06:20.556 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.556 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.556 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.556 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.556 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.556 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135296 9.15 9:ae091abe:::benchmark_data_smithi008_502820_object51123:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e477)", 2023-10-09T06:06:20.563 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:06.289091+0000", 2023-10-09T06:06:20.563 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 374.24503492000002, 2023-10-09T06:06:20.563 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 11.558454653, 2023-10-09T06:06:20.563 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135296 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289091+0000", 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.564 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289091+0000", 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289088+0000", 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999976 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.565 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289100+0000", 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.208e-05 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289120+0000", 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9913999999999999e-05 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289135+0000", 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4450000000000001e-05 2023-10-09T06:06:20.566 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289222+0000", 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7608999999999994e-05 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:06.289238+0000", 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6174000000000001e-05 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.567 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.794145+0000", 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 11.504906586000001 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.796652+0000", 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025069179999999999 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.801845+0000", 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051928349999999998 2023-10-09T06:06:20.568 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.814352+0000", 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012507364999999999 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.816332+0000", 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019798580000000001 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.569 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.570 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.841671+0000", 2023-10-09T06:06:20.570 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.025339265 2023-10-09T06:06:20.570 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.570 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.570 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.570 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.842816+0000", 2023-10-09T06:06:20.570 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011452649999999999 2023-10-09T06:06:20.570 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.571 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.571 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.571 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.847545+0000", 2023-10-09T06:06:20.571 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0047286979999999999 2023-10-09T06:06:20.571 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.571 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.571 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.571 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.571 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.571 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135428 9.5 9:a3b356e8:::benchmark_data_smithi008_502820_object51255:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:06:20.571 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.442799+0000", 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 363.09132632000001, 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.46811048300000002, 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135428 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.572 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442799+0000", 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442799+0000", 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442798+0000", 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:20.573 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442805+0000", 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8600000000000004e-06 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442821+0000", 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6042e-05 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.574 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.442829+0000", 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1389999999999995e-06 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.443441+0000", 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000612033 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.443654+0000", 2023-10-09T06:06:20.575 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000213434 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.832185+0000", 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.38853078499999999 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.834228+0000", 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020425539999999998 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.576 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:06:20.577 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.837657+0000", 2023-10-09T06:06:20.577 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034290000000000002 2023-10-09T06:06:20.577 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.577 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.577 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.577 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.848361+0000", 2023-10-09T06:06:20.577 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010704959999999999 2023-10-09T06:06:20.577 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.577 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.577 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.577 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.866209+0000", 2023-10-09T06:06:20.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017847569000000001 2023-10-09T06:06:20.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.578 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.905664+0000", 2023-10-09T06:06:20.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.039454522999999998 2023-10-09T06:06:20.578 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.578 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.578 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.578 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.906752+0000", 2023-10-09T06:06:20.578 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001088807 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.910910+0000", 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041572320000000003 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135438 9.15 9:ac71c127:::benchmark_data_smithi008_502820_object51265:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:06:20.579 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.467551+0000", 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 363.06657439399999, 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.424989799, 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135438 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.580 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467551+0000", 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467551+0000", 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467550+0000", 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:06:20.581 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.582 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.582 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.582 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467555+0000", 2023-10-09T06:06:20.582 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3240000000000002e-06 2023-10-09T06:06:20.582 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.582 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.582 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.582 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467566+0000", 2023-10-09T06:06:20.582 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0946e-05 2023-10-09T06:06:20.582 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.582 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.467572+0000", 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6130000000000003e-06 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.468100+0000", 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052845400000000003 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.468325+0000", 2023-10-09T06:06:20.583 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00022488200000000001 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.816574+0000", 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.34824860000000002 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.819442+0000", 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002867882 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.584 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.824069+0000", 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0046273820000000002 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.829908+0000", 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0058389480000000001 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.847904+0000", 2023-10-09T06:06:20.585 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017996504 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.887093+0000", 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.039188134999999999 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.888018+0000", 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00092505699999999996 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.586 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.892541+0000", 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0045231919999999997 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135455 9.5 9:a744df51:::benchmark_data_smithi008_502820_object51282:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.555805+0000", 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 362.97832055600003, 2023-10-09T06:06:20.587 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.36574448700000001, 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135455 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555805+0000", 2023-10-09T06:06:20.588 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555805+0000", 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555804+0000", 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.589 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.590 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555809+0000", 2023-10-09T06:06:20.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7189999999999998e-06 2023-10-09T06:06:20.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.590 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.590 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555822+0000", 2023-10-09T06:06:20.590 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2437e-05 2023-10-09T06:06:20.590 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.590 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.590 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.593 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.555828+0000", 2023-10-09T06:06:20.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9009999999999999e-06 2023-10-09T06:06:20.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.594 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.556356+0000", 2023-10-09T06:06:20.594 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052816600000000005 2023-10-09T06:06:20.594 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.594 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.594 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:20.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.556533+0000", 2023-10-09T06:06:20.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017680899999999999 2023-10-09T06:06:20.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.595 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.848754+0000", 2023-10-09T06:06:20.595 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29222136599999998 2023-10-09T06:06:20.595 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.595 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.595 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.596 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.850864+0000", 2023-10-09T06:06:20.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021101840000000002 2023-10-09T06:06:20.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.596 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:06:20.596 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.854254+0000", 2023-10-09T06:06:20.596 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003389909 2023-10-09T06:06:20.596 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.596 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.597 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.597 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.866794+0000", 2023-10-09T06:06:20.597 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012540112000000001 2023-10-09T06:06:20.597 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.597 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.597 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.597 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.886075+0000", 2023-10-09T06:06:20.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019280761 2023-10-09T06:06:20.598 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.598 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.598 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.598 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.916939+0000", 2023-10-09T06:06:20.598 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030863607000000001 2023-10-09T06:06:20.599 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.599 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.599 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.917782+0000", 2023-10-09T06:06:20.599 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00084381300000000001 2023-10-09T06:06:20.599 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.599 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.600 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.600 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.921549+0000", 2023-10-09T06:06:20.600 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003766892 2023-10-09T06:06:20.600 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.600 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.600 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.600 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.601 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.601 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135464 9.5 9:a4c9db9f:::benchmark_data_smithi008_502820_object51291:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:06:20.601 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.581489+0000", 2023-10-09T06:06:20.601 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 362.952636934, 2023-10-09T06:06:20.601 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35698092199999998, 2023-10-09T06:06:20.601 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.601 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.602 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.602 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:20.602 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:20.602 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135464 2023-10-09T06:06:20.602 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.602 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.603 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581489+0000", 2023-10-09T06:06:20.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.603 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.603 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.603 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581489+0000", 2023-10-09T06:06:20.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581487+0000", 2023-10-09T06:06:20.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:20.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581494+0000", 2023-10-09T06:06:20.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8299999999999998e-06 2023-10-09T06:06:20.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581506+0000", 2023-10-09T06:06:20.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2165e-05 2023-10-09T06:06:20.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.581513+0000", 2023-10-09T06:06:20.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5320000000000003e-06 2023-10-09T06:06:20.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.582012+0000", 2023-10-09T06:06:20.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00049926799999999998 2023-10-09T06:06:20.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:20.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.582190+0000", 2023-10-09T06:06:20.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017807399999999999 2023-10-09T06:06:20.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.857269+0000", 2023-10-09T06:06:20.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27507849099999998 2023-10-09T06:06:20.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.859832+0000", 2023-10-09T06:06:20.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0025629759999999998 2023-10-09T06:06:20.610 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:06:20.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.863254+0000", 2023-10-09T06:06:20.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034221020000000002 2023-10-09T06:06:20.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.867308+0000", 2023-10-09T06:06:20.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040540230000000003 2023-10-09T06:06:20.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.612 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.612 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.886247+0000", 2023-10-09T06:06:20.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018939448000000001 2023-10-09T06:06:20.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.933550+0000", 2023-10-09T06:06:20.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.047303167 2023-10-09T06:06:20.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.934616+0000", 2023-10-09T06:06:20.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001065489 2023-10-09T06:06:20.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.938469+0000", 2023-10-09T06:06:20.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038536320000000001 2023-10-09T06:06:20.615 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.615 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.615 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.615 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135469 9.5 9:a4af4507:::benchmark_data_smithi008_502820_object51296:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:06:20.615 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.593956+0000", 2023-10-09T06:06:20.615 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 362.94016908399999, 2023-10-09T06:06:20.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35374935200000002, 2023-10-09T06:06:20.616 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.616 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.616 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.616 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:20.616 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:20.616 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135469 2023-10-09T06:06:20.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.617 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593956+0000", 2023-10-09T06:06:20.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.618 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.618 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593956+0000", 2023-10-09T06:06:20.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593955+0000", 2023-10-09T06:06:20.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:06:20.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593961+0000", 2023-10-09T06:06:20.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1500000000000004e-06 2023-10-09T06:06:20.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593973+0000", 2023-10-09T06:06:20.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2228e-05 2023-10-09T06:06:20.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.593979+0000", 2023-10-09T06:06:20.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0220000000000001e-06 2023-10-09T06:06:20.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.594482+0000", 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00050292100000000003 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.594661+0000", 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017859899999999999 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.867482+0000", 2023-10-09T06:06:20.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.27282152999999998 2023-10-09T06:06:20.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.869478+0000", 2023-10-09T06:06:20.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019958799999999998 2023-10-09T06:06:20.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.622 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:06:20.622 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.872688+0000", 2023-10-09T06:06:20.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032103909999999999 2023-10-09T06:06:20.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.886833+0000", 2023-10-09T06:06:20.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014144818 2023-10-09T06:06:20.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.893123+0000", 2023-10-09T06:06:20.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0062893610000000003 2023-10-09T06:06:20.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.939942+0000", 2023-10-09T06:06:20.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.046819500999999999 2023-10-09T06:06:20.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.941017+0000", 2023-10-09T06:06:20.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001075211 2023-10-09T06:06:20.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.947706+0000", 2023-10-09T06:06:20.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0066884229999999998 2023-10-09T06:06:20.626 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.626 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.626 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.627 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:135471 9.5 9:a1af57c1:::benchmark_data_smithi008_502820_object51298:head [delete] snapc 0=[] ondisk+write+known_if_redirected e480)", 2023-10-09T06:06:20.627 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:00:17.615863+0000", 2023-10-09T06:06:20.627 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 362.91826289199997, 2023-10-09T06:06:20.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33742538799999999, 2023-10-09T06:06:20.627 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.628 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.628 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.628 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:20.628 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:20.628 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 135471 2023-10-09T06:06:20.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.628 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615863+0000", 2023-10-09T06:06:20.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615863+0000", 2023-10-09T06:06:20.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615861+0000", 2023-10-09T06:06:20.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:20.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.631 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.631 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615868+0000", 2023-10-09T06:06:20.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7599999999999997e-06 2023-10-09T06:06:20.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615880+0000", 2023-10-09T06:06:20.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2174999999999999e-05 2023-10-09T06:06:20.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.615888+0000", 2023-10-09T06:06:20.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.525e-06 2023-10-09T06:06:20.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.876726+0000", 2023-10-09T06:06:20.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26083806399999998 2023-10-09T06:06:20.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.879008+0000", 2023-10-09T06:06:20.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022821320000000001 2023-10-09T06:06:20.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:06:20.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.882350+0000", 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033419690000000002 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.887338+0000", 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049886599999999998 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.893331+0000", 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0059928400000000001 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.948636+0000", 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.055305193000000002 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.949599+0000", 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00096221399999999997 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:00:17.953288+0000", 2023-10-09T06:06:20.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036893070000000002 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:153220 9.15 9:ac0b5463:::benchmark_data_smithi008_502820_object69047:head [delete] snapc 0=[] ondisk+write+known_if_redirected e495)", 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:01:28.845935+0000", 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 291.68819027699999, 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33700997300000002, 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.637 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:20.638 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:20.638 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 153220 2023-10-09T06:06:20.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.638 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.845935+0000", 2023-10-09T06:06:20.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.845935+0000", 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.845934+0000", 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.845940+0000", 2023-10-09T06:06:20.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4190000000000001e-06 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.845948+0000", 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5799999999999992e-06 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.846112+0000", 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016346700000000001 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.846185+0000", 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3405999999999995e-05 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.846286+0000", 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010111300000000001 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:06:20.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.846679+0000", 2023-10-09T06:06:20.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00039266000000000003 2023-10-09T06:06:20.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.848724+0000", 2023-10-09T06:06:20.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020446890000000001 2023-10-09T06:06:20.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:28.849533+0000", 2023-10-09T06:06:20.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00080934699999999997 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:29.182412+0000", 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33287920199999999 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:29.182471+0000", 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8455000000000002e-05 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:01:29.182945+0000", 2023-10-09T06:06:20.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000474625 2023-10-09T06:06:20.644 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.644 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.644 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.644 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:166957 9.8 9.f7ad5128 (undecoded) ondisk+write+known_if_redirected e504)", 2023-10-09T06:06:20.644 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:02:07.989499+0000", 2023-10-09T06:06:20.644 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 252.54462595300001, 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33934386999999999, 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 166957 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989499+0000", 2023-10-09T06:06:20.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989499+0000", 2023-10-09T06:06:20.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989497+0000", 2023-10-09T06:06:20.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:06:20.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989508+0000", 2023-10-09T06:06:20.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0900999999999999e-05 2023-10-09T06:06:20.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989533+0000", 2023-10-09T06:06:20.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4547999999999999e-05 2023-10-09T06:06:20.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.989546+0000", 2023-10-09T06:06:20.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3089e-05 2023-10-09T06:06:20.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.994259+0000", 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0047132529999999997 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:07.994272+0000", 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3169999999999999e-05 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:08.328828+0000", 2023-10-09T06:06:20.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33455541799999999 2023-10-09T06:06:20.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:02:08.328843+0000", 2023-10-09T06:06:20.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5511000000000001e-05 2023-10-09T06:06:20.654 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.654 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.654 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:63139 10.0 10:04e05b18:::benchmark_data_smithi008_634586_object63138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:19.662902+0000", 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 180.87122328500001, 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.32210230099999998, 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 63139 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.655 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662902+0000", 2023-10-09T06:06:20.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662902+0000", 2023-10-09T06:06:20.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662902+0000", 2023-10-09T06:06:20.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:20.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662925+0000", 2023-10-09T06:06:20.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3309e-05 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662932+0000", 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8290000000000002e-06 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662936+0000", 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.2729999999999996e-06 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.969240+0000", 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.306303885 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.969387+0000", 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000147045 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.969516+0000", 2023-10-09T06:06:20.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012888400000000001 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.977219+0000", 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0077028369999999997 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.982574+0000", 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005355355 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.982632+0000", 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8121000000000001e-05 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.982689+0000", 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6557000000000001e-05 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.985004+0000", 2023-10-09T06:06:20.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002315774 2023-10-09T06:06:20.662 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.662 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.662 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.662 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:63140 10.0 10:083e2f66:::benchmark_data_smithi008_634586_object63139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:06:20.662 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:19.662941+0000", 2023-10-09T06:06:20.662 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 180.87118460100001, 2023-10-09T06:06:20.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.32458843900000001, 2023-10-09T06:06:20.662 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 63140 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662941+0000", 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662941+0000", 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662941+0000", 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662957+0000", 2023-10-09T06:06:20.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6189999999999999e-05 2023-10-09T06:06:20.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662961+0000", 2023-10-09T06:06:20.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3409999999999996e-06 2023-10-09T06:06:20.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.662964+0000", 2023-10-09T06:06:20.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2499999999999998e-06 2023-10-09T06:06:20.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.971739+0000", 2023-10-09T06:06:20.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.30877492499999998 2023-10-09T06:06:20.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.666 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.666 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.971940+0000", 2023-10-09T06:06:20.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000200454 2023-10-09T06:06:20.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:06:20.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.972155+0000", 2023-10-09T06:06:20.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00021504299999999999 2023-10-09T06:06:20.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.977263+0000", 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0051080709999999996 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.985094+0000", 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078314360000000006 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.985107+0000", 2023-10-09T06:06:20.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3104e-05 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.985194+0000", 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7143000000000002e-05 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:19.987529+0000", 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0023347950000000002 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.669 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.47036.0:83465 10.4 e526/520 10:236eb373:::benchmark_data_smithi008_634586_object83464:head v 526'5078, mlcod=0'0)", 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:03:49.961347+0000", 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 150.57277891800001, 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26935263100000001, 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:49.961347+0000", 2023-10-09T06:06:20.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:49.961347+0000", 2023-10-09T06:06:20.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:49.961346+0000", 2023-10-09T06:06:20.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:06:20.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:49.961490+0000", 2023-10-09T06:06:20.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014486800000000001 2023-10-09T06:06:20.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:49.961505+0000", 2023-10-09T06:06:20.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4072e-05 2023-10-09T06:06:20.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:49.961512+0000", 2023-10-09T06:06:20.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3710000000000002e-06 2023-10-09T06:06:20.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:49.963741+0000", 2023-10-09T06:06:20.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022288870000000001 2023-10-09T06:06:20.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:49.963763+0000", 2023-10-09T06:06:20.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2124e-05 2023-10-09T06:06:20.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:50.230658+0000", 2023-10-09T06:06:20.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.266895347 2023-10-09T06:06:20.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.675 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.675 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:03:50.230699+0000", 2023-10-09T06:06:20.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0861000000000001e-05 2023-10-09T06:06:20.676 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.676 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.676 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.676 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:106717 10.b 10:d4341d20:::benchmark_data_smithi008_634586_object19552:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:06:20.676 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:04:59.775437+0000", 2023-10-09T06:06:20.676 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 80.758688550000002, 2023-10-09T06:06:20.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.22605266500000001, 2023-10-09T06:06:20.677 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.677 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.677 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.677 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:20.677 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:20.677 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 106717 2023-10-09T06:06:20.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.678 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775437+0000", 2023-10-09T06:06:20.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775437+0000", 2023-10-09T06:06:20.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775436+0000", 2023-10-09T06:06:20.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:20.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775440+0000", 2023-10-09T06:06:20.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2279999999999998e-06 2023-10-09T06:06:20.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775446+0000", 2023-10-09T06:06:20.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5309999999999998e-06 2023-10-09T06:06:20.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775451+0000", 2023-10-09T06:06:20.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1930000000000001e-06 2023-10-09T06:06:20.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775578+0000", 2023-10-09T06:06:20.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012634699999999999 2023-10-09T06:06:20.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775709+0000", 2023-10-09T06:06:20.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013135999999999999 2023-10-09T06:06:20.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.684 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:06:20.684 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775837+0000", 2023-10-09T06:06:20.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012809200000000001 2023-10-09T06:06:20.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.783076+0000", 2023-10-09T06:06:20.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0072391929999999997 2023-10-09T06:06:20.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.783099+0000", 2023-10-09T06:06:20.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2688e-05 2023-10-09T06:06:20.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.790677+0000", 2023-10-09T06:06:20.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0075776089999999999 2023-10-09T06:06:20.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.790732+0000", 2023-10-09T06:06:20.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5614e-05 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001490+0000", 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.21075724200000001 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:106719 10.b 10:dc12a0b7:::benchmark_data_smithi008_634586_object19554:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:04:59.775465+0000", 2023-10-09T06:06:20.688 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 80.758660302999999, 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.22648402300000001, 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 106719 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775465+0000", 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775465+0000", 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775465+0000", 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:20.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775468+0000", 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0149999999999999e-06 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775474+0000", 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.9179999999999999e-06 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775479+0000", 2023-10-09T06:06:20.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0660000000000003e-06 2023-10-09T06:06:20.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.779615+0000", 2023-10-09T06:06:20.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041366229999999999 2023-10-09T06:06:20.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.779746+0000", 2023-10-09T06:06:20.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013059399999999999 2023-10-09T06:06:20.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:06:20.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.779874+0000", 2023-10-09T06:06:20.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012812300000000001 2023-10-09T06:06:20.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.783184+0000", 2023-10-09T06:06:20.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033095339999999998 2023-10-09T06:06:20.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.789472+0000", 2023-10-09T06:06:20.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0062885980000000003 2023-10-09T06:06:20.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001785+0000", 2023-10-09T06:06:20.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.212312854 2023-10-09T06:06:20.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001829+0000", 2023-10-09T06:06:20.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4029999999999997e-05 2023-10-09T06:06:20.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001949+0000", 2023-10-09T06:06:20.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00012003299999999999 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:106721 10.2 10:4d63a1e3:::benchmark_data_smithi008_634586_object19556:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:04:59.775483+0000", 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 80.758642330000001, 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.22630935899999999, 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.696 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.697 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:20.697 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:20.697 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 106721 2023-10-09T06:06:20.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.697 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775483+0000", 2023-10-09T06:06:20.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775483+0000", 2023-10-09T06:06:20.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775483+0000", 2023-10-09T06:06:20.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:20.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775485+0000", 2023-10-09T06:06:20.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2879999999999999e-06 2023-10-09T06:06:20.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775490+0000", 2023-10-09T06:06:20.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2970000000000003e-06 2023-10-09T06:06:20.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.775495+0000", 2023-10-09T06:06:20.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6240000000000001e-06 2023-10-09T06:06:20.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.782220+0000", 2023-10-09T06:06:20.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.006724697 2023-10-09T06:06:20.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.782356+0000", 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000136501 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.782477+0000", 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000120283 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.787402+0000", 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0049253109999999999 2023-10-09T06:06:20.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.790635+0000", 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003232829 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001698+0000", 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.21106379 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001749+0000", 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0899000000000002e-05 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001792+0000", 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3137999999999999e-05 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.47036.0:106723 10.1 e543/528 10:8d75b5a8:::benchmark_data_smithi008_634586_object19558:head v 543'6531, mlcod=0'0)", 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:04:59.782399+0000", 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 80.751726156999993, 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.22227101799999999, 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.782399+0000", 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.782399+0000", 2023-10-09T06:06:20.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.782399+0000", 2023-10-09T06:06:20.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:20.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.782405+0000", 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7209999999999999e-06 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.782411+0000", 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.545e-06 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.782416+0000", 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.7899999999999999e-06 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.782500+0000", 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4264000000000005e-05 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.782527+0000", 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6261000000000001e-05 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.799226+0000", 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016699259000000001 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.004670+0000", 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.205444495 2023-10-09T06:06:20.708 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_repop(client.47036.0:106725 10.6 e543/512 10:6c78e38f:::benchmark_data_smithi008_634586_object19560:head v 543'6658, mlcod=543'6656)", 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:04:59.786723+0000", 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 80.747402077000004, 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.21506362400000001, 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.786723+0000", 2023-10-09T06:06:20.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.710 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.786723+0000", 2023-10-09T06:06:20.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.710 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.786723+0000", 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.786730+0000", 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6699999999999994e-06 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.786742+0000", 2023-10-09T06:06:20.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1280999999999999e-05 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.786749+0000", 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3529999999999997e-06 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.786809+0000", 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0186999999999998e-05 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.786829+0000", 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9949000000000002e-05 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.799347+0000", 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012518118999999999 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001787+0000", 2023-10-09T06:06:20.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.202439854 2023-10-09T06:06:20.714 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.714 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.714 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.714 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:106727 10.2 10:49a6c67a:::benchmark_data_smithi008_634586_object19562:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:06:20.714 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:04:59.790995+0000", 2023-10-09T06:06:20.714 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 80.743130863000005, 2023-10-09T06:06:20.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.221308015, 2023-10-09T06:06:20.714 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:20.714 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 106727 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.790995+0000", 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.790995+0000", 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.790993+0000", 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:04:59.790999+0000", 2023-10-09T06:06:20.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3630000000000002e-06 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001462+0000", 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.210463343 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001478+0000", 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6376e-05 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001820+0000", 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00034174100000000001 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.001934+0000", 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000113861 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.002047+0000", 2023-10-09T06:06:20.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011291299999999999 2023-10-09T06:06:20.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:20.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.009055+0000", 2023-10-09T06:06:20.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0070079160000000003 2023-10-09T06:06:20.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.719 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.719 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.011722+0000", 2023-10-09T06:06:20.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026670190000000001 2023-10-09T06:06:20.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.012209+0000", 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00048696600000000003 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.012259+0000", 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0167999999999998e-05 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:20.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:20.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:00.012303+0000", 2023-10-09T06:06:20.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.3519000000000001e-05 2023-10-09T06:06:20.721 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.721 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.721 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.721 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:20.721 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:20.721 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:06:20.791 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:20 smithi008 ceph-mon[184418]: pgmap v1693: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 15794/115299 objects degraded (13.698%); 2664/115299 objects misplaced (2.311%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:06:20.792 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:06:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:06:20] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.33.4" 2023-10-09T06:06:20.792 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:20 smithi008 ceph-mon[191227]: pgmap v1693: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 15794/115299 objects degraded (13.698%); 2664/115299 objects misplaced (2.311%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:06:20.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:20 smithi191 ceph-mon[161669]: pgmap v1693: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 15794/115299 objects degraded (13.698%); 2664/115299 objects misplaced (2.311%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:06:20.800 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:21.913 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:21.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:21.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:22.027 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:22.044 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:06:22.078 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:22.120 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:22.152 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:22.359 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:22.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:22.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:22.463 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:22.497 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:22.557 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:22.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:22.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:22.663 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:22.751 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:22 smithi008 ceph-mon[184418]: pgmap v1694: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 15794/112992 objects degraded (13.978%); 2550/112992 objects misplaced (2.257%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:06:22.752 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:22 smithi008 ceph-mon[191227]: pgmap v1694: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 15794/112992 objects degraded (13.978%); 2550/112992 objects misplaced (2.257%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:06:22.761 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:22.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:22 smithi191 ceph-mon[161669]: pgmap v1694: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 15794/112992 objects degraded (13.978%); 2550/112992 objects misplaced (2.257%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:06:23.078 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:23.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:23.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:23.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:23.225 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:23.285 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:23.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:23.395 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:23.438 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:23.513 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:23.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:23.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:23.534 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:06:23.621 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:23.655 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:23.718 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4140986457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:23.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:23 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:23.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15794/112992 objects degraded (13.978%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:23.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4140986457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:23.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:23 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:23.722 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15794/112992 objects degraded (13.978%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:23.728 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:23.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:23.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:23.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4140986457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:23.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:23 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:23.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15794/112992 objects degraded (13.978%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:23.842 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:23.936 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:24.699 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:24.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:24.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:24 smithi191 ceph-mon[161669]: pgmap v1695: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 15492/108222 objects degraded (14.315%); 2392/108222 objects misplaced (2.210%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:06:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:24 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:24 smithi191 ceph-mon[161669]: failed to encode map e558 with expected crc 2023-10-09T06:06:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:24 smithi191 ceph-mon[161669]: failed to encode map e558 with expected crc 2023-10-09T06:06:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:24 smithi191 ceph-mon[161669]: failed to encode map e558 with expected crc 2023-10-09T06:06:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:24 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:06:24.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:24 smithi191 ceph-mon[161669]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T06:06:24.807 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:24.850 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[184418]: pgmap v1695: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 15492/108222 objects degraded (14.315%); 2392/108222 objects misplaced (2.210%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:06:24.850 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:24.850 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[184418]: failed to encode map e558 with expected crc 2023-10-09T06:06:24.850 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[184418]: failed to encode map e558 with expected crc 2023-10-09T06:06:24.850 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[184418]: failed to encode map e558 with expected crc 2023-10-09T06:06:24.850 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:06:24.850 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[184418]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T06:06:24.850 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[191227]: pgmap v1695: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 15492/108222 objects degraded (14.315%); 2392/108222 objects misplaced (2.210%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:06:24.851 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:24.851 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[191227]: failed to encode map e558 with expected crc 2023-10-09T06:06:24.851 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[191227]: failed to encode map e558 with expected crc 2023-10-09T06:06:24.851 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[191227]: failed to encode map e558 with expected crc 2023-10-09T06:06:24.851 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:06:24.851 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:24 smithi008 ceph-mon[191227]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T06:06:24.852 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:25.272 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:25.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:25.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:25.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:25.422 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:25.488 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:25.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:25.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:25.535 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:25.597 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:25.735 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:25.869 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:25.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:25.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:25.977 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:26.072 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:26.124 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:06:26.179 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:06:26.242 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:06:26.256 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:26.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:26.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:26.281 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:06:26.319 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:06:26.372 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:06:26.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:26.410 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:26.428 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:06:26.464 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:06:26.478 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:26.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:26.485 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:06:26.485 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:06:26.486 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:06:26.486 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:06:26.486 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:06:26.486 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:06:26.486 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:06:26.486 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:06:26.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:26.616 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:26.674 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:26.679 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:26 smithi191 ceph-mon[161669]: pgmap v1697: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 15492/105975 objects degraded (14.619%); 2282/105975 objects misplaced (2.153%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:06:26.679 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3190440038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:26.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:26.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:26.781 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:26.793 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:26 smithi008 ceph-mon[184418]: pgmap v1697: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 15492/105975 objects degraded (14.619%); 2282/105975 objects misplaced (2.153%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:06:26.794 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3190440038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:26.794 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:26 smithi008 ceph-mon[191227]: pgmap v1697: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 15492/105975 objects degraded (14.619%); 2282/105975 objects misplaced (2.153%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:06:26.794 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3190440038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:26.883 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:26.932 INFO:teuthology.orchestra.run.smithi008.stdout:moved 0 / 267 (0%) 2023-10-09T06:06:26.932 INFO:teuthology.orchestra.run.smithi008.stdout:avg 38.1429 2023-10-09T06:06:26.932 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 8.49249 -> 8.49249 (expected baseline 5.71786) 2023-10-09T06:06:26.932 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.0 with 25 -> 25 pgs (0.655431 -> 0.655431 * mean) 2023-10-09T06:06:26.932 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.1 with 51 -> 51 pgs (1.33708 -> 1.33708 * mean) 2023-10-09T06:06:26.932 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:06:26.932 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T06:06:26.932 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 3 2023-10-09T06:06:26.932 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T06:06:26.933 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0228 2023-10-09T06:06:26.933 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.2279 2023-10-09T06:06:26.959 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T06:06:27.366 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:06:27.921 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:27.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:27.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:28.029 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:28.056 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:28.120 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:28.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:28.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:28.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:28.271 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:28.331 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:28.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:28.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:28.443 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:28.552 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:28 smithi191 ceph-mon[161669]: pgmap v1698: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 15179/101913 objects degraded (14.894%); 2282/101913 objects misplaced (2.239%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:06:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:28 smithi191 ceph-mon[161669]: from='client.57133 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15179/101913 objects degraded (14.894%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:28.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:28 smithi008 ceph-mon[184418]: pgmap v1698: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 15179/101913 objects degraded (14.894%); 2282/101913 objects misplaced (2.239%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:06:28.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:28 smithi008 ceph-mon[184418]: from='client.57133 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:28.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15179/101913 objects degraded (14.894%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:28.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:28 smithi008 ceph-mon[191227]: pgmap v1698: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 15179/101913 objects degraded (14.894%); 2282/101913 objects misplaced (2.239%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:06:28.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:28 smithi008 ceph-mon[191227]: from='client.57133 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:28.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15179/101913 objects degraded (14.894%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:29.184 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:29.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:29.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:29.290 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:29.335 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:06:29.335 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-10-09T06:06:29.431 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:29.705 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:29.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:29.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:29.789 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:29.824 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:29.926 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:06:29.926 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:06:29.949 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:29.949 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:29.967 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:30.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:30.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:30 smithi008 ceph-mon[184418]: pgmap v1699: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 15179/101067 objects degraded (15.019%); 2101/101067 objects misplaced (2.079%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T06:06:30.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:30 smithi008 ceph-mon[191227]: pgmap v1699: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 15179/101067 objects degraded (15.019%); 2101/101067 objects misplaced (2.079%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T06:06:30.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:06:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:06:30] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.33.4" 2023-10-09T06:06:31.029 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:06:31.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:30 smithi191 ceph-mon[161669]: pgmap v1699: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 15179/101067 objects degraded (15.019%); 2101/101067 objects misplaced (2.079%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T06:06:31.276 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:31.286 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:06:31.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:31.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:31.398 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:31.508 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:31.808 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:31.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:31.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:31.913 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:32.004 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:32.454 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:32.479 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:32.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:32.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:32.595 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:32.629 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:32.698 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:32.702 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:32.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:32 smithi008 ceph-mon[184418]: pgmap v1700: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 15189/98739 objects degraded (15.383%); 1978/98739 objects misplaced (2.003%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T06:06:32.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:32 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:32.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:32 smithi008 ceph-mon[191227]: pgmap v1700: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 15189/98739 objects degraded (15.383%); 1978/98739 objects misplaced (2.003%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T06:06:32.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:32 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:32.735 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:32.815 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:32.910 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:32 smithi191 ceph-mon[161669]: pgmap v1700: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 15189/98739 objects degraded (15.383%); 1978/98739 objects misplaced (2.003%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T06:06:32.910 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:32 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:32.914 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:32.984 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:32.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:32.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:33.095 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:33.157 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:33.270 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:33.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:33.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:33.376 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:33.487 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:33.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 15189/98739 objects degraded (15.383%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:33.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:33 smithi008 ceph-mon[184418]: failed to encode map e559 with expected crc 2023-10-09T06:06:33.724 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:33 smithi008 ceph-mon[184418]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T06:06:33.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 15189/98739 objects degraded (15.383%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:33.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:33 smithi008 ceph-mon[191227]: failed to encode map e559 with expected crc 2023-10-09T06:06:33.724 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:33 smithi008 ceph-mon[191227]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T06:06:34.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 15189/98739 objects degraded (15.383%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:33 smithi191 ceph-mon[161669]: failed to encode map e559 with expected crc 2023-10-09T06:06:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:33 smithi191 ceph-mon[161669]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T06:06:34.435 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:06:34.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[184418]: pgmap v1701: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 14842/93378 objects degraded (15.895%); 1878/93378 objects misplaced (2.011%); 7.3 MiB/s, 147 objects/s recovering 2023-10-09T06:06:34.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[184418]: failed to encode map e559 with expected crc 2023-10-09T06:06:34.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[184418]: failed to encode map e559 with expected crc 2023-10-09T06:06:34.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2347564747' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:34.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:06:34.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[184418]: failed to encode map e560 with expected crc 2023-10-09T06:06:34.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2347564747' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:34.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[184418]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T06:06:34.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[191227]: pgmap v1701: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 14842/93378 objects degraded (15.895%); 1878/93378 objects misplaced (2.011%); 7.3 MiB/s, 147 objects/s recovering 2023-10-09T06:06:34.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[191227]: failed to encode map e559 with expected crc 2023-10-09T06:06:34.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[191227]: failed to encode map e559 with expected crc 2023-10-09T06:06:34.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2347564747' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:34.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:06:34.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[191227]: failed to encode map e560 with expected crc 2023-10-09T06:06:34.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2347564747' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:34.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:34 smithi008 ceph-mon[191227]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T06:06:34.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:34 smithi191 ceph-mon[161669]: pgmap v1701: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 14842/93378 objects degraded (15.895%); 1878/93378 objects misplaced (2.011%); 7.3 MiB/s, 147 objects/s recovering 2023-10-09T06:06:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:34 smithi191 ceph-mon[161669]: failed to encode map e559 with expected crc 2023-10-09T06:06:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:34 smithi191 ceph-mon[161669]: failed to encode map e559 with expected crc 2023-10-09T06:06:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2347564747' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:34 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:06:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:34 smithi191 ceph-mon[161669]: failed to encode map e560 with expected crc 2023-10-09T06:06:34.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2347564747' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:34.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:34 smithi191 ceph-mon[161669]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T06:06:34.941 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:34.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:34.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:35.046 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:35.138 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:35.925 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:35 smithi008 ceph-mon[184418]: failed to encode map e560 with expected crc 2023-10-09T06:06:35.925 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:35 smithi008 ceph-mon[184418]: pgmap v1704: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 14879/91263 objects degraded (16.303%); 1729/91263 objects misplaced (1.895%); 6.7 MiB/s, 145 objects/s recovering 2023-10-09T06:06:35.925 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:35 smithi008 ceph-mon[191227]: failed to encode map e560 with expected crc 2023-10-09T06:06:35.925 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:35 smithi008 ceph-mon[191227]: pgmap v1704: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 14879/91263 objects degraded (16.303%); 1729/91263 objects misplaced (1.895%); 6.7 MiB/s, 145 objects/s recovering 2023-10-09T06:06:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:35 smithi191 ceph-mon[161669]: failed to encode map e560 with expected crc 2023-10-09T06:06:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:35 smithi191 ceph-mon[161669]: pgmap v1704: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 14879/91263 objects degraded (16.303%); 1729/91263 objects misplaced (1.895%); 6.7 MiB/s, 145 objects/s recovering 2023-10-09T06:06:36.242 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:36.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:36.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:36.359 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:36.465 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:37.383 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:37.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:37.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:37.490 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:37.539 INFO:teuthology.orchestra.run.smithi008.stdout:moved 0 / 267 (0%) 2023-10-09T06:06:37.540 INFO:teuthology.orchestra.run.smithi008.stdout:avg 38.1429 2023-10-09T06:06:37.540 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 8.49249 -> 8.49249 (expected baseline 5.71786) 2023-10-09T06:06:37.540 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.0 with 25 -> 25 pgs (0.655431 -> 0.655431 * mean) 2023-10-09T06:06:37.540 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.1 with 51 -> 51 pgs (1.33708 -> 1.33708 * mean) 2023-10-09T06:06:37.540 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:06:37.540 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T06:06:37.540 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 0.05 2023-10-09T06:06:37.540 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T06:06:37.540 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0214 2023-10-09T06:06:37.540 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.2143 2023-10-09T06:06:37.556 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:37.581 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T06:06:37.662 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:37.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:37.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:37.773 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:37.901 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:38.200 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:38.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:38.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:38.213 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:37 smithi008 ceph-mon[184418]: pgmap v1705: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 14500/86811 objects degraded (16.703%); 1729/86811 objects misplaced (1.992%); 6.8 MiB/s, 171 objects/s recovering 2023-10-09T06:06:38.213 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:37 smithi008 ceph-mon[184418]: from='client.47297 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:38.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:37 smithi008 ceph-mon[191227]: pgmap v1705: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 14500/86811 objects degraded (16.703%); 1729/86811 objects misplaced (1.992%); 6.8 MiB/s, 171 objects/s recovering 2023-10-09T06:06:38.213 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:37 smithi008 ceph-mon[191227]: from='client.47297 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:38.240 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:06:38.240 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:06:38.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:37 smithi191 ceph-mon[161669]: pgmap v1705: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 14500/86811 objects degraded (16.703%); 1729/86811 objects misplaced (1.992%); 6.8 MiB/s, 171 objects/s recovering 2023-10-09T06:06:38.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:37 smithi191 ceph-mon[161669]: from='client.47297 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:38.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:38.310 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:06:38.349 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:38.353 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:06:38.399 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:06:38.443 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:38.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:38.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:38.463 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:06:38.511 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:06:38.543 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:06:38.549 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:38.595 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:06:38.628 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:06:38.634 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:38.650 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.650 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.651 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.651 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.651 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.651 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.651 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.651 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:06:39.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:39 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:39.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:39 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:39.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/10879351' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:39.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:39 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14500/86811 objects degraded (16.703%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:39.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:39 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:39.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:39 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:39.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/10879351' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:39.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:39 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14500/86811 objects degraded (16.703%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:39 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:39 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/10879351' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:39.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:39 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14500/86811 objects degraded (16.703%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:39.604 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:06:39.807 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T06:06:39.808 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-10-09T06:06:39.817 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:39.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:39.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:39.855 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:06:39.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:39.970 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:40.043 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:40.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:40.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:40.149 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:40.306 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:40.433 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:40 smithi008 ceph-mon[184418]: pgmap v1706: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 14500/86355 objects degraded (16.791%); 1729/86355 objects misplaced (2.002%); 5.9 MiB/s, 157 objects/s recovering 2023-10-09T06:06:40.434 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:40 smithi008 ceph-mon[191227]: pgmap v1706: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 14500/86355 objects degraded (16.791%); 1729/86355 objects misplaced (2.002%); 5.9 MiB/s, 157 objects/s recovering 2023-10-09T06:06:40.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:06:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:06:40] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.33.4" 2023-10-09T06:06:40.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:40 smithi191 ceph-mon[161669]: pgmap v1706: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 14500/86355 objects degraded (16.791%); 1729/86355 objects misplaced (2.002%); 5.9 MiB/s, 157 objects/s recovering 2023-10-09T06:06:41.120 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:41.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:41.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:41.232 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:41.309 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:41.373 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:41.407 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:41.449 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:41.524 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:41.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:41.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:41.635 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:41.666 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:41.720 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:41.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:41.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:41.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:41.854 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:41.911 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:41.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:41.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:42.022 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:42.053 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:42.101 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:42.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:42.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:42.206 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:42.231 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:42.275 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:42.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:42.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:42.380 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:42.406 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:42.462 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:42.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:42.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:42.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:42 smithi191 ceph-mon[161669]: pgmap v1707: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 14561/83799 objects degraded (17.376%); 1554/83799 objects misplaced (1.854%); 3.3 MiB/s, 77 objects/s recovering 2023-10-09T06:06:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:42 smithi191 ceph-mon[161669]: failed to encode map e561 with expected crc 2023-10-09T06:06:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:42 smithi191 ceph-mon[161669]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T06:06:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:42 smithi191 ceph-mon[161669]: failed to encode map e561 with expected crc 2023-10-09T06:06:42.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:42.618 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:42.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:42 smithi008 ceph-mon[191227]: pgmap v1707: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 14561/83799 objects degraded (17.376%); 1554/83799 objects misplaced (1.854%); 3.3 MiB/s, 77 objects/s recovering 2023-10-09T06:06:42.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:42 smithi008 ceph-mon[191227]: failed to encode map e561 with expected crc 2023-10-09T06:06:42.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:42 smithi008 ceph-mon[191227]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T06:06:42.679 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:42 smithi008 ceph-mon[191227]: failed to encode map e561 with expected crc 2023-10-09T06:06:42.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:42 smithi008 ceph-mon[184418]: pgmap v1707: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 14561/83799 objects degraded (17.376%); 1554/83799 objects misplaced (1.854%); 3.3 MiB/s, 77 objects/s recovering 2023-10-09T06:06:42.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:42 smithi008 ceph-mon[184418]: failed to encode map e561 with expected crc 2023-10-09T06:06:42.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:42 smithi008 ceph-mon[184418]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T06:06:42.681 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:42 smithi008 ceph-mon[184418]: failed to encode map e561 with expected crc 2023-10-09T06:06:42.684 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:42.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:42.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:42.791 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:42.875 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:43.167 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:06:43.481 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:43 smithi008 ceph-mon[184418]: failed to encode map e562 with expected crc 2023-10-09T06:06:43.481 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:43 smithi008 ceph-mon[184418]: failed to encode map e562 with expected crc 2023-10-09T06:06:43.481 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:43 smithi008 ceph-mon[184418]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T06:06:43.482 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:43 smithi008 ceph-mon[184418]: failed to encode map e562 with expected crc 2023-10-09T06:06:43.482 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:43 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14561/83799 objects degraded (17.376%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:43 smithi191 ceph-mon[161669]: failed to encode map e562 with expected crc 2023-10-09T06:06:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:43 smithi191 ceph-mon[161669]: failed to encode map e562 with expected crc 2023-10-09T06:06:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:43 smithi191 ceph-mon[161669]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T06:06:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:43 smithi191 ceph-mon[161669]: failed to encode map e562 with expected crc 2023-10-09T06:06:43.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:43 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14561/83799 objects degraded (17.376%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:43.872 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:43.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:43.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:43.882 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:43 smithi008 ceph-mon[191227]: failed to encode map e562 with expected crc 2023-10-09T06:06:43.882 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:43 smithi008 ceph-mon[191227]: failed to encode map e562 with expected crc 2023-10-09T06:06:43.883 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:43 smithi008 ceph-mon[191227]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T06:06:43.883 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:43 smithi008 ceph-mon[191227]: failed to encode map e562 with expected crc 2023-10-09T06:06:43.883 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:43 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14561/83799 objects degraded (17.376%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:43.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:44.012 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:44.085 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:44.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:44.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:44.191 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:44.318 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:44.599 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:44.605 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:44 smithi008 ceph-mon[184418]: pgmap v1710: 89 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 14403/78267 objects degraded (18.402%); 1554/78267 objects misplaced (1.986%); 3.7 MiB/s, 97 objects/s recovering 2023-10-09T06:06:44.605 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:44.605 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:44.605 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:44 smithi008 ceph-mon[191227]: pgmap v1710: 89 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 14403/78267 objects degraded (18.402%); 1554/78267 objects misplaced (1.986%); 3.7 MiB/s, 97 objects/s recovering 2023-10-09T06:06:44.605 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:44.605 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:44.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:44.707 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:44 smithi191 ceph-mon[161669]: pgmap v1710: 89 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 14403/78267 objects degraded (18.402%); 1554/78267 objects misplaced (1.986%); 3.7 MiB/s, 97 objects/s recovering 2023-10-09T06:06:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:44.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:44.813 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:45.280 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:45.286 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:45.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:45.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:45.403 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:45.429 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:45.484 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:45.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:45.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:45.623 INFO:teuthology.orchestra.run.smithi008.stdout:moved 0 / 267 (0%) 2023-10-09T06:06:45.623 INFO:teuthology.orchestra.run.smithi008.stdout:avg 38.1429 2023-10-09T06:06:45.624 INFO:teuthology.orchestra.run.smithi008.stdout:stddev 8.49249 -> 8.49249 (expected baseline 5.71786) 2023-10-09T06:06:45.624 INFO:teuthology.orchestra.run.smithi008.stdout:min osd.0 with 25 -> 25 pgs (0.655431 -> 0.655431 * mean) 2023-10-09T06:06:45.624 INFO:teuthology.orchestra.run.smithi008.stdout:max osd.1 with 51 -> 51 pgs (1.33708 -> 1.33708 * mean) 2023-10-09T06:06:45.624 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:06:45.624 INFO:teuthology.orchestra.run.smithi008.stdout:oload 1000 2023-10-09T06:06:45.624 INFO:teuthology.orchestra.run.smithi008.stdout:max_change 1 2023-10-09T06:06:45.624 INFO:teuthology.orchestra.run.smithi008.stdout:max_change_osds 4 2023-10-09T06:06:45.624 INFO:teuthology.orchestra.run.smithi008.stdout:average_utilization 0.0203 2023-10-09T06:06:45.624 INFO:teuthology.orchestra.run.smithi008.stdout:overload_utilization 0.2034 2023-10-09T06:06:45.625 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:45.657 INFO:teuthology.orchestra.run.smithi008.stderr:no change 2023-10-09T06:06:45.697 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:45.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:45.806 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:45.896 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:06:45.897 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T06:06:45.897 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T06:06:45.897 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T06:06:45.897 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.897 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117296 9.10 9:0bb04225:::benchmark_data_smithi008_502820_object33123:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-10-09T06:06:45.897 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:56.263811+0000", 2023-10-09T06:06:45.897 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 529.618498499, 2023-10-09T06:06:45.897 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.170647588, 2023-10-09T06:06:45.897 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.897 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:45.898 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.898 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:45.898 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:45.898 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117296 2023-10-09T06:06:45.898 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.898 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.898 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.898 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.898 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263811+0000", 2023-10-09T06:06:45.899 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.899 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.899 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.899 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.899 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263811+0000", 2023-10-09T06:06:45.899 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.899 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.899 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.899 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.900 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263809+0000", 2023-10-09T06:06:45.900 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:06:45.900 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.900 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.900 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.900 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263819+0000", 2023-10-09T06:06:45.900 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0018e-05 2023-10-09T06:06:45.900 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.900 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.900 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.900 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263842+0000", 2023-10-09T06:06:45.901 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3445999999999999e-05 2023-10-09T06:06:45.901 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.901 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.901 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.901 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263852+0000", 2023-10-09T06:06:45.901 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9650000000000003e-06 2023-10-09T06:06:45.901 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.901 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.901 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.902 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263925+0000", 2023-10-09T06:06:45.902 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2465999999999994e-05 2023-10-09T06:06:45.902 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.902 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.902 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.902 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.263942+0000", 2023-10-09T06:06:45.902 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7008e-05 2023-10-09T06:06:45.903 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.903 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.903 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.903 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.429345+0000", 2023-10-09T06:06:45.903 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1654032759999999 2023-10-09T06:06:45.903 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.904 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.904 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:45.904 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.429511+0000", 2023-10-09T06:06:45.904 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016623000000000001 2023-10-09T06:06:45.904 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.905 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.905 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:06:45.905 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.429618+0000", 2023-10-09T06:06:45.906 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000106716 2023-10-09T06:06:45.906 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.906 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.906 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:45.906 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.432625+0000", 2023-10-09T06:06:45.906 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030070420000000001 2023-10-09T06:06:45.906 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.907 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.907 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:45.907 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.432632+0000", 2023-10-09T06:06:45.907 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3440000000000003e-06 2023-10-09T06:06:45.907 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.907 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.908 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:45.908 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.433415+0000", 2023-10-09T06:06:45.908 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00078227500000000003 2023-10-09T06:06:45.908 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.908 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.908 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:45.908 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.433475+0000", 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0384000000000003e-05 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.434458+0000", 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00098339400000000006 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117298 9.6 9:64c5977e:::benchmark_data_smithi008_502820_object33125:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-10-09T06:06:45.909 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:56.293379+0000", 2023-10-09T06:06:45.910 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 529.58893049799997, 2023-10-09T06:06:45.910 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4447610179999999, 2023-10-09T06:06:45.910 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.910 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:45.910 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.910 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:45.910 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:45.910 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117298 2023-10-09T06:06:45.910 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.910 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.911 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.911 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.911 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293379+0000", 2023-10-09T06:06:45.911 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.911 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.911 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.911 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.911 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293379+0000", 2023-10-09T06:06:45.911 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.911 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.912 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.912 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.912 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293377+0000", 2023-10-09T06:06:45.912 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:45.912 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.912 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.912 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.912 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293386+0000", 2023-10-09T06:06:45.912 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2519999999999999e-06 2023-10-09T06:06:45.913 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.913 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.913 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.913 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293406+0000", 2023-10-09T06:06:45.913 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.0307000000000001e-05 2023-10-09T06:06:45.913 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.913 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.913 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.914 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293417+0000", 2023-10-09T06:06:45.914 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1223000000000001e-05 2023-10-09T06:06:45.914 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.914 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.914 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.914 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293499+0000", 2023-10-09T06:06:45.914 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1650999999999995e-05 2023-10-09T06:06:45.914 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.914 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.914 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.914 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.293513+0000", 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3808999999999999e-05 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.244709+0000", 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.95119675999999997 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.244723+0000", 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3230999999999999e-05 2023-10-09T06:06:45.915 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.916 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.916 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.916 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.722284+0000", 2023-10-09T06:06:45.916 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4775617670000001 2023-10-09T06:06:45.916 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.916 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.916 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:45.916 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.722454+0000", 2023-10-09T06:06:45.916 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016943700000000001 2023-10-09T06:06:45.916 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.917 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.917 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:06:45.917 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.722557+0000", 2023-10-09T06:06:45.917 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000102985 2023-10-09T06:06:45.917 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.917 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.917 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:45.917 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.725888+0000", 2023-10-09T06:06:45.917 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003330937 2023-10-09T06:06:45.918 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.918 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.918 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:45.918 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.725901+0000", 2023-10-09T06:06:45.918 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.288e-05 2023-10-09T06:06:45.918 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.918 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.918 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:45.918 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.736945+0000", 2023-10-09T06:06:45.919 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.011044221999999999 2023-10-09T06:06:45.919 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.919 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.919 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:45.919 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.737001+0000", 2023-10-09T06:06:45.919 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6258000000000002e-05 2023-10-09T06:06:45.920 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.920 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.920 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:45.920 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.738140+0000", 2023-10-09T06:06:45.920 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011387680000000001 2023-10-09T06:06:45.920 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.920 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:45.921 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.921 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.921 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.921 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.36258.0:117300 9.16 9:6b0df22d:::benchmark_data_smithi008_502820_object33127:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-10-09T06:06:45.921 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T05:57:56.313799+0000", 2023-10-09T06:06:45.921 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 529.56851000200004, 2023-10-09T06:06:45.921 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.4264962080000001, 2023-10-09T06:06:45.922 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.922 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:45.922 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.922 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.36258", 2023-10-09T06:06:45.922 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:06:45.922 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 117300 2023-10-09T06:06:45.923 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.923 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.923 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.923 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.923 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313799+0000", 2023-10-09T06:06:45.924 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.924 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.924 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.924 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.924 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313799+0000", 2023-10-09T06:06:45.924 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.925 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.925 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.925 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.925 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313798+0000", 2023-10-09T06:06:45.925 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:45.925 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.925 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.926 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.926 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313809+0000", 2023-10-09T06:06:45.926 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1031e-05 2023-10-09T06:06:45.926 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.926 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.926 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.927 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313823+0000", 2023-10-09T06:06:45.927 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3562e-05 2023-10-09T06:06:45.927 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.927 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.927 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.928 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313830+0000", 2023-10-09T06:06:45.928 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.7360000000000001e-06 2023-10-09T06:06:45.928 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.928 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.928 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.928 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313887+0000", 2023-10-09T06:06:45.929 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6759999999999999e-05 2023-10-09T06:06:45.929 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.929 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.929 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.929 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:56.313899+0000", 2023-10-09T06:06:45.929 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1528999999999999e-05 2023-10-09T06:06:45.930 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.930 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.930 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.930 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.244751+0000", 2023-10-09T06:06:45.930 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93085188200000002 2023-10-09T06:06:45.930 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.930 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.931 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.931 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:57.244760+0000", 2023-10-09T06:06:45.931 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7780000000000002e-06 2023-10-09T06:06:45.931 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.931 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.931 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.931 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.734776+0000", 2023-10-09T06:06:45.931 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.490015206 2023-10-09T06:06:45.932 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.932 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.932 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:45.932 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.734873+0000", 2023-10-09T06:06:45.932 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7702999999999996e-05 2023-10-09T06:06:45.932 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.932 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.933 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:06:45.933 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.734974+0000", 2023-10-09T06:06:45.933 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010043999999999999 2023-10-09T06:06:45.933 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.933 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.933 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:45.933 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.738071+0000", 2023-10-09T06:06:45.933 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030975210000000002 2023-10-09T06:06:45.934 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.934 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.934 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:45.934 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.738084+0000", 2023-10-09T06:06:45.934 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2621e-05 2023-10-09T06:06:45.934 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.935 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.935 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:45.935 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.739642+0000", 2023-10-09T06:06:45.935 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001558282 2023-10-09T06:06:45.935 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.935 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.935 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:45.936 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.739707+0000", 2023-10-09T06:06:45.936 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4712e-05 2023-10-09T06:06:45.936 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.936 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.936 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:45.936 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T05:57:58.740296+0000", 2023-10-09T06:06:45.936 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00058872799999999995 2023-10-09T06:06:45.936 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.937 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:45.937 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.937 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.937 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.937 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:124844 10.7 10.9ab82d7 (undecoded) ondisk+retry+write+known_if_redirected e552)", 2023-10-09T06:06:45.937 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:43.605228+0000", 2023-10-09T06:06:45.937 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 62.277081260999999, 2023-10-09T06:06:45.938 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90999754700000002, 2023-10-09T06:06:45.938 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.938 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:06:45.938 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.938 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:45.938 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:45.938 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 124844 2023-10-09T06:06:45.939 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.939 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.939 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.939 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.939 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.605228+0000", 2023-10-09T06:06:45.939 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.939 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.940 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.940 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.940 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.605228+0000", 2023-10-09T06:06:45.940 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.940 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.940 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.940 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.940 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.605227+0000", 2023-10-09T06:06:45.940 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:45.941 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.941 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.605233+0000", 2023-10-09T06:06:45.941 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7429999999999996e-06 2023-10-09T06:06:45.941 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.941 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.941 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.941 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.605249+0000", 2023-10-09T06:06:45.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5481999999999998e-05 2023-10-09T06:06:45.942 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.942 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.942 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.942 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.605260+0000", 2023-10-09T06:06:45.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1236000000000001e-05 2023-10-09T06:06:45.942 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.942 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.942 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.942 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.784933+0000", 2023-10-09T06:06:45.942 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.179672888 2023-10-09T06:06:45.943 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.943 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.943 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.943 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.784946+0000", 2023-10-09T06:06:45.943 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3385e-05 2023-10-09T06:06:45.943 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.943 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.943 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.943 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515206+0000", 2023-10-09T06:06:45.943 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73025978999999996 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515226+0000", 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9555000000000002e-05 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:124863 10.7 10.acb8b997 (undecoded) ondisk+retry+write+known_if_redirected e552)", 2023-10-09T06:06:45.944 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:43.606038+0000", 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 62.276271864999998, 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90922767999999998, 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 124863 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.945 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606038+0000", 2023-10-09T06:06:45.946 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.946 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.946 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.946 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606038+0000", 2023-10-09T06:06:45.946 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.946 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.946 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.946 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.946 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606036+0000", 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606042+0000", 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4330000000000003e-06 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606053+0000", 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0759e-05 2023-10-09T06:06:45.947 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606062+0000", 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.679e-06 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.784977+0000", 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17891423300000001 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.948 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.784985+0000", 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5499999999999995e-06 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515256+0000", 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73027141500000003 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515265+0000", 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.687e-06 2023-10-09T06:06:45.949 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:124875 10.7 10.9b2dd607 (undecoded) ondisk+retry+write+known_if_redirected e552)", 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:43.606068+0000", 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 62.276241290000002, 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.909226435, 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.950 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:45.951 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:45.951 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 124875 2023-10-09T06:06:45.951 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.951 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.951 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.951 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.951 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606068+0000", 2023-10-09T06:06:45.951 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.951 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.951 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.951 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.952 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606068+0000", 2023-10-09T06:06:45.952 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.952 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.952 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.952 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.952 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606068+0000", 2023-10-09T06:06:45.952 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:45.952 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.952 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.952 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.952 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606071+0000", 2023-10-09T06:06:45.953 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9699999999999999e-06 2023-10-09T06:06:45.953 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.953 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.953 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.953 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606077+0000", 2023-10-09T06:06:45.953 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.7549999999999997e-06 2023-10-09T06:06:45.953 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.954 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.954 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.954 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606083+0000", 2023-10-09T06:06:45.954 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8769999999999998e-06 2023-10-09T06:06:45.954 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.954 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.954 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.954 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.785008+0000", 2023-10-09T06:06:45.954 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.178924574 2023-10-09T06:06:45.955 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.955 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.955 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.785016+0000", 2023-10-09T06:06:45.955 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.1370000000000002e-06 2023-10-09T06:06:45.955 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.955 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.955 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.955 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515287+0000", 2023-10-09T06:06:45.955 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73027065300000005 2023-10-09T06:06:45.956 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.956 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.956 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:45.956 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515295+0000", 2023-10-09T06:06:45.956 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.8499999999999994e-06 2023-10-09T06:06:45.956 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.956 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:45.956 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.956 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.957 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.957 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:124906 10.7 10.64842f47 (undecoded) ondisk+retry+write+known_if_redirected e552)", 2023-10-09T06:06:45.957 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:43.606088+0000", 2023-10-09T06:06:45.957 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 62.276221356999997, 2023-10-09T06:06:45.957 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90924048999999996, 2023-10-09T06:06:45.957 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.957 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:06:45.957 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.957 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:45.957 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:45.958 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 124906 2023-10-09T06:06:45.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.958 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.958 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.958 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.958 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606088+0000", 2023-10-09T06:06:45.958 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.958 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.959 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.959 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606088+0000", 2023-10-09T06:06:45.959 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.959 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.959 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.959 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.959 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606088+0000", 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606090+0000", 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5000000000000002e-06 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606097+0000", 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4180000000000002e-06 2023-10-09T06:06:45.960 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.961 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.961 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.961 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606102+0000", 2023-10-09T06:06:45.961 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5829999999999997e-06 2023-10-09T06:06:45.961 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.961 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.961 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.961 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.785030+0000", 2023-10-09T06:06:45.961 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17892798200000001 2023-10-09T06:06:45.962 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.962 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.962 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.962 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.785037+0000", 2023-10-09T06:06:45.962 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.584e-06 2023-10-09T06:06:45.962 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.962 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.962 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.962 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515319+0000", 2023-10-09T06:06:45.962 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73028198300000002 2023-10-09T06:06:45.963 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.963 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.963 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:45.963 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515329+0000", 2023-10-09T06:06:45.963 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.8500000000000006e-06 2023-10-09T06:06:45.963 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.963 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:45.963 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.964 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.964 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.964 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:124913 10.7 10.1e023087 (undecoded) ondisk+retry+write+known_if_redirected e552)", 2023-10-09T06:06:45.964 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:43.606107+0000", 2023-10-09T06:06:45.964 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 62.276202779000002, 2023-10-09T06:06:45.964 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90925188300000004, 2023-10-09T06:06:45.964 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.964 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:06:45.964 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.964 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:45.965 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:45.965 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 124913 2023-10-09T06:06:45.965 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.965 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.965 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.965 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.965 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606107+0000", 2023-10-09T06:06:45.965 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.965 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.965 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.965 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.966 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606107+0000", 2023-10-09T06:06:45.966 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.966 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.966 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.966 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.966 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606106+0000", 2023-10-09T06:06:45.966 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:45.966 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.966 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.966 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.966 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606109+0000", 2023-10-09T06:06:45.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5909999999999998e-06 2023-10-09T06:06:45.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.967 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.967 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.967 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606115+0000", 2023-10-09T06:06:45.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4160000000000001e-06 2023-10-09T06:06:45.967 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.967 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.967 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.967 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606121+0000", 2023-10-09T06:06:45.967 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3870000000000003e-06 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.785054+0000", 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.178933751 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.785058+0000", 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0999999999999997e-06 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.968 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.969 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515350+0000", 2023-10-09T06:06:45.969 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73029111700000005 2023-10-09T06:06:45.969 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.969 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.969 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:45.969 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515358+0000", 2023-10-09T06:06:45.969 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9039999999999998e-06 2023-10-09T06:06:45.969 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.969 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:45.970 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.970 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.970 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.970 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:124931 10.7 10.40cd1c77 (undecoded) ondisk+retry+write+known_if_redirected e552)", 2023-10-09T06:06:45.970 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:43.606125+0000", 2023-10-09T06:06:45.970 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 62.276184196999999, 2023-10-09T06:06:45.970 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90926337199999996, 2023-10-09T06:06:45.970 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.970 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:06:45.971 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.971 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:45.971 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:45.971 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 124931 2023-10-09T06:06:45.971 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.971 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.971 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.971 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.972 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606125+0000", 2023-10-09T06:06:45.972 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.972 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.972 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.972 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.972 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606125+0000", 2023-10-09T06:06:45.972 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.972 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.972 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.972 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.972 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606125+0000", 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606127+0000", 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.401e-06 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606133+0000", 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.0809999999999999e-06 2023-10-09T06:06:45.973 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.974 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.606139+0000", 2023-10-09T06:06:45.974 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.536e-06 2023-10-09T06:06:45.974 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.974 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.974 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.785070+0000", 2023-10-09T06:06:45.974 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.17893098700000001 2023-10-09T06:06:45.974 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.974 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.975 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.975 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.785074+0000", 2023-10-09T06:06:45.975 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.9960000000000004e-06 2023-10-09T06:06:45.975 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.975 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.975 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.975 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515380+0000", 2023-10-09T06:06:45.975 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.73030640700000005 2023-10-09T06:06:45.975 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.515389+0000", 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3760000000000008e-06 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:124939 10.e 10:757b0941:::benchmark_data_smithi008_634586_object37774:head [delete] snapc 0=[] ondisk+write+known_if_redirected e552)", 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:43.773985+0000", 2023-10-09T06:06:45.976 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 62.108324793999998, 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.78666757700000001, 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 124939 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.977 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.773985+0000", 2023-10-09T06:06:45.978 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.978 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.978 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.978 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.978 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.773985+0000", 2023-10-09T06:06:45.978 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.978 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.978 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.978 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.978 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.773983+0000", 2023-10-09T06:06:45.978 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:45.979 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.979 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.979 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.979 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.773990+0000", 2023-10-09T06:06:45.979 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3830000000000003e-06 2023-10-09T06:06:45.979 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.979 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.979 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.980 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.774007+0000", 2023-10-09T06:06:45.980 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6208000000000001e-05 2023-10-09T06:06:45.980 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.980 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.980 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.980 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.774015+0000", 2023-10-09T06:06:45.980 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2989999999999997e-06 2023-10-09T06:06:45.980 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.980 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.774085+0000", 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.0075999999999998e-05 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:43.774098+0000", 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2933e-05 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.981 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.552506+0000", 2023-10-09T06:06:45.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.778408196 2023-10-09T06:06:45.982 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.982 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:45.982 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.552608+0000", 2023-10-09T06:06:45.982 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00010207 2023-10-09T06:06:45.982 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.982 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.982 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:06:45.982 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.552688+0000", 2023-10-09T06:06:45.983 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0277000000000004e-05 2023-10-09T06:06:45.983 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.983 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.983 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:45.983 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.556745+0000", 2023-10-09T06:06:45.983 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040564879999999996 2023-10-09T06:06:45.983 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.983 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.983 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:45.983 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.556926+0000", 2023-10-09T06:06:45.983 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018139400000000001 2023-10-09T06:06:45.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.984 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:45.984 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.560459+0000", 2023-10-09T06:06:45.984 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035322320000000002 2023-10-09T06:06:45.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.984 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.984 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:45.984 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.560511+0000", 2023-10-09T06:06:45.984 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.2210999999999999e-05 2023-10-09T06:06:45.984 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.985 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.985 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:45.985 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:44.560652+0000", 2023-10-09T06:06:45.985 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014137199999999999 2023-10-09T06:06:45.985 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.985 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:45.985 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.985 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.985 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.985 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:148607 10.a 10:52694b4d:::benchmark_data_smithi008_634586_object61442:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e561)", 2023-10-09T06:06:45.985 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:41.471335+0000", 2023-10-09T06:06:45.986 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.41097392, 2023-10-09T06:06:45.986 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.017429457, 2023-10-09T06:06:45.986 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.986 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "reached pg", 2023-10-09T06:06:45.986 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.986 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:45.986 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:45.986 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 148607 2023-10-09T06:06:45.986 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.986 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.987 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.987 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.987 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.471335+0000", 2023-10-09T06:06:45.987 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.987 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.987 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.987 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.987 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.471335+0000", 2023-10-09T06:06:45.987 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.987 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.987 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.471333+0000", 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.471344+0000", 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0623e-05 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.988 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.471360+0000", 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6022e-05 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.471371+0000", 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0608000000000001e-05 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.474454+0000", 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0030831349999999999 2023-10-09T06:06:45.989 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.990 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.990 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.990 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.474467+0000", 2023-10-09T06:06:45.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2887e-05 2023-10-09T06:06:45.990 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.990 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.990 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.990 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.488649+0000", 2023-10-09T06:06:45.990 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0141822540000001 2023-10-09T06:06:45.991 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.991 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.991 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:45.991 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.488765+0000", 2023-10-09T06:06:45.991 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011603 2023-10-09T06:06:45.991 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.991 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:45.991 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:45.991 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.991 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:148626 10.a 10:53493b32:::benchmark_data_smithi008_634586_object61461:head [delete] snapc 0=[] ondisk+write+known_if_redirected e561)", 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:41.485767+0000", 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.3965427210000003, 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0181855369999999, 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 148626 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.992 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:45.993 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.993 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:45.993 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.485767+0000", 2023-10-09T06:06:45.993 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.993 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.993 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.993 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:45.993 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.485767+0000", 2023-10-09T06:06:45.993 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:45.993 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.993 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.994 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:45.994 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.485765+0000", 2023-10-09T06:06:45.994 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:06:45.994 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.994 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.994 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:45.994 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.485777+0000", 2023-10-09T06:06:45.994 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2951e-05 2023-10-09T06:06:45.994 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.994 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.994 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.485803+0000", 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.5522000000000001e-05 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.485817+0000", 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3888e-05 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.485913+0000", 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6211000000000004e-05 2023-10-09T06:06:45.995 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:45.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.485931+0000", 2023-10-09T06:06:45.996 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.8175000000000001e-05 2023-10-09T06:06:45.996 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.996 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.996 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:45.996 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.489212+0000", 2023-10-09T06:06:45.996 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0032803239999999 2023-10-09T06:06:45.996 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.489322+0000", 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000110813 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.489454+0000", 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013164000000000001 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.997 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:45.998 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.497303+0000", 2023-10-09T06:06:45.998 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0078487650000000006 2023-10-09T06:06:45.998 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.998 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.998 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:45.998 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.497357+0000", 2023-10-09T06:06:45.998 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4630999999999999e-05 2023-10-09T06:06:45.998 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.998 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.999 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:45.999 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.503894+0000", 2023-10-09T06:06:45.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0065367100000000003 2023-10-09T06:06:45.999 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.999 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:45.999 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.503925+0000", 2023-10-09T06:06:45.999 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0589000000000001e-05 2023-10-09T06:06:45.999 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:45.999 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:45.999 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:46.000 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.503952+0000", 2023-10-09T06:06:46.000 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.7457000000000002e-05 2023-10-09T06:06:46.000 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.000 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:46.000 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.000 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.000 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.000 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:148633 10.a 10:5491636e:::benchmark_data_smithi008_634586_object61468:head [delete] snapc 0=[] ondisk+write+known_if_redirected e561)", 2023-10-09T06:06:46.001 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:41.498441+0000", 2023-10-09T06:06:46.001 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.383868734, 2023-10-09T06:06:46.001 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.00562967, 2023-10-09T06:06:46.001 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:46.001 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.001 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:46.001 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:46.001 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:46.001 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 148633 2023-10-09T06:06:46.001 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.002 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:46.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.002 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:46.002 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498441+0000", 2023-10-09T06:06:46.002 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.002 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.002 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:46.003 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498441+0000", 2023-10-09T06:06:46.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.003 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.003 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:46.003 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498439+0000", 2023-10-09T06:06:46.003 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:06:46.003 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.003 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.003 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:46.004 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498448+0000", 2023-10-09T06:06:46.004 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.329e-06 2023-10-09T06:06:46.004 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.004 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.004 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:46.004 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498471+0000", 2023-10-09T06:06:46.004 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3289e-05 2023-10-09T06:06:46.004 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.004 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.004 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.004 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498481+0000", 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7010000000000008e-06 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498544+0000", 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2581000000000003e-05 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498563+0000", 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9366000000000001e-05 2023-10-09T06:06:46.005 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.006 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.006 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.006 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.493331+0000", 2023-10-09T06:06:46.006 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99476832199999998 2023-10-09T06:06:46.006 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.006 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.006 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:46.006 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.493451+0000", 2023-10-09T06:06:46.006 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000119894 2023-10-09T06:06:46.006 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.006 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.493544+0000", 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3035000000000006e-05 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.497365+0000", 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038204760000000002 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.497403+0000", 2023-10-09T06:06:46.008 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.8260000000000003e-05 2023-10-09T06:06:46.008 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.008 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.008 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.008 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.503997+0000", 2023-10-09T06:06:46.008 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0065944030000000004 2023-10-09T06:06:46.008 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.008 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.008 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:46.008 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.504038+0000", 2023-10-09T06:06:46.008 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1043e-05 2023-10-09T06:06:46.009 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.009 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.009 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:46.009 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.504070+0000", 2023-10-09T06:06:46.009 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1949999999999997e-05 2023-10-09T06:06:46.009 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.009 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:46.009 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.009 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.009 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.009 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:148634 10.a 10:59dc0ef4:::benchmark_data_smithi008_634586_object61469:head [delete] snapc 0=[] ondisk+write+known_if_redirected e561)", 2023-10-09T06:06:46.010 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:41.498496+0000", 2023-10-09T06:06:46.010 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.3838135249999999, 2023-10-09T06:06:46.010 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0057433250000001, 2023-10-09T06:06:46.010 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:46.010 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.010 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:46.010 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:46.010 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:46.010 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 148634 2023-10-09T06:06:46.011 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.011 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:46.011 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.011 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:46.011 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498496+0000", 2023-10-09T06:06:46.011 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.011 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.011 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.011 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:46.011 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498496+0000", 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498495+0000", 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498500+0000", 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5299999999999998e-06 2023-10-09T06:06:46.012 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498507+0000", 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.109e-06 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498512+0000", 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4060000000000004e-06 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.013 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498592+0000", 2023-10-09T06:06:46.014 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.9783999999999994e-05 2023-10-09T06:06:46.014 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.014 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.014 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.014 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.498602+0000", 2023-10-09T06:06:46.014 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0321e-05 2023-10-09T06:06:46.014 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.014 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.014 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.014 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.497430+0000", 2023-10-09T06:06:46.014 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.99882805699999999 2023-10-09T06:06:46.015 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.015 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.015 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:46.015 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.497573+0000", 2023-10-09T06:06:46.015 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000142991 2023-10-09T06:06:46.015 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.015 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.015 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.015 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.497710+0000", 2023-10-09T06:06:46.015 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013682700000000001 2023-10-09T06:06:46.015 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.016 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.503864+0000", 2023-10-09T06:06:46.016 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0061542130000000004 2023-10-09T06:06:46.016 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.016 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:46.016 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.503964+0000", 2023-10-09T06:06:46.016 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9376000000000005e-05 2023-10-09T06:06:46.016 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.016 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.504134+0000", 2023-10-09T06:06:46.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00017003299999999999 2023-10-09T06:06:46.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.017 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:46.017 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.504202+0000", 2023-10-09T06:06:46.017 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.8459000000000003e-05 2023-10-09T06:06:46.017 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.017 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.017 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:46.018 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.504239+0000", 2023-10-09T06:06:46.018 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6831e-05 2023-10-09T06:06:46.018 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.018 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:46.018 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.018 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.018 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.018 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:148645 10.a 10:5211b289:::benchmark_data_smithi008_634586_object61480:head [delete] snapc 0=[] ondisk+write+known_if_redirected e561)", 2023-10-09T06:06:46.018 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:41.519421+0000", 2023-10-09T06:06:46.018 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.3628888440000004, 2023-10-09T06:06:46.018 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0089739069999999, 2023-10-09T06:06:46.019 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:46.019 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.019 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:46.019 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:46.019 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:46.019 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 148645 2023-10-09T06:06:46.019 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.019 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:46.019 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.019 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.519421+0000", 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.519421+0000", 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.519420+0000", 2023-10-09T06:06:46.020 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:46.021 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.021 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.021 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:46.021 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.519423+0000", 2023-10-09T06:06:46.021 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2789999999999999e-06 2023-10-09T06:06:46.021 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.021 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.021 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:46.021 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.519431+0000", 2023-10-09T06:06:46.021 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6009999999999997e-06 2023-10-09T06:06:46.022 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.022 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.022 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.022 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.519437+0000", 2023-10-09T06:06:46.022 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.2319999999999998e-06 2023-10-09T06:06:46.022 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.022 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.022 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.022 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.523271+0000", 2023-10-09T06:06:46.022 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038335489999999999 2023-10-09T06:06:46.022 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.523287+0000", 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6362999999999999e-05 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.500893+0000", 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97760577100000001 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.023 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.501022+0000", 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000129006 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.501098+0000", 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.6452999999999995e-05 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.504253+0000", 2023-10-09T06:06:46.024 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031550800000000002 2023-10-09T06:06:46.025 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.025 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.025 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.025 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.504294+0000", 2023-10-09T06:06:46.025 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.0735999999999998e-05 2023-10-09T06:06:46.025 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.025 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.025 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.025 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.528306+0000", 2023-10-09T06:06:46.025 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024012084999999999 2023-10-09T06:06:46.025 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.026 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.026 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:46.026 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.528353+0000", 2023-10-09T06:06:46.026 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6381999999999997e-05 2023-10-09T06:06:46.026 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.026 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.026 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:46.026 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.528394+0000", 2023-10-09T06:06:46.026 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1841999999999997e-05 2023-10-09T06:06:46.026 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.026 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:46.027 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.027 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.027 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.027 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:148664 10.a 10:52817ebf:::benchmark_data_smithi008_634586_object61499:head [delete] snapc 0=[] ondisk+write+known_if_redirected e561)", 2023-10-09T06:06:46.027 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:41.570297+0000", 2023-10-09T06:06:46.027 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.3120125189999996, 2023-10-09T06:06:46.027 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.97821766200000004, 2023-10-09T06:06:46.027 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:46.027 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.027 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:46.028 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:46.028 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:46.028 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 148664 2023-10-09T06:06:46.028 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.028 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:46.028 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.028 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:46.028 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.570297+0000", 2023-10-09T06:06:46.028 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.028 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.028 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.029 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:46.029 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.570297+0000", 2023-10-09T06:06:46.029 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.029 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.029 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.029 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:46.029 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.570296+0000", 2023-10-09T06:06:46.029 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.029 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.029 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.029 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:46.030 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.570302+0000", 2023-10-09T06:06:46.030 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7200000000000003e-06 2023-10-09T06:06:46.030 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.030 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.030 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:46.030 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.570313+0000", 2023-10-09T06:06:46.030 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1201e-05 2023-10-09T06:06:46.030 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.030 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.030 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.030 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.570321+0000", 2023-10-09T06:06:46.031 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0839999999999992e-06 2023-10-09T06:06:46.031 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.031 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.031 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.031 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.570378+0000", 2023-10-09T06:06:46.031 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.6941000000000003e-05 2023-10-09T06:06:46.031 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.031 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.031 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.570389+0000", 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1582e-05 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.504317+0000", 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93392772499999999 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.504432+0000", 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00011479799999999999 2023-10-09T06:06:46.032 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.504536+0000", 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000104503 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.528441+0000", 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.023904667000000001 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.033 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.528475+0000", 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4112000000000001e-05 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548400+0000", 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019925040000000001 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548459+0000", 2023-10-09T06:06:46.034 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8801000000000002e-05 2023-10-09T06:06:46.035 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.035 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.035 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:46.035 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548515+0000", 2023-10-09T06:06:46.035 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5486999999999999e-05 2023-10-09T06:06:46.035 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.035 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:46.035 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.035 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.035 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.035 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:148691 10.a 10:5e4fe2f2:::benchmark_data_smithi008_634586_object61526:head [delete] snapc 0=[] ondisk+write+known_if_redirected e561)", 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:41.610845+0000", 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.2714644870000003, 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.93775340200000001, 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 148691 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:46.036 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.610845+0000", 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.610845+0000", 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.610844+0000", 2023-10-09T06:06:46.037 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.610850+0000", 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.8000000000000004e-06 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.610862+0000", 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2088e-05 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.038 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.610870+0000", 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.3210000000000005e-06 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.610935+0000", 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4809999999999998e-05 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.039 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.610949+0000", 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3952e-05 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.508736+0000", 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.897786686 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.508835+0000", 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.9308999999999994e-05 2023-10-09T06:06:46.040 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.508932+0000", 2023-10-09T06:06:46.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.7275000000000006e-05 2023-10-09T06:06:46.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.041 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.041 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.528507+0000", 2023-10-09T06:06:46.041 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019574435000000001 2023-10-09T06:06:46.041 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.041 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.042 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.528537+0000", 2023-10-09T06:06:46.042 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.0102000000000001e-05 2023-10-09T06:06:46.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:46.042 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548523+0000", 2023-10-09T06:06:46.042 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019986133 2023-10-09T06:06:46.042 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.042 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.042 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548565+0000", 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.1643000000000001e-05 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548598+0000", 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3781000000000001e-05 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.043 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:148702 10.a 10:503dca62:::benchmark_data_smithi008_634586_object61537:head [delete] snapc 0=[] ondisk+write+known_if_redirected e561)", 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:41.641477+0000", 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.240831901, 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.90719194400000003, 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 148702 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.044 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.641477+0000", 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.641477+0000", 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.045 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.641476+0000", 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.641483+0000", 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.9999999999999999e-06 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.641494+0000", 2023-10-09T06:06:46.046 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.129e-05 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.641500+0000", 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5609999999999998e-06 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.641551+0000", 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0949999999999998e-05 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.047 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.641560+0000", 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8249999999999994e-06 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.512602+0000", 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.87104234199999997 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:46.048 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.512740+0000", 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000137372 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.512876+0000", 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000136835 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.528567+0000", 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015690322 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.049 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.528595+0000", 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.8515999999999999e-05 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548604+0000", 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020009064 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:46.050 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548636+0000", 2023-10-09T06:06:46.051 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.2147000000000002e-05 2023-10-09T06:06:46.051 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.051 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.051 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:46.051 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548669+0000", 2023-10-09T06:06:46.051 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.3006999999999998e-05 2023-10-09T06:06:46.051 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.051 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:46.051 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.051 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.051 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.052 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:148724 10.a 10:5070a54b:::benchmark_data_smithi008_634586_object61559:head [delete] snapc 0=[] ondisk+write+known_if_redirected e561)", 2023-10-09T06:06:46.052 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:41.683848+0000", 2023-10-09T06:06:46.052 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.1984609989999999, 2023-10-09T06:06:46.052 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.86489456899999995, 2023-10-09T06:06:46.052 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:46.052 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.052 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:46.052 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:46.052 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 148724 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.683848+0000", 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.683848+0000", 2023-10-09T06:06:46.053 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.054 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.054 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.054 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:46.054 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.683847+0000", 2023-10-09T06:06:46.054 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.054 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.054 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:46.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.683853+0000", 2023-10-09T06:06:46.055 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4870000000000002e-06 2023-10-09T06:06:46.055 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.055 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.055 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:46.055 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.683865+0000", 2023-10-09T06:06:46.055 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.1962e-05 2023-10-09T06:06:46.055 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.056 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.056 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.056 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.683871+0000", 2023-10-09T06:06:46.056 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4389999999999997e-06 2023-10-09T06:06:46.056 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.056 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.056 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.056 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.683923+0000", 2023-10-09T06:06:46.057 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1845999999999998e-05 2023-10-09T06:06:46.057 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.057 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.057 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.057 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.683946+0000", 2023-10-09T06:06:46.057 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.2982000000000001e-05 2023-10-09T06:06:46.058 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.058 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.058 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.058 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.516415+0000", 2023-10-09T06:06:46.058 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.83246880199999995 2023-10-09T06:06:46.058 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.058 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.059 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:46.059 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.516510+0000", 2023-10-09T06:06:46.059 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4876000000000004e-05 2023-10-09T06:06:46.059 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.059 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.059 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.059 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.516599+0000", 2023-10-09T06:06:46.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.9361e-05 2023-10-09T06:06:46.060 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.060 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.060 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.060 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.528626+0000", 2023-10-09T06:06:46.060 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012027072999999999 2023-10-09T06:06:46.060 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.061 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.061 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.536277+0000", 2023-10-09T06:06:46.061 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0076504260000000001 2023-10-09T06:06:46.061 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.061 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.062 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:46.062 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548675+0000", 2023-10-09T06:06:46.062 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012398573 2023-10-09T06:06:46.062 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.062 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.063 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:46.063 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548711+0000", 2023-10-09T06:06:46.063 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.6152999999999998e-05 2023-10-09T06:06:46.063 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.063 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.063 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:46.064 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548743+0000", 2023-10-09T06:06:46.064 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1642e-05 2023-10-09T06:06:46.064 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.064 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:46.064 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.064 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.065 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.065 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:148745 10.a 10:57c5b249:::benchmark_data_smithi008_634586_object61580:head [delete] snapc 0=[] ondisk+write+known_if_redirected e561)", 2023-10-09T06:06:46.065 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:41.720044+0000", 2023-10-09T06:06:46.065 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 4.1622657360000002, 2023-10-09T06:06:46.065 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.82877396000000003, 2023-10-09T06:06:46.065 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:06:46.065 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.066 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:06:46.066 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:06:46.066 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:06:46.066 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 148745 2023-10-09T06:06:46.066 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.066 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:06:46.066 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.720044+0000", 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.720044+0000", 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:06:46.067 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.720043+0000", 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.720047+0000", 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5199999999999999e-06 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.720055+0000", 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0839999999999992e-06 2023-10-09T06:06:46.068 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.720060+0000", 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0730000000000004e-06 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.720106+0000", 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.6060000000000003e-05 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.069 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:41.720114+0000", 2023-10-09T06:06:46.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.4580000000000004e-06 2023-10-09T06:06:46.070 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.070 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.070 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:06:46.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.520289+0000", 2023-10-09T06:06:46.070 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.80017549200000004 2023-10-09T06:06:46.070 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.070 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.070 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:06:46.070 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.520414+0000", 2023-10-09T06:06:46.071 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000124423 2023-10-09T06:06:46.071 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.071 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.071 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.071 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.520529+0000", 2023-10-09T06:06:46.071 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000115238 2023-10-09T06:06:46.071 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.071 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.071 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.536215+0000", 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015686037999999999 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.536308+0000", 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2658999999999995e-05 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548751+0000", 2023-10-09T06:06:46.072 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012443246999999999 2023-10-09T06:06:46.073 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.073 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.073 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:06:46.073 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548786+0000", 2023-10-09T06:06:46.073 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.5284999999999998e-05 2023-10-09T06:06:46.073 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:06:46.073 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:06:46.073 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:06:46.073 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:42.548818+0000", 2023-10-09T06:06:46.073 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.1340000000000001e-05 2023-10-09T06:06:46.073 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.074 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:46.074 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.074 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:06:46.074 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:06:46.074 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:06:46.075 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:46.172 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:46.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:46.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:46.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:46.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:46.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:46.490 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:46.592 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:46.592 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:06:46.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:46 smithi191 ceph-mon[161669]: pgmap v1711: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 444 op/s; 14434/76143 objects degraded (18.956%); 1411/76143 objects misplaced (1.853%); 3.0 MiB/s, 62 objects/s recovering 2023-10-09T06:06:46.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:46 smithi191 ceph-mon[161669]: from='client.47306 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:46.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:46 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2857443021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:46.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:46 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:46.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:46 smithi008 ceph-mon[184418]: pgmap v1711: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 444 op/s; 14434/76143 objects degraded (18.956%); 1411/76143 objects misplaced (1.853%); 3.0 MiB/s, 62 objects/s recovering 2023-10-09T06:06:46.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:46 smithi008 ceph-mon[184418]: from='client.47306 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:46.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:46 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2857443021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:46.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:46 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:46.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:46 smithi008 ceph-mon[191227]: pgmap v1711: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 444 op/s; 14434/76143 objects degraded (18.956%); 1411/76143 objects misplaced (1.853%); 3.0 MiB/s, 62 objects/s recovering 2023-10-09T06:06:46.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:46 smithi008 ceph-mon[191227]: from='client.47306 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:46.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:46 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2857443021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:46.969 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:46 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:47.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:47.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:06:47.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[184418]: osdmap e563: 8 total, 8 up, 7 in 2023-10-09T06:06:47.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[184418]: failed to encode map e563 with expected crc 2023-10-09T06:06:47.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[184418]: failed to encode map e563 with expected crc 2023-10-09T06:06:47.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[184418]: failed to encode map e563 with expected crc 2023-10-09T06:06:47.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[184418]: failed to encode map e563 with expected crc 2023-10-09T06:06:47.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:47.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:06:47.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[191227]: osdmap e563: 8 total, 8 up, 7 in 2023-10-09T06:06:47.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[191227]: failed to encode map e563 with expected crc 2023-10-09T06:06:47.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[191227]: failed to encode map e563 with expected crc 2023-10-09T06:06:47.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[191227]: failed to encode map e563 with expected crc 2023-10-09T06:06:47.968 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:47 smithi008 ceph-mon[191227]: failed to encode map e563 with expected crc 2023-10-09T06:06:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:47 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:47 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:06:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:47 smithi191 ceph-mon[161669]: osdmap e563: 8 total, 8 up, 7 in 2023-10-09T06:06:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:47 smithi191 ceph-mon[161669]: failed to encode map e563 with expected crc 2023-10-09T06:06:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:47 smithi191 ceph-mon[161669]: failed to encode map e563 with expected crc 2023-10-09T06:06:48.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:47 smithi191 ceph-mon[161669]: failed to encode map e563 with expected crc 2023-10-09T06:06:48.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:47 smithi191 ceph-mon[161669]: failed to encode map e563 with expected crc 2023-10-09T06:06:48.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:48 smithi191 ceph-mon[161669]: pgmap v1713: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 14278/71856 objects degraded (19.870%); 1411/71856 objects misplaced (1.964%); 4.4 MiB/s, 89 objects/s recovering 2023-10-09T06:06:48.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:48 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14278/71856 objects degraded (19.870%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:48.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:48 smithi008 ceph-mon[184418]: pgmap v1713: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 14278/71856 objects degraded (19.870%); 1411/71856 objects misplaced (1.964%); 4.4 MiB/s, 89 objects/s recovering 2023-10-09T06:06:48.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:48 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14278/71856 objects degraded (19.870%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:48.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:48 smithi008 ceph-mon[191227]: pgmap v1713: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 14278/71856 objects degraded (19.870%); 1411/71856 objects misplaced (1.964%); 4.4 MiB/s, 89 objects/s recovering 2023-10-09T06:06:48.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:48 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14278/71856 objects degraded (19.870%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:49.103 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:49.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:49.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:49.154 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:06:49.212 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:49.329 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:49.846 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:49.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:49.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:49.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:49.996 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:50.072 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:50.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:50.082 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:06:50.089 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:50.137 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:06:50.179 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:06:50.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:50.222 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:06:50.223 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:50.254 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:06:50.300 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:50.302 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:06:50.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:50.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:50.351 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:06:50.388 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:06:50.390 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:06:50.390 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:06:50.390 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:06:50.390 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:06:50.390 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:06:50.390 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:06:50.390 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:06:50.391 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:06:50.405 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:50.449 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:50.518 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:50.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:50.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:50.626 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:50.671 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:50.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:50 smithi008 ceph-mon[184418]: pgmap v1714: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 14278/71343 objects degraded (20.013%); 1411/71343 objects misplaced (1.978%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:06:50.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1918524044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:50.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:50 smithi008 ceph-mon[191227]: pgmap v1714: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 14278/71343 objects degraded (20.013%); 1411/71343 objects misplaced (1.978%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:06:50.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1918524044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:50.715 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:06:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:06:50] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.33.4" 2023-10-09T06:06:50.769 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:50.776 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:50 smithi191 ceph-mon[161669]: pgmap v1714: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 14278/71343 objects degraded (20.013%); 1411/71343 objects misplaced (1.978%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:06:50.776 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1918524044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:50.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:50.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:50.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:50.916 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:50.999 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:51.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:51.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:51.112 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:51.120 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:06:51.224 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:52.088 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:52.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:52.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:52.199 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:52.274 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:52.689 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:52.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:52.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:52.805 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:52.912 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:52 smithi008 ceph-mon[184418]: pgmap v1715: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 14313/68808 objects degraded (20.801%); 1261/68808 objects misplaced (1.833%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:06:52.912 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:52 smithi008 ceph-mon[191227]: pgmap v1715: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 14313/68808 objects degraded (20.801%); 1261/68808 objects misplaced (1.833%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:06:52.926 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:53.028 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-09T06:06:53.029 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:06:53.029 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T06:06:53.029 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:06:53.029 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:06:53.029 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T06:06:53.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:52 smithi191 ceph-mon[161669]: pgmap v1715: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 14313/68808 objects degraded (20.801%); 1261/68808 objects misplaced (1.833%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:06:53.162 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:53.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:53.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:53.244 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:53.271 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:53.304 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:53.354 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:53.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:53.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:53.460 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:53.558 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:53.691 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:06:53.691 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:06:53.900 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:53 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14313/68808 objects degraded (20.801%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:53.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:53.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:53 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:53.903 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:53 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14313/68808 objects degraded (20.801%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:53.903 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:53.903 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:53 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:53.926 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:53.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:53.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:53.970 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:06:54.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:54.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:53 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14313/68808 objects degraded (20.801%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:54.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:54.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:53 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:54.087 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:54.151 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:54.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:54.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:54.258 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:54.286 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:54.340 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:54.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:54.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:54.446 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:54.490 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:54.551 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:54.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:54.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:54.642 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:54 smithi191 ceph-mon[161669]: pgmap v1716: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 14156/62949 objects degraded (22.488%); 1261/62949 objects misplaced (2.003%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:06:54.663 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:54.727 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:54.905 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:54 smithi008 ceph-mon[184418]: pgmap v1716: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 14156/62949 objects degraded (22.488%); 1261/62949 objects misplaced (2.003%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:06:54.905 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:54 smithi008 ceph-mon[191227]: pgmap v1716: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 14156/62949 objects degraded (22.488%); 1261/62949 objects misplaced (2.003%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:06:54.939 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:06:54.954 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:54.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:54.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:55.065 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:55.163 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:55.354 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:55.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:55.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:55.463 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:55.494 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:55.934 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:55.946 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:55 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:55.947 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:55 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:55.947 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:55 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:55.947 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:55 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:55.947 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:55 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:55.947 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:55 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:55.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:56.030 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:56.048 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:56.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:55 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:56.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:55 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:55 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:56.101 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:56.181 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:56.316 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:56.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:56.419 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:56.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:56.520 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:56.572 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:56.712 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:56 smithi008 ceph-mon[184418]: pgmap v1717: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 14199/60528 objects degraded (23.459%); 1107/60528 objects misplaced (1.829%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:06:56.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:56 smithi008 ceph-mon[191227]: pgmap v1717: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 14199/60528 objects degraded (23.459%); 1107/60528 objects misplaced (1.829%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:06:56.994 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:57.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:57.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:57.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:56 smithi191 ceph-mon[161669]: pgmap v1717: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 14199/60528 objects degraded (23.459%); 1107/60528 objects misplaced (1.829%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:06:57.105 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:57.221 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:57.690 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:57 smithi008 ceph-mon[184418]: pgmap v1718: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 14031/55782 objects degraded (25.153%); 1107/55782 objects misplaced (1.985%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:06:57.927 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:57.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:57.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:57.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:57 smithi008 ceph-mon[191227]: pgmap v1718: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 14031/55782 objects degraded (25.153%); 1107/55782 objects misplaced (1.985%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:06:58.036 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:58.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:57 smithi191 ceph-mon[161669]: pgmap v1718: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 14031/55782 objects degraded (25.153%); 1107/55782 objects misplaced (1.985%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:06:58.072 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:58.144 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:58.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:58.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:58.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:58.291 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:58.353 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:58.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:58.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:58.461 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:58.558 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:58.712 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1503239270' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:58.712 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:58 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:58.712 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:58 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 14031/55782 objects degraded (25.153%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:58.766 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:06:58.801 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1503239270' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:58.802 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:58 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:58.802 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:58 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 14031/55782 objects degraded (25.153%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:58.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1503239270' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:58.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:58 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:58.802 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:58 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 14031/55782 objects degraded (25.153%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:06:59.656 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:59.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:59.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:59.772 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:59.834 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[184418]: osdmap e564: 8 total, 8 up, 7 in 2023-10-09T06:06:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[184418]: failed to encode map e564 with expected crc 2023-10-09T06:06:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[184418]: failed to encode map e564 with expected crc 2023-10-09T06:06:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[184418]: failed to encode map e564 with expected crc 2023-10-09T06:06:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[184418]: pgmap v1720: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 14031/55272 objects degraded (25.385%); 1107/55272 objects misplaced (2.003%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T06:06:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:59.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:59.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:59.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[191227]: osdmap e564: 8 total, 8 up, 7 in 2023-10-09T06:06:59.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[191227]: failed to encode map e564 with expected crc 2023-10-09T06:06:59.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[191227]: failed to encode map e564 with expected crc 2023-10-09T06:06:59.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[191227]: failed to encode map e564 with expected crc 2023-10-09T06:06:59.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[191227]: pgmap v1720: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 14031/55272 objects degraded (25.385%); 1107/55272 objects misplaced (2.003%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T06:06:59.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:06:59.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:06:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:59 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:59 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:59 smithi191 ceph-mon[161669]: osdmap e564: 8 total, 8 up, 7 in 2023-10-09T06:07:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:59 smithi191 ceph-mon[161669]: failed to encode map e564 with expected crc 2023-10-09T06:07:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:59 smithi191 ceph-mon[161669]: failed to encode map e564 with expected crc 2023-10-09T06:07:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:59 smithi191 ceph-mon[161669]: failed to encode map e564 with expected crc 2023-10-09T06:07:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:59 smithi191 ceph-mon[161669]: pgmap v1720: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 14031/55272 objects degraded (25.385%); 1107/55272 objects misplaced (2.003%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T06:07:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:06:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:00.540 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:00.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:00.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:00.651 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:00.783 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:00 smithi008 ceph-mon[184418]: failed to encode map e564 with expected crc 2023-10-09T06:07:00.784 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:07:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:07:00] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.33.4" 2023-10-09T06:07:00.784 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:00 smithi008 ceph-mon[191227]: failed to encode map e564 with expected crc 2023-10-09T06:07:00.792 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:01.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:00 smithi191 ceph-mon[161669]: failed to encode map e564 with expected crc 2023-10-09T06:07:01.157 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:07:01.175 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:01.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:01.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:01.206 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:07:01.265 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:07:01.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:01.321 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:07:01.329 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:01.355 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:07:01.393 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:01.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:01.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:01.403 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:07:01.448 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:07:01.492 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:07:01.500 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:01.516 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:07:01.516 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:07:01.516 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:07:01.516 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:07:01.517 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:07:01.517 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:07:01.517 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:07:01.517 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:07:01.614 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:01.649 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:07:01.649 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:07:01.649 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T06:07:01.649 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T06:07:01.649 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:07:01.764 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:07:01.764 INFO:teuthology.orchestra.run.smithi008.stdout: "success": "" 2023-10-09T06:07:01.764 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:07:01.768 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:01 smithi191 ceph-mon[161669]: pgmap v1721: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 14075/52902 objects degraded (26.606%); 950/52902 objects misplaced (1.796%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:07:01.768 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:01 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2105613636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:01.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:01 smithi008 ceph-mon[184418]: pgmap v1721: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 14075/52902 objects degraded (26.606%); 950/52902 objects misplaced (1.796%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:07:01.933 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:01 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2105613636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:01.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:01 smithi008 ceph-mon[191227]: pgmap v1721: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 14075/52902 objects degraded (26.606%); 950/52902 objects misplaced (1.796%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:07:01.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:01 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2105613636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:02.216 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:02.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:02.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:02.324 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:02.353 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:02.820 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:07:04.070 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:07:04.087 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:04.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:04.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:04.198 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:04.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:04 smithi008 ceph-mon[184418]: pgmap v1722: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 984 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 13903/47115 objects degraded (29.509%); 950/47115 objects misplaced (2.016%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:07:04.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:04 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13903/47115 objects degraded (29.509%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:04.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:04 smithi008 ceph-mon[191227]: pgmap v1722: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 984 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 13903/47115 objects degraded (29.509%); 950/47115 objects misplaced (2.016%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:07:04.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:04 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13903/47115 objects degraded (29.509%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:04.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:04 smithi191 ceph-mon[161669]: pgmap v1722: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 984 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 13903/47115 objects degraded (29.509%); 950/47115 objects misplaced (2.016%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:07:04.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:04 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13903/47115 objects degraded (29.509%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:04.325 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:04.365 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:04.528 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:04.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:04.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:04.638 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:04.708 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:04.865 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:04.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:04.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:04.984 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:05.024 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:05.101 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:05.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:05.239 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:05.324 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:05.443 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:05.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:05.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:05.567 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:05.617 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:05.641 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:07:05.686 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:05.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:05.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:05.797 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:05.877 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:06.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:06 smithi191 ceph-mon[161669]: pgmap v1723: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 938 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 13930/44910 objects degraded (31.018%); 813/44910 objects misplaced (1.810%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:07:06.029 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:06 smithi008 ceph-mon[191227]: pgmap v1723: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 938 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 13930/44910 objects degraded (31.018%); 813/44910 objects misplaced (1.810%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:07:06.029 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:06 smithi008 ceph-mon[184418]: pgmap v1723: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 938 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 13930/44910 objects degraded (31.018%); 813/44910 objects misplaced (1.810%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:07:06.116 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:06.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:06.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:06.206 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:06.224 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:06.335 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:06.427 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:06.514 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:06.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:06.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:06.625 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:06.662 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:06.740 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:06.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:06.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:06.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:06.891 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:06.943 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:06.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:06.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:07.053 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:07.091 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:07.147 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:07.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:07.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:07.259 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:07.290 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:07.353 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:07.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:07.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:07.461 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:07.586 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:07.859 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:07.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:07.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:07.969 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:08.062 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:08.098 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:07:08.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:08 smithi008 ceph-mon[184418]: pgmap v1724: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 837 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 13755/40023 objects degraded (34.368%); 813/40023 objects misplaced (2.031%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:07:08.124 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2280208396' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:08.124 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:08 smithi008 ceph-mon[191227]: pgmap v1724: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 837 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 13755/40023 objects degraded (34.368%); 813/40023 objects misplaced (2.031%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:07:08.124 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2280208396' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:08.176 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:08.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:08.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:08.289 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:08.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:08 smithi191 ceph-mon[161669]: pgmap v1724: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 837 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 13755/40023 objects degraded (34.368%); 813/40023 objects misplaced (2.031%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:07:08.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2280208396' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:08.361 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:09.005 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-09T06:07:09.005 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:07:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:09 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:09 smithi191 ceph-mon[161669]: failed to encode map e565 with expected crc 2023-10-09T06:07:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:09 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2280208396' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:07:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:09 smithi191 ceph-mon[161669]: osdmap e565: 8 total, 8 up, 7 in 2023-10-09T06:07:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:09 smithi191 ceph-mon[161669]: failed to encode map e565 with expected crc 2023-10-09T06:07:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:09 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13755/40023 objects degraded (34.368%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:09.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:09.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[184418]: failed to encode map e565 with expected crc 2023-10-09T06:07:09.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2280208396' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:07:09.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[184418]: osdmap e565: 8 total, 8 up, 7 in 2023-10-09T06:07:09.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[184418]: failed to encode map e565 with expected crc 2023-10-09T06:07:09.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13755/40023 objects degraded (34.368%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[191227]: failed to encode map e565 with expected crc 2023-10-09T06:07:09.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2280208396' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:07:09.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[191227]: osdmap e565: 8 total, 8 up, 7 in 2023-10-09T06:07:09.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[191227]: failed to encode map e565 with expected crc 2023-10-09T06:07:09.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:09 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13755/40023 objects degraded (34.368%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:09.733 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:09.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:09.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:09.858 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:09.914 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:10.104 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:10.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:10.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:10.168 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:10 smithi008 ceph-mon[184418]: failed to encode map e565 with expected crc 2023-10-09T06:07:10.169 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:10 smithi008 ceph-mon[184418]: pgmap v1726: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 827 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 13755/39546 objects degraded (34.782%); 813/39546 objects misplaced (2.056%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:07:10.169 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:10 smithi008 ceph-mon[191227]: failed to encode map e565 with expected crc 2023-10-09T06:07:10.169 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:10 smithi008 ceph-mon[191227]: pgmap v1726: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 827 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 13755/39546 objects degraded (34.782%); 813/39546 objects misplaced (2.056%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:07:10.221 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:10.260 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:10.326 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:10.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:10.337 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:10 smithi191 ceph-mon[161669]: failed to encode map e565 with expected crc 2023-10-09T06:07:10.337 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:10 smithi191 ceph-mon[161669]: pgmap v1726: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 827 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 13755/39546 objects degraded (34.782%); 813/39546 objects misplaced (2.056%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:07:10.437 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:10.478 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:10.550 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:10.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:10.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:10.602 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:07:10.602 INFO:teuthology.orchestra.run.smithi008.stdout: "size": 20, 2023-10-09T06:07:10.602 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 600, 2023-10-09T06:07:10.602 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [ 2023-10-09T06:07:10.602 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.602 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:129596 10.7 10:eb1a002a:::benchmark_data_smithi008_634586_object42431:head [delete] snapc 0=[] ondisk+write+known_if_redirected e555)", 2023-10-09T06:07:10.603 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:05:59.853302+0000", 2023-10-09T06:07:10.603 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 70.740113352999998, 2023-10-09T06:07:10.603 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.50119338599999996, 2023-10-09T06:07:10.603 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.603 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.603 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 129596 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:59.853302+0000", 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.604 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:59.853302+0000", 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:59.853300+0000", 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:59.853309+0000", 2023-10-09T06:07:10.605 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7399999999999993e-06 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:59.853323+0000", 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.403e-05 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:59.853332+0000", 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6930000000000002e-06 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.606 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:59.876666+0000", 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.023333553999999999 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:59.878602+0000", 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019363819999999999 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.607 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:59.900770+0000", 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022167165999999999 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:05:59.902960+0000", 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021908309999999999 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.191607+0000", 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.28864632899999998 2023-10-09T06:07:10.608 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.192960+0000", 2023-10-09T06:07:10.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013530009999999999 2023-10-09T06:07:10.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.609 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.609 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.195630+0000", 2023-10-09T06:07:10.609 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026705880000000002 2023-10-09T06:07:10.609 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.609 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.322466+0000", 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.12683562200000001 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.323074+0000", 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000608521 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.610 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.349401+0000", 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.026326423000000002 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.350478+0000", 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010768959999999999 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.354495+0000", 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0040173759999999996 2023-10-09T06:07:10.611 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:129683 10.7 10:ef5908e0:::benchmark_data_smithi008_634586_object42518:head [delete] snapc 0=[] ondisk+write+known_if_redirected e555)", 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:00.040053+0000", 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 70.553361808999995, 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.44232768700000002, 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.612 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 129683 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.040053+0000", 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.613 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.040053+0000", 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.040051+0000", 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.040068+0000", 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.6719e-05 2023-10-09T06:07:10.614 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.040084+0000", 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5588000000000001e-05 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.040092+0000", 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4009999999999997e-06 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.615 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.048378+0000", 2023-10-09T06:07:10.616 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0082856070000000004 2023-10-09T06:07:10.616 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.616 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.616 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.616 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.050196+0000", 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018185199999999999 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.094901+0000", 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.044704502 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.097141+0000", 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022403420000000002 2023-10-09T06:07:10.617 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.618 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.618 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.422868+0000", 2023-10-09T06:07:10.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.32572685499999998 2023-10-09T06:07:10.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.618 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.618 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.424982+0000", 2023-10-09T06:07:10.618 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002114243 2023-10-09T06:07:10.618 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.618 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.428310+0000", 2023-10-09T06:07:10.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003327993 2023-10-09T06:07:10.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.619 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.476554+0000", 2023-10-09T06:07:10.619 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.048244060999999998 2023-10-09T06:07:10.619 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.619 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.619 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.477178+0000", 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00062365200000000004 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.477381+0000", 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000202981 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.478407+0000", 2023-10-09T06:07:10.620 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0010262310000000001 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.482381+0000", 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039737710000000001 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:129733 10.7 10:e4af4ae6:::benchmark_data_smithi008_634586_object42568:head [delete] snapc 0=[] ondisk+write+known_if_redirected e555)", 2023-10-09T06:07:10.621 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:00.151537+0000", 2023-10-09T06:07:10.622 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 70.441878052000007, 2023-10-09T06:07:10.622 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.463417615, 2023-10-09T06:07:10.622 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.622 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.622 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.622 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.622 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.622 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 129733 2023-10-09T06:07:10.622 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.622 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.622 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.151537+0000", 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.151537+0000", 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.623 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.151536+0000", 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.151541+0000", 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.0649999999999998e-06 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.151550+0000", 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6099999999999995e-06 2023-10-09T06:07:10.624 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.151557+0000", 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.5849999999999996e-06 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.185648+0000", 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.034090585 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.625 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.187617+0000", 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001969181 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.319585+0000", 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.131968111 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.321833+0000", 2023-10-09T06:07:10.626 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002248655 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.589577+0000", 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26774347399999998 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.590932+0000", 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001354907 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.627 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.593879+0000", 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029470659999999999 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.609788+0000", 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.015909494999999999 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.628 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.609991+0000", 2023-10-09T06:07:10.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000202654 2023-10-09T06:07:10.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.610589+0000", 2023-10-09T06:07:10.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00059817400000000004 2023-10-09T06:07:10.629 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.629 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.629 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.629 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.611496+0000", 2023-10-09T06:07:10.629 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00090676500000000002 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.614955+0000", 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0034586339999999999 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:129887 10.7 10:e8587876:::benchmark_data_smithi008_634586_object42722:head [delete] snapc 0=[] ondisk+write+known_if_redirected e555)", 2023-10-09T06:07:10.630 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:00.591300+0000", 2023-10-09T06:07:10.631 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 70.002115435999997, 2023-10-09T06:07:10.631 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.44252047, 2023-10-09T06:07:10.631 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.631 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.631 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.631 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.631 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.631 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 129887 2023-10-09T06:07:10.631 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.631 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.631 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.591300+0000", 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.591300+0000", 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.591298+0000", 2023-10-09T06:07:10.632 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.591308+0000", 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.4730000000000006e-06 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.591322+0000", 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4083999999999999e-05 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.633 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.591330+0000", 2023-10-09T06:07:10.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2099999999999993e-06 2023-10-09T06:07:10.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.634 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.615516+0000", 2023-10-09T06:07:10.634 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.024186361999999999 2023-10-09T06:07:10.634 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.634 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.634 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.617428+0000", 2023-10-09T06:07:10.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019119880000000001 2023-10-09T06:07:10.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.628213+0000", 2023-10-09T06:07:10.635 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010784373999999999 2023-10-09T06:07:10.635 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.635 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.635 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.635 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.630424+0000", 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002211705 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.002742+0000", 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37231797500000002 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.005675+0000", 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029327820000000001 2023-10-09T06:07:10.636 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.008981+0000", 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033054030000000002 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.029135+0000", 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020154504 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.637 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.029295+0000", 2023-10-09T06:07:10.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00016014600000000001 2023-10-09T06:07:10.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.029745+0000", 2023-10-09T06:07:10.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00044973600000000001 2023-10-09T06:07:10.638 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.638 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.638 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.638 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.030512+0000", 2023-10-09T06:07:10.638 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00076729599999999999 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.033820+0000", 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033078199999999999 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:129915 10.7 10:ec748b8c:::benchmark_data_smithi008_634586_object42750:head [delete] snapc 0=[] ondisk+write+known_if_redirected e555)", 2023-10-09T06:07:10.639 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:00.662658+0000", 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 69.930756677000005, 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.44793470699999999, 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 129915 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.640 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.662658+0000", 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.662658+0000", 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.662657+0000", 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:10.641 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.662666+0000", 2023-10-09T06:07:10.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.5189999999999998e-06 2023-10-09T06:07:10.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.642 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.642 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.662690+0000", 2023-10-09T06:07:10.642 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.3272999999999999e-05 2023-10-09T06:07:10.642 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.642 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.662700+0000", 2023-10-09T06:07:10.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.01e-05 2023-10-09T06:07:10.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.643 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.682144+0000", 2023-10-09T06:07:10.643 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019443986999999999 2023-10-09T06:07:10.643 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.643 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.643 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.684519+0000", 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002374821 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.693554+0000", 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0090350059999999999 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.695432+0000", 2023-10-09T06:07:10.644 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018786580000000001 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.081264+0000", 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.38583231200000001 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.082825+0000", 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015604379999999999 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.645 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.085576+0000", 2023-10-09T06:07:10.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027511440000000001 2023-10-09T06:07:10.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.646 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.104142+0000", 2023-10-09T06:07:10.646 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018566282 2023-10-09T06:07:10.646 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.646 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.646 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.104746+0000", 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00060349000000000004 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.105907+0000", 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0011612409999999999 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.106853+0000", 2023-10-09T06:07:10.647 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00094572500000000002 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.110593+0000", 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0037402949999999998 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:129925 10.7 10:eadb7590:::benchmark_data_smithi008_634586_object42760:head [delete] snapc 0=[] ondisk+write+known_if_redirected e555)", 2023-10-09T06:07:10.648 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:00.701906+0000", 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 69.891509167999999, 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.45682184799999997, 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 129925 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.649 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.701906+0000", 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.701906+0000", 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.701905+0000", 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:10.650 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.701911+0000", 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.1750000000000002e-06 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.701921+0000", 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0348e-05 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.651 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.701927+0000", 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.4600000000000002e-06 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.732352+0000", 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030425331999999999 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.734648+0000", 2023-10-09T06:07:10.652 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022963039999999999 2023-10-09T06:07:10.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.751889+0000", 2023-10-09T06:07:10.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.017240411000000001 2023-10-09T06:07:10.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.653 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.653 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.653 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.754051+0000", 2023-10-09T06:07:10.653 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021619510000000001 2023-10-09T06:07:10.653 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.120679+0000", 2023-10-09T06:07:10.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.36662892899999999 2023-10-09T06:07:10.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.654 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.654 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.122523+0000", 2023-10-09T06:07:10.654 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001843115 2023-10-09T06:07:10.654 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.654 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.126056+0000", 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035332699999999998 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.152605+0000", 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.026548855 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.153203+0000", 2023-10-09T06:07:10.655 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00059820499999999998 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.153799+0000", 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00059586100000000005 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.154827+0000", 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001028686 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.656 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.158728+0000", 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039003039999999998 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:129981 10.7 10:e9ed194e:::benchmark_data_smithi008_634586_object42816:head [delete] snapc 0=[] ondisk+write+known_if_redirected e555)", 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:00.888028+0000", 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 69.705386950999994, 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.443765346, 2023-10-09T06:07:10.657 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 129981 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.888028+0000", 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.658 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.888028+0000", 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.888027+0000", 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.659 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.888033+0000", 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.545e-06 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.888046+0000", 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2257e-05 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.888052+0000", 2023-10-09T06:07:10.660 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.4169999999999997e-06 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.897735+0000", 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0096829370000000008 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.899534+0000", 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017988819999999999 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.661 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.936377+0000", 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.036842786000000002 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.938619+0000", 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0022422259999999999 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.290650+0000", 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35203162399999999 2023-10-09T06:07:10.662 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.292433+0000", 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001782608 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.295621+0000", 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031878060000000001 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.663 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.316591+0000", 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.020969970000000001 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.317182+0000", 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00059095700000000003 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.326777+0000", 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0095951079999999998 2023-10-09T06:07:10.664 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.327650+0000", 2023-10-09T06:07:10.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00087356300000000005 2023-10-09T06:07:10.665 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.665 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.665 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.665 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.331793+0000", 2023-10-09T06:07:10.665 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0041430460000000001 2023-10-09T06:07:10.665 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.665 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:129990 10.7 10:eca81164:::benchmark_data_smithi008_634586_object42825:head [delete] snapc 0=[] ondisk+write+known_if_redirected e555)", 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:00.912281+0000", 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 69.681134053999997, 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.47555610100000001, 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.666 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 129990 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.912281+0000", 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.912281+0000", 2023-10-09T06:07:10.667 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.912279+0000", 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.912287+0000", 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5979999999999999e-06 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.668 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.912303+0000", 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.5862000000000002e-05 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.912311+0000", 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2600000000000005e-06 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.948058+0000", 2023-10-09T06:07:10.669 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.035746880000000002 2023-10-09T06:07:10.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.950039+0000", 2023-10-09T06:07:10.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019812639999999999 2023-10-09T06:07:10.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.670 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.670 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.670 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.995081+0000", 2023-10-09T06:07:10.670 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.045041957000000001 2023-10-09T06:07:10.670 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:00.997213+0000", 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0021322630000000001 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.336716+0000", 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.33950248100000002 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.671 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.338505+0000", 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001789014 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.341903+0000", 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0033980049999999999 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.379700+0000", 2023-10-09T06:07:10.672 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037796757 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.380329+0000", 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00062980000000000002 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.383065+0000", 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002736048 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.673 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.384001+0000", 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00093541000000000002 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:01.387837+0000", 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0038363540000000002 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.674 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131116 10.7 10:e33ebdbf:::benchmark_data_smithi008_634586_object43951:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:03.762291+0000", 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.831124303999999, 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.470304528, 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131116 2023-10-09T06:07:10.675 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762291+0000", 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762291+0000", 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.676 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762290+0000", 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762293+0000", 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.6419999999999999e-06 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762298+0000", 2023-10-09T06:07:10.677 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.3820000000000003e-06 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762303+0000", 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.5700000000000003e-06 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.803103+0000", 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.040799996999999998 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.678 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.804840+0000", 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001736783 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.842059+0000", 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037219575999999997 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.679 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.843826+0000", 2023-10-09T06:07:10.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017666940000000001 2023-10-09T06:07:10.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.195979+0000", 2023-10-09T06:07:10.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.35215278999999999 2023-10-09T06:07:10.680 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.680 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.680 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.680 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.197315+0000", 2023-10-09T06:07:10.680 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013359349999999999 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.199790+0000", 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0024753660000000001 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.216752+0000", 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.016961798 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.681 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.225991+0000", 2023-10-09T06:07:10.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0092386180000000005 2023-10-09T06:07:10.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.228985+0000", 2023-10-09T06:07:10.682 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029945319999999998 2023-10-09T06:07:10.682 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.682 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.682 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.682 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.229712+0000", 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00072722200000000002 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.232595+0000", 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028829459999999999 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.683 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131119 10.7 10:e505ebc7:::benchmark_data_smithi008_634586_object43954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:03.762307+0000", 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.831108044000004, 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.51268815499999998, 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131119 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.684 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762307+0000", 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762307+0000", 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762307+0000", 2023-10-09T06:07:10.685 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:10.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762309+0000", 2023-10-09T06:07:10.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9879999999999999e-06 2023-10-09T06:07:10.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.686 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.686 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.686 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762314+0000", 2023-10-09T06:07:10.686 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.9030000000000003e-06 2023-10-09T06:07:10.686 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.762318+0000", 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4.4750000000000004e-06 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.805073+0000", 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.042755023000000003 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.687 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.807075+0000", 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020021000000000001 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.862428+0000", 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.055353254999999997 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.864261+0000", 2023-10-09T06:07:10.688 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00183285 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.241449+0000", 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37718754999999998 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.243102+0000", 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016533030000000001 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.689 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.245866+0000", 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027641250000000001 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.260091+0000", 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014224982000000001 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.690 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.260547+0000", 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00045532199999999998 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.270714+0000", 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010167168000000001 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.271400+0000", 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00068646199999999999 2023-10-09T06:07:10.691 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.274995+0000", 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003594917 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131153 10.7 10:ebccfc82:::benchmark_data_smithi008_634586_object43988:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:03.819275+0000", 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.774140236999997, 2023-10-09T06:07:10.692 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.49852025799999999, 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131153 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.819275+0000", 2023-10-09T06:07:10.693 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.819275+0000", 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.819273+0000", 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.694 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.819282+0000", 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.8079999999999994e-06 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.819296+0000", 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.4374e-05 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.819305+0000", 2023-10-09T06:07:10.695 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0299999999999999e-06 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.865209+0000", 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.045903397999999998 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.867098+0000", 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001888895 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.696 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.016220+0000", 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.14912220000000001 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.017972+0000", 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017526779999999999 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.284742+0000", 2023-10-09T06:07:10.697 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.26676951399999999 2023-10-09T06:07:10.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.286204+0000", 2023-10-09T06:07:10.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0014623189999999999 2023-10-09T06:07:10.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.698 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.698 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.698 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.288666+0000", 2023-10-09T06:07:10.698 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002461815 2023-10-09T06:07:10.698 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.303526+0000", 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014860207 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.304048+0000", 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00052213599999999998 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.699 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.314251+0000", 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.010202243 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.314941+0000", 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00069047399999999995 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.317795+0000", 2023-10-09T06:07:10.700 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0028539450000000001 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131164 10.7 10:e4f06df6:::benchmark_data_smithi008_634586_object43999:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:03.836467+0000", 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.756948480999995, 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.51162761499999998, 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.701 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131164 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.836467+0000", 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.702 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.836467+0000", 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.836465+0000", 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.836472+0000", 2023-10-09T06:07:10.703 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2130000000000002e-06 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.836484+0000", 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2184000000000001e-05 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.836493+0000", 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.7390000000000005e-06 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.704 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.867327+0000", 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.030833719999999998 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.869332+0000", 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020055419999999999 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.705 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.027212+0000", 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.157879993 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.028981+0000", 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0017689260000000001 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.326420+0000", 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.29743834400000002 2023-10-09T06:07:10.706 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.327719+0000", 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001298974 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.330187+0000", 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002467964 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.707 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.343899+0000", 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013711872 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.344040+0000", 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00014133700000000001 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.344746+0000", 2023-10-09T06:07:10.708 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00070587099999999995 2023-10-09T06:07:10.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.345433+0000", 2023-10-09T06:07:10.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00068696000000000002 2023-10-09T06:07:10.709 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.709 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.709 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.709 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.348094+0000", 2023-10-09T06:07:10.709 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0026614770000000002 2023-10-09T06:07:10.709 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.710 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.710 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.710 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.710 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.710 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131166 10.7 10:e10f4c8e:::benchmark_data_smithi008_634586_object44001:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.710 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:03.841115+0000", 2023-10-09T06:07:10.710 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.752300051999995, 2023-10-09T06:07:10.710 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.54074385899999999, 2023-10-09T06:07:10.710 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.710 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.710 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131166 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.841115+0000", 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.711 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.841115+0000", 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.841113+0000", 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.841121+0000", 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.3239999999999997e-06 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.712 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.841134+0000", 2023-10-09T06:07:10.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.3242e-05 2023-10-09T06:07:10.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.713 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.841142+0000", 2023-10-09T06:07:10.713 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.4670000000000004e-06 2023-10-09T06:07:10.713 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.713 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.713 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.029606+0000", 2023-10-09T06:07:10.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.188463885 2023-10-09T06:07:10.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.031489+0000", 2023-10-09T06:07:10.714 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0018828849999999999 2023-10-09T06:07:10.714 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.714 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.714 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.714 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.045236+0000", 2023-10-09T06:07:10.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013747149 2023-10-09T06:07:10.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.046995+0000", 2023-10-09T06:07:10.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001758243 2023-10-09T06:07:10.715 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.715 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.715 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.715 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.361518+0000", 2023-10-09T06:07:10.715 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.31452322300000002 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.363008+0000", 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001489907 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.366353+0000", 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003345013 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.716 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.376251+0000", 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0098983669999999999 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.376740+0000", 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00048925499999999996 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.377387+0000", 2023-10-09T06:07:10.717 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00064639500000000002 2023-10-09T06:07:10.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.378328+0000", 2023-10-09T06:07:10.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00094134100000000005 2023-10-09T06:07:10.718 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.718 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.718 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.718 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.381859+0000", 2023-10-09T06:07:10.718 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035307400000000001 2023-10-09T06:07:10.718 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131169 10.7 10:e20fbaed:::benchmark_data_smithi008_634586_object44004:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:03.849859+0000", 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.743555927000003, 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.56159287899999999, 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.719 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131169 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.849859+0000", 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.849859+0000", 2023-10-09T06:07:10.720 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.849858+0000", 2023-10-09T06:07:10.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:10.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.721 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.721 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.721 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.849866+0000", 2023-10-09T06:07:10.721 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.0460000000000006e-06 2023-10-09T06:07:10.721 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.849879+0000", 2023-10-09T06:07:10.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.2954e-05 2023-10-09T06:07:10.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.722 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.722 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.722 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.849887+0000", 2023-10-09T06:07:10.722 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.2460000000000003e-06 2023-10-09T06:07:10.722 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.031676+0000", 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.18178882099999999 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.033343+0000", 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001666949 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.723 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.056258+0000", 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022915517 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.058187+0000", 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019293089999999999 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.382086+0000", 2023-10-09T06:07:10.724 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.32389831000000002 2023-10-09T06:07:10.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.383751+0000", 2023-10-09T06:07:10.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016656609999999999 2023-10-09T06:07:10.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.725 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.725 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.725 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.386575+0000", 2023-10-09T06:07:10.725 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002823997 2023-10-09T06:07:10.725 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.406254+0000", 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019678975000000001 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.406693+0000", 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00043808900000000001 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.726 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.407390+0000", 2023-10-09T06:07:10.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00069752699999999998 2023-10-09T06:07:10.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.727 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.408213+0000", 2023-10-09T06:07:10.727 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00082323599999999998 2023-10-09T06:07:10.727 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.727 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.727 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.727 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.411452+0000", 2023-10-09T06:07:10.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032387330000000001 2023-10-09T06:07:10.728 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.728 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.728 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.728 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.728 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.728 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131174 10.7 10:efdfa8af:::benchmark_data_smithi008_634586_object44009:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.728 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:03.866216+0000", 2023-10-09T06:07:10.728 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.727198598000001, 2023-10-09T06:07:10.728 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.54982194699999998, 2023-10-09T06:07:10.728 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.729 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.729 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.729 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.729 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.729 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131174 2023-10-09T06:07:10.729 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.729 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.729 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.729 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.729 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.866216+0000", 2023-10-09T06:07:10.729 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.866216+0000", 2023-10-09T06:07:10.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.730 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.730 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.730 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.866215+0000", 2023-10-09T06:07:10.730 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:10.730 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.866222+0000", 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 6.72e-06 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.866233+0000", 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.0861e-05 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.731 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:03.866240+0000", 2023-10-09T06:07:10.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.1080000000000004e-06 2023-10-09T06:07:10.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.033562+0000", 2023-10-09T06:07:10.732 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.167322463 2023-10-09T06:07:10.732 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.732 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.732 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.732 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.035637+0000", 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020753099999999999 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.066773+0000", 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.031135975 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.068396+0000", 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016224500000000001 2023-10-09T06:07:10.733 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.396328+0000", 2023-10-09T06:07:10.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.327932218 2023-10-09T06:07:10.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.734 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.734 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.397987+0000", 2023-10-09T06:07:10.734 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016590140000000001 2023-10-09T06:07:10.734 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.734 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.401231+0000", 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032437939999999999 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.406823+0000", 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.005592535 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.411902+0000", 2023-10-09T06:07:10.735 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0050781979999999999 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.412353+0000", 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00045165299999999999 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.413142+0000", 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00078896399999999996 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.736 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.416038+0000", 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.002896123 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131253 10.7 10:eda6af56:::benchmark_data_smithi008_634586_object44088:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:04.244569+0000", 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.348846218999995, 2023-10-09T06:07:10.737 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48870293199999998, 2023-10-09T06:07:10.738 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.738 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.738 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.738 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.738 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.738 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131253 2023-10-09T06:07:10.738 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.738 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.738 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.738 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.738 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.244569+0000", 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.244569+0000", 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.244568+0000", 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.999999 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.739 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.740 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.244573+0000", 2023-10-09T06:07:10.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.1399999999999999e-06 2023-10-09T06:07:10.740 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.740 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.740 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.244582+0000", 2023-10-09T06:07:10.740 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.0790000000000007e-06 2023-10-09T06:07:10.740 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.740 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.740 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.244588+0000", 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.5620000000000003e-06 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.278082+0000", 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.033494838999999998 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.280113+0000", 2023-10-09T06:07:10.741 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020300919999999998 2023-10-09T06:07:10.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.742 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.742 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.742 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.312312+0000", 2023-10-09T06:07:10.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.032199604999999999 2023-10-09T06:07:10.742 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.742 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.742 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.742 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.313821+0000", 2023-10-09T06:07:10.742 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001508459 2023-10-09T06:07:10.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.743 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.691914+0000", 2023-10-09T06:07:10.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.37809322000000001 2023-10-09T06:07:10.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.743 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.743 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.743 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.693751+0000", 2023-10-09T06:07:10.743 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001837001 2023-10-09T06:07:10.743 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.697308+0000", 2023-10-09T06:07:10.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035569999999999998 2023-10-09T06:07:10.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.744 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.744 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.715809+0000", 2023-10-09T06:07:10.744 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018501231 2023-10-09T06:07:10.744 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.744 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.715946+0000", 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00013690499999999999 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.729403+0000", 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.013457258 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.730108+0000", 2023-10-09T06:07:10.745 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00070428199999999998 2023-10-09T06:07:10.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.746 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.746 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.733272+0000", 2023-10-09T06:07:10.746 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031642689999999999 2023-10-09T06:07:10.746 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.746 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.746 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.746 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.746 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.746 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131261 10.7 10:e4f35c83:::benchmark_data_smithi008_634586_object44096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:04.292278+0000", 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.301137319999995, 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.51015021599999999, 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131261 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.747 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.292278+0000", 2023-10-09T06:07:10.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.748 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.292278+0000", 2023-10-09T06:07:10.748 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.748 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.748 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.748 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.292276+0000", 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.292284+0000", 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.5410000000000003e-06 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.292303+0000", 2023-10-09T06:07:10.749 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.9111999999999999e-05 2023-10-09T06:07:10.750 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.750 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.750 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.750 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.292311+0000", 2023-10-09T06:07:10.750 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.6950000000000006e-06 2023-10-09T06:07:10.750 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.750 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.750 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.750 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.318085+0000", 2023-10-09T06:07:10.750 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.025773602999999999 2023-10-09T06:07:10.750 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.320118+0000", 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0020333759999999999 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.356502+0000", 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.036383703000000003 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.751 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.752 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.357994+0000", 2023-10-09T06:07:10.752 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00149194 2023-10-09T06:07:10.752 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.752 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.752 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.752 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.764734+0000", 2023-10-09T06:07:10.752 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.406739984 2023-10-09T06:07:10.752 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.752 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.752 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.752 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.766410+0000", 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016758420000000001 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.770407+0000", 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0039972419999999998 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.796816+0000", 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.026409294 2023-10-09T06:07:10.753 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.754 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.797753+0000", 2023-10-09T06:07:10.754 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0009366 2023-10-09T06:07:10.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.754 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.754 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.798333+0000", 2023-10-09T06:07:10.754 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00057983999999999998 2023-10-09T06:07:10.754 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.754 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.799289+0000", 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00095595900000000004 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.802428+0000", 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0031392410000000001 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.755 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.756 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131263 10.7 10:e54e5e35:::benchmark_data_smithi008_634586_object44098:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.756 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:04.298143+0000", 2023-10-09T06:07:10.756 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.295271881999994, 2023-10-09T06:07:10.756 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48799337700000001, 2023-10-09T06:07:10.756 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.756 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.756 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.756 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.756 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.756 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131263 2023-10-09T06:07:10.756 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.298143+0000", 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.298143+0000", 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.757 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.298142+0000", 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.298149+0000", 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 7.2280000000000001e-06 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.758 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.298179+0000", 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 2.9929000000000001e-05 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.298188+0000", 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.2590000000000006e-06 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.320798+0000", 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.022609614 2023-10-09T06:07:10.759 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.322476+0000", 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016779150000000001 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.341916+0000", 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019440341 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.760 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.343456+0000", 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001540198 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.749732+0000", 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.40627536400000003 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.751127+0000", 2023-10-09T06:07:10.761 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0013956629999999999 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.753831+0000", 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0027034049999999999 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.759511+0000", 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0056805079999999999 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.762 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.760096+0000", 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00058495900000000002 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.781530+0000", 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021433473000000001 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.763 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.782444+0000", 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.000913966 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.786137+0000", 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0036929530000000001 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.764 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131274 10.7 10:ef4fb14c:::benchmark_data_smithi008_634586_object44109:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:04.396000+0000", 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.197415145999997, 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.48089165499999997, 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131274 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.765 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.766 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.766 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.396000+0000", 2023-10-09T06:07:10.766 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.766 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.766 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.766 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.766 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.396000+0000", 2023-10-09T06:07:10.766 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.766 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.766 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.766 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.767 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.395998+0000", 2023-10-09T06:07:10.767 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:10.767 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.767 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.767 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.767 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.396006+0000", 2023-10-09T06:07:10.767 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 8.5380000000000002e-06 2023-10-09T06:07:10.767 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.767 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.767 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.767 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.396024+0000", 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 1.7689e-05 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.396033+0000", 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.3810000000000005e-06 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.417927+0000", 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.021893320000000001 2023-10-09T06:07:10.768 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.419439+0000", 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001512427 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.448243+0000", 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.028803553999999999 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.769 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.449825+0000", 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001581743 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.835437+0000", 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.38561250200000002 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.837075+0000", 2023-10-09T06:07:10.770 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0016383560000000001 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.840306+0000", 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0032306309999999999 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.852364+0000", 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.012057966 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.771 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.852907+0000", 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00054256499999999995 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.872432+0000", 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.019525672000000001 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.873330+0000", 2023-10-09T06:07:10.772 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00089779599999999997 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.876892+0000", 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0035615629999999998 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: "description": "osd_op(client.47036.0:131279 10.7 10:e895ee83:::benchmark_data_smithi008_634586_object44114:head [delete] snapc 0=[] ondisk+write+known_if_redirected e556)", 2023-10-09T06:07:10.773 INFO:teuthology.orchestra.run.smithi008.stdout: "initiated_at": "2023-10-09T06:06:04.413386+0000", 2023-10-09T06:07:10.774 INFO:teuthology.orchestra.run.smithi008.stdout: "age": 66.180029359000002, 2023-10-09T06:07:10.774 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.46934516599999998, 2023-10-09T06:07:10.774 INFO:teuthology.orchestra.run.smithi008.stdout: "type_data": { 2023-10-09T06:07:10.774 INFO:teuthology.orchestra.run.smithi008.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:10.774 INFO:teuthology.orchestra.run.smithi008.stdout: "client_info": { 2023-10-09T06:07:10.774 INFO:teuthology.orchestra.run.smithi008.stdout: "client": "client.47036", 2023-10-09T06:07:10.774 INFO:teuthology.orchestra.run.smithi008.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:10.774 INFO:teuthology.orchestra.run.smithi008.stdout: "tid": 131279 2023-10-09T06:07:10.774 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.774 INFO:teuthology.orchestra.run.smithi008.stdout: "events": [ 2023-10-09T06:07:10.774 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "initiated", 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.413386+0000", 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "throttled", 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.413386+0000", 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "header_read", 2023-10-09T06:07:10.775 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.413385+0000", 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "all_read", 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.413389+0000", 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 3.4199999999999999e-06 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "dispatched", 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.413398+0000", 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 9.6600000000000007e-06 2023-10-09T06:07:10.776 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "queued_for_pg", 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.413404+0000", 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 5.7529999999999998e-06 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.450499+0000", 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.037095040000000003 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.777 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for missing object", 2023-10-09T06:07:10.778 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.452045+0000", 2023-10-09T06:07:10.778 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0015464739999999999 2023-10-09T06:07:10.778 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.778 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.778 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.778 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.470996+0000", 2023-10-09T06:07:10.778 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.018950993999999999 2023-10-09T06:07:10.778 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.778 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.778 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:10.778 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.472584+0000", 2023-10-09T06:07:10.779 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.001587286 2023-10-09T06:07:10.779 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.779 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.779 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "reached_pg", 2023-10-09T06:07:10.779 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.858341+0000", 2023-10-09T06:07:10.779 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.38575729800000003 2023-10-09T06:07:10.779 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.779 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.779 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "started", 2023-10-09T06:07:10.779 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.860323+0000", 2023-10-09T06:07:10.779 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0019820060000000001 2023-10-09T06:07:10.780 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.780 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.780 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:07:10.780 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.863272+0000", 2023-10-09T06:07:10.780 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.0029486590000000002 2023-10-09T06:07:10.780 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.780 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.780 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.780 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.877470+0000", 2023-10-09T06:07:10.780 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.014197914000000001 2023-10-09T06:07:10.780 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "op_commit", 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.877656+0000", 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00018596300000000001 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.878245+0000", 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00058946599999999997 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.781 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "commit_sent", 2023-10-09T06:07:10.782 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.879108+0000", 2023-10-09T06:07:10.782 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.00086279100000000001 2023-10-09T06:07:10.782 INFO:teuthology.orchestra.run.smithi008.stdout: }, 2023-10-09T06:07:10.782 INFO:teuthology.orchestra.run.smithi008.stdout: { 2023-10-09T06:07:10.782 INFO:teuthology.orchestra.run.smithi008.stdout: "event": "done", 2023-10-09T06:07:10.782 INFO:teuthology.orchestra.run.smithi008.stdout: "time": "2023-10-09T06:06:04.882731+0000", 2023-10-09T06:07:10.782 INFO:teuthology.orchestra.run.smithi008.stdout: "duration": 0.003623018 2023-10-09T06:07:10.782 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.782 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.782 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.782 INFO:teuthology.orchestra.run.smithi008.stdout: } 2023-10-09T06:07:10.783 INFO:teuthology.orchestra.run.smithi008.stdout: ] 2023-10-09T06:07:10.783 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:07:10.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:10.784 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:07:10.784 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:10.784 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:07:10.814 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:10.842 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:07:10.885 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:10.889 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:07:10.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:10.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:10.907 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:07:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:07:10] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.33.4" 2023-10-09T06:07:10.926 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:07:10.976 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:07:10.992 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:11.026 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:07:11.028 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:11.063 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:07:11.070 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:07:11.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:07:11.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:07:11.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:07:11.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:07:11.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:07:11.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:07:11.071 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:07:11.081 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:11.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:11.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:11.186 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:11.217 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:11 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4106593543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:11.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:11 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4106593543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:11.288 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:11.387 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:11.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:11.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:11.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:11.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:11 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4106593543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:11.555 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:11.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:11.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:11.696 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:07:11.722 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:11.811 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:11.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:11.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:11.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:11.959 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:12.006 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:12.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:12.106 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:12 smithi008 ceph-mon[184418]: pgmap v1727: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 774 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 13803/37005 objects degraded (37.300%); 652/37005 objects misplaced (1.762%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:07:12.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:12.144 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:12.232 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:12.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:12.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:12.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:12.373 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:12 smithi191 ceph-mon[161669]: pgmap v1727: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 774 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 13803/37005 objects degraded (37.300%); 652/37005 objects misplaced (1.762%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:07:12.376 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:12.451 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:12.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:12.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:12.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:12 smithi008 ceph-mon[191227]: pgmap v1727: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 774 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 529 op/s; 13803/37005 objects degraded (37.300%); 652/37005 objects misplaced (1.762%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:07:12.559 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:12.635 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:12.837 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:07:12.901 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:12.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:12.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:13.015 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:13.117 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:13.255 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:13.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:13.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:13.322 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:13.362 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:13.401 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:13.447 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:13.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:13.551 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:13.589 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:13.688 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:13.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:13.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:13.793 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:13.861 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:13.982 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:14.036 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:14.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:14.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:14.128 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:14 smithi191 ceph-mon[161669]: pgmap v1728: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 654 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 13622/31266 objects degraded (43.568%); 652/31266 objects misplaced (2.085%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:07:14.129 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:14 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13622/31266 objects degraded (43.568%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:14.129 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:14.149 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:14.189 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:14.268 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:14.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:14.374 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:14.408 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:14 smithi008 ceph-mon[184418]: pgmap v1728: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 654 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 13622/31266 objects degraded (43.568%); 652/31266 objects misplaced (2.085%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:07:14.408 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:14 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13622/31266 objects degraded (43.568%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:14.408 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:14.409 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:14 smithi008 ceph-mon[191227]: pgmap v1728: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 654 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 13622/31266 objects degraded (43.568%); 652/31266 objects misplaced (2.085%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:07:14.409 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:14 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13622/31266 objects degraded (43.568%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:14.409 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:14.483 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:14.757 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:07:14.881 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:14.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:14.998 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:15.165 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:15.846 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:15.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:15.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:15.957 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:15.991 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:16.069 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:16.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:16.174 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:16.219 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:07:16.245 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:16.262 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:16.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:16 smithi191 ceph-mon[161669]: pgmap v1729: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 603 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 13666/28791 objects degraded (47.466%); 493/28791 objects misplaced (1.712%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T06:07:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:16 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/684902605' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:07:16.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:16 smithi008 ceph-mon[184418]: pgmap v1729: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 603 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 13666/28791 objects degraded (47.466%); 493/28791 objects misplaced (1.712%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T06:07:16.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:16 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/684902605' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:07:16.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:16 smithi008 ceph-mon[191227]: pgmap v1729: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 603 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 13666/28791 objects degraded (47.466%); 493/28791 objects misplaced (1.712%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T06:07:16.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:16 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/684902605' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:07:16.509 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:16.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:16.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:16.623 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:16.708 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:16.708 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:16.750 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:17.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:17 smithi008 ceph-mon[184418]: failed to encode map e566 with expected crc 2023-10-09T06:07:17.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:17 smithi008 ceph-mon[184418]: failed to encode map e566 with expected crc 2023-10-09T06:07:17.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/684902605' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:07:17.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:17 smithi008 ceph-mon[184418]: osdmap e566: 8 total, 8 up, 7 in 2023-10-09T06:07:17.220 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:17 smithi008 ceph-mon[184418]: failed to encode map e566 with expected crc 2023-10-09T06:07:17.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:17 smithi008 ceph-mon[191227]: failed to encode map e566 with expected crc 2023-10-09T06:07:17.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:17 smithi008 ceph-mon[191227]: failed to encode map e566 with expected crc 2023-10-09T06:07:17.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/684902605' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:07:17.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:17 smithi008 ceph-mon[191227]: osdmap e566: 8 total, 8 up, 7 in 2023-10-09T06:07:17.220 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:17 smithi008 ceph-mon[191227]: failed to encode map e566 with expected crc 2023-10-09T06:07:17.461 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:17.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:17.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:17 smithi191 ceph-mon[161669]: failed to encode map e566 with expected crc 2023-10-09T06:07:17.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:17 smithi191 ceph-mon[161669]: failed to encode map e566 with expected crc 2023-10-09T06:07:17.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/684902605' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:07:17.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:17 smithi191 ceph-mon[161669]: osdmap e566: 8 total, 8 up, 7 in 2023-10-09T06:07:17.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:17 smithi191 ceph-mon[161669]: failed to encode map e566 with expected crc 2023-10-09T06:07:17.569 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:17.587 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:07:17.606 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:17.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:07:17.703 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:17.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:17.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:17.816 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:17.961 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:18.072 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:18.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:18.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:18.159 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:18.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:18.209 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:18.223 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:07:18.279 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:18.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:18.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:18.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:18.429 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:18 smithi008 ceph-mon[184418]: pgmap v1731: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 503 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 13500/23991 objects degraded (56.271%); 493/23991 objects misplaced (2.055%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:07:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:18 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/241555369' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:18.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:18 smithi008 ceph-mon[191227]: pgmap v1731: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 503 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 13500/23991 objects degraded (56.271%); 493/23991 objects misplaced (2.055%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:07:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:18 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/241555369' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:18.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:18.482 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:18.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:18.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:18.488 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:18 smithi191 ceph-mon[161669]: pgmap v1731: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 503 MiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 13500/23991 objects degraded (56.271%); 493/23991 objects misplaced (2.055%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:07:18.489 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:18 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/241555369' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:18.489 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:18.589 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:18.692 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:18.805 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:18.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:18.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:18.924 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:18.952 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:19.360 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:19.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:19.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:19.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:19.484 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:07:19.484 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[184418]: failed to encode map e567 with expected crc 2023-10-09T06:07:19.484 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[184418]: failed to encode map e567 with expected crc 2023-10-09T06:07:19.484 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:19.484 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[184418]: osdmap e567: 8 total, 8 up, 7 in 2023-10-09T06:07:19.485 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[184418]: failed to encode map e567 with expected crc 2023-10-09T06:07:19.485 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13500/23991 objects degraded (56.271%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:19.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:07:19.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[191227]: failed to encode map e567 with expected crc 2023-10-09T06:07:19.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[191227]: failed to encode map e567 with expected crc 2023-10-09T06:07:19.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:19.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[191227]: osdmap e567: 8 total, 8 up, 7 in 2023-10-09T06:07:19.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[191227]: failed to encode map e567 with expected crc 2023-10-09T06:07:19.485 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:19 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13500/23991 objects degraded (56.271%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:19.531 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:19 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:07:19.531 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:19 smithi191 ceph-mon[161669]: failed to encode map e567 with expected crc 2023-10-09T06:07:19.531 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:19 smithi191 ceph-mon[161669]: failed to encode map e567 with expected crc 2023-10-09T06:07:19.531 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:19 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:19.531 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:19 smithi191 ceph-mon[161669]: osdmap e567: 8 total, 8 up, 7 in 2023-10-09T06:07:19.531 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:19 smithi191 ceph-mon[161669]: failed to encode map e567 with expected crc 2023-10-09T06:07:19.531 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:19 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13500/23991 objects degraded (56.271%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:19.537 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:19.651 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:19.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:19.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:19.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:19.802 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:19.880 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:19.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:19.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:19.911 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:19.990 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:20.038 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:20.039 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:20.090 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:07:20.102 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:20.130 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:07:20.191 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:07:20.243 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:07:20.290 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:07:20.335 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:07:20.351 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:20.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:20.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:20.399 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:07:20.444 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:07:20.458 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:20.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:07:20.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:07:20.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:07:20.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:07:20.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:07:20.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:07:20.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:07:20.464 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:07:20.484 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:20.527 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:20.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:20.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:20.631 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:20.676 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:20.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:20 smithi008 ceph-mon[184418]: pgmap v1733: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 493 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 13500/23508 objects degraded (57.427%); 493/23508 objects misplaced (2.097%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:07:20.719 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:20 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/377838778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:20.719 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:07:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:07:20] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.33.4" 2023-10-09T06:07:20.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:20 smithi008 ceph-mon[191227]: pgmap v1733: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 493 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 13500/23508 objects degraded (57.427%); 493/23508 objects misplaced (2.097%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:07:20.720 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:20 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/377838778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:20 smithi191 ceph-mon[161669]: pgmap v1733: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 493 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 13500/23508 objects degraded (57.427%); 493/23508 objects misplaced (2.097%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:07:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:20 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/377838778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:20.946 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:20.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:20.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:21.032 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:07:21.055 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:21.135 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:21.200 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:21.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:21.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:21.305 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:21.424 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:21.487 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:07:21.514 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:21.986 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:22.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:22.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:22.101 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:22.186 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:22.382 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:22.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:22.463 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:22.486 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:22.502 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:22 smithi191 ceph-mon[161669]: pgmap v1734: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 446 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 415 op/s; 13533/21285 objects degraded (63.580%); 352/21285 objects misplaced (1.654%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:07:22.584 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:22.641 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:22 smithi008 ceph-mon[184418]: pgmap v1734: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 446 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 415 op/s; 13533/21285 objects degraded (63.580%); 352/21285 objects misplaced (1.654%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:07:22.642 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:22 smithi008 ceph-mon[191227]: pgmap v1734: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 446 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 415 op/s; 13533/21285 objects degraded (63.580%); 352/21285 objects misplaced (1.654%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:07:22.802 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:22.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:22.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:22.954 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:23.025 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:23.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:23.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:23.131 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:23.180 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:23.308 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:23.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:23.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:23.415 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:23.465 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:07:23.465 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T06:07:23.465 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T06:07:23.465 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T06:07:23.465 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.465 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:63144 10.4 10:2f798f30:::benchmark_data_smithi008_634586_object63143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:07:23.465 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:19.670240+0000", 2023-10-09T06:07:23.465 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 243.79027316899999, 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.321828318, 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 63144 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670240+0000", 2023-10-09T06:07:23.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.467 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.467 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670240+0000", 2023-10-09T06:07:23.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:23.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.467 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.467 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670238+0000", 2023-10-09T06:07:23.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8511000000000003e-05 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670297+0000", 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.396e-05 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670311+0000", 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4899999999999997e-06 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.468 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670317+0000", 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4267000000000002e-05 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670362+0000", 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000140066 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.469 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670502+0000", 2023-10-09T06:07:23.470 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000169261 2023-10-09T06:07:23.470 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.470 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.470 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:07:23.470 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.670671+0000", 2023-10-09T06:07:23.470 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020061380000000002 2023-10-09T06:07:23.470 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.470 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.470 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.470 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.672677+0000", 2023-10-09T06:07:23.470 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041772579999999997 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.676854+0000", 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31327599900000003 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.990130+0000", 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7629000000000002e-05 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.472 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.472 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.990168+0000", 2023-10-09T06:07:23.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019004899999999999 2023-10-09T06:07:23.472 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.472 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.472 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.472 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.992068+0000", 2023-10-09T06:07:23.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.321828318 2023-10-09T06:07:23.472 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.472 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.473 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.473 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.473 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:63154 10.4 10:2bf45174:::benchmark_data_smithi008_634586_object63153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:07:23.473 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:19.676995+0000", 2023-10-09T06:07:23.473 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 243.78351805299999, 2023-10-09T06:07:23.473 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31648402199999998, 2023-10-09T06:07:23.473 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.473 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.473 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.473 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.474 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.474 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 63154 2023-10-09T06:07:23.474 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.474 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.474 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.474 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.676995+0000", 2023-10-09T06:07:23.474 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.474 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.676995+0000", 2023-10-09T06:07:23.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:23.475 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.676994+0000", 2023-10-09T06:07:23.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2188000000000003e-05 2023-10-09T06:07:23.475 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.476 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677036+0000", 2023-10-09T06:07:23.476 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2265e-05 2023-10-09T06:07:23.476 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.476 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.476 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.476 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677048+0000", 2023-10-09T06:07:23.476 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.242e-06 2023-10-09T06:07:23.476 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.476 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.476 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677054+0000", 2023-10-09T06:07:23.477 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9261999999999998e-05 2023-10-09T06:07:23.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.477 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.477 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677093+0000", 2023-10-09T06:07:23.477 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013400700000000001 2023-10-09T06:07:23.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.477 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.477 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677227+0000", 2023-10-09T06:07:23.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014552799999999999 2023-10-09T06:07:23.478 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.478 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.478 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:07:23.478 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.677372+0000", 2023-10-09T06:07:23.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001802884 2023-10-09T06:07:23.478 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.478 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.478 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.478 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.679175+0000", 2023-10-09T06:07:23.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010563809999999999 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.680232+0000", 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.311896966 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.992129+0000", 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6134000000000002e-05 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.479 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.480 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.480 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.992185+0000", 2023-10-09T06:07:23.480 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012944129999999999 2023-10-09T06:07:23.480 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.480 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.480 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.480 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.993479+0000", 2023-10-09T06:07:23.480 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.31648402199999998 2023-10-09T06:07:23.480 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.480 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.480 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:63165 10.4 10:2b8ca226:::benchmark_data_smithi008_634586_object63164:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:19.690274+0000", 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 243.77023887499999, 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30505157300000002, 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.481 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 63165 2023-10-09T06:07:23.482 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.482 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.482 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.482 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.482 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690274+0000", 2023-10-09T06:07:23.482 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.482 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.482 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.482 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.482 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690274+0000", 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690273+0000", 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6658e-05 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690309+0000", 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5274e-05 2023-10-09T06:07:23.483 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.484 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.484 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.484 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690325+0000", 2023-10-09T06:07:23.484 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6409999999999996e-06 2023-10-09T06:07:23.484 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.484 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.484 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.484 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690331+0000", 2023-10-09T06:07:23.484 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2237000000000006e-05 2023-10-09T06:07:23.484 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.484 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690413+0000", 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018728399999999999 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690601+0000", 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000158632 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:07:23.485 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.690759+0000", 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001824401 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.692584+0000", 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094702999999999996 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.693531+0000", 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.300027186 2023-10-09T06:07:23.486 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.487 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.487 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.487 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.993558+0000", 2023-10-09T06:07:23.487 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6448999999999999e-05 2023-10-09T06:07:23.487 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.487 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.487 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.487 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.993634+0000", 2023-10-09T06:07:23.487 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016915840000000001 2023-10-09T06:07:23.487 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.487 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.995326+0000", 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.30505157300000002 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:63172 10.4 10:2721e6a1:::benchmark_data_smithi008_634586_object63171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:19.698313+0000", 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 243.76220072999999, 2023-10-09T06:07:23.488 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29883361000000003, 2023-10-09T06:07:23.489 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.489 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.489 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.489 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.489 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.489 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 63172 2023-10-09T06:07:23.489 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.489 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.489 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.489 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.489 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698313+0000", 2023-10-09T06:07:23.490 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.490 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.490 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.490 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.490 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698313+0000", 2023-10-09T06:07:23.490 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:23.490 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.490 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.490 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.490 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698311+0000", 2023-10-09T06:07:23.490 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5070999999999997e-05 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698346+0000", 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.279e-05 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698359+0000", 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3580000000000002e-06 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.491 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698364+0000", 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0896999999999998e-05 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698405+0000", 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000130159 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.492 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698535+0000", 2023-10-09T06:07:23.493 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019234499999999999 2023-10-09T06:07:23.493 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.493 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.493 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:07:23.493 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.698727+0000", 2023-10-09T06:07:23.493 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001770095 2023-10-09T06:07:23.493 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.493 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.493 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.493 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.700498+0000", 2023-10-09T06:07:23.493 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00108668 2023-10-09T06:07:23.494 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.494 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.494 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.494 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.701584+0000", 2023-10-09T06:07:23.494 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29380753199999998 2023-10-09T06:07:23.494 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.494 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.494 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.494 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.995392+0000", 2023-10-09T06:07:23.494 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.004e-05 2023-10-09T06:07:23.494 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.995462+0000", 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016843330000000001 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:19.997146+0000", 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29883361000000003 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.495 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:83465 10.4 10:236eb373:::benchmark_data_smithi008_634586_object83464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e526)", 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:49.960539+0000", 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 213.499974425, 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27214083900000002, 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.496 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 83465 2023-10-09T06:07:23.497 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.497 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.497 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.497 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.497 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960539+0000", 2023-10-09T06:07:23.497 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.497 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.497 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.497 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.497 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960539+0000", 2023-10-09T06:07:23.497 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960537+0000", 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000148379 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960685+0000", 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6019e-05 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.498 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960701+0000", 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.708e-06 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960708+0000", 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5919000000000004e-05 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.499 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960774+0000", 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000146866 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.960921+0000", 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020136000000000001 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.961122+0000", 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0062320800000000001 2023-10-09T06:07:23.500 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.501 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.501 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.501 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.967354+0000", 2023-10-09T06:07:23.501 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2861000000000001e-05 2023-10-09T06:07:23.501 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.501 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.501 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.501 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.967407+0000", 2023-10-09T06:07:23.501 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.26340967399999998 2023-10-09T06:07:23.501 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.501 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.502 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.502 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.230817+0000", 2023-10-09T06:07:23.502 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5763999999999997e-05 2023-10-09T06:07:23.502 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.502 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.502 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.502 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.230873+0000", 2023-10-09T06:07:23.502 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018070499999999999 2023-10-09T06:07:23.502 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.505 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.505 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.505 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.232680+0000", 2023-10-09T06:07:23.505 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27214083900000002 2023-10-09T06:07:23.505 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.505 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.505 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.506 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.506 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.506 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:83467 10.4 10:28157da3:::benchmark_data_smithi008_634586_object83466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e526)", 2023-10-09T06:07:23.506 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:03:49.965092+0000", 2023-10-09T06:07:23.506 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 213.495421478, 2023-10-09T06:07:23.506 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.269243485, 2023-10-09T06:07:23.506 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.506 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.506 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.506 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.506 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 83467 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965092+0000", 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965092+0000", 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:23.507 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965090+0000", 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7626000000000003e-05 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965188+0000", 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2163e-05 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.508 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965200+0000", 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3829999999999997e-06 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.965207+0000", 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022278089999999999 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.967434+0000", 2023-10-09T06:07:23.509 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017571899999999999 2023-10-09T06:07:23.510 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.510 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.510 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.510 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.967610+0000", 2023-10-09T06:07:23.510 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021979599999999999 2023-10-09T06:07:23.510 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.510 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.510 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:07:23.510 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.967830+0000", 2023-10-09T06:07:23.510 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026064780000000002 2023-10-09T06:07:23.510 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.511 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.511 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.511 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.970436+0000", 2023-10-09T06:07:23.511 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033157719999999998 2023-10-09T06:07:23.511 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.511 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.511 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.511 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:49.973752+0000", 2023-10-09T06:07:23.511 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.258994418 2023-10-09T06:07:23.511 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.511 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.512 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.512 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.232747+0000", 2023-10-09T06:07:23.512 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4076999999999998e-05 2023-10-09T06:07:23.512 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.512 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.512 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.512 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.232791+0000", 2023-10-09T06:07:23.512 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015446609999999999 2023-10-09T06:07:23.512 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.512 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:03:50.234335+0000", 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.269243485 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:96490 10.d 10:bd3938ce:::benchmark_data_smithi008_634586_object9325:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:04:39.104315+0000", 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 164.35619839500001, 2023-10-09T06:07:23.513 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.256447588, 2023-10-09T06:07:23.514 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.514 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.514 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.514 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.514 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.514 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 96490 2023-10-09T06:07:23.514 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.514 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.514 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.514 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.514 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104315+0000", 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104315+0000", 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104314+0000", 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8289999999999999e-06 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.515 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104317+0000", 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7299999999999994e-06 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104326+0000", 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8e-06 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104330+0000", 2023-10-09T06:07:23.516 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002211845 2023-10-09T06:07:23.517 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.517 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.517 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.517 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.106542+0000", 2023-10-09T06:07:23.517 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00065125600000000001 2023-10-09T06:07:23.517 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.517 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.517 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.517 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.107193+0000", 2023-10-09T06:07:23.517 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.03810931 2023-10-09T06:07:23.517 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.145302+0000", 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5633e-05 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.145368+0000", 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010242300000000001 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.518 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.145470+0000", 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014231242999999999 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.159701+0000", 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028590120000000002 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.162560+0000", 2023-10-09T06:07:23.519 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.194264938 2023-10-09T06:07:23.520 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.520 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.520 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.520 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.356825+0000", 2023-10-09T06:07:23.520 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7241999999999998e-05 2023-10-09T06:07:23.520 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.520 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.520 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.520 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.356883+0000", 2023-10-09T06:07:23.520 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038798109999999999 2023-10-09T06:07:23.520 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.360762+0000", 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.256447588 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:96496 10.d 10:b1be06e8:::benchmark_data_smithi008_634586_object9331:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:04:39.104798+0000", 2023-10-09T06:07:23.521 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 164.35571536699999, 2023-10-09T06:07:23.522 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25924583099999998, 2023-10-09T06:07:23.522 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.522 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.522 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.523 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.523 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.523 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 96496 2023-10-09T06:07:23.523 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.523 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.523 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.523 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.523 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104798+0000", 2023-10-09T06:07:23.524 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.524 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.524 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.524 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.524 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104798+0000", 2023-10-09T06:07:23.524 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:07:23.524 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.524 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.524 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.524 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104797+0000", 2023-10-09T06:07:23.524 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4190000000000004e-06 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104802+0000", 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0399999999999998e-06 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104811+0000", 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3870000000000003e-06 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.525 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.104816+0000", 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024788470000000002 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.107295+0000", 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00066429100000000001 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.526 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.107959+0000", 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.044045075000000003 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.152004+0000", 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2082999999999995e-05 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.152066+0000", 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4959999999999999e-05 2023-10-09T06:07:23.527 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.152161+0000", 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010370736 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.162532+0000", 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5365999999999996e-05 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.528 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.162607+0000", 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.198236308 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.360844+0000", 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5483999999999998e-05 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.360909+0000", 2023-10-09T06:07:23.529 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031345639999999998 2023-10-09T06:07:23.530 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.530 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.530 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.530 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.364044+0000", 2023-10-09T06:07:23.530 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25924583099999998 2023-10-09T06:07:23.530 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.530 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.530 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.530 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.530 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.530 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:96499 10.d 10:b047fea9:::benchmark_data_smithi008_634586_object9334:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:04:39.114743+0000", 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 164.34576978699999, 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25254888800000003, 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 96499 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.531 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114743+0000", 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114743+0000", 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114742+0000", 2023-10-09T06:07:23.532 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8010000000000004e-06 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114749+0000", 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2295e-05 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114762+0000", 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6520000000000006e-06 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.533 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114769+0000", 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3637000000000002e-05 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.114823+0000", 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000755129 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.534 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.115578+0000", 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.044154153000000002 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.159732+0000", 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6231999999999998e-05 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.159809+0000", 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108804 2023-10-09T06:07:23.535 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.159917+0000", 2023-10-09T06:07:23.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004328068 2023-10-09T06:07:23.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.536 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.536 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.164245+0000", 2023-10-09T06:07:23.536 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5066999999999999e-05 2023-10-09T06:07:23.536 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.536 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.164271+0000", 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19981747799999999 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.364088+0000", 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8321000000000001e-05 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.364126+0000", 2023-10-09T06:07:23.537 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031660410000000001 2023-10-09T06:07:23.538 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.538 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.538 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.538 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:04:39.367292+0000", 2023-10-09T06:07:23.538 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25254888800000003 2023-10-09T06:07:23.538 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.538 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.538 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.538 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.538 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.538 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:108082 10.d 10:be6cf74f:::benchmark_data_smithi008_634586_object20917:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:02.567231+0000", 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 140.89328272899999, 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23090850800000001, 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 108082 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.539 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567231+0000", 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567231+0000", 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.540 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567229+0000", 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4339999999999998e-06 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567239+0000", 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5401999999999999e-05 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567254+0000", 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1210000000000003e-06 2023-10-09T06:07:23.541 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.567260+0000", 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016035051000000002 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.583295+0000", 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00086684900000000005 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.542 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.584162+0000", 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.031746462000000003 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.615908+0000", 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0943000000000001e-05 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.615989+0000", 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111888 2023-10-09T06:07:23.543 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.616101+0000", 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006815644 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.622917+0000", 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002131675 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.544 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.625049+0000", 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17035199300000001 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.795401+0000", 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2364e-05 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.795473+0000", 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026661060000000001 2023-10-09T06:07:23.545 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.798139+0000", 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23090850800000001 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:108119 10.d 10:b9c774ff:::benchmark_data_smithi008_634586_object20954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:02.621972+0000", 2023-10-09T06:07:23.546 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 140.83854114499999, 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22555293000000001, 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 108119 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.547 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.621972+0000", 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.621972+0000", 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.621970+0000", 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1619999999999997e-06 2023-10-09T06:07:23.548 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.621979+0000", 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5625999999999999e-05 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.621995+0000", 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3629999999999999e-06 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.549 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.622001+0000", 2023-10-09T06:07:23.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00095310800000000004 2023-10-09T06:07:23.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.622954+0000", 2023-10-09T06:07:23.550 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00054953100000000002 2023-10-09T06:07:23.550 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.550 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.550 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.550 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.623504+0000", 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18212404900000001 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.805628+0000", 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000104003 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.805732+0000", 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122263 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.551 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.552 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.805854+0000", 2023-10-09T06:07:23.552 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017202920999999999 2023-10-09T06:07:23.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.552 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.823057+0000", 2023-10-09T06:07:23.552 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021373919000000002 2023-10-09T06:07:23.552 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.552 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.552 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.844431+0000", 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2620000000000002e-05 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.844473+0000", 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6733000000000001e-05 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.844520+0000", 2023-10-09T06:07:23.553 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030050229999999999 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.847525+0000", 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22555293000000001 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:108122 10.d 10:b44ab0c2:::benchmark_data_smithi008_634586_object20957:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:07:23.554 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:02.627034+0000", 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 140.833479622, 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22281990500000001, 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 108122 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.555 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627034+0000", 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627034+0000", 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627032+0000", 2023-10-09T06:07:23.556 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8639999999999996e-06 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627040+0000", 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5248e-05 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627055+0000", 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2979999999999997e-06 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.557 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627061+0000", 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8594999999999998e-05 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627110+0000", 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00050693799999999996 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.558 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.627617+0000", 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18666287200000001 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.814280+0000", 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011455600000000001 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.814394+0000", 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000195837 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.559 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.814590+0000", 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021439581999999999 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.836030+0000", 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011599135 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.560 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.847629+0000", 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7178e-05 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.847686+0000", 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6995e-05 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.847733+0000", 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021206210000000001 2023-10-09T06:07:23.561 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:02.849854+0000", 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22281990500000001 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:109672 10.d 10:ba5db7b9:::benchmark_data_smithi008_634586_object22507:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:06.043231+0000", 2023-10-09T06:07:23.562 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 137.41728226999999, 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.225129208, 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 109672 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.563 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043231+0000", 2023-10-09T06:07:23.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043231+0000", 2023-10-09T06:07:23.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:23.564 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.564 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.564 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.564 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043230+0000", 2023-10-09T06:07:23.564 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8650000000000001e-06 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043238+0000", 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1722999999999999e-05 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043249+0000", 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2510000000000002e-06 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.565 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043254+0000", 2023-10-09T06:07:23.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024191780999999999 2023-10-09T06:07:23.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.067446+0000", 2023-10-09T06:07:23.566 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00061352599999999998 2023-10-09T06:07:23.566 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.566 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.566 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.566 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.068060+0000", 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019991198000000002 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.088051+0000", 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102199 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.088153+0000", 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016991400000000001 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.567 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.568 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.088323+0000", 2023-10-09T06:07:23.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017905299 2023-10-09T06:07:23.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.568 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.106228+0000", 2023-10-09T06:07:23.568 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071681599999999998 2023-10-09T06:07:23.568 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.568 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.568 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.113397+0000", 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.152913827 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.266310+0000", 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6059000000000002e-05 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.266346+0000", 2023-10-09T06:07:23.569 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020137430000000001 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.268360+0000", 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.225129208 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:109678 10.d 10:bc52feb1:::benchmark_data_smithi008_634586_object22513:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:07:23.570 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:06.043464+0000", 2023-10-09T06:07:23.571 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 137.417049195, 2023-10-09T06:07:23.571 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23864223700000001, 2023-10-09T06:07:23.571 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.571 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.571 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.571 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.571 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.571 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 109678 2023-10-09T06:07:23.571 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.571 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.571 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043464+0000", 2023-10-09T06:07:23.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.572 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043464+0000", 2023-10-09T06:07:23.572 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:07:23.572 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.572 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.572 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043463+0000", 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4280000000000001e-06 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043470+0000", 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1616e-05 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043481+0000", 2023-10-09T06:07:23.573 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9960000000000001e-06 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.043486+0000", 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024607812999999999 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.068094+0000", 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00048847400000000003 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.574 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.068582+0000", 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.027396343 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.095979+0000", 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119375 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.575 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.096098+0000", 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020468999999999999 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.096303+0000", 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017102452000000001 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.113405+0000", 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6369000000000001e-05 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.576 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.113432+0000", 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.166464106 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.279896+0000", 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7088999999999997e-05 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.577 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.279933+0000", 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021733949999999998 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.282106+0000", 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23864223700000001 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.578 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:109681 10.d 10:bbcf7de4:::benchmark_data_smithi008_634586_object22516:head [delete] snapc 0=[] ondisk+write+known_if_redirected e543)", 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:06.055511+0000", 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 137.40500204, 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28364116, 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 109681 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.579 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.055511+0000", 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.055511+0000", 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.055510+0000", 2023-10-09T06:07:23.580 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.878e-06 2023-10-09T06:07:23.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.055516+0000", 2023-10-09T06:07:23.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3209e-05 2023-10-09T06:07:23.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.581 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.581 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.581 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.055530+0000", 2023-10-09T06:07:23.581 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9190000000000004e-06 2023-10-09T06:07:23.581 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.055536+0000", 2023-10-09T06:07:23.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013101843 2023-10-09T06:07:23.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.582 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.582 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.068637+0000", 2023-10-09T06:07:23.582 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000512318 2023-10-09T06:07:23.582 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.582 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.069150+0000", 2023-10-09T06:07:23.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.037124996 2023-10-09T06:07:23.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.583 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.583 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.106275+0000", 2023-10-09T06:07:23.583 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9877999999999997e-05 2023-10-09T06:07:23.583 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.583 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.106345+0000", 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010122499999999999 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.106446+0000", 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.007008752 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.113455+0000", 2023-10-09T06:07:23.584 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1709999999999999e-06 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.113461+0000", 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.168686998 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.282148+0000", 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1881999999999997e-05 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.585 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.282180+0000", 2023-10-09T06:07:23.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.056972706999999997 2023-10-09T06:07:23.586 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.586 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.586 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:06.339152+0000", 2023-10-09T06:07:23.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.28364116 2023-10-09T06:07:23.586 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.586 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.586 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:117569 10.d 10:be8322cd:::benchmark_data_smithi008_634586_object30404:head [delete] snapc 0=[] ondisk+write+known_if_redirected e546)", 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:22.683605+0000", 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 120.776907806, 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.247550621, 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117569 2023-10-09T06:07:23.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.588 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.683605+0000", 2023-10-09T06:07:23.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.683605+0000", 2023-10-09T06:07:23.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.683604+0000", 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3690000000000007e-06 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.683612+0000", 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2147000000000001e-05 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.683624+0000", 2023-10-09T06:07:23.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7499999999999998e-06 2023-10-09T06:07:23.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.683634+0000", 2023-10-09T06:07:23.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042013160000000001 2023-10-09T06:07:23.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.687835+0000", 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00069289799999999995 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.688528+0000", 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022816203 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.711345+0000", 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7485999999999997e-05 2023-10-09T06:07:23.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.711412+0000", 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000101817 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.711514+0000", 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21742718899999999 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.928941+0000", 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0954e-05 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.928992+0000", 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8702000000000002e-05 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.929041+0000", 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1474999999999998e-05 2023-10-09T06:07:23.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.929082+0000", 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020739249999999999 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.931156+0000", 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.247550621 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.594 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:117576 10.d 10:b7c8f2b4:::benchmark_data_smithi008_634586_object30411:head [delete] snapc 0=[] ondisk+write+known_if_redirected e546)", 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:22.695739+0000", 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 120.764774366, 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23681416399999999, 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117576 2023-10-09T06:07:23.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.596 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.695739+0000", 2023-10-09T06:07:23.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.695739+0000", 2023-10-09T06:07:23.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:07:23.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.695738+0000", 2023-10-09T06:07:23.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7100000000000004e-06 2023-10-09T06:07:23.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.695744+0000", 2023-10-09T06:07:23.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9089999999999995e-06 2023-10-09T06:07:23.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.695753+0000", 2023-10-09T06:07:23.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5459999999999999e-06 2023-10-09T06:07:23.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.695759+0000", 2023-10-09T06:07:23.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026010159999999998 2023-10-09T06:07:23.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.698360+0000", 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00065713299999999996 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.699017+0000", 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019023258000000001 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.718040+0000", 2023-10-09T06:07:23.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0915000000000002e-05 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.718101+0000", 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4718999999999999e-05 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.718196+0000", 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.21078781399999999 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.928984+0000", 2023-10-09T06:07:23.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5722e-05 2023-10-09T06:07:23.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.929000+0000", 2023-10-09T06:07:23.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002203483 2023-10-09T06:07:23.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.931203+0000", 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2858999999999997e-05 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.931246+0000", 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013071510000000001 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.932553+0000", 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23681416399999999 2023-10-09T06:07:23.602 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:117581 10.d 10:b3c94648:::benchmark_data_smithi008_634586_object30416:head [delete] snapc 0=[] ondisk+write+known_if_redirected e546)", 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:22.699490+0000", 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 120.761023729, 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25939459399999998, 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.603 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117581 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.699490+0000", 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.699490+0000", 2023-10-09T06:07:23.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:07:23.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.699489+0000", 2023-10-09T06:07:23.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9540000000000001e-06 2023-10-09T06:07:23.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.699494+0000", 2023-10-09T06:07:23.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1330000000000005e-06 2023-10-09T06:07:23.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.699504+0000", 2023-10-09T06:07:23.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4059999999999998e-06 2023-10-09T06:07:23.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.699508+0000", 2023-10-09T06:07:23.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9994000000000001e-05 2023-10-09T06:07:23.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.699538+0000", 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00046576700000000001 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.700004+0000", 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.225426397 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.925430+0000", 2023-10-09T06:07:23.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000125726 2023-10-09T06:07:23.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.925556+0000", 2023-10-09T06:07:23.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000212547 2023-10-09T06:07:23.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.925768+0000", 2023-10-09T06:07:23.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00687427 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.932643+0000", 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060138599999999998 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.933244+0000", 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023584193 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.956828+0000", 2023-10-09T06:07:23.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6779999999999997e-05 2023-10-09T06:07:23.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.956865+0000", 2023-10-09T06:07:23.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020190550000000001 2023-10-09T06:07:23.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:22.958884+0000", 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25939459399999998 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:119227 10.d 10:b7760fae:::benchmark_data_smithi008_634586_object32062:head [delete] snapc 0=[] ondisk+write+known_if_redirected e547)", 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:26.463283+0000", 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 116.997230034, 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23434908900000001, 2023-10-09T06:07:23.611 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119227 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463283+0000", 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463283+0000", 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463283+0000", 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1500000000000002e-06 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463285+0000", 2023-10-09T06:07:23.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.703e-06 2023-10-09T06:07:23.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463289+0000", 2023-10-09T06:07:23.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5069999999999999e-06 2023-10-09T06:07:23.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.614 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.614 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463291+0000", 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.1544268 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.617718+0000", 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00056229100000000003 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.618280+0000", 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033375831000000002 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.651656+0000", 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011137 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.651768+0000", 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017087800000000001 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.651938+0000", 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020196266000000001 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.672135+0000", 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0065831199999999996 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.678718+0000", 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016043136999999999 2023-10-09T06:07:23.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.694761+0000", 2023-10-09T06:07:23.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1495999999999997e-05 2023-10-09T06:07:23.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.694802+0000", 2023-10-09T06:07:23.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028299029999999999 2023-10-09T06:07:23.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.697632+0000", 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23434908900000001 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.47036.0:119228 10.d 10:b0bd352a:::benchmark_data_smithi008_634586_object32063:head [delete] snapc 0=[] ondisk+write+known_if_redirected e547)", 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:05:26.463294+0000", 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 116.997219278, 2023-10-09T06:07:23.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23690137, 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.47036", 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/21882678", 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 119228 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463294+0000", 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:23.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463294+0000", 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463294+0000", 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6649999999999999e-06 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463295+0000", 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9890000000000001e-06 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463298+0000", 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0360000000000001e-06 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.463300+0000", 2023-10-09T06:07:23.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15502785499999999 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.618328+0000", 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000448448 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.618777+0000", 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.039898386000000001 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.658675+0000", 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106069 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.658781+0000", 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000200681 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:07:23.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.658982+0000", 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016308368 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.675290+0000", 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0058231619999999998 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.681113+0000", 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016603697000000001 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.697717+0000", 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3943000000000007e-05 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.697791+0000", 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024043490000000001 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:23.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:05:26.700195+0000", 2023-10-09T06:07:23.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23690137 2023-10-09T06:07:23.627 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.627 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.627 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.627 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:23.627 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:23.627 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:07:23.628 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:23.628 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:07:23.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:23.629 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:23.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:23.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:23.755 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:23 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2255720590' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:07:23.755 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:23 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:07:23.756 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:23 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 13533/21285 objects degraded (63.580%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:23.758 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:23.844 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:23.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:23.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:23.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:23.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:23 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2255720590' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:07:23.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:23 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:07:23.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:23 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 13533/21285 objects degraded (63.580%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:23 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2255720590' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:07:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:23 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:07:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:23 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 13533/21285 objects degraded (63.580%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:23.977 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:24.024 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:24.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:24.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:24.089 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:07:24.128 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:24.153 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:24.314 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:07:24.330 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:24.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:24.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:24.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:24.464 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:24.516 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:24.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:24.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:24 smithi191 ceph-mon[161669]: pgmap v1735: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 340 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 13384/16182 objects degraded (82.709%); 352/16182 objects misplaced (2.175%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:07:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:24 smithi191 ceph-mon[161669]: failed to encode map e568 with expected crc 2023-10-09T06:07:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:24 smithi191 ceph-mon[161669]: failed to encode map e568 with expected crc 2023-10-09T06:07:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:24 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:07:24.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:24 smithi191 ceph-mon[161669]: osdmap e568: 8 total, 8 up, 7 in 2023-10-09T06:07:24.622 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:24.674 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:24.735 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:24.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:24.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:24.795 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:24.844 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:24.847 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:24 smithi008 ceph-mon[184418]: pgmap v1735: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 340 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 13384/16182 objects degraded (82.709%); 352/16182 objects misplaced (2.175%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:07:24.848 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:24 smithi008 ceph-mon[184418]: failed to encode map e568 with expected crc 2023-10-09T06:07:24.848 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:24 smithi008 ceph-mon[184418]: failed to encode map e568 with expected crc 2023-10-09T06:07:24.848 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:24 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:07:24.848 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:24 smithi008 ceph-mon[184418]: osdmap e568: 8 total, 8 up, 7 in 2023-10-09T06:07:24.848 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:24 smithi008 ceph-mon[191227]: pgmap v1735: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 340 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 13384/16182 objects degraded (82.709%); 352/16182 objects misplaced (2.175%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:07:24.848 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:24 smithi008 ceph-mon[191227]: failed to encode map e568 with expected crc 2023-10-09T06:07:24.848 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:24 smithi008 ceph-mon[191227]: failed to encode map e568 with expected crc 2023-10-09T06:07:24.849 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:24 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:07:24.849 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:24 smithi008 ceph-mon[191227]: osdmap e568: 8 total, 8 up, 7 in 2023-10-09T06:07:24.893 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:24.942 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:07:24.965 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:24.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:24.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:25.069 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:25.101 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:25.175 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:25.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:25.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:25.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:25.309 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:25.362 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:25.459 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:25.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:25.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:25.480 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:25.562 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:25.637 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:25.740 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:25.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:25.843 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:25.869 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:25 smithi008 ceph-mon[184418]: failed to encode map e569 with expected crc 2023-10-09T06:07:25.869 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:25 smithi008 ceph-mon[184418]: failed to encode map e569 with expected crc 2023-10-09T06:07:25.869 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:25 smithi008 ceph-mon[184418]: osdmap e569: 8 total, 8 up, 7 in 2023-10-09T06:07:25.869 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:25 smithi008 ceph-mon[184418]: failed to encode map e569 with expected crc 2023-10-09T06:07:25.869 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:25 smithi008 ceph-mon[184418]: failed to encode map e569 with expected crc 2023-10-09T06:07:25.869 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:25 smithi008 ceph-mon[191227]: failed to encode map e569 with expected crc 2023-10-09T06:07:25.870 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:25 smithi008 ceph-mon[191227]: failed to encode map e569 with expected crc 2023-10-09T06:07:25.870 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:25 smithi008 ceph-mon[191227]: osdmap e569: 8 total, 8 up, 7 in 2023-10-09T06:07:25.870 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:25 smithi008 ceph-mon[191227]: failed to encode map e569 with expected crc 2023-10-09T06:07:25.870 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:25 smithi008 ceph-mon[191227]: failed to encode map e569 with expected crc 2023-10-09T06:07:25.871 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:25.919 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:25.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:25.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:26.023 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:26.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:25 smithi191 ceph-mon[161669]: failed to encode map e569 with expected crc 2023-10-09T06:07:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:25 smithi191 ceph-mon[161669]: failed to encode map e569 with expected crc 2023-10-09T06:07:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:25 smithi191 ceph-mon[161669]: osdmap e569: 8 total, 8 up, 7 in 2023-10-09T06:07:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:25 smithi191 ceph-mon[161669]: failed to encode map e569 with expected crc 2023-10-09T06:07:26.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:25 smithi191 ceph-mon[161669]: failed to encode map e569 with expected crc 2023-10-09T06:07:26.069 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:26.557 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:07:26.596 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:07:26.596 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:07:26.596 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T06:07:26.596 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:07:26.731 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:26.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:26.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:26.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:26 smithi191 ceph-mon[161669]: pgmap v1738: 89 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 309 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 15499/14682 objects degraded (105.565%); 2/14682 objects misplaced (0.014%); 3.1 MiB/s, 1 keys/s, 71 objects/s recovering 2023-10-09T06:07:26.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:26 smithi191 ceph-mon[161669]: failed to encode map e570 with expected crc 2023-10-09T06:07:26.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:26 smithi191 ceph-mon[161669]: failed to encode map e570 with expected crc 2023-10-09T06:07:26.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:26 smithi191 ceph-mon[161669]: osdmap e570: 8 total, 8 up, 7 in 2023-10-09T06:07:26.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:26 smithi191 ceph-mon[161669]: failed to encode map e570 with expected crc 2023-10-09T06:07:26.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:26 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/78381804' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:07:26.867 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:26.912 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:26.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:26.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:26.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[184418]: pgmap v1738: 89 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 309 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 15499/14682 objects degraded (105.565%); 2/14682 objects misplaced (0.014%); 3.1 MiB/s, 1 keys/s, 71 objects/s recovering 2023-10-09T06:07:26.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[184418]: failed to encode map e570 with expected crc 2023-10-09T06:07:26.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[184418]: failed to encode map e570 with expected crc 2023-10-09T06:07:26.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[184418]: osdmap e570: 8 total, 8 up, 7 in 2023-10-09T06:07:26.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[184418]: failed to encode map e570 with expected crc 2023-10-09T06:07:26.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/78381804' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:07:26.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[191227]: pgmap v1738: 89 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 309 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 15499/14682 objects degraded (105.565%); 2/14682 objects misplaced (0.014%); 3.1 MiB/s, 1 keys/s, 71 objects/s recovering 2023-10-09T06:07:26.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[191227]: failed to encode map e570 with expected crc 2023-10-09T06:07:26.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[191227]: failed to encode map e570 with expected crc 2023-10-09T06:07:26.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[191227]: osdmap e570: 8 total, 8 up, 7 in 2023-10-09T06:07:26.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[191227]: failed to encode map e570 with expected crc 2023-10-09T06:07:26.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:26 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/78381804' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:07:27.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:27.043 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:27.085 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:27.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:27.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:27.188 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:27.215 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:27.490 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:27.493 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:07:27.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:27.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:27.597 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:27.630 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:27.695 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:27.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:27.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:27 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:27 smithi191 ceph-mon[161669]: failed to encode map e571 with expected crc 2023-10-09T06:07:27.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:27 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/78381804' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:07:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:27 smithi191 ceph-mon[161669]: osdmap e571: 8 total, 8 up, 7 in 2023-10-09T06:07:27.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:27 smithi191 ceph-mon[161669]: failed to encode map e571 with expected crc 2023-10-09T06:07:27.801 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:27.829 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:27 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:27.829 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:27 smithi008 ceph-mon[184418]: failed to encode map e571 with expected crc 2023-10-09T06:07:27.829 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:27 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/78381804' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:07:27.829 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:27 smithi008 ceph-mon[184418]: osdmap e571: 8 total, 8 up, 7 in 2023-10-09T06:07:27.829 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:27 smithi008 ceph-mon[184418]: failed to encode map e571 with expected crc 2023-10-09T06:07:27.830 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:27 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:27.830 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:27 smithi008 ceph-mon[191227]: failed to encode map e571 with expected crc 2023-10-09T06:07:27.830 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:27 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/78381804' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:07:27.830 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:27 smithi008 ceph-mon[191227]: osdmap e571: 8 total, 8 up, 7 in 2023-10-09T06:07:27.830 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:27 smithi008 ceph-mon[191227]: failed to encode map e571 with expected crc 2023-10-09T06:07:27.833 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:27.967 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:28.214 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:28.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:28.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:28.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:28.347 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:28.399 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:28.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:28.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:28.503 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:28.531 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:28.681 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:28.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:28.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:28.786 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:28 smithi191 ceph-mon[161669]: pgmap v1741: 89 pgs: 2 active+recovering+undersized+remapped, 3 peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 267 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 19463/12684 objects degraded (153.445%); 210/12684 objects misplaced (1.656%); 3.1 MiB/s, 2 keys/s, 96 objects/s recovering 2023-10-09T06:07:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:28 smithi191 ceph-mon[161669]: failed to encode map e571 with expected crc 2023-10-09T06:07:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:28 smithi191 ceph-mon[161669]: failed to encode map e571 with expected crc 2023-10-09T06:07:28.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:28 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 19463/12684 objects degraded (153.445%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:28.814 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:28 smithi008 ceph-mon[184418]: pgmap v1741: 89 pgs: 2 active+recovering+undersized+remapped, 3 peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 267 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 19463/12684 objects degraded (153.445%); 210/12684 objects misplaced (1.656%); 3.1 MiB/s, 2 keys/s, 96 objects/s recovering 2023-10-09T06:07:28.814 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:28 smithi008 ceph-mon[184418]: failed to encode map e571 with expected crc 2023-10-09T06:07:28.815 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:28 smithi008 ceph-mon[184418]: failed to encode map e571 with expected crc 2023-10-09T06:07:28.815 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:28 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 19463/12684 objects degraded (153.445%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:28.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:28 smithi008 ceph-mon[191227]: pgmap v1741: 89 pgs: 2 active+recovering+undersized+remapped, 3 peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 267 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 19463/12684 objects degraded (153.445%); 210/12684 objects misplaced (1.656%); 3.1 MiB/s, 2 keys/s, 96 objects/s recovering 2023-10-09T06:07:28.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:28 smithi008 ceph-mon[191227]: failed to encode map e571 with expected crc 2023-10-09T06:07:28.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:28 smithi008 ceph-mon[191227]: failed to encode map e571 with expected crc 2023-10-09T06:07:28.815 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:28 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 19463/12684 objects degraded (153.445%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:28.818 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:28.971 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:28.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:28.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:29.076 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:29.111 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:29.203 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:29.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:29.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:29.284 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:07:29.310 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:29.327 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:07:29.375 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:07:29.389 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:29.414 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:07:29.452 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:07:29.488 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:07:29.518 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:29.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:29.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:29.534 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:07:29.585 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:07:29.614 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:07:29.614 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:07:29.614 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:07:29.615 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:07:29.615 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:07:29.615 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:07:29.615 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:07:29.615 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:07:29.624 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:07:29.626 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:29.651 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:29.651 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1863515323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:07:29.652 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/311166186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:29.652 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:29.652 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1863515323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:07:29.652 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/311166186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:29.692 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:29.692 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1863515323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:07:29.692 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/311166186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:29.702 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:29.703 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:29.713 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:29.903 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:29.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:29.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:30.012 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:30.094 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:30.240 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:07:30.727 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:07:30.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:07:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:07:30] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.33.4" 2023-10-09T06:07:30.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:30 smithi008 ceph-mon[191227]: pgmap v1742: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 251 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 24494/11922 objects degraded (205.452%); 210/11922 objects misplaced (1.761%); 2.3 MiB/s, 1 keys/s, 71 objects/s recovering 2023-10-09T06:07:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:30 smithi008 ceph-mon[191227]: failed to encode map e572 with expected crc 2023-10-09T06:07:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1863515323' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:07:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:30 smithi008 ceph-mon[191227]: osdmap e572: 8 total, 8 up, 7 in 2023-10-09T06:07:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:30 smithi008 ceph-mon[191227]: failed to encode map e572 with expected crc 2023-10-09T06:07:30.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:30 smithi008 ceph-mon[184418]: pgmap v1742: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 251 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 24494/11922 objects degraded (205.452%); 210/11922 objects misplaced (1.761%); 2.3 MiB/s, 1 keys/s, 71 objects/s recovering 2023-10-09T06:07:30.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:30 smithi008 ceph-mon[184418]: failed to encode map e572 with expected crc 2023-10-09T06:07:30.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1863515323' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:07:30.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:30 smithi008 ceph-mon[184418]: osdmap e572: 8 total, 8 up, 7 in 2023-10-09T06:07:30.967 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:30 smithi008 ceph-mon[184418]: failed to encode map e572 with expected crc 2023-10-09T06:07:31.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:30 smithi191 ceph-mon[161669]: pgmap v1742: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 251 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 24494/11922 objects degraded (205.452%); 210/11922 objects misplaced (1.761%); 2.3 MiB/s, 1 keys/s, 71 objects/s recovering 2023-10-09T06:07:31.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:30 smithi191 ceph-mon[161669]: failed to encode map e572 with expected crc 2023-10-09T06:07:31.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1863515323' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:07:31.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:30 smithi191 ceph-mon[161669]: osdmap e572: 8 total, 8 up, 7 in 2023-10-09T06:07:31.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:30 smithi191 ceph-mon[161669]: failed to encode map e572 with expected crc 2023-10-09T06:07:31.100 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:31.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:31.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:31.207 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:31.216 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:07:31.285 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:31.374 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:31.466 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:31.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:31.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:31.579 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:31.594 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:31.726 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:31.853 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:31.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:31.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:31.970 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:32.045 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:32.203 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:32.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:32.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:32.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:32.342 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:32.408 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:32.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:32.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:32.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:32.561 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:32.635 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:32 smithi191 ceph-mon[161669]: pgmap v1744: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 245 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 169 op/s; 24494/11628 objects degraded (210.647%); 208/11628 objects misplaced (1.789%); 1.2 MiB/s, 70 objects/s recovering 2023-10-09T06:07:32.691 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:32.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:32.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:32.720 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:32 smithi008 ceph-mon[184418]: pgmap v1744: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 245 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 169 op/s; 24494/11628 objects degraded (210.647%); 208/11628 objects misplaced (1.789%); 1.2 MiB/s, 70 objects/s recovering 2023-10-09T06:07:32.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:32 smithi008 ceph-mon[191227]: pgmap v1744: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 245 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 169 op/s; 24494/11628 objects degraded (210.647%); 208/11628 objects misplaced (1.789%); 1.2 MiB/s, 70 objects/s recovering 2023-10-09T06:07:32.805 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:32.892 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:33.283 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:33.308 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:33.312 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:07:33.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:33.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:33.377 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:33.377 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:33.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:33.497 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:33.633 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:33.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:33.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:33.739 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:33.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:33 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24494/11628 objects degraded (210.647%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:33.828 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:33.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:33 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24494/11628 objects degraded (210.647%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:33.929 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:33 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24494/11628 objects degraded (210.647%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:33.948 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:33.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:34.056 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:34.105 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:34.378 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:34.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:07:34.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:34.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:34.490 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:34.526 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:34.937 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:34 smithi008 ceph-mon[184418]: pgmap v1745: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 170 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 24411/8019 objects degraded (304.415%); 91/8019 objects misplaced (1.135%); 2.7 MiB/s, 84 objects/s recovering 2023-10-09T06:07:34.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:34 smithi008 ceph-mon[191227]: pgmap v1745: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 170 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 24411/8019 objects degraded (304.415%); 91/8019 objects misplaced (1.135%); 2.7 MiB/s, 84 objects/s recovering 2023-10-09T06:07:35.030 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:35.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:35.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:34 smithi191 ceph-mon[161669]: pgmap v1745: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 170 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 24411/8019 objects degraded (304.415%); 91/8019 objects misplaced (1.135%); 2.7 MiB/s, 84 objects/s recovering 2023-10-09T06:07:35.138 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:35.256 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:35.349 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:35.459 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:35.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:35.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:35.564 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:35.648 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:35.705 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:07:35.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2601067692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:35.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:35 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:35.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2601067692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:35.721 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:35 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:35.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2601067692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:35.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:35 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:35.867 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:35.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:35.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:35.978 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:36.007 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Removed 87162 objects 2023-10-09T06:07:36.007 INFO:tasks.radosbench.radosbench.0.smithi008.stdout:Clean up completed and total clean up time :198.796 2023-10-09T06:07:36.012 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:36.057 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:36.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:36.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:36.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:36.186 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:36.353 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:36.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:36.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:36.459 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:36.486 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:36.583 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:36.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:36.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:36.690 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:36.690 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:36.766 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:36.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:36.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:36.778 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:36.870 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:36.898 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[184418]: pgmap v1746: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 125 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 24411/5889 objects degraded (414.519%); 91/5889 objects misplaced (1.545%); 2.3 MiB/s, 74 objects/s recovering 2023-10-09T06:07:36.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:36.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[184418]: failed to encode map e573 with expected crc 2023-10-09T06:07:36.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:36.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[184418]: osdmap e573: 8 total, 8 up, 7 in 2023-10-09T06:07:36.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[184418]: failed to encode map e573 with expected crc 2023-10-09T06:07:36.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3896389601' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:07:36.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:07:36.899 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[191227]: pgmap v1746: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 125 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 24411/5889 objects degraded (414.519%); 91/5889 objects misplaced (1.545%); 2.3 MiB/s, 74 objects/s recovering 2023-10-09T06:07:36.899 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:36.900 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[191227]: failed to encode map e573 with expected crc 2023-10-09T06:07:36.900 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:36.900 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[191227]: osdmap e573: 8 total, 8 up, 7 in 2023-10-09T06:07:36.900 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[191227]: failed to encode map e573 with expected crc 2023-10-09T06:07:36.900 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3896389601' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:07:36.900 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:36 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:07:36.902 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:36.930 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:36 smithi191 ceph-mon[161669]: pgmap v1746: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 125 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 348 op/s; 24411/5889 objects degraded (414.519%); 91/5889 objects misplaced (1.545%); 2.3 MiB/s, 74 objects/s recovering 2023-10-09T06:07:36.931 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:36 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:36.931 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:36 smithi191 ceph-mon[161669]: failed to encode map e573 with expected crc 2023-10-09T06:07:36.931 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:36 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:36.931 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:36 smithi191 ceph-mon[161669]: osdmap e573: 8 total, 8 up, 7 in 2023-10-09T06:07:36.931 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:36 smithi191 ceph-mon[161669]: failed to encode map e573 with expected crc 2023-10-09T06:07:36.931 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:36 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3896389601' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:07:36.931 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:36 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:07:36.955 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:36.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:36.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:37.059 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:37.086 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:37.470 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:07:37.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:37 smithi191 ceph-mon[161669]: failed to encode map e574 with expected crc 2023-10-09T06:07:37.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:37 smithi191 ceph-mon[161669]: failed to encode map e574 with expected crc 2023-10-09T06:07:37.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:37 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:07:37.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:37 smithi191 ceph-mon[161669]: osdmap e574: 8 total, 8 up, 7 in 2023-10-09T06:07:37.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:37 smithi191 ceph-mon[161669]: pgmap v1749: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 37 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 24297/1635 objects degraded (1486.055%); 3.4 MiB/s, 60 objects/s recovering 2023-10-09T06:07:37.868 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:37.953 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:37.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:37 smithi008 ceph-mon[184418]: failed to encode map e574 with expected crc 2023-10-09T06:07:37.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:37 smithi008 ceph-mon[184418]: failed to encode map e574 with expected crc 2023-10-09T06:07:37.955 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:37 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:07:37.955 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:37 smithi008 ceph-mon[184418]: osdmap e574: 8 total, 8 up, 7 in 2023-10-09T06:07:37.955 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:37 smithi008 ceph-mon[184418]: pgmap v1749: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 37 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 24297/1635 objects degraded (1486.055%); 3.4 MiB/s, 60 objects/s recovering 2023-10-09T06:07:37.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:37 smithi008 ceph-mon[191227]: failed to encode map e574 with expected crc 2023-10-09T06:07:37.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:37 smithi008 ceph-mon[191227]: failed to encode map e574 with expected crc 2023-10-09T06:07:37.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:37 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:07:37.955 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:37 smithi008 ceph-mon[191227]: osdmap e574: 8 total, 8 up, 7 in 2023-10-09T06:07:37.956 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:37 smithi008 ceph-mon[191227]: pgmap v1749: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 37 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 24297/1635 objects degraded (1486.055%); 3.4 MiB/s, 60 objects/s recovering 2023-10-09T06:07:37.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:37.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:38.057 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:38.089 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:38.208 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:38.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:38.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:38.222 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:07:38.313 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:38.342 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:38.439 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:38.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:38.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:38.543 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:38.563 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:07:38.576 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:38.593 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:07:38.599 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T06:07:38.599 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T06:07:38.622 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:07:38.636 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:38.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:38.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:38.648 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:07:38.660 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:38.668 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:07:38.695 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:07:38.716 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:07:38.735 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:07:38.737 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:07:38.737 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:07:38.737 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:07:38.737 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:07:38.737 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:07:38.737 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:07:38.737 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:07:38.737 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:07:38.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:38.767 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:38.919 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:38.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:38.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:38.955 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:38 smithi191 ceph-mon[161669]: failed to encode map e574 with expected crc 2023-10-09T06:07:38.955 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:38 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24297/1635 objects degraded (1486.055%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:38.955 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:38 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3570559858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:38.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:38 smithi008 ceph-mon[191227]: failed to encode map e574 with expected crc 2023-10-09T06:07:38.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:38 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24297/1635 objects degraded (1486.055%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:38 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3570559858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:38 smithi008 ceph-mon[184418]: failed to encode map e574 with expected crc 2023-10-09T06:07:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:38 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24297/1635 objects degraded (1486.055%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:38 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3570559858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:39.026 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:39.055 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:39.413 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:07:39.414 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:07:39.414 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2023-10-09T06:07:39.414 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2023-10-09T06:07:39.414 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:07:39.936 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:39 smithi008 ceph-mon[184418]: pgmap v1750: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 17 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 24297/657 objects degraded (3698.174%); 3.2 MiB/s, 55 objects/s recovering 2023-10-09T06:07:39.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:39 smithi008 ceph-mon[191227]: pgmap v1750: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 17 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 24297/657 objects degraded (3698.174%); 3.2 MiB/s, 55 objects/s recovering 2023-10-09T06:07:40.006 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:40.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:40.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:40.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:39 smithi191 ceph-mon[161669]: pgmap v1750: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 17 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 24297/657 objects degraded (3698.174%); 3.2 MiB/s, 55 objects/s recovering 2023-10-09T06:07:40.095 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:07:40.100 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:07:40.110 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:40.171 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:40.431 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:40.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:40.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:40.474 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:40.496 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:40.536 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:40.565 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:40.607 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:40.615 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:40.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:40.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:40.703 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:07:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:07:40] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.33.4" 2023-10-09T06:07:40.719 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:40.746 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:40.790 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:40.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:40.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:40.895 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:40.924 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:41.274 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:41.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:41.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:41.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:41.406 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:41.561 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:41.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:41.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:41.625 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:07:41.668 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:41.696 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:41.993 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:07:41.994 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2023-10-09T06:07:41.994 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2023-10-09T06:07:41.994 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2023-10-09T06:07:41.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:41.994 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117314 9.19 9:9d16215d:::benchmark_data_smithi008_502820_object33141:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:07:41.994 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.240001+0000", 2023-10-09T06:07:41.995 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 584.75029891600002, 2023-10-09T06:07:41.995 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.143222604999998, 2023-10-09T06:07:41.995 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:41.995 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:41.995 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:41.995 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:41.995 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:41.995 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117314 2023-10-09T06:07:41.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:41.996 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:41.996 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:41.996 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:41.996 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240001+0000", 2023-10-09T06:07:41.996 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:41.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:41.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:41.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:41.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240001+0000", 2023-10-09T06:07:41.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:41.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:41.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:41.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:41.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240000+0000", 2023-10-09T06:07:41.998 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.96e-06 2023-10-09T06:07:41.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:41.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:41.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:41.998 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240003+0000", 2023-10-09T06:07:41.998 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0399999999999998e-06 2023-10-09T06:07:41.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:41.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:41.999 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:41.999 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240012+0000", 2023-10-09T06:07:41.999 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6589999999999998e-06 2023-10-09T06:07:41.999 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:41.999 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:41.999 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:41.999 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240017+0000", 2023-10-09T06:07:41.999 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000471003 2023-10-09T06:07:42.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240488+0000", 2023-10-09T06:07:42.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0127e-05 2023-10-09T06:07:42.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2023-10-09T06:07:42.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240498+0000", 2023-10-09T06:07:42.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001336481 2023-10-09T06:07:42.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.001 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.241834+0000", 2023-10-09T06:07:42.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6310000000000003e-06 2023-10-09T06:07:42.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.002 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.241842+0000", 2023-10-09T06:07:42.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4456201419999992 2023-10-09T06:07:42.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.687462+0000", 2023-10-09T06:07:42.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073848840000000004 2023-10-09T06:07:42.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:42.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.694847+0000", 2023-10-09T06:07:42.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99199552499999999 2023-10-09T06:07:42.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686843+0000", 2023-10-09T06:07:42.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3572e-05 2023-10-09T06:07:42.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686856+0000", 2023-10-09T06:07:42.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.69965971100000002 2023-10-09T06:07:42.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386516+0000", 2023-10-09T06:07:42.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2209999999999997e-06 2023-10-09T06:07:42.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.006 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.006 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.006 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386524+0000", 2023-10-09T06:07:42.006 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3956454790000006 2023-10-09T06:07:42.006 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.006 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.006 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.007 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.782170+0000", 2023-10-09T06:07:42.007 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003363689 2023-10-09T06:07:42.007 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.007 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.007 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.007 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.785533+0000", 2023-10-09T06:07:42.007 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0053417940000000004 2023-10-09T06:07:42.007 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.007 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.790875+0000", 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.040818535000000003 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.831694+0000", 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5428561090000001 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.374550+0000", 2023-10-09T06:07:42.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00093305999999999999 2023-10-09T06:07:42.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.375483+0000", 2023-10-09T06:07:42.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001490901 2023-10-09T06:07:42.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.376974+0000", 2023-10-09T06:07:42.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006249714 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.383223+0000", 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.143222604999998 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117319 9.19 9:991b5772:::benchmark_data_smithi008_502820_object33146:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:07:42.010 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:07:42.011 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 584.750278202, 2023-10-09T06:07:42.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.166676882000001, 2023-10-09T06:07:42.011 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.011 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.011 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.011 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.011 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.011 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117319 2023-10-09T06:07:42.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.011 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:07:42.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:07:42.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:42.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240021+0000", 2023-10-09T06:07:42.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6680000000000001e-06 2023-10-09T06:07:42.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240024+0000", 2023-10-09T06:07:42.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2449999999999998e-06 2023-10-09T06:07:42.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240029+0000", 2023-10-09T06:07:42.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4620000000000001e-06 2023-10-09T06:07:42.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240032+0000", 2023-10-09T06:07:42.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021777950000000002 2023-10-09T06:07:42.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242210+0000", 2023-10-09T06:07:42.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3260000000000002e-06 2023-10-09T06:07:42.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242219+0000", 2023-10-09T06:07:42.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4529842770000005 2023-10-09T06:07:42.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.695204+0000", 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099633949999999999 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.705167+0000", 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98175812600000001 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686925+0000", 2023-10-09T06:07:42.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0090000000000001e-06 2023-10-09T06:07:42.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686932+0000", 2023-10-09T06:07:42.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.69967429400000003 2023-10-09T06:07:42.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386607+0000", 2023-10-09T06:07:42.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6420000000000001e-06 2023-10-09T06:07:42.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386613+0000", 2023-10-09T06:07:42.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4280742919999998 2023-10-09T06:07:42.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.814688+0000", 2023-10-09T06:07:42.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046527440000000003 2023-10-09T06:07:42.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.819340+0000", 2023-10-09T06:07:42.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068654730000000004 2023-10-09T06:07:42.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.020 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:07:42.020 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.826206+0000", 2023-10-09T06:07:42.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.025179699999999999 2023-10-09T06:07:42.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.851385+0000", 2023-10-09T06:07:42.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5449409780000001 2023-10-09T06:07:42.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.396326+0000", 2023-10-09T06:07:42.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001055615 2023-10-09T06:07:42.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.397382+0000", 2023-10-09T06:07:42.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017229529999999999 2023-10-09T06:07:42.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.023 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.023 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.023 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.399105+0000", 2023-10-09T06:07:42.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075933620000000002 2023-10-09T06:07:42.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.406698+0000", 2023-10-09T06:07:42.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.166676882000001 2023-10-09T06:07:42.024 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.024 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.024 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.025 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117320 9.19 9:9ca20909:::benchmark_data_smithi008_502820_object33147:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e444)", 2023-10-09T06:07:42.025 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:07:42.025 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 584.75026360000004, 2023-10-09T06:07:42.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.155182235000002, 2023-10-09T06:07:42.025 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.025 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.025 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.025 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.025 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.025 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117320 2023-10-09T06:07:42.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.026 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:07:42.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:07:42.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240036+0000", 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.638e-06 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240038+0000", 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0300000000000001e-06 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240043+0000", 2023-10-09T06:07:42.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8990000000000003e-06 2023-10-09T06:07:42.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.240047+0000", 2023-10-09T06:07:42.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021981399999999999 2023-10-09T06:07:42.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242245+0000", 2023-10-09T06:07:42.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2699999999999999e-06 2023-10-09T06:07:42.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:57.242253+0000", 2023-10-09T06:07:42.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4632743759999993 2023-10-09T06:07:42.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.705527+0000", 2023-10-09T06:07:42.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075602960000000002 2023-10-09T06:07:42.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:42.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.713087+0000", 2023-10-09T06:07:42.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97380087699999995 2023-10-09T06:07:42.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686888+0000", 2023-10-09T06:07:42.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8900000000000001e-06 2023-10-09T06:07:42.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.686895+0000", 2023-10-09T06:07:42.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.69966168100000004 2023-10-09T06:07:42.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386557+0000", 2023-10-09T06:07:42.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3550000000000005e-06 2023-10-09T06:07:42.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386565+0000", 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4098301660000008 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.796395+0000", 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048933850000000001 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.801289+0000", 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071789369999999998 2023-10-09T06:07:42.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:07:42.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.808468+0000", 2023-10-09T06:07:42.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.042643954999999997 2023-10-09T06:07:42.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.851112+0000", 2023-10-09T06:07:42.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.532899402 2023-10-09T06:07:42.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.384011+0000", 2023-10-09T06:07:42.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010206099999999999 2023-10-09T06:07:42.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.385032+0000", 2023-10-09T06:07:42.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017808520000000001 2023-10-09T06:07:42.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.386812+0000", 2023-10-09T06:07:42.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0084058440000000009 2023-10-09T06:07:42.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.395218+0000", 2023-10-09T06:07:42.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 23.155182235000002 2023-10-09T06:07:42.037 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.037 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.037 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.037 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117329 9.19 9:98a1b605:::benchmark_data_smithi008_502820_object33156:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-09T06:07:42.037 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:58.537538+0000", 2023-10-09T06:07:42.037 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 583.45276125700002, 2023-10-09T06:07:42.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 22.047981301, 2023-10-09T06:07:42.037 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.038 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.038 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.038 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.038 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.038 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117329 2023-10-09T06:07:42.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.038 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537538+0000", 2023-10-09T06:07:42.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537538+0000", 2023-10-09T06:07:42.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:42.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537537+0000", 2023-10-09T06:07:42.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.357e-06 2023-10-09T06:07:42.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537544+0000", 2023-10-09T06:07:42.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5744e-05 2023-10-09T06:07:42.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537560+0000", 2023-10-09T06:07:42.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1219999999999995e-06 2023-10-09T06:07:42.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.537568+0000", 2023-10-09T06:07:42.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1875399160000004 2023-10-09T06:07:42.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725108+0000", 2023-10-09T06:07:42.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7539999999999995e-06 2023-10-09T06:07:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725117+0000", 2023-10-09T06:07:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66151549200000004 2023-10-09T06:07:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386632+0000", 2023-10-09T06:07:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.083e-06 2023-10-09T06:07:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386638+0000", 2023-10-09T06:07:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4453952789999995 2023-10-09T06:07:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.832033+0000", 2023-10-09T06:07:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0096413320000000007 2023-10-09T06:07:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.841675+0000", 2023-10-09T06:07:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5839331400000001 2023-10-09T06:07:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.425608+0000", 2023-10-09T06:07:42.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020811639999999999 2023-10-09T06:07:42.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.427689+0000", 2023-10-09T06:07:42.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052414740000000003 2023-10-09T06:07:42.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:07:42.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.432931+0000", 2023-10-09T06:07:42.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.046699651000000002 2023-10-09T06:07:42.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.479630+0000", 2023-10-09T06:07:42.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.094573824000000001 2023-10-09T06:07:42.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.574204+0000", 2023-10-09T06:07:42.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001875704 2023-10-09T06:07:42.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.576080+0000", 2023-10-09T06:07:42.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018718509999999999 2023-10-09T06:07:42.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.577952+0000", 2023-10-09T06:07:42.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.007568165 2023-10-09T06:07:42.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.585520+0000", 2023-10-09T06:07:42.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 22.047981301 2023-10-09T06:07:42.048 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.048 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.048 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.049 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117332 9.19 9:9d36e61d:::benchmark_data_smithi008_502820_object33159:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-09T06:07:42.049 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:57:58.661367+0000", 2023-10-09T06:07:42.049 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 583.32893291000005, 2023-10-09T06:07:42.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 21.935008709000002, 2023-10-09T06:07:42.049 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.049 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.049 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.049 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.049 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.050 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117332 2023-10-09T06:07:42.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.050 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661367+0000", 2023-10-09T06:07:42.050 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661367+0000", 2023-10-09T06:07:42.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:42.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661365+0000", 2023-10-09T06:07:42.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8889999999999997e-06 2023-10-09T06:07:42.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661375+0000", 2023-10-09T06:07:42.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2055999999999999e-05 2023-10-09T06:07:42.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661387+0000", 2023-10-09T06:07:42.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9869999999999996e-06 2023-10-09T06:07:42.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:57:58.661393+0000", 2023-10-09T06:07:42.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0637442499999992 2023-10-09T06:07:42.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725137+0000", 2023-10-09T06:07:42.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0648e-05 2023-10-09T06:07:42.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725148+0000", 2023-10-09T06:07:42.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66150936400000004 2023-10-09T06:07:42.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386657+0000", 2023-10-09T06:07:42.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.976e-06 2023-10-09T06:07:42.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386663+0000", 2023-10-09T06:07:42.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4554108919999997 2023-10-09T06:07:42.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.842074+0000", 2023-10-09T06:07:42.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087053159999999994 2023-10-09T06:07:42.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:42.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.850779+0000", 2023-10-09T06:07:42.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6107857050000001 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.461565+0000", 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023301989999999998 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.463895+0000", 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052078059999999997 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:07:42.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.469103+0000", 2023-10-09T06:07:42.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010784145 2023-10-09T06:07:42.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.479887+0000", 2023-10-09T06:07:42.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.106784905 2023-10-09T06:07:42.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.586672+0000", 2023-10-09T06:07:42.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011016520000000001 2023-10-09T06:07:42.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.587774+0000", 2023-10-09T06:07:42.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001720217 2023-10-09T06:07:42.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.589494+0000", 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068816320000000004 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.596375+0000", 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 21.935008709000002 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117337 9.1 9:857ac274:::benchmark_data_smithi008_502820_object33164:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:02.529277+0000", 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 579.46102299799998, 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 18.012440933000001, 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117337 2023-10-09T06:07:42.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.061 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529277+0000", 2023-10-09T06:07:42.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529277+0000", 2023-10-09T06:07:42.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:42.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529275+0000", 2023-10-09T06:07:42.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6810000000000001e-06 2023-10-09T06:07:42.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529282+0000", 2023-10-09T06:07:42.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5460000000000001e-05 2023-10-09T06:07:42.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529297+0000", 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0799999999999996e-06 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.529302+0000", 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1958720979999997 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725174+0000", 2023-10-09T06:07:42.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9480000000000003e-06 2023-10-09T06:07:42.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725184+0000", 2023-10-09T06:07:42.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66691750900000002 2023-10-09T06:07:42.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392102+0000", 2023-10-09T06:07:42.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8009999999999998e-06 2023-10-09T06:07:42.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392112+0000", 2023-10-09T06:07:42.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.865323069 2023-10-09T06:07:42.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.257435+0000", 2023-10-09T06:07:42.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015459999999999999 2023-10-09T06:07:42.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.257589+0000", 2023-10-09T06:07:42.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028764399999999999 2023-10-09T06:07:42.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:07:42.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.257877+0000", 2023-10-09T06:07:42.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035365399999999999 2023-10-09T06:07:42.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.261413+0000", 2023-10-09T06:07:42.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27957103 2023-10-09T06:07:42.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.540984+0000", 2023-10-09T06:07:42.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1287999999999998e-05 2023-10-09T06:07:42.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.068 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541066+0000", 2023-10-09T06:07:42.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0819999999999998e-05 2023-10-09T06:07:42.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.068 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541117+0000", 2023-10-09T06:07:42.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060101399999999998 2023-10-09T06:07:42.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541718+0000", 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 18.012440933000001 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117340 9.9 9:91f28550:::benchmark_data_smithi008_502820_object33167:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:02.596319+0000", 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 579.39398039800005, 2023-10-09T06:07:42.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 17.955864099999999, 2023-10-09T06:07:42.070 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.070 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.070 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.070 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.070 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.070 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117340 2023-10-09T06:07:42.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.070 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596319+0000", 2023-10-09T06:07:42.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596319+0000", 2023-10-09T06:07:42.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:42.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596318+0000", 2023-10-09T06:07:42.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0000000000000002e-06 2023-10-09T06:07:42.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596324+0000", 2023-10-09T06:07:42.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3848e-05 2023-10-09T06:07:42.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596338+0000", 2023-10-09T06:07:42.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3350000000000003e-06 2023-10-09T06:07:42.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:02.596344+0000", 2023-10-09T06:07:42.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1288643570000003 2023-10-09T06:07:42.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725209+0000", 2023-10-09T06:07:42.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0413e-05 2023-10-09T06:07:42.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725219+0000", 2023-10-09T06:07:42.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65901121200000001 2023-10-09T06:07:42.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384230+0000", 2023-10-09T06:07:42.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3641e-05 2023-10-09T06:07:42.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384244+0000", 2023-10-09T06:07:42.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4740669539999995 2023-10-09T06:07:42.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.858311+0000", 2023-10-09T06:07:42.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086265019999999994 2023-10-09T06:07:42.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:42.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.866937+0000", 2023-10-09T06:07:42.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4266018129999996 2023-10-09T06:07:42.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.293539+0000", 2023-10-09T06:07:42.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002220236 2023-10-09T06:07:42.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.295759+0000", 2023-10-09T06:07:42.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050211379999999996 2023-10-09T06:07:42.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:07:42.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.300781+0000", 2023-10-09T06:07:42.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033493818000000002 2023-10-09T06:07:42.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.334274+0000", 2023-10-09T06:07:42.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20914828399999999 2023-10-09T06:07:42.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.543423+0000", 2023-10-09T06:07:42.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00094928299999999998 2023-10-09T06:07:42.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.544372+0000", 2023-10-09T06:07:42.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013903489999999999 2023-10-09T06:07:42.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.545762+0000", 2023-10-09T06:07:42.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0064211219999999996 2023-10-09T06:07:42.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.552183+0000", 2023-10-09T06:07:42.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 17.955864099999999 2023-10-09T06:07:42.080 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.080 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.080 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.080 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117343 9.9 9:978d0f59:::benchmark_data_smithi008_502820_object33170:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:07:42.080 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.479396+0000", 2023-10-09T06:07:42.080 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 576.51090362499997, 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.081940300999999, 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117343 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479396+0000", 2023-10-09T06:07:42.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479396+0000", 2023-10-09T06:07:42.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999967 2023-10-09T06:07:42.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479393+0000", 2023-10-09T06:07:42.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1915e-05 2023-10-09T06:07:42.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479405+0000", 2023-10-09T06:07:42.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4262e-05 2023-10-09T06:07:42.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479419+0000", 2023-10-09T06:07:42.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3119999999999994e-06 2023-10-09T06:07:42.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.479427+0000", 2023-10-09T06:07:42.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2458186229999999 2023-10-09T06:07:42.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725246+0000", 2023-10-09T06:07:42.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0760000000000003e-06 2023-10-09T06:07:42.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725254+0000", 2023-10-09T06:07:42.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65942001299999997 2023-10-09T06:07:42.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384674+0000", 2023-10-09T06:07:42.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1280999999999999e-05 2023-10-09T06:07:42.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384685+0000", 2023-10-09T06:07:42.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4825987420000004 2023-10-09T06:07:42.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.867284+0000", 2023-10-09T06:07:42.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0098823669999999995 2023-10-09T06:07:42.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.877166+0000", 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4447639959999998 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.321930+0000", 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020355619999999999 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.323966+0000", 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059662459999999997 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.329932+0000", 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022155676999999999 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.352088+0000", 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.201059509 2023-10-09T06:07:42.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.553147+0000", 2023-10-09T06:07:42.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00084139399999999997 2023-10-09T06:07:42.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.553989+0000", 2023-10-09T06:07:42.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001347577 2023-10-09T06:07:42.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.555336+0000", 2023-10-09T06:07:42.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060000390000000004 2023-10-09T06:07:42.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.561336+0000", 2023-10-09T06:07:42.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.081940300999999 2023-10-09T06:07:42.091 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.092 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.092 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.092 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117347 9.1 9:83732981:::benchmark_data_smithi008_502820_object33174:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:07:42.092 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.499313+0000", 2023-10-09T06:07:42.092 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 576.49098709299994, 2023-10-09T06:07:42.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.043182195, 2023-10-09T06:07:42.092 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.093 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.093 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.093 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.093 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.093 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117347 2023-10-09T06:07:42.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.093 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499313+0000", 2023-10-09T06:07:42.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499313+0000", 2023-10-09T06:07:42.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:07:42.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499310+0000", 2023-10-09T06:07:42.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6179999999999995e-06 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499319+0000", 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5282e-05 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499334+0000", 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1779999999999997e-06 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.499341+0000", 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.225918622 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725260+0000", 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4930000000000001e-06 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725268+0000", 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66689868200000002 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392166+0000", 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6649999999999992e-06 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.392175+0000", 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.866980590000001 2023-10-09T06:07:42.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.259156+0000", 2023-10-09T06:07:42.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000185317 2023-10-09T06:07:42.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.259341+0000", 2023-10-09T06:07:42.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00037832499999999999 2023-10-09T06:07:42.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:07:42.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.259719+0000", 2023-10-09T06:07:42.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035454420000000002 2023-10-09T06:07:42.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.263265+0000", 2023-10-09T06:07:42.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.277776933 2023-10-09T06:07:42.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541042+0000", 2023-10-09T06:07:42.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00079637999999999996 2023-10-09T06:07:42.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541838+0000", 2023-10-09T06:07:42.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5203999999999994e-05 2023-10-09T06:07:42.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.541903+0000", 2023-10-09T06:07:42.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000591623 2023-10-09T06:07:42.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.542495+0000", 2023-10-09T06:07:42.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 15.043182195 2023-10-09T06:07:42.101 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.101 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.101 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.102 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117351 9.11 9:8879fcf3:::benchmark_data_smithi008_502820_object33178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:07:42.102 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.521160+0000", 2023-10-09T06:07:42.102 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 576.46913972599998, 2023-10-09T06:07:42.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.849076910000001, 2023-10-09T06:07:42.102 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.102 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.102 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.102 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.102 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.102 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117351 2023-10-09T06:07:42.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521160+0000", 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521160+0000", 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521158+0000", 2023-10-09T06:07:42.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.622e-06 2023-10-09T06:07:42.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521166+0000", 2023-10-09T06:07:42.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2873e-05 2023-10-09T06:07:42.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521179+0000", 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7890000000000006e-06 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.521189+0000", 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2040851159999999 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725274+0000", 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8170000000000001e-06 2023-10-09T06:07:42.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725281+0000", 2023-10-09T06:07:42.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66564084199999995 2023-10-09T06:07:42.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.390922+0000", 2023-10-09T06:07:42.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0733999999999999e-05 2023-10-09T06:07:42.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.390932+0000", 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5330637649999996 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.923996+0000", 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013707300000000001 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.924133+0000", 2023-10-09T06:07:42.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00038352900000000002 2023-10-09T06:07:42.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4,6", 2023-10-09T06:07:42.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.924517+0000", 2023-10-09T06:07:42.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.87137725899999996 2023-10-09T06:07:42.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.795894+0000", 2023-10-09T06:07:42.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5739970530000003 2023-10-09T06:07:42.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.369891+0000", 2023-10-09T06:07:42.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6699000000000002e-05 2023-10-09T06:07:42.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.369928+0000", 2023-10-09T06:07:42.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010414800000000001 2023-10-09T06:07:42.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.370032+0000", 2023-10-09T06:07:42.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5777000000000004e-05 2023-10-09T06:07:42.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.370098+0000", 2023-10-09T06:07:42.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013928500000000001 2023-10-09T06:07:42.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.370237+0000", 2023-10-09T06:07:42.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.849076910000001 2023-10-09T06:07:42.111 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.111 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.111 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.111 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117352 9.9 9:93c88060:::benchmark_data_smithi008_502820_object33179:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:07:42.111 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.574636+0000", 2023-10-09T06:07:42.112 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 576.415663645, 2023-10-09T06:07:42.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.996202152, 2023-10-09T06:07:42.112 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.112 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.112 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.112 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.112 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.112 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117352 2023-10-09T06:07:42.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.112 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574636+0000", 2023-10-09T06:07:42.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574636+0000", 2023-10-09T06:07:42.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:42.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574634+0000", 2023-10-09T06:07:42.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2150000000000001e-06 2023-10-09T06:07:42.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574643+0000", 2023-10-09T06:07:42.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6385e-05 2023-10-09T06:07:42.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574659+0000", 2023-10-09T06:07:42.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6510000000000001e-06 2023-10-09T06:07:42.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.574667+0000", 2023-10-09T06:07:42.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.150622016 2023-10-09T06:07:42.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725289+0000", 2023-10-09T06:07:42.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1060000000000002e-06 2023-10-09T06:07:42.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725296+0000", 2023-10-09T06:07:42.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.65942802199999995 2023-10-09T06:07:42.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384724+0000", 2023-10-09T06:07:42.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1769999999999998e-06 2023-10-09T06:07:42.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.384732+0000", 2023-10-09T06:07:42.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4928461019999997 2023-10-09T06:07:42.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.877578+0000", 2023-10-09T06:07:42.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011095785 2023-10-09T06:07:42.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:42.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:15.888674+0000", 2023-10-09T06:07:42.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4637061429999996 2023-10-09T06:07:42.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.352380+0000", 2023-10-09T06:07:42.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020417460000000001 2023-10-09T06:07:42.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.354422+0000", 2023-10-09T06:07:42.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0058679479999999996 2023-10-09T06:07:42.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:07:42.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.360290+0000", 2023-10-09T06:07:42.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.036684833 2023-10-09T06:07:42.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.396974+0000", 2023-10-09T06:07:42.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16525932500000001 2023-10-09T06:07:42.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.562234+0000", 2023-10-09T06:07:42.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00104692 2023-10-09T06:07:42.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.563281+0000", 2023-10-09T06:07:42.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014355430000000001 2023-10-09T06:07:42.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.564716+0000", 2023-10-09T06:07:42.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0061219309999999997 2023-10-09T06:07:42.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.570838+0000", 2023-10-09T06:07:42.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.996202152 2023-10-09T06:07:42.122 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.122 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.122 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.122 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:117356 9.19 9:9e99d7bd:::benchmark_data_smithi008_502820_object33183:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-09T06:07:42.122 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T05:58:05.607515+0000", 2023-10-09T06:07:42.122 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 576.38278500800004, 2023-10-09T06:07:42.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.999065100999999, 2023-10-09T06:07:42.122 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 117356 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607515+0000", 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607515+0000", 2023-10-09T06:07:42.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:42.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607513+0000", 2023-10-09T06:07:42.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5640000000000002e-06 2023-10-09T06:07:42.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607520+0000", 2023-10-09T06:07:42.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1073000000000001e-05 2023-10-09T06:07:42.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607531+0000", 2023-10-09T06:07:42.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9660000000000001e-06 2023-10-09T06:07:42.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:05.607537+0000", 2023-10-09T06:07:42.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1177633890000001 2023-10-09T06:07:42.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725300+0000", 2023-10-09T06:07:42.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7010000000000001e-06 2023-10-09T06:07:42.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:06.725306+0000", 2023-10-09T06:07:42.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.66137529500000003 2023-10-09T06:07:42.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386681+0000", 2023-10-09T06:07:42.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0240000000000003e-06 2023-10-09T06:07:42.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:07.386687+0000", 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4650221170000002 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.851709+0000", 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0099491639999999999 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for degraded object", 2023-10-09T06:07:42.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:16.861658+0000", 2023-10-09T06:07:42.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6314088509999998 2023-10-09T06:07:42.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.493067+0000", 2023-10-09T06:07:42.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002267879 2023-10-09T06:07:42.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.495335+0000", 2023-10-09T06:07:42.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0054393109999999996 2023-10-09T06:07:42.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:07:42.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.500774+0000", 2023-10-09T06:07:42.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024364858999999999 2023-10-09T06:07:42.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.525139+0000", 2023-10-09T06:07:42.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.072382734000000004 2023-10-09T06:07:42.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.597522+0000", 2023-10-09T06:07:42.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001197832 2023-10-09T06:07:42.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.598720+0000", 2023-10-09T06:07:42.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001605965 2023-10-09T06:07:42.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.600326+0000", 2023-10-09T06:07:42.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0062539149999999996 2023-10-09T06:07:42.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T05:58:20.606580+0000", 2023-10-09T06:07:42.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 14.999065100999999 2023-10-09T06:07:42.132 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135245 9.d 9:b964d666:::benchmark_data_smithi008_502820_object51072:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.596664+0000", 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 458.39363578400003, 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789031075, 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.133 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135245 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596664+0000", 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596664+0000", 2023-10-09T06:07:42.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:07:42.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596663+0000", 2023-10-09T06:07:42.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.011e-06 2023-10-09T06:07:42.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596668+0000", 2023-10-09T06:07:42.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0095000000000001e-05 2023-10-09T06:07:42.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596678+0000", 2023-10-09T06:07:42.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5299999999999998e-06 2023-10-09T06:07:42.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.596682+0000", 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.063468346999999994 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.660151+0000", 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033404179999999999 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.663491+0000", 2023-10-09T06:07:42.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017397383999999998 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.680889+0000", 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004312265 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.685201+0000", 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.61805938000000005 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.303260+0000", 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035192460000000002 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.306780+0000", 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0085941420000000008 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:07:42.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.315374+0000", 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.057904045000000001 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373278+0000", 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0834e-05 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373289+0000", 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91257296300000001 2023-10-09T06:07:42.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285861+0000", 2023-10-09T06:07:42.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8009999999999998e-06 2023-10-09T06:07:42.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285868+0000", 2023-10-09T06:07:42.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.830804881000001 2023-10-09T06:07:42.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.116673+0000", 2023-10-09T06:07:42.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050648639999999997 2023-10-09T06:07:42.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:07:42.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.121738+0000", 2023-10-09T06:07:42.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.26043395800000002 2023-10-09T06:07:42.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.382172+0000", 2023-10-09T06:07:42.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035229559999999998 2023-10-09T06:07:42.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.385695+0000", 2023-10-09T06:07:42.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789031075 2023-10-09T06:07:42.144 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.144 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.144 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135269 9.d 9:b1952556:::benchmark_data_smithi008_502820_object51096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.688239+0000", 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 458.30206087900001, 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.735737110000001, 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135269 2023-10-09T06:07:42.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688239+0000", 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688239+0000", 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688238+0000", 2023-10-09T06:07:42.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5710000000000003e-06 2023-10-09T06:07:42.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688244+0000", 2023-10-09T06:07:42.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8319999999999995e-06 2023-10-09T06:07:42.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688253+0000", 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7010000000000001e-06 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.688259+0000", 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.024856844999999999 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.713116+0000", 2023-10-09T06:07:42.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041163090000000003 2023-10-09T06:07:42.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:07:42.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.717232+0000", 2023-10-09T06:07:42.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.032543962000000003 2023-10-09T06:07:42.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749776+0000", 2023-10-09T06:07:42.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042646100000000003 2023-10-09T06:07:42.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:42.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.754041+0000", 2023-10-09T06:07:42.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.57617059599999998 2023-10-09T06:07:42.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.330211+0000", 2023-10-09T06:07:42.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003060962 2023-10-09T06:07:42.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.333272+0000", 2023-10-09T06:07:42.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059605719999999999 2023-10-09T06:07:42.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:07:42.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.339233+0000", 2023-10-09T06:07:42.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034085450000000003 2023-10-09T06:07:42.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373318+0000", 2023-10-09T06:07:42.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4910000000000002e-06 2023-10-09T06:07:42.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373325+0000", 2023-10-09T06:07:42.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91256626500000004 2023-10-09T06:07:42.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285891+0000", 2023-10-09T06:07:42.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2830000000000004e-06 2023-10-09T06:07:42.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285898+0000", 2023-10-09T06:07:42.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.836523452 2023-10-09T06:07:42.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.122422+0000", 2023-10-09T06:07:42.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0067783660000000001 2023-10-09T06:07:42.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:07:42.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.129200+0000", 2023-10-09T06:07:42.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.29146132499999999 2023-10-09T06:07:42.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.420661+0000", 2023-10-09T06:07:42.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033146180000000001 2023-10-09T06:07:42.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.423976+0000", 2023-10-09T06:07:42.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.735737110000001 2023-10-09T06:07:42.155 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.155 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.155 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135271 9.d 9:ba573113:::benchmark_data_smithi008_502820_object51098:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.694068+0000", 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 458.29623193200001, 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.803375333, 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.156 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135271 2023-10-09T06:07:42.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.157 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694068+0000", 2023-10-09T06:07:42.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694068+0000", 2023-10-09T06:07:42.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999976 2023-10-09T06:07:42.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694065+0000", 2023-10-09T06:07:42.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2420000000000006e-06 2023-10-09T06:07:42.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694075+0000", 2023-10-09T06:07:42.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5207e-05 2023-10-09T06:07:42.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694090+0000", 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.721e-06 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.694099+0000", 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.060414994999999999 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.754514+0000", 2023-10-09T06:07:42.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047620040000000002 2023-10-09T06:07:42.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:07:42.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.759276+0000", 2023-10-09T06:07:42.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01924886 2023-10-09T06:07:42.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.778524+0000", 2023-10-09T06:07:42.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045597650000000003 2023-10-09T06:07:42.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:42.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.783084+0000", 2023-10-09T06:07:42.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.59039565999999999 2023-10-09T06:07:42.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373480+0000", 2023-10-09T06:07:42.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1539999999999999e-06 2023-10-09T06:07:42.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373486+0000", 2023-10-09T06:07:42.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91251702300000004 2023-10-09T06:07:42.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286003+0000", 2023-10-09T06:07:42.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.336e-06 2023-10-09T06:07:42.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286009+0000", 2023-10-09T06:07:42.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.906405661999999 2023-10-09T06:07:42.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.192415+0000", 2023-10-09T06:07:42.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00484973 2023-10-09T06:07:42.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.197265+0000", 2023-10-09T06:07:42.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0089266180000000007 2023-10-09T06:07:42.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:07:42.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.206191+0000", 2023-10-09T06:07:42.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.065920287999999994 2023-10-09T06:07:42.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.272112+0000", 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.20899689099999999 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.481109+0000", 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001501549 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.482610+0000", 2023-10-09T06:07:42.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015254660000000001 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.484136+0000", 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026156030000000002 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.486751+0000", 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010691813999999999 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.497443+0000", 2023-10-09T06:07:42.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.803375333 2023-10-09T06:07:42.167 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.167 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.167 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.167 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135277 9.d 9:bcf8be17:::benchmark_data_smithi008_502820_object51104:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:07:42.167 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.729022+0000", 2023-10-09T06:07:42.167 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 458.26127721099999, 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.750575383999999, 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135277 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729022+0000", 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729022+0000", 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729020+0000", 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0536e-05 2023-10-09T06:07:42.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729031+0000", 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2653999999999999e-05 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729044+0000", 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6559999999999992e-06 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.729051+0000", 2023-10-09T06:07:42.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.031414678000000001 2023-10-09T06:07:42.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.760466+0000", 2023-10-09T06:07:42.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032615449999999998 2023-10-09T06:07:42.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:07:42.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.763727+0000", 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.036828547000000003 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.800556+0000", 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032710500000000002 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.803827+0000", 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.56961666799999999 2023-10-09T06:07:42.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373444+0000", 2023-10-09T06:07:42.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1970000000000001e-06 2023-10-09T06:07:42.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373450+0000", 2023-10-09T06:07:42.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91252582299999996 2023-10-09T06:07:42.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285976+0000", 2023-10-09T06:07:42.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5559999999999998e-06 2023-10-09T06:07:42.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.285981+0000", 2023-10-09T06:07:42.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.882853229 2023-10-09T06:07:42.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.168835+0000", 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0048070370000000001 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.173642+0000", 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086679089999999997 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.182309+0000", 2023-10-09T06:07:42.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.051578453000000003 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.233888+0000", 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22473416600000001 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.458622+0000", 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001297997 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.459920+0000", 2023-10-09T06:07:42.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014969269999999999 2023-10-09T06:07:42.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.461417+0000", 2023-10-09T06:07:42.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025559990000000002 2023-10-09T06:07:42.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.463973+0000", 2023-10-09T06:07:42.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015624812 2023-10-09T06:07:42.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.479598+0000", 2023-10-09T06:07:42.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.750575383999999 2023-10-09T06:07:42.178 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135285 9.d 9:b8ae6d39:::benchmark_data_smithi008_502820_object51112:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 458.240461326, 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789996110000001, 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.179 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.180 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135285 2023-10-09T06:07:42.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.180 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:07:42.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749838+0000", 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.281e-06 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749841+0000", 2023-10-09T06:07:42.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0190000000000001e-06 2023-10-09T06:07:42.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749848+0000", 2023-10-09T06:07:42.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0799999999999999e-06 2023-10-09T06:07:42.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.749852+0000", 2023-10-09T06:07:42.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033638386999999999 2023-10-09T06:07:42.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.783491+0000", 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031708800000000001 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.786661+0000", 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034315245000000001 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.820977+0000", 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032606850000000001 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.824237+0000", 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.54932086199999997 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373558+0000", 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1770000000000003e-06 2023-10-09T06:07:42.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373564+0000", 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91249455899999998 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286059+0000", 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7699999999999998e-06 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286065+0000", 2023-10-09T06:07:42.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.948433641999999 2023-10-09T06:07:42.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.234498+0000", 2023-10-09T06:07:42.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042428049999999997 2023-10-09T06:07:42.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.238741+0000", 2023-10-09T06:07:42.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0073975059999999999 2023-10-09T06:07:42.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:07:42.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.246139+0000", 2023-10-09T06:07:42.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.045528540999999999 2023-10-09T06:07:42.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.291667+0000", 2023-10-09T06:07:42.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22418721699999999 2023-10-09T06:07:42.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.515854+0000", 2023-10-09T06:07:42.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001393544 2023-10-09T06:07:42.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.517248+0000", 2023-10-09T06:07:42.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001617402 2023-10-09T06:07:42.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.518865+0000", 2023-10-09T06:07:42.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0061180770000000004 2023-10-09T06:07:42.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.524983+0000", 2023-10-09T06:07:42.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014850993999999999 2023-10-09T06:07:42.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.539834+0000", 2023-10-09T06:07:42.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.789996110000001 2023-10-09T06:07:42.192 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.192 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.192 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.192 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135292 9.d 9:b1d0c439:::benchmark_data_smithi008_502820_object51119:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:07:42.192 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:03.788529+0000", 2023-10-09T06:07:42.192 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 458.20177055699997, 2023-10-09T06:07:42.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.832817245999999, 2023-10-09T06:07:42.192 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.193 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.193 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.193 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.193 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.193 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135292 2023-10-09T06:07:42.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.193 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788529+0000", 2023-10-09T06:07:42.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788529+0000", 2023-10-09T06:07:42.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:42.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788528+0000", 2023-10-09T06:07:42.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8979999999999998e-06 2023-10-09T06:07:42.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788532+0000", 2023-10-09T06:07:42.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3579999999999999e-06 2023-10-09T06:07:42.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788539+0000", 2023-10-09T06:07:42.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.58e-06 2023-10-09T06:07:42.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.788542+0000", 2023-10-09T06:07:42.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.037230805999999998 2023-10-09T06:07:42.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.825773+0000", 2023-10-09T06:07:42.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045452899999999996 2023-10-09T06:07:42.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:07:42.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.830318+0000", 2023-10-09T06:07:42.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.044094162999999999 2023-10-09T06:07:42.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.874413+0000", 2023-10-09T06:07:42.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033226699999999998 2023-10-09T06:07:42.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:42.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:03.877735+0000", 2023-10-09T06:07:42.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.496008956 2023-10-09T06:07:42.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373744+0000", 2023-10-09T06:07:42.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2219999999999999e-06 2023-10-09T06:07:42.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373750+0000", 2023-10-09T06:07:42.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91544140500000004 2023-10-09T06:07:42.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289192+0000", 2023-10-09T06:07:42.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5060000000000001e-06 2023-10-09T06:07:42.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289198+0000", 2023-10-09T06:07:42.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.022931246000001 2023-10-09T06:07:42.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.312130+0000", 2023-10-09T06:07:42.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041899379999999998 2023-10-09T06:07:42.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.316319+0000", 2023-10-09T06:07:42.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0083299189999999999 2023-10-09T06:07:42.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:07:42.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.324649+0000", 2023-10-09T06:07:42.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0088983910000000003 2023-10-09T06:07:42.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.333548+0000", 2023-10-09T06:07:42.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.27093540199999999 2023-10-09T06:07:42.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.604483+0000", 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001811764 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.606295+0000", 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002186805 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.608482+0000", 2023-10-09T06:07:42.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024063980000000001 2023-10-09T06:07:42.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.610888+0000", 2023-10-09T06:07:42.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010458169999999999 2023-10-09T06:07:42.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.621346+0000", 2023-10-09T06:07:42.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.832817245999999 2023-10-09T06:07:42.209 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.209 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.209 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.209 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135349 9.d 9:b7ad6e1c:::benchmark_data_smithi008_502820_object51176:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:07:42.209 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:04.105455+0000", 2023-10-09T06:07:42.209 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 457.88484431299997, 2023-10-09T06:07:42.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.459172209, 2023-10-09T06:07:42.209 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.209 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.210 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.210 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.210 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.210 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135349 2023-10-09T06:07:42.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.210 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105455+0000", 2023-10-09T06:07:42.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105455+0000", 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105453+0000", 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0417999999999999e-05 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105464+0000", 2023-10-09T06:07:42.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.025e-05 2023-10-09T06:07:42.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105474+0000", 2023-10-09T06:07:42.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0610000000000001e-06 2023-10-09T06:07:42.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.105480+0000", 2023-10-09T06:07:42.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.055762855 2023-10-09T06:07:42.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.161243+0000", 2023-10-09T06:07:42.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031189049999999999 2023-10-09T06:07:42.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:07:42.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.164362+0000", 2023-10-09T06:07:42.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019388989999999998 2023-10-09T06:07:42.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.183751+0000", 2023-10-09T06:07:42.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003301151 2023-10-09T06:07:42.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:42.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.187052+0000", 2023-10-09T06:07:42.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18655844199999999 2023-10-09T06:07:42.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373610+0000", 2023-10-09T06:07:42.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1009999999999997e-06 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373617+0000", 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91246933500000005 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286086+0000", 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2680000000000002e-06 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.286091+0000", 2023-10-09T06:07:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 11.966898898 2023-10-09T06:07:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.252990+0000", 2023-10-09T06:07:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047099940000000003 2023-10-09T06:07:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.257700+0000", 2023-10-09T06:07:42.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0077358369999999997 2023-10-09T06:07:42.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:07:42.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.265436+0000", 2023-10-09T06:07:42.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.026715319000000001 2023-10-09T06:07:42.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.292151+0000", 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.249149184 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.541300+0000", 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014578600000000001 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.542758+0000", 2023-10-09T06:07:42.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0046799320000000004 2023-10-09T06:07:42.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.547438+0000", 2023-10-09T06:07:42.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026429719999999999 2023-10-09T06:07:42.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.550081+0000", 2023-10-09T06:07:42.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014546408 2023-10-09T06:07:42.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.564628+0000", 2023-10-09T06:07:42.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.459172209 2023-10-09T06:07:42.222 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.222 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.222 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.223 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36258.0:135364 9.d 9:b4fd37ff:::benchmark_data_smithi008_502820_object51191:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-10-09T06:07:42.223 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2023-10-09T06:00:04.153115+0000", 2023-10-09T06:07:42.223 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 457.83718478600002, 2023-10-09T06:07:42.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.449789453999999, 2023-10-09T06:07:42.223 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2023-10-09T06:07:42.223 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:42.223 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2023-10-09T06:07:42.223 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36258", 2023-10-09T06:07:42.223 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.8:0/725355002", 2023-10-09T06:07:42.223 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 135364 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153115+0000", 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153115+0000", 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2023-10-09T06:07:42.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2023-10-09T06:07:42.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153114+0000", 2023-10-09T06:07:42.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1359999999999999e-06 2023-10-09T06:07:42.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2023-10-09T06:07:42.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153118+0000", 2023-10-09T06:07:42.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6540000000000005e-06 2023-10-09T06:07:42.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2023-10-09T06:07:42.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153127+0000", 2023-10-09T06:07:42.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9570000000000001e-06 2023-10-09T06:07:42.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2023-10-09T06:07:42.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.153132+0000", 2023-10-09T06:07:42.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011677977000000001 2023-10-09T06:07:42.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.164810+0000", 2023-10-09T06:07:42.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040417029999999998 2023-10-09T06:07:42.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2023-10-09T06:07:42.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.168851+0000", 2023-10-09T06:07:42.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034353937000000001 2023-10-09T06:07:42.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.203205+0000", 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030759429999999998 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.206281+0000", 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.16741861399999999 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373700+0000", 2023-10-09T06:07:42.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9889999999999992e-06 2023-10-09T06:07:42.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:04.373708+0000", 2023-10-09T06:07:42.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91545159799999998 2023-10-09T06:07:42.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289159+0000", 2023-10-09T06:07:42.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7560000000000002e-06 2023-10-09T06:07:42.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2023-10-09T06:07:42.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:05.289166+0000", 2023-10-09T06:07:42.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 12.003512383 2023-10-09T06:07:42.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2023-10-09T06:07:42.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.292679+0000", 2023-10-09T06:07:42.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059218559999999996 2023-10-09T06:07:42.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.298600+0000", 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071744410000000002 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.305775+0000", 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.026882515999999999 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2023-10-09T06:07:42.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.332657+0000", 2023-10-09T06:07:42.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.25068875899999998 2023-10-09T06:07:42.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.583346+0000", 2023-10-09T06:07:42.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016443880000000001 2023-10-09T06:07:42.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.584991+0000", 2023-10-09T06:07:42.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014049830000000001 2023-10-09T06:07:42.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:42.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.586396+0000", 2023-10-09T06:07:42.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025583310000000001 2023-10-09T06:07:42.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2023-10-09T06:07:42.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.588954+0000", 2023-10-09T06:07:42.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01395045 2023-10-09T06:07:42.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-10-09T06:07:42.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-10-09T06:07:42.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2023-10-09T06:07:42.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2023-10-09T06:00:17.602904+0000", 2023-10-09T06:07:42.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 13.449789453999999 2023-10-09T06:07:42.234 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.234 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.234 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.234 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-10-09T06:07:42.235 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-10-09T06:07:42.235 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:07:42.236 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:07:42.236 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:42 smithi191 ceph-mon[161669]: pgmap v1751: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 330 op/s; 24297/84 objects degraded (28925.000%); 1.6 MiB/s, 30 objects/s recovering 2023-10-09T06:07:42.236 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2485693261' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:07:42.285 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:42 smithi008 ceph-mon[184418]: pgmap v1751: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 330 op/s; 24297/84 objects degraded (28925.000%); 1.6 MiB/s, 30 objects/s recovering 2023-10-09T06:07:42.285 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2485693261' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:07:42.286 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:42 smithi008 ceph-mon[191227]: pgmap v1751: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 330 op/s; 24297/84 objects degraded (28925.000%); 1.6 MiB/s, 30 objects/s recovering 2023-10-09T06:07:42.286 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2485693261' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:07:42.622 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:42.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:42.728 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:42.737 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:07:42.758 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:43.079 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:43.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:43.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:43.095 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:07:43.107 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:43.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:43.211 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:43 smithi191 ceph-mon[161669]: failed to encode map e575 with expected crc 2023-10-09T06:07:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:43 smithi191 ceph-mon[161669]: failed to encode map e575 with expected crc 2023-10-09T06:07:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:43 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2485693261' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:07:43.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:43 smithi191 ceph-mon[161669]: osdmap e575: 8 total, 8 up, 7 in 2023-10-09T06:07:43.304 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:43.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:43.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:43.341 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:43.409 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:43.422 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:43 smithi008 ceph-mon[184418]: failed to encode map e575 with expected crc 2023-10-09T06:07:43.422 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:43 smithi008 ceph-mon[184418]: failed to encode map e575 with expected crc 2023-10-09T06:07:43.422 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:43 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2485693261' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:07:43.422 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:43 smithi008 ceph-mon[184418]: osdmap e575: 8 total, 8 up, 7 in 2023-10-09T06:07:43.422 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:43 smithi008 ceph-mon[191227]: failed to encode map e575 with expected crc 2023-10-09T06:07:43.422 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:43 smithi008 ceph-mon[191227]: failed to encode map e575 with expected crc 2023-10-09T06:07:43.423 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:43 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2485693261' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:07:43.423 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:43 smithi008 ceph-mon[191227]: osdmap e575: 8 total, 8 up, 7 in 2023-10-09T06:07:43.449 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:43.701 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:43.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:43.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:43.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:43.833 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:43.876 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:43.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:43.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:43.980 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:44.018 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:44.067 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:44.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:44.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:44.161 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:44.177 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:44.210 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:44 smithi191 ceph-mon[161669]: failed to encode map e575 with expected crc 2023-10-09T06:07:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:44 smithi191 ceph-mon[161669]: pgmap v1753: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 24297/84 objects degraded (28925.000%); 1.8 MiB/s, 49 objects/s recovering 2023-10-09T06:07:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:44 smithi191 ceph-mon[161669]: Health check update: Degraded data redundancy: 24297/84 objects degraded (28925.000%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3136168662' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:44 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:44.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:44.357 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:44.359 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[184418]: failed to encode map e575 with expected crc 2023-10-09T06:07:44.359 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[184418]: pgmap v1753: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 24297/84 objects degraded (28925.000%); 1.8 MiB/s, 49 objects/s recovering 2023-10-09T06:07:44.359 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[184418]: Health check update: Degraded data redundancy: 24297/84 objects degraded (28925.000%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:44.359 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3136168662' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:44.359 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:44.359 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:44.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:44.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:44.361 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[191227]: failed to encode map e575 with expected crc 2023-10-09T06:07:44.361 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[191227]: pgmap v1753: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 24297/84 objects degraded (28925.000%); 1.8 MiB/s, 49 objects/s recovering 2023-10-09T06:07:44.361 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[191227]: Health check update: Degraded data redundancy: 24297/84 objects degraded (28925.000%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:07:44.361 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3136168662' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:44.361 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:44.361 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:44.461 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:44.484 INFO:teuthology.orchestra.run.smithi008.stderr:pool 'unique_pool_5' removed 2023-10-09T06:07:44.489 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:07:44.516 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:44.617 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:07:44.617 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2023-10-09T06:07:44.617 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2023-10-09T06:07:44.617 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:07:44.910 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:44.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:44.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:45.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:45.046 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:45.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3179979017' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[184418]: failed to encode map e576 with expected crc 2023-10-09T06:07:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[184418]: failed to encode map e576 with expected crc 2023-10-09T06:07:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:07:45.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:07:45.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[184418]: osdmap e576: 8 total, 8 up, 7 in 2023-10-09T06:07:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3179979017' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[191227]: failed to encode map e576 with expected crc 2023-10-09T06:07:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[191227]: failed to encode map e576 with expected crc 2023-10-09T06:07:45.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:07:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:07:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:45 smithi008 ceph-mon[191227]: osdmap e576: 8 total, 8 up, 7 in 2023-10-09T06:07:45.298 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:45.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3179979017' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:45.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:45 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:45 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:45 smithi191 ceph-mon[161669]: failed to encode map e576 with expected crc 2023-10-09T06:07:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:45 smithi191 ceph-mon[161669]: failed to encode map e576 with expected crc 2023-10-09T06:07:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:45 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:07:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:45 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:07:45.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:45 smithi191 ceph-mon[161669]: osdmap e576: 8 total, 8 up, 7 in 2023-10-09T06:07:45.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:45.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:45.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:07:45.402 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:45.430 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:45.474 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:45.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:45.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:45.580 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:45.607 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:45.652 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:45.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:45.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:45.756 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:45.767 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:45.784 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:46.116 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:46.121 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:07:46.122 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T06:07:46.122 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:07:46.122 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T06:07:46.123 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:07:46.123 DEBUG:teuthology.orchestra.run.smithi008:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:07:46.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:46.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:46.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:07:46.201 INFO:teuthology.orchestra.run.smithi008.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:07:46.201 DEBUG:teuthology.orchestra.run.smithi008:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:07:46.225 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:46.250 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:46 smithi191 ceph-mon[161669]: pgmap v1755: 73 pgs: 73 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:46.251 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]: dispatch 2023-10-09T06:07:46.251 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 7]}]: dispatch 2023-10-09T06:07:46.251 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T06:07:46.251 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 6]}]: dispatch 2023-10-09T06:07:46.251 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]: dispatch 2023-10-09T06:07:46.251 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2023-10-09T06:07:46.251 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:46 smithi191 ceph-mon[161669]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24297/84 objects degraded (28925.000%), 7 pgs degraded, 2 pgs undersized) 2023-10-09T06:07:46.253 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:46.276 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:07:46.277 DEBUG:teuthology.orchestra.run.smithi008:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:07:46.296 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:46.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:46.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:46.332 INFO:tasks.workunit:timeout=3h 2023-10-09T06:07:46.333 INFO:tasks.workunit:cleanup=True 2023-10-09T06:07:46.333 DEBUG:teuthology.orchestra.run.smithi008:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:07:46.394 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[184418]: pgmap v1755: 73 pgs: 73 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:46.395 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]: dispatch 2023-10-09T06:07:46.395 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 7]}]: dispatch 2023-10-09T06:07:46.395 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T06:07:46.395 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 6]}]: dispatch 2023-10-09T06:07:46.395 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]: dispatch 2023-10-09T06:07:46.395 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2023-10-09T06:07:46.395 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[184418]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24297/84 objects degraded (28925.000%), 7 pgs degraded, 2 pgs undersized) 2023-10-09T06:07:46.395 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[191227]: pgmap v1755: 73 pgs: 73 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:46.396 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]: dispatch 2023-10-09T06:07:46.396 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 7]}]: dispatch 2023-10-09T06:07:46.396 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T06:07:46.396 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 6]}]: dispatch 2023-10-09T06:07:46.396 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]: dispatch 2023-10-09T06:07:46.396 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2023-10-09T06:07:46.396 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:46 smithi008 ceph-mon[191227]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24297/84 objects degraded (28925.000%), 7 pgs degraded, 2 pgs undersized) 2023-10-09T06:07:46.400 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:46.401 INFO:tasks.workunit.client.0.smithi008.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T06:07:46.434 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:46.631 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:46.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:46.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:46.735 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:46.763 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:46.868 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:07:46.887 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:07:46.905 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:07:46.925 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:07:46.943 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:07:46.961 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:07:46.965 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:46.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:46.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:46.980 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:07:46.999 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:07:47.000 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:07:47.000 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:07:47.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:07:47.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:07:47.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:07:47.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:07:47.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:07:47.001 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:07:47.070 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:47.098 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:47.119 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:47.120 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:47.143 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:47.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:47.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:47.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:47.277 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:47.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:47 smithi191 ceph-mon[161669]: failed to encode map e577 with expected crc 2023-10-09T06:07:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:47 smithi191 ceph-mon[161669]: failed to encode map e577 with expected crc 2023-10-09T06:07:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:47 smithi191 ceph-mon[161669]: failed to encode map e577 with expected crc 2023-10-09T06:07:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]': finished 2023-10-09T06:07:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 7]}]': finished 2023-10-09T06:07:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-09T06:07:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 6]}]': finished 2023-10-09T06:07:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]': finished 2023-10-09T06:07:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2023-10-09T06:07:47.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:47 smithi191 ceph-mon[161669]: osdmap e577: 8 total, 8 up, 7 in 2023-10-09T06:07:47.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:47 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1140428072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:47.372 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:47.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:47.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:47.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[184418]: failed to encode map e577 with expected crc 2023-10-09T06:07:47.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[184418]: failed to encode map e577 with expected crc 2023-10-09T06:07:47.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[184418]: failed to encode map e577 with expected crc 2023-10-09T06:07:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]': finished 2023-10-09T06:07:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 7]}]': finished 2023-10-09T06:07:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-09T06:07:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 6]}]': finished 2023-10-09T06:07:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]': finished 2023-10-09T06:07:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2023-10-09T06:07:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[184418]: osdmap e577: 8 total, 8 up, 7 in 2023-10-09T06:07:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1140428072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:47.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[191227]: failed to encode map e577 with expected crc 2023-10-09T06:07:47.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[191227]: failed to encode map e577 with expected crc 2023-10-09T06:07:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[191227]: failed to encode map e577 with expected crc 2023-10-09T06:07:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]': finished 2023-10-09T06:07:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 7]}]': finished 2023-10-09T06:07:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-09T06:07:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 6]}]': finished 2023-10-09T06:07:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]': finished 2023-10-09T06:07:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2023-10-09T06:07:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[191227]: osdmap e577: 8 total, 8 up, 7 in 2023-10-09T06:07:47.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:47 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1140428072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:47.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:47.503 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:47.793 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:07:47.807 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:47.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:47.913 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:47.939 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:47.982 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:47.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:47.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:48.029 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:07:48.089 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:48.117 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:48.162 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:48.164 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:48.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:48.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:48.271 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:48.298 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[184418]: pgmap v1757: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:48.298 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:07:48.299 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[184418]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.299 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[184418]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.299 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[184418]: osdmap e578: 8 total, 8 up, 7 in 2023-10-09T06:07:48.299 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[184418]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.299 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[184418]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.299 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[184418]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.299 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[184418]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.300 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[191227]: pgmap v1757: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:48.300 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:07:48.300 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[191227]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.300 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[191227]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.300 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[191227]: osdmap e578: 8 total, 8 up, 7 in 2023-10-09T06:07:48.300 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[191227]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.300 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[191227]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.300 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[191227]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.301 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:48 smithi008 ceph-mon[191227]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:48 smithi191 ceph-mon[161669]: pgmap v1757: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:48 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:07:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:48 smithi191 ceph-mon[161669]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:48 smithi191 ceph-mon[161669]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:48 smithi191 ceph-mon[161669]: osdmap e578: 8 total, 8 up, 7 in 2023-10-09T06:07:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:48 smithi191 ceph-mon[161669]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:48 smithi191 ceph-mon[161669]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:48 smithi191 ceph-mon[161669]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.302 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:48 smithi191 ceph-mon[161669]: failed to encode map e578 with expected crc 2023-10-09T06:07:48.302 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:48.503 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:48.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:48.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:48.607 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:48.612 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:48.638 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:48.692 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:48.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:48.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:48.795 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:48.825 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:48.875 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:48.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:48.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:48.979 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:49.007 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:49.177 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:49.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:49.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:49.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:49.314 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:49.408 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:49.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:49.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:49.482 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:07:49.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:49.539 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:49.587 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:49.589 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4190929821' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:07:49.589 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:49 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:07:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:49.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:49.634 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:49.634 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:49.691 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:49.718 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:49.815 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:49.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:49.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:49.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:49.947 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:49.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4190929821' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:07:49.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:49 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:07:49.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4190929821' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:07:49.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:49 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:07:49.989 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:49.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:49.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:50.093 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:50.119 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:50.170 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:07:50.352 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:07:50.365 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:50.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:50.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:50.471 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:50.501 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:50.546 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:50.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:50.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:50.651 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:50.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[184418]: pgmap v1759: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:50.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[184418]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.665 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:07:50.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[184418]: osdmap e579: 8 total, 8 up, 7 in 2023-10-09T06:07:50.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[184418]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[184418]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[184418]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[184418]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[184418]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.666 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[184418]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.666 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:07:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:07:50] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.33.4" 2023-10-09T06:07:50.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[191227]: pgmap v1759: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:50.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[191227]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:07:50.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[191227]: osdmap e579: 8 total, 8 up, 7 in 2023-10-09T06:07:50.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[191227]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[191227]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[191227]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[191227]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[191227]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.667 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:50 smithi008 ceph-mon[191227]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.675 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:50 smithi191 ceph-mon[161669]: pgmap v1759: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:50.676 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:50 smithi191 ceph-mon[161669]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.676 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:50 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:07:50.676 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:50 smithi191 ceph-mon[161669]: osdmap e579: 8 total, 8 up, 7 in 2023-10-09T06:07:50.676 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:50 smithi191 ceph-mon[161669]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.676 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:50 smithi191 ceph-mon[161669]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.676 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:50 smithi191 ceph-mon[161669]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.676 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:50 smithi191 ceph-mon[161669]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.676 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:50 smithi191 ceph-mon[161669]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.676 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:50 smithi191 ceph-mon[161669]: failed to encode map e579 with expected crc 2023-10-09T06:07:50.678 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:50.686 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:07:50.720 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:50.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:50.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:50.826 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:50.857 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:51.121 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:51.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:51.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:51.227 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:51.262 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:51.277 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:51.326 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:51.329 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:51.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:51.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:51.430 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:51.458 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:51.461 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:51.499 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:07:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[184418]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[184418]: osdmap e580: 8 total, 8 up, 7 in 2023-10-09T06:07:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[184418]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[184418]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[184418]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[184418]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[184418]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[184418]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/714908520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:51.716 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:51.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[191227]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[191227]: osdmap e580: 8 total, 8 up, 7 in 2023-10-09T06:07:51.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[191227]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[191227]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.716 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[191227]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[191227]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[191227]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[191227]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/714908520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:51.717 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:51 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:51 smithi191 ceph-mon[161669]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:51 smithi191 ceph-mon[161669]: osdmap e580: 8 total, 8 up, 7 in 2023-10-09T06:07:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:51 smithi191 ceph-mon[161669]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:51 smithi191 ceph-mon[161669]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:51 smithi191 ceph-mon[161669]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:51 smithi191 ceph-mon[161669]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:51 smithi191 ceph-mon[161669]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:51 smithi191 ceph-mon[161669]: failed to encode map e580 with expected crc 2023-10-09T06:07:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:51 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/714908520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:51.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:51 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:51.866 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:51.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:51.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:51.996 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:52.027 DEBUG:teuthology.orchestra.run.smithi008:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T06:07:52.038 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:52.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:52.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:52.060 INFO:tasks.workunit.client.0.smithi008.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T06:07:52.063 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:07:52.063 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T06:07:52.144 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:52.173 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:52.289 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T06:07:52.507 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T06:07:52.650 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:07:52.652 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:07:52.652 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T06:07:52.778 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:07:52.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:52 smithi191 ceph-mon[161669]: pgmap v1762: 73 pgs: 10 peering, 63 active+clean; 4.6 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:52.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:52 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:07:52.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:52 smithi191 ceph-mon[161669]: failed to encode map e581 with expected crc 2023-10-09T06:07:52.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:52 smithi191 ceph-mon[161669]: failed to encode map e581 with expected crc 2023-10-09T06:07:52.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:52 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:52.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:52 smithi191 ceph-mon[161669]: osdmap e581: 8 total, 8 up, 7 in 2023-10-09T06:07:52.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:52 smithi191 ceph-mon[161669]: failed to encode map e581 with expected crc 2023-10-09T06:07:52.963 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T06:07:52.963 DEBUG:teuthology.orchestra.run.smithi008:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T06:07:52.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[191227]: pgmap v1762: 73 pgs: 10 peering, 63 active+clean; 4.6 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:52.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:07:52.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[191227]: failed to encode map e581 with expected crc 2023-10-09T06:07:52.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[191227]: failed to encode map e581 with expected crc 2023-10-09T06:07:52.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[191227]: osdmap e581: 8 total, 8 up, 7 in 2023-10-09T06:07:52.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[191227]: failed to encode map e581 with expected crc 2023-10-09T06:07:52.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[184418]: pgmap v1762: 73 pgs: 10 peering, 63 active+clean; 4.6 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:07:52.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:07:52.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[184418]: failed to encode map e581 with expected crc 2023-10-09T06:07:52.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[184418]: failed to encode map e581 with expected crc 2023-10-09T06:07:52.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:52.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[184418]: osdmap e581: 8 total, 8 up, 7 in 2023-10-09T06:07:52.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:52 smithi008 ceph-mon[184418]: failed to encode map e581 with expected crc 2023-10-09T06:07:53.006 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T06:07:53.008 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T06:07:53.008 DEBUG:teuthology.orchestra.run.smithi008:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T06:07:53.160 INFO:tasks.workunit.client.0.smithi008.stdout:Running main() from gmock_main.cc 2023-10-09T06:07:53.160 INFO:tasks.workunit.client.0.smithi008.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T06:07:53.160 INFO:tasks.workunit.client.0.smithi008.stdout:[----------] Global test environment set-up. 2023-10-09T06:07:53.160 INFO:tasks.workunit.client.0.smithi008.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T06:07:53.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:53 smithi008 ceph-mon[184418]: failed to encode map e582 with expected crc 2023-10-09T06:07:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:53 smithi008 ceph-mon[184418]: failed to encode map e582 with expected crc 2023-10-09T06:07:53.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:53 smithi008 ceph-mon[184418]: osdmap e582: 8 total, 8 up, 7 in 2023-10-09T06:07:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:53 smithi008 ceph-mon[191227]: failed to encode map e582 with expected crc 2023-10-09T06:07:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:53 smithi008 ceph-mon[191227]: failed to encode map e582 with expected crc 2023-10-09T06:07:53.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:53 smithi008 ceph-mon[191227]: osdmap e582: 8 total, 8 up, 7 in 2023-10-09T06:07:54.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:53 smithi191 ceph-mon[161669]: failed to encode map e582 with expected crc 2023-10-09T06:07:54.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:53 smithi191 ceph-mon[161669]: failed to encode map e582 with expected crc 2023-10-09T06:07:54.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:53 smithi191 ceph-mon[161669]: osdmap e582: 8 total, 8 up, 7 in 2023-10-09T06:07:54.648 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T06:07:54.778 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.get_all_features (131 ms) 2023-10-09T06:07:54.779 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T06:07:54.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: failed to encode map e582 with expected crc 2023-10-09T06:07:54.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: failed to encode map e582 with expected crc 2023-10-09T06:07:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: pgmap v1765: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 138 B/s, 1 objects/s recovering 2023-10-09T06:07:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:07:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T06:07:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: failed to encode map e583 with expected crc 2023-10-09T06:07:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: failed to encode map e583 with expected crc 2023-10-09T06:07:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: osdmap e583: 8 total, 8 up, 7 in 2023-10-09T06:07:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: failed to encode map e583 with expected crc 2023-10-09T06:07:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1308642939' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi008-700076-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: failed to encode map e583 with expected crc 2023-10-09T06:07:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: failed to encode map e583 with expected crc 2023-10-09T06:07:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi008-700076-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: failed to encode map e582 with expected crc 2023-10-09T06:07:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: failed to encode map e582 with expected crc 2023-10-09T06:07:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: pgmap v1765: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 138 B/s, 1 objects/s recovering 2023-10-09T06:07:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:07:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T06:07:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: failed to encode map e583 with expected crc 2023-10-09T06:07:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: failed to encode map e583 with expected crc 2023-10-09T06:07:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: osdmap e583: 8 total, 8 up, 7 in 2023-10-09T06:07:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: failed to encode map e583 with expected crc 2023-10-09T06:07:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1308642939' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi008-700076-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:54.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: failed to encode map e583 with expected crc 2023-10-09T06:07:54.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: failed to encode map e583 with expected crc 2023-10-09T06:07:54.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:54 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi008-700076-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:54.989 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.copyup (211 ms) 2023-10-09T06:07:54.989 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T06:07:55.004 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.sparse_copyup (15 ms) 2023-10-09T06:07:55.005 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T06:07:55.013 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.get_and_set_id (9 ms) 2023-10-09T06:07:55.013 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T06:07:55.024 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2023-10-09T06:07:55.024 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T06:07:55.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: failed to encode map e582 with expected crc 2023-10-09T06:07:55.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: failed to encode map e582 with expected crc 2023-10-09T06:07:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: pgmap v1765: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 138 B/s, 1 objects/s recovering 2023-10-09T06:07:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:07:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T06:07:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: failed to encode map e583 with expected crc 2023-10-09T06:07:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: failed to encode map e583 with expected crc 2023-10-09T06:07:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: osdmap e583: 8 total, 8 up, 7 in 2023-10-09T06:07:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: failed to encode map e583 with expected crc 2023-10-09T06:07:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1308642939' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi008-700076-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: failed to encode map e583 with expected crc 2023-10-09T06:07:55.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: failed to encode map e583 with expected crc 2023-10-09T06:07:55.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:54 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi008-700076-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:55.239 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.directory_methods (215 ms) 2023-10-09T06:07:55.239 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.create 2023-10-09T06:07:55.254 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.create (15 ms) 2023-10-09T06:07:55.254 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T06:07:55.256 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2023-10-09T06:07:55.256 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T06:07:55.257 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-10-09T06:07:55.257 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T06:07:55.259 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-10-09T06:07:55.259 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T06:07:55.260 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-10-09T06:07:55.260 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T06:07:55.262 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-10-09T06:07:55.262 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T06:07:55.265 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-10-09T06:07:55.265 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T06:07:55.312 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.get_size (47 ms) 2023-10-09T06:07:55.312 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T06:07:55.318 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.set_size (6 ms) 2023-10-09T06:07:55.318 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T06:07:55.338 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.protection_status (20 ms) 2023-10-09T06:07:55.338 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T06:07:55.345 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-10-09T06:07:55.345 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T06:07:55.395 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.parents_v1 (50 ms) 2023-10-09T06:07:55.395 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T06:07:55.428 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.parents_v2 (33 ms) 2023-10-09T06:07:55.428 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T06:07:55.449 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.snapshots (21 ms) 2023-10-09T06:07:55.450 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T06:07:55.453 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.snapid_race (4 ms) 2023-10-09T06:07:55.453 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T06:07:55.462 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2023-10-09T06:07:55.462 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T06:07:55.464 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.object_map_save (1 ms) 2023-10-09T06:07:55.464 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T06:07:55.469 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2023-10-09T06:07:55.469 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T06:07:55.472 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.object_map_update (4 ms) 2023-10-09T06:07:55.472 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T06:07:55.472 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T06:07:55.473 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T06:07:55.479 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.object_map_snap_add (6 ms) 2023-10-09T06:07:55.479 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T06:07:55.486 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.object_map_snap_remove (8 ms) 2023-10-09T06:07:55.486 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T06:07:55.491 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-10-09T06:07:55.491 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T06:07:55.515 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.metadata (23 ms) 2023-10-09T06:07:55.515 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T06:07:55.523 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.set_features (8 ms) 2023-10-09T06:07:55.523 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T06:07:55.557 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.mirror (35 ms) 2023-10-09T06:07:55.557 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T06:07:55.572 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.mirror_image (14 ms) 2023-10-09T06:07:55.572 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T06:07:55.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[184418]: failed to encode map e584 with expected crc 2023-10-09T06:07:55.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[184418]: failed to encode map e584 with expected crc 2023-10-09T06:07:55.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi008-700076-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:07:55.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[184418]: osdmap e584: 8 total, 8 up, 7 in 2023-10-09T06:07:55.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[184418]: failed to encode map e584 with expected crc 2023-10-09T06:07:55.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[184418]: failed to encode map e584 with expected crc 2023-10-09T06:07:55.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[184418]: failed to encode map e584 with expected crc 2023-10-09T06:07:55.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:55.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[191227]: failed to encode map e584 with expected crc 2023-10-09T06:07:55.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[191227]: failed to encode map e584 with expected crc 2023-10-09T06:07:55.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi008-700076-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:07:55.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[191227]: osdmap e584: 8 total, 8 up, 7 in 2023-10-09T06:07:55.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[191227]: failed to encode map e584 with expected crc 2023-10-09T06:07:55.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[191227]: failed to encode map e584 with expected crc 2023-10-09T06:07:55.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[191227]: failed to encode map e584 with expected crc 2023-10-09T06:07:55.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:55 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:56.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:55 smithi191 ceph-mon[161669]: failed to encode map e584 with expected crc 2023-10-09T06:07:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:55 smithi191 ceph-mon[161669]: failed to encode map e584 with expected crc 2023-10-09T06:07:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:55 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi008-700076-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:07:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:55 smithi191 ceph-mon[161669]: osdmap e584: 8 total, 8 up, 7 in 2023-10-09T06:07:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:55 smithi191 ceph-mon[161669]: failed to encode map e584 with expected crc 2023-10-09T06:07:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:55 smithi191 ceph-mon[161669]: failed to encode map e584 with expected crc 2023-10-09T06:07:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:55 smithi191 ceph-mon[161669]: failed to encode map e584 with expected crc 2023-10-09T06:07:56.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:55 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:56.066 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:56.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:56.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:56.171 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:56.212 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:56.256 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:56.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:56.362 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:56.405 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:56.606 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:56.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:56.713 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:56.740 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:56.837 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:56.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:56.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:56.942 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:56.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[184418]: pgmap v1768: 105 pgs: 1 creating+activating, 3 creating+peering, 17 unknown, 84 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s; 144 KiB/s, 6 objects/s recovering 2023-10-09T06:07:56.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[184418]: failed to encode map e585 with expected crc 2023-10-09T06:07:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[184418]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T06:07:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[184418]: failed to encode map e585 with expected crc 2023-10-09T06:07:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[184418]: failed to encode map e585 with expected crc 2023-10-09T06:07:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[184418]: failed to encode map e585 with expected crc 2023-10-09T06:07:56.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[191227]: pgmap v1768: 105 pgs: 1 creating+activating, 3 creating+peering, 17 unknown, 84 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s; 144 KiB/s, 6 objects/s recovering 2023-10-09T06:07:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[191227]: failed to encode map e585 with expected crc 2023-10-09T06:07:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[191227]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T06:07:56.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[191227]: failed to encode map e585 with expected crc 2023-10-09T06:07:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[191227]: failed to encode map e585 with expected crc 2023-10-09T06:07:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[191227]: failed to encode map e585 with expected crc 2023-10-09T06:07:56.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:56 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:56.974 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:56 smithi191 ceph-mon[161669]: pgmap v1768: 105 pgs: 1 creating+activating, 3 creating+peering, 17 unknown, 84 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s; 144 KiB/s, 6 objects/s recovering 2023-10-09T06:07:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:56 smithi191 ceph-mon[161669]: failed to encode map e585 with expected crc 2023-10-09T06:07:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:56 smithi191 ceph-mon[161669]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T06:07:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:56 smithi191 ceph-mon[161669]: failed to encode map e585 with expected crc 2023-10-09T06:07:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:56 smithi191 ceph-mon[161669]: failed to encode map e585 with expected crc 2023-10-09T06:07:57.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:56 smithi191 ceph-mon[161669]: failed to encode map e585 with expected crc 2023-10-09T06:07:57.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:56 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:58.067 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:58.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:58.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:58.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:58.200 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:58.246 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:58.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:58.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:58.354 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:58.392 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:58.480 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:58.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:58.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:58.585 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:58.596 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:07:58.634 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:07:58.640 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:58.659 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:07:58.691 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:07:58.715 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:07:58.739 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:07:58.763 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:07:58.790 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:07:58.791 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:07:58.792 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:07:58.792 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:07:58.792 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:07:58.792 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:07:58.792 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:07:58.792 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:07:58.792 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:07:58.819 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:58 smithi008 ceph-mon[184418]: pgmap v1770: 105 pgs: 1 creating+activating, 3 creating+peering, 11 unknown, 90 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 43 KiB/s rd, 8.0 KiB/s wr, 62 op/s; 129 KiB/s, 5 objects/s recovering 2023-10-09T06:07:58.819 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:58 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:58.819 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:58 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:58.819 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:58 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2959160148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:58.819 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:58 smithi008 ceph-mon[191227]: pgmap v1770: 105 pgs: 1 creating+activating, 3 creating+peering, 11 unknown, 90 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 43 KiB/s rd, 8.0 KiB/s wr, 62 op/s; 129 KiB/s, 5 objects/s recovering 2023-10-09T06:07:58.819 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:58 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:58.819 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:58 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:58.819 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:58 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2959160148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:58 smithi191 ceph-mon[161669]: pgmap v1770: 105 pgs: 1 creating+activating, 3 creating+peering, 11 unknown, 90 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 43 KiB/s rd, 8.0 KiB/s wr, 62 op/s; 129 KiB/s, 5 objects/s recovering 2023-10-09T06:07:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:58 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:58 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:59.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:58 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2959160148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:59.068 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:07:59.275 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.mirror_image_status (3704 ms) 2023-10-09T06:07:59.276 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T06:07:59.373 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.mirror_image_map (98 ms) 2023-10-09T06:07:59.373 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T06:07:59.383 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.mirror_instances (10 ms) 2023-10-09T06:07:59.383 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T06:07:59.396 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.mirror_snapshot (12 ms) 2023-10-09T06:07:59.396 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T06:07:59.398 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2023-10-09T06:07:59.398 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T06:07:59.400 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-10-09T06:07:59.400 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T06:07:59.403 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-10-09T06:07:59.403 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T06:07:59.409 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-10-09T06:07:59.409 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T06:07:59.412 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-10-09T06:07:59.413 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T06:07:59.417 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-10-09T06:07:59.417 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T06:07:59.419 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-10-09T06:07:59.419 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T06:07:59.506 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_image_remove (87 ms) 2023-10-09T06:07:59.507 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T06:07:59.509 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-10-09T06:07:59.510 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T06:07:59.514 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2023-10-09T06:07:59.514 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T06:07:59.517 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-10-09T06:07:59.517 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T06:07:59.520 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2023-10-09T06:07:59.520 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T06:07:59.523 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-10-09T06:07:59.523 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T06:07:59.524 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-10-09T06:07:59.524 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T06:07:59.526 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-09T06:07:59.526 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T06:07:59.529 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-10-09T06:07:59.529 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T06:07:59.532 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-10-09T06:07:59.533 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T06:07:59.535 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-10-09T06:07:59.535 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T06:07:59.538 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2023-10-09T06:07:59.538 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T06:07:59.554 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (15 ms) 2023-10-09T06:07:59.554 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T06:07:59.669 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:07:59.669 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:07:59.669 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T06:07:59.669 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:07:59.697 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:59.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:59.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:59.801 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:59.830 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:59.876 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:59.877 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (324 ms) 2023-10-09T06:07:59.877 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T06:07:59.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:59.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:59.881 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-10-09T06:07:59.882 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T06:07:59.883 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-10-09T06:07:59.883 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T06:07:59.892 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.trash_methods (9 ms) 2023-10-09T06:07:59.892 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T06:07:59.899 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2023-10-09T06:07:59.899 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T06:07:59.922 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.clone_parent (22 ms) 2023-10-09T06:07:59.922 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T06:07:59.930 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-10-09T06:07:59.931 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T06:07:59.943 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.clone_child (13 ms) 2023-10-09T06:07:59.943 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T06:07:59.951 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.namespace_methods (8 ms) 2023-10-09T06:07:59.951 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T06:07:59.952 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:59.952 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:59.953 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:59 smithi008 ceph-mon[184418]: pgmap v1771: 105 pgs: 1 creating+activating, 3 creating+peering, 101 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 542 KiB/s rd, 344 KiB/s wr, 916 op/s; 96 KiB/s, 2 objects/s recovering 2023-10-09T06:07:59.953 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:07:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/479114237' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:07:59.953 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:07:59.953 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:59.953 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:59 smithi008 ceph-mon[191227]: pgmap v1771: 105 pgs: 1 creating+activating, 3 creating+peering, 101 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 542 KiB/s rd, 344 KiB/s wr, 916 op/s; 96 KiB/s, 2 objects/s recovering 2023-10-09T06:07:59.953 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:07:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/479114237' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:07:59.959 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:07:59.965 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.migration (13 ms) 2023-10-09T06:07:59.965 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T06:07:59.975 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.migration_v1 (11 ms) 2023-10-09T06:07:59.975 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T06:07:59.982 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:00.011 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:59 smithi191 ceph-mon[161669]: pgmap v1771: 105 pgs: 1 creating+activating, 3 creating+peering, 101 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 542 KiB/s rd, 344 KiB/s wr, 916 op/s; 96 KiB/s, 2 objects/s recovering 2023-10-09T06:08:00.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:07:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/479114237' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:08:00.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:08:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:08:00] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.33.4" 2023-10-09T06:08:01.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:00 smithi008 ceph-mon[184418]: failed to encode map e586 with expected crc 2023-10-09T06:08:01.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:00 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/479114237' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:08:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:00 smithi008 ceph-mon[184418]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T06:08:01.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:00 smithi008 ceph-mon[184418]: failed to encode map e586 with expected crc 2023-10-09T06:08:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:00 smithi008 ceph-mon[191227]: failed to encode map e586 with expected crc 2023-10-09T06:08:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:00 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/479114237' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:08:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:00 smithi008 ceph-mon[191227]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T06:08:01.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:00 smithi008 ceph-mon[191227]: failed to encode map e586 with expected crc 2023-10-09T06:08:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:00 smithi191 ceph-mon[161669]: failed to encode map e586 with expected crc 2023-10-09T06:08:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:00 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/479114237' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:08:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:00 smithi191 ceph-mon[161669]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T06:08:01.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:00 smithi191 ceph-mon[161669]: failed to encode map e586 with expected crc 2023-10-09T06:08:01.359 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:08:01.360 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:08:01.480 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:01.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:01.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:01.584 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:01.613 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:01.712 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:01.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:01.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:01.787 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:08:01.816 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:01.849 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:01.899 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:01.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:01.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:01.938 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:01.972 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1996 ms) 2023-10-09T06:08:01.972 INFO:tasks.workunit.client.0.smithi008.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T06:08:02.004 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:02.019 INFO:tasks.workunit.client.0.smithi008.stdout:[ OK ] TestClsRbd.sparsify (48 ms) 2023-10-09T06:08:02.031 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:02.073 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:02.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:02.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:02.178 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:02.200 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[184418]: failed to encode map e586 with expected crc 2023-10-09T06:08:02.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[184418]: failed to encode map e586 with expected crc 2023-10-09T06:08:02.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[184418]: failed to encode map e587 with expected crc 2023-10-09T06:08:02.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[184418]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T06:08:02.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[184418]: failed to encode map e587 with expected crc 2023-10-09T06:08:02.201 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[184418]: pgmap v1774: 105 pgs: 105 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 960 op/s 2023-10-09T06:08:02.201 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[191227]: failed to encode map e586 with expected crc 2023-10-09T06:08:02.201 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[191227]: failed to encode map e586 with expected crc 2023-10-09T06:08:02.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[191227]: failed to encode map e587 with expected crc 2023-10-09T06:08:02.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[191227]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T06:08:02.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[191227]: failed to encode map e587 with expected crc 2023-10-09T06:08:02.202 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:01 smithi008 ceph-mon[191227]: pgmap v1774: 105 pgs: 105 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 960 op/s 2023-10-09T06:08:02.211 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:02.260 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:02.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:02.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:02.267 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:02.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:01 smithi191 ceph-mon[161669]: failed to encode map e586 with expected crc 2023-10-09T06:08:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:01 smithi191 ceph-mon[161669]: failed to encode map e586 with expected crc 2023-10-09T06:08:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:01 smithi191 ceph-mon[161669]: failed to encode map e587 with expected crc 2023-10-09T06:08:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:01 smithi191 ceph-mon[161669]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T06:08:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:01 smithi191 ceph-mon[161669]: failed to encode map e587 with expected crc 2023-10-09T06:08:02.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:01 smithi191 ceph-mon[161669]: pgmap v1774: 105 pgs: 105 active+clean; 4.6 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 960 op/s 2023-10-09T06:08:02.364 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:02.394 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:02.696 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:02.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:02.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:02.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:02.832 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:02.881 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:02.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:02.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:02.976 INFO:tasks.workunit.client.0.smithi008.stdout:[----------] 70 tests from TestClsRbd (7372 ms total) 2023-10-09T06:08:02.976 INFO:tasks.workunit.client.0.smithi008.stdout: 2023-10-09T06:08:02.976 INFO:tasks.workunit.client.0.smithi008.stdout:[----------] Global test environment tear-down 2023-10-09T06:08:02.976 INFO:tasks.workunit.client.0.smithi008.stdout:[==========] 70 tests from 1 test suite ran. (9817 ms total) 2023-10-09T06:08:02.977 INFO:tasks.workunit.client.0.smithi008.stdout:[ PASSED ] 70 tests. 2023-10-09T06:08:02.978 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T06:08:02.978 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T06:08:02.989 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:03.007 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T06:08:03.007 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:08:03.016 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:03.060 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:03.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:03.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:03.165 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:03.192 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:03.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[184418]: failed to encode map e587 with expected crc 2023-10-09T06:08:03.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[184418]: failed to encode map e587 with expected crc 2023-10-09T06:08:03.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[184418]: failed to encode map e588 with expected crc 2023-10-09T06:08:03.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[184418]: failed to encode map e588 with expected crc 2023-10-09T06:08:03.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[184418]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T06:08:03.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[184418]: failed to encode map e588 with expected crc 2023-10-09T06:08:03.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[191227]: failed to encode map e587 with expected crc 2023-10-09T06:08:03.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[191227]: failed to encode map e587 with expected crc 2023-10-09T06:08:03.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[191227]: failed to encode map e588 with expected crc 2023-10-09T06:08:03.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[191227]: failed to encode map e588 with expected crc 2023-10-09T06:08:03.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[191227]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T06:08:03.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:02 smithi008 ceph-mon[191227]: failed to encode map e588 with expected crc 2023-10-09T06:08:03.266 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:03.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:03.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:03.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:02 smithi191 ceph-mon[161669]: failed to encode map e587 with expected crc 2023-10-09T06:08:03.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:02 smithi191 ceph-mon[161669]: failed to encode map e587 with expected crc 2023-10-09T06:08:03.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:02 smithi191 ceph-mon[161669]: failed to encode map e588 with expected crc 2023-10-09T06:08:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:02 smithi191 ceph-mon[161669]: failed to encode map e588 with expected crc 2023-10-09T06:08:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:02 smithi191 ceph-mon[161669]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T06:08:03.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:02 smithi191 ceph-mon[161669]: failed to encode map e588 with expected crc 2023-10-09T06:08:03.363 DEBUG:teuthology.parallel:result is None 2023-10-09T06:08:03.363 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:08:03.371 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:03.381 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:03.396 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:08:03.397 DEBUG:teuthology.orchestra.run.smithi008:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:08:03.415 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:03.439 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:08:03.439 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:08:03.439 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T06:08:03.439 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:08:03.440 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T06:08:03.440 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:08:03.440 DEBUG:teuthology.orchestra.run.smithi008:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:08:03.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:08:03.499 INFO:teuthology.orchestra.run.smithi008.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:08:03.499 DEBUG:teuthology.orchestra.run.smithi008:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:08:03.522 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:03.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:03.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:03.542 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:08:03.543 DEBUG:teuthology.orchestra.run.smithi008:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:08:03.603 INFO:tasks.workunit:timeout=3h 2023-10-09T06:08:03.604 INFO:tasks.workunit:cleanup=True 2023-10-09T06:08:03.604 DEBUG:teuthology.orchestra.run.smithi008:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:08:03.630 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:03.657 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:03.662 INFO:tasks.workunit.client.0.smithi008.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T06:08:03.691 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:08:03.700 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:03.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:03.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:03.805 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:03.838 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:04.194 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:04.196 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:03 smithi008 ceph-mon[184418]: failed to encode map e589 with expected crc 2023-10-09T06:08:04.196 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:03 smithi008 ceph-mon[184418]: osdmap e589: 8 total, 8 up, 7 in 2023-10-09T06:08:04.196 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:03 smithi008 ceph-mon[184418]: failed to encode map e589 with expected crc 2023-10-09T06:08:04.196 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:03 smithi008 ceph-mon[184418]: pgmap v1777: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:04.196 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:03 smithi008 ceph-mon[184418]: failed to encode map e589 with expected crc 2023-10-09T06:08:04.196 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:03 smithi008 ceph-mon[191227]: failed to encode map e589 with expected crc 2023-10-09T06:08:04.197 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:03 smithi008 ceph-mon[191227]: osdmap e589: 8 total, 8 up, 7 in 2023-10-09T06:08:04.197 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:03 smithi008 ceph-mon[191227]: failed to encode map e589 with expected crc 2023-10-09T06:08:04.197 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:03 smithi008 ceph-mon[191227]: pgmap v1777: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:04.197 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:03 smithi008 ceph-mon[191227]: failed to encode map e589 with expected crc 2023-10-09T06:08:04.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:04.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:04.299 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:03 smithi191 ceph-mon[161669]: failed to encode map e589 with expected crc 2023-10-09T06:08:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:03 smithi191 ceph-mon[161669]: osdmap e589: 8 total, 8 up, 7 in 2023-10-09T06:08:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:03 smithi191 ceph-mon[161669]: failed to encode map e589 with expected crc 2023-10-09T06:08:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:03 smithi191 ceph-mon[161669]: pgmap v1777: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:04.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:03 smithi191 ceph-mon[161669]: failed to encode map e589 with expected crc 2023-10-09T06:08:04.327 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:04.983 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is set 2023-10-09T06:08:05.177 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:05.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:05.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:05.281 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:05.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:04 smithi191 ceph-mon[161669]: failed to encode map e589 with expected crc 2023-10-09T06:08:05.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/170471578' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:05.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:04 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:05.313 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:05.360 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:05.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:05.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:05.395 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:04 smithi008 ceph-mon[184418]: failed to encode map e589 with expected crc 2023-10-09T06:08:05.395 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/170471578' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:05.395 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:04 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:05.395 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:04 smithi008 ceph-mon[191227]: failed to encode map e589 with expected crc 2023-10-09T06:08:05.395 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/170471578' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:05.395 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:04 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:05.466 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:05.496 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:06.197 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:06.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:06.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:06.269 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[191227]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:06.269 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[191227]: pgmap v1779: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:06.269 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[191227]: failed to encode map e590 with expected crc 2023-10-09T06:08:06.269 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[191227]: failed to encode map e590 with expected crc 2023-10-09T06:08:06.269 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:08:06.269 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[191227]: osdmap e590: 8 total, 8 up, 7 in 2023-10-09T06:08:06.270 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[184418]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:06.270 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[184418]: pgmap v1779: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:06.270 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[184418]: failed to encode map e590 with expected crc 2023-10-09T06:08:06.270 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[184418]: failed to encode map e590 with expected crc 2023-10-09T06:08:06.270 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:08:06.270 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:05 smithi008 ceph-mon[184418]: osdmap e590: 8 total, 8 up, 7 in 2023-10-09T06:08:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:05 smithi191 ceph-mon[161669]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:05 smithi191 ceph-mon[161669]: pgmap v1779: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:05 smithi191 ceph-mon[161669]: failed to encode map e590 with expected crc 2023-10-09T06:08:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:05 smithi191 ceph-mon[161669]: failed to encode map e590 with expected crc 2023-10-09T06:08:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:05 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:08:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:05 smithi191 ceph-mon[161669]: osdmap e590: 8 total, 8 up, 7 in 2023-10-09T06:08:06.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:06.333 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:06.381 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:06.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:06.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:06.487 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:06.515 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:06.561 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:06.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:06.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:06.667 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:06.694 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:06.737 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:06.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:06.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:06.843 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:06.874 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:07.125 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:06 smithi008 ceph-mon[184418]: failed to encode map e590 with expected crc 2023-10-09T06:08:07.125 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:06 smithi008 ceph-mon[184418]: failed to encode map e590 with expected crc 2023-10-09T06:08:07.125 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:06 smithi008 ceph-mon[191227]: failed to encode map e590 with expected crc 2023-10-09T06:08:07.125 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:06 smithi008 ceph-mon[191227]: failed to encode map e590 with expected crc 2023-10-09T06:08:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:06 smithi191 ceph-mon[161669]: failed to encode map e590 with expected crc 2023-10-09T06:08:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:06 smithi191 ceph-mon[161669]: failed to encode map e590 with expected crc 2023-10-09T06:08:07.339 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:07.474 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:07.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:07.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:07.576 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:08:07.577 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T06:08:07.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:07.611 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:07.656 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:07.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:07.763 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:07.795 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:08.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:07 smithi008 ceph-mon[184418]: pgmap v1780: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:08.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:07 smithi008 ceph-mon[184418]: from='client.37449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:08.134 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:07 smithi008 ceph-mon[191227]: pgmap v1780: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:08.135 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:07 smithi008 ceph-mon[191227]: from='client.37449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:08.150 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:08.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:08.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:08.254 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:08.283 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:08.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:07 smithi191 ceph-mon[161669]: pgmap v1780: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:08.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:07 smithi191 ceph-mon[161669]: from='client.37449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:08.972 DEBUG:teuthology.orchestra.run.smithi008:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T06:08:08.979 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:08.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:08.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:08.996 INFO:tasks.workunit.client.0.smithi008.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T06:08:08.999 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:08:08.999 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T06:08:09.055 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T06:08:09.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:09.116 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:09.126 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T06:08:09.165 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:09.165 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:08:09.167 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:08:09.167 INFO:tasks.workunit.client.0.smithi008.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T06:08:09.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:09.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:09.222 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:08:09.222 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:08:09.222 INFO:teuthology.orchestra.run.smithi008.stdout: "complaint_time": 30, 2023-10-09T06:08:09.222 INFO:teuthology.orchestra.run.smithi008.stdout: "num_blocked_ops": 0 2023-10-09T06:08:09.222 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:08:09.230 INFO:tasks.workunit.client.0.smithi008.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:08:09.234 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2023-10-09T06:08:09.234 DEBUG:teuthology.orchestra.run.smithi008:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T06:08:09.272 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1780,"stamp":"2023-10-09T06:08:06.977080+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6288,"num_read_kb":13711,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":266,"num_bytes_recovered":10777762,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8902196,"kb_used_data":1541308,"kb_used_omap":59,"kb_used_meta":7360772,"kb_avail":647313868,"statfs":{"total":671965249536,"available":662849400832,"internally_reserved":0,"allocated":1578299392,"data_stored":1565267671,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":60463,"internal_metadata":7537431505},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-374,"num_objects":-40,"num_object_clones":0,"num_object_copies":-120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-40,"num_whiteouts":0,"num_read":-3562,"num_read_kb":-11651,"num_write":-2208,"num_write_kb":-6249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-31,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001671"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":441,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337044+0000","last_change":"2023-10-09T06:05:30.517344+0000","last_active":"2023-10-09T06:08:02.337044+0000","last_peered":"2023-10-09T06:08:02.337044+0000","last_clean":"2023-10-09T06:08:02.337044+0000","last_became_active":"2023-10-09T06:05:30.516920+0000","last_became_peered":"2023-10-09T06:05:30.516920+0000","last_unstale":"2023-10-09T06:08:02.337044+0000","last_undegraded":"2023-10-09T06:08:02.337044+0000","last_fullsized":"2023-10-09T06:08:02.337044+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.804649+0000","last_change":"2023-10-09T06:07:47.056618+0000","last_active":"2023-10-09T06:08:03.804649+0000","last_peered":"2023-10-09T06:08:03.804649+0000","last_clean":"2023-10-09T06:08:03.804649+0000","last_became_active":"2023-10-09T06:07:47.056311+0000","last_became_peered":"2023-10-09T06:07:47.056311+0000","last_unstale":"2023-10-09T06:08:03.804649+0000","last_undegraded":"2023-10-09T06:08:03.804649+0000","last_fullsized":"2023-10-09T06:08:03.804649+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:28.019550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":328,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337371+0000","last_change":"2023-10-09T06:07:24.553497+0000","last_active":"2023-10-09T06:08:02.337371+0000","last_peered":"2023-10-09T06:08:02.337371+0000","last_clean":"2023-10-09T06:08:02.337371+0000","last_became_active":"2023-10-09T06:07:24.553189+0000","last_became_peered":"2023-10-09T06:07:24.553189+0000","last_unstale":"2023-10-09T06:08:02.337371+0000","last_undegraded":"2023-10-09T06:08:02.337371+0000","last_fullsized":"2023-10-09T06:08:02.337371+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:53.168858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":656,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968938+0000","last_change":"2023-10-09T06:03:56.489473+0000","last_active":"2023-10-09T06:08:01.968938+0000","last_peered":"2023-10-09T06:08:01.968938+0000","last_clean":"2023-10-09T06:08:01.968938+0000","last_became_active":"2023-10-09T06:03:56.489102+0000","last_became_peered":"2023-10-09T06:03:56.489102+0000","last_unstale":"2023-10-09T06:08:01.968938+0000","last_undegraded":"2023-10-09T06:08:01.968938+0000","last_fullsized":"2023-10-09T06:08:01.968938+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":19,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.960263+0000","last_change":"2023-10-09T06:07:50.492752+0000","last_active":"2023-10-09T06:08:00.960263+0000","last_peered":"2023-10-09T06:08:00.960263+0000","last_clean":"2023-10-09T06:08:00.960263+0000","last_became_active":"2023-10-09T06:07:50.492443+0000","last_became_peered":"2023-10-09T06:07:50.492443+0000","last_unstale":"2023-10-09T06:08:00.960263+0000","last_undegraded":"2023-10-09T06:08:00.960263+0000","last_fullsized":"2023-10-09T06:08:00.960263+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:02.416416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":150,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.961107+0000","last_change":"2023-10-09T06:04:33.125479+0000","last_active":"2023-10-09T06:08:00.961107+0000","last_peered":"2023-10-09T06:08:00.961107+0000","last_clean":"2023-10-09T06:08:00.961107+0000","last_became_active":"2023-10-09T06:04:33.125135+0000","last_became_peered":"2023-10-09T06:04:33.125135+0000","last_unstale":"2023-10-09T06:08:00.961107+0000","last_undegraded":"2023-10-09T06:08:00.961107+0000","last_fullsized":"2023-10-09T06:08:00.961107+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:33.288989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":545,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.961198+0000","last_change":"2023-10-09T06:04:33.126133+0000","last_active":"2023-10-09T06:08:00.961198+0000","last_peered":"2023-10-09T06:08:00.961198+0000","last_clean":"2023-10-09T06:08:00.961198+0000","last_became_active":"2023-10-09T06:04:33.125857+0000","last_became_peered":"2023-10-09T06:04:33.125857+0000","last_unstale":"2023-10-09T06:08:00.961198+0000","last_undegraded":"2023-10-09T06:08:00.961198+0000","last_fullsized":"2023-10-09T06:08:00.961198+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:04.644424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":221,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.238331+0000","last_change":"2023-10-09T06:04:32.950240+0000","last_active":"2023-10-09T06:08:01.238331+0000","last_peered":"2023-10-09T06:08:01.238331+0000","last_clean":"2023-10-09T06:08:01.238331+0000","last_became_active":"2023-10-09T06:04:32.949852+0000","last_became_peered":"2023-10-09T06:04:32.949852+0000","last_unstale":"2023-10-09T06:08:01.238331+0000","last_undegraded":"2023-10-09T06:08:01.238331+0000","last_fullsized":"2023-10-09T06:08:01.238331+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:04.138187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":255,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.969265+0000","last_change":"2023-10-09T06:03:09.369870+0000","last_active":"2023-10-09T06:08:01.969265+0000","last_peered":"2023-10-09T06:08:01.969265+0000","last_clean":"2023-10-09T06:08:01.969265+0000","last_became_active":"2023-10-09T06:03:09.369216+0000","last_became_peered":"2023-10-09T06:03:09.369216+0000","last_unstale":"2023-10-09T06:08:01.969265+0000","last_undegraded":"2023-10-09T06:08:01.969265+0000","last_fullsized":"2023-10-09T06:08:01.969265+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":255,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968350+0000","last_change":"2023-10-09T06:04:32.956316+0000","last_active":"2023-10-09T06:08:01.968350+0000","last_peered":"2023-10-09T06:08:01.968350+0000","last_clean":"2023-10-09T06:08:01.968350+0000","last_became_active":"2023-10-09T06:04:32.955969+0000","last_became_peered":"2023-10-09T06:04:32.955969+0000","last_unstale":"2023-10-09T06:08:01.968350+0000","last_undegraded":"2023-10-09T06:08:01.968350+0000","last_fullsized":"2023-10-09T06:08:01.968350+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":221,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968414+0000","last_change":"2023-10-09T05:56:00.348771+0000","last_active":"2023-10-09T06:08:01.968414+0000","last_peered":"2023-10-09T06:08:01.968414+0000","last_clean":"2023-10-09T06:08:01.968414+0000","last_became_active":"2023-10-09T05:56:00.224347+0000","last_became_peered":"2023-10-09T05:56:00.224347+0000","last_unstale":"2023-10-09T06:08:01.968414+0000","last_undegraded":"2023-10-09T06:08:01.968414+0000","last_fullsized":"2023-10-09T06:08:01.968414+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":399,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337468+0000","last_change":"2023-10-09T06:05:15.568750+0000","last_active":"2023-10-09T06:08:02.337468+0000","last_peered":"2023-10-09T06:08:02.337468+0000","last_clean":"2023-10-09T06:08:02.337468+0000","last_became_active":"2023-10-09T06:05:15.568039+0000","last_became_peered":"2023-10-09T06:05:15.568039+0000","last_unstale":"2023-10-09T06:08:02.337468+0000","last_undegraded":"2023-10-09T06:08:02.337468+0000","last_fullsized":"2023-10-09T06:08:02.337468+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:38.828502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":281,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968568+0000","last_change":"2023-10-09T06:07:24.905311+0000","last_active":"2023-10-09T06:08:01.968568+0000","last_peered":"2023-10-09T06:08:01.968568+0000","last_clean":"2023-10-09T06:08:01.968568+0000","last_became_active":"2023-10-09T06:07:24.576154+0000","last_became_peered":"2023-10-09T06:07:24.576154+0000","last_unstale":"2023-10-09T06:08:01.968568+0000","last_undegraded":"2023-10-09T06:08:01.968568+0000","last_fullsized":"2023-10-09T06:08:01.968568+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":606,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.804595+0000","last_change":"2023-10-09T06:05:44.525598+0000","last_active":"2023-10-09T06:08:03.804595+0000","last_peered":"2023-10-09T06:08:03.804595+0000","last_clean":"2023-10-09T06:08:03.804595+0000","last_became_active":"2023-10-09T06:05:44.524943+0000","last_became_peered":"2023-10-09T06:05:44.524943+0000","last_unstale":"2023-10-09T06:08:03.804595+0000","last_undegraded":"2023-10-09T06:08:03.804595+0000","last_fullsized":"2023-10-09T06:08:03.804595+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:27.889121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":253,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968536+0000","last_change":"2023-10-09T06:07:24.732171+0000","last_active":"2023-10-09T06:08:01.968536+0000","last_peered":"2023-10-09T06:08:01.968536+0000","last_clean":"2023-10-09T06:08:01.968536+0000","last_became_active":"2023-10-09T06:07:24.576067+0000","last_became_peered":"2023-10-09T06:07:24.576067+0000","last_unstale":"2023-10-09T06:08:01.968536+0000","last_undegraded":"2023-10-09T06:08:01.968536+0000","last_fullsized":"2023-10-09T06:08:01.968536+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.804293+0000","last_change":"2023-10-09T06:07:47.252267+0000","last_active":"2023-10-09T06:08:03.804293+0000","last_peered":"2023-10-09T06:08:03.804293+0000","last_clean":"2023-10-09T06:08:03.804293+0000","last_became_active":"2023-10-09T06:07:47.251957+0000","last_became_peered":"2023-10-09T06:07:47.251957+0000","last_unstale":"2023-10-09T06:08:03.804293+0000","last_undegraded":"2023-10-09T06:08:03.804293+0000","last_fullsized":"2023-10-09T06:08:03.804293+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:03.068994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":639,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968197+0000","last_change":"2023-10-09T06:07:50.491749+0000","last_active":"2023-10-09T06:08:01.968197+0000","last_peered":"2023-10-09T06:08:01.968197+0000","last_clean":"2023-10-09T06:08:01.968197+0000","last_became_active":"2023-10-09T06:07:50.491494+0000","last_became_peered":"2023-10-09T06:07:50.491494+0000","last_unstale":"2023-10-09T06:08:01.968197+0000","last_undegraded":"2023-10-09T06:08:01.968197+0000","last_fullsized":"2023-10-09T06:08:01.968197+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":307,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968707+0000","last_change":"2023-10-09T06:04:32.952030+0000","last_active":"2023-10-09T06:08:01.968707+0000","last_peered":"2023-10-09T06:08:01.968707+0000","last_clean":"2023-10-09T06:08:01.968707+0000","last_became_active":"2023-10-09T06:04:32.951603+0000","last_became_peered":"2023-10-09T06:04:32.951603+0000","last_unstale":"2023-10-09T06:08:01.968707+0000","last_undegraded":"2023-10-09T06:08:01.968707+0000","last_fullsized":"2023-10-09T06:08:01.968707+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":237,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968130+0000","last_change":"2023-10-09T06:04:32.956628+0000","last_active":"2023-10-09T06:08:01.968130+0000","last_peered":"2023-10-09T06:08:01.968130+0000","last_clean":"2023-10-09T06:08:01.968130+0000","last_became_active":"2023-10-09T06:04:32.956260+0000","last_became_peered":"2023-10-09T06:04:32.956260+0000","last_unstale":"2023-10-09T06:08:01.968130+0000","last_undegraded":"2023-10-09T06:08:01.968130+0000","last_fullsized":"2023-10-09T06:08:01.968130+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":609,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.969020+0000","last_change":"2023-10-09T06:07:50.490355+0000","last_active":"2023-10-09T06:08:01.969020+0000","last_peered":"2023-10-09T06:08:01.969020+0000","last_clean":"2023-10-09T06:08:01.969020+0000","last_became_active":"2023-10-09T06:07:50.490061+0000","last_became_peered":"2023-10-09T06:07:50.490061+0000","last_unstale":"2023-10-09T06:08:01.969020+0000","last_undegraded":"2023-10-09T06:08:01.969020+0000","last_fullsized":"2023-10-09T06:08:01.969020+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":29,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.960369+0000","last_change":"2023-10-09T06:07:50.508117+0000","last_active":"2023-10-09T06:08:00.960369+0000","last_peered":"2023-10-09T06:08:00.960369+0000","last_clean":"2023-10-09T06:08:00.960369+0000","last_became_active":"2023-10-09T06:07:50.494658+0000","last_became_peered":"2023-10-09T06:07:50.494658+0000","last_unstale":"2023-10-09T06:08:00.960369+0000","last_undegraded":"2023-10-09T06:08:00.960369+0000","last_fullsized":"2023-10-09T06:08:00.960369+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:24.081528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":232,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.969529+0000","last_change":"2023-10-09T06:03:09.369599+0000","last_active":"2023-10-09T06:08:01.969529+0000","last_peered":"2023-10-09T06:08:01.969529+0000","last_clean":"2023-10-09T06:08:01.969529+0000","last_became_active":"2023-10-09T06:03:09.369274+0000","last_became_peered":"2023-10-09T06:03:09.369274+0000","last_unstale":"2023-10-09T06:08:01.969529+0000","last_undegraded":"2023-10-09T06:08:01.969529+0000","last_fullsized":"2023-10-09T06:08:01.969529+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":269,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.969547+0000","last_change":"2023-10-09T06:07:50.541376+0000","last_active":"2023-10-09T06:08:01.969547+0000","last_peered":"2023-10-09T06:08:01.969547+0000","last_clean":"2023-10-09T06:08:01.969547+0000","last_became_active":"2023-10-09T06:07:50.489742+0000","last_became_peered":"2023-10-09T06:07:50.489742+0000","last_unstale":"2023-10-09T06:08:01.969547+0000","last_undegraded":"2023-10-09T06:08:01.969547+0000","last_fullsized":"2023-10-09T06:08:01.969547+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":276,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968096+0000","last_change":"2023-10-09T06:04:32.969529+0000","last_active":"2023-10-09T06:08:01.968096+0000","last_peered":"2023-10-09T06:08:01.968096+0000","last_clean":"2023-10-09T06:08:01.968096+0000","last_became_active":"2023-10-09T06:04:32.969220+0000","last_became_peered":"2023-10-09T06:04:32.969220+0000","last_unstale":"2023-10-09T06:08:01.968096+0000","last_undegraded":"2023-10-09T06:08:01.968096+0000","last_fullsized":"2023-10-09T06:08:01.968096+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":227,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.969135+0000","last_change":"2023-10-09T06:07:24.631948+0000","last_active":"2023-10-09T06:08:01.969135+0000","last_peered":"2023-10-09T06:08:01.969135+0000","last_clean":"2023-10-09T06:08:01.969135+0000","last_became_active":"2023-10-09T06:07:24.564922+0000","last_became_peered":"2023-10-09T06:07:24.564922+0000","last_unstale":"2023-10-09T06:08:01.969135+0000","last_undegraded":"2023-10-09T06:08:01.969135+0000","last_fullsized":"2023-10-09T06:08:01.969135+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":21,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.967990+0000","last_change":"2023-10-09T06:07:47.252223+0000","last_active":"2023-10-09T06:08:01.967990+0000","last_peered":"2023-10-09T06:08:01.967990+0000","last_clean":"2023-10-09T06:08:01.967990+0000","last_became_active":"2023-10-09T06:07:47.251804+0000","last_became_peered":"2023-10-09T06:07:47.251804+0000","last_unstale":"2023-10-09T06:08:01.967990+0000","last_undegraded":"2023-10-09T06:08:01.967990+0000","last_fullsized":"2023-10-09T06:08:01.967990+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":238,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968809+0000","last_change":"2023-10-09T06:04:32.951565+0000","last_active":"2023-10-09T06:08:01.968809+0000","last_peered":"2023-10-09T06:08:01.968809+0000","last_clean":"2023-10-09T06:08:01.968809+0000","last_became_active":"2023-10-09T06:04:32.951221+0000","last_became_peered":"2023-10-09T06:04:32.951221+0000","last_unstale":"2023-10-09T06:08:01.968809+0000","last_undegraded":"2023-10-09T06:08:01.968809+0000","last_fullsized":"2023-10-09T06:08:01.968809+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":355,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337672+0000","last_change":"2023-10-09T06:05:15.553522+0000","last_active":"2023-10-09T06:08:02.337672+0000","last_peered":"2023-10-09T06:08:02.337672+0000","last_clean":"2023-10-09T06:08:02.337672+0000","last_became_active":"2023-10-09T06:05:15.553151+0000","last_became_peered":"2023-10-09T06:05:15.553151+0000","last_unstale":"2023-10-09T06:08:02.337672+0000","last_undegraded":"2023-10-09T06:08:02.337672+0000","last_fullsized":"2023-10-09T06:08:02.337672+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:41.893526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":524,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337624+0000","last_change":"2023-10-09T06:05:15.589640+0000","last_active":"2023-10-09T06:08:02.337624+0000","last_peered":"2023-10-09T06:08:02.337624+0000","last_clean":"2023-10-09T06:08:02.337624+0000","last_became_active":"2023-10-09T06:05:15.589350+0000","last_became_peered":"2023-10-09T06:05:15.589350+0000","last_unstale":"2023-10-09T06:08:02.337624+0000","last_undegraded":"2023-10-09T06:08:02.337624+0000","last_fullsized":"2023-10-09T06:08:02.337624+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:31.476101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":522,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.960696+0000","last_change":"2023-10-09T06:00:05.352723+0000","last_active":"2023-10-09T06:08:00.960696+0000","last_peered":"2023-10-09T06:08:00.960696+0000","last_clean":"2023-10-09T06:08:00.960696+0000","last_became_active":"2023-10-09T06:00:05.352008+0000","last_became_peered":"2023-10-09T06:00:05.352008+0000","last_unstale":"2023-10-09T06:08:00.960696+0000","last_undegraded":"2023-10-09T06:08:00.960696+0000","last_fullsized":"2023-10-09T06:08:00.960696+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:25.302918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":452,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.238821+0000","last_change":"2023-10-09T06:07:50.544943+0000","last_active":"2023-10-09T06:08:01.238821+0000","last_peered":"2023-10-09T06:08:01.238821+0000","last_clean":"2023-10-09T06:08:01.238821+0000","last_became_active":"2023-10-09T06:07:50.489809+0000","last_became_peered":"2023-10-09T06:07:50.489809+0000","last_unstale":"2023-10-09T06:08:01.238821+0000","last_undegraded":"2023-10-09T06:08:01.238821+0000","last_fullsized":"2023-10-09T06:08:01.238821+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:32.160515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":602,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337327+0000","last_change":"2023-10-09T06:04:32.947237+0000","last_active":"2023-10-09T06:08:02.337327+0000","last_peered":"2023-10-09T06:08:02.337327+0000","last_clean":"2023-10-09T06:08:02.337327+0000","last_became_active":"2023-10-09T06:04:32.946963+0000","last_became_peered":"2023-10-09T06:04:32.946963+0000","last_unstale":"2023-10-09T06:08:02.337327+0000","last_undegraded":"2023-10-09T06:08:02.337327+0000","last_fullsized":"2023-10-09T06:08:02.337327+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":116,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.961206+0000","last_change":"2023-10-09T06:04:32.950583+0000","last_active":"2023-10-09T06:08:00.961206+0000","last_peered":"2023-10-09T06:08:00.961206+0000","last_clean":"2023-10-09T06:08:00.961206+0000","last_became_active":"2023-10-09T06:04:32.950108+0000","last_became_peered":"2023-10-09T06:04:32.950108+0000","last_unstale":"2023-10-09T06:08:00.961206+0000","last_undegraded":"2023-10-09T06:08:00.961206+0000","last_fullsized":"2023-10-09T06:08:00.961206+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":561,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.804947+0000","last_change":"2023-10-09T06:07:24.555315+0000","last_active":"2023-10-09T06:08:03.804947+0000","last_peered":"2023-10-09T06:08:03.804947+0000","last_clean":"2023-10-09T06:08:03.804947+0000","last_became_active":"2023-10-09T06:07:24.554902+0000","last_became_peered":"2023-10-09T06:07:24.554902+0000","last_unstale":"2023-10-09T06:08:03.804947+0000","last_undegraded":"2023-10-09T06:08:03.804947+0000","last_fullsized":"2023-10-09T06:08:03.804947+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:33.021735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":619,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.977548+0000","last_change":"2023-10-09T06:07:24.592232+0000","last_active":"2023-10-09T06:08:04.977548+0000","last_peered":"2023-10-09T06:08:04.977548+0000","last_clean":"2023-10-09T06:08:04.977548+0000","last_became_active":"2023-10-09T06:07:24.591849+0000","last_became_peered":"2023-10-09T06:07:24.591849+0000","last_unstale":"2023-10-09T06:08:04.977548+0000","last_undegraded":"2023-10-09T06:08:04.977548+0000","last_fullsized":"2023-10-09T06:08:04.977548+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:56.410869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":573,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337826+0000","last_change":"2023-10-09T06:05:15.553450+0000","last_active":"2023-10-09T06:08:02.337826+0000","last_peered":"2023-10-09T06:08:02.337826+0000","last_clean":"2023-10-09T06:08:02.337826+0000","last_became_active":"2023-10-09T06:05:15.553129+0000","last_became_peered":"2023-10-09T06:05:15.553129+0000","last_unstale":"2023-10-09T06:08:02.337826+0000","last_undegraded":"2023-10-09T06:08:02.337826+0000","last_fullsized":"2023-10-09T06:08:02.337826+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:21.096565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":368,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337536+0000","last_change":"2023-10-09T06:05:15.576694+0000","last_active":"2023-10-09T06:08:02.337536+0000","last_peered":"2023-10-09T06:08:02.337536+0000","last_clean":"2023-10-09T06:08:02.337536+0000","last_became_active":"2023-10-09T06:05:15.576437+0000","last_became_peered":"2023-10-09T06:05:15.576437+0000","last_unstale":"2023-10-09T06:08:02.337536+0000","last_undegraded":"2023-10-09T06:08:02.337536+0000","last_fullsized":"2023-10-09T06:08:02.337536+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:29.917899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":289,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.338112+0000","last_change":"2023-10-09T06:07:47.057804+0000","last_active":"2023-10-09T06:08:02.338112+0000","last_peered":"2023-10-09T06:08:02.338112+0000","last_clean":"2023-10-09T06:08:02.338112+0000","last_became_active":"2023-10-09T06:07:47.057508+0000","last_became_peered":"2023-10-09T06:07:47.057508+0000","last_unstale":"2023-10-09T06:08:02.338112+0000","last_undegraded":"2023-10-09T06:08:02.338112+0000","last_fullsized":"2023-10-09T06:08:02.338112+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:45.041480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"76'147","reported_seq":996,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968235+0000","last_change":"2023-10-09T06:02:08.341435+0000","last_active":"2023-10-09T06:08:01.968235+0000","last_peered":"2023-10-09T06:08:01.968235+0000","last_clean":"2023-10-09T06:08:01.968235+0000","last_became_active":"2023-10-09T06:02:08.340863+0000","last_became_peered":"2023-10-09T06:02:08.340863+0000","last_unstale":"2023-10-09T06:08:01.968235+0000","last_undegraded":"2023-10-09T06:08:01.968235+0000","last_fullsized":"2023-10-09T06:08:01.968235+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":769,"num_read_kb":1595,"num_write":136,"num_write_kb":506,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":591,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.239068+0000","last_change":"2023-10-09T06:05:30.448881+0000","last_active":"2023-10-09T06:08:01.239068+0000","last_peered":"2023-10-09T06:08:01.239068+0000","last_clean":"2023-10-09T06:08:01.239068+0000","last_became_active":"2023-10-09T06:05:30.448534+0000","last_became_peered":"2023-10-09T06:05:30.448534+0000","last_unstale":"2023-10-09T06:08:01.239068+0000","last_undegraded":"2023-10-09T06:08:01.239068+0000","last_fullsized":"2023-10-09T06:08:01.239068+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:10.307670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":138,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337459+0000","last_change":"2023-10-09T06:07:24.553776+0000","last_active":"2023-10-09T06:08:02.337459+0000","last_peered":"2023-10-09T06:08:02.337459+0000","last_clean":"2023-10-09T06:08:02.337459+0000","last_became_active":"2023-10-09T06:07:24.553439+0000","last_became_peered":"2023-10-09T06:07:24.553439+0000","last_unstale":"2023-10-09T06:08:02.337459+0000","last_undegraded":"2023-10-09T06:08:02.337459+0000","last_fullsized":"2023-10-09T06:08:02.337459+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:03.005009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"76'62","reported_seq":924,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337587+0000","last_change":"2023-10-09T06:04:32.958038+0000","last_active":"2023-10-09T06:08:02.337587+0000","last_peered":"2023-10-09T06:08:02.337587+0000","last_clean":"2023-10-09T06:08:02.337587+0000","last_became_active":"2023-10-09T06:04:32.957507+0000","last_became_peered":"2023-10-09T06:04:32.957507+0000","last_unstale":"2023-10-09T06:08:02.337587+0000","last_undegraded":"2023-10-09T06:08:02.337587+0000","last_fullsized":"2023-10-09T06:08:02.337587+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:33.323626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":688,"num_read_kb":1956,"num_write":62,"num_write_kb":1224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":533,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337393+0000","last_change":"2023-10-09T06:04:32.951743+0000","last_active":"2023-10-09T06:08:02.337393+0000","last_peered":"2023-10-09T06:08:02.337393+0000","last_clean":"2023-10-09T06:08:02.337393+0000","last_became_active":"2023-10-09T06:04:32.951415+0000","last_became_peered":"2023-10-09T06:04:32.951415+0000","last_unstale":"2023-10-09T06:08:02.337393+0000","last_undegraded":"2023-10-09T06:08:02.337393+0000","last_fullsized":"2023-10-09T06:08:02.337393+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":461,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.806223+0000","last_change":"2023-10-09T06:05:44.522337+0000","last_active":"2023-10-09T06:08:03.806223+0000","last_peered":"2023-10-09T06:08:03.806223+0000","last_clean":"2023-10-09T06:08:03.806223+0000","last_became_active":"2023-10-09T06:05:44.522025+0000","last_became_peered":"2023-10-09T06:05:44.522025+0000","last_unstale":"2023-10-09T06:08:03.806223+0000","last_undegraded":"2023-10-09T06:08:03.806223+0000","last_fullsized":"2023-10-09T06:08:03.806223+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:52.064736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"76'86","reported_seq":954,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.805101+0000","last_change":"2023-10-09T06:05:44.560082+0000","last_active":"2023-10-09T06:08:03.805101+0000","last_peered":"2023-10-09T06:08:03.805101+0000","last_clean":"2023-10-09T06:08:03.805101+0000","last_became_active":"2023-10-09T06:05:44.534078+0000","last_became_peered":"2023-10-09T06:05:44.534078+0000","last_unstale":"2023-10-09T06:08:03.805101+0000","last_undegraded":"2023-10-09T06:08:03.805101+0000","last_fullsized":"2023-10-09T06:08:03.805101+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:45.519427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":507,"num_read_kb":1403,"num_write":82,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":411,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337421+0000","last_change":"2023-10-09T06:05:30.447496+0000","last_active":"2023-10-09T06:08:02.337421+0000","last_peered":"2023-10-09T06:08:02.337421+0000","last_clean":"2023-10-09T06:08:02.337421+0000","last_became_active":"2023-10-09T06:05:30.447219+0000","last_became_peered":"2023-10-09T06:05:30.447219+0000","last_unstale":"2023-10-09T06:08:02.337421+0000","last_undegraded":"2023-10-09T06:08:02.337421+0000","last_fullsized":"2023-10-09T06:08:02.337421+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:08.869779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":82,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337986+0000","last_change":"2023-10-09T06:07:50.496727+0000","last_active":"2023-10-09T06:08:02.337986+0000","last_peered":"2023-10-09T06:08:02.337986+0000","last_clean":"2023-10-09T06:08:02.337986+0000","last_became_active":"2023-10-09T06:07:50.496503+0000","last_became_peered":"2023-10-09T06:07:50.496503+0000","last_unstale":"2023-10-09T06:08:02.337986+0000","last_undegraded":"2023-10-09T06:08:02.337986+0000","last_fullsized":"2023-10-09T06:08:02.337986+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:17.224957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"76'180","reported_seq":838,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.804698+0000","last_change":"2023-10-09T06:05:44.537575+0000","last_active":"2023-10-09T06:08:03.804698+0000","last_peered":"2023-10-09T06:08:03.804698+0000","last_clean":"2023-10-09T06:08:03.804698+0000","last_became_active":"2023-10-09T06:05:44.533477+0000","last_became_peered":"2023-10-09T06:05:44.533477+0000","last_unstale":"2023-10-09T06:08:03.804698+0000","last_undegraded":"2023-10-09T06:08:03.804698+0000","last_fullsized":"2023-10-09T06:08:03.804698+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:44.900250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":887,"num_read_kb":802,"num_write":153,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":810,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968118+0000","last_change":"2023-10-09T06:07:53.650105+0000","last_active":"2023-10-09T06:08:01.968118+0000","last_peered":"2023-10-09T06:08:01.968118+0000","last_clean":"2023-10-09T06:08:01.968118+0000","last_became_active":"2023-10-09T06:07:53.649614+0000","last_became_peered":"2023-10-09T06:07:53.649614+0000","last_unstale":"2023-10-09T06:08:01.968118+0000","last_undegraded":"2023-10-09T06:08:01.968118+0000","last_fullsized":"2023-10-09T06:08:01.968118+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":485,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.238376+0000","last_change":"2023-10-09T06:04:32.942556+0000","last_active":"2023-10-09T06:08:01.238376+0000","last_peered":"2023-10-09T06:08:01.238376+0000","last_clean":"2023-10-09T06:08:01.238376+0000","last_became_active":"2023-10-09T06:04:32.942269+0000","last_became_peered":"2023-10-09T06:04:32.942269+0000","last_unstale":"2023-10-09T06:08:01.238376+0000","last_undegraded":"2023-10-09T06:08:01.238376+0000","last_fullsized":"2023-10-09T06:08:01.238376+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:53.383296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":257,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968283+0000","last_change":"2023-10-09T06:07:24.596895+0000","last_active":"2023-10-09T06:08:01.968283+0000","last_peered":"2023-10-09T06:08:01.968283+0000","last_clean":"2023-10-09T06:08:01.968283+0000","last_became_active":"2023-10-09T06:07:24.596620+0000","last_became_peered":"2023-10-09T06:07:24.596620+0000","last_unstale":"2023-10-09T06:08:01.968283+0000","last_undegraded":"2023-10-09T06:08:01.968283+0000","last_fullsized":"2023-10-09T06:08:01.968283+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"76'137","reported_seq":1179,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968247+0000","last_change":"2023-10-09T06:03:12.481829+0000","last_active":"2023-10-09T06:08:01.968247+0000","last_peered":"2023-10-09T06:08:01.968247+0000","last_clean":"2023-10-09T06:08:01.968247+0000","last_became_active":"2023-10-09T06:03:12.479898+0000","last_became_peered":"2023-10-09T06:03:12.479898+0000","last_unstale":"2023-10-09T06:08:01.968247+0000","last_undegraded":"2023-10-09T06:08:01.968247+0000","last_fullsized":"2023-10-09T06:08:01.968247+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1132,"num_read_kb":3426,"num_write":98,"num_write_kb":2273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":577,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.960490+0000","last_change":"2023-10-09T06:07:24.555346+0000","last_active":"2023-10-09T06:08:00.960490+0000","last_peered":"2023-10-09T06:08:00.960490+0000","last_clean":"2023-10-09T06:08:00.960490+0000","last_became_active":"2023-10-09T06:07:24.554992+0000","last_became_peered":"2023-10-09T06:07:24.554992+0000","last_unstale":"2023-10-09T06:08:00.960490+0000","last_undegraded":"2023-10-09T06:08:00.960490+0000","last_fullsized":"2023-10-09T06:08:00.960490+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:43.884493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.961001+0000","last_change":"2023-10-09T06:07:50.496088+0000","last_active":"2023-10-09T06:08:00.961001+0000","last_peered":"2023-10-09T06:08:00.961001+0000","last_clean":"2023-10-09T06:08:00.961001+0000","last_became_active":"2023-10-09T06:07:50.495249+0000","last_became_peered":"2023-10-09T06:07:50.495249+0000","last_unstale":"2023-10-09T06:08:00.961001+0000","last_undegraded":"2023-10-09T06:08:00.961001+0000","last_fullsized":"2023-10-09T06:08:00.961001+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:52.044105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"76'103","reported_seq":858,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968856+0000","last_change":"2023-10-09T06:07:53.651882+0000","last_active":"2023-10-09T06:08:01.968856+0000","last_peered":"2023-10-09T06:08:01.968856+0000","last_clean":"2023-10-09T06:08:01.968856+0000","last_became_active":"2023-10-09T06:07:53.651256+0000","last_became_peered":"2023-10-09T06:07:53.651256+0000","last_unstale":"2023-10-09T06:08:01.968856+0000","last_undegraded":"2023-10-09T06:08:01.968856+0000","last_fullsized":"2023-10-09T06:08:01.968856+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":2349,"num_write":106,"num_write_kb":3131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":465,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.806332+0000","last_change":"2023-10-09T06:07:50.491120+0000","last_active":"2023-10-09T06:08:03.806332+0000","last_peered":"2023-10-09T06:08:03.806332+0000","last_clean":"2023-10-09T06:08:03.806332+0000","last_became_active":"2023-10-09T06:07:50.490772+0000","last_became_peered":"2023-10-09T06:07:50.490772+0000","last_unstale":"2023-10-09T06:08:03.806332+0000","last_undegraded":"2023-10-09T06:08:03.806332+0000","last_fullsized":"2023-10-09T06:08:03.806332+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:31.534859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":527,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968697+0000","last_change":"2023-10-09T06:02:08.340506+0000","last_active":"2023-10-09T06:08:01.968697+0000","last_peered":"2023-10-09T06:08:01.968697+0000","last_clean":"2023-10-09T06:08:01.968697+0000","last_became_active":"2023-10-09T06:02:08.340152+0000","last_became_peered":"2023-10-09T06:02:08.340152+0000","last_unstale":"2023-10-09T06:08:01.968697+0000","last_undegraded":"2023-10-09T06:08:01.968697+0000","last_fullsized":"2023-10-09T06:08:01.968697+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":255,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.969730+0000","last_change":"2023-10-09T06:00:05.329139+0000","last_active":"2023-10-09T06:08:01.969730+0000","last_peered":"2023-10-09T06:08:01.969730+0000","last_clean":"2023-10-09T06:08:01.969730+0000","last_became_active":"2023-10-09T06:00:05.328811+0000","last_became_peered":"2023-10-09T06:00:05.328811+0000","last_unstale":"2023-10-09T06:08:01.969730+0000","last_undegraded":"2023-10-09T06:08:01.969730+0000","last_fullsized":"2023-10-09T06:08:01.969730+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"76'145","reported_seq":631,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337911+0000","last_change":"2023-10-09T06:04:32.959605+0000","last_active":"2023-10-09T06:08:02.337911+0000","last_peered":"2023-10-09T06:08:02.337911+0000","last_clean":"2023-10-09T06:08:02.337911+0000","last_became_active":"2023-10-09T06:04:32.959073+0000","last_became_peered":"2023-10-09T06:04:32.959073+0000","last_unstale":"2023-10-09T06:08:02.337911+0000","last_undegraded":"2023-10-09T06:08:02.337911+0000","last_fullsized":"2023-10-09T06:08:02.337911+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:00.843837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":699,"num_read_kb":868,"num_write":109,"num_write_kb":205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":277,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.968005+0000","last_change":"2023-10-09T06:07:50.490552+0000","last_active":"2023-10-09T06:08:01.968005+0000","last_peered":"2023-10-09T06:08:01.968005+0000","last_clean":"2023-10-09T06:08:01.968005+0000","last_became_active":"2023-10-09T06:07:50.490268+0000","last_became_peered":"2023-10-09T06:07:50.490268+0000","last_unstale":"2023-10-09T06:08:01.968005+0000","last_undegraded":"2023-10-09T06:08:01.968005+0000","last_fullsized":"2023-10-09T06:08:01.968005+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"76'116","reported_seq":792,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:01.969632+0000","last_change":"2023-10-09T06:04:32.961494+0000","last_active":"2023-10-09T06:08:01.969632+0000","last_peered":"2023-10-09T06:08:01.969632+0000","last_clean":"2023-10-09T06:08:01.969632+0000","last_became_active":"2023-10-09T06:04:32.961137+0000","last_became_peered":"2023-10-09T06:04:32.961137+0000","last_unstale":"2023-10-09T06:08:01.969632+0000","last_undegraded":"2023-10-09T06:08:01.969632+0000","last_fullsized":"2023-10-09T06:08:01.969632+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":116,"log_dups_size":0,"ondisk_log_size":116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":812,"num_write":100,"num_write_kb":4164,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":23,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.960996+0000","last_change":"2023-10-09T06:07:50.543307+0000","last_active":"2023-10-09T06:08:00.960996+0000","last_peered":"2023-10-09T06:08:00.960996+0000","last_clean":"2023-10-09T06:08:00.960996+0000","last_became_active":"2023-10-09T06:07:50.495736+0000","last_became_peered":"2023-10-09T06:07:50.495736+0000","last_unstale":"2023-10-09T06:08:00.960996+0000","last_undegraded":"2023-10-09T06:08:00.960996+0000","last_fullsized":"2023-10-09T06:08:00.960996+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:47.121228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":168,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.960350+0000","last_change":"2023-10-09T06:07:24.557878+0000","last_active":"2023-10-09T06:08:00.960350+0000","last_peered":"2023-10-09T06:08:00.960350+0000","last_clean":"2023-10-09T06:08:00.960350+0000","last_became_active":"2023-10-09T06:07:24.557528+0000","last_became_peered":"2023-10-09T06:07:24.557528+0000","last_unstale":"2023-10-09T06:08:00.960350+0000","last_undegraded":"2023-10-09T06:08:00.960350+0000","last_fullsized":"2023-10-09T06:08:00.960350+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:57.735693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":19,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.960796+0000","last_change":"2023-10-09T06:07:50.489321+0000","last_active":"2023-10-09T06:08:00.960796+0000","last_peered":"2023-10-09T06:08:00.960796+0000","last_clean":"2023-10-09T06:08:00.960796+0000","last_became_active":"2023-10-09T06:07:50.488938+0000","last_became_peered":"2023-10-09T06:07:50.488938+0000","last_unstale":"2023-10-09T06:08:00.960796+0000","last_undegraded":"2023-10-09T06:08:00.960796+0000","last_fullsized":"2023-10-09T06:08:00.960796+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:00.393060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":294,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.804192+0000","last_change":"2023-10-09T06:07:47.252116+0000","last_active":"2023-10-09T06:08:03.804192+0000","last_peered":"2023-10-09T06:08:03.804192+0000","last_clean":"2023-10-09T06:08:03.804192+0000","last_became_active":"2023-10-09T06:07:47.251804+0000","last_became_peered":"2023-10-09T06:07:47.251804+0000","last_unstale":"2023-10-09T06:08:03.804192+0000","last_undegraded":"2023-10-09T06:08:03.804192+0000","last_fullsized":"2023-10-09T06:08:03.804192+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:02.563612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":149,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337357+0000","last_change":"2023-10-09T06:04:32.957514+0000","last_active":"2023-10-09T06:08:02.337357+0000","last_peered":"2023-10-09T06:08:02.337357+0000","last_clean":"2023-10-09T06:08:02.337357+0000","last_became_active":"2023-10-09T06:04:32.957114+0000","last_became_peered":"2023-10-09T06:04:32.957114+0000","last_unstale":"2023-10-09T06:08:02.337357+0000","last_undegraded":"2023-10-09T06:08:02.337357+0000","last_fullsized":"2023-10-09T06:08:02.337357+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337167+0000","last_change":"2023-10-09T06:07:47.252501+0000","last_active":"2023-10-09T06:08:02.337167+0000","last_peered":"2023-10-09T06:08:02.337167+0000","last_clean":"2023-10-09T06:08:02.337167+0000","last_became_active":"2023-10-09T06:07:47.251841+0000","last_became_peered":"2023-10-09T06:07:47.251841+0000","last_unstale":"2023-10-09T06:08:02.337167+0000","last_undegraded":"2023-10-09T06:08:02.337167+0000","last_fullsized":"2023-10-09T06:08:02.337167+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":149,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.337421+0000","last_change":"2023-10-09T06:04:32.957504+0000","last_active":"2023-10-09T06:08:02.337421+0000","last_peered":"2023-10-09T06:08:02.337421+0000","last_clean":"2023-10-09T06:08:02.337421+0000","last_became_active":"2023-10-09T06:04:32.957148+0000","last_became_peered":"2023-10-09T06:04:32.957148+0000","last_unstale":"2023-10-09T06:08:02.337421+0000","last_undegraded":"2023-10-09T06:08:02.337421+0000","last_fullsized":"2023-10-09T06:08:02.337421+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":30,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.336877+0000","last_change":"2023-10-09T06:07:24.555569+0000","last_active":"2023-10-09T06:08:02.336877+0000","last_peered":"2023-10-09T06:08:02.336877+0000","last_clean":"2023-10-09T06:08:02.336877+0000","last_became_active":"2023-10-09T06:07:24.555251+0000","last_became_peered":"2023-10-09T06:07:24.555251+0000","last_unstale":"2023-10-09T06:08:02.336877+0000","last_undegraded":"2023-10-09T06:08:02.336877+0000","last_fullsized":"2023-10-09T06:08:02.336877+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":371,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.805460+0000","last_change":"2023-10-09T06:05:44.522492+0000","last_active":"2023-10-09T06:08:03.805460+0000","last_peered":"2023-10-09T06:08:03.805460+0000","last_clean":"2023-10-09T06:08:03.805460+0000","last_became_active":"2023-10-09T06:05:44.522205+0000","last_became_peered":"2023-10-09T06:05:44.522205+0000","last_unstale":"2023-10-09T06:08:03.805460+0000","last_undegraded":"2023-10-09T06:08:03.805460+0000","last_fullsized":"2023-10-09T06:08:03.805460+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:58.840737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":455,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.805200+0000","last_change":"2023-10-09T06:05:44.527302+0000","last_active":"2023-10-09T06:08:03.805200+0000","last_peered":"2023-10-09T06:08:03.805200+0000","last_clean":"2023-10-09T06:08:03.805200+0000","last_became_active":"2023-10-09T06:05:44.526986+0000","last_became_peered":"2023-10-09T06:05:44.526986+0000","last_unstale":"2023-10-09T06:08:03.805200+0000","last_undegraded":"2023-10-09T06:08:03.805200+0000","last_fullsized":"2023-10-09T06:08:03.805200+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:13.582935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":25,"reported_epoch":587,"state":"active+clean","last_fresh":"2023-10-09T06:08:00.961064+0000","last_change":"2023-10-09T06:07:50.500816+0000","last_active":"2023-10-09T06:08:00.961064+0000","last_peered":"2023-10-09T06:08:00.961064+0000","last_clean":"2023-10-09T06:08:00.961064+0000","last_became_active":"2023-10-09T06:07:50.494692+0000","last_became_peered":"2023-10-09T06:07:50.494692+0000","last_unstale":"2023-10-09T06:08:00.961064+0000","last_undegraded":"2023-10-09T06:08:00.961064+0000","last_fullsized":"2023-10-09T06:08:00.961064+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:44.716403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":581,"reported_epoch":588,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.336978+0000","last_change":"2023-10-09T06:04:32.949651+0000","last_active":"2023-10-09T06:08:02.336978+0000","last_peered":"2023-10-09T06:08:02.336978+0000","last_clean":"2023-10-09T06:08:02.336978+0000","last_became_active":"2023-10-09T06:04:32.949288+0000","last_became_peered":"2023-10-09T06:04:32.949288+0000","last_unstale":"2023-10-09T06:08:02.336978+0000","last_undegraded":"2023-10-09T06:08:02.336978+0000","last_fullsized":"2023-10-09T06:08:02.336978+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":86,"num_bytes_recovered":27644,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":52606,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":13211,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846988,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1510992,"kb_used_data":10248,"kb_used_omap":14,"kb_used_meta":1500721,"kb_avail":92234160,"statfs":{"total":95995035648,"available":94447779840,"internally_reserved":0,"allocated":10493952,"data_stored":9302138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":1536739162},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":10.419,"5min":3.879,"15min":4.016},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":167.06200000000001,"5min":167.06200000000001,"15min":212.64400000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":9.9399999999999995,"5min":3.7709999999999999,"15min":4.1150000000000002},"min":{"1min":0.27300000000000002,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":167.077,"5min":167.077,"15min":212.655},"last":0.44700000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":10.201000000000001,"5min":6.5940000000000003,"15min":4.593},"min":{"1min":0.32000000000000001,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":169.98400000000001,"5min":169.98400000000001,"15min":176.90600000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":11.323,"5min":7.0979999999999999,"15min":4.7080000000000002},"min":{"1min":0.32200000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":172.821,"5min":172.821,"15min":176.86799999999999},"last":0.42399999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":11.234999999999999,"5min":9.0069999999999997,"15min":5.891},"min":{"1min":0.316,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":177.65100000000001,"5min":302.01799999999997,"15min":302.01799999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":10.667,"5min":8.6080000000000005,"15min":5.5919999999999996},"min":{"1min":0.374,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":175.762,"5min":304.988,"15min":304.988},"last":0.54000000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":11.079000000000001,"5min":4.5090000000000003,"15min":3.472},"min":{"1min":0.36899999999999999,"5min":0.34399999999999997,"15min":0.30499999999999999},"max":{"1min":167.64699999999999,"5min":167.64699999999999,"15min":176.958},"last":0.38},{"interface":"front","average":{"1min":10.465,"5min":4.0590000000000002,"15min":3.4420000000000002},"min":{"1min":0.38300000000000001,"5min":0.32900000000000001,"15min":0.29699999999999999},"max":{"1min":169.52199999999999,"5min":169.52199999999999,"15min":176.91800000000001},"last":0.48299999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.747,"15min":1.855},"min":{"1min":0.311,"5min":0.252,"15min":0.252},"max":{"1min":3.7130000000000001,"5min":5.6340000000000003,"15min":178.995},"last":0.495},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.79100000000000004,"15min":1.913},"min":{"1min":0.32700000000000001,"5min":0.246,"15min":0.246},"max":{"1min":4.9349999999999996,"5min":4.9349999999999996,"15min":178.899},"last":0.32300000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.70999999999999996,"15min":2.3050000000000002},"min":{"1min":0.35799999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.9089999999999998,"5min":3.1299999999999999,"15min":176.82400000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.755,"15min":2.2890000000000001},"min":{"1min":0.34399999999999997,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.1210000000000004,"5min":4.242,"15min":176.79499999999999},"last":0.436}]},{"osd":6,"last update":"Mon Oct 9 06:07:41 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.1120000000000001,"15min":1.127},"min":{"1min":0.33200000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":13.193,"5min":13.193,"15min":13.193},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.448,"5min":0.93899999999999995,"15min":0.79000000000000004},"min":{"1min":0.42599999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.247,"5min":13.247,"15min":13.247},"last":0.50900000000000001}]}]},{"osd":6,"up_from":537,"seq":2306397437995,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":381828,"kb_used_data":10236,"kb_used_omap":13,"kb_used_meta":371570,"kb_avail":93363324,"statfs":{"total":95995035648,"available":95604043776,"internally_reserved":0,"allocated":10481664,"data_stored":9286464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13650,"internal_metadata":380488366},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":8.0039999999999996,"5min":5.069,"15min":5.2469999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":126.914,"5min":126.914,"15min":126.914},"last":1.143},{"interface":"front","average":{"1min":8.6780000000000008,"5min":5.4690000000000003,"15min":5.6150000000000002},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":126.925,"5min":126.925,"15min":126.925},"last":1.1519999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":8.5830000000000002,"5min":9.3740000000000006,"15min":11.186},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":130.28100000000001,"5min":130.28100000000001,"15min":130.28100000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":8.4410000000000007,"5min":9.2609999999999992,"15min":11.09},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":135.06100000000001,"5min":135.06100000000001,"15min":135.06100000000001},"last":1.264}]},{"osd":2,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":10.076000000000001,"5min":3.8130000000000002,"15min":2.8170000000000002},"min":{"1min":0.313,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":133.917,"5min":133.917,"15min":133.917},"last":2.0009999999999999},{"interface":"front","average":{"1min":9.9930000000000003,"5min":4.1109999999999998,"15min":3.2149999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":133.92699999999999,"5min":133.92699999999999,"15min":133.92699999999999},"last":2.0110000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":8.298,"5min":5.7149999999999999,"15min":5.9569999999999999},"min":{"1min":0.377,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":129.375,"5min":129.375,"15min":129.375},"last":0.497},{"interface":"front","average":{"1min":8.0820000000000007,"5min":5.6749999999999998,"15min":5.9349999999999996},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":129.38900000000001,"5min":129.38900000000001,"15min":129.38900000000001},"last":0.51900000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.1100000000000001,"15min":1.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":11.103999999999999,"5min":11.103999999999999,"15min":11.103999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.92600000000000005,"15min":1.0600000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.1669999999999998,"5min":8.1229999999999993,"15min":8.1229999999999993},"last":0.43099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":0.94899999999999995,"15min":0.92400000000000004},"min":{"1min":0.314,"5min":0.313,"15min":0.313},"max":{"1min":11.218,"5min":11.218,"15min":11.218},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.878,"15min":0.89900000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":6.5389999999999997,"5min":6.5389999999999997,"15min":6.5389999999999997},"last":0.50900000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":4.2720000000000002,"15min":5.7990000000000004},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.4319999999999999,"5min":124.27200000000001,"15min":124.27200000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.4259999999999999,"5min":4.2039999999999997,"15min":5.5549999999999997},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":11.339,"5min":121.136,"15min":121.136},"last":0.48799999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958980,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1033336,"kb_used_data":6704,"kb_used_omap":7,"kb_used_meta":1026616,"kb_avail":92711816,"statfs":{"total":95995035648,"available":94936899584,"internally_reserved":0,"allocated":6864896,"data_stored":4629162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":1051255310},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":2.5430000000000001,"15min":2.024},"min":{"1min":0.253,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":7.3789999999999996,"5min":27.045999999999999,"15min":131.91300000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":2.7679999999999998,"5min":3.141,"15min":2.3490000000000002},"min":{"1min":0.30599999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.163,"5min":27.344000000000001,"15min":132.191},"last":0.32100000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:08:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":5.5289999999999999,"15min":3.077},"min":{"1min":0.23000000000000001,"5min":0.21299999999999999,"15min":0.20899999999999999},"max":{"1min":16.183,"5min":186.84200000000001,"15min":186.84200000000001},"last":0.26200000000000001},{"interface":"front","average":{"1min":2.621,"5min":5.2910000000000004,"15min":3.081},"min":{"1min":0.26200000000000001,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":20.803999999999998,"5min":186.75800000000001,"15min":186.75800000000001},"last":0.36899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":3.0800000000000001,"5min":4.9500000000000002,"15min":2.9700000000000002},"min":{"1min":0.26000000000000001,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":10.404,"5min":177.70599999999999,"15min":177.70599999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":3.4529999999999998,"5min":4.8470000000000004,"15min":2.915},"min":{"1min":0.36699999999999999,"5min":0.28299999999999997,"15min":0.24099999999999999},"max":{"1min":14.023,"5min":177.82400000000001,"15min":177.82400000000001},"last":0.29399999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":3.5379999999999998,"15min":2.4489999999999998},"min":{"1min":0.35199999999999998,"5min":0.29699999999999999,"15min":0.25800000000000001},"max":{"1min":10.129,"5min":86.483999999999995,"15min":131.636},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.3300000000000001,"5min":2.5099999999999998,"15min":2.0049999999999999},"min":{"1min":0.41199999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.4209999999999998,"5min":86.441999999999993,"15min":131.75999999999999},"last":0.53200000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.9279999999999999,"15min":2.2759999999999998},"min":{"1min":0.41299999999999998,"5min":0.314,"15min":0.313},"max":{"1min":8.6769999999999996,"5min":14.019,"15min":132.291},"last":0.54600000000000004},{"interface":"front","average":{"1min":2.4100000000000001,"5min":3.5350000000000001,"15min":3.04},"min":{"1min":0.39300000000000002,"5min":0.34999999999999998,"15min":0.33700000000000002},"max":{"1min":10.063000000000001,"5min":99.769999999999996,"15min":131.738},"last":0.51000000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:07:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":3.5190000000000001,"15min":3.8149999999999999},"min":{"1min":0.374,"5min":0.372,"15min":0.372},"max":{"1min":13.205,"5min":13.205,"15min":13.205},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.7629999999999999,"5min":2.206,"15min":2.4020000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":7.3259999999999996,"5min":13.685,"15min":13.685},"last":0.56999999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":2.7610000000000001,"15min":2.4380000000000002},"min":{"1min":0.44800000000000001,"5min":0.379,"15min":0.31},"max":{"1min":6.6639999999999997,"5min":86.911000000000001,"15min":131.821},"last":0.62},{"interface":"front","average":{"1min":1.7290000000000001,"5min":2.8530000000000002,"15min":2.6030000000000002},"min":{"1min":0.52200000000000002,"5min":0.38400000000000001,"15min":0.35199999999999998},"max":{"1min":4.7300000000000004,"5min":86.814999999999998,"15min":131.852},"last":0.496}]}]},{"osd":0,"up_from":88,"seq":377957122505,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1607040,"kb_used_data":604984,"kb_used_omap":8,"kb_used_meta":1002039,"kb_avail":92138112,"statfs":{"total":95995035648,"available":94349426688,"internally_reserved":0,"allocated":619503616,"data_stored":617201286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":1026088712},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:07:34 2023","interfaces":[{"interface":"back","average":{"1min":3.8410000000000002,"5min":4.0999999999999996,"15min":3.548},"min":{"1min":0.215,"5min":0.19400000000000001,"15min":0.188},"max":{"1min":15.27,"5min":93.971999999999994,"15min":163.572},"last":0.43099999999999999},{"interface":"front","average":{"1min":5.1630000000000003,"5min":4.4640000000000004,"15min":3.8450000000000002},"min":{"1min":0.28499999999999998,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":16.222000000000001,"5min":94.007000000000005,"15min":162.48699999999999},"last":0.52500000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:07:13 2023","interfaces":[{"interface":"back","average":{"1min":5.5540000000000003,"5min":3.452,"15min":2.2519999999999998},"min":{"1min":0.23400000000000001,"5min":0.22700000000000001,"15min":0.20599999999999999},"max":{"1min":27.338000000000001,"5min":95.512,"15min":95.512},"last":0.45400000000000001},{"interface":"front","average":{"1min":5.3579999999999997,"5min":3.5870000000000002,"15min":2.1749999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.221},"max":{"1min":31.065999999999999,"5min":95.462999999999994,"15min":95.462999999999994},"last":0.48499999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:07:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1760000000000002,"5min":2.8679999999999999,"15min":1.8680000000000001},"min":{"1min":0.318,"5min":0.255,"15min":0.21299999999999999},"max":{"1min":12.247999999999999,"5min":12.304,"15min":62.698999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.7559999999999998,"5min":2.5790000000000002,"15min":1.9339999999999999},"min":{"1min":0.33100000000000002,"5min":0.29899999999999999,"15min":0.28199999999999997},"max":{"1min":8.5090000000000003,"5min":13.632,"15min":60.383000000000003},"last":0.503}]},{"osd":4,"last update":"Mon Oct 9 06:07:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":1.6899999999999999,"15min":1.129},"min":{"1min":0.32200000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.965999999999999,"5min":13.592000000000001,"15min":13.592000000000001},"last":0.75700000000000001},{"interface":"front","average":{"1min":2.8999999999999999,"5min":1.7529999999999999,"15min":1.3700000000000001},"min":{"1min":0.32200000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":13.775,"5min":13.775,"15min":13.775},"last":0.746}]},{"osd":5,"last update":"Mon Oct 9 06:07:13 2023","interfaces":[{"interface":"back","average":{"1min":3.5329999999999999,"5min":2.0089999999999999,"15min":3.4489999999999998},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.27900000000000003},"max":{"1min":13.746,"5min":13.746,"15min":246.761},"last":0.77000000000000002},{"interface":"front","average":{"1min":2.7480000000000002,"5min":1.9510000000000001,"15min":3.5249999999999999},"min":{"1min":0.374,"5min":0.34499999999999997,"15min":0.30399999999999999},"max":{"1min":8.6039999999999992,"5min":11.52,"15min":247.48400000000001},"last":0.68200000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:07:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.9650000000000001,"15min":1.9670000000000001},"min":{"1min":0.39000000000000001,"5min":0.314,"15min":0.314},"max":{"1min":12.929,"5min":12.929,"15min":12.929},"last":0.64200000000000002},{"interface":"front","average":{"1min":3.2330000000000001,"5min":1.996,"15min":1.5920000000000001},"min":{"1min":0.40400000000000003,"5min":0.372,"15min":0.372},"max":{"1min":8.6780000000000008,"5min":13.608000000000001,"15min":13.608000000000001},"last":0.79300000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:07:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":3.8479999999999999,"15min":3.238},"min":{"1min":0.41699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":12.987,"5min":168.98599999999999,"15min":168.98599999999999},"last":0.67200000000000004},{"interface":"front","average":{"1min":2.5049999999999999,"5min":3.9359999999999999,"15min":3.3450000000000002},"min":{"1min":0.47699999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.9909999999999997,"5min":169.03800000000001,"15min":169.03800000000001},"last":0.81299999999999994}]}]},{"osd":2,"up_from":98,"seq":420906795454,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1080924,"kb_used_data":11220,"kb_used_omap":11,"kb_used_meta":1069684,"kb_avail":92664228,"statfs":{"total":95995035648,"available":94888169472,"internally_reserved":0,"allocated":11489280,"data_stored":8879870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":1095357013},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:17 2023","interfaces":[{"interface":"back","average":{"1min":3.9809999999999999,"5min":3.0009999999999999,"15min":2.0779999999999998},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.19600000000000001},"max":{"1min":11.599,"5min":27.292000000000002,"15min":140.71100000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.3919999999999999,"5min":2.7679999999999998,"15min":2.274},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19600000000000001},"max":{"1min":9.7590000000000003,"5min":14.050000000000001,"15min":142.542},"last":0.47699999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:07:17 2023","interfaces":[{"interface":"back","average":{"1min":5.1749999999999998,"5min":4.6449999999999996,"15min":2.8730000000000002},"min":{"1min":0.30099999999999999,"5min":0.25900000000000001,"15min":0.19},"max":{"1min":26.869,"5min":133.42500000000001,"15min":133.42500000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":4.9589999999999996,"5min":5.2160000000000002,"15min":3.2490000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":13.238,"5min":133.40799999999999,"15min":133.40799999999999},"last":0.44700000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.016,"5min":5.181,"15min":2.9009999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.23300000000000001},"max":{"1min":6.8019999999999996,"5min":142.26300000000001,"15min":142.26300000000001},"last":0.503},{"interface":"front","average":{"1min":1.722,"5min":4.1760000000000002,"15min":2.4609999999999999},"min":{"1min":0.30399999999999999,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":8.266,"5min":141.477,"15min":141.477},"last":0.53300000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:07:17 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":2.5369999999999999,"15min":1.5780000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":11.132999999999999,"5min":11.132999999999999,"15min":11.132999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":2.871,"5min":1.7549999999999999,"15min":1.208},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.3100000000000005,"5min":16.457000000000001,"15min":16.457000000000001},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:07:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":2.093,"15min":2.778},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.33000000000000002},"max":{"1min":9.577,"5min":10.852,"15min":207.80500000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":3.4790000000000001,"5min":2.7080000000000002,"15min":3.1589999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.169,"5min":15.781000000000001,"15min":207.54900000000001},"last":0.84899999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:07:43 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":2.1459999999999999,"15min":2.0289999999999999},"min":{"1min":0.47099999999999997,"5min":0.375,"15min":0.375},"max":{"1min":8.4589999999999996,"5min":16.422000000000001,"15min":16.422000000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":2.2839999999999998,"5min":2.0659999999999998,"15min":1.905},"min":{"1min":0.41899999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":9.6210000000000004,"5min":10.801,"15min":10.801},"last":0.74099999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":2.0779999999999998,"15min":2.4590000000000001},"min":{"1min":0.46400000000000002,"5min":0.39900000000000002,"15min":0.34399999999999997},"max":{"1min":8.3640000000000008,"5min":16.382999999999999,"15min":162.09200000000001},"last":0.81399999999999995},{"interface":"front","average":{"1min":2.0099999999999998,"5min":2.3250000000000002,"15min":2.6629999999999998},"min":{"1min":0.45000000000000001,"5min":0.39600000000000002,"15min":0.36099999999999999},"max":{"1min":9.5990000000000002,"5min":10.890000000000001,"15min":162.35400000000001},"last":0.78400000000000003}]}]},{"osd":3,"up_from":103,"seq":442381631928,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1211204,"kb_used_data":6780,"kb_used_omap":4,"kb_used_meta":1204411,"kb_avail":92533948,"statfs":{"total":95995035648,"available":94754762752,"internally_reserved":0,"allocated":6942720,"data_stored":4629723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1233317470},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":6.3620000000000001,"5min":5.5300000000000002,"15min":4.4160000000000004},"min":{"1min":0.26100000000000001,"5min":0.217,"15min":0.20399999999999999},"max":{"1min":50.546999999999997,"5min":135.501,"15min":312.56599999999997},"last":0.32200000000000001},{"interface":"front","average":{"1min":5.9340000000000002,"5min":5.266,"15min":4.3760000000000003},"min":{"1min":0.23300000000000001,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":55.683,"5min":135.99199999999999,"15min":311.02300000000002},"last":0.53100000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":8.6780000000000008,"5min":5.5750000000000002,"15min":5.1840000000000002},"min":{"1min":0.29799999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":55.277000000000001,"5min":89.165000000000006,"15min":142.32599999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":7.5510000000000002,"5min":4.9089999999999998,"15min":4.9960000000000004},"min":{"1min":0.33500000000000002,"5min":0.23899999999999999,"15min":0.224},"max":{"1min":55.209000000000003,"5min":89.093999999999994,"15min":139.33199999999999},"last":0.34200000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":9.1489999999999991,"5min":7.165,"15min":4.0069999999999997},"min":{"1min":0.314,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":45.183,"5min":89.024000000000001,"15min":89.024000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":9.327,"5min":5.6779999999999999,"15min":3.4300000000000002},"min":{"1min":0.433,"5min":0.25900000000000001,"15min":0.248},"max":{"1min":51.189999999999998,"5min":90.850999999999999,"15min":90.850999999999999},"last":0.38400000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":2.2000000000000002,"5min":3.0459999999999998,"15min":1.8759999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.9930000000000003,"5min":74.957999999999998,"15min":74.957999999999998},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.4140000000000001,"5min":3.0680000000000001,"15min":1.909},"min":{"1min":0.315,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":9.7309999999999999,"5min":75.022999999999996,"15min":75.022999999999996},"last":0.60399999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":1.5329999999999999,"15min":1.5920000000000001},"min":{"1min":0.38200000000000001,"5min":0.32900000000000001,"15min":0.309},"max":{"1min":6.8449999999999998,"5min":8.8450000000000006,"15min":71.203999999999994},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.2250000000000001,"5min":1.849,"15min":1.6599999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.309},"max":{"1min":9.8109999999999999,"5min":15.932,"15min":70.397000000000006},"last":0.57399999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":1.7589999999999999,"15min":1.591},"min":{"1min":0.45900000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.7939999999999996,"5min":8.9149999999999991,"15min":8.9149999999999991},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.6749999999999998,"5min":2.71,"15min":2.8500000000000001},"min":{"1min":0.40100000000000002,"5min":0.38,"15min":0.38},"max":{"1min":9.7780000000000005,"5min":15.954000000000001,"15min":15.954000000000001},"last":0.63600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:07:58 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":3.1949999999999998,"15min":1.952},"min":{"1min":0.42999999999999999,"5min":0.375,"15min":0.375},"max":{"1min":8.0090000000000003,"5min":79.403000000000006,"15min":79.403000000000006},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.895,"5min":3.4060000000000001,"15min":1.986},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":12.242000000000001,"5min":79.203000000000003,"15min":79.203000000000003},"last":0.65500000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019474,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2076872,"kb_used_data":891136,"kb_used_omap":0,"kb_used_meta":1185728,"kb_avail":91668280,"statfs":{"total":95995035648,"available":93868318720,"internally_reserved":0,"allocated":912523264,"data_stored":911339028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1214185472},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:19 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":3.7410000000000001,"15min":2.8460000000000001},"min":{"1min":0.23799999999999999,"5min":0.22600000000000001,"15min":0.21299999999999999},"max":{"1min":17.248999999999999,"5min":86.509,"15min":95.466999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":3.6419999999999999,"5min":4.3250000000000002,"15min":3.2730000000000001},"min":{"1min":0.27600000000000002,"5min":0.246,"15min":0.23699999999999999},"max":{"1min":18.664000000000001,"5min":86.233999999999995,"15min":95.367999999999995},"last":0.46100000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:07:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3650000000000002,"5min":2.9260000000000002,"15min":2.6480000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.7050000000000001,"5min":34.911000000000001,"15min":96.033000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.145,"5min":4.0999999999999996,"15min":3.04},"min":{"1min":0.33200000000000002,"5min":0.26700000000000002,"15min":0.255},"max":{"1min":10.766,"5min":108.45699999999999,"15min":108.45699999999999},"last":0.35699999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:08:02 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":4.7510000000000003,"15min":2.8650000000000002},"min":{"1min":0.35299999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.1029999999999998,"5min":105.157,"15min":143.71000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.117,"5min":4.4409999999999998,"15min":3.133},"min":{"1min":0.35599999999999998,"5min":0.29599999999999999,"15min":0.27700000000000002},"max":{"1min":3.573,"5min":107.851,"15min":143.768},"last":0.64100000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":2.492,"5min":3.4830000000000001,"15min":2.9700000000000002},"min":{"1min":0.35499999999999998,"5min":0.254,"15min":0.254},"max":{"1min":8.1159999999999997,"5min":107.352,"15min":107.352},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.5510000000000002,"5min":3.0499999999999998,"15min":2.77},"min":{"1min":0.35399999999999998,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":8.3919999999999995,"5min":105.336,"15min":105.336},"last":0.52400000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":1.1419999999999999,"15min":1.093},"min":{"1min":0.32700000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.9349999999999996,"5min":7.1070000000000002,"15min":15.597},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.262,"5min":1.004,"15min":1.0369999999999999},"min":{"1min":0.378,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":6.2990000000000004,"5min":7.069,"15min":17.882000000000001},"last":0.61799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.94499999999999995,"15min":0.88},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.9400000000000004,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.57599999999999996},{"interface":"front","average":{"1min":2.306,"5min":1.2869999999999999,"15min":1.0549999999999999},"min":{"1min":0.38300000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.0340000000000007,"5min":8.9909999999999997,"15min":8.9909999999999997},"last":0.54000000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:07:54 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":0.90900000000000003,"15min":1.1890000000000001},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":4.9610000000000003,"5min":6.6040000000000001,"15min":36.729999999999997},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.986,"5min":1.226,"15min":1.2050000000000001},"min":{"1min":0.41299999999999998,"5min":0.33000000000000002,"15min":0.32200000000000001},"max":{"1min":7.96,"5min":8.9580000000000002,"15min":36.424999999999997},"last":0.59699999999999998}]}]},{"osd":5,"up_from":0,"seq":154618823248,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13650,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:08:09.277 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:08:09.278 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:09.316 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:08:09.338 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T06:08:09.340 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T06:08:09.340 DEBUG:teuthology.orchestra.run.smithi008:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T06:08:09.360 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:09.389 INFO:tasks.workunit.client.0.smithi008.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T06:08:09.389 INFO:tasks.workunit.client.0.smithi008.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-10-09T06:08:09.389 INFO:tasks.workunit.client.0.smithi008.stderr:+ TMPDIR=/tmp/rbd_import_export_704372 2023-10-09T06:08:09.389 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm -rf /tmp/rbd_import_export_704372 2023-10-09T06:08:09.390 INFO:tasks.workunit.client.0.smithi008.stderr:+ mkdir /tmp/rbd_import_export_704372 2023-10-09T06:08:09.391 INFO:tasks.workunit.client.0.smithi008.stderr:+ trap 'rm -rf /tmp/rbd_import_export_704372' INT TERM EXIT 2023-10-09T06:08:09.391 INFO:tasks.workunit.client.0.smithi008.stderr:+ mkdir foo.704372 2023-10-09T06:08:09.392 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import foo.704372 foo.dir 2023-10-09T06:08:09.468 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: cannot import a directory 2023-10-09T06:08:09.468 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 0% complete...failed. 2023-10-09T06:08:09.468 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: import failed: (21) Is a directory 2023-10-09T06:08:09.476 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T06:08:09.476 INFO:tasks.workunit.client.0.smithi008.stderr:+ rmdir foo.704372 2023-10-09T06:08:09.477 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_704372/img bs=1k count=1 seek=10 2023-10-09T06:08:09.478 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records in 2023-10-09T06:08:09.478 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records out 2023-10-09T06:08:09.478 INFO:tasks.workunit.client.0.smithi008.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 5.2577e-05 s, 19.5 MB/s 2023-10-09T06:08:09.478 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_704372/img bs=1k count=10 seek=100 2023-10-09T06:08:09.479 INFO:tasks.workunit.client.0.smithi008.stderr:10+0 records in 2023-10-09T06:08:09.479 INFO:tasks.workunit.client.0.smithi008.stderr:10+0 records out 2023-10-09T06:08:09.479 INFO:tasks.workunit.client.0.smithi008.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.1664e-05 s, 166 MB/s 2023-10-09T06:08:09.479 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_704372/img bs=1k count=100 seek=1000 2023-10-09T06:08:09.479 INFO:tasks.workunit.client.0.smithi008.stderr:70+1 records in 2023-10-09T06:08:09.480 INFO:tasks.workunit.client.0.smithi008.stderr:70+1 records out 2023-10-09T06:08:09.480 INFO:tasks.workunit.client.0.smithi008.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000202258 s, 356 MB/s 2023-10-09T06:08:09.480 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_704372/img bs=1k seek=10000 2023-10-09T06:08:09.480 INFO:tasks.workunit.client.0.smithi008.stderr:139+1 records in 2023-10-09T06:08:09.481 INFO:tasks.workunit.client.0.smithi008.stderr:139+1 records out 2023-10-09T06:08:09.481 INFO:tasks.workunit.client.0.smithi008.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000360804 s, 397 MB/s 2023-10-09T06:08:09.481 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_704372/img bs=1k seek=100000 2023-10-09T06:08:09.481 INFO:tasks.workunit.client.0.smithi008.stderr:70+1 records in 2023-10-09T06:08:09.482 INFO:tasks.workunit.client.0.smithi008.stderr:70+1 records out 2023-10-09T06:08:09.482 INFO:tasks.workunit.client.0.smithi008.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.000233486 s, 309 MB/s 2023-10-09T06:08:09.482 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_704372/img bs=1k seek=1000000 2023-10-09T06:08:09.483 INFO:tasks.workunit.client.0.smithi008.stderr:189+1 records in 2023-10-09T06:08:09.483 INFO:tasks.workunit.client.0.smithi008.stderr:189+1 records out 2023-10-09T06:08:09.483 INFO:tasks.workunit.client.0.smithi008.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000576134 s, 336 MB/s 2023-10-09T06:08:09.483 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm testimg 2023-10-09T06:08:09.530 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 0% complete...failed. 2023-10-09T06:08:09.530 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:08:09.533 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T06:08:09.533 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format /tmp/rbd_import_export_704372/img testimg 2023-10-09T06:08:09.878 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:09.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:09.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:09.987 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:10.021 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:10.167 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T06:08:10.168 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:08:10.170 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg /tmp/rbd_import_export_704372/img2 2023-10-09T06:08:10.650 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:10.653 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg - 2023-10-09T06:08:10.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:10 smithi008 ceph-mon[184418]: pgmap v1781: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:10.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:10 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:10.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:08:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:08:10] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.33.4" 2023-10-09T06:08:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:10 smithi008 ceph-mon[191227]: pgmap v1781: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:10.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:10 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:11.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:10 smithi191 ceph-mon[161669]: pgmap v1781: 73 pgs: 73 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:08:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:10 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:11.431 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:11.435 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm testimg 2023-10-09T06:08:11.531 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T06:08:11.534 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp /tmp/rbd_import_export_704372/img /tmp/rbd_import_export_704372/img2 2023-10-09T06:08:12.317 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp /tmp/rbd_import_export_704372/img /tmp/rbd_import_export_704372/img3 2023-10-09T06:08:12.840 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/img2 /tmp/rbd_import_export_704372/img3 2023-10-09T06:08:12.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:12 smithi008 ceph-mon[184418]: pgmap v1782: 73 pgs: 73 active+clean; 8.2 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 5.0 KiB/s rd, 28 KiB/s wr, 7 op/s 2023-10-09T06:08:12.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:12 smithi008 ceph-mon[191227]: pgmap v1782: 73 pgs: 73 active+clean; 8.2 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 5.0 KiB/s rd, 28 KiB/s wr, 7 op/s 2023-10-09T06:08:13.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:12 smithi191 ceph-mon[161669]: pgmap v1782: 73 pgs: 73 active+clean; 8.2 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 5.0 KiB/s rd, 28 KiB/s wr, 7 op/s 2023-10-09T06:08:13.272 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format - testimg 2023-10-09T06:08:13.923 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:13.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:13.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:14.003 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T06:08:14.003 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:08:14.006 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg /tmp/rbd_import_export_704372/img2 2023-10-09T06:08:14.029 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:14.060 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:14.144 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:14.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:14.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:14.160 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:08:14.259 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:14.289 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:14.304 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:14.366 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:14.371 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg - 2023-10-09T06:08:14.402 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:14.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:14.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:14.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:14 smithi008 ceph-mon[184418]: pgmap v1783: 73 pgs: 73 active+clean; 8.2 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 23 KiB/s rd, 31 KiB/s wr, 9 op/s 2023-10-09T06:08:14.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:14.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:14.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:14.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:14.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:14 smithi008 ceph-mon[191227]: pgmap v1783: 73 pgs: 73 active+clean; 8.2 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 23 KiB/s rd, 31 KiB/s wr, 9 op/s 2023-10-09T06:08:14.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:14.467 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:14.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:14.468 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:14.508 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:14.539 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:14.637 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:14.782 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:08:14.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:14 smithi191 ceph-mon[161669]: pgmap v1783: 73 pgs: 73 active+clean; 8.2 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 23 KiB/s rd, 31 KiB/s wr, 9 op/s 2023-10-09T06:08:14.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:14.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:14.801 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:08:14.805 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:14.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:14.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:14.822 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:08:14.842 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:08:14.861 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:08:14.880 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:08:14.904 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:08:14.911 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:14.923 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:08:14.924 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:08:14.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:08:14.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:08:14.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:08:14.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:08:14.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:08:14.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:08:14.925 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:08:14.942 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:14.990 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:14.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:14.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:15.095 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:15.136 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:15.211 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:15.215 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm testimg 2023-10-09T06:08:15.317 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T06:08:15.320 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp /tmp/rbd_import_export_704372/img /tmp/rbd_import_export_704372/img2 2023-10-09T06:08:15.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:15 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:15.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:15 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:15.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:15 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:15.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:15 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1631006874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:15.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:15 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:15.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:15 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:15.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:15 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:15.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:15 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1631006874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:15 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:15 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:15 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:15.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:15 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1631006874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:15.895 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:15.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:15.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:15.999 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:16.019 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp /tmp/rbd_import_export_704372/img /tmp/rbd_import_export_704372/img3 2023-10-09T06:08:16.029 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:16.380 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:16.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:16.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:16.483 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:16.513 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:16.565 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/img /tmp/rbd_import_export_704372/img2 /tmp/rbd_import_export_704372/img3 2023-10-09T06:08:16.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:16 smithi008 ceph-mon[184418]: pgmap v1784: 73 pgs: 73 active+clean; 10 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 189 KiB/s rd, 103 KiB/s wr, 63 op/s 2023-10-09T06:08:16.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:16 smithi008 ceph-mon[191227]: pgmap v1784: 73 pgs: 73 active+clean; 10 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 189 KiB/s rd, 103 KiB/s wr, 63 op/s 2023-10-09T06:08:16.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:16 smithi191 ceph-mon[161669]: pgmap v1784: 73 pgs: 73 active+clean; 10 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 189 KiB/s rd, 103 KiB/s wr, 63 op/s 2023-10-09T06:08:18.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:18 smithi008 ceph-mon[184418]: pgmap v1785: 73 pgs: 73 active+clean; 10 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 159 KiB/s rd, 86 KiB/s wr, 54 op/s 2023-10-09T06:08:18.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:18 smithi008 ceph-mon[191227]: pgmap v1785: 73 pgs: 73 active+clean; 10 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 159 KiB/s rd, 86 KiB/s wr, 54 op/s 2023-10-09T06:08:18.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:18 smithi191 ceph-mon[161669]: pgmap v1785: 73 pgs: 73 active+clean; 10 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 159 KiB/s rd, 86 KiB/s wr, 54 op/s 2023-10-09T06:08:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:20 smithi008 ceph-mon[184418]: pgmap v1786: 73 pgs: 73 active+clean; 10 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 159 KiB/s rd, 86 KiB/s wr, 54 op/s 2023-10-09T06:08:20.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:08:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:08:20] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.33.4" 2023-10-09T06:08:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:20 smithi008 ceph-mon[191227]: pgmap v1786: 73 pgs: 73 active+clean; 10 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 159 KiB/s rd, 86 KiB/s wr, 54 op/s 2023-10-09T06:08:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:20 smithi191 ceph-mon[161669]: pgmap v1786: 73 pgs: 73 active+clean; 10 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 159 KiB/s rd, 86 KiB/s wr, 54 op/s 2023-10-09T06:08:22.668 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd help export 2023-10-09T06:08:22.668 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -q export-format 2023-10-09T06:08:22.688 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_704372/img bs=1k count=10 seek=100 2023-10-09T06:08:22.689 INFO:tasks.workunit.client.0.smithi008.stderr:10+0 records in 2023-10-09T06:08:22.689 INFO:tasks.workunit.client.0.smithi008.stderr:10+0 records out 2023-10-09T06:08:22.689 INFO:tasks.workunit.client.0.smithi008.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.1869e-05 s, 142 MB/s 2023-10-09T06:08:22.689 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format /tmp/rbd_import_export_704372/img testimg 2023-10-09T06:08:22.745 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T06:08:22.745 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:08:22.747 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap create testimg@snap 2023-10-09T06:08:22.963 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:22 smithi008 ceph-mon[184418]: pgmap v1787: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 211 KiB/s rd, 86 KiB/s wr, 67 op/s 2023-10-09T06:08:22.963 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:22 smithi008 ceph-mon[191227]: pgmap v1787: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 211 KiB/s rd, 86 KiB/s wr, 67 op/s 2023-10-09T06:08:23.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:22 smithi191 ceph-mon[161669]: pgmap v1787: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 211 KiB/s rd, 86 KiB/s wr, 67 op/s 2023-10-09T06:08:23.575 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:23.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:23.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:23.659 INFO:tasks.workunit.client.0.smithi008.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:08:23.666 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T06:08:23.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:23.710 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:23.730 INFO:tasks.workunit.client.0.smithi008.stderr:++ rbd image-meta list testimg 2023-10-09T06:08:23.758 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:23.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:23.766 INFO:tasks.workunit.client.0.smithi008.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-10-09T06:08:23.766 INFO:tasks.workunit.client.0.smithi008.stderr: 2023-10-09T06:08:23.766 INFO:tasks.workunit.client.0.smithi008.stderr:Key Value 2023-10-09T06:08:23.766 INFO:tasks.workunit.client.0.smithi008.stderr:key1 value1' 2023-10-09T06:08:23.766 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_704372/img_v2 2023-10-09T06:08:23.817 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:23.820 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_704372/img_v2 testimg_import 2023-10-09T06:08:23.863 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:23.886 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:23.889 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:23.931 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:23.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:23.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:24.037 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:24.069 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:24.217 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:24.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:24.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:24.321 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:24.348 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:24.390 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:24.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:24.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:24.495 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:24.526 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:24.660 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is set 2023-10-09T06:08:24.672 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:08:24.675 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import 2023-10-09T06:08:24.709 INFO:tasks.workunit.client.0.smithi008.stdout:rbd image 'testimg_import': 2023-10-09T06:08:24.709 INFO:tasks.workunit.client.0.smithi008.stdout: size 110 KiB in 1 objects 2023-10-09T06:08:24.709 INFO:tasks.workunit.client.0.smithi008.stdout: order 22 (4 MiB objects) 2023-10-09T06:08:24.709 INFO:tasks.workunit.client.0.smithi008.stdout: snapshot_count: 1 2023-10-09T06:08:24.709 INFO:tasks.workunit.client.0.smithi008.stdout: id: b9a5a37d4ac4 2023-10-09T06:08:24.709 INFO:tasks.workunit.client.0.smithi008.stdout: block_name_prefix: rbd_data.b9a5a37d4ac4 2023-10-09T06:08:24.710 INFO:tasks.workunit.client.0.smithi008.stdout: format: 2 2023-10-09T06:08:24.710 INFO:tasks.workunit.client.0.smithi008.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T06:08:24.710 INFO:tasks.workunit.client.0.smithi008.stdout: op_features: 2023-10-09T06:08:24.710 INFO:tasks.workunit.client.0.smithi008.stdout: flags: 2023-10-09T06:08:24.710 INFO:tasks.workunit.client.0.smithi008.stdout: create_timestamp: Mon Oct 9 06:08:23 2023 2023-10-09T06:08:24.710 INFO:tasks.workunit.client.0.smithi008.stdout: access_timestamp: Mon Oct 9 06:08:23 2023 2023-10-09T06:08:24.710 INFO:tasks.workunit.client.0.smithi008.stdout: modify_timestamp: Mon Oct 9 06:08:23 2023 2023-10-09T06:08:24.712 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import@snap 2023-10-09T06:08:24.748 INFO:tasks.workunit.client.0.smithi008.stdout:rbd image 'testimg_import': 2023-10-09T06:08:24.748 INFO:tasks.workunit.client.0.smithi008.stdout: size 110 KiB in 1 objects 2023-10-09T06:08:24.748 INFO:tasks.workunit.client.0.smithi008.stdout: order 22 (4 MiB objects) 2023-10-09T06:08:24.748 INFO:tasks.workunit.client.0.smithi008.stdout: snapshot_count: 1 2023-10-09T06:08:24.748 INFO:tasks.workunit.client.0.smithi008.stdout: id: b9a5a37d4ac4 2023-10-09T06:08:24.748 INFO:tasks.workunit.client.0.smithi008.stdout: block_name_prefix: rbd_data.b9a5a37d4ac4 2023-10-09T06:08:24.748 INFO:tasks.workunit.client.0.smithi008.stdout: format: 2 2023-10-09T06:08:24.748 INFO:tasks.workunit.client.0.smithi008.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T06:08:24.748 INFO:tasks.workunit.client.0.smithi008.stdout: op_features: 2023-10-09T06:08:24.748 INFO:tasks.workunit.client.0.smithi008.stdout: flags: 2023-10-09T06:08:24.749 INFO:tasks.workunit.client.0.smithi008.stdout: create_timestamp: Mon Oct 9 06:08:23 2023 2023-10-09T06:08:24.749 INFO:tasks.workunit.client.0.smithi008.stdout: access_timestamp: Mon Oct 9 06:08:23 2023 2023-10-09T06:08:24.749 INFO:tasks.workunit.client.0.smithi008.stdout: modify_timestamp: Mon Oct 9 06:08:23 2023 2023-10-09T06:08:24.749 INFO:tasks.workunit.client.0.smithi008.stdout: protected: False 2023-10-09T06:08:24.751 INFO:tasks.workunit.client.0.smithi008.stderr:++ rbd image-meta list testimg_import 2023-10-09T06:08:24.796 INFO:tasks.workunit.client.0.smithi008.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-10-09T06:08:24.797 INFO:tasks.workunit.client.0.smithi008.stderr: 2023-10-09T06:08:24.797 INFO:tasks.workunit.client.0.smithi008.stderr:Key Value 2023-10-09T06:08:24.797 INFO:tasks.workunit.client.0.smithi008.stderr:key1 value1' 2023-10-09T06:08:24.797 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 'There is 1 metadatum on this image: 2023-10-09T06:08:24.797 INFO:tasks.workunit.client.0.smithi008.stderr: 2023-10-09T06:08:24.797 INFO:tasks.workunit.client.0.smithi008.stderr:Key Value 2023-10-09T06:08:24.797 INFO:tasks.workunit.client.0.smithi008.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-10-09T06:08:24.797 INFO:tasks.workunit.client.0.smithi008.stderr: 2023-10-09T06:08:24.797 INFO:tasks.workunit.client.0.smithi008.stderr:Key Value 2023-10-09T06:08:24.797 INFO:tasks.workunit.client.0.smithi008.stderr:key1 value1' ']' 2023-10-09T06:08:24.797 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg_import /tmp/rbd_import_export_704372/img_import 2023-10-09T06:08:24.831 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete...done. 2023-10-09T06:08:24.834 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_704372/img /tmp/rbd_import_export_704372/img_import 2023-10-09T06:08:24.834 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_704372/img /tmp/rbd_import_export_704372/img_import 2023-10-09T06:08:24.835 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/img --format %b 2023-10-09T06:08:24.836 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=24 2023-10-09T06:08:24.836 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/img_import --format %b 2023-10-09T06:08:24.837 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=224 2023-10-09T06:08:24.837 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=200 2023-10-09T06:08:24.837 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=200 2023-10-09T06:08:24.837 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T06:08:24.837 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_704372/img_snap 2023-10-09T06:08:24.874 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete...done. 2023-10-09T06:08:24.876 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_704372/img_snap_import 2023-10-09T06:08:24.911 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete...done. 2023-10-09T06:08:24.914 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_704372/img_snap /tmp/rbd_import_export_704372/img_snap_import 2023-10-09T06:08:24.914 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_704372/img_snap /tmp/rbd_import_export_704372/img_snap_import 2023-10-09T06:08:24.915 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/img_snap --format %b 2023-10-09T06:08:24.916 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=224 2023-10-09T06:08:24.916 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/img_snap_import --format %b 2023-10-09T06:08:24.917 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=224 2023-10-09T06:08:24.917 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T06:08:24.918 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T06:08:24.918 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:08:24.918 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/img_v2 2023-10-09T06:08:24.946 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/img_import 2023-10-09T06:08:24.949 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/img_snap 2023-10-09T06:08:24.950 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/img_snap_import 2023-10-09T06:08:24.951 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T06:08:24.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:24 smithi008 ceph-mon[184418]: pgmap v1788: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 207 KiB/s rd, 68 KiB/s wr, 61 op/s 2023-10-09T06:08:24.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:24 smithi008 ceph-mon[184418]: failed to encode map e591 with expected crc 2023-10-09T06:08:24.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:24 smithi008 ceph-mon[184418]: failed to encode map e591 with expected crc 2023-10-09T06:08:24.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:24 smithi008 ceph-mon[184418]: osdmap e591: 8 total, 8 up, 7 in 2023-10-09T06:08:24.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:24 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1921262217' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:24 smithi008 ceph-mon[191227]: pgmap v1788: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 207 KiB/s rd, 68 KiB/s wr, 61 op/s 2023-10-09T06:08:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:24 smithi008 ceph-mon[191227]: failed to encode map e591 with expected crc 2023-10-09T06:08:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:24 smithi008 ceph-mon[191227]: failed to encode map e591 with expected crc 2023-10-09T06:08:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:24 smithi008 ceph-mon[191227]: osdmap e591: 8 total, 8 up, 7 in 2023-10-09T06:08:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:24 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1921262217' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:25.021 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:08:25.022 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T06:08:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:24 smithi191 ceph-mon[161669]: pgmap v1788: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 207 KiB/s rd, 68 KiB/s wr, 61 op/s 2023-10-09T06:08:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:24 smithi191 ceph-mon[161669]: failed to encode map e591 with expected crc 2023-10-09T06:08:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:24 smithi191 ceph-mon[161669]: failed to encode map e591 with expected crc 2023-10-09T06:08:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:24 smithi191 ceph-mon[161669]: osdmap e591: 8 total, 8 up, 7 in 2023-10-09T06:08:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:24 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1921262217' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:25.664 INFO:tasks.workunit.client.0.smithi008.stderr: Removing snap: 100% complete...done. 2023-10-09T06:08:25.675 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg_import 2023-10-09T06:08:25.765 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T06:08:25.768 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap rm testimg@snap 2023-10-09T06:08:25.808 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:08:25.963 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[184418]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:25.963 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[184418]: failed to encode map e592 with expected crc 2023-10-09T06:08:25.963 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[184418]: failed to encode map e592 with expected crc 2023-10-09T06:08:25.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[184418]: failed to encode map e592 with expected crc 2023-10-09T06:08:25.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1921262217' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:25.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[184418]: osdmap e592: 8 total, 8 up, 7 in 2023-10-09T06:08:25.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[184418]: failed to encode map e592 with expected crc 2023-10-09T06:08:25.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[191227]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:25.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[191227]: failed to encode map e592 with expected crc 2023-10-09T06:08:25.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[191227]: failed to encode map e592 with expected crc 2023-10-09T06:08:25.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[191227]: failed to encode map e592 with expected crc 2023-10-09T06:08:25.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1921262217' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:25.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[191227]: osdmap e592: 8 total, 8 up, 7 in 2023-10-09T06:08:25.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:25 smithi008 ceph-mon[191227]: failed to encode map e592 with expected crc 2023-10-09T06:08:26.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:25 smithi191 ceph-mon[161669]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:25 smithi191 ceph-mon[161669]: failed to encode map e592 with expected crc 2023-10-09T06:08:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:25 smithi191 ceph-mon[161669]: failed to encode map e592 with expected crc 2023-10-09T06:08:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:25 smithi191 ceph-mon[161669]: failed to encode map e592 with expected crc 2023-10-09T06:08:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:25 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1921262217' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:25 smithi191 ceph-mon[161669]: osdmap e592: 8 total, 8 up, 7 in 2023-10-09T06:08:26.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:25 smithi191 ceph-mon[161669]: failed to encode map e592 with expected crc 2023-10-09T06:08:26.432 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:26.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:26.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:26.536 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:26.565 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:26.654 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1791,"stamp":"2023-10-09T06:08:24.981835+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5015631,"num_objects":36,"num_object_clones":0,"num_object_copies":108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36,"num_whiteouts":0,"num_read":7195,"num_read_kb":16433,"num_write":1259,"num_write_kb":16214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":266,"num_bytes_recovered":10777762,"num_keys_recovered":30,"num_objects_omap":17,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1340,"ondisk_log_size":1340,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":226,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8666188,"kb_used_data":1305300,"kb_used_omap":59,"kb_used_meta":7360772,"kb_avail":647549876,"statfs":{"total":671965249536,"available":663091073024,"internally_reserved":0,"allocated":1336627200,"data_stored":1323571971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":60463,"internal_metadata":7537431505},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-5461164,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":345,"num_read_kb":819,"num_write":56,"num_write_kb":41,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001870"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":442,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.379754+0000","last_change":"2023-10-09T06:05:30.517344+0000","last_active":"2023-10-09T06:08:03.379754+0000","last_peered":"2023-10-09T06:08:03.379754+0000","last_clean":"2023-10-09T06:08:03.379754+0000","last_became_active":"2023-10-09T06:05:30.516920+0000","last_became_peered":"2023-10-09T06:05:30.516920+0000","last_unstale":"2023-10-09T06:08:03.379754+0000","last_undegraded":"2023-10-09T06:08:03.379754+0000","last_fullsized":"2023-10-09T06:08:03.379754+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.804649+0000","last_change":"2023-10-09T06:07:47.056618+0000","last_active":"2023-10-09T06:08:03.804649+0000","last_peered":"2023-10-09T06:08:03.804649+0000","last_clean":"2023-10-09T06:08:03.804649+0000","last_became_active":"2023-10-09T06:07:47.056311+0000","last_became_peered":"2023-10-09T06:07:47.056311+0000","last_unstale":"2023-10-09T06:08:03.804649+0000","last_undegraded":"2023-10-09T06:08:03.804649+0000","last_fullsized":"2023-10-09T06:08:03.804649+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:28.019550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":330,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:05.399711+0000","last_change":"2023-10-09T06:07:24.553497+0000","last_active":"2023-10-09T06:08:05.399711+0000","last_peered":"2023-10-09T06:08:05.399711+0000","last_clean":"2023-10-09T06:08:05.399711+0000","last_became_active":"2023-10-09T06:07:24.553189+0000","last_became_peered":"2023-10-09T06:07:24.553189+0000","last_unstale":"2023-10-09T06:08:05.399711+0000","last_undegraded":"2023-10-09T06:08:05.399711+0000","last_fullsized":"2023-10-09T06:08:05.399711+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:53.168858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":659,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653025+0000","last_change":"2023-10-09T06:03:56.489473+0000","last_active":"2023-10-09T06:08:23.653025+0000","last_peered":"2023-10-09T06:08:23.653025+0000","last_clean":"2023-10-09T06:08:23.653025+0000","last_became_active":"2023-10-09T06:03:56.489102+0000","last_became_peered":"2023-10-09T06:03:56.489102+0000","last_unstale":"2023-10-09T06:08:23.653025+0000","last_undegraded":"2023-10-09T06:08:23.653025+0000","last_fullsized":"2023-10-09T06:08:23.653025+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.111404+0000","last_change":"2023-10-09T06:07:50.492752+0000","last_active":"2023-10-09T06:08:04.111404+0000","last_peered":"2023-10-09T06:08:04.111404+0000","last_clean":"2023-10-09T06:08:04.111404+0000","last_became_active":"2023-10-09T06:07:50.492443+0000","last_became_peered":"2023-10-09T06:07:50.492443+0000","last_unstale":"2023-10-09T06:08:04.111404+0000","last_undegraded":"2023-10-09T06:08:04.111404+0000","last_fullsized":"2023-10-09T06:08:04.111404+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:02.416416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":152,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.111616+0000","last_change":"2023-10-09T06:04:33.125479+0000","last_active":"2023-10-09T06:08:04.111616+0000","last_peered":"2023-10-09T06:08:04.111616+0000","last_clean":"2023-10-09T06:08:04.111616+0000","last_became_active":"2023-10-09T06:04:33.125135+0000","last_became_peered":"2023-10-09T06:04:33.125135+0000","last_unstale":"2023-10-09T06:08:04.111616+0000","last_undegraded":"2023-10-09T06:08:04.111616+0000","last_fullsized":"2023-10-09T06:08:04.111616+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:33.288989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":547,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.111751+0000","last_change":"2023-10-09T06:04:33.126133+0000","last_active":"2023-10-09T06:08:04.111751+0000","last_peered":"2023-10-09T06:08:04.111751+0000","last_clean":"2023-10-09T06:08:04.111751+0000","last_became_active":"2023-10-09T06:04:33.125857+0000","last_became_peered":"2023-10-09T06:04:33.125857+0000","last_unstale":"2023-10-09T06:08:04.111751+0000","last_undegraded":"2023-10-09T06:08:04.111751+0000","last_fullsized":"2023-10-09T06:08:04.111751+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:04.644424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":223,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.971824+0000","last_change":"2023-10-09T06:04:32.950240+0000","last_active":"2023-10-09T06:08:02.971824+0000","last_peered":"2023-10-09T06:08:02.971824+0000","last_clean":"2023-10-09T06:08:02.971824+0000","last_became_active":"2023-10-09T06:04:32.949852+0000","last_became_peered":"2023-10-09T06:04:32.949852+0000","last_unstale":"2023-10-09T06:08:02.971824+0000","last_undegraded":"2023-10-09T06:08:02.971824+0000","last_fullsized":"2023-10-09T06:08:02.971824+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:04.138187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":258,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653151+0000","last_change":"2023-10-09T06:03:09.369870+0000","last_active":"2023-10-09T06:08:23.653151+0000","last_peered":"2023-10-09T06:08:23.653151+0000","last_clean":"2023-10-09T06:08:23.653151+0000","last_became_active":"2023-10-09T06:03:09.369216+0000","last_became_peered":"2023-10-09T06:03:09.369216+0000","last_unstale":"2023-10-09T06:08:23.653151+0000","last_undegraded":"2023-10-09T06:08:23.653151+0000","last_fullsized":"2023-10-09T06:08:23.653151+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":258,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653026+0000","last_change":"2023-10-09T06:04:32.956316+0000","last_active":"2023-10-09T06:08:23.653026+0000","last_peered":"2023-10-09T06:08:23.653026+0000","last_clean":"2023-10-09T06:08:23.653026+0000","last_became_active":"2023-10-09T06:04:32.955969+0000","last_became_peered":"2023-10-09T06:04:32.955969+0000","last_unstale":"2023-10-09T06:08:23.653026+0000","last_undegraded":"2023-10-09T06:08:23.653026+0000","last_fullsized":"2023-10-09T06:08:23.653026+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":224,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.652931+0000","last_change":"2023-10-09T05:56:00.348771+0000","last_active":"2023-10-09T06:08:23.652931+0000","last_peered":"2023-10-09T06:08:23.652931+0000","last_clean":"2023-10-09T06:08:23.652931+0000","last_became_active":"2023-10-09T05:56:00.224347+0000","last_became_peered":"2023-10-09T05:56:00.224347+0000","last_unstale":"2023-10-09T06:08:23.652931+0000","last_undegraded":"2023-10-09T06:08:23.652931+0000","last_fullsized":"2023-10-09T06:08:23.652931+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":401,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:05.399735+0000","last_change":"2023-10-09T06:05:15.568750+0000","last_active":"2023-10-09T06:08:05.399735+0000","last_peered":"2023-10-09T06:08:05.399735+0000","last_clean":"2023-10-09T06:08:05.399735+0000","last_became_active":"2023-10-09T06:05:15.568039+0000","last_became_peered":"2023-10-09T06:05:15.568039+0000","last_unstale":"2023-10-09T06:08:05.399735+0000","last_undegraded":"2023-10-09T06:08:05.399735+0000","last_fullsized":"2023-10-09T06:08:05.399735+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:38.828502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":284,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653075+0000","last_change":"2023-10-09T06:07:24.905311+0000","last_active":"2023-10-09T06:08:23.653075+0000","last_peered":"2023-10-09T06:08:23.653075+0000","last_clean":"2023-10-09T06:08:23.653075+0000","last_became_active":"2023-10-09T06:07:24.576154+0000","last_became_peered":"2023-10-09T06:07:24.576154+0000","last_unstale":"2023-10-09T06:08:23.653075+0000","last_undegraded":"2023-10-09T06:08:23.653075+0000","last_fullsized":"2023-10-09T06:08:23.653075+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":606,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.804595+0000","last_change":"2023-10-09T06:05:44.525598+0000","last_active":"2023-10-09T06:08:03.804595+0000","last_peered":"2023-10-09T06:08:03.804595+0000","last_clean":"2023-10-09T06:08:03.804595+0000","last_became_active":"2023-10-09T06:05:44.524943+0000","last_became_peered":"2023-10-09T06:05:44.524943+0000","last_unstale":"2023-10-09T06:08:03.804595+0000","last_undegraded":"2023-10-09T06:08:03.804595+0000","last_fullsized":"2023-10-09T06:08:03.804595+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:27.889121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":256,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.652837+0000","last_change":"2023-10-09T06:07:24.732171+0000","last_active":"2023-10-09T06:08:23.652837+0000","last_peered":"2023-10-09T06:08:23.652837+0000","last_clean":"2023-10-09T06:08:23.652837+0000","last_became_active":"2023-10-09T06:07:24.576067+0000","last_became_peered":"2023-10-09T06:07:24.576067+0000","last_unstale":"2023-10-09T06:08:23.652837+0000","last_undegraded":"2023-10-09T06:08:23.652837+0000","last_fullsized":"2023-10-09T06:08:23.652837+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.804293+0000","last_change":"2023-10-09T06:07:47.252267+0000","last_active":"2023-10-09T06:08:03.804293+0000","last_peered":"2023-10-09T06:08:03.804293+0000","last_clean":"2023-10-09T06:08:03.804293+0000","last_became_active":"2023-10-09T06:07:47.251957+0000","last_became_peered":"2023-10-09T06:07:47.251957+0000","last_unstale":"2023-10-09T06:08:03.804293+0000","last_undegraded":"2023-10-09T06:08:03.804293+0000","last_fullsized":"2023-10-09T06:08:03.804293+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:03.068994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":642,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.652723+0000","last_change":"2023-10-09T06:07:50.491749+0000","last_active":"2023-10-09T06:08:23.652723+0000","last_peered":"2023-10-09T06:08:23.652723+0000","last_clean":"2023-10-09T06:08:23.652723+0000","last_became_active":"2023-10-09T06:07:50.491494+0000","last_became_peered":"2023-10-09T06:07:50.491494+0000","last_unstale":"2023-10-09T06:08:23.652723+0000","last_undegraded":"2023-10-09T06:08:23.652723+0000","last_fullsized":"2023-10-09T06:08:23.652723+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":310,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653204+0000","last_change":"2023-10-09T06:04:32.952030+0000","last_active":"2023-10-09T06:08:23.653204+0000","last_peered":"2023-10-09T06:08:23.653204+0000","last_clean":"2023-10-09T06:08:23.653204+0000","last_became_active":"2023-10-09T06:04:32.951603+0000","last_became_peered":"2023-10-09T06:04:32.951603+0000","last_unstale":"2023-10-09T06:08:23.653204+0000","last_undegraded":"2023-10-09T06:08:23.653204+0000","last_fullsized":"2023-10-09T06:08:23.653204+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":240,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.652749+0000","last_change":"2023-10-09T06:04:32.956628+0000","last_active":"2023-10-09T06:08:23.652749+0000","last_peered":"2023-10-09T06:08:23.652749+0000","last_clean":"2023-10-09T06:08:23.652749+0000","last_became_active":"2023-10-09T06:04:32.956260+0000","last_became_peered":"2023-10-09T06:04:32.956260+0000","last_unstale":"2023-10-09T06:08:23.652749+0000","last_undegraded":"2023-10-09T06:08:23.652749+0000","last_fullsized":"2023-10-09T06:08:23.652749+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":612,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653036+0000","last_change":"2023-10-09T06:07:50.490355+0000","last_active":"2023-10-09T06:08:23.653036+0000","last_peered":"2023-10-09T06:08:23.653036+0000","last_clean":"2023-10-09T06:08:23.653036+0000","last_became_active":"2023-10-09T06:07:50.490061+0000","last_became_peered":"2023-10-09T06:07:50.490061+0000","last_unstale":"2023-10-09T06:08:23.653036+0000","last_undegraded":"2023-10-09T06:08:23.653036+0000","last_fullsized":"2023-10-09T06:08:23.653036+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":31,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.111501+0000","last_change":"2023-10-09T06:07:50.508117+0000","last_active":"2023-10-09T06:08:04.111501+0000","last_peered":"2023-10-09T06:08:04.111501+0000","last_clean":"2023-10-09T06:08:04.111501+0000","last_became_active":"2023-10-09T06:07:50.494658+0000","last_became_peered":"2023-10-09T06:07:50.494658+0000","last_unstale":"2023-10-09T06:08:04.111501+0000","last_undegraded":"2023-10-09T06:08:04.111501+0000","last_fullsized":"2023-10-09T06:08:04.111501+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:24.081528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":235,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653160+0000","last_change":"2023-10-09T06:03:09.369599+0000","last_active":"2023-10-09T06:08:23.653160+0000","last_peered":"2023-10-09T06:08:23.653160+0000","last_clean":"2023-10-09T06:08:23.653160+0000","last_became_active":"2023-10-09T06:03:09.369274+0000","last_became_peered":"2023-10-09T06:03:09.369274+0000","last_unstale":"2023-10-09T06:08:23.653160+0000","last_undegraded":"2023-10-09T06:08:23.653160+0000","last_fullsized":"2023-10-09T06:08:23.653160+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":272,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653185+0000","last_change":"2023-10-09T06:07:50.541376+0000","last_active":"2023-10-09T06:08:23.653185+0000","last_peered":"2023-10-09T06:08:23.653185+0000","last_clean":"2023-10-09T06:08:23.653185+0000","last_became_active":"2023-10-09T06:07:50.489742+0000","last_became_peered":"2023-10-09T06:07:50.489742+0000","last_unstale":"2023-10-09T06:08:23.653185+0000","last_undegraded":"2023-10-09T06:08:23.653185+0000","last_fullsized":"2023-10-09T06:08:23.653185+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":279,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.652610+0000","last_change":"2023-10-09T06:04:32.969529+0000","last_active":"2023-10-09T06:08:23.652610+0000","last_peered":"2023-10-09T06:08:23.652610+0000","last_clean":"2023-10-09T06:08:23.652610+0000","last_became_active":"2023-10-09T06:04:32.969220+0000","last_became_peered":"2023-10-09T06:04:32.969220+0000","last_unstale":"2023-10-09T06:08:23.652610+0000","last_undegraded":"2023-10-09T06:08:23.652610+0000","last_fullsized":"2023-10-09T06:08:23.652610+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":230,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653148+0000","last_change":"2023-10-09T06:07:24.631948+0000","last_active":"2023-10-09T06:08:23.653148+0000","last_peered":"2023-10-09T06:08:23.653148+0000","last_clean":"2023-10-09T06:08:23.653148+0000","last_became_active":"2023-10-09T06:07:24.564922+0000","last_became_peered":"2023-10-09T06:07:24.564922+0000","last_unstale":"2023-10-09T06:08:23.653148+0000","last_undegraded":"2023-10-09T06:08:23.653148+0000","last_fullsized":"2023-10-09T06:08:23.653148+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":24,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.652649+0000","last_change":"2023-10-09T06:07:47.252223+0000","last_active":"2023-10-09T06:08:23.652649+0000","last_peered":"2023-10-09T06:08:23.652649+0000","last_clean":"2023-10-09T06:08:23.652649+0000","last_became_active":"2023-10-09T06:07:47.251804+0000","last_became_peered":"2023-10-09T06:07:47.251804+0000","last_unstale":"2023-10-09T06:08:23.652649+0000","last_undegraded":"2023-10-09T06:08:23.652649+0000","last_fullsized":"2023-10-09T06:08:23.652649+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":241,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653232+0000","last_change":"2023-10-09T06:04:32.951565+0000","last_active":"2023-10-09T06:08:23.653232+0000","last_peered":"2023-10-09T06:08:23.653232+0000","last_clean":"2023-10-09T06:08:23.653232+0000","last_became_active":"2023-10-09T06:04:32.951221+0000","last_became_peered":"2023-10-09T06:04:32.951221+0000","last_unstale":"2023-10-09T06:08:23.653232+0000","last_undegraded":"2023-10-09T06:08:23.653232+0000","last_fullsized":"2023-10-09T06:08:23.653232+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":357,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:05.399948+0000","last_change":"2023-10-09T06:05:15.553522+0000","last_active":"2023-10-09T06:08:05.399948+0000","last_peered":"2023-10-09T06:08:05.399948+0000","last_clean":"2023-10-09T06:08:05.399948+0000","last_became_active":"2023-10-09T06:05:15.553151+0000","last_became_peered":"2023-10-09T06:05:15.553151+0000","last_unstale":"2023-10-09T06:08:05.399948+0000","last_undegraded":"2023-10-09T06:08:05.399948+0000","last_fullsized":"2023-10-09T06:08:05.399948+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:41.893526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":526,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:05.399937+0000","last_change":"2023-10-09T06:05:15.589640+0000","last_active":"2023-10-09T06:08:05.399937+0000","last_peered":"2023-10-09T06:08:05.399937+0000","last_clean":"2023-10-09T06:08:05.399937+0000","last_became_active":"2023-10-09T06:05:15.589350+0000","last_became_peered":"2023-10-09T06:05:15.589350+0000","last_unstale":"2023-10-09T06:08:05.399937+0000","last_undegraded":"2023-10-09T06:08:05.399937+0000","last_fullsized":"2023-10-09T06:08:05.399937+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:31.476101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":524,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.111740+0000","last_change":"2023-10-09T06:00:05.352723+0000","last_active":"2023-10-09T06:08:04.111740+0000","last_peered":"2023-10-09T06:08:04.111740+0000","last_clean":"2023-10-09T06:08:04.111740+0000","last_became_active":"2023-10-09T06:00:05.352008+0000","last_became_peered":"2023-10-09T06:00:05.352008+0000","last_unstale":"2023-10-09T06:08:04.111740+0000","last_undegraded":"2023-10-09T06:08:04.111740+0000","last_fullsized":"2023-10-09T06:08:04.111740+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:25.302918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":454,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.972607+0000","last_change":"2023-10-09T06:07:50.544943+0000","last_active":"2023-10-09T06:08:02.972607+0000","last_peered":"2023-10-09T06:08:02.972607+0000","last_clean":"2023-10-09T06:08:02.972607+0000","last_became_active":"2023-10-09T06:07:50.489809+0000","last_became_peered":"2023-10-09T06:07:50.489809+0000","last_unstale":"2023-10-09T06:08:02.972607+0000","last_undegraded":"2023-10-09T06:08:02.972607+0000","last_fullsized":"2023-10-09T06:08:02.972607+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:32.160515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":603,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.380026+0000","last_change":"2023-10-09T06:04:32.947237+0000","last_active":"2023-10-09T06:08:03.380026+0000","last_peered":"2023-10-09T06:08:03.380026+0000","last_clean":"2023-10-09T06:08:03.380026+0000","last_became_active":"2023-10-09T06:04:32.946963+0000","last_became_peered":"2023-10-09T06:04:32.946963+0000","last_unstale":"2023-10-09T06:08:03.380026+0000","last_undegraded":"2023-10-09T06:08:03.380026+0000","last_fullsized":"2023-10-09T06:08:03.380026+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":118,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.977920+0000","last_change":"2023-10-09T06:04:32.950583+0000","last_active":"2023-10-09T06:08:02.977920+0000","last_peered":"2023-10-09T06:08:02.977920+0000","last_clean":"2023-10-09T06:08:02.977920+0000","last_became_active":"2023-10-09T06:04:32.950108+0000","last_became_peered":"2023-10-09T06:04:32.950108+0000","last_unstale":"2023-10-09T06:08:02.977920+0000","last_undegraded":"2023-10-09T06:08:02.977920+0000","last_fullsized":"2023-10-09T06:08:02.977920+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":561,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.804947+0000","last_change":"2023-10-09T06:07:24.555315+0000","last_active":"2023-10-09T06:08:03.804947+0000","last_peered":"2023-10-09T06:08:03.804947+0000","last_clean":"2023-10-09T06:08:03.804947+0000","last_became_active":"2023-10-09T06:07:24.554902+0000","last_became_peered":"2023-10-09T06:07:24.554902+0000","last_unstale":"2023-10-09T06:08:03.804947+0000","last_undegraded":"2023-10-09T06:08:03.804947+0000","last_fullsized":"2023-10-09T06:08:03.804947+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:33.021735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":619,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.977548+0000","last_change":"2023-10-09T06:07:24.592232+0000","last_active":"2023-10-09T06:08:04.977548+0000","last_peered":"2023-10-09T06:08:04.977548+0000","last_clean":"2023-10-09T06:08:04.977548+0000","last_became_active":"2023-10-09T06:07:24.591849+0000","last_became_peered":"2023-10-09T06:07:24.591849+0000","last_unstale":"2023-10-09T06:08:04.977548+0000","last_undegraded":"2023-10-09T06:08:04.977548+0000","last_fullsized":"2023-10-09T06:08:04.977548+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:56.410869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":575,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:05.400130+0000","last_change":"2023-10-09T06:05:15.553450+0000","last_active":"2023-10-09T06:08:05.400130+0000","last_peered":"2023-10-09T06:08:05.400130+0000","last_clean":"2023-10-09T06:08:05.400130+0000","last_became_active":"2023-10-09T06:05:15.553129+0000","last_became_peered":"2023-10-09T06:05:15.553129+0000","last_unstale":"2023-10-09T06:08:05.400130+0000","last_undegraded":"2023-10-09T06:08:05.400130+0000","last_fullsized":"2023-10-09T06:08:05.400130+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:21.096565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":370,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:05.399800+0000","last_change":"2023-10-09T06:05:15.576694+0000","last_active":"2023-10-09T06:08:05.399800+0000","last_peered":"2023-10-09T06:08:05.399800+0000","last_clean":"2023-10-09T06:08:05.399800+0000","last_became_active":"2023-10-09T06:05:15.576437+0000","last_became_peered":"2023-10-09T06:05:15.576437+0000","last_unstale":"2023-10-09T06:08:05.399800+0000","last_undegraded":"2023-10-09T06:08:05.399800+0000","last_fullsized":"2023-10-09T06:08:05.399800+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:29.917899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":291,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:05.400287+0000","last_change":"2023-10-09T06:07:47.057804+0000","last_active":"2023-10-09T06:08:05.400287+0000","last_peered":"2023-10-09T06:08:05.400287+0000","last_clean":"2023-10-09T06:08:05.400287+0000","last_became_active":"2023-10-09T06:07:47.057508+0000","last_became_peered":"2023-10-09T06:07:47.057508+0000","last_unstale":"2023-10-09T06:08:05.400287+0000","last_undegraded":"2023-10-09T06:08:05.400287+0000","last_fullsized":"2023-10-09T06:08:05.400287+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:45.041480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"592'184","reported_seq":1120,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:08:24.663371+0000","last_change":"2023-10-09T06:02:08.341435+0000","last_active":"2023-10-09T06:08:24.663371+0000","last_peered":"2023-10-09T06:08:24.663371+0000","last_clean":"2023-10-09T06:08:24.663371+0000","last_became_active":"2023-10-09T06:02:08.340863+0000","last_became_peered":"2023-10-09T06:02:08.340863+0000","last_unstale":"2023-10-09T06:08:24.663371+0000","last_undegraded":"2023-10-09T06:08:24.663371+0000","last_fullsized":"2023-10-09T06:08:24.663371+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":184,"log_dups_size":0,"ondisk_log_size":184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":973,"num_read_kb":1914,"num_write":167,"num_write_kb":591,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":593,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.973279+0000","last_change":"2023-10-09T06:05:30.448881+0000","last_active":"2023-10-09T06:08:02.973279+0000","last_peered":"2023-10-09T06:08:02.973279+0000","last_clean":"2023-10-09T06:08:02.973279+0000","last_became_active":"2023-10-09T06:05:30.448534+0000","last_became_peered":"2023-10-09T06:05:30.448534+0000","last_unstale":"2023-10-09T06:08:02.973279+0000","last_undegraded":"2023-10-09T06:08:02.973279+0000","last_fullsized":"2023-10-09T06:08:02.973279+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:10.307670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":140,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:05.399711+0000","last_change":"2023-10-09T06:07:24.553776+0000","last_active":"2023-10-09T06:08:05.399711+0000","last_peered":"2023-10-09T06:08:05.399711+0000","last_clean":"2023-10-09T06:08:05.399711+0000","last_became_active":"2023-10-09T06:07:24.553439+0000","last_became_peered":"2023-10-09T06:07:24.553439+0000","last_unstale":"2023-10-09T06:08:05.399711+0000","last_undegraded":"2023-10-09T06:08:05.399711+0000","last_fullsized":"2023-10-09T06:08:05.399711+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:03.005009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"590'89","reported_seq":997,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.875959+0000","last_change":"2023-10-09T06:04:32.958038+0000","last_active":"2023-10-09T06:08:23.875959+0000","last_peered":"2023-10-09T06:08:23.875959+0000","last_clean":"2023-10-09T06:08:23.875959+0000","last_became_active":"2023-10-09T06:04:32.957507+0000","last_became_peered":"2023-10-09T06:04:32.957507+0000","last_unstale":"2023-10-09T06:08:23.875959+0000","last_undegraded":"2023-10-09T06:08:23.875959+0000","last_fullsized":"2023-10-09T06:08:23.875959+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:33.323626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":808,"num_read_kb":2333,"num_write":95,"num_write_kb":1386,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":534,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.380791+0000","last_change":"2023-10-09T06:04:32.951743+0000","last_active":"2023-10-09T06:08:03.380791+0000","last_peered":"2023-10-09T06:08:03.380791+0000","last_clean":"2023-10-09T06:08:03.380791+0000","last_became_active":"2023-10-09T06:04:32.951415+0000","last_became_peered":"2023-10-09T06:04:32.951415+0000","last_unstale":"2023-10-09T06:08:03.380791+0000","last_undegraded":"2023-10-09T06:08:03.380791+0000","last_fullsized":"2023-10-09T06:08:03.380791+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":461,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.806223+0000","last_change":"2023-10-09T06:05:44.522337+0000","last_active":"2023-10-09T06:08:03.806223+0000","last_peered":"2023-10-09T06:08:03.806223+0000","last_clean":"2023-10-09T06:08:03.806223+0000","last_became_active":"2023-10-09T06:05:44.522025+0000","last_became_peered":"2023-10-09T06:05:44.522025+0000","last_unstale":"2023-10-09T06:08:03.806223+0000","last_undegraded":"2023-10-09T06:08:03.806223+0000","last_fullsized":"2023-10-09T06:08:03.806223+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:52.064736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"590'95","reported_seq":977,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:14.419700+0000","last_change":"2023-10-09T06:05:44.560082+0000","last_active":"2023-10-09T06:08:14.419700+0000","last_peered":"2023-10-09T06:08:14.419700+0000","last_clean":"2023-10-09T06:08:14.419700+0000","last_became_active":"2023-10-09T06:05:44.534078+0000","last_became_peered":"2023-10-09T06:05:44.534078+0000","last_unstale":"2023-10-09T06:08:14.419700+0000","last_undegraded":"2023-10-09T06:08:14.419700+0000","last_fullsized":"2023-10-09T06:08:14.419700+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":95,"log_dups_size":0,"ondisk_log_size":95,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:45.519427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":1585,"num_write":90,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":413,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:05.399745+0000","last_change":"2023-10-09T06:05:30.447496+0000","last_active":"2023-10-09T06:08:05.399745+0000","last_peered":"2023-10-09T06:08:05.399745+0000","last_clean":"2023-10-09T06:08:05.399745+0000","last_became_active":"2023-10-09T06:05:30.447219+0000","last_became_peered":"2023-10-09T06:05:30.447219+0000","last_unstale":"2023-10-09T06:08:05.399745+0000","last_undegraded":"2023-10-09T06:08:05.399745+0000","last_fullsized":"2023-10-09T06:08:05.399745+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:08.869779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:19.354714+0000","last_change":"2023-10-09T06:07:50.496727+0000","last_active":"2023-10-09T06:08:19.354714+0000","last_peered":"2023-10-09T06:08:19.354714+0000","last_clean":"2023-10-09T06:08:19.354714+0000","last_became_active":"2023-10-09T06:07:50.496503+0000","last_became_peered":"2023-10-09T06:07:50.496503+0000","last_unstale":"2023-10-09T06:08:19.354714+0000","last_undegraded":"2023-10-09T06:08:19.354714+0000","last_fullsized":"2023-10-09T06:08:19.354714+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:17.224957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"590'189","reported_seq":856,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:15.317544+0000","last_change":"2023-10-09T06:05:44.537575+0000","last_active":"2023-10-09T06:08:15.317544+0000","last_peered":"2023-10-09T06:08:15.317544+0000","last_clean":"2023-10-09T06:08:15.317544+0000","last_became_active":"2023-10-09T06:05:44.533477+0000","last_became_peered":"2023-10-09T06:05:44.533477+0000","last_unstale":"2023-10-09T06:08:15.317544+0000","last_undegraded":"2023-10-09T06:08:15.317544+0000","last_fullsized":"2023-10-09T06:08:15.317544+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":189,"log_dups_size":0,"ondisk_log_size":189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:44.900250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":900,"num_read_kb":815,"num_write":161,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":813,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.652651+0000","last_change":"2023-10-09T06:07:53.650105+0000","last_active":"2023-10-09T06:08:23.652651+0000","last_peered":"2023-10-09T06:08:23.652651+0000","last_clean":"2023-10-09T06:08:23.652651+0000","last_became_active":"2023-10-09T06:07:53.649614+0000","last_became_peered":"2023-10-09T06:07:53.649614+0000","last_unstale":"2023-10-09T06:08:23.652651+0000","last_undegraded":"2023-10-09T06:08:23.652651+0000","last_fullsized":"2023-10-09T06:08:23.652651+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":487,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:02.973859+0000","last_change":"2023-10-09T06:04:32.942556+0000","last_active":"2023-10-09T06:08:02.973859+0000","last_peered":"2023-10-09T06:08:02.973859+0000","last_clean":"2023-10-09T06:08:02.973859+0000","last_became_active":"2023-10-09T06:04:32.942269+0000","last_became_peered":"2023-10-09T06:04:32.942269+0000","last_unstale":"2023-10-09T06:08:02.973859+0000","last_undegraded":"2023-10-09T06:08:02.973859+0000","last_fullsized":"2023-10-09T06:08:02.973859+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:53.383296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":260,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.652906+0000","last_change":"2023-10-09T06:07:24.596895+0000","last_active":"2023-10-09T06:08:23.652906+0000","last_peered":"2023-10-09T06:08:23.652906+0000","last_clean":"2023-10-09T06:08:23.652906+0000","last_became_active":"2023-10-09T06:07:24.596620+0000","last_became_peered":"2023-10-09T06:07:24.596620+0000","last_unstale":"2023-10-09T06:08:23.652906+0000","last_undegraded":"2023-10-09T06:08:23.652906+0000","last_fullsized":"2023-10-09T06:08:23.652906+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"591'167","reported_seq":1335,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.815890+0000","last_change":"2023-10-09T06:03:12.481829+0000","last_active":"2023-10-09T06:08:23.815890+0000","last_peered":"2023-10-09T06:08:23.815890+0000","last_clean":"2023-10-09T06:08:23.815890+0000","last_became_active":"2023-10-09T06:03:12.479898+0000","last_became_peered":"2023-10-09T06:03:12.479898+0000","last_unstale":"2023-10-09T06:08:23.815890+0000","last_undegraded":"2023-10-09T06:08:23.815890+0000","last_fullsized":"2023-10-09T06:08:23.815890+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1433,"num_read_kb":3693,"num_write":118,"num_write_kb":2284,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":579,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.110689+0000","last_change":"2023-10-09T06:07:24.555346+0000","last_active":"2023-10-09T06:08:04.110689+0000","last_peered":"2023-10-09T06:08:04.110689+0000","last_clean":"2023-10-09T06:08:04.110689+0000","last_became_active":"2023-10-09T06:07:24.554992+0000","last_became_peered":"2023-10-09T06:07:24.554992+0000","last_unstale":"2023-10-09T06:08:04.110689+0000","last_undegraded":"2023-10-09T06:08:04.110689+0000","last_fullsized":"2023-10-09T06:08:04.110689+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:43.884493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":21,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.110543+0000","last_change":"2023-10-09T06:07:50.496088+0000","last_active":"2023-10-09T06:08:04.110543+0000","last_peered":"2023-10-09T06:08:04.110543+0000","last_clean":"2023-10-09T06:08:04.110543+0000","last_became_active":"2023-10-09T06:07:50.495249+0000","last_became_peered":"2023-10-09T06:07:50.495249+0000","last_unstale":"2023-10-09T06:08:04.110543+0000","last_undegraded":"2023-10-09T06:08:04.110543+0000","last_fullsized":"2023-10-09T06:08:04.110543+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:52.044105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"590'144","reported_seq":982,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653298+0000","last_change":"2023-10-09T06:07:53.651882+0000","last_active":"2023-10-09T06:08:23.653298+0000","last_peered":"2023-10-09T06:08:23.653298+0000","last_clean":"2023-10-09T06:08:23.653298+0000","last_became_active":"2023-10-09T06:07:53.651256+0000","last_became_peered":"2023-10-09T06:07:53.651256+0000","last_unstale":"2023-10-09T06:08:23.653298+0000","last_undegraded":"2023-10-09T06:08:23.653298+0000","last_fullsized":"2023-10-09T06:08:23.653298+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":602,"num_read_kb":2854,"num_write":144,"num_write_kb":3305,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":465,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.806332+0000","last_change":"2023-10-09T06:07:50.491120+0000","last_active":"2023-10-09T06:08:03.806332+0000","last_peered":"2023-10-09T06:08:03.806332+0000","last_clean":"2023-10-09T06:08:03.806332+0000","last_became_active":"2023-10-09T06:07:50.490772+0000","last_became_peered":"2023-10-09T06:07:50.490772+0000","last_unstale":"2023-10-09T06:08:03.806332+0000","last_undegraded":"2023-10-09T06:08:03.806332+0000","last_fullsized":"2023-10-09T06:08:03.806332+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:31.534859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":530,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.652985+0000","last_change":"2023-10-09T06:02:08.340506+0000","last_active":"2023-10-09T06:08:23.652985+0000","last_peered":"2023-10-09T06:08:23.652985+0000","last_clean":"2023-10-09T06:08:23.652985+0000","last_became_active":"2023-10-09T06:02:08.340152+0000","last_became_peered":"2023-10-09T06:02:08.340152+0000","last_unstale":"2023-10-09T06:08:23.652985+0000","last_undegraded":"2023-10-09T06:08:23.652985+0000","last_fullsized":"2023-10-09T06:08:23.652985+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":258,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.653266+0000","last_change":"2023-10-09T06:00:05.329139+0000","last_active":"2023-10-09T06:08:23.653266+0000","last_peered":"2023-10-09T06:08:23.653266+0000","last_clean":"2023-10-09T06:08:23.653266+0000","last_became_active":"2023-10-09T06:00:05.328811+0000","last_became_peered":"2023-10-09T06:00:05.328811+0000","last_unstale":"2023-10-09T06:08:23.653266+0000","last_undegraded":"2023-10-09T06:08:23.653266+0000","last_fullsized":"2023-10-09T06:08:23.653266+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"591'158","reported_seq":653,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.878547+0000","last_change":"2023-10-09T06:04:32.959605+0000","last_active":"2023-10-09T06:08:23.878547+0000","last_peered":"2023-10-09T06:08:23.878547+0000","last_clean":"2023-10-09T06:08:23.878547+0000","last_became_active":"2023-10-09T06:04:32.959073+0000","last_became_peered":"2023-10-09T06:04:32.959073+0000","last_unstale":"2023-10-09T06:08:23.878547+0000","last_undegraded":"2023-10-09T06:08:23.878547+0000","last_fullsized":"2023-10-09T06:08:23.878547+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":158,"log_dups_size":0,"ondisk_log_size":158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:00.843837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":721,"num_read_kb":1637,"num_write":120,"num_write_kb":590,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":280,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.652548+0000","last_change":"2023-10-09T06:07:50.490552+0000","last_active":"2023-10-09T06:08:23.652548+0000","last_peered":"2023-10-09T06:08:23.652548+0000","last_clean":"2023-10-09T06:08:23.652548+0000","last_became_active":"2023-10-09T06:07:50.490268+0000","last_became_peered":"2023-10-09T06:07:50.490268+0000","last_unstale":"2023-10-09T06:08:23.652548+0000","last_undegraded":"2023-10-09T06:08:23.652548+0000","last_fullsized":"2023-10-09T06:08:23.652548+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"591'120","reported_seq":804,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:08:23.885811+0000","last_change":"2023-10-09T06:04:32.961494+0000","last_active":"2023-10-09T06:08:23.885811+0000","last_peered":"2023-10-09T06:08:23.885811+0000","last_clean":"2023-10-09T06:08:23.885811+0000","last_became_active":"2023-10-09T06:04:32.961137+0000","last_became_peered":"2023-10-09T06:04:32.961137+0000","last_unstale":"2023-10-09T06:08:23.885811+0000","last_undegraded":"2023-10-09T06:08:23.885811+0000","last_fullsized":"2023-10-09T06:08:23.885811+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4419584,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":933,"num_read_kb":1102,"num_write":104,"num_write_kb":4324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":25,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.110459+0000","last_change":"2023-10-09T06:07:50.543307+0000","last_active":"2023-10-09T06:08:04.110459+0000","last_peered":"2023-10-09T06:08:04.110459+0000","last_clean":"2023-10-09T06:08:04.110459+0000","last_became_active":"2023-10-09T06:07:50.495736+0000","last_became_peered":"2023-10-09T06:07:50.495736+0000","last_unstale":"2023-10-09T06:08:04.110459+0000","last_undegraded":"2023-10-09T06:08:04.110459+0000","last_fullsized":"2023-10-09T06:08:04.110459+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:47.121228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":170,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.110501+0000","last_change":"2023-10-09T06:07:24.557878+0000","last_active":"2023-10-09T06:08:04.110501+0000","last_peered":"2023-10-09T06:08:04.110501+0000","last_clean":"2023-10-09T06:08:04.110501+0000","last_became_active":"2023-10-09T06:07:24.557528+0000","last_became_peered":"2023-10-09T06:07:24.557528+0000","last_unstale":"2023-10-09T06:08:04.110501+0000","last_undegraded":"2023-10-09T06:08:04.110501+0000","last_fullsized":"2023-10-09T06:08:04.110501+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:57.735693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":21,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.112450+0000","last_change":"2023-10-09T06:07:50.489321+0000","last_active":"2023-10-09T06:08:04.112450+0000","last_peered":"2023-10-09T06:08:04.112450+0000","last_clean":"2023-10-09T06:08:04.112450+0000","last_became_active":"2023-10-09T06:07:50.488938+0000","last_became_peered":"2023-10-09T06:07:50.488938+0000","last_unstale":"2023-10-09T06:08:04.112450+0000","last_undegraded":"2023-10-09T06:08:04.112450+0000","last_fullsized":"2023-10-09T06:08:04.112450+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:00.393060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":294,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.804192+0000","last_change":"2023-10-09T06:07:47.252116+0000","last_active":"2023-10-09T06:08:03.804192+0000","last_peered":"2023-10-09T06:08:03.804192+0000","last_clean":"2023-10-09T06:08:03.804192+0000","last_became_active":"2023-10-09T06:07:47.251804+0000","last_became_peered":"2023-10-09T06:07:47.251804+0000","last_unstale":"2023-10-09T06:08:03.804192+0000","last_undegraded":"2023-10-09T06:08:03.804192+0000","last_fullsized":"2023-10-09T06:08:03.804192+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:02.563612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":150,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.380795+0000","last_change":"2023-10-09T06:04:32.957514+0000","last_active":"2023-10-09T06:08:03.380795+0000","last_peered":"2023-10-09T06:08:03.380795+0000","last_clean":"2023-10-09T06:08:03.380795+0000","last_became_active":"2023-10-09T06:04:32.957114+0000","last_became_peered":"2023-10-09T06:04:32.957114+0000","last_unstale":"2023-10-09T06:08:03.380795+0000","last_undegraded":"2023-10-09T06:08:03.380795+0000","last_fullsized":"2023-10-09T06:08:03.380795+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.379435+0000","last_change":"2023-10-09T06:07:47.252501+0000","last_active":"2023-10-09T06:08:03.379435+0000","last_peered":"2023-10-09T06:08:03.379435+0000","last_clean":"2023-10-09T06:08:03.379435+0000","last_became_active":"2023-10-09T06:07:47.251841+0000","last_became_peered":"2023-10-09T06:07:47.251841+0000","last_unstale":"2023-10-09T06:08:03.379435+0000","last_undegraded":"2023-10-09T06:08:03.379435+0000","last_fullsized":"2023-10-09T06:08:03.379435+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":150,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.380934+0000","last_change":"2023-10-09T06:04:32.957504+0000","last_active":"2023-10-09T06:08:03.380934+0000","last_peered":"2023-10-09T06:08:03.380934+0000","last_clean":"2023-10-09T06:08:03.380934+0000","last_became_active":"2023-10-09T06:04:32.957148+0000","last_became_peered":"2023-10-09T06:04:32.957148+0000","last_unstale":"2023-10-09T06:08:03.380934+0000","last_undegraded":"2023-10-09T06:08:03.380934+0000","last_fullsized":"2023-10-09T06:08:03.380934+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":31,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.379662+0000","last_change":"2023-10-09T06:07:24.555569+0000","last_active":"2023-10-09T06:08:03.379662+0000","last_peered":"2023-10-09T06:08:03.379662+0000","last_clean":"2023-10-09T06:08:03.379662+0000","last_became_active":"2023-10-09T06:07:24.555251+0000","last_became_peered":"2023-10-09T06:07:24.555251+0000","last_unstale":"2023-10-09T06:08:03.379662+0000","last_undegraded":"2023-10-09T06:08:03.379662+0000","last_fullsized":"2023-10-09T06:08:03.379662+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":371,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.805460+0000","last_change":"2023-10-09T06:05:44.522492+0000","last_active":"2023-10-09T06:08:03.805460+0000","last_peered":"2023-10-09T06:08:03.805460+0000","last_clean":"2023-10-09T06:08:03.805460+0000","last_became_active":"2023-10-09T06:05:44.522205+0000","last_became_peered":"2023-10-09T06:05:44.522205+0000","last_unstale":"2023-10-09T06:08:03.805460+0000","last_undegraded":"2023-10-09T06:08:03.805460+0000","last_fullsized":"2023-10-09T06:08:03.805460+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:58.840737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":456,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.977639+0000","last_change":"2023-10-09T06:05:44.527302+0000","last_active":"2023-10-09T06:08:04.977639+0000","last_peered":"2023-10-09T06:08:04.977639+0000","last_clean":"2023-10-09T06:08:04.977639+0000","last_became_active":"2023-10-09T06:05:44.526986+0000","last_became_peered":"2023-10-09T06:05:44.526986+0000","last_unstale":"2023-10-09T06:08:04.977639+0000","last_undegraded":"2023-10-09T06:08:04.977639+0000","last_fullsized":"2023-10-09T06:08:04.977639+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:13.582935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":27,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.111531+0000","last_change":"2023-10-09T06:07:50.500816+0000","last_active":"2023-10-09T06:08:04.111531+0000","last_peered":"2023-10-09T06:08:04.111531+0000","last_clean":"2023-10-09T06:08:04.111531+0000","last_became_active":"2023-10-09T06:07:50.494692+0000","last_became_peered":"2023-10-09T06:07:50.494692+0000","last_unstale":"2023-10-09T06:08:04.111531+0000","last_undegraded":"2023-10-09T06:08:04.111531+0000","last_fullsized":"2023-10-09T06:08:04.111531+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:44.716403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":582,"reported_epoch":589,"state":"active+clean","last_fresh":"2023-10-09T06:08:03.379592+0000","last_change":"2023-10-09T06:04:32.949651+0000","last_active":"2023-10-09T06:08:03.379592+0000","last_peered":"2023-10-09T06:08:03.379592+0000","last_clean":"2023-10-09T06:08:03.379592+0000","last_became_active":"2023-10-09T06:04:32.949288+0000","last_became_peered":"2023-10-09T06:04:32.949288+0000","last_unstale":"2023-10-09T06:08:03.379592+0000","last_undegraded":"2023-10-09T06:08:03.379592+0000","last_fullsized":"2023-10-09T06:08:03.379592+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":86,"num_bytes_recovered":27644,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":52606,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4419759,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":6901,"num_read_kb":15933,"num_write":999,"num_write_kb":13769,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12660736,"data_stored":12603744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":1146,"ondisk_log_size":1146,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460846992,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511040,"kb_used_data":10296,"kb_used_omap":14,"kb_used_meta":1500721,"kb_avail":92234112,"statfs":{"total":95995035648,"available":94447730688,"internally_reserved":0,"allocated":10543104,"data_stored":9337982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":1536739162},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":10.419,"5min":3.879,"15min":4.016},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":167.06200000000001,"5min":167.06200000000001,"15min":212.64400000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":9.9399999999999995,"5min":3.7709999999999999,"15min":4.1150000000000002},"min":{"1min":0.27300000000000002,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":167.077,"5min":167.077,"15min":212.655},"last":0.58299999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":10.201000000000001,"5min":6.5940000000000003,"15min":4.593},"min":{"1min":0.32000000000000001,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":169.98400000000001,"5min":169.98400000000001,"15min":176.90600000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":11.323,"5min":7.0979999999999999,"15min":4.7080000000000002},"min":{"1min":0.32200000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":172.821,"5min":172.821,"15min":176.86799999999999},"last":0.66800000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":11.234999999999999,"5min":9.0069999999999997,"15min":5.891},"min":{"1min":0.316,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":177.65100000000001,"5min":302.01799999999997,"15min":302.01799999999997},"last":0.59599999999999997},{"interface":"front","average":{"1min":10.667,"5min":8.6080000000000005,"15min":5.5919999999999996},"min":{"1min":0.374,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":175.762,"5min":304.988,"15min":304.988},"last":0.69599999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":11.079000000000001,"5min":4.5090000000000003,"15min":3.472},"min":{"1min":0.36899999999999999,"5min":0.34399999999999997,"15min":0.30499999999999999},"max":{"1min":167.64699999999999,"5min":167.64699999999999,"15min":176.958},"last":0.63800000000000001},{"interface":"front","average":{"1min":10.465,"5min":4.0590000000000002,"15min":3.4420000000000002},"min":{"1min":0.38300000000000001,"5min":0.32900000000000001,"15min":0.29699999999999999},"max":{"1min":169.52199999999999,"5min":169.52199999999999,"15min":176.91800000000001},"last":0.65700000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.747,"15min":1.855},"min":{"1min":0.311,"5min":0.252,"15min":0.252},"max":{"1min":3.7130000000000001,"5min":5.6340000000000003,"15min":178.995},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.79100000000000004,"15min":1.913},"min":{"1min":0.32700000000000001,"5min":0.246,"15min":0.246},"max":{"1min":4.9349999999999996,"5min":4.9349999999999996,"15min":178.899},"last":0.624}]},{"osd":5,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.70999999999999996,"15min":2.3050000000000002},"min":{"1min":0.35799999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.9089999999999998,"5min":3.1299999999999999,"15min":176.82400000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.755,"15min":2.2890000000000001},"min":{"1min":0.34399999999999997,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.1210000000000004,"5min":4.242,"15min":176.79499999999999},"last":0.495}]},{"osd":6,"last update":"Mon Oct 9 06:07:41 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.1120000000000001,"15min":1.127},"min":{"1min":0.33200000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":13.193,"5min":13.193,"15min":13.193},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.448,"5min":0.93899999999999995,"15min":0.79000000000000004},"min":{"1min":0.42599999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.247,"5min":13.247,"15min":13.247},"last":0.53200000000000003}]}]},{"osd":6,"up_from":537,"seq":2306397437999,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":381840,"kb_used_data":10248,"kb_used_omap":13,"kb_used_meta":371570,"kb_avail":93363312,"statfs":{"total":95995035648,"available":95604031488,"internally_reserved":0,"allocated":10493952,"data_stored":9309947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13650,"internal_metadata":380488366},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":8.0039999999999996,"5min":5.069,"15min":5.2469999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":126.914,"5min":126.914,"15min":126.914},"last":0.69899999999999995},{"interface":"front","average":{"1min":8.6780000000000008,"5min":5.4690000000000003,"15min":5.6150000000000002},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":126.925,"5min":126.925,"15min":126.925},"last":0.65900000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":8.5830000000000002,"5min":9.3740000000000006,"15min":11.186},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":130.28100000000001,"5min":130.28100000000001,"15min":130.28100000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":8.4410000000000007,"5min":9.2609999999999992,"15min":11.09},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":135.06100000000001,"5min":135.06100000000001,"15min":135.06100000000001},"last":0.503}]},{"osd":2,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":10.076000000000001,"5min":3.8130000000000002,"15min":2.8170000000000002},"min":{"1min":0.313,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":133.917,"5min":133.917,"15min":133.917},"last":0.56699999999999995},{"interface":"front","average":{"1min":9.9930000000000003,"5min":4.1109999999999998,"15min":3.2149999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":133.92699999999999,"5min":133.92699999999999,"15min":133.92699999999999},"last":0.626}]},{"osd":3,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":8.298,"5min":5.7149999999999999,"15min":5.9569999999999999},"min":{"1min":0.377,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":129.375,"5min":129.375,"15min":129.375},"last":0.58699999999999997},{"interface":"front","average":{"1min":8.0820000000000007,"5min":5.6749999999999998,"15min":5.9349999999999996},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":129.38900000000001,"5min":129.38900000000001,"15min":129.38900000000001},"last":0.63800000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.1100000000000001,"15min":1.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":11.103999999999999,"5min":11.103999999999999,"15min":11.103999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.92600000000000005,"15min":1.0600000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.1669999999999998,"5min":8.1229999999999993,"15min":8.1229999999999993},"last":0.61099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":0.94899999999999995,"15min":0.92400000000000004},"min":{"1min":0.314,"5min":0.313,"15min":0.313},"max":{"1min":11.218,"5min":11.218,"15min":11.218},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.878,"15min":0.89900000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":6.5389999999999997,"5min":6.5389999999999997,"15min":6.5389999999999997},"last":0.67000000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":4.2720000000000002,"15min":5.7990000000000004},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.4319999999999999,"5min":124.27200000000001,"15min":124.27200000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.4259999999999999,"5min":4.2039999999999997,"15min":5.5549999999999997},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":11.339,"5min":121.136,"15min":121.136},"last":0.59899999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958984,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1033368,"kb_used_data":6736,"kb_used_omap":7,"kb_used_meta":1026616,"kb_avail":92711784,"statfs":{"total":95995035648,"available":94936866816,"internally_reserved":0,"allocated":6897664,"data_stored":4653356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":1051255310},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":2.5430000000000001,"15min":2.024},"min":{"1min":0.253,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":7.3789999999999996,"5min":27.045999999999999,"15min":131.91300000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":2.7679999999999998,"5min":3.141,"15min":2.3490000000000002},"min":{"1min":0.30599999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.163,"5min":27.344000000000001,"15min":132.191},"last":0.52600000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:08:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":5.5289999999999999,"15min":3.077},"min":{"1min":0.23000000000000001,"5min":0.21299999999999999,"15min":0.20899999999999999},"max":{"1min":16.183,"5min":186.84200000000001,"15min":186.84200000000001},"last":0.502},{"interface":"front","average":{"1min":2.621,"5min":5.2910000000000004,"15min":3.081},"min":{"1min":0.26200000000000001,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":20.803999999999998,"5min":186.75800000000001,"15min":186.75800000000001},"last":0.47699999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":3.0800000000000001,"5min":4.9500000000000002,"15min":2.9700000000000002},"min":{"1min":0.26000000000000001,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":10.404,"5min":177.70599999999999,"15min":177.70599999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":3.4529999999999998,"5min":4.8470000000000004,"15min":2.915},"min":{"1min":0.36699999999999999,"5min":0.28299999999999997,"15min":0.24099999999999999},"max":{"1min":14.023,"5min":177.82400000000001,"15min":177.82400000000001},"last":0.55200000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":3.5379999999999998,"15min":2.4489999999999998},"min":{"1min":0.35199999999999998,"5min":0.29699999999999999,"15min":0.25800000000000001},"max":{"1min":10.129,"5min":86.483999999999995,"15min":131.636},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.3300000000000001,"5min":2.5099999999999998,"15min":2.0049999999999999},"min":{"1min":0.41199999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.4209999999999998,"5min":86.441999999999993,"15min":131.75999999999999},"last":0.753}]},{"osd":5,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.9279999999999999,"15min":2.2759999999999998},"min":{"1min":0.41299999999999998,"5min":0.314,"15min":0.313},"max":{"1min":8.6769999999999996,"5min":14.019,"15min":132.291},"last":0.628},{"interface":"front","average":{"1min":2.4100000000000001,"5min":3.5350000000000001,"15min":3.04},"min":{"1min":0.39300000000000002,"5min":0.34999999999999998,"15min":0.33700000000000002},"max":{"1min":10.063000000000001,"5min":99.769999999999996,"15min":131.738},"last":0.66500000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:07:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":3.5190000000000001,"15min":3.8149999999999999},"min":{"1min":0.374,"5min":0.372,"15min":0.372},"max":{"1min":13.205,"5min":13.205,"15min":13.205},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.7629999999999999,"5min":2.206,"15min":2.4020000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":7.3259999999999996,"5min":13.685,"15min":13.685},"last":0.65200000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":2.7610000000000001,"15min":2.4380000000000002},"min":{"1min":0.44800000000000001,"5min":0.379,"15min":0.31},"max":{"1min":6.6639999999999997,"5min":86.911000000000001,"15min":131.821},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.7290000000000001,"5min":2.8530000000000002,"15min":2.6030000000000002},"min":{"1min":0.52200000000000002,"5min":0.38400000000000001,"15min":0.35199999999999998},"max":{"1min":4.7300000000000004,"5min":86.814999999999998,"15min":131.852},"last":0.71899999999999997}]}]},{"osd":0,"up_from":88,"seq":377957122509,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1486104,"kb_used_data":484048,"kb_used_omap":8,"kb_used_meta":1002039,"kb_avail":92259048,"statfs":{"total":95995035648,"available":94473265152,"internally_reserved":0,"allocated":495665152,"data_stored":493354649,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":1026088712},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:07:34 2023","interfaces":[{"interface":"back","average":{"1min":3.8410000000000002,"5min":4.0999999999999996,"15min":3.548},"min":{"1min":0.215,"5min":0.19400000000000001,"15min":0.188},"max":{"1min":15.27,"5min":93.971999999999994,"15min":163.572},"last":0.53600000000000003},{"interface":"front","average":{"1min":5.1630000000000003,"5min":4.4640000000000004,"15min":3.8450000000000002},"min":{"1min":0.28499999999999998,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":16.222000000000001,"5min":94.007000000000005,"15min":162.48699999999999},"last":0.50900000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:08:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":3.383,"15min":1.778},"min":{"1min":0.252,"5min":0.22700000000000001,"15min":0.20599999999999999},"max":{"1min":6.7370000000000001,"5min":95.512,"15min":95.512},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.258,"5min":3.6600000000000001,"15min":1.7869999999999999},"min":{"1min":0.32600000000000001,"5min":0.254,"15min":0.221},"max":{"1min":5.6710000000000003,"5min":95.462999999999994,"15min":95.462999999999994},"last":0.64200000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:07:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1760000000000002,"5min":2.8679999999999999,"15min":1.8680000000000001},"min":{"1min":0.318,"5min":0.255,"15min":0.21299999999999999},"max":{"1min":12.247999999999999,"5min":12.304,"15min":62.698999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.7559999999999998,"5min":2.5790000000000002,"15min":1.9339999999999999},"min":{"1min":0.33100000000000002,"5min":0.29899999999999999,"15min":0.28199999999999997},"max":{"1min":8.5090000000000003,"5min":13.632,"15min":60.383000000000003},"last":0.48599999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:08:16 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.7230000000000001,"15min":1.145},"min":{"1min":0.28299999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.4870000000000001,"5min":13.592000000000001,"15min":13.592000000000001},"last":0.78100000000000003},{"interface":"front","average":{"1min":1.514,"5min":1.8360000000000001,"15min":1.339},"min":{"1min":0.45100000000000001,"5min":0.32200000000000001,"15min":0.29399999999999998},"max":{"1min":6.7919999999999998,"5min":13.775,"15min":13.775},"last":0.79500000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":2.0859999999999999,"15min":3.4169999999999998},"min":{"1min":0.442,"5min":0.34999999999999998,"15min":0.27900000000000003},"max":{"1min":6.8230000000000004,"5min":13.746,"15min":246.761},"last":0.625},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.9590000000000001,"15min":3.5139999999999998},"min":{"1min":0.40500000000000003,"5min":0.34499999999999997,"15min":0.30599999999999999},"max":{"1min":6.6699999999999999,"5min":11.52,"15min":247.48400000000001},"last":0.70099999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:07:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.9650000000000001,"15min":1.9670000000000001},"min":{"1min":0.39000000000000001,"5min":0.314,"15min":0.314},"max":{"1min":12.929,"5min":12.929,"15min":12.929},"last":0.86299999999999999},{"interface":"front","average":{"1min":3.2330000000000001,"5min":1.996,"15min":1.5920000000000001},"min":{"1min":0.40400000000000003,"5min":0.372,"15min":0.372},"max":{"1min":8.6780000000000008,"5min":13.608000000000001,"15min":13.608000000000001},"last":0.88600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:07:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":3.8479999999999999,"15min":3.238},"min":{"1min":0.41699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":12.987,"5min":168.98599999999999,"15min":168.98599999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":2.5049999999999999,"5min":3.9359999999999999,"15min":3.3450000000000002},"min":{"1min":0.47699999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.9909999999999997,"5min":169.03800000000001,"15min":169.03800000000001},"last":0.84999999999999998}]}]},{"osd":2,"up_from":98,"seq":420906795457,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1080924,"kb_used_data":11220,"kb_used_omap":11,"kb_used_meta":1069684,"kb_avail":92664228,"statfs":{"total":95995035648,"available":94888169472,"internally_reserved":0,"allocated":11489280,"data_stored":8879870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":1095357013},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":2.851,"15min":2.008},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19600000000000001},"max":{"1min":8.1560000000000006,"5min":27.292000000000002,"15min":140.71100000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.80700000000000005,"5min":2.6539999999999999,"15min":2.1960000000000002},"min":{"1min":0.30299999999999999,"5min":0.19700000000000001,"15min":0.19600000000000001},"max":{"1min":7.923,"5min":14.050000000000001,"15min":142.542},"last":0.71799999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":2.907,"15min":2.7160000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19},"max":{"1min":2.6659999999999999,"5min":26.869,"15min":133.42500000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.873,"5min":3.5339999999999998,"15min":3.1629999999999998},"min":{"1min":0.23100000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.5570000000000004,"5min":15.83,"15min":133.40799999999999},"last":0.60899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.016,"5min":5.181,"15min":2.9009999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.23300000000000001},"max":{"1min":6.8019999999999996,"5min":142.26300000000001,"15min":142.26300000000001},"last":0.753},{"interface":"front","average":{"1min":1.722,"5min":4.1760000000000002,"15min":2.4609999999999999},"min":{"1min":0.30399999999999999,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":8.266,"5min":141.477,"15min":141.477},"last":0.84999999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.524,"15min":1.5329999999999999},"min":{"1min":0.36099999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.5060000000000002,"5min":11.132999999999999,"15min":11.132999999999999},"last":0.88600000000000001},{"interface":"front","average":{"1min":1.111,"5min":1.778,"15min":1.149},"min":{"1min":0.35699999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.7649999999999997,"5min":16.457000000000001,"15min":16.457000000000001},"last":0.93100000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":2.101,"15min":2.7240000000000002},"min":{"1min":0.437,"5min":0.35499999999999998,"15min":0.33000000000000002},"max":{"1min":2.8399999999999999,"5min":10.852,"15min":207.80500000000001},"last":0.81200000000000006},{"interface":"front","average":{"1min":1.2450000000000001,"5min":2.7229999999999999,"15min":3.1200000000000001},"min":{"1min":0.40300000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.5419999999999998,"5min":15.781000000000001,"15min":207.54900000000001},"last":1.0269999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:07:43 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":2.1459999999999999,"15min":2.0289999999999999},"min":{"1min":0.47099999999999997,"5min":0.375,"15min":0.375},"max":{"1min":8.4589999999999996,"5min":16.422000000000001,"15min":16.422000000000001},"last":1.004},{"interface":"front","average":{"1min":2.2839999999999998,"5min":2.0659999999999998,"15min":1.905},"min":{"1min":0.41899999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":9.6210000000000004,"5min":10.801,"15min":10.801},"last":0.91400000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":2.0779999999999998,"15min":2.4590000000000001},"min":{"1min":0.46400000000000002,"5min":0.39900000000000002,"15min":0.34399999999999997},"max":{"1min":8.3640000000000008,"5min":16.382999999999999,"15min":162.09200000000001},"last":1.0529999999999999},{"interface":"front","average":{"1min":2.0099999999999998,"5min":2.3250000000000002,"15min":2.6629999999999998},"min":{"1min":0.45000000000000001,"5min":0.39600000000000002,"15min":0.36099999999999999},"max":{"1min":9.5990000000000002,"5min":10.890000000000001,"15min":162.35400000000001},"last":0.96599999999999997}]}]},{"osd":3,"up_from":103,"seq":442381631932,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1211232,"kb_used_data":6808,"kb_used_omap":4,"kb_used_meta":1204411,"kb_avail":92533920,"statfs":{"total":95995035648,"available":94754734080,"internally_reserved":0,"allocated":6971392,"data_stored":4654292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1233317470},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":6.3620000000000001,"5min":5.5300000000000002,"15min":4.4160000000000004},"min":{"1min":0.26100000000000001,"5min":0.217,"15min":0.20399999999999999},"max":{"1min":50.546999999999997,"5min":135.501,"15min":312.56599999999997},"last":0.43099999999999999},{"interface":"front","average":{"1min":5.9340000000000002,"5min":5.266,"15min":4.3760000000000003},"min":{"1min":0.23300000000000001,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":55.683,"5min":135.99199999999999,"15min":311.02300000000002},"last":0.40000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":8.6780000000000008,"5min":5.5750000000000002,"15min":5.1840000000000002},"min":{"1min":0.29799999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":55.277000000000001,"5min":89.165000000000006,"15min":142.32599999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":7.5510000000000002,"5min":4.9089999999999998,"15min":4.9960000000000004},"min":{"1min":0.33500000000000002,"5min":0.23899999999999999,"15min":0.224},"max":{"1min":55.209000000000003,"5min":89.093999999999994,"15min":139.33199999999999},"last":0.372}]},{"osd":2,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":9.1489999999999991,"5min":7.165,"15min":4.0069999999999997},"min":{"1min":0.314,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":45.183,"5min":89.024000000000001,"15min":89.024000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":9.327,"5min":5.6779999999999999,"15min":3.4300000000000002},"min":{"1min":0.433,"5min":0.25900000000000001,"15min":0.248},"max":{"1min":51.189999999999998,"5min":90.850999999999999,"15min":90.850999999999999},"last":0.495}]},{"osd":4,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":2.2000000000000002,"5min":3.0459999999999998,"15min":1.8759999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.9930000000000003,"5min":74.957999999999998,"15min":74.957999999999998},"last":0.65800000000000003},{"interface":"front","average":{"1min":2.4140000000000001,"5min":3.0680000000000001,"15min":1.909},"min":{"1min":0.315,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":9.7309999999999999,"5min":75.022999999999996,"15min":75.022999999999996},"last":0.63200000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:07:29 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":1.5329999999999999,"15min":1.5920000000000001},"min":{"1min":0.38200000000000001,"5min":0.32900000000000001,"15min":0.309},"max":{"1min":6.8449999999999998,"5min":8.8450000000000006,"15min":71.203999999999994},"last":0.68400000000000005},{"interface":"front","average":{"1min":2.2250000000000001,"5min":1.849,"15min":1.6599999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.309},"max":{"1min":9.8109999999999999,"5min":15.932,"15min":70.397000000000006},"last":0.70899999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":1.7589999999999999,"15min":1.591},"min":{"1min":0.45900000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.7939999999999996,"5min":8.9149999999999991,"15min":8.9149999999999991},"last":0.745},{"interface":"front","average":{"1min":2.6749999999999998,"5min":2.71,"15min":2.8500000000000001},"min":{"1min":0.40100000000000002,"5min":0.38,"15min":0.38},"max":{"1min":9.7780000000000005,"5min":15.954000000000001,"15min":15.954000000000001},"last":0.59499999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:07:58 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":3.1949999999999998,"15min":1.952},"min":{"1min":0.42999999999999999,"5min":0.375,"15min":0.375},"max":{"1min":8.0090000000000003,"5min":79.403000000000006,"15min":79.403000000000006},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.895,"5min":3.4060000000000001,"15min":1.986},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":12.242000000000001,"5min":79.203000000000003,"15min":79.203000000000003},"last":0.72599999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019477,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1961680,"kb_used_data":775944,"kb_used_omap":0,"kb_used_meta":1185728,"kb_avail":91783472,"statfs":{"total":95995035648,"available":93986275328,"internally_reserved":0,"allocated":794566656,"data_stored":793381875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1214185472},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:08:20 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":3.589,"15min":2.7829999999999999},"min":{"1min":0.28899999999999998,"5min":0.22600000000000001,"15min":0.21299999999999999},"max":{"1min":3.4340000000000002,"5min":86.509,"15min":95.466999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.236,"5min":4.2069999999999999,"15min":3.202},"min":{"1min":0.32700000000000001,"5min":0.27600000000000002,"15min":0.24299999999999999},"max":{"1min":4.7389999999999999,"5min":86.233999999999995,"15min":95.367999999999995},"last":0.57599999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:07:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3650000000000002,"5min":2.9260000000000002,"15min":2.6480000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.7050000000000001,"5min":34.911000000000001,"15min":96.033000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.145,"5min":4.0999999999999996,"15min":3.04},"min":{"1min":0.33200000000000002,"5min":0.26700000000000002,"15min":0.255},"max":{"1min":10.766,"5min":108.45699999999999,"15min":108.45699999999999},"last":0.629}]},{"osd":2,"last update":"Mon Oct 9 06:08:02 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":4.7510000000000003,"15min":2.8650000000000002},"min":{"1min":0.35299999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.1029999999999998,"5min":105.157,"15min":143.71000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.117,"5min":4.4409999999999998,"15min":3.133},"min":{"1min":0.35599999999999998,"5min":0.29599999999999999,"15min":0.27700000000000002},"max":{"1min":3.573,"5min":107.851,"15min":143.768},"last":0.746}]},{"osd":3,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":2.492,"5min":3.4830000000000001,"15min":2.9700000000000002},"min":{"1min":0.35499999999999998,"5min":0.254,"15min":0.254},"max":{"1min":8.1159999999999997,"5min":107.352,"15min":107.352},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.5510000000000002,"5min":3.0499999999999998,"15min":2.77},"min":{"1min":0.35399999999999998,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":8.3919999999999995,"5min":105.336,"15min":105.336},"last":0.71299999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":1.1419999999999999,"15min":1.093},"min":{"1min":0.32700000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.9349999999999996,"5min":7.1070000000000002,"15min":15.597},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.262,"5min":1.004,"15min":1.0369999999999999},"min":{"1min":0.378,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":6.2990000000000004,"5min":7.069,"15min":17.882000000000001},"last":0.66900000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:07:35 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.94499999999999995,"15min":0.88},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.9400000000000004,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.54300000000000004},{"interface":"front","average":{"1min":2.306,"5min":1.2869999999999999,"15min":1.0549999999999999},"min":{"1min":0.38300000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.0340000000000007,"5min":8.9909999999999997,"15min":8.9909999999999997},"last":0.64900000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:07:54 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":0.90900000000000003,"15min":1.1890000000000001},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":4.9610000000000003,"5min":6.6040000000000001,"15min":36.729999999999997},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.986,"5min":1.226,"15min":1.2050000000000001},"min":{"1min":0.41299999999999998,"5min":0.33000000000000002,"15min":0.32200000000000001},"max":{"1min":7.96,"5min":8.9580000000000002,"15min":36.424999999999997},"last":0.69599999999999995}]}]},{"osd":5,"up_from":0,"seq":154618823252,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":69,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4231168,"data_stored":4214889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13650,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:08:26.662 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:08:26.662 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:08:26.663 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:08:26.663 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.000000 2023-10-09T06:08:26.663 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 0 2023-10-09T06:08:26.677 INFO:tasks.workunit.client.0.smithi008.stderr: Removing snap: 100% complete...done. 2023-10-09T06:08:26.702 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm testimg 2023-10-09T06:08:26.752 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:26.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:26.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:26.813 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T06:08:26.816 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --order 20 /tmp/rbd_import_export_704372/img testimg 2023-10-09T06:08:26.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:26.875 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T06:08:26.876 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:08:26.878 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_704372/img_v2 2023-10-09T06:08:26.887 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:26.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[184418]: pgmap v1791: 73 pgs: 73 active+clean; 4.8 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 102 KiB/s rd, 5.1 KiB/s wr, 50 op/s 2023-10-09T06:08:26.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[184418]: from='client.47555 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:26.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[184418]: failed to encode map e593 with expected crc 2023-10-09T06:08:26.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[184418]: osdmap e593: 8 total, 8 up, 7 in 2023-10-09T06:08:26.909 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[184418]: failed to encode map e593 with expected crc 2023-10-09T06:08:26.910 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[184418]: failed to encode map e593 with expected crc 2023-10-09T06:08:26.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[191227]: pgmap v1791: 73 pgs: 73 active+clean; 4.8 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 102 KiB/s rd, 5.1 KiB/s wr, 50 op/s 2023-10-09T06:08:26.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[191227]: from='client.47555 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:26.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[191227]: failed to encode map e593 with expected crc 2023-10-09T06:08:26.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[191227]: osdmap e593: 8 total, 8 up, 7 in 2023-10-09T06:08:26.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[191227]: failed to encode map e593 with expected crc 2023-10-09T06:08:26.910 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:26 smithi008 ceph-mon[191227]: failed to encode map e593 with expected crc 2023-10-09T06:08:26.920 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:26.924 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_704372/img_v2 testimg_import 2023-10-09T06:08:26.932 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:26.938 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:26 smithi191 ceph-mon[161669]: pgmap v1791: 73 pgs: 73 active+clean; 4.8 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 102 KiB/s rd, 5.1 KiB/s wr, 50 op/s 2023-10-09T06:08:26.938 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:26 smithi191 ceph-mon[161669]: from='client.47555 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:26.938 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:26 smithi191 ceph-mon[161669]: failed to encode map e593 with expected crc 2023-10-09T06:08:26.938 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:26 smithi191 ceph-mon[161669]: osdmap e593: 8 total, 8 up, 7 in 2023-10-09T06:08:26.938 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:26 smithi191 ceph-mon[161669]: failed to encode map e593 with expected crc 2023-10-09T06:08:26.939 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:26 smithi191 ceph-mon[161669]: failed to encode map e593 with expected crc 2023-10-09T06:08:26.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:26.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:26.991 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:08:26.994 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import 2023-10-09T06:08:26.994 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep order 2023-10-09T06:08:26.995 INFO:tasks.workunit.client.0.smithi008.stderr:+ awk '{print $2}' 2023-10-09T06:08:26.995 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 20 2023-10-09T06:08:27.032 INFO:tasks.workunit.client.0.smithi008.stdout:20 2023-10-09T06:08:27.032 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/img_v2 2023-10-09T06:08:27.033 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg_import 2023-10-09T06:08:27.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:27.066 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:27.108 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:27.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:27.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:27.123 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:27.149 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T06:08:27.153 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg 2023-10-09T06:08:27.171 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:27.171 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:27.214 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:27.254 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:27.259 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T06:08:27.264 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_704372/img testimg 2023-10-09T06:08:27.322 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T06:08:27.325 INFO:tasks.workunit.client.0.smithi008.stderr:++ rbd info testimg 2023-10-09T06:08:27.325 INFO:tasks.workunit.client.0.smithi008.stderr:++ grep features 2023-10-09T06:08:27.361 INFO:tasks.workunit.client.0.smithi008.stderr:+ FEATURES_BEFORE=' features: layering 2023-10-09T06:08:27.361 INFO:tasks.workunit.client.0.smithi008.stderr: op_features: ' 2023-10-09T06:08:27.361 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_704372/img_v2 2023-10-09T06:08:27.397 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:27.400 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_704372/img_v2 testimg_import 2023-10-09T06:08:27.454 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:08:27.458 INFO:tasks.workunit.client.0.smithi008.stderr:++ rbd info testimg_import 2023-10-09T06:08:27.458 INFO:tasks.workunit.client.0.smithi008.stderr:++ grep features 2023-10-09T06:08:27.494 INFO:tasks.workunit.client.0.smithi008.stderr:+ FEATURES_AFTER=' features: layering 2023-10-09T06:08:27.494 INFO:tasks.workunit.client.0.smithi008.stderr: op_features: ' 2023-10-09T06:08:27.494 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' ' features: layering 2023-10-09T06:08:27.494 INFO:tasks.workunit.client.0.smithi008.stderr: op_features: ' '!=' ' features: layering 2023-10-09T06:08:27.494 INFO:tasks.workunit.client.0.smithi008.stderr: op_features: ' ']' 2023-10-09T06:08:27.494 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/img_v2 2023-10-09T06:08:27.495 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg_import 2023-10-09T06:08:27.564 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T06:08:27.566 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg 2023-10-09T06:08:27.756 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T06:08:27.759 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_704372/img testimg 2023-10-09T06:08:27.823 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T06:08:27.826 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_704372/img_v2 2023-10-09T06:08:27.868 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:27.871 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_704372/img_v2 testimg_import 2023-10-09T06:08:27.933 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:08:27.936 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import 2023-10-09T06:08:27.936 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 'stripe unit' 2023-10-09T06:08:27.936 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -Ei '(4 KiB|4096)' 2023-10-09T06:08:27.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:27 smithi008 ceph-mon[184418]: failed to encode map e594 with expected crc 2023-10-09T06:08:27.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:27 smithi008 ceph-mon[184418]: osdmap e594: 8 total, 8 up, 7 in 2023-10-09T06:08:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:27 smithi008 ceph-mon[184418]: failed to encode map e594 with expected crc 2023-10-09T06:08:27.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:27 smithi008 ceph-mon[184418]: failed to encode map e594 with expected crc 2023-10-09T06:08:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:27 smithi008 ceph-mon[191227]: failed to encode map e594 with expected crc 2023-10-09T06:08:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:27 smithi008 ceph-mon[191227]: osdmap e594: 8 total, 8 up, 7 in 2023-10-09T06:08:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:27 smithi008 ceph-mon[191227]: failed to encode map e594 with expected crc 2023-10-09T06:08:27.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:27 smithi008 ceph-mon[191227]: failed to encode map e594 with expected crc 2023-10-09T06:08:27.971 INFO:tasks.workunit.client.0.smithi008.stdout: stripe unit: 4 KiB 2023-10-09T06:08:27.972 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import 2023-10-09T06:08:27.972 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 'stripe count' 2023-10-09T06:08:27.972 INFO:tasks.workunit.client.0.smithi008.stderr:+ awk '{print $3}' 2023-10-09T06:08:27.972 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 1000 2023-10-09T06:08:28.008 INFO:tasks.workunit.client.0.smithi008.stdout:1000 2023-10-09T06:08:28.008 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/img_v2 2023-10-09T06:08:28.009 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg_import 2023-10-09T06:08:28.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:27 smithi191 ceph-mon[161669]: failed to encode map e594 with expected crc 2023-10-09T06:08:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:27 smithi191 ceph-mon[161669]: osdmap e594: 8 total, 8 up, 7 in 2023-10-09T06:08:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:27 smithi191 ceph-mon[161669]: failed to encode map e594 with expected crc 2023-10-09T06:08:28.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:27 smithi191 ceph-mon[161669]: failed to encode map e594 with expected crc 2023-10-09T06:08:28.101 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T06:08:28.104 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg 2023-10-09T06:08:28.114 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:28.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:28.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:28.190 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T06:08:28.193 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_704372/img testimg 2023-10-09T06:08:28.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:28.245 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:28.254 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T06:08:28.256 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap create testimg@snap1 2023-10-09T06:08:28.270 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:28.288 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:28.292 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:08:28.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:28.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:28.394 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:28.420 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:28.462 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:28.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:28.569 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:28.598 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:28.685 INFO:tasks.workunit.client.0.smithi008.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:08:28.691 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap create testimg@snap2 2023-10-09T06:08:28.776 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:08:28.805 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:28 smithi008 ceph-mon[184418]: pgmap v1794: 73 pgs: 73 active+clean; 4.8 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 134 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-10-09T06:08:28.805 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:28 smithi008 ceph-mon[191227]: pgmap v1794: 73 pgs: 73 active+clean; 4.8 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 134 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-10-09T06:08:28.997 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:29.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:29.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:29.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:28 smithi191 ceph-mon[161669]: pgmap v1794: 73 pgs: 73 active+clean; 4.8 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 134 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-10-09T06:08:29.102 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:29.129 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:29.172 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:29.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:29.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:29.278 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:29.309 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:29.613 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:29.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:29.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:29.694 INFO:tasks.workunit.client.0.smithi008.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:08:29.700 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap protect testimg@snap2 2023-10-09T06:08:29.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:29.739 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_704372/snap_protect 2023-10-09T06:08:29.744 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:29.778 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:29.781 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_704372/snap_protect testimg_import 2023-10-09T06:08:29.787 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:29.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:29.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:29.893 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:29.919 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:29.961 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:29 smithi008 ceph-mon[184418]: failed to encode map e595 with expected crc 2023-10-09T06:08:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:29 smithi008 ceph-mon[184418]: failed to encode map e595 with expected crc 2023-10-09T06:08:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:29 smithi008 ceph-mon[184418]: osdmap e595: 8 total, 8 up, 7 in 2023-10-09T06:08:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:29 smithi008 ceph-mon[184418]: failed to encode map e595 with expected crc 2023-10-09T06:08:29.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:29 smithi008 ceph-mon[191227]: failed to encode map e595 with expected crc 2023-10-09T06:08:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:29 smithi008 ceph-mon[191227]: failed to encode map e595 with expected crc 2023-10-09T06:08:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:29 smithi008 ceph-mon[191227]: osdmap e595: 8 total, 8 up, 7 in 2023-10-09T06:08:29.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:29 smithi008 ceph-mon[191227]: failed to encode map e595 with expected crc 2023-10-09T06:08:29.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:29.966 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:29 smithi191 ceph-mon[161669]: failed to encode map e595 with expected crc 2023-10-09T06:08:29.966 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:29 smithi191 ceph-mon[161669]: failed to encode map e595 with expected crc 2023-10-09T06:08:29.966 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:29 smithi191 ceph-mon[161669]: osdmap e595: 8 total, 8 up, 7 in 2023-10-09T06:08:29.966 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:29 smithi191 ceph-mon[161669]: failed to encode map e595 with expected crc 2023-10-09T06:08:29.967 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:29.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:29.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:30.068 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:30.094 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:30.136 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:30.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:30.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:30.241 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:30.274 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:30.279 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:30.451 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:30.460 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:30.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:30.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:30.465 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:08:30.492 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:08:30.512 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:08:30.532 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:08:30.552 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:08:30.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:30.573 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:08:30.591 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:30.592 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:08:30.611 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:08:30.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:08:30.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:08:30.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:08:30.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:08:30.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:08:30.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:08:30.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:08:30.613 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:08:30.633 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:30.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:30.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:30.768 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:30.811 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:30.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:30.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:30.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:30.941 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:30 smithi191 ceph-mon[161669]: pgmap v1796: 73 pgs: 73 active+clean; 4.8 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 119 KiB/s rd, 10 KiB/s wr, 150 op/s 2023-10-09T06:08:30.941 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:30 smithi191 ceph-mon[161669]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.941 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:30 smithi191 ceph-mon[161669]: osdmap e596: 8 total, 8 up, 7 in 2023-10-09T06:08:30.941 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:30 smithi191 ceph-mon[161669]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.941 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:30 smithi191 ceph-mon[161669]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.941 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:30 smithi191 ceph-mon[161669]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.941 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:30 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3411543300' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:30.943 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:30.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[184418]: pgmap v1796: 73 pgs: 73 active+clean; 4.8 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 119 KiB/s rd, 10 KiB/s wr, 150 op/s 2023-10-09T06:08:30.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[184418]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[184418]: osdmap e596: 8 total, 8 up, 7 in 2023-10-09T06:08:30.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[184418]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[184418]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[184418]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3411543300' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:30.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:08:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:08:30] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.33.4" 2023-10-09T06:08:30.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[191227]: pgmap v1796: 73 pgs: 73 active+clean; 4.8 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 119 KiB/s rd, 10 KiB/s wr, 150 op/s 2023-10-09T06:08:30.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[191227]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[191227]: osdmap e596: 8 total, 8 up, 7 in 2023-10-09T06:08:30.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[191227]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[191227]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[191227]: failed to encode map e596 with expected crc 2023-10-09T06:08:30.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:30 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3411543300' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:30.985 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:30.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:30.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:31.091 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:31.117 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:31.159 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:31.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:31.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:31.265 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:31.297 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:31.713 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:08:31.715 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import@snap1 2023-10-09T06:08:31.715 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 'protected: False' 2023-10-09T06:08:31.758 INFO:tasks.workunit.client.0.smithi008.stdout: protected: False 2023-10-09T06:08:31.758 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd info testimg_import@snap2 2023-10-09T06:08:31.758 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep 'protected: True' 2023-10-09T06:08:31.802 INFO:tasks.workunit.client.0.smithi008.stdout: protected: True 2023-10-09T06:08:31.802 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/snap_protect 2023-10-09T06:08:31.803 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T06:08:31.845 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T06:08:31.887 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap purge testimg 2023-10-09T06:08:31.901 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:31.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:31.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:31 smithi008 ceph-mon[184418]: failed to encode map e597 with expected crc 2023-10-09T06:08:31.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:31 smithi008 ceph-mon[184418]: failed to encode map e597 with expected crc 2023-10-09T06:08:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:31 smithi008 ceph-mon[184418]: osdmap e597: 8 total, 8 up, 7 in 2023-10-09T06:08:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:31 smithi008 ceph-mon[184418]: failed to encode map e597 with expected crc 2023-10-09T06:08:31.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:31 smithi008 ceph-mon[184418]: failed to encode map e597 with expected crc 2023-10-09T06:08:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:31 smithi008 ceph-mon[191227]: failed to encode map e597 with expected crc 2023-10-09T06:08:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:31 smithi008 ceph-mon[191227]: failed to encode map e597 with expected crc 2023-10-09T06:08:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:31 smithi008 ceph-mon[191227]: osdmap e597: 8 total, 8 up, 7 in 2023-10-09T06:08:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:31 smithi008 ceph-mon[191227]: failed to encode map e597 with expected crc 2023-10-09T06:08:31.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:31 smithi008 ceph-mon[191227]: failed to encode map e597 with expected crc 2023-10-09T06:08:32.006 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:32.035 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:32.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:31 smithi191 ceph-mon[161669]: failed to encode map e597 with expected crc 2023-10-09T06:08:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:31 smithi191 ceph-mon[161669]: failed to encode map e597 with expected crc 2023-10-09T06:08:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:31 smithi191 ceph-mon[161669]: osdmap e597: 8 total, 8 up, 7 in 2023-10-09T06:08:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:31 smithi191 ceph-mon[161669]: failed to encode map e597 with expected crc 2023-10-09T06:08:32.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:31 smithi191 ceph-mon[161669]: failed to encode map e597 with expected crc 2023-10-09T06:08:32.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:32 smithi008 ceph-mon[184418]: pgmap v1799: 73 pgs: 73 active+clean; 4.7 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 278 KiB/s rd, 27 KiB/s wr, 363 op/s 2023-10-09T06:08:32.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:32 smithi008 ceph-mon[184418]: failed to encode map e598 with expected crc 2023-10-09T06:08:32.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:32 smithi008 ceph-mon[184418]: failed to encode map e598 with expected crc 2023-10-09T06:08:32.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:32 smithi008 ceph-mon[184418]: osdmap e598: 8 total, 8 up, 7 in 2023-10-09T06:08:32.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:32 smithi008 ceph-mon[184418]: failed to encode map e598 with expected crc 2023-10-09T06:08:32.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:32 smithi008 ceph-mon[191227]: pgmap v1799: 73 pgs: 73 active+clean; 4.7 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 278 KiB/s rd, 27 KiB/s wr, 363 op/s 2023-10-09T06:08:32.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:32 smithi008 ceph-mon[191227]: failed to encode map e598 with expected crc 2023-10-09T06:08:32.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:32 smithi008 ceph-mon[191227]: failed to encode map e598 with expected crc 2023-10-09T06:08:32.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:32 smithi008 ceph-mon[191227]: osdmap e598: 8 total, 8 up, 7 in 2023-10-09T06:08:32.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:32 smithi008 ceph-mon[191227]: failed to encode map e598 with expected crc 2023-10-09T06:08:33.037 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:33.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:33.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:32 smithi191 ceph-mon[161669]: pgmap v1799: 73 pgs: 73 active+clean; 4.7 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 278 KiB/s rd, 27 KiB/s wr, 363 op/s 2023-10-09T06:08:33.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:32 smithi191 ceph-mon[161669]: failed to encode map e598 with expected crc 2023-10-09T06:08:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:32 smithi191 ceph-mon[161669]: failed to encode map e598 with expected crc 2023-10-09T06:08:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:32 smithi191 ceph-mon[161669]: osdmap e598: 8 total, 8 up, 7 in 2023-10-09T06:08:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:32 smithi191 ceph-mon[161669]: failed to encode map e598 with expected crc 2023-10-09T06:08:33.141 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:33.172 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:33.650 INFO:tasks.workunit.client.0.smithi008.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T06:08:33.658 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.7 primary-affinity to 0 (802) 2023-10-09T06:08:33.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: failed to encode map e598 with expected crc 2023-10-09T06:08:33.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.899 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: osdmap e599: 8 total, 8 up, 7 in 2023-10-09T06:08:33.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: pgmap v1802: 73 pgs: 73 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 418 KiB/s rd, 48 KiB/s wr, 558 op/s 2023-10-09T06:08:33.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1380099752' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T06:08:33.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T06:08:33.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-09T06:08:33.900 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[184418]: osdmap e600: 8 total, 8 up, 7 in 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: failed to encode map e598 with expected crc 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: osdmap e599: 8 total, 8 up, 7 in 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: failed to encode map e599 with expected crc 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: pgmap v1802: 73 pgs: 73 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 418 KiB/s rd, 48 KiB/s wr, 558 op/s 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1380099752' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T06:08:33.901 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-09T06:08:33.902 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:33 smithi008 ceph-mon[191227]: osdmap e600: 8 total, 8 up, 7 in 2023-10-09T06:08:33.974 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:33.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:33.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:34.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: failed to encode map e598 with expected crc 2023-10-09T06:08:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: failed to encode map e599 with expected crc 2023-10-09T06:08:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: osdmap e599: 8 total, 8 up, 7 in 2023-10-09T06:08:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: failed to encode map e599 with expected crc 2023-10-09T06:08:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: failed to encode map e599 with expected crc 2023-10-09T06:08:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: failed to encode map e599 with expected crc 2023-10-09T06:08:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: failed to encode map e599 with expected crc 2023-10-09T06:08:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: failed to encode map e599 with expected crc 2023-10-09T06:08:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: pgmap v1802: 73 pgs: 73 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 418 KiB/s rd, 48 KiB/s wr, 558 op/s 2023-10-09T06:08:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1380099752' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T06:08:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T06:08:34.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-09T06:08:34.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:33 smithi191 ceph-mon[161669]: osdmap e600: 8 total, 8 up, 7 in 2023-10-09T06:08:34.077 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:34.108 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:34.156 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:34.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:34.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:34.166 INFO:teuthology.orchestra.run.smithi008.stdout:{ 2023-10-09T06:08:34.166 INFO:teuthology.orchestra.run.smithi008.stdout: "ops": [], 2023-10-09T06:08:34.166 INFO:teuthology.orchestra.run.smithi008.stdout: "num_ops": 0 2023-10-09T06:08:34.167 INFO:teuthology.orchestra.run.smithi008.stdout:} 2023-10-09T06:08:34.261 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:34.289 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:34.587 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:34.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:34.652 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:08:34.683 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd snap purge testimg_import 2023-10-09T06:08:34.690 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:34.721 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:34.767 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:34.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:34.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:34.843 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[184418]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.843 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[184418]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.843 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[184418]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.843 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[184418]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.843 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[184418]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.843 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[184418]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.843 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/103647107' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:34.843 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:34.843 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[184418]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:34.843 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:34.843 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[184418]: osdmap e601: 8 total, 8 up, 7 in 2023-10-09T06:08:34.844 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[191227]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.844 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[191227]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.844 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[191227]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.844 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[191227]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.844 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[191227]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.844 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[191227]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.844 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/103647107' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:34.844 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:34.844 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[191227]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:34.844 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:34.845 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:34 smithi008 ceph-mon[191227]: osdmap e601: 8 total, 8 up, 7 in 2023-10-09T06:08:34.871 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:34.903 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:34.951 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:34.955 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:34 smithi191 ceph-mon[161669]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.955 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:34 smithi191 ceph-mon[161669]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.955 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:34 smithi191 ceph-mon[161669]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.955 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:34 smithi191 ceph-mon[161669]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.956 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:34 smithi191 ceph-mon[161669]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.956 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:34 smithi191 ceph-mon[161669]: failed to encode map e600 with expected crc 2023-10-09T06:08:34.956 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/103647107' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:34.956 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:34 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:34.956 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:34 smithi191 ceph-mon[161669]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:34.956 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:34 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:34.956 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:34 smithi191 ceph-mon[161669]: osdmap e601: 8 total, 8 up, 7 in 2023-10-09T06:08:34.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:34.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:35.058 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:35.085 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:35.130 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:35.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:35.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:35.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:35.263 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:35.304 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:35.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:35.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:35.410 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:35.439 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[191227]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.706 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[191227]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[191227]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[191227]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[191227]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[191227]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.707 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[191227]: pgmap v1805: 73 pgs: 3 peering, 70 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 127 KiB/s rd, 20 KiB/s wr, 177 op/s 2023-10-09T06:08:35.737 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:08:35.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[184418]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[184418]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[184418]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[184418]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[184418]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[184418]: failed to encode map e601 with expected crc 2023-10-09T06:08:35.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:35 smithi008 ceph-mon[184418]: pgmap v1805: 73 pgs: 3 peering, 70 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 127 KiB/s rd, 20 KiB/s wr, 177 op/s 2023-10-09T06:08:36.024 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:36.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:36.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:36.032 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:08:36.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:35 smithi191 ceph-mon[161669]: failed to encode map e601 with expected crc 2023-10-09T06:08:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:35 smithi191 ceph-mon[161669]: failed to encode map e601 with expected crc 2023-10-09T06:08:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:35 smithi191 ceph-mon[161669]: failed to encode map e601 with expected crc 2023-10-09T06:08:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:35 smithi191 ceph-mon[161669]: failed to encode map e601 with expected crc 2023-10-09T06:08:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:35 smithi191 ceph-mon[161669]: failed to encode map e601 with expected crc 2023-10-09T06:08:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:35 smithi191 ceph-mon[161669]: failed to encode map e601 with expected crc 2023-10-09T06:08:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:35 smithi191 ceph-mon[161669]: pgmap v1805: 73 pgs: 3 peering, 70 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 127 KiB/s rd, 20 KiB/s wr, 177 op/s 2023-10-09T06:08:36.128 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:36.182 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:36.219 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:36.234 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:36.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:36.341 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:36.368 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:36.520 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:36.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:36.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:36.623 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:36.653 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:36.717 INFO:tasks.workunit.client.0.smithi008.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T06:08:36.725 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg 2023-10-09T06:08:36.748 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:36.803 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:36.806 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:08:36.806 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[184418]: failed to encode map e602 with expected crc 2023-10-09T06:08:36.806 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[184418]: failed to encode map e602 with expected crc 2023-10-09T06:08:36.807 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[184418]: osdmap e602: 8 total, 8 up, 7 in 2023-10-09T06:08:36.807 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[184418]: failed to encode map e602 with expected crc 2023-10-09T06:08:36.807 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[184418]: failed to encode map e602 with expected crc 2023-10-09T06:08:36.807 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:08:36.807 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[191227]: failed to encode map e602 with expected crc 2023-10-09T06:08:36.807 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[191227]: failed to encode map e602 with expected crc 2023-10-09T06:08:36.807 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[191227]: osdmap e602: 8 total, 8 up, 7 in 2023-10-09T06:08:36.807 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[191227]: failed to encode map e602 with expected crc 2023-10-09T06:08:36.808 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:36 smithi008 ceph-mon[191227]: failed to encode map e602 with expected crc 2023-10-09T06:08:36.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:36.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:36.830 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T06:08:36.833 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd remove testimg_import 2023-10-09T06:08:36.910 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:36.937 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:36 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:08:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:36 smithi191 ceph-mon[161669]: failed to encode map e602 with expected crc 2023-10-09T06:08:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:36 smithi191 ceph-mon[161669]: failed to encode map e602 with expected crc 2023-10-09T06:08:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:36 smithi191 ceph-mon[161669]: osdmap e602: 8 total, 8 up, 7 in 2023-10-09T06:08:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:36 smithi191 ceph-mon[161669]: failed to encode map e602 with expected crc 2023-10-09T06:08:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:36 smithi191 ceph-mon[161669]: failed to encode map e602 with expected crc 2023-10-09T06:08:37.113 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T06:08:37.116 INFO:tasks.workunit.client.0.smithi008.stderr:+ tiered=0 2023-10-09T06:08:37.116 INFO:tasks.workunit.client.0.smithi008.stderr:+ ceph osd dump 2023-10-09T06:08:37.116 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep '^pool' 2023-10-09T06:08:37.116 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep ''\''rbd'\''' 2023-10-09T06:08:37.116 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep tier 2023-10-09T06:08:37.465 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:37.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:37.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:37.569 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:37.600 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:37.650 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:37.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:37.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[184418]: failed to encode map e603 with expected crc 2023-10-09T06:08:37.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[184418]: failed to encode map e603 with expected crc 2023-10-09T06:08:37.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[184418]: failed to encode map e603 with expected crc 2023-10-09T06:08:37.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[184418]: osdmap e603: 8 total, 8 up, 7 in 2023-10-09T06:08:37.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[184418]: failed to encode map e603 with expected crc 2023-10-09T06:08:37.717 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[184418]: pgmap v1808: 73 pgs: 3 peering, 70 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 93 KiB/s rd, 6.7 KiB/s wr, 123 op/s 2023-10-09T06:08:37.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[191227]: failed to encode map e603 with expected crc 2023-10-09T06:08:37.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[191227]: failed to encode map e603 with expected crc 2023-10-09T06:08:37.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[191227]: failed to encode map e603 with expected crc 2023-10-09T06:08:37.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[191227]: osdmap e603: 8 total, 8 up, 7 in 2023-10-09T06:08:37.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[191227]: failed to encode map e603 with expected crc 2023-10-09T06:08:37.718 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:37 smithi008 ceph-mon[191227]: pgmap v1808: 73 pgs: 3 peering, 70 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 93 KiB/s rd, 6.7 KiB/s wr, 123 op/s 2023-10-09T06:08:37.755 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:37.782 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:37.825 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:37.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:37.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:37.931 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:37.962 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:37.974 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_704372/sparse1 2023-10-09T06:08:37.990 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records in 2023-10-09T06:08:37.990 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records out 2023-10-09T06:08:37.990 INFO:tasks.workunit.client.0.smithi008.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00725587 s, 145 MB/s 2023-10-09T06:08:37.990 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_704372/sparse2 2023-10-09T06:08:37.995 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:08:37.998 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records in 2023-10-09T06:08:37.998 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records out 2023-10-09T06:08:37.998 INFO:tasks.workunit.client.0.smithi008.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00543286 s, 193 MB/s 2023-10-09T06:08:37.998 INFO:tasks.workunit.client.0.smithi008.stderr:+ truncate /tmp/rbd_import_export_704372/sparse2 -s 2M 2023-10-09T06:08:38.009 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse1 2023-10-09T06:08:38.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:37 smithi191 ceph-mon[161669]: failed to encode map e603 with expected crc 2023-10-09T06:08:38.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:37 smithi191 ceph-mon[161669]: failed to encode map e603 with expected crc 2023-10-09T06:08:38.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:37 smithi191 ceph-mon[161669]: failed to encode map e603 with expected crc 2023-10-09T06:08:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:37 smithi191 ceph-mon[161669]: osdmap e603: 8 total, 8 up, 7 in 2023-10-09T06:08:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:37 smithi191 ceph-mon[161669]: failed to encode map e603 with expected crc 2023-10-09T06:08:38.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:37 smithi191 ceph-mon[161669]: pgmap v1808: 73 pgs: 3 peering, 70 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 93 KiB/s rd, 6.7 KiB/s wr, 123 op/s 2023-10-09T06:08:38.054 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 0% complete...failed. 2023-10-09T06:08:38.054 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:08:38.057 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T06:08:38.057 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_704372/sparse1 2023-10-09T06:08:38.065 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:38.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:38.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:38.125 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T06:08:38.126 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:08:38.126 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:08:38.129 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd ls -l 2023-10-09T06:08:38.129 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep sparse1 2023-10-09T06:08:38.129 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T06:08:38.166 INFO:tasks.workunit.client.0.smithi008.stdout:sparse1 2 MiB 2 2023-10-09T06:08:38.167 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse1 2023-10-09T06:08:38.167 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse1 2023-10-09T06:08:38.168 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse1 2023-10-09T06:08:38.168 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T06:08:38.168 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T06:08:38.171 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:38.209 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:38.217 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.badaa3dbab99 2023-10-09T06:08:38.218 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.badaa3dbab99 2023-10-09T06:08:38.218 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse1 2023-10-09T06:08:38.218 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse1 2023-10-09T06:08:38.218 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.badaa3dbab99\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:08:38.219 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T06:08:38.219 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse1 2023-10-09T06:08:38.220 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:08:38.220 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:08:38.223 INFO:teuthology.orchestra.run.smithi008.stdout:osd.0: {} 2023-10-09T06:08:38.259 INFO:teuthology.orchestra.run.smithi008.stdout:osd.1: {} 2023-10-09T06:08:38.263 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T06:08:38.263 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T06:08:38.263 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T06:08:38.263 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T06:08:38.263 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T06:08:38.282 INFO:teuthology.orchestra.run.smithi008.stdout:osd.2: {} 2023-10-09T06:08:38.305 INFO:teuthology.orchestra.run.smithi008.stdout:osd.3: {} 2023-10-09T06:08:38.316 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:38.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:38.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:38.323 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects=1 2023-10-09T06:08:38.323 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 1 2023-10-09T06:08:38.323 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T06:08:38.324 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export sparse1 /tmp/rbd_import_export_704372/sparse1.out 2023-10-09T06:08:38.329 INFO:teuthology.orchestra.run.smithi008.stdout:osd.4: {} 2023-10-09T06:08:38.347 INFO:teuthology.orchestra.run.smithi008.stdout:osd.5: {} 2023-10-09T06:08:38.358 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:38.361 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_704372/sparse1 /tmp/rbd_import_export_704372/sparse1.out 2023-10-09T06:08:38.362 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_704372/sparse1 /tmp/rbd_import_export_704372/sparse1.out 2023-10-09T06:08:38.364 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/sparse1 --format %b 2023-10-09T06:08:38.365 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=2048 2023-10-09T06:08:38.365 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/sparse1.out --format %b 2023-10-09T06:08:38.366 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=2048 2023-10-09T06:08:38.366 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T06:08:38.366 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T06:08:38.366 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:08:38.366 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/sparse1.out 2023-10-09T06:08:38.367 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse1 2023-10-09T06:08:38.367 INFO:teuthology.orchestra.run.smithi008.stdout:osd.6: {} 2023-10-09T06:08:38.386 INFO:teuthology.orchestra.run.smithi008.stdout:osd.7: {} 2023-10-09T06:08:38.388 INFO:teuthology.orchestra.run.smithi008.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:08:38.388 INFO:teuthology.orchestra.run.smithi008.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:08:38.388 INFO:teuthology.orchestra.run.smithi008.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:08:38.388 INFO:teuthology.orchestra.run.smithi008.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:08:38.388 INFO:teuthology.orchestra.run.smithi008.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:08:38.388 INFO:teuthology.orchestra.run.smithi008.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:08:38.389 INFO:teuthology.orchestra.run.smithi008.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:08:38.389 INFO:teuthology.orchestra.run.smithi008.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:08:38.421 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:38.447 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:38.558 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T06:08:38.560 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse2 2023-10-09T06:08:38.596 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 0% complete...failed. 2023-10-09T06:08:38.596 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:08:38.599 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T06:08:38.599 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_704372/sparse2 2023-10-09T06:08:38.681 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T06:08:38.682 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:08:38.682 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:08:38.685 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd ls -l 2023-10-09T06:08:38.685 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep sparse2 2023-10-09T06:08:38.685 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T06:08:38.720 INFO:tasks.workunit.client.0.smithi008.stdout:sparse2 2 MiB 2 2023-10-09T06:08:38.720 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse2 2023-10-09T06:08:38.721 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse2 2023-10-09T06:08:38.721 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse2 2023-10-09T06:08:38.721 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T06:08:38.721 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T06:08:38.755 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.e14992f479c2 2023-10-09T06:08:38.756 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.e14992f479c2 2023-10-09T06:08:38.756 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse2 2023-10-09T06:08:38.757 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse2 2023-10-09T06:08:38.757 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.e14992f479c2\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:08:38.757 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T06:08:38.757 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse2 2023-10-09T06:08:38.757 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:08:38.757 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:08:38.791 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T06:08:38.791 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T06:08:38.791 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T06:08:38.791 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T06:08:38.791 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T06:08:38.823 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects=0 2023-10-09T06:08:38.823 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 0 2023-10-09T06:08:38.823 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T06:08:38.823 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export sparse2 /tmp/rbd_import_export_704372/sparse2.out 2023-10-09T06:08:38.858 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:38.861 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_704372/sparse2 /tmp/rbd_import_export_704372/sparse2.out 2023-10-09T06:08:38.861 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_704372/sparse2 /tmp/rbd_import_export_704372/sparse2.out 2023-10-09T06:08:38.863 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/sparse2 --format %b 2023-10-09T06:08:38.864 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=2048 2023-10-09T06:08:38.864 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/sparse2.out --format %b 2023-10-09T06:08:38.865 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=2048 2023-10-09T06:08:38.865 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T06:08:38.865 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T06:08:38.866 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:08:38.866 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/sparse2.out 2023-10-09T06:08:38.866 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse2 2023-10-09T06:08:38.952 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T06:08:38.955 INFO:tasks.workunit.client.0.smithi008.stderr:+ truncate /tmp/rbd_import_export_704372/sparse1 -s 10M 2023-10-09T06:08:38.956 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T06:08:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:38 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/690232405' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:08:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:38 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/690232405' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:08:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:38 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2395892312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:38.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:38 smithi008 ceph-mon[184418]: osdmap e604: 8 total, 8 up, 7 in 2023-10-09T06:08:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:38 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/690232405' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:08:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:38 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/690232405' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:08:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:38 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2395892312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:38 smithi008 ceph-mon[191227]: osdmap e604: 8 total, 8 up, 7 in 2023-10-09T06:08:39.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:38 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/690232405' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:08:39.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:38 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/690232405' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:08:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:38 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2395892312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:39.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:38 smithi191 ceph-mon[161669]: osdmap e604: 8 total, 8 up, 7 in 2023-10-09T06:08:39.054 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T06:08:39.055 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:08:39.055 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:08:39.057 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd ls -l 2023-10-09T06:08:39.058 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-10-09T06:08:39.058 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep sparse1 2023-10-09T06:08:39.100 INFO:tasks.workunit.client.0.smithi008.stdout:sparse1 10 MiB 2 2023-10-09T06:08:39.101 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse1 2023-10-09T06:08:39.101 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse1 2023-10-09T06:08:39.101 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse1 2023-10-09T06:08:39.101 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T06:08:39.101 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T06:08:39.138 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.944a98fb1ee3 2023-10-09T06:08:39.139 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.944a98fb1ee3 2023-10-09T06:08:39.139 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.944a98fb1ee3\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:08:39.139 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse1 2023-10-09T06:08:39.139 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse1 2023-10-09T06:08:39.139 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T06:08:39.140 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse1 2023-10-09T06:08:39.140 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:08:39.140 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:08:39.146 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:39.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:39.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:39.174 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T06:08:39.174 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T06:08:39.174 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T06:08:39.174 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T06:08:39.174 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T06:08:39.205 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects=1 2023-10-09T06:08:39.205 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 1 2023-10-09T06:08:39.205 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T06:08:39.205 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export sparse1 /tmp/rbd_import_export_704372/sparse1.out 2023-10-09T06:08:39.240 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:39.244 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_704372/sparse1 /tmp/rbd_import_export_704372/sparse1.out 2023-10-09T06:08:39.244 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_704372/sparse1 /tmp/rbd_import_export_704372/sparse1.out 2023-10-09T06:08:39.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:39.252 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/sparse1 --format %b 2023-10-09T06:08:39.253 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=2048 2023-10-09T06:08:39.253 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/sparse1.out --format %b 2023-10-09T06:08:39.254 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=2048 2023-10-09T06:08:39.254 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T06:08:39.254 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T06:08:39.255 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:08:39.255 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/sparse1.out 2023-10-09T06:08:39.256 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse1 2023-10-09T06:08:39.277 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:39.319 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:39.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:39.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:39.406 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T06:08:39.409 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_704372/sparse2 oflag=append conv=notrunc 2023-10-09T06:08:39.420 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records in 2023-10-09T06:08:39.420 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records out 2023-10-09T06:08:39.420 INFO:tasks.workunit.client.0.smithi008.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0100855 s, 208 MB/s 2023-10-09T06:08:39.420 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T06:08:39.424 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:39.454 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:39.502 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T06:08:39.503 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:08:39.503 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:08:39.506 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd ls -l 2023-10-09T06:08:39.506 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep sparse2 2023-10-09T06:08:39.506 INFO:tasks.workunit.client.0.smithi008.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-10-09T06:08:39.540 INFO:tasks.workunit.client.0.smithi008.stdout:sparse2 4 MiB 2 2023-10-09T06:08:39.540 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse2 2023-10-09T06:08:39.541 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse2 2023-10-09T06:08:39.541 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse2 2023-10-09T06:08:39.541 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T06:08:39.541 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T06:08:39.574 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.9477d93d08b8 2023-10-09T06:08:39.575 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.9477d93d08b8 2023-10-09T06:08:39.575 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse2 2023-10-09T06:08:39.575 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.9477d93d08b8\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:08:39.575 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse2 2023-10-09T06:08:39.575 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T06:08:39.576 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse2 2023-10-09T06:08:39.576 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:08:39.576 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:08:39.610 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T06:08:39.610 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T06:08:39.610 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T06:08:39.610 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T06:08:39.610 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T06:08:39.641 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects='0 2023-10-09T06:08:39.641 INFO:tasks.workunit.client.0.smithi008.stderr:2 2023-10-09T06:08:39.641 INFO:tasks.workunit.client.0.smithi008.stderr:3' 2023-10-09T06:08:39.641 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 0 2 3 2023-10-09T06:08:39.641 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-10-09T06:08:39.641 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export sparse2 /tmp/rbd_import_export_704372/sparse2.out 2023-10-09T06:08:39.677 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T06:08:39.681 INFO:tasks.workunit.client.0.smithi008.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_704372/sparse2 /tmp/rbd_import_export_704372/sparse2.out 2023-10-09T06:08:39.681 INFO:tasks.workunit.client.0.smithi008.stderr:+ cmp -l /tmp/rbd_import_export_704372/sparse2 /tmp/rbd_import_export_704372/sparse2.out 2023-10-09T06:08:39.683 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/sparse2 --format %b 2023-10-09T06:08:39.684 INFO:tasks.workunit.client.0.smithi008.stderr:+ origsize=6144 2023-10-09T06:08:39.685 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/sparse2.out --format %b 2023-10-09T06:08:39.686 INFO:tasks.workunit.client.0.smithi008.stderr:+ exportsize=6144 2023-10-09T06:08:39.686 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T06:08:39.686 INFO:tasks.workunit.client.0.smithi008.stderr:+ difference=0 2023-10-09T06:08:39.686 INFO:tasks.workunit.client.0.smithi008.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:08:39.686 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/sparse2.out 2023-10-09T06:08:39.687 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse2 2023-10-09T06:08:39.773 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T06:08:39.775 INFO:tasks.workunit.client.0.smithi008.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-10-09T06:08:39.776 INFO:tasks.workunit.client.0.smithi008.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T06:08:39.776 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_704372/sparse1 sparse 2023-10-09T06:08:39.810 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:39.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:39.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:39.839 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T06:08:39.839 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:08:39.839 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:08:39.842 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse 2023-10-09T06:08:39.842 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse 2023-10-09T06:08:39.843 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse 2023-10-09T06:08:39.843 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T06:08:39.843 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T06:08:39.877 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.e1ca6534045f 2023-10-09T06:08:39.878 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.e1ca6534045f 2023-10-09T06:08:39.878 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse 2023-10-09T06:08:39.878 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse 2023-10-09T06:08:39.878 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.e1ca6534045f\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:08:39.878 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T06:08:39.879 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse 2023-10-09T06:08:39.879 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:08:39.879 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:08:39.914 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:39.917 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T06:08:39.917 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T06:08:39.917 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T06:08:39.917 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T06:08:39.917 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T06:08:39.944 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:39.950 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects=1 2023-10-09T06:08:39.950 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 1 2023-10-09T06:08:39.951 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T06:08:39.951 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse 2023-10-09T06:08:39.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:39 smithi008 ceph-mon[184418]: failed to encode map e604 with expected crc 2023-10-09T06:08:39.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:39 smithi008 ceph-mon[184418]: failed to encode map e604 with expected crc 2023-10-09T06:08:39.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:39 smithi008 ceph-mon[184418]: failed to encode map e604 with expected crc 2023-10-09T06:08:39.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:39 smithi008 ceph-mon[184418]: failed to encode map e604 with expected crc 2023-10-09T06:08:39.982 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:39 smithi008 ceph-mon[184418]: pgmap v1810: 73 pgs: 3 peering, 70 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 84 KiB/s rd, 8.8 KiB/s wr, 114 op/s 2023-10-09T06:08:39.982 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:39 smithi008 ceph-mon[191227]: failed to encode map e604 with expected crc 2023-10-09T06:08:39.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:39 smithi008 ceph-mon[191227]: failed to encode map e604 with expected crc 2023-10-09T06:08:39.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:39 smithi008 ceph-mon[191227]: failed to encode map e604 with expected crc 2023-10-09T06:08:39.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:39 smithi008 ceph-mon[191227]: failed to encode map e604 with expected crc 2023-10-09T06:08:39.983 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:39 smithi008 ceph-mon[191227]: pgmap v1810: 73 pgs: 3 peering, 70 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 84 KiB/s rd, 8.8 KiB/s wr, 114 op/s 2023-10-09T06:08:40.046 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T06:08:40.049 INFO:tasks.workunit.client.0.smithi008.stdout:zeros import through stdin to sparse image 2023-10-09T06:08:40.049 INFO:tasks.workunit.client.0.smithi008.stderr:+ echo 'zeros import through stdin to sparse image' 2023-10-09T06:08:40.050 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T06:08:40.050 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd import --new-format - sparse 2023-10-09T06:08:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:39 smithi191 ceph-mon[161669]: failed to encode map e604 with expected crc 2023-10-09T06:08:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:39 smithi191 ceph-mon[161669]: failed to encode map e604 with expected crc 2023-10-09T06:08:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:39 smithi191 ceph-mon[161669]: failed to encode map e604 with expected crc 2023-10-09T06:08:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:39 smithi191 ceph-mon[161669]: failed to encode map e604 with expected crc 2023-10-09T06:08:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:39 smithi191 ceph-mon[161669]: pgmap v1810: 73 pgs: 3 peering, 70 active+clean; 4.7 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 84 KiB/s rd, 8.8 KiB/s wr, 114 op/s 2023-10-09T06:08:40.097 INFO:tasks.workunit.client.0.smithi008.stderr:4+0 records in 2023-10-09T06:08:40.097 INFO:tasks.workunit.client.0.smithi008.stderr:4+0 records out 2023-10-09T06:08:40.097 INFO:tasks.workunit.client.0.smithi008.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0471169 s, 89.0 MB/s 2023-10-09T06:08:40.103 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:40.109 INFO:tasks.workunit.client.0.smithi008.stderr: Importing image: 100% complete...done. 2023-10-09T06:08:40.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:40.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:40.111 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:08:40.114 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse 2023-10-09T06:08:40.114 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse 2023-10-09T06:08:40.115 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse 2023-10-09T06:08:40.115 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T06:08:40.115 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T06:08:40.158 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.bbb2c710c8ea 2023-10-09T06:08:40.158 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.bbb2c710c8ea 2023-10-09T06:08:40.158 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse 2023-10-09T06:08:40.158 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse 2023-10-09T06:08:40.158 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.bbb2c710c8ea\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:08:40.158 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T06:08:40.159 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse 2023-10-09T06:08:40.159 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:08:40.159 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:08:40.183 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:40.199 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T06:08:40.199 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T06:08:40.199 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T06:08:40.199 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T06:08:40.199 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T06:08:40.212 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:40.238 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects= 2023-10-09T06:08:40.238 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 2023-10-09T06:08:40.238 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-10-09T06:08:40.239 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse 2023-10-09T06:08:40.249 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:40.258 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:40.258 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:40.332 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T06:08:40.334 INFO:tasks.workunit.client.0.smithi008.stdout:zeros export to sparse file 2023-10-09T06:08:40.335 INFO:tasks.workunit.client.0.smithi008.stderr:+ echo 'zeros export to sparse file' 2023-10-09T06:08:40.335 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T06:08:40.378 INFO:tasks.workunit.client.0.smithi008.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:08:40.381 INFO:tasks.workunit.client.0.smithi008.stderr:++ rbd info sparse 2023-10-09T06:08:40.381 INFO:tasks.workunit.client.0.smithi008.stderr:++ grep block_name_prefix 2023-10-09T06:08:40.381 INFO:tasks.workunit.client.0.smithi008.stderr:++ awk '{print $NF;}' 2023-10-09T06:08:40.416 INFO:tasks.workunit.client.0.smithi008.stderr:+ prefix=rbd_data.94e322a01362 2023-10-09T06:08:40.416 INFO:tasks.workunit.client.0.smithi008.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T06:08:40.416 INFO:tasks.workunit.client.0.smithi008.stderr:++ get_image_data_pool sparse 2023-10-09T06:08:40.417 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse 2023-10-09T06:08:40.417 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse 2023-10-09T06:08:40.417 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep 'data_pool: ' 2023-10-09T06:08:40.417 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk -F: '{ print $NF }' 2023-10-09T06:08:40.452 INFO:tasks.workunit.client.0.smithi008.stderr:++ data_pool= 2023-10-09T06:08:40.452 INFO:tasks.workunit.client.0.smithi008.stderr:++ '[' -z ']' 2023-10-09T06:08:40.452 INFO:tasks.workunit.client.0.smithi008.stderr:++ data_pool=rbd 2023-10-09T06:08:40.452 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo rbd 2023-10-09T06:08:40.452 INFO:tasks.workunit.client.0.smithi008.stderr:+ rados -p rbd put rbd_data.94e322a01362.000000000000 - 2023-10-09T06:08:40.482 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records in 2023-10-09T06:08:40.482 INFO:tasks.workunit.client.0.smithi008.stderr:1+0 records out 2023-10-09T06:08:40.482 INFO:tasks.workunit.client.0.smithi008.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0649171 s, 64.6 MB/s 2023-10-09T06:08:40.497 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects sparse 2023-10-09T06:08:40.497 INFO:tasks.workunit.client.0.smithi008.stderr:++ image=sparse 2023-10-09T06:08:40.498 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rbd info sparse 2023-10-09T06:08:40.498 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep block_name_prefix 2023-10-09T06:08:40.498 INFO:tasks.workunit.client.0.smithi008.stderr:+++ awk '{print $NF;}' 2023-10-09T06:08:40.532 INFO:tasks.workunit.client.0.smithi008.stderr:++ prefix=rbd_data.94e322a01362 2023-10-09T06:08:40.533 INFO:tasks.workunit.client.0.smithi008.stderr:+++ grep rbd_data.94e322a01362 2023-10-09T06:08:40.533 INFO:tasks.workunit.client.0.smithi008.stderr:++++ get_image_data_pool sparse 2023-10-09T06:08:40.533 INFO:tasks.workunit.client.0.smithi008.stderr:++++ image=sparse 2023-10-09T06:08:40.533 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sed -e 's/rbd_data.94e322a01362\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:08:40.533 INFO:tasks.workunit.client.0.smithi008.stderr:+++ sort -u 2023-10-09T06:08:40.533 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ rbd info sparse 2023-10-09T06:08:40.534 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:08:40.534 INFO:tasks.workunit.client.0.smithi008.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:08:40.567 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool= 2023-10-09T06:08:40.567 INFO:tasks.workunit.client.0.smithi008.stderr:++++ '[' -z ']' 2023-10-09T06:08:40.568 INFO:tasks.workunit.client.0.smithi008.stderr:++++ data_pool=rbd 2023-10-09T06:08:40.568 INFO:tasks.workunit.client.0.smithi008.stderr:++++ echo rbd 2023-10-09T06:08:40.568 INFO:tasks.workunit.client.0.smithi008.stderr:+++ rados ls -p rbd 2023-10-09T06:08:40.598 INFO:tasks.workunit.client.0.smithi008.stderr:++ objects=0 2023-10-09T06:08:40.598 INFO:tasks.workunit.client.0.smithi008.stderr:++ echo 0 2023-10-09T06:08:40.599 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T06:08:40.599 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/sparse 2023-10-09T06:08:40.599 INFO:tasks.workunit.client.0.smithi008.stderr:rm: cannot remove '/tmp/rbd_import_export_704372/sparse': No such file or directory 2023-10-09T06:08:40.599 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T06:08:40.599 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd export sparse /tmp/rbd_import_export_704372/sparse 2023-10-09T06:08:40.634 INFO:tasks.workunit.client.0.smithi008.stderr: Exporting image: 100% complete...done. 2023-10-09T06:08:40.637 INFO:tasks.workunit.client.0.smithi008.stderr:++ stat /tmp/rbd_import_export_704372/sparse --format=%b 2023-10-09T06:08:40.638 INFO:tasks.workunit.client.0.smithi008.stderr:+ '[' 0 = 0 ']' 2023-10-09T06:08:40.638 INFO:tasks.workunit.client.0.smithi008.stderr:+ rbd rm sparse 2023-10-09T06:08:40.702 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:40.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:40.722 INFO:tasks.workunit.client.0.smithi008.stderr: Removing image: 100% complete...done. 2023-10-09T06:08:40.725 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm /tmp/rbd_import_export_704372/sparse /tmp/rbd_import_export_704372/sparse1 /tmp/rbd_import_export_704372/sparse2 /tmp/rbd_import_export_704372/sparse3 2023-10-09T06:08:40.727 INFO:tasks.workunit.client.0.smithi008.stderr:rm: cannot remove '/tmp/rbd_import_export_704372/sparse3': No such file or directory 2023-10-09T06:08:40.728 INFO:tasks.workunit.client.0.smithi008.stderr:+ true 2023-10-09T06:08:40.728 INFO:tasks.workunit.client.0.smithi008.stdout:OK 2023-10-09T06:08:40.729 INFO:tasks.workunit.client.0.smithi008.stderr:+ echo OK 2023-10-09T06:08:40.729 INFO:tasks.workunit.client.0.smithi008.stderr:+ rm -rf /tmp/rbd_import_export_704372 2023-10-09T06:08:40.729 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T06:08:40.729 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T06:08:40.794 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:08:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:08:40] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.33.4" 2023-10-09T06:08:40.806 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:40.833 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:40.946 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T06:08:40.946 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:08:41.112 INFO:tasks.thrashosds.thrasher:in_osds: [2, 4, 1, 7, 3, 0, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-09T06:08:41.113 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:08:41.113 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T06:08:41.113 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:08:41.113 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:08:41.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T06:08:41.425 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:41 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:08:41.425 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:41 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:08:41.446 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:41.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:41.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:41.461 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:08:41.541 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:08:41.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:41 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:08:41.555 DEBUG:teuthology.orchestra.run.smithi008:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:08:41.612 INFO:teuthology.orchestra.run.smithi008.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:41.617 DEBUG:teuthology.parallel:result is None 2023-10-09T06:08:41.617 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:08:41.697 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:08:41.698 DEBUG:teuthology.orchestra.run.smithi008:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:08:41.717 INFO:teuthology.orchestra.run.smithi008.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:08:41.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:41.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi008 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:08:41.738 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:08:41.738 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:08:41.738 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T06:08:41.738 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:08:41.739 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T06:08:41.739 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:08:41.739 DEBUG:teuthology.orchestra.run.smithi008:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:08:41.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:08:41.797 INFO:teuthology.orchestra.run.smithi008.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:08:41.798 DEBUG:teuthology.orchestra.run.smithi008:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:08:41.821 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:41.850 INFO:teuthology.orchestra.run.smithi191.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:41.856 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:08:41.856 DEBUG:teuthology.orchestra.run.smithi008:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:08:41.914 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T06:08:41.996 INFO:teuthology.orchestra.run.smithi191.stderr:Error: no container with name or ID ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:41.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:41.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 125: 'sudo docker kill -s 1 ceph-9e85b874-6662-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:42.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:42 smithi191 ceph-mon[161669]: pgmap v1811: 73 pgs: 73 active+clean; 4.6 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 415 KiB/s rd, 348 KiB/s wr, 100 op/s 2023-10-09T06:08:42.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:42 smithi008 ceph-mon[184418]: pgmap v1811: 73 pgs: 73 active+clean; 4.6 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 415 KiB/s rd, 348 KiB/s wr, 100 op/s 2023-10-09T06:08:42.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:42 smithi008 ceph-mon[191227]: pgmap v1811: 73 pgs: 73 active+clean; 4.6 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 415 KiB/s rd, 348 KiB/s wr, 100 op/s 2023-10-09T06:08:42.616 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:43.094 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-10-09T06:08:43.094 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "" 2023-10-09T06:08:43.094 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-10-09T06:08:43.653 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:08:44.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[184418]: pgmap v1812: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.4 MiB/s wr, 364 op/s 2023-10-09T06:08:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1888589126' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[184418]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:08:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T06:08:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[184418]: failed to encode map e605 with expected crc 2023-10-09T06:08:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1888589126' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[184418]: osdmap e605: 8 total, 8 up, 7 in 2023-10-09T06:08:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:44.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[191227]: pgmap v1812: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.4 MiB/s wr, 364 op/s 2023-10-09T06:08:44.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1888589126' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[191227]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:08:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T06:08:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[191227]: failed to encode map e605 with expected crc 2023-10-09T06:08:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1888589126' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[191227]: osdmap e605: 8 total, 8 up, 7 in 2023-10-09T06:08:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:44.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:44.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:44 smithi191 ceph-mon[161669]: pgmap v1812: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.4 MiB/s wr, 364 op/s 2023-10-09T06:08:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1888589126' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:44 smithi191 ceph-mon[161669]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:08:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:44 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T06:08:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:44 smithi191 ceph-mon[161669]: failed to encode map e605 with expected crc 2023-10-09T06:08:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1888589126' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:44 smithi191 ceph-mon[161669]: osdmap e605: 8 total, 8 up, 7 in 2023-10-09T06:08:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:45.226 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:45.226 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:45.452 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:45 smithi008 ceph-mon[184418]: failed to encode map e605 with expected crc 2023-10-09T06:08:45.453 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:45 smithi008 ceph-mon[184418]: failed to encode map e605 with expected crc 2023-10-09T06:08:45.453 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:45 smithi008 ceph-mon[191227]: failed to encode map e605 with expected crc 2023-10-09T06:08:45.453 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:45 smithi008 ceph-mon[191227]: failed to encode map e605 with expected crc 2023-10-09T06:08:45.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:45 smithi191 ceph-mon[161669]: failed to encode map e605 with expected crc 2023-10-09T06:08:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:45 smithi191 ceph-mon[161669]: failed to encode map e605 with expected crc 2023-10-09T06:08:45.978 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:08:46.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:46 smithi008 ceph-mon[184418]: pgmap v1814: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 1012 KiB/s rd, 1.4 MiB/s wr, 326 op/s 2023-10-09T06:08:46.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-10-09T06:08:46.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:46 smithi008 ceph-mon[191227]: pgmap v1814: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 1012 KiB/s rd, 1.4 MiB/s wr, 326 op/s 2023-10-09T06:08:46.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-10-09T06:08:46.350 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:46 smithi191 ceph-mon[161669]: pgmap v1814: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 1012 KiB/s rd, 1.4 MiB/s wr, 326 op/s 2023-10-09T06:08:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-10-09T06:08:46.827 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:08:47.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:47 smithi008 ceph-mon[184418]: failed to encode map e606 with expected crc 2023-10-09T06:08:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:47 smithi008 ceph-mon[184418]: failed to encode map e606 with expected crc 2023-10-09T06:08:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-10-09T06:08:47.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:47 smithi008 ceph-mon[184418]: osdmap e606: 8 total, 8 up, 7 in 2023-10-09T06:08:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:47 smithi008 ceph-mon[191227]: failed to encode map e606 with expected crc 2023-10-09T06:08:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:47 smithi008 ceph-mon[191227]: failed to encode map e606 with expected crc 2023-10-09T06:08:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-10-09T06:08:47.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:47 smithi008 ceph-mon[191227]: osdmap e606: 8 total, 8 up, 7 in 2023-10-09T06:08:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:47 smithi191 ceph-mon[161669]: failed to encode map e606 with expected crc 2023-10-09T06:08:47.600 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:47 smithi191 ceph-mon[161669]: failed to encode map e606 with expected crc 2023-10-09T06:08:47.600 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-10-09T06:08:47.600 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:47 smithi191 ceph-mon[161669]: osdmap e606: 8 total, 8 up, 7 in 2023-10-09T06:08:47.606 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:48 smithi008 ceph-mon[184418]: pgmap v1816: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 1009 KiB/s rd, 1.4 MiB/s wr, 320 op/s 2023-10-09T06:08:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:48 smithi008 ceph-mon[184418]: failed to encode map e607 with expected crc 2023-10-09T06:08:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:48 smithi008 ceph-mon[184418]: failed to encode map e607 with expected crc 2023-10-09T06:08:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:48 smithi008 ceph-mon[184418]: osdmap e607: 8 total, 8 up, 7 in 2023-10-09T06:08:48.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:48 smithi008 ceph-mon[184418]: failed to encode map e607 with expected crc 2023-10-09T06:08:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:48 smithi008 ceph-mon[191227]: pgmap v1816: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 1009 KiB/s rd, 1.4 MiB/s wr, 320 op/s 2023-10-09T06:08:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:48 smithi008 ceph-mon[191227]: failed to encode map e607 with expected crc 2023-10-09T06:08:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:48 smithi008 ceph-mon[191227]: failed to encode map e607 with expected crc 2023-10-09T06:08:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:48 smithi008 ceph-mon[191227]: osdmap e607: 8 total, 8 up, 7 in 2023-10-09T06:08:48.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:48 smithi008 ceph-mon[191227]: failed to encode map e607 with expected crc 2023-10-09T06:08:48.284 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:48.284 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:48 smithi191 ceph-mon[161669]: pgmap v1816: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 1009 KiB/s rd, 1.4 MiB/s wr, 320 op/s 2023-10-09T06:08:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:48 smithi191 ceph-mon[161669]: failed to encode map e607 with expected crc 2023-10-09T06:08:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:48 smithi191 ceph-mon[161669]: failed to encode map e607 with expected crc 2023-10-09T06:08:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:48 smithi191 ceph-mon[161669]: osdmap e607: 8 total, 8 up, 7 in 2023-10-09T06:08:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:48 smithi191 ceph-mon[161669]: failed to encode map e607 with expected crc 2023-10-09T06:08:48.693 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T06:08:48.693 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-10-09T06:08:48.694 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd in 5 2023-10-09T06:08:49.719 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:08:49.805 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:50.822 INFO:teuthology.orchestra.run.smithi008.stderr:noscrub is unset 2023-10-09T06:08:50.848 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:50 smithi008 ceph-mon[184418]: pgmap v1818: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 47 KiB/s rd, 174 KiB/s wr, 68 op/s 2023-10-09T06:08:50.848 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:50 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:50.848 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:50 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4138996582' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:50.849 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:08:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:08:50] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.33.4" 2023-10-09T06:08:50.849 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:50 smithi008 ceph-mon[191227]: pgmap v1818: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 47 KiB/s rd, 174 KiB/s wr, 68 op/s 2023-10-09T06:08:50.849 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:50 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:50.849 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:50 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4138996582' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:51.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:50 smithi191 ceph-mon[161669]: pgmap v1818: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 47 KiB/s rd, 174 KiB/s wr, 68 op/s 2023-10-09T06:08:51.062 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:50 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:08:51.062 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:50 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4138996582' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:51.526 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:51.726 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:08:51.946 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:51 smithi008 ceph-mon[184418]: failed to encode map e608 with expected crc 2023-10-09T06:08:51.946 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:51 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4138996582' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:51.946 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:51 smithi008 ceph-mon[184418]: osdmap e608: 8 total, 8 up, 7 in 2023-10-09T06:08:51.947 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:51 smithi008 ceph-mon[184418]: failed to encode map e608 with expected crc 2023-10-09T06:08:51.947 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:51 smithi008 ceph-mon[184418]: pgmap v1820: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:08:51.947 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:51 smithi008 ceph-mon[191227]: failed to encode map e608 with expected crc 2023-10-09T06:08:51.947 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:51 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4138996582' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:51.948 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:51 smithi008 ceph-mon[191227]: osdmap e608: 8 total, 8 up, 7 in 2023-10-09T06:08:51.948 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:51 smithi008 ceph-mon[191227]: failed to encode map e608 with expected crc 2023-10-09T06:08:51.948 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:51 smithi008 ceph-mon[191227]: pgmap v1820: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:08:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:51 smithi191 ceph-mon[161669]: failed to encode map e608 with expected crc 2023-10-09T06:08:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:51 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4138996582' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:51 smithi191 ceph-mon[161669]: osdmap e608: 8 total, 8 up, 7 in 2023-10-09T06:08:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:51 smithi191 ceph-mon[161669]: failed to encode map e608 with expected crc 2023-10-09T06:08:52.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:51 smithi191 ceph-mon[161669]: pgmap v1820: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:08:52.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:52 smithi008 conmon[184394]: 2023-10-09T06:08:52.451+0000 7f8a66521700 -1 mon.a@0(leader).osd e608 definitely_dead 0 2023-10-09T06:08:52.841 INFO:teuthology.orchestra.run.smithi008.stderr:marked in osd.5. 2023-10-09T06:08:53.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:52 smithi008 ceph-mon[184418]: failed to encode map e608 with expected crc 2023-10-09T06:08:53.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:52 smithi008 ceph-mon[184418]: failed to encode map e608 with expected crc 2023-10-09T06:08:53.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3355160576' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:08:53.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:52 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:08:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:52 smithi008 ceph-mon[191227]: failed to encode map e608 with expected crc 2023-10-09T06:08:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:52 smithi008 ceph-mon[191227]: failed to encode map e608 with expected crc 2023-10-09T06:08:53.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3355160576' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:08:53.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:52 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:08:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:52 smithi191 ceph-mon[161669]: failed to encode map e608 with expected crc 2023-10-09T06:08:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:52 smithi191 ceph-mon[161669]: failed to encode map e608 with expected crc 2023-10-09T06:08:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3355160576' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:08:53.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:52 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:08:53.853 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-10-09T06:08:53.853 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T06:08:53.854 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 81 2023-10-09T06:08:53.971 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:54.003 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:54.003 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:54.031 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[184418]: failed to encode map e609 with expected crc 2023-10-09T06:08:54.031 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[184418]: failed to encode map e609 with expected crc 2023-10-09T06:08:54.031 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[184418]: failed to encode map e609 with expected crc 2023-10-09T06:08:54.031 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:08:54.032 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[184418]: osdmap e609: 8 total, 8 up, 8 in 2023-10-09T06:08:54.032 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[184418]: pgmap v1822: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:08:54.032 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[184418]: osdmap e610: 8 total, 8 up, 8 in 2023-10-09T06:08:54.032 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[191227]: failed to encode map e609 with expected crc 2023-10-09T06:08:54.032 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[191227]: failed to encode map e609 with expected crc 2023-10-09T06:08:54.032 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[191227]: failed to encode map e609 with expected crc 2023-10-09T06:08:54.032 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:08:54.032 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[191227]: osdmap e609: 8 total, 8 up, 8 in 2023-10-09T06:08:54.033 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[191227]: pgmap v1822: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:08:54.033 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:53 smithi008 ceph-mon[191227]: osdmap e610: 8 total, 8 up, 8 in 2023-10-09T06:08:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:53 smithi191 ceph-mon[161669]: failed to encode map e609 with expected crc 2023-10-09T06:08:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:53 smithi191 ceph-mon[161669]: failed to encode map e609 with expected crc 2023-10-09T06:08:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:53 smithi191 ceph-mon[161669]: failed to encode map e609 with expected crc 2023-10-09T06:08:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:53 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:08:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:53 smithi191 ceph-mon[161669]: osdmap e609: 8 total, 8 up, 8 in 2023-10-09T06:08:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:53 smithi191 ceph-mon[161669]: pgmap v1822: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail 2023-10-09T06:08:54.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:53 smithi191 ceph-mon[161669]: osdmap e610: 8 total, 8 up, 8 in 2023-10-09T06:08:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[184418]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[184418]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[184418]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[184418]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[184418]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[184418]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[184418]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[184418]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[184418]: osdmap e611: 8 total, 8 up, 8 in 2023-10-09T06:08:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[191227]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[191227]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[191227]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[191227]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[191227]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[191227]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[191227]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[191227]: failed to encode map e610 with expected crc 2023-10-09T06:08:54.967 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:54 smithi008 ceph-mon[191227]: osdmap e611: 8 total, 8 up, 8 in 2023-10-09T06:08:55.018 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:08:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:54 smithi191 ceph-mon[161669]: failed to encode map e610 with expected crc 2023-10-09T06:08:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:54 smithi191 ceph-mon[161669]: failed to encode map e610 with expected crc 2023-10-09T06:08:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:54 smithi191 ceph-mon[161669]: failed to encode map e610 with expected crc 2023-10-09T06:08:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:54 smithi191 ceph-mon[161669]: failed to encode map e610 with expected crc 2023-10-09T06:08:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:54 smithi191 ceph-mon[161669]: failed to encode map e610 with expected crc 2023-10-09T06:08:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:54 smithi191 ceph-mon[161669]: failed to encode map e610 with expected crc 2023-10-09T06:08:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:54 smithi191 ceph-mon[161669]: failed to encode map e610 with expected crc 2023-10-09T06:08:55.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:54 smithi191 ceph-mon[161669]: failed to encode map e610 with expected crc 2023-10-09T06:08:55.301 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:54 smithi191 ceph-mon[161669]: osdmap e611: 8 total, 8 up, 8 in 2023-10-09T06:08:55.707 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:55.957 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:55 smithi008 ceph-mon[184418]: failed to encode map e611 with expected crc 2023-10-09T06:08:55.957 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:55 smithi008 ceph-mon[184418]: failed to encode map e611 with expected crc 2023-10-09T06:08:55.957 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:55 smithi008 ceph-mon[184418]: pgmap v1825: 73 pgs: 1 remapped+peering, 72 active+clean; 8.6 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:08:55.957 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:55 smithi008 ceph-mon[191227]: failed to encode map e611 with expected crc 2023-10-09T06:08:55.957 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:55 smithi008 ceph-mon[191227]: failed to encode map e611 with expected crc 2023-10-09T06:08:55.957 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:55 smithi008 ceph-mon[191227]: pgmap v1825: 73 pgs: 1 remapped+peering, 72 active+clean; 8.6 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:08:56.104 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:55 smithi191 ceph-mon[161669]: failed to encode map e611 with expected crc 2023-10-09T06:08:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:55 smithi191 ceph-mon[161669]: failed to encode map e611 with expected crc 2023-10-09T06:08:56.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:55 smithi191 ceph-mon[161669]: pgmap v1825: 73 pgs: 1 remapped+peering, 72 active+clean; 8.6 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:08:56.882 INFO:teuthology.orchestra.run.smithi008.stderr:nodeep-scrub is unset 2023-10-09T06:08:57.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[191227]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[191227]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[191227]: osdmap e612: 8 total, 8 up, 8 in 2023-10-09T06:08:57.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[191227]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[191227]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[191227]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4144205757' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[184418]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[184418]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[184418]: osdmap e612: 8 total, 8 up, 8 in 2023-10-09T06:08:57.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[184418]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[184418]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[184418]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.216 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:56 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4144205757' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:57.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:56 smithi191 ceph-mon[161669]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:56 smithi191 ceph-mon[161669]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:56 smithi191 ceph-mon[161669]: osdmap e612: 8 total, 8 up, 8 in 2023-10-09T06:08:57.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:56 smithi191 ceph-mon[161669]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:56 smithi191 ceph-mon[161669]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:56 smithi191 ceph-mon[161669]: failed to encode map e612 with expected crc 2023-10-09T06:08:57.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:56 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4144205757' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:58.068 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[184418]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.068 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[184418]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.068 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[184418]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.068 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4144205757' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:58.068 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[184418]: osdmap e613: 8 total, 8 up, 8 in 2023-10-09T06:08:58.068 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[184418]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.068 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[184418]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.069 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[184418]: pgmap v1828: 73 pgs: 1 remapped+peering, 72 active+clean; 8.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:08:58.069 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[191227]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.069 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[191227]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.069 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[191227]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.069 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4144205757' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:58.070 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[191227]: osdmap e613: 8 total, 8 up, 8 in 2023-10-09T06:08:58.070 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[191227]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.070 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[191227]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.070 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:57 smithi008 ceph-mon[191227]: pgmap v1828: 73 pgs: 1 remapped+peering, 72 active+clean; 8.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:08:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:57 smithi191 ceph-mon[161669]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:57 smithi191 ceph-mon[161669]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:57 smithi191 ceph-mon[161669]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:57 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4144205757' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:57 smithi191 ceph-mon[161669]: osdmap e613: 8 total, 8 up, 8 in 2023-10-09T06:08:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:57 smithi191 ceph-mon[161669]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:57 smithi191 ceph-mon[161669]: failed to encode map e613 with expected crc 2023-10-09T06:08:58.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:57 smithi191 ceph-mon[161669]: pgmap v1828: 73 pgs: 1 remapped+peering, 72 active+clean; 8.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:08:58.940 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:58.940 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:59.169 INFO:teuthology.orchestra.run.smithi008.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T06:08:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:08:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/450173474' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T06:08:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:08:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/450173474' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T06:09:00.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:00.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:08:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/450173474' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T06:09:00.083 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:09:00.248 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T06:09:00.249 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T06:09:00.466 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:00.862 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:00 smithi008 ceph-mon[184418]: pgmap v1829: 73 pgs: 1 remapped+peering, 72 active+clean; 8.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 103 B/s, 0 objects/s recovering 2023-10-09T06:09:00.862 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:00 smithi008 ceph-mon[191227]: pgmap v1829: 73 pgs: 1 remapped+peering, 72 active+clean; 8.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 103 B/s, 0 objects/s recovering 2023-10-09T06:09:00.863 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:09:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:09:00] "GET /metrics HTTP/1.1" 200 34175 "" "Prometheus/2.33.4" 2023-10-09T06:09:00.901 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:00 smithi191 ceph-mon[161669]: pgmap v1829: 73 pgs: 1 remapped+peering, 72 active+clean; 8.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 103 B/s, 0 objects/s recovering 2023-10-09T06:09:02.539 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:02.539 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:02.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:02 smithi008 ceph-mon[184418]: pgmap v1830: 73 pgs: 73 active+clean; 8.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 90 B/s, 5 objects/s recovering 2023-10-09T06:09:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:02 smithi008 ceph-mon[191227]: pgmap v1830: 73 pgs: 73 active+clean; 8.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 90 B/s, 5 objects/s recovering 2023-10-09T06:09:03.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:02 smithi191 ceph-mon[161669]: pgmap v1830: 73 pgs: 73 active+clean; 8.6 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 90 B/s, 5 objects/s recovering 2023-10-09T06:09:03.725 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:09:04.032 INFO:teuthology.orchestra.run.smithi008.stdout:mon.a: {} 2023-10-09T06:09:04.043 INFO:teuthology.orchestra.run.smithi008.stdout:mon.b: {} 2023-10-09T06:09:04.053 INFO:teuthology.orchestra.run.smithi008.stdout:mon.c: {} 2023-10-09T06:09:04.055 INFO:teuthology.orchestra.run.smithi008.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:09:04.055 INFO:teuthology.orchestra.run.smithi008.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:09:04.055 INFO:teuthology.orchestra.run.smithi008.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:09:04.095 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:04.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:04 smithi008 ceph-mon[184418]: pgmap v1831: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 71 B/s, 4 objects/s recovering 2023-10-09T06:09:04.929 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2644960366' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:09:04.930 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:04 smithi008 ceph-mon[191227]: pgmap v1831: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 71 B/s, 4 objects/s recovering 2023-10-09T06:09:04.930 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2644960366' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:09:04.951 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:09:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:04 smithi191 ceph-mon[161669]: pgmap v1831: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 71 B/s, 4 objects/s recovering 2023-10-09T06:09:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2644960366' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:09:05.863 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:06.304 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:06.304 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:06.663 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:06 smithi008 ceph-mon[191227]: pgmap v1832: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 62 B/s, 3 objects/s recovering 2023-10-09T06:09:06.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:06 smithi008 ceph-mon[184418]: pgmap v1832: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 62 B/s, 3 objects/s recovering 2023-10-09T06:09:07.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:06 smithi191 ceph-mon[161669]: pgmap v1832: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 62 B/s, 3 objects/s recovering 2023-10-09T06:09:07.690 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:09:08.184 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:08.671 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:09:08.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:08 smithi008 ceph-mon[184418]: pgmap v1833: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 56 B/s, 3 objects/s recovering 2023-10-09T06:09:08.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:08 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1926650624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:08.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:08 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:08.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:08 smithi008 ceph-mon[191227]: pgmap v1833: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 56 B/s, 3 objects/s recovering 2023-10-09T06:09:08.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:08 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1926650624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:08.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:08 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:09.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:08 smithi191 ceph-mon[161669]: pgmap v1833: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 56 B/s, 3 objects/s recovering 2023-10-09T06:09:09.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:08 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1926650624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:09.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:08 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:09.789 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T06:09:09.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:09 smithi008 ceph-mon[184418]: failed to encode map e614 with expected crc 2023-10-09T06:09:09.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:09 smithi008 ceph-mon[184418]: failed to encode map e614 with expected crc 2023-10-09T06:09:09.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:09 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:09:09.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:09 smithi008 ceph-mon[184418]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T06:09:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:09 smithi008 ceph-mon[191227]: failed to encode map e614 with expected crc 2023-10-09T06:09:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:09 smithi008 ceph-mon[191227]: failed to encode map e614 with expected crc 2023-10-09T06:09:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:09 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:09:09.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:09 smithi008 ceph-mon[191227]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T06:09:10.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:09 smithi191 ceph-mon[161669]: failed to encode map e614 with expected crc 2023-10-09T06:09:10.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:09 smithi191 ceph-mon[161669]: failed to encode map e614 with expected crc 2023-10-09T06:09:10.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:09 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:09:10.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:09 smithi191 ceph-mon[161669]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T06:09:10.218 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:10.222 INFO:teuthology.orchestra.run.smithi008.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:10.222 INFO:teuthology.orchestra.run.smithi008.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:10 smithi008 ceph-mon[184418]: pgmap v1835: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 1 B/s, 3 objects/s recovering 2023-10-09T06:09:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:10 smithi008 ceph-mon[184418]: failed to encode map e614 with expected crc 2023-10-09T06:09:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:10 smithi008 ceph-mon[184418]: failed to encode map e614 with expected crc 2023-10-09T06:09:10.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:10 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:10.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:09:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:09:10] "GET /metrics HTTP/1.1" 200 34175 "" "Prometheus/2.33.4" 2023-10-09T06:09:10.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:10 smithi008 ceph-mon[191227]: pgmap v1835: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 1 B/s, 3 objects/s recovering 2023-10-09T06:09:10.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:10 smithi008 ceph-mon[191227]: failed to encode map e614 with expected crc 2023-10-09T06:09:10.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:10 smithi008 ceph-mon[191227]: failed to encode map e614 with expected crc 2023-10-09T06:09:10.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:10 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:10 smithi191 ceph-mon[161669]: pgmap v1835: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 1 B/s, 3 objects/s recovering 2023-10-09T06:09:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:10 smithi191 ceph-mon[161669]: failed to encode map e614 with expected crc 2023-10-09T06:09:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:10 smithi191 ceph-mon[161669]: failed to encode map e614 with expected crc 2023-10-09T06:09:11.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:10 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:11.259 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:09:11.622 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:12.939 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:12 smithi191 ceph-mon[161669]: pgmap v1836: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:12.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:12 smithi008 ceph-mon[184418]: pgmap v1836: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:12.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:12 smithi008 ceph-mon[191227]: pgmap v1836: 73 pgs: 73 active+clean; 8.6 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:13.142 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:13.143 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:13.693 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T06:09:13.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:09:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:13 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/360077018' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:13.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:13 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:13.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:13 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/360077018' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:13.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:13 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:13 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/360077018' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:14.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:13 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:09:14.111 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:14.521 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:09:14.925 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:14.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[184418]: pgmap v1837: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:14.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[184418]: failed to encode map e615 with expected crc 2023-10-09T06:09:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[184418]: failed to encode map e615 with expected crc 2023-10-09T06:09:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:09:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[184418]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T06:09:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[184418]: failed to encode map e615 with expected crc 2023-10-09T06:09:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[184418]: failed to encode map e615 with expected crc 2023-10-09T06:09:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:14.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:14.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[191227]: pgmap v1837: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:14.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[191227]: failed to encode map e615 with expected crc 2023-10-09T06:09:14.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[191227]: failed to encode map e615 with expected crc 2023-10-09T06:09:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:09:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[191227]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T06:09:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[191227]: failed to encode map e615 with expected crc 2023-10-09T06:09:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[191227]: failed to encode map e615 with expected crc 2023-10-09T06:09:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:14.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:15.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:14 smithi191 ceph-mon[161669]: pgmap v1837: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:14 smithi191 ceph-mon[161669]: failed to encode map e615 with expected crc 2023-10-09T06:09:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:14 smithi191 ceph-mon[161669]: failed to encode map e615 with expected crc 2023-10-09T06:09:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:14 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:09:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:14 smithi191 ceph-mon[161669]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T06:09:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:14 smithi191 ceph-mon[161669]: failed to encode map e615 with expected crc 2023-10-09T06:09:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:14 smithi191 ceph-mon[161669]: failed to encode map e615 with expected crc 2023-10-09T06:09:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:15.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:15.577 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:15.577 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:15.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:15 smithi008 ceph-mon[191227]: failed to encode map e616 with expected crc 2023-10-09T06:09:15.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:15 smithi008 ceph-mon[191227]: failed to encode map e616 with expected crc 2023-10-09T06:09:15.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:15 smithi008 ceph-mon[191227]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T06:09:15.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:15 smithi008 ceph-mon[191227]: failed to encode map e616 with expected crc 2023-10-09T06:09:15.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:15 smithi008 ceph-mon[191227]: failed to encode map e616 with expected crc 2023-10-09T06:09:15.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:15 smithi008 ceph-mon[184418]: failed to encode map e616 with expected crc 2023-10-09T06:09:15.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:15 smithi008 ceph-mon[184418]: failed to encode map e616 with expected crc 2023-10-09T06:09:15.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:15 smithi008 ceph-mon[184418]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T06:09:15.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:15 smithi008 ceph-mon[184418]: failed to encode map e616 with expected crc 2023-10-09T06:09:15.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:15 smithi008 ceph-mon[184418]: failed to encode map e616 with expected crc 2023-10-09T06:09:16.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:15 smithi191 ceph-mon[161669]: failed to encode map e616 with expected crc 2023-10-09T06:09:16.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:15 smithi191 ceph-mon[161669]: failed to encode map e616 with expected crc 2023-10-09T06:09:16.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:15 smithi191 ceph-mon[161669]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T06:09:16.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:15 smithi191 ceph-mon[161669]: failed to encode map e616 with expected crc 2023-10-09T06:09:16.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:15 smithi191 ceph-mon[161669]: failed to encode map e616 with expected crc 2023-10-09T06:09:16.385 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:09:16.929 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:16.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:16 smithi008 ceph-mon[184418]: pgmap v1840: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:16.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:16 smithi008 ceph-mon[191227]: pgmap v1840: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:17.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:16 smithi191 ceph-mon[161669]: pgmap v1840: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:17.693 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:17 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4252270239' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:09:17.694 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:17 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:09:17.706 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:09:17.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:17 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4252270239' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:09:17.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:17 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:09:17.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:17 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4252270239' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:09:17.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:17 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:09:18.604 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:18.604 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:18.752 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T06:09:18.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:18 smithi008 ceph-mon[184418]: pgmap v1841: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:18.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:18 smithi008 ceph-mon[184418]: failed to encode map e617 with expected crc 2023-10-09T06:09:18.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:18 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:09:18.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:18 smithi008 ceph-mon[184418]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T06:09:18.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:18 smithi008 ceph-mon[184418]: failed to encode map e617 with expected crc 2023-10-09T06:09:18.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:18 smithi008 ceph-mon[191227]: pgmap v1841: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:18.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:18 smithi008 ceph-mon[191227]: failed to encode map e617 with expected crc 2023-10-09T06:09:18.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:18 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:09:18.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:18 smithi008 ceph-mon[191227]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T06:09:18.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:18 smithi008 ceph-mon[191227]: failed to encode map e617 with expected crc 2023-10-09T06:09:19.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:18 smithi191 ceph-mon[161669]: pgmap v1841: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:18 smithi191 ceph-mon[161669]: failed to encode map e617 with expected crc 2023-10-09T06:09:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:18 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:09:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:18 smithi191 ceph-mon[161669]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T06:09:19.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:18 smithi191 ceph-mon[161669]: failed to encode map e617 with expected crc 2023-10-09T06:09:19.300 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:19.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:09:19.799 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:19 smithi191 ceph-mon[161669]: failed to encode map e617 with expected crc 2023-10-09T06:09:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:19 smithi191 ceph-mon[161669]: failed to encode map e617 with expected crc 2023-10-09T06:09:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:19 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:19.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:19 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:19.872 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:19 smithi008 ceph-mon[184418]: failed to encode map e617 with expected crc 2023-10-09T06:09:19.872 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:19 smithi008 ceph-mon[184418]: failed to encode map e617 with expected crc 2023-10-09T06:09:19.872 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:19 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:19.872 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:19 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:19.872 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:19 smithi008 ceph-mon[191227]: failed to encode map e617 with expected crc 2023-10-09T06:09:19.872 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:19 smithi008 ceph-mon[191227]: failed to encode map e617 with expected crc 2023-10-09T06:09:19.873 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:19 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:19.873 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:19 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:20.012 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:20.674 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:09:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:09:20] "GET /metrics HTTP/1.1" 200 34173 "" "Prometheus/2.33.4" 2023-10-09T06:09:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:20 smithi191 ceph-mon[161669]: pgmap v1843: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:09:20.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[184418]: pgmap v1843: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:20.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:09:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[191227]: pgmap v1843: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:09:20.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:21.508 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:21.508 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:21.716 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T06:09:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:21 smithi191 ceph-mon[161669]: pgmap v1844: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:21.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:21 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1449741730' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:09:21.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:21 smithi008 ceph-mon[184418]: pgmap v1844: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:21.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:21 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1449741730' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:09:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:21 smithi008 ceph-mon[191227]: pgmap v1844: 73 pgs: 73 active+clean; 8.6 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:21.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:21 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1449741730' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:09:22.203 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:09:22.553 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:22.610 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:09:22.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[191227]: failed to encode map e618 with expected crc 2023-10-09T06:09:22.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[191227]: failed to encode map e618 with expected crc 2023-10-09T06:09:22.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1449741730' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:09:22.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[191227]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T06:09:22.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[191227]: failed to encode map e618 with expected crc 2023-10-09T06:09:22.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[191227]: failed to encode map e618 with expected crc 2023-10-09T06:09:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[184418]: failed to encode map e618 with expected crc 2023-10-09T06:09:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[184418]: failed to encode map e618 with expected crc 2023-10-09T06:09:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1449741730' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:09:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[184418]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T06:09:22.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[184418]: failed to encode map e618 with expected crc 2023-10-09T06:09:22.966 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:22 smithi008 ceph-mon[184418]: failed to encode map e618 with expected crc 2023-10-09T06:09:22.991 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:22 smithi191 ceph-mon[161669]: failed to encode map e618 with expected crc 2023-10-09T06:09:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:22 smithi191 ceph-mon[161669]: failed to encode map e618 with expected crc 2023-10-09T06:09:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:22 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1449741730' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:09:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:22 smithi191 ceph-mon[161669]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T06:09:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:22 smithi191 ceph-mon[161669]: failed to encode map e618 with expected crc 2023-10-09T06:09:23.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:22 smithi191 ceph-mon[161669]: failed to encode map e618 with expected crc 2023-10-09T06:09:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:23 smithi191 ceph-mon[161669]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:23 smithi191 ceph-mon[161669]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:23 smithi191 ceph-mon[161669]: osdmap e619: 8 total, 8 up, 8 in 2023-10-09T06:09:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:23 smithi191 ceph-mon[161669]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:23 smithi191 ceph-mon[161669]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:23 smithi191 ceph-mon[161669]: pgmap v1847: 73 pgs: 73 active+clean; 8.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:23.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[184418]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[184418]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[184418]: osdmap e619: 8 total, 8 up, 8 in 2023-10-09T06:09:23.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[184418]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[184418]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[184418]: pgmap v1847: 73 pgs: 73 active+clean; 8.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[191227]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[191227]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[191227]: osdmap e619: 8 total, 8 up, 8 in 2023-10-09T06:09:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[191227]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[191227]: failed to encode map e619 with expected crc 2023-10-09T06:09:23.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:23 smithi008 ceph-mon[191227]: pgmap v1847: 73 pgs: 73 active+clean; 8.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:23.994 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:23.994 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:24.652 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:09:24.943 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:24 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1705165451' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:09:24.943 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:24 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:09:24.944 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:09:25.006 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:25.202 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:24 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1705165451' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:09:25.203 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:24 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:09:25.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:24 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1705165451' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:09:25.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:24 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:09:25.766 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T06:09:26.135 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:26.212 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:25 smithi191 ceph-mon[161669]: failed to encode map e620 with expected crc 2023-10-09T06:09:26.212 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:25 smithi191 ceph-mon[161669]: failed to encode map e620 with expected crc 2023-10-09T06:09:26.212 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:25 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:09:26.212 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:25 smithi191 ceph-mon[161669]: osdmap e620: 8 total, 8 up, 8 in 2023-10-09T06:09:26.212 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:25 smithi191 ceph-mon[161669]: pgmap v1849: 73 pgs: 73 active+clean; 8.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:26.212 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:25 smithi191 ceph-mon[161669]: failed to encode map e620 with expected crc 2023-10-09T06:09:26.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[184418]: failed to encode map e620 with expected crc 2023-10-09T06:09:26.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[184418]: failed to encode map e620 with expected crc 2023-10-09T06:09:26.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:09:26.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[184418]: osdmap e620: 8 total, 8 up, 8 in 2023-10-09T06:09:26.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[184418]: pgmap v1849: 73 pgs: 73 active+clean; 8.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:26.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[184418]: failed to encode map e620 with expected crc 2023-10-09T06:09:26.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[191227]: failed to encode map e620 with expected crc 2023-10-09T06:09:26.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[191227]: failed to encode map e620 with expected crc 2023-10-09T06:09:26.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:09:26.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[191227]: osdmap e620: 8 total, 8 up, 8 in 2023-10-09T06:09:26.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[191227]: pgmap v1849: 73 pgs: 73 active+clean; 8.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:26.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:25 smithi008 ceph-mon[191227]: failed to encode map e620 with expected crc 2023-10-09T06:09:26.409 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:26.410 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:27.052 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:09:27.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:26 smithi008 ceph-mon[184418]: failed to encode map e620 with expected crc 2023-10-09T06:09:27.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:26 smithi008 ceph-mon[191227]: failed to encode map e620 with expected crc 2023-10-09T06:09:27.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:26 smithi191 ceph-mon[161669]: failed to encode map e620 with expected crc 2023-10-09T06:09:27.408 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:28.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:27 smithi008 ceph-mon[184418]: pgmap v1850: 73 pgs: 73 active+clean; 8.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:28.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:27 smithi008 ceph-mon[191227]: pgmap v1850: 73 pgs: 73 active+clean; 8.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:28.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:27 smithi191 ceph-mon[161669]: pgmap v1850: 73 pgs: 73 active+clean; 8.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:28.801 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:28.802 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:28.967 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T06:09:29.204 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:28 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/570262682' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:09:29.204 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:28 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:29.205 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:28 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/570262682' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:09:29.205 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:28 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:29.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:28 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/570262682' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:09:29.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:28 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:29.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:09:29.785 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:09:29.814 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:30.158 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:30.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:29 smithi008 ceph-mon[184418]: failed to encode map e621 with expected crc 2023-10-09T06:09:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:29 smithi008 ceph-mon[184418]: failed to encode map e621 with expected crc 2023-10-09T06:09:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:29 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/570262682' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:09:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:29 smithi008 ceph-mon[184418]: osdmap e621: 8 total, 8 up, 8 in 2023-10-09T06:09:30.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:29 smithi008 ceph-mon[184418]: pgmap v1852: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:30.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:29 smithi008 ceph-mon[191227]: failed to encode map e621 with expected crc 2023-10-09T06:09:30.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:29 smithi008 ceph-mon[191227]: failed to encode map e621 with expected crc 2023-10-09T06:09:30.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:29 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/570262682' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:09:30.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:29 smithi008 ceph-mon[191227]: osdmap e621: 8 total, 8 up, 8 in 2023-10-09T06:09:30.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:29 smithi008 ceph-mon[191227]: pgmap v1852: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:29 smithi191 ceph-mon[161669]: failed to encode map e621 with expected crc 2023-10-09T06:09:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:29 smithi191 ceph-mon[161669]: failed to encode map e621 with expected crc 2023-10-09T06:09:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:29 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/570262682' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:09:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:29 smithi191 ceph-mon[161669]: osdmap e621: 8 total, 8 up, 8 in 2023-10-09T06:09:30.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:29 smithi191 ceph-mon[161669]: pgmap v1852: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:30.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:09:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:09:30] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.33.4" 2023-10-09T06:09:31.095 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:30 smithi191 ceph-mon[161669]: failed to encode map e621 with expected crc 2023-10-09T06:09:31.095 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:30 smithi191 ceph-mon[161669]: failed to encode map e621 with expected crc 2023-10-09T06:09:31.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:30 smithi008 ceph-mon[184418]: failed to encode map e621 with expected crc 2023-10-09T06:09:31.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:30 smithi008 ceph-mon[184418]: failed to encode map e621 with expected crc 2023-10-09T06:09:31.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:30 smithi008 ceph-mon[191227]: failed to encode map e621 with expected crc 2023-10-09T06:09:31.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:30 smithi008 ceph-mon[191227]: failed to encode map e621 with expected crc 2023-10-09T06:09:31.259 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:31.260 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:31.959 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:09:32.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:31 smithi191 ceph-mon[161669]: pgmap v1853: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:32.310 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:32.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:31 smithi008 ceph-mon[184418]: pgmap v1853: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:32.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:31 smithi008 ceph-mon[191227]: pgmap v1853: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:32.980 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:09:33.218 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:32 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2554585415' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:09:33.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:32 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2554585415' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:09:33.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:32 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2554585415' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:09:33.670 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:33.671 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:33.767 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T06:09:34.139 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:33 smithi008 ceph-mon[184418]: failed to encode map e622 with expected crc 2023-10-09T06:09:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:33 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2554585415' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:09:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:33 smithi008 ceph-mon[184418]: osdmap e622: 8 total, 8 up, 8 in 2023-10-09T06:09:34.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:33 smithi008 ceph-mon[184418]: pgmap v1855: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:33 smithi008 ceph-mon[191227]: failed to encode map e622 with expected crc 2023-10-09T06:09:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:33 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2554585415' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:09:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:33 smithi008 ceph-mon[191227]: osdmap e622: 8 total, 8 up, 8 in 2023-10-09T06:09:34.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:33 smithi008 ceph-mon[191227]: pgmap v1855: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:34.244 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:33 smithi191 ceph-mon[161669]: failed to encode map e622 with expected crc 2023-10-09T06:09:34.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:33 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2554585415' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:09:34.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:33 smithi191 ceph-mon[161669]: osdmap e622: 8 total, 8 up, 8 in 2023-10-09T06:09:34.245 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:33 smithi191 ceph-mon[161669]: pgmap v1855: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:34.259 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:09:34.607 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:35.125 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:34 smithi191 ceph-mon[161669]: failed to encode map e622 with expected crc 2023-10-09T06:09:35.125 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:34 smithi191 ceph-mon[161669]: failed to encode map e622 with expected crc 2023-10-09T06:09:35.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:34 smithi008 ceph-mon[184418]: failed to encode map e622 with expected crc 2023-10-09T06:09:35.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:34 smithi008 ceph-mon[184418]: failed to encode map e622 with expected crc 2023-10-09T06:09:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:34 smithi008 ceph-mon[191227]: failed to encode map e622 with expected crc 2023-10-09T06:09:35.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:34 smithi008 ceph-mon[191227]: failed to encode map e622 with expected crc 2023-10-09T06:09:35.994 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T06:09:36.011 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:36.012 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:35 smithi191 ceph-mon[161669]: pgmap v1856: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:35 smithi191 ceph-mon[161669]: failed to encode map e622 with expected crc 2023-10-09T06:09:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3018980351' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:09:36.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:35 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:09:36.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:35 smithi008 ceph-mon[184418]: pgmap v1856: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:36.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:35 smithi008 ceph-mon[184418]: failed to encode map e622 with expected crc 2023-10-09T06:09:36.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3018980351' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:09:36.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:35 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:09:36.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:35 smithi008 ceph-mon[191227]: pgmap v1856: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:36.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:35 smithi008 ceph-mon[191227]: failed to encode map e622 with expected crc 2023-10-09T06:09:36.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3018980351' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:09:36.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:35 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:09:36.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:09:36.780 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:09:36.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:36 smithi191 ceph-mon[161669]: failed to encode map e623 with expected crc 2023-10-09T06:09:36.985 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:36 smithi191 ceph-mon[161669]: failed to encode map e623 with expected crc 2023-10-09T06:09:36.985 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:36 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:09:36.985 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:36 smithi191 ceph-mon[161669]: osdmap e623: 8 total, 8 up, 8 in 2023-10-09T06:09:36.985 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:36 smithi191 ceph-mon[161669]: failed to encode map e623 with expected crc 2023-10-09T06:09:36.985 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:36 smithi191 ceph-mon[161669]: failed to encode map e623 with expected crc 2023-10-09T06:09:37.002 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:37.159 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[184418]: failed to encode map e623 with expected crc 2023-10-09T06:09:37.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[184418]: failed to encode map e623 with expected crc 2023-10-09T06:09:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:09:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[184418]: osdmap e623: 8 total, 8 up, 8 in 2023-10-09T06:09:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[184418]: failed to encode map e623 with expected crc 2023-10-09T06:09:37.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[184418]: failed to encode map e623 with expected crc 2023-10-09T06:09:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[191227]: failed to encode map e623 with expected crc 2023-10-09T06:09:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[191227]: failed to encode map e623 with expected crc 2023-10-09T06:09:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:09:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[191227]: osdmap e623: 8 total, 8 up, 8 in 2023-10-09T06:09:37.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[191227]: failed to encode map e623 with expected crc 2023-10-09T06:09:37.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:36 smithi008 ceph-mon[191227]: failed to encode map e623 with expected crc 2023-10-09T06:09:38.203 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:37 smithi191 ceph-mon[161669]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.203 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:37 smithi191 ceph-mon[161669]: osdmap e624: 8 total, 8 up, 8 in 2023-10-09T06:09:38.203 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:37 smithi191 ceph-mon[161669]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.203 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:37 smithi191 ceph-mon[161669]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.204 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:37 smithi191 ceph-mon[161669]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.204 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:37 smithi191 ceph-mon[161669]: pgmap v1859: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:38.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[184418]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[184418]: osdmap e624: 8 total, 8 up, 8 in 2023-10-09T06:09:38.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[184418]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[184418]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[184418]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[184418]: pgmap v1859: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[191227]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[191227]: osdmap e624: 8 total, 8 up, 8 in 2023-10-09T06:09:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[191227]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[191227]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[191227]: failed to encode map e624 with expected crc 2023-10-09T06:09:38.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:37 smithi008 ceph-mon[191227]: pgmap v1859: 73 pgs: 73 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:38.367 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:38.367 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:39.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:09:39.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:38 smithi191 ceph-mon[161669]: failed to encode map e624 with expected crc 2023-10-09T06:09:39.397 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:39.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:38 smithi008 ceph-mon[184418]: failed to encode map e624 with expected crc 2023-10-09T06:09:39.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:38 smithi008 ceph-mon[191227]: failed to encode map e624 with expected crc 2023-10-09T06:09:40.008 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:09:40.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:40 smithi191 ceph-mon[161669]: pgmap v1860: 73 pgs: 1 active, 11 activating, 61 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:40.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:40 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/316334657' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:09:40.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:40 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:09:40.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:40 smithi008 ceph-mon[184418]: pgmap v1860: 73 pgs: 1 active, 11 activating, 61 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:40.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:40 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/316334657' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:09:40.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:40 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:09:40.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:40 smithi008 ceph-mon[191227]: pgmap v1860: 73 pgs: 1 active, 11 activating, 61 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:40.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:40 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/316334657' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:09:40.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:40 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:09:40.828 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:09:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:09:40] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.33.4" 2023-10-09T06:09:40.839 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:40.839 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:40.848 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T06:09:41.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:41 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 8 pgs inactive (PG_AVAILABILITY) 2023-10-09T06:09:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:41 smithi008 ceph-mon[184418]: failed to encode map e625 with expected crc 2023-10-09T06:09:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:41 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:09:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:41 smithi008 ceph-mon[184418]: osdmap e625: 8 total, 8 up, 8 in 2023-10-09T06:09:41.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:41 smithi008 ceph-mon[184418]: failed to encode map e625 with expected crc 2023-10-09T06:09:41.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:41 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 8 pgs inactive (PG_AVAILABILITY) 2023-10-09T06:09:41.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:41 smithi008 ceph-mon[191227]: failed to encode map e625 with expected crc 2023-10-09T06:09:41.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:41 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:09:41.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:41 smithi008 ceph-mon[191227]: osdmap e625: 8 total, 8 up, 8 in 2023-10-09T06:09:41.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:41 smithi008 ceph-mon[191227]: failed to encode map e625 with expected crc 2023-10-09T06:09:41.222 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:41 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 8 pgs inactive (PG_AVAILABILITY) 2023-10-09T06:09:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:41 smithi191 ceph-mon[161669]: failed to encode map e625 with expected crc 2023-10-09T06:09:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:41 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:09:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:41 smithi191 ceph-mon[161669]: osdmap e625: 8 total, 8 up, 8 in 2023-10-09T06:09:41.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:41 smithi191 ceph-mon[161669]: failed to encode map e625 with expected crc 2023-10-09T06:09:42.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:42 smithi008 ceph-mon[184418]: failed to encode map e625 with expected crc 2023-10-09T06:09:42.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:42 smithi008 ceph-mon[184418]: failed to encode map e625 with expected crc 2023-10-09T06:09:42.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:42 smithi008 ceph-mon[184418]: pgmap v1862: 73 pgs: 1 active, 11 activating, 61 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:42.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:42 smithi008 ceph-mon[191227]: failed to encode map e625 with expected crc 2023-10-09T06:09:42.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:42 smithi008 ceph-mon[191227]: failed to encode map e625 with expected crc 2023-10-09T06:09:42.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:42 smithi008 ceph-mon[191227]: pgmap v1862: 73 pgs: 1 active, 11 activating, 61 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:42 smithi191 ceph-mon[161669]: failed to encode map e625 with expected crc 2023-10-09T06:09:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:42 smithi191 ceph-mon[161669]: failed to encode map e625 with expected crc 2023-10-09T06:09:42.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:42 smithi191 ceph-mon[161669]: pgmap v1862: 73 pgs: 1 active, 11 activating, 61 active+clean; 8.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:43.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:43 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive) 2023-10-09T06:09:43.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:43 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T06:09:43.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:43 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive) 2023-10-09T06:09:43.466 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:43 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T06:09:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:43 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive) 2023-10-09T06:09:43.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:43 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T06:09:44.116 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T06:09:44.464 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:44 smithi008 ceph-mon[191227]: pgmap v1863: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:44.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:44 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1012087166' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:09:44.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:44 smithi008 ceph-mon[184418]: pgmap v1863: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:44 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1012087166' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:09:44.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:44 smithi191 ceph-mon[161669]: pgmap v1863: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:44 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1012087166' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:09:44.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:44.935 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:09:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:45 smithi008 ceph-mon[191227]: failed to encode map e626 with expected crc 2023-10-09T06:09:45.217 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:45 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1012087166' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:09:45.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:45 smithi008 ceph-mon[191227]: osdmap e626: 8 total, 8 up, 8 in 2023-10-09T06:09:45.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:45 smithi008 ceph-mon[191227]: failed to encode map e626 with expected crc 2023-10-09T06:09:45.218 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:45 smithi008 ceph-mon[191227]: failed to encode map e626 with expected crc 2023-10-09T06:09:45.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:45 smithi008 ceph-mon[184418]: failed to encode map e626 with expected crc 2023-10-09T06:09:45.218 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:45 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1012087166' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:09:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:45 smithi008 ceph-mon[184418]: osdmap e626: 8 total, 8 up, 8 in 2023-10-09T06:09:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:45 smithi008 ceph-mon[184418]: failed to encode map e626 with expected crc 2023-10-09T06:09:45.219 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:45 smithi008 ceph-mon[184418]: failed to encode map e626 with expected crc 2023-10-09T06:09:45.316 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:45 smithi191 ceph-mon[161669]: failed to encode map e626 with expected crc 2023-10-09T06:09:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:45 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1012087166' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:09:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:45 smithi191 ceph-mon[161669]: osdmap e626: 8 total, 8 up, 8 in 2023-10-09T06:09:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:45 smithi191 ceph-mon[161669]: failed to encode map e626 with expected crc 2023-10-09T06:09:45.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:45 smithi191 ceph-mon[161669]: failed to encode map e626 with expected crc 2023-10-09T06:09:46.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[184418]: pgmap v1865: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:46.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[184418]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[184418]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[184418]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[184418]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[184418]: osdmap e627: 8 total, 8 up, 8 in 2023-10-09T06:09:46.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[184418]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [6, 5]}]: dispatch 2023-10-09T06:09:46.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[191227]: pgmap v1865: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:46.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[191227]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[191227]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[191227]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[191227]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[191227]: osdmap e627: 8 total, 8 up, 8 in 2023-10-09T06:09:46.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[191227]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:46 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [6, 5]}]: dispatch 2023-10-09T06:09:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:46 smithi191 ceph-mon[161669]: pgmap v1865: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:46 smithi191 ceph-mon[161669]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:46 smithi191 ceph-mon[161669]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:46 smithi191 ceph-mon[161669]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:46 smithi191 ceph-mon[161669]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:46 smithi191 ceph-mon[161669]: osdmap e627: 8 total, 8 up, 8 in 2023-10-09T06:09:46.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:46 smithi191 ceph-mon[161669]: failed to encode map e627 with expected crc 2023-10-09T06:09:46.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:46 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [6, 5]}]: dispatch 2023-10-09T06:09:47.128 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:47 smithi008 ceph-mon[191227]: failed to encode map e628 with expected crc 2023-10-09T06:09:47.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:47 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [6, 5]}]': finished 2023-10-09T06:09:47.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:47 smithi008 ceph-mon[191227]: osdmap e628: 8 total, 8 up, 8 in 2023-10-09T06:09:47.129 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:47 smithi008 ceph-mon[191227]: failed to encode map e628 with expected crc 2023-10-09T06:09:47.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:47 smithi008 ceph-mon[184418]: failed to encode map e628 with expected crc 2023-10-09T06:09:47.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:47 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [6, 5]}]': finished 2023-10-09T06:09:47.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:47 smithi008 ceph-mon[184418]: osdmap e628: 8 total, 8 up, 8 in 2023-10-09T06:09:47.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:47 smithi008 ceph-mon[184418]: failed to encode map e628 with expected crc 2023-10-09T06:09:47.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:47 smithi191 ceph-mon[161669]: failed to encode map e628 with expected crc 2023-10-09T06:09:47.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:47 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [6, 5]}]': finished 2023-10-09T06:09:47.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:47 smithi191 ceph-mon[161669]: osdmap e628: 8 total, 8 up, 8 in 2023-10-09T06:09:47.551 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:47 smithi191 ceph-mon[161669]: failed to encode map e628 with expected crc 2023-10-09T06:09:48.140 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:09:48.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[184418]: pgmap v1868: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 7 B/s, 3 keys/s, 0 objects/s recovering 2023-10-09T06:09:48.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[184418]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[184418]: osdmap e629: 8 total, 8 up, 8 in 2023-10-09T06:09:48.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[184418]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[184418]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.402 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4080843498' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:09:48.403 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[184418]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[191227]: pgmap v1868: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 7 B/s, 3 keys/s, 0 objects/s recovering 2023-10-09T06:09:48.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[191227]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[191227]: osdmap e629: 8 total, 8 up, 8 in 2023-10-09T06:09:48.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[191227]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[191227]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4080843498' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:09:48.403 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:48 smithi008 ceph-mon[191227]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:48 smithi191 ceph-mon[161669]: pgmap v1868: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 7 B/s, 3 keys/s, 0 objects/s recovering 2023-10-09T06:09:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:48 smithi191 ceph-mon[161669]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:48 smithi191 ceph-mon[161669]: osdmap e629: 8 total, 8 up, 8 in 2023-10-09T06:09:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:48 smithi191 ceph-mon[161669]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:48 smithi191 ceph-mon[161669]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:48 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4080843498' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:09:48.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:48 smithi191 ceph-mon[161669]: failed to encode map e629 with expected crc 2023-10-09T06:09:48.997 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T06:09:49.372 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:49.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[184418]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.501 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/4080843498' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:09:49.501 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[184418]: osdmap e630: 8 total, 8 up, 8 in 2023-10-09T06:09:49.501 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[184418]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.501 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[184418]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.501 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[184418]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.501 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[184418]: osdmap e631: 8 total, 8 up, 8 in 2023-10-09T06:09:49.502 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[191227]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.502 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/4080843498' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:09:49.502 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[191227]: osdmap e630: 8 total, 8 up, 8 in 2023-10-09T06:09:49.502 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[191227]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.502 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[191227]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.502 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[191227]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.502 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:49 smithi008 ceph-mon[191227]: osdmap e631: 8 total, 8 up, 8 in 2023-10-09T06:09:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:49 smithi191 ceph-mon[161669]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:49 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/4080843498' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:09:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:49 smithi191 ceph-mon[161669]: osdmap e630: 8 total, 8 up, 8 in 2023-10-09T06:09:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:49 smithi191 ceph-mon[161669]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:49 smithi191 ceph-mon[161669]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:49 smithi191 ceph-mon[161669]: failed to encode map e630 with expected crc 2023-10-09T06:09:49.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:49 smithi191 ceph-mon[161669]: osdmap e631: 8 total, 8 up, 8 in 2023-10-09T06:09:50.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:50 smithi008 ceph-mon[184418]: pgmap v1871: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 11 B/s, 4 keys/s, 0 objects/s recovering 2023-10-09T06:09:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:50 smithi008 ceph-mon[184418]: failed to encode map e631 with expected crc 2023-10-09T06:09:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:50 smithi008 ceph-mon[184418]: failed to encode map e631 with expected crc 2023-10-09T06:09:50.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:50 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:50 smithi008 ceph-mon[191227]: pgmap v1871: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 11 B/s, 4 keys/s, 0 objects/s recovering 2023-10-09T06:09:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:50 smithi008 ceph-mon[191227]: failed to encode map e631 with expected crc 2023-10-09T06:09:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:50 smithi008 ceph-mon[191227]: failed to encode map e631 with expected crc 2023-10-09T06:09:50.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:50 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:50.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:50 smithi191 ceph-mon[161669]: pgmap v1871: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 11 B/s, 4 keys/s, 0 objects/s recovering 2023-10-09T06:09:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:50 smithi191 ceph-mon[161669]: failed to encode map e631 with expected crc 2023-10-09T06:09:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:50 smithi191 ceph-mon[161669]: failed to encode map e631 with expected crc 2023-10-09T06:09:50.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:50 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:09:50.820 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:09:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:09:50] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.33.4" 2023-10-09T06:09:52.158 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T06:09:52.410 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:52 smithi008 ceph-mon[184418]: pgmap v1873: 73 pgs: 1 activating, 72 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 23 B/s, 4 keys/s, 1 objects/s recovering 2023-10-09T06:09:52.410 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:52 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3445426404' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:09:52.410 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:52 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:09:52.411 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:52 smithi008 ceph-mon[191227]: pgmap v1873: 73 pgs: 1 activating, 72 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 23 B/s, 4 keys/s, 1 objects/s recovering 2023-10-09T06:09:52.411 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:52 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3445426404' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:09:52.411 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:52 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:09:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:52 smithi191 ceph-mon[161669]: pgmap v1873: 73 pgs: 1 activating, 72 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 23 B/s, 4 keys/s, 1 objects/s recovering 2023-10-09T06:09:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:52 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3445426404' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:09:52.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:52 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:09:53.016 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:09:53.394 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:53.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:53 smithi008 ceph-mon[184418]: failed to encode map e632 with expected crc 2023-10-09T06:09:53.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:53 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:09:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:53 smithi008 ceph-mon[184418]: osdmap e632: 8 total, 8 up, 8 in 2023-10-09T06:09:53.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:53 smithi008 ceph-mon[184418]: failed to encode map e632 with expected crc 2023-10-09T06:09:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:53 smithi008 ceph-mon[191227]: failed to encode map e632 with expected crc 2023-10-09T06:09:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:53 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:09:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:53 smithi008 ceph-mon[191227]: osdmap e632: 8 total, 8 up, 8 in 2023-10-09T06:09:53.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:53 smithi008 ceph-mon[191227]: failed to encode map e632 with expected crc 2023-10-09T06:09:53.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:53 smithi191 ceph-mon[161669]: failed to encode map e632 with expected crc 2023-10-09T06:09:53.549 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:53 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:09:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:53 smithi191 ceph-mon[161669]: osdmap e632: 8 total, 8 up, 8 in 2023-10-09T06:09:53.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:53 smithi191 ceph-mon[161669]: failed to encode map e632 with expected crc 2023-10-09T06:09:54.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:54 smithi008 ceph-mon[184418]: pgmap v1875: 73 pgs: 1 activating, 72 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-10-09T06:09:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:54 smithi008 ceph-mon[184418]: failed to encode map e633 with expected crc 2023-10-09T06:09:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:54 smithi008 ceph-mon[184418]: osdmap e633: 8 total, 8 up, 8 in 2023-10-09T06:09:54.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:54 smithi008 ceph-mon[184418]: failed to encode map e633 with expected crc 2023-10-09T06:09:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:54 smithi008 ceph-mon[191227]: pgmap v1875: 73 pgs: 1 activating, 72 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-10-09T06:09:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:54 smithi008 ceph-mon[191227]: failed to encode map e633 with expected crc 2023-10-09T06:09:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:54 smithi008 ceph-mon[191227]: osdmap e633: 8 total, 8 up, 8 in 2023-10-09T06:09:54.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:54 smithi008 ceph-mon[191227]: failed to encode map e633 with expected crc 2023-10-09T06:09:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:54 smithi191 ceph-mon[161669]: pgmap v1875: 73 pgs: 1 activating, 72 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-10-09T06:09:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:54 smithi191 ceph-mon[161669]: failed to encode map e633 with expected crc 2023-10-09T06:09:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:54 smithi191 ceph-mon[161669]: osdmap e633: 8 total, 8 up, 8 in 2023-10-09T06:09:54.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:54 smithi191 ceph-mon[161669]: failed to encode map e633 with expected crc 2023-10-09T06:09:56.163 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:09:56.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:56 smithi008 ceph-mon[184418]: pgmap v1877: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T06:09:56.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:56 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/2798475330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:09:56.464 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:56 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:09:56.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:56 smithi008 ceph-mon[191227]: pgmap v1877: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T06:09:56.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:56 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/2798475330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:09:56.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:56 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:09:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:56 smithi191 ceph-mon[161669]: pgmap v1877: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T06:09:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:56 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/2798475330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:09:56.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:56 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:09:56.983 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T06:09:57.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:57 smithi008 ceph-mon[191227]: failed to encode map e634 with expected crc 2023-10-09T06:09:57.214 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:57 smithi008 ceph-mon[191227]: failed to encode map e634 with expected crc 2023-10-09T06:09:57.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:57 smithi008 ceph-mon[191227]: failed to encode map e634 with expected crc 2023-10-09T06:09:57.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:57 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:09:57.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:57 smithi008 ceph-mon[191227]: osdmap e634: 8 total, 8 up, 8 in 2023-10-09T06:09:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:57 smithi008 ceph-mon[184418]: failed to encode map e634 with expected crc 2023-10-09T06:09:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:57 smithi008 ceph-mon[184418]: failed to encode map e634 with expected crc 2023-10-09T06:09:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:57 smithi008 ceph-mon[184418]: failed to encode map e634 with expected crc 2023-10-09T06:09:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:57 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:09:57.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:57 smithi008 ceph-mon[184418]: osdmap e634: 8 total, 8 up, 8 in 2023-10-09T06:09:57.355 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:57 smithi191 ceph-mon[161669]: failed to encode map e634 with expected crc 2023-10-09T06:09:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:57 smithi191 ceph-mon[161669]: failed to encode map e634 with expected crc 2023-10-09T06:09:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:57 smithi191 ceph-mon[161669]: failed to encode map e634 with expected crc 2023-10-09T06:09:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:57 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:09:57.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:57 smithi191 ceph-mon[161669]: osdmap e634: 8 total, 8 up, 8 in 2023-10-09T06:09:58.465 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:58 smithi008 ceph-mon[184418]: pgmap v1879: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:58.465 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:58 smithi008 ceph-mon[191227]: pgmap v1879: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:58.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:58 smithi191 ceph-mon[161669]: pgmap v1879: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:09:59.681 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T06:09:59.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:59 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:59 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/910493527' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:09:59.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:09:59 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:09:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:59 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:59 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/910493527' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:09:59.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:09:59 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:10:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:59 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:59 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/910493527' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:10:00.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:09:59 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:10:00.489 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:10:00.872 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:00.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:00 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:10:00] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.33.4" 2023-10-09T06:10:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[184418]: pgmap v1880: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[184418]: failed to encode map e635 with expected crc 2023-10-09T06:10:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[184418]: failed to encode map e635 with expected crc 2023-10-09T06:10:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[184418]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:10:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[184418]: osdmap e635: 8 total, 8 up, 8 in 2023-10-09T06:10:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[184418]: failed to encode map e635 with expected crc 2023-10-09T06:10:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[184418]: failed to encode map e635 with expected crc 2023-10-09T06:10:00.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[184418]: overall HEALTH_OK 2023-10-09T06:10:00.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[191227]: pgmap v1880: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:00.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[191227]: failed to encode map e635 with expected crc 2023-10-09T06:10:00.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[191227]: failed to encode map e635 with expected crc 2023-10-09T06:10:00.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[191227]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:10:00.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[191227]: osdmap e635: 8 total, 8 up, 8 in 2023-10-09T06:10:00.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[191227]: failed to encode map e635 with expected crc 2023-10-09T06:10:00.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[191227]: failed to encode map e635 with expected crc 2023-10-09T06:10:00.966 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:00 smithi008 ceph-mon[191227]: overall HEALTH_OK 2023-10-09T06:10:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:00 smithi191 ceph-mon[161669]: pgmap v1880: 73 pgs: 73 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:00 smithi191 ceph-mon[161669]: failed to encode map e635 with expected crc 2023-10-09T06:10:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:00 smithi191 ceph-mon[161669]: failed to encode map e635 with expected crc 2023-10-09T06:10:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:00 smithi191 ceph-mon[161669]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:10:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:00 smithi191 ceph-mon[161669]: osdmap e635: 8 total, 8 up, 8 in 2023-10-09T06:10:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:00 smithi191 ceph-mon[161669]: failed to encode map e635 with expected crc 2023-10-09T06:10:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:00 smithi191 ceph-mon[161669]: failed to encode map e635 with expected crc 2023-10-09T06:10:01.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:00 smithi191 ceph-mon[161669]: overall HEALTH_OK 2023-10-09T06:10:01.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:01 smithi008 ceph-mon[184418]: failed to encode map e636 with expected crc 2023-10-09T06:10:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:01 smithi008 ceph-mon[184418]: failed to encode map e636 with expected crc 2023-10-09T06:10:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:01 smithi008 ceph-mon[184418]: failed to encode map e636 with expected crc 2023-10-09T06:10:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:01 smithi008 ceph-mon[184418]: osdmap e636: 8 total, 8 up, 8 in 2023-10-09T06:10:01.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:01 smithi008 ceph-mon[184418]: failed to encode map e636 with expected crc 2023-10-09T06:10:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:01 smithi008 ceph-mon[191227]: failed to encode map e636 with expected crc 2023-10-09T06:10:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:01 smithi008 ceph-mon[191227]: failed to encode map e636 with expected crc 2023-10-09T06:10:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:01 smithi008 ceph-mon[191227]: failed to encode map e636 with expected crc 2023-10-09T06:10:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:01 smithi008 ceph-mon[191227]: osdmap e636: 8 total, 8 up, 8 in 2023-10-09T06:10:01.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:01 smithi008 ceph-mon[191227]: failed to encode map e636 with expected crc 2023-10-09T06:10:02.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:01 smithi191 ceph-mon[161669]: failed to encode map e636 with expected crc 2023-10-09T06:10:02.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:01 smithi191 ceph-mon[161669]: failed to encode map e636 with expected crc 2023-10-09T06:10:02.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:01 smithi191 ceph-mon[161669]: failed to encode map e636 with expected crc 2023-10-09T06:10:02.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:01 smithi191 ceph-mon[161669]: osdmap e636: 8 total, 8 up, 8 in 2023-10-09T06:10:02.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:01 smithi191 ceph-mon[161669]: failed to encode map e636 with expected crc 2023-10-09T06:10:02.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:02 smithi008 ceph-mon[184418]: pgmap v1883: 73 pgs: 10 peering, 63 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 96 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T06:10:02.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:02 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 1 pg inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T06:10:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:02 smithi008 ceph-mon[191227]: pgmap v1883: 73 pgs: 10 peering, 63 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 96 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T06:10:02.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:02 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 1 pg inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T06:10:03.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:02 smithi191 ceph-mon[161669]: pgmap v1883: 73 pgs: 10 peering, 63 active+clean; 8.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 96 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T06:10:03.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:02 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 1 pg inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T06:10:03.696 INFO:teuthology.orchestra.run.smithi008.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:10:03.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:03 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3727875464' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:10:03.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:03 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3727875464' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:10:04.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:03 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3727875464' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:10:04.465 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T06:10:04.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[184418]: pgmap v1884: 73 pgs: 10 peering, 63 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 149 B/s rd, 0 op/s; 84 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T06:10:04.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[184418]: failed to encode map e637 with expected crc 2023-10-09T06:10:04.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[184418]: failed to encode map e637 with expected crc 2023-10-09T06:10:04.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[184418]: failed to encode map e637 with expected crc 2023-10-09T06:10:04.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3727875464' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:10:04.715 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[184418]: osdmap e637: 8 total, 8 up, 8 in 2023-10-09T06:10:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[191227]: pgmap v1884: 73 pgs: 10 peering, 63 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 149 B/s rd, 0 op/s; 84 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T06:10:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[191227]: failed to encode map e637 with expected crc 2023-10-09T06:10:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[191227]: failed to encode map e637 with expected crc 2023-10-09T06:10:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[191227]: failed to encode map e637 with expected crc 2023-10-09T06:10:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3727875464' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:10:04.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:04 smithi008 ceph-mon[191227]: osdmap e637: 8 total, 8 up, 8 in 2023-10-09T06:10:04.844 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:04 smithi191 ceph-mon[161669]: pgmap v1884: 73 pgs: 10 peering, 63 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 149 B/s rd, 0 op/s; 84 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T06:10:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:04 smithi191 ceph-mon[161669]: failed to encode map e637 with expected crc 2023-10-09T06:10:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:04 smithi191 ceph-mon[161669]: failed to encode map e637 with expected crc 2023-10-09T06:10:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:04 smithi191 ceph-mon[161669]: failed to encode map e637 with expected crc 2023-10-09T06:10:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:04 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3727875464' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:10:05.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:04 smithi191 ceph-mon[161669]: osdmap e637: 8 total, 8 up, 8 in 2023-10-09T06:10:06.210 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:05 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:06.211 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:05 smithi008 ceph-mon[184418]: pgmap v1886: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 96 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T06:10:06.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:05 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:06.211 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:05 smithi008 ceph-mon[191227]: pgmap v1886: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 96 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T06:10:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:05 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:06.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:05 smithi191 ceph-mon[161669]: pgmap v1886: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 96 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T06:10:06.861 INFO:teuthology.orchestra.run.smithi008.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T06:10:07.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:06 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 8 pgs peering) 2023-10-09T06:10:07.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:06 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T06:10:07.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:06 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1696739727' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:10:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:06 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 8 pgs peering) 2023-10-09T06:10:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:06 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T06:10:07.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:06 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1696739727' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:10:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:06 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 8 pgs peering) 2023-10-09T06:10:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:06 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T06:10:07.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:06 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1696739727' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:10:07.635 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T06:10:07.635 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T06:10:07.635 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T06:10:07.635 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T06:10:07.636 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T06:10:07.636 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:10:07.869 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[191227]: failed to encode map e638 with expected crc 2023-10-09T06:10:07.869 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[191227]: failed to encode map e638 with expected crc 2023-10-09T06:10:07.870 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1696739727' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:10:07.870 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[191227]: osdmap e638: 8 total, 8 up, 8 in 2023-10-09T06:10:07.870 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[191227]: failed to encode map e638 with expected crc 2023-10-09T06:10:07.870 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[191227]: pgmap v1888: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:08.013 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:08.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[184418]: failed to encode map e638 with expected crc 2023-10-09T06:10:08.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[184418]: failed to encode map e638 with expected crc 2023-10-09T06:10:08.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1696739727' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:10:08.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[184418]: osdmap e638: 8 total, 8 up, 8 in 2023-10-09T06:10:08.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[184418]: failed to encode map e638 with expected crc 2023-10-09T06:10:08.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:07 smithi008 ceph-mon[184418]: pgmap v1888: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:07 smithi191 ceph-mon[161669]: failed to encode map e638 with expected crc 2023-10-09T06:10:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:07 smithi191 ceph-mon[161669]: failed to encode map e638 with expected crc 2023-10-09T06:10:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:07 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1696739727' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:10:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:07 smithi191 ceph-mon[161669]: osdmap e638: 8 total, 8 up, 8 in 2023-10-09T06:10:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:07 smithi191 ceph-mon[161669]: failed to encode map e638 with expected crc 2023-10-09T06:10:08.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:07 smithi191 ceph-mon[161669]: pgmap v1888: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:08.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:08 smithi008 ceph-mon[184418]: failed to encode map e639 with expected crc 2023-10-09T06:10:08.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:08 smithi008 ceph-mon[184418]: failed to encode map e639 with expected crc 2023-10-09T06:10:08.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:08 smithi008 ceph-mon[184418]: failed to encode map e639 with expected crc 2023-10-09T06:10:08.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:08 smithi008 ceph-mon[184418]: osdmap e639: 8 total, 8 up, 8 in 2023-10-09T06:10:08.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:08 smithi008 ceph-mon[184418]: failed to encode map e639 with expected crc 2023-10-09T06:10:08.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:08 smithi008 ceph-mon[191227]: failed to encode map e639 with expected crc 2023-10-09T06:10:08.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:08 smithi008 ceph-mon[191227]: failed to encode map e639 with expected crc 2023-10-09T06:10:08.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:08 smithi008 ceph-mon[191227]: failed to encode map e639 with expected crc 2023-10-09T06:10:08.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:08 smithi008 ceph-mon[191227]: osdmap e639: 8 total, 8 up, 8 in 2023-10-09T06:10:08.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:08 smithi008 ceph-mon[191227]: failed to encode map e639 with expected crc 2023-10-09T06:10:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:08 smithi191 ceph-mon[161669]: failed to encode map e639 with expected crc 2023-10-09T06:10:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:08 smithi191 ceph-mon[161669]: failed to encode map e639 with expected crc 2023-10-09T06:10:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:08 smithi191 ceph-mon[161669]: failed to encode map e639 with expected crc 2023-10-09T06:10:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:08 smithi191 ceph-mon[161669]: osdmap e639: 8 total, 8 up, 8 in 2023-10-09T06:10:09.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:08 smithi191 ceph-mon[161669]: failed to encode map e639 with expected crc 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":639,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:14:56.160913+0000","modified":"2023-10-09T06:10:07.863217+0000","last_up_change":"2023-10-09T06:04:31.912288+0000","last_in_change":"2023-10-09T06:08:52.820741+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:17:53.386952+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:20:22.166676+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"603","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":603,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:21:15.712971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:21:16.208554+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":630,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6802","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6803","nonce":633445692}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6804","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6805","nonce":633445692}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6808","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6809","nonce":633445692}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6806","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6807","nonce":633445692}]},"public_addr":"172.21.15.8:6803/633445692","cluster_addr":"172.21.15.8:6805/633445692","heartbeat_back_addr":"172.21.15.8:6809/633445692","heartbeat_front_addr":"172.21.15.8:6807/633445692","state":["exists","up"]},{"osd":1,"uuid":"55f3290e-b82d-4839-a28d-1123a20c3a4c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":630,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6810","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6811","nonce":417852095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6812","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6813","nonce":417852095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6816","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6817","nonce":417852095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6814","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6815","nonce":417852095}]},"public_addr":"172.21.15.8:6811/417852095","cluster_addr":"172.21.15.8:6813/417852095","heartbeat_back_addr":"172.21.15.8:6817/417852095","heartbeat_front_addr":"172.21.15.8:6815/417852095","state":["exists","up"]},{"osd":2,"uuid":"720b47df-d743-46cf-ad04-36e113c8fcb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":611,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6818","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6819","nonce":1092325054}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6820","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6821","nonce":1092325054}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6824","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6825","nonce":1092325054}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6822","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6823","nonce":1092325054}]},"public_addr":"172.21.15.8:6819/1092325054","cluster_addr":"172.21.15.8:6821/1092325054","heartbeat_back_addr":"172.21.15.8:6825/1092325054","heartbeat_front_addr":"172.21.15.8:6823/1092325054","state":["exists","up"]},{"osd":3,"uuid":"861bf05d-69f8-4000-999e-ab708531edb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":623,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6826","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6827","nonce":4118391505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6828","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6829","nonce":4118391505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6832","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6833","nonce":4118391505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6830","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6831","nonce":4118391505}]},"public_addr":"172.21.15.8:6827/4118391505","cluster_addr":"172.21.15.8:6829/4118391505","heartbeat_back_addr":"172.21.15.8:6833/4118391505","heartbeat_front_addr":"172.21.15.8:6831/4118391505","state":["exists","up"]},{"osd":4,"uuid":"b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":626,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6801","nonce":3473028939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6803","nonce":3473028939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6807","nonce":3473028939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6805","nonce":3473028939}]},"public_addr":"172.21.15.191:6801/3473028939","cluster_addr":"172.21.15.191:6803/3473028939","heartbeat_back_addr":"172.21.15.191:6807/3473028939","heartbeat_front_addr":"172.21.15.191:6805/3473028939","state":["exists","up"]},{"osd":5,"uuid":"01507478-481e-4646-a30d-542184240f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":632,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6809","nonce":1592222523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6811","nonce":1592222523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6815","nonce":1592222523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6813","nonce":1592222523}]},"public_addr":"172.21.15.191:6809/1592222523","cluster_addr":"172.21.15.191:6811/1592222523","heartbeat_back_addr":"172.21.15.191:6815/1592222523","heartbeat_front_addr":"172.21.15.191:6813/1592222523","state":["exists","up"]},{"osd":6,"uuid":"8bc75ec3-46ee-4d36-989b-4e65e330cfb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":531,"up_from":537,"up_thru":638,"down_at":532,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2409825908},{"type":"v1","addr":"172.21.15.191:6817","nonce":2409825908}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2409825908},{"type":"v1","addr":"172.21.15.191:6819","nonce":2409825908}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2409825908},{"type":"v1","addr":"172.21.15.191:6823","nonce":2409825908}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2409825908},{"type":"v1","addr":"172.21.15.191:6821","nonce":2409825908}]},"public_addr":"172.21.15.191:6817/2409825908","cluster_addr":"172.21.15.191:6819/2409825908","heartbeat_back_addr":"172.21.15.191:6823/2409825908","heartbeat_front_addr":"172.21.15.191:6821/2409825908","state":["exists","up"]},{"osd":7,"uuid":"5ac2e382-7341-47dd-b6c9-0fbdba12829c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":111,"up_from":127,"up_thru":635,"down_at":112,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":859245059},{"type":"v1","addr":"172.21.15.191:6825","nonce":859245059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":859245059},{"type":"v1","addr":"172.21.15.191:6827","nonce":859245059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":859245059},{"type":"v1","addr":"172.21.15.191:6831","nonce":859245059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":859245059},{"type":"v1","addr":"172.21.15.191:6829","nonce":859245059}]},"public_addr":"172.21.15.191:6825/859245059","cluster_addr":"172.21.15.191:6827/859245059","heartbeat_back_addr":"172.21.15.191:6831/859245059","heartbeat_front_addr":"172.21.15.191:6829/859245059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:06.542686+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:28.674405+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:51.044955+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:14.395353+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:36.486099+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:58.485442+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:20.015429+0000","dead_epoch":531},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:42.014771+0000","dead_epoch":111}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.1b","mappings":[{"from":4,"to":2}]},{"pgid":"4.5","mappings":[{"from":6,"to":5}]},{"pgid":"4.9","mappings":[{"from":1,"to":7}]},{"pgid":"4.f","mappings":[{"from":3,"to":7}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":6}]},{"pgid":"4.1c","mappings":[{"from":1,"to":6}]},{"pgid":"4.1d","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.191:6835/1974805222":"2023-10-10T05:25:00.410091+0000","172.21.15.191:6834/1974805222":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6800/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/1497753039":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/1669399762":"2023-10-10T05:23:46.082746+0000","172.21.15.8:6800/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/357500347":"2023-10-10T05:23:46.082746+0000","172.21.15.8:0/1025807783":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/69531294":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/3932209052":"2023-10-10T05:25:43.728914+0000","172.21.15.191:0/3608336050":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6801/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/711490379":"2023-10-10T05:25:43.728914+0000","172.21.15.191:0/1074075163":"2023-10-10T05:25:00.410091+0000","172.21.15.191:0/3279324894":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/2854879000":"2023-10-10T05:23:46.082746+0000","172.21.15.191:0/1645031799":"2023-10-10T05:25:00.410091+0000","172.21.15.8:0/2096086544":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/819405330":"2023-10-10T05:25:43.728914+0000","172.21.15.8:6800/2853847379":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/2360193392":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6801/2853847379":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/3599326555":"2023-10-10T05:25:43.728914+0000","172.21.15.191:0/465298627":"2023-10-10T05:25:00.410091+0000","172.21.15.8:0/2178079664":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6800/2854879000":"2023-10-10T05:23:46.082746+0000","172.21.15.8:6800/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.191:0/2331494796":"2023-10-10T05:25:00.410091+0000","172.21.15.8:0/1936221066":"2023-10-10T05:20:01.878268+0000","172.21.15.8:0/1062877078":"2023-10-10T05:15:49.270008+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:10:10.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:09 smithi008 ceph-mon[184418]: pgmap v1890: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:10.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:09 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3361347170' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:10:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:09 smithi008 ceph-mon[191227]: pgmap v1890: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:10.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:09 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3361347170' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:10:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:09 smithi191 ceph-mon[161669]: pgmap v1890: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:10.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:09 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3361347170' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:10:10.608 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T06:10:10.609 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:10:10.874 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:10 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:10:10] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.33.4" 2023-10-09T06:10:10.976 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:11.199 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:10 smithi008 ceph-mon[184418]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T06:10:11.200 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:10 smithi008 ceph-mon[191227]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T06:10:11.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:10 smithi191 ceph-mon[161669]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T06:10:11.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:11 smithi008 ceph-mon[184418]: pgmap v1891: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:11.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:11 smithi008 ceph-mon[191227]: pgmap v1891: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:12.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:11 smithi191 ceph-mon[161669]: pgmap v1891: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:12.821 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:10:12.821 INFO:teuthology.orchestra.run.smithi008.stdout:{"epoch":639,"fsid":"9e85b874-6662-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:14:56.160913+0000","modified":"2023-10-09T06:10:07.863217+0000","last_up_change":"2023-10-09T06:04:31.912288+0000","last_in_change":"2023-10-09T06:08:52.820741+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:17:53.386952+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:20:22.166676+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"603","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":603,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:21:15.712971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:21:16.208554+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"457d6ec5-1fb1-4e54-a1cd-6b1db2de15fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":630,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6802","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6803","nonce":633445692}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6804","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6805","nonce":633445692}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6808","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6809","nonce":633445692}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6806","nonce":633445692},{"type":"v1","addr":"172.21.15.8:6807","nonce":633445692}]},"public_addr":"172.21.15.8:6803/633445692","cluster_addr":"172.21.15.8:6805/633445692","heartbeat_back_addr":"172.21.15.8:6809/633445692","heartbeat_front_addr":"172.21.15.8:6807/633445692","state":["exists","up"]},{"osd":1,"uuid":"55f3290e-b82d-4839-a28d-1123a20c3a4c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":630,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6810","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6811","nonce":417852095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6812","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6813","nonce":417852095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6816","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6817","nonce":417852095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6814","nonce":417852095},{"type":"v1","addr":"172.21.15.8:6815","nonce":417852095}]},"public_addr":"172.21.15.8:6811/417852095","cluster_addr":"172.21.15.8:6813/417852095","heartbeat_back_addr":"172.21.15.8:6817/417852095","heartbeat_front_addr":"172.21.15.8:6815/417852095","state":["exists","up"]},{"osd":2,"uuid":"720b47df-d743-46cf-ad04-36e113c8fcb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":611,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6818","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6819","nonce":1092325054}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6820","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6821","nonce":1092325054}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6824","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6825","nonce":1092325054}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6822","nonce":1092325054},{"type":"v1","addr":"172.21.15.8:6823","nonce":1092325054}]},"public_addr":"172.21.15.8:6819/1092325054","cluster_addr":"172.21.15.8:6821/1092325054","heartbeat_back_addr":"172.21.15.8:6825/1092325054","heartbeat_front_addr":"172.21.15.8:6823/1092325054","state":["exists","up"]},{"osd":3,"uuid":"861bf05d-69f8-4000-999e-ab708531edb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":623,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6826","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6827","nonce":4118391505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6828","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6829","nonce":4118391505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6832","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6833","nonce":4118391505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.8:6830","nonce":4118391505},{"type":"v1","addr":"172.21.15.8:6831","nonce":4118391505}]},"public_addr":"172.21.15.8:6827/4118391505","cluster_addr":"172.21.15.8:6829/4118391505","heartbeat_back_addr":"172.21.15.8:6833/4118391505","heartbeat_front_addr":"172.21.15.8:6831/4118391505","state":["exists","up"]},{"osd":4,"uuid":"b7ff9c7e-d3d6-4b0c-b35c-74d5765fbbf9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":626,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6801","nonce":3473028939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6803","nonce":3473028939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6807","nonce":3473028939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":3473028939},{"type":"v1","addr":"172.21.15.191:6805","nonce":3473028939}]},"public_addr":"172.21.15.191:6801/3473028939","cluster_addr":"172.21.15.191:6803/3473028939","heartbeat_back_addr":"172.21.15.191:6807/3473028939","heartbeat_front_addr":"172.21.15.191:6805/3473028939","state":["exists","up"]},{"osd":5,"uuid":"01507478-481e-4646-a30d-542184240f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":632,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6809","nonce":1592222523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6811","nonce":1592222523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6815","nonce":1592222523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":1592222523},{"type":"v1","addr":"172.21.15.191:6813","nonce":1592222523}]},"public_addr":"172.21.15.191:6809/1592222523","cluster_addr":"172.21.15.191:6811/1592222523","heartbeat_back_addr":"172.21.15.191:6815/1592222523","heartbeat_front_addr":"172.21.15.191:6813/1592222523","state":["exists","up"]},{"osd":6,"uuid":"8bc75ec3-46ee-4d36-989b-4e65e330cfb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":531,"up_from":537,"up_thru":638,"down_at":532,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2409825908},{"type":"v1","addr":"172.21.15.191:6817","nonce":2409825908}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2409825908},{"type":"v1","addr":"172.21.15.191:6819","nonce":2409825908}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2409825908},{"type":"v1","addr":"172.21.15.191:6823","nonce":2409825908}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2409825908},{"type":"v1","addr":"172.21.15.191:6821","nonce":2409825908}]},"public_addr":"172.21.15.191:6817/2409825908","cluster_addr":"172.21.15.191:6819/2409825908","heartbeat_back_addr":"172.21.15.191:6823/2409825908","heartbeat_front_addr":"172.21.15.191:6821/2409825908","state":["exists","up"]},{"osd":7,"uuid":"5ac2e382-7341-47dd-b6c9-0fbdba12829c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":111,"up_from":127,"up_thru":635,"down_at":112,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":859245059},{"type":"v1","addr":"172.21.15.191:6825","nonce":859245059}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":859245059},{"type":"v1","addr":"172.21.15.191:6827","nonce":859245059}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":859245059},{"type":"v1","addr":"172.21.15.191:6831","nonce":859245059}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":859245059},{"type":"v1","addr":"172.21.15.191:6829","nonce":859245059}]},"public_addr":"172.21.15.191:6825/859245059","cluster_addr":"172.21.15.191:6827/859245059","heartbeat_back_addr":"172.21.15.191:6831/859245059","heartbeat_front_addr":"172.21.15.191:6829/859245059","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:06.542686+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:28.674405+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:17:51.044955+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:14.395353+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:36.486099+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:18:58.485442+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:20.015429+0000","dead_epoch":531},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:19:42.014771+0000","dead_epoch":111}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.1b","mappings":[{"from":4,"to":2}]},{"pgid":"4.5","mappings":[{"from":6,"to":5}]},{"pgid":"4.9","mappings":[{"from":1,"to":7}]},{"pgid":"4.f","mappings":[{"from":3,"to":7}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":6}]},{"pgid":"4.1c","mappings":[{"from":1,"to":6}]},{"pgid":"4.1d","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.8:0/1497753039":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/1025807783":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/3932209052":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/3599326555":"2023-10-10T05:25:43.728914+0000","172.21.15.8:6800/2853847379":"2023-10-10T05:25:43.728914+0000","172.21.15.191:6835/1974805222":"2023-10-10T05:25:00.410091+0000","172.21.15.191:6834/1974805222":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6800/2430836187":"2023-10-10T05:15:24.205194+0000","172.21.15.8:0/1669399762":"2023-10-10T05:23:46.082746+0000","172.21.15.8:6800/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/357500347":"2023-10-10T05:23:46.082746+0000","172.21.15.8:0/711490379":"2023-10-10T05:25:43.728914+0000","172.21.15.8:6801/310049223":"2023-10-10T05:15:49.270008+0000","172.21.15.191:0/1074075163":"2023-10-10T05:25:00.410091+0000","172.21.15.191:0/3279324894":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/2854879000":"2023-10-10T05:23:46.082746+0000","172.21.15.8:0/819405330":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/2096086544":"2023-10-10T05:15:49.270008+0000","172.21.15.191:0/1645031799":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6801/2853847379":"2023-10-10T05:25:43.728914+0000","172.21.15.8:0/2360193392":"2023-10-10T05:15:24.205194+0000","172.21.15.191:0/2331494796":"2023-10-10T05:25:00.410091+0000","172.21.15.8:0/1936221066":"2023-10-10T05:20:01.878268+0000","172.21.15.8:0/1062877078":"2023-10-10T05:15:49.270008+0000","172.21.15.8:0/69531294":"2023-10-10T05:15:24.205194+0000","172.21.15.8:6800/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.191:0/465298627":"2023-10-10T05:25:00.410091+0000","172.21.15.8:6800/2854879000":"2023-10-10T05:23:46.082746+0000","172.21.15.8:0/2178079664":"2023-10-10T05:20:01.878268+0000","172.21.15.8:6801/1748522337":"2023-10-10T05:20:01.878268+0000","172.21.15.191:0/3608336050":"2023-10-10T05:25:00.410091+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:10:13.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:12 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3106398905' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:10:13.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:12 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3106398905' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:10:13.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:12 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3106398905' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:10:13.635 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T06:10:13.636 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T06:10:13.636 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T06:10:13.636 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T06:10:13.637 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T06:10:13.637 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T06:10:13.638 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T06:10:13.638 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T06:10:14.531 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:14.583 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:14.892 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:14.935 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:14 smithi008 ceph-mon[184418]: pgmap v1892: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:14.935 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:14 smithi008 ceph-mon[184418]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2023-10-09T06:10:14.935 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:14 smithi008 ceph-mon[184418]: Cluster is now healthy 2023-10-09T06:10:14.935 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:14 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:14.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:14 smithi008 ceph-mon[191227]: pgmap v1892: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:14.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:14 smithi008 ceph-mon[191227]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2023-10-09T06:10:14.935 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:14 smithi008 ceph-mon[191227]: Cluster is now healthy 2023-10-09T06:10:14.936 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:14 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:14.937 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:15.038 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:15.463 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:14 smithi191 ceph-mon[161669]: pgmap v1892: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:15.463 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:14 smithi191 ceph-mon[161669]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2023-10-09T06:10:15.464 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:14 smithi191 ceph-mon[161669]: Cluster is now healthy 2023-10-09T06:10:15.464 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:14 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:15.464 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:15.864 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:15 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:15.864 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:15 smithi008 ceph-mon[191227]: pgmap v1893: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:16.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:15 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:16.134 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:15 smithi008 ceph-mon[184418]: pgmap v1893: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:15 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:16.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:15 smithi191 ceph-mon[161669]: pgmap v1893: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:16.454 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:17.376 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:18.174 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:18 smithi008 ceph-mon[184418]: pgmap v1894: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:18.175 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:18 smithi008 ceph-mon[191227]: pgmap v1894: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:18.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:18 smithi191 ceph-mon[161669]: pgmap v1894: 73 pgs: 73 active+clean; 8.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:20.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:20 smithi008 ceph-mon[191227]: pgmap v1895: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:20.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:20 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:20 smithi008 ceph-mon[184418]: pgmap v1895: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:20.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:20 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:20.965 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:20 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:10:20] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.33.4" 2023-10-09T06:10:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:20 smithi191 ceph-mon[161669]: pgmap v1895: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:21.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:20 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:21.501 INFO:teuthology.orchestra.run.smithi008.stdout:377957122533 2023-10-09T06:10:21.502 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T06:10:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:21 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:21 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:22.215 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:21 smithi008 ceph-mon[184418]: pgmap v1896: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:21 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:22.215 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:21 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:22.216 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:21 smithi008 ceph-mon[191227]: pgmap v1896: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:22.299 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:21 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:21 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:22.300 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:21 smithi191 ceph-mon[161669]: pgmap v1896: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:22.646 INFO:teuthology.orchestra.run.smithi008.stdout:128849019501 2023-10-09T06:10:22.647 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T06:10:24.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:24 smithi008 ceph-mon[184418]: pgmap v1897: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:24.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:24 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:24.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:24 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:24 smithi008 ceph-mon[191227]: pgmap v1897: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:24 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:24.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:24 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:25.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:24 smithi191 ceph-mon[161669]: pgmap v1897: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:24 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:25.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:24 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:26.199 INFO:teuthology.orchestra.run.smithi008.stdout:399431959008 2023-10-09T06:10:26.200 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T06:10:26.300 INFO:teuthology.orchestra.run.smithi008.stdout:442381631957 2023-10-09T06:10:26.301 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T06:10:26.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:26 smithi008 ceph-mon[184418]: pgmap v1898: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:26.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:26 smithi008 ceph-mon[191227]: pgmap v1898: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:27.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:26 smithi191 ceph-mon[161669]: pgmap v1898: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi008.stdout:2306397438024 2023-10-09T06:10:28.160 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T06:10:28.267 INFO:teuthology.orchestra.run.smithi008.stdout:154618823276 2023-10-09T06:10:28.268 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T06:10:28.706 INFO:teuthology.orchestra.run.smithi008.stdout:420906795482 2023-10-09T06:10:28.706 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T06:10:28.895 INFO:teuthology.orchestra.run.smithi008.stdout:545460847017 2023-10-09T06:10:28.895 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T06:10:28.953 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:28 smithi008 ceph-mon[191227]: pgmap v1899: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:28.954 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:28 smithi008 ceph-mon[184418]: pgmap v1899: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:29.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:28 smithi191 ceph-mon[161669]: pgmap v1899: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:29.058 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:29.226 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:29.684 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:29 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:29.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:29 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:30.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:29 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:30.685 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:30 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:10:30] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.33.4" 2023-10-09T06:10:30.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:30 smithi008 ceph-mon[184418]: pgmap v1900: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:30.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:30 smithi008 ceph-mon[191227]: pgmap v1900: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:31.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:30 smithi191 ceph-mon[161669]: pgmap v1900: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:31.081 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:31.345 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:31.347 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:31.541 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:31.599 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:31.990 INFO:teuthology.orchestra.run.smithi008.stdout:377957122535 2023-10-09T06:10:32.011 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:32.947 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:32 smithi008 ceph-mon[191227]: pgmap v1901: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:32.947 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:32 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3702179220' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:10:32.947 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:32 smithi008 ceph-mon[184418]: pgmap v1901: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:32.947 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:32 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3702179220' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:10:33.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:32 smithi191 ceph-mon[161669]: pgmap v1901: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:33.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:32 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3702179220' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:10:33.512 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122533 got 377957122535 for osd.0 2023-10-09T06:10:33.512 DEBUG:teuthology.parallel:result is None 2023-10-09T06:10:34.474 INFO:teuthology.orchestra.run.smithi008.stdout:128849019504 2023-10-09T06:10:34.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:34 smithi008 ceph-mon[184418]: pgmap v1902: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:34.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:34 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1227536244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:10:34.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:34 smithi008 ceph-mon[191227]: pgmap v1902: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:34.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:34 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1227536244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:10:35.012 INFO:teuthology.orchestra.run.smithi008.stdout:154618823278 2023-10-09T06:10:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:34 smithi191 ceph-mon[161669]: pgmap v1902: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:35.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:34 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1227536244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:10:35.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:35 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/3783461209' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:10:35.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:35 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/3783461209' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:10:36.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:35 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/3783461209' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:10:36.962 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:36 smithi008 ceph-mon[184418]: pgmap v1903: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:36.963 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:36 smithi008 ceph-mon[191227]: pgmap v1903: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:37.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:36 smithi191 ceph-mon[161669]: pgmap v1903: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:38.061 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618823276 got 154618823278 for osd.5 2023-10-09T06:10:38.062 DEBUG:teuthology.parallel:result is None 2023-10-09T06:10:38.453 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019501 got 128849019504 for osd.4 2023-10-09T06:10:38.454 DEBUG:teuthology.parallel:result is None 2023-10-09T06:10:38.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:38 smithi008 ceph-mon[184418]: pgmap v1904: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:38.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:38 smithi008 ceph-mon[191227]: pgmap v1904: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:39.049 INFO:teuthology.orchestra.run.smithi008.stdout:420906795485 2023-10-09T06:10:39.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:38 smithi191 ceph-mon[161669]: pgmap v1904: 73 pgs: 73 active+clean; 8.6 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:39.233 INFO:teuthology.orchestra.run.smithi008.stdout:545460847019 2023-10-09T06:10:39.885 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1343181216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:10:39.885 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1804884687' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:10:39.885 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:39 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/284900087' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:10:39.885 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1343181216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:10:39.885 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1804884687' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:10:39.886 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:39 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/284900087' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:10:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1343181216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:10:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1804884687' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:10:40.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:39 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/284900087' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:10:40.714 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:40 smithi008 ceph-mon[184418]: pgmap v1905: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:40.714 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:40 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:10:40] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.33.4" 2023-10-09T06:10:40.715 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:40 smithi008 ceph-mon[191227]: pgmap v1905: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:41.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:40 smithi191 ceph-mon[161669]: pgmap v1905: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:41.121 INFO:tasks.cephadm.ceph_manager.ceph:need seq 545460847017 got 545460847019 for osd.7 2023-10-09T06:10:41.122 DEBUG:teuthology.parallel:result is None 2023-10-09T06:10:41.169 INFO:teuthology.orchestra.run.smithi008.stdout:2306397438026 2023-10-09T06:10:41.335 INFO:teuthology.orchestra.run.smithi008.stdout:442381631960 2023-10-09T06:10:41.486 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795482 got 420906795485 for osd.2 2023-10-09T06:10:41.487 DEBUG:teuthology.parallel:result is None 2023-10-09T06:10:41.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:41 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/263495508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:10:41.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:41 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/263495508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:10:42.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:41 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/263495508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:10:42.703 INFO:teuthology.orchestra.run.smithi008.stdout:399431959012 2023-10-09T06:10:42.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:42 smithi008 ceph-mon[191227]: pgmap v1906: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:42.710 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:42 smithi008 ceph-mon[191227]: from='client.? 172.21.15.8:0/1256344281' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:10:42.811 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2306397438024 got 2306397438026 for osd.6 2023-10-09T06:10:42.811 DEBUG:teuthology.parallel:result is None 2023-10-09T06:10:42.865 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381631957 got 442381631960 for osd.3 2023-10-09T06:10:42.865 DEBUG:teuthology.parallel:result is None 2023-10-09T06:10:42.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:42 smithi008 ceph-mon[184418]: pgmap v1906: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:42.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:42 smithi008 ceph-mon[184418]: from='client.? 172.21.15.8:0/1256344281' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:10:43.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:42 smithi191 ceph-mon[161669]: pgmap v1906: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:43.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:42 smithi191 ceph-mon[161669]: from='client.? 172.21.15.8:0/1256344281' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:10:43.501 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431959008 got 399431959012 for osd.1 2023-10-09T06:10:43.502 DEBUG:teuthology.parallel:result is None 2023-10-09T06:10:43.502 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:10:43.502 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:10:44.050 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:44.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:44 smithi008 ceph-mon[184418]: pgmap v1907: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:44.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:44.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:44 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:44 smithi008 ceph-mon[191227]: pgmap v1907: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:44.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:44 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:45.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:44 smithi191 ceph-mon[161669]: pgmap v1907: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:45.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:44 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:45.871 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:10:45.871 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:10:45.871 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:45 smithi008 ceph-mon[184418]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:45.872 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:10:45.872 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:10:45.872 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:45 smithi008 ceph-mon[191227]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:46.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:10:46.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:10:46.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:45 smithi191 ceph-mon[161669]: from='mgr.25913 172.21.15.8:0/2624747576' entity='mgr.y' 2023-10-09T06:10:46.312 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:10:46.313 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T06:10:46.965 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:46 smithi008 ceph-mon[184418]: pgmap v1908: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:46.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:46 smithi008 ceph-mon[191227]: pgmap v1908: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:47.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:46 smithi191 ceph-mon[161669]: pgmap v1908: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:47.104 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1908,"stamp":"2023-10-09T06:10:45.015483+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12282,"num_read_kb":26921,"num_write":1950,"num_write_kb":27835,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":268,"num_bytes_recovered":10773582,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":220,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8331132,"kb_used_data":79740,"kb_used_omap":59,"kb_used_meta":8251268,"kb_avail":741630084,"statfs":{"total":767960285184,"available":759429206016,"internally_reserved":0,"allocated":81653760,"data_stored":66770426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":60463,"internal_metadata":8449299409},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002953"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":492,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861493+0000","last_change":"2023-10-09T06:09:45.128308+0000","last_active":"2023-10-09T06:10:06.861493+0000","last_peered":"2023-10-09T06:10:06.861493+0000","last_clean":"2023-10-09T06:10:06.861493+0000","last_became_active":"2023-10-09T06:09:45.128032+0000","last_became_peered":"2023-10-09T06:09:45.128032+0000","last_unstale":"2023-10-09T06:10:06.861493+0000","last_undegraded":"2023-10-09T06:10:06.861493+0000","last_fullsized":"2023-10-09T06:10:06.861493+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.869740+0000","last_change":"2023-10-09T06:07:47.056618+0000","last_active":"2023-10-09T06:10:07.869740+0000","last_peered":"2023-10-09T06:10:07.869740+0000","last_clean":"2023-10-09T06:10:07.869740+0000","last_became_active":"2023-10-09T06:07:47.056311+0000","last_became_peered":"2023-10-09T06:07:47.056311+0000","last_unstale":"2023-10-09T06:10:07.869740+0000","last_undegraded":"2023-10-09T06:10:07.869740+0000","last_fullsized":"2023-10-09T06:10:07.869740+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:28.019550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":379,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864188+0000","last_change":"2023-10-09T06:07:24.553497+0000","last_active":"2023-10-09T06:10:06.864188+0000","last_peered":"2023-10-09T06:10:06.864188+0000","last_clean":"2023-10-09T06:10:06.864188+0000","last_became_active":"2023-10-09T06:07:24.553189+0000","last_became_peered":"2023-10-09T06:07:24.553189+0000","last_unstale":"2023-10-09T06:10:06.864188+0000","last_undegraded":"2023-10-09T06:10:06.864188+0000","last_fullsized":"2023-10-09T06:10:06.864188+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:53.168858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":637,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.875402+0000","last_change":"2023-10-09T06:08:34.675487+0000","last_active":"2023-10-09T06:10:07.875402+0000","last_peered":"2023-10-09T06:10:07.875402+0000","last_clean":"2023-10-09T06:10:07.875402+0000","last_became_active":"2023-10-09T06:08:34.674645+0000","last_became_peered":"2023-10-09T06:08:34.674645+0000","last_unstale":"2023-10-09T06:10:07.875402+0000","last_undegraded":"2023-10-09T06:10:07.875402+0000","last_fullsized":"2023-10-09T06:10:07.875402+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:50.956195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":486,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029129+0000","last_change":"2023-10-09T06:09:37.000349+0000","last_active":"2023-10-09T06:10:07.029129+0000","last_peered":"2023-10-09T06:10:07.029129+0000","last_clean":"2023-10-09T06:10:07.029129+0000","last_became_active":"2023-10-09T06:09:36.999813+0000","last_became_peered":"2023-10-09T06:09:36.999813+0000","last_unstale":"2023-10-09T06:10:07.029129+0000","last_undegraded":"2023-10-09T06:10:07.029129+0000","last_fullsized":"2023-10-09T06:10:07.029129+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:07.235570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":230,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.939303+0000","last_change":"2023-10-09T06:10:07.939303+0000","last_active":"2023-10-09T06:10:07.939303+0000","last_peered":"2023-10-09T06:10:07.939303+0000","last_clean":"2023-10-09T06:10:07.939303+0000","last_became_active":"2023-10-09T06:10:07.938952+0000","last_became_peered":"2023-10-09T06:10:07.938952+0000","last_unstale":"2023-10-09T06:10:07.939303+0000","last_undegraded":"2023-10-09T06:10:07.939303+0000","last_fullsized":"2023-10-09T06:10:07.939303+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":599,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.865748+0000","last_change":"2023-10-09T06:09:14.937602+0000","last_active":"2023-10-09T06:10:06.865748+0000","last_peered":"2023-10-09T06:10:06.865748+0000","last_clean":"2023-10-09T06:10:06.865748+0000","last_became_active":"2023-10-09T06:09:14.937271+0000","last_became_peered":"2023-10-09T06:09:14.937271+0000","last_unstale":"2023-10-09T06:10:06.865748+0000","last_undegraded":"2023-10-09T06:10:06.865748+0000","last_fullsized":"2023-10-09T06:10:06.865748+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:04.497640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":321,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029180+0000","last_change":"2023-10-09T06:09:37.000097+0000","last_active":"2023-10-09T06:10:07.029180+0000","last_peered":"2023-10-09T06:10:07.029180+0000","last_clean":"2023-10-09T06:10:07.029180+0000","last_became_active":"2023-10-09T06:09:36.999415+0000","last_became_peered":"2023-10-09T06:09:36.999415+0000","last_unstale":"2023-10-09T06:10:07.029180+0000","last_undegraded":"2023-10-09T06:10:07.029180+0000","last_fullsized":"2023-10-09T06:10:07.029180+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:59.199203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":39,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859478+0000","last_change":"2023-10-09T06:08:53.670871+0000","last_active":"2023-10-09T06:10:06.859478+0000","last_peered":"2023-10-09T06:10:06.859478+0000","last_clean":"2023-10-09T06:10:06.859478+0000","last_became_active":"2023-10-09T06:08:53.670472+0000","last_became_peered":"2023-10-09T06:08:53.670472+0000","last_unstale":"2023-10-09T06:10:06.859478+0000","last_undegraded":"2023-10-09T06:10:06.859478+0000","last_fullsized":"2023-10-09T06:10:06.859478+0000","mapping_epoch":609,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":245,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029225+0000","last_change":"2023-10-09T06:08:34.670870+0000","last_active":"2023-10-09T06:10:07.029225+0000","last_peered":"2023-10-09T06:10:07.029225+0000","last_clean":"2023-10-09T06:10:07.029225+0000","last_became_active":"2023-10-09T06:08:34.670528+0000","last_became_peered":"2023-10-09T06:08:34.670528+0000","last_unstale":"2023-10-09T06:10:07.029225+0000","last_undegraded":"2023-10-09T06:10:07.029225+0000","last_fullsized":"2023-10-09T06:10:07.029225+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:14:05.732701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":268,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868788+0000","last_change":"2023-10-09T06:10:00.686515+0000","last_active":"2023-10-09T06:10:06.868788+0000","last_peered":"2023-10-09T06:10:06.868788+0000","last_clean":"2023-10-09T06:10:06.868788+0000","last_became_active":"2023-10-09T06:10:00.686115+0000","last_became_peered":"2023-10-09T06:10:00.686115+0000","last_unstale":"2023-10-09T06:10:06.868788+0000","last_undegraded":"2023-10-09T06:10:06.868788+0000","last_fullsized":"2023-10-09T06:10:06.868788+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":450,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864721+0000","last_change":"2023-10-09T06:05:15.568750+0000","last_active":"2023-10-09T06:10:06.864721+0000","last_peered":"2023-10-09T06:10:06.864721+0000","last_clean":"2023-10-09T06:10:06.864721+0000","last_became_active":"2023-10-09T06:05:15.568039+0000","last_became_peered":"2023-10-09T06:05:15.568039+0000","last_unstale":"2023-10-09T06:10:06.864721+0000","last_undegraded":"2023-10-09T06:10:06.864721+0000","last_fullsized":"2023-10-09T06:10:06.864721+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:38.828502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":292,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861684+0000","last_change":"2023-10-09T06:08:34.656541+0000","last_active":"2023-10-09T06:10:06.861684+0000","last_peered":"2023-10-09T06:10:06.861684+0000","last_clean":"2023-10-09T06:10:06.861684+0000","last_became_active":"2023-10-09T06:08:34.656094+0000","last_became_peered":"2023-10-09T06:08:34.656094+0000","last_unstale":"2023-10-09T06:10:06.861684+0000","last_undegraded":"2023-10-09T06:10:06.861684+0000","last_fullsized":"2023-10-09T06:10:06.861684+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":583,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861215+0000","last_change":"2023-10-09T06:09:45.134387+0000","last_active":"2023-10-09T06:10:06.861215+0000","last_peered":"2023-10-09T06:10:06.861215+0000","last_clean":"2023-10-09T06:10:06.861215+0000","last_became_active":"2023-10-09T06:09:45.133037+0000","last_became_peered":"2023-10-09T06:09:45.133037+0000","last_unstale":"2023-10-09T06:10:06.861215+0000","last_undegraded":"2023-10-09T06:10:06.861215+0000","last_fullsized":"2023-10-09T06:10:06.861215+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":277,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.869304+0000","last_change":"2023-10-09T06:10:00.686444+0000","last_active":"2023-10-09T06:10:06.869304+0000","last_peered":"2023-10-09T06:10:06.869304+0000","last_clean":"2023-10-09T06:10:06.869304+0000","last_became_active":"2023-10-09T06:10:00.686078+0000","last_became_peered":"2023-10-09T06:10:00.686078+0000","last_unstale":"2023-10-09T06:10:06.869304+0000","last_undegraded":"2023-10-09T06:10:06.869304+0000","last_fullsized":"2023-10-09T06:10:06.869304+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":190,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029323+0000","last_change":"2023-10-09T06:09:37.001713+0000","last_active":"2023-10-09T06:10:07.029323+0000","last_peered":"2023-10-09T06:10:07.029323+0000","last_clean":"2023-10-09T06:10:07.029323+0000","last_became_active":"2023-10-09T06:09:37.001410+0000","last_became_peered":"2023-10-09T06:09:37.001410+0000","last_unstale":"2023-10-09T06:10:07.029323+0000","last_undegraded":"2023-10-09T06:10:07.029323+0000","last_fullsized":"2023-10-09T06:10:07.029323+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:49.657779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":671,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.865973+0000","last_change":"2023-10-09T06:08:34.676004+0000","last_active":"2023-10-09T06:10:06.865973+0000","last_peered":"2023-10-09T06:10:06.865973+0000","last_clean":"2023-10-09T06:10:06.865973+0000","last_became_active":"2023-10-09T06:08:34.674879+0000","last_became_peered":"2023-10-09T06:08:34.674879+0000","last_unstale":"2023-10-09T06:10:06.865973+0000","last_undegraded":"2023-10-09T06:10:06.865973+0000","last_fullsized":"2023-10-09T06:10:06.865973+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:27.794738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":18,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859703+0000","last_change":"2023-10-09T06:09:49.140421+0000","last_active":"2023-10-09T06:10:06.859703+0000","last_peered":"2023-10-09T06:10:06.859703+0000","last_clean":"2023-10-09T06:10:06.859703+0000","last_became_active":"2023-10-09T06:09:49.139538+0000","last_became_peered":"2023-10-09T06:09:49.139538+0000","last_unstale":"2023-10-09T06:10:06.859703+0000","last_undegraded":"2023-10-09T06:10:06.859703+0000","last_fullsized":"2023-10-09T06:10:06.859703+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":287,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868482+0000","last_change":"2023-10-09T06:10:00.690974+0000","last_active":"2023-10-09T06:10:06.868482+0000","last_peered":"2023-10-09T06:10:06.868482+0000","last_clean":"2023-10-09T06:10:06.868482+0000","last_became_active":"2023-10-09T06:10:00.690593+0000","last_became_peered":"2023-10-09T06:10:00.690593+0000","last_unstale":"2023-10-09T06:10:06.868482+0000","last_undegraded":"2023-10-09T06:10:06.868482+0000","last_fullsized":"2023-10-09T06:10:06.868482+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":641,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.866086+0000","last_change":"2023-10-09T06:08:34.674326+0000","last_active":"2023-10-09T06:10:06.866086+0000","last_peered":"2023-10-09T06:10:06.866086+0000","last_clean":"2023-10-09T06:10:06.866086+0000","last_became_active":"2023-10-09T06:08:34.673989+0000","last_became_peered":"2023-10-09T06:08:34.673989+0000","last_unstale":"2023-10-09T06:10:06.866086+0000","last_undegraded":"2023-10-09T06:10:06.866086+0000","last_fullsized":"2023-10-09T06:10:06.866086+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:12.836741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":290,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.876836+0000","last_change":"2023-10-09T06:10:07.876836+0000","last_active":"2023-10-09T06:10:07.876836+0000","last_peered":"2023-10-09T06:10:07.876836+0000","last_clean":"2023-10-09T06:10:07.876836+0000","last_became_active":"2023-10-09T06:10:07.876099+0000","last_became_peered":"2023-10-09T06:10:07.876099+0000","last_unstale":"2023-10-09T06:10:07.876836+0000","last_undegraded":"2023-10-09T06:10:07.876836+0000","last_fullsized":"2023-10-09T06:10:07.876836+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":330,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029479+0000","last_change":"2023-10-09T06:09:37.001585+0000","last_active":"2023-10-09T06:10:07.029479+0000","last_peered":"2023-10-09T06:10:07.029479+0000","last_clean":"2023-10-09T06:10:07.029479+0000","last_became_active":"2023-10-09T06:09:37.001273+0000","last_became_peered":"2023-10-09T06:09:37.001273+0000","last_unstale":"2023-10-09T06:10:07.029479+0000","last_undegraded":"2023-10-09T06:10:07.029479+0000","last_fullsized":"2023-10-09T06:10:07.029479+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:35.765491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":293,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.869778+0000","last_change":"2023-10-09T06:10:00.689010+0000","last_active":"2023-10-09T06:10:06.869778+0000","last_peered":"2023-10-09T06:10:06.869778+0000","last_clean":"2023-10-09T06:10:06.869778+0000","last_became_active":"2023-10-09T06:10:00.688613+0000","last_became_peered":"2023-10-09T06:10:00.688613+0000","last_unstale":"2023-10-09T06:10:06.869778+0000","last_undegraded":"2023-10-09T06:10:06.869778+0000","last_fullsized":"2023-10-09T06:10:06.869778+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":18,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859590+0000","last_change":"2023-10-09T06:09:49.140131+0000","last_active":"2023-10-09T06:10:06.859590+0000","last_peered":"2023-10-09T06:10:06.859590+0000","last_clean":"2023-10-09T06:10:06.859590+0000","last_became_active":"2023-10-09T06:09:49.139438+0000","last_became_peered":"2023-10-09T06:09:49.139438+0000","last_unstale":"2023-10-09T06:10:06.859590+0000","last_undegraded":"2023-10-09T06:10:06.859590+0000","last_fullsized":"2023-10-09T06:10:06.859590+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":251,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868882+0000","last_change":"2023-10-09T06:10:00.685537+0000","last_active":"2023-10-09T06:10:06.868882+0000","last_peered":"2023-10-09T06:10:06.868882+0000","last_clean":"2023-10-09T06:10:06.868882+0000","last_became_active":"2023-10-09T06:10:00.685103+0000","last_became_peered":"2023-10-09T06:10:00.685103+0000","last_unstale":"2023-10-09T06:10:06.868882+0000","last_undegraded":"2023-10-09T06:10:06.868882+0000","last_fullsized":"2023-10-09T06:10:06.868882+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":638,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861144+0000","last_change":"2023-10-09T06:09:45.404191+0000","last_active":"2023-10-09T06:10:06.861144+0000","last_peered":"2023-10-09T06:10:06.861144+0000","last_clean":"2023-10-09T06:10:06.861144+0000","last_became_active":"2023-10-09T06:09:45.403145+0000","last_became_peered":"2023-10-09T06:09:45.403145+0000","last_unstale":"2023-10-09T06:10:06.861144+0000","last_undegraded":"2023-10-09T06:10:06.861144+0000","last_fullsized":"2023-10-09T06:10:06.861144+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":285,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868900+0000","last_change":"2023-10-09T06:10:00.688504+0000","last_active":"2023-10-09T06:10:06.868900+0000","last_peered":"2023-10-09T06:10:06.868900+0000","last_clean":"2023-10-09T06:10:06.868900+0000","last_became_active":"2023-10-09T06:10:00.688167+0000","last_became_peered":"2023-10-09T06:10:00.688167+0000","last_unstale":"2023-10-09T06:10:06.868900+0000","last_undegraded":"2023-10-09T06:10:06.868900+0000","last_fullsized":"2023-10-09T06:10:06.868900+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":313,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.939491+0000","last_change":"2023-10-09T06:10:07.939491+0000","last_active":"2023-10-09T06:10:07.939491+0000","last_peered":"2023-10-09T06:10:07.939491+0000","last_clean":"2023-10-09T06:10:07.939491+0000","last_became_active":"2023-10-09T06:10:07.939133+0000","last_became_peered":"2023-10-09T06:10:07.939133+0000","last_unstale":"2023-10-09T06:10:07.939491+0000","last_undegraded":"2023-10-09T06:10:07.939491+0000","last_fullsized":"2023-10-09T06:10:07.939491+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":502,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859509+0000","last_change":"2023-10-09T06:09:53.246986+0000","last_active":"2023-10-09T06:10:06.859509+0000","last_peered":"2023-10-09T06:10:06.859509+0000","last_clean":"2023-10-09T06:10:06.859509+0000","last_became_active":"2023-10-09T06:09:53.246690+0000","last_became_peered":"2023-10-09T06:09:53.246690+0000","last_unstale":"2023-10-09T06:10:06.859509+0000","last_undegraded":"2023-10-09T06:10:06.859509+0000","last_fullsized":"2023-10-09T06:10:06.859509+0000","mapping_epoch":632,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":574,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.865226+0000","last_change":"2023-10-09T06:00:05.352723+0000","last_active":"2023-10-09T06:10:06.865226+0000","last_peered":"2023-10-09T06:10:06.865226+0000","last_clean":"2023-10-09T06:10:06.865226+0000","last_became_active":"2023-10-09T06:00:05.352008+0000","last_became_peered":"2023-10-09T06:00:05.352008+0000","last_unstale":"2023-10-09T06:10:06.865226+0000","last_undegraded":"2023-10-09T06:10:06.865226+0000","last_fullsized":"2023-10-09T06:10:06.865226+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:25.302918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":504,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029315+0000","last_change":"2023-10-09T06:09:37.004819+0000","last_active":"2023-10-09T06:10:07.029315+0000","last_peered":"2023-10-09T06:10:07.029315+0000","last_clean":"2023-10-09T06:10:07.029315+0000","last_became_active":"2023-10-09T06:09:37.003299+0000","last_became_peered":"2023-10-09T06:09:37.003299+0000","last_unstale":"2023-10-09T06:10:07.029315+0000","last_undegraded":"2023-10-09T06:10:07.029315+0000","last_fullsized":"2023-10-09T06:10:07.029315+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:36.536844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":658,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861304+0000","last_change":"2023-10-09T06:09:45.404027+0000","last_active":"2023-10-09T06:10:06.861304+0000","last_peered":"2023-10-09T06:10:06.861304+0000","last_clean":"2023-10-09T06:10:06.861304+0000","last_became_active":"2023-10-09T06:09:45.403071+0000","last_became_peered":"2023-10-09T06:09:45.403071+0000","last_unstale":"2023-10-09T06:10:06.861304+0000","last_undegraded":"2023-10-09T06:10:06.861304+0000","last_fullsized":"2023-10-09T06:10:06.861304+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":286,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.955878+0000","last_change":"2023-10-09T06:10:07.955878+0000","last_active":"2023-10-09T06:10:07.955878+0000","last_peered":"2023-10-09T06:10:07.955878+0000","last_clean":"2023-10-09T06:10:07.955878+0000","last_became_active":"2023-10-09T06:10:07.955458+0000","last_became_peered":"2023-10-09T06:10:07.955458+0000","last_unstale":"2023-10-09T06:10:07.955878+0000","last_undegraded":"2023-10-09T06:10:07.955878+0000","last_fullsized":"2023-10-09T06:10:07.955878+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":554,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861412+0000","last_change":"2023-10-09T06:09:45.132016+0000","last_active":"2023-10-09T06:10:06.861412+0000","last_peered":"2023-10-09T06:10:06.861412+0000","last_clean":"2023-10-09T06:10:06.861412+0000","last_became_active":"2023-10-09T06:09:45.131661+0000","last_became_peered":"2023-10-09T06:09:45.131661+0000","last_unstale":"2023-10-09T06:10:06.861412+0000","last_undegraded":"2023-10-09T06:10:06.861412+0000","last_fullsized":"2023-10-09T06:10:06.861412+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":612,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.860828+0000","last_change":"2023-10-09T06:09:45.132179+0000","last_active":"2023-10-09T06:10:06.860828+0000","last_peered":"2023-10-09T06:10:06.860828+0000","last_clean":"2023-10-09T06:10:06.860828+0000","last_became_active":"2023-10-09T06:09:45.131540+0000","last_became_peered":"2023-10-09T06:09:45.131540+0000","last_unstale":"2023-10-09T06:10:06.860828+0000","last_undegraded":"2023-10-09T06:10:06.860828+0000","last_fullsized":"2023-10-09T06:10:06.860828+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":555,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861540+0000","last_change":"2023-10-09T06:09:45.130020+0000","last_active":"2023-10-09T06:10:06.861540+0000","last_peered":"2023-10-09T06:10:06.861540+0000","last_clean":"2023-10-09T06:10:06.861540+0000","last_became_active":"2023-10-09T06:09:45.129685+0000","last_became_peered":"2023-10-09T06:09:45.129685+0000","last_unstale":"2023-10-09T06:10:06.861540+0000","last_undegraded":"2023-10-09T06:10:06.861540+0000","last_fullsized":"2023-10-09T06:10:06.861540+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":341,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029078+0000","last_change":"2023-10-09T06:09:37.005423+0000","last_active":"2023-10-09T06:10:07.029078+0000","last_peered":"2023-10-09T06:10:07.029078+0000","last_clean":"2023-10-09T06:10:07.029078+0000","last_became_active":"2023-10-09T06:09:37.002797+0000","last_became_peered":"2023-10-09T06:09:37.002797+0000","last_unstale":"2023-10-09T06:10:07.029078+0000","last_undegraded":"2023-10-09T06:10:07.029078+0000","last_fullsized":"2023-10-09T06:10:07.029078+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:52.098169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":340,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.866157+0000","last_change":"2023-10-09T06:07:47.057804+0000","last_active":"2023-10-09T06:10:06.866157+0000","last_peered":"2023-10-09T06:10:06.866157+0000","last_clean":"2023-10-09T06:10:06.866157+0000","last_became_active":"2023-10-09T06:07:47.057508+0000","last_became_peered":"2023-10-09T06:07:47.057508+0000","last_unstale":"2023-10-09T06:10:06.866157+0000","last_undegraded":"2023-10-09T06:10:06.866157+0000","last_fullsized":"2023-10-09T06:10:06.866157+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:45.041480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"604'322","reported_seq":1646,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.938088+0000","last_change":"2023-10-09T06:09:22.719273+0000","last_active":"2023-10-09T06:10:07.938088+0000","last_peered":"2023-10-09T06:10:07.938088+0000","last_clean":"2023-10-09T06:10:07.938088+0000","last_became_active":"2023-10-09T06:09:22.718603+0000","last_became_peered":"2023-10-09T06:09:22.718603+0000","last_unstale":"2023-10-09T06:10:07.938088+0000","last_undegraded":"2023-10-09T06:10:07.938088+0000","last_fullsized":"2023-10-09T06:10:07.938088+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":322,"log_dups_size":0,"ondisk_log_size":322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:06.922542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062772900000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1736,"num_read_kb":2559,"num_write":305,"num_write_kb":642,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":39,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859908+0000","last_change":"2023-10-09T06:08:53.661676+0000","last_active":"2023-10-09T06:10:06.859908+0000","last_peered":"2023-10-09T06:10:06.859908+0000","last_clean":"2023-10-09T06:10:06.859908+0000","last_became_active":"2023-10-09T06:08:53.661314+0000","last_became_peered":"2023-10-09T06:08:53.661314+0000","last_unstale":"2023-10-09T06:10:06.859908+0000","last_undegraded":"2023-10-09T06:10:06.859908+0000","last_fullsized":"2023-10-09T06:10:06.859908+0000","mapping_epoch":609,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":198,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864646+0000","last_change":"2023-10-09T06:09:49.952540+0000","last_active":"2023-10-09T06:10:06.864646+0000","last_peered":"2023-10-09T06:10:06.864646+0000","last_clean":"2023-10-09T06:10:06.864646+0000","last_became_active":"2023-10-09T06:09:49.952226+0000","last_became_peered":"2023-10-09T06:09:49.952226+0000","last_unstale":"2023-10-09T06:10:06.864646+0000","last_undegraded":"2023-10-09T06:10:06.864646+0000","last_fullsized":"2023-10-09T06:10:06.864646+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:21.483423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"603'130","reported_seq":1362,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859470+0000","last_change":"2023-10-09T06:09:53.163090+0000","last_active":"2023-10-09T06:10:06.859470+0000","last_peered":"2023-10-09T06:10:06.859470+0000","last_clean":"2023-10-09T06:10:06.859470+0000","last_became_active":"2023-10-09T06:09:53.162471+0000","last_became_peered":"2023-10-09T06:09:53.162471+0000","last_unstale":"2023-10-09T06:10:06.859470+0000","last_undegraded":"2023-10-09T06:10:06.859470+0000","last_fullsized":"2023-10-09T06:10:06.859470+0000","mapping_epoch":632,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1200,"num_read_kb":3722,"num_write":133,"num_write_kb":2436,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":581,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.862322+0000","last_change":"2023-10-09T06:09:45.135716+0000","last_active":"2023-10-09T06:10:06.862322+0000","last_peered":"2023-10-09T06:10:06.862322+0000","last_clean":"2023-10-09T06:10:06.862322+0000","last_became_active":"2023-10-09T06:09:45.133517+0000","last_became_peered":"2023-10-09T06:09:45.133517+0000","last_unstale":"2023-10-09T06:10:06.862322+0000","last_undegraded":"2023-10-09T06:10:06.862322+0000","last_fullsized":"2023-10-09T06:10:06.862322+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":445,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864293+0000","last_change":"2023-10-09T06:09:22.950106+0000","last_active":"2023-10-09T06:10:06.864293+0000","last_peered":"2023-10-09T06:10:06.864293+0000","last_clean":"2023-10-09T06:10:06.864293+0000","last_became_active":"2023-10-09T06:09:22.949792+0000","last_became_peered":"2023-10-09T06:09:22.949792+0000","last_unstale":"2023-10-09T06:10:06.864293+0000","last_undegraded":"2023-10-09T06:10:06.864293+0000","last_fullsized":"2023-10-09T06:10:06.864293+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:53.990328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"604'210","reported_seq":1465,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.869402+0000","last_change":"2023-10-09T06:08:36.715763+0000","last_active":"2023-10-09T06:10:07.869402+0000","last_peered":"2023-10-09T06:10:07.869402+0000","last_clean":"2023-10-09T06:10:07.869402+0000","last_became_active":"2023-10-09T06:05:44.534078+0000","last_became_peered":"2023-10-09T06:05:44.534078+0000","last_unstale":"2023-10-09T06:10:07.869402+0000","last_undegraded":"2023-10-09T06:10:07.869402+0000","last_fullsized":"2023-10-09T06:10:07.869402+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":210,"log_dups_size":0,"ondisk_log_size":210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:45.519427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042835799999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1263,"num_read_kb":5231,"num_write":190,"num_write_kb":8397,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":462,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864529+0000","last_change":"2023-10-09T06:05:30.447496+0000","last_active":"2023-10-09T06:10:06.864529+0000","last_peered":"2023-10-09T06:10:06.864529+0000","last_clean":"2023-10-09T06:10:06.864529+0000","last_became_active":"2023-10-09T06:05:30.447219+0000","last_became_peered":"2023-10-09T06:05:30.447219+0000","last_unstale":"2023-10-09T06:10:06.864529+0000","last_undegraded":"2023-10-09T06:10:06.864529+0000","last_fullsized":"2023-10-09T06:10:06.864529+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:08.869779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":143,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.937936+0000","last_change":"2023-10-09T06:08:53.663036+0000","last_active":"2023-10-09T06:10:07.937936+0000","last_peered":"2023-10-09T06:10:07.937936+0000","last_clean":"2023-10-09T06:10:07.937936+0000","last_became_active":"2023-10-09T06:08:53.662753+0000","last_became_peered":"2023-10-09T06:08:53.662753+0000","last_unstale":"2023-10-09T06:10:07.937936+0000","last_undegraded":"2023-10-09T06:10:07.937936+0000","last_fullsized":"2023-10-09T06:10:07.937936+0000","mapping_epoch":609,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:37.084661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"604'451","reported_seq":1887,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859419+0000","last_change":"2023-10-09T06:08:56.855202+0000","last_active":"2023-10-09T06:10:06.859419+0000","last_peered":"2023-10-09T06:10:06.859419+0000","last_clean":"2023-10-09T06:10:06.859419+0000","last_became_active":"2023-10-09T06:08:56.854446+0000","last_became_peered":"2023-10-09T06:08:56.854446+0000","last_unstale":"2023-10-09T06:10:06.859419+0000","last_undegraded":"2023-10-09T06:10:06.859419+0000","last_fullsized":"2023-10-09T06:10:06.859419+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":451,"log_dups_size":0,"ondisk_log_size":451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2690,"num_read_kb":2377,"num_write":394,"num_write_kb":242,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":834,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868263+0000","last_change":"2023-10-09T06:10:00.691112+0000","last_active":"2023-10-09T06:10:06.868263+0000","last_peered":"2023-10-09T06:10:06.868263+0000","last_clean":"2023-10-09T06:10:06.868263+0000","last_became_active":"2023-10-09T06:10:00.690656+0000","last_became_peered":"2023-10-09T06:10:00.690656+0000","last_unstale":"2023-10-09T06:10:06.868263+0000","last_undegraded":"2023-10-09T06:10:06.868263+0000","last_fullsized":"2023-10-09T06:10:06.868263+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":532,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.954636+0000","last_change":"2023-10-09T06:09:37.911323+0000","last_active":"2023-10-09T06:10:07.954636+0000","last_peered":"2023-10-09T06:10:07.954636+0000","last_clean":"2023-10-09T06:10:07.954636+0000","last_became_active":"2023-10-09T06:09:37.910959+0000","last_became_peered":"2023-10-09T06:09:37.910959+0000","last_unstale":"2023-10-09T06:10:07.954636+0000","last_undegraded":"2023-10-09T06:10:07.954636+0000","last_fullsized":"2023-10-09T06:10:07.954636+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:17.207997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":20,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859798+0000","last_change":"2023-10-09T06:09:47.257362+0000","last_active":"2023-10-09T06:10:06.859798+0000","last_peered":"2023-10-09T06:10:06.859798+0000","last_clean":"2023-10-09T06:10:06.859798+0000","last_became_active":"2023-10-09T06:09:47.256947+0000","last_became_peered":"2023-10-09T06:09:47.256947+0000","last_unstale":"2023-10-09T06:10:06.859798+0000","last_undegraded":"2023-10-09T06:10:06.859798+0000","last_fullsized":"2023-10-09T06:10:06.859798+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"603'179","reported_seq":1384,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868281+0000","last_change":"2023-10-09T06:10:00.689307+0000","last_active":"2023-10-09T06:10:06.868281+0000","last_peered":"2023-10-09T06:10:06.868281+0000","last_clean":"2023-10-09T06:10:06.868281+0000","last_became_active":"2023-10-09T06:10:00.688637+0000","last_became_peered":"2023-10-09T06:10:00.688637+0000","last_unstale":"2023-10-09T06:10:06.868281+0000","last_undegraded":"2023-10-09T06:10:06.868281+0000","last_fullsized":"2023-10-09T06:10:06.868281+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":179,"log_dups_size":0,"ondisk_log_size":179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1445,"num_read_kb":3702,"num_write":128,"num_write_kb":2304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":630,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.875543+0000","last_change":"2023-10-09T06:07:24.555346+0000","last_active":"2023-10-09T06:10:07.875543+0000","last_peered":"2023-10-09T06:10:07.875543+0000","last_clean":"2023-10-09T06:10:07.875543+0000","last_became_active":"2023-10-09T06:07:24.554992+0000","last_became_peered":"2023-10-09T06:07:24.554992+0000","last_unstale":"2023-10-09T06:10:07.875543+0000","last_undegraded":"2023-10-09T06:10:07.875543+0000","last_fullsized":"2023-10-09T06:10:07.875543+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:43.884493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":71,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.866262+0000","last_change":"2023-10-09T06:07:50.496088+0000","last_active":"2023-10-09T06:10:06.866262+0000","last_peered":"2023-10-09T06:10:06.866262+0000","last_clean":"2023-10-09T06:10:06.866262+0000","last_became_active":"2023-10-09T06:07:50.495249+0000","last_became_peered":"2023-10-09T06:07:50.495249+0000","last_unstale":"2023-10-09T06:10:06.866262+0000","last_undegraded":"2023-10-09T06:10:06.866262+0000","last_fullsized":"2023-10-09T06:10:06.866262+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:52.044105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"604'221","reported_seq":1288,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868948+0000","last_change":"2023-10-09T06:10:00.689601+0000","last_active":"2023-10-09T06:10:06.868948+0000","last_peered":"2023-10-09T06:10:06.868948+0000","last_clean":"2023-10-09T06:10:06.868948+0000","last_became_active":"2023-10-09T06:10:00.688878+0000","last_became_peered":"2023-10-09T06:10:00.688878+0000","last_unstale":"2023-10-09T06:10:06.868948+0000","last_undegraded":"2023-10-09T06:10:06.868948+0000","last_fullsized":"2023-10-09T06:10:06.868948+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":221,"log_dups_size":0,"ondisk_log_size":221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1081,"num_read_kb":4262,"num_write":208,"num_write_kb":4363,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":517,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864446+0000","last_change":"2023-10-09T06:09:22.714892+0000","last_active":"2023-10-09T06:10:06.864446+0000","last_peered":"2023-10-09T06:10:06.864446+0000","last_clean":"2023-10-09T06:10:06.864446+0000","last_became_active":"2023-10-09T06:09:22.714584+0000","last_became_peered":"2023-10-09T06:09:22.714584+0000","last_unstale":"2023-10-09T06:10:06.864446+0000","last_undegraded":"2023-10-09T06:10:06.864446+0000","last_fullsized":"2023-10-09T06:10:06.864446+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:29.022515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":506,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.875487+0000","last_change":"2023-10-09T06:09:14.937686+0000","last_active":"2023-10-09T06:10:07.875487+0000","last_peered":"2023-10-09T06:10:07.875487+0000","last_clean":"2023-10-09T06:10:07.875487+0000","last_became_active":"2023-10-09T06:09:14.937378+0000","last_became_peered":"2023-10-09T06:09:14.937378+0000","last_unstale":"2023-10-09T06:10:07.875487+0000","last_undegraded":"2023-10-09T06:10:07.875487+0000","last_fullsized":"2023-10-09T06:10:07.875487+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:10.880068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":67,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029533+0000","last_change":"2023-10-09T06:09:37.001519+0000","last_active":"2023-10-09T06:10:07.029533+0000","last_peered":"2023-10-09T06:10:07.029533+0000","last_clean":"2023-10-09T06:10:07.029533+0000","last_became_active":"2023-10-09T06:09:37.001149+0000","last_became_peered":"2023-10-09T06:09:37.001149+0000","last_unstale":"2023-10-09T06:10:07.029533+0000","last_undegraded":"2023-10-09T06:10:07.029533+0000","last_fullsized":"2023-10-09T06:10:07.029533+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:04.062745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"604'253","reported_seq":943,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864382+0000","last_change":"2023-10-09T06:08:36.755071+0000","last_active":"2023-10-09T06:10:06.864382+0000","last_peered":"2023-10-09T06:10:06.864382+0000","last_clean":"2023-10-09T06:10:06.864382+0000","last_became_active":"2023-10-09T06:04:32.959073+0000","last_became_peered":"2023-10-09T06:04:32.959073+0000","last_unstale":"2023-10-09T06:10:06.864382+0000","last_undegraded":"2023-10-09T06:10:06.864382+0000","last_fullsized":"2023-10-09T06:10:06.864382+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":253,"log_dups_size":0,"ondisk_log_size":253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:00.843837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043321599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1018,"num_read_kb":2880,"num_write":192,"num_write_kb":2673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":310,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.954417+0000","last_change":"2023-10-09T06:09:37.003040+0000","last_active":"2023-10-09T06:10:07.954417+0000","last_peered":"2023-10-09T06:10:07.954417+0000","last_clean":"2023-10-09T06:10:07.954417+0000","last_became_active":"2023-10-09T06:09:37.001673+0000","last_became_peered":"2023-10-09T06:09:37.001673+0000","last_unstale":"2023-10-09T06:10:07.954417+0000","last_undegraded":"2023-10-09T06:10:07.954417+0000","last_fullsized":"2023-10-09T06:10:07.954417+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:41.397215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"604'184","reported_seq":1186,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.878173+0000","last_change":"2023-10-09T06:10:07.878173+0000","last_active":"2023-10-09T06:10:07.878173+0000","last_peered":"2023-10-09T06:10:07.878173+0000","last_clean":"2023-10-09T06:10:07.878173+0000","last_became_active":"2023-10-09T06:10:07.877472+0000","last_became_peered":"2023-10-09T06:10:07.877472+0000","last_unstale":"2023-10-09T06:10:07.878173+0000","last_undegraded":"2023-10-09T06:10:07.878173+0000","last_fullsized":"2023-10-09T06:10:07.878173+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":184,"log_dups_size":0,"ondisk_log_size":184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1555,"num_read_kb":1688,"num_write":140,"num_write_kb":4333,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":75,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.866181+0000","last_change":"2023-10-09T06:07:50.543307+0000","last_active":"2023-10-09T06:10:06.866181+0000","last_peered":"2023-10-09T06:10:06.866181+0000","last_clean":"2023-10-09T06:10:06.866181+0000","last_became_active":"2023-10-09T06:07:50.495736+0000","last_became_peered":"2023-10-09T06:07:50.495736+0000","last_unstale":"2023-10-09T06:10:06.866181+0000","last_undegraded":"2023-10-09T06:10:06.866181+0000","last_fullsized":"2023-10-09T06:10:06.866181+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:47.121228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":164,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861030+0000","last_change":"2023-10-09T06:09:45.130162+0000","last_active":"2023-10-09T06:10:06.861030+0000","last_peered":"2023-10-09T06:10:06.861030+0000","last_clean":"2023-10-09T06:10:06.861030+0000","last_became_active":"2023-10-09T06:09:45.129813+0000","last_became_peered":"2023-10-09T06:09:45.129813+0000","last_unstale":"2023-10-09T06:10:06.861030+0000","last_undegraded":"2023-10-09T06:10:06.861030+0000","last_fullsized":"2023-10-09T06:10:06.861030+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":80,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.875422+0000","last_change":"2023-10-09T06:08:47.115790+0000","last_active":"2023-10-09T06:10:07.875422+0000","last_peered":"2023-10-09T06:10:07.875422+0000","last_clean":"2023-10-09T06:10:07.875422+0000","last_became_active":"2023-10-09T06:08:47.115423+0000","last_became_peered":"2023-10-09T06:08:47.115423+0000","last_unstale":"2023-10-09T06:10:07.875422+0000","last_undegraded":"2023-10-09T06:10:07.875422+0000","last_fullsized":"2023-10-09T06:10:07.875422+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:10.939752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":344,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.865585+0000","last_change":"2023-10-09T06:07:47.252116+0000","last_active":"2023-10-09T06:10:06.865585+0000","last_peered":"2023-10-09T06:10:06.865585+0000","last_clean":"2023-10-09T06:10:06.865585+0000","last_became_active":"2023-10-09T06:07:47.251804+0000","last_became_peered":"2023-10-09T06:07:47.251804+0000","last_unstale":"2023-10-09T06:10:06.865585+0000","last_undegraded":"2023-10-09T06:10:06.865585+0000","last_fullsized":"2023-10-09T06:10:06.865585+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:02.563612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":38,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859396+0000","last_change":"2023-10-09T06:09:53.163027+0000","last_active":"2023-10-09T06:10:06.859396+0000","last_peered":"2023-10-09T06:10:06.859396+0000","last_clean":"2023-10-09T06:10:06.859396+0000","last_became_active":"2023-10-09T06:09:53.162617+0000","last_became_peered":"2023-10-09T06:09:53.162617+0000","last_unstale":"2023-10-09T06:10:06.859396+0000","last_undegraded":"2023-10-09T06:10:06.859396+0000","last_fullsized":"2023-10-09T06:10:06.859396+0000","mapping_epoch":632,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":435,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861025+0000","last_change":"2023-10-09T06:09:45.403512+0000","last_active":"2023-10-09T06:10:06.861025+0000","last_peered":"2023-10-09T06:10:06.861025+0000","last_clean":"2023-10-09T06:10:06.861025+0000","last_became_active":"2023-10-09T06:09:45.403092+0000","last_became_peered":"2023-10-09T06:09:45.403092+0000","last_unstale":"2023-10-09T06:10:06.861025+0000","last_undegraded":"2023-10-09T06:10:06.861025+0000","last_fullsized":"2023-10-09T06:10:06.861025+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":46,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859893+0000","last_change":"2023-10-09T06:09:53.163877+0000","last_active":"2023-10-09T06:10:06.859893+0000","last_peered":"2023-10-09T06:10:06.859893+0000","last_clean":"2023-10-09T06:10:06.859893+0000","last_became_active":"2023-10-09T06:09:53.163479+0000","last_became_peered":"2023-10-09T06:09:53.163479+0000","last_unstale":"2023-10-09T06:10:06.859893+0000","last_undegraded":"2023-10-09T06:10:06.859893+0000","last_fullsized":"2023-10-09T06:10:06.859893+0000","mapping_epoch":632,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":277,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861047+0000","last_change":"2023-10-09T06:09:45.403838+0000","last_active":"2023-10-09T06:10:06.861047+0000","last_peered":"2023-10-09T06:10:06.861047+0000","last_clean":"2023-10-09T06:10:06.861047+0000","last_became_active":"2023-10-09T06:09:45.403249+0000","last_became_peered":"2023-10-09T06:09:45.403249+0000","last_unstale":"2023-10-09T06:10:06.861047+0000","last_undegraded":"2023-10-09T06:10:06.861047+0000","last_fullsized":"2023-10-09T06:10:06.861047+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":185,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029507+0000","last_change":"2023-10-09T06:09:36.997962+0000","last_active":"2023-10-09T06:10:07.029507+0000","last_peered":"2023-10-09T06:10:07.029507+0000","last_clean":"2023-10-09T06:10:07.029507+0000","last_became_active":"2023-10-09T06:09:36.997617+0000","last_became_peered":"2023-10-09T06:09:36.997617+0000","last_unstale":"2023-10-09T06:10:07.029507+0000","last_undegraded":"2023-10-09T06:10:07.029507+0000","last_fullsized":"2023-10-09T06:10:07.029507+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:33.665893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":435,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029305+0000","last_change":"2023-10-09T06:09:37.000176+0000","last_active":"2023-10-09T06:10:07.029305+0000","last_peered":"2023-10-09T06:10:07.029305+0000","last_clean":"2023-10-09T06:10:07.029305+0000","last_became_active":"2023-10-09T06:09:36.999854+0000","last_became_peered":"2023-10-09T06:09:36.999854+0000","last_unstale":"2023-10-09T06:10:07.029305+0000","last_undegraded":"2023-10-09T06:10:07.029305+0000","last_fullsized":"2023-10-09T06:10:07.029305+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:24.485275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":92,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.865597+0000","last_change":"2023-10-09T06:08:53.670793+0000","last_active":"2023-10-09T06:10:06.865597+0000","last_peered":"2023-10-09T06:10:06.865597+0000","last_clean":"2023-10-09T06:10:06.865597+0000","last_became_active":"2023-10-09T06:08:53.662715+0000","last_became_peered":"2023-10-09T06:08:53.662715+0000","last_unstale":"2023-10-09T06:10:06.865597+0000","last_undegraded":"2023-10-09T06:10:06.865597+0000","last_fullsized":"2023-10-09T06:10:06.865597+0000","mapping_epoch":609,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:42.974962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":632,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.860834+0000","last_change":"2023-10-09T06:09:45.132231+0000","last_active":"2023-10-09T06:10:06.860834+0000","last_peered":"2023-10-09T06:10:06.860834+0000","last_clean":"2023-10-09T06:10:06.860834+0000","last_became_active":"2023-10-09T06:09:45.131062+0000","last_became_peered":"2023-10-09T06:09:45.131062+0000","last_unstale":"2023-10-09T06:10:06.860834+0000","last_undegraded":"2023-10-09T06:10:06.860834+0000","last_fullsized":"2023-10-09T06:10:06.860834+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":71,"num_bytes_recovered":23464,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":52606,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11988,"num_read_kb":26421,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":189,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460847021,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515488,"kb_used_data":10648,"kb_used_omap":14,"kb_used_meta":1504817,"kb_avail":92229664,"statfs":{"total":95995035648,"available":94443175936,"internally_reserved":0,"allocated":10903552,"data_stored":9686606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":1540933466},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":2.8730000000000002,"15min":2.8650000000000002},"min":{"1min":0.39400000000000002,"5min":0.26300000000000001,"15min":0.23200000000000001},"max":{"1min":0.76000000000000001,"5min":167.06200000000001,"15min":176.89500000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":2.7400000000000002,"15min":2.9430000000000001},"min":{"1min":0.42999999999999999,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":0.83199999999999996,"5min":167.077,"15min":176.947},"last":0.32800000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":3.4060000000000001,"15min":4.2750000000000004},"min":{"1min":0.39300000000000002,"5min":0.28100000000000003,"15min":0.252},"max":{"1min":0.72199999999999998,"5min":169.98400000000001,"15min":176.90600000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":3.855,"15min":4.3869999999999996},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.25900000000000001},"max":{"1min":0.71399999999999997,"5min":172.821,"15min":176.86799999999999},"last":0.499}]},{"osd":2,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":5.8250000000000002,"15min":5.593},"min":{"1min":0.41399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.81599999999999995,"5min":184.27500000000001,"15min":302.01799999999997},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":5.5259999999999998,"15min":5.3019999999999996},"min":{"1min":0.42199999999999999,"5min":0.35099999999999998,"15min":0.30299999999999999},"max":{"1min":0.84299999999999997,"5min":184.315,"15min":304.988},"last":0.54400000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":3.415,"15min":3.2090000000000001},"min":{"1min":0.45200000000000001,"5min":0.35099999999999998,"15min":0.30499999999999999},"max":{"1min":0.80500000000000005,"5min":167.64699999999999,"15min":176.958},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":3.0859999999999999,"15min":3.1360000000000001},"min":{"1min":0.437,"5min":0.34499999999999997,"15min":0.29699999999999999},"max":{"1min":0.79700000000000004,"5min":169.52199999999999,"15min":176.91800000000001},"last":0.53600000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.74299999999999999,"15min":1.492},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.252},"max":{"1min":0.68400000000000005,"5min":5.6340000000000003,"15min":178.995},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.73799999999999999,"15min":1.5920000000000001},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.246},"max":{"1min":0.70799999999999996,"5min":4.9349999999999996,"15min":178.899},"last":0.48899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.67200000000000004,"15min":1.9259999999999999},"min":{"1min":0.372,"5min":0.35199999999999998,"15min":0.29399999999999998},"max":{"1min":0.74399999999999999,"5min":3.1299999999999999,"15min":176.82400000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.70499999999999996,"15min":1.887},"min":{"1min":0.39700000000000002,"5min":0.33200000000000002,"15min":0.28999999999999998},"max":{"1min":0.73899999999999999,"5min":4.1210000000000004,"15min":176.79499999999999},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.78800000000000003,"15min":1.0189999999999999},"min":{"1min":0.34300000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.81599999999999995,"5min":13.193,"15min":13.193},"last":0.75},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.86799999999999999,"15min":0.76600000000000001},"min":{"1min":0.36799999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.749,"5min":13.247,"15min":13.247},"last":0.56899999999999995}]}]},{"osd":6,"up_from":537,"seq":2306397438028,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":382228,"kb_used_data":10636,"kb_used_omap":13,"kb_used_meta":371570,"kb_avail":93362924,"statfs":{"total":95995035648,"available":95603634176,"internally_reserved":0,"allocated":10891264,"data_stored":9686214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13650,"internal_metadata":380488366},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":2.1890000000000001,"15min":4.2869999999999999},"min":{"1min":0.30099999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.745,"5min":126.914,"15min":126.914},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.60999999999999999,"5min":2.4289999999999998,"15min":4.6020000000000003},"min":{"1min":0.45700000000000002,"5min":0.33900000000000002,"15min":0.32000000000000001},"max":{"1min":0.82499999999999996,"5min":126.925,"15min":126.925},"last":0.56999999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":2.4289999999999998,"15min":8.8710000000000004},"min":{"1min":0.29399999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.70199999999999996,"5min":130.28100000000001,"15min":130.28100000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":2.431,"15min":8.8130000000000006},"min":{"1min":0.371,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.75,"5min":135.06100000000001,"15min":135.06100000000001},"last":0.60499999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":2.786,"15min":2.4750000000000001},"min":{"1min":0.33300000000000002,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":0.73499999999999999,"5min":133.917,"15min":133.917},"last":0.63},{"interface":"front","average":{"1min":0.63,"5min":2.8730000000000002,"15min":2.8029999999999999},"min":{"1min":0.42599999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.91600000000000004,"5min":133.92699999999999,"15min":133.92699999999999},"last":0.64200000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":2.4369999999999998,"15min":4.8639999999999999},"min":{"1min":0.34100000000000003,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.76700000000000002,"5min":129.375,"15min":129.375},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":2.3849999999999998,"15min":4.8390000000000004},"min":{"1min":0.45700000000000002,"5min":0.308,"15min":0.308},"max":{"1min":0.75,"5min":129.38900000000001,"15min":129.38900000000001},"last":0.45700000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.74399999999999999,"15min":0.99399999999999999},"min":{"1min":0.36399999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.76100000000000001,"5min":11.103999999999999,"15min":11.103999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57899999999999996,"15min":0.94399999999999995},"min":{"1min":0.39200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.80800000000000005,"5min":4.1669999999999998,"15min":8.1229999999999993},"last":0.46899999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.73599999999999999,"15min":0.85299999999999998},"min":{"1min":0.437,"5min":0.314,"15min":0.313},"max":{"1min":0.79700000000000004,"5min":11.218,"15min":11.218},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.624,"5min":0.68700000000000006,"15min":0.83499999999999996},"min":{"1min":0.46700000000000003,"5min":0.315,"15min":0.315},"max":{"1min":0.89800000000000002,"5min":6.5389999999999997,"15min":6.5389999999999997},"last":0.54300000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.70099999999999996,"15min":4.609},"min":{"1min":0.40999999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.96999999999999997,"5min":3.4369999999999998,"15min":124.27200000000001},"last":0.432},{"interface":"front","average":{"1min":0.625,"5min":0.86299999999999999,"15min":4.4409999999999998},"min":{"1min":0.45200000000000001,"5min":0.31,"15min":0.308},"max":{"1min":0.83899999999999997,"5min":11.339,"15min":121.136},"last":0.49299999999999999}]}]},{"osd":1,"up_from":93,"seq":399431959013,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1037972,"kb_used_data":7244,"kb_used_omap":7,"kb_used_meta":1030712,"kb_avail":92707180,"statfs":{"total":95995035648,"available":94932152320,"internally_reserved":0,"allocated":7417856,"data_stored":5042705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":1055449614},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":1.1970000000000001,"15min":1.292},"min":{"1min":0.27400000000000002,"5min":0.23699999999999999,"15min":0.21199999999999999},"max":{"1min":0.67700000000000005,"5min":17.465,"15min":27.045999999999999},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.39900000000000002,"5min":1.468,"15min":1.4410000000000001},"min":{"1min":0.31,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.60999999999999999,"5min":15.682,"15min":27.344000000000001},"last":0.33700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:10:07 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":3.718,"15min":2.3490000000000002},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.20899999999999999},"max":{"1min":0.56699999999999995,"5min":186.84200000000001,"15min":186.84200000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":3.6890000000000001,"15min":2.4009999999999998},"min":{"1min":0.32300000000000001,"5min":0.26200000000000001,"15min":0.23699999999999999},"max":{"1min":0.72599999999999998,"5min":186.75800000000001,"15min":186.75800000000001},"last":0.36699999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":3.4790000000000001,"15min":2.2349999999999999},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.23799999999999999},"max":{"1min":0.625,"5min":177.70599999999999,"15min":177.70599999999999},"last":0.313},{"interface":"front","average":{"1min":0.46899999999999997,"5min":3.4630000000000001,"15min":2.2120000000000002},"min":{"1min":0.375,"5min":0.29399999999999998,"15min":0.24099999999999999},"max":{"1min":0.63900000000000001,"5min":177.82400000000001,"15min":177.82400000000001},"last":0.41099999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":1.548,"15min":1.706},"min":{"1min":0.42099999999999999,"5min":0.29699999999999999,"15min":0.25800000000000001},"max":{"1min":0.98599999999999999,"5min":10.907999999999999,"15min":86.483999999999995},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.95299999999999996,"15min":1.407},"min":{"1min":0.41399999999999998,"5min":0.33900000000000002,"15min":0.32000000000000001},"max":{"1min":0.96499999999999997,"5min":6.6580000000000004,"15min":86.441999999999993},"last":0.52900000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":1.0149999999999999,"15min":1.244},"min":{"1min":0.46899999999999997,"5min":0.314,"15min":0.313},"max":{"1min":0.89100000000000001,"5min":8.6769999999999996,"15min":14.019},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.66900000000000004,"5min":2.5310000000000001,"15min":1.887},"min":{"1min":0.38800000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.94899999999999995,"5min":99.769999999999996,"15min":99.769999999999996},"last":0.56899999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":1.5189999999999999,"15min":3.1480000000000001},"min":{"1min":0.54900000000000004,"5min":0.372,"15min":0.372},"max":{"1min":0.85599999999999998,"5min":13.205,"15min":13.205},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.72499999999999998,"5min":1.097,"15min":2.0329999999999999},"min":{"1min":0.502,"5min":0.375,"15min":0.375},"max":{"1min":0.97199999999999998,"5min":7.3259999999999996,"15min":13.685},"last":0.60999999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":1.2150000000000001,"15min":1.8620000000000001},"min":{"1min":0.49199999999999999,"5min":0.379,"15min":0.31},"max":{"1min":1.0089999999999999,"5min":6.7190000000000003,"15min":89.787000000000006},"last":0.628},{"interface":"front","average":{"1min":0.70299999999999996,"5min":1.238,"15min":1.966},"min":{"1min":0.52800000000000002,"5min":0.443,"15min":0.35199999999999998},"max":{"1min":0.89700000000000002,"5min":7.3310000000000004,"15min":89.869},"last":0.58299999999999996}]}]},{"osd":0,"up_from":88,"seq":377957122538,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1001752,"kb_used_data":11984,"kb_used_omap":8,"kb_used_meta":989751,"kb_avail":92743400,"statfs":{"total":95995035648,"available":94969241600,"internally_reserved":0,"allocated":12271616,"data_stored":9833357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":1013505800},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":1.5600000000000001,"15min":3.4129999999999998},"min":{"1min":0.249,"5min":0.19400000000000001,"15min":0.191},"max":{"1min":0.47799999999999998,"5min":15.27,"15min":163.572},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.39300000000000002,"5min":1.8029999999999999,"15min":3.5779999999999998},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.19700000000000001},"max":{"1min":0.46100000000000002,"5min":16.222000000000001,"15min":162.48699999999999},"last":0.35799999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.75,"15min":1.5840000000000001},"min":{"1min":0.25,"5min":0.215,"15min":0.20599999999999999},"max":{"1min":0.99399999999999999,"5min":27.338000000000001,"15min":95.512},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.40000000000000002,"5min":2.024,"15min":1.661},"min":{"1min":0.26200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.51700000000000002,"5min":31.065999999999999,"15min":95.462999999999994},"last":0.371}]},{"osd":3,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":1.607,"15min":1.3340000000000001},"min":{"1min":0.28599999999999998,"5min":0.221,"15min":0.21299999999999999},"max":{"1min":0.53600000000000003,"5min":12.304,"15min":12.304},"last":0.441},{"interface":"front","average":{"1min":0.44900000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"min":{"1min":0.35099999999999998,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":1.0169999999999999,"5min":8.5090000000000003,"15min":13.632},"last":0.42899999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.2949999999999999,"15min":1.0920000000000001},"min":{"1min":0.42299999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.85899999999999999,"5min":12.965999999999999,"15min":13.592000000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.498,"15min":1.1990000000000001},"min":{"1min":0.45500000000000002,"5min":0.32200000000000001,"15min":0.29399999999999998},"max":{"1min":0.93400000000000005,"5min":13.775,"15min":13.775},"last":0.57299999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.6930000000000001,"15min":2.3730000000000002},"min":{"1min":0.41199999999999998,"5min":0.318,"15min":0.27900000000000003},"max":{"1min":0.84899999999999998,"5min":13.746,"15min":210.77799999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.6399999999999999,"15min":2.448},"min":{"1min":0.36599999999999999,"5min":0.34499999999999997,"15min":0.30599999999999999},"max":{"1min":1.2430000000000001,"5min":11.52,"15min":209.91},"last":0.48799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:09:43 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.4259999999999999,"15min":1.788},"min":{"1min":0.44700000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.90600000000000003,"5min":12.929,"15min":12.929},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.629,"5min":1.7050000000000001,"15min":1.4950000000000001},"min":{"1min":0.42299999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.89100000000000001,"5min":13.608000000000001,"15min":13.608000000000001},"last":0.499}]},{"osd":7,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":1.321,"15min":3.1499999999999999},"min":{"1min":0.442,"5min":0.373,"15min":0.32400000000000001},"max":{"1min":0.91200000000000003,"5min":12.987,"15min":168.98599999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.607,"15min":3.1930000000000001},"min":{"1min":0.52000000000000002,"5min":0.39100000000000001,"15min":0.32800000000000001},"max":{"1min":0.86399999999999999,"5min":13.654999999999999,"15min":169.03800000000001},"last":0.52600000000000002}]}]},{"osd":2,"up_from":98,"seq":420906795486,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1073300,"kb_used_data":15884,"kb_used_omap":11,"kb_used_meta":1057396,"kb_avail":92671852,"statfs":{"total":95995035648,"available":94895976448,"internally_reserved":0,"allocated":16265216,"data_stored":13476950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":1082774101},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":1.611,"15min":1.405},"min":{"1min":0.216,"5min":0.20499999999999999,"15min":0.19600000000000001},"max":{"1min":0.67000000000000004,"5min":11.928000000000001,"15min":27.292000000000002},"last":0.432},{"interface":"front","average":{"1min":0.39400000000000002,"5min":1.3939999999999999,"15min":1.4330000000000001},"min":{"1min":0.22600000000000001,"5min":0.19700000000000001,"15min":0.19600000000000001},"max":{"1min":0.64700000000000002,"5min":10.958,"15min":17.109999999999999},"last":0.309}]},{"osd":1,"last update":"Mon Oct 9 06:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.7749999999999999,"15min":2.387},"min":{"1min":0.253,"5min":0.22700000000000001,"15min":0.19},"max":{"1min":0.71899999999999997,"5min":26.869,"15min":133.42500000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.39000000000000001,"5min":2.1000000000000001,"15min":2.6269999999999998},"min":{"1min":0.23499999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.64500000000000002,"5min":13.238,"15min":133.40799999999999},"last":0.41299999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":2.1850000000000001,"15min":2.5379999999999998},"min":{"1min":0.28799999999999998,"5min":0.252,"15min":0.23300000000000001},"max":{"1min":0.81799999999999995,"5min":13.285,"15min":142.26300000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":1.6279999999999999,"15min":2.1030000000000002},"min":{"1min":0.29799999999999999,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":0.78700000000000003,"5min":15.108000000000001,"15min":141.477},"last":0.39200000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.7649999999999999,"15min":1.3939999999999999},"min":{"1min":0.47399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.76100000000000001,"5min":11.132999999999999,"15min":11.132999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.65100000000000002,"5min":1.319,"15min":1.0800000000000001},"min":{"1min":0.496,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.86499999999999999,"5min":8.3100000000000005,"15min":16.457000000000001},"last":0.58099999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.496,"15min":1.304},"min":{"1min":0.40899999999999997,"5min":0.35499999999999998,"15min":0.33000000000000002},"max":{"1min":0.82799999999999996,"5min":10.852,"15min":18.350000000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.8009999999999999,"15min":1.6559999999999999},"min":{"1min":0.44400000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.874,"5min":11.169,"15min":15.781000000000001},"last":0.65800000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:09:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":1.619,"15min":1.853},"min":{"1min":0.434,"5min":0.375,"15min":0.375},"max":{"1min":1.1619999999999999,"5min":16.422000000000001,"15min":16.422000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.6080000000000001,"15min":1.752},"min":{"1min":0.52800000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.93799999999999994,"5min":10.801,"15min":10.801},"last":0.72899999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":1.6279999999999999,"15min":1.883},"min":{"1min":0.50800000000000001,"5min":0.39900000000000002,"15min":0.34399999999999997},"max":{"1min":1.111,"5min":8.3640000000000008,"15min":122.827},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.71799999999999997,"5min":1.7829999999999999,"15min":2.0430000000000001},"min":{"1min":0.55300000000000005,"5min":0.39600000000000002,"15min":0.36099999999999999},"max":{"1min":0.91800000000000004,"5min":10.890000000000001,"15min":123.56399999999999},"last":0.75900000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631961,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1215848,"kb_used_data":11424,"kb_used_omap":4,"kb_used_meta":1204411,"kb_avail":92529304,"statfs":{"total":95995035648,"available":94750007296,"internally_reserved":0,"allocated":11698176,"data_stored":9245418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1233317470},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":1.8879999999999999,"15min":2.194},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20399999999999999},"max":{"1min":0.49199999999999999,"5min":50.546999999999997,"15min":135.501},"last":0.433},{"interface":"front","average":{"1min":0.34899999999999998,"5min":1.8520000000000001,"15min":2.1360000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.20899999999999999},"max":{"1min":0.50600000000000001,"5min":55.683,"15min":135.99199999999999},"last":0.28399999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":3.6150000000000002,"15min":3.71},"min":{"1min":0.245,"5min":0.24299999999999999,"15min":0.20100000000000001},"max":{"1min":0.65400000000000003,"5min":137.679,"15min":137.679},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.373,"5min":3.5859999999999999,"15min":3.504},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":0.53600000000000003,"5min":137.46799999999999,"15min":137.46799999999999},"last":0.36799999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":3.5419999999999998,"15min":3.2749999999999999},"min":{"1min":0.27500000000000002,"5min":0.23999999999999999,"15min":0.22700000000000001},"max":{"1min":0.66000000000000003,"5min":71.018000000000001,"15min":89.024000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":3.2690000000000001,"15min":2.8029999999999999},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.607,"5min":69.918999999999997,"15min":90.850999999999999},"last":0.40400000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.135,"15min":1.4470000000000001},"min":{"1min":0.41499999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.82699999999999996,"5min":8.9930000000000003,"15min":74.957999999999998},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.1659999999999999,"15min":1.484},"min":{"1min":0.45000000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.071,"5min":9.7309999999999999,"15min":75.022999999999996},"last":0.52100000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.0960000000000001,"15min":0.95999999999999996},"min":{"1min":0.51100000000000001,"5min":0.38,"15min":0.32700000000000001},"max":{"1min":1,"5min":6.8449999999999998,"15min":8.8450000000000006},"last":0.503},{"interface":"front","average":{"1min":0.74399999999999999,"5min":1.1419999999999999,"15min":1.1479999999999999},"min":{"1min":0.45700000000000002,"5min":0.34999999999999998,"15min":0.33400000000000002},"max":{"1min":2.6200000000000001,"5min":9.8109999999999999,"15min":15.932},"last":0.58599999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.2230000000000001,"15min":1.413},"min":{"1min":0.47299999999999998,"5min":0.40699999999999997,"15min":0.34300000000000003},"max":{"1min":0.89100000000000001,"5min":6.7939999999999996,"15min":8.9149999999999991},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.3360000000000001,"15min":2.3919999999999999},"min":{"1min":0.41699999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.94199999999999995,"5min":9.7780000000000005,"15min":15.954000000000001},"last":0.61099999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":2.6680000000000001,"15min":1.8100000000000001},"min":{"1min":0.44,"5min":0.41599999999999998,"15min":0.375},"max":{"1min":0.92400000000000004,"5min":79.403000000000006,"15min":79.403000000000006},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.66800000000000004,"5min":2.851,"15min":1.8779999999999999},"min":{"1min":0.505,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.022,"5min":79.203000000000003,"15min":79.203000000000003},"last":0.55900000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019506,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195800,"kb_used_data":5968,"kb_used_omap":0,"kb_used_meta":1189824,"kb_avail":92549352,"statfs":{"total":95995035648,"available":94770536448,"internally_reserved":0,"allocated":6111232,"data_stored":4901864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1218379776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.073,"15min":1.9099999999999999},"min":{"1min":0.315,"5min":0.23799999999999999,"15min":0.21299999999999999},"max":{"1min":1.002,"5min":17.248999999999999,"15min":86.509},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.619,"15min":2.2799999999999998},"min":{"1min":0.35999999999999999,"5min":0.27600000000000002,"15min":0.24299999999999999},"max":{"1min":3.661,"5min":18.664000000000001,"15min":86.233999999999995},"last":0.435}]},{"osd":1,"last update":"Mon Oct 9 06:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.821,"15min":2.331},"min":{"1min":0.435,"5min":0.253,"15min":0.253},"max":{"1min":0.749,"5min":18.951000000000001,"15min":96.033000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":2.9750000000000001,"15min":2.8130000000000002},"min":{"1min":0.35899999999999999,"5min":0.26700000000000002,"15min":0.255},"max":{"1min":0.77200000000000002,"5min":108.45699999999999,"15min":108.45699999999999},"last":0.51600000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:10:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":4.3959999999999999,"15min":2.8460000000000001},"min":{"1min":0.38800000000000001,"5min":0.33900000000000002,"15min":0.27200000000000002},"max":{"1min":0.79600000000000004,"5min":105.157,"15min":143.71000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":3.6930000000000001,"15min":3.137},"min":{"1min":0.309,"5min":0.309,"15min":0.27700000000000002},"max":{"1min":0.95799999999999996,"5min":107.851,"15min":143.768},"last":0.505}]},{"osd":3,"last update":"Mon Oct 9 06:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":3.0089999999999999,"15min":2.5449999999999999},"min":{"1min":0.38200000000000001,"5min":0.30199999999999999,"15min":0.254},"max":{"1min":0.65300000000000002,"5min":107.352,"15min":107.352},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":2.7229999999999999,"15min":2.3769999999999998},"min":{"1min":0.41299999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68400000000000005,"5min":105.336,"15min":105.336},"last":0.54600000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.0429999999999999,"15min":1.0369999999999999},"min":{"1min":0.317,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.70199999999999996,"5min":7.1070000000000002,"15min":15.597},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.90300000000000002,"15min":0.96699999999999997},"min":{"1min":0.42899999999999999,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":0.83899999999999997,"5min":7.069,"15min":17.882000000000001},"last":0.66100000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.81399999999999995,"15min":0.83599999999999997},"min":{"1min":0.34999999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.79100000000000004,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":1.1220000000000001,"15min":1},"min":{"1min":0.377,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.78000000000000003,"5min":8.9909999999999997,"15min":8.9909999999999997},"last":0.68000000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.83899999999999997,"15min":0.98599999999999999},"min":{"1min":0.39500000000000002,"5min":0.33600000000000002,"15min":0.29799999999999999},"max":{"1min":0.75600000000000001,"5min":6.6040000000000001,"15min":17.832000000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.145,"15min":1.038},"min":{"1min":0.38400000000000001,"5min":0.35299999999999998,"15min":0.32900000000000001},"max":{"1min":0.71299999999999997,"5min":8.9580000000000002,"15min":17.742999999999999},"last":0.63}]}]},{"osd":5,"up_from":36,"seq":154618823281,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908744,"kb_used_data":5952,"kb_used_omap":0,"kb_used_meta":902784,"kb_avail":92836408,"statfs":{"total":95995035648,"available":95064481792,"internally_reserved":0,"allocated":6094848,"data_stored":4897312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924450816},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.94799999999999995,"15min":1.71},"min":{"1min":0.309,"5min":0.25900000000000001,"15min":0.223},"max":{"1min":1.081,"5min":10.618,"15min":104.842},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.87,"15min":1.837},"min":{"1min":0.37,"5min":0.29199999999999998,"15min":0.254},"max":{"1min":1.1040000000000001,"5min":7.5460000000000003,"15min":104.813},"last":0.64500000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.3759999999999999,"15min":2.0339999999999998},"min":{"1min":0.42099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.048,"5min":26.544,"15min":80.003},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.3740000000000001,"15min":1.9850000000000001},"min":{"1min":0.34000000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.1930000000000001,"5min":25.946999999999999,"15min":79.930000000000007},"last":0.67400000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.9510000000000001,"15min":2.613},"min":{"1min":0.36899999999999999,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":1.0309999999999999,"5min":40.969999999999999,"15min":151.06100000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.62,"5min":1.6359999999999999,"15min":2.5790000000000002},"min":{"1min":0.35899999999999999,"5min":0.34200000000000003,"15min":0.30299999999999999},"max":{"1min":1.1619999999999999,"5min":40.816000000000003,"15min":151.46299999999999},"last":0.621}]},{"osd":3,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":1.339,"15min":1.659},"min":{"1min":0.39500000000000002,"5min":0.33000000000000002,"15min":0.29899999999999999},"max":{"1min":1.2230000000000001,"5min":21.91,"15min":102.768},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.6000000000000001,"15min":1.637},"min":{"1min":0.51000000000000001,"5min":0.36099999999999999,"15min":0.33500000000000002},"max":{"1min":1.1220000000000001,"5min":21.608000000000001,"15min":102.806},"last":0.69399999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.78800000000000003,"15min":0.79400000000000004},"min":{"1min":0.375,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":1.3089999999999999,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61099999999999999,"15min":0.754},"min":{"1min":0.40300000000000002,"5min":0.311,"15min":0.27000000000000002},"max":{"1min":1.2110000000000001,"5min":4.8650000000000002,"15min":10.154},"last":0.42999999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.72799999999999998,"15min":0.57699999999999996},"min":{"1min":0.39800000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.258,"5min":13.098000000000001,"15min":13.098000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.61399999999999999,"15min":0.52200000000000002},"min":{"1min":0.373,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.274,"5min":3.3420000000000001,"15min":3.3420000000000001},"last":0.47899999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.75900000000000001,"15min":0.91700000000000004},"min":{"1min":0.41799999999999998,"5min":0.35599999999999998,"15min":0.32000000000000001},"max":{"1min":0.92500000000000004,"5min":7.8470000000000004,"15min":21.068000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.751,"15min":0.94799999999999995},"min":{"1min":0.44400000000000001,"5min":0.35699999999999998,"15min":0.34799999999999998},"max":{"1min":0.93300000000000005,"5min":5.2409999999999997,"15min":21.106000000000002},"last":0.66000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13650,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:10:47.110 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:10:47.477 INFO:teuthology.orchestra.run.smithi008.stderr:Inferring config /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:47.964 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:47 smithi008 ceph-mon[184418]: from='client.48389 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:10:47.965 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:47 smithi008 ceph-mon[191227]: from='client.48389 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:10:48.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:47 smithi191 ceph-mon[161669]: from='client.48389 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:10:48.838 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:48 smithi008 ceph-mon[184418]: pgmap v1909: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:48.838 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:48 smithi008 ceph-mon[191227]: pgmap v1909: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:49.049 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:48 smithi191 ceph-mon[161669]: pgmap v1909: 73 pgs: 73 active+clean; 8.6 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:10:49.206 INFO:teuthology.orchestra.run.smithi008.stdout: 2023-10-09T06:10:49.207 INFO:teuthology.orchestra.run.smithi008.stderr:dumped all 2023-10-09T06:10:50.012 INFO:teuthology.orchestra.run.smithi008.stdout:{"pg_ready":true,"pg_map":{"version":1910,"stamp":"2023-10-09T06:10:49.016399+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12282,"num_read_kb":26921,"num_write":1950,"num_write_kb":27835,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":268,"num_bytes_recovered":10773582,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8331132,"kb_used_data":79740,"kb_used_omap":59,"kb_used_meta":8251268,"kb_avail":741630084,"statfs":{"total":767960285184,"available":759429206016,"internally_reserved":0,"allocated":81653760,"data_stored":66770426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":60463,"internal_metadata":8449299409},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002886"},"pg_stats":[{"pgid":"3.1a","version":"59'1","reported_seq":492,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861493+0000","last_change":"2023-10-09T06:09:45.128308+0000","last_active":"2023-10-09T06:10:06.861493+0000","last_peered":"2023-10-09T06:10:06.861493+0000","last_clean":"2023-10-09T06:10:06.861493+0000","last_became_active":"2023-10-09T06:09:45.128032+0000","last_became_peered":"2023-10-09T06:09:45.128032+0000","last_unstale":"2023-10-09T06:10:06.861493+0000","last_undegraded":"2023-10-09T06:10:06.861493+0000","last_fullsized":"2023-10-09T06:10:06.861493+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.869740+0000","last_change":"2023-10-09T06:07:47.056618+0000","last_active":"2023-10-09T06:10:07.869740+0000","last_peered":"2023-10-09T06:10:07.869740+0000","last_clean":"2023-10-09T06:10:07.869740+0000","last_became_active":"2023-10-09T06:07:47.056311+0000","last_became_peered":"2023-10-09T06:07:47.056311+0000","last_unstale":"2023-10-09T06:10:07.869740+0000","last_undegraded":"2023-10-09T06:10:07.869740+0000","last_fullsized":"2023-10-09T06:10:07.869740+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:28.019550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":379,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864188+0000","last_change":"2023-10-09T06:07:24.553497+0000","last_active":"2023-10-09T06:10:06.864188+0000","last_peered":"2023-10-09T06:10:06.864188+0000","last_clean":"2023-10-09T06:10:06.864188+0000","last_became_active":"2023-10-09T06:07:24.553189+0000","last_became_peered":"2023-10-09T06:07:24.553189+0000","last_unstale":"2023-10-09T06:10:06.864188+0000","last_undegraded":"2023-10-09T06:10:06.864188+0000","last_fullsized":"2023-10-09T06:10:06.864188+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:53.168858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":637,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.875402+0000","last_change":"2023-10-09T06:08:34.675487+0000","last_active":"2023-10-09T06:10:07.875402+0000","last_peered":"2023-10-09T06:10:07.875402+0000","last_clean":"2023-10-09T06:10:07.875402+0000","last_became_active":"2023-10-09T06:08:34.674645+0000","last_became_peered":"2023-10-09T06:08:34.674645+0000","last_unstale":"2023-10-09T06:10:07.875402+0000","last_undegraded":"2023-10-09T06:10:07.875402+0000","last_fullsized":"2023-10-09T06:10:07.875402+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:50.956195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":486,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029129+0000","last_change":"2023-10-09T06:09:37.000349+0000","last_active":"2023-10-09T06:10:07.029129+0000","last_peered":"2023-10-09T06:10:07.029129+0000","last_clean":"2023-10-09T06:10:07.029129+0000","last_became_active":"2023-10-09T06:09:36.999813+0000","last_became_peered":"2023-10-09T06:09:36.999813+0000","last_unstale":"2023-10-09T06:10:07.029129+0000","last_undegraded":"2023-10-09T06:10:07.029129+0000","last_fullsized":"2023-10-09T06:10:07.029129+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:07.235570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":230,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.939303+0000","last_change":"2023-10-09T06:10:07.939303+0000","last_active":"2023-10-09T06:10:07.939303+0000","last_peered":"2023-10-09T06:10:07.939303+0000","last_clean":"2023-10-09T06:10:07.939303+0000","last_became_active":"2023-10-09T06:10:07.938952+0000","last_became_peered":"2023-10-09T06:10:07.938952+0000","last_unstale":"2023-10-09T06:10:07.939303+0000","last_undegraded":"2023-10-09T06:10:07.939303+0000","last_fullsized":"2023-10-09T06:10:07.939303+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":599,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.865748+0000","last_change":"2023-10-09T06:09:14.937602+0000","last_active":"2023-10-09T06:10:06.865748+0000","last_peered":"2023-10-09T06:10:06.865748+0000","last_clean":"2023-10-09T06:10:06.865748+0000","last_became_active":"2023-10-09T06:09:14.937271+0000","last_became_peered":"2023-10-09T06:09:14.937271+0000","last_unstale":"2023-10-09T06:10:06.865748+0000","last_undegraded":"2023-10-09T06:10:06.865748+0000","last_fullsized":"2023-10-09T06:10:06.865748+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:04.497640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":321,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029180+0000","last_change":"2023-10-09T06:09:37.000097+0000","last_active":"2023-10-09T06:10:07.029180+0000","last_peered":"2023-10-09T06:10:07.029180+0000","last_clean":"2023-10-09T06:10:07.029180+0000","last_became_active":"2023-10-09T06:09:36.999415+0000","last_became_peered":"2023-10-09T06:09:36.999415+0000","last_unstale":"2023-10-09T06:10:07.029180+0000","last_undegraded":"2023-10-09T06:10:07.029180+0000","last_fullsized":"2023-10-09T06:10:07.029180+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:59.199203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":39,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859478+0000","last_change":"2023-10-09T06:08:53.670871+0000","last_active":"2023-10-09T06:10:06.859478+0000","last_peered":"2023-10-09T06:10:06.859478+0000","last_clean":"2023-10-09T06:10:06.859478+0000","last_became_active":"2023-10-09T06:08:53.670472+0000","last_became_peered":"2023-10-09T06:08:53.670472+0000","last_unstale":"2023-10-09T06:10:06.859478+0000","last_undegraded":"2023-10-09T06:10:06.859478+0000","last_fullsized":"2023-10-09T06:10:06.859478+0000","mapping_epoch":609,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":245,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029225+0000","last_change":"2023-10-09T06:08:34.670870+0000","last_active":"2023-10-09T06:10:07.029225+0000","last_peered":"2023-10-09T06:10:07.029225+0000","last_clean":"2023-10-09T06:10:07.029225+0000","last_became_active":"2023-10-09T06:08:34.670528+0000","last_became_peered":"2023-10-09T06:08:34.670528+0000","last_unstale":"2023-10-09T06:10:07.029225+0000","last_undegraded":"2023-10-09T06:10:07.029225+0000","last_fullsized":"2023-10-09T06:10:07.029225+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:14:05.732701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"59'3","reported_seq":268,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868788+0000","last_change":"2023-10-09T06:10:00.686515+0000","last_active":"2023-10-09T06:10:06.868788+0000","last_peered":"2023-10-09T06:10:06.868788+0000","last_clean":"2023-10-09T06:10:06.868788+0000","last_became_active":"2023-10-09T06:10:00.686115+0000","last_became_peered":"2023-10-09T06:10:00.686115+0000","last_unstale":"2023-10-09T06:10:06.868788+0000","last_undegraded":"2023-10-09T06:10:06.868788+0000","last_fullsized":"2023-10-09T06:10:06.868788+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":450,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864721+0000","last_change":"2023-10-09T06:05:15.568750+0000","last_active":"2023-10-09T06:10:06.864721+0000","last_peered":"2023-10-09T06:10:06.864721+0000","last_clean":"2023-10-09T06:10:06.864721+0000","last_became_active":"2023-10-09T06:05:15.568039+0000","last_became_peered":"2023-10-09T06:05:15.568039+0000","last_unstale":"2023-10-09T06:10:06.864721+0000","last_undegraded":"2023-10-09T06:10:06.864721+0000","last_fullsized":"2023-10-09T06:10:06.864721+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:38.828502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":292,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861684+0000","last_change":"2023-10-09T06:08:34.656541+0000","last_active":"2023-10-09T06:10:06.861684+0000","last_peered":"2023-10-09T06:10:06.861684+0000","last_clean":"2023-10-09T06:10:06.861684+0000","last_became_active":"2023-10-09T06:08:34.656094+0000","last_became_peered":"2023-10-09T06:08:34.656094+0000","last_unstale":"2023-10-09T06:10:06.861684+0000","last_undegraded":"2023-10-09T06:10:06.861684+0000","last_fullsized":"2023-10-09T06:10:06.861684+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":583,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861215+0000","last_change":"2023-10-09T06:09:45.134387+0000","last_active":"2023-10-09T06:10:06.861215+0000","last_peered":"2023-10-09T06:10:06.861215+0000","last_clean":"2023-10-09T06:10:06.861215+0000","last_became_active":"2023-10-09T06:09:45.133037+0000","last_became_peered":"2023-10-09T06:09:45.133037+0000","last_unstale":"2023-10-09T06:10:06.861215+0000","last_undegraded":"2023-10-09T06:10:06.861215+0000","last_fullsized":"2023-10-09T06:10:06.861215+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"59'2","reported_seq":277,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.869304+0000","last_change":"2023-10-09T06:10:00.686444+0000","last_active":"2023-10-09T06:10:06.869304+0000","last_peered":"2023-10-09T06:10:06.869304+0000","last_clean":"2023-10-09T06:10:06.869304+0000","last_became_active":"2023-10-09T06:10:00.686078+0000","last_became_peered":"2023-10-09T06:10:00.686078+0000","last_unstale":"2023-10-09T06:10:06.869304+0000","last_undegraded":"2023-10-09T06:10:06.869304+0000","last_fullsized":"2023-10-09T06:10:06.869304+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":190,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029323+0000","last_change":"2023-10-09T06:09:37.001713+0000","last_active":"2023-10-09T06:10:07.029323+0000","last_peered":"2023-10-09T06:10:07.029323+0000","last_clean":"2023-10-09T06:10:07.029323+0000","last_became_active":"2023-10-09T06:09:37.001410+0000","last_became_peered":"2023-10-09T06:09:37.001410+0000","last_unstale":"2023-10-09T06:10:07.029323+0000","last_undegraded":"2023-10-09T06:10:07.029323+0000","last_fullsized":"2023-10-09T06:10:07.029323+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:49.657779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":671,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.865973+0000","last_change":"2023-10-09T06:08:34.676004+0000","last_active":"2023-10-09T06:10:06.865973+0000","last_peered":"2023-10-09T06:10:06.865973+0000","last_clean":"2023-10-09T06:10:06.865973+0000","last_became_active":"2023-10-09T06:08:34.674879+0000","last_became_peered":"2023-10-09T06:08:34.674879+0000","last_unstale":"2023-10-09T06:10:06.865973+0000","last_undegraded":"2023-10-09T06:10:06.865973+0000","last_fullsized":"2023-10-09T06:10:06.865973+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:27.794738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":18,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859703+0000","last_change":"2023-10-09T06:09:49.140421+0000","last_active":"2023-10-09T06:10:06.859703+0000","last_peered":"2023-10-09T06:10:06.859703+0000","last_clean":"2023-10-09T06:10:06.859703+0000","last_became_active":"2023-10-09T06:09:49.139538+0000","last_became_peered":"2023-10-09T06:09:49.139538+0000","last_unstale":"2023-10-09T06:10:06.859703+0000","last_undegraded":"2023-10-09T06:10:06.859703+0000","last_fullsized":"2023-10-09T06:10:06.859703+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":287,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868482+0000","last_change":"2023-10-09T06:10:00.690974+0000","last_active":"2023-10-09T06:10:06.868482+0000","last_peered":"2023-10-09T06:10:06.868482+0000","last_clean":"2023-10-09T06:10:06.868482+0000","last_became_active":"2023-10-09T06:10:00.690593+0000","last_became_peered":"2023-10-09T06:10:00.690593+0000","last_unstale":"2023-10-09T06:10:06.868482+0000","last_undegraded":"2023-10-09T06:10:06.868482+0000","last_fullsized":"2023-10-09T06:10:06.868482+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":641,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.866086+0000","last_change":"2023-10-09T06:08:34.674326+0000","last_active":"2023-10-09T06:10:06.866086+0000","last_peered":"2023-10-09T06:10:06.866086+0000","last_clean":"2023-10-09T06:10:06.866086+0000","last_became_active":"2023-10-09T06:08:34.673989+0000","last_became_peered":"2023-10-09T06:08:34.673989+0000","last_unstale":"2023-10-09T06:10:06.866086+0000","last_undegraded":"2023-10-09T06:10:06.866086+0000","last_fullsized":"2023-10-09T06:10:06.866086+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:12.836741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"59'8","reported_seq":290,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.876836+0000","last_change":"2023-10-09T06:10:07.876836+0000","last_active":"2023-10-09T06:10:07.876836+0000","last_peered":"2023-10-09T06:10:07.876836+0000","last_clean":"2023-10-09T06:10:07.876836+0000","last_became_active":"2023-10-09T06:10:07.876099+0000","last_became_peered":"2023-10-09T06:10:07.876099+0000","last_unstale":"2023-10-09T06:10:07.876836+0000","last_undegraded":"2023-10-09T06:10:07.876836+0000","last_fullsized":"2023-10-09T06:10:07.876836+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":330,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029479+0000","last_change":"2023-10-09T06:09:37.001585+0000","last_active":"2023-10-09T06:10:07.029479+0000","last_peered":"2023-10-09T06:10:07.029479+0000","last_clean":"2023-10-09T06:10:07.029479+0000","last_became_active":"2023-10-09T06:09:37.001273+0000","last_became_peered":"2023-10-09T06:09:37.001273+0000","last_unstale":"2023-10-09T06:10:07.029479+0000","last_undegraded":"2023-10-09T06:10:07.029479+0000","last_fullsized":"2023-10-09T06:10:07.029479+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:35.765491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"59'1","reported_seq":293,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.869778+0000","last_change":"2023-10-09T06:10:00.689010+0000","last_active":"2023-10-09T06:10:06.869778+0000","last_peered":"2023-10-09T06:10:06.869778+0000","last_clean":"2023-10-09T06:10:06.869778+0000","last_became_active":"2023-10-09T06:10:00.688613+0000","last_became_peered":"2023-10-09T06:10:00.688613+0000","last_unstale":"2023-10-09T06:10:06.869778+0000","last_undegraded":"2023-10-09T06:10:06.869778+0000","last_fullsized":"2023-10-09T06:10:06.869778+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":18,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859590+0000","last_change":"2023-10-09T06:09:49.140131+0000","last_active":"2023-10-09T06:10:06.859590+0000","last_peered":"2023-10-09T06:10:06.859590+0000","last_clean":"2023-10-09T06:10:06.859590+0000","last_became_active":"2023-10-09T06:09:49.139438+0000","last_became_peered":"2023-10-09T06:09:49.139438+0000","last_unstale":"2023-10-09T06:10:06.859590+0000","last_undegraded":"2023-10-09T06:10:06.859590+0000","last_fullsized":"2023-10-09T06:10:06.859590+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"59'2","reported_seq":251,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868882+0000","last_change":"2023-10-09T06:10:00.685537+0000","last_active":"2023-10-09T06:10:06.868882+0000","last_peered":"2023-10-09T06:10:06.868882+0000","last_clean":"2023-10-09T06:10:06.868882+0000","last_became_active":"2023-10-09T06:10:00.685103+0000","last_became_peered":"2023-10-09T06:10:00.685103+0000","last_unstale":"2023-10-09T06:10:06.868882+0000","last_undegraded":"2023-10-09T06:10:06.868882+0000","last_fullsized":"2023-10-09T06:10:06.868882+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":638,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861144+0000","last_change":"2023-10-09T06:09:45.404191+0000","last_active":"2023-10-09T06:10:06.861144+0000","last_peered":"2023-10-09T06:10:06.861144+0000","last_clean":"2023-10-09T06:10:06.861144+0000","last_became_active":"2023-10-09T06:09:45.403145+0000","last_became_peered":"2023-10-09T06:09:45.403145+0000","last_unstale":"2023-10-09T06:10:06.861144+0000","last_undegraded":"2023-10-09T06:10:06.861144+0000","last_fullsized":"2023-10-09T06:10:06.861144+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"59'1","reported_seq":285,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868900+0000","last_change":"2023-10-09T06:10:00.688504+0000","last_active":"2023-10-09T06:10:06.868900+0000","last_peered":"2023-10-09T06:10:06.868900+0000","last_clean":"2023-10-09T06:10:06.868900+0000","last_became_active":"2023-10-09T06:10:00.688167+0000","last_became_peered":"2023-10-09T06:10:00.688167+0000","last_unstale":"2023-10-09T06:10:06.868900+0000","last_undegraded":"2023-10-09T06:10:06.868900+0000","last_fullsized":"2023-10-09T06:10:06.868900+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":313,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.939491+0000","last_change":"2023-10-09T06:10:07.939491+0000","last_active":"2023-10-09T06:10:07.939491+0000","last_peered":"2023-10-09T06:10:07.939491+0000","last_clean":"2023-10-09T06:10:07.939491+0000","last_became_active":"2023-10-09T06:10:07.939133+0000","last_became_peered":"2023-10-09T06:10:07.939133+0000","last_unstale":"2023-10-09T06:10:07.939491+0000","last_undegraded":"2023-10-09T06:10:07.939491+0000","last_fullsized":"2023-10-09T06:10:07.939491+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":502,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859509+0000","last_change":"2023-10-09T06:09:53.246986+0000","last_active":"2023-10-09T06:10:06.859509+0000","last_peered":"2023-10-09T06:10:06.859509+0000","last_clean":"2023-10-09T06:10:06.859509+0000","last_became_active":"2023-10-09T06:09:53.246690+0000","last_became_peered":"2023-10-09T06:09:53.246690+0000","last_unstale":"2023-10-09T06:10:06.859509+0000","last_undegraded":"2023-10-09T06:10:06.859509+0000","last_fullsized":"2023-10-09T06:10:06.859509+0000","mapping_epoch":632,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":574,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.865226+0000","last_change":"2023-10-09T06:00:05.352723+0000","last_active":"2023-10-09T06:10:06.865226+0000","last_peered":"2023-10-09T06:10:06.865226+0000","last_clean":"2023-10-09T06:10:06.865226+0000","last_became_active":"2023-10-09T06:00:05.352008+0000","last_became_peered":"2023-10-09T06:00:05.352008+0000","last_unstale":"2023-10-09T06:10:06.865226+0000","last_undegraded":"2023-10-09T06:10:06.865226+0000","last_fullsized":"2023-10-09T06:10:06.865226+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:25.302918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"59'2","reported_seq":504,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029315+0000","last_change":"2023-10-09T06:09:37.004819+0000","last_active":"2023-10-09T06:10:07.029315+0000","last_peered":"2023-10-09T06:10:07.029315+0000","last_clean":"2023-10-09T06:10:07.029315+0000","last_became_active":"2023-10-09T06:09:37.003299+0000","last_became_peered":"2023-10-09T06:09:37.003299+0000","last_unstale":"2023-10-09T06:10:07.029315+0000","last_undegraded":"2023-10-09T06:10:07.029315+0000","last_fullsized":"2023-10-09T06:10:07.029315+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:36.536844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":658,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861304+0000","last_change":"2023-10-09T06:09:45.404027+0000","last_active":"2023-10-09T06:10:06.861304+0000","last_peered":"2023-10-09T06:10:06.861304+0000","last_clean":"2023-10-09T06:10:06.861304+0000","last_became_active":"2023-10-09T06:09:45.403071+0000","last_became_peered":"2023-10-09T06:09:45.403071+0000","last_unstale":"2023-10-09T06:10:06.861304+0000","last_undegraded":"2023-10-09T06:10:06.861304+0000","last_fullsized":"2023-10-09T06:10:06.861304+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":286,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.955878+0000","last_change":"2023-10-09T06:10:07.955878+0000","last_active":"2023-10-09T06:10:07.955878+0000","last_peered":"2023-10-09T06:10:07.955878+0000","last_clean":"2023-10-09T06:10:07.955878+0000","last_became_active":"2023-10-09T06:10:07.955458+0000","last_became_peered":"2023-10-09T06:10:07.955458+0000","last_unstale":"2023-10-09T06:10:07.955878+0000","last_undegraded":"2023-10-09T06:10:07.955878+0000","last_fullsized":"2023-10-09T06:10:07.955878+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":554,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861412+0000","last_change":"2023-10-09T06:09:45.132016+0000","last_active":"2023-10-09T06:10:06.861412+0000","last_peered":"2023-10-09T06:10:06.861412+0000","last_clean":"2023-10-09T06:10:06.861412+0000","last_became_active":"2023-10-09T06:09:45.131661+0000","last_became_peered":"2023-10-09T06:09:45.131661+0000","last_unstale":"2023-10-09T06:10:06.861412+0000","last_undegraded":"2023-10-09T06:10:06.861412+0000","last_fullsized":"2023-10-09T06:10:06.861412+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":612,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.860828+0000","last_change":"2023-10-09T06:09:45.132179+0000","last_active":"2023-10-09T06:10:06.860828+0000","last_peered":"2023-10-09T06:10:06.860828+0000","last_clean":"2023-10-09T06:10:06.860828+0000","last_became_active":"2023-10-09T06:09:45.131540+0000","last_became_peered":"2023-10-09T06:09:45.131540+0000","last_unstale":"2023-10-09T06:10:06.860828+0000","last_undegraded":"2023-10-09T06:10:06.860828+0000","last_fullsized":"2023-10-09T06:10:06.860828+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":555,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861540+0000","last_change":"2023-10-09T06:09:45.130020+0000","last_active":"2023-10-09T06:10:06.861540+0000","last_peered":"2023-10-09T06:10:06.861540+0000","last_clean":"2023-10-09T06:10:06.861540+0000","last_became_active":"2023-10-09T06:09:45.129685+0000","last_became_peered":"2023-10-09T06:09:45.129685+0000","last_unstale":"2023-10-09T06:10:06.861540+0000","last_undegraded":"2023-10-09T06:10:06.861540+0000","last_fullsized":"2023-10-09T06:10:06.861540+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":341,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029078+0000","last_change":"2023-10-09T06:09:37.005423+0000","last_active":"2023-10-09T06:10:07.029078+0000","last_peered":"2023-10-09T06:10:07.029078+0000","last_clean":"2023-10-09T06:10:07.029078+0000","last_became_active":"2023-10-09T06:09:37.002797+0000","last_became_peered":"2023-10-09T06:09:37.002797+0000","last_unstale":"2023-10-09T06:10:07.029078+0000","last_undegraded":"2023-10-09T06:10:07.029078+0000","last_fullsized":"2023-10-09T06:10:07.029078+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:52.098169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":340,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.866157+0000","last_change":"2023-10-09T06:07:47.057804+0000","last_active":"2023-10-09T06:10:06.866157+0000","last_peered":"2023-10-09T06:10:06.866157+0000","last_clean":"2023-10-09T06:10:06.866157+0000","last_became_active":"2023-10-09T06:07:47.057508+0000","last_became_peered":"2023-10-09T06:07:47.057508+0000","last_unstale":"2023-10-09T06:10:06.866157+0000","last_undegraded":"2023-10-09T06:10:06.866157+0000","last_fullsized":"2023-10-09T06:10:06.866157+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:45.041480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"604'322","reported_seq":1646,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.938088+0000","last_change":"2023-10-09T06:09:22.719273+0000","last_active":"2023-10-09T06:10:07.938088+0000","last_peered":"2023-10-09T06:10:07.938088+0000","last_clean":"2023-10-09T06:10:07.938088+0000","last_became_active":"2023-10-09T06:09:22.718603+0000","last_became_peered":"2023-10-09T06:09:22.718603+0000","last_unstale":"2023-10-09T06:10:07.938088+0000","last_undegraded":"2023-10-09T06:10:07.938088+0000","last_fullsized":"2023-10-09T06:10:07.938088+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":322,"log_dups_size":0,"ondisk_log_size":322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:06.922542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062772900000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1736,"num_read_kb":2559,"num_write":305,"num_write_kb":642,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":39,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859908+0000","last_change":"2023-10-09T06:08:53.661676+0000","last_active":"2023-10-09T06:10:06.859908+0000","last_peered":"2023-10-09T06:10:06.859908+0000","last_clean":"2023-10-09T06:10:06.859908+0000","last_became_active":"2023-10-09T06:08:53.661314+0000","last_became_peered":"2023-10-09T06:08:53.661314+0000","last_unstale":"2023-10-09T06:10:06.859908+0000","last_undegraded":"2023-10-09T06:10:06.859908+0000","last_fullsized":"2023-10-09T06:10:06.859908+0000","mapping_epoch":609,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":198,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864646+0000","last_change":"2023-10-09T06:09:49.952540+0000","last_active":"2023-10-09T06:10:06.864646+0000","last_peered":"2023-10-09T06:10:06.864646+0000","last_clean":"2023-10-09T06:10:06.864646+0000","last_became_active":"2023-10-09T06:09:49.952226+0000","last_became_peered":"2023-10-09T06:09:49.952226+0000","last_unstale":"2023-10-09T06:10:06.864646+0000","last_undegraded":"2023-10-09T06:10:06.864646+0000","last_fullsized":"2023-10-09T06:10:06.864646+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:21.483423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"603'130","reported_seq":1362,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859470+0000","last_change":"2023-10-09T06:09:53.163090+0000","last_active":"2023-10-09T06:10:06.859470+0000","last_peered":"2023-10-09T06:10:06.859470+0000","last_clean":"2023-10-09T06:10:06.859470+0000","last_became_active":"2023-10-09T06:09:53.162471+0000","last_became_peered":"2023-10-09T06:09:53.162471+0000","last_unstale":"2023-10-09T06:10:06.859470+0000","last_undegraded":"2023-10-09T06:10:06.859470+0000","last_fullsized":"2023-10-09T06:10:06.859470+0000","mapping_epoch":632,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1200,"num_read_kb":3722,"num_write":133,"num_write_kb":2436,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":581,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.862322+0000","last_change":"2023-10-09T06:09:45.135716+0000","last_active":"2023-10-09T06:10:06.862322+0000","last_peered":"2023-10-09T06:10:06.862322+0000","last_clean":"2023-10-09T06:10:06.862322+0000","last_became_active":"2023-10-09T06:09:45.133517+0000","last_became_peered":"2023-10-09T06:09:45.133517+0000","last_unstale":"2023-10-09T06:10:06.862322+0000","last_undegraded":"2023-10-09T06:10:06.862322+0000","last_fullsized":"2023-10-09T06:10:06.862322+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":445,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864293+0000","last_change":"2023-10-09T06:09:22.950106+0000","last_active":"2023-10-09T06:10:06.864293+0000","last_peered":"2023-10-09T06:10:06.864293+0000","last_clean":"2023-10-09T06:10:06.864293+0000","last_became_active":"2023-10-09T06:09:22.949792+0000","last_became_peered":"2023-10-09T06:09:22.949792+0000","last_unstale":"2023-10-09T06:10:06.864293+0000","last_undegraded":"2023-10-09T06:10:06.864293+0000","last_fullsized":"2023-10-09T06:10:06.864293+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:53.990328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"604'210","reported_seq":1465,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.869402+0000","last_change":"2023-10-09T06:08:36.715763+0000","last_active":"2023-10-09T06:10:07.869402+0000","last_peered":"2023-10-09T06:10:07.869402+0000","last_clean":"2023-10-09T06:10:07.869402+0000","last_became_active":"2023-10-09T06:05:44.534078+0000","last_became_peered":"2023-10-09T06:05:44.534078+0000","last_unstale":"2023-10-09T06:10:07.869402+0000","last_undegraded":"2023-10-09T06:10:07.869402+0000","last_fullsized":"2023-10-09T06:10:07.869402+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":210,"log_dups_size":0,"ondisk_log_size":210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:45.519427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042835799999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1263,"num_read_kb":5231,"num_write":190,"num_write_kb":8397,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":462,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864529+0000","last_change":"2023-10-09T06:05:30.447496+0000","last_active":"2023-10-09T06:10:06.864529+0000","last_peered":"2023-10-09T06:10:06.864529+0000","last_clean":"2023-10-09T06:10:06.864529+0000","last_became_active":"2023-10-09T06:05:30.447219+0000","last_became_peered":"2023-10-09T06:05:30.447219+0000","last_unstale":"2023-10-09T06:10:06.864529+0000","last_undegraded":"2023-10-09T06:10:06.864529+0000","last_fullsized":"2023-10-09T06:10:06.864529+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:08.869779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":143,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.937936+0000","last_change":"2023-10-09T06:08:53.663036+0000","last_active":"2023-10-09T06:10:07.937936+0000","last_peered":"2023-10-09T06:10:07.937936+0000","last_clean":"2023-10-09T06:10:07.937936+0000","last_became_active":"2023-10-09T06:08:53.662753+0000","last_became_peered":"2023-10-09T06:08:53.662753+0000","last_unstale":"2023-10-09T06:10:07.937936+0000","last_undegraded":"2023-10-09T06:10:07.937936+0000","last_fullsized":"2023-10-09T06:10:07.937936+0000","mapping_epoch":609,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:37.084661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"604'451","reported_seq":1887,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859419+0000","last_change":"2023-10-09T06:08:56.855202+0000","last_active":"2023-10-09T06:10:06.859419+0000","last_peered":"2023-10-09T06:10:06.859419+0000","last_clean":"2023-10-09T06:10:06.859419+0000","last_became_active":"2023-10-09T06:08:56.854446+0000","last_became_peered":"2023-10-09T06:08:56.854446+0000","last_unstale":"2023-10-09T06:10:06.859419+0000","last_undegraded":"2023-10-09T06:10:06.859419+0000","last_fullsized":"2023-10-09T06:10:06.859419+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":451,"log_dups_size":0,"ondisk_log_size":451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2690,"num_read_kb":2377,"num_write":394,"num_write_kb":242,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":834,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868263+0000","last_change":"2023-10-09T06:10:00.691112+0000","last_active":"2023-10-09T06:10:06.868263+0000","last_peered":"2023-10-09T06:10:06.868263+0000","last_clean":"2023-10-09T06:10:06.868263+0000","last_became_active":"2023-10-09T06:10:00.690656+0000","last_became_peered":"2023-10-09T06:10:00.690656+0000","last_unstale":"2023-10-09T06:10:06.868263+0000","last_undegraded":"2023-10-09T06:10:06.868263+0000","last_fullsized":"2023-10-09T06:10:06.868263+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:17:53.921965+0000","last_clean_scrub_stamp":"2023-10-09T05:17:53.921965+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":532,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.954636+0000","last_change":"2023-10-09T06:09:37.911323+0000","last_active":"2023-10-09T06:10:07.954636+0000","last_peered":"2023-10-09T06:10:07.954636+0000","last_clean":"2023-10-09T06:10:07.954636+0000","last_became_active":"2023-10-09T06:09:37.910959+0000","last_became_peered":"2023-10-09T06:09:37.910959+0000","last_unstale":"2023-10-09T06:10:07.954636+0000","last_undegraded":"2023-10-09T06:10:07.954636+0000","last_fullsized":"2023-10-09T06:10:07.954636+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:17.207997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":20,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859798+0000","last_change":"2023-10-09T06:09:47.257362+0000","last_active":"2023-10-09T06:10:06.859798+0000","last_peered":"2023-10-09T06:10:06.859798+0000","last_clean":"2023-10-09T06:10:06.859798+0000","last_became_active":"2023-10-09T06:09:47.256947+0000","last_became_peered":"2023-10-09T06:09:47.256947+0000","last_unstale":"2023-10-09T06:10:06.859798+0000","last_undegraded":"2023-10-09T06:10:06.859798+0000","last_fullsized":"2023-10-09T06:10:06.859798+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"603'179","reported_seq":1384,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868281+0000","last_change":"2023-10-09T06:10:00.689307+0000","last_active":"2023-10-09T06:10:06.868281+0000","last_peered":"2023-10-09T06:10:06.868281+0000","last_clean":"2023-10-09T06:10:06.868281+0000","last_became_active":"2023-10-09T06:10:00.688637+0000","last_became_peered":"2023-10-09T06:10:00.688637+0000","last_unstale":"2023-10-09T06:10:06.868281+0000","last_undegraded":"2023-10-09T06:10:06.868281+0000","last_fullsized":"2023-10-09T06:10:06.868281+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":179,"log_dups_size":0,"ondisk_log_size":179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1445,"num_read_kb":3702,"num_write":128,"num_write_kb":2304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":630,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.875543+0000","last_change":"2023-10-09T06:07:24.555346+0000","last_active":"2023-10-09T06:10:07.875543+0000","last_peered":"2023-10-09T06:10:07.875543+0000","last_clean":"2023-10-09T06:10:07.875543+0000","last_became_active":"2023-10-09T06:07:24.554992+0000","last_became_peered":"2023-10-09T06:07:24.554992+0000","last_unstale":"2023-10-09T06:10:07.875543+0000","last_undegraded":"2023-10-09T06:10:07.875543+0000","last_fullsized":"2023-10-09T06:10:07.875543+0000","mapping_epoch":568,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:43.884493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":71,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.866262+0000","last_change":"2023-10-09T06:07:50.496088+0000","last_active":"2023-10-09T06:10:06.866262+0000","last_peered":"2023-10-09T06:10:06.866262+0000","last_clean":"2023-10-09T06:10:06.866262+0000","last_became_active":"2023-10-09T06:07:50.495249+0000","last_became_peered":"2023-10-09T06:07:50.495249+0000","last_unstale":"2023-10-09T06:10:06.866262+0000","last_undegraded":"2023-10-09T06:10:06.866262+0000","last_fullsized":"2023-10-09T06:10:06.866262+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:52.044105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"604'221","reported_seq":1288,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.868948+0000","last_change":"2023-10-09T06:10:00.689601+0000","last_active":"2023-10-09T06:10:06.868948+0000","last_peered":"2023-10-09T06:10:06.868948+0000","last_clean":"2023-10-09T06:10:06.868948+0000","last_became_active":"2023-10-09T06:10:00.688878+0000","last_became_peered":"2023-10-09T06:10:00.688878+0000","last_unstale":"2023-10-09T06:10:06.868948+0000","last_undegraded":"2023-10-09T06:10:06.868948+0000","last_fullsized":"2023-10-09T06:10:06.868948+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":221,"log_dups_size":0,"ondisk_log_size":221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1081,"num_read_kb":4262,"num_write":208,"num_write_kb":4363,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":517,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864446+0000","last_change":"2023-10-09T06:09:22.714892+0000","last_active":"2023-10-09T06:10:06.864446+0000","last_peered":"2023-10-09T06:10:06.864446+0000","last_clean":"2023-10-09T06:10:06.864446+0000","last_became_active":"2023-10-09T06:09:22.714584+0000","last_became_peered":"2023-10-09T06:09:22.714584+0000","last_unstale":"2023-10-09T06:10:06.864446+0000","last_undegraded":"2023-10-09T06:10:06.864446+0000","last_fullsized":"2023-10-09T06:10:06.864446+0000","mapping_epoch":618,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:29.022515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":506,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.875487+0000","last_change":"2023-10-09T06:09:14.937686+0000","last_active":"2023-10-09T06:10:07.875487+0000","last_peered":"2023-10-09T06:10:07.875487+0000","last_clean":"2023-10-09T06:10:07.875487+0000","last_became_active":"2023-10-09T06:09:14.937378+0000","last_became_peered":"2023-10-09T06:09:14.937378+0000","last_unstale":"2023-10-09T06:10:07.875487+0000","last_undegraded":"2023-10-09T06:10:07.875487+0000","last_fullsized":"2023-10-09T06:10:07.875487+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:10.880068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"59'2","reported_seq":67,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029533+0000","last_change":"2023-10-09T06:09:37.001519+0000","last_active":"2023-10-09T06:10:07.029533+0000","last_peered":"2023-10-09T06:10:07.029533+0000","last_clean":"2023-10-09T06:10:07.029533+0000","last_became_active":"2023-10-09T06:09:37.001149+0000","last_became_peered":"2023-10-09T06:09:37.001149+0000","last_unstale":"2023-10-09T06:10:07.029533+0000","last_undegraded":"2023-10-09T06:10:07.029533+0000","last_fullsized":"2023-10-09T06:10:07.029533+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:04.062745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"604'253","reported_seq":943,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.864382+0000","last_change":"2023-10-09T06:08:36.755071+0000","last_active":"2023-10-09T06:10:06.864382+0000","last_peered":"2023-10-09T06:10:06.864382+0000","last_clean":"2023-10-09T06:10:06.864382+0000","last_became_active":"2023-10-09T06:04:32.959073+0000","last_became_peered":"2023-10-09T06:04:32.959073+0000","last_unstale":"2023-10-09T06:10:06.864382+0000","last_undegraded":"2023-10-09T06:10:06.864382+0000","last_fullsized":"2023-10-09T06:10:06.864382+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":253,"log_dups_size":0,"ondisk_log_size":253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:00.843837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043321599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1018,"num_read_kb":2880,"num_write":192,"num_write_kb":2673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":310,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.954417+0000","last_change":"2023-10-09T06:09:37.003040+0000","last_active":"2023-10-09T06:10:07.954417+0000","last_peered":"2023-10-09T06:10:07.954417+0000","last_clean":"2023-10-09T06:10:07.954417+0000","last_became_active":"2023-10-09T06:09:37.001673+0000","last_became_peered":"2023-10-09T06:09:37.001673+0000","last_unstale":"2023-10-09T06:10:07.954417+0000","last_undegraded":"2023-10-09T06:10:07.954417+0000","last_fullsized":"2023-10-09T06:10:07.954417+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:41.397215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"604'184","reported_seq":1186,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.878173+0000","last_change":"2023-10-09T06:10:07.878173+0000","last_active":"2023-10-09T06:10:07.878173+0000","last_peered":"2023-10-09T06:10:07.878173+0000","last_clean":"2023-10-09T06:10:07.878173+0000","last_became_active":"2023-10-09T06:10:07.877472+0000","last_became_peered":"2023-10-09T06:10:07.877472+0000","last_unstale":"2023-10-09T06:10:07.878173+0000","last_undegraded":"2023-10-09T06:10:07.878173+0000","last_fullsized":"2023-10-09T06:10:07.878173+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:20:22.216869+0000","last_clean_scrub_stamp":"2023-10-09T05:20:22.216869+0000","objects_scrubbed":0,"log_size":184,"log_dups_size":0,"ondisk_log_size":184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1555,"num_read_kb":1688,"num_write":140,"num_write_kb":4333,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"59'1","reported_seq":75,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.866181+0000","last_change":"2023-10-09T06:07:50.543307+0000","last_active":"2023-10-09T06:10:06.866181+0000","last_peered":"2023-10-09T06:10:06.866181+0000","last_clean":"2023-10-09T06:10:06.866181+0000","last_became_active":"2023-10-09T06:07:50.495736+0000","last_became_peered":"2023-10-09T06:07:50.495736+0000","last_unstale":"2023-10-09T06:10:06.866181+0000","last_undegraded":"2023-10-09T06:10:06.866181+0000","last_fullsized":"2023-10-09T06:10:06.866181+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:47.121228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":164,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861030+0000","last_change":"2023-10-09T06:09:45.130162+0000","last_active":"2023-10-09T06:10:06.861030+0000","last_peered":"2023-10-09T06:10:06.861030+0000","last_clean":"2023-10-09T06:10:06.861030+0000","last_became_active":"2023-10-09T06:09:45.129813+0000","last_became_peered":"2023-10-09T06:09:45.129813+0000","last_unstale":"2023-10-09T06:10:06.861030+0000","last_undegraded":"2023-10-09T06:10:06.861030+0000","last_fullsized":"2023-10-09T06:10:06.861030+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"59'1","reported_seq":80,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.875422+0000","last_change":"2023-10-09T06:08:47.115790+0000","last_active":"2023-10-09T06:10:07.875422+0000","last_peered":"2023-10-09T06:10:07.875422+0000","last_clean":"2023-10-09T06:10:07.875422+0000","last_became_active":"2023-10-09T06:08:47.115423+0000","last_became_peered":"2023-10-09T06:08:47.115423+0000","last_unstale":"2023-10-09T06:10:07.875422+0000","last_undegraded":"2023-10-09T06:10:07.875422+0000","last_fullsized":"2023-10-09T06:10:07.875422+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:10.939752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":344,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.865585+0000","last_change":"2023-10-09T06:07:47.252116+0000","last_active":"2023-10-09T06:10:06.865585+0000","last_peered":"2023-10-09T06:10:06.865585+0000","last_clean":"2023-10-09T06:10:06.865585+0000","last_became_active":"2023-10-09T06:07:47.251804+0000","last_became_peered":"2023-10-09T06:07:47.251804+0000","last_unstale":"2023-10-09T06:10:06.865585+0000","last_undegraded":"2023-10-09T06:10:06.865585+0000","last_fullsized":"2023-10-09T06:10:06.865585+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:02.563612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"59'2","reported_seq":38,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859396+0000","last_change":"2023-10-09T06:09:53.163027+0000","last_active":"2023-10-09T06:10:06.859396+0000","last_peered":"2023-10-09T06:10:06.859396+0000","last_clean":"2023-10-09T06:10:06.859396+0000","last_became_active":"2023-10-09T06:09:53.162617+0000","last_became_peered":"2023-10-09T06:09:53.162617+0000","last_unstale":"2023-10-09T06:10:06.859396+0000","last_undegraded":"2023-10-09T06:10:06.859396+0000","last_fullsized":"2023-10-09T06:10:06.859396+0000","mapping_epoch":632,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":435,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861025+0000","last_change":"2023-10-09T06:09:45.403512+0000","last_active":"2023-10-09T06:10:06.861025+0000","last_peered":"2023-10-09T06:10:06.861025+0000","last_clean":"2023-10-09T06:10:06.861025+0000","last_became_active":"2023-10-09T06:09:45.403092+0000","last_became_peered":"2023-10-09T06:09:45.403092+0000","last_unstale":"2023-10-09T06:10:06.861025+0000","last_undegraded":"2023-10-09T06:10:06.861025+0000","last_fullsized":"2023-10-09T06:10:06.861025+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"59'2","reported_seq":46,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.859893+0000","last_change":"2023-10-09T06:09:53.163877+0000","last_active":"2023-10-09T06:10:06.859893+0000","last_peered":"2023-10-09T06:10:06.859893+0000","last_clean":"2023-10-09T06:10:06.859893+0000","last_became_active":"2023-10-09T06:09:53.163479+0000","last_became_peered":"2023-10-09T06:09:53.163479+0000","last_unstale":"2023-10-09T06:10:06.859893+0000","last_undegraded":"2023-10-09T06:10:06.859893+0000","last_fullsized":"2023-10-09T06:10:06.859893+0000","mapping_epoch":632,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":277,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.861047+0000","last_change":"2023-10-09T06:09:45.403838+0000","last_active":"2023-10-09T06:10:06.861047+0000","last_peered":"2023-10-09T06:10:06.861047+0000","last_clean":"2023-10-09T06:10:06.861047+0000","last_became_active":"2023-10-09T06:09:45.403249+0000","last_became_peered":"2023-10-09T06:09:45.403249+0000","last_unstale":"2023-10-09T06:10:06.861047+0000","last_undegraded":"2023-10-09T06:10:06.861047+0000","last_fullsized":"2023-10-09T06:10:06.861047+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"59'1","reported_seq":185,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029507+0000","last_change":"2023-10-09T06:09:36.997962+0000","last_active":"2023-10-09T06:10:07.029507+0000","last_peered":"2023-10-09T06:10:07.029507+0000","last_clean":"2023-10-09T06:10:07.029507+0000","last_became_active":"2023-10-09T06:09:36.997617+0000","last_became_peered":"2023-10-09T06:09:36.997617+0000","last_unstale":"2023-10-09T06:10:07.029507+0000","last_undegraded":"2023-10-09T06:10:07.029507+0000","last_fullsized":"2023-10-09T06:10:07.029507+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:33.665893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":435,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:07.029305+0000","last_change":"2023-10-09T06:09:37.000176+0000","last_active":"2023-10-09T06:10:07.029305+0000","last_peered":"2023-10-09T06:10:07.029305+0000","last_clean":"2023-10-09T06:10:07.029305+0000","last_became_active":"2023-10-09T06:09:36.999854+0000","last_became_peered":"2023-10-09T06:09:36.999854+0000","last_unstale":"2023-10-09T06:10:07.029305+0000","last_undegraded":"2023-10-09T06:10:07.029305+0000","last_fullsized":"2023-10-09T06:10:07.029305+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:24.485275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"59'4","reported_seq":92,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.865597+0000","last_change":"2023-10-09T06:08:53.670793+0000","last_active":"2023-10-09T06:10:06.865597+0000","last_peered":"2023-10-09T06:10:06.865597+0000","last_clean":"2023-10-09T06:10:06.865597+0000","last_became_active":"2023-10-09T06:08:53.662715+0000","last_became_peered":"2023-10-09T06:08:53.662715+0000","last_unstale":"2023-10-09T06:10:06.865597+0000","last_undegraded":"2023-10-09T06:10:06.865597+0000","last_fullsized":"2023-10-09T06:10:06.865597+0000","mapping_epoch":609,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:15.763192+0000","last_clean_scrub_stamp":"2023-10-09T05:21:15.763192+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:42.974962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":632,"reported_epoch":638,"state":"active+clean","last_fresh":"2023-10-09T06:10:06.860834+0000","last_change":"2023-10-09T06:09:45.132231+0000","last_active":"2023-10-09T06:10:06.860834+0000","last_peered":"2023-10-09T06:10:06.860834+0000","last_clean":"2023-10-09T06:10:06.860834+0000","last_became_active":"2023-10-09T06:09:45.131062+0000","last_became_peered":"2023-10-09T06:09:45.131062+0000","last_unstale":"2023-10-09T06:10:06.860834+0000","last_undegraded":"2023-10-09T06:10:06.860834+0000","last_fullsized":"2023-10-09T06:10:06.860834+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:21:16.764798+0000","last_clean_scrub_stamp":"2023-10-09T05:21:16.764798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":71,"num_bytes_recovered":23464,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":52606,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11988,"num_read_kb":26421,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":189,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":127,"seq":545460847021,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515488,"kb_used_data":10648,"kb_used_omap":14,"kb_used_meta":1504817,"kb_avail":92229664,"statfs":{"total":95995035648,"available":94443175936,"internally_reserved":0,"allocated":10903552,"data_stored":9686606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14502,"internal_metadata":1540933466},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":2.8730000000000002,"15min":2.8650000000000002},"min":{"1min":0.39400000000000002,"5min":0.26300000000000001,"15min":0.23200000000000001},"max":{"1min":0.76000000000000001,"5min":167.06200000000001,"15min":176.89500000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":2.7400000000000002,"15min":2.9430000000000001},"min":{"1min":0.42999999999999999,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":0.83199999999999996,"5min":167.077,"15min":176.947},"last":0.32800000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":3.4060000000000001,"15min":4.2750000000000004},"min":{"1min":0.39300000000000002,"5min":0.28100000000000003,"15min":0.252},"max":{"1min":0.72199999999999998,"5min":169.98400000000001,"15min":176.90600000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":3.855,"15min":4.3869999999999996},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.25900000000000001},"max":{"1min":0.71399999999999997,"5min":172.821,"15min":176.86799999999999},"last":0.499}]},{"osd":2,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":5.8250000000000002,"15min":5.593},"min":{"1min":0.41399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.81599999999999995,"5min":184.27500000000001,"15min":302.01799999999997},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":5.5259999999999998,"15min":5.3019999999999996},"min":{"1min":0.42199999999999999,"5min":0.35099999999999998,"15min":0.30299999999999999},"max":{"1min":0.84299999999999997,"5min":184.315,"15min":304.988},"last":0.54400000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":3.415,"15min":3.2090000000000001},"min":{"1min":0.45200000000000001,"5min":0.35099999999999998,"15min":0.30499999999999999},"max":{"1min":0.80500000000000005,"5min":167.64699999999999,"15min":176.958},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":3.0859999999999999,"15min":3.1360000000000001},"min":{"1min":0.437,"5min":0.34499999999999997,"15min":0.29699999999999999},"max":{"1min":0.79700000000000004,"5min":169.52199999999999,"15min":176.91800000000001},"last":0.53600000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.74299999999999999,"15min":1.492},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.252},"max":{"1min":0.68400000000000005,"5min":5.6340000000000003,"15min":178.995},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.73799999999999999,"15min":1.5920000000000001},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.246},"max":{"1min":0.70799999999999996,"5min":4.9349999999999996,"15min":178.899},"last":0.48899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:09:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.67200000000000004,"15min":1.9259999999999999},"min":{"1min":0.372,"5min":0.35199999999999998,"15min":0.29399999999999998},"max":{"1min":0.74399999999999999,"5min":3.1299999999999999,"15min":176.82400000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.70499999999999996,"15min":1.887},"min":{"1min":0.39700000000000002,"5min":0.33200000000000002,"15min":0.28999999999999998},"max":{"1min":0.73899999999999999,"5min":4.1210000000000004,"15min":176.79499999999999},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.78800000000000003,"15min":1.0189999999999999},"min":{"1min":0.34300000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.81599999999999995,"5min":13.193,"15min":13.193},"last":0.75},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.86799999999999999,"15min":0.76600000000000001},"min":{"1min":0.36799999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.749,"5min":13.247,"15min":13.247},"last":0.56899999999999995}]}]},{"osd":6,"up_from":537,"seq":2306397438029,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":382228,"kb_used_data":10636,"kb_used_omap":13,"kb_used_meta":371570,"kb_avail":93362924,"statfs":{"total":95995035648,"available":95603634176,"internally_reserved":0,"allocated":10891264,"data_stored":9686214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13650,"internal_metadata":380488366},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":2.1890000000000001,"15min":4.2869999999999999},"min":{"1min":0.30099999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.745,"5min":126.914,"15min":126.914},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.60999999999999999,"5min":2.4289999999999998,"15min":4.6020000000000003},"min":{"1min":0.45700000000000002,"5min":0.33900000000000002,"15min":0.32000000000000001},"max":{"1min":0.82499999999999996,"5min":126.925,"15min":126.925},"last":0.53700000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":2.4289999999999998,"15min":8.8710000000000004},"min":{"1min":0.29399999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.70199999999999996,"5min":130.28100000000001,"15min":130.28100000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.52600000000000002,"5min":2.431,"15min":8.8130000000000006},"min":{"1min":0.371,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.75,"5min":135.06100000000001,"15min":135.06100000000001},"last":0.57699999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":2.786,"15min":2.4750000000000001},"min":{"1min":0.33300000000000002,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":0.73499999999999999,"5min":133.917,"15min":133.917},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.63,"5min":2.8730000000000002,"15min":2.8029999999999999},"min":{"1min":0.42599999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.91600000000000004,"5min":133.92699999999999,"15min":133.92699999999999},"last":0.62}]},{"osd":3,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":2.4369999999999998,"15min":4.8639999999999999},"min":{"1min":0.34100000000000003,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.76700000000000002,"5min":129.375,"15min":129.375},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.56299999999999994,"5min":2.3849999999999998,"15min":4.8390000000000004},"min":{"1min":0.45700000000000002,"5min":0.308,"15min":0.308},"max":{"1min":0.75,"5min":129.38900000000001,"15min":129.38900000000001},"last":0.59099999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.74399999999999999,"15min":0.99399999999999999},"min":{"1min":0.36399999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.76100000000000001,"5min":11.103999999999999,"15min":11.103999999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57899999999999996,"15min":0.94399999999999995},"min":{"1min":0.39200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.80800000000000005,"5min":4.1669999999999998,"15min":8.1229999999999993},"last":0.56499999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.73599999999999999,"15min":0.85299999999999998},"min":{"1min":0.437,"5min":0.314,"15min":0.313},"max":{"1min":0.79700000000000004,"5min":11.218,"15min":11.218},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.624,"5min":0.68700000000000006,"15min":0.83499999999999996},"min":{"1min":0.46700000000000003,"5min":0.315,"15min":0.315},"max":{"1min":0.89800000000000002,"5min":6.5389999999999997,"15min":6.5389999999999997},"last":0.60999999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.70099999999999996,"15min":4.609},"min":{"1min":0.40999999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.96999999999999997,"5min":3.4369999999999998,"15min":124.27200000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.625,"5min":0.86299999999999999,"15min":4.4409999999999998},"min":{"1min":0.45200000000000001,"5min":0.31,"15min":0.308},"max":{"1min":0.83899999999999997,"5min":11.339,"15min":121.136},"last":0.52300000000000002}]}]},{"osd":1,"up_from":93,"seq":399431959013,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1037972,"kb_used_data":7244,"kb_used_omap":7,"kb_used_meta":1030712,"kb_avail":92707180,"statfs":{"total":95995035648,"available":94932152320,"internally_reserved":0,"allocated":7417856,"data_stored":5042705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":1055449614},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":1.1970000000000001,"15min":1.292},"min":{"1min":0.27400000000000002,"5min":0.23699999999999999,"15min":0.21199999999999999},"max":{"1min":0.67700000000000005,"5min":17.465,"15min":27.045999999999999},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.39900000000000002,"5min":1.468,"15min":1.4410000000000001},"min":{"1min":0.31,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.60999999999999999,"5min":15.682,"15min":27.344000000000001},"last":0.33700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:10:07 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":3.718,"15min":2.3490000000000002},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.20899999999999999},"max":{"1min":0.56699999999999995,"5min":186.84200000000001,"15min":186.84200000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":3.6890000000000001,"15min":2.4009999999999998},"min":{"1min":0.32300000000000001,"5min":0.26200000000000001,"15min":0.23699999999999999},"max":{"1min":0.72599999999999998,"5min":186.75800000000001,"15min":186.75800000000001},"last":0.36699999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":3.4790000000000001,"15min":2.2349999999999999},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.23799999999999999},"max":{"1min":0.625,"5min":177.70599999999999,"15min":177.70599999999999},"last":0.313},{"interface":"front","average":{"1min":0.46899999999999997,"5min":3.4630000000000001,"15min":2.2120000000000002},"min":{"1min":0.375,"5min":0.29399999999999998,"15min":0.24099999999999999},"max":{"1min":0.63900000000000001,"5min":177.82400000000001,"15min":177.82400000000001},"last":0.41099999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":1.548,"15min":1.706},"min":{"1min":0.42099999999999999,"5min":0.29699999999999999,"15min":0.25800000000000001},"max":{"1min":0.98599999999999999,"5min":10.907999999999999,"15min":86.483999999999995},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.95299999999999996,"15min":1.407},"min":{"1min":0.41399999999999998,"5min":0.33900000000000002,"15min":0.32000000000000001},"max":{"1min":0.96499999999999997,"5min":6.6580000000000004,"15min":86.441999999999993},"last":0.52900000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":1.0149999999999999,"15min":1.244},"min":{"1min":0.46899999999999997,"5min":0.314,"15min":0.313},"max":{"1min":0.89100000000000001,"5min":8.6769999999999996,"15min":14.019},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.66900000000000004,"5min":2.5310000000000001,"15min":1.887},"min":{"1min":0.38800000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.94899999999999995,"5min":99.769999999999996,"15min":99.769999999999996},"last":0.56899999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":1.5189999999999999,"15min":3.1480000000000001},"min":{"1min":0.54900000000000004,"5min":0.372,"15min":0.372},"max":{"1min":0.85599999999999998,"5min":13.205,"15min":13.205},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.72499999999999998,"5min":1.097,"15min":2.0329999999999999},"min":{"1min":0.502,"5min":0.375,"15min":0.375},"max":{"1min":0.97199999999999998,"5min":7.3259999999999996,"15min":13.685},"last":0.60999999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":1.2150000000000001,"15min":1.8620000000000001},"min":{"1min":0.49199999999999999,"5min":0.379,"15min":0.31},"max":{"1min":1.0089999999999999,"5min":6.7190000000000003,"15min":89.787000000000006},"last":0.628},{"interface":"front","average":{"1min":0.70299999999999996,"5min":1.238,"15min":1.966},"min":{"1min":0.52800000000000002,"5min":0.443,"15min":0.35199999999999998},"max":{"1min":0.89700000000000002,"5min":7.3310000000000004,"15min":89.869},"last":0.58299999999999996}]}]},{"osd":0,"up_from":88,"seq":377957122539,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1001752,"kb_used_data":11984,"kb_used_omap":8,"kb_used_meta":989751,"kb_avail":92743400,"statfs":{"total":95995035648,"available":94969241600,"internally_reserved":0,"allocated":12271616,"data_stored":9833357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":1013505800},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":1.5600000000000001,"15min":3.4129999999999998},"min":{"1min":0.249,"5min":0.19400000000000001,"15min":0.191},"max":{"1min":0.47799999999999998,"5min":15.27,"15min":163.572},"last":0.31},{"interface":"front","average":{"1min":0.39300000000000002,"5min":1.8029999999999999,"15min":3.5779999999999998},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.19700000000000001},"max":{"1min":0.46100000000000002,"5min":16.222000000000001,"15min":162.48699999999999},"last":0.40400000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.75,"15min":1.5840000000000001},"min":{"1min":0.25,"5min":0.215,"15min":0.20599999999999999},"max":{"1min":0.99399999999999999,"5min":27.338000000000001,"15min":95.512},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.40000000000000002,"5min":2.024,"15min":1.661},"min":{"1min":0.26200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.51700000000000002,"5min":31.065999999999999,"15min":95.462999999999994},"last":0.36299999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":1.607,"15min":1.3340000000000001},"min":{"1min":0.28599999999999998,"5min":0.221,"15min":0.21299999999999999},"max":{"1min":0.53600000000000003,"5min":12.304,"15min":12.304},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"min":{"1min":0.35099999999999998,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":1.0169999999999999,"5min":8.5090000000000003,"15min":13.632},"last":0.378}]},{"osd":4,"last update":"Mon Oct 9 06:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.2949999999999999,"15min":1.0920000000000001},"min":{"1min":0.42299999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.85899999999999999,"5min":12.965999999999999,"15min":13.592000000000001},"last":0.83699999999999997},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.498,"15min":1.1990000000000001},"min":{"1min":0.45500000000000002,"5min":0.32200000000000001,"15min":0.29399999999999998},"max":{"1min":0.93400000000000005,"5min":13.775,"15min":13.775},"last":0.84799999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.6930000000000001,"15min":2.3730000000000002},"min":{"1min":0.41199999999999998,"5min":0.318,"15min":0.27900000000000003},"max":{"1min":0.84899999999999998,"5min":13.746,"15min":210.77799999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.6399999999999999,"15min":2.448},"min":{"1min":0.36599999999999999,"5min":0.34499999999999997,"15min":0.30599999999999999},"max":{"1min":1.2430000000000001,"5min":11.52,"15min":209.91},"last":0.72499999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.155,"15min":1.6970000000000001},"min":{"1min":0.48399999999999999,"5min":0.32800000000000001,"15min":0.314},"max":{"1min":0.80300000000000005,"5min":12.929,"15min":12.929},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.546,"15min":1.4419999999999999},"min":{"1min":0.44400000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.81999999999999995,"5min":13.608000000000001,"15min":13.608000000000001},"last":0.746}]},{"osd":7,"last update":"Mon Oct 9 06:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":1.321,"15min":3.1499999999999999},"min":{"1min":0.442,"5min":0.373,"15min":0.32400000000000001},"max":{"1min":0.91200000000000003,"5min":12.987,"15min":168.98599999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.607,"15min":3.1930000000000001},"min":{"1min":0.52000000000000002,"5min":0.39100000000000001,"15min":0.32800000000000001},"max":{"1min":0.86399999999999999,"5min":13.654999999999999,"15min":169.03800000000001},"last":0.77700000000000002}]}]},{"osd":2,"up_from":98,"seq":420906795487,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1073300,"kb_used_data":15884,"kb_used_omap":11,"kb_used_meta":1057396,"kb_avail":92671852,"statfs":{"total":95995035648,"available":94895976448,"internally_reserved":0,"allocated":16265216,"data_stored":13476950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11691,"internal_metadata":1082774101},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":1.611,"15min":1.405},"min":{"1min":0.216,"5min":0.20499999999999999,"15min":0.19600000000000001},"max":{"1min":0.67000000000000004,"5min":11.928000000000001,"15min":27.292000000000002},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.39400000000000002,"5min":1.3939999999999999,"15min":1.4330000000000001},"min":{"1min":0.22600000000000001,"5min":0.19700000000000001,"15min":0.19600000000000001},"max":{"1min":0.64700000000000002,"5min":10.958,"15min":17.109999999999999},"last":0.34599999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.7749999999999999,"15min":2.387},"min":{"1min":0.253,"5min":0.22700000000000001,"15min":0.19},"max":{"1min":0.71899999999999997,"5min":26.869,"15min":133.42500000000001},"last":0.307},{"interface":"front","average":{"1min":0.39000000000000001,"5min":2.1000000000000001,"15min":2.6269999999999998},"min":{"1min":0.23499999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.64500000000000002,"5min":13.238,"15min":133.40799999999999},"last":0.32300000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":2.1850000000000001,"15min":2.5379999999999998},"min":{"1min":0.28799999999999998,"5min":0.252,"15min":0.23300000000000001},"max":{"1min":0.81799999999999995,"5min":13.285,"15min":142.26300000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":1.6279999999999999,"15min":2.1030000000000002},"min":{"1min":0.29799999999999999,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":0.78700000000000003,"5min":15.108000000000001,"15min":141.477},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.7649999999999999,"15min":1.3939999999999999},"min":{"1min":0.47399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.76100000000000001,"5min":11.132999999999999,"15min":11.132999999999999},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.65100000000000002,"5min":1.319,"15min":1.0800000000000001},"min":{"1min":0.496,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.86499999999999999,"5min":8.3100000000000005,"15min":16.457000000000001},"last":0.58799999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.496,"15min":1.304},"min":{"1min":0.40899999999999997,"5min":0.35499999999999998,"15min":0.33000000000000002},"max":{"1min":0.82799999999999996,"5min":10.852,"15min":18.350000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.8009999999999999,"15min":1.6559999999999999},"min":{"1min":0.44400000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.874,"5min":11.169,"15min":15.781000000000001},"last":0.65600000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:09:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":1.619,"15min":1.853},"min":{"1min":0.434,"5min":0.375,"15min":0.375},"max":{"1min":1.1619999999999999,"5min":16.422000000000001,"15min":16.422000000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.6080000000000001,"15min":1.752},"min":{"1min":0.52800000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.93799999999999994,"5min":10.801,"15min":10.801},"last":0.64500000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":1.6279999999999999,"15min":1.883},"min":{"1min":0.50800000000000001,"5min":0.39900000000000002,"15min":0.34399999999999997},"max":{"1min":1.111,"5min":8.3640000000000008,"15min":122.827},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.71799999999999997,"5min":1.7829999999999999,"15min":2.0430000000000001},"min":{"1min":0.55300000000000005,"5min":0.39600000000000002,"15min":0.36099999999999999},"max":{"1min":0.91800000000000004,"5min":10.890000000000001,"15min":123.56399999999999},"last":0.66900000000000004}]}]},{"osd":3,"up_from":103,"seq":442381631962,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1215848,"kb_used_data":11424,"kb_used_omap":4,"kb_used_meta":1204411,"kb_avail":92529304,"statfs":{"total":95995035648,"available":94750007296,"internally_reserved":0,"allocated":11698176,"data_stored":9245418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1233317470},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":1.8879999999999999,"15min":2.194},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20399999999999999},"max":{"1min":0.49199999999999999,"5min":50.546999999999997,"15min":135.501},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.34899999999999998,"5min":1.8520000000000001,"15min":2.1360000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.20899999999999999},"max":{"1min":0.50600000000000001,"5min":55.683,"15min":135.99199999999999},"last":0.29999999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":3.6150000000000002,"15min":3.71},"min":{"1min":0.245,"5min":0.24299999999999999,"15min":0.20100000000000001},"max":{"1min":0.65400000000000003,"5min":137.679,"15min":137.679},"last":0.246},{"interface":"front","average":{"1min":0.373,"5min":3.5859999999999999,"15min":3.504},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23799999999999999},"max":{"1min":0.53600000000000003,"5min":137.46799999999999,"15min":137.46799999999999},"last":0.26500000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":3.5419999999999998,"15min":3.2749999999999999},"min":{"1min":0.27500000000000002,"5min":0.23999999999999999,"15min":0.22700000000000001},"max":{"1min":0.66000000000000003,"5min":71.018000000000001,"15min":89.024000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.45700000000000002,"5min":3.2690000000000001,"15min":2.8029999999999999},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.607,"5min":69.918999999999997,"15min":90.850999999999999},"last":0.39700000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.135,"15min":1.4470000000000001},"min":{"1min":0.41499999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.82699999999999996,"5min":8.9930000000000003,"15min":74.957999999999998},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.1659999999999999,"15min":1.484},"min":{"1min":0.45000000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.071,"5min":9.7309999999999999,"15min":75.022999999999996},"last":0.60499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.0960000000000001,"15min":0.95999999999999996},"min":{"1min":0.51100000000000001,"5min":0.38,"15min":0.32700000000000001},"max":{"1min":1,"5min":6.8449999999999998,"15min":8.8450000000000006},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.74399999999999999,"5min":1.1419999999999999,"15min":1.1479999999999999},"min":{"1min":0.45700000000000002,"5min":0.34999999999999998,"15min":0.33400000000000002},"max":{"1min":2.6200000000000001,"5min":9.8109999999999999,"15min":15.932},"last":0.63800000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.2230000000000001,"15min":1.413},"min":{"1min":0.47299999999999998,"5min":0.40699999999999997,"15min":0.34300000000000003},"max":{"1min":0.89100000000000001,"5min":6.7939999999999996,"15min":8.9149999999999991},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.3360000000000001,"15min":2.3919999999999999},"min":{"1min":0.41699999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.94199999999999995,"5min":9.7780000000000005,"15min":15.954000000000001},"last":0.67400000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:10:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":2.6680000000000001,"15min":1.8100000000000001},"min":{"1min":0.44,"5min":0.41599999999999998,"15min":0.375},"max":{"1min":0.92400000000000004,"5min":79.403000000000006,"15min":79.403000000000006},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.66800000000000004,"5min":2.851,"15min":1.8779999999999999},"min":{"1min":0.505,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.022,"5min":79.203000000000003,"15min":79.203000000000003},"last":0.59899999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019507,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195800,"kb_used_data":5968,"kb_used_omap":0,"kb_used_meta":1189824,"kb_avail":92549352,"statfs":{"total":95995035648,"available":94770536448,"internally_reserved":0,"allocated":6111232,"data_stored":4901864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1218379776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.073,"15min":1.9099999999999999},"min":{"1min":0.315,"5min":0.23799999999999999,"15min":0.21299999999999999},"max":{"1min":1.002,"5min":17.248999999999999,"15min":86.509},"last":0.502},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.619,"15min":2.2799999999999998},"min":{"1min":0.35999999999999999,"5min":0.27600000000000002,"15min":0.24299999999999999},"max":{"1min":3.661,"5min":18.664000000000001,"15min":86.233999999999995},"last":0.442}]},{"osd":1,"last update":"Mon Oct 9 06:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.821,"15min":2.331},"min":{"1min":0.435,"5min":0.253,"15min":0.253},"max":{"1min":0.749,"5min":18.951000000000001,"15min":96.033000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":2.9750000000000001,"15min":2.8130000000000002},"min":{"1min":0.35899999999999999,"5min":0.26700000000000002,"15min":0.255},"max":{"1min":0.77200000000000002,"5min":108.45699999999999,"15min":108.45699999999999},"last":0.46300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:10:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":4.3959999999999999,"15min":2.8460000000000001},"min":{"1min":0.38800000000000001,"5min":0.33900000000000002,"15min":0.27200000000000002},"max":{"1min":0.79600000000000004,"5min":105.157,"15min":143.71000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.59199999999999997,"5min":3.6930000000000001,"15min":3.137},"min":{"1min":0.309,"5min":0.309,"15min":0.27700000000000002},"max":{"1min":0.95799999999999996,"5min":107.851,"15min":143.768},"last":0.60299999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":2.3559999999999999,"15min":1.762},"min":{"1min":0.40699999999999997,"5min":0.30199999999999999,"15min":0.254},"max":{"1min":3.7029999999999998,"5min":107.352,"15min":107.352},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":2.278,"15min":1.5600000000000001},"min":{"1min":0.45100000000000001,"5min":0.33400000000000002,"15min":0.32700000000000001},"max":{"1min":0.88600000000000001,"5min":105.336,"15min":105.336},"last":0.63900000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.84699999999999998,"15min":1.008},"min":{"1min":0.36099999999999999,"5min":0.317,"15min":0.26100000000000001},"max":{"1min":0.65300000000000002,"5min":7.1070000000000002,"15min":15.597},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.80500000000000005,"15min":0.96999999999999997},"min":{"1min":0.48699999999999999,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":0.80500000000000005,"5min":7.069,"15min":17.882000000000001},"last":0.56000000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.76500000000000001,"15min":0.81999999999999995},"min":{"1min":0.378,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.80300000000000005,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.054,"15min":0.97799999999999998},"min":{"1min":0.40200000000000002,"5min":0.34200000000000003,"15min":0.33900000000000002},"max":{"1min":0.72999999999999998,"5min":8.9909999999999997,"15min":8.9909999999999997},"last":0.54100000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.83899999999999997,"15min":0.98599999999999999},"min":{"1min":0.39500000000000002,"5min":0.33600000000000002,"15min":0.29799999999999999},"max":{"1min":0.75600000000000001,"5min":6.6040000000000001,"15min":17.832000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.145,"15min":1.038},"min":{"1min":0.38400000000000001,"5min":0.35299999999999998,"15min":0.32900000000000001},"max":{"1min":0.71299999999999997,"5min":8.9580000000000002,"15min":17.742999999999999},"last":0.626}]}]},{"osd":5,"up_from":36,"seq":154618823281,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908744,"kb_used_data":5952,"kb_used_omap":0,"kb_used_meta":902784,"kb_avail":92836408,"statfs":{"total":95995035648,"available":95064481792,"internally_reserved":0,"allocated":6094848,"data_stored":4897312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924450816},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.94799999999999995,"15min":1.71},"min":{"1min":0.309,"5min":0.25900000000000001,"15min":0.223},"max":{"1min":1.081,"5min":10.618,"15min":104.842},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.87,"15min":1.837},"min":{"1min":0.37,"5min":0.29199999999999998,"15min":0.254},"max":{"1min":1.1040000000000001,"5min":7.5460000000000003,"15min":104.813},"last":0.64500000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.3759999999999999,"15min":2.0339999999999998},"min":{"1min":0.42099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.048,"5min":26.544,"15min":80.003},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.3740000000000001,"15min":1.9850000000000001},"min":{"1min":0.34000000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.1930000000000001,"5min":25.946999999999999,"15min":79.930000000000007},"last":0.67400000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.9510000000000001,"15min":2.613},"min":{"1min":0.36899999999999999,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":1.0309999999999999,"5min":40.969999999999999,"15min":151.06100000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.62,"5min":1.6359999999999999,"15min":2.5790000000000002},"min":{"1min":0.35899999999999999,"5min":0.34200000000000003,"15min":0.30299999999999999},"max":{"1min":1.1619999999999999,"5min":40.816000000000003,"15min":151.46299999999999},"last":0.621}]},{"osd":3,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":1.339,"15min":1.659},"min":{"1min":0.39500000000000002,"5min":0.33000000000000002,"15min":0.29899999999999999},"max":{"1min":1.2230000000000001,"5min":21.91,"15min":102.768},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.6000000000000001,"15min":1.637},"min":{"1min":0.51000000000000001,"5min":0.36099999999999999,"15min":0.33500000000000002},"max":{"1min":1.1220000000000001,"5min":21.608000000000001,"15min":102.806},"last":0.69399999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.78800000000000003,"15min":0.79400000000000004},"min":{"1min":0.375,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":1.3089999999999999,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61099999999999999,"15min":0.754},"min":{"1min":0.40300000000000002,"5min":0.311,"15min":0.27000000000000002},"max":{"1min":1.2110000000000001,"5min":4.8650000000000002,"15min":10.154},"last":0.42999999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.72799999999999998,"15min":0.57699999999999996},"min":{"1min":0.39800000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.258,"5min":13.098000000000001,"15min":13.098000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.61399999999999999,"15min":0.52200000000000002},"min":{"1min":0.373,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.274,"5min":3.3420000000000001,"15min":3.3420000000000001},"last":0.47899999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.75900000000000001,"15min":0.91700000000000004},"min":{"1min":0.41799999999999998,"5min":0.35599999999999998,"15min":0.32000000000000001},"max":{"1min":0.92500000000000004,"5min":7.8470000000000004,"15min":21.068000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.751,"15min":0.94799999999999995},"min":{"1min":0.44400000000000001,"5min":0.35699999999999998,"15min":0.34799999999999998},"max":{"1min":0.93300000000000005,"5min":5.2409999999999997,"15min":21.106000000000002},"last":0.66000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2297,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8440,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7666,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13650,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12205,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:10:50.020 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:10:50.020 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T06:10:50.089 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=b49b9fbe5efd4c8ca20e2c2a42334fed Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T06:10:50.091 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T06:10:50.102 INFO:tasks.cephadm:Teardown begin 2023-10-09T06:10:50.102 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:10:50.178 DEBUG:teuthology.orchestra.run.smithi191:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:10:50.252 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T06:10:50.252 DEBUG:teuthology.orchestra.run.smithi008:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T06:10:50.271 DEBUG:teuthology.orchestra.run.smithi191:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T06:10:50.290 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T06:10:50.291 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T06:10:50.291 DEBUG:teuthology.orchestra.run.smithi008:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T06:10:50.595 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:50 smithi008 systemd[1]: Stopping Ceph mon.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:10:50.595 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:50 smithi008 conmon[184394]: 2023-10-09T06:10:50.582+0000 7f8a6bd2c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:10:50.595 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:50 smithi008 conmon[184394]: 2023-10-09T06:10:50.582+0000 7f8a6bd2c700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T06:10:50.595 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:50 smithi008 conmon[177229]: ::ffff:172.21.15.191 - - [09/Oct/2023:06:10:50] "GET /metrics HTTP/1.1" 200 34158 "" "Prometheus/2.33.4" 2023-10-09T06:10:50.871 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:50 smithi008 conmon[177229]: [09/Oct/2023:06:10:50] ENGINE Bus STOPPING 2023-10-09T06:10:50.871 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:50 smithi008 conmon[177229]: [09/Oct/2023:06:10:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:10:50.871 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:50 smithi008 conmon[177229]: [09/Oct/2023:06:10:50] ENGINE Bus STOPPED 2023-10-09T06:10:50.872 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:50 smithi008 conmon[177229]: [09/Oct/2023:06:10:50] ENGINE Bus STARTING 2023-10-09T06:10:50.872 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:50 smithi008 conmon[177229]: [09/Oct/2023:06:10:50] ENGINE Serving on http://:::9283 2023-10-09T06:10:50.872 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:50 smithi008 conmon[177229]: [09/Oct/2023:06:10:50] ENGINE Bus STARTED 2023-10-09T06:10:51.214 INFO:journalctl@ceph.mon.a.smithi008.stdout:Oct 09 06:10:50 smithi008 bash[735039]: dbfd9e97488a61d7a86a687a3d4165e63ed21d6ca27ba60945fd121f6d2f2e4c 2023-10-09T06:10:51.678 DEBUG:teuthology.orchestra.run.smithi008:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T06:10:51.741 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:10:51.741 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T06:10:51.742 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T06:10:51.742 DEBUG:teuthology.orchestra.run.smithi008:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T06:10:51.964 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:51 smithi008 systemd[1]: Stopping Ceph mon.c for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:10:52.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:52 smithi008 conmon[191203]: 2023-10-09T06:10:52.060+0000 7fe9f8ac6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:10:52.363 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:52 smithi008 conmon[191203]: 2023-10-09T06:10:52.060+0000 7fe9f8ac6700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T06:10:52.714 INFO:journalctl@ceph.mon.c.smithi008.stdout:Oct 09 06:10:52 smithi008 bash[735349]: b504573c5a12362e1d77a42d2a3d88325112768e70599400b5b68aee10b7be71 2023-10-09T06:10:53.027 DEBUG:teuthology.orchestra.run.smithi008:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T06:10:53.070 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:10:53.070 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T06:10:53.070 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T06:10:53.071 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T06:10:53.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:53 smithi191 systemd[1]: Stopping Ceph mon.b for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:10:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:53 smithi191 conmon[161644]: 2023-10-09T06:10:53.361+0000 7fa33254e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:10:53.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Oct 09 06:10:53 smithi191 conmon[161644]: 2023-10-09T06:10:53.361+0000 7fa33254e700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T06:10:54.229 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T06:10:54.269 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:10:54.269 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T06:10:54.269 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T06:10:54.269 DEBUG:teuthology.orchestra.run.smithi008:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T06:10:54.560 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:54 smithi008 systemd[1]: Stopping Ceph mgr.y for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:10:54.964 INFO:journalctl@ceph.mgr.y.smithi008.stdout:Oct 09 06:10:54 smithi008 conmon[177229]: 2023-10-09T06:10:54.559+0000 7f023dc02700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:10:55.401 DEBUG:teuthology.orchestra.run.smithi008:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T06:10:55.440 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:10:55.440 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T06:10:55.441 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T06:10:55.441 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T06:10:55.800 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Oct 09 06:10:55 smithi191 systemd[1]: Stopping Ceph mgr.x for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:10:56.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T06:10:56.444 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:10:56.445 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T06:10:56.445 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T06:10:56.445 DEBUG:teuthology.orchestra.run.smithi008:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T06:10:56.743 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 06:10:56 smithi008 systemd[1]: Stopping Ceph osd.0 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:10:57.215 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 06:10:56 smithi008 conmon[203840]: 2023-10-09T06:10:56.742+0000 7fbbc4745700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:10:57.215 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 06:10:56 smithi008 conmon[203840]: 2023-10-09T06:10:56.742+0000 7fbbc4745700 -1 osd.0 639 *** Got signal Terminated *** 2023-10-09T06:10:57.215 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 06:10:56 smithi008 conmon[203840]: 2023-10-09T06:10:56.742+0000 7fbbc4745700 -1 osd.0 639 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:11:02.715 INFO:journalctl@ceph.osd.0.smithi008.stdout:Oct 09 06:11:02 smithi008 bash[735956]: 288560b20015cf290be5de477400f55b8ae6c80106ed4bb67ea16a85eba03bf8 2023-10-09T06:11:04.868 DEBUG:teuthology.orchestra.run.smithi008:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T06:11:04.910 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:11:04.910 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T06:11:04.911 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T06:11:04.911 DEBUG:teuthology.orchestra.run.smithi008:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T06:11:05.465 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 06:11:04 smithi008 systemd[1]: Stopping Ceph osd.1 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:11:05.466 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 06:11:05 smithi008 conmon[208712]: 2023-10-09T06:11:05.222+0000 7f05c0ded700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:11:05.467 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 06:11:05 smithi008 conmon[208712]: 2023-10-09T06:11:05.222+0000 7f05c0ded700 -1 osd.1 639 *** Got signal Terminated *** 2023-10-09T06:11:05.467 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 06:11:05 smithi008 conmon[208712]: 2023-10-09T06:11:05.222+0000 7f05c0ded700 -1 osd.1 639 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:11:11.714 INFO:journalctl@ceph.osd.1.smithi008.stdout:Oct 09 06:11:11 smithi008 bash[736537]: 086a4a5d61a57bc68de83968d6d5a43d356dbd5d37239172fbf9a9bfb9f5e1e6 2023-10-09T06:11:13.034 DEBUG:teuthology.orchestra.run.smithi008:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T06:11:13.078 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:11:13.078 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T06:11:13.078 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T06:11:13.078 DEBUG:teuthology.orchestra.run.smithi008:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T06:11:13.422 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 06:11:13 smithi008 systemd[1]: Stopping Ceph osd.2 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:11:13.714 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 06:11:13 smithi008 conmon[213723]: 2023-10-09T06:11:13.421+0000 7f2a8ba90700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:11:13.715 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 06:11:13 smithi008 conmon[213723]: 2023-10-09T06:11:13.421+0000 7f2a8ba90700 -1 osd.2 639 *** Got signal Terminated *** 2023-10-09T06:11:13.715 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 06:11:13 smithi008 conmon[213723]: 2023-10-09T06:11:13.421+0000 7f2a8ba90700 -1 osd.2 639 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:11:15.800 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:15 smithi191 conmon[151041]: ts=2023-10-09T06:11:15.442Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.8:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.8:8765: connect: connection refused" 2023-10-09T06:11:15.801 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:15 smithi191 conmon[151041]: ts=2023-10-09T06:11:15.439Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.8:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.8:8765: connect: connection refused" 2023-10-09T06:11:15.801 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:15 smithi191 conmon[151041]: ts=2023-10-09T06:11:15.439Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.8:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.8:8765: connect: connection refused" 2023-10-09T06:11:15.801 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:15 smithi191 conmon[151041]: ts=2023-10-09T06:11:15.442Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.8:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.8:8765: connect: connection refused" 2023-10-09T06:11:19.464 INFO:journalctl@ceph.osd.2.smithi008.stdout:Oct 09 06:11:19 smithi008 bash[737110]: 5a87289cd2f00cf82084594b83a07ccfd8d376bc5e03d2af3c09c1458df9ff14 2023-10-09T06:11:21.116 DEBUG:teuthology.orchestra.run.smithi008:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T06:11:21.159 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:11:21.159 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T06:11:21.159 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T06:11:21.160 DEBUG:teuthology.orchestra.run.smithi008:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T06:11:21.464 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 06:11:21 smithi008 systemd[1]: Stopping Ceph osd.3 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:11:21.964 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 06:11:21 smithi008 conmon[218691]: 2023-10-09T06:11:21.474+0000 7fbdcfad8700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:11:21.965 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 06:11:21 smithi008 conmon[218691]: 2023-10-09T06:11:21.474+0000 7fbdcfad8700 -1 osd.3 639 *** Got signal Terminated *** 2023-10-09T06:11:21.965 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 06:11:21 smithi008 conmon[218691]: 2023-10-09T06:11:21.474+0000 7fbdcfad8700 -1 osd.3 639 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:11:24.300 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:23 smithi191 conmon[181254]: debug 2023-10-09T06:11:23.858+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:24.714 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 06:11:24 smithi008 conmon[218691]: 2023-10-09T06:11:24.261+0000 7fbdcc0ce700 -1 osd.3 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:00.611735+0000 front 2023-10-09T06:11:00.611915+0000 (oldest deadline 2023-10-09T06:11:23.511948+0000) 2023-10-09T06:11:25.300 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:24 smithi191 conmon[181254]: debug 2023-10-09T06:11:24.826+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:25.300 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:24 smithi191 conmon[113449]: debug 2023-10-09T06:11:24.882+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:25.714 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 06:11:25 smithi008 conmon[218691]: 2023-10-09T06:11:25.301+0000 7fbdcc0ce700 -1 osd.3 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:00.611735+0000 front 2023-10-09T06:11:00.611915+0000 (oldest deadline 2023-10-09T06:11:23.511948+0000) 2023-10-09T06:11:25.799 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:25 smithi191 conmon[392089]: debug 2023-10-09T06:11:25.412+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:26.299 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:25 smithi191 conmon[181254]: debug 2023-10-09T06:11:25.803+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:26.300 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:25 smithi191 conmon[113449]: debug 2023-10-09T06:11:25.918+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:26.714 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 06:11:26 smithi008 conmon[218691]: 2023-10-09T06:11:26.298+0000 7fbdcc0ce700 -1 osd.3 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:00.611735+0000 front 2023-10-09T06:11:00.611915+0000 (oldest deadline 2023-10-09T06:11:23.511948+0000) 2023-10-09T06:11:26.799 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:26 smithi191 conmon[392089]: debug 2023-10-09T06:11:26.385+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:27.300 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:26 smithi191 conmon[113449]: debug 2023-10-09T06:11:26.885+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:27.300 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:26 smithi191 conmon[181254]: debug 2023-10-09T06:11:26.823+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:27.465 INFO:journalctl@ceph.osd.3.smithi008.stdout:Oct 09 06:11:27 smithi008 bash[737688]: 8625f8d5125ed07dd80332db22cb6d0f24261fe49c267ff682faa79d0b44009b 2023-10-09T06:11:27.799 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:27 smithi191 conmon[392089]: debug 2023-10-09T06:11:27.355+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:28.300 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:27 smithi191 conmon[113449]: debug 2023-10-09T06:11:27.863+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:28.300 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:27 smithi191 conmon[181254]: debug 2023-10-09T06:11:27.843+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:28.300 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:28 smithi191 conmon[107592]: debug 2023-10-09T06:11:28.000+0000 7f5b36a5d700 -1 osd.4 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.253281+0000 front 2023-10-09T06:11:01.253174+0000 (oldest deadline 2023-10-09T06:11:27.153381+0000) 2023-10-09T06:11:28.799 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:28 smithi191 conmon[392089]: debug 2023-10-09T06:11:28.399+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:29.156 DEBUG:teuthology.orchestra.run.smithi008:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T06:11:29.198 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:11:29.198 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T06:11:29.199 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T06:11:29.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T06:11:29.230 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:29 smithi191 conmon[107592]: debug 2023-10-09T06:11:29.031+0000 7f5b36a5d700 -1 osd.4 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.253281+0000 front 2023-10-09T06:11:01.253174+0000 (oldest deadline 2023-10-09T06:11:27.153381+0000) 2023-10-09T06:11:29.231 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:28 smithi191 conmon[113449]: debug 2023-10-09T06:11:28.907+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:29.231 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:28 smithi191 conmon[181254]: debug 2023-10-09T06:11:28.799+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:29.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:29 smithi191 conmon[392089]: debug 2023-10-09T06:11:29.360+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:29.550 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:29 smithi191 systemd[1]: Stopping Ceph osd.4 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:11:29.550 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:29 smithi191 conmon[107592]: debug 2023-10-09T06:11:29.478+0000 7f5b39ae0700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:11:29.550 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:29 smithi191 conmon[107592]: debug 2023-10-09T06:11:29.478+0000 7f5b39ae0700 -1 osd.4 639 *** Got signal Terminated *** 2023-10-09T06:11:29.551 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:29 smithi191 conmon[107592]: debug 2023-10-09T06:11:29.478+0000 7f5b39ae0700 -1 osd.4 639 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:11:30.050 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:29 smithi191 conmon[113449]: debug 2023-10-09T06:11:29.872+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:30.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:29 smithi191 conmon[181254]: debug 2023-10-09T06:11:29.759+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:30.405 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:30 smithi191 conmon[107592]: debug 2023-10-09T06:11:30.079+0000 7f5b36a5d700 -1 osd.4 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.253281+0000 front 2023-10-09T06:11:01.253174+0000 (oldest deadline 2023-10-09T06:11:27.153381+0000) 2023-10-09T06:11:30.747 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:30 smithi191 conmon[392089]: debug 2023-10-09T06:11:30.404+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:31.049 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:30 smithi191 conmon[113449]: debug 2023-10-09T06:11:30.846+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:31.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:30 smithi191 conmon[181254]: debug 2023-10-09T06:11:30.746+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:31.363 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:31 smithi191 conmon[107592]: debug 2023-10-09T06:11:31.067+0000 7f5b36a5d700 -1 osd.4 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.253281+0000 front 2023-10-09T06:11:01.253174+0000 (oldest deadline 2023-10-09T06:11:27.153381+0000) 2023-10-09T06:11:31.728 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:31 smithi191 conmon[392089]: debug 2023-10-09T06:11:31.362+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:32.049 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:31 smithi191 conmon[113449]: debug 2023-10-09T06:11:31.882+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:32.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:31 smithi191 conmon[181254]: debug 2023-10-09T06:11:31.726+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:32.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:31 smithi191 conmon[181254]: debug 2023-10-09T06:11:31.726+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:32.380 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:32 smithi191 conmon[107592]: debug 2023-10-09T06:11:32.068+0000 7f5b36a5d700 -1 osd.4 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.253281+0000 front 2023-10-09T06:11:01.253174+0000 (oldest deadline 2023-10-09T06:11:27.153381+0000) 2023-10-09T06:11:32.380 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:32 smithi191 conmon[107592]: debug 2023-10-09T06:11:32.068+0000 7f5b36a5d700 -1 osd.4 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:07.154022+0000 front 2023-10-09T06:11:07.153980+0000 (oldest deadline 2023-10-09T06:11:31.253890+0000) 2023-10-09T06:11:32.692 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:32 smithi191 conmon[392089]: debug 2023-10-09T06:11:32.379+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:32.692 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:32 smithi191 conmon[392089]: debug 2023-10-09T06:11:32.379+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:33.049 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:32 smithi191 conmon[113449]: debug 2023-10-09T06:11:32.882+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:33.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:32 smithi191 conmon[181254]: debug 2023-10-09T06:11:32.690+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:33.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:32 smithi191 conmon[181254]: debug 2023-10-09T06:11:32.690+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:33.390 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:33 smithi191 conmon[107592]: debug 2023-10-09T06:11:33.061+0000 7f5b36a5d700 -1 osd.4 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.253281+0000 front 2023-10-09T06:11:01.253174+0000 (oldest deadline 2023-10-09T06:11:27.153381+0000) 2023-10-09T06:11:33.390 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:33 smithi191 conmon[107592]: debug 2023-10-09T06:11:33.061+0000 7f5b36a5d700 -1 osd.4 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:07.154022+0000 front 2023-10-09T06:11:07.153980+0000 (oldest deadline 2023-10-09T06:11:31.253890+0000) 2023-10-09T06:11:33.672 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:33 smithi191 conmon[392089]: debug 2023-10-09T06:11:33.389+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:33.672 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:33 smithi191 conmon[392089]: debug 2023-10-09T06:11:33.389+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:34.049 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:33 smithi191 conmon[113449]: debug 2023-10-09T06:11:33.843+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:34.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:33 smithi191 conmon[181254]: debug 2023-10-09T06:11:33.670+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:34.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:33 smithi191 conmon[181254]: debug 2023-10-09T06:11:33.670+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:34.373 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:34 smithi191 conmon[107592]: debug 2023-10-09T06:11:34.061+0000 7f5b36a5d700 -1 osd.4 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.253281+0000 front 2023-10-09T06:11:01.253174+0000 (oldest deadline 2023-10-09T06:11:27.153381+0000) 2023-10-09T06:11:34.373 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:34 smithi191 conmon[107592]: debug 2023-10-09T06:11:34.061+0000 7f5b36a5d700 -1 osd.4 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:07.154022+0000 front 2023-10-09T06:11:07.153980+0000 (oldest deadline 2023-10-09T06:11:31.253890+0000) 2023-10-09T06:11:34.625 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:34 smithi191 conmon[392089]: debug 2023-10-09T06:11:34.372+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:34.625 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:34 smithi191 conmon[392089]: debug 2023-10-09T06:11:34.372+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:34.887 INFO:journalctl@ceph.osd.4.smithi191.stdout:Oct 09 06:11:34 smithi191 bash[431910]: 57a8d325e472a2970c5d444ed5cf2a095c51752e36a0b4036cacc85bd44ef284 2023-10-09T06:11:34.888 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:34 smithi191 conmon[181254]: debug 2023-10-09T06:11:34.623+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:34.888 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:34 smithi191 conmon[181254]: debug 2023-10-09T06:11:34.623+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:35.299 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:34 smithi191 conmon[113449]: debug 2023-10-09T06:11:34.886+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:35.641 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:35 smithi191 conmon[392089]: debug 2023-10-09T06:11:35.339+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:35.641 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:35 smithi191 conmon[392089]: debug 2023-10-09T06:11:35.339+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:36.050 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:35 smithi191 conmon[113449]: debug 2023-10-09T06:11:35.854+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:36.050 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:35 smithi191 conmon[113449]: debug 2023-10-09T06:11:35.854+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:09.074660+0000 front 2023-10-09T06:11:09.074592+0000 (oldest deadline 2023-10-09T06:11:34.974564+0000) 2023-10-09T06:11:36.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:35 smithi191 conmon[181254]: debug 2023-10-09T06:11:35.641+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:36.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:35 smithi191 conmon[181254]: debug 2023-10-09T06:11:35.641+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:36.652 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:36 smithi191 conmon[392089]: debug 2023-10-09T06:11:36.304+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:36.653 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:36 smithi191 conmon[392089]: debug 2023-10-09T06:11:36.304+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:36.905 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:36 smithi191 conmon[113449]: debug 2023-10-09T06:11:36.860+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:36.905 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:36 smithi191 conmon[113449]: debug 2023-10-09T06:11:36.860+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:09.074660+0000 front 2023-10-09T06:11:09.074592+0000 (oldest deadline 2023-10-09T06:11:34.974564+0000) 2023-10-09T06:11:36.905 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:36 smithi191 conmon[181254]: debug 2023-10-09T06:11:36.651+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:36.905 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:36 smithi191 conmon[181254]: debug 2023-10-09T06:11:36.651+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:36.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T06:11:36.993 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:11:36.993 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T06:11:36.993 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T06:11:36.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T06:11:37.288 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:37 smithi191 systemd[1]: Stopping Ceph osd.5 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:11:37.549 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:37 smithi191 conmon[392089]: debug 2023-10-09T06:11:37.287+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:37.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:37 smithi191 conmon[392089]: debug 2023-10-09T06:11:37.287+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:37.550 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:37 smithi191 conmon[113449]: debug 2023-10-09T06:11:37.291+0000 7fed1a651700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:11:37.550 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:37 smithi191 conmon[113449]: debug 2023-10-09T06:11:37.291+0000 7fed1a651700 -1 osd.5 639 *** Got signal Terminated *** 2023-10-09T06:11:37.550 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:37 smithi191 conmon[113449]: debug 2023-10-09T06:11:37.291+0000 7fed1a651700 -1 osd.5 639 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:11:38.049 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:37 smithi191 conmon[181254]: debug 2023-10-09T06:11:37.635+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:38.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:37 smithi191 conmon[181254]: debug 2023-10-09T06:11:37.635+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:38.050 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:37 smithi191 conmon[113449]: debug 2023-10-09T06:11:37.882+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:38.050 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:37 smithi191 conmon[113449]: debug 2023-10-09T06:11:37.882+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:09.074660+0000 front 2023-10-09T06:11:09.074592+0000 (oldest deadline 2023-10-09T06:11:34.974564+0000) 2023-10-09T06:11:38.549 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:38 smithi191 conmon[392089]: debug 2023-10-09T06:11:38.246+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:38.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:38 smithi191 conmon[392089]: debug 2023-10-09T06:11:38.246+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:39.050 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:38 smithi191 conmon[113449]: debug 2023-10-09T06:11:38.835+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:39.050 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:38 smithi191 conmon[113449]: debug 2023-10-09T06:11:38.835+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:09.074660+0000 front 2023-10-09T06:11:09.074592+0000 (oldest deadline 2023-10-09T06:11:34.974564+0000) 2023-10-09T06:11:39.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:38 smithi191 conmon[181254]: debug 2023-10-09T06:11:38.668+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:39.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:38 smithi191 conmon[181254]: debug 2023-10-09T06:11:38.668+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:39.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:39 smithi191 conmon[392089]: debug 2023-10-09T06:11:39.239+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:39.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:39 smithi191 conmon[392089]: debug 2023-10-09T06:11:39.239+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:40.049 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:39 smithi191 conmon[113449]: debug 2023-10-09T06:11:39.850+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:40.050 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:39 smithi191 conmon[113449]: debug 2023-10-09T06:11:39.850+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:09.074660+0000 front 2023-10-09T06:11:09.074592+0000 (oldest deadline 2023-10-09T06:11:34.974564+0000) 2023-10-09T06:11:40.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:39 smithi191 conmon[181254]: debug 2023-10-09T06:11:39.666+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:40.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:39 smithi191 conmon[181254]: debug 2023-10-09T06:11:39.666+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:40.549 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:40 smithi191 conmon[392089]: debug 2023-10-09T06:11:40.246+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:40.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:40 smithi191 conmon[392089]: debug 2023-10-09T06:11:40.246+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:41.049 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:40 smithi191 conmon[113449]: debug 2023-10-09T06:11:40.878+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:41.050 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:40 smithi191 conmon[113449]: debug 2023-10-09T06:11:40.878+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:09.074660+0000 front 2023-10-09T06:11:09.074592+0000 (oldest deadline 2023-10-09T06:11:34.974564+0000) 2023-10-09T06:11:41.050 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:40 smithi191 conmon[113449]: debug 2023-10-09T06:11:40.878+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:18.376340+0000 front 2023-10-09T06:11:18.376144+0000 (oldest deadline 2023-10-09T06:11:40.076011+0000) 2023-10-09T06:11:41.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:40 smithi191 conmon[181254]: debug 2023-10-09T06:11:40.666+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:41.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:40 smithi191 conmon[181254]: debug 2023-10-09T06:11:40.666+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:41.549 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:41 smithi191 conmon[392089]: debug 2023-10-09T06:11:41.251+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:41.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:41 smithi191 conmon[392089]: debug 2023-10-09T06:11:41.251+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:41.884 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:41 smithi191 conmon[181254]: debug 2023-10-09T06:11:41.617+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:41.885 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:41 smithi191 conmon[181254]: debug 2023-10-09T06:11:41.617+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:42.267 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:41 smithi191 conmon[113449]: debug 2023-10-09T06:11:41.884+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.773303+0000 front 2023-10-09T06:10:59.773274+0000 (oldest deadline 2023-10-09T06:11:24.473210+0000) 2023-10-09T06:11:42.267 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:41 smithi191 conmon[113449]: debug 2023-10-09T06:11:41.884+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:09.074660+0000 front 2023-10-09T06:11:09.074592+0000 (oldest deadline 2023-10-09T06:11:34.974564+0000) 2023-10-09T06:11:42.267 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:41 smithi191 conmon[113449]: debug 2023-10-09T06:11:41.884+0000 7fed14dc9700 -1 osd.5 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:18.376340+0000 front 2023-10-09T06:11:18.376144+0000 (oldest deadline 2023-10-09T06:11:40.076011+0000) 2023-10-09T06:11:42.545 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:42 smithi191 conmon[392089]: debug 2023-10-09T06:11:42.266+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:42.546 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:42 smithi191 conmon[392089]: debug 2023-10-09T06:11:42.266+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:42.546 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:42 smithi191 conmon[392089]: debug 2023-10-09T06:11:42.266+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.862327+0000 front 2023-10-09T06:11:16.862349+0000 (oldest deadline 2023-10-09T06:11:42.062730+0000) 2023-10-09T06:11:42.799 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:42 smithi191 conmon[181254]: debug 2023-10-09T06:11:42.643+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:42.800 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:42 smithi191 conmon[181254]: debug 2023-10-09T06:11:42.643+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:42.800 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:42 smithi191 conmon[181254]: debug 2023-10-09T06:11:42.643+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:43.299 INFO:journalctl@ceph.osd.5.smithi191.stdout:Oct 09 06:11:43 smithi191 bash[432491]: 141ad4d0ecc572a3985518a6c76198c1ab37210ab7fd070436d6bfd3badcd088 2023-10-09T06:11:43.300 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:43 smithi191 conmon[392089]: debug 2023-10-09T06:11:43.228+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:43.300 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:43 smithi191 conmon[392089]: debug 2023-10-09T06:11:43.228+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:43.300 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:43 smithi191 conmon[392089]: debug 2023-10-09T06:11:43.228+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.862327+0000 front 2023-10-09T06:11:16.862349+0000 (oldest deadline 2023-10-09T06:11:42.062730+0000) 2023-10-09T06:11:43.987 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:43 smithi191 conmon[181254]: debug 2023-10-09T06:11:43.658+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:43.987 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:43 smithi191 conmon[181254]: debug 2023-10-09T06:11:43.658+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:43.987 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:43 smithi191 conmon[181254]: debug 2023-10-09T06:11:43.658+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:44.300 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:44 smithi191 conmon[392089]: debug 2023-10-09T06:11:44.209+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:44.300 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:44 smithi191 conmon[392089]: debug 2023-10-09T06:11:44.209+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:44.300 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:44 smithi191 conmon[392089]: debug 2023-10-09T06:11:44.209+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.862327+0000 front 2023-10-09T06:11:16.862349+0000 (oldest deadline 2023-10-09T06:11:42.062730+0000) 2023-10-09T06:11:44.585 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T06:11:44.624 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:11:44.625 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T06:11:44.625 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T06:11:44.625 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T06:11:44.928 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:44 smithi191 systemd[1]: Stopping Ceph osd.6 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:11:44.928 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:44 smithi191 conmon[181254]: debug 2023-10-09T06:11:44.691+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:44.928 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:44 smithi191 conmon[181254]: debug 2023-10-09T06:11:44.691+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:44.928 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:44 smithi191 conmon[181254]: debug 2023-10-09T06:11:44.691+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:45.240 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:44 smithi191 conmon[392089]: debug 2023-10-09T06:11:44.927+0000 7fe0e1f98700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:11:45.241 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:44 smithi191 conmon[392089]: debug 2023-10-09T06:11:44.927+0000 7fe0e1f98700 -1 osd.6 639 *** Got signal Terminated *** 2023-10-09T06:11:45.241 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:44 smithi191 conmon[392089]: debug 2023-10-09T06:11:44.927+0000 7fe0e1f98700 -1 osd.6 639 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:11:45.549 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:45 smithi191 conmon[392089]: debug 2023-10-09T06:11:45.240+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:45.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:45 smithi191 conmon[392089]: debug 2023-10-09T06:11:45.240+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:45.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:45 smithi191 conmon[392089]: debug 2023-10-09T06:11:45.240+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.862327+0000 front 2023-10-09T06:11:16.862349+0000 (oldest deadline 2023-10-09T06:11:42.062730+0000) 2023-10-09T06:11:46.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:45 smithi191 conmon[181254]: debug 2023-10-09T06:11:45.718+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:46.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:45 smithi191 conmon[181254]: debug 2023-10-09T06:11:45.718+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:46.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:45 smithi191 conmon[181254]: debug 2023-10-09T06:11:45.718+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:46.549 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:46 smithi191 conmon[392089]: debug 2023-10-09T06:11:46.273+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:46.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:46 smithi191 conmon[392089]: debug 2023-10-09T06:11:46.273+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:46.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:46 smithi191 conmon[392089]: debug 2023-10-09T06:11:46.273+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.862327+0000 front 2023-10-09T06:11:16.862349+0000 (oldest deadline 2023-10-09T06:11:42.062730+0000) 2023-10-09T06:11:47.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:46 smithi191 conmon[181254]: debug 2023-10-09T06:11:46.680+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:47.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:46 smithi191 conmon[181254]: debug 2023-10-09T06:11:46.680+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:47.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:46 smithi191 conmon[181254]: debug 2023-10-09T06:11:46.680+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:47.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:47 smithi191 conmon[392089]: debug 2023-10-09T06:11:47.253+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:47.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:47 smithi191 conmon[392089]: debug 2023-10-09T06:11:47.253+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:47.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:47 smithi191 conmon[392089]: debug 2023-10-09T06:11:47.253+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.862327+0000 front 2023-10-09T06:11:16.862349+0000 (oldest deadline 2023-10-09T06:11:42.062730+0000) 2023-10-09T06:11:48.049 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:47 smithi191 conmon[181254]: debug 2023-10-09T06:11:47.634+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:48.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:47 smithi191 conmon[181254]: debug 2023-10-09T06:11:47.634+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:48.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:47 smithi191 conmon[181254]: debug 2023-10-09T06:11:47.634+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:48.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:47 smithi191 conmon[181254]: debug 2023-10-09T06:11:47.634+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6830 osd.3 since back 2023-10-09T06:11:22.224051+0000 front 2023-10-09T06:11:22.224059+0000 (oldest deadline 2023-10-09T06:11:47.523964+0000) 2023-10-09T06:11:48.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:48 smithi191 conmon[392089]: debug 2023-10-09T06:11:48.211+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:48.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:48 smithi191 conmon[392089]: debug 2023-10-09T06:11:48.211+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:48.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:48 smithi191 conmon[392089]: debug 2023-10-09T06:11:48.211+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.862327+0000 front 2023-10-09T06:11:16.862349+0000 (oldest deadline 2023-10-09T06:11:42.062730+0000) 2023-10-09T06:11:49.049 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:48 smithi191 conmon[181254]: debug 2023-10-09T06:11:48.642+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:49.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:48 smithi191 conmon[181254]: debug 2023-10-09T06:11:48.642+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:49.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:48 smithi191 conmon[181254]: debug 2023-10-09T06:11:48.642+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:49.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:48 smithi191 conmon[181254]: debug 2023-10-09T06:11:48.642+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6830 osd.3 since back 2023-10-09T06:11:22.224051+0000 front 2023-10-09T06:11:22.224059+0000 (oldest deadline 2023-10-09T06:11:47.523964+0000) 2023-10-09T06:11:49.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:49 smithi191 conmon[392089]: debug 2023-10-09T06:11:49.229+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:10:59.360391+0000 front 2023-10-09T06:10:59.360378+0000 (oldest deadline 2023-10-09T06:11:25.260038+0000) 2023-10-09T06:11:49.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:49 smithi191 conmon[392089]: debug 2023-10-09T06:11:49.229+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:08.760991+0000 front 2023-10-09T06:11:08.761033+0000 (oldest deadline 2023-10-09T06:11:32.260874+0000) 2023-10-09T06:11:49.550 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:49 smithi191 conmon[392089]: debug 2023-10-09T06:11:49.229+0000 7fe0dd712700 -1 osd.6 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.862327+0000 front 2023-10-09T06:11:16.862349+0000 (oldest deadline 2023-10-09T06:11:42.062730+0000) 2023-10-09T06:11:50.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:49 smithi191 conmon[181254]: debug 2023-10-09T06:11:49.683+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:50.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:49 smithi191 conmon[181254]: debug 2023-10-09T06:11:49.683+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:50.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:49 smithi191 conmon[181254]: debug 2023-10-09T06:11:49.683+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:50.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:49 smithi191 conmon[181254]: debug 2023-10-09T06:11:49.683+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6830 osd.3 since back 2023-10-09T06:11:22.224051+0000 front 2023-10-09T06:11:22.224059+0000 (oldest deadline 2023-10-09T06:11:47.523964+0000) 2023-10-09T06:11:50.549 INFO:journalctl@ceph.osd.6.smithi191.stdout:Oct 09 06:11:50 smithi191 bash[433068]: 4feaa20b7a1a2c1f899a3d8bc59ad3a6f7dc9e42046714f508e5786ae09e6adc 2023-10-09T06:11:50.715 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:50 smithi191 conmon[181254]: debug 2023-10-09T06:11:50.712+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:50.715 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:50 smithi191 conmon[181254]: debug 2023-10-09T06:11:50.712+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:51.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:50 smithi191 conmon[181254]: debug 2023-10-09T06:11:50.712+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:51.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:50 smithi191 conmon[181254]: debug 2023-10-09T06:11:50.712+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6830 osd.3 since back 2023-10-09T06:11:22.224051+0000 front 2023-10-09T06:11:22.224059+0000 (oldest deadline 2023-10-09T06:11:47.523964+0000) 2023-10-09T06:11:51.799 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:51 smithi191 conmon[181254]: debug 2023-10-09T06:11:51.688+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:51.800 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:51 smithi191 conmon[181254]: debug 2023-10-09T06:11:51.688+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:51.800 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:51 smithi191 conmon[181254]: debug 2023-10-09T06:11:51.688+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:51.800 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:51 smithi191 conmon[181254]: debug 2023-10-09T06:11:51.688+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6830 osd.3 since back 2023-10-09T06:11:22.224051+0000 front 2023-10-09T06:11:22.224059+0000 (oldest deadline 2023-10-09T06:11:47.523964+0000) 2023-10-09T06:11:51.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T06:11:51.967 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:11:51.967 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T06:11:51.968 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T06:11:51.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T06:11:52.267 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:52 smithi191 systemd[1]: Stopping Ceph osd.7 for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:11:52.549 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:52 smithi191 conmon[181254]: debug 2023-10-09T06:11:52.267+0000 7efc02e17700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:11:52.550 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:52 smithi191 conmon[181254]: debug 2023-10-09T06:11:52.267+0000 7efc02e17700 -1 osd.7 639 *** Got signal Terminated *** 2023-10-09T06:11:52.550 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:52 smithi191 conmon[181254]: debug 2023-10-09T06:11:52.267+0000 7efc02e17700 -1 osd.7 639 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:11:53.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:52 smithi191 conmon[181254]: debug 2023-10-09T06:11:52.658+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:53.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:52 smithi191 conmon[181254]: debug 2023-10-09T06:11:52.658+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:53.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:52 smithi191 conmon[181254]: debug 2023-10-09T06:11:52.658+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:53.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:52 smithi191 conmon[181254]: debug 2023-10-09T06:11:52.658+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6830 osd.3 since back 2023-10-09T06:11:22.224051+0000 front 2023-10-09T06:11:22.224059+0000 (oldest deadline 2023-10-09T06:11:47.523964+0000) 2023-10-09T06:11:54.049 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:53 smithi191 conmon[181254]: debug 2023-10-09T06:11:53.699+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:54.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:53 smithi191 conmon[181254]: debug 2023-10-09T06:11:53.699+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:54.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:53 smithi191 conmon[181254]: debug 2023-10-09T06:11:53.699+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:54.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:53 smithi191 conmon[181254]: debug 2023-10-09T06:11:53.699+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6830 osd.3 since back 2023-10-09T06:11:22.224051+0000 front 2023-10-09T06:11:22.224059+0000 (oldest deadline 2023-10-09T06:11:47.523964+0000) 2023-10-09T06:11:55.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:54 smithi191 conmon[181254]: debug 2023-10-09T06:11:54.668+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:55.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:54 smithi191 conmon[181254]: debug 2023-10-09T06:11:54.668+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:55.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:54 smithi191 conmon[181254]: debug 2023-10-09T06:11:54.668+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:55.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:54 smithi191 conmon[181254]: debug 2023-10-09T06:11:54.668+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6830 osd.3 since back 2023-10-09T06:11:22.224051+0000 front 2023-10-09T06:11:22.224059+0000 (oldest deadline 2023-10-09T06:11:47.523964+0000) 2023-10-09T06:11:56.049 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:55 smithi191 conmon[181254]: debug 2023-10-09T06:11:55.630+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:56.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:55 smithi191 conmon[181254]: debug 2023-10-09T06:11:55.630+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:56.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:55 smithi191 conmon[181254]: debug 2023-10-09T06:11:55.630+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:56.053 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:55 smithi191 conmon[181254]: debug 2023-10-09T06:11:55.630+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6830 osd.3 since back 2023-10-09T06:11:22.224051+0000 front 2023-10-09T06:11:22.224059+0000 (oldest deadline 2023-10-09T06:11:47.523964+0000) 2023-10-09T06:11:56.053 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:55 smithi191 conmon[181254]: debug 2023-10-09T06:11:55.630+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.191:6804 osd.4 since back 2023-10-09T06:11:31.024973+0000 front 2023-10-09T06:11:31.024990+0000 (oldest deadline 2023-10-09T06:11:55.124976+0000) 2023-10-09T06:11:57.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:56 smithi191 conmon[181254]: debug 2023-10-09T06:11:56.612+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6806 osd.0 since back 2023-10-09T06:11:01.121584+0000 front 2023-10-09T06:11:01.121519+0000 (oldest deadline 2023-10-09T06:11:22.821409+0000) 2023-10-09T06:11:57.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:56 smithi191 conmon[181254]: debug 2023-10-09T06:11:56.612+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6814 osd.1 since back 2023-10-09T06:11:05.122595+0000 front 2023-10-09T06:11:05.122672+0000 (oldest deadline 2023-10-09T06:11:31.022484+0000) 2023-10-09T06:11:57.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:56 smithi191 conmon[181254]: debug 2023-10-09T06:11:56.612+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6822 osd.2 since back 2023-10-09T06:11:16.923507+0000 front 2023-10-09T06:11:16.923412+0000 (oldest deadline 2023-10-09T06:11:42.223458+0000) 2023-10-09T06:11:57.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:56 smithi191 conmon[181254]: debug 2023-10-09T06:11:56.612+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.8:6830 osd.3 since back 2023-10-09T06:11:22.224051+0000 front 2023-10-09T06:11:22.224059+0000 (oldest deadline 2023-10-09T06:11:47.523964+0000) 2023-10-09T06:11:57.050 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:56 smithi191 conmon[181254]: debug 2023-10-09T06:11:56.612+0000 7efbfbd8c700 -1 osd.7 639 heartbeat_check: no reply from 172.21.15.191:6804 osd.4 since back 2023-10-09T06:11:31.024973+0000 front 2023-10-09T06:11:31.024990+0000 (oldest deadline 2023-10-09T06:11:55.124976+0000) 2023-10-09T06:11:58.049 INFO:journalctl@ceph.osd.7.smithi191.stdout:Oct 09 06:11:57 smithi191 bash[433648]: 6282488b13e674a60c927473c4d58e5b5597ad6c56ef3352eec88ff1e16bb2b4 2023-10-09T06:11:59.298 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T06:11:59.338 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:11:59.338 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T06:11:59.338 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T06:11:59.341 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T06:11:59.550 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 systemd[1]: Stopping Ceph prometheus.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:12:00.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.641Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:12:00.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.641Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T06:12:00.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.641Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T06:12:00.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.641Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T06:12:00.050 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.641Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T06:12:00.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.641Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T06:12:00.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.641Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T06:12:00.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.641Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T06:12:00.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.641Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T06:12:00.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.643Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T06:12:00.051 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.643Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T06:12:00.052 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Oct 09 06:11:59 smithi191 conmon[151041]: ts=2023-10-09T06:11:59.643Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T06:12:00.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T06:12:00.343 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:12:00.343 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T06:12:00.343 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T06:12:01.715 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 06:12:01 smithi008 systemd[1]: Stopping Ceph node-exporter.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:12:02.562 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 06:12:02 smithi008 bash[738578]: 0b0d19fe4991526ba7d8d317472327fcdade1f66addd61b34db3a862e3e88c0e 2023-10-09T06:12:02.563 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 06:12:02 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:12:02.563 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 06:12:02 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T06:12:02.563 INFO:journalctl@ceph.node-exporter.a.smithi008.stdout:Oct 09 06:12:02 smithi008 systemd[1]: Stopped Ceph node-exporter.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T06:12:02.952 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 06:12:02 smithi008 systemd[1]: Stopping Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:12:02.952 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 06:12:02 smithi008 conmon[175748]: level=info ts=2023-10-09T06:12:02.801Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:12:03.846 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 06:12:03 smithi008 bash[738856]: 172e511a984d8f143d866d4460c2dcae82ad2ebf33dab50c4132adc1992badb8 2023-10-09T06:12:03.846 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 06:12:03 smithi008 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T06:12:03.847 INFO:journalctl@ceph.alertmanager.a.smithi008.stdout:Oct 09 06:12:03 smithi008 systemd[1]: Stopped Ceph alertmanager.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T06:12:12.896 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T06:12:14.300 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 06:12:14 smithi191 systemd[1]: Stopping Ceph node-exporter.b for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:12:15.000 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 06:12:14 smithi191 bash[434833]: 05f0cea760c5fae5d2f19492ac44e265c35330e0219c4170185d5aa8d94a88e1 2023-10-09T06:12:15.001 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 06:12:14 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:12:15.001 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 06:12:14 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T06:12:15.001 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Oct 09 06:12:14 smithi191 systemd[1]: Stopped Ceph node-exporter.b for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T06:12:15.300 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 06:12:15 smithi191 systemd[1]: Stopping Ceph grafana.a for 9e85b874-6662-11ee-8db5-212e2dc638e7... 2023-10-09T06:12:15.300 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 06:12:15 smithi191 conmon[146508]: t=2023-10-09T06:12:15+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T06:12:15.973 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 06:12:15 smithi191 bash[435113]: 66b01c4fecc1c055ed262a624227b278f869cca63e49773a544458edb32fd83f 2023-10-09T06:12:15.974 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 06:12:15 smithi191 systemd[1]: ceph-9e85b874-6662-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T06:12:15.974 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Oct 09 06:12:15 smithi191 systemd[1]: Stopped Ceph grafana.a for 9e85b874-6662-11ee-8db5-212e2dc638e7. 2023-10-09T06:12:20.114 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:12:20.166 DEBUG:teuthology.orchestra.run.smithi191:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:12:20.196 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T06:12:20.198 DEBUG:teuthology.misc:Transferring archived files from smithi008:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492/remote/smithi008/crash 2023-10-09T06:12:20.199 DEBUG:teuthology.orchestra.run.smithi008:> sudo tar c -f - -C /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T06:12:20.892 INFO:teuthology.orchestra.run.smithi008.stderr:tar: /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T06:12:20.893 INFO:teuthology.orchestra.run.smithi008.stderr:tar: Error is not recoverable: exiting now 2023-10-09T06:12:20.894 DEBUG:teuthology.misc:Transferring archived files from smithi191:/var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492/remote/smithi191/crash 2023-10-09T06:12:20.895 DEBUG:teuthology.orchestra.run.smithi191:> sudo tar c -f - -C /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T06:12:20.946 INFO:teuthology.orchestra.run.smithi191.stderr:tar: /var/lib/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T06:12:20.947 INFO:teuthology.orchestra.run.smithi191.stderr:tar: Error is not recoverable: exiting now 2023-10-09T06:12:20.948 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T06:12:20.948 DEBUG:teuthology.orchestra.run.smithi008:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T06:12:21.004 INFO:teuthology.orchestra.run.smithi008.stderr:grep: /var/log/ceph/9e85b874-6662-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T06:12:21.008 INFO:tasks.cephadm:Compressing logs... 2023-10-09T06:12:21.008 DEBUG:teuthology.orchestra.run.smithi008:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:12:21.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:12:21.047 INFO:teuthology.orchestra.run.smithi191.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T06:12:21.059 INFO:teuthology.orchestra.run.smithi008.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T06:23:11.804 INFO:tasks.cephadm:Archiving logs... 2023-10-09T06:23:11.828 DEBUG:teuthology.misc:Transferring archived files from smithi008:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492/remote/smithi008/log 2023-10-09T06:23:11.829 DEBUG:teuthology.orchestra.run.smithi008:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T06:24:29.314 DEBUG:teuthology.misc:Transferring archived files from smithi191:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492/remote/smithi191/log 2023-10-09T06:24:29.316 DEBUG:teuthology.orchestra.run.smithi191:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T06:25:22.970 INFO:tasks.cephadm:Removing cluster... 2023-10-09T06:25:22.970 DEBUG:teuthology.orchestra.run.smithi008:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 --force 2023-10-09T06:25:24.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9e85b874-6662-11ee-8db5-212e2dc638e7 --force 2023-10-09T06:25:25.403 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T06:25:25.403 DEBUG:teuthology.orchestra.run.smithi008:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T06:25:25.425 DEBUG:teuthology.orchestra.run.smithi191:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T06:25:25.460 INFO:tasks.cephadm:Teardown complete 2023-10-09T06:25:25.461 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T06:25:25.477 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T06:25:25.478 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T06:25:25.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T06:25:25.608 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T06:25:25.620 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T06:25:25.620 DEBUG:teuthology.orchestra.run.smithi008:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T06:25:25.623 DEBUG:teuthology.orchestra.run.smithi191:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T06:25:25.639 INFO:teuthology.orchestra.run.smithi191.stderr:bash: ntpq: command not found 2023-10-09T06:25:25.648 INFO:teuthology.orchestra.run.smithi008.stderr:bash: ntpq: command not found 2023-10-09T06:25:25.661 INFO:teuthology.orchestra.run.smithi191.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T06:25:25.661 INFO:teuthology.orchestra.run.smithi191.stdout:=============================================================================== 2023-10-09T06:25:25.662 INFO:teuthology.orchestra.run.smithi191.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 53 -649us[ -649us] +/- 83ms 2023-10-09T06:25:25.662 INFO:teuthology.orchestra.run.smithi191.stdout:^* hv02.front.sepia.ceph.com 3 6 377 53 +296us[ +320us] +/- 77ms 2023-10-09T06:25:25.662 INFO:teuthology.orchestra.run.smithi191.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 52 +406us[ +406us] +/- 63ms 2023-10-09T06:25:25.662 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T06:25:25.675 INFO:teuthology.orchestra.run.smithi008.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T06:25:25.675 INFO:teuthology.orchestra.run.smithi008.stdout:=============================================================================== 2023-10-09T06:25:25.676 INFO:teuthology.orchestra.run.smithi008.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 53 -700us[ -698us] +/- 83ms 2023-10-09T06:25:25.676 INFO:teuthology.orchestra.run.smithi008.stdout:^* hv02.front.sepia.ceph.com 3 6 377 51 +232us[ +233us] +/- 77ms 2023-10-09T06:25:25.676 INFO:teuthology.orchestra.run.smithi008.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 52 +354us[ +356us] +/- 63ms 2023-10-09T06:25:25.676 INFO:teuthology.orchestra.run.smithi008.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T06:25:25.677 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T06:25:25.687 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T06:25:25.688 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T06:25:25.702 DEBUG:teuthology.orchestra.run.smithi008:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T06:25:25.861 DEBUG:teuthology.orchestra.run.smithi191:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T06:25:25.987 DEBUG:teuthology.orchestra.run.smithi008:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T06:25:26.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:26.024 DEBUG:teuthology.orchestra.run.smithi191:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T06:25:26.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:26.056 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T06:25:26.066 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T06:25:26.079 INFO:teuthology.task.internal:Duration was 5194.343263 seconds 2023-10-09T06:25:26.080 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T06:25:26.090 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T06:25:26.090 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T06:25:26.093 DEBUG:teuthology.orchestra.run.smithi191:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T06:25:26.161 INFO:teuthology.orchestra.run.smithi008.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T06:25:26.161 INFO:teuthology.orchestra.run.smithi191.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T06:25:26.503 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T06:25:26.503 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi008.front.sepia.ceph.com 2023-10-09T06:25:26.503 DEBUG:teuthology.orchestra.run.smithi008:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T06:25:26.532 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi191.front.sepia.ceph.com 2023-10-09T06:25:26.532 DEBUG:teuthology.orchestra.run.smithi191:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T06:25:26.562 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T06:25:26.562 DEBUG:teuthology.orchestra.run.smithi008:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:25:26.575 DEBUG:teuthology.orchestra.run.smithi191:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:25:26.741 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T06:25:26.741 DEBUG:teuthology.orchestra.run.smithi008:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T06:25:26.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T06:25:27.182 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T06:25:27.209 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T06:25:27.210 DEBUG:teuthology.orchestra.run.smithi008:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T06:25:27.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T06:25:27.318 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T06:25:27.341 DEBUG:teuthology.orchestra.run.smithi008:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:25:27.344 DEBUG:teuthology.orchestra.run.smithi191:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:25:27.395 INFO:teuthology.orchestra.run.smithi008.stdout:kernel.core_pattern = core 2023-10-09T06:25:27.396 INFO:teuthology.orchestra.run.smithi191.stdout:kernel.core_pattern = core 2023-10-09T06:25:27.428 DEBUG:teuthology.orchestra.run.smithi008:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:25:27.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:27.446 DEBUG:teuthology.orchestra.run.smithi191:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:25:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:27.485 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T06:25:27.516 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T06:25:27.522 DEBUG:teuthology.misc:Transferring archived files from smithi008:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492/remote/smithi008 2023-10-09T06:25:27.522 DEBUG:teuthology.orchestra.run.smithi008:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T06:25:27.605 DEBUG:teuthology.misc:Transferring archived files from smithi191:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492/remote/smithi191 2023-10-09T06:25:27.605 DEBUG:teuthology.orchestra.run.smithi191:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T06:25:27.668 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T06:25:27.668 DEBUG:teuthology.orchestra.run.smithi008:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T06:25:27.671 DEBUG:teuthology.orchestra.run.smithi191:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T06:25:27.716 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T06:25:27.784 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T06:25:27.784 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T06:25:27.808 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T06:25:27.808 DEBUG:teuthology.orchestra.run.smithi008:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T06:25:27.811 DEBUG:teuthology.orchestra.run.smithi191:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T06:25:27.827 INFO:teuthology.orchestra.run.smithi191.stdout: 262307 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 06:25 /home/ubuntu/cephtest 2023-10-09T06:25:27.856 INFO:teuthology.orchestra.run.smithi008.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 06:25 /home/ubuntu/cephtest 2023-10-09T06:25:27.856 INFO:teuthology.orchestra.run.smithi008.stdout: 4456449 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 06:08 /home/ubuntu/cephtest/mnt.0 2023-10-09T06:25:27.856 INFO:teuthology.orchestra.run.smithi008.stdout: 4456450 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 06:08 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:25:27.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:27.864 INFO:teuthology.orchestra.run.smithi008.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T06:25:27.864 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi008 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T06:25:27.871 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T06:25:27.962 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T06:25:27.977 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi008 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T06:25:27.978 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T06:25:28.007 DEBUG:teuthology.nuke:shortname: smithi008 2023-10-09T06:25:28.007 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T06:25:28.029 DEBUG:teuthology.nuke:shortname: smithi191 2023-10-09T06:25:28.029 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T06:25:28.034 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi008.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 04:53:34.770646', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T06:25:28.062 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi191.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417492', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 04:53:34.772192', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T06:25:28.090 INFO:teuthology.orchestra.console.smithi008:Power off 2023-10-09T06:25:28.090 DEBUG:teuthology.orchestra.console.smithi008:pexpect command: ipmitool -H smithi008.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T06:25:28.117 DEBUG:teuthology.orchestra.console.smithi008:power off output: Chassis Power Control: Down/Off 2023-10-09T06:25:28.117 DEBUG:teuthology.orchestra.console.smithi008:pexpect command: ipmitool -H smithi008.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:25:28.138 INFO:teuthology.orchestra.console.smithi191:Power off 2023-10-09T06:25:28.139 DEBUG:teuthology.orchestra.console.smithi191:pexpect command: ipmitool -H smithi191.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T06:25:28.154 DEBUG:teuthology.orchestra.console.smithi008:check power output: Chassis Power is on 2023-10-09T06:25:28.166 DEBUG:teuthology.orchestra.console.smithi191:power off output: Chassis Power Control: Down/Off 2023-10-09T06:25:28.166 DEBUG:teuthology.orchestra.console.smithi191:pexpect command: ipmitool -H smithi191.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:25:28.193 DEBUG:teuthology.orchestra.console.smithi191:check power output: Chassis Power is on 2023-10-09T06:25:32.156 DEBUG:teuthology.orchestra.console.smithi008:pexpect command: ipmitool -H smithi008.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:25:32.195 DEBUG:teuthology.orchestra.console.smithi191:pexpect command: ipmitool -H smithi191.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:25:32.267 DEBUG:teuthology.orchestra.console.smithi008:check power output: Chassis Power is on 2023-10-09T06:25:32.305 DEBUG:teuthology.orchestra.console.smithi191:check power output: Chassis Power is on 2023-10-09T06:25:36.272 DEBUG:teuthology.orchestra.console.smithi008:pexpect command: ipmitool -H smithi008.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:25:36.306 DEBUG:teuthology.orchestra.console.smithi191:pexpect command: ipmitool -H smithi191.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:25:36.382 DEBUG:teuthology.orchestra.console.smithi008:check power output: Chassis Power is off 2023-10-09T06:25:36.416 DEBUG:teuthology.orchestra.console.smithi191:check power output: Chassis Power is off 2023-10-09T06:25:36.483 INFO:teuthology.orchestra.console.smithi008:Power off completed 2023-10-09T06:25:36.517 INFO:teuthology.orchestra.console.smithi191:Power off completed 2023-10-09T06:25:36.621 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 5194.343262672424 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=b49b9fbe5efd4c8ca20e2c2a42334fed status: fail success: false 2023-10-09T06:25:36.622 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T06:25:36.709 INFO:teuthology.run:FAIL